Samples Java Spring (JDK17)

name: Samples Java Spring (JDK17)

on:
  push:
    paths:
      - samples/openapi3/client/petstore/spring-cloud-3-with-optional
      - samples/openapi3/server/petstore/springboot-3
  pull_request:
    paths:
      - samples/openapi3/client/petstore/spring-cloud-3-with-optional
      - samples/openapi3/server/petstore/springboot-3
      - samples/server/petstore/springboot-lombok-data
      - samples/server/petstore/springboot-lombok-tostring
jobs:
  build:
    name: Build Java Spring (JDK17)
    runs-on: ubuntu-latest
    strategy:
      fail-fast: false
      matrix:
        sample:
          # clients
          - samples/openapi3/client/petstore/spring-cloud-3-with-optional
          # servers
          - samples/openapi3/server/petstore/springboot-3
          - samples/server/petstore/springboot-lombok-data
          - samples/server/petstore/springboot-lombok-tostring
    steps:
      - uses: actions/checkout@v4
      - uses: actions/setup-java@v4
        with:
          distribution: 'temurin'
          java-version: 17
      - name: Cache maven dependencies
        uses: actions/cache@v4
        env:
          cache-name: maven-repository
        with:
          path: |
            ~/.m2
          key: ${{ runner.os }}-${{ github.job }}-${{ env.cache-name }}-${{ hashFiles('**/pom.xml') }}
      - name: Build
        working-directory: ${{ matrix.sample }}
        run: mvn clean package --no-transfer-progress


Samples Java Spring

name: Samples Java Spring

on:
  push:
    paths:
      # clients
      - 'samples/client/petstore/spring*/**'
      # servers
      - 'samples/server/petstore/spring*/**'
      - 'samples/openapi3/server/petstore/spring*/**'
  pull_request:
    paths:
      # clients
      - 'samples/client/petstore/spring*/**'
      # servers
      - 'samples/server/petstore/spring*/**'
      - 'samples/openapi3/server/petstore/spring*/**'
jobs:
  build:
    name: Build Java Spring
    runs-on: ubuntu-latest
    strategy:
      fail-fast: false
      matrix:
        sample:
          # clients
          - samples/client/petstore/spring-cloud
          - samples/client/petstore/spring-cloud-auth
          - samples/client/petstore/spring-cloud-date-time
          - samples/client/petstore/spring-cloud-deprecated
          - samples/client/petstore/spring-cloud-tags
          - samples/openapi3/client/petstore/spring-cloud
          - samples/openapi3/client/petstore/spring-cloud-async
          - samples/openapi3/client/petstore/spring-cloud-date-time
          - samples/openapi3/client/petstore/spring-cloud-http-basic
          - samples/openapi3/client/petstore/spring-cloud-spring-pageable
          - samples/openapi3/client/petstore/spring-stubs
          - samples/openapi3/client/petstore/spring-stubs-skip-default-interface
          # servers
          - samples/openapi3/server/petstore/springboot
          - samples/openapi3/server/petstore/springboot-implicitHeaders
          - samples/openapi3/server/petstore/springboot-delegate
          - samples/openapi3/server/petstore/spring-boot-oneof
          - samples/server/petstore/spring-boot-nullable-set
          - samples/server/petstore/spring-boot-defaultInterface-unhandledException
          - samples/server/petstore/springboot
          - samples/server/petstore/springboot-beanvalidation
          - samples/server/petstore/springboot-delegate
          - samples/server/petstore/springboot-delegate-no-response-entity
          - samples/server/petstore/springboot-implicitHeaders
          - samples/server/petstore/springboot-implicitHeaders-annotationLibrary
          - samples/server/petstore/springboot-lombok-data
          - samples/server/petstore/springboot-reactive
          - samples/server/petstore/springboot-reactive-noResponseEntity
          - samples/server/petstore/springboot-spring-pageable-delegatePattern-without-j8
          - samples/server/petstore/springboot-spring-pageable-delegatePattern
          - samples/server/petstore/springboot-spring-pageable-without-j8
          - samples/server/petstore/springboot-spring-pageable
          - samples/server/petstore/springboot-spring-provide-args
          - samples/server/petstore/springboot-useoptional
          - samples/server/petstore/springboot-virtualan
    steps:
      - uses: actions/checkout@v4
      - uses: actions/setup-java@v4
        with:
          distribution: 'temurin'
          java-version: 8
      - name: Cache maven dependencies
        uses: actions/cache@v4
        env:
          cache-name: maven-repository
        with:
          path: |
            ~/.m2
          key: ${{ runner.os }}-${{ github.job }}-${{ env.cache-name }}-${{ hashFiles('**/pom.xml') }}
      - name: Build
        working-directory: ${{ matrix.sample }}
        run: mvn clean package --no-transfer-progress


Samples Swift

name: Samples Swift

on:
  push:
    paths:
      - samples/client/petstore/swift5/**
  pull_request:
    paths:
      - samples/client/petstore/swift5/**

jobs:
  build:
    strategy:
      fail-fast: false
      matrix:
        os: [macos-latest]
        sample:
          - samples/client/petstore/swift5/alamofireLibrary
          - samples/client/petstore/swift5/anycodableLibrary
          - samples/client/petstore/swift5/asyncAwaitLibrary
          - samples/client/petstore/swift5/combineLibrary
          - samples/client/petstore/swift5/default
          - samples/client/petstore/swift5/deprecated
          - samples/client/petstore/swift5/frozenEnums
          - samples/client/petstore/swift5/nonPublicApi
          - samples/client/petstore/swift5/objcCompatible
          - samples/client/petstore/swift5/oneOf
          - samples/client/petstore/swift5/promisekitLibrary
          - samples/client/petstore/swift5/readonlyProperties
          - samples/client/petstore/swift5/resultLibrary
          - samples/client/petstore/swift5/rxswiftLibrary
          - samples/client/petstore/swift5/urlsessionLibrary
          - samples/client/petstore/swift5/validation
          # NOTE: disabled as vaporLibrary doesnt build when adding this CI config
          #- samples/client/petstore/swift5/vaporLibrary
          - samples/client/petstore/swift5/x-swift-hashable
        include:
        - os: ubuntu-latest
          sample: samples/client/petstore/swift5/alamofireLibrary
    name: Build Swift samples
    runs-on: ${{ matrix.os }}
    steps:
      - uses: actions/checkout@v4
      - uses: swift-actions/setup-swift@v2
        if: ${{ matrix.os == 'ubuntu-latest' }}
        with:
          swift-version: '5'
      - name: Build
        working-directory: ${{ matrix.sample }}
        run: swift build


TypeScript Axios Client (Echo API)

name: TypeScript Axios Client (Echo API)

on:
  pull_request:
    paths:
      - samples/client/echo_api/typescript-axios/**
      - .github/workflows/samples-typescript-axios-echo-api.yaml
jobs:
  build:
    name: Test TypeScript Axios client
    runs-on: ubuntu-latest
    strategy:
      fail-fast: false
      matrix:
        sample:
          # clients
          - samples/client/echo_api/typescript-axios/test
        node-version:
          - 16
          - 18
          - 20
    steps:
      - uses: actions/checkout@v4

      - uses: actions/setup-node@v4
        with:
          node-version: ${{ matrix.node-version }}

      - name: Run echo server
        run: |
          git clone https://github.com/wing328/http-echo-server -b openapi-generator-test-server
          (cd http-echo-server && git checkout 15a684c41d4de692878636f47fcc1670af309a0f && npm install && npm start &)

      - name: Install
        working-directory: ${{ matrix.sample }}
        run: |
          npm run preinstall
          npm i

      - name: Test
        working-directory: ${{ matrix.sample }}
        run: npm test


Samples Zapier

name: Samples Zapier

on:
  push:
    branches:
      - 'samples/client/petstore/zapier/**'
  pull_request:
    paths:
      - 'samples/client/petstore/zapier/**'

jobs:
  validate:
    name: Test zapier generation
    runs-on: ubuntu-latest
    strategy:
      fail-fast: false
      matrix:
        sample:
          - samples/client/petstore/zapier
    steps:
    - uses: actions/checkout@v4
    - name: Setup node
      uses: actions/setup-node@v4
    - name: Install dependencies
      working-directory: ${{ matrix.sample }}
      run: |
        npm install
    - name: Validate
      working-directory: ${{ matrix.sample }}
      run: node index.js


Windows tests

name: Windows tests

on:
  push:
    branches:
      - master
      - '[5-9]+.[0-9]+.x'
  pull_request:
    branches:
      - master
      - '[5-9]+.[0-9]+.x'

jobs:
  build:
    name: Build
    runs-on: windows-latest
    strategy:
      matrix:
        java: [11, 17]
    steps:
      - uses: actions/checkout@v4
      - name: Set up JDK ${{ matrix.java }}
        uses: actions/setup-java@v4
        with:
          java-version: ${{ matrix.java }}
          distribution: 'temurin'
      - name: Cache maven dependencies
        uses: actions/cache@v4
        env:
          cache-name: cache-maven-repository
        with:
          path: |
            ~/.m2/repository
            ~/.gradle
            !~/.gradle/caches/*/plugin-resolution/
            !~/.m2/repository/org/openapitools/
          key: ${{ runner.os }}-build-${{ env.cache-name }}-${{ hashFiles('**/pom.xml') }}
          restore-keys: |
            ${{ runner.os }}-build-${{ env.cache-name }}-
            ${{ runner.os }}-build-
      - name: Setup Maven
        uses: s4u/setup-maven-action@v1.13.0
        with:
          java-version: ${{ matrix.java }}
          maven-version: 3.8.8
      - name: Run maven
        run: ./mvnw clean --no-snapshot-updates --batch-mode --quiet install
        env:
          GRADLE_ENTERPRISE_ACCESS_KEY: ${{ secrets.GRADLE_ENTERPRISE_ACCESS_KEY }}
      - name: Test Template Retrieval
        run: |
          cd modules/openapi-generator-cli/target
          # generator names containing "-" caused problems in the past, see https://github.com/OpenAPITools/openapi-generator/issues/9528
          java -jar ./openapi-generator-cli.jar author template --verbose  -g jaxrs-spec --library quarkus
      - name: Setup Gradle
        uses: gradle/gradle-build-action@v3
      - name: Gradle tests
        run: |
          gradle -b modules/openapi-generator-gradle-plugin/samples/local-spec/build.gradle buildGoSdk --stacktrace


CI - Integration - Backwards Compatibility


Pulsar Bot

#
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements.  See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership.  The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License.  You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied.  See the License for the
# specific language governing permissions and limitations
# under the License.
#

name: Pulsar Bot
on:
  issue_comment:
    types: [created]

jobs:
  pulsarbot:
    runs-on: ubuntu-22.04
    timeout-minutes: 10
    if: github.event_name == 'issue_comment' && contains(github.event.comment.body, '/pulsarbot')
    steps:
      - name: Execute pulsarbot command
        id: pulsarbot
        env:
          GITHUB_TOKEN: ${{ secrets.PULSARBOT_TOKEN }}
        uses: apache/pulsar-test-infra/pulsarbot@master


Pulsar CI

#
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements.  See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership.  The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License.  You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied.  See the License for the
# specific language governing permissions and limitations
# under the License.
#

name: Pulsar CI
on:
  pull_request:
    branches:
      - master
      - branch-*
      - pulsar-*
  schedule:
    # scheduled job with JDK 17
    - cron: '0 12 * * *'
    # scheduled job with JDK 21
    # if cron expression is changed, make sure to update the expression in jdk_major_version step in preconditions job
    - cron: '0 6 * * *'
  workflow_dispatch:
    inputs:
      collect_coverage:
        description: 'Collect test coverage and upload to Codecov'
        required: true
        type: boolean
        default: true
      jdk_major_version:
        description: 'JDK major version to use for the build'
        required: true
        type: choice
        options:
          - '17'
          - '21'
        default: '17'
      trace_test_resource_cleanup:
        description: 'Collect thread & heap information before exiting a test JVM. When set to "on", thread dump and heap histogram will be collected. When set to "full", a heap dump will also be collected.'
        required: true
        type: choice
        options:
          - 'off'
          - 'on'
          - 'full'
        default: 'off'
      thread_leak_detector_wait_millis:
        description: 'Duration in ms to wait for threads to exit in thread leak detection between test classes. It is necessary to wait for threads to complete before they are determined to be leaked threads.'
        required: true
        type: number
        default: 10000

concurrency:
  group: ${{ github.workflow }}-${{ github.ref }}-${{ github.event_name }}${{ github.event_name == 'workflow_dispatch' && github.event.inputs.jdk_major_version || '' }}
  cancel-in-progress: true

env:
  MAVEN_OPTS: -Xss1500k -Xmx1500m -Daether.connector.http.reuseConnections=false -Daether.connector.requestTimeout=60000 -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false -Dmaven.wagon.http.retryHandler.class=standard -Dmaven.wagon.http.retryHandler.count=3 -Dmaven.wagon.http.retryHandler.requestSentEnabled=true -Dmaven.wagon.http.serviceUnavailableRetryStrategy.class=standard -Dmaven.wagon.rto=60000
  # defines the retention period for the intermediate build artifacts needed for rerunning a failed build job
  # it's possible to rerun individual failed jobs when the build artifacts are available
  # if the artifacts have already been expired, the complete workflow can be rerun by closing and reopening the PR or by rebasing the PR
  ARTIFACT_RETENTION_DAYS: 3
  JDK_DISTRIBUTION: corretto

jobs:
  preconditions:
    name: Preconditions
    runs-on: ubuntu-22.04
    outputs:
      docs_only: ${{ steps.check_changes.outputs.docs_only }}
      changed_tests: ${{ steps.changes.outputs.tests_files }}
      need_owasp: ${{ steps.changes.outputs.need_owasp }}
      collect_coverage: ${{ steps.check_coverage.outputs.collect_coverage }}
      jdk_major_version: ${{ steps.jdk_major_version.outputs.jdk_major_version }}
      java_non_tests: ${{ steps.changes.outputs.java_non_tests }}
    steps:
      - name: Cancel scheduled jobs in forks by default
        if: ${{ github.repository != 'apache/pulsar' && github.event_name == 'schedule' }}
        uses: actions/github-script@v6
        with:
          script: |
            await github.rest.actions.cancelWorkflowRun({owner: context.repo.owner, repo: context.repo.repo, run_id: context.runId});
            process.exit(1);

      - name: Select JDK major version
        id: jdk_major_version
        run: |
          # use JDK 21 for the scheduled build with cron expression '0 6 * * *'
          if [[ "${{ github.event_name == 'schedule' && github.event.schedule == '0 6 * * *' && 'true' || 'false' }}" == "true" ]]; then
            echo "jdk_major_version=21" >> $GITHUB_OUTPUT
            exit 0
          fi
          # use JDK 17 for build unless overridden with workflow_dispatch input
          echo "jdk_major_version=${{ github.event_name == 'workflow_dispatch' && github.event.inputs.jdk_major_version || '17'}}" >> $GITHUB_OUTPUT

      - name: checkout
        if: ${{ github.event_name == 'pull_request' }}
        uses: actions/checkout@v4

      - name: Detect changed files
        if: ${{ github.event_name == 'pull_request' }}
        id:   changes
        uses: apache/pulsar-test-infra/paths-filter@master
        with:
          filters: .github/changes-filter.yaml
          list-files: csv

      - name: Check changed files
        if: ${{ github.event_name == 'pull_request' }}
        id: check_changes
        run: |
          if [[ "${GITHUB_EVENT_NAME}" != "schedule" && "${GITHUB_EVENT_NAME}" != "workflow_dispatch" ]]; then
            echo "docs_only=${{ fromJSON(steps.changes.outputs.all_count) == fromJSON(steps.changes.outputs.docs_count) && fromJSON(steps.changes.outputs.docs_count) > 0 }}" >> $GITHUB_OUTPUT
          else
            echo docs_only=false >> $GITHUB_OUTPUT
          fi

      - name: Check if coverage should be collected
        id: check_coverage
        run: |
          echo "collect_coverage=${{
          (steps.check_changes.outputs.docs_only != 'true' && github.event_name != 'workflow_dispatch'
            && (github.base_ref == 'master' || github.ref_name == 'master'))
          || (github.event_name == 'workflow_dispatch' && github.event.inputs.collect_coverage == 'true')
          }}"  >> $GITHUB_OUTPUT

      - name: Check if the PR has been approved for testing
        if: ${{ steps.check_changes.outputs.docs_only != 'true' && github.repository == 'apache/pulsar' && github.event_name == 'pull_request' }}
        env:
          GITHUB_RUN_ATTEMPT: ${{ github.run_attempt }}
          GITHUB_TOKEN: ${{ github.token }}
        run: |
          build/pulsar_ci_tool.sh check_ready_to_test

  build-and-license-check:
    needs: preconditions
    name: Build and License check
    env:
      JOB_NAME: Build and License check
      GRADLE_ENTERPRISE_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }}
      CI_JDK_MAJOR_VERSION: ${{ needs.preconditions.outputs.jdk_major_version }}
    runs-on: ubuntu-22.04
    timeout-minutes: 60
    if: ${{ needs.preconditions.outputs.docs_only != 'true' }}
    steps:
      - name: checkout
        uses: actions/checkout@v4

      - name: Tune Runner VM
        uses: ./.github/actions/tune-runner-vm

      - name: Setup ssh access to build runner VM
        # ssh access is enabled for builds in own forks
        if: ${{ github.repository != 'apache/pulsar' && github.event_name == 'pull_request' }}
        uses: ./.github/actions/ssh-access
        continue-on-error: true
        with:
          limit-access-to-actor: true

      - name: Cache local Maven repository
        uses: actions/cache@v4
        timeout-minutes: 5
        with:
          path: |
            ~/.m2/repository/*/*/*
            !~/.m2/repository/org/apache/pulsar
          key: ${{ runner.os }}-m2-dependencies-core-modules-${{ hashFiles('**/pom.xml') }}
          restore-keys: |
            ${{ runner.os }}-m2-dependencies-core-modules-

      - name: Set up JDK ${{ env.CI_JDK_MAJOR_VERSION }}
        uses: actions/setup-java@v4
        with:
          distribution: ${{ env.JDK_DISTRIBUTION }}
          java-version: ${{ env.CI_JDK_MAJOR_VERSION }}

      - name: Check source code license headers
        run: mvn -B -T 8 -ntp initialize apache-rat:check license:check

      - name: Check source code style
        run: mvn -B -T 8 -ntp initialize checkstyle:check

      - name: Build core-modules
        run: |
          mvn -B -T 1C -ntp -Pcore-modules,-main clean install -DskipTests -Dlicense.skip=true -Drat.skip=true -Dcheckstyle.skip=true

      - name: Check binary licenses
        run: src/check-binary-license.sh ./distribution/server/target/apache-pulsar-*-bin.tar.gz

      - name: Install gh-actions-artifact-client.js
        uses: apache/pulsar-test-infra/gh-actions-artifact-client/dist@master

      - name: Save maven build results to Github artifact cache so that the results can be reused
        run: |
          cd $HOME
          $GITHUB_WORKSPACE/build/pulsar_ci_tool.sh store_tar_to_github_actions_artifacts pulsar-maven-repository-binaries \
          tar --exclude '.m2/repository/org/apache/pulsar/pulsar-*-distribution' \
              -I zstd -cf - .m2/repository/org/apache/pulsar
          cd $GITHUB_WORKSPACE
          $GITHUB_WORKSPACE/build/pulsar_ci_tool.sh store_tar_to_github_actions_artifacts pulsar-server-distribution \
          tar -I zstd -cf - distribution/server/target/apache-pulsar-*-bin.tar.gz

      - name: Wait for ssh connection when build fails
        # ssh access is enabled for builds in own forks
        uses: ./.github/actions/ssh-access
        if: ${{ failure() && github.repository != 'apache/pulsar' && github.event_name == 'pull_request' }}
        continue-on-error: true
        with:
          action: wait

  unit-tests:
    name: CI - Unit - ${{ matrix.name }}
    env:
      JOB_NAME: CI - Unit - ${{ matrix.name }}
      COLLECT_COVERAGE: "${{ needs.preconditions.outputs.collect_coverage }}"
      GRADLE_ENTERPRISE_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }}
      CI_JDK_MAJOR_VERSION: ${{ needs.preconditions.outputs.jdk_major_version }}
      TRACE_TEST_RESOURCE_CLEANUP: ${{ github.event_name == 'workflow_dispatch' && github.event.inputs.trace_test_resource_cleanup || 'off' }}
      TRACE_TEST_RESOURCE_CLEANUP_DIR: ${{ github.workspace }}/target/trace-test-resource-cleanup
      THREAD_LEAK_DETECTOR_WAIT_MILLIS: ${{ github.event_name == 'workflow_dispatch' && github.event.inputs.thread_leak_detector_wait_millis || 10000 }}
      THREAD_LEAK_DETECTOR_DIR: ${{ github.workspace }}/target/thread-leak-dumps
    runs-on: ubuntu-22.04
    timeout-minutes: ${{ matrix.timeout || 60 }}
    needs: ['preconditions', 'build-and-license-check']
    if: ${{ needs.preconditions.outputs.docs_only != 'true' }}
    strategy:
      fail-fast: false
      matrix:
        include:
          - name: Other
            group: OTHER
            timeout: 75
          - name: Brokers - Broker Group 1
            group: BROKER_GROUP_1
          - name: Brokers - Broker Group 2
            group: BROKER_GROUP_2
          - name: Brokers - Broker Group 3
            group: BROKER_GROUP_3
          - name: Brokers - Broker Group 4
            group: BROKER_GROUP_4
          - name: Brokers - Client Api
            group: BROKER_CLIENT_API
          - name: Brokers - Client Impl
            group: BROKER_CLIENT_IMPL
          - name: Proxy
            group: PROXY
          - name: Pulsar IO
            group: PULSAR_IO
            timeout: 75
          - name: Pulsar IO - Elastic Search
            group: PULSAR_IO_ELASTIC
          - name: Pulsar IO - Kafka Connect Adaptor
            group: PULSAR_IO_KAFKA_CONNECT
          - name: Pulsar Client
            group: CLIENT
          - name: Pulsar Metadata
            group: METADATA

    steps:
      - name: checkout
        uses: actions/checkout@v4

      - name: Tune Runner VM
        uses: ./.github/actions/tune-runner-vm

      - name: Clean Disk when tracing test resource cleanup
        if: ${{ env.TRACE_TEST_RESOURCE_CLEANUP != 'off' }}
        uses: ./.github/actions/clean-disk

      - name: Setup ssh access to build runner VM
        # ssh access is enabled for builds in own forks
        if: ${{ github.repository != 'apache/pulsar' && github.event_name == 'pull_request' }}
        uses: ./.github/actions/ssh-access
        continue-on-error: true
        with:
          limit-access-to-actor: true

      - name: Cache Maven dependencies
        uses: actions/cache@v4
        timeout-minutes: 5
        with:
          path: |
            ~/.m2/repository/*/*/*
            !~/.m2/repository/org/apache/pulsar
          key: ${{ runner.os }}-m2-dependencies-core-modules-${{ hashFiles('**/pom.xml') }}
          restore-keys: |
            ${{ runner.os }}-m2-dependencies-core-modules-

      - name: Set up JDK ${{ matrix.jdk || env.CI_JDK_MAJOR_VERSION }}
        uses: actions/setup-java@v4
        with:
          distribution: ${{ env.JDK_DISTRIBUTION }}
          java-version: ${{ matrix.jdk || env.CI_JDK_MAJOR_VERSION }}

      - name: Install gh-actions-artifact-client.js
        uses: apache/pulsar-test-infra/gh-actions-artifact-client/dist@master

      - name: Restore maven build results from Github artifact cache
        run: |
          cd $HOME
          $GITHUB_WORKSPACE/build/pulsar_ci_tool.sh restore_tar_from_github_actions_artifacts pulsar-maven-repository-binaries
          $GITHUB_WORKSPACE/build/pulsar_ci_tool.sh snapshot_pulsar_maven_artifacts

      - name: Run setup commands
        run: |
          ${{ matrix.setup }}

      - name: Run unit test group '${{ matrix.group }}'
        run: |
          CHANGED_TESTS="${{ needs.preconditions.outputs.tests_files }}" ./build/run_unit_group.sh ${{ matrix.group }}

      - name: Upload coverage to build artifacts
        if: ${{ needs.preconditions.outputs.collect_coverage == 'true' }}
        run: $GITHUB_WORKSPACE/build/pulsar_ci_tool.sh upload_unittest_coverage_files ${{ matrix.group }}

      - name: print JVM thread dumps when cancelled
        if: cancelled()
        run: $GITHUB_WORKSPACE/build/pulsar_ci_tool.sh print_thread_dumps

      - name: Aggregates all test reports to ./test-reports and ./surefire-reports directories
        if: ${{ always() }}
        uses: ./.github/actions/copy-test-reports

      - name: Publish Test Report
        uses: apache/pulsar-test-infra/action-junit-report@master
        if: ${{ always() }}
        with:
          report_paths: 'test-reports/TEST-*.xml'
          annotate_only: 'true'

      - name: Report detected thread leaks
        if: ${{ always() }}
        run: |
          if [ -d "$THREAD_LEAK_DETECTOR_DIR" ]; then
            cd "$THREAD_LEAK_DETECTOR_DIR"
            cat threadleak*.txt | awk '/^Summary:/ {print "::warning::" $0 "\n"; next} {print}'
          fi

      - name: Upload Surefire reports
        uses: actions/upload-artifact@v4
        if: ${{ !success() || env.TRACE_TEST_RESOURCE_CLEANUP != 'off' }}
        with:
          name: Unit-${{ matrix.group }}-surefire-reports
          path: surefire-reports
          retention-days: 7

      - name: Upload possible heap dump, core dump or crash files
        uses: actions/upload-artifact@v4
        if: ${{ always() }}
        with:
          name: Unit-${{ matrix.group }}-dumps
          path: |
            /tmp/*.hprof
            **/hs_err_*.log
            **/core.*
            ${{ env.TRACE_TEST_RESOURCE_CLEANUP_DIR }}/*
            ${{ env.THREAD_LEAK_DETECTOR_DIR }}/*
          retention-days: 7
          if-no-files-found: ignore

      - name: Wait for ssh connection when build fails
        # ssh access is enabled for builds in own forks
        uses: ./.github/actions/ssh-access
        if: ${{ failure() && github.repository != 'apache/pulsar' && github.event_name == 'pull_request' }}
        continue-on-error: true
        with:
          action: wait

  unit-tests-upload-coverage:
    name: CI - Unit - Upload Coverage
    runs-on: ubuntu-22.04
    timeout-minutes: 30
    needs: ['preconditions', 'unit-tests']
    env:
      CI_JDK_MAJOR_VERSION: ${{ needs.preconditions.outputs.jdk_major_version }}
    if: ${{ needs.preconditions.outputs.collect_coverage == 'true' }}

    steps:
      - name: checkout
        uses: actions/checkout@v4

      - name: Tune Runner VM
        uses: ./.github/actions/tune-runner-vm

      - name: Setup ssh access to build runner VM
        # ssh access is enabled for builds in own forks
        if: ${{ github.repository != 'apache/pulsar' && github.event_name == 'pull_request' }}
        uses: ./.github/actions/ssh-access
        continue-on-error: true
        with:
          limit-access-to-actor: true

      - name: Cache Maven dependencies
        uses: actions/cache@v4
        timeout-minutes: 5
        with:
          path: |
            ~/.m2/repository/*/*/*
            !~/.m2/repository/org/apache/pulsar
          key: ${{ runner.os }}-m2-dependencies-core-modules-${{ hashFiles('**/pom.xml') }}
          restore-keys: |
            ${{ runner.os }}-m2-dependencies-core-modules-

      - name: Set up JDK ${{ matrix.jdk || env.CI_JDK_MAJOR_VERSION }}
        uses: actions/setup-java@v4
        with:
          distribution: ${{ env.JDK_DISTRIBUTION }}
          java-version: ${{ matrix.jdk || env.CI_JDK_MAJOR_VERSION }}

      - name: Install gh-actions-artifact-client.js
        uses: apache/pulsar-test-infra/gh-actions-artifact-client/dist@master

      - name: Restore maven build results from Github artifact cache
        run: |
          cd $HOME
          $GITHUB_WORKSPACE/build/pulsar_ci_tool.sh restore_tar_from_github_actions_artifacts pulsar-maven-repository-binaries
          $GITHUB_WORKSPACE/build/pulsar_ci_tool.sh snapshot_pulsar_maven_artifacts

      - name: Restore coverage files from build artifacts and create Jacoco reports
        run: |
          $GITHUB_WORKSPACE/build/pulsar_ci_tool.sh restore_unittest_coverage_files
          $GITHUB_WORKSPACE/build/pulsar_ci_tool.sh create_test_coverage_report
          cd $GITHUB_WORKSPACE/target
          zip -qr jacoco_test_coverage_report_unittests.zip jacoco_test_coverage_report || true

      - name: Upload Jacoco report files to build artifacts
        uses: actions/upload-artifact@v4
        with:
          name: Jacoco-coverage-report-unittests
          path: target/jacoco_test_coverage_report_unittests.zip
          retention-days: 3

      - name: Upload to Codecov
        uses: ./.github/actions/upload-coverage
        with:
          flags: unittests

      - name: Delete coverage files from build artifacts
        run: |
          $GITHUB_WORKSPACE/build/pulsar_ci_tool.sh delete_unittest_coverage_files

      - name: Wait for ssh connection when build fails
        # ssh access is enabled for builds in own forks
        uses: ./.github/actions/ssh-access
        if: ${{ failure() && github.repository != 'apache/pulsar' && github.event_name == 'pull_request' }}
        continue-on-error: true
        with:
          action: wait

  pulsar-java-test-image:
    name: Build Pulsar java-test-image docker image
    runs-on: ubuntu-22.04
    timeout-minutes: 60
    needs: ['preconditions', 'build-and-license-check']
    if: ${{ needs.preconditions.outputs.docs_only != 'true'}}
    strategy:
      fail-fast: false
      matrix:
        platform:
          - linux/amd64
          - linux/arm64
    env:
      GRADLE_ENTERPRISE_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }}
      CI_JDK_MAJOR_VERSION: ${{ needs.preconditions.outputs.jdk_major_version }}
      IMAGE_JDK_MAJOR_VERSION: ${{ needs.preconditions.outputs.jdk_major_version }}
    steps:
      - name: checkout
        uses: actions/checkout@v4

      - name: Tune Runner VM
        uses: ./.github/actions/tune-runner-vm

      - name: Setup ssh access to build runner VM
        # ssh access is enabled for builds in own forks
        if: ${{ github.repository != 'apache/pulsar' && github.event_name == 'pull_request' }}
        uses: ./.github/actions/ssh-access
        continue-on-error: true
        with:
          limit-access-to-actor: true

      - name: Cache Maven dependencies
        uses: actions/cache@v4
        timeout-minutes: 5
        with:
          path: |
            ~/.m2/repository/*/*/*
            !~/.m2/repository/org/apache/pulsar
          key: ${{ runner.os }}-m2-dependencies-core-modules-${{ hashFiles('**/pom.xml') }}
          restore-keys: |
            ${{ runner.os }}-m2-dependencies-core-modules-

      - name: Set up JDK ${{ env.CI_JDK_MAJOR_VERSION }}
        uses: actions/setup-java@v4
        with:
          distribution: ${{ env.JDK_DISTRIBUTION }}
          java-version: ${{ env.CI_JDK_MAJOR_VERSION }}

      - name: Install gh-actions-artifact-client.js
        uses: apache/pulsar-test-infra/gh-actions-artifact-client/dist@master

      - name: restore maven build results from Github artifact cache
        run: |
          cd $HOME
          $GITHUB_WORKSPACE/build/pulsar_ci_tool.sh restore_tar_from_github_actions_artifacts pulsar-maven-repository-binaries

      - name: Set up QEMU
        uses: docker/setup-qemu-action@v3
        with:
          platforms: arm64

      - name: Build java-test-image docker image - ${{ matrix.platform }}
        run: |
          # build docker image
          DOCKER_CLI_EXPERIMENTAL=enabled mvn -B -am -pl docker/pulsar,tests/docker-images/java-test-image install -Pcore-modules,-main,integrationTests,docker \
          -Ddocker.platforms=${{ matrix.platform }} \
          -Dmaven.test.skip=true -DskipSourceReleaseAssembly=true \
          -Dspotbugs.skip=true  -Dlicense.skip=true -Dcheckstyle.skip=true -Drat.skip=true

      - name: save docker image apachepulsar/java-test-image:latest to Github artifact cache
        if: ${{ matrix.platform == 'linux/amd64' }}
        run: |
          $GITHUB_WORKSPACE/build/pulsar_ci_tool.sh docker_save_image_to_github_actions_artifacts apachepulsar/java-test-image:latest pulsar-java-test-image

      - name: Wait for ssh connection when build fails
        # ssh access is enabled for builds in own forks
        uses: ./.github/actions/ssh-access
        if: ${{ failure() && github.repository != 'apache/pulsar' && github.event_name == 'pull_request' }}
        continue-on-error: true
        with:
          action: wait

  integration-tests:
    name: CI - Integration - ${{ matrix.name }}
    runs-on: ubuntu-22.04
    timeout-minutes: ${{ matrix.timeout || 60 }}
    needs: ['preconditions', 'pulsar-java-test-image']
    if: ${{ needs.preconditions.outputs.docs_only != 'true' }}
    env:
      JOB_NAME: CI - Integration - ${{ matrix.name }}
      PULSAR_TEST_IMAGE_NAME: apachepulsar/java-test-image:latest
      GRADLE_ENTERPRISE_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }}
      CI_JDK_MAJOR_VERSION: ${{ needs.preconditions.outputs.jdk_major_version }}
    strategy:
      fail-fast: false
      matrix:
        include:
          - name: Backwards Compatibility
            group: BACKWARDS_COMPAT
            no_coverage: true

          - name: Cli
            group: CLI

          - name: Messaging
            group: MESSAGING

          - name: LoadBalance
            group: LOADBALANCE
            no_coverage: true

          - name: Shade on Java 8
            group: SHADE_RUN
            runtime_jdk: 8
            setup: ./build/run_integration_group.sh SHADE_BUILD
            no_coverage: true

          - name: Shade on Java 11
            group: SHADE_RUN
            runtime_jdk: 11
            setup: ./build/run_integration_group.sh SHADE_BUILD
            no_coverage: true

          - name: Shade on Java 17
            group: SHADE_RUN
            runtime_jdk: 17
            setup: ./build/run_integration_group.sh SHADE_BUILD
            no_coverage: true

          - name: Shade on Java 21
            group: SHADE_RUN
            runtime_jdk: 21
            setup: ./build/run_integration_group.sh SHADE_BUILD
            no_coverage: true

          - name: Standalone
            group: STANDALONE

          - name: Transaction
            group: TRANSACTION

          - name: Metrics
            group: METRICS

    steps:
      - name: checkout
        uses: actions/checkout@v4

      - name: Tune Runner VM
        uses: ./.github/actions/tune-runner-vm

      - name: Setup ssh access to build runner VM
        # ssh access is enabled for builds in own forks
        if: ${{ github.repository != 'apache/pulsar' && github.event_name == 'pull_request' }}
        uses: ./.github/actions/ssh-access
        continue-on-error: true
        with:
          limit-access-to-actor: true

      - name: Cache Maven dependencies
        uses: actions/cache@v4
        timeout-minutes: 5
        with:
          path: |
            ~/.m2/repository/*/*/*
            !~/.m2/repository/org/apache/pulsar
          key: ${{ runner.os }}-m2-dependencies-core-modules-${{ hashFiles('**/pom.xml') }}
          restore-keys: |
            ${{ runner.os }}-m2-dependencies-core-modules-

      - name: Set up JDK ${{ env.CI_JDK_MAJOR_VERSION }}
        uses: actions/setup-java@v4
        with:
          distribution: ${{ env.JDK_DISTRIBUTION }}
          java-version: ${{ env.CI_JDK_MAJOR_VERSION }}

      - name: Install gh-actions-artifact-client.js
        uses: apache/pulsar-test-infra/gh-actions-artifact-client/dist@master

      - name: Restore maven build results from Github artifact cache
        run: |
          cd $HOME
          $GITHUB_WORKSPACE/build/pulsar_ci_tool.sh restore_tar_from_github_actions_artifacts pulsar-maven-repository-binaries
          $GITHUB_WORKSPACE/build/pulsar_ci_tool.sh snapshot_pulsar_maven_artifacts

      - name: Load docker image apachepulsar/java-test-image:latest from Github artifact cache
        run: |
          $GITHUB_WORKSPACE/build/pulsar_ci_tool.sh docker_load_image_from_github_actions_artifacts pulsar-java-test-image

      - name: Run setup commands
        if: ${{ matrix.setup }}
        run: |
          ${{ matrix.setup }}

      - name: Set up runtime JDK ${{ matrix.runtime_jdk }}
        uses: actions/setup-java@v4
        if: ${{ matrix.runtime_jdk }}
        with:
          distribution: ${{ env.JDK_DISTRIBUTION }}
          java-version: ${{ matrix.runtime_jdk }}

      - name: Run integration test group '${{ matrix.group }}'
        run: |
          if [[ "${{ matrix.no_coverage }}" != "true" && "${{ needs.preconditions.outputs.collect_coverage }}" == "true" ]]; then
            coverage_args="--coverage"
          fi
          ./build/run_integration_group.sh ${{ matrix.group }} $coverage_args

      - name: Upload coverage to build artifacts
        if: ${{ !matrix.no_coverage && needs.preconditions.outputs.collect_coverage == 'true' }}
        run: $GITHUB_WORKSPACE/build/pulsar_ci_tool.sh upload_inttest_coverage_files ${{ matrix.group }}

      - name: print JVM thread dumps when cancelled
        if: cancelled()
        run: $GITHUB_WORKSPACE/build/pulsar_ci_tool.sh print_thread_dumps

      - name: Aggregates all test reports to ./test-reports and ./surefire-reports directories
        if: ${{ always() }}
        uses: ./.github/actions/copy-test-reports

      - name: Publish Test Report
        uses: apache/pulsar-test-infra/action-junit-report@master
        if: ${{ always() }}
        with:
          report_paths: 'test-reports/TEST-*.xml'
          annotate_only: 'true'

      - name: Upload Surefire reports
        uses: actions/upload-artifact@v4
        if: ${{ !success() }}
        with:
          name: Integration-${{ matrix.group }}-surefire-reports
          path: surefire-reports
          retention-days: 7

      - name: Upload container logs
        uses: actions/upload-artifact@v4
        if: ${{ !success() }}
        continue-on-error: true
        with:
          name: Integration-${{ matrix.group }}-container-logs
          path: tests/integration/target/container-logs
          retention-days: 7

      - name: Wait for ssh connection when build fails
        # ssh access is enabled for builds in own forks
        uses: ./.github/actions/ssh-access
        if: ${{ failure() && github.repository != 'apache/pulsar' && github.event_name == 'pull_request' }}
        continue-on-error: true
        with:
          action: wait

  integration-tests-upload-coverage:
    name: CI - Integration - Upload Coverage
    runs-on: ubuntu-22.04
    timeout-minutes: 30
    needs: ['preconditions', 'integration-tests']
    if: ${{ needs.preconditions.outputs.collect_coverage == 'true' }}
    env:
      PULSAR_TEST_IMAGE_NAME: apachepulsar/java-test-image:latest
      CI_JDK_MAJOR_VERSION: ${{ needs.preconditions.outputs.jdk_major_version }}
    steps:
      - name: checkout
        uses: actions/checkout@v4

      - name: Tune Runner VM
        uses: ./.github/actions/tune-runner-vm

      - name: Setup ssh access to build runner VM
        # ssh access is enabled for builds in own forks
        if: ${{ github.repository != 'apache/pulsar' && github.event_name == 'pull_request' }}
        uses: ./.github/actions/ssh-access
        continue-on-error: true
        with:
          limit-access-to-actor: true

      - name: Cache Maven dependencies
        uses: actions/cache@v4
        timeout-minutes: 5
        with:
          path: |
            ~/.m2/repository/*/*/*
            !~/.m2/repository/org/apache/pulsar
          key: ${{ runner.os }}-m2-dependencies-core-modules-${{ hashFiles('**/pom.xml') }}
          restore-keys: |
            ${{ runner.os }}-m2-dependencies-core-modules-

      - name: Set up JDK ${{ env.CI_JDK_MAJOR_VERSION }}
        uses: actions/setup-java@v4
        with:
          distribution: ${{ env.JDK_DISTRIBUTION }}
          java-version: ${{ env.CI_JDK_MAJOR_VERSION }}

      - name: Install gh-actions-artifact-client.js
        uses: apache/pulsar-test-infra/gh-actions-artifact-client/dist@master

      - name: Restore maven build results from Github artifact cache
        run: |
          cd $HOME
          $GITHUB_WORKSPACE/build/pulsar_ci_tool.sh restore_tar_from_github_actions_artifacts pulsar-maven-repository-binaries
          $GITHUB_WORKSPACE/build/pulsar_ci_tool.sh snapshot_pulsar_maven_artifacts

      - name: Load docker image apachepulsar/java-test-image:latest from Github artifact cache
        run: |
          $GITHUB_WORKSPACE/build/pulsar_ci_tool.sh docker_load_image_from_github_actions_artifacts pulsar-java-test-image

      - name: Restore coverage files from build artifacts and create Jacoco reports
        run: |
          $GITHUB_WORKSPACE/build/pulsar_ci_tool.sh restore_inttest_coverage_files
          $GITHUB_WORKSPACE/build/pulsar_ci_tool.sh create_inttest_coverage_report
          $GITHUB_WORKSPACE/build/pulsar_ci_tool.sh create_test_coverage_report
          cd $GITHUB_WORKSPACE/target
          zip -qr jacoco_test_coverage_report_inttests.zip jacoco_test_coverage_report jacoco_inttest_coverage_report || true

      - name: Upload Jacoco report files to build artifacts
        uses: actions/upload-artifact@v4
        with:
          name: Jacoco-coverage-report-inttests
          path: target/jacoco_test_coverage_report_inttests.zip
          retention-days: 3

      - name: Upload to Codecov
        uses: ./.github/actions/upload-coverage
        with:
          flags: inttests

      - name: Delete coverage files from build artifacts
        run: |
          $GITHUB_WORKSPACE/build/pulsar_ci_tool.sh delete_inttest_coverage_files

      - name: Wait for ssh connection when build fails
        # ssh access is enabled for builds in own forks
        uses: ./.github/actions/ssh-access
        if: ${{ failure() && github.repository != 'apache/pulsar' && github.event_name == 'pull_request' }}
        continue-on-error: true
        with:
          action: wait

  delete-integration-test-docker-image-artifact:
    name: "Delete integration test docker image artifact"
    runs-on: ubuntu-22.04
    timeout-minutes: 10
    needs: [
      'preconditions',
      'integration-tests',
      'integration-tests-upload-coverage'
    ]
    if: ${{ needs.preconditions.outputs.docs_only != 'true' }}
    steps:
      - name: checkout
        uses: actions/checkout@v4

      - name: Tune Runner VM
        uses: ./.github/actions/tune-runner-vm

      - name: Install gh-actions-artifact-client.js
        uses: apache/pulsar-test-infra/gh-actions-artifact-client/dist@master

      - name: Delete docker image from GitHub Actions Artifacts
        run: |
          gh-actions-artifact-client.js delete pulsar-java-test-image.zst

  pulsar-test-latest-version-image:
    name: Build Pulsar docker image
    runs-on: ubuntu-22.04
    timeout-minutes: 60
    needs: ['preconditions', 'build-and-license-check']
    if: ${{ needs.preconditions.outputs.docs_only != 'true' }}
    env:
      GRADLE_ENTERPRISE_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }}
      CI_JDK_MAJOR_VERSION: ${{ needs.preconditions.outputs.jdk_major_version }}
      IMAGE_JDK_MAJOR_VERSION: ${{ needs.preconditions.outputs.jdk_major_version }}
    steps:
      - name: checkout
        uses: actions/checkout@v4

      - name: Tune Runner VM
        uses: ./.github/actions/tune-runner-vm

      - name: Setup ssh access to build runner VM
        # ssh access is enabled for builds in own forks
        if: ${{ github.repository != 'apache/pulsar' && github.event_name == 'pull_request' }}
        uses: ./.github/actions/ssh-access
        continue-on-error: true
        with:
          limit-access-to-actor: true

      - name: Clean Disk
        uses: ./.github/actions/clean-disk
        with:
          mode: full

      - name: Cache local Maven repository
        uses: actions/cache@v4
        timeout-minutes: 5
        with:
          path: |
            ~/.m2/repository/*/*/*
            !~/.m2/repository/org/apache/pulsar
          key: ${{ runner.os }}-m2-dependencies-all-${{ hashFiles('**/pom.xml') }}
          restore-keys: |
            ${{ runner.os }}-m2-dependencies-core-modules-${{ hashFiles('**/pom.xml') }}
            ${{ runner.os }}-m2-dependencies-core-modules-

      - name: Set up JDK ${{ env.CI_JDK_MAJOR_VERSION }}
        uses: actions/setup-java@v4
        with:
          distribution: ${{ env.JDK_DISTRIBUTION }}
          java-version: ${{ env.CI_JDK_MAJOR_VERSION }}

      - name: Install gh-actions-artifact-client.js
        uses: apache/pulsar-test-infra/gh-actions-artifact-client/dist@master

      - name: restore maven build results from Github artifact cache
        run: |
          cd $HOME
          $GITHUB_WORKSPACE/build/pulsar_ci_tool.sh restore_tar_from_github_actions_artifacts pulsar-maven-repository-binaries

      - name: Build latest-version-image docker image
        run: |
          # build docker image
          # include building of Connectors, Offloaders and server distros
          DOCKER_CLI_EXPERIMENTAL=enabled mvn -B -am -pl distribution/io,distribution/offloaders,distribution/server,distribution/shell,tests/docker-images/latest-version-image install \
          -Pmain,docker -Dmaven.test.skip=true \
          -Dspotbugs.skip=true -Dlicense.skip=true -Dcheckstyle.skip=true -Drat.skip=true

      # check full build artifacts licenses
      - name: Check binary licenses
        run: src/check-binary-license.sh ./distribution/server/target/apache-pulsar-*-bin.tar.gz && src/check-binary-license.sh ./distribution/shell/target/apache-pulsar-shell-*-bin.tar.gz

      - name: Run Trivy container scan
        uses: aquasecurity/trivy-action@master
        if: ${{ github.repository == 'apache/pulsar' && github.event_name != 'pull_request' }}
        with:
          image-ref: "apachepulsar/pulsar:latest"
          scanners: vuln
          severity: CRITICAL,HIGH,MEDIUM,LOW
          limit-severities-for-sarif: true
          format: 'sarif'
          output: 'trivy-results.sarif'

      - name: Upload Trivy scan results to GitHub Security tab
        uses: github/codeql-action/upload-sarif@v3
        if: ${{ github.repository == 'apache/pulsar' && github.event_name != 'pull_request' }}
        with:
          sarif_file: 'trivy-results.sarif'

      - name: Clean up disk space
        run: |
          # release disk space since saving docker image consumes local disk space
          #
          echo "::group::Available diskspace before cleaning"
          time df -BM / /mnt
          echo "::endgroup::"
          echo "::group::Clean build directory"
          # docker build changes some files to root ownership, fix this before deleting files
          sudo chown -R $USER:$GROUP .
          # clean build directories
          time git clean -fdx
          echo "::endgroup::"
          echo "::group::Available diskspace after cleaning build directory"
          time df -BM / /mnt
          echo "::endgroup::"
          echo "::group::Delete maven repository"
          # delete maven repository
          time rm -rf ~/.m2/repository
          echo "::endgroup::"
          echo "::group::Available diskspace after cleaning maven repository"
          time df -BM / /mnt
          echo "::endgroup::"

      - name: save docker image apachepulsar/pulsar-test-latest-version:latest to Github artifact cache
        run: |
          $GITHUB_WORKSPACE/build/pulsar_ci_tool.sh docker_save_image_to_github_actions_artifacts apachepulsar/pulsar-test-latest-version:latest pulsar-test-latest-version-image

      - name: Wait for ssh connection when build fails
        # ssh access is enabled for builds in own forks
        uses: ./.github/actions/ssh-access
        if: ${{ failure() && github.repository != 'apache/pulsar' && github.event_name == 'pull_request' }}
        continue-on-error: true
        with:
          action: wait

  system-tests:
    name: CI - System - ${{ matrix.name }}
    runs-on: ubuntu-22.04
    timeout-minutes: 60
    needs: ['preconditions', 'pulsar-test-latest-version-image']
    if: ${{ needs.preconditions.outputs.docs_only != 'true' }}
    env:
      JOB_NAME: CI - System - ${{ matrix.name }}
      PULSAR_TEST_IMAGE_NAME: apachepulsar/pulsar-test-latest-version:latest
      GRADLE_ENTERPRISE_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }}
      CI_JDK_MAJOR_VERSION: ${{ needs.preconditions.outputs.jdk_major_version }}
    strategy:
      fail-fast: false
      matrix:
        include:
          - name: Tiered FileSystem
            group: TIERED_FILESYSTEM

          - name: Tiered JCloud
            group: TIERED_JCLOUD

          - name: Function
            group: FUNCTION

          - name: Schema
            group: SCHEMA

          - name: Pulsar Connectors - Thread
            group: PULSAR_CONNECTORS_THREAD

          - name: Pulsar Connectors - Process
            group: PULSAR_CONNECTORS_PROCESS

          - name: Pulsar IO
            group: PULSAR_IO
            clean_disk: true

    steps:
      - name: checkout
        uses: actions/checkout@v4

      - name: Tune Runner VM
        uses: ./.github/actions/tune-runner-vm

      - name: Clean Disk when needed
        if: ${{ matrix.clean_disk }}
        uses: ./.github/actions/clean-disk

      - name: Setup ssh access to build runner VM
        # ssh access is enabled for builds in own forks
        if: ${{ github.repository != 'apache/pulsar' && github.event_name == 'pull_request' }}
        uses: ./.github/actions/ssh-access
        continue-on-error: true
        with:
          limit-access-to-actor: true

      - name: Cache local Maven repository
        uses: actions/cache@v4
        timeout-minutes: 5
        with:
          path: |
            ~/.m2/repository/*/*/*
            !~/.m2/repository/org/apache/pulsar
          key: ${{ runner.os }}-m2-dependencies-all-${{ hashFiles('**/pom.xml') }}
          restore-keys: |
            ${{ runner.os }}-m2-dependencies-core-modules-${{ hashFiles('**/pom.xml') }}
            ${{ runner.os }}-m2-dependencies-core-modules-

      - name: Set up JDK ${{ env.CI_JDK_MAJOR_VERSION }}
        uses: actions/setup-java@v4
        with:
          distribution: ${{ env.JDK_DISTRIBUTION }}
          java-version: ${{ env.CI_JDK_MAJOR_VERSION }}

      - name: Install gh-actions-artifact-client.js
        uses: apache/pulsar-test-infra/gh-actions-artifact-client/dist@master

      - name: Restore maven build results from Github artifact cache
        run: |
          cd $HOME
          $GITHUB_WORKSPACE/build/pulsar_ci_tool.sh restore_tar_from_github_actions_artifacts pulsar-maven-repository-binaries
          $GITHUB_WORKSPACE/build/pulsar_ci_tool.sh snapshot_pulsar_maven_artifacts

      - name: Load docker image apachepulsar/pulsar-test-latest-version:latest from Github artifact cache
        run: |
          $GITHUB_WORKSPACE/build/pulsar_ci_tool.sh docker_load_image_from_github_actions_artifacts pulsar-test-latest-version-image

      - name: Run setup commands
        if: ${{ matrix.setup }}
        run: |
          ${{ matrix.setup }}

      - name: Run system test group '${{ matrix.group }}'
        run: |
          if [[ "${{ matrix.no_coverage }}" != "true" && "${{ needs.preconditions.outputs.collect_coverage }}" == "true" ]]; then
            coverage_args="--coverage"
          fi
          ./build/run_integration_group.sh ${{ matrix.group }} $coverage_args

      - name: Upload coverage to build artifacts
        if: ${{ !matrix.no_coverage && needs.preconditions.outputs.collect_coverage == 'true' }}
        run: $GITHUB_WORKSPACE/build/pulsar_ci_tool.sh upload_systest_coverage_files ${{ matrix.group }}

      - name: print JVM thread dumps when cancelled
        if: cancelled()
        run: $GITHUB_WORKSPACE/build/pulsar_ci_tool.sh print_thread_dumps

      - name: Aggregates all test reports to ./test-reports and ./surefire-reports directories
        if: ${{ always() }}
        uses: ./.github/actions/copy-test-reports

      - name: Publish Test Report
        uses: apache/pulsar-test-infra/action-junit-report@master
        if: ${{ always() }}
        with:
          report_paths: 'test-reports/TEST-*.xml'
          annotate_only: 'true'

      - name: Upload container logs
        uses: actions/upload-artifact@v4
        if: ${{ !success() }}
        continue-on-error: true
        with:
          name: System-${{ matrix.group }}-container-logs
          path: tests/integration/target/container-logs
          retention-days: 7

      - name: Upload Surefire reports
        uses: actions/upload-artifact@v4
        if: ${{ !success() }}
        with:
          name: System-${{ matrix.name }}-surefire-reports
          path: surefire-reports
          retention-days: 7

      - name: Wait for ssh connection when build fails
        # ssh access is enabled for builds in own forks
        uses: ./.github/actions/ssh-access
        if: ${{ failure() && github.repository != 'apache/pulsar' && github.event_name == 'pull_request' }}
        continue-on-error: true
        with:
          action: wait

  system-tests-upload-coverage:
    name: CI - System - Upload Coverage
    runs-on: ubuntu-22.04
    timeout-minutes: 30
    needs: ['preconditions', 'system-tests']
    if: ${{ needs.preconditions.outputs.collect_coverage == 'true' }}
    env:
      PULSAR_TEST_IMAGE_NAME: apachepulsar/pulsar-test-latest-version:latest
      CI_JDK_MAJOR_VERSION: ${{ needs.preconditions.outputs.jdk_major_version }}

    steps:
      - name: checkout
        uses: actions/checkout@v4

      - name: Tune Runner VM
        uses: ./.github/actions/tune-runner-vm

      - name: Setup ssh access to build runner VM
        # ssh access is enabled for builds in own forks
        if: ${{ github.repository != 'apache/pulsar' && github.event_name == 'pull_request' }}
        uses: ./.github/actions/ssh-access
        continue-on-error: true
        with:
          limit-access-to-actor: true

      - name: Cache local Maven repository
        uses: actions/cache@v4
        timeout-minutes: 5
        with:
          path: |
            ~/.m2/repository/*/*/*
            !~/.m2/repository/org/apache/pulsar
          key: ${{ runner.os }}-m2-dependencies-all-${{ hashFiles('**/pom.xml') }}
          restore-keys: |
            ${{ runner.os }}-m2-dependencies-core-modules-${{ hashFiles('**/pom.xml') }}
            ${{ runner.os }}-m2-dependencies-core-modules-

      - name: Set up JDK ${{ env.CI_JDK_MAJOR_VERSION }}
        uses: actions/setup-java@v4
        with:
          distribution: ${{ env.JDK_DISTRIBUTION }}
          java-version: ${{ env.CI_JDK_MAJOR_VERSION }}

      - name: Install gh-actions-artifact-client.js
        uses: apache/pulsar-test-infra/gh-actions-artifact-client/dist@master

      - name: Restore maven build results from Github artifact cache
        run: |
          cd $HOME
          $GITHUB_WORKSPACE/build/pulsar_ci_tool.sh restore_tar_from_github_actions_artifacts pulsar-maven-repository-binaries

      - name: Load docker image apachepulsar/pulsar-test-latest-version:latest from Github artifact cache
        run: |
          $GITHUB_WORKSPACE/build/pulsar_ci_tool.sh docker_load_image_from_github_actions_artifacts pulsar-test-latest-version-image

      - name: Restore coverage files from build artifacts and create Jacoco reports
        run: |
          $GITHUB_WORKSPACE/build/pulsar_ci_tool.sh restore_systest_coverage_files
          $GITHUB_WORKSPACE/build/pulsar_ci_tool.sh create_inttest_coverage_report
          $GITHUB_WORKSPACE/build/pulsar_ci_tool.sh create_test_coverage_report
          cd $GITHUB_WORKSPACE/target
          zip -qr jacoco_test_coverage_report_systests.zip jacoco_test_coverage_report jacoco_inttest_coverage_report || true

      - name: Upload Jacoco report files to build artifacts
        uses: actions/upload-artifact@v4
        with:
          name: Jacoco-coverage-report-systests
          path: target/jacoco_test_coverage_report_systests.zip
          retention-days: 3

      - name: Upload to Codecov
        uses: ./.github/actions/upload-coverage
        with:
          flags: systests

      - name: Delete coverage files from build artifacts
        run: |
          $GITHUB_WORKSPACE/build/pulsar_ci_tool.sh delete_systest_coverage_files

      - name: Wait for ssh connection when build fails
        # ssh access is enabled for builds in own forks
        uses: ./.github/actions/ssh-access
        if: ${{ failure() && github.repository != 'apache/pulsar' && github.event_name == 'pull_request' }}
        continue-on-error: true
        with:
          action: wait

  flaky-system-tests:
    name: CI Flaky - System - ${{ matrix.name }}
    runs-on: ubuntu-22.04
    timeout-minutes: 60
    needs: [ 'preconditions', 'pulsar-test-latest-version-image' ]
    if: ${{ needs.preconditions.outputs.docs_only != 'true' }}
    env:
      JOB_NAME: CI Flaky - System - ${{ matrix.name }}
      PULSAR_TEST_IMAGE_NAME: apachepulsar/pulsar-test-latest-version:latest
      GRADLE_ENTERPRISE_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }}
      CI_JDK_MAJOR_VERSION: ${{ needs.preconditions.outputs.jdk_major_version }}
    strategy:
      fail-fast: false
      matrix:
        include:
          - name: Plugin
            group: PLUGIN

          - name: Pulsar IO - Oracle
            group: PULSAR_IO_ORA
            clean_disk: true

    steps:
      - name: checkout
        uses: actions/checkout@v4

      - name: Tune Runner VM
        uses: ./.github/actions/tune-runner-vm

      - name: Clean Disk when needed
        if: ${{ matrix.clean_disk }}
        uses: ./.github/actions/clean-disk

      - name: Setup ssh access to build runner VM
        # ssh access is enabled for builds in own forks
        if: ${{ github.repository != 'apache/pulsar' && github.event_name == 'pull_request' }}
        uses: ./.github/actions/ssh-access
        continue-on-error: true
        with:
          limit-access-to-actor: true

      - name: Cache local Maven repository
        uses: actions/cache@v4
        timeout-minutes: 5
        with:
          path: |
            ~/.m2/repository/*/*/*
            !~/.m2/repository/org/apache/pulsar
          key: ${{ runner.os }}-m2-dependencies-all-${{ hashFiles('**/pom.xml') }}
          restore-keys: |
            ${{ runner.os }}-m2-dependencies-core-modules-${{ hashFiles('**/pom.xml') }}
            ${{ runner.os }}-m2-dependencies-core-modules-

      - name: Set up JDK ${{ env.CI_JDK_MAJOR_VERSION }}
        uses: actions/setup-java@v4
        with:
          distribution: ${{ env.JDK_DISTRIBUTION }}
          java-version: ${{ env.CI_JDK_MAJOR_VERSION }}

      - name: Install gh-actions-artifact-client.js
        uses: apache/pulsar-test-infra/gh-actions-artifact-client/dist@master

      - name: Restore maven build results from Github artifact cache
        run: |
          cd $HOME
          $GITHUB_WORKSPACE/build/pulsar_ci_tool.sh restore_tar_from_github_actions_artifacts pulsar-maven-repository-binaries

      - name: Load docker image apachepulsar/pulsar-test-latest-version:latest from Github artifact cache
        run: |
          $GITHUB_WORKSPACE/build/pulsar_ci_tool.sh docker_load_image_from_github_actions_artifacts pulsar-test-latest-version-image

      - name: Run setup commands
        if: ${{ matrix.setup }}
        run: |
          ${{ matrix.setup }}

      - name: Run system test group '${{ matrix.group }}'
        run: |
          ./build/run_integration_group.sh ${{ matrix.group }}

      - name: print JVM thread dumps when cancelled
        if: cancelled()
        run: $GITHUB_WORKSPACE/build/pulsar_ci_tool.sh print_thread_dumps

      - name: Aggregates all test reports to ./test-reports and ./surefire-reports directories
        if: ${{ always() }}
        uses: ./.github/actions/copy-test-reports

      - name: Publish Test Report
        uses: apache/pulsar-test-infra/action-junit-report@master
        if: ${{ always() }}
        with:
          report_paths: 'test-reports/TEST-*.xml'
          annotate_only: 'true'

      - name: Upload container logs
        uses: actions/upload-artifact@v4
        if: ${{ !success() }}
        continue-on-error: true
        with:
          name: System-${{ matrix.group }}-container-logs
          path: tests/integration/target/container-logs
          retention-days: 7

      - name: Upload Surefire reports
        uses: actions/upload-artifact@v4
        if: ${{ !success() }}
        with:
          name: System-${{ matrix.name }}-surefire-reports
          path: surefire-reports
          retention-days: 7

      - name: Wait for ssh connection when build fails
        # ssh access is enabled for builds in own forks
        uses: ./.github/actions/ssh-access
        if: ${{ failure() && github.repository != 'apache/pulsar' && github.event_name == 'pull_request' }}
        continue-on-error: true
        with:
          action: wait

  delete-system-test-docker-image-artifact:
    name: "Delete system test docker image artifact"
    runs-on: ubuntu-22.04
    timeout-minutes: 10
    needs: [
      'preconditions',
      'system-tests',
      'system-tests-upload-coverage',
      'flaky-system-tests'
    ]
    if: ${{ needs.preconditions.outputs.docs_only != 'true' }}
    steps:
      - name: checkout
        uses: actions/checkout@v4

      - name: Tune Runner VM
        uses: ./.github/actions/tune-runner-vm

      - name: Install gh-actions-artifact-client.js
        uses: apache/pulsar-test-infra/gh-actions-artifact-client/dist@master

      - name: Delete docker image from GitHub Actions Artifacts
        run: |
          gh-actions-artifact-client.js delete pulsar-test-latest-version-image.zst

  macos-build:
    name: Build Pulsar on MacOS
    runs-on: macos-11
    timeout-minutes: 120
    needs: ['preconditions', 'integration-tests']
    if: ${{ needs.preconditions.outputs.docs_only != 'true' }}
    env:
      GRADLE_ENTERPRISE_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }}
      CI_JDK_MAJOR_VERSION: ${{ needs.preconditions.outputs.jdk_major_version }}
    steps:
      - name: checkout
        uses: actions/checkout@v4

      - name: Tune Runner VM
        uses: ./.github/actions/tune-runner-vm

      - name: Cache Maven dependencies
        uses: actions/cache@v4
        timeout-minutes: 5
        with:
          path: |
            ~/.m2/repository/*/*/*
            !~/.m2/repository/org/apache/pulsar
          key: ${{ runner.os }}-m2-dependencies-all-${{ hashFiles('**/pom.xml') }}
          restore-keys: |
            ${{ runner.os }}-m2-dependencies-all-

      - name: Set up JDK ${{ env.CI_JDK_MAJOR_VERSION }}
        uses: actions/setup-java@v4
        with:
          distribution: ${{ env.JDK_DISTRIBUTION }}
          java-version: ${{ env.CI_JDK_MAJOR_VERSION }}

      - name: build package
        run: mvn -B clean package -DskipTests -T 1C -ntp

  codeql:
    name: Run CodeQL Analysis
    runs-on: ubuntu-22.04
    timeout-minutes: 60
    needs: ['preconditions', 'unit-tests']
    if: ${{ (needs.preconditions.outputs.java_non_tests == 'true' || github.event_name != 'pull_request') && ((github.event_name == 'pull_request' && github.base_ref == 'master') || (github.event_name != 'pull_request' && github.ref_name == 'master')) }}
    permissions:
      actions: read
      contents: read
      security-events: write
    env:
      GRADLE_ENTERPRISE_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }}
      CI_JDK_MAJOR_VERSION: ${{ needs.preconditions.outputs.jdk_major_version }}
      CODEQL_LANGUAGE: java-kotlin
    steps:
      - name: checkout
        uses: actions/checkout@v4

      - name: Tune Runner VM
        uses: ./.github/actions/tune-runner-vm

      - name: Clean Disk when needed
        if: ${{ matrix.clean_disk }}
        uses: ./.github/actions/clean-disk

      - name: Setup ssh access to build runner VM
        # ssh access is enabled for builds in own forks
        if: ${{ github.repository != 'apache/pulsar' && github.event_name == 'pull_request' }}
        uses: ./.github/actions/ssh-access
        continue-on-error: true
        with:
          limit-access-to-actor: true

      - name: Cache local Maven repository
        uses: actions/cache@v4
        timeout-minutes: 5
        with:
          path: |
            ~/.m2/repository/*/*/*
            !~/.m2/repository/org/apache/pulsar
          key: ${{ runner.os }}-m2-dependencies-all-${{ hashFiles('**/pom.xml') }}
          restore-keys: |
            ${{ runner.os }}-m2-dependencies-core-modules-${{ hashFiles('**/pom.xml') }}
            ${{ runner.os }}-m2-dependencies-core-modules-

      - name: Set up JDK ${{ env.CI_JDK_MAJOR_VERSION }}
        uses: actions/setup-java@v4
        with:
          distribution: ${{ env.JDK_DISTRIBUTION }}
          java-version: ${{ env.CI_JDK_MAJOR_VERSION }}

      - name: Initialize CodeQL
        uses: github/codeql-action/init@v3
        with:
          languages: ${{ env.CODEQL_LANGUAGE }}

      - name: Build Java code
        run: |
          mvn -B -ntp -Pcore-modules,-main install -DskipTests -Dlicense.skip=true -Drat.skip=true -Dcheckstyle.skip=true

      - name: Perform CodeQL Analysis
        uses: github/codeql-action/analyze@v3
        with:
          category: "/language:${{ env.CODEQL_LANGUAGE }}"

  owasp-dep-check:
    name: OWASP dependency check
    runs-on: ubuntu-22.04
    timeout-minutes: 120
    needs: [ 'preconditions', 'integration-tests' ]
    if: ${{ needs.preconditions.outputs.need_owasp == 'true' }}
    env:
      GRADLE_ENTERPRISE_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }}
      CI_JDK_MAJOR_VERSION: ${{ needs.preconditions.outputs.jdk_major_version }}
    steps:
      - name: checkout
        uses: actions/checkout@v4

      - name: Tune Runner VM
        uses: ./.github/actions/tune-runner-vm

      - name: Setup ssh access to build runner VM
        # ssh access is enabled for builds in own forks
        if: ${{ github.repository != 'apache/pulsar' && github.event_name == 'pull_request' }}
        uses: ./.github/actions/ssh-access
        continue-on-error: true
        with:
          limit-access-to-actor: true

      - name: Cache Maven dependencies
        uses: actions/cache@v4
        timeout-minutes: 5
        with:
          path: |
            ~/.m2/repository/*/*/*
            !~/.m2/repository/org/apache/pulsar
            !~/.m2/repository/org/owasp/dependency-check-data
          key: ${{ runner.os }}-m2-dependencies-core-modules-${{ hashFiles('**/pom.xml') }}
          lookup-only: true
          restore-keys: |
            ${{ runner.os }}-m2-dependencies-core-modules-

      - name: Set up JDK ${{ matrix.jdk || env.CI_JDK_MAJOR_VERSION }}
        uses: actions/setup-java@v4
        with:
          distribution: ${{ env.JDK_DISTRIBUTION }}
          java-version: ${{ matrix.jdk || env.CI_JDK_MAJOR_VERSION }}

      - name: Clean Disk
        uses: ./.github/actions/clean-disk

      - name: Install gh-actions-artifact-client.js
        uses: apache/pulsar-test-infra/gh-actions-artifact-client/dist@master

      - name: Restore maven build results from Github artifact cache
        run: |
          cd $HOME
          $GITHUB_WORKSPACE/build/pulsar_ci_tool.sh restore_tar_from_github_actions_artifacts pulsar-maven-repository-binaries

      - name: OWASP cache key weeknum
        id: get-weeknum
        run: |
          echo "weeknum=$(date -u +"%Y-%U")" >> $GITHUB_OUTPUT
        shell: bash

      - name: Restore OWASP Dependency Check data
        id: restore-owasp-dependency-check-data
        uses: actions/cache/restore@v3
        timeout-minutes: 5
        with:
          path: ~/.m2/repository/org/owasp/dependency-check-data
          key: owasp-dependency-check-data-${{ steps.get-weeknum.outputs.weeknum }}
          enableCrossOsArchive: true
          restore-keys: |
            owasp-dependency-check-data-

      # Projects dependent on flume, hdfs, and hbase currently excluded from the scan.
      - name: trigger dependency check
        run: |
          mvn -B -ntp verify -PskipDocker,skip-all,owasp-dependency-check -Dcheckstyle.skip=true -DskipTests \
            -pl '!distribution/server,!distribution/io,!distribution/offloaders,!tiered-storage/file-system,!pulsar-io/flume,!pulsar-io/hbase,!pulsar-io/hdfs2,!pulsar-io/hdfs3,!pulsar-io/docs,!pulsar-io/jdbc/openmldb'

      - name: Upload report
        uses: actions/upload-artifact@v4
        if: ${{ cancelled() || failure() }}
        continue-on-error: true
        with:
          name: dependency report
          path: target/dependency-check-report.html

      - name: Wait for ssh connection when build fails
        # ssh access is enabled for builds in own forks
        uses: ./.github/actions/ssh-access
        if: ${{ failure() && github.repository != 'apache/pulsar' && github.event_name == 'pull_request' }}
        continue-on-error: true
        with:
          action: wait

  # This job is required for pulls to be merged. This job is referenced by name in .asf.yaml file in the
  # protected_branches section for master branch required_status_checks.
  # It depends on all other jobs in this workflow.
  # This job also cleans up the binaries at the end of the workflow.
  pulsar-ci-checks-completed:
    name: "Pulsar CI checks completed"
    # run always, but skip for other repositories than apache/pulsar when a scheduled workflow is cancelled
    # this is to allow the workflow scheduled jobs to show as cancelled instead of failed since scheduled
    # jobs are not enabled for other than apache/pulsar repository.
    if: ${{ always() && !(cancelled() && github.repository != 'apache/pulsar' && github.event_name == 'schedule') }}
    runs-on: ubuntu-22.04
    timeout-minutes: 10
    needs: [
      'preconditions',
      'unit-tests',
      'integration-tests',
      'system-tests',
      'flaky-system-tests',
      'macos-build',
      'unit-tests-upload-coverage',
      'integration-tests-upload-coverage',
      'system-tests-upload-coverage',
      'owasp-dep-check',
      'codeql'
    ]
    steps:
      - name: Check that all required jobs were completed successfully
        if: ${{ needs.preconditions.result != 'success' || needs.preconditions.outputs.docs_only != 'true'  }}
        run: |
          if [[ ! ( \
                   "${{ needs.preconditions.result }}" == "success" \
                && "${{ needs.unit-tests.result }}" == "success" \
                && "${{ needs.integration-tests.result }}" == "success" \
                && "${{ needs.system-tests.result }}" == "success" \
                && "${{ needs.macos-build.result }}" == "success" \
                && ( "${{ needs.codeql.result }}" == "success" || "${{ needs.codeql.result }}" == "skipped" ) \
               ) ]]; then
            echo "Required jobs haven't been completed successfully."
            exit 1
          fi

      - name: checkout
        if: ${{ needs.preconditions.outputs.docs_only != 'true' }}
        uses: actions/checkout@v4

      - name: Tune Runner VM
        if: ${{ needs.preconditions.outputs.docs_only != 'true' }}
        uses: ./.github/actions/tune-runner-vm

      - name: Install gh-actions-artifact-client.js
        if: ${{ needs.preconditions.outputs.docs_only != 'true' }}
        uses: apache/pulsar-test-infra/gh-actions-artifact-client/dist@master

      - name: Delete maven repository binaries from GitHub Actions Artifacts
        if: ${{ needs.preconditions.outputs.docs_only != 'true' && !contains(needs.*.result, 'failure') && !contains(needs.*.result, 'cancelled') }}
        run: |
          gh-actions-artifact-client.js delete pulsar-maven-repository-binaries.tar.zst || true
          gh-actions-artifact-client.js delete pulsar-server-distribution.tar.zst || true


CI - Maven Dependency Cache Update

#
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements.  See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership.  The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License.  You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied.  See the License for the
# specific language governing permissions and limitations
# under the License.
#

# This workflow keeps the GitHub Action Caches up-to-date in the repository.
#
# A pull request build cannot update the cache of the upstream repository. Pull
# requests have a cache in the context of the fork of the upstream repository.
# A pull request build will lookup cache entries from the cache of the upstream
# repository in the case that a cache entry is missing in the pull request source
# repository's cache.
# To reduce cache misses for pull request builds, it is necessary that the
# caches in the upstream repository are up-to-date.
# If the cache entry already exists, the cache won't be updated. This will keep the
# update job very efficient and the downloading and updating will only run if one of the pom.xml
# files has been modified or the cache entry expires.
#

name: CI - Maven Dependency Cache Update
on:
  # trigger on every commit to given branches
  push:
    branches:
      - master
  # trigger on a schedule so that the cache will be rebuilt if it happens to expire
  schedule:
    - cron: '30 */12 * * *'

env:
  MAVEN_OPTS: -Xss1500k -Xmx1024m -Daether.connector.http.reuseConnections=false -Daether.connector.requestTimeout=60000 -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false -Dmaven.wagon.http.retryHandler.class=standard -Dmaven.wagon.http.retryHandler.count=3 -Dmaven.wagon.http.retryHandler.requestSentEnabled=true -Dmaven.wagon.http.serviceUnavailableRetryStrategy.class=standard -Dmaven.wagon.rto=60000
  JDK_DISTRIBUTION: corretto

jobs:
  update-maven-dependencies-cache:
    name: Update Maven dependency cache for ${{ matrix.name }}
    env:
      JOB_NAME: Update Maven dependency cache for ${{ matrix.name }}
      GRADLE_ENTERPRISE_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }}
    runs-on: ${{ matrix.runs-on }}
    timeout-minutes: 45

    strategy:
      fail-fast: false
      matrix:
        include:
          - name: all modules
            runs-on: ubuntu-22.04
            cache_name: 'm2-dependencies-all'
            mvn_arguments: ''

          - name: all modules - macos
            runs-on: macos-11
            cache_name: 'm2-dependencies-all'

          - name: core-modules
            runs-on: ubuntu-22.04
            cache_name: 'm2-dependencies-core-modules'
            mvn_arguments: '-Pcore-modules,-main'

    steps:
      - name: checkout
        uses: actions/checkout@v4

      - name: Tune Runner VM
        uses: ./.github/actions/tune-runner-vm

      - name: Detect changed files
        if: ${{ github.event_name != 'schedule' }}
        id:   changes
        uses: apache/pulsar-test-infra/paths-filter@master
        with:
          filters: |
            poms:
              - 'pom.xml'
              - '**/pom.xml'

      - name: Cache local Maven repository
        if: ${{ github.event_name == 'schedule' || steps.changes.outputs.poms == 'true' }}
        id: cache
        uses: actions/cache@v4
        timeout-minutes: 5
        with:
          path: |
            ~/.m2/repository/*/*/*
            !~/.m2/repository/org/apache/pulsar
          key: ${{ runner.os }}-${{ matrix.cache_name }}-${{ hashFiles('**/pom.xml') }}
          # there is no restore-keys here so that the cache size doesn't keep
          # on growing from old entries which wouldn't never expire if the old
          # cache would be used as the starting point for a new cache entry

      - name: Set up JDK 17
        uses: actions/setup-java@v4
        if: ${{ (github.event_name == 'schedule' || steps.changes.outputs.poms == 'true') && steps.cache.outputs.cache-hit != 'true' }}
        with:
          distribution: ${{ env.JDK_DISTRIBUTION }}
          java-version: 17

      - name: Download dependencies
        if: ${{ (github.event_name == 'schedule' || steps.changes.outputs.poms == 'true') && steps.cache.outputs.cache-hit != 'true' }}
        run: |
          # download dependencies, ignore errors
          mvn -B -fn -ntp ${{ matrix.mvn_arguments }} dependency:go-offline


CI - OWASP Dependency Check

#
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements.  See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership.  The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License.  You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied.  See the License for the
# specific language governing permissions and limitations
# under the License.
#

name: CI - OWASP Dependency Check
on:
  schedule:
    - cron: '15 0 * * *'
  workflow_dispatch:

env:
  MAVEN_OPTS: -Xss1500k -Xmx1024m -Daether.connector.http.reuseConnections=false -Daether.connector.requestTimeout=60000 -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false -Dmaven.wagon.http.retryHandler.class=standard -Dmaven.wagon.http.retryHandler.count=3 -Dmaven.wagon.http.retryHandler.requestSentEnabled=true -Dmaven.wagon.http.serviceUnavailableRetryStrategy.class=standard -Dmaven.wagon.rto=60000
  JDK_DISTRIBUTION: corretto

jobs:
  run-owasp-dependency-check:
    if: ${{ github.repository == 'apache/pulsar' || github.event_name == 'workflow_dispatch' }}
    name: Check ${{ matrix.branch }}
    env:
      JOB_NAME: Check ${{ matrix.branch }}
      GRADLE_ENTERPRISE_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }}
    runs-on: ubuntu-22.04
    timeout-minutes: 75
    strategy:
      fail-fast: false
      max-parallel: 1
      matrix:
        include:
          - branch: master
          - branch: branch-3.2
          - branch: branch-3.1
          - branch: branch-3.0
          - branch: branch-2.11
          - branch: branch-2.10
            jdk: 11

    steps:
      - name: checkout
        uses: actions/checkout@v4
        with:
          ref: ${{ matrix.branch }}

      - name: Tune Runner VM
        uses: ./.github/actions/tune-runner-vm

      - name: Cache local Maven repository
        uses: actions/cache@v4
        timeout-minutes: 5
        with:
          path: |
            ~/.m2/repository/*/*/*
            !~/.m2/repository/org/apache/pulsar
            !~/.m2/repository/org/owasp/dependency-check-data
          key: ${{ runner.os }}-m2-dependencies-all-${{ hashFiles('**/pom.xml') }}
          lookup-only: true
          restore-keys: |
            ${{ runner.os }}-m2-dependencies-core-modules-${{ hashFiles('**/pom.xml') }}
            ${{ runner.os }}-m2-dependencies-core-modules-

      - name: Set up JDK ${{ matrix.jdk || '17' }}
        uses: actions/setup-java@v4
        with:
          distribution: ${{ env.JDK_DISTRIBUTION }}
          java-version: ${{ matrix.jdk || '17' }}

      - name: run install by skip tests
        run: mvn -B -ntp clean install -DskipTests -Dspotbugs.skip=true  -Dlicense.skip=true -Dcheckstyle.skip=true -Drat.skip=true -DskipDocker=true

      - name: OWASP cache key weeknum
        id: get-weeknum
        run: |
          echo "weeknum=$(date -u +"%Y-%U")" >> $GITHUB_OUTPUT
        shell: bash

      - name: Restore OWASP Dependency Check data
        id: restore-owasp-dependency-check-data
        uses: actions/cache/restore@v3
        timeout-minutes: 5
        with:
          path: ~/.m2/repository/org/owasp/dependency-check-data
          key: owasp-dependency-check-data-${{ steps.get-weeknum.outputs.weeknum }}
          enableCrossOsArchive: true
          restore-keys: |
            owasp-dependency-check-data-

      - name: Update OWASP Dependency Check data
        id: update-owasp-dependency-check-data
        if: ${{ matrix.branch == 'master' && (steps.restore-owasp-dependency-check-data.outputs.cache-hit != 'true' || steps.restore-owasp-dependency-check-data.outputs.cache-matched-key != steps.restore-owasp-dependency-check-data.outputs.cache-primary-key) }}
        run: mvn -B -ntp -Powasp-dependency-check initialize -pl . dependency-check:update-only

      - name: Save OWASP Dependency Check data
        if: ${{ steps.update-owasp-dependency-check-data.outcome == 'success' }}
        uses: actions/cache/save@v3
        timeout-minutes: 5
        with:
          path: ~/.m2/repository/org/owasp/dependency-check-data
          key: ${{ steps.restore-owasp-dependency-check-data.outputs.cache-primary-key }}
          enableCrossOsArchive: true

      - name: run OWASP Dependency Check for distribution/server (-DfailBuildOnAnyVulnerability=true)
        run: mvn -B -ntp -Pmain,skip-all,skipDocker,owasp-dependency-check initialize verify -pl distribution/server -DfailBuildOnAnyVulnerability=true

      - name: run OWASP Dependency Check for offloaders/tiered-storage and pulsar-io connectors (-DfailOnError=false)
        if: ${{ !cancelled() }}
        run: |
          mvnprojects=$(mvn -B -ntp -Dscan=false initialize \
            | grep -- "-< .* >-" \
            | sed -E 's/.*-< (.*) >-.*/\1/' \
            | grep -E 'pulsar-io-|tiered-storage-|offloader' \
            | tr '\n' ',' | sed 's/,$/\n/' )
          set -xe
          mvn --fail-at-end -B -ntp -Pmain,skip-all,skipDocker,owasp-dependency-check initialize verify -DfailOnError=false -pl "${mvnprojects}"

      - name: Upload OWASP Dependency Check reports
        uses: actions/upload-artifact@v4
        if: always()
        with:
          name: owasp-dependency-check-reports-${{ matrix.branch }}
          path: |
            **/target/dependency-check-report.html


CI - Integration - Pulsar-IO Sinks and Sources


CI - Integration - Pulsar-IO Oracle Source


Documentation Bot

#
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements.  See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership.  The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License.  You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied.  See the License for the
# specific language governing permissions and limitations
# under the License.
#

name: Documentation Bot

on:
  pull_request_target :
    types:
      - opened
      - edited
      - labeled
      - unlabeled

concurrency:
  group: ${{ github.workflow }}-${{ github.ref }}-${{ github.event.number }}
  cancel-in-progress: true

jobs:
  label:
    if: (github.repository == 'apache/pulsar') && (github.event.pull_request.state == 'open')
    permissions:
      pull-requests: write
    runs-on: ubuntu-22.04
    steps:
      - name: Labeling
        uses: apache/pulsar-test-infra/docbot@master
        env:
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
          LABEL_WATCH_LIST: 'doc,doc-required,doc-not-needed,doc-complete'
          LABEL_MISSING: 'doc-label-missing'


Add flaky test issues and PRs to flaky test project


CI - Go Functions

#
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements.  See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership.  The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License.  You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied.  See the License for the
# specific language governing permissions and limitations
# under the License.
#

name: CI - Go Functions
on:
  pull_request:
    branches:
      - master
    paths:
      - '.github/workflows/**'
      - 'pulsar-function-go/**'
  workflow_dispatch:

concurrency:
  group: ${{ github.workflow }}-${{ github.ref }}
  cancel-in-progress: true

env:
  MAVEN_OPTS: -Xss1500k -Xmx1024m -Daether.connector.http.reuseConnections=false -Daether.connector.requestTimeout=60000 -Dhttp.keepAlive=false -Dmaven.wagon.http.pool=false -Dmaven.wagon.http.retryHandler.class=standard -Dmaven.wagon.http.retryHandler.count=3 -Dmaven.wagon.http.retryHandler.requestSentEnabled=true -Dmaven.wagon.http.serviceUnavailableRetryStrategy.class=standard -Dmaven.wagon.rto=60000

jobs:
  preconditions:
    name: Preconditions
    runs-on: ubuntu-22.04
    outputs:
      docs_only: ${{ steps.check_changes.outputs.docs_only }}
    steps:
      - name: checkout
        uses: actions/checkout@v4

      - name: Detect changed files
        id: changes
        uses: apache/pulsar-test-infra/paths-filter@master
        with:
          filters: .github/changes-filter.yaml
          list-files: csv

      - name: Check changed files
        id: check_changes
        run: |
          if [[ "${GITHUB_EVENT_NAME}" != "schedule" ]]; then
            echo "docs_only=${{ fromJSON(steps.changes.outputs.all_count) == fromJSON(steps.changes.outputs.docs_count) && fromJSON(steps.changes.outputs.docs_count) > 0 }}" >> $GITHUB_OUTPUT
          else
            echo docs_only=false >> $GITHUB_OUTPUT
          fi

      - name: Check if the PR has been approved for testing
        if: ${{ steps.check_changes.outputs.docs_only != 'true' && github.repository == 'apache/pulsar' && github.event_name == 'pull_request' }}
        env:
          GITHUB_RUN_ATTEMPT: ${{ github.run_attempt }}
          GITHUB_TOKEN: ${{ github.token }}
        run: |
          build/pulsar_ci_tool.sh check_ready_to_test

  check-style:
    needs: preconditions
    if: ${{ needs.preconditions.outputs.docs_only != 'true' }}
    name: Go ${{ matrix.go-version }} Functions style check
    runs-on: ubuntu-22.04
    strategy:
      matrix:
        go-version: ['1.21']

    steps:
      - name: Check out code into the Go module directory
        uses: actions/checkout@v4

      - name: Tune Runner VM
        uses: ./.github/actions/tune-runner-vm

      - name: Set up Go
        uses: actions/setup-go@v5
        with:
          go-version: ${{ matrix.go-version }}
        id: go

      - name: InstallTool
        run: |
          cd pulsar-function-go
          wget -O - -q https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh| sh -s v1.55.2
          ./bin/golangci-lint --version

      - name: Build
        run: |
          cd pulsar-function-go
          export GO111MODULE=on
          go build ./...

      - name: Test
        run: |
          cd pulsar-function-go
          export GO111MODULE=on
          ./bin/golangci-lint run -c ./golangci.yml ./pf
          go test -v $(go list ./... | grep -v examples)