.github/workflows/.test.yml

# reusable workflow
name: .test

# TODO: hide reusable workflow from the UI. Tracked in https://github.com/community/community/discussions/12025

permissions:
  contents: read

on:
  workflow_call:
    inputs:
      storage:
        required: true
        type: string
        default: "graphdriver"
    secrets:
      CODECOV_TOKEN:
        required: false

env:
  GO_VERSION: "1.26.2"
  GOTESTLIST_VERSION: v0.3.1
  TESTSTAT_VERSION: v0.1.25
  ITG_CLI_MATRIX_SIZE: 6
  DOCKER_EXPERIMENTAL: 1
  DOCKER_GRAPHDRIVER: ${{ inputs.storage == 'snapshotter' && 'overlayfs' || 'overlay2' }}
  TEST_INTEGRATION_USE_GRAPHDRIVER: ${{ inputs.storage == 'graphdriver' && '1' || '' }}
  SETUP_BUILDX_VERSION: edge
  SETUP_BUILDKIT_IMAGE: moby/buildkit:latest

jobs:
  docker-py:
    runs-on: ubuntu-24.04
    timeout-minutes: 120 # guardrails timeout for the whole job
    continue-on-error: ${{ github.event_name != 'pull_request' }}
    steps:
      -
        name: Checkout
        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
      -
        name: Set up runner
        uses: ./.github/actions/setup-runner
      -
        name: Set up tracing
        uses: ./.github/actions/setup-tracing
      -
        name: Set up Docker Buildx
        uses: docker/setup-buildx-action@4d04d5d9486b7bd6fa91e7baf45bbb4f8b9deedd # v4.0.0
        with:
          version: ${{ env.SETUP_BUILDX_VERSION }}
          driver-opts: image=${{ env.SETUP_BUILDKIT_IMAGE }}
          buildkitd-flags: --debug
      -
        name: Build dev image
        uses: docker/bake-action@a66e1c87e2eca0503c343edf1d208c716d54b8a8 # v7.1.0
        with:
          targets: dev
          set: |
            dev.cache-from=type=gha,scope=dev
      -
        name: Test
        run: |
          make TEST_SKIP_INTEGRATION_CLI=1 -o build test-docker-py
      -
        name: Prepare reports
        if: always()
        run: |
          docker stop otelcol
          mkdir -p bundles /tmp/reports
          find bundles \( -path '*/root' -o -path '*/root/*' \) -prune -o -type f \( -name '*-report.json' -o -name '*.log' -o -name '*.out' -o -name '*.prof' -o -name '*-report.xml' \) -print0 | sudo tar --null --files-from=- -czf /tmp/reports.tar.gz
          tar -xzf /tmp/reports.tar.gz -C /tmp/reports
          sudo chown -R $(id -u):$(id -g) /tmp/reports
          tree -nh /tmp/reports
      -
        name: Test daemon logs
        if: always()
        run: |
          cat bundles/test-docker-py/docker.log
      -
        name: Upload reports
        if: always()
        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
        with:
          name: test-reports-docker-py-${{ inputs.storage }}
          path: /tmp/reports/*
          retention-days: 1

  integration-flaky:
    runs-on: ubuntu-24.04
    timeout-minutes: 120 # guardrails timeout for the whole job
    continue-on-error: ${{ github.event_name != 'pull_request' }}
    steps:
      -
        name: Checkout
        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
      -
        name: Set up runner
        uses: ./.github/actions/setup-runner
      -
        name: Set up Docker Buildx
        uses: docker/setup-buildx-action@4d04d5d9486b7bd6fa91e7baf45bbb4f8b9deedd # v4.0.0
        with:
          version: ${{ env.SETUP_BUILDX_VERSION }}
          driver-opts: image=${{ env.SETUP_BUILDKIT_IMAGE }}
          buildkitd-flags: --debug
      -
        name: Build dev image
        uses: docker/bake-action@a66e1c87e2eca0503c343edf1d208c716d54b8a8 # v7.1.0
        with:
          targets: dev
          set: |
            dev.cache-from=type=gha,scope=dev
      -
        name: Test
        run: |
          make -o build test-integration-flaky
        env:
          TEST_SKIP_INTEGRATION_CLI: 1

  integration-prepare:
    runs-on: ubuntu-24.04
    timeout-minutes: 10 # guardrails timeout for the whole job
    continue-on-error: ${{ github.event_name != 'pull_request' }}
    outputs:
      includes: ${{ steps.set.outputs.includes }}
    steps:
      -
        name: Create matrix includes
        id: set
        uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0
        env:
          INPUT_STORAGE: ${{ inputs.storage }}
        with:
          script: |
            const storage = core.getInput('storage', { required: true });
            let includes = [
              { os: 'ubuntu-22.04', mode: '' },
              { os: 'ubuntu-22.04', mode: 'rootless' },
              { os: 'ubuntu-22.04', mode: 'systemd' },
              { os: 'ubuntu-24.04', mode: '' },
              { os: 'ubuntu-24.04', mode: 'rootless' },
              { os: 'ubuntu-24.04', mode: 'systemd' },
              // { os: 'ubuntu-24.04', mode: 'rootless-systemd' }, // FIXME: https://github.com/moby/moby/issues/44084
            ];
            if (storage == "snapshotter") {
              includes.push({ os: 'ubuntu-24.04', mode: 'iptables+firewalld' });
              includes.push({ os: 'ubuntu-24.04', mode: 'nftables' });
              includes.push({ os: 'ubuntu-24.04', mode: 'nftables+firewalld' });
            }
            await core.group(`Set matrix`, async () => {
              core.info(`matrix: ${JSON.stringify(includes)}`);
              core.setOutput('includes', JSON.stringify(includes));
            });

  integration:
    runs-on: ${{ matrix.os }}
    timeout-minutes: 120 # guardrails timeout for the whole job
    continue-on-error: ${{ github.event_name != 'pull_request' }}
    needs:
      - integration-prepare
    strategy:
      fail-fast: false
      matrix:
        include: ${{ fromJson(needs.integration-prepare.outputs.includes) }}
    steps:
      -
        name: Checkout
        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
      -
        name: Set up runner
        uses: ./.github/actions/setup-runner
      -
        name: Set up tracing
        uses: ./.github/actions/setup-tracing
      -
        name: Prepare
        env:
          MODE: ${{ matrix.mode }}
        run: |
          CACHE_DEV_SCOPE=dev
          if [[ "$MODE" == *"rootless"* ]]; then
            echo "DOCKER_ROOTLESS=1" >> $GITHUB_ENV
          fi
          if [[ "$MODE" == *"systemd"* ]]; then
            echo "SYSTEMD=true" >> $GITHUB_ENV
            CACHE_DEV_SCOPE="${CACHE_DEV_SCOPE}systemd"
          fi
          if [[ "$MODE" == *"firewalld"* ]]; then
            echo "FIREWALLD=true" >> $GITHUB_ENV
            CACHE_DEV_SCOPE="${CACHE_DEV_SCOPE}firewalld"
          fi
          if [[ "$MODE" == *"nftables"* ]]; then
            echo "DOCKER_FIREWALL_BACKEND=nftables" >> $GITHUB_ENV
          fi
          echo "CACHE_DEV_SCOPE=${CACHE_DEV_SCOPE}" >> $GITHUB_ENV
      -
        name: Set up AppArmor for RootlessKit
        if: contains(matrix.mode, 'rootless') && matrix.os != 'ubuntu-22.04'
        run: |
          cat <<EOT | sudo tee "/etc/apparmor.d/usr.local.bin.rootlesskit"
          abi <abi/4.0>,
          include <tunables/global>

          /usr/local/bin/rootlesskit flags=(unconfined) {
            userns,
          }
          EOT
          sudo systemctl restart apparmor.service
      -
        name: Set up Docker Buildx
        uses: docker/setup-buildx-action@4d04d5d9486b7bd6fa91e7baf45bbb4f8b9deedd # v4.0.0
        with:
          version: ${{ env.SETUP_BUILDX_VERSION }}
          driver-opts: image=${{ env.SETUP_BUILDKIT_IMAGE }}
          buildkitd-flags: --debug
      -
        name: Build dev image
        uses: docker/bake-action@a66e1c87e2eca0503c343edf1d208c716d54b8a8 # v7.1.0
        with:
          targets: dev
          set: |
            dev.cache-from=type=gha,scope=${{ env.CACHE_DEV_SCOPE }}
      -
        name: Test
        run: |
          make -o build test-integration
        env:
          TEST_SKIP_INTEGRATION_CLI: 1
          TESTCOVERAGE: 1
      -
        name: Prepare reports
        if: always()
        env:
          MATRIX_MODE: ${{ matrix.mode }}
          MATRIX_OS: ${{ matrix.os }}
        run: |
          reportsName=$MATRIX_OS
          if [ -n "$MATRIX_MODE" ]; then
            reportsName="$reportsName-$MATRIX_MODE"
          fi
          reportsPath="/tmp/reports/$reportsName"
          echo "TESTREPORTS_NAME=$reportsName" >> $GITHUB_ENV
          
          docker stop otelcol
          mkdir -p bundles $reportsPath
          find bundles \( -path '*/root' -o -path '*/root/*' \) -prune -o -type f \( -name '*-report.json' -o -name '*.log' -o -name '*.out' -o -name '*.prof' -o -name '*-report.xml' \) -print0 | sudo tar --null --files-from=- -czf /tmp/reports.tar.gz
          tar -xzf /tmp/reports.tar.gz -C $reportsPath
          sudo mv /tmp/reports/otel-trace*.jsonl $reportsPath/ 2>/dev/null || true
          sudo chown -R $(id -u):$(id -g) $reportsPath
          tree -nh $reportsPath
      -
        name: Send to Codecov
        uses: codecov/codecov-action@57e3a136b779b570ffcdbf80b3bdc90e7fab3de2 # v6.0.0
        with:
          directory: ./bundles/test-integration
          env_vars: RUNNER_OS
          flags: integration,${{ matrix.mode }}
          token: ${{ secrets.CODECOV_TOKEN }}  # used to upload coverage reports: https://github.com/moby/buildkit/pull/4660#issue-2142122533
      -
        name: Test daemon logs
        if: always()
        run: |
          cat bundles/test-integration/docker.log
      -
        name: Upload reports
        if: always()
        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
        with:
          name: test-reports-integration-${{ inputs.storage }}-${{ env.TESTREPORTS_NAME }}
          path: /tmp/reports/*
          retention-days: 1

  integration-report:
    runs-on: ubuntu-24.04
    timeout-minutes: 10
    continue-on-error: ${{ github.event_name != 'pull_request' }}
    if: always()
    needs:
      - integration
    steps:
      -
        name: Set up Go
        uses: actions/setup-go@4a3601121dd01d1626a1e23e37211e3254c1c06c # v6.4.0
        with:
          go-version: ${{ env.GO_VERSION }}
          cache: false
      -
        name: Download reports
        uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
        with:
          path: /tmp/reports
          pattern: test-reports-integration-${{ inputs.storage }}-*
          merge-multiple: true
      -
        name: Install teststat
        run: |
          go install github.com/vearutop/teststat@${{ env.TESTSTAT_VERSION }}
      -
        name: Create summary
        run: |
          find /tmp/reports -type f -name '*-go-test-report.json' -exec teststat -markdown {} \+ >> $GITHUB_STEP_SUMMARY

  integration-cli-prepare:
    runs-on: ubuntu-24.04
    timeout-minutes: 120 # guardrails timeout for the whole job
    continue-on-error: ${{ github.event_name != 'pull_request' }}
    outputs:
      matrix: ${{ steps.set.outputs.matrix }}
    steps:
      -
        name: Checkout
        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
      -
        name: Set up Go
        uses: actions/setup-go@4a3601121dd01d1626a1e23e37211e3254c1c06c # v6.4.0
        with:
          go-version: ${{ env.GO_VERSION }}
          cache: false
      -
        name: Install gotestlist
        run:
          go install github.com/crazy-max/gotestlist/cmd/gotestlist@${{ env.GOTESTLIST_VERSION }}
      -
        name: Create test matrix
        id: tests
        working-directory: ./integration-cli
        run: |
          # This step creates a matrix for integration-cli tests. Tests suites
          # are distributed in integration-cli job through a matrix. There is
          # also overrides being added to the matrix like "./..." to run
          # "Test integration" step exclusively and specific tests suites that
          # take a long time to run.
          matrix="$(gotestlist -d ${{ env.ITG_CLI_MATRIX_SIZE }} -o "./..." -o "DockerSwarmSuite" -o "DockerNetworkSuite|DockerExternalVolumeSuite" ./...)"
          echo "matrix=$matrix" >> $GITHUB_OUTPUT
      -
        name: Create gha matrix
        id: set
        uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0
        env:
          INPUT_STORAGE: ${{ inputs.storage }}
          INPUT_TEST-MATRIX: ${{ steps.tests.outputs.matrix }}
        with:
          script: |
            const storage = core.getInput('storage', { required: true });
            let matrix = {
              test: JSON.parse(core.getInput('test-matrix', { required: true })),
              include: [],
            };
            // For some reasons, GHA doesn't combine a dynamically defined
            // 'include' with other matrix variables that aren't part of the
            // include items.
            // Moreover, since the goal is to run only relevant tests with
            // firewalld/nftables enabled to minimize the number of CI jobs, we
            // statically define the list of test suites that we want to run.
            if (storage == "snapshotter") {
              matrix.include.push({
                'mode': 'iptables+firewalld',
                'test': 'DockerCLINetworkSuite|DockerCLIPortSuite|DockerDaemonSuite'
              });
              matrix.include.push({
                'mode': 'iptables+firewalld',
                'test': 'DockerSwarmSuite'
              });
              matrix.include.push({
                'mode': 'iptables+firewalld',
                'test': 'DockerNetworkSuite'
              });
              matrix.include.push({
                'mode': 'nftables',
                'test': 'DockerCLINetworkSuite|DockerCLIPortSuite|DockerDaemonSuite'
              });
              matrix.include.push({
                'mode': 'nftables',
                'test': 'DockerSwarmSuite'
              });
              matrix.include.push({
                'mode': 'nftables',
                'test': 'DockerNetworkSuite'
              });
              matrix.include.push({
                'mode': 'nftables+firewalld',
                'test': 'DockerCLINetworkSuite|DockerCLIPortSuite|DockerDaemonSuite'
              });
              matrix.include.push({
                'mode': 'nftables+firewalld',
                'test': 'DockerSwarmSuite'
              });
              matrix.include.push({
                'mode': 'nftables+firewalld',
                'test': 'DockerNetworkSuite'
              });
            }
            await core.group(`Set matrix`, async () => {
              core.info(`matrix: ${JSON.stringify(matrix)}`);
              core.setOutput('matrix', JSON.stringify(matrix));
            });

  integration-cli:
    runs-on: ubuntu-24.04
    timeout-minutes: 120 # guardrails timeout for the whole job
    continue-on-error: ${{ github.event_name != 'pull_request' }}
    needs:
      - integration-cli-prepare
    strategy:
      fail-fast: false
      matrix: ${{ fromJson(needs.integration-cli-prepare.outputs.matrix) }}
    steps:
      -
        name: Checkout
        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
      -
        name: Set up runner
        uses: ./.github/actions/setup-runner
      -
        name: Set up tracing
        uses: ./.github/actions/setup-tracing
      -
        name: Prepare
        env:
          MODE: ${{ matrix.mode }}
        run: |
          CACHE_DEV_SCOPE=dev
          if [[ "$MODE" == *"firewalld"* ]]; then
            echo "FIREWALLD=true" >> $GITHUB_ENV
            CACHE_DEV_SCOPE="${CACHE_DEV_SCOPE}firewalld"
          fi
          if [[ "$MODE" == *"nftables"* ]]; then
            echo "DOCKER_FIREWALL_BACKEND=nftables" >> $GITHUB_ENV
          fi
          echo "CACHE_DEV_SCOPE=${CACHE_DEV_SCOPE}" >> $GITHUB_ENV
      -
        name: Set up Docker Buildx
        uses: docker/setup-buildx-action@4d04d5d9486b7bd6fa91e7baf45bbb4f8b9deedd # v4.0.0
        with:
          version: ${{ env.SETUP_BUILDX_VERSION }}
          driver-opts: image=${{ env.SETUP_BUILDKIT_IMAGE }}
          buildkitd-flags: --debug
      -
        name: Build dev image
        uses: docker/bake-action@a66e1c87e2eca0503c343edf1d208c716d54b8a8 # v7.1.0
        with:
          targets: dev
          set: |
            dev.cache-from=type=gha,scope=dev
      -
        name: Test
        run: |
          make -o build test-integration
        env:
          TEST_SKIP_INTEGRATION: 1
          TESTCOVERAGE: 1
          TESTFLAGS: "-test.run (${{ matrix.test }})/"
      -
        name: Prepare reports
        if: always()
        env:
          MATRIX_TEST: ${{ matrix.test }}
        run: |
          reportsName=$(echo -n "$MATRIX_TEST" | sha256sum | cut -d " " -f 1)
          reportsPath=/tmp/reports/$reportsName
          echo "TESTREPORTS_NAME=$reportsName" >> $GITHUB_ENV
          
          docker stop otelcol
          mkdir -p bundles $reportsPath
          echo "$MATRIX_TEST" | tr -s '|' '\n' | tee -a "$reportsPath/tests.txt"
          find bundles \( -path '*/root' -o -path '*/root/*' \) -prune -o -type f \( -name '*-report.json' -o -name '*.log' -o -name '*.out' -o -name '*.prof' -o -name '*-report.xml' \) -print0 | sudo tar --null --files-from=- -czf /tmp/reports.tar.gz
          tar -xzf /tmp/reports.tar.gz -C $reportsPath
          sudo mv /tmp/reports/otel-trace*.jsonl $reportsPath/ 2>/dev/null || true
          sudo chown -R $(id -u):$(id -g) $reportsPath
          tree -nh $reportsPath
      -
        name: Send to Codecov
        uses: codecov/codecov-action@57e3a136b779b570ffcdbf80b3bdc90e7fab3de2 # v6.0.0
        with:
          directory: ./bundles/test-integration
          env_vars: RUNNER_OS
          flags: integration-cli
          token: ${{ secrets.CODECOV_TOKEN }}  # used to upload coverage reports: https://github.com/moby/buildkit/pull/4660#issue-2142122533
      -
        name: Test daemon logs
        if: always()
        run: |
          cat bundles/test-integration/docker.log
      -
        name: Upload reports
        if: always()
        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
        with:
          name: test-reports-integration-cli-${{ inputs.storage }}-${{ matrix.mode }}-${{ env.TESTREPORTS_NAME }}
          path: /tmp/reports/*
          retention-days: 1

  integration-cli-report:
    runs-on: ubuntu-24.04
    timeout-minutes: 10
    continue-on-error: ${{ github.event_name != 'pull_request' }}
    if: always()
    needs:
      - integration-cli
    steps:
      -
        name: Set up Go
        uses: actions/setup-go@4a3601121dd01d1626a1e23e37211e3254c1c06c # v6.4.0
        with:
          go-version: ${{ env.GO_VERSION }}
          cache: false
      -
        name: Download reports
        uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
        with:
          path: /tmp/reports
          pattern: test-reports-integration-cli-${{ inputs.storage }}-${{ matrix.mode }}-*
          merge-multiple: true
      -
        name: Install teststat
        run: |
          go install github.com/vearutop/teststat@${{ env.TESTSTAT_VERSION }}
      -
        name: Create summary
        run: |
          find /tmp/reports -type f -name '*-go-test-report.json' -exec teststat -markdown {} \+ >> $GITHUB_STEP_SUMMARY