containerd/.github/workflows/ci.yml

name: CI
on:
  # When added to a merge queue.
  # See https://docs.github.com/en/repositories/configuring-branches-and-merges-in-your-repository/configuring-pull-request-merges/managing-a-merge-queue#triggering-merge-group-checks-with-github-actions
  merge_group:
  pull_request:
    branches: ['main', 'release/**', 'dev/**']

permissions: # added using https://github.com/step-security/secure-workflows
  contents: read

env:
  GOTOOLCHAIN: local

jobs:
  #
  # golangci-lint
  #
  linters:
    permissions:
      contents: read # for actions/checkout to fetch code
      pull-requests: read # for golangci/golangci-lint-action to fetch pull requests
    name: Linters
    runs-on: ${{ matrix.os }}
    timeout-minutes: 10

    strategy:
      matrix:
        os: [ubuntu-24.04, ubuntu-24.04-arm, macos-latest, windows-latest]
        exclude:
          - os: ${{ github.event.repository.private && 'ubuntu-24.04-arm' || '' }}
          - os: ${{ github.event.repository.private && 'macos-latest' || '' }}
          - os: ${{ github.event.repository.private && 'windows-latest' || '' }}


    steps:
      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
        if: ${{ !github.event.repository.private }}
        with:
          persist-credentials: false
      - uses: ./.github/actions/install-go
        if: ${{ !github.event.repository.private }}
      - uses: golangci/golangci-lint-action@82606bf257cbaff209d206a39f5134f0cfbfd2ee  # v9.2.1
        if: ${{ !github.event.repository.private }}
        with:
          version: v2.14.0
          skip-cache: true
          args: --timeout=8m

  #
  # Project checks
  #
  project:
    name: Project Checks
    runs-on: ubuntu-24.04
    timeout-minutes: 5

    steps:
      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
        with:
          path: src/github.com/containerd/containerd
          fetch-depth: 100
          persist-credentials: false

      - uses: ./src/github.com/containerd/containerd/.github/actions/install-go

      - uses: containerd/project-checks@9d887fad80ae4e40d6f10a68529082e7fc06a9af # v1.2.3
        if: github.repository == 'containerd/containerd'
        with:
          working-directory: src/github.com/containerd/containerd
          repo-access-token: ${{ secrets.GITHUB_TOKEN }}
          # github.com/cyphar/filepath-securejoin: MPL-2.0 AND BSD-3-Clause
          # Approval: https://github.com/cncf/foundation/issues/1154#issuecomment-3562385979
          go-licenses-ignore: |
            github.com/cyphar/filepath-securejoin

      - name: verify go modules and vendor directory
        run: |
          sudo apt-get install -y jq
          make verify-vendor
        working-directory: src/github.com/containerd/containerd

  #
  # Protobuf checks
  #
  protos:
    name: Protobuf
    runs-on: ubuntu-24.04
    timeout-minutes: 5

    defaults:
      run:
        working-directory: src/github.com/containerd/containerd

    steps:
      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
        with:
          path: src/github.com/containerd/containerd
          persist-credentials: false

      - uses: ./src/github.com/containerd/containerd/.github/actions/install-go

      - name: Set env
        shell: bash
        run: |
          echo "GOPATH=${{ github.workspace }}" >> $GITHUB_ENV
          echo "${{ github.workspace }}/bin" >> $GITHUB_PATH

      - run: script/setup/install-dev-tools
      - run: make check-protos

  man:
    name: Manpages
    runs-on: ubuntu-24.04
    timeout-minutes: 5

    steps:
      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
        with:
          persist-credentials: false
      - uses: ./.github/actions/install-go
      - run: go install github.com/cpuguy83/go-md2man/v2@v2.0.7
      - run: make man

  # Make sure binaries compile with other minor platforms.
  # Well-known architectures are covered in release.yml.
  crossbuild:
    name: Crossbuild Binaries
    needs: [project, linters, protos, man]
    runs-on: ubuntu-24.04
    timeout-minutes: 10
    strategy:
      fail-fast: false
      matrix:
        include:
          - goos: linux
            goarch: arm
            goarm: "7"
          - goos: linux
            goarch: arm
            goarm: "5"
          - goos: freebsd
            goarch: amd64
          - goos: freebsd
            goarch: arm64
          - goos: linux
            goarch: loong64
          - goos: windows
            goarch: arm64

    steps:
      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
        with:
          persist-credentials: false
      - uses: ./.github/actions/install-go
      - run: |
          set -e -x

          packages=""
          platform="${{matrix.goos}}/${{matrix.goarch}}"
          if [ -n "${{matrix.goarm}}" ]; then
            platform+="/v${{matrix.goarm}}"
          fi

          case "${platform}" in
          linux/arm/v5)
            packages+=" crossbuild-essential-armel"
            echo "CGO_ENABLED=1" >> $GITHUB_ENV
            echo "CC=arm-linux-gnueabi-gcc" >> $GITHUB_ENV
            ;;
          linux/arm/v7)
            packages+=" crossbuild-essential-armhf"
            echo "CGO_ENABLED=1" >> $GITHUB_ENV
            echo "CC=arm-linux-gnueabihf-gcc" >> $GITHUB_ENV
            ;;
          windows/arm64)
            echo "CGO_ENABLED=0" >> $GITHUB_ENV
            ;;
          linux/loong64)
            echo "CGO_ENABLED=0" >> $GITHUB_ENV
            ;;
          esac

          if [ -n "${packages}" ]; then
            sudo apt-get update && sudo apt-get install -y ${packages}
          fi
        name: Install deps
      - name: Build
        env:
          GOOS: ${{matrix.goos}}
          GOARCH: ${{matrix.goarch}}
          GOARM: ${{matrix.goarm}}
        run: |
          make build
          make binaries

  #
  # Build containerd binaries
  #
  binaries:
    name: Binaries
    runs-on: ${{ matrix.os }}
    timeout-minutes: 20
    needs: [project, linters, protos, man]

    strategy:
      matrix:
        os: [ubuntu-24.04, ubuntu-24.04-arm, macos-latest, windows-latest]
        go-version: ["1.27.3"] # 1.26 is not supported due to k8s 1.38 requiring 1.27
        exclude:
          - os: ${{ github.event.repository.private && 'ubuntu-24.04-arm' || '' }}
    steps:
      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
        with:
          persist-credentials: false
      - uses: ./.github/actions/install-go
        with:
          go-version: ${{ matrix.go-version }}
      - name: Make
        run: |
          make build
          make binaries

  #
  # Integration and CRI tests
  #
  integration-windows:
    name: Windows Integration
    if: github.event_name != 'merge_group'
    runs-on: ${{ matrix.os }}
    timeout-minutes: 90
    needs: [project, linters, protos, man]
    env:
      GOTEST: gotestsum --

    strategy:
      fail-fast: false
      matrix:
        os: [windows-2022]
        cgroup_driver: [cgroupfs]

    defaults:
      run:
        shell: bash
        working-directory: src/github.com/containerd/containerd

    steps:
      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
        with:
          path: src/github.com/containerd/containerd
          persist-credentials: false

      - uses: ./src/github.com/containerd/containerd/.github/actions/install-go

      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
        with:
          repository: kubernetes-sigs/cri-tools
          path: src/github.com/kubernetes-sigs/cri-tools
          fetch-depth: 0
          persist-credentials: false

      - name: Set env
        run: |
          echo "GOPATH=${{ github.workspace }}" >> $GITHUB_ENV
          echo "${{ github.workspace }}/bin" >> $GITHUB_PATH
          echo "${{ github.workspace }}/src/github.com/containerd/containerd/bin" >> $GITHUB_PATH
          echo "${{ github.workspace }}/src/github.com/kubernetes-sigs/cri-tools/build/bin/windows/amd64" >> $GITHUB_PATH
          echo "CRITOOLS_VERSION=$(cat script/setup/critools-version)" >> $GITHUB_ENV

      - run: script/setup/install-dev-tools

      - name: Binaries
        shell: bash
        env:
          CGO_ENABLED: 1
        run: |
          set -o xtrace
          mingw32-make.exe binaries
          cd ../../kubernetes-sigs/cri-tools
          git checkout "${CRITOOLS_VERSION}"
          make critest

      # Docker creates the "nat" HNS network that install-cni-windows needs, but
      # on windows-2022 it intermittently fails to start. Start it explicitly.
      # https://github.com/actions/runner-images/issues/13729#issuecomment-4183956802
      - name: Ensure Docker is running
        shell: powershell
        run: |
          Start-Service -Name "docker"

      - run: script/setup/install-cni-windows

      - env:
          TEST_IMAGE_LIST: ${{github.workspace}}/repolist.toml
          CRI_TEST_IMAGES: ${{github.workspace}}/cri-test-images.yaml
          BUSYBOX_TESTING_IMAGE_REF: "registry.k8s.io/e2e-test-images/busybox:1.29-2"
          RESOURCE_CONSUMER_TESTING_IMAGE_REF: "registry.k8s.io/e2e-test-images/resource-consumer:1.10"
          WEBSERVER_TESTING_IMAGE_REF: "registry.k8s.io/e2e-test-images/nginx:1.14-2"
        run: |
          cat > "${{ env.TEST_IMAGE_LIST }}" << EOF
          busybox = "${{ env.BUSYBOX_TESTING_IMAGE_REF }}"
          ResourceConsumer = "${{ env.RESOURCE_CONSUMER_TESTING_IMAGE_REF }}"
          EOF
          cat > "${{ env.CRI_TEST_IMAGES }}" << EOF
          defaultTestContainerImage: ${{ env.BUSYBOX_TESTING_IMAGE_REF }}
          webServerTestImage: ${{ env.WEBSERVER_TESTING_IMAGE_REF }}
          EOF

      - name: Get crictl tool
        shell: powershell
        run: |
          # Get critctl tool. Used for cri-integration tests
          $CRICTL_DOWNLOAD_URL="https://github.com/kubernetes-sigs/cri-tools/releases/download/$env:CRITOOLS_VERSION/crictl-$env:CRITOOLS_VERSION-windows-amd64.tar.gz"
          curl.exe -L $CRICTL_DOWNLOAD_URL -o c:\crictl.tar.gz
          tar -xvf c:\crictl.tar.gz
          mv crictl.exe "${{ github.workspace }}/bin/crictl.exe" # Move crictl somewhere in path

      - run: script/setup/install-gotestsum
      - run: script/setup/install-teststat
      - name: Tests
        env:
          CGO_ENABLED: 1
          GOTESTSUM_JUNITFILE: ${{github.workspace}}/test-unit-root.xml
          GOTESTSUM_JSONFILE: ${{github.workspace}}/test-unit-root-gotest.json
        run: mingw32-make.exe test root-test
      - run: if [ -f *-gotest.json ]; then echo '# Root Test' >> $GITHUB_STEP_SUMMARY; teststat -markdown *-gotest.json >> $GITHUB_STEP_SUMMARY; fi
        if: always()
      - run: script/test/test2annotation.sh ${TESTFILE}
        env:
          TESTFILE: ${{github.workspace}}/test-unit-root-gotest.json
        if: always()

      - name: Integration 1
        env:
          CGO_ENABLED: 1
          GOTESTSUM_JUNITFILE: ${{github.workspace}}/test-integration-serial-junit.xml
          GOTESTSUM_JSONFILE: ${{github.workspace}}/test-integration-serial-gotest.json
          EXTRA_TESTFLAGS: "-timeout=20m"
        run: mingw32-make.exe integration
      - run: if [ -f *-gotest.json ]; then echo '# Integration 1' >> $GITHUB_STEP_SUMMARY; teststat -markdown *-gotest.json >> $GITHUB_STEP_SUMMARY; fi
        if: always()
      - run: script/test/test2annotation.sh "$TESTFILE"
        env:
          TESTFILE: ${{github.workspace}}/test-integration-serial-gotest.json
        if: success()

      # Run the integration suite a second time. See discussion in github.com/containerd/containerd/pull/1759
      - name: Integration 2
        env:
          TESTFLAGS_PARALLEL: 1
          EXTRA_TESTFLAGS: "-short"
          CGO_ENABLED: 1
          GOTESTSUM_JUNITFILE: ${{github.workspace}}/test-integration-parallel-junit.xml
          GOTESTSUM_JSONFILE: ${{github.workspace}}/test-integration-parallel-gotest.json
        run: mingw32-make.exe integration
      - run: if [ -f *-gotest.json ]; then echo '# Integration 2' >> $GITHUB_STEP_SUMMARY; teststat -markdown *-gotest.json >> $GITHUB_STEP_SUMMARY; fi
        if: always()
      - run: script/test/test2annotation.sh "$TESTFILE"
        env:
          TESTFILE: ${{github.workspace}}/test-integration-parallel-gotest.json
        if: success()

      - name: CRI Integration Test
        env:
          TEST_IMAGE_LIST: ${{github.workspace}}/repolist.toml
          CGROUP_DRIVER: ${{ matrix.cgroup_driver }}
        run: |
          make cri-integration

      - name: cri-tools critest
        env:
          CRI_TEST_IMAGES: ${{github.workspace}}/cri-test-images.yaml
          CGROUP_DRIVER: ${{ matrix.cgroup_driver }}
        shell: powershell
        run: |
          Start-Process -FilePath containerd.exe -NoNewWindow -RedirectStandardError true -PassThru
          get-process | sls containerd
          start-sleep 5
          if( '${{ matrix.os }}' -eq 'windows-2022' )
          {
            $skipTests = @(
              # This test is exceedingly flaky only on ws2022 so skip for now to keep CI happy.
              # Info: https://github.com/containerd/containerd/issues/6652
              "runtime should support exec with tty=true and stdin=true"
            )
            $skip = $skipTests -join "|"
          }
          Write-Host "Skip parameters: $skip"
          critest.exe --runtime-endpoint=npipe://.//pipe//containerd-containerd --test-images-file='${{env.CRI_TEST_IMAGES}}' --report-dir='${{github.workspace}}/critestreport' --ginkgo.skip="$skip"

      - uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2
        if: always()
        with:
          name: TestResults ${{ matrix.os }} ${{ matrix.cgroup_driver }}
          path: |
            ${{github.workspace}}/*-junit.xml
            ${{github.workspace}}/*-gotest.json
            ${{github.workspace}}/report/*.log

  integration-linux:
    name: Linux Integration
    runs-on: ${{ matrix.os }}
    timeout-minutes: 60
    needs: [project, linters, protos, man]

    strategy:
      fail-fast: false
      matrix:
        runtime:
          - io.containerd.runc.v2
          - io.containerd.runc.v2-rs
        runc: [runc]  # crun can be added here to debug crun issues
        os: [ubuntu-22.04, ubuntu-24.04, ubuntu-24.04-arm]
        cgroup_driver: [cgroupfs, systemd]
        exclude:
          - os: ${{ github.event.repository.private && 'ubuntu-24.04-arm' || '' }}
          - runtime: io.containerd.runc.v2-rs
            os: ubuntu-22.04
          - runtime: io.containerd.runc.v2-rs
            os: ubuntu-24.04-arm
        include:
          # Not supported by the Rust shim yet: checkpoints, trace propagation,
          # State during an exec(TestIssue10589), and the idmapped mounts user namespaces need.
          - runtime: io.containerd.runc.v2-rs
            # CRI only types the options of the runc runtime, so it gets the
            # Rust shim as that runtime's binary, the way a drop-in shim is configured.
            cri_runtime: io.containerd.runc.v2
            cri_runtime_path: /usr/local/bin/containerd-shim-runc-v2-rs
            extra_testflags: "-no-criu -test.skip='(TestIssue10589|TestShimTraceContextPropagation)'"
            cri_skip: "(TestPodUserNS|TestUsernsVolumeCopyUp|TestImageVolumeWithUserNamespace|TestIssue10598)"

    env:
      GOTEST: gotestsum --
      # Build the shim with tracing enabled, so that the integration suite
      # covers trace context propagation to runc and the OCI hooks.
      BUILDTAGS: shim_tracing
      RUST_SHIM_VERSION: "0.2.0"
      RUST_TOOLCHAIN: "1.91"
    steps:
      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
        with:
          persist-credentials: false
      - uses: ./.github/actions/install-go

      - name: Install containerd dependencies
        env:
          RUNC_FLAVOR: ${{ matrix.runc }}
        run: |
          sudo apt-get update
          sudo apt-get install -y gperf dmsetup strace xfsprogs
          script/setup/install-seccomp
          script/setup/install-runc
          script/setup/install-cni $(grep containernetworking/plugins go.mod | awk '{print $2}')
          script/setup/install-critools
          script/setup/install-failpoint-binaries

      # Disable criu testing on arm64 until we can solve the consistent failures of restore testing
      - if: matrix.os != 'ubuntu-24.04-arm'
        name: Install criu
        run: |
          sudo add-apt-repository -y ppa:criu/ppa
          sudo apt-get update
          sudo apt-get install -y criu

      # "apt-get install erofs-utils" installs an older version (1.7.1 as of 2025-09-18)
      # Build and install erofs-utils 1.8.10 from source to get a newer version
      - name: Build and install erofs-utils from source
        run: |
          # Install dependencies
          sudo apt-get update
          sudo apt-get install -y wget make gcc libz-dev liblz4-dev libfuse-dev autoconf libtool pkg-config uuid-dev

          # Create temporary build directory
          BUILD_DIR="$(mktemp -d)"
          cd "$BUILD_DIR"

          # get erofs-utils-1.8.10.tar.gz
          curl -L https://github.com/erofs/erofs-utils/archive/refs/tags/v1.8.10.tar.gz | tar -xzf -
          cd erofs-utils-1.8.10

          # Build and install erofs-utils
          ./autogen.sh
          ./configure --enable-lz4 --enable-fuse --with-uuid
          make -j"$(nproc)"
          sudo make install
          sudo ldconfig

          # Clean up
          cd -
          rm -rf "$BUILD_DIR"

          # Verify installation
          mkfs.erofs --version

      - name: Load EROFS kernel module
        run: |
          sudo modprobe erofs

      - name: Load dm-verity kernel module
        run: |
          sudo modprobe dm_verity

      - name: Cache the Rust runc shim
        if: matrix.runtime == 'io.containerd.runc.v2-rs'
        id: rust-shim
        uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0
        with:
          path: ${{ runner.temp }}/rust-shim/bin
          key: rust-shim-${{ matrix.os }}-${{ env.RUST_SHIM_VERSION }}-rust-${{ env.RUST_TOOLCHAIN }}

      - name: Install the Rust toolchain
        if: matrix.runtime == 'io.containerd.runc.v2-rs' && steps.rust-shim.outputs.cache-hit != 'true'
        uses: dtolnay/rust-toolchain@683b588c487a69d01f6c6ddf40f036ff682cc2c8 # v1
        with:
          toolchain: ${{ env.RUST_TOOLCHAIN }}

      - name: Install the Rust runc shim
        if: matrix.runtime == 'io.containerd.runc.v2-rs'
        run: |
          if [ "${{ steps.rust-shim.outputs.cache-hit }}" != "true" ]; then
            cargo install containerd-runc-shim --version "$RUST_SHIM_VERSION" --locked --root "$RUNNER_TEMP/rust-shim"
          fi
          sudo install -D "$RUNNER_TEMP/rust-shim/bin/containerd-shim-runc-v2-rs" /usr/local/bin/

      - name: Install containerd
        env:
          CGO_ENABLED: 1
        run: |
          make binaries GO_BUILD_FLAGS="-mod=vendor"
          sudo -E PATH=$PATH make install

      - run: script/setup/install-gotestsum
      - run: script/setup/install-teststat
      - name: Tests
        env:
          GOTESTSUM_JUNITFILE: ${{github.workspace}}/test-unit-root-junit.xml
          GOTESTSUM_JSONFILE: ${{github.workspace}}/test-unit-root-gotest.json
        run: |
          # Prevent udevd worker exhaustion on larger runners caused by parallel loopback creations.
          sudo udevadm control --children-max=500
          make test
          sudo -E PATH=$PATH make root-test
      - run: if [ -f *-gotest.json ]; then echo '# Root Test' >> $GITHUB_STEP_SUMMARY; teststat -markdown *-gotest.json >> $GITHUB_STEP_SUMMARY; fi
        if: always()
      - run: script/test/test2annotation.sh *-gotest.json
        if: always()

      - name: Integration 1
        env:
          TEST_RUNTIME: ${{ matrix.runtime }}
          RUNC_FLAVOR: ${{ matrix.runc }}
          EXTRA_TESTFLAGS: ${{ matrix.extra_testflags }}
          GOTESTSUM_JUNITFILE: ${{github.workspace}}/test-integration-serial-junit.xml
          GOTESTSUM_JSONFILE: ${{github.workspace}}/test-integration-serial-gotest.json
        run: |
          extraflags=""
          [ "${RUNC_FLAVOR}" == "crun" ] && {
                  extraflags="EXTRA_TESTFLAGS=-no-criu";
          }
          sudo -E PATH=$PATH make integration ${extraflags} TESTFLAGS_RACE=-race
      - run: if [ -f *-gotest.json ]; then echo '# Integration 1' >> $GITHUB_STEP_SUMMARY; teststat -markdown *-gotest.json >> $GITHUB_STEP_SUMMARY; fi
        if: always()
      - run: script/test/test2annotation.sh *-gotest.json
        if: always()

      # Run the integration suite a second time. See discussion in github.com/containerd/containerd/pull/1759
      - name: Integration 2
        env:
          TEST_RUNTIME: ${{ matrix.runtime }}
          RUNC_FLAVOR: ${{ matrix.runc }}
          EXTRA_TESTFLAGS: ${{ matrix.extra_testflags }}
          GOTESTSUM_JUNITFILE: ${{github.workspace}}/test-integration-parallel-junit.xml
          GOTESTSUM_JSONFILE: ${{github.workspace}}/test-integration-parallel-gotest.json
        run: |
          extraflags=""
          [ "${RUNC_FLAVOR}" == "crun" ] && {
                  extraflags="EXTRA_TESTFLAGS=-no-criu";
          }
          sudo -E PATH=$PATH TESTFLAGS_PARALLEL=1 make integration ${extraflags}
      - run: if [ -f *-gotest.json ]; then echo '# Integration 2' >> $GITHUB_STEP_SUMMARY; teststat -markdown *-gotest.json >> $GITHUB_STEP_SUMMARY; fi
        if: always()
      - run: script/test/test2annotation.sh *-gotest.json
        if: always()

      - name: CRI Integration Test
        env:
          TEST_RUNTIME: ${{ matrix.cri_runtime || matrix.runtime }}
          CONTAINERD_RUNTIME_PATH: ${{ matrix.cri_runtime_path }}
          CGROUP_DRIVER: ${{ matrix.cgroup_driver }}
          RUNC_FLAVOR: ${{ matrix.runc }}
          SKIP: ${{ matrix.cri_skip }}
        run: |
          cat /sys/fs/cgroup/cgroup.controllers
          systemctl status
          [ "${RUNC_FLAVOR}" == "crun" ] && {
             export XDG_RUNTIME_DIR=/run/user/$(id -u)
          }
          runc --version
          CONTAINERD_RUNTIME=$TEST_RUNTIME make cri-integration

      - name: cri-tools critest
        env:
          TEST_RUNTIME: ${{ matrix.cri_runtime || matrix.runtime }}
          TEST_RUNTIME_PATH: ${{ matrix.cri_runtime_path }}
          CGROUP_DRIVER: ${{ matrix.cgroup_driver }}
        run: |
          env
          sudo -E PATH=$PATH ./script/critest.sh "${{github.workspace}}/report"

      # Experimental: the same critest suite through the pause-less sandbox
      # shim (containerd-shim-sandboxed-runc-v2 selected with runtime_path,
      # with the shim sandboxer). Specs that need what the shim does not
      # support yet are skipped by their full name; the list shrinks as the
      # features land:
      #  - "runtime should support PodPID": a shared pod PID namespace needs
      #    a durable PID 1 (sandbox-init).
      # The user namespace specs skip themselves: the handler does not
      # advertise user namespaces until the shim supports them.
      # The sandboxed shim is a Go shim: it runs on the runc.v2 legs only.
      - if: matrix.runtime == 'io.containerd.runc.v2'
        name: cri-tools critest (sandbox shim)
        env:
          TEST_RUNTIME: ${{ matrix.runtime }}
          TEST_RUNTIME_PATH: /usr/local/bin/containerd-shim-sandboxed-runc-v2
          SANDBOXER: shim
          CGROUP_DRIVER: ${{ matrix.cgroup_driver }}
          SKIP_TEST: "NamespaceOption runtime should support PodPID"
        run: |
          env
          sudo -E PATH=$PATH ./script/critest.sh "${{github.workspace}}/report-sandbox-shim"

      # The Rust shim does not implement checkpoint yet.
      - if: matrix.os != 'ubuntu-24.04-arm' && matrix.runtime != 'io.containerd.runc.v2-rs'
        name: Checkpoint/Restore Disable via CRI
        env:
          TEST_RUNTIME: ${{ matrix.runtime }}
          CGROUP_DRIVER: ${{ matrix.cgroup_driver }}
        run: |
          env
          sudo -E PATH=$PATH ./contrib/checkpoint/checkpoint-restore-cri-disable-test.sh

      # Log the status of this VM to investigate issues like
      # https://github.com/containerd/containerd/issues/4969
      - name: Host Status
        if: always()
        run: |
          set -x
          mount
          df
          losetup -l
      - name: Kernel Message
        if: failure()
        run: |
          sudo lsmod
          sudo dmesg -T -f kern

      - uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2
        if: always()
        with:
          name: TestResults ${{ matrix.runtime }} ${{matrix.runc}} ${{ matrix.os }} ${{ matrix.cgroup_driver }}
          path: |
            *-junit.xml
            *-gotest.json
            ${{github.workspace}}/report/*.xml
            ${{github.workspace}}/report/*.log
            ${{github.workspace}}/report-sandbox-shim/*.xml
            ${{github.workspace}}/report-sandbox-shim/*.log

  integration-vm:
    name: VM integration
    if: github.event_name != 'merge_group'
    runs-on: ubuntu-24.04
    timeout-minutes: 60
    needs: [project, linters, protos, man]

    strategy:
      fail-fast: false
      matrix:
        include:
          - template: fedora-44
            cgroup_driver: cgroupfs
            runc: runc
          - template: fedora-44
            cgroup_driver: systemd
            runc: runc
          - template: fedora-44
            cgroup_driver: cgroupfs
            runc: crun
          - template: fedora-44
            cgroup_driver: systemd
            runc: crun
            # We have to keep EL8 to test old glibc, cgroup, kernel, etc.
          - template: almalinux-8
            cgroup_driver: cgroupfs
            runc: runc
          - template: almalinux-8
            cgroup_driver: systemd
            runc: runc
          - template: almalinux-9
            cgroup_driver: cgroupfs
            runc: runc
          - template: almalinux-9
            cgroup_driver: systemd
            runc: runc
          - template: almalinux-10
            cgroup_driver: cgroupfs
            runc: runc
          - template: almalinux-10
            cgroup_driver: systemd
            runc: runc
    env:
      LIMA_TEMPLATE: ${{ matrix.template }}
      CGROUP_DRIVER: ${{ matrix.cgroup_driver }}
      RUNC_FLAVOR: ${{ matrix.runc }}
      LIMA_WORKDIR: /go/src/github.com/containerd/containerd
    steps:
      - name: Show the host info
        run: |
          set -x
          uname -a
          cat /etc/os-release
          cat /proc/cpuinfo
          free -mt
      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
        with:
          persist-credentials: false
      - uses: lima-vm/lima-actions/setup@55627e31b78637bf254a8b2a14da8ea7d12564e5 # v1.1.0
        id: lima-actions-setup
      - uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0
        with:
          path: ~/.cache/lima
          key: lima-${{ steps.lima-actions-setup.outputs.version }}-${{ matrix.template }}
      - name: Boot VM
        # --plain keeps the guest pristine (no guest agent, no mounts, no port forwards)
        run: |
          limactl start --plain --name=default --cpus=2 --memory=4 --disk=60 "template:$LIMA_TEMPLATE" || {
            # limactl can fail to download the VM image due to flakiness on the
            # upstream mirrors, wait a bit and retry
            limactl delete -f default || true
            sleep 60
            limactl start --plain --name=default --cpus=2 --memory=4 --disk=60 "template:$LIMA_TEMPLATE"
          }
      - name: Copy the source code to the VM
        env:
          # $LIMA_WORKDIR does not exist yet
          LIMA_WORKDIR: /
        run: |
          limactl cp -r . default:containerd
          lima bash -c 'sudo mkdir -p /go/src/github.com/containerd && sudo mv ~/containerd /go/src/github.com/containerd/containerd'
      - name: Provision the VM
        run: lima sudo RUNC_FLAVOR="$RUNC_FLAVOR" script/vm/provision.sh
      - name: test-integration
        run: lima sudo RUNC_FLAVOR="$RUNC_FLAVOR" script/vm/test-integration.sh
      - name: test-cri-integration
        run: lima sudo CGROUP_DRIVER="$CGROUP_DRIVER" RUNC_FLAVOR="$RUNC_FLAVOR" script/vm/test-cri-integration.sh
      - name: test-cri
        run: lima sudo CGROUP_DRIVER="$CGROUP_DRIVER" script/vm/test-cri.sh
      - name: dump-dmesg
        if: always()
        env:
          # $LIMA_WORKDIR may not exist if an earlier step failed
          LIMA_WORKDIR: /
        run: lima sudo dmesg -T -f kern


  integration-runsc:
    name: runsc (gVisor) integration
    runs-on: ubuntu-latest
    timeout-minutes: 60
    needs: [project, linters, protos, man]

    strategy:
      fail-fast: false
      matrix:
        cgroup_driver: [systemd, cgroupfs]

    env:
      CGROUP_DRIVER: ${{ matrix.cgroup_driver }}
      TEST_RUNTIME: io.containerd.runsc.v1

    steps:
      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
        with:
          persist-credentials: false
      - uses: ./.github/actions/install-go

      - name: Install containerd dependencies
        run: |
          sudo apt-get update
          sudo apt-get install -y gperf dmsetup strace xfsprogs
          script/setup/install-seccomp
          script/setup/install-runc
          script/setup/install-cni $(grep containernetworking/plugins go.mod | awk '{print $2}')
          script/setup/install-critools
          script/setup/install-failpoint-binaries

      - name: Install runsc (gVisor)
        run: script/runsc/install-runsc

      - name: Build containerd
        env:
          CGO_ENABLED: 1
        run: |
          make binaries GO_BUILD_FLAGS="-mod=vendor"
          sudo -E PATH=$PATH make install

      - name: CRI integration (runsc)
        run: |
          # Generate the runsc containerd config and pass it to cri-integration
          # so utils.sh skips its own config generation.
          cfg=$(mktemp /tmp/containerd-config-runsc-XXXXXX.toml)
          trap 'rm -f "$cfg"' EXIT
          script/runsc/write-runsc-shim-config.sh "$CGROUP_DRIVER" systrap
          script/runsc/write-containerd-runsc-config.sh "$cfg" "$CGROUP_DRIVER"
          # The external config and runsc shim config already contain the cgroup settings.
          # Keep CGROUP_DRIVER set but empty so utils.sh does not append runc.options again.
          CGROUP_DRIVER= CONTAINERD_CONFIG_FILE="$cfg" RUNTIME=runsc RUNTIME_TYPE=io.containerd.runsc.v1 make cri-integration

      - name: Runsc memory baseline
        run: |
          set -eux
          free -h
          cat /proc/meminfo | grep -E 'MemAvailable|SwapFree|SwapTotal'
          systemctl show containerd --property=MemoryMax --property=MemoryHigh || true
          sudo systemd-cgtop -b -n 1 || true

      - name: critest (runsc)
        env:
          GOMAXPROCS: "2"
          GINKGO_NODES: "2"
          GOMEMLIMIT: "4GiB"
        run: |
          mkdir -p "${{github.workspace}}/report"
          cfg=$(mktemp /tmp/containerd-config-runsc-XXXXXX.toml)
          trap 'rm -f "$cfg"' EXIT
          script/runsc/write-containerd-runsc-config.sh "$cfg" "$CGROUP_DRIVER"
          sudo \
            CGROUP_DRIVER="$CGROUP_DRIVER" \
            TEST_RUNTIME="$TEST_RUNTIME" \
            CONTAINERD_CONFIG_FILE="$cfg" \
            GOMAXPROCS="$GOMAXPROCS" \
            GOMEMLIMIT="$GOMEMLIMIT" \
            GINKGO_NODES="$GINKGO_NODES" \
            PATH="$PATH" \
            script/runsc/critest-runsc.sh "${{github.workspace}}/report"

      - name: Host Status
        if: always()
        run: |
          free -h
          df -h
          ps aux --sort=-%mem | head -n 30
          sudo systemctl status containerd --no-pager || true
          sudo journalctl -k --no-pager -n 200 || true
          sudo dmesg -T | grep -iE 'oom|out of memory|killed process' || true

      - name: Kernel Message
        if: failure()
        run: |
          sudo dmesg -T | tail -n 200 || true

      - uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2
        if: always()
        with:
          name: TestResults-runsc-${{ matrix.cgroup_driver }}
          path: |
            *-junit.xml
            *-gotest.json
            ${{github.workspace}}/report/*.xml
            ${{github.workspace}}/report/*.log


  tests-cri-in-userns:
    name: "CRI-in-UserNS"

    runs-on: ubuntu-24.04
    timeout-minutes: 40
    needs: [project, linters, protos, man]

    strategy:
      fail-fast: false
      matrix:
        cgroup_driver: [cgroupfs, systemd]

    steps:
      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
        with:
          persist-credentials: false
      - name: Set up cgroup v2 delegation
        run: |
          sudo mkdir -p /etc/systemd/system/user@.service.d
          cat <<EOF | sudo tee /etc/systemd/system/user@.service.d/delegate.conf
          [Service]
          Delegate=cpu cpuset io memory pids
          EOF
          sudo systemctl daemon-reload
      - name: Build cri-in-userns image
        env:
          CGROUP_DRIVER: ${{ matrix.cgroup_driver }}
        run: |
          if [ "$CGROUP_DRIVER" = "systemd" ];then
            podman build --target cri-in-userns-systemd -t cri-in-userns-systemd -f ./contrib/Dockerfile.test .
          else
            podman build --target cri-in-userns -t cri-in-userns -f ./contrib/Dockerfile.test .
          fi

      - name: Run cri-in-userns image
        env:
          CGROUP_DRIVER: ${{ matrix.cgroup_driver }}
        # Rootless Podman is used for testing CRI-in-UserNS
        # (We could use rootless Docker or rootless nerdctl, but we are using Podman here because it is preinstalled)
        run: |
          if [ "$CGROUP_DRIVER" = "systemd" ];then
              set +e
              touch ./critest_exit_code.txt
              podman run --rm --privileged --group-add keep-groups -v ./critest_exit_code.txt:/tmp/critest_exit_code.txt cri-in-userns-systemd
              exit_code=`cat ./critest_exit_code.txt`
              echo "exit_code:"$exit_code
              if [ "$exit_code" -gt 0 ]; then
                exit 1
              else
                exit 0
              fi
          else
            podman run --rm --privileged cri-in-userns
          fi

  tests-mac-os:
    name: MacOS unit tests
    runs-on: macos-latest
    timeout-minutes: 10
    needs: [project, linters, protos, man]
    env:
      GOTEST: gotestsum --

    steps:
      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
        with:
          persist-credentials: false
      - uses: ./.github/actions/install-go
      - run: script/setup/install-gotestsum
      - run: script/setup/install-teststat
      - name: Tests
        env:
          GOTESTSUM_JUNITFILE: "${{ github.workspace }}/macos-test-junit.xml"
          GOTESTSUM_JSONFILE: "${{ github.workspace }}/macos-test-gotest.json"
        run: make test
      - run: if [ -f *-gotest.json ]; then echo '# Unit Tests' >> $GITHUB_STEP_SUMMARY; teststat -markdown *-gotest.json >> $GITHUB_STEP_SUMMARY; fi
        if: always()
      - run: script/test/test2annotation.sh *-gotest.json
        if: always()
      - uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2
        if: always()
        with:
          name: TestResults MacOS
          path: |
            *-junit.xml
            *-gotest.json

  node-e2e:
    name: E2E
    if: github.event_name != 'merge_group'
    uses: ./.github/workflows/node-e2e.yml

  # Currently Github actions UI supports no masks to mark matrix jobs as required to pass status checks.
  # This means that every time version of Go, containerd, or OS is changed, a corresponding job should
  # be added to the list of required checks. Which is not very convenient.
  # To workaround this, a special job is added to report statuses of all other jobs, with fixed title.
  # So it needs to be added to the list of required checks only once.
  #
  # See https://github.com/orgs/community/discussions/26822
  results:
    name: Report required job statuses
    runs-on: ubuntu-24.04
    # List job dependencies which are required to pass status checks in order to be merged via merge queue.
    needs: [linters, project, protos, binaries, integration-linux, integration-windows, integration-runsc, tests-cri-in-userns, tests-mac-os, node-e2e]
    if: ${{ always() }}
    steps:
      - run: exit 1
        if: ${{ contains(needs.*.result, 'failure') || contains(needs.*.result, 'cancelled') }}