integration: build the whiteout-test image locally

Build the image for TestIssue13030 locally using pkg/archive,
instead of pulling the prebuilt ghcr.io/containerd/whiteout-test image.

The image is built by appending layers generated with archive.WriteDiff
on top of the busybox base image, served with the new
testutil.ServeImage helper on a localhost HTTP registry, and pulled
through the regular client.Pull path with WithPullUnpack. This removes
the need to publish the multi-arch whiteout-test image, so the
Dockerfile and Makefile added in PR 13704 are removed along with the
image list entry.

Assisted-by: Claude Fable 5 <noreply@anthropic.com>
Signed-off-by: Akihiro Suda <akihiro.suda.cz@hco.ntt.co.jp>

integration/client/container_linux_test.go

@@ -19,6 +19,7 @@ package client
 import (
 	"bytes"
 	"context"
+	"encoding/json"
 	"fmt"
 	"io"
 	"os"
@@ -40,15 +41,20 @@ import (
 
 	. "github.com/containerd/containerd/v2/client"
 	"github.com/containerd/containerd/v2/core/containers"
+	coreimages "github.com/containerd/containerd/v2/core/images"
 	"github.com/containerd/containerd/v2/integration/failpoint"
 	"github.com/containerd/containerd/v2/integration/images"
+	"github.com/containerd/containerd/v2/pkg/archive"
 	"github.com/containerd/containerd/v2/pkg/cio"
 	"github.com/containerd/containerd/v2/pkg/fifosync"
 	"github.com/containerd/containerd/v2/pkg/oci"
 	"github.com/containerd/containerd/v2/pkg/shim"
 	"github.com/containerd/containerd/v2/pkg/sys"
+	"github.com/containerd/containerd/v2/pkg/testutil"
 	"github.com/containerd/containerd/v2/plugins"
 
+	"github.com/opencontainers/go-digest"
+	ocispec "github.com/opencontainers/image-spec/specs-go/v1"
 	"github.com/opencontainers/runtime-spec/specs-go"
 	"github.com/stretchr/testify/require"
 	"golang.org/x/sync/semaphore"
@@ -1815,6 +1821,79 @@ func TestIssue10589(t *testing.T) {
 	assert.Equal(t, Stopped, status.Status)
 }
 
+// buildWhiteoutImage builds an image on top of the base image, appending
+// layers generated with archive.WriteDiff that create files and then delete
+// them again with whiteouts:
+//
+//	touch /file-to-delete
+//	rm /file-to-delete
+//	mkdir /dir-to-delete && touch /dir-to-delete/foo
+//	rm -rf /dir-to-delete
+//
+// The image is returned as the blobs of the new layers, config, and manifest
+// keyed by digest, together with the manifest descriptor, to be served with
+// testutil.ServeImage. The blobs of the base image are not included; they
+// are expected to be present in the content store already.
+func buildWhiteoutImage(ctx context.Context, t *testing.T, client *Client, base Image) (map[digest.Digest][]byte, ocispec.Descriptor) {
+	blobs := map[digest.Digest][]byte{}
+
+	empty := t.TempDir()
+	withFile := t.TempDir()
+	require.NoError(t, os.WriteFile(filepath.Join(withFile, "file-to-delete"), nil, 0644))
+	withDir := t.TempDir()
+	require.NoError(t, os.Mkdir(filepath.Join(withDir, "dir-to-delete"), 0755))
+	require.NoError(t, os.WriteFile(filepath.Join(withDir, "dir-to-delete", "foo"), nil, 0644))
+
+	config, err := base.Spec(ctx)
+	require.NoError(t, err)
+	manifest, err := coreimages.Manifest(ctx, client.ContentStore(), base.Target(), platforms.Default())
+	require.NoError(t, err)
+
+	for _, l := range []struct {
+		createdBy    string
+		lower, upper string
+	}{
+		{"touch /file-to-delete", empty, withFile},
+		{"rm /file-to-delete", withFile, empty},
+		{"mkdir /dir-to-delete && touch /dir-to-delete/foo", empty, withDir},
+		{"rm -rf /dir-to-delete", withDir, empty},
+	} {
+		var buf bytes.Buffer
+		require.NoError(t, archive.WriteDiff(ctx, &buf, l.lower, l.upper))
+		desc := ocispec.Descriptor{
+			MediaType: ocispec.MediaTypeImageLayer,
+			Digest:    digest.FromBytes(buf.Bytes()),
+			Size:      int64(buf.Len()),
+		}
+		blobs[desc.Digest] = buf.Bytes()
+		manifest.Layers = append(manifest.Layers, desc)
+		// The layer is uncompressed, so its diff ID is its blob digest.
+		config.RootFS.DiffIDs = append(config.RootFS.DiffIDs, desc.Digest)
+		config.History = append(config.History, ocispec.History{CreatedBy: l.createdBy})
+	}
+
+	configBlob, err := json.Marshal(config)
+	require.NoError(t, err)
+	manifest.Config = ocispec.Descriptor{
+		MediaType: ocispec.MediaTypeImageConfig,
+		Digest:    digest.FromBytes(configBlob),
+		Size:      int64(len(configBlob)),
+	}
+	blobs[manifest.Config.Digest] = configBlob
+
+	manifest.MediaType = ocispec.MediaTypeImageManifest
+	manifestBlob, err := json.Marshal(manifest)
+	require.NoError(t, err)
+	target := ocispec.Descriptor{
+		MediaType: ocispec.MediaTypeImageManifest,
+		Digest:    digest.FromBytes(manifestBlob),
+		Size:      int64(len(manifestBlob)),
+	}
+	blobs[target.Digest] = manifestBlob
+
+	return blobs, target
+}
+
 // TestIssue13030 is a regression test for parallel image unpacking.
 // The test validates that when multiple layers are unpacked in parallel,
 // that whiteout files are properly processed and do not cause files to
@@ -1831,14 +1910,22 @@ func TestIssue13030(t *testing.T) {
 	ctx, cancel := testContext(t)
 	t.Cleanup(cancel)
 
+	// Pull the base image without unpacking it, so that all the layers of
+	// the derived image built below are unpacked in parallel.
+	base, err := client.Pull(ctx, images.Get(images.BusyBox), WithPlatformMatcher(platforms.Default()))
+	require.NoError(t, err)
+
+	blobs, target := buildWhiteoutImage(ctx, t, client, base)
+	ref := testutil.ServeImage(t, "whiteout-test", "latest", target, blobs)
+
 	image, err := client.Pull(ctx,
-		images.Get(images.Whiteout),
+		ref,
 		WithPlatformMatcher(platforms.Default()),
 		WithPullUnpack,
 		WithUnpackOpts([]UnpackOpt{WithUnpackLimiter(semaphore.NewWeighted(3))}),
 	)
 	t.Cleanup(func() {
-		client.ImageService().Delete(ctx, images.Get(images.Whiteout))
+		client.ImageService().Delete(ctx, ref)
 	})
 	if err != nil {
 		t.Fatal(err)

integration/images/image_list.go

@@ -39,7 +39,6 @@ type ImageList struct {
 	VolumeOwnership  string
 	ArgsEscaped      string
 	Nginx            string
-	Whiteout         string
 }
 
 var (
@@ -60,7 +59,6 @@ func initImages(imageListFile string) {
 		VolumeOwnership:  "ghcr.io/containerd/volume-ownership:2.1",
 		ArgsEscaped:      "cplatpublic.azurecr.io/args-escaped-test-image-ns:1.0",
 		Nginx:            "ghcr.io/containerd/nginx:1.27.0",
-		Whiteout:         "ghcr.io/containerd/whiteout-test:1.0",
 	}
 
 	if imageListFile != "" {
@@ -102,8 +100,6 @@ const (
 	ArgsEscaped
 	// Nginx image
 	Nginx
-	// Whiteout image
-	Whiteout
 )
 
 func initImageMap(imageList ImageList) map[int]string {
@@ -117,7 +113,6 @@ func initImageMap(imageList ImageList) map[int]string {
 	images[VolumeOwnership] = imageList.VolumeOwnership
 	images[ArgsEscaped] = imageList.ArgsEscaped
 	images[Nginx] = imageList.Nginx
-	images[Whiteout] = imageList.Whiteout
 	return images
 }
 

integration/images/whiteout-test/Dockerfile

@@ -1,21 +0,0 @@
-#   Copyright The containerd Authors.
-
-#   Licensed under the Apache License, Version 2.0 (the "License");
-#   you may not use this file except in compliance with the License.
-#   You may obtain a copy of the License at
-
-#       http://www.apache.org/licenses/LICENSE-2.0
-
-#   Unless required by applicable law or agreed to in writing, software
-#   distributed under the License is distributed on an "AS IS" BASIS,
-#   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-#   See the License for the specific language governing permissions and
-#   limitations under the License.
-
-FROM alpine
-
-RUN touch /file-to-delete
-RUN rm /file-to-delete
-
-RUN mkdir /dir-to-delete && touch /dir-to-delete/foo
-RUN rm -rf /dir-to-delete

integration/images/whiteout-test/Makefile

@@ -1,60 +0,0 @@
-#   Copyright The containerd Authors.
-
-#   Licensed under the Apache License, Version 2.0 (the "License");
-#   you may not use this file except in compliance with the License.
-#   You may obtain a copy of the License at
-
-#       http://www.apache.org/licenses/LICENSE-2.0
-
-#   Unless required by applicable law or agreed to in writing, software
-#   distributed under the License is distributed on an "AS IS" BASIS,
-#   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-#   See the License for the specific language governing permissions and
-#   limitations under the License.
-
-all: build
-
-PROJ ?= ghcr.io/containerd
-NAME ?= whiteout-test
-VERSION ?= 1.0
-IMAGE ?= $(PROJ)/$(NAME):$(VERSION)
-
-DOCKER ?= docker
-MIN_DOCKER_MAJOR ?= 29
-DOCKER_CLIENT_VERSION = $(shell $(DOCKER) version --format '{{.Client.Version}}' 2>/dev/null)
-DOCKER_CLIENT_MAJOR = $(firstword $(subst ., ,$(DOCKER_CLIENT_VERSION)))
-
-# Use the same helper image as https://github.com/containerd/containerd/blob/main/test/init-buildx.sh#L78
-BINFMT_IMAGE ?= multiarch/qemu-user-static@sha256:c772ee1965aa0be9915ee1b018a0dd92ea361b4fa1bcab5bbc033517749b2af4
-PLATFORMS ?= linux/386,linux/amd64,linux/arm/v6,linux/arm/v7,linux/arm64/v8,linux/ppc64le,linux/riscv64,linux/s390x
-
-# Optional/manual: register binfmt handlers for cross-arch emulation.
-setup-binfmt:
-	$(DOCKER) run --privileged --rm $(BINFMT_IMAGE) --reset -p yes
-
-check-docker-version:
-	@if [ -z "$(DOCKER_CLIENT_VERSION)" ]; then \
-		echo "error: failed to detect Docker client version"; \
-		exit 1; \
-	elif ! [ "$(DOCKER_CLIENT_MAJOR)" -eq "$(DOCKER_CLIENT_MAJOR)" ] 2>/dev/null; then \
-		echo "error: failed to parse Docker major version from $(DOCKER_CLIENT_VERSION)"; \
-		exit 1; \
-	elif [ "$(DOCKER_CLIENT_MAJOR)" -lt "$(MIN_DOCKER_MAJOR)" ]; then \
-		echo "error: this Makefile requires Docker >= $(MIN_DOCKER_MAJOR); found $(DOCKER_CLIENT_VERSION)"; \
-		exit 1; \
-	fi
-
-build: check-docker-version
-	$(DOCKER) buildx build --platform $(PLATFORMS) -t $(IMAGE) .
-
-push: build
-	$(DOCKER) push $(IMAGE)
-
-print-config:
-	@echo "DOCKER=$(DOCKER)"
-	@echo "MIN_DOCKER_MAJOR=$(MIN_DOCKER_MAJOR)"
-	@echo "DOCKER_CLIENT_VERSION=$(DOCKER_CLIENT_VERSION)"
-	@echo "IMAGE=$(IMAGE)"
-	@echo "PLATFORMS=$(PLATFORMS)"
-
-.PHONY: all setup-binfmt check-docker-version build push print-config

pkg/testutil/registry.go

@@ -0,0 +1,92 @@
+/*
+   Copyright The containerd Authors.
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.
+*/
+
+package testutil
+
+import (
+	"net/http"
+	"net/http/httptest"
+	"strconv"
+	"strings"
+	"testing"
+
+	"github.com/opencontainers/go-digest"
+	ocispec "github.com/opencontainers/image-spec/specs-go/v1"
+)
+
+// ServeImage serves an image from a read-only HTTP registry listening on
+// localhost, and returns a reference to pull the image from.
+//
+// blobs must be keyed by digest and contain the manifest blob referenced by
+// target, along with the blobs it references, except for those already
+// present in the puller's content store. The registry is shut down when the
+// test completes.
+func ServeImage(t *testing.T, name, tag string, target ocispec.Descriptor, blobs map[digest.Digest][]byte) string {
+	manifestPath := "/v2/" + name + "/manifests/"
+	blobPath := "/v2/" + name + "/blobs/"
+	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		if r.Method != http.MethodGet && r.Method != http.MethodHead {
+			w.WriteHeader(http.StatusMethodNotAllowed)
+			return
+		}
+		var b []byte
+		mediaType := "application/octet-stream"
+		switch {
+		case strings.HasPrefix(r.URL.Path, manifestPath):
+			// A manifest may be requested by the tag, by the target
+			// digest, or, for a multi-platform image, by the digest of
+			// a platform manifest.
+			dgst := target.Digest
+			if ref := strings.TrimPrefix(r.URL.Path, manifestPath); ref != tag {
+				var err error
+				if dgst, err = digest.Parse(ref); err != nil {
+					w.WriteHeader(http.StatusNotFound)
+					return
+				}
+			}
+			var ok bool
+			if b, ok = blobs[dgst]; !ok {
+				w.WriteHeader(http.StatusNotFound)
+				return
+			}
+			if dgst == target.Digest {
+				mediaType = target.MediaType
+			}
+			w.Header().Set("Docker-Content-Digest", dgst.String())
+		case strings.HasPrefix(r.URL.Path, blobPath):
+			dgst, err := digest.Parse(strings.TrimPrefix(r.URL.Path, blobPath))
+			if err != nil {
+				w.WriteHeader(http.StatusBadRequest)
+				return
+			}
+			var ok bool
+			if b, ok = blobs[dgst]; !ok {
+				w.WriteHeader(http.StatusNotFound)
+				return
+			}
+		default:
+			w.WriteHeader(http.StatusNotFound)
+			return
+		}
+		w.Header().Set("Content-Type", mediaType)
+		w.Header().Set("Content-Length", strconv.Itoa(len(b)))
+		if r.Method == http.MethodGet {
+			w.Write(b)
+		}
+	}))
+	t.Cleanup(srv.Close)
+	return strings.TrimPrefix(srv.URL, "http://") + "/" + name + ":" + tag
+}