From c4198e6053b1d4840f9c6e6dd152314548539356 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pawe=C5=82=20Gronowski?= Date: Wed, 19 Apr 2023 14:48:47 +0200 Subject: [PATCH 1/4] Dockerfile: Use separate cli for shell and integration-cli MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Use separate cli for integration-cli to allow use newer CLI for interactive dev shell usage. Both versions can be overriden with DOCKERCLI_VERSION or DOCKERCLI_INTEGRATION_VERSION. Binary is downloaded from download.docker.com if it's available, otherwise it's built from the source. For backwards compatibility DOCKER_CLI_PATH overrides BOTH clis. Signed-off-by: Paweł Gronowski (cherry picked from commit 17c99f716458575cdfb6311067720e3b6e5f8bc0) Signed-off-by: Paweł Gronowski --- Dockerfile | 50 ++++++++++++++--------------- Makefile | 8 +++++ hack/dockerfile/cli.sh | 29 +++++++++++++++++ hack/make/.integration-daemon-start | 6 ++-- integration-cli/check_test.go | 14 ++++++++ 5 files changed, 79 insertions(+), 28 deletions(-) create mode 100755 hack/dockerfile/cli.sh diff --git a/Dockerfile b/Dockerfile index a9f9725e61..4b3171c0ef 100644 --- a/Dockerfile +++ b/Dockerfile @@ -6,7 +6,12 @@ ARG GOLANG_IMAGE="golang:${GO_VERSION}-${BASE_DEBIAN_DISTRO}" ARG XX_VERSION=1.2.1 ARG VPNKIT_VERSION=0.5.0 -ARG DOCKERCLI_VERSION=v17.06.2-ce + +ARG DOCKERCLI_REPOSITORY="https://github.com/docker/cli.git" +ARG DOCKERCLI_VERSION=v24.0.2 +# cli version used for integration-cli tests +ARG DOCKERCLI_INTEGRATION_REPOSITORY="https://github.com/docker/cli.git" +ARG DOCKERCLI_INTEGRATION_VERSION=v17.06.2-ce ARG SYSTEMD="false" ARG DEBIAN_FRONTEND=noninteractive @@ -243,34 +248,25 @@ RUN --mount=type=cache,target=/root/.cache/go-build \ GOBIN=/build/ GO111MODULE=on go install "mvdan.cc/sh/v3/cmd/shfmt@${SHFMT_VERSION}" \ && /build/shfmt --version -# dockercli -FROM base AS dockercli-src -WORKDIR /tmp/dockercli -RUN git init . && git remote add origin "https://github.com/docker/cli.git" -ARG DOCKERCLI_VERSION -RUN git fetch -q --depth 1 origin "${DOCKERCLI_VERSION}" +refs/tags/*:refs/tags/* && git checkout -q FETCH_HEAD -RUN [ -d ./components/cli ] && mv ./components/cli /usr/src/dockercli || mv /tmp/dockercli /usr/src/dockercli -WORKDIR /usr/src/dockercli - FROM base AS dockercli WORKDIR /go/src/github.com/docker/cli +COPY hack/dockerfile/cli.sh /download-or-build-cli.sh +ARG DOCKERCLI_REPOSITORY ARG DOCKERCLI_VERSION -ARG DOCKERCLI_CHANNEL=stable ARG TARGETPLATFORM -RUN xx-apt-get install -y --no-install-recommends gcc libc6-dev -RUN --mount=from=dockercli-src,src=/usr/src/dockercli,rw \ - --mount=type=cache,target=/root/.cache/go-build,id=dockercli-build-$TARGETPLATFORM </dev/null 2>&1; then - mkdir /build - curl -Ls "${DOWNLOAD_URL}" | tar -xz docker/docker - mv docker/docker /build/docker - else - CGO_ENABLED=0 xx-go build -o /build/docker ./cmd/docker - fi - xx-verify /build/docker -EOT +RUN --mount=type=cache,id=dockercli-git-$TARGETPLATFORM,target=./.git \ + --mount=type=cache,target=/root/.cache/go-build,id=dockercli-build-$TARGETPLATFORM \ + /download-or-build-cli.sh ${DOCKERCLI_VERSION} ${DOCKERCLI_REPOSITORY} /build + +FROM base AS dockercli-integration +WORKDIR /go/src/github.com/docker/cli +COPY hack/dockerfile/cli.sh /download-or-build-cli.sh +ARG DOCKERCLI_INTEGRATION_REPOSITORY +ARG DOCKERCLI_INTEGRATION_VERSION +ARG TARGETPLATFORM +RUN --mount=type=cache,id=dockercli-integration-git-$TARGETPLATFORM,target=./.git \ + --mount=type=cache,target=/root/.cache/go-build,id=dockercli-integration-build-$TARGETPLATFORM \ + /download-or-build-cli.sh ${DOCKERCLI_INTEGRATION_VERSION} ${DOCKERCLI_INTEGRATION_REPOSITORY} /build # runc FROM base AS runc-src @@ -439,7 +435,6 @@ FROM containerutil-windows-${TARGETARCH} AS containerutil-windows FROM containerutil-${TARGETOS} AS containerutil FROM base AS dev-systemd-false -COPY --link --from=dockercli /build/ /usr/local/cli COPY --link --from=frozen-images /build/ /docker-frozen-images COPY --link --from=swagger /build/ /usr/local/bin/ COPY --link --from=delve /build/ /usr/local/bin/ @@ -464,11 +459,14 @@ COPY --link --from=containerutil /build/ /usr/local/bin/ COPY --link --from=crun /build/ /usr/local/bin/ COPY --link hack/dockerfile/etc/docker/ /etc/docker/ ENV PATH=/usr/local/cli:$PATH +ENV TEST_CLIENT_BINARY=/usr/local/cli-integration/docker ENV CONTAINERD_ADDRESS=/run/docker/containerd/containerd.sock ENV CONTAINERD_NAMESPACE=moby WORKDIR /go/src/github.com/docker/docker VOLUME /var/lib/docker VOLUME /home/unprivilegeduser/.local/share/docker +COPY --link --from=dockercli /build/ /usr/local/cli +COPY --link --from=dockercli-integration /build/ /usr/local/cli-integration # Wrap all commands in the "docker-in-docker" script to allow nested containers ENTRYPOINT ["hack/dind"] diff --git a/Makefile b/Makefile index 97ad5915f3..17aebbd26c 100644 --- a/Makefile +++ b/Makefile @@ -41,6 +41,10 @@ DOCKER_ENVS := \ -e DOCKER_BUILDKIT \ -e DOCKER_BASH_COMPLETION_PATH \ -e DOCKER_CLI_PATH \ + -e DOCKERCLI_VERSION \ + -e DOCKERCLI_REPOSITORY \ + -e DOCKERCLI_INTEGRATION_VERSION \ + -e DOCKERCLI_INTEGRATION_REPOSITORY \ -e DOCKER_DEBUG \ -e DOCKER_EXPERIMENTAL \ -e DOCKER_GITCOMMIT \ @@ -136,6 +140,10 @@ endif DOCKER_RUN_DOCKER := $(DOCKER_FLAGS) "$(DOCKER_IMAGE)" DOCKER_BUILD_ARGS += --build-arg=GO_VERSION +DOCKER_BUILD_ARGS += --build-arg=DOCKERCLI_VERSION +DOCKER_BUILD_ARGS += --build-arg=DOCKERCLI_REPOSITORY +DOCKER_BUILD_ARGS += --build-arg=DOCKERCLI_INTEGRATION_VERSION +DOCKER_BUILD_ARGS += --build-arg=DOCKERCLI_INTEGRATION_REPOSITORY ifdef DOCKER_SYSTEMD DOCKER_BUILD_ARGS += --build-arg=SYSTEMD=true endif diff --git a/hack/dockerfile/cli.sh b/hack/dockerfile/cli.sh new file mode 100755 index 0000000000..6af829210a --- /dev/null +++ b/hack/dockerfile/cli.sh @@ -0,0 +1,29 @@ +#!/bin/sh + +set -e +version="$1" +repository="$2" +outdir="$3" + +DOWNLOAD_URL="https://download.docker.com/linux/static/stable/$(xx-info march)/docker-${version#v}.tgz" + +mkdir "$outdir" +if curl --head --silent --fail "${DOWNLOAD_URL}" 1> /dev/null 2>&1; then + curl -Ls "${DOWNLOAD_URL}" | tar -xz docker/docker + mv docker/docker "${outdir}/docker" +else + git init -q . + git remote remove origin || true + git remote add origin "${repository}" + git fetch -q --depth 1 origin "${version}" +refs/tags/*:refs/tags/* + git checkout -fq "${version}" + if [ -d ./components/cli ]; then + mv ./components/cli/* ./ + CGO_ENABLED=0 xx-go build -o "${outdir}/docker" ./cmd/docker + git reset --hard "${version}" + else + xx-go --wrap && CGO_ENABLED=0 TARGET="${outdir}" ./scripts/build/binary + fi +fi + +xx-verify "${outdir}/docker" diff --git a/hack/make/.integration-daemon-start b/hack/make/.integration-daemon-start index 766e09f7fb..89e38993db 100644 --- a/hack/make/.integration-daemon-start +++ b/hack/make/.integration-daemon-start @@ -5,9 +5,11 @@ base="$ABS_DEST/.." export PATH="$base/dynbinary-daemon:$base/binary-daemon:$PATH" -export TEST_CLIENT_BINARY=docker - +if [ -z "$TEST_CLIENT_BINARY" ]; then + export TEST_CLIENT_BINARY=docker +fi if [ -n "$DOCKER_CLI_PATH" ]; then + # /usr/local/cli is a bind mount to the base dir of DOCKER_CLI_PATH (if used) export TEST_CLIENT_BINARY=/usr/local/cli/$(basename "$DOCKER_CLI_PATH") fi diff --git a/integration-cli/check_test.go b/integration-cli/check_test.go index db6683709d..dd036f3f0e 100644 --- a/integration-cli/check_test.go +++ b/integration-cli/check_test.go @@ -67,9 +67,23 @@ func TestMain(m *testing.M) { } testEnv.Print() + printCliVersion() os.Exit(m.Run()) } +func printCliVersion() { + // Print output of "docker version" + cli.SetTestEnvironment(testEnv) + cmd := cli.Docker(cli.Args("version")) + if cmd.Error != nil { + fmt.Printf("WARNING: Failed to run \"docker version\": %+v\n", cmd.Error) + return + } + + fmt.Println("INFO: Testing with docker cli version:") + fmt.Println(cmd.Stdout()) +} + func ensureTestEnvSetup(t *testing.T) { testEnvOnce.Do(func() { cli.SetTestEnvironment(testEnv) From 1a078977e1e5d02883917c1ac1c7054825bf4314 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pawe=C5=82=20Gronowski?= Date: Wed, 1 Feb 2023 14:17:36 +0100 Subject: [PATCH 2/4] Dockerfile/shell: Install buildx cli plugin MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Installs the buildx cli plugin in the container shell by default. Previously user had to manually download the buildx binary to use buildkit. Signed-off-by: Paweł Gronowski (cherry picked from commit 49f76a34b53700a4724dd71ed4348f6068c4c70b) Signed-off-by: Paweł Gronowski --- Dockerfile | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/Dockerfile b/Dockerfile index 4b3171c0ef..38bde1c40d 100644 --- a/Dockerfile +++ b/Dockerfile @@ -12,6 +12,7 @@ ARG DOCKERCLI_VERSION=v24.0.2 # cli version used for integration-cli tests ARG DOCKERCLI_INTEGRATION_REPOSITORY="https://github.com/docker/cli.git" ARG DOCKERCLI_INTEGRATION_VERSION=v17.06.2-ce +ARG BUILDX_VERSION=0.10.5 ARG SYSTEMD="false" ARG DEBIAN_FRONTEND=noninteractive @@ -433,6 +434,7 @@ FROM binary-dummy AS containerutil-linux FROM containerutil-build AS containerutil-windows-amd64 FROM containerutil-windows-${TARGETARCH} AS containerutil-windows FROM containerutil-${TARGETOS} AS containerutil +FROM docker/buildx-bin:${BUILDX_VERSION} as buildx FROM base AS dev-systemd-false COPY --link --from=frozen-images /build/ /docker-frozen-images @@ -458,6 +460,8 @@ COPY --link --from=vpnkit / /usr/local/bin/ COPY --link --from=containerutil /build/ /usr/local/bin/ COPY --link --from=crun /build/ /usr/local/bin/ COPY --link hack/dockerfile/etc/docker/ /etc/docker/ +COPY --link --from=buildx /buildx /usr/local/libexec/docker/cli-plugins/docker-buildx + ENV PATH=/usr/local/cli:$PATH ENV TEST_CLIENT_BINARY=/usr/local/cli-integration/docker ENV CONTAINERD_ADDRESS=/run/docker/containerd/containerd.sock From e5fbc3f75ae155651a073ace8341d2d0571e36d9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pawe=C5=82=20Gronowski?= Date: Mon, 5 Jun 2023 16:31:15 +0200 Subject: [PATCH 3/4] hack/cli.sh: Quiet origin cleanup MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Don't show `error: No such remote: 'origin'` error when building for the first time and the cached git repository doesn't a remote yet. Signed-off-by: Paweł Gronowski (cherry picked from commit 5433b88e2dbd43852a0f1f9625fefcf69a6ea688) Signed-off-by: Paweł Gronowski --- hack/dockerfile/cli.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hack/dockerfile/cli.sh b/hack/dockerfile/cli.sh index 6af829210a..f821994650 100755 --- a/hack/dockerfile/cli.sh +++ b/hack/dockerfile/cli.sh @@ -13,7 +13,7 @@ if curl --head --silent --fail "${DOWNLOAD_URL}" 1> /dev/null 2>&1; then mv docker/docker "${outdir}/docker" else git init -q . - git remote remove origin || true + git remote remove origin 2> /dev/null || true git remote add origin "${repository}" git fetch -q --depth 1 origin "${version}" +refs/tags/*:refs/tags/* git checkout -fq "${version}" From 61d547fd06cbbe362dd94c22d85d5e44ae635611 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pawe=C5=82=20Gronowski?= Date: Mon, 5 Jun 2023 16:32:24 +0200 Subject: [PATCH 4/4] Dockerfile: Move dockercli to base-dev MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Avoids invalidation of dev-systemd-true and dev-base when changing the CLI version/repository. Signed-off-by: Paweł Gronowski (cherry picked from commit 0f9c8e684a3132b0cfe1f3a04fec68e5f8f6bfc6) Signed-off-by: Paweł Gronowski --- Dockerfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index 38bde1c40d..04a48239ce 100644 --- a/Dockerfile +++ b/Dockerfile @@ -469,8 +469,6 @@ ENV CONTAINERD_NAMESPACE=moby WORKDIR /go/src/github.com/docker/docker VOLUME /var/lib/docker VOLUME /home/unprivilegeduser/.local/share/docker -COPY --link --from=dockercli /build/ /usr/local/cli -COPY --link --from=dockercli-integration /build/ /usr/local/cli-integration # Wrap all commands in the "docker-in-docker" script to allow nested containers ENTRYPOINT ["hack/dind"] @@ -551,6 +549,8 @@ RUN --mount=type=cache,sharing=locked,id=moby-dev-aptlib,target=/var/lib/apt \ libsecret-1-dev \ libsystemd-dev \ libudev-dev +COPY --link --from=dockercli /build/ /usr/local/cli +COPY --link --from=dockercli-integration /build/ /usr/local/cli-integration FROM base AS build COPY --from=gowinres /build/ /usr/local/bin/