From 394eaa8495f4906ca0223d905d1c5ea504a1eabb Mon Sep 17 00:00:00 2001 From: Bjorn Neergaard Date: Mon, 14 Nov 2022 14:54:35 -0700 Subject: [PATCH 1/6] hack/vendor.sh: allow running tidy & vendor separately Signed-off-by: Bjorn Neergaard (cherry picked from commit 72568286abcd7e2feb6625b58757dc2924c2b44f) Signed-off-by: Bjorn Neergaard --- hack/vendor.sh | 33 ++++++++++++++++++++++++++++----- 1 file changed, 28 insertions(+), 5 deletions(-) diff --git a/hack/vendor.sh b/hack/vendor.sh index 52eeda2460..d106494be9 100755 --- a/hack/vendor.sh +++ b/hack/vendor.sh @@ -1,14 +1,37 @@ #!/usr/bin/env bash - -# This file is just wrapper around 'go mod vendor' tool. +# +# This file is just a wrapper around the 'go mod vendor' tool. # For updating dependencies you should change `vendor.mod` file in root of the # project. set -e -set -x SCRIPTDIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" "${SCRIPTDIR}"/go-mod-prepare.sh -GO111MODULE=auto go mod tidy -modfile 'vendor.mod' -compat 1.18 -GO111MODULE=auto go mod vendor -modfile vendor.mod +export GO111MODULE=on + +tidy() ( + set -x + go mod tidy -modfile vendor.mod -compat 1.18 +) + +vendor() ( + set -x + go mod vendor -modfile vendor.mod +) + +help() { + printf "%s:\n" "$(basename "$0")" + echo " - tidy: run go mod tidy" + echo " - vendor: run go mod vendor" + echo " - all: run tidy && vendor" + echo " - help: show this help" +} + +case "$1" in + tidy) tidy ;; + vendor) vendor ;; + ""|all) tidy && vendor ;; + *) help ;; +esac From a4957d2585db4ad5da519985d4584bbed6c57a65 Mon Sep 17 00:00:00 2001 From: Bjorn Neergaard Date: Mon, 14 Nov 2022 14:56:53 -0700 Subject: [PATCH 2/6] hack/validate/vendor: split tidy from vendor Signed-off-by: Bjorn Neergaard (cherry picked from commit dbd76848ca32187e4e8538252d30a97a5a0a452a) Signed-off-by: Bjorn Neergaard --- hack/validate/vendor | 63 ++++++++++++++++++++++---------------------- 1 file changed, 32 insertions(+), 31 deletions(-) diff --git a/hack/validate/vendor b/hack/validate/vendor index 851bf6e249..8c589b0e59 100755 --- a/hack/validate/vendor +++ b/hack/validate/vendor @@ -1,53 +1,54 @@ #!/usr/bin/env bash +set -e + SCRIPTDIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" source "${SCRIPTDIR}/.validate" +tidy_files=('vendor.mod' 'vendor.sum') +vendor_files=("${tidy_files[@]}" 'vendor/') + +validate_vendor_tidy() { + # run mod tidy + ./hack/vendor.sh tidy + # check if any files have changed + git diff --quiet HEAD -- "${tidy_files[@]}" +} + validate_vendor_diff() { - IFS=$'\n' - check_files=('vendor.sum' 'vendor.mod' 'vendor/') - # shellcheck disable=SC2207 - changed_files=($(validate_diff --diff-filter=ACMR --name-only -- "${check_files[@]}" || true)) - unset IFS + mapfile -t changed_files < <(validate_diff --diff-filter=ACMR --name-only -- "${vendor_files[@]}") if [ -n "${TEST_FORCE_VALIDATE:-}" ] || [ "${#changed_files[@]}" -gt 0 ]; then # recreate vendor/ - ./hack/vendor.sh + ./hack/vendor.sh vendor # check if any files have changed - diffs="$(git status --porcelain -- "${check_files[@]}" 2> /dev/null)" - mfiles="$(echo "$diffs" | awk '/^ M / {print $2}')" - if [ "$diffs" ]; then - { - echo 'The result of go mod vendor differs' - echo - echo "$diffs" - echo - echo 'Please vendor your package with hack/vendor.sh.' - echo - if [ -n "$mfiles" ]; then - git diff -- "$mfiles" - fi - } >&2 - false - else - echo 'Congratulations! All vendoring changes are done the right way.' - fi + git diff --quiet HEAD -- "${vendor_files[@]}" else - echo 'No vendor changes in diff.' + echo >&2 'No vendor changes in diff; skipping vendor check.' fi } -# 1. make sure all the vendored packages are used -# 2. make sure all the packages contain license information (just warning, because it can cause false-positive) -validate_vendor_used() { +validate_vendor_license() { for f in $(mawk '$1 = "#" { print $2 }' 'vendor/modules.txt'); do if [ -d "vendor/$f" ]; then if ! echo "vendor/$f"/* | grep -qiEc '/(LICENSE|COPYING)'; then - echo "WARNING: could not find copyright information for $f" + echo >&2 "WARNING: could not find copyright information for $f" fi fi done } -validate_vendor_diff -validate_vendor_used +if validate_vendor_tidy && validate_vendor_diff && validate_vendor_license; then + echo >&2 'Vendoring has been performed correctly!' +else + { + echo 'Vendoring was not performed correctly; the following files changed during re-vendor:' + echo + git diff --name-status HEAD -- "${vendor_files[@]}" + echo + echo 'Please revendor with hack/vendor.sh' + echo + git diff --diff-filter=M -- "${vendor_files[@]}" + } >&2 + exit 1 +fi From 036398f512621b4128942ff62f697cfb4b4b9bc6 Mon Sep 17 00:00:00 2001 From: Bjorn Neergaard Date: Mon, 14 Nov 2022 15:20:01 -0700 Subject: [PATCH 3/6] hack/validate/vendor: clean up license validation step Signed-off-by: Bjorn Neergaard (cherry picked from commit 9f004830a5c5c742ef5c2dec5468956f729b59a0) Signed-off-by: Bjorn Neergaard --- hack/validate/vendor | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/hack/validate/vendor b/hack/validate/vendor index 8c589b0e59..a0b35d3f25 100755 --- a/hack/validate/vendor +++ b/hack/validate/vendor @@ -24,25 +24,26 @@ validate_vendor_diff() { # check if any files have changed git diff --quiet HEAD -- "${vendor_files[@]}" else - echo >&2 'No vendor changes in diff; skipping vendor check.' + echo >&2 'INFO: no vendor changes in diff; skipping vendor check.' fi } validate_vendor_license() { - for f in $(mawk '$1 = "#" { print $2 }' 'vendor/modules.txt'); do - if [ -d "vendor/$f" ]; then - if ! echo "vendor/$f"/* | grep -qiEc '/(LICENSE|COPYING)'; then - echo >&2 "WARNING: could not find copyright information for $f" - fi + while IFS= read -r module; do + test -d "vendor/$module" || continue + if ! compgen -G "vendor/$module/*" | grep -qEi '/(LICENSE|COPYING)[^/]*$'; then + echo >&2 "WARNING: could not find copyright information for $module" fi - done + done < <(awk '/^# /{ print $2 }' vendor/modules.txt) } if validate_vendor_tidy && validate_vendor_diff && validate_vendor_license; then - echo >&2 'Vendoring has been performed correctly!' + echo >&2 'PASS: Vendoring has been performed correctly!' else { - echo 'Vendoring was not performed correctly; the following files changed during re-vendor:' + echo 'FAIL: Vendoring was not performed correctly!' + echo + echo 'The following files changed during re-vendor:' echo git diff --name-status HEAD -- "${vendor_files[@]}" echo From c508d133727db7068305cebe770e24bbc1d38289 Mon Sep 17 00:00:00 2001 From: Bjorn Neergaard Date: Mon, 14 Nov 2022 18:32:30 -0700 Subject: [PATCH 4/6] hack/go-mod-prepare.sh: find root robustly; make steps transparent Signed-off-by: Bjorn Neergaard (cherry picked from commit af8e955e8fadc44b65c0297e90bfb25efe3177f0) Signed-off-by: Bjorn Neergaard --- hack/go-mod-prepare.sh | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/hack/go-mod-prepare.sh b/hack/go-mod-prepare.sh index f1fb655a5c..e5552aaa5f 100755 --- a/hack/go-mod-prepare.sh +++ b/hack/go-mod-prepare.sh @@ -1,8 +1,13 @@ #!/usr/bin/env bash -ROOTDIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +set -e -cat > "${ROOTDIR}/go.mod" << EOF +SCRIPTDIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +ROOTDIR="$(git -C "$SCRIPTDIR" rev-parse --show-toplevel)" + +set -x + +tee "${ROOTDIR}/go.mod" << EOF module github.com/docker/docker go 1.18 From 45fe08c93c4571fd2992ecf75fec86e1b21d841e Mon Sep 17 00:00:00 2001 From: Bjorn Neergaard Date: Fri, 18 Nov 2022 08:29:06 -0700 Subject: [PATCH 5/6] hack: replace go-mod-prepare.sh with wrapper script To make the local build environment more correct and consistent, we should never leave an uncommitted go.mod in the tree; however, we need a go.mod for certain commands to work properly. Use a wrapper script to create and destroy the go.mod as needed instead of potentially changing tooling behavior by leaving it. If a go.mod already exists, this script will warn and call the wrapped command with GO111MODULE=on. Signed-off-by: Bjorn Neergaard (cherry picked from commit a449f77774e4747ad42bfd004d7160a08cb17ff5) Signed-off-by: Bjorn Neergaard --- .github/workflows/buildkit.yml | 1 - hack/buildkit-ref | 5 +---- hack/go-mod-prepare.sh | 14 -------------- hack/vendor.sh | 7 ++----- hack/with-go-mod.sh | 31 +++++++++++++++++++++++++++++++ 5 files changed, 34 insertions(+), 24 deletions(-) delete mode 100755 hack/go-mod-prepare.sh create mode 100755 hack/with-go-mod.sh diff --git a/.github/workflows/buildkit.yml b/.github/workflows/buildkit.yml index d9de83a471..e1242b3136 100644 --- a/.github/workflows/buildkit.yml +++ b/.github/workflows/buildkit.yml @@ -69,7 +69,6 @@ jobs: - name: BuildKit ref run: | - ./hack/go-mod-prepare.sh # FIXME(thaJeztah) temporarily overriding version to use for tests; remove with the next release of buildkit # echo "BUILDKIT_REF=$(./hack/buildkit-ref)" >> $GITHUB_ENV echo "BUILDKIT_REF=0bfcd83e6db95e6c6877ee6e5224b994cea62ba1" >> $GITHUB_ENV diff --git a/hack/buildkit-ref b/hack/buildkit-ref index 3c3809b52b..6f497c29ae 100755 --- a/hack/buildkit-ref +++ b/hack/buildkit-ref @@ -9,11 +9,8 @@ if [ -n "$BUILDKIT_REF" ]; then exit 0 fi -# prepare go mod -./hack/go-mod-prepare.sh - # get buildkit version from vendor.mod -BUILDKIT_REF=$(GO111MODULE=on go list -mod=mod -modfile=vendor.mod -u -m -f '{{.Version}}' "github.com/${BUILDKIT_REPO}") +BUILDKIT_REF=$(./hack/with-go-mod.sh go list -mod=mod -modfile=vendor.mod -u -m -f '{{.Version}}' "github.com/${BUILDKIT_REPO}") if [[ "${BUILDKIT_REF}" == *-*-* ]]; then # if pseudo-version, figure out just the uncommon sha (https://github.com/golang/go/issues/34745) BUILDKIT_REF=$(echo "${BUILDKIT_REF}" | awk -F"-" '{print $NF}' | awk 'BEGIN{FIELDWIDTHS="7"} {print $1}') diff --git a/hack/go-mod-prepare.sh b/hack/go-mod-prepare.sh deleted file mode 100755 index e5552aaa5f..0000000000 --- a/hack/go-mod-prepare.sh +++ /dev/null @@ -1,14 +0,0 @@ -#!/usr/bin/env bash - -set -e - -SCRIPTDIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -ROOTDIR="$(git -C "$SCRIPTDIR" rev-parse --show-toplevel)" - -set -x - -tee "${ROOTDIR}/go.mod" << EOF -module github.com/docker/docker - -go 1.18 -EOF diff --git a/hack/vendor.sh b/hack/vendor.sh index d106494be9..32538a3dc1 100755 --- a/hack/vendor.sh +++ b/hack/vendor.sh @@ -7,18 +7,15 @@ set -e SCRIPTDIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -"${SCRIPTDIR}"/go-mod-prepare.sh - -export GO111MODULE=on tidy() ( set -x - go mod tidy -modfile vendor.mod -compat 1.18 + "${SCRIPTDIR}"/with-go-mod.sh go mod tidy -modfile vendor.mod -compat 1.18 ) vendor() ( set -x - go mod vendor -modfile vendor.mod + "${SCRIPTDIR}"/with-go-mod.sh go mod vendor -modfile vendor.mod ) help() { diff --git a/hack/with-go-mod.sh b/hack/with-go-mod.sh new file mode 100755 index 0000000000..911eb3b50b --- /dev/null +++ b/hack/with-go-mod.sh @@ -0,0 +1,31 @@ +#!/usr/bin/env bash +# +# This script is used to coerce certain commands which rely on the presence of +# a go.mod into working with our repository. It works by creating a fake +# go.mod, running a specified command (passed via arguments), and removing it +# when the command is finished. This script should be dropped when this +# repository is a proper Go module with a permanent go.mod. + +set -e + +SCRIPTDIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +ROOTDIR="$(git -C "$SCRIPTDIR" rev-parse --show-toplevel)" + +if test -e "${ROOTDIR}/go.mod"; then + { + scriptname=$(basename "$0") + echo "${scriptname}: WARN: go.mod exists in the repository root!" + echo "${scriptname}: WARN: Using your go.mod instead of our generated version -- this may misbehave!" + } >&2 +else + set -x + + tee "${ROOTDIR}/go.mod" >&2 <<- EOF + module github.com/docker/docker + + go 1.18 + EOF + trap 'rm -f "${ROOTDIR}/go.mod"' EXIT +fi + +GO111MODULE=on "$@" From a6818fd4cbac7ee3468d86eb9a066fc0da4a2651 Mon Sep 17 00:00:00 2001 From: Bjorn Neergaard Date: Fri, 18 Nov 2022 15:55:26 -0700 Subject: [PATCH 6/6] hack: introduce validate/no-module Moby is not a Go module; to prevent anyone from mistakenly trying to convert it to one before we are ready, introduce a check (usable in CI and locally) for a go.mod file. This is preferable to trying to .gitignore the file as we can ensure that a mistakenly created go.mod is surfaced by Git-based tooling and is less likely to surprise a contributor. Signed-off-by: Bjorn Neergaard (cherry picked from commit 25c3421802af2c008998b5ad664b5043a0db33fc) Signed-off-by: Bjorn Neergaard --- .gitignore | 2 -- hack/validate/no-module | 17 +++++++++++++++++ 2 files changed, 17 insertions(+), 2 deletions(-) create mode 100755 hack/validate/no-module diff --git a/.gitignore b/.gitignore index d3f896b19e..9b2c8b9c51 100644 --- a/.gitignore +++ b/.gitignore @@ -13,8 +13,6 @@ thumbs.db .bashrc .editorconfig -# top-level go.mod is not meant to be checked in -/go.mod # build artifacts bundles/ cli/winresources/*/*.syso diff --git a/hack/validate/no-module b/hack/validate/no-module new file mode 100755 index 0000000000..67a9c559ad --- /dev/null +++ b/hack/validate/no-module @@ -0,0 +1,17 @@ +#!/usr/bin/env bash +# +# Check that no one is trying to commit a go.mod. + +SCRIPTDIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +ROOTDIR="$(git -C "$SCRIPTDIR" rev-parse --show-toplevel)" + +if test -e "${ROOTDIR}/go.mod"; then + { + echo 'FAIL: go.mod found in repository root!' + echo + echo ' Moby is not a Go module; please delete go.mod and try again.' + } >&2 + exit 1 +else + echo 'PASS: No go.mod found in repository root!' +fi