From b4bdad2bd1175afbc9834b7a6c7cacae128de2fe Mon Sep 17 00:00:00 2001 From: CrazyMax Date: Sun, 22 Nov 2020 01:34:31 +0100 Subject: [PATCH] Refactor generators for GitHub Actions Signed-off-by: CrazyMax --- .github/workflows/validate.yml | 88 +++++++++++++++++----------------- .travis.yml | 3 +- hack/shfmt | 28 +++-------- hack/update-generated-files | 43 ++++------------- hack/update-vendor | 45 ++++------------- hack/util | 24 +++++++++- 6 files changed, 95 insertions(+), 136 deletions(-) diff --git a/.github/workflows/validate.yml b/.github/workflows/validate.yml index c5525f276..e54d535a7 100644 --- a/.github/workflows/validate.yml +++ b/.github/workflows/validate.yml @@ -1,44 +1,44 @@ -name: validate - -on: - push: - branches: - - 'master' - tags: - - 'v*' - - 'dockerfile/*' - pull_request: - branches: - - 'master' - -env: - REPO_SLUG_ORIGIN: "moby/buildkit:latest" - -jobs: - validate: - runs-on: ubuntu-latest - strategy: - fail-fast: false - matrix: - include: - - script: ./hack/lint - - script: ./hack/validate-vendor - - script: ./hack/validate-generated-files - - script: ./hack/validate-shfmt - steps: - - - name: Checkout - uses: actions/checkout@v2 - - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v1 - with: - driver-opts: image=${{ env.REPO_SLUG_ORIGIN }} - - - name: Run - run: | - ${{ matrix.script }} - - - name: Dump context - if: always() - uses: crazy-max/ghaction-dump-context@v1 +name: validate + +on: + push: + branches: + - 'master' + tags: + - 'v*' + - 'dockerfile/*' + pull_request: + branches: + - 'master' + +env: + REPO_SLUG_ORIGIN: "moby/buildkit:latest" + +jobs: + validate: + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + include: + - script: ./hack/lint + - script: ./hack/validate-vendor + - script: ./hack/validate-generated-files + - script: ./hack/validate-shfmt + steps: + - + name: Checkout + uses: actions/checkout@v2 + - + name: Set up Docker Buildx + uses: docker/setup-buildx-action@v1 + with: + driver-opts: image=${{ env.REPO_SLUG_ORIGIN }} + - + name: Run + run: | + ${{ matrix.script }} + - + name: Dump context + if: always() + uses: crazy-max/ghaction-dump-context@v1 diff --git a/.travis.yml b/.travis.yml index 9f1a2e195..f02f2b2ca 100644 --- a/.travis.yml +++ b/.travis.yml @@ -4,6 +4,7 @@ language: minimal env: global: - DOCKER_CHANNEL="stable" + - DOCKER_CLI_EXPERIMENTAL="enabled" - PLATFORMS="linux/amd64,linux/arm/v7,linux/arm64,linux/s390x,linux/ppc64le" - PREFER_BUILDCTL="1" @@ -46,7 +47,7 @@ jobs: - TESTPKGS=./frontend ./hack/test name: "Unit Tests & Lint & Vendor & Proto" - script: - - TESTPKGS=./frontend/dockerfile TESTFLAGS='-v --parallel=6 --timeout=20m' ./hack/test + - TESTPKGS=./frontend/dockerfile TESTFLAGS='-v --parallel=5 --timeout=30m' ./hack/test name: "Dockerfile integration tests" - script: TESTPKGS=./frontend/dockerfile ./hack/test dockerfile name: "External Dockerfile tests" diff --git a/hack/shfmt b/hack/shfmt index b268d817a..0368e680e 100755 --- a/hack/shfmt +++ b/hack/shfmt @@ -1,26 +1,10 @@ #!/usr/bin/env bash . $(dirname $0)/util -set -eu -o pipefail -x +set -e -: ${CONTINUOUS_INTEGRATION=} - -progressFlag="" -if [ "$CONTINUOUS_INTEGRATION" == "true" ]; then progressFlag="--progress=plain"; fi - -case $buildmode in - "buildkit") - buildctl build $progressFlag --frontend=dockerfile.v0 --local context=. --local dockerfile=. \ - --opt target=update \ - --opt filename=./hack/dockerfiles/shfmt.Dockerfile \ - --output type=local,dest=. - ;; - "docker-buildkit") - export DOCKER_BUILDKIT=1 - docker build $progressFlag -f ./hack/dockerfiles/shfmt.Dockerfile --target update -o . . - ;; - *) - echo "Unsupported build mode: $buildmode" >&2 - exit 1 - ;; -esac +buildxCmd build \ + --target "update" \ + --output "type=local,dest=." \ + --file "./hack/dockerfiles/shfmt.Dockerfile" \ + . diff --git a/hack/update-generated-files b/hack/update-generated-files index 007bf8c06..a58262028 100755 --- a/hack/update-generated-files +++ b/hack/update-generated-files @@ -1,40 +1,17 @@ #!/usr/bin/env bash . $(dirname $0)/util -set -eu -o pipefail -x - -: ${CONTINUOUS_INTEGRATION=} - -progressFlag="" -if [ "$CONTINUOUS_INTEGRATION" == "true" ]; then progressFlag="--progress=plain"; fi +set -eu gogo_version=$(awk '$1 == "github.com/gogo/protobuf" { print $2 }' go.mod) -case $buildmode in - "buildkit") - output=$(mktemp -d -t buildctl-output.XXXXXXXXXX) - buildctl build $progressFlag --frontend=dockerfile.v0 --local context=. --local dockerfile=. \ - --opt build-arg:GOGO_VERSION=$gogo_version \ - --opt target=update \ - --opt filename=./hack/dockerfiles/generated-files.Dockerfile \ - --output type=local,dest=$output - cp -R "$output/generated-files/" . - rm -rf $output - ;; - "docker-buildkit") - iidfile=$(mktemp -t docker-iidfile.XXXXXXXXXX) - export DOCKER_BUILDKIT=1 - docker build --build-arg GOGO_VERSION=$gogo_version --iidfile $iidfile -f ./hack/dockerfiles/generated-files.Dockerfile --target update --force-rm . - iid=$(cat $iidfile) - cid=$(docker create $iid noop) +output=$(mktemp -d -t buildctl-output.XXXXXXXXXX) - docker export $cid | tar -xf - --strip-components=1 generated-files +buildxCmd build \ + --target "update" \ + --build-arg "GOGO_VERSION=$gogo_version" \ + --output "type=local,dest=$output" \ + --file "./hack/dockerfiles/generated-files.Dockerfile" \ + . - docker rm $cid - - rm -f $iidfile - ;; - *) - echo "Unsupported build mode: $buildmode" >&2 - exit 1 - ;; -esac +cp -R "$output/generated-files/" . +rm -rf $output diff --git a/hack/update-vendor b/hack/update-vendor index a2e0aa236..18dd3237c 100755 --- a/hack/update-vendor +++ b/hack/update-vendor @@ -1,41 +1,16 @@ #!/usr/bin/env bash . $(dirname $0)/util -set -eu -o pipefail -x +set -eu -: ${CONTINUOUS_INTEGRATION=} +output=$(mktemp -d -t buildx-output.XXXXXXXXXX) -progressFlag="" -if [ "$CONTINUOUS_INTEGRATION" == "true" ]; then progressFlag="--progress=plain"; fi +buildxCmd build \ + --target "update" \ + --output "type=local,dest=$output" \ + --file "./hack/dockerfiles/vendor.Dockerfile" \ + . -case $buildmode in - "buildkit") - output=$(mktemp -d -t buildctl-output.XXXXXXXXXX) - buildctl build $progressFlag --frontend=dockerfile.v0 --local context=. --local dockerfile=. \ - --opt target=update \ - --opt filename=./hack/dockerfiles/vendor.Dockerfile \ - --output type=local,dest=$output - rm -rf ./vendor - cp -R "$output"/out/* . - rm -rf $output - ;; - "docker-buildkit") - iidfile=$(mktemp -t docker-iidfile.XXXXXXXXXX) - export DOCKER_BUILDKIT=1 - docker build --iidfile $iidfile -f ./hack/dockerfiles/vendor.Dockerfile --target update --force-rm . - iid=$(cat $iidfile) - cid=$(docker create $iid noop) - rm -rf ./vendor - - docker cp $cid:/out/go.mod . - docker cp $cid:/out/go.sum . - docker cp $cid:/out/vendor . - - docker rm $cid - rm -f $iidfile - ;; - *) - echo "Unsupported build mode: $buildmode" >&2 - exit 1 - ;; -esac +rm -rf ./vendor +cp -R "$output"/out/* . +rm -rf $output diff --git a/hack/util b/hack/util index d6d342d1e..7fafa98d6 100755 --- a/hack/util +++ b/hack/util @@ -1,6 +1,8 @@ #!/usr/bin/env sh export BUILDX_NO_DEFAULT_LOAD=true +: ${PREFER_BUILDCTL=} +: ${PREFER_LEGACY=} : ${CONTINUOUS_INTEGRATION=} : ${CI=} : ${GITHUB_ACTIONS=} @@ -14,7 +16,27 @@ if [ "$CONTINUOUS_INTEGRATION" = "true" ] || [ "$CI" = "true" ]; then progressFlag="--progress=plain" fi -buildmode="buildkit" +newerEqualThan() { # $1=minimum wanted version $2=actual-version + [ "$1" = "$(printf "$1\n$2" | sort -V | head -n 1)" ] +} + +buildmode="legacy" +if [ "$PREFER_BUILDCTL" = "1" ]; then + buildmode="buildkit" +else + serverVersion=$(docker info --format '{{.ServerVersion}}') + experimental=$(docker info --format '{{.ExperimentalBuild}}') + if [ "$PREFER_LEGACY" != "1" ] && (newerEqualThan "18.09" $serverVersion || + (newerEqualThan "18.06" $serverVersion && [ "true" = "$experimental" ]) || + [ "$DOCKER_BUILDKIT" = "1" ]); then + buildmode="docker-buildkit" + fi +fi +if [ "$buildmode" = "legacy" ]; then + echo >&2 "ERROR: \"legacy\" buildmode is no longer supported. Please enable DOCKER_BUILDKIT or install standalone BuildKit" + exit 1 +fi + buildxCmd() { if docker buildx version >/dev/null 2>&1; then set -x