Merge pull request #2588 from tonistiigi/amd64-variants-support

amd64 variants support
This commit is contained in:
Tõnis Tiigi
2022-02-14 11:45:56 -08:00
committed by GitHub
50 changed files with 627 additions and 171 deletions

View File

@@ -407,10 +407,10 @@ func setDefaultConfig(cfg *config.Config) {
}
if cfg.Workers.OCI.Platforms == nil {
cfg.Workers.OCI.Platforms = archutil.SupportedPlatforms(false)
cfg.Workers.OCI.Platforms = formatPlatforms(archutil.SupportedPlatforms(false))
}
if cfg.Workers.Containerd.Platforms == nil {
cfg.Workers.Containerd.Platforms = archutil.SupportedPlatforms(false)
cfg.Workers.Containerd.Platforms = formatPlatforms(archutil.SupportedPlatforms(false))
}
cfg.Workers.OCI.NetworkConfig = setDefaultNetworkConfig(cfg.Workers.OCI.NetworkConfig)
@@ -700,8 +700,8 @@ func newWorkerController(c *cli.Context, wiOpt workerInitializerOpt) (*worker.Co
return nil, err
}
for _, w := range ws {
p := formatPlatforms(w.Platforms(false))
logrus.Infof("found worker %q, labels=%v, platforms=%v", w.ID(), w.Labels(), p)
p := w.Platforms(false)
logrus.Infof("found worker %q, labels=%v, platforms=%v", w.ID(), w.Labels(), formatPlatforms(p))
archutil.WarnIfUnsupported(p)
if err = wc.Add(w); err != nil {
return nil, err

3
go.mod
View File

@@ -8,7 +8,7 @@ require (
github.com/agext/levenshtein v1.2.3
github.com/armon/circbuf v0.0.0-20190214190532-5111143e8da2
github.com/containerd/console v1.0.3
github.com/containerd/containerd v1.6.0-rc.1
github.com/containerd/containerd v1.6.0-rc.1.0.20220127150749-a43703fcba54
github.com/containerd/continuity v0.2.2
github.com/containerd/fuse-overlayfs-snapshotter v1.0.2
github.com/containerd/go-cni v1.1.2
@@ -54,6 +54,7 @@ require (
github.com/stretchr/testify v1.7.0
github.com/tonistiigi/fsutil v0.0.0-20220115021204-b19f7f9cb274
github.com/tonistiigi/go-actions-cache v0.0.0-20211202175116-9642704158ff
github.com/tonistiigi/go-archvariant v1.0.0
github.com/tonistiigi/units v0.0.0-20180711220420-6950e57a87ea
github.com/tonistiigi/vt100 v0.0.0-20210615222946-8066bb97264f
github.com/urfave/cli v1.22.4

5
go.sum
View File

@@ -305,8 +305,9 @@ github.com/containerd/containerd v1.5.0-rc.0/go.mod h1:V/IXoMqNGgBlabz3tHD2TWDoT
github.com/containerd/containerd v1.5.1/go.mod h1:0DOxVqwDy2iZvrZp2JUx/E+hS0UNTVn7dJnIOwtYR4g=
github.com/containerd/containerd v1.5.7/go.mod h1:gyvv6+ugqY25TiXxcZC3L5yOeYgEw0QMhscqVp1AR9c=
github.com/containerd/containerd v1.5.8/go.mod h1:YdFSv5bTFLpG2HIYmfqDpSYYTDX+mc5qtSuYx1YUb/s=
github.com/containerd/containerd v1.6.0-rc.1 h1:bFSTay5ZHCSmAE/hzubBXX+eUpptHprbXi0QELuUk0Y=
github.com/containerd/containerd v1.6.0-rc.1/go.mod h1:jdeOXdXovU4bWF2Bd2hJjqpcOrrpSC1wmFCTtf6LD+A=
github.com/containerd/containerd v1.6.0-rc.1.0.20220127150749-a43703fcba54 h1:ZQqTZnRLXn44/VOq0LBjnH3vMOgGoEUic+8b1a1TBx4=
github.com/containerd/containerd v1.6.0-rc.1.0.20220127150749-a43703fcba54/go.mod h1:jdeOXdXovU4bWF2Bd2hJjqpcOrrpSC1wmFCTtf6LD+A=
github.com/containerd/continuity v0.0.0-20190426062206-aaeac12a7ffc/go.mod h1:GL3xCUCBDV3CZiTSEKksMWbLE66hEyuu9qyDOOqM47Y=
github.com/containerd/continuity v0.0.0-20190815185530-f2a389ac0a02/go.mod h1:GL3xCUCBDV3CZiTSEKksMWbLE66hEyuu9qyDOOqM47Y=
github.com/containerd/continuity v0.0.0-20191127005431-f65d91d395eb/go.mod h1:GL3xCUCBDV3CZiTSEKksMWbLE66hEyuu9qyDOOqM47Y=
@@ -1236,6 +1237,8 @@ github.com/tonistiigi/fsutil v0.0.0-20220115021204-b19f7f9cb274 h1:wbyZxD6IPFp0s
github.com/tonistiigi/fsutil v0.0.0-20220115021204-b19f7f9cb274/go.mod h1:oPAfvw32vlUJSjyDcQ3Bu0nb2ON2B+G0dtVN/SZNJiA=
github.com/tonistiigi/go-actions-cache v0.0.0-20211202175116-9642704158ff h1:n8i1G5sBFmY8aDteg5Kf2rdU15KnFcS807QrYRM9/yQ=
github.com/tonistiigi/go-actions-cache v0.0.0-20211202175116-9642704158ff/go.mod h1:qqvyZqkfwkoJuPU/bw61bItaoO0SJ8YSW0vSVRRvsRg=
github.com/tonistiigi/go-archvariant v1.0.0 h1:5LC1eDWiBNflnTF1prCiX09yfNHIxDC/aukdhCdTyb0=
github.com/tonistiigi/go-archvariant v1.0.0/go.mod h1:TxFmO5VS6vMq2kvs3ht04iPXtu2rUT/erOnGFYfk5Ho=
github.com/tonistiigi/units v0.0.0-20180711220420-6950e57a87ea h1:SXhTLE6pb6eld/v/cCndK0AMpt1wiVFb/YYmqB3/QG0=
github.com/tonistiigi/units v0.0.0-20180711220420-6950e57a87ea/go.mod h1:WPnis/6cRcDZSUvVmezrxJPkiO87ThFYsoUiMwWNDJk=
github.com/tonistiigi/vt100 v0.0.0-20210615222946-8066bb97264f h1:DLpt6B5oaaS8jyXHa9VA4rrZloBVPVXeCtrOsrFauxc=

View File

@@ -21,7 +21,6 @@ import (
"github.com/moby/buildkit/solver/llbsolver/errdefs"
"github.com/moby/buildkit/solver/llbsolver/mounts"
"github.com/moby/buildkit/solver/pb"
"github.com/moby/buildkit/util/bklog"
"github.com/moby/buildkit/util/progress/logs"
utilsystem "github.com/moby/buildkit/util/system"
"github.com/moby/buildkit/worker"
@@ -296,8 +295,11 @@ func (e *execOp) Exec(ctx context.Context, g session.Group, inputs []solver.Resu
return nil, err
}
emu, err := getEmulator(e.platform, e.cm.IdentityMapping())
if err == nil && emu != nil {
emu, err := getEmulator(ctx, e.platform, e.cm.IdentityMapping())
if err != nil {
return nil, err
}
if emu != nil {
e.op.Meta.Args = append([]string{qemuMountName}, e.op.Meta.Args...)
p.Mounts = append(p.Mounts, executor.Mount{
@@ -306,9 +308,6 @@ func (e *execOp) Exec(ctx context.Context, g session.Group, inputs []solver.Resu
Dest: qemuMountName,
})
}
if err != nil {
bklog.G(ctx).Warn(err.Error()) // TODO: remove this with pull support
}
meta := executor.Meta{
Args: e.op.Meta.Args,

View File

@@ -6,6 +6,7 @@ import (
"os"
"os/exec"
"path/filepath"
"strings"
"github.com/containerd/containerd/mount"
"github.com/containerd/containerd/platforms"
@@ -13,6 +14,7 @@ import (
"github.com/moby/buildkit/snapshot"
"github.com/moby/buildkit/solver/pb"
"github.com/moby/buildkit/util/archutil"
"github.com/moby/buildkit/util/bklog"
ocispecs "github.com/opencontainers/image-spec/specs-go/v1"
"github.com/pkg/errors"
copy "github.com/tonistiigi/fsutil/copy"
@@ -83,7 +85,7 @@ func (m *staticEmulatorMount) IdentityMapping() *idtools.IdentityMapping {
return m.idmap
}
func getEmulator(p *pb.Platform, idmap *idtools.IdentityMapping) (*emulator, error) {
func getEmulator(ctx context.Context, p *pb.Platform, idmap *idtools.IdentityMapping) (*emulator, error) {
all := archutil.SupportedPlatforms(false)
pp := platforms.Normalize(ocispecs.Platform{
Architecture: p.Architecture,
@@ -91,11 +93,21 @@ func getEmulator(p *pb.Platform, idmap *idtools.IdentityMapping) (*emulator, err
Variant: p.Variant,
})
for _, ps := range all {
if p, err := platforms.Parse(ps); err == nil {
if platforms.Only(p).Match(pp) {
return nil, nil
for _, p := range all {
if platforms.Only(p).Match(pp) {
return nil, nil
}
}
if pp.Architecture == "amd64" {
if pp.Variant != "" && pp.Variant != "v2" {
var supported []string
for _, p := range all {
if p.Architecture == "amd64" {
supported = append(supported, platforms.Format(p))
}
}
return nil, errors.Errorf("no support for running processes with %s platform, supported: %s", platforms.Format(pp), strings.Join(supported, ", "))
}
}
@@ -106,7 +118,8 @@ func getEmulator(p *pb.Platform, idmap *idtools.IdentityMapping) (*emulator, err
fn, err := exec.LookPath("buildkit-qemu-" + a)
if err != nil {
return nil, errors.Errorf("no emulator available for %v", pp.OS)
bklog.G(ctx).Warn(err.Error()) // TODO: remove this with pull support
return nil, nil // no emulator available
}
return &emulator{path: fn}, nil

View File

@@ -3,6 +3,6 @@
package archutil
func i386Supported() error {
return check(Binary386)
func i386Supported() (string, error) {
return check("386", Binary386)
}

View File

@@ -3,6 +3,6 @@
package archutil
func i386Supported() error {
return nil
func i386Supported() (string, error) {
return "", nil
}

View File

@@ -1,8 +1,8 @@
FROM debian:bullseye-slim AS base
RUN apt-get update && apt-get --no-install-recommends install -y \
gcc-x86-64-linux-gnu \
binutils-arm-linux-gnueabihf \
binutils-aarch64-linux-gnu \
binutils-x86-64-linux-gnu \
binutils-i686-linux-gnu \
binutils-riscv64-linux-gnu \
binutils-s390x-linux-gnu \
@@ -13,8 +13,8 @@ WORKDIR /src
FROM base AS exit-amd64
COPY fixtures/exit.amd64.s .
RUN x86_64-linux-gnu-as --noexecstack -o exit.o exit.amd64.s && x86_64-linux-gnu-ld -o exit -s exit.o
COPY fixtures/exit.amd64.S .
RUN x86_64-linux-gnu-gcc -static -nostdlib -o exit exit.amd64.S
FROM base AS exit-386
COPY fixtures/exit.386.s .

View File

@@ -6,4 +6,4 @@ package archutil
// This file is generated by running make inside the archutil package.
// Do not edit manually.
const Binaryamd64 = "\x1f\x8b\x08\x00\x00\x00\x00\x00\x02\xff\xaa\x77\xf5\x71\x63\x62\x64\x64\x80\x01\x26\x06\x3b\x06\x30\x4f\xc0\x01\xcc\x77\x80\x8a\x2b\x08\xc0\x95\x30\x38\x30\x58\x30\x30\x33\x38\x30\x30\x33\x30\x81\xd5\xb2\x30\x20\x03\x07\x14\xfa\x05\x94\x07\xa3\x19\xa0\xe6\x80\xf4\xb1\x22\xf1\x61\xf6\xc1\x68\x1e\xa8\x30\x0f\x9a\xbe\xc0\xa7\x25\x29\x6c\x0c\xc4\x03\x01\x86\x51\x30\x0a\x46\xc1\x28\x18\x05\xa3\x60\x14\x8c\x82\x51\x30\x0a\x46\xc1\x28\x18\x05\xa3\x60\x14\x78\x1c\x3f\x60\x03\xa2\x0c\xff\xf3\xb3\x32\xe8\x15\x67\x14\x97\x14\x95\x24\x26\x31\xe8\x95\xa4\x56\x94\x50\xc3\x7c\x6e\x68\x5f\x1f\xde\x67\x87\xf7\xf3\xd1\xfa\xf7\x0c\x88\x71\x01\x74\x3e\x33\x16\x73\x79\xa0\xfa\x05\x09\xe8\x07\x04\x00\x00\xff\xff\x6d\x7a\x0b\x2f\xe0\x10\x00\x00"
const Binaryamd64 = "\x1f\x8b\x08\x00\x00\x00\x00\x00\x02\xff\xec\x98\x3f\x8b\x13\x41\x18\xc6\x9f\xd9\x6c\xee\x0f\x08\xb9\xe0\x81\xc2\x59\xa8\x58\xd8\xb8\xe1\x54\x38\x05\x95\x55\x50\x07\xf1\xec\xce\xd2\x65\x93\x5d\xce\x80\x97\x5b\xb2\xb3\xe7\x0a\x07\x77\x47\xbe\x83\x58\x07\x14\x0b\x4b\xc1\x94\xa9\x5c\x2c\x6d\xac\x4d\x21\x04\xad\xec\x0c\x28\xca\x6e\xde\x49\xb2\x63\xa2\x82\xdd\x31\x3f\x08\xcf\xbc\xcf\xbe\x33\xef\x90\xa4\xd9\x67\xef\xe6\xdd\x5b\x06\x63\x90\x18\xb8\x86\xac\x5a\xb2\xb3\xda\x26\xff\x63\x79\xd4\x02\x1b\x97\x50\x80\x8d\x39\x14\xb3\x5e\x13\x93\xd8\x39\x3d\x42\x47\x4b\xc5\xd2\x50\xd2\xb2\x38\x51\xcb\x79\x52\x03\xb2\xa5\xca\x3e\x93\x3e\x7d\xb2\xfb\x34\x47\xea\x19\xf2\xa5\x9a\x13\xba\x0c\xa0\x00\xe0\xf6\xbd\x0d\xf8\xcf\xdf\x5c\xfe\xf0\xf2\xd3\xfd\xf5\x1f\x4f\x8f\x7e\x7f\xf1\x79\xef\xed\x85\xf7\xaf\xa1\xd1\x68\x34\x1a\x8d\x46\xa3\xd1\x68\x34\x1a\xcd\x21\x85\xaf\x76\x4b\x6d\xde\xfa\x36\xbf\x7b\x87\x77\x00\xec\xa7\x66\xa9\x7d\x95\x01\xfb\xbb\x6b\x3c\xe9\xb2\xac\xe9\x5d\xa9\xcd\x0f\x7a\xec\xf4\x33\xf0\x83\x41\x2a\xd1\x0a\xef\x30\x6a\x1f\x3e\x6e\xf5\x18\x6f\x0d\x58\x64\x7c\x59\xe4\x49\x72\x1d\x40\xb6\xb8\x91\x2e\xc0\x93\xee\x95\xf4\xe0\xe2\x9f\xee\x52\x00\x1b\xbd\xc7\xe7\x7d\x63\x9c\x0f\x60\x9c\x1f\x98\xf8\xfa\x53\xed\x2d\x53\x8a\xb1\xa1\xf4\x2f\x93\xff\x40\xf1\x8f\x91\xbf\xa9\xf8\xa7\xb2\xc8\xe1\xf7\xb9\x27\xa4\x7f\x32\xef\x9f\x9d\xe1\x57\x66\xf8\xa8\x88\xad\xa0\x52\xab\xc5\xb5\xb5\xc7\xcd\x75\x6b\x1b\x3b\xab\xd8\x39\x0f\x3f\xae\x0b\x38\x4e\x35\x0c\x9d\x50\xb8\x4d\x01\xc7\xf7\x5c\xe1\xc2\xf1\x1b\x1e\x60\x85\x4f\xb6\x84\x5b\x85\x15\x8a\xe6\x50\x1f\xca\x55\x63\x5b\xf8\xd6\x66\x23\xb2\xaa\x51\xfd\x91\x77\xae\xee\xc1\x12\x7e\x2c\xfe\xfb\xff\xb1\x02\x60\x3e\xfb\x86\xd4\xbc\x25\x9f\xb3\x40\xc9\x5b\x24\x16\xfd\x56\x73\xa3\x1c\xc7\xce\xe5\x39\x81\xd2\xcf\xa6\xd4\xc6\x94\x7b\x05\xb4\x7f\x81\x8d\xe7\xa6\xf7\x5c\xa0\xe7\xc7\x49\x17\x29\xf3\x51\x89\x29\xcf\xba\xf8\x97\xf9\xe5\x19\xfb\x5f\xfd\xe3\xfe\x5f\x01\x00\x00\xff\xff\xa5\x58\xb1\x16\x60\x13\x00\x00"

View File

@@ -3,6 +3,6 @@
package archutil
func amd64Supported() error {
return check(Binaryamd64)
func amd64Supported() (string, error) {
return check("amd64", Binaryamd64)
}

View File

@@ -3,6 +3,10 @@
package archutil
func amd64Supported() error {
return nil
import (
archvariant "github.com/tonistiigi/go-archvariant"
)
func amd64Supported() (string, error) {
return archvariant.AMD64Variant(), nil
}

View File

@@ -3,6 +3,6 @@
package archutil
func arm64Supported() error {
return check(Binaryarm64)
func arm64Supported() (string, error) {
return check("arm64", Binaryarm64)
}

View File

@@ -3,6 +3,6 @@
package archutil
func arm64Supported() error {
return nil
func arm64Supported() (string, error) {
return "", nil
}

View File

@@ -3,6 +3,6 @@
package archutil
func armSupported() error {
return check(Binaryarm)
func armSupported() (string, error) {
return check("arm", Binaryarm)
}

View File

@@ -3,6 +3,6 @@
package archutil
func armSupported() error {
return nil
func armSupported() (string, error) {
return "", nil
}

View File

@@ -12,6 +12,8 @@ import (
"os/exec"
"path/filepath"
"syscall"
"github.com/pkg/errors"
)
func withChroot(cmd *exec.Cmd, dir string) {
@@ -20,33 +22,49 @@ func withChroot(cmd *exec.Cmd, dir string) {
}
}
func check(bin string) error {
func check(arch, bin string) (string, error) {
tmpdir, err := ioutil.TempDir("", "qemu-check")
if err != nil {
return err
return "", err
}
defer os.RemoveAll(tmpdir)
pp := filepath.Join(tmpdir, "check")
r, err := gzip.NewReader(bytes.NewReader([]byte(bin)))
if err != nil {
return err
return "", err
}
defer r.Close()
f, err := os.OpenFile(pp, os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0700)
if err != nil {
return err
return "", err
}
if _, err := io.Copy(f, r); err != nil {
f.Close()
return err
return "", err
}
f.Close()
cmd := exec.Command("/check")
withChroot(cmd, tmpdir)
err = cmd.Run()
return err
if arch != "amd64" {
return "", err
}
// special handling for amd64. Exit code is 64 + amd64 variant
if err == nil {
return "", errors.Errorf("invalid zero exit code")
}
if exitError, ok := err.(*exec.ExitError); ok {
switch exitError.ExitCode() {
case 65:
return "v1", nil
case 66:
return "v2", nil
}
}
return "", err
}

View File

@@ -11,6 +11,6 @@ import (
func withChroot(cmd *exec.Cmd, dir string) {
}
func check(bin string) error {
return errors.New("binfmt is not supported on Windows")
func check(arch, bin string) (string, error) {
return "", errors.New("binfmt is not supported on Windows")
}

View File

@@ -1,6 +1,7 @@
package archutil
import (
"sort"
"strings"
"sync"
@@ -10,130 +11,131 @@ import (
)
var mu sync.Mutex
var arr []string
var arr []ocispecs.Platform
func SupportedPlatforms(noCache bool) []string {
func SupportedPlatforms(noCache bool) []ocispecs.Platform {
mu.Lock()
defer mu.Unlock()
if !noCache && arr != nil {
return arr
}
def := defaultPlatform()
arr = append([]string{}, def)
if p := "linux/amd64"; def != p && amd64Supported() == nil {
arr = append(arr, p)
def := nativePlatform()
arr = append([]ocispecs.Platform{}, def)
if def.OS != "linux" {
return arr
}
if p := "linux/arm64"; def != p && arm64Supported() == nil {
arr = append(arr, p)
if variant, err := amd64Supported(); err == nil {
p := "amd64"
if def.Architecture != p {
arr = append(arr, linux(p))
}
for _, v := range amd64vector(variant) {
p := linux(p)
p.Variant = v
arr = append(arr, p)
}
}
if p := "linux/riscv64"; def != p && riscv64Supported() == nil {
arr = append(arr, p)
if p := "arm64"; def.Architecture != p {
if _, err := arm64Supported(); err == nil {
arr = append(arr, linux(p))
}
}
if p := "linux/ppc64le"; def != p && ppc64leSupported() == nil {
arr = append(arr, p)
if p := "riscv64"; def.Architecture != p {
if _, err := riscv64Supported(); err == nil {
arr = append(arr, linux(p))
}
}
if p := "linux/s390x"; def != p && s390xSupported() == nil {
arr = append(arr, p)
if p := "ppc64le"; def.Architecture != p {
if _, err := ppc64leSupported(); err == nil {
arr = append(arr, linux(p))
}
}
if p := "linux/386"; def != p && i386Supported() == nil {
arr = append(arr, p)
if p := "s390x"; def.Architecture != p {
if _, err := s390xSupported(); err == nil {
arr = append(arr, linux(p))
}
}
if p := "linux/mips64le"; def != p && mips64leSupported() == nil {
arr = append(arr, p)
if p := "386"; def.Architecture != p {
if _, err := i386Supported(); err == nil {
arr = append(arr, linux(p))
}
}
if p := "linux/mips64"; def != p && mips64Supported() == nil {
arr = append(arr, p)
if p := "mips64le"; def.Architecture != p {
if _, err := mips64leSupported(); err == nil {
arr = append(arr, linux(p))
}
}
if !strings.HasPrefix(def, "linux/arm/") && armSupported() == nil {
arr = append(arr, "linux/arm/v7", "linux/arm/v6")
} else if def == "linux/arm/v7" {
arr = append(arr, "linux/arm/v6")
if p := "mips64"; def.Architecture != p {
if _, err := mips64Supported(); err == nil {
arr = append(arr, linux(p))
}
}
if p := "arm"; def.Architecture != p {
if _, err := armSupported(); err == nil {
p := linux("arm")
p.Variant = "v6"
arr = append(arr, linux("arm"), p)
}
} else if def.Variant == "" {
p := linux("arm")
p.Variant = "v6"
arr = append(arr, p)
}
return arr
}
func Check(pp ocispecs.Platform) bool {
p := platforms.Format(pp)
if p == "linux/amd64" && amd64Supported() == nil {
return true
}
if p == "linux/arm64" && arm64Supported() == nil {
return true
}
if p == "linux/riscv64" && riscv64Supported() == nil {
return true
}
if p == "linux/ppc64le" && ppc64leSupported() == nil {
return true
}
if p == "linux/s390x" && s390xSupported() == nil {
return true
}
if p == "linux/386" && i386Supported() == nil {
return true
}
if p == "linux/mips64le" && mips64leSupported() == nil {
return true
}
if p == "linux/mips64" && mips64Supported() == nil {
return true
}
if !strings.HasPrefix(p, "linux/arm/") && armSupported() == nil {
return true
}
return false
}
//WarnIfUnsupported validates the platforms and show warning message if there is,
//the end user could fix the issue based on those warning, and thus no need to drop
//the platform from the candidates.
func WarnIfUnsupported(pfs []string) {
def := defaultPlatform()
func WarnIfUnsupported(pfs []ocispecs.Platform) {
def := nativePlatform()
for _, p := range pfs {
if p != def {
if p == "linux/amd64" {
if err := amd64Supported(); err != nil {
if p.Architecture != def.Architecture {
if p.Architecture == "amd64" {
if _, err := amd64Supported(); err != nil {
printPlatformWarning(p, err)
}
}
if p == "linux/arm64" {
if err := arm64Supported(); err != nil {
if p.Architecture == "arm64" {
if _, err := arm64Supported(); err != nil {
printPlatformWarning(p, err)
}
}
if p == "linux/riscv64" {
if err := riscv64Supported(); err != nil {
if p.Architecture == "riscv64" {
if _, err := riscv64Supported(); err != nil {
printPlatformWarning(p, err)
}
}
if p == "linux/ppc64le" {
if err := ppc64leSupported(); err != nil {
if p.Architecture == "ppc64le" {
if _, err := ppc64leSupported(); err != nil {
printPlatformWarning(p, err)
}
}
if p == "linux/s390x" {
if err := s390xSupported(); err != nil {
if p.Architecture == "s390x" {
if _, err := s390xSupported(); err != nil {
printPlatformWarning(p, err)
}
}
if p == "linux/386" {
if err := i386Supported(); err != nil {
if p.Architecture == "386" {
if _, err := i386Supported(); err != nil {
printPlatformWarning(p, err)
}
}
if p == "linux/mips64le" {
if err := mips64leSupported(); err != nil {
if p.Architecture == "mips64le" {
if _, err := mips64leSupported(); err != nil {
printPlatformWarning(p, err)
}
}
if p == "linux/mips64" {
if err := mips64Supported(); err != nil {
if p.Architecture == "mips64" {
if _, err := mips64Supported(); err != nil {
printPlatformWarning(p, err)
}
}
if strings.HasPrefix(p, "linux/arm/v6") || strings.HasPrefix(p, "linux/arm/v7") {
if err := armSupported(); err != nil {
if p.Architecture == "arm" {
if _, err := armSupported(); err != nil {
printPlatformWarning(p, err)
}
}
@@ -141,16 +143,38 @@ func WarnIfUnsupported(pfs []string) {
}
}
func defaultPlatform() string {
return platforms.Format(platforms.Normalize(platforms.DefaultSpec()))
func nativePlatform() ocispecs.Platform {
return platforms.Normalize(platforms.DefaultSpec())
}
func printPlatformWarning(p string, err error) {
if strings.Contains(err.Error(), "exec format error") {
logrus.Warnf("platform %s cannot pass the validation, kernel support for miscellaneous binary may have not enabled.", p)
} else if strings.Contains(err.Error(), "no such file or directory") {
logrus.Warnf("platforms %s cannot pass the validation, '-F' flag might have not set for 'archutil'.", p)
} else {
logrus.Warnf("platforms %s cannot pass the validation: %s", p, err.Error())
func linux(arch string) ocispecs.Platform {
return ocispecs.Platform{
OS: "linux",
Architecture: arch,
}
}
func amd64vector(v string) (out []string) {
switch v {
case "v4":
out = append(out, "v4")
fallthrough
case "v3":
out = append(out, "v3")
fallthrough
case "v2":
out = append(out, "v2")
}
sort.Strings(out)
return
}
func printPlatformWarning(p ocispecs.Platform, err error) {
if strings.Contains(err.Error(), "exec format error") {
logrus.Warnf("platform %s cannot pass the validation, kernel support for miscellaneous binary may have not enabled.", platforms.Format(p))
} else if strings.Contains(err.Error(), "no such file or directory") {
logrus.Warnf("platforms %s cannot pass the validation, '-F' flag might have not set for 'archutil'.", platforms.Format(p))
} else {
logrus.Warnf("platforms %s cannot pass the validation: %s", platforms.Format(p), err.Error())
}
}

View File

@@ -1,6 +1,50 @@
.global _start
.text
// SSE3 SSSE3 CMPXCHNG16 SSE4.1 SSE4.2 POPCNT
#define V2_FEATURES_CX (1 << 0 | 1 << 9 | 1 << 13 | 1 << 19 | 1 << 20 | 1 << 23)
// LAHF/SAHF
#define V2_EXT_FEATURES_CX (1 << 0)
// 64-113 is user-defined exit codes
// https://tldp.org/LDP/abs/html/exitcodes.html
#define exitcode(x) $((x) + 64)
.global _start
.text
_start:
mov $60, %rax
xor %rdi, %rdi
syscall
// highest basic calling parameter
xor %rax, %rax
cpuid
cmp $7, %rax
jl v1
// highest extended calling parameter
mov $0x80000000, %rax
cpuid
cmp $0x80000001, %eax
jl v1
// feature bits
mov $1, %rax
xor %rcx, %rcx
cpuid
and $V2_FEATURES_CX, %rcx
cmp $V2_FEATURES_CX, %rcx
jne v1
// extended feature bits
mov $0x80000001, %rax
xor %rcx, %rcx
cpuid
and $V2_EXT_FEATURES_CX, %rcx
cmp $V2_EXT_FEATURES_CX, %rcx
jne v1
jmp v2
v1:
mov exitcode(1), %rdi
jmp exit
v2:
mov exitcode(2), %rdi
jmp exit
exit:
mov $60, %rax
syscall

View File

@@ -3,6 +3,6 @@
package archutil
func mips64Supported() error {
return check(Binarymips64)
func mips64Supported() (string, error) {
return check("mips64", Binarymips64)
}

View File

@@ -3,6 +3,6 @@
package archutil
func mips64Supported() error {
return nil
func mips64Supported() (string, error) {
return "", nil
}

View File

@@ -3,6 +3,6 @@
package archutil
func mips64leSupported() error {
return check(Binarymips64le)
func mips64leSupported() (string, error) {
return check("mips64le", Binarymips64le)
}

View File

@@ -3,6 +3,6 @@
package archutil
func mips64leSupported() error {
return nil
func mips64leSupported() (string, error) {
return "", nil
}

View File

@@ -3,6 +3,6 @@
package archutil
func ppc64leSupported() error {
return check(Binaryppc64le)
func ppc64leSupported() (string, error) {
return check("ppc64le", Binaryppc64le)
}

View File

@@ -3,6 +3,6 @@
package archutil
func ppc64leSupported() error {
return nil
func ppc64leSupported() (string, error) {
return "", nil
}

View File

@@ -3,6 +3,6 @@
package archutil
func riscv64Supported() error {
return check(Binaryriscv64)
func riscv64Supported() (string, error) {
return check("riscv64", Binaryriscv64)
}

View File

@@ -3,6 +3,6 @@
package archutil
func riscv64Supported() error {
return nil
func riscv64Supported() (string, error) {
return "", nil
}

View File

@@ -3,6 +3,6 @@
package archutil
func s390xSupported() error {
return check(Binarys390x)
func s390xSupported() (string, error) {
return check("390x", Binarys390x)
}

View File

@@ -3,6 +3,6 @@
package archutil
func s390xSupported() error {
return nil
func s390xSupported() (string, error) {
return "", nil
}

View File

@@ -59,6 +59,8 @@ type Stats interface {
//
// We can probably use this to inform a design for incremental GC by injecting
// callbacks to the set modification algorithms.
//
// https://en.wikipedia.org/wiki/Tracing_garbage_collection#Tri-color_marking
func Tricolor(roots []Node, refs func(ref Node) ([]Node, error)) (map[Node]struct{}, error) {
var (
grays []Node // maintain a gray "stack"

View File

@@ -772,6 +772,7 @@ func writeExpireAt(expire time.Time, bkt *bolt.Bucket) error {
return bkt.Put(bucketKeyExpireAt, expireAt)
}
// garbageCollect removes all contents that are no longer used.
func (cs *contentStore) garbageCollect(ctx context.Context) (d time.Duration, err error) {
cs.l.Lock()
t1 := time.Now()

View File

@@ -277,7 +277,7 @@ func (s GCStats) Elapsed() time.Duration {
return s.MetaD
}
// GarbageCollect starts garbage collection
// GarbageCollect removes resources (snapshots, contents, ...) that are no longer used.
func (m *DB) GarbageCollect(ctx context.Context) (gc.Stats, error) {
m.wlock.Lock()
t1 := time.Now()
@@ -363,6 +363,7 @@ func (m *DB) GarbageCollect(ctx context.Context) (gc.Stats, error) {
return stats, err
}
// getMarked returns all resources that are used.
func (m *DB) getMarked(ctx context.Context) (map[gc.Node]struct{}, error) {
var marked map[gc.Node]struct{}
if err := m.db.View(func(tx *bolt.Tx) error {

View File

@@ -58,6 +58,8 @@ var (
labelGCFlat = []byte("containerd.io/gc.flat")
)
// scanRoots sends the given channel "root" resources that are certainly used.
// The caller could look the references of the resources to find all resources that are used.
func scanRoots(ctx context.Context, tx *bolt.Tx, nc chan<- gc.Node) error {
v1bkt := tx.Bucket(bucketKeyVersion)
if v1bkt == nil {
@@ -276,6 +278,7 @@ func scanRoots(ctx context.Context, tx *bolt.Tx, nc chan<- gc.Node) error {
return cerr
}
// references finds the resources that are reachable from the given node.
func references(ctx context.Context, tx *bolt.Tx, node gc.Node, fn func(gc.Node)) error {
switch node.Type {
case ResourceContent:
@@ -328,6 +331,7 @@ func references(ctx context.Context, tx *bolt.Tx, node gc.Node, fn func(gc.Node)
return nil
}
// scanAll finds all resources regardless whether the resources are used or not.
func scanAll(ctx context.Context, tx *bolt.Tx, fn func(ctx context.Context, n gc.Node) error) error {
v1bkt := tx.Bucket(bucketKeyVersion)
if v1bkt == nil {
@@ -408,6 +412,7 @@ func scanAll(ctx context.Context, tx *bolt.Tx, fn func(ctx context.Context, n gc
return nil
}
// remove all buckets for the given node.
func remove(ctx context.Context, tx *bolt.Tx, node gc.Node) error {
v1bkt := tx.Bucket(bucketKeyVersion)
if v1bkt == nil {

View File

@@ -790,6 +790,7 @@ func validateSnapshot(info *snapshots.Info) error {
return nil
}
// garbageCollect removes all snapshots that are no longer used.
func (s *snapshotter) garbageCollect(ctx context.Context) (d time.Duration, err error) {
s.l.Lock()
t1 := time.Now()

View File

@@ -38,12 +38,22 @@ func platformVector(platform specs.Platform) []specs.Platform {
switch platform.Architecture {
case "amd64":
if amd64Version, err := strconv.Atoi(strings.TrimPrefix(platform.Variant, "v")); err == nil && amd64Version > 1 {
for amd64Version--; amd64Version >= 1; amd64Version-- {
vector = append(vector, specs.Platform{
Architecture: platform.Architecture,
OS: platform.OS,
OSVersion: platform.OSVersion,
OSFeatures: platform.OSFeatures,
Variant: "v" + strconv.Itoa(amd64Version),
})
}
}
vector = append(vector, specs.Platform{
Architecture: "386",
OS: platform.OS,
OSVersion: platform.OSVersion,
OSFeatures: platform.OSFeatures,
Variant: platform.Variant,
})
case "arm":
if armVersion, err := strconv.Atoi(strings.TrimPrefix(platform.Variant, "v")); err == nil && armVersion > 5 {

View File

@@ -86,9 +86,11 @@ func normalizeArch(arch, variant string) (string, string) {
case "i386":
arch = "386"
variant = ""
case "x86_64", "x86-64":
case "x86_64", "x86-64", "amd64":
arch = "amd64"
variant = ""
if variant == "v1" {
variant = ""
}
case "aarch64", "arm64":
arch = "arm64"
switch variant {

View File

@@ -74,7 +74,7 @@ func ContextWithAppendPullRepositoryScope(ctx context.Context, repo string) cont
// GetTokenScopes returns deduplicated and sorted scopes from ctx.Value(tokenScopesKey{}) and common scopes.
func GetTokenScopes(ctx context.Context, common []string) []string {
var scopes []string
scopes := []string{}
if x := ctx.Value(tokenScopesKey{}); x != nil {
scopes = append(scopes, x.([]string)...)
}
@@ -82,6 +82,10 @@ func GetTokenScopes(ctx context.Context, common []string) []string {
scopes = append(scopes, common...)
sort.Strings(scopes)
if len(scopes) == 0 {
return scopes
}
l := 0
for idx := 1; idx < len(scopes); idx++ {
// Note: this comparison is unaware of the scope grammar (https://docs.docker.com/registry/spec/auth/scope/)

View File

@@ -0,0 +1,20 @@
run:
timeout: 10m
skip-files:
- ".*\\.pb\\.go$"
linters:
enable:
- gofmt
- govet
- deadcode
- goimports
- ineffassign
- misspell
- unused
- varcheck
- revive
- staticcheck
- typecheck
- structcheck
disable-all: true

15
vendor/github.com/tonistiigi/go-archvariant/Dockerfile generated vendored Normal file
View File

@@ -0,0 +1,15 @@
ARG GO_VERSION=1.17
FROM --platform=$BUILDPLATFORM tonistiigi/xx AS xx
FROM --platform=$BUILDPLATFORM golang:${GO_VERSION}-alpine AS build
COPY --from=xx / /
RUN apk add --no-cache git
WORKDIR /src
ARG TARGETPLATFORM
RUN --mount=target=. \
TARGETPLATFORM=$TARGETPLATFORM xx-go build -o /out/amd64variant ./cmd/amd64variant && \
xx-verify --static /out/amd64variant
FROM scratch
COPY --from=build /out/amd64variant .

21
vendor/github.com/tonistiigi/go-archvariant/LICENSE generated vendored Normal file
View File

@@ -0,0 +1,21 @@
MIT License
Copyright (c) 2022 Tõnis Tiigi
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.

16
vendor/github.com/tonistiigi/go-archvariant/README.md generated vendored Normal file
View File

@@ -0,0 +1,16 @@
# go-archvariant
[![Go Reference](https://pkg.go.dev/badge/github.com/tonistiigi/go-archvariant.svg)](https://pkg.go.dev/github.com/tonistiigi/go-archvariant)
[![Build Status](https://github.com/tonistiigi/go-archvariant/workflows/ci/badge.svg)](https://github.com/tonistiigi/go-archvariant/actions)
Go package for determining the maximum compatibility version of the current system. The main use case is to use this value in container [platform definitions](https://github.com/containerd/containerd/blob/v1.5.9/platforms/platforms.go#L55).
On x86-64 platforms this package returns the maximum current microarchitecture level as defined in https://en.wikipedia.org/wiki/X86-64#Microarchitecture_levels . This value can be used to configure compiler in [LLVM since 12.0](https://github.com/llvm/llvm-project/commit/012dd42e027e2ff3d183cc9dcf27004cf9711720) and [GCC since 11.0](https://github.com/gcc-mirror/gcc/commit/324bec558e95584e8c1997575ae9d75978af59f1). [Go1.18+](https://tip.golang.org/doc/go1.18#amd64) uses `GOAMD64` environemnt to configure Go compiler with this value.
#### Scope
The goal of this repository is to only provide the variant with minimal external dependencies. If you need more specific CPU features detection you should look at [`golang.org/x/sys/cpu`](https://pkg.go.dev/golang.org/x/sys/cpu) or [`github.com/klauspost/cpuid`](https://pkg.go.dev/github.com/klauspost/cpuid/v2) instead.
#### Credits
The checks in this repository are based on the checks Go runtime does [on startup](https://github.com/golang/go/blob/go1.18beta1/src/runtime/asm_amd64.s#L95-L96).

View File

@@ -0,0 +1,140 @@
//go:build amd64
// +build amd64
package archvariant
import (
"fmt"
"sync"
)
var cacheOnce sync.Once
var amdVariantCache string
func cpuid(ax, cx uint32) (eax, ebx, ecx, edx uint32)
func xgetbv() (eax uint32)
const (
sse3 = 0
ssse3 = 9
cx16 = 13
sse4_1 = 19
sse4_2 = 20
popcnt = 23
fma = 12
movbe = 22
xsave = 26
osxsave = 27
avx = 28
f16c = 29
v2Features = 1<<sse3 | 1<<ssse3 | 1<<cx16 | 1<<sse4_1 | 1<<sse4_2 | 1<<popcnt
v3Features = v2Features | 1<<fma | 1<<movbe | 1<<xsave | 1<<osxsave | 1<<avx | 1<<f16c
)
const (
bmi1 = 3
avx2 = 5
bmi2 = 8
avx512F = 16
avx512Dq = 17
avx512Cd = 28
avx512Bw = 30
avx512Vl = 31
v3ExtFeaturesBX = 1<<bmi1 | 1<<avx2 | 1<<bmi2
v4ExtFeaturesBX = v3ExtFeaturesBX | 1<<avx512F | 1<<avx512Dq | 1<<avx512Cd | 1<<avx512Bw | 1<<avx512Vl
)
const (
lahfLm = 0
abm = 5
v2ExtFeatureCX = 1 << lahfLm
v3ExtFeatureCX = v2ExtFeatureCX | 1<<abm
)
const (
// XCR0
xmm = 1
ymm = 2
opmask = 5
zmmHi16 = 6
zmmHi256 = 7
v3OSSupport = 1<<xmm | 1<<ymm
v4OSSupport = v3OSSupport | 1<<opmask | 1<<zmmHi16 | 1<<zmmHi256
)
func detectVersion() int {
// highest basic calling parameter
ax, _, _, _ := cpuid(0, 0)
if ax < 7 {
return 1
}
// highest extended calling parameter
ax, _, _, _ = cpuid(0x80000000, 0)
if ax < 0x80000001 {
return 1
}
version := 4 // initialize to highest version
// feature bits
_, _, cx, _ := cpuid(1, 0)
if cx&v3Features != v3Features {
version = 2
if cx&v2Features != v2Features {
return 1
}
}
// extended features
_, bx, _, _ := cpuid(7, 0)
if version == 4 {
if bx&v4ExtFeaturesBX != v4ExtFeaturesBX {
version = 3
}
}
if version == 3 {
if bx&v3ExtFeaturesBX != v3ExtFeaturesBX {
version = 2
}
}
// extended processor info and feature bits
_, _, cx, _ = cpuid(0x80000001, 0)
if version >= 3 {
if cx&v3ExtFeatureCX != v3ExtFeatureCX {
version = 2
}
}
if version == 2 {
if cx&v2ExtFeatureCX != v2ExtFeatureCX {
return 1
}
}
if version >= 3 {
ax = xgetbv()
if version == 4 {
if !osAVX512Supported(ax) {
version = 3
}
}
if ax&v3OSSupport != v3OSSupport {
version = 2
}
}
return version
}
func AMD64Variant() string {
cacheOnce.Do(func() {
amdVariantCache = "v" + fmt.Sprintf("%d", detectVersion())
})
return amdVariantCache
}

View File

@@ -0,0 +1,21 @@
//go:build amd64
// +build amd64
#include "textflag.h"
TEXT ·cpuid(SB),NOSPLIT,$0-24
MOVL ax+0(FP),AX
MOVL cx+4(FP), CX
CPUID
MOVL AX,eax+8(FP)
MOVL BX,ebx+12(FP)
MOVL CX,ecx+16(FP)
MOVL DX,edx+20(FP)
RET
TEXT ·xgetbv(SB),NOSPLIT,$0-8
XORL CX, CX
XGETBV
MOVL AX, eax+0(FP)
RET

View File

@@ -0,0 +1,10 @@
//go:build amd64 && darwin
// +build amd64,darwin
package archvariant
func darwinSupportsAVX512() bool
func osAVX512Supported(ax uint32) bool {
return ax&v3OSSupport == v3OSSupport && darwinSupportsAVX512()
}

View File

@@ -0,0 +1,30 @@
//go:build amd64 && darwin
// +build amd64,darwin
#include "textflag.h"
// Based on https://github.com/golang/sys/blob/ae416a5f93c7892a9dce2d607fc2479eabbacd70/cpu/cpu_x86.s#L31
// Note that Go currently has problems with AVX512 on darwin due to signal handling conflict in kernel and
// has been disabled in some libraries for runtime detection. We keep it on as this is unlikely to be used
// in runtime checks. golang/go#49233
// func darwinSupportsAVX512() bool
TEXT ·darwinSupportsAVX512(SB), NOSPLIT, $0-1
MOVB $0, ret+0(FP) // default to false
// These values from:
// https://github.com/apple/darwin-xnu/blob/xnu-4570.1.46/osfmk/i386/cpu_capabilities.h
#define commpage64_base_address 0x00007fffffe00000
#define commpage64_cpu_capabilities64 (commpage64_base_address+0x010)
#define commpage64_version (commpage64_base_address+0x01E)
#define hasAVX512F 0x0000004000000000
MOVQ $commpage64_version, BX
CMPW (BX), $13 // cpu_capabilities64 undefined in versions < 13
JL no_avx512
MOVQ $commpage64_cpu_capabilities64, BX
MOVQ $hasAVX512F, CX
TESTQ (BX), CX
JZ no_avx512
MOVB $1, ret+0(FP)
no_avx512:
RET

View File

@@ -0,0 +1,8 @@
//go:build amd64 && !darwin
// +build amd64,!darwin
package archvariant
func osAVX512Supported(ax uint32) bool {
return ax&v4OSSupport == v4OSSupport
}

View File

@@ -0,0 +1,8 @@
//go:build !amd64
// +build !amd64
package archvariant
func AMD64Variant() string {
return "v1"
}

View File

@@ -0,0 +1,30 @@
variable GO_VERSION {
default = "1.17"
}
target "_base" {
args = {
GO_VERSION = GO_VERSION
}
}
target "binary" {
inherits = ["_base"]
platforms = ["local"]
output = ["bin"]
}
target "all-arch" {
inherits = ["_base"]
platforms = [
"linux/amd64",
"linux/arm64",
"linux/arm",
"linux/riscv64",
"linux/386",
"windows/amd64",
"windows/arm64",
"darwin/amd64",
"darwin/arm64",
]
}

5
vendor/modules.txt vendored
View File

@@ -57,7 +57,7 @@ github.com/containerd/cgroups/stats/v1
# github.com/containerd/console v1.0.3
## explicit; go 1.13
github.com/containerd/console
# github.com/containerd/containerd v1.6.0-rc.1
# github.com/containerd/containerd v1.6.0-rc.1.0.20220127150749-a43703fcba54
## explicit; go 1.16
github.com/containerd/containerd
github.com/containerd/containerd/api/services/containers/v1
@@ -474,6 +474,9 @@ github.com/tonistiigi/fsutil/types
# github.com/tonistiigi/go-actions-cache v0.0.0-20211202175116-9642704158ff
## explicit; go 1.16
github.com/tonistiigi/go-actions-cache
# github.com/tonistiigi/go-archvariant v1.0.0
## explicit; go 1.17
github.com/tonistiigi/go-archvariant
# github.com/tonistiigi/units v0.0.0-20180711220420-6950e57a87ea
## explicit
github.com/tonistiigi/units

View File

@@ -198,14 +198,16 @@ func (w *Worker) Labels() map[string]string {
func (w *Worker) Platforms(noCache bool) []ocispecs.Platform {
if noCache {
pm := make(map[string]struct{}, len(w.WorkerOpt.Platforms))
for _, p := range w.WorkerOpt.Platforms {
pm[platforms.Format(p)] = struct{}{}
}
for _, p := range archutil.SupportedPlatforms(noCache) {
if _, ok := pm[p]; !ok {
pp, _ := platforms.Parse(p)
w.WorkerOpt.Platforms = append(w.WorkerOpt.Platforms, pp)
exists := false
for _, pp := range w.WorkerOpt.Platforms {
if platforms.Only(pp).Match(p) {
exists = true
break
}
}
if !exists {
w.WorkerOpt.Platforms = append(w.WorkerOpt.Platforms, p)
}
}
}