From fdfa6bcb59fad9179ad16f0e652a433f213a8e4b Mon Sep 17 00:00:00 2001 From: Tonis Tiigi Date: Mon, 4 Dec 2017 23:34:25 -0800 Subject: [PATCH] vendor: update containerd to 1.0.0 Signed-off-by: Tonis Tiigi --- README.md | 2 +- cache/cacheimport/import.go | 2 +- cache/contenthash/checksum_test.go | 2 +- cache/manager.go | 2 +- cache/manager_test.go | 2 +- cache/refs.go | 2 +- control/control.go | 4 +- control/control_default.go | 2 +- control/control_standalone.go | 4 +- examples/buildkit0/buildkit.go | 2 +- examples/buildkit1/buildkit.go | 2 +- examples/buildkit2/buildkit.go | 2 +- examples/buildkit3/buildkit.go | 2 +- hack/dockerfiles/test.Dockerfile | 2 +- snapshot/blobmapping/snapshotter.go | 14 +- snapshot/snapshotter.go | 4 +- source/containerimage/pull.go | 6 +- source/git/gitsource_test.go | 2 +- source/http/httpsource_test.go | 2 +- vendor.conf | 4 +- .../containerd/containerd/README.md | 76 +- .../api/services/images/v1/images.pb.go | 124 ++- .../api/services/images/v1/images.proto | 6 + .../v1/snapshots.pb.go | 136 +-- .../v1/snapshots.proto | 2 +- .../containerd/containerd/archive/tar.go | 45 +- .../containerd/containerd/client.go | 182 ++-- .../containerd/containerd/container.go | 8 +- .../containerd/containerd/container_opts.go | 28 + .../containerd/container_opts_unix.go | 96 ++ .../containerd/containerd/content/helpers.go | 17 +- .../containerd/content/local/store.go | 19 +- .../containerd/diff/walking/differ.go | 32 +- .../containerd/containerd/events/events.go | 31 + .../containerd/containerd/export.go | 189 ---- .../containerd/containerd/filters/parser.go | 22 +- .../containerd/containerd/filters/quote.go | 237 +++++ .../containerd/containerd/filters/scanner.go | 5 +- .../containerd/containerd/fs/copy.go | 13 +- .../containerd/containerd/fs/copy_linux.go | 4 +- .../containerd/containerd/fs/copy_unix.go | 4 +- .../containerd/containerd/fs/copy_windows.go | 4 +- .../containerd/containerd/fs/diff.go | 83 +- .../github.com/containerd/containerd/fs/du.go | 9 + .../containerd/containerd/fs/du_unix.go | 46 +- .../containerd/containerd/fs/du_windows.go | 27 + .../containerd/fs/fstest/testsuite.go | 23 + .../github.com/containerd/containerd/image.go | 18 +- .../containerd/containerd/image_store.go | 9 +- .../containerd/containerd/images/image.go | 27 +- .../containerd/images/importexport.go | 21 + .../containerd/containerd/import.go | 120 --- .../github.com/containerd/containerd/lease.go | 3 +- .../containerd/metadata/containers.go | 20 +- .../containerd/containerd/metadata/content.go | 13 +- .../containerd/containerd/metadata/db.go | 120 ++- .../containerd/containerd/metadata/gc.go | 4 +- .../containerd/containerd/metadata/images.go | 8 +- .../containerd/metadata/snapshot.go | 85 +- .../containerd/mount/mount_linux.go | 27 +- .../containerd/containerd/oci/client.go | 22 + .../containerd/containerd/{ => oci}/spec.go | 4 +- .../containerd/containerd/oci/spec_opts.go | 35 + .../containerd/{ => oci}/spec_opts_unix.go | 152 +--- .../containerd/{ => oci}/spec_opts_windows.go | 25 +- .../containerd/{ => oci}/spec_unix.go | 37 +- .../containerd/{ => oci}/spec_windows.go | 2 +- .../containerd/containerd/plugin/plugin.go | 2 + .../containerd/plugin/plugin_go18.go | 2 +- .../containerd/plugin/plugin_other.go | 2 +- .../containerd/containerd/remotes/handlers.go | 1 + .../containerd/containerd/rootfs/apply.go | 14 +- .../containerd/containerd/rootfs/diff.go | 6 +- .../containerd/containerd/rootfs/init.go | 14 +- .../containerd/containerd/snapshot.go | 88 +- .../{snapshot => snapshots}/naive/naive.go | 66 +- .../overlay/overlay.go | 88 +- .../{snapshot => snapshots}/snapshotter.go | 2 +- .../{snapshot => snapshots}/storage/bolt.go | 90 +- .../storage/metastore.go | 4 +- .../containerd/containerd/spec_opts.go | 74 -- .../github.com/containerd/containerd/task.go | 10 +- .../containerd/containerd/task_opts_linux.go | 15 + .../containerd/task_opts_windows.go | 15 + .../containerd/containerd/vendor.conf | 7 +- vendor/github.com/dmcgowan/go-tar/common.go | 706 +++++++++++++-- vendor/github.com/dmcgowan/go-tar/format.go | 170 +++- vendor/github.com/dmcgowan/go-tar/reader.go | 842 +++++++++-------- .../github.com/dmcgowan/go-tar/sparse_unix.go | 77 ++ .../dmcgowan/go-tar/sparse_windows.go | 129 +++ .../go-tar/{stat_atim.go => stat_actime1.go} | 0 .../{stat_atimespec.go => stat_actime2.go} | 0 .../github.com/dmcgowan/go-tar/stat_unix.go | 72 +- vendor/github.com/dmcgowan/go-tar/strconv.go | 126 ++- vendor/github.com/dmcgowan/go-tar/writer.go | 845 ++++++++++++------ worker/oci/spec_unix.go | 10 +- 96 files changed, 3694 insertions(+), 2067 deletions(-) rename vendor/github.com/containerd/containerd/api/services/{snapshot => snapshots}/v1/snapshots.pb.go (94%) rename vendor/github.com/containerd/containerd/api/services/{snapshot => snapshots}/v1/snapshots.proto (99%) delete mode 100644 vendor/github.com/containerd/containerd/export.go create mode 100644 vendor/github.com/containerd/containerd/filters/quote.go create mode 100644 vendor/github.com/containerd/containerd/images/importexport.go delete mode 100644 vendor/github.com/containerd/containerd/import.go create mode 100644 vendor/github.com/containerd/containerd/oci/client.go rename vendor/github.com/containerd/containerd/{ => oci}/spec.go (75%) create mode 100644 vendor/github.com/containerd/containerd/oci/spec_opts.go rename vendor/github.com/containerd/containerd/{ => oci}/spec_opts_unix.go (66%) rename vendor/github.com/containerd/containerd/{ => oci}/spec_opts_windows.go (65%) rename vendor/github.com/containerd/containerd/{ => oci}/spec_unix.go (79%) rename vendor/github.com/containerd/containerd/{ => oci}/spec_windows.go (96%) rename vendor/github.com/containerd/containerd/{snapshot => snapshots}/naive/naive.go (79%) rename vendor/github.com/containerd/containerd/{snapshot => snapshots}/overlay/overlay.go (80%) rename vendor/github.com/containerd/containerd/{snapshot => snapshots}/snapshotter.go (99%) rename vendor/github.com/containerd/containerd/{snapshot => snapshots}/storage/bolt.go (83%) rename vendor/github.com/containerd/containerd/{snapshot => snapshots}/storage/metastore.go (97%) delete mode 100644 vendor/github.com/containerd/containerd/spec_opts.go create mode 100644 vendor/github.com/containerd/containerd/task_opts_linux.go create mode 100644 vendor/github.com/containerd/containerd/task_opts_windows.go create mode 100644 vendor/github.com/dmcgowan/go-tar/sparse_unix.go create mode 100644 vendor/github.com/dmcgowan/go-tar/sparse_windows.go rename vendor/github.com/dmcgowan/go-tar/{stat_atim.go => stat_actime1.go} (100%) rename vendor/github.com/dmcgowan/go-tar/{stat_atimespec.go => stat_actime2.go} (100%) diff --git a/README.md b/README.md index b0e5379cd..e3a5e830e 100644 --- a/README.md +++ b/README.md @@ -139,7 +139,7 @@ The `tonistiigi/buildkit:standalone` image can be built locally using the Docker #### Supported runc version -During development buildkit is tested with the version of runc that is being used by the containerd repository. Please refer to [runc.md](https://github.com/containerd/containerd/blob/4af5f657526a8aa5eedef734f5f29152031b1d3a/RUNC.md) for more information. +During development buildkit is tested with the version of runc that is being used by the containerd repository. Please refer to [runc.md](https://github.com/containerd/containerd/blob/v1.0.0/RUNC.md) for more information. #### Contributing diff --git a/cache/cacheimport/import.go b/cache/cacheimport/import.go index 5b3e73988..ad0a97bcc 100644 --- a/cache/cacheimport/import.go +++ b/cache/cacheimport/import.go @@ -11,7 +11,7 @@ import ( "github.com/containerd/containerd/remotes" "github.com/containerd/containerd/remotes/docker" "github.com/containerd/containerd/rootfs" - cdsnapshot "github.com/containerd/containerd/snapshot" + cdsnapshot "github.com/containerd/containerd/snapshots" "github.com/moby/buildkit/cache" "github.com/moby/buildkit/cache/blobs" "github.com/moby/buildkit/client" diff --git a/cache/contenthash/checksum_test.go b/cache/contenthash/checksum_test.go index 773f1ab70..042ddb40a 100644 --- a/cache/contenthash/checksum_test.go +++ b/cache/contenthash/checksum_test.go @@ -11,7 +11,7 @@ import ( "testing" "time" - "github.com/containerd/containerd/snapshot/naive" + "github.com/containerd/containerd/snapshots/naive" "github.com/moby/buildkit/cache" "github.com/moby/buildkit/cache/metadata" "github.com/moby/buildkit/snapshot" diff --git a/cache/manager.go b/cache/manager.go index 60e3b1ac6..29866bf62 100644 --- a/cache/manager.go +++ b/cache/manager.go @@ -6,7 +6,7 @@ import ( "sync" "time" - cdsnapshot "github.com/containerd/containerd/snapshot" + cdsnapshot "github.com/containerd/containerd/snapshots" "github.com/moby/buildkit/cache/metadata" "github.com/moby/buildkit/client" "github.com/moby/buildkit/identity" diff --git a/cache/manager_test.go b/cache/manager_test.go index e9603c6cb..526a3cf37 100644 --- a/cache/manager_test.go +++ b/cache/manager_test.go @@ -9,7 +9,7 @@ import ( "testing" "github.com/containerd/containerd/namespaces" - "github.com/containerd/containerd/snapshot/naive" + "github.com/containerd/containerd/snapshots/naive" "github.com/moby/buildkit/cache/metadata" "github.com/moby/buildkit/client" "github.com/moby/buildkit/snapshot" diff --git a/cache/refs.go b/cache/refs.go index 66a30fdcc..6fbe69f2f 100644 --- a/cache/refs.go +++ b/cache/refs.go @@ -5,7 +5,7 @@ import ( "time" "github.com/containerd/containerd/mount" - cdsnapshot "github.com/containerd/containerd/snapshot" + cdsnapshot "github.com/containerd/containerd/snapshots" "github.com/moby/buildkit/cache/metadata" "github.com/moby/buildkit/identity" "github.com/moby/buildkit/util/flightcontrol" diff --git a/control/control.go b/control/control.go index 4ac599d4b..b62fae012 100644 --- a/control/control.go +++ b/control/control.go @@ -1,7 +1,7 @@ package control import ( - "github.com/containerd/containerd/snapshot" + "github.com/containerd/containerd/snapshots" "github.com/docker/distribution/reference" controlapi "github.com/moby/buildkit/api/services/control" "github.com/moby/buildkit/cache" @@ -22,7 +22,7 @@ import ( ) type Opt struct { - Snapshotter snapshot.Snapshotter + Snapshotter snapshots.Snapshotter CacheManager cache.Manager Worker worker.Worker SourceManager *source.Manager diff --git a/control/control_default.go b/control/control_default.go index 3840928a0..cee931a66 100644 --- a/control/control_default.go +++ b/control/control_default.go @@ -8,7 +8,7 @@ import ( "github.com/containerd/containerd/content" "github.com/containerd/containerd/diff" "github.com/containerd/containerd/images" - ctdsnapshot "github.com/containerd/containerd/snapshot" + ctdsnapshot "github.com/containerd/containerd/snapshots" "github.com/moby/buildkit/cache" "github.com/moby/buildkit/cache/cacheimport" "github.com/moby/buildkit/cache/instructioncache" diff --git a/control/control_standalone.go b/control/control_standalone.go index 25ea7af6a..ea5983d94 100644 --- a/control/control_standalone.go +++ b/control/control_standalone.go @@ -14,8 +14,8 @@ import ( "github.com/containerd/containerd/metadata" "github.com/containerd/containerd/mount" "github.com/containerd/containerd/namespaces" - ctdsnapshot "github.com/containerd/containerd/snapshot" - "github.com/containerd/containerd/snapshot/overlay" + ctdsnapshot "github.com/containerd/containerd/snapshots" + "github.com/containerd/containerd/snapshots/overlay" "github.com/moby/buildkit/worker/runcworker" digest "github.com/opencontainers/go-digest" "github.com/pkg/errors" diff --git a/examples/buildkit0/buildkit.go b/examples/buildkit0/buildkit.go index 6b87eb341..e29216265 100644 --- a/examples/buildkit0/buildkit.go +++ b/examples/buildkit0/buildkit.go @@ -17,7 +17,7 @@ type buildOpt struct { func main() { var opt buildOpt flag.StringVar(&opt.target, "target", "containerd", "target (standalone, containerd)") - flag.StringVar(&opt.containerd, "containerd", "4af5f657526a8aa5eedef734f5f29152031b1d3a", "containerd version") + flag.StringVar(&opt.containerd, "containerd", "v1.0.0", "containerd version") flag.StringVar(&opt.runc, "runc", "74a17296470088de3805e138d3d87c62e613dfc4", "runc version") flag.Parse() diff --git a/examples/buildkit1/buildkit.go b/examples/buildkit1/buildkit.go index 1567836ff..29948a0ad 100644 --- a/examples/buildkit1/buildkit.go +++ b/examples/buildkit1/buildkit.go @@ -17,7 +17,7 @@ type buildOpt struct { func main() { var opt buildOpt flag.StringVar(&opt.target, "target", "containerd", "target (standalone, containerd)") - flag.StringVar(&opt.containerd, "containerd", "4af5f657526a8aa5eedef734f5f29152031b1d3a", "containerd version") + flag.StringVar(&opt.containerd, "containerd", "v1.0.0", "containerd version") flag.StringVar(&opt.runc, "runc", "74a17296470088de3805e138d3d87c62e613dfc4", "runc version") flag.Parse() diff --git a/examples/buildkit2/buildkit.go b/examples/buildkit2/buildkit.go index 3f50e2572..1dd3aa2f1 100644 --- a/examples/buildkit2/buildkit.go +++ b/examples/buildkit2/buildkit.go @@ -17,7 +17,7 @@ type buildOpt struct { func main() { var opt buildOpt flag.StringVar(&opt.target, "target", "containerd", "target (standalone, containerd)") - flag.StringVar(&opt.containerd, "containerd", "4af5f657526a8aa5eedef734f5f29152031b1d3a", "containerd version") + flag.StringVar(&opt.containerd, "containerd", "v1.0.0", "containerd version") flag.StringVar(&opt.runc, "runc", "74a17296470088de3805e138d3d87c62e613dfc4", "runc version") flag.Parse() diff --git a/examples/buildkit3/buildkit.go b/examples/buildkit3/buildkit.go index 2655574bf..c1c8bf5b8 100644 --- a/examples/buildkit3/buildkit.go +++ b/examples/buildkit3/buildkit.go @@ -18,7 +18,7 @@ type buildOpt struct { func main() { var opt buildOpt flag.StringVar(&opt.target, "target", "containerd", "target (standalone, containerd)") - flag.StringVar(&opt.containerd, "containerd", "4af5f657526a8aa5eedef734f5f29152031b1d3a", "containerd version") + flag.StringVar(&opt.containerd, "containerd", "v1.0.0", "containerd version") flag.StringVar(&opt.runc, "runc", "74a17296470088de3805e138d3d87c62e613dfc4", "runc version") flag.StringVar(&opt.buildkit, "buildkit", "master", "buildkit version") flag.Parse() diff --git a/hack/dockerfiles/test.Dockerfile b/hack/dockerfiles/test.Dockerfile index eed3d4b35..4e6e05db9 100644 --- a/hack/dockerfiles/test.Dockerfile +++ b/hack/dockerfiles/test.Dockerfile @@ -1,5 +1,5 @@ ARG RUNC_VERSION=74a17296470088de3805e138d3d87c62e613dfc4 -ARG CONTAINERD_VERSION=9649a428e8c470cddc0d5a3c394f36111f940adc +ARG CONTAINERD_VERSION=v1.0.0 ARG BUILDKIT_TARGET=standalone FROM golang:1.9-alpine AS gobuild-base diff --git a/snapshot/blobmapping/snapshotter.go b/snapshot/blobmapping/snapshotter.go index fb66dca55..c759fdbfa 100644 --- a/snapshot/blobmapping/snapshotter.go +++ b/snapshot/blobmapping/snapshotter.go @@ -5,7 +5,7 @@ import ( "github.com/boltdb/bolt" "github.com/containerd/containerd/content" - "github.com/containerd/containerd/snapshot" + "github.com/containerd/containerd/snapshots" "github.com/moby/buildkit/cache/metadata" digest "github.com/opencontainers/go-digest" "github.com/sirupsen/logrus" @@ -15,12 +15,12 @@ const blobKey = "blobmapping.blob" type Opt struct { Content content.Store - Snapshotter snapshot.Snapshotter + Snapshotter snapshots.Snapshotter MetadataStore *metadata.Store } type Info struct { - snapshot.Info + snapshots.Info Blob string } @@ -32,7 +32,7 @@ type DiffPair struct { // this snapshotter keeps an internal mapping between a snapshot and a blob type Snapshotter struct { - snapshot.Snapshotter + snapshots.Snapshotter opt Opt } @@ -70,10 +70,10 @@ func (s *Snapshotter) Remove(ctx context.Context, key string) error { return nil } -func (s *Snapshotter) Usage(ctx context.Context, key string) (snapshot.Usage, error) { +func (s *Snapshotter) Usage(ctx context.Context, key string) (snapshots.Usage, error) { u, err := s.Snapshotter.Usage(ctx, key) if err != nil { - return snapshot.Usage{}, err + return snapshots.Usage{}, err } _, blob, err := s.GetBlob(ctx, key) if err != nil { @@ -84,7 +84,7 @@ func (s *Snapshotter) Usage(ctx context.Context, key string) (snapshot.Usage, er if err != nil { return u, err } - (&u).Add(snapshot.Usage{Size: info.Size, Inodes: 1}) + (&u).Add(snapshots.Usage{Size: info.Size, Inodes: 1}) } return u, nil } diff --git a/snapshot/snapshotter.go b/snapshot/snapshotter.go index 9fc1397b1..66ade92c7 100644 --- a/snapshot/snapshotter.go +++ b/snapshot/snapshotter.go @@ -1,10 +1,10 @@ package snapshot import ( - "github.com/containerd/containerd/snapshot" + "github.com/containerd/containerd/snapshots" ) // Snapshotter defines interface that any snapshot implementation should satisfy type Snapshotter interface { - snapshot.Snapshotter + snapshots.Snapshotter } diff --git a/source/containerimage/pull.go b/source/containerimage/pull.go index 6e146dbf2..c76116194 100644 --- a/source/containerimage/pull.go +++ b/source/containerimage/pull.go @@ -16,7 +16,7 @@ import ( "github.com/containerd/containerd/remotes/docker" "github.com/containerd/containerd/remotes/docker/schema1" "github.com/containerd/containerd/rootfs" - "github.com/containerd/containerd/snapshot" + "github.com/containerd/containerd/snapshots" "github.com/moby/buildkit/cache" "github.com/moby/buildkit/session" "github.com/moby/buildkit/session/auth" @@ -35,7 +35,7 @@ import ( type SourceOpt struct { SessionManager *session.Manager - Snapshotter snapshot.Snapshotter + Snapshotter snapshots.Snapshotter ContentStore content.Store Applier diff.Differ CacheAccessor cache.Accessor @@ -243,7 +243,7 @@ func (is *imageSource) unpack(ctx context.Context, desc ocispec.Descriptor) (str "containerd.io/gc.root": time.Now().UTC().Format(time.RFC3339Nano), "containerd.io/uncompressed": layer.Diff.Digest.String(), } - if _, err := rootfs.ApplyLayer(ctx, layer, chain, is.Snapshotter, is.Applier, snapshot.WithLabels(labels)); err != nil { + if _, err := rootfs.ApplyLayer(ctx, layer, chain, is.Snapshotter, is.Applier, snapshots.WithLabels(labels)); err != nil { return "", err } chain = append(chain, layer.Diff.Digest) diff --git a/source/git/gitsource_test.go b/source/git/gitsource_test.go index 952162ed5..0c3c989fd 100644 --- a/source/git/gitsource_test.go +++ b/source/git/gitsource_test.go @@ -10,7 +10,7 @@ import ( "testing" "github.com/containerd/containerd/namespaces" - "github.com/containerd/containerd/snapshot/naive" + "github.com/containerd/containerd/snapshots/naive" "github.com/moby/buildkit/cache" "github.com/moby/buildkit/cache/metadata" "github.com/moby/buildkit/snapshot" diff --git a/source/http/httpsource_test.go b/source/http/httpsource_test.go index a3b4074d1..d25a5ec57 100644 --- a/source/http/httpsource_test.go +++ b/source/http/httpsource_test.go @@ -7,7 +7,7 @@ import ( "path/filepath" "testing" - "github.com/containerd/containerd/snapshot/naive" + "github.com/containerd/containerd/snapshots/naive" "github.com/moby/buildkit/cache" "github.com/moby/buildkit/cache/metadata" "github.com/moby/buildkit/identity" diff --git a/vendor.conf b/vendor.conf index 9a8eb6546..8e4b188b8 100644 --- a/vendor.conf +++ b/vendor.conf @@ -6,7 +6,7 @@ github.com/davecgh/go-spew v1.1.0 github.com/pmezard/go-difflib v1.0.0 golang.org/x/sys 9aade4d3a3b7e6d876cd3823ad20ec45fc035402 # update to 7ddbeae9ae08c6a06a59597f0c9edbc5ff2444ce wiht pkg/signal change -github.com/containerd/containerd 4af5f657526a8aa5eedef734f5f29152031b1d3a +github.com/containerd/containerd v1.0.0 github.com/containerd/typeurl f6943554a7e7e88b3c14aad190bf05932da84788 golang.org/x/sync 450f422ab23cf9881c94e2db30cac0eb1b7cf80c github.com/sirupsen/logrus v1.0.0 @@ -37,7 +37,7 @@ github.com/docker/docker 6f723db8c6f0c7f0b252674a9673a25b5978db04 https://github github.com/pkg/profile 5b67d428864e92711fcbd2f8629456121a56d91f github.com/tonistiigi/fsutil efacb76f1192fb1d48cdaf9763ff9f31760481ce -github.com/dmcgowan/go-tar 2e2c51242e8993c50445dab7c03c8e7febddd0cf +github.com/dmcgowan/go-tar go1.10 github.com/hashicorp/go-immutable-radix 826af9ccf0feeee615d546d69b11f8e98da8c8f1 git://github.com/tonistiigi/go-immutable-radix.git github.com/hashicorp/golang-lru a0d98a5f288019575c6d1f4bb1573fef2d1fcdc4 github.com/mitchellh/hashstructure 2bca23e0e452137f789efbc8610126fd8b94f73b diff --git a/vendor/github.com/containerd/containerd/README.md b/vendor/github.com/containerd/containerd/README.md index 5d4c507fc..84d1eec8a 100644 --- a/vendor/github.com/containerd/containerd/README.md +++ b/vendor/github.com/containerd/containerd/README.md @@ -13,7 +13,37 @@ containerd is designed to be embedded into a larger system, rather than being us ## Getting Started -If you are interested in trying out containerd please see our [Getting Started Guide](docs/getting-started.md). +See our documentation on [containerd.io](containerd.io): +* [for ops and admins](docs/ops.md) +* [namespaces](docs/namespaces.md) +* [client options](docs/client-opts.md) + +See how to build containerd from source at [BUILDING](BUILDING.md). + +If you are interested in trying out containerd see our example at [Getting Started](docs/getting-started.md). + + +## Runtime Requirements + +Runtime requirements for containerd are very minimal. Most interactions with +the Linux and Windows container feature sets are handled via [runc](https://github.com/opencontainers/runc) and/or +OS-specific libraries (e.g. [hcsshim](https://github.com/Microsoft/hcsshim) for Microsoft). The current required version of `runc` is always listed in [RUNC.md](/RUNC.md). + +There are specific features +used by containerd core code and snapshotters that will require a minimum kernel +version on Linux. With the understood caveat of distro kernel versioning, a +reasonable starting point for Linux is a minimum 4.x kernel version. + +The overlay filesystem snapshotter, used by default, uses features that were +finalized in the 4.x kernel series. If you choose to use btrfs, there may +be more flexibility in kernel version (minimum recommended is 3.18), but will +require the btrfs kernel module and btrfs tools to be installed on your Linux +distribution. + +To use Linux checkpoint and restore features, you will need `criu` installed on +your system. See more details in [Checkpoint and Restore](#checkpoint-and-restore). + +Build requirements for developers are listed in [BUILDING](BUILDING.md). ## Features @@ -23,7 +53,11 @@ containerd offers a full client package to help you integrate containerd into yo ```go -import "github.com/containerd/containerd" +import ( + "github.com/containerd/containerd" + "github.com/containerd/containerd/cio" +) + func main() { client, err := containerd.New("/run/containerd/containerd.sock") @@ -39,7 +73,7 @@ Namespaces allow multiple consumers to use the same containerd without conflicti To set a namespace for requests to the API: ```go -context = context.Background() +context = context.Background() // create a context for docker docker = namespaces.WithNamespace(context, "docker") @@ -78,7 +112,7 @@ containerd fully supports the OCI runtime specification for running containers. You can specify options when creating a container about how to modify the specification. ```go -redis, err := client.NewContainer(context, "redis-master", containerd.WithNewSpec(containerd.WithImageConfig(image))) +redis, err := client.NewContainer(context, "redis-master", containerd.WithNewSpec(oci.WithImageConfig(image))) ``` ### Root Filesystems @@ -92,8 +126,7 @@ image, err := client.Pull(context, "docker.io/library/redis:latest", containerd. // allocate a new RW root filesystem for a container based on the image redis, err := client.NewContainer(context, "redis-master", containerd.WithNewSnapshot("redis-rootfs", image), - containerd.WithNewSpec(containerd.WithImageConfig(image)), - + containerd.WithNewSpec(oci.WithImageConfig(image)), ) // use a readonly filesystem with multiple containers @@ -101,7 +134,7 @@ for i := 0; i < 10; i++ { id := fmt.Sprintf("id-%s", i) container, err := client.NewContainer(ctx, id, containerd.WithNewSnapshotView(id, image), - containerd.WithNewSpec(containerd.WithImageConfig(image)), + containerd.WithNewSpec(oci.WithImageConfig(image)), ) } ``` @@ -112,7 +145,7 @@ Taking a container object and turning it into a runnable process on a system is ```go // create a new task -task, err := redis.NewTask(context, containerd.Stdio) +task, err := redis.NewTask(context, cio.Stdio) defer task.Delete(context) // the task is now running and has a pid that can be use to setup networking @@ -144,37 +177,12 @@ checkpoint := image.Target() redis, err = client.NewContainer(context, "redis-master", containerd.WithCheckpoint(checkpoint, "redis-rootfs")) defer container.Delete(context) -task, err = redis.NewTask(context, containerd.Stdio, containerd.WithTaskCheckpoint(checkpoint)) +task, err = redis.NewTask(context, cio.Stdio, containerd.WithTaskCheckpoint(checkpoint)) defer task.Delete(context) err := task.Start(context) ``` -## Developer Quick-Start - -To build the daemon and `ctr` simple test client, the following build system dependencies are required: - -* Go 1.9.x or above -* Protoc 3.x compiler and headers (download at the [Google protobuf releases page](https://github.com/google/protobuf/releases)) -* Btrfs headers and libraries for your distribution. Note that building the btrfs driver can be disabled via build tag removing this dependency. - -For proper results, install the `protoc` release into `/usr/local` on your build system. For example, the following commands will download and install the 3.5.0 release for a 64-bit Linux host: - -``` -$ wget -c https://github.com/google/protobuf/releases/download/v3.5.0/protoc-3.5.0-linux-x86_64.zip -$ sudo unzip protoc-3.5.0-linux-x86_64.zip -d /usr/local -``` - -With the required dependencies installed, the `Makefile` target named **binaries** will compile the `ctr` and `containerd` binaries and place them in the `bin/` directory. Using `sudo make install` will place the binaries in `/usr/local/bin`. When making any changes to the gRPC API, `make generate` will use the installed `protoc` compiler to regenerate the API generated code packages. - -> *Note*: A build tag is currently available to disable building the btrfs snapshot driver. -> Adding `BUILDTAGS=no_btrfs` to your environment before calling the **binaries** -> Makefile target will disable the btrfs driver within the containerd Go build. - -Vendoring of external imports uses the [`vndr` tool](https://github.com/LK4D4/vndr) which uses a simple config file, `vendor.conf`, to provide the URL and version or hash details for each vendored import. After modifying `vendor.conf` run the `vndr` tool to update the `vendor/` directory contents. Combining the `vendor.conf` update with the changeset in `vendor/` after running `vndr` should become a single commit for a PR which relies on vendored updates. - -Please refer to [RUNC.md](/RUNC.md) for the currently supported version of `runc` that is used by containerd. - ### Releases and API Stability Please see [RELEASES.md](RELEASES.md) for details on versioning and stability diff --git a/vendor/github.com/containerd/containerd/api/services/images/v1/images.pb.go b/vendor/github.com/containerd/containerd/api/services/images/v1/images.pb.go index 4c9f0f982..4577eb089 100644 --- a/vendor/github.com/containerd/containerd/api/services/images/v1/images.pb.go +++ b/vendor/github.com/containerd/containerd/api/services/images/v1/images.pb.go @@ -163,6 +163,11 @@ func (*ListImagesResponse) Descriptor() ([]byte, []int) { return fileDescriptorI type DeleteImageRequest struct { Name string `protobuf:"bytes,1,opt,name=name,proto3" json:"name,omitempty"` + // Sync indicates that the delete and cleanup should be done + // synchronously before returning to the caller + // + // Default is false + Sync bool `protobuf:"varint,2,opt,name=sync,proto3" json:"sync,omitempty"` } func (m *DeleteImageRequest) Reset() { *m = DeleteImageRequest{} } @@ -717,6 +722,16 @@ func (m *DeleteImageRequest) MarshalTo(dAtA []byte) (int, error) { i = encodeVarintImages(dAtA, i, uint64(len(m.Name))) i += copy(dAtA[i:], m.Name) } + if m.Sync { + dAtA[i] = 0x10 + i++ + if m.Sync { + dAtA[i] = 1 + } else { + dAtA[i] = 0 + } + i++ + } return i, nil } @@ -840,6 +855,9 @@ func (m *DeleteImageRequest) Size() (n int) { if l > 0 { n += 1 + l + sovImages(uint64(l)) } + if m.Sync { + n += 2 + } return n } @@ -967,6 +985,7 @@ func (this *DeleteImageRequest) String() string { } s := strings.Join([]string{`&DeleteImageRequest{`, `Name:` + fmt.Sprintf("%v", this.Name) + `,`, + `Sync:` + fmt.Sprintf("%v", this.Sync) + `,`, `}`, }, "") return s @@ -1999,6 +2018,26 @@ func (m *DeleteImageRequest) Unmarshal(dAtA []byte) error { } m.Name = string(dAtA[iNdEx:postIndex]) iNdEx = postIndex + case 2: + if wireType != 0 { + return fmt.Errorf("proto: wrong wireType = %d for field Sync", wireType) + } + var v int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowImages + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + v |= (int(b) & 0x7F) << shift + if b < 0x80 { + break + } + } + m.Sync = bool(v != 0) default: iNdEx = preIndex skippy, err := skipImages(dAtA[iNdEx:]) @@ -2130,46 +2169,47 @@ func init() { } var fileDescriptorImages = []byte{ - // 650 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xac, 0x55, 0x4f, 0x6f, 0xd3, 0x4e, - 0x10, 0x8d, 0x93, 0xd4, 0x6d, 0x27, 0x87, 0x5f, 0x7f, 0x4b, 0x85, 0x2c, 0x03, 0x69, 0x14, 0x81, - 0x94, 0x0b, 0x6b, 0x1a, 0x2e, 0xd0, 0x4a, 0x88, 0xa6, 0x2d, 0x05, 0xa9, 0x70, 0x30, 0xff, 0x2a, - 0x2e, 0xd5, 0x26, 0x99, 0x18, 0x2b, 0x76, 0x6c, 0xbc, 0x9b, 0x48, 0xb9, 0xf1, 0x11, 0x90, 0xe0, - 0x43, 0xf5, 0xc8, 0x91, 0x13, 0xd0, 0x1c, 0xf8, 0x1c, 0xc8, 0xbb, 0x1b, 0x9a, 0x26, 0x11, 0x6e, - 0x4a, 0x6f, 0xe3, 0xf8, 0xbd, 0x79, 0x33, 0x6f, 0x66, 0x62, 0xd8, 0xf3, 0x7c, 0xf1, 0xbe, 0xdf, - 0xa4, 0xad, 0x28, 0x74, 0x5a, 0x51, 0x4f, 0x30, 0xbf, 0x87, 0x49, 0x7b, 0x32, 0x64, 0xb1, 0xef, - 0x70, 0x4c, 0x06, 0x7e, 0x0b, 0xb9, 0xe3, 0x87, 0xcc, 0x43, 0xee, 0x0c, 0x36, 0x75, 0x44, 0xe3, - 0x24, 0x12, 0x11, 0xb9, 0x75, 0x86, 0xa7, 0x63, 0x2c, 0xd5, 0x88, 0xc1, 0xa6, 0xbd, 0xee, 0x45, - 0x5e, 0x24, 0x91, 0x4e, 0x1a, 0x29, 0x92, 0x7d, 0xc3, 0x8b, 0x22, 0x2f, 0x40, 0x47, 0x3e, 0x35, - 0xfb, 0x1d, 0x07, 0xc3, 0x58, 0x0c, 0xf5, 0xcb, 0xca, 0xf4, 0xcb, 0x8e, 0x8f, 0x41, 0xfb, 0x38, - 0x64, 0xbc, 0xab, 0x11, 0x1b, 0xd3, 0x08, 0xe1, 0x87, 0xc8, 0x05, 0x0b, 0x63, 0x0d, 0xd8, 0xbe, - 0x50, 0x6b, 0x62, 0x18, 0x23, 0x77, 0xda, 0xc8, 0x5b, 0x89, 0x1f, 0x8b, 0x28, 0x51, 0xe4, 0xea, - 0xaf, 0x3c, 0x2c, 0x3d, 0x4b, 0x1b, 0x20, 0x04, 0x8a, 0x3d, 0x16, 0xa2, 0x65, 0x54, 0x8c, 0xda, - 0xaa, 0x2b, 0x63, 0xf2, 0x14, 0xcc, 0x80, 0x35, 0x31, 0xe0, 0x56, 0xbe, 0x52, 0xa8, 0x95, 0xea, - 0xf7, 0xe8, 0x5f, 0x0d, 0xa0, 0x32, 0x13, 0x3d, 0x94, 0x94, 0xfd, 0x9e, 0x48, 0x86, 0xae, 0xe6, - 0x93, 0x2d, 0x30, 0x05, 0x4b, 0x3c, 0x14, 0x56, 0xa1, 0x62, 0xd4, 0x4a, 0xf5, 0x9b, 0x93, 0x99, - 0x64, 0x6d, 0x74, 0xef, 0x4f, 0x6d, 0x8d, 0xe2, 0xc9, 0xf7, 0x8d, 0x9c, 0xab, 0x19, 0x64, 0x17, - 0xa0, 0x95, 0x20, 0x13, 0xd8, 0x3e, 0x66, 0xc2, 0x5a, 0x96, 0x7c, 0x9b, 0x2a, 0x5b, 0xe8, 0xd8, - 0x16, 0xfa, 0x6a, 0x6c, 0x4b, 0x63, 0x25, 0x65, 0x7f, 0xfa, 0xb1, 0x61, 0xb8, 0xab, 0x9a, 0xb7, - 0x23, 0x93, 0xf4, 0xe3, 0xf6, 0x38, 0xc9, 0xca, 0x22, 0x49, 0x34, 0x6f, 0x47, 0xd8, 0x0f, 0xa1, - 0x34, 0xd1, 0x1c, 0x59, 0x83, 0x42, 0x17, 0x87, 0xda, 0xb1, 0x34, 0x24, 0xeb, 0xb0, 0x34, 0x60, - 0x41, 0x1f, 0xad, 0xbc, 0xfc, 0x4d, 0x3d, 0x6c, 0xe5, 0x1f, 0x18, 0xd5, 0x3b, 0xf0, 0xdf, 0x01, - 0x0a, 0x69, 0x90, 0x8b, 0x1f, 0xfa, 0xc8, 0xc5, 0x3c, 0xc7, 0xab, 0x2f, 0x60, 0xed, 0x0c, 0xc6, - 0xe3, 0xa8, 0xc7, 0x91, 0x6c, 0xc1, 0x92, 0xb4, 0x58, 0x02, 0x4b, 0xf5, 0xdb, 0x17, 0x19, 0x82, - 0xab, 0x28, 0xd5, 0x37, 0x40, 0x76, 0xa5, 0x07, 0xe7, 0x94, 0x1f, 0x5f, 0x22, 0xa3, 0x1e, 0x8a, - 0xce, 0xfb, 0x16, 0xae, 0x9d, 0xcb, 0xab, 0x4b, 0xfd, 0xf7, 0xc4, 0x9f, 0x0d, 0x20, 0xaf, 0xa5, - 0xe1, 0x57, 0x5b, 0x31, 0xd9, 0x86, 0x92, 0x1a, 0xa4, 0x3c, 0x2e, 0x39, 0xa0, 0x79, 0x1b, 0xf0, - 0x24, 0xbd, 0xbf, 0xe7, 0x8c, 0x77, 0x5d, 0xbd, 0x2f, 0x69, 0x9c, 0xb6, 0x7b, 0xae, 0xa8, 0x2b, - 0x6b, 0xf7, 0x2e, 0xfc, 0x7f, 0xe8, 0x73, 0x35, 0x70, 0x3e, 0x6e, 0xd6, 0x82, 0xe5, 0x8e, 0x1f, - 0x08, 0x4c, 0xb8, 0x65, 0x54, 0x0a, 0xb5, 0x55, 0x77, 0xfc, 0x58, 0x3d, 0x02, 0x32, 0x09, 0xd7, - 0x65, 0x34, 0xc0, 0x54, 0x22, 0x12, 0xbe, 0x58, 0x1d, 0x9a, 0x59, 0xad, 0x01, 0xd9, 0xc3, 0x00, - 0xa7, 0x6c, 0x9f, 0xb3, 0xa2, 0xf5, 0x2f, 0x45, 0x30, 0x55, 0x01, 0xa4, 0x03, 0x85, 0x03, 0x14, - 0x84, 0x66, 0xe8, 0x4d, 0x2d, 0xbe, 0xed, 0x5c, 0x18, 0xaf, 0x1b, 0xec, 0x42, 0x31, 0x6d, 0x9b, - 0x64, 0xfd, 0xff, 0xcc, 0x58, 0x69, 0x6f, 0x2e, 0xc0, 0xd0, 0x62, 0x11, 0x98, 0x6a, 0xb5, 0x49, - 0x16, 0x79, 0xf6, 0xb2, 0xec, 0xfa, 0x22, 0x94, 0x33, 0x41, 0xb5, 0x5c, 0x99, 0x82, 0xb3, 0x87, - 0x91, 0x29, 0x38, 0x6f, 0x6d, 0x5f, 0x82, 0xa9, 0x66, 0x9d, 0x29, 0x38, 0xbb, 0x12, 0xf6, 0xf5, - 0x99, 0x93, 0xd9, 0x4f, 0xbf, 0x67, 0x8d, 0xa3, 0x93, 0xd3, 0x72, 0xee, 0xdb, 0x69, 0x39, 0xf7, - 0x71, 0x54, 0x36, 0x4e, 0x46, 0x65, 0xe3, 0xeb, 0xa8, 0x6c, 0xfc, 0x1c, 0x95, 0x8d, 0x77, 0x8f, - 0x2e, 0xf9, 0xed, 0xdd, 0x56, 0xd1, 0x51, 0xae, 0x69, 0x4a, 0xad, 0xfb, 0xbf, 0x03, 0x00, 0x00, - 0xff, 0xff, 0x86, 0xe6, 0x32, 0x0a, 0xc6, 0x07, 0x00, 0x00, + // 659 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xac, 0x55, 0xcd, 0x6e, 0xd3, 0x40, + 0x10, 0x8e, 0x93, 0xd4, 0x6d, 0x27, 0x07, 0xca, 0x52, 0x21, 0xcb, 0x40, 0x1a, 0x45, 0x20, 0xe5, + 0xc2, 0x9a, 0x86, 0x0b, 0xb4, 0x08, 0xd1, 0xb4, 0xa5, 0x20, 0x15, 0x0e, 0xe6, 0xaf, 0xe2, 0x52, + 0x6d, 0x92, 0x89, 0xb1, 0x62, 0xc7, 0xc6, 0xbb, 0x89, 0x94, 0x1b, 0x8f, 0x80, 0x04, 0x0f, 0xd5, + 0x23, 0x47, 0x4e, 0x40, 0x73, 0xe0, 0x39, 0x90, 0x77, 0x37, 0x34, 0x4d, 0x22, 0x92, 0x94, 0xde, + 0x66, 0xed, 0xef, 0x9b, 0x9f, 0x6f, 0x66, 0x76, 0x61, 0xcf, 0xf3, 0xc5, 0x87, 0x6e, 0x9d, 0x36, + 0xa2, 0xd0, 0x69, 0x44, 0x1d, 0xc1, 0xfc, 0x0e, 0x26, 0xcd, 0x51, 0x93, 0xc5, 0xbe, 0xc3, 0x31, + 0xe9, 0xf9, 0x0d, 0xe4, 0x8e, 0x1f, 0x32, 0x0f, 0xb9, 0xd3, 0xdb, 0xd4, 0x16, 0x8d, 0x93, 0x48, + 0x44, 0xe4, 0xd6, 0x19, 0x9e, 0x0e, 0xb1, 0x54, 0x23, 0x7a, 0x9b, 0xf6, 0xba, 0x17, 0x79, 0x91, + 0x44, 0x3a, 0xa9, 0xa5, 0x48, 0xf6, 0x0d, 0x2f, 0x8a, 0xbc, 0x00, 0x1d, 0x79, 0xaa, 0x77, 0x5b, + 0x0e, 0x86, 0xb1, 0xe8, 0xeb, 0x9f, 0xa5, 0xf1, 0x9f, 0x2d, 0x1f, 0x83, 0xe6, 0x71, 0xc8, 0x78, + 0x5b, 0x23, 0x36, 0xc6, 0x11, 0xc2, 0x0f, 0x91, 0x0b, 0x16, 0xc6, 0x1a, 0xb0, 0x3d, 0x57, 0x69, + 0xa2, 0x1f, 0x23, 0x77, 0x9a, 0xc8, 0x1b, 0x89, 0x1f, 0x8b, 0x28, 0x51, 0xe4, 0xf2, 0xef, 0x2c, + 0x2c, 0x3d, 0x4f, 0x0b, 0x20, 0x04, 0xf2, 0x1d, 0x16, 0xa2, 0x65, 0x94, 0x8c, 0xca, 0xaa, 0x2b, + 0x6d, 0xf2, 0x0c, 0xcc, 0x80, 0xd5, 0x31, 0xe0, 0x56, 0xb6, 0x94, 0xab, 0x14, 0xaa, 0xf7, 0xe8, + 0x3f, 0x05, 0xa0, 0xd2, 0x13, 0x3d, 0x94, 0x94, 0xfd, 0x8e, 0x48, 0xfa, 0xae, 0xe6, 0x93, 0x2d, + 0x30, 0x05, 0x4b, 0x3c, 0x14, 0x56, 0xae, 0x64, 0x54, 0x0a, 0xd5, 0x9b, 0xa3, 0x9e, 0x64, 0x6e, + 0x74, 0xef, 0x6f, 0x6e, 0xb5, 0xfc, 0xc9, 0x8f, 0x8d, 0x8c, 0xab, 0x19, 0x64, 0x17, 0xa0, 0x91, + 0x20, 0x13, 0xd8, 0x3c, 0x66, 0xc2, 0x5a, 0x96, 0x7c, 0x9b, 0x2a, 0x59, 0xe8, 0x50, 0x16, 0xfa, + 0x7a, 0x28, 0x4b, 0x6d, 0x25, 0x65, 0x7f, 0xfe, 0xb9, 0x61, 0xb8, 0xab, 0x9a, 0xb7, 0x23, 0x9d, + 0x74, 0xe3, 0xe6, 0xd0, 0xc9, 0xca, 0x22, 0x4e, 0x34, 0x6f, 0x47, 0xd8, 0x0f, 0xa1, 0x30, 0x52, + 0x1c, 0x59, 0x83, 0x5c, 0x1b, 0xfb, 0x5a, 0xb1, 0xd4, 0x24, 0xeb, 0xb0, 0xd4, 0x63, 0x41, 0x17, + 0xad, 0xac, 0xfc, 0xa6, 0x0e, 0x5b, 0xd9, 0x07, 0x46, 0xf9, 0x0e, 0x5c, 0x39, 0x40, 0x21, 0x05, + 0x72, 0xf1, 0x63, 0x17, 0xb9, 0x98, 0xa6, 0x78, 0xf9, 0x25, 0xac, 0x9d, 0xc1, 0x78, 0x1c, 0x75, + 0x38, 0x92, 0x2d, 0x58, 0x92, 0x12, 0x4b, 0x60, 0xa1, 0x7a, 0x7b, 0x9e, 0x26, 0xb8, 0x8a, 0x52, + 0x7e, 0x0b, 0x64, 0x57, 0x6a, 0x70, 0x2e, 0xf2, 0x93, 0x0b, 0x78, 0xd4, 0x4d, 0xd1, 0x7e, 0xdf, + 0xc1, 0xb5, 0x73, 0x7e, 0x75, 0xaa, 0xff, 0xef, 0xf8, 0x8b, 0x01, 0xe4, 0x8d, 0x14, 0xfc, 0x72, + 0x33, 0x26, 0xdb, 0x50, 0x50, 0x8d, 0x94, 0xcb, 0x25, 0x1b, 0x34, 0x6d, 0x02, 0x9e, 0xa6, 0xfb, + 0xf7, 0x82, 0xf1, 0xb6, 0xab, 0xe7, 0x25, 0xb5, 0xd3, 0x72, 0xcf, 0x25, 0x75, 0x69, 0xe5, 0xde, + 0x85, 0xab, 0x87, 0x3e, 0x57, 0x0d, 0xe7, 0xc3, 0x62, 0x2d, 0x58, 0x6e, 0xf9, 0x81, 0xc0, 0x84, + 0x5b, 0x46, 0x29, 0x57, 0x59, 0x75, 0x87, 0xc7, 0xf2, 0x11, 0x90, 0x51, 0xb8, 0x4e, 0xa3, 0x06, + 0xa6, 0x0a, 0x22, 0xe1, 0x8b, 0xe5, 0xa1, 0x99, 0xe5, 0x47, 0x40, 0xf6, 0x30, 0xc0, 0x31, 0xd9, + 0xa7, 0x5d, 0x0a, 0x04, 0xf2, 0xbc, 0xdf, 0x69, 0x48, 0x05, 0x57, 0x5c, 0x69, 0x57, 0xbf, 0xe6, + 0xc1, 0x54, 0x49, 0x91, 0x16, 0xe4, 0x0e, 0x50, 0x10, 0x3a, 0x23, 0x87, 0xb1, 0x65, 0xb0, 0x9d, + 0xb9, 0xf1, 0xba, 0xe8, 0x36, 0xe4, 0x53, 0x29, 0xc8, 0xac, 0x3b, 0x69, 0x42, 0x5e, 0x7b, 0x73, + 0x01, 0x86, 0x0e, 0x16, 0x81, 0xa9, 0xc6, 0x9d, 0xcc, 0x22, 0x4f, 0x6e, 0x9b, 0x5d, 0x5d, 0x84, + 0x72, 0x16, 0x50, 0x0d, 0xdc, 0xcc, 0x80, 0x93, 0xcb, 0x32, 0x33, 0xe0, 0xb4, 0x51, 0x7e, 0x05, + 0xa6, 0xea, 0xff, 0xcc, 0x80, 0x93, 0x63, 0x62, 0x5f, 0x9f, 0x58, 0xa3, 0xfd, 0xf4, 0x8d, 0xab, + 0x1d, 0x9d, 0x9c, 0x16, 0x33, 0xdf, 0x4f, 0x8b, 0x99, 0x4f, 0x83, 0xa2, 0x71, 0x32, 0x28, 0x1a, + 0xdf, 0x06, 0x45, 0xe3, 0xd7, 0xa0, 0x68, 0xbc, 0x7f, 0x7c, 0xc1, 0xf7, 0x78, 0x5b, 0x59, 0x47, + 0x99, 0xba, 0x29, 0x63, 0xdd, 0xff, 0x13, 0x00, 0x00, 0xff, 0xff, 0x24, 0x4e, 0xca, 0x64, 0xda, + 0x07, 0x00, 0x00, } diff --git a/vendor/github.com/containerd/containerd/api/services/images/v1/images.proto b/vendor/github.com/containerd/containerd/api/services/images/v1/images.proto index 9e6444c27..152ade2a0 100644 --- a/vendor/github.com/containerd/containerd/api/services/images/v1/images.proto +++ b/vendor/github.com/containerd/containerd/api/services/images/v1/images.proto @@ -115,4 +115,10 @@ message ListImagesResponse { message DeleteImageRequest { string name = 1; + + // Sync indicates that the delete and cleanup should be done + // synchronously before returning to the caller + // + // Default is false + bool sync = 2; } diff --git a/vendor/github.com/containerd/containerd/api/services/snapshot/v1/snapshots.pb.go b/vendor/github.com/containerd/containerd/api/services/snapshots/v1/snapshots.pb.go similarity index 94% rename from vendor/github.com/containerd/containerd/api/services/snapshot/v1/snapshots.pb.go rename to vendor/github.com/containerd/containerd/api/services/snapshots/v1/snapshots.pb.go index bfd4a0438..4cf68f0d8 100644 --- a/vendor/github.com/containerd/containerd/api/services/snapshot/v1/snapshots.pb.go +++ b/vendor/github.com/containerd/containerd/api/services/snapshots/v1/snapshots.pb.go @@ -1,11 +1,11 @@ // Code generated by protoc-gen-gogo. DO NOT EDIT. -// source: github.com/containerd/containerd/api/services/snapshot/v1/snapshots.proto +// source: github.com/containerd/containerd/api/services/snapshots/v1/snapshots.proto /* - Package snapshot is a generated protocol buffer package. + Package snapshots is a generated protocol buffer package. It is generated from these files: - github.com/containerd/containerd/api/services/snapshot/v1/snapshots.proto + github.com/containerd/containerd/api/services/snapshots/v1/snapshots.proto It has these top-level messages: PrepareSnapshotRequest @@ -26,7 +26,7 @@ UsageRequest UsageResponse */ -package snapshot +package snapshots import proto "github.com/gogo/protobuf/proto" import fmt "fmt" @@ -651,7 +651,7 @@ var _Snapshots_serviceDesc = grpc.ServiceDesc{ ServerStreams: true, }, }, - Metadata: "github.com/containerd/containerd/api/services/snapshot/v1/snapshots.proto", + Metadata: "github.com/containerd/containerd/api/services/snapshots/v1/snapshots.proto", } func (m *PrepareSnapshotRequest) Marshal() (dAtA []byte, err error) { @@ -4194,72 +4194,72 @@ var ( ) func init() { - proto.RegisterFile("github.com/containerd/containerd/api/services/snapshot/v1/snapshots.proto", fileDescriptorSnapshots) + proto.RegisterFile("github.com/containerd/containerd/api/services/snapshots/v1/snapshots.proto", fileDescriptorSnapshots) } var fileDescriptorSnapshots = []byte{ - // 1006 bytes of a gzipped FileDescriptorProto + // 1007 bytes of a gzipped FileDescriptorProto 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xb4, 0x57, 0x4f, 0x6f, 0x1a, 0x47, 0x14, 0x67, 0x60, 0x8d, 0xe3, 0x87, 0xed, 0xd2, 0x09, 0x26, 0x68, 0x5b, 0xe1, 0x15, 0x87, 0xca, - 0xea, 0x61, 0x37, 0xa1, 0x6a, 0xe2, 0xc4, 0x97, 0x02, 0xa1, 0x15, 0x71, 0xec, 0x54, 0x1b, 0xdb, - 0x89, 0xd3, 0x48, 0xd1, 0x1a, 0xc6, 0x78, 0x05, 0xbb, 0x4b, 0x99, 0x81, 0x88, 0x56, 0xaa, 0x7a, - 0x8c, 0x7c, 0xea, 0x17, 0xf0, 0xa9, 0xfd, 0x10, 0x55, 0x3f, 0x81, 0x8f, 0x3d, 0xf6, 0xd4, 0x36, - 0xfe, 0x12, 0x3d, 0xf5, 0x8f, 0x66, 0x76, 0x16, 0x30, 0xa6, 0x62, 0xc1, 0xe4, 0xf6, 0x76, 0x66, - 0xde, 0x7b, 0xbf, 0xf7, 0x7b, 0xf3, 0xde, 0x9b, 0x85, 0x4a, 0xdd, 0x66, 0x27, 0x9d, 0x23, 0xbd, - 0xea, 0x39, 0x46, 0xd5, 0x73, 0x99, 0x65, 0xbb, 0xa4, 0x5d, 0x1b, 0x16, 0xad, 0x96, 0x6d, 0x50, - 0xd2, 0xee, 0xda, 0x55, 0x42, 0x0d, 0xea, 0x5a, 0x2d, 0x7a, 0xe2, 0x31, 0xa3, 0x7b, 0xa7, 0x2f, - 0x53, 0xbd, 0xd5, 0xf6, 0x98, 0x87, 0xb5, 0x81, 0x92, 0x1e, 0x28, 0xe8, 0x83, 0x43, 0xdd, 0x3b, - 0x6a, 0xaa, 0xee, 0xd5, 0x3d, 0x71, 0xd8, 0xe0, 0x92, 0xaf, 0xa7, 0x7e, 0x50, 0xf7, 0xbc, 0x7a, - 0x93, 0x18, 0xe2, 0xeb, 0xa8, 0x73, 0x6c, 0x10, 0xa7, 0xc5, 0x7a, 0x72, 0x53, 0x1b, 0xdd, 0x3c, - 0xb6, 0x49, 0xb3, 0xf6, 0xca, 0xb1, 0x68, 0x43, 0x9e, 0x58, 0x1f, 0x3d, 0xc1, 0x6c, 0x87, 0x50, - 0x66, 0x39, 0x2d, 0x79, 0xe0, 0x6e, 0xa8, 0x10, 0x59, 0xaf, 0x45, 0xa8, 0xe1, 0x78, 0x1d, 0x97, - 0xf9, 0x7a, 0xb9, 0xbf, 0x11, 0xa4, 0xbf, 0x6c, 0x93, 0x96, 0xd5, 0x26, 0x4f, 0x65, 0x14, 0x26, - 0xf9, 0xba, 0x43, 0x28, 0xc3, 0x1a, 0x24, 0x82, 0xc0, 0x18, 0x69, 0x67, 0x90, 0x86, 0x36, 0x96, - 0xcc, 0xe1, 0x25, 0x9c, 0x84, 0x58, 0x83, 0xf4, 0x32, 0x51, 0xb1, 0xc3, 0x45, 0x9c, 0x86, 0x38, - 0x37, 0xe5, 0xb2, 0x4c, 0x4c, 0x2c, 0xca, 0x2f, 0xfc, 0x12, 0xe2, 0x4d, 0xeb, 0x88, 0x34, 0x69, - 0x46, 0xd1, 0x62, 0x1b, 0x89, 0xfc, 0x43, 0x7d, 0x12, 0x8f, 0xfa, 0x78, 0x54, 0xfa, 0x63, 0x61, - 0xa6, 0xec, 0xb2, 0x76, 0xcf, 0x94, 0x36, 0xd5, 0xfb, 0x90, 0x18, 0x5a, 0x0e, 0x60, 0xa1, 0x01, - 0xac, 0x14, 0x2c, 0x74, 0xad, 0x66, 0x87, 0x48, 0xa8, 0xfe, 0xc7, 0x83, 0xe8, 0x26, 0xca, 0x3d, - 0x82, 0x5b, 0x57, 0x1c, 0xd1, 0x96, 0xe7, 0x52, 0x82, 0x0d, 0x88, 0x0b, 0xa6, 0x68, 0x06, 0x09, - 0xcc, 0xb7, 0x86, 0x31, 0x0b, 0x26, 0xf5, 0x1d, 0xbe, 0x6f, 0xca, 0x63, 0xb9, 0xbf, 0x10, 0xdc, - 0x3c, 0xb0, 0xc9, 0xeb, 0x77, 0x49, 0xe4, 0xe1, 0x08, 0x91, 0x85, 0xc9, 0x44, 0x8e, 0x81, 0x34, - 0x6f, 0x16, 0xbf, 0x80, 0xd4, 0x65, 0x2f, 0xb3, 0x52, 0x58, 0x82, 0x15, 0xb1, 0x40, 0xaf, 0xc1, - 0x5d, 0xae, 0x00, 0xab, 0x81, 0x91, 0x59, 0x71, 0x6c, 0xc3, 0x9a, 0x49, 0x1c, 0xaf, 0x3b, 0x8f, - 0xa2, 0xe0, 0xf7, 0x62, 0xad, 0xe4, 0x39, 0x8e, 0xcd, 0xa6, 0xb7, 0x86, 0x41, 0x71, 0x2d, 0x27, - 0xa0, 0x5c, 0xc8, 0x81, 0x87, 0xd8, 0x20, 0x33, 0x5f, 0x8d, 0xdc, 0x8a, 0xd2, 0xe4, 0x5b, 0x31, - 0x16, 0xd0, 0xbc, 0xef, 0x45, 0x05, 0x6e, 0x3e, 0x65, 0x16, 0x9b, 0x07, 0x89, 0xff, 0x46, 0x41, - 0xa9, 0xb8, 0xc7, 0x5e, 0x9f, 0x11, 0x34, 0xc4, 0xc8, 0xa0, 0x5a, 0xa2, 0x97, 0xaa, 0xe5, 0x01, - 0x28, 0x0d, 0xdb, 0xad, 0x09, 0xaa, 0x56, 0xf3, 0x1f, 0x4d, 0x66, 0x65, 0xdb, 0x76, 0x6b, 0xa6, - 0xd0, 0xc1, 0x25, 0x80, 0x6a, 0x9b, 0x58, 0x8c, 0xd4, 0x5e, 0x59, 0x2c, 0xa3, 0x68, 0x68, 0x23, - 0x91, 0x57, 0x75, 0xbf, 0x0f, 0xeb, 0x41, 0x1f, 0xd6, 0xf7, 0x82, 0x3e, 0x5c, 0xbc, 0x71, 0xfe, - 0xfb, 0x7a, 0xe4, 0x87, 0x3f, 0xd6, 0x91, 0xb9, 0x24, 0xf5, 0x0a, 0x8c, 0x1b, 0xe9, 0xb4, 0x6a, - 0x81, 0x91, 0x85, 0x69, 0x8c, 0x48, 0xbd, 0x02, 0xc3, 0x8f, 0xfa, 0xd9, 0x8d, 0x8b, 0xec, 0xe6, - 0x27, 0xc7, 0xc1, 0x99, 0x9a, 0x77, 0x32, 0x9f, 0x43, 0xea, 0x72, 0x32, 0x65, 0x71, 0x7d, 0x06, - 0x8a, 0xed, 0x1e, 0x7b, 0xc2, 0x48, 0x22, 0x0c, 0xc9, 0x1c, 0x5c, 0x51, 0xe1, 0x91, 0x9a, 0x42, - 0x33, 0xf7, 0x33, 0x82, 0xb5, 0x7d, 0x11, 0xee, 0xf4, 0x37, 0x25, 0xf0, 0x1e, 0x9d, 0xd5, 0x3b, - 0xde, 0x82, 0x84, 0xcf, 0xb5, 0x18, 0xb8, 0xe2, 0xae, 0x8c, 0x4b, 0xd2, 0xe7, 0x7c, 0x26, 0xef, - 0x58, 0xb4, 0x61, 0xca, 0x94, 0x72, 0x39, 0xf7, 0x02, 0xd2, 0xa3, 0xc8, 0xe7, 0x46, 0xcb, 0x26, - 0xa4, 0x1e, 0xdb, 0xb4, 0x4f, 0x78, 0xf8, 0x9e, 0x98, 0x3b, 0x84, 0xb5, 0x11, 0xcd, 0x2b, 0xa0, - 0x62, 0x33, 0x82, 0x2a, 0xc2, 0xf2, 0x3e, 0xb5, 0xea, 0xe4, 0x3a, 0xb5, 0xbc, 0x05, 0x2b, 0xd2, - 0x86, 0x84, 0x85, 0x41, 0xa1, 0xf6, 0x37, 0x7e, 0x4d, 0xc7, 0x4c, 0x21, 0xf3, 0x9a, 0xb6, 0x5d, - 0xaf, 0x46, 0xa8, 0xd0, 0x8c, 0x99, 0xf2, 0xeb, 0xe3, 0x37, 0x08, 0x14, 0x5e, 0xa6, 0xf8, 0x43, - 0x58, 0xdc, 0xdf, 0xdd, 0xde, 0x7d, 0xf2, 0x6c, 0x37, 0x19, 0x51, 0xdf, 0x3b, 0x3d, 0xd3, 0x12, - 0x7c, 0x79, 0xdf, 0x6d, 0xb8, 0xde, 0x6b, 0x17, 0xa7, 0x41, 0x39, 0xa8, 0x94, 0x9f, 0x25, 0x91, - 0xba, 0x7c, 0x7a, 0xa6, 0xdd, 0xe0, 0x5b, 0x7c, 0x44, 0x61, 0x15, 0xe2, 0x85, 0xd2, 0x5e, 0xe5, - 0xa0, 0x9c, 0x8c, 0xaa, 0xab, 0xa7, 0x67, 0x1a, 0xf0, 0x9d, 0x42, 0x95, 0xd9, 0x5d, 0x82, 0x35, - 0x58, 0x2a, 0x3d, 0xd9, 0xd9, 0xa9, 0xec, 0xed, 0x95, 0x1f, 0x26, 0x63, 0xea, 0xfb, 0xa7, 0x67, - 0xda, 0x0a, 0xdf, 0xf6, 0x7b, 0x25, 0x23, 0x35, 0x75, 0xf9, 0xcd, 0x8f, 0xd9, 0xc8, 0x2f, 0x3f, - 0x65, 0x05, 0x82, 0xfc, 0x3f, 0x8b, 0xb0, 0xd4, 0xe7, 0x18, 0x7f, 0x07, 0x8b, 0xf2, 0x29, 0x81, - 0x37, 0x67, 0x7d, 0xde, 0xa8, 0xf7, 0x67, 0xd0, 0x94, 0x24, 0x76, 0x40, 0x11, 0x11, 0x7e, 0x3a, - 0xd3, 0x93, 0x40, 0xbd, 0x3b, 0xad, 0x9a, 0x74, 0xdb, 0x80, 0xb8, 0x3f, 0x6d, 0xb1, 0x31, 0xd9, - 0xc2, 0xa5, 0xe1, 0xae, 0xde, 0x0e, 0xaf, 0x20, 0x9d, 0x1d, 0x42, 0xdc, 0x4f, 0x06, 0xbe, 0x37, - 0xe3, 0x88, 0x53, 0xd3, 0x57, 0x2a, 0xbb, 0xcc, 0x9f, 0xe2, 0xdc, 0xb4, 0x3f, 0xf2, 0xc3, 0x98, - 0x1e, 0xfb, 0x38, 0xf8, 0x5f, 0xd3, 0x1d, 0x50, 0x78, 0xe7, 0x0c, 0x93, 0x99, 0x31, 0xe3, 0x32, - 0x4c, 0x66, 0xc6, 0x36, 0xe6, 0x6f, 0x21, 0xee, 0xf7, 0xa6, 0x30, 0x11, 0x8d, 0xed, 0xbf, 0xea, - 0xe6, 0xf4, 0x8a, 0xd2, 0x79, 0x0f, 0x14, 0xde, 0x82, 0x70, 0x08, 0xf0, 0xe3, 0x9a, 0x9c, 0x7a, - 0x6f, 0x6a, 0x3d, 0xdf, 0xf1, 0x6d, 0x84, 0x4f, 0x60, 0x41, 0xb4, 0x17, 0xac, 0x87, 0x40, 0x3f, - 0xd4, 0xcb, 0x54, 0x23, 0xf4, 0x79, 0xdf, 0x57, 0xf1, 0xe5, 0xf9, 0xdb, 0x6c, 0xe4, 0xb7, 0xb7, - 0xd9, 0xc8, 0xf7, 0x17, 0x59, 0x74, 0x7e, 0x91, 0x45, 0xbf, 0x5e, 0x64, 0xd1, 0x9f, 0x17, 0x59, - 0xf4, 0xa2, 0x38, 0xf3, 0x2f, 0xe7, 0x56, 0x20, 0x3f, 0x8f, 0x1c, 0xc5, 0xc5, 0x45, 0xfa, 0xe4, - 0xbf, 0x00, 0x00, 0x00, 0xff, 0xff, 0xed, 0x58, 0xc4, 0xcf, 0xc1, 0x0e, 0x00, 0x00, + 0xea, 0x61, 0x37, 0xa1, 0x6a, 0xe2, 0xc4, 0x97, 0x62, 0x4c, 0x2b, 0xec, 0xd8, 0xa9, 0x36, 0xb6, + 0x13, 0xa7, 0x55, 0xa3, 0x35, 0x8c, 0xf1, 0x0a, 0x76, 0x97, 0x32, 0x03, 0x11, 0xad, 0x54, 0xf5, + 0x18, 0xf9, 0xd4, 0x2f, 0xe0, 0x53, 0xfb, 0x21, 0xaa, 0x7e, 0x02, 0x1f, 0x7b, 0xec, 0xa9, 0x6d, + 0xfc, 0x25, 0x7a, 0xea, 0x1f, 0xcd, 0xec, 0x2c, 0x60, 0x4c, 0xc5, 0x82, 0xc9, 0x6d, 0x66, 0x67, + 0x7e, 0xef, 0xfd, 0xe6, 0xf7, 0xe6, 0xbd, 0x37, 0x0b, 0xdb, 0x35, 0x9b, 0x9d, 0xb6, 0x8f, 0xf5, + 0x8a, 0xe7, 0x18, 0x15, 0xcf, 0x65, 0x96, 0xed, 0x92, 0x56, 0x75, 0x70, 0x68, 0x35, 0x6d, 0x83, + 0x92, 0x56, 0xc7, 0xae, 0x10, 0x6a, 0x50, 0xd7, 0x6a, 0xd2, 0x53, 0x8f, 0x51, 0xa3, 0x73, 0xaf, + 0x3f, 0xd1, 0x9b, 0x2d, 0x8f, 0x79, 0x58, 0xeb, 0xa3, 0xf4, 0x00, 0xa1, 0xf7, 0x37, 0x75, 0xee, + 0xa9, 0xa9, 0x9a, 0x57, 0xf3, 0xc4, 0x66, 0x83, 0x8f, 0x7c, 0x9c, 0xfa, 0x5e, 0xcd, 0xf3, 0x6a, + 0x0d, 0x62, 0x88, 0xd9, 0x71, 0xfb, 0xc4, 0x20, 0x4e, 0x93, 0x75, 0xe5, 0xa2, 0x36, 0xbc, 0x78, + 0x62, 0x93, 0x46, 0xf5, 0xa5, 0x63, 0xd1, 0xba, 0xdc, 0xb1, 0x3a, 0xbc, 0x83, 0xd9, 0x0e, 0xa1, + 0xcc, 0x72, 0x9a, 0x72, 0xc3, 0xfd, 0x50, 0x67, 0x64, 0xdd, 0x26, 0xa1, 0x86, 0xe3, 0xb5, 0x5d, + 0xe6, 0xe3, 0x72, 0x7f, 0x23, 0x48, 0x7f, 0xde, 0x22, 0x4d, 0xab, 0x45, 0x9e, 0xca, 0x53, 0x98, + 0xe4, 0xeb, 0x36, 0xa1, 0x0c, 0x6b, 0x90, 0x08, 0x0e, 0xc6, 0x48, 0x2b, 0x83, 0x34, 0xb4, 0xb6, + 0x60, 0x0e, 0x7e, 0xc2, 0x49, 0x88, 0xd5, 0x49, 0x37, 0x13, 0x15, 0x2b, 0x7c, 0x88, 0xd3, 0x10, + 0xe7, 0xa6, 0x5c, 0x96, 0x89, 0x89, 0x8f, 0x72, 0x86, 0xbf, 0x84, 0x78, 0xc3, 0x3a, 0x26, 0x0d, + 0x9a, 0x51, 0xb4, 0xd8, 0x5a, 0x22, 0xbf, 0xa5, 0x8f, 0xd3, 0x51, 0x1f, 0xcd, 0x4a, 0x7f, 0x2c, + 0xcc, 0x94, 0x5c, 0xd6, 0xea, 0x9a, 0xd2, 0xa6, 0xfa, 0x10, 0x12, 0x03, 0x9f, 0x03, 0x5a, 0xa8, + 0x4f, 0x2b, 0x05, 0x73, 0x1d, 0xab, 0xd1, 0x26, 0x92, 0xaa, 0x3f, 0x79, 0x14, 0x5d, 0x47, 0xb9, + 0x6d, 0xb8, 0x73, 0xcd, 0x11, 0x6d, 0x7a, 0x2e, 0x25, 0xd8, 0x80, 0xb8, 0x50, 0x8a, 0x66, 0x90, + 0xe0, 0x7c, 0x67, 0x90, 0xb3, 0x50, 0x52, 0xdf, 0xe5, 0xeb, 0xa6, 0xdc, 0x96, 0xfb, 0x0b, 0xc1, + 0xed, 0x43, 0x9b, 0xbc, 0x7a, 0x9b, 0x42, 0x1e, 0x0d, 0x09, 0x59, 0x18, 0x2f, 0xe4, 0x08, 0x4a, + 0xb3, 0x56, 0xf1, 0x33, 0x48, 0x5d, 0xf5, 0x32, 0xad, 0x84, 0x45, 0x58, 0x12, 0x1f, 0xe8, 0x0d, + 0xb4, 0xcb, 0x15, 0x60, 0x39, 0x30, 0x32, 0x2d, 0x8f, 0x1d, 0x58, 0x31, 0x89, 0xe3, 0x75, 0x66, + 0x91, 0x14, 0xfc, 0x5e, 0xac, 0x14, 0x3d, 0xc7, 0xb1, 0xd9, 0xe4, 0xd6, 0x30, 0x28, 0xae, 0xe5, + 0x04, 0x92, 0x8b, 0x71, 0xe0, 0x21, 0xd6, 0x8f, 0xcc, 0x17, 0x43, 0xb7, 0xa2, 0x38, 0xfe, 0x56, + 0x8c, 0x24, 0x34, 0xeb, 0x7b, 0x51, 0x86, 0xdb, 0x4f, 0x99, 0xc5, 0x66, 0x21, 0xe2, 0xbf, 0x51, + 0x50, 0xca, 0xee, 0x89, 0xd7, 0x53, 0x04, 0x0d, 0x28, 0xd2, 0xcf, 0x96, 0xe8, 0x95, 0x6c, 0x79, + 0x04, 0x4a, 0xdd, 0x76, 0xab, 0x42, 0xaa, 0xe5, 0xfc, 0x07, 0xe3, 0x55, 0xd9, 0xb1, 0xdd, 0xaa, + 0x29, 0x30, 0xb8, 0x08, 0x50, 0x69, 0x11, 0x8b, 0x91, 0xea, 0x4b, 0x8b, 0x65, 0x14, 0x0d, 0xad, + 0x25, 0xf2, 0xaa, 0xee, 0xd7, 0x61, 0x3d, 0xa8, 0xc3, 0xfa, 0x7e, 0x50, 0x87, 0x37, 0x6f, 0x5d, + 0xfc, 0xbe, 0x1a, 0xf9, 0xe1, 0x8f, 0x55, 0x64, 0x2e, 0x48, 0x5c, 0x81, 0x71, 0x23, 0xed, 0x66, + 0x35, 0x30, 0x32, 0x37, 0x89, 0x11, 0x89, 0x2b, 0x30, 0xbc, 0xdd, 0x8b, 0x6e, 0x5c, 0x44, 0x37, + 0x3f, 0xfe, 0x1c, 0x5c, 0xa9, 0x59, 0x07, 0xf3, 0x39, 0xa4, 0xae, 0x06, 0x53, 0x26, 0xd7, 0x27, + 0xa0, 0xd8, 0xee, 0x89, 0x27, 0x8c, 0x24, 0xc2, 0x88, 0xcc, 0xc9, 0x6d, 0x2a, 0xfc, 0xa4, 0xa6, + 0x40, 0xe6, 0x7e, 0x46, 0xb0, 0x72, 0x20, 0x8e, 0x3b, 0xf9, 0x4d, 0x09, 0xbc, 0x47, 0xa7, 0xf5, + 0x8e, 0x37, 0x20, 0xe1, 0x6b, 0x2d, 0x1a, 0xae, 0xb8, 0x2b, 0xa3, 0x82, 0xf4, 0x29, 0xef, 0xc9, + 0xbb, 0x16, 0xad, 0x9b, 0x32, 0xa4, 0x7c, 0x9c, 0x7b, 0x01, 0xe9, 0x61, 0xe6, 0x33, 0x93, 0x65, + 0x1d, 0x52, 0x8f, 0x6d, 0xda, 0x13, 0x3c, 0x7c, 0x4d, 0xcc, 0x1d, 0xc1, 0xca, 0x10, 0xf2, 0x1a, + 0xa9, 0xd8, 0x94, 0xa4, 0x36, 0x61, 0xf1, 0x80, 0x5a, 0x35, 0x72, 0x93, 0x5c, 0xde, 0x80, 0x25, + 0x69, 0x43, 0xd2, 0xc2, 0xa0, 0x50, 0xfb, 0x1b, 0x3f, 0xa7, 0x63, 0xa6, 0x18, 0xf3, 0x9c, 0xb6, + 0x5d, 0xaf, 0x4a, 0xa8, 0x40, 0xc6, 0x4c, 0x39, 0xfb, 0xf0, 0x35, 0x02, 0x85, 0xa7, 0x29, 0x7e, + 0x1f, 0xe6, 0x0f, 0xf6, 0x76, 0xf6, 0x9e, 0x3c, 0xdb, 0x4b, 0x46, 0xd4, 0x77, 0xce, 0xce, 0xb5, + 0x04, 0xff, 0x7c, 0xe0, 0xd6, 0x5d, 0xef, 0x95, 0x8b, 0xd3, 0xa0, 0x1c, 0x96, 0x4b, 0xcf, 0x92, + 0x48, 0x5d, 0x3c, 0x3b, 0xd7, 0x6e, 0xf1, 0x25, 0xde, 0xa2, 0xb0, 0x0a, 0xf1, 0x42, 0x71, 0xbf, + 0x7c, 0x58, 0x4a, 0x46, 0xd5, 0xe5, 0xb3, 0x73, 0x0d, 0xf8, 0x4a, 0xa1, 0xc2, 0xec, 0x0e, 0xc1, + 0x1a, 0x2c, 0x14, 0x9f, 0xec, 0xee, 0x96, 0xf7, 0xf7, 0x4b, 0x5b, 0xc9, 0x98, 0xfa, 0xee, 0xd9, + 0xb9, 0xb6, 0xc4, 0x97, 0xfd, 0x5a, 0xc9, 0x48, 0x55, 0x5d, 0x7c, 0xfd, 0x63, 0x36, 0xf2, 0xcb, + 0x4f, 0x59, 0xc1, 0x20, 0xff, 0xcf, 0x3c, 0x2c, 0xf4, 0x34, 0xc6, 0xdf, 0xc1, 0xbc, 0x7c, 0x4a, + 0xe0, 0xf5, 0x69, 0x9f, 0x37, 0xea, 0xc3, 0x29, 0x90, 0x52, 0xc4, 0x36, 0x28, 0xe2, 0x84, 0x1f, + 0x4f, 0xf5, 0x24, 0x50, 0xef, 0x4f, 0x0a, 0x93, 0x6e, 0xeb, 0x10, 0xf7, 0xbb, 0x2d, 0x36, 0xc6, + 0x5b, 0xb8, 0xd2, 0xdc, 0xd5, 0xbb, 0xe1, 0x01, 0xd2, 0xd9, 0x11, 0xc4, 0xfd, 0x60, 0xe0, 0x07, + 0x53, 0xb6, 0x38, 0x35, 0x7d, 0x2d, 0xb3, 0x4b, 0xfc, 0x29, 0xce, 0x4d, 0xfb, 0x2d, 0x3f, 0x8c, + 0xe9, 0x91, 0x8f, 0x83, 0xff, 0x35, 0xdd, 0x06, 0x85, 0x57, 0xce, 0x30, 0x91, 0x19, 0xd1, 0x2e, + 0xc3, 0x44, 0x66, 0x64, 0x61, 0xfe, 0x16, 0xe2, 0x7e, 0x6d, 0x0a, 0x73, 0xa2, 0x91, 0xf5, 0x57, + 0x5d, 0x9f, 0x1c, 0x28, 0x9d, 0x77, 0x41, 0xe1, 0x25, 0x08, 0x87, 0x20, 0x3f, 0xaa, 0xc8, 0xa9, + 0x0f, 0x26, 0xc6, 0xf9, 0x8e, 0xef, 0x22, 0x7c, 0x0a, 0x73, 0xa2, 0xbc, 0x60, 0x3d, 0x04, 0xfb, + 0x81, 0x5a, 0xa6, 0x1a, 0xa1, 0xf7, 0xfb, 0xbe, 0x36, 0xbf, 0xba, 0x78, 0x93, 0x8d, 0xfc, 0xf6, + 0x26, 0x1b, 0xf9, 0xfe, 0x32, 0x8b, 0x2e, 0x2e, 0xb3, 0xe8, 0xd7, 0xcb, 0x2c, 0xfa, 0xf3, 0x32, + 0x8b, 0x5e, 0x6c, 0x4d, 0xff, 0xcf, 0xb9, 0xd1, 0x9b, 0x3c, 0x8f, 0x1c, 0xc7, 0xc5, 0x55, 0xfa, + 0xe8, 0xbf, 0x00, 0x00, 0x00, 0xff, 0xff, 0x8e, 0xa0, 0xb2, 0xda, 0xc4, 0x0e, 0x00, 0x00, } diff --git a/vendor/github.com/containerd/containerd/api/services/snapshot/v1/snapshots.proto b/vendor/github.com/containerd/containerd/api/services/snapshots/v1/snapshots.proto similarity index 99% rename from vendor/github.com/containerd/containerd/api/services/snapshot/v1/snapshots.proto rename to vendor/github.com/containerd/containerd/api/services/snapshots/v1/snapshots.proto index 8caafc904..0e62add3b 100644 --- a/vendor/github.com/containerd/containerd/api/services/snapshot/v1/snapshots.proto +++ b/vendor/github.com/containerd/containerd/api/services/snapshots/v1/snapshots.proto @@ -8,7 +8,7 @@ import "google/protobuf/field_mask.proto"; import "google/protobuf/timestamp.proto"; import "github.com/containerd/containerd/api/types/mount.proto"; -option go_package = "github.com/containerd/containerd/api/services/snapshot/v1;snapshot"; +option go_package = "github.com/containerd/containerd/api/services/snapshots/v1;snapshots"; // Snapshot service manages snapshots service Snapshots { diff --git a/vendor/github.com/containerd/containerd/archive/tar.go b/vendor/github.com/containerd/containerd/archive/tar.go index 0550f218c..c12d4f280 100644 --- a/vendor/github.com/containerd/containerd/archive/tar.go +++ b/vendor/github.com/containerd/containerd/archive/tar.go @@ -19,13 +19,12 @@ import ( "github.com/pkg/errors" ) -var ( - bufferPool = &sync.Pool{ - New: func() interface{} { - return make([]byte, 32*1024) - }, - } -) +var bufferPool = &sync.Pool{ + New: func() interface{} { + buffer := make([]byte, 32*1024) + return &buffer + }, +} // Diff returns a tar stream of the computed filesystem // difference between the provided directories. @@ -82,6 +81,8 @@ const ( // whiteoutOpaqueDir file means directory has been made opaque - meaning // readdir calls to this directory do not follow to lower layers. whiteoutOpaqueDir = whiteoutMetaPrefix + ".opq" + + paxSchilyXattr = "SCHILY.xattrs." ) // Apply applies a tar stream of an OCI style diff tar. @@ -388,9 +389,10 @@ func (cw *changeWriter) HandleChange(k fs.ChangeKind, p string, f os.FileInfo, e if capability, err := getxattr(source, "security.capability"); err != nil { return errors.Wrap(err, "failed to get capabilities xattr") } else if capability != nil { - hdr.Xattrs = map[string]string{ - "security.capability": string(capability), + if hdr.PAXRecords == nil { + hdr.PAXRecords = map[string]string{} } + hdr.PAXRecords[paxSchilyXattr+"security.capability"] = string(capability) } if err := cw.tw.WriteHeader(hdr); err != nil { @@ -404,8 +406,8 @@ func (cw *changeWriter) HandleChange(k fs.ChangeKind, p string, f os.FileInfo, e } defer file.Close() - buf := bufferPool.Get().([]byte) - n, err := io.CopyBuffer(cw.tw, file, buf) + buf := bufferPool.Get().(*[]byte) + n, err := io.CopyBuffer(cw.tw, file, *buf) bufferPool.Put(buf) if err != nil { return errors.Wrap(err, "failed to copy") @@ -509,13 +511,16 @@ func createTarFile(ctx context.Context, path, extractDir string, hdr *tar.Header } } - for key, value := range hdr.Xattrs { - if err := setxattr(path, key, value); err != nil { - if errors.Cause(err) == syscall.ENOTSUP { - log.G(ctx).WithError(err).Warnf("ignored xattr %s in archive", key) - continue + for key, value := range hdr.PAXRecords { + if strings.HasPrefix(key, paxSchilyXattr) { + key = key[len(paxSchilyXattr):] + if err := setxattr(path, key, value); err != nil { + if errors.Cause(err) == syscall.ENOTSUP { + log.G(ctx).WithError(err).Warnf("ignored xattr %s in archive", key) + continue + } + return err } - return err } } @@ -529,7 +534,7 @@ func createTarFile(ctx context.Context, path, extractDir string, hdr *tar.Header } func copyBuffered(ctx context.Context, dst io.Writer, src io.Reader) (written int64, err error) { - buf := bufferPool.Get().([]byte) + buf := bufferPool.Get().(*[]byte) defer bufferPool.Put(buf) for { @@ -540,9 +545,9 @@ func copyBuffered(ctx context.Context, dst io.Writer, src io.Reader) (written in default: } - nr, er := src.Read(buf) + nr, er := src.Read(*buf) if nr > 0 { - nw, ew := dst.Write(buf[0:nr]) + nw, ew := dst.Write((*buf)[0:nr]) if nw > 0 { written += int64(nw) } diff --git a/vendor/github.com/containerd/containerd/client.go b/vendor/github.com/containerd/containerd/client.go index 6c5814425..39547f589 100644 --- a/vendor/github.com/containerd/containerd/client.go +++ b/vendor/github.com/containerd/containerd/client.go @@ -7,6 +7,7 @@ import ( "net/http" "runtime" "strconv" + "strings" "sync" "time" @@ -17,7 +18,7 @@ import ( imagesapi "github.com/containerd/containerd/api/services/images/v1" introspectionapi "github.com/containerd/containerd/api/services/introspection/v1" namespacesapi "github.com/containerd/containerd/api/services/namespaces/v1" - snapshotapi "github.com/containerd/containerd/api/services/snapshot/v1" + snapshotsapi "github.com/containerd/containerd/api/services/snapshots/v1" "github.com/containerd/containerd/api/services/tasks/v1" versionservice "github.com/containerd/containerd/api/services/version/v1" "github.com/containerd/containerd/containers" @@ -29,11 +30,10 @@ import ( "github.com/containerd/containerd/namespaces" "github.com/containerd/containerd/platforms" "github.com/containerd/containerd/plugin" - "github.com/containerd/containerd/reference" "github.com/containerd/containerd/remotes" "github.com/containerd/containerd/remotes/docker" "github.com/containerd/containerd/remotes/docker/schema1" - "github.com/containerd/containerd/snapshot" + "github.com/containerd/containerd/snapshots" "github.com/containerd/typeurl" ptypes "github.com/gogo/protobuf/types" ocispec "github.com/opencontainers/image-spec/specs-go/v1" @@ -133,7 +133,7 @@ func (c *Client) Containers(ctx context.Context, filters ...string) ([]Container // NewContainer will create a new container in container with the provided id // the id must be unique within the namespace func (c *Client) NewContainer(ctx context.Context, id string, opts ...NewContainerOpts) (Container, error) { - ctx, done, err := c.withLease(ctx) + ctx, done, err := c.WithLease(ctx) if err != nil { return nil, err } @@ -214,7 +214,7 @@ func (c *Client) Pull(ctx context.Context, ref string, opts ...RemoteOpt) (Image } store := c.ContentStore() - ctx, done, err := c.withLease(ctx) + ctx, done, err := c.WithLease(ctx) if err != nil { return nil, err } @@ -334,6 +334,14 @@ func (c *Client) Push(ctx context.Context, ref string, desc ocispec.Descriptor, for i := len(manifestStack) - 1; i >= 0; i-- { _, err := pushHandler(ctx, manifestStack[i]) if err != nil { + // TODO(estesp): until we have a more complete method for index push, we need to report + // missing dependencies in an index/manifest list by sensing the "400 Bad Request" + // as a marker for this problem + if (manifestStack[i].MediaType == ocispec.MediaTypeImageIndex || + manifestStack[i].MediaType == images.MediaTypeDockerSchema2ManifestList) && + errors.Cause(err) != nil && strings.Contains(errors.Cause(err).Error(), "400 Bad Request") { + return errors.Wrap(err, "manifest list/index references to blobs and/or manifests are missing in your target registry") + } return err } } @@ -435,8 +443,8 @@ func (c *Client) ContentStore() content.Store { } // SnapshotService returns the underlying snapshotter for the provided snapshotter name -func (c *Client) SnapshotService(snapshotterName string) snapshot.Snapshotter { - return NewSnapshotterFromClient(snapshotapi.NewSnapshotsClient(c.conn), snapshotterName) +func (c *Client) SnapshotService(snapshotterName string) snapshots.Snapshotter { + return NewSnapshotterFromClient(snapshotsapi.NewSnapshotsClient(c.conn), snapshotterName) } // TaskService returns the underlying TasksClient @@ -494,157 +502,97 @@ func (c *Client) Version(ctx context.Context) (Version, error) { }, nil } -type imageFormat string - -const ( - ociImageFormat imageFormat = "oci" -) - type importOpts struct { - format imageFormat - refObject string - labels map[string]string } // ImportOpt allows the caller to specify import specific options type ImportOpt func(c *importOpts) error -// WithImportLabel sets a label to be associated with an imported image -func WithImportLabel(key, value string) ImportOpt { - return func(opts *importOpts) error { - if opts.labels == nil { - opts.labels = make(map[string]string) - } - - opts.labels[key] = value - return nil - } -} - -// WithImportLabels associates a set of labels to an imported image -func WithImportLabels(labels map[string]string) ImportOpt { - return func(opts *importOpts) error { - if opts.labels == nil { - opts.labels = make(map[string]string) - } - - for k, v := range labels { - opts.labels[k] = v - } - return nil - } -} - -// WithOCIImportFormat sets the import format for an OCI image format -func WithOCIImportFormat() ImportOpt { - return func(c *importOpts) error { - if c.format != "" { - return errors.New("format already set") - } - c.format = ociImageFormat - return nil - } -} - -// WithRefObject specifies the ref object to import. -// If refObject is empty, it is copied from the ref argument of Import(). -func WithRefObject(refObject string) ImportOpt { - return func(c *importOpts) error { - c.refObject = refObject - return nil - } -} - -func resolveImportOpt(ref string, opts ...ImportOpt) (importOpts, error) { +func resolveImportOpt(opts ...ImportOpt) (importOpts, error) { var iopts importOpts for _, o := range opts { if err := o(&iopts); err != nil { return iopts, err } } - // use OCI as the default format - if iopts.format == "" { - iopts.format = ociImageFormat - } - // if refObject is not explicitly specified, use the one specified in ref - if iopts.refObject == "" { - refSpec, err := reference.Parse(ref) - if err != nil { - return iopts, err - } - iopts.refObject = refSpec.Object - } return iopts, nil } // Import imports an image from a Tar stream using reader. -// OCI format is assumed by default. -// -// Note that unreferenced blobs are imported to the content store as well. -func (c *Client) Import(ctx context.Context, ref string, reader io.Reader, opts ...ImportOpt) (Image, error) { - iopts, err := resolveImportOpt(ref, opts...) +// Caller needs to specify importer. Future version may use oci.v1 as the default. +// Note that unreferrenced blobs may be imported to the content store as well. +func (c *Client) Import(ctx context.Context, importer images.Importer, reader io.Reader, opts ...ImportOpt) ([]Image, error) { + _, err := resolveImportOpt(opts...) // unused now if err != nil { return nil, err } - ctx, done, err := c.withLease(ctx) + ctx, done, err := c.WithLease(ctx) if err != nil { return nil, err } defer done() - switch iopts.format { - case ociImageFormat: - return c.importFromOCITar(ctx, ref, reader, iopts) - default: - return nil, errors.Errorf("unsupported format: %s", iopts.format) + imgrecs, err := importer.Import(ctx, c.ContentStore(), reader) + if err != nil { + // is.Update() is not called on error + return nil, err } + + is := c.ImageService() + var images []Image + for _, imgrec := range imgrecs { + if updated, err := is.Update(ctx, imgrec, "target"); err != nil { + if !errdefs.IsNotFound(err) { + return nil, err + } + + created, err := is.Create(ctx, imgrec) + if err != nil { + return nil, err + } + + imgrec = created + } else { + imgrec = updated + } + + images = append(images, &image{ + client: c, + i: imgrec, + }) + } + return images, nil } type exportOpts struct { - format imageFormat } -// ExportOpt allows callers to set export options +// ExportOpt allows the caller to specify export-specific options type ExportOpt func(c *exportOpts) error -// WithOCIExportFormat sets the OCI image format as the export target -func WithOCIExportFormat() ExportOpt { - return func(c *exportOpts) error { - if c.format != "" { - return errors.New("format already set") +func resolveExportOpt(opts ...ExportOpt) (exportOpts, error) { + var eopts exportOpts + for _, o := range opts { + if err := o(&eopts); err != nil { + return eopts, err } - c.format = ociImageFormat - return nil } + return eopts, nil } -// TODO: add WithMediaTypeTranslation that transforms media types according to the format. -// e.g. application/vnd.docker.image.rootfs.diff.tar.gzip -// -> application/vnd.oci.image.layer.v1.tar+gzip - // Export exports an image to a Tar stream. // OCI format is used by default. // It is up to caller to put "org.opencontainers.image.ref.name" annotation to desc. -func (c *Client) Export(ctx context.Context, desc ocispec.Descriptor, opts ...ExportOpt) (io.ReadCloser, error) { - var eopts exportOpts - for _, o := range opts { - if err := o(&eopts); err != nil { - return nil, err - } - } - // use OCI as the default format - if eopts.format == "" { - eopts.format = ociImageFormat +// TODO(AkihiroSuda): support exporting multiple descriptors at once to a single archive stream. +func (c *Client) Export(ctx context.Context, exporter images.Exporter, desc ocispec.Descriptor, opts ...ExportOpt) (io.ReadCloser, error) { + _, err := resolveExportOpt(opts...) // unused now + if err != nil { + return nil, err } pr, pw := io.Pipe() - switch eopts.format { - case ociImageFormat: - go func() { - pw.CloseWithError(c.exportToOCITar(ctx, desc, pw, eopts)) - }() - default: - return nil, errors.Errorf("unsupported format: %s", eopts.format) - } + go func() { + pw.CloseWithError(exporter.Export(ctx, c.ContentStore(), desc, pw)) + }() return pr, nil } diff --git a/vendor/github.com/containerd/containerd/container.go b/vendor/github.com/containerd/containerd/container.go index 2d5c9aedb..716e0456c 100644 --- a/vendor/github.com/containerd/containerd/container.go +++ b/vendor/github.com/containerd/containerd/container.go @@ -162,11 +162,17 @@ func (c *container) Image(ctx context.Context) (Image, error) { }, nil } -func (c *container) NewTask(ctx context.Context, ioCreate cio.Creation, opts ...NewTaskOpts) (Task, error) { +func (c *container) NewTask(ctx context.Context, ioCreate cio.Creation, opts ...NewTaskOpts) (_ Task, err error) { i, err := ioCreate(c.id) if err != nil { return nil, err } + defer func() { + if err != nil && i != nil { + i.Cancel() + i.Close() + } + }() cfg := i.Config() request := &tasks.CreateTaskRequest{ ContainerID: c.id, diff --git a/vendor/github.com/containerd/containerd/container_opts.go b/vendor/github.com/containerd/containerd/container_opts.go index 4c534fe12..fb22a9096 100644 --- a/vendor/github.com/containerd/containerd/container_opts.go +++ b/vendor/github.com/containerd/containerd/container_opts.go @@ -5,10 +5,12 @@ import ( "github.com/containerd/containerd/containers" "github.com/containerd/containerd/errdefs" + "github.com/containerd/containerd/oci" "github.com/containerd/containerd/platforms" "github.com/containerd/typeurl" "github.com/gogo/protobuf/types" "github.com/opencontainers/image-spec/identity" + specs "github.com/opencontainers/runtime-spec/specs-go" "github.com/pkg/errors" ) @@ -164,3 +166,29 @@ func WithContainerExtension(name string, extension interface{}) NewContainerOpts return nil } } + +// WithNewSpec generates a new spec for a new container +func WithNewSpec(opts ...oci.SpecOpts) NewContainerOpts { + return func(ctx context.Context, client *Client, c *containers.Container) error { + s, err := oci.GenerateSpec(ctx, client, c, opts...) + if err != nil { + return err + } + c.Spec, err = typeurl.MarshalAny(s) + return err + } +} + +// WithSpec sets the provided spec on the container +func WithSpec(s *specs.Spec, opts ...oci.SpecOpts) NewContainerOpts { + return func(ctx context.Context, client *Client, c *containers.Container) error { + for _, o := range opts { + if err := o(ctx, client, c, s); err != nil { + return err + } + } + var err error + c.Spec, err = typeurl.MarshalAny(s) + return err + } +} diff --git a/vendor/github.com/containerd/containerd/container_opts_unix.go b/vendor/github.com/containerd/containerd/container_opts_unix.go index e2cd7c939..b678033b7 100644 --- a/vendor/github.com/containerd/containerd/container_opts_unix.go +++ b/vendor/github.com/containerd/containerd/container_opts_unix.go @@ -6,12 +6,17 @@ import ( "context" "encoding/json" "fmt" + "io/ioutil" + "os" + "path/filepath" + "syscall" "github.com/containerd/containerd/api/types" "github.com/containerd/containerd/containers" "github.com/containerd/containerd/content" "github.com/containerd/containerd/errdefs" "github.com/containerd/containerd/images" + "github.com/containerd/containerd/mount" "github.com/containerd/containerd/platforms" "github.com/gogo/protobuf/proto" protobuf "github.com/gogo/protobuf/types" @@ -122,3 +127,94 @@ func decodeIndex(ctx context.Context, store content.Store, id digest.Digest) (*v return &index, nil } + +// WithRemappedSnapshot creates a new snapshot and remaps the uid/gid for the +// filesystem to be used by a container with user namespaces +func WithRemappedSnapshot(id string, i Image, uid, gid uint32) NewContainerOpts { + return withRemappedSnapshotBase(id, i, uid, gid, false) +} + +// WithRemappedSnapshotView is similar to WithRemappedSnapshot but rootfs is mounted as read-only. +func WithRemappedSnapshotView(id string, i Image, uid, gid uint32) NewContainerOpts { + return withRemappedSnapshotBase(id, i, uid, gid, true) +} + +func withRemappedSnapshotBase(id string, i Image, uid, gid uint32, readonly bool) NewContainerOpts { + return func(ctx context.Context, client *Client, c *containers.Container) error { + diffIDs, err := i.(*image).i.RootFS(ctx, client.ContentStore(), platforms.Default()) + if err != nil { + return err + } + + setSnapshotterIfEmpty(c) + + var ( + snapshotter = client.SnapshotService(c.Snapshotter) + parent = identity.ChainID(diffIDs).String() + usernsID = fmt.Sprintf("%s-%d-%d", parent, uid, gid) + ) + if _, err := snapshotter.Stat(ctx, usernsID); err == nil { + if _, err := snapshotter.Prepare(ctx, id, usernsID); err == nil { + c.SnapshotKey = id + c.Image = i.Name() + return nil + } else if !errdefs.IsNotFound(err) { + return err + } + } + mounts, err := snapshotter.Prepare(ctx, usernsID+"-remap", parent) + if err != nil { + return err + } + if err := remapRootFS(mounts, uid, gid); err != nil { + snapshotter.Remove(ctx, usernsID) + return err + } + if err := snapshotter.Commit(ctx, usernsID, usernsID+"-remap"); err != nil { + return err + } + if readonly { + _, err = snapshotter.View(ctx, id, usernsID) + } else { + _, err = snapshotter.Prepare(ctx, id, usernsID) + } + if err != nil { + return err + } + c.SnapshotKey = id + c.Image = i.Name() + return nil + } +} + +func remapRootFS(mounts []mount.Mount, uid, gid uint32) error { + root, err := ioutil.TempDir("", "ctd-remap") + if err != nil { + return err + } + defer os.Remove(root) + for _, m := range mounts { + if err := m.Mount(root); err != nil { + return err + } + } + err = filepath.Walk(root, incrementFS(root, uid, gid)) + if uerr := mount.Unmount(root, 0); err == nil { + err = uerr + } + return err +} + +func incrementFS(root string, uidInc, gidInc uint32) filepath.WalkFunc { + return func(path string, info os.FileInfo, err error) error { + if err != nil { + return err + } + var ( + stat = info.Sys().(*syscall.Stat_t) + u, g = int(stat.Uid + uidInc), int(stat.Gid + gidInc) + ) + // be sure the lchown the path as to not de-reference the symlink to a host file + return os.Lchown(path, u, g) + } +} diff --git a/vendor/github.com/containerd/containerd/content/helpers.go b/vendor/github.com/containerd/containerd/content/helpers.go index 775583a90..86b853685 100644 --- a/vendor/github.com/containerd/containerd/content/helpers.go +++ b/vendor/github.com/containerd/containerd/content/helpers.go @@ -10,13 +10,12 @@ import ( "github.com/pkg/errors" ) -var ( - bufPool = sync.Pool{ - New: func() interface{} { - return make([]byte, 1<<20) - }, - } -) +var bufPool = sync.Pool{ + New: func() interface{} { + buffer := make([]byte, 1<<20) + return &buffer + }, +} // NewReader returns a io.Reader from a ReaderAt func NewReader(ra ReaderAt) io.Reader { @@ -88,10 +87,10 @@ func Copy(ctx context.Context, cw Writer, r io.Reader, size int64, expected dige } } - buf := bufPool.Get().([]byte) + buf := bufPool.Get().(*[]byte) defer bufPool.Put(buf) - if _, err := io.CopyBuffer(cw, r, buf); err != nil { + if _, err := io.CopyBuffer(cw, r, *buf); err != nil { return err } diff --git a/vendor/github.com/containerd/containerd/content/local/store.go b/vendor/github.com/containerd/containerd/content/local/store.go index e449bad5e..9ff95de45 100644 --- a/vendor/github.com/containerd/containerd/content/local/store.go +++ b/vendor/github.com/containerd/containerd/content/local/store.go @@ -21,13 +21,12 @@ import ( "github.com/pkg/errors" ) -var ( - bufPool = sync.Pool{ - New: func() interface{} { - return make([]byte, 1<<20) - }, - } -) +var bufPool = sync.Pool{ + New: func() interface{} { + buffer := make([]byte, 1<<20) + return &buffer + }, +} // LabelStore is used to store mutable labels for digests type LabelStore interface { @@ -63,7 +62,7 @@ func NewStore(root string) (content.Store, error) { // require labels and should use `NewStore`. `NewLabeledStore` is primarily // useful for tests or standalone implementations. func NewLabeledStore(root string, ls LabelStore) (content.Store, error) { - if err := os.MkdirAll(filepath.Join(root, "ingest"), 0777); err != nil && !os.IsExist(err) { + if err := os.MkdirAll(filepath.Join(root, "ingest"), 0777); err != nil { return nil, err } @@ -463,10 +462,10 @@ func (s *store) writer(ctx context.Context, ref string, total int64, expected di } defer fp.Close() - p := bufPool.Get().([]byte) + p := bufPool.Get().(*[]byte) defer bufPool.Put(p) - offset, err = io.CopyBuffer(digester.Hash(), fp, p) + offset, err = io.CopyBuffer(digester.Hash(), fp, *p) if err != nil { return nil, err } diff --git a/vendor/github.com/containerd/containerd/diff/walking/differ.go b/vendor/github.com/containerd/containerd/diff/walking/differ.go index c94e5592c..b1afbe930 100644 --- a/vendor/github.com/containerd/containerd/diff/walking/differ.go +++ b/vendor/github.com/containerd/containerd/diff/walking/differ.go @@ -94,12 +94,20 @@ func (s *walkingDiff) Apply(ctx context.Context, desc ocispec.Descriptor, mounts if err != nil { return emptyDesc, errors.Wrap(err, "failed to create temporary directory") } - defer os.RemoveAll(dir) + // We change RemoveAll to Remove so that we either leak a temp dir + // if it fails but not RM snapshot data. refer to #1868 #1785 + defer os.Remove(dir) if err := mount.All(mounts, dir); err != nil { return emptyDesc, errors.Wrap(err, "failed to mount") } - defer mount.Unmount(dir, 0) + defer func() { + if uerr := mount.Unmount(dir, 0); uerr != nil { + if err == nil { + err = uerr + } + } + }() ra, err := s.store.ReaderAt(ctx, desc.Digest) if err != nil { @@ -164,23 +172,35 @@ func (s *walkingDiff) DiffMounts(ctx context.Context, lower, upper []mount.Mount if err != nil { return emptyDesc, errors.Wrap(err, "failed to create temporary directory") } - defer os.RemoveAll(aDir) + defer os.Remove(aDir) bDir, err := ioutil.TempDir("", "right-") if err != nil { return emptyDesc, errors.Wrap(err, "failed to create temporary directory") } - defer os.RemoveAll(bDir) + defer os.Remove(bDir) if err := mount.All(lower, aDir); err != nil { return emptyDesc, errors.Wrap(err, "failed to mount") } - defer mount.Unmount(aDir, 0) + defer func() { + if uerr := mount.Unmount(aDir, 0); uerr != nil { + if err == nil { + err = uerr + } + } + }() if err := mount.All(upper, bDir); err != nil { return emptyDesc, errors.Wrap(err, "failed to mount") } - defer mount.Unmount(bDir, 0) + defer func() { + if uerr := mount.Unmount(bDir, 0); uerr != nil { + if err == nil { + err = uerr + } + } + }() var newReference bool if config.Reference == "" { diff --git a/vendor/github.com/containerd/containerd/events/events.go b/vendor/github.com/containerd/containerd/events/events.go index 87b94c310..2bfedb1e8 100644 --- a/vendor/github.com/containerd/containerd/events/events.go +++ b/vendor/github.com/containerd/containerd/events/events.go @@ -4,6 +4,7 @@ import ( "context" "time" + "github.com/containerd/typeurl" "github.com/gogo/protobuf/types" ) @@ -15,6 +16,36 @@ type Envelope struct { Event *types.Any } +// Field returns the value for the given fieldpath as a string, if defined. +// If the value is not defined, the second value will be false. +func (e *Envelope) Field(fieldpath []string) (string, bool) { + if len(fieldpath) == 0 { + return "", false + } + + switch fieldpath[0] { + // unhandled: timestamp + case "namespace": + return string(e.Namespace), len(e.Namespace) > 0 + case "topic": + return string(e.Topic), len(e.Topic) > 0 + case "event": + decoded, err := typeurl.UnmarshalAny(e.Event) + if err != nil { + return "", false + } + + adaptor, ok := decoded.(interface { + Field([]string) (string, bool) + }) + if !ok { + return "", false + } + return adaptor.Field(fieldpath[1:]) + } + return "", false +} + // Event is a generic interface for any type of event type Event interface{} diff --git a/vendor/github.com/containerd/containerd/export.go b/vendor/github.com/containerd/containerd/export.go deleted file mode 100644 index 76bebe3cd..000000000 --- a/vendor/github.com/containerd/containerd/export.go +++ /dev/null @@ -1,189 +0,0 @@ -package containerd - -import ( - "archive/tar" - "context" - "encoding/json" - "io" - "sort" - - "github.com/containerd/containerd/content" - "github.com/containerd/containerd/images" - "github.com/containerd/containerd/platforms" - ocispecs "github.com/opencontainers/image-spec/specs-go" - ocispec "github.com/opencontainers/image-spec/specs-go/v1" - "github.com/pkg/errors" -) - -func (c *Client) exportToOCITar(ctx context.Context, desc ocispec.Descriptor, writer io.Writer, eopts exportOpts) error { - tw := tar.NewWriter(writer) - defer tw.Close() - - records := []tarRecord{ - ociLayoutFile(""), - ociIndexRecord(desc), - } - - cs := c.ContentStore() - algorithms := map[string]struct{}{} - exportHandler := func(ctx context.Context, desc ocispec.Descriptor) ([]ocispec.Descriptor, error) { - records = append(records, blobRecord(cs, desc)) - algorithms[desc.Digest.Algorithm().String()] = struct{}{} - return nil, nil - } - - handlers := images.Handlers( - images.ChildrenHandler(cs, platforms.Default()), - images.HandlerFunc(exportHandler), - ) - - // Walk sequentially since the number of fetchs is likely one and doing in - // parallel requires locking the export handler - if err := images.Walk(ctx, handlers, desc); err != nil { - return err - } - - if len(algorithms) > 0 { - records = append(records, directoryRecord("blobs/", 0755)) - for alg := range algorithms { - records = append(records, directoryRecord("blobs/"+alg+"/", 0755)) - } - } - - return writeTar(ctx, tw, records) -} - -type tarRecord struct { - Header *tar.Header - CopyTo func(context.Context, io.Writer) (int64, error) -} - -func blobRecord(cs content.Store, desc ocispec.Descriptor) tarRecord { - path := "blobs/" + desc.Digest.Algorithm().String() + "/" + desc.Digest.Hex() - return tarRecord{ - Header: &tar.Header{ - Name: path, - Mode: 0444, - Size: desc.Size, - Typeflag: tar.TypeReg, - }, - CopyTo: func(ctx context.Context, w io.Writer) (int64, error) { - r, err := cs.ReaderAt(ctx, desc.Digest) - if err != nil { - return 0, err - } - defer r.Close() - - // Verify digest - dgstr := desc.Digest.Algorithm().Digester() - - n, err := io.Copy(io.MultiWriter(w, dgstr.Hash()), content.NewReader(r)) - if err != nil { - return 0, err - } - if dgstr.Digest() != desc.Digest { - return 0, errors.Errorf("unexpected digest %s copied", dgstr.Digest()) - } - return n, nil - }, - } -} - -func directoryRecord(name string, mode int64) tarRecord { - return tarRecord{ - Header: &tar.Header{ - Name: name, - Mode: mode, - Typeflag: tar.TypeDir, - }, - } -} - -func ociLayoutFile(version string) tarRecord { - if version == "" { - version = ocispec.ImageLayoutVersion - } - layout := ocispec.ImageLayout{ - Version: version, - } - - b, err := json.Marshal(layout) - if err != nil { - panic(err) - } - - return tarRecord{ - Header: &tar.Header{ - Name: ocispec.ImageLayoutFile, - Mode: 0444, - Size: int64(len(b)), - Typeflag: tar.TypeReg, - }, - CopyTo: func(ctx context.Context, w io.Writer) (int64, error) { - n, err := w.Write(b) - return int64(n), err - }, - } - -} - -func ociIndexRecord(manifests ...ocispec.Descriptor) tarRecord { - index := ocispec.Index{ - Versioned: ocispecs.Versioned{ - SchemaVersion: 2, - }, - Manifests: manifests, - } - - b, err := json.Marshal(index) - if err != nil { - panic(err) - } - - return tarRecord{ - Header: &tar.Header{ - Name: "index.json", - Mode: 0644, - Size: int64(len(b)), - Typeflag: tar.TypeReg, - }, - CopyTo: func(ctx context.Context, w io.Writer) (int64, error) { - n, err := w.Write(b) - return int64(n), err - }, - } -} - -func writeTar(ctx context.Context, tw *tar.Writer, records []tarRecord) error { - sort.Sort(tarRecordsByName(records)) - - for _, record := range records { - if err := tw.WriteHeader(record.Header); err != nil { - return err - } - if record.CopyTo != nil { - n, err := record.CopyTo(ctx, tw) - if err != nil { - return err - } - if n != record.Header.Size { - return errors.Errorf("unexpected copy size for %s", record.Header.Name) - } - } else if record.Header.Size > 0 { - return errors.Errorf("no content to write to record with non-zero size for %s", record.Header.Name) - } - } - return nil -} - -type tarRecordsByName []tarRecord - -func (t tarRecordsByName) Len() int { - return len(t) -} -func (t tarRecordsByName) Swap(i, j int) { - t[i], t[j] = t[j], t[i] -} -func (t tarRecordsByName) Less(i, j int) bool { - return t[i].Header.Name < t[j].Header.Name -} diff --git a/vendor/github.com/containerd/containerd/filters/parser.go b/vendor/github.com/containerd/containerd/filters/parser.go index c9b09847b..c765ea00c 100644 --- a/vendor/github.com/containerd/containerd/filters/parser.go +++ b/vendor/github.com/containerd/containerd/filters/parser.go @@ -3,7 +3,6 @@ package filters import ( "fmt" "io" - "strconv" "github.com/containerd/containerd/errdefs" "github.com/pkg/errors" @@ -134,7 +133,12 @@ func (p *parser) selector() (selector, error) { return selector{}, err } - value, err := p.value() + var allowAltQuotes bool + if op == operatorMatches { + allowAltQuotes = true + } + + value, err := p.value(allowAltQuotes) if err != nil { if err == io.EOF { return selector{}, io.ErrUnexpectedEOF @@ -188,7 +192,7 @@ func (p *parser) field() (string, error) { case tokenField: return s, nil case tokenQuoted: - return p.unquote(pos, s) + return p.unquote(pos, s, false) } return "", p.mkerr(pos, "expected field or quoted") @@ -213,21 +217,25 @@ func (p *parser) operator() (operator, error) { return 0, p.mkerr(pos, `expected an operator ("=="|"!="|"~=")`) } -func (p *parser) value() (string, error) { +func (p *parser) value(allowAltQuotes bool) (string, error) { pos, tok, s := p.scanner.scan() switch tok { case tokenValue, tokenField: return s, nil case tokenQuoted: - return p.unquote(pos, s) + return p.unquote(pos, s, allowAltQuotes) } return "", p.mkerr(pos, "expected value or quoted") } -func (p *parser) unquote(pos int, s string) (string, error) { - uq, err := strconv.Unquote(s) +func (p *parser) unquote(pos int, s string, allowAlts bool) (string, error) { + if !allowAlts && s[0] != '\'' && s[0] != '"' { + return "", p.mkerr(pos, "invalid quote encountered") + } + + uq, err := unquote(s) if err != nil { return "", p.mkerr(pos, "unquoting failed: %v", err) } diff --git a/vendor/github.com/containerd/containerd/filters/quote.go b/vendor/github.com/containerd/containerd/filters/quote.go new file mode 100644 index 000000000..08698e1ba --- /dev/null +++ b/vendor/github.com/containerd/containerd/filters/quote.go @@ -0,0 +1,237 @@ +package filters + +import ( + "unicode/utf8" + + "github.com/pkg/errors" +) + +// NOTE(stevvooe): Most of this code in this file is copied from the stdlib +// strconv package and modified to be able to handle quoting with `/` and `|` +// as delimiters. The copyright is held by the Go authors. + +var errQuoteSyntax = errors.New("quote syntax error") + +// UnquoteChar decodes the first character or byte in the escaped string +// or character literal represented by the string s. +// It returns four values: +// +// 1) value, the decoded Unicode code point or byte value; +// 2) multibyte, a boolean indicating whether the decoded character requires a multibyte UTF-8 representation; +// 3) tail, the remainder of the string after the character; and +// 4) an error that will be nil if the character is syntactically valid. +// +// The second argument, quote, specifies the type of literal being parsed +// and therefore which escaped quote character is permitted. +// If set to a single quote, it permits the sequence \' and disallows unescaped '. +// If set to a double quote, it permits \" and disallows unescaped ". +// If set to zero, it does not permit either escape and allows both quote characters to appear unescaped. +// +// This is from Go strconv package, modified to support `|` and `/` as double +// quotes for use with regular expressions. +func unquoteChar(s string, quote byte) (value rune, multibyte bool, tail string, err error) { + // easy cases + switch c := s[0]; { + case c == quote && (quote == '\'' || quote == '"' || quote == '/' || quote == '|'): + err = errQuoteSyntax + return + case c >= utf8.RuneSelf: + r, size := utf8.DecodeRuneInString(s) + return r, true, s[size:], nil + case c != '\\': + return rune(s[0]), false, s[1:], nil + } + + // hard case: c is backslash + if len(s) <= 1 { + err = errQuoteSyntax + return + } + c := s[1] + s = s[2:] + + switch c { + case 'a': + value = '\a' + case 'b': + value = '\b' + case 'f': + value = '\f' + case 'n': + value = '\n' + case 'r': + value = '\r' + case 't': + value = '\t' + case 'v': + value = '\v' + case 'x', 'u', 'U': + n := 0 + switch c { + case 'x': + n = 2 + case 'u': + n = 4 + case 'U': + n = 8 + } + var v rune + if len(s) < n { + err = errQuoteSyntax + return + } + for j := 0; j < n; j++ { + x, ok := unhex(s[j]) + if !ok { + err = errQuoteSyntax + return + } + v = v<<4 | x + } + s = s[n:] + if c == 'x' { + // single-byte string, possibly not UTF-8 + value = v + break + } + if v > utf8.MaxRune { + err = errQuoteSyntax + return + } + value = v + multibyte = true + case '0', '1', '2', '3', '4', '5', '6', '7': + v := rune(c) - '0' + if len(s) < 2 { + err = errQuoteSyntax + return + } + for j := 0; j < 2; j++ { // one digit already; two more + x := rune(s[j]) - '0' + if x < 0 || x > 7 { + err = errQuoteSyntax + return + } + v = (v << 3) | x + } + s = s[2:] + if v > 255 { + err = errQuoteSyntax + return + } + value = v + case '\\': + value = '\\' + case '\'', '"', '|', '/': + if c != quote { + err = errQuoteSyntax + return + } + value = rune(c) + default: + err = errQuoteSyntax + return + } + tail = s + return +} + +// unquote interprets s as a single-quoted, double-quoted, +// or backquoted Go string literal, returning the string value +// that s quotes. (If s is single-quoted, it would be a Go +// character literal; Unquote returns the corresponding +// one-character string.) +// +// This is modified from the standard library to support `|` and `/` as quote +// characters for use with regular expressions. +func unquote(s string) (string, error) { + n := len(s) + if n < 2 { + return "", errQuoteSyntax + } + quote := s[0] + if quote != s[n-1] { + return "", errQuoteSyntax + } + s = s[1 : n-1] + + if quote == '`' { + if contains(s, '`') { + return "", errQuoteSyntax + } + if contains(s, '\r') { + // -1 because we know there is at least one \r to remove. + buf := make([]byte, 0, len(s)-1) + for i := 0; i < len(s); i++ { + if s[i] != '\r' { + buf = append(buf, s[i]) + } + } + return string(buf), nil + } + return s, nil + } + if quote != '"' && quote != '\'' && quote != '|' && quote != '/' { + return "", errQuoteSyntax + } + if contains(s, '\n') { + return "", errQuoteSyntax + } + + // Is it trivial? Avoid allocation. + if !contains(s, '\\') && !contains(s, quote) { + switch quote { + case '"', '/', '|': // pipe and slash are treated like double quote + return s, nil + case '\'': + r, size := utf8.DecodeRuneInString(s) + if size == len(s) && (r != utf8.RuneError || size != 1) { + return s, nil + } + } + } + + var runeTmp [utf8.UTFMax]byte + buf := make([]byte, 0, 3*len(s)/2) // Try to avoid more allocations. + for len(s) > 0 { + c, multibyte, ss, err := unquoteChar(s, quote) + if err != nil { + return "", err + } + s = ss + if c < utf8.RuneSelf || !multibyte { + buf = append(buf, byte(c)) + } else { + n := utf8.EncodeRune(runeTmp[:], c) + buf = append(buf, runeTmp[:n]...) + } + if quote == '\'' && len(s) != 0 { + // single-quoted must be single character + return "", errQuoteSyntax + } + } + return string(buf), nil +} + +// contains reports whether the string contains the byte c. +func contains(s string, c byte) bool { + for i := 0; i < len(s); i++ { + if s[i] == c { + return true + } + } + return false +} + +func unhex(b byte) (v rune, ok bool) { + c := rune(b) + switch { + case '0' <= c && c <= '9': + return c - '0', true + case 'a' <= c && c <= 'f': + return c - 'a' + 10, true + case 'A' <= c && c <= 'F': + return c - 'A' + 10, true + } + return +} diff --git a/vendor/github.com/containerd/containerd/filters/scanner.go b/vendor/github.com/containerd/containerd/filters/scanner.go index 5a55e0abf..3a8e72395 100644 --- a/vendor/github.com/containerd/containerd/filters/scanner.go +++ b/vendor/github.com/containerd/containerd/filters/scanner.go @@ -87,7 +87,7 @@ func (s *scanner) peek() rune { return ch } -func (s *scanner) scan() (int, token, string) { +func (s *scanner) scan() (nextp int, tk token, text string) { var ( ch = s.next() pos = s.pos @@ -101,6 +101,7 @@ chomp: s.scanQuoted(ch) return pos, tokenQuoted, s.input[pos:s.ppos] case isSeparatorRune(ch): + s.value = false return pos, tokenSeparator, s.input[pos:s.ppos] case isOperatorRune(ch): s.scanOperator() @@ -241,7 +242,7 @@ func isOperatorRune(r rune) bool { func isQuoteRune(r rune) bool { switch r { - case '"': // maybe add single quoting? + case '/', '|', '"': // maybe add single quoting? return true } diff --git a/vendor/github.com/containerd/containerd/fs/copy.go b/vendor/github.com/containerd/containerd/fs/copy.go index 0d11fa527..e8f452819 100644 --- a/vendor/github.com/containerd/containerd/fs/copy.go +++ b/vendor/github.com/containerd/containerd/fs/copy.go @@ -9,13 +9,12 @@ import ( "github.com/pkg/errors" ) -var ( - bufferPool = &sync.Pool{ - New: func() interface{} { - return make([]byte, 32*1024) - }, - } -) +var bufferPool = &sync.Pool{ + New: func() interface{} { + buffer := make([]byte, 32*1024) + return &buffer + }, +} // CopyDir copies the directory from src to dst. // Most efficient copy of files is attempted. diff --git a/vendor/github.com/containerd/containerd/fs/copy_linux.go b/vendor/github.com/containerd/containerd/fs/copy_linux.go index efe4753e0..c1fb2d1c4 100644 --- a/vendor/github.com/containerd/containerd/fs/copy_linux.go +++ b/vendor/github.com/containerd/containerd/fs/copy_linux.go @@ -43,8 +43,8 @@ func copyFileContent(dst, src *os.File) error { return errors.Wrap(err, "copy file range failed") } - buf := bufferPool.Get().([]byte) - _, err = io.CopyBuffer(dst, src, buf) + buf := bufferPool.Get().(*[]byte) + _, err = io.CopyBuffer(dst, src, *buf) bufferPool.Put(buf) return err } diff --git a/vendor/github.com/containerd/containerd/fs/copy_unix.go b/vendor/github.com/containerd/containerd/fs/copy_unix.go index 6234f3da3..b31a14fcd 100644 --- a/vendor/github.com/containerd/containerd/fs/copy_unix.go +++ b/vendor/github.com/containerd/containerd/fs/copy_unix.go @@ -34,8 +34,8 @@ func copyFileInfo(fi os.FileInfo, name string) error { } func copyFileContent(dst, src *os.File) error { - buf := bufferPool.Get().([]byte) - _, err := io.CopyBuffer(dst, src, buf) + buf := bufferPool.Get().(*[]byte) + _, err := io.CopyBuffer(dst, src, *buf) bufferPool.Put(buf) return err diff --git a/vendor/github.com/containerd/containerd/fs/copy_windows.go b/vendor/github.com/containerd/containerd/fs/copy_windows.go index fb4933c25..6fb3de571 100644 --- a/vendor/github.com/containerd/containerd/fs/copy_windows.go +++ b/vendor/github.com/containerd/containerd/fs/copy_windows.go @@ -18,8 +18,8 @@ func copyFileInfo(fi os.FileInfo, name string) error { } func copyFileContent(dst, src *os.File) error { - buf := bufferPool.Get().([]byte) - _, err := io.CopyBuffer(dst, src, buf) + buf := bufferPool.Get().(*[]byte) + _, err := io.CopyBuffer(dst, src, *buf) bufferPool.Put(buf) return err } diff --git a/vendor/github.com/containerd/containerd/fs/diff.go b/vendor/github.com/containerd/containerd/fs/diff.go index 9073d0d92..3a53f4215 100644 --- a/vendor/github.com/containerd/containerd/fs/diff.go +++ b/vendor/github.com/containerd/containerd/fs/diff.go @@ -222,8 +222,10 @@ func doubleWalkDiff(ctx context.Context, changeFn ChangeFunc, a, b string) (err c1 = make(chan *currentPath) c2 = make(chan *currentPath) - f1, f2 *currentPath - rmdir string + f1, f2 *currentPath + rmdir string + lastEmittedDir = string(filepath.Separator) + parents []os.FileInfo ) g.Go(func() error { defer close(c1) @@ -258,7 +260,10 @@ func doubleWalkDiff(ctx context.Context, changeFn ChangeFunc, a, b string) (err continue } - var f os.FileInfo + var ( + f os.FileInfo + emit = true + ) k, p := pathChange(f1, f2) switch k { case ChangeKindAdd: @@ -294,13 +299,35 @@ func doubleWalkDiff(ctx context.Context, changeFn ChangeFunc, a, b string) (err f2 = nil if same { if !isLinked(f) { - continue + emit = false } k = ChangeKindUnmodified } } - if err := changeFn(k, p, f, nil); err != nil { - return err + if emit { + emittedDir, emitParents := commonParents(lastEmittedDir, p, parents) + for _, pf := range emitParents { + p := filepath.Join(emittedDir, pf.Name()) + if err := changeFn(ChangeKindUnmodified, p, pf, nil); err != nil { + return err + } + emittedDir = p + } + + if err := changeFn(k, p, f, nil); err != nil { + return err + } + + if f != nil && f.IsDir() { + lastEmittedDir = p + } else { + lastEmittedDir = emittedDir + } + + parents = parents[:0] + } else if f.IsDir() { + lastEmittedDir, parents = commonParents(lastEmittedDir, p, parents) + parents = append(parents, f) } } return nil @@ -308,3 +335,47 @@ func doubleWalkDiff(ctx context.Context, changeFn ChangeFunc, a, b string) (err return g.Wait() } + +func commonParents(base, updated string, dirs []os.FileInfo) (string, []os.FileInfo) { + if basePrefix := makePrefix(base); strings.HasPrefix(updated, basePrefix) { + var ( + parents []os.FileInfo + last = base + ) + for _, d := range dirs { + next := filepath.Join(last, d.Name()) + if strings.HasPrefix(updated, makePrefix(last)) { + parents = append(parents, d) + last = next + } else { + break + } + } + return base, parents + } + + baseS := strings.Split(base, string(filepath.Separator)) + updatedS := strings.Split(updated, string(filepath.Separator)) + commonS := []string{string(filepath.Separator)} + + min := len(baseS) + if len(updatedS) < min { + min = len(updatedS) + } + for i := 0; i < min; i++ { + if baseS[i] == updatedS[i] { + commonS = append(commonS, baseS[i]) + } else { + break + } + } + + return filepath.Join(commonS...), []os.FileInfo{} +} + +func makePrefix(d string) string { + if d == "" || d[len(d)-1] != filepath.Separator { + return d + string(filepath.Separator) + } + return d +} diff --git a/vendor/github.com/containerd/containerd/fs/du.go b/vendor/github.com/containerd/containerd/fs/du.go index 61f439d39..26f533315 100644 --- a/vendor/github.com/containerd/containerd/fs/du.go +++ b/vendor/github.com/containerd/containerd/fs/du.go @@ -1,5 +1,7 @@ package fs +import "context" + // Usage of disk information type Usage struct { Inodes int64 @@ -11,3 +13,10 @@ type Usage struct { func DiskUsage(roots ...string) (Usage, error) { return diskUsage(roots...) } + +// DiffUsage counts the numbers of inodes and disk usage in the +// diff between the 2 directories. The first path is intended +// as the base directory and the second as the changed directory. +func DiffUsage(ctx context.Context, a, b string) (Usage, error) { + return diffUsage(ctx, a, b) +} diff --git a/vendor/github.com/containerd/containerd/fs/du_unix.go b/vendor/github.com/containerd/containerd/fs/du_unix.go index d8654d32f..6328e80f3 100644 --- a/vendor/github.com/containerd/containerd/fs/du_unix.go +++ b/vendor/github.com/containerd/containerd/fs/du_unix.go @@ -3,17 +3,19 @@ package fs import ( + "context" "os" "path/filepath" "syscall" ) +type inode struct { + // TODO(stevvooe): Can probably reduce memory usage by not tracking + // device, but we can leave this right for now. + dev, ino uint64 +} + func diskUsage(roots ...string) (Usage, error) { - type inode struct { - // TODO(stevvooe): Can probably reduce memory usage by not tracking - // device, but we can leave this right for now. - dev, ino uint64 - } var ( size int64 @@ -45,3 +47,37 @@ func diskUsage(roots ...string) (Usage, error) { Size: size, }, nil } + +func diffUsage(ctx context.Context, a, b string) (Usage, error) { + var ( + size int64 + inodes = map[inode]struct{}{} // expensive! + ) + + if err := Changes(ctx, a, b, func(kind ChangeKind, _ string, fi os.FileInfo, err error) error { + if err != nil { + return err + } + + if kind == ChangeKindAdd || kind == ChangeKindModify { + stat := fi.Sys().(*syscall.Stat_t) + + inoKey := inode{dev: uint64(stat.Dev), ino: uint64(stat.Ino)} + if _, ok := inodes[inoKey]; !ok { + inodes[inoKey] = struct{}{} + size += fi.Size() + } + + return nil + + } + return nil + }); err != nil { + return Usage{}, err + } + + return Usage{ + Inodes: int64(len(inodes)), + Size: size, + }, nil +} diff --git a/vendor/github.com/containerd/containerd/fs/du_windows.go b/vendor/github.com/containerd/containerd/fs/du_windows.go index 4a0363c06..3f852fc15 100644 --- a/vendor/github.com/containerd/containerd/fs/du_windows.go +++ b/vendor/github.com/containerd/containerd/fs/du_windows.go @@ -3,6 +3,7 @@ package fs import ( + "context" "os" "path/filepath" ) @@ -31,3 +32,29 @@ func diskUsage(roots ...string) (Usage, error) { Size: size, }, nil } + +func diffUsage(ctx context.Context, a, b string) (Usage, error) { + var ( + size int64 + ) + + if err := Changes(ctx, a, b, func(kind ChangeKind, _ string, fi os.FileInfo, err error) error { + if err != nil { + return err + } + + if kind == ChangeKindAdd || kind == ChangeKindModify { + size += fi.Size() + + return nil + + } + return nil + }); err != nil { + return Usage{}, err + } + + return Usage{ + Size: size, + }, nil +} diff --git a/vendor/github.com/containerd/containerd/fs/fstest/testsuite.go b/vendor/github.com/containerd/containerd/fs/fstest/testsuite.go index 546be89d1..a26e8bbd8 100644 --- a/vendor/github.com/containerd/containerd/fs/fstest/testsuite.go +++ b/vendor/github.com/containerd/containerd/fs/fstest/testsuite.go @@ -19,6 +19,7 @@ func FSSuite(t *testing.T, a TestApplier) { t.Run("Deletion", makeTest(t, a, deletionTest)) t.Run("Update", makeTest(t, a, updateTest)) t.Run("DirectoryPermission", makeTest(t, a, directoryPermissionsTest)) + t.Run("ParentDirectoryPermission", makeTest(t, a, parentDirectoryPermissionsTest)) t.Run("HardlinkUnmodified", makeTest(t, a, hardlinkUnmodified)) t.Run("HardlinkBeforeUnmodified", makeTest(t, a, hardlinkBeforeUnmodified)) t.Run("HardlinkBeforeModified", makeTest(t, a, hardlinkBeforeModified)) @@ -159,6 +160,28 @@ var ( ), } + // parentDirectoryPermissionsTest covers directory permissions for updated + // files + parentDirectoryPermissionsTest = []Applier{ + Apply( + CreateDir("/d1", 0700), + CreateDir("/d1/a", 0700), + CreateDir("/d1/a/b", 0700), + CreateDir("/d1/a/b/c", 0700), + CreateFile("/d1/a/b/f", []byte("content1"), 0644), + CreateDir("/d2", 0751), + CreateDir("/d2/a/b", 0751), + CreateDir("/d2/a/b/c", 0751), + CreateFile("/d2/a/b/f", []byte("content1"), 0644), + ), + Apply( + CreateFile("/d1/a/b/f", []byte("content1"), 0644), + Chmod("/d1/a/b/c", 0700), + CreateFile("/d2/a/b/f", []byte("content2"), 0644), + Chmod("/d2/a/b/c", 0751), + ), + } + hardlinkUnmodified = []Applier{ baseApplier, Apply( diff --git a/vendor/github.com/containerd/containerd/image.go b/vendor/github.com/containerd/containerd/image.go index 4f978e2ea..6e9f4bd19 100644 --- a/vendor/github.com/containerd/containerd/image.go +++ b/vendor/github.com/containerd/containerd/image.go @@ -9,7 +9,7 @@ import ( "github.com/containerd/containerd/images" "github.com/containerd/containerd/platforms" "github.com/containerd/containerd/rootfs" - "github.com/containerd/containerd/snapshot" + "github.com/containerd/containerd/snapshots" digest "github.com/opencontainers/go-digest" "github.com/opencontainers/image-spec/identity" ocispec "github.com/opencontainers/image-spec/specs-go/v1" @@ -32,6 +32,8 @@ type Image interface { Config(ctx context.Context) (ocispec.Descriptor, error) // IsUnpacked returns whether or not an image is unpacked. IsUnpacked(context.Context, string) (bool, error) + // ContentStore provides a content store which contains image blob data + ContentStore() content.Store } var _ = (Image)(&image{}) @@ -86,6 +88,12 @@ func (i *image) IsUnpacked(ctx context.Context, snapshotterName string) (bool, e } func (i *image) Unpack(ctx context.Context, snapshotterName string) error { + ctx, done, err := i.client.WithLease(ctx) + if err != nil { + return err + } + defer done() + layers, err := i.getLayers(ctx, platforms.Default()) if err != nil { return err @@ -104,7 +112,7 @@ func (i *image) Unpack(ctx context.Context, snapshotterName string) error { "containerd.io/uncompressed": layer.Diff.Digest.String(), } - unpacked, err = rootfs.ApplyLayer(ctx, layer, chain, sn, a, snapshot.WithLabels(labels)) + unpacked, err = rootfs.ApplyLayer(ctx, layer, chain, sn, a, snapshots.WithLabels(labels)) if err != nil { return err } @@ -139,7 +147,7 @@ func (i *image) getLayers(ctx context.Context, platform string) ([]rootfs.Layer, manifest, err := images.Manifest(ctx, cs, i.i.Target, platform) if err != nil { - return nil, errors.Wrap(err, "") + return nil, err } diffIDs, err := i.i.RootFS(ctx, cs, platform) @@ -160,3 +168,7 @@ func (i *image) getLayers(ctx context.Context, platform string) ([]rootfs.Layer, } return layers, nil } + +func (i *image) ContentStore() content.Store { + return i.client.ContentStore() +} diff --git a/vendor/github.com/containerd/containerd/image_store.go b/vendor/github.com/containerd/containerd/image_store.go index daa18f7f5..9a3aafc84 100644 --- a/vendor/github.com/containerd/containerd/image_store.go +++ b/vendor/github.com/containerd/containerd/image_store.go @@ -74,9 +74,16 @@ func (s *remoteImages) Update(ctx context.Context, image images.Image, fieldpath return imageFromProto(&updated.Image), nil } -func (s *remoteImages) Delete(ctx context.Context, name string) error { +func (s *remoteImages) Delete(ctx context.Context, name string, opts ...images.DeleteOpt) error { + var do images.DeleteOptions + for _, opt := range opts { + if err := opt(ctx, &do); err != nil { + return err + } + } _, err := s.client.Delete(ctx, &imagesapi.DeleteImageRequest{ Name: name, + Sync: do.Synchronous, }) return errdefs.FromGRPC(err) diff --git a/vendor/github.com/containerd/containerd/images/image.go b/vendor/github.com/containerd/containerd/images/image.go index 4c78c6cc2..7b4215faf 100644 --- a/vendor/github.com/containerd/containerd/images/image.go +++ b/vendor/github.com/containerd/containerd/images/image.go @@ -38,6 +38,23 @@ type Image struct { CreatedAt, UpdatedAt time.Time } +// DeleteOptions provide options on image delete +type DeleteOptions struct { + Synchronous bool +} + +// DeleteOpt allows configuring a delete operation +type DeleteOpt func(context.Context, *DeleteOptions) error + +// SynchronousDelete is used to indicate that an image deletion and removal of +// the image resources should occur synchronously before returning a result. +func SynchronousDelete() DeleteOpt { + return func(ctx context.Context, o *DeleteOptions) error { + o.Synchronous = true + return nil + } +} + // Store and interact with images type Store interface { Get(ctx context.Context, name string) (Image, error) @@ -48,7 +65,7 @@ type Store interface { // one or more fieldpaths are provided, only those fields will be updated. Update(ctx context.Context, image Image, fieldpaths ...string) (Image, error) - Delete(ctx context.Context, name string) error + Delete(ctx context.Context, name string, opts ...DeleteOpt) error } // TODO(stevvooe): Many of these functions make strong platform assumptions, @@ -170,13 +187,13 @@ func Manifest(ctx context.Context, provider content.Provider, image ocispec.Desc return descs, nil } - return nil, errors.Wrap(errdefs.ErrNotFound, "could not resolve manifest") + return nil, errors.Wrapf(errdefs.ErrNotFound, "unexpected media type %v for %v", desc.MediaType, desc.Digest) }), image); err != nil { return ocispec.Manifest{}, err } if m == nil { - return ocispec.Manifest{}, errors.Wrap(errdefs.ErrNotFound, "manifest not found") + return ocispec.Manifest{}, errors.Wrapf(errdefs.ErrNotFound, "manifest %v", image.Digest) } return *m, nil @@ -240,7 +257,7 @@ func Check(ctx context.Context, provider content.Provider, image ocispec.Descrip return false, []ocispec.Descriptor{image}, nil, []ocispec.Descriptor{image}, nil } - return false, nil, nil, nil, errors.Wrap(err, "image check failed") + return false, nil, nil, nil, errors.Wrapf(err, "failed to check image %v", image.Digest) } // TODO(stevvooe): It is possible that referenced conponents could have @@ -255,7 +272,7 @@ func Check(ctx context.Context, provider content.Provider, image ocispec.Descrip missing = append(missing, desc) continue } else { - return false, nil, nil, nil, err + return false, nil, nil, nil, errors.Wrapf(err, "failed to check image %v", desc.Digest) } } ra.Close() diff --git a/vendor/github.com/containerd/containerd/images/importexport.go b/vendor/github.com/containerd/containerd/images/importexport.go new file mode 100644 index 000000000..f8cf742ba --- /dev/null +++ b/vendor/github.com/containerd/containerd/images/importexport.go @@ -0,0 +1,21 @@ +package images + +import ( + "context" + "io" + + "github.com/containerd/containerd/content" + ocispec "github.com/opencontainers/image-spec/specs-go/v1" +) + +// Importer is the interface for image importer. +type Importer interface { + // Import imports an image from a tar stream. + Import(ctx context.Context, store content.Store, reader io.Reader) ([]Image, error) +} + +// Exporter is the interface for image exporter. +type Exporter interface { + // Export exports an image to a tar stream. + Export(ctx context.Context, store content.Store, desc ocispec.Descriptor, writer io.Writer) error +} diff --git a/vendor/github.com/containerd/containerd/import.go b/vendor/github.com/containerd/containerd/import.go deleted file mode 100644 index 9f8f9af7d..000000000 --- a/vendor/github.com/containerd/containerd/import.go +++ /dev/null @@ -1,120 +0,0 @@ -package containerd - -import ( - "archive/tar" - "context" - "encoding/json" - "io" - "io/ioutil" - "strings" - - "github.com/containerd/containerd/content" - "github.com/containerd/containerd/errdefs" - "github.com/containerd/containerd/images" - "github.com/containerd/containerd/reference" - digest "github.com/opencontainers/go-digest" - ocispec "github.com/opencontainers/image-spec/specs-go/v1" - "github.com/pkg/errors" -) - -func resolveOCIIndex(idx ocispec.Index, refObject string) (*ocispec.Descriptor, error) { - tag, dgst := reference.SplitObject(refObject) - if tag == "" && dgst == "" { - return nil, errors.Errorf("unexpected object: %q", refObject) - } - for _, m := range idx.Manifests { - if m.Digest == dgst { - return &m, nil - } - annot, ok := m.Annotations[ocispec.AnnotationRefName] - if ok && annot == tag && tag != "" { - return &m, nil - } - } - return nil, errors.Errorf("not found: %q", refObject) -} - -func (c *Client) importFromOCITar(ctx context.Context, ref string, reader io.Reader, iopts importOpts) (Image, error) { - tr := tar.NewReader(reader) - store := c.ContentStore() - var desc *ocispec.Descriptor - for { - hdr, err := tr.Next() - if err == io.EOF { - break - } - if err != nil { - return nil, err - } - if hdr.Typeflag != tar.TypeReg && hdr.Typeflag != tar.TypeRegA { - continue - } - if hdr.Name == "index.json" { - desc, err = onUntarIndexJSON(tr, iopts.refObject) - if err != nil { - return nil, err - } - continue - } - if strings.HasPrefix(hdr.Name, "blobs/") { - if err := onUntarBlob(ctx, tr, store, hdr.Name, hdr.Size); err != nil { - return nil, err - } - } - } - if desc == nil { - return nil, errors.Errorf("no descriptor found for reference object %q", iopts.refObject) - } - imgrec := images.Image{ - Name: ref, - Target: *desc, - Labels: iopts.labels, - } - is := c.ImageService() - if updated, err := is.Update(ctx, imgrec, "target"); err != nil { - if !errdefs.IsNotFound(err) { - return nil, err - } - - created, err := is.Create(ctx, imgrec) - if err != nil { - return nil, err - } - - imgrec = created - } else { - imgrec = updated - } - - img := &image{ - client: c, - i: imgrec, - } - return img, nil -} - -func onUntarIndexJSON(r io.Reader, refObject string) (*ocispec.Descriptor, error) { - b, err := ioutil.ReadAll(r) - if err != nil { - return nil, err - } - var idx ocispec.Index - if err := json.Unmarshal(b, &idx); err != nil { - return nil, err - } - return resolveOCIIndex(idx, refObject) -} - -func onUntarBlob(ctx context.Context, r io.Reader, store content.Store, name string, size int64) error { - // name is like "blobs/sha256/deadbeef" - split := strings.Split(name, "/") - if len(split) != 3 { - return errors.Errorf("unexpected name: %q", name) - } - algo := digest.Algorithm(split[1]) - if !algo.Available() { - return errors.Errorf("unsupported algorithm: %s", algo) - } - dgst := digest.NewDigestFromHex(algo.String(), split[2]) - return content.WriteBlob(ctx, store, "unknown-"+dgst.String(), r, size, dgst) -} diff --git a/vendor/github.com/containerd/containerd/lease.go b/vendor/github.com/containerd/containerd/lease.go index 6ecc58dec..8eb3bc0fe 100644 --- a/vendor/github.com/containerd/containerd/lease.go +++ b/vendor/github.com/containerd/containerd/lease.go @@ -51,7 +51,8 @@ func (c *Client) ListLeases(ctx context.Context) ([]Lease, error) { return leases, nil } -func (c *Client) withLease(ctx context.Context) (context.Context, func() error, error) { +// WithLease attaches a lease on the context +func (c *Client) WithLease(ctx context.Context) (context.Context, func() error, error) { _, ok := leases.Lease(ctx) if ok { return ctx, func() error { diff --git a/vendor/github.com/containerd/containerd/metadata/containers.go b/vendor/github.com/containerd/containerd/metadata/containers.go index 4cca5f69f..32f339afe 100644 --- a/vendor/github.com/containerd/containerd/metadata/containers.go +++ b/vendor/github.com/containerd/containerd/metadata/containers.go @@ -37,12 +37,12 @@ func (s *containerStore) Get(ctx context.Context, id string) (containers.Contain bkt := getContainerBucket(s.tx, namespace, id) if bkt == nil { - return containers.Container{}, errors.Wrapf(errdefs.ErrNotFound, "bucket name %q:%q", namespace, id) + return containers.Container{}, errors.Wrapf(errdefs.ErrNotFound, "container %q in namespace %q", id, namespace) } container := containers.Container{ID: id} if err := readContainer(&container, bkt); err != nil { - return containers.Container{}, errors.Wrapf(err, "failed to read container %v", id) + return containers.Container{}, errors.Wrapf(err, "failed to read container %q", id) } return container, nil @@ -61,7 +61,7 @@ func (s *containerStore) List(ctx context.Context, fs ...string) ([]containers.C bkt := getContainersBucket(s.tx, namespace) if bkt == nil { - return nil, nil + return nil, nil // empty store } var m []containers.Container @@ -73,7 +73,7 @@ func (s *containerStore) List(ctx context.Context, fs ...string) ([]containers.C container := containers.Container{ID: string(k)} if err := readContainer(&container, cbkt); err != nil { - return errors.Wrap(err, "failed to read container") + return errors.Wrapf(err, "failed to read container %q", string(k)) } if filter.Match(adaptContainer(container)) { @@ -113,7 +113,7 @@ func (s *containerStore) Create(ctx context.Context, container containers.Contai container.CreatedAt = time.Now().UTC() container.UpdatedAt = container.CreatedAt if err := writeContainer(cbkt, &container); err != nil { - return containers.Container{}, errors.Wrap(err, "failed to write container") + return containers.Container{}, errors.Wrapf(err, "failed to write container %q", container.ID) } return container, nil @@ -131,7 +131,7 @@ func (s *containerStore) Update(ctx context.Context, container containers.Contai bkt := getContainersBucket(s.tx, namespace) if bkt == nil { - return containers.Container{}, errors.Wrapf(errdefs.ErrNotFound, "container %q", container.ID) + return containers.Container{}, errors.Wrapf(errdefs.ErrNotFound, "cannot update container %q in namespace %q", container.ID, namespace) } cbkt := bkt.Bucket([]byte(container.ID)) @@ -141,7 +141,7 @@ func (s *containerStore) Update(ctx context.Context, container containers.Contai var updated containers.Container if err := readContainer(&updated, cbkt); err != nil { - return updated, errors.Wrapf(err, "failed to read container from bucket") + return updated, errors.Wrapf(err, "failed to read container %q", container.ID) } createdat := updated.CreatedAt updated.ID = container.ID @@ -211,7 +211,7 @@ func (s *containerStore) Update(ctx context.Context, container containers.Contai updated.CreatedAt = createdat updated.UpdatedAt = time.Now().UTC() if err := writeContainer(cbkt, &updated); err != nil { - return containers.Container{}, errors.Wrap(err, "failed to write container") + return containers.Container{}, errors.Wrapf(err, "failed to write container %q", container.ID) } return updated, nil @@ -225,7 +225,7 @@ func (s *containerStore) Delete(ctx context.Context, id string) error { bkt := getContainersBucket(s.tx, namespace) if bkt == nil { - return errors.Wrapf(errdefs.ErrNotFound, "cannot delete container %v, bucket not present", id) + return errors.Wrapf(errdefs.ErrNotFound, "cannot delete container %q in namespace %q", id, namespace) } if err := bkt.DeleteBucket([]byte(id)); err == bolt.ErrBucketNotFound { @@ -236,7 +236,7 @@ func (s *containerStore) Delete(ctx context.Context, id string) error { func validateContainer(container *containers.Container) error { if err := identifiers.Validate(container.ID); err != nil { - return errors.Wrapf(err, "container.ID validation error") + return errors.Wrap(err, "container.ID") } for k := range container.Extensions { diff --git a/vendor/github.com/containerd/containerd/metadata/content.go b/vendor/github.com/containerd/containerd/metadata/content.go index 1a9b16af9..c13f7867e 100644 --- a/vendor/github.com/containerd/containerd/metadata/content.go +++ b/vendor/github.com/containerd/containerd/metadata/content.go @@ -530,12 +530,14 @@ func writeInfo(info *content.Info, bkt *bolt.Bucket) error { return bkt.Put(bucketKeySize, sizeEncoded) } -func (cs *contentStore) garbageCollect(ctx context.Context) error { - lt1 := time.Now() +func (cs *contentStore) garbageCollect(ctx context.Context) (d time.Duration, err error) { cs.l.Lock() + t1 := time.Now() defer func() { + if err == nil { + d = time.Now().Sub(t1) + } cs.l.Unlock() - log.G(ctx).WithField("t", time.Now().Sub(lt1)).Debugf("content garbage collected") }() seen := map[string]struct{}{} @@ -570,10 +572,10 @@ func (cs *contentStore) garbageCollect(ctx context.Context) error { return nil }); err != nil { - return err + return 0, err } - return cs.Store.Walk(ctx, func(info content.Info) error { + err = cs.Store.Walk(ctx, func(info content.Info) error { if _, ok := seen[info.Digest.String()]; !ok { if err := cs.Store.Delete(ctx, info.Digest); err != nil { return err @@ -582,4 +584,5 @@ func (cs *contentStore) garbageCollect(ctx context.Context) error { } return nil }) + return } diff --git a/vendor/github.com/containerd/containerd/metadata/db.go b/vendor/github.com/containerd/containerd/metadata/db.go index 7c366ebcc..8be62a95c 100644 --- a/vendor/github.com/containerd/containerd/metadata/db.go +++ b/vendor/github.com/containerd/containerd/metadata/db.go @@ -11,7 +11,7 @@ import ( "github.com/containerd/containerd/content" "github.com/containerd/containerd/gc" "github.com/containerd/containerd/log" - "github.com/containerd/containerd/snapshot" + "github.com/containerd/containerd/snapshots" "github.com/pkg/errors" ) @@ -53,13 +53,14 @@ type DB struct { dirtySS map[string]struct{} dirtyCS bool - // TODO: Keep track of stats such as pause time, number of collected objects, errors - lastCollection time.Time + // mutationCallbacks are called after each mutation with the flag + // set indicating whether any dirty flags are set + mutationCallbacks []func(bool) } // NewDB creates a new metadata database using the provided // bolt database, content store, and snapshotters. -func NewDB(db *bolt.DB, cs content.Store, ss map[string]snapshot.Snapshotter) *DB { +func NewDB(db *bolt.DB, cs content.Store, ss map[string]snapshots.Snapshotter) *DB { m := &DB{ db: db, ss: make(map[string]*snapshotter, len(ss)), @@ -137,7 +138,7 @@ func (m *DB) Init(ctx context.Context) error { if err := m.migrate(tx); err != nil { return errors.Wrapf(err, "failed to migrate to %s.%d", m.schema, m.version) } - log.G(ctx).WithField("d", time.Now().Sub(t0)).Debugf("database migration to %s.%d finished", m.schema, m.version) + log.G(ctx).WithField("d", time.Now().Sub(t0)).Debugf("finished database migration to %s.%d", m.schema, m.version) } } @@ -170,7 +171,7 @@ func (m *DB) ContentStore() content.Store { // Snapshotter returns a namespaced content store for // the requested snapshotter name proxied to a snapshotter. -func (m *DB) Snapshotter(name string) snapshot.Snapshotter { +func (m *DB) Snapshotter(name string) snapshots.Snapshotter { sn, ok := m.ss[name] if !ok { return nil @@ -183,29 +184,53 @@ func (m *DB) View(fn func(*bolt.Tx) error) error { return m.db.View(fn) } -// Update runs a writable transation on the metadata store. +// Update runs a writable transaction on the metadata store. func (m *DB) Update(fn func(*bolt.Tx) error) error { m.wlock.RLock() defer m.wlock.RUnlock() - return m.db.Update(fn) + err := m.db.Update(fn) + if err == nil { + m.dirtyL.Lock() + dirty := m.dirtyCS || len(m.dirtySS) > 0 + for _, fn := range m.mutationCallbacks { + fn(dirty) + } + m.dirtyL.Unlock() + } + + return err +} + +// RegisterMutationCallback registers a function to be called after a metadata +// mutations has been performed. +// +// The callback function in an argument for whether a deletion has occurred +// since the last garbage collection. +func (m *DB) RegisterMutationCallback(fn func(bool)) { + m.dirtyL.Lock() + m.mutationCallbacks = append(m.mutationCallbacks, fn) + m.dirtyL.Unlock() +} + +// GCStats holds the duration for the different phases of the garbage collector +type GCStats struct { + MetaD time.Duration + ContentD time.Duration + SnapshotD map[string]time.Duration } // GarbageCollect starts garbage collection -func (m *DB) GarbageCollect(ctx context.Context) error { - lt1 := time.Now() +func (m *DB) GarbageCollect(ctx context.Context) (stats GCStats, err error) { m.wlock.Lock() - defer func() { - m.wlock.Unlock() - log.G(ctx).WithField("d", time.Now().Sub(lt1)).Debug("metadata garbage collected") - }() + t1 := time.Now() marked, err := m.getMarked(ctx) if err != nil { - return err + m.wlock.Unlock() + return GCStats{}, err } m.dirtyL.Lock() - defer m.dirtyL.Unlock() if err := m.db.Update(func(tx *bolt.Tx) error { ctx, cancel := context.WithCancel(ctx) @@ -232,26 +257,53 @@ func (m *DB) GarbageCollect(ctx context.Context) error { return nil }); err != nil { - return err + m.dirtyL.Unlock() + m.wlock.Unlock() + return GCStats{}, err } - m.lastCollection = time.Now() + var wg sync.WaitGroup if len(m.dirtySS) > 0 { + var sl sync.Mutex + stats.SnapshotD = map[string]time.Duration{} + wg.Add(len(m.dirtySS)) for snapshotterName := range m.dirtySS { - log.G(ctx).WithField("snapshotter", snapshotterName).Debug("scheduling snapshotter cleanup") - go m.cleanupSnapshotter(snapshotterName) + log.G(ctx).WithField("snapshotter", snapshotterName).Debug("schedule snapshotter cleanup") + go func(snapshotterName string) { + st1 := time.Now() + m.cleanupSnapshotter(snapshotterName) + + sl.Lock() + stats.SnapshotD[snapshotterName] = time.Now().Sub(st1) + sl.Unlock() + + wg.Done() + }(snapshotterName) } m.dirtySS = map[string]struct{}{} } if m.dirtyCS { - log.G(ctx).Debug("scheduling content cleanup") - go m.cleanupContent() + wg.Add(1) + log.G(ctx).Debug("schedule content cleanup") + go func() { + ct1 := time.Now() + m.cleanupContent() + stats.ContentD = time.Now().Sub(ct1) + wg.Done() + }() m.dirtyCS = false } - return nil + m.dirtyL.Unlock() + + stats.MetaD = time.Now().Sub(t1) + m.wlock.Unlock() + + wg.Wait() + + return } func (m *DB) getMarked(ctx context.Context) (map[gc.Node]struct{}, error) { @@ -302,27 +354,35 @@ func (m *DB) getMarked(ctx context.Context) (map[gc.Node]struct{}, error) { return marked, nil } -func (m *DB) cleanupSnapshotter(name string) { +func (m *DB) cleanupSnapshotter(name string) (time.Duration, error) { ctx := context.Background() sn, ok := m.ss[name] if !ok { - return + return 0, nil } - err := sn.garbageCollect(ctx) + d, err := sn.garbageCollect(ctx) + logger := log.G(ctx).WithField("snapshotter", name) if err != nil { - log.G(ctx).WithError(err).WithField("snapshotter", name).Warn("garbage collection failed") + logger.WithError(err).Warn("snapshot garbage collection failed") + } else { + logger.WithField("d", d).Debugf("snapshot garbage collected") } + return d, err } -func (m *DB) cleanupContent() { +func (m *DB) cleanupContent() (time.Duration, error) { ctx := context.Background() if m.cs == nil { - return + return 0, nil } - err := m.cs.garbageCollect(ctx) + d, err := m.cs.garbageCollect(ctx) if err != nil { log.G(ctx).WithError(err).Warn("content garbage collection failed") + } else { + log.G(ctx).WithField("d", d).Debugf("content garbage collected") } + + return d, err } diff --git a/vendor/github.com/containerd/containerd/metadata/gc.go b/vendor/github.com/containerd/containerd/metadata/gc.go index 7fe6f7da2..186f350ee 100644 --- a/vendor/github.com/containerd/containerd/metadata/gc.go +++ b/vendor/github.com/containerd/containerd/metadata/gc.go @@ -301,7 +301,7 @@ func remove(ctx context.Context, tx *bolt.Tx, node gc.Node) error { cbkt = cbkt.Bucket(bucketKeyObjectBlob) } if cbkt != nil { - log.G(ctx).WithField("key", node.Key).Debug("delete content") + log.G(ctx).WithField("key", node.Key).Debug("remove content") return cbkt.DeleteBucket([]byte(node.Key)) } case ResourceSnapshot: @@ -313,7 +313,7 @@ func remove(ctx context.Context, tx *bolt.Tx, node gc.Node) error { } ssbkt := sbkt.Bucket([]byte(parts[0])) if ssbkt != nil { - log.G(ctx).WithField("key", parts[1]).WithField("snapshotter", parts[0]).Debug("delete snapshot") + log.G(ctx).WithField("key", parts[1]).WithField("snapshotter", parts[0]).Debug("remove snapshot") return ssbkt.DeleteBucket([]byte(parts[1])) } } diff --git a/vendor/github.com/containerd/containerd/metadata/images.go b/vendor/github.com/containerd/containerd/metadata/images.go index 7e5e3c76e..070439a8c 100644 --- a/vendor/github.com/containerd/containerd/metadata/images.go +++ b/vendor/github.com/containerd/containerd/metadata/images.go @@ -100,10 +100,6 @@ func (s *imageStore) Create(ctx context.Context, image images.Image) (images.Ima return images.Image{}, err } - if image.Name == "" { - return images.Image{}, errors.Wrapf(errdefs.ErrInvalidArgument, "image name is required for create") - } - if err := validateImage(&image); err != nil { return images.Image{}, err } @@ -177,7 +173,7 @@ func (s *imageStore) Update(ctx context.Context, image images.Image, fieldpaths updated = image } - if err := validateImage(&image); err != nil { + if err := validateImage(&updated); err != nil { return err } @@ -187,7 +183,7 @@ func (s *imageStore) Update(ctx context.Context, image images.Image, fieldpaths }) } -func (s *imageStore) Delete(ctx context.Context, name string) error { +func (s *imageStore) Delete(ctx context.Context, name string, opts ...images.DeleteOpt) error { namespace, err := namespaces.NamespaceRequired(ctx) if err != nil { return err diff --git a/vendor/github.com/containerd/containerd/metadata/snapshot.go b/vendor/github.com/containerd/containerd/metadata/snapshot.go index 760cac80e..6c34e49c2 100644 --- a/vendor/github.com/containerd/containerd/metadata/snapshot.go +++ b/vendor/github.com/containerd/containerd/metadata/snapshot.go @@ -14,12 +14,12 @@ import ( "github.com/containerd/containerd/metadata/boltutil" "github.com/containerd/containerd/mount" "github.com/containerd/containerd/namespaces" - "github.com/containerd/containerd/snapshot" + "github.com/containerd/containerd/snapshots" "github.com/pkg/errors" ) type snapshotter struct { - snapshot.Snapshotter + snapshots.Snapshotter name string db *DB l sync.RWMutex @@ -27,7 +27,7 @@ type snapshotter struct { // newSnapshotter returns a new Snapshotter which namespaces the given snapshot // using the provided name and database. -func newSnapshotter(db *DB, name string, sn snapshot.Snapshotter) *snapshotter { +func newSnapshotter(db *DB, name string, sn snapshots.Snapshotter) *snapshotter { return &snapshotter{ Snapshotter: sn, name: name, @@ -75,15 +75,15 @@ func (s *snapshotter) resolveKey(ctx context.Context, key string) (string, error return id, nil } -func (s *snapshotter) Stat(ctx context.Context, key string) (snapshot.Info, error) { +func (s *snapshotter) Stat(ctx context.Context, key string) (snapshots.Info, error) { ns, err := namespaces.NamespaceRequired(ctx) if err != nil { - return snapshot.Info{}, err + return snapshots.Info{}, err } var ( bkey string - local = snapshot.Info{ + local = snapshots.Info{ Name: key, } ) @@ -108,33 +108,33 @@ func (s *snapshotter) Stat(ctx context.Context, key string) (snapshot.Info, erro return nil }); err != nil { - return snapshot.Info{}, err + return snapshots.Info{}, err } info, err := s.Snapshotter.Stat(ctx, bkey) if err != nil { - return snapshot.Info{}, err + return snapshots.Info{}, err } return overlayInfo(info, local), nil } -func (s *snapshotter) Update(ctx context.Context, info snapshot.Info, fieldpaths ...string) (snapshot.Info, error) { +func (s *snapshotter) Update(ctx context.Context, info snapshots.Info, fieldpaths ...string) (snapshots.Info, error) { s.l.RLock() defer s.l.RUnlock() ns, err := namespaces.NamespaceRequired(ctx) if err != nil { - return snapshot.Info{}, err + return snapshots.Info{}, err } if info.Name == "" { - return snapshot.Info{}, errors.Wrap(errdefs.ErrInvalidArgument, "") + return snapshots.Info{}, errors.Wrap(errdefs.ErrInvalidArgument, "") } var ( bkey string - local = snapshot.Info{ + local = snapshots.Info{ Name: info.Name, } ) @@ -195,18 +195,18 @@ func (s *snapshotter) Update(ctx context.Context, info snapshot.Info, fieldpaths return nil }); err != nil { - return snapshot.Info{}, err + return snapshots.Info{}, err } info, err = s.Snapshotter.Stat(ctx, bkey) if err != nil { - return snapshot.Info{}, err + return snapshots.Info{}, err } return overlayInfo(info, local), nil } -func overlayInfo(info, overlay snapshot.Info) snapshot.Info { +func overlayInfo(info, overlay snapshots.Info) snapshots.Info { // Merge info info.Name = overlay.Name info.Created = overlay.Created @@ -222,10 +222,10 @@ func overlayInfo(info, overlay snapshot.Info) snapshot.Info { return info } -func (s *snapshotter) Usage(ctx context.Context, key string) (snapshot.Usage, error) { +func (s *snapshotter) Usage(ctx context.Context, key string) (snapshots.Usage, error) { bkey, err := s.resolveKey(ctx, key) if err != nil { - return snapshot.Usage{}, err + return snapshots.Usage{}, err } return s.Snapshotter.Usage(ctx, bkey) } @@ -238,15 +238,15 @@ func (s *snapshotter) Mounts(ctx context.Context, key string) ([]mount.Mount, er return s.Snapshotter.Mounts(ctx, bkey) } -func (s *snapshotter) Prepare(ctx context.Context, key, parent string, opts ...snapshot.Opt) ([]mount.Mount, error) { +func (s *snapshotter) Prepare(ctx context.Context, key, parent string, opts ...snapshots.Opt) ([]mount.Mount, error) { return s.createSnapshot(ctx, key, parent, false, opts) } -func (s *snapshotter) View(ctx context.Context, key, parent string, opts ...snapshot.Opt) ([]mount.Mount, error) { +func (s *snapshotter) View(ctx context.Context, key, parent string, opts ...snapshots.Opt) ([]mount.Mount, error) { return s.createSnapshot(ctx, key, parent, true, opts) } -func (s *snapshotter) createSnapshot(ctx context.Context, key, parent string, readonly bool, opts []snapshot.Opt) ([]mount.Mount, error) { +func (s *snapshotter) createSnapshot(ctx context.Context, key, parent string, readonly bool, opts []snapshots.Opt) ([]mount.Mount, error) { s.l.RLock() defer s.l.RUnlock() @@ -255,7 +255,7 @@ func (s *snapshotter) createSnapshot(ctx context.Context, key, parent string, re return nil, err } - var base snapshot.Info + var base snapshots.Info for _, opt := range opts { if err := opt(&base); err != nil { return nil, err @@ -336,7 +336,7 @@ func (s *snapshotter) createSnapshot(ctx context.Context, key, parent string, re return m, nil } -func (s *snapshotter) Commit(ctx context.Context, name, key string, opts ...snapshot.Opt) error { +func (s *snapshotter) Commit(ctx context.Context, name, key string, opts ...snapshots.Opt) error { s.l.RLock() defer s.l.RUnlock() @@ -345,7 +345,7 @@ func (s *snapshotter) Commit(ctx context.Context, name, key string, opts ...snap return err } - var base snapshot.Info + var base snapshots.Info for _, opt := range opts { if err := opt(&base); err != nil { return err @@ -359,7 +359,8 @@ func (s *snapshotter) Commit(ctx context.Context, name, key string, opts ...snap return update(ctx, s.db, func(tx *bolt.Tx) error { bkt := getSnapshotterBucket(tx, ns, s.name) if bkt == nil { - return errors.Wrapf(errdefs.ErrNotFound, "snapshot %v does not exist", key) + return errors.Wrapf(errdefs.ErrNotFound, + "can not find snapshotter %q", s.name) } bbkt, err := bkt.CreateBucket([]byte(name)) @@ -493,10 +494,10 @@ func (s *snapshotter) Remove(ctx context.Context, key string) error { type infoPair struct { bkey string - info snapshot.Info + info snapshots.Info } -func (s *snapshotter) Walk(ctx context.Context, fn func(context.Context, snapshot.Info) error) error { +func (s *snapshotter) Walk(ctx context.Context, fn func(context.Context, snapshots.Info) error) error { ns, err := namespaces.NamespaceRequired(ctx) if err != nil { return err @@ -533,7 +534,7 @@ func (s *snapshotter) Walk(ctx context.Context, fn func(context.Context, snapsho pair := infoPair{ bkey: string(sbkt.Get(bucketKeyName)), - info: snapshot.Info{ + info: snapshots.Info{ Name: string(k), Parent: string(sbkt.Get(bucketKeyParent)), }, @@ -586,7 +587,7 @@ func (s *snapshotter) Walk(ctx context.Context, fn func(context.Context, snapsho return nil } -func validateSnapshot(info *snapshot.Info) error { +func validateSnapshot(info *snapshots.Info) error { for k, v := range info.Labels { if err := labels.Validate(k, v); err != nil { return errors.Wrapf(err, "info.Labels") @@ -596,13 +597,14 @@ func validateSnapshot(info *snapshot.Info) error { return nil } -func (s *snapshotter) garbageCollect(ctx context.Context) error { - logger := log.G(ctx).WithField("snapshotter", s.name) - lt1 := time.Now() +func (s *snapshotter) garbageCollect(ctx context.Context) (d time.Duration, err error) { s.l.Lock() + t1 := time.Now() defer func() { + if err == nil { + d = time.Now().Sub(t1) + } s.l.Unlock() - logger.WithField("t", time.Now().Sub(lt1)).Debugf("garbage collected") }() seen := map[string]struct{}{} @@ -646,27 +648,30 @@ func (s *snapshotter) garbageCollect(ctx context.Context) error { return nil }); err != nil { - return err + return 0, err } roots, err := s.walkTree(ctx, seen) if err != nil { - return err + return 0, err } - // TODO: Unlock before prune (once nodes are fully unavailable) + // TODO: Unlock before removal (once nodes are fully unavailable). + // This could be achieved through doing prune inside the lock + // and having a cleanup method which actually performs the + // deletions on the snapshotters which support it. for _, node := range roots { if err := s.pruneBranch(ctx, node); err != nil { - return err + return 0, err } } - return nil + return } type treeNode struct { - info snapshot.Info + info snapshots.Info remove bool children []*treeNode } @@ -675,7 +680,7 @@ func (s *snapshotter) walkTree(ctx context.Context, seen map[string]struct{}) ([ roots := []*treeNode{} nodes := map[string]*treeNode{} - if err := s.Snapshotter.Walk(ctx, func(ctx context.Context, info snapshot.Info) error { + if err := s.Snapshotter.Walk(ctx, func(ctx context.Context, info snapshots.Info) error { _, isSeen := seen[info.Name] node, ok := nodes[info.Name] if !ok { @@ -718,7 +723,7 @@ func (s *snapshotter) pruneBranch(ctx context.Context, node *treeNode) error { if !errdefs.IsFailedPrecondition(err) { return err } - logger.WithError(err).WithField("key", node.info.Name).Warnf("snapshot removal failed") + logger.WithError(err).WithField("key", node.info.Name).Warnf("failed to remove snapshot") } else { logger.WithField("key", node.info.Name).Debug("removed snapshot") } diff --git a/vendor/github.com/containerd/containerd/mount/mount_linux.go b/vendor/github.com/containerd/containerd/mount/mount_linux.go index 474792d8e..de2e8bb7d 100644 --- a/vendor/github.com/containerd/containerd/mount/mount_linux.go +++ b/vendor/github.com/containerd/containerd/mount/mount_linux.go @@ -2,7 +2,9 @@ package mount import ( "strings" + "time" + "github.com/pkg/errors" "golang.org/x/sys/unix" ) @@ -42,8 +44,27 @@ func (m *Mount) Mount(target string) error { } // Unmount the provided mount path with the flags -func Unmount(mount string, flags int) error { - return unix.Unmount(mount, flags) +func Unmount(target string, flags int) error { + if err := unmount(target, flags); err != nil && err != unix.EINVAL { + return err + } + return nil +} + +func unmount(target string, flags int) error { + for i := 0; i < 50; i++ { + if err := unix.Unmount(target, flags); err != nil { + switch err { + case unix.EBUSY: + time.Sleep(50 * time.Millisecond) + continue + default: + return err + } + } + return nil + } + return errors.Wrapf(unix.EBUSY, "failed to unmount target %s", target) } // UnmountAll repeatedly unmounts the given mount point until there @@ -51,7 +72,7 @@ func Unmount(mount string, flags int) error { // useful for undoing a stack of mounts on the same mount point. func UnmountAll(mount string, flags int) error { for { - if err := Unmount(mount, flags); err != nil { + if err := unmount(mount, flags); err != nil { // EINVAL is returned if the target is not a // mount point, indicating that we are // done. It can also indicate a few other diff --git a/vendor/github.com/containerd/containerd/oci/client.go b/vendor/github.com/containerd/containerd/oci/client.go new file mode 100644 index 000000000..d2cd355c5 --- /dev/null +++ b/vendor/github.com/containerd/containerd/oci/client.go @@ -0,0 +1,22 @@ +package oci + +import ( + "context" + + "github.com/containerd/containerd/content" + "github.com/containerd/containerd/snapshots" + ocispec "github.com/opencontainers/image-spec/specs-go/v1" +) + +// Client interface used by SpecOpt +type Client interface { + SnapshotService(snapshotterName string) snapshots.Snapshotter +} + +// Image interface used by some SpecOpt to query image configuration +type Image interface { + // Config descriptor for the image. + Config(ctx context.Context) (ocispec.Descriptor, error) + // ContentStore provides a content store which contains image blob data + ContentStore() content.Store +} diff --git a/vendor/github.com/containerd/containerd/spec.go b/vendor/github.com/containerd/containerd/oci/spec.go similarity index 75% rename from vendor/github.com/containerd/containerd/spec.go rename to vendor/github.com/containerd/containerd/oci/spec.go index 850f470a0..558a3570f 100644 --- a/vendor/github.com/containerd/containerd/spec.go +++ b/vendor/github.com/containerd/containerd/oci/spec.go @@ -1,4 +1,4 @@ -package containerd +package oci import ( "context" @@ -9,7 +9,7 @@ import ( // GenerateSpec will generate a default spec from the provided image // for use as a containerd container -func GenerateSpec(ctx context.Context, client *Client, c *containers.Container, opts ...SpecOpts) (*specs.Spec, error) { +func GenerateSpec(ctx context.Context, client Client, c *containers.Container, opts ...SpecOpts) (*specs.Spec, error) { s, err := createDefaultSpec(ctx, c.ID) if err != nil { return nil, err diff --git a/vendor/github.com/containerd/containerd/oci/spec_opts.go b/vendor/github.com/containerd/containerd/oci/spec_opts.go new file mode 100644 index 000000000..c940c7aab --- /dev/null +++ b/vendor/github.com/containerd/containerd/oci/spec_opts.go @@ -0,0 +1,35 @@ +package oci + +import ( + "context" + + "github.com/containerd/containerd/containers" + specs "github.com/opencontainers/runtime-spec/specs-go" +) + +// SpecOpts sets spec specific information to a newly generated OCI spec +type SpecOpts func(context.Context, Client, *containers.Container, *specs.Spec) error + +// WithProcessArgs replaces the args on the generated spec +func WithProcessArgs(args ...string) SpecOpts { + return func(_ context.Context, _ Client, _ *containers.Container, s *specs.Spec) error { + s.Process.Args = args + return nil + } +} + +// WithProcessCwd replaces the current working directory on the generated spec +func WithProcessCwd(cwd string) SpecOpts { + return func(_ context.Context, _ Client, _ *containers.Container, s *specs.Spec) error { + s.Process.Cwd = cwd + return nil + } +} + +// WithHostname sets the container's hostname +func WithHostname(name string) SpecOpts { + return func(_ context.Context, _ Client, _ *containers.Container, s *specs.Spec) error { + s.Hostname = name + return nil + } +} diff --git a/vendor/github.com/containerd/containerd/spec_opts_unix.go b/vendor/github.com/containerd/containerd/oci/spec_opts_unix.go similarity index 66% rename from vendor/github.com/containerd/containerd/spec_opts_unix.go rename to vendor/github.com/containerd/containerd/oci/spec_opts_unix.go index 01d5121d4..865aff29a 100644 --- a/vendor/github.com/containerd/containerd/spec_opts_unix.go +++ b/vendor/github.com/containerd/containerd/oci/spec_opts_unix.go @@ -1,6 +1,6 @@ // +build !windows -package containerd +package oci import ( "context" @@ -12,16 +12,12 @@ import ( "strconv" "strings" - "golang.org/x/sys/unix" - "github.com/containerd/containerd/containers" "github.com/containerd/containerd/content" - "github.com/containerd/containerd/errdefs" "github.com/containerd/containerd/fs" "github.com/containerd/containerd/images" + "github.com/containerd/containerd/mount" "github.com/containerd/containerd/namespaces" - "github.com/containerd/containerd/platforms" - "github.com/opencontainers/image-spec/identity" "github.com/opencontainers/image-spec/specs-go/v1" "github.com/opencontainers/runc/libcontainer/user" specs "github.com/opencontainers/runtime-spec/specs-go" @@ -30,7 +26,7 @@ import ( // WithTTY sets the information on the spec as well as the environment variables for // using a TTY -func WithTTY(_ context.Context, _ *Client, _ *containers.Container, s *specs.Spec) error { +func WithTTY(_ context.Context, _ Client, _ *containers.Container, s *specs.Spec) error { s.Process.Terminal = true s.Process.Env = append(s.Process.Env, "TERM=xterm") return nil @@ -38,7 +34,7 @@ func WithTTY(_ context.Context, _ *Client, _ *containers.Container, s *specs.Spe // WithHostNamespace allows a task to run inside the host's linux namespace func WithHostNamespace(ns specs.LinuxNamespaceType) SpecOpts { - return func(_ context.Context, _ *Client, _ *containers.Container, s *specs.Spec) error { + return func(_ context.Context, _ Client, _ *containers.Container, s *specs.Spec) error { for i, n := range s.Linux.Namespaces { if n.Type == ns { s.Linux.Namespaces = append(s.Linux.Namespaces[:i], s.Linux.Namespaces[i+1:]...) @@ -52,7 +48,7 @@ func WithHostNamespace(ns specs.LinuxNamespaceType) SpecOpts { // WithLinuxNamespace uses the passed in namespace for the spec. If a namespace of the same type already exists in the // spec, the existing namespace is replaced by the one provided. func WithLinuxNamespace(ns specs.LinuxNamespace) SpecOpts { - return func(_ context.Context, _ *Client, _ *containers.Container, s *specs.Spec) error { + return func(_ context.Context, _ Client, _ *containers.Container, s *specs.Spec) error { for i, n := range s.Linux.Namespaces { if n.Type == ns.Type { before := s.Linux.Namespaces[:i] @@ -68,13 +64,9 @@ func WithLinuxNamespace(ns specs.LinuxNamespace) SpecOpts { } // WithImageConfig configures the spec to from the configuration of an Image -func WithImageConfig(i Image) SpecOpts { - return func(ctx context.Context, client *Client, c *containers.Container, s *specs.Spec) error { - var ( - image = i.(*image) - store = client.ContentStore() - ) - ic, err := image.i.Config(ctx, store, platforms.Default()) +func WithImageConfig(image Image) SpecOpts { + return func(ctx context.Context, client Client, c *containers.Container, s *specs.Spec) error { + ic, err := image.Config(ctx) if err != nil { return err } @@ -84,7 +76,7 @@ func WithImageConfig(i Image) SpecOpts { ) switch ic.MediaType { case v1.MediaTypeImageConfig, images.MediaTypeDockerSchema2Config: - p, err := content.ReadBlob(ctx, store, ic.Digest) + p, err := content.ReadBlob(ctx, image.ContentStore(), ic.Digest) if err != nil { return err } @@ -96,6 +88,11 @@ func WithImageConfig(i Image) SpecOpts { default: return fmt.Errorf("unknown image config media type %s", ic.MediaType) } + + if s.Process == nil { + s.Process = &specs.Process{} + } + s.Process.Env = append(s.Process.Env, config.Env...) cmd := config.Cmd s.Process.Args = append(config.Entrypoint, cmd...) @@ -103,7 +100,7 @@ func WithImageConfig(i Image) SpecOpts { parts := strings.Split(config.User, ":") switch len(parts) { case 1: - v, err := strconv.ParseUint(parts[0], 0, 10) + v, err := strconv.Atoi(parts[0]) if err != nil { // if we cannot parse as a uint they try to see if it is a username if err := WithUsername(config.User)(ctx, client, c, s); err != nil { @@ -115,13 +112,13 @@ func WithImageConfig(i Image) SpecOpts { return err } case 2: - v, err := strconv.ParseUint(parts[0], 0, 10) + v, err := strconv.Atoi(parts[0]) if err != nil { - return err + return errors.Wrapf(err, "parse uid %s", parts[0]) } uid := uint32(v) - if v, err = strconv.ParseUint(parts[1], 0, 10); err != nil { - return err + if v, err = strconv.Atoi(parts[1]); err != nil { + return errors.Wrapf(err, "parse gid %s", parts[1]) } gid := uint32(v) s.Process.User.UID, s.Process.User.GID = uid, gid @@ -140,7 +137,7 @@ func WithImageConfig(i Image) SpecOpts { // WithRootFSPath specifies unmanaged rootfs path. func WithRootFSPath(path string) SpecOpts { - return func(_ context.Context, _ *Client, _ *containers.Container, s *specs.Spec) error { + return func(_ context.Context, _ Client, _ *containers.Container, s *specs.Spec) error { if s.Root == nil { s.Root = &specs.Root{} } @@ -152,7 +149,7 @@ func WithRootFSPath(path string) SpecOpts { // WithRootFSReadonly sets specs.Root.Readonly to true func WithRootFSReadonly() SpecOpts { - return func(_ context.Context, _ *Client, _ *containers.Container, s *specs.Spec) error { + return func(_ context.Context, _ Client, _ *containers.Container, s *specs.Spec) error { if s.Root == nil { s.Root = &specs.Root{} } @@ -161,22 +158,14 @@ func WithRootFSReadonly() SpecOpts { } } -// WithResources sets the provided resources on the spec for task updates -func WithResources(resources *specs.LinuxResources) UpdateTaskOpts { - return func(ctx context.Context, client *Client, r *UpdateTaskInfo) error { - r.Resources = resources - return nil - } -} - // WithNoNewPrivileges sets no_new_privileges on the process for the container -func WithNoNewPrivileges(_ context.Context, _ *Client, _ *containers.Container, s *specs.Spec) error { +func WithNoNewPrivileges(_ context.Context, _ Client, _ *containers.Container, s *specs.Spec) error { s.Process.NoNewPrivileges = true return nil } // WithHostHostsFile bind-mounts the host's /etc/hosts into the container as readonly -func WithHostHostsFile(_ context.Context, _ *Client, _ *containers.Container, s *specs.Spec) error { +func WithHostHostsFile(_ context.Context, _ Client, _ *containers.Container, s *specs.Spec) error { s.Mounts = append(s.Mounts, specs.Mount{ Destination: "/etc/hosts", Type: "bind", @@ -187,7 +176,7 @@ func WithHostHostsFile(_ context.Context, _ *Client, _ *containers.Container, s } // WithHostResolvconf bind-mounts the host's /etc/resolv.conf into the container as readonly -func WithHostResolvconf(_ context.Context, _ *Client, _ *containers.Container, s *specs.Spec) error { +func WithHostResolvconf(_ context.Context, _ Client, _ *containers.Container, s *specs.Spec) error { s.Mounts = append(s.Mounts, specs.Mount{ Destination: "/etc/resolv.conf", Type: "bind", @@ -198,7 +187,7 @@ func WithHostResolvconf(_ context.Context, _ *Client, _ *containers.Container, s } // WithHostLocaltime bind-mounts the host's /etc/localtime into the container as readonly -func WithHostLocaltime(_ context.Context, _ *Client, _ *containers.Container, s *specs.Spec) error { +func WithHostLocaltime(_ context.Context, _ Client, _ *containers.Container, s *specs.Spec) error { s.Mounts = append(s.Mounts, specs.Mount{ Destination: "/etc/localtime", Type: "bind", @@ -211,7 +200,7 @@ func WithHostLocaltime(_ context.Context, _ *Client, _ *containers.Container, s // WithUserNamespace sets the uid and gid mappings for the task // this can be called multiple times to add more mappings to the generated spec func WithUserNamespace(container, host, size uint32) SpecOpts { - return func(_ context.Context, _ *Client, _ *containers.Container, s *specs.Spec) error { + return func(_ context.Context, _ Client, _ *containers.Container, s *specs.Spec) error { var hasUserns bool for _, ns := range s.Linux.Namespaces { if ns.Type == specs.UserNamespace { @@ -235,68 +224,9 @@ func WithUserNamespace(container, host, size uint32) SpecOpts { } } -// WithRemappedSnapshot creates a new snapshot and remaps the uid/gid for the -// filesystem to be used by a container with user namespaces -func WithRemappedSnapshot(id string, i Image, uid, gid uint32) NewContainerOpts { - return withRemappedSnapshotBase(id, i, uid, gid, false) -} - -// WithRemappedSnapshotView is similar to WithRemappedSnapshot but rootfs is mounted as read-only. -func WithRemappedSnapshotView(id string, i Image, uid, gid uint32) NewContainerOpts { - return withRemappedSnapshotBase(id, i, uid, gid, true) -} - -func withRemappedSnapshotBase(id string, i Image, uid, gid uint32, readonly bool) NewContainerOpts { - return func(ctx context.Context, client *Client, c *containers.Container) error { - diffIDs, err := i.(*image).i.RootFS(ctx, client.ContentStore(), platforms.Default()) - if err != nil { - return err - } - - setSnapshotterIfEmpty(c) - - var ( - snapshotter = client.SnapshotService(c.Snapshotter) - parent = identity.ChainID(diffIDs).String() - usernsID = fmt.Sprintf("%s-%d-%d", parent, uid, gid) - ) - if _, err := snapshotter.Stat(ctx, usernsID); err == nil { - if _, err := snapshotter.Prepare(ctx, id, usernsID); err == nil { - c.SnapshotKey = id - c.Image = i.Name() - return nil - } else if !errdefs.IsNotFound(err) { - return err - } - } - mounts, err := snapshotter.Prepare(ctx, usernsID+"-remap", parent) - if err != nil { - return err - } - if err := remapRootFS(mounts, uid, gid); err != nil { - snapshotter.Remove(ctx, usernsID) - return err - } - if err := snapshotter.Commit(ctx, usernsID, usernsID+"-remap"); err != nil { - return err - } - if readonly { - _, err = snapshotter.View(ctx, id, usernsID) - } else { - _, err = snapshotter.Prepare(ctx, id, usernsID) - } - if err != nil { - return err - } - c.SnapshotKey = id - c.Image = i.Name() - return nil - } -} - // WithCgroup sets the container's cgroup path func WithCgroup(path string) SpecOpts { - return func(_ context.Context, _ *Client, _ *containers.Container, s *specs.Spec) error { + return func(_ context.Context, _ Client, _ *containers.Container, s *specs.Spec) error { s.Linux.CgroupsPath = path return nil } @@ -305,7 +235,7 @@ func WithCgroup(path string) SpecOpts { // WithNamespacedCgroup uses the namespace set on the context to create a // root directory for containers in the cgroup with the id as the subcgroup func WithNamespacedCgroup() SpecOpts { - return func(ctx context.Context, _ *Client, c *containers.Container, s *specs.Spec) error { + return func(ctx context.Context, _ Client, c *containers.Container, s *specs.Spec) error { namespace, err := namespaces.NamespaceRequired(ctx) if err != nil { return err @@ -317,7 +247,7 @@ func WithNamespacedCgroup() SpecOpts { // WithUIDGID allows the UID and GID for the Process to be set func WithUIDGID(uid, gid uint32) SpecOpts { - return func(_ context.Context, _ *Client, _ *containers.Container, s *specs.Spec) error { + return func(_ context.Context, _ Client, _ *containers.Container, s *specs.Spec) error { s.Process.User.UID = uid s.Process.User.GID = gid return nil @@ -329,7 +259,7 @@ func WithUIDGID(uid, gid uint32) SpecOpts { // or uid is not found in /etc/passwd, it sets gid to be the same with // uid, and not returns error. func WithUserID(uid uint32) SpecOpts { - return func(ctx context.Context, client *Client, c *containers.Container, s *specs.Spec) error { + return func(ctx context.Context, client Client, c *containers.Container, s *specs.Spec) (err error) { if c.Snapshotter == "" { return errors.Errorf("no snapshotter set for container") } @@ -345,13 +275,19 @@ func WithUserID(uid uint32) SpecOpts { if err != nil { return err } - defer os.RemoveAll(root) + defer os.Remove(root) for _, m := range mounts { if err := m.Mount(root); err != nil { return err } } - defer unix.Unmount(root, 0) + defer func() { + if uerr := mount.Unmount(root, 0); uerr != nil { + if err == nil { + err = uerr + } + } + }() ppath, err := fs.RootPath(root, "/etc/passwd") if err != nil { return err @@ -386,7 +322,7 @@ func WithUserID(uid uint32) SpecOpts { // does not exist, or the username is not found in /etc/passwd, // it returns error. func WithUsername(username string) SpecOpts { - return func(ctx context.Context, client *Client, c *containers.Container, s *specs.Spec) error { + return func(ctx context.Context, client Client, c *containers.Container, s *specs.Spec) (err error) { if c.Snapshotter == "" { return errors.Errorf("no snapshotter set for container") } @@ -402,13 +338,19 @@ func WithUsername(username string) SpecOpts { if err != nil { return err } - defer os.RemoveAll(root) + defer os.Remove(root) for _, m := range mounts { if err := m.Mount(root); err != nil { return err } } - defer unix.Unmount(root, 0) + defer func() { + if uerr := mount.Unmount(root, 0); uerr != nil { + if err == nil { + err = uerr + } + } + }() ppath, err := fs.RootPath(root, "/etc/passwd") if err != nil { return err diff --git a/vendor/github.com/containerd/containerd/spec_opts_windows.go b/vendor/github.com/containerd/containerd/oci/spec_opts_windows.go similarity index 65% rename from vendor/github.com/containerd/containerd/spec_opts_windows.go rename to vendor/github.com/containerd/containerd/oci/spec_opts_windows.go index 1fc5d5e37..796ad5598 100644 --- a/vendor/github.com/containerd/containerd/spec_opts_windows.go +++ b/vendor/github.com/containerd/containerd/oci/spec_opts_windows.go @@ -1,6 +1,6 @@ // +build windows -package containerd +package oci import ( "context" @@ -10,19 +10,14 @@ import ( "github.com/containerd/containerd/containers" "github.com/containerd/containerd/content" "github.com/containerd/containerd/images" - "github.com/containerd/containerd/platforms" "github.com/opencontainers/image-spec/specs-go/v1" specs "github.com/opencontainers/runtime-spec/specs-go" ) // WithImageConfig configures the spec to from the configuration of an Image -func WithImageConfig(i Image) SpecOpts { - return func(ctx context.Context, client *Client, _ *containers.Container, s *specs.Spec) error { - var ( - image = i.(*image) - store = client.ContentStore() - ) - ic, err := image.i.Config(ctx, store, platforms.Default()) +func WithImageConfig(image Image) SpecOpts { + return func(ctx context.Context, client Client, _ *containers.Container, s *specs.Spec) error { + ic, err := image.Config(ctx) if err != nil { return err } @@ -32,7 +27,7 @@ func WithImageConfig(i Image) SpecOpts { ) switch ic.MediaType { case v1.MediaTypeImageConfig, images.MediaTypeDockerSchema2Config: - p, err := content.ReadBlob(ctx, store, ic.Digest) + p, err := content.ReadBlob(ctx, image.ContentStore(), ic.Digest) if err != nil { return err } @@ -55,7 +50,7 @@ func WithImageConfig(i Image) SpecOpts { // WithTTY sets the information on the spec as well as the environment variables for // using a TTY func WithTTY(width, height int) SpecOpts { - return func(_ context.Context, _ *Client, _ *containers.Container, s *specs.Spec) error { + return func(_ context.Context, _ Client, _ *containers.Container, s *specs.Spec) error { s.Process.Terminal = true if s.Process.ConsoleSize == nil { s.Process.ConsoleSize = &specs.Box{} @@ -66,10 +61,10 @@ func WithTTY(width, height int) SpecOpts { } } -// WithResources sets the provided resources on the spec for task updates -func WithResources(resources *specs.WindowsResources) UpdateTaskOpts { - return func(ctx context.Context, client *Client, r *UpdateTaskInfo) error { - r.Resources = resources +// WithUsername sets the username on the process +func WithUsername(username string) SpecOpts { + return func(ctx context.Context, client Client, c *containers.Container, s *specs.Spec) error { + s.Process.User.Username = username return nil } } diff --git a/vendor/github.com/containerd/containerd/spec_unix.go b/vendor/github.com/containerd/containerd/oci/spec_unix.go similarity index 79% rename from vendor/github.com/containerd/containerd/spec_unix.go rename to vendor/github.com/containerd/containerd/oci/spec_unix.go index 00e814962..c8f3b37af 100644 --- a/vendor/github.com/containerd/containerd/spec_unix.go +++ b/vendor/github.com/containerd/containerd/oci/spec_unix.go @@ -1,17 +1,11 @@ // +build !windows -package containerd +package oci import ( "context" - "io/ioutil" - "os" "path/filepath" - "syscall" - "golang.org/x/sys/unix" - - "github.com/containerd/containerd/mount" "github.com/containerd/containerd/namespaces" specs "github.com/opencontainers/runtime-spec/specs-go" ) @@ -173,32 +167,3 @@ func createDefaultSpec(ctx context.Context, id string) (*specs.Spec, error) { } return s, nil } - -func remapRootFS(mounts []mount.Mount, uid, gid uint32) error { - root, err := ioutil.TempDir("", "ctd-remap") - if err != nil { - return err - } - defer os.RemoveAll(root) - for _, m := range mounts { - if err := m.Mount(root); err != nil { - return err - } - } - defer unix.Unmount(root, 0) - return filepath.Walk(root, incrementFS(root, uid, gid)) -} - -func incrementFS(root string, uidInc, gidInc uint32) filepath.WalkFunc { - return func(path string, info os.FileInfo, err error) error { - if err != nil { - return err - } - var ( - stat = info.Sys().(*syscall.Stat_t) - u, g = int(stat.Uid + uidInc), int(stat.Gid + gidInc) - ) - // be sure the lchown the path as to not de-reference the symlink to a host file - return os.Lchown(path, u, g) - } -} diff --git a/vendor/github.com/containerd/containerd/spec_windows.go b/vendor/github.com/containerd/containerd/oci/spec_windows.go similarity index 96% rename from vendor/github.com/containerd/containerd/spec_windows.go rename to vendor/github.com/containerd/containerd/oci/spec_windows.go index 16a58b446..64c228883 100644 --- a/vendor/github.com/containerd/containerd/spec_windows.go +++ b/vendor/github.com/containerd/containerd/oci/spec_windows.go @@ -1,4 +1,4 @@ -package containerd +package oci import ( "context" diff --git a/vendor/github.com/containerd/containerd/plugin/plugin.go b/vendor/github.com/containerd/containerd/plugin/plugin.go index 9bda46cbf..5746bf72d 100644 --- a/vendor/github.com/containerd/containerd/plugin/plugin.go +++ b/vendor/github.com/containerd/containerd/plugin/plugin.go @@ -54,6 +54,8 @@ const ( MetadataPlugin Type = "io.containerd.metadata.v1" // ContentPlugin implements a content store ContentPlugin Type = "io.containerd.content.v1" + // GCPlugin implements garbage collection policy + GCPlugin Type = "io.containerd.gc.v1" ) // Registration contains information for registering a plugin diff --git a/vendor/github.com/containerd/containerd/plugin/plugin_go18.go b/vendor/github.com/containerd/containerd/plugin/plugin_go18.go index d9101246e..eee0e3fdb 100644 --- a/vendor/github.com/containerd/containerd/plugin/plugin_go18.go +++ b/vendor/github.com/containerd/containerd/plugin/plugin_go18.go @@ -1,4 +1,4 @@ -// +build go1.8,!windows,amd64 +// +build go1.8,!windows,amd64,!static_build package plugin diff --git a/vendor/github.com/containerd/containerd/plugin/plugin_other.go b/vendor/github.com/containerd/containerd/plugin/plugin_other.go index 21a457040..180917af8 100644 --- a/vendor/github.com/containerd/containerd/plugin/plugin_other.go +++ b/vendor/github.com/containerd/containerd/plugin/plugin_other.go @@ -1,4 +1,4 @@ -// +build !go1.8 windows !amd64 +// +build !go1.8 windows !amd64 static_build package plugin diff --git a/vendor/github.com/containerd/containerd/remotes/handlers.go b/vendor/github.com/containerd/containerd/remotes/handlers.go index e583391d8..ad4cd9f31 100644 --- a/vendor/github.com/containerd/containerd/remotes/handlers.go +++ b/vendor/github.com/containerd/containerd/remotes/handlers.go @@ -114,6 +114,7 @@ func fetch(ctx context.Context, ingester content.Ingester, fetcher Fetcher, desc func commitOpts(desc ocispec.Descriptor, r io.Reader) (io.Reader, []content.Opt) { var childrenF func(r io.Reader) ([]ocispec.Descriptor, error) + // TODO(AkihiroSuda): use images/oci.GetChildrenDescriptors? switch desc.MediaType { case images.MediaTypeDockerSchema2Manifest, ocispec.MediaTypeImageManifest: childrenF = func(r io.Reader) ([]ocispec.Descriptor, error) { diff --git a/vendor/github.com/containerd/containerd/rootfs/apply.go b/vendor/github.com/containerd/containerd/rootfs/apply.go index a198c99f9..405129564 100644 --- a/vendor/github.com/containerd/containerd/rootfs/apply.go +++ b/vendor/github.com/containerd/containerd/rootfs/apply.go @@ -9,7 +9,7 @@ import ( "github.com/containerd/containerd/diff" "github.com/containerd/containerd/errdefs" "github.com/containerd/containerd/log" - "github.com/containerd/containerd/snapshot" + "github.com/containerd/containerd/snapshots" "github.com/opencontainers/go-digest" "github.com/opencontainers/image-spec/identity" ocispec "github.com/opencontainers/image-spec/specs-go/v1" @@ -30,7 +30,7 @@ type Layer struct { // The returned result is a chain id digest representing all the applied layers. // Layers are applied in order they are given, making the first layer the // bottom-most layer in the layer chain. -func ApplyLayers(ctx context.Context, layers []Layer, sn snapshot.Snapshotter, a diff.Differ) (digest.Digest, error) { +func ApplyLayers(ctx context.Context, layers []Layer, sn snapshots.Snapshotter, a diff.Differ) (digest.Digest, error) { var chain []digest.Digest for _, layer := range layers { if _, err := ApplyLayer(ctx, layer, chain, sn, a); err != nil { @@ -46,7 +46,7 @@ func ApplyLayers(ctx context.Context, layers []Layer, sn snapshot.Snapshotter, a // ApplyLayer applies a single layer on top of the given provided layer chain, // using the provided snapshotter and applier. If the layer was unpacked true // is returned, if the layer already exists false is returned. -func ApplyLayer(ctx context.Context, layer Layer, chain []digest.Digest, sn snapshot.Snapshotter, a diff.Differ, opts ...snapshot.Opt) (bool, error) { +func ApplyLayer(ctx context.Context, layer Layer, chain []digest.Digest, sn snapshots.Snapshotter, a diff.Differ, opts ...snapshots.Opt) (bool, error) { var ( parent = identity.ChainID(chain) chainID = identity.ChainID(append(chain, layer.Diff.Digest)) @@ -55,10 +55,10 @@ func ApplyLayer(ctx context.Context, layer Layer, chain []digest.Digest, sn snap _, err := sn.Stat(ctx, chainID.String()) if err == nil { - log.G(ctx).Debugf("Extraction not needed, layer snapshot exists") + log.G(ctx).Debugf("Extraction not needed, layer snapshot %s exists", chainID) return false, nil } else if !errdefs.IsNotFound(err) { - return false, errors.Wrap(err, "failed to stat snapshot") + return false, errors.Wrapf(err, "failed to stat snapshot %s", chainID) } key := fmt.Sprintf("extract-%s %s", uniquePart(), chainID) @@ -67,7 +67,7 @@ func ApplyLayer(ctx context.Context, layer Layer, chain []digest.Digest, sn snap mounts, err := sn.Prepare(ctx, key, parent.String(), opts...) if err != nil { //TODO: If is snapshot exists error, retry - return false, errors.Wrap(err, "failed to prepare extraction layer") + return false, errors.Wrapf(err, "failed to prepare extraction snapshot %q", key) } defer func() { if err != nil { @@ -89,7 +89,7 @@ func ApplyLayer(ctx context.Context, layer Layer, chain []digest.Digest, sn snap if err = sn.Commit(ctx, chainID.String(), key, opts...); err != nil { if !errdefs.IsAlreadyExists(err) { - return false, errors.Wrapf(err, "failed to commit snapshot %s", parent) + return false, errors.Wrapf(err, "failed to commit snapshot %s", key) } // Destination already exists, cleanup key and return without error diff --git a/vendor/github.com/containerd/containerd/rootfs/diff.go b/vendor/github.com/containerd/containerd/rootfs/diff.go index 035eb3026..bab7a3cca 100644 --- a/vendor/github.com/containerd/containerd/rootfs/diff.go +++ b/vendor/github.com/containerd/containerd/rootfs/diff.go @@ -5,7 +5,7 @@ import ( "github.com/containerd/containerd/diff" "github.com/containerd/containerd/mount" - "github.com/containerd/containerd/snapshot" + "github.com/containerd/containerd/snapshots" ocispec "github.com/opencontainers/image-spec/specs-go/v1" "golang.org/x/net/context" ) @@ -14,7 +14,7 @@ import ( // of the snapshot. A content ref is provided to track the progress of the // content creation and the provided snapshotter and mount differ are used // for calculating the diff. The descriptor for the layer diff is returned. -func Diff(ctx context.Context, snapshotID string, sn snapshot.Snapshotter, d diff.Differ, opts ...diff.Opt) (ocispec.Descriptor, error) { +func Diff(ctx context.Context, snapshotID string, sn snapshots.Snapshotter, d diff.Differ, opts ...diff.Opt) (ocispec.Descriptor, error) { info, err := sn.Stat(ctx, snapshotID) if err != nil { return ocispec.Descriptor{}, err @@ -28,7 +28,7 @@ func Diff(ctx context.Context, snapshotID string, sn snapshot.Snapshotter, d dif defer sn.Remove(ctx, lowerKey) var upper []mount.Mount - if info.Kind == snapshot.KindActive { + if info.Kind == snapshots.KindActive { upper, err = sn.Mounts(ctx, snapshotID) if err != nil { return ocispec.Descriptor{}, err diff --git a/vendor/github.com/containerd/containerd/rootfs/init.go b/vendor/github.com/containerd/containerd/rootfs/init.go index 271e6cee5..4f32f11ae 100644 --- a/vendor/github.com/containerd/containerd/rootfs/init.go +++ b/vendor/github.com/containerd/containerd/rootfs/init.go @@ -8,7 +8,7 @@ import ( "github.com/containerd/containerd/log" "github.com/containerd/containerd/mount" - "github.com/containerd/containerd/snapshot" + "github.com/containerd/containerd/snapshots" digest "github.com/opencontainers/go-digest" "github.com/pkg/errors" ) @@ -26,7 +26,7 @@ type Mounter interface { } // InitRootFS initializes the snapshot for use as a rootfs -func InitRootFS(ctx context.Context, name string, parent digest.Digest, readonly bool, snapshotter snapshot.Snapshotter, mounter Mounter) ([]mount.Mount, error) { +func InitRootFS(ctx context.Context, name string, parent digest.Digest, readonly bool, snapshotter snapshots.Snapshotter, mounter Mounter) ([]mount.Mount, error) { _, err := snapshotter.Stat(ctx, name) if err == nil { return nil, errors.Errorf("rootfs already exists") @@ -51,7 +51,7 @@ func InitRootFS(ctx context.Context, name string, parent digest.Digest, readonly return snapshotter.Prepare(ctx, name, parentS) } -func createInitLayer(ctx context.Context, parent, initName string, initFn func(string) error, snapshotter snapshot.Snapshotter, mounter Mounter) (string, error) { +func createInitLayer(ctx context.Context, parent, initName string, initFn func(string) error, snapshotter snapshots.Snapshotter, mounter Mounter) (string, error) { initS := fmt.Sprintf("%s %s", parent, initName) if _, err := snapshotter.Stat(ctx, initS); err == nil { return initS, nil @@ -69,12 +69,12 @@ func createInitLayer(ctx context.Context, parent, initName string, initFn func(s if err != nil { return "", err } + defer func() { if err != nil { - // TODO: once implemented uncomment - //if rerr := snapshotter.Remove(ctx, td); rerr != nil { - // log.G(ctx).Errorf("Failed to remove snapshot %s: %v", td, merr) - //} + if rerr := snapshotter.Remove(ctx, td); rerr != nil { + log.G(ctx).Errorf("Failed to remove snapshot %s: %v", td, rerr) + } } }() diff --git a/vendor/github.com/containerd/containerd/snapshot.go b/vendor/github.com/containerd/containerd/snapshot.go index e29d21ca3..85bdba1b6 100644 --- a/vendor/github.com/containerd/containerd/snapshot.go +++ b/vendor/github.com/containerd/containerd/snapshot.go @@ -4,17 +4,17 @@ import ( "context" "io" - snapshotapi "github.com/containerd/containerd/api/services/snapshot/v1" + snapshotsapi "github.com/containerd/containerd/api/services/snapshots/v1" "github.com/containerd/containerd/api/types" "github.com/containerd/containerd/errdefs" "github.com/containerd/containerd/mount" - "github.com/containerd/containerd/snapshot" + "github.com/containerd/containerd/snapshots" protobuftypes "github.com/gogo/protobuf/types" ) // NewSnapshotterFromClient returns a new Snapshotter which communicates // over a GRPC connection. -func NewSnapshotterFromClient(client snapshotapi.SnapshotsClient, snapshotterName string) snapshot.Snapshotter { +func NewSnapshotterFromClient(client snapshotsapi.SnapshotsClient, snapshotterName string) snapshots.Snapshotter { return &remoteSnapshotter{ client: client, snapshotterName: snapshotterName, @@ -22,25 +22,25 @@ func NewSnapshotterFromClient(client snapshotapi.SnapshotsClient, snapshotterNam } type remoteSnapshotter struct { - client snapshotapi.SnapshotsClient + client snapshotsapi.SnapshotsClient snapshotterName string } -func (r *remoteSnapshotter) Stat(ctx context.Context, key string) (snapshot.Info, error) { +func (r *remoteSnapshotter) Stat(ctx context.Context, key string) (snapshots.Info, error) { resp, err := r.client.Stat(ctx, - &snapshotapi.StatSnapshotRequest{ + &snapshotsapi.StatSnapshotRequest{ Snapshotter: r.snapshotterName, Key: key, }) if err != nil { - return snapshot.Info{}, errdefs.FromGRPC(err) + return snapshots.Info{}, errdefs.FromGRPC(err) } return toInfo(resp.Info), nil } -func (r *remoteSnapshotter) Update(ctx context.Context, info snapshot.Info, fieldpaths ...string) (snapshot.Info, error) { +func (r *remoteSnapshotter) Update(ctx context.Context, info snapshots.Info, fieldpaths ...string) (snapshots.Info, error) { resp, err := r.client.Update(ctx, - &snapshotapi.UpdateSnapshotRequest{ + &snapshotsapi.UpdateSnapshotRequest{ Snapshotter: r.snapshotterName, Info: fromInfo(info), UpdateMask: &protobuftypes.FieldMask{ @@ -48,24 +48,24 @@ func (r *remoteSnapshotter) Update(ctx context.Context, info snapshot.Info, fiel }, }) if err != nil { - return snapshot.Info{}, errdefs.FromGRPC(err) + return snapshots.Info{}, errdefs.FromGRPC(err) } return toInfo(resp.Info), nil } -func (r *remoteSnapshotter) Usage(ctx context.Context, key string) (snapshot.Usage, error) { - resp, err := r.client.Usage(ctx, &snapshotapi.UsageRequest{ +func (r *remoteSnapshotter) Usage(ctx context.Context, key string) (snapshots.Usage, error) { + resp, err := r.client.Usage(ctx, &snapshotsapi.UsageRequest{ Snapshotter: r.snapshotterName, Key: key, }) if err != nil { - return snapshot.Usage{}, errdefs.FromGRPC(err) + return snapshots.Usage{}, errdefs.FromGRPC(err) } return toUsage(resp), nil } func (r *remoteSnapshotter) Mounts(ctx context.Context, key string) ([]mount.Mount, error) { - resp, err := r.client.Mounts(ctx, &snapshotapi.MountsRequest{ + resp, err := r.client.Mounts(ctx, &snapshotsapi.MountsRequest{ Snapshotter: r.snapshotterName, Key: key, }) @@ -75,14 +75,14 @@ func (r *remoteSnapshotter) Mounts(ctx context.Context, key string) ([]mount.Mou return toMounts(resp.Mounts), nil } -func (r *remoteSnapshotter) Prepare(ctx context.Context, key, parent string, opts ...snapshot.Opt) ([]mount.Mount, error) { - var local snapshot.Info +func (r *remoteSnapshotter) Prepare(ctx context.Context, key, parent string, opts ...snapshots.Opt) ([]mount.Mount, error) { + var local snapshots.Info for _, opt := range opts { if err := opt(&local); err != nil { return nil, err } } - resp, err := r.client.Prepare(ctx, &snapshotapi.PrepareSnapshotRequest{ + resp, err := r.client.Prepare(ctx, &snapshotsapi.PrepareSnapshotRequest{ Snapshotter: r.snapshotterName, Key: key, Parent: parent, @@ -94,14 +94,14 @@ func (r *remoteSnapshotter) Prepare(ctx context.Context, key, parent string, opt return toMounts(resp.Mounts), nil } -func (r *remoteSnapshotter) View(ctx context.Context, key, parent string, opts ...snapshot.Opt) ([]mount.Mount, error) { - var local snapshot.Info +func (r *remoteSnapshotter) View(ctx context.Context, key, parent string, opts ...snapshots.Opt) ([]mount.Mount, error) { + var local snapshots.Info for _, opt := range opts { if err := opt(&local); err != nil { return nil, err } } - resp, err := r.client.View(ctx, &snapshotapi.ViewSnapshotRequest{ + resp, err := r.client.View(ctx, &snapshotsapi.ViewSnapshotRequest{ Snapshotter: r.snapshotterName, Key: key, Parent: parent, @@ -113,14 +113,14 @@ func (r *remoteSnapshotter) View(ctx context.Context, key, parent string, opts . return toMounts(resp.Mounts), nil } -func (r *remoteSnapshotter) Commit(ctx context.Context, name, key string, opts ...snapshot.Opt) error { - var local snapshot.Info +func (r *remoteSnapshotter) Commit(ctx context.Context, name, key string, opts ...snapshots.Opt) error { + var local snapshots.Info for _, opt := range opts { if err := opt(&local); err != nil { return err } } - _, err := r.client.Commit(ctx, &snapshotapi.CommitSnapshotRequest{ + _, err := r.client.Commit(ctx, &snapshotsapi.CommitSnapshotRequest{ Snapshotter: r.snapshotterName, Name: name, Key: key, @@ -130,15 +130,15 @@ func (r *remoteSnapshotter) Commit(ctx context.Context, name, key string, opts . } func (r *remoteSnapshotter) Remove(ctx context.Context, key string) error { - _, err := r.client.Remove(ctx, &snapshotapi.RemoveSnapshotRequest{ + _, err := r.client.Remove(ctx, &snapshotsapi.RemoveSnapshotRequest{ Snapshotter: r.snapshotterName, Key: key, }) return errdefs.FromGRPC(err) } -func (r *remoteSnapshotter) Walk(ctx context.Context, fn func(context.Context, snapshot.Info) error) error { - sc, err := r.client.List(ctx, &snapshotapi.ListSnapshotsRequest{ +func (r *remoteSnapshotter) Walk(ctx context.Context, fn func(context.Context, snapshots.Info) error) error { + sc, err := r.client.List(ctx, &snapshotsapi.ListSnapshotsRequest{ Snapshotter: r.snapshotterName, }) if err != nil { @@ -167,18 +167,18 @@ func (r *remoteSnapshotter) Close() error { return nil } -func toKind(kind snapshotapi.Kind) snapshot.Kind { - if kind == snapshotapi.KindActive { - return snapshot.KindActive +func toKind(kind snapshotsapi.Kind) snapshots.Kind { + if kind == snapshotsapi.KindActive { + return snapshots.KindActive } - if kind == snapshotapi.KindView { - return snapshot.KindView + if kind == snapshotsapi.KindView { + return snapshots.KindView } - return snapshot.KindCommitted + return snapshots.KindCommitted } -func toInfo(info snapshotapi.Info) snapshot.Info { - return snapshot.Info{ +func toInfo(info snapshotsapi.Info) snapshots.Info { + return snapshots.Info{ Name: info.Name, Parent: info.Parent, Kind: toKind(info.Kind), @@ -188,8 +188,8 @@ func toInfo(info snapshotapi.Info) snapshot.Info { } } -func toUsage(resp *snapshotapi.UsageResponse) snapshot.Usage { - return snapshot.Usage{ +func toUsage(resp *snapshotsapi.UsageResponse) snapshots.Usage { + return snapshots.Usage{ Inodes: resp.Inodes, Size: resp.Size_, } @@ -207,18 +207,18 @@ func toMounts(mm []*types.Mount) []mount.Mount { return mounts } -func fromKind(kind snapshot.Kind) snapshotapi.Kind { - if kind == snapshot.KindActive { - return snapshotapi.KindActive +func fromKind(kind snapshots.Kind) snapshotsapi.Kind { + if kind == snapshots.KindActive { + return snapshotsapi.KindActive } - if kind == snapshot.KindView { - return snapshotapi.KindView + if kind == snapshots.KindView { + return snapshotsapi.KindView } - return snapshotapi.KindCommitted + return snapshotsapi.KindCommitted } -func fromInfo(info snapshot.Info) snapshotapi.Info { - return snapshotapi.Info{ +func fromInfo(info snapshots.Info) snapshotsapi.Info { + return snapshotsapi.Info{ Name: info.Name, Parent: info.Parent, Kind: fromKind(info.Kind), diff --git a/vendor/github.com/containerd/containerd/snapshot/naive/naive.go b/vendor/github.com/containerd/containerd/snapshots/naive/naive.go similarity index 79% rename from vendor/github.com/containerd/containerd/snapshot/naive/naive.go rename to vendor/github.com/containerd/containerd/snapshots/naive/naive.go index bcd4c35fd..e6e13cc63 100644 --- a/vendor/github.com/containerd/containerd/snapshot/naive/naive.go +++ b/vendor/github.com/containerd/containerd/snapshots/naive/naive.go @@ -11,8 +11,8 @@ import ( "github.com/containerd/containerd/mount" "github.com/containerd/containerd/platforms" "github.com/containerd/containerd/plugin" - "github.com/containerd/containerd/snapshot" - "github.com/containerd/containerd/snapshot/storage" + "github.com/containerd/containerd/snapshots" + "github.com/containerd/containerd/snapshots/storage" "github.com/pkg/errors" ) @@ -34,7 +34,7 @@ type snapshotter struct { // NewSnapshotter returns a Snapshotter which copies layers on the underlying // file system. A metadata file is stored under the root. -func NewSnapshotter(root string) (snapshot.Snapshotter, error) { +func NewSnapshotter(root string) (snapshots.Snapshotter, error) { if err := os.MkdirAll(root, 0700); err != nil { return nil, err } @@ -58,68 +58,68 @@ func NewSnapshotter(root string) (snapshot.Snapshotter, error) { // // Should be used for parent resolution, existence checks and to discern // the kind of snapshot. -func (o *snapshotter) Stat(ctx context.Context, key string) (snapshot.Info, error) { +func (o *snapshotter) Stat(ctx context.Context, key string) (snapshots.Info, error) { ctx, t, err := o.ms.TransactionContext(ctx, false) if err != nil { - return snapshot.Info{}, err + return snapshots.Info{}, err } defer t.Rollback() _, info, _, err := storage.GetInfo(ctx, key) if err != nil { - return snapshot.Info{}, err + return snapshots.Info{}, err } return info, nil } -func (o *snapshotter) Update(ctx context.Context, info snapshot.Info, fieldpaths ...string) (snapshot.Info, error) { +func (o *snapshotter) Update(ctx context.Context, info snapshots.Info, fieldpaths ...string) (snapshots.Info, error) { ctx, t, err := o.ms.TransactionContext(ctx, true) if err != nil { - return snapshot.Info{}, err + return snapshots.Info{}, err } info, err = storage.UpdateInfo(ctx, info, fieldpaths...) if err != nil { t.Rollback() - return snapshot.Info{}, err + return snapshots.Info{}, err } if err := t.Commit(); err != nil { - return snapshot.Info{}, err + return snapshots.Info{}, err } return info, nil } -func (o *snapshotter) Usage(ctx context.Context, key string) (snapshot.Usage, error) { +func (o *snapshotter) Usage(ctx context.Context, key string) (snapshots.Usage, error) { ctx, t, err := o.ms.TransactionContext(ctx, false) if err != nil { - return snapshot.Usage{}, err + return snapshots.Usage{}, err } defer t.Rollback() id, info, usage, err := storage.GetInfo(ctx, key) if err != nil { - return snapshot.Usage{}, err + return snapshots.Usage{}, err } - if info.Kind == snapshot.KindActive { + if info.Kind == snapshots.KindActive { du, err := fs.DiskUsage(o.getSnapshotDir(id)) if err != nil { - return snapshot.Usage{}, err + return snapshots.Usage{}, err } - usage = snapshot.Usage(du) + usage = snapshots.Usage(du) } return usage, nil } -func (o *snapshotter) Prepare(ctx context.Context, key, parent string, opts ...snapshot.Opt) ([]mount.Mount, error) { - return o.createSnapshot(ctx, snapshot.KindActive, key, parent, opts) +func (o *snapshotter) Prepare(ctx context.Context, key, parent string, opts ...snapshots.Opt) ([]mount.Mount, error) { + return o.createSnapshot(ctx, snapshots.KindActive, key, parent, opts) } -func (o *snapshotter) View(ctx context.Context, key, parent string, opts ...snapshot.Opt) ([]mount.Mount, error) { - return o.createSnapshot(ctx, snapshot.KindView, key, parent, opts) +func (o *snapshotter) View(ctx context.Context, key, parent string, opts ...snapshots.Opt) ([]mount.Mount, error) { + return o.createSnapshot(ctx, snapshots.KindView, key, parent, opts) } // Mounts returns the mounts for the transaction identified by key. Can be @@ -139,7 +139,7 @@ func (o *snapshotter) Mounts(ctx context.Context, key string) ([]mount.Mount, er return o.mounts(s), nil } -func (o *snapshotter) Commit(ctx context.Context, name, key string, opts ...snapshot.Opt) error { +func (o *snapshotter) Commit(ctx context.Context, name, key string, opts ...snapshots.Opt) error { ctx, t, err := o.ms.TransactionContext(ctx, true) if err != nil { return err @@ -155,9 +155,9 @@ func (o *snapshotter) Commit(ctx context.Context, name, key string, opts ...snap return err } - if _, err := storage.CommitActive(ctx, key, name, snapshot.Usage(usage), opts...); err != nil { + if _, err := storage.CommitActive(ctx, key, name, snapshots.Usage(usage), opts...); err != nil { if rerr := t.Rollback(); rerr != nil { - log.G(ctx).WithError(rerr).Warn("Failure rolling back transaction") + log.G(ctx).WithError(rerr).Warn("failed to rollback transaction") } return errors.Wrap(err, "failed to commit snapshot") } @@ -174,7 +174,7 @@ func (o *snapshotter) Remove(ctx context.Context, key string) (err error) { defer func() { if err != nil && t != nil { if rerr := t.Rollback(); rerr != nil { - log.G(ctx).WithError(rerr).Warn("Failure rolling back transaction") + log.G(ctx).WithError(rerr).Warn("failed to rollback transaction") } } }() @@ -199,7 +199,7 @@ func (o *snapshotter) Remove(ctx context.Context, key string) (err error) { if renamed != "" { if err1 := os.Rename(renamed, path); err1 != nil { // May cause inconsistent data on disk - log.G(ctx).WithError(err1).WithField("path", renamed).Errorf("Failed to rename after failed commit") + log.G(ctx).WithError(err1).WithField("path", renamed).Errorf("failed to rename after failed commit") } } return errors.Wrap(err, "failed to commit") @@ -207,7 +207,7 @@ func (o *snapshotter) Remove(ctx context.Context, key string) (err error) { if renamed != "" { if err := os.RemoveAll(renamed); err != nil { // Must be cleaned up, any "rm-*" could be removed if no active transactions - log.G(ctx).WithError(err).WithField("path", renamed).Warnf("Failed to remove root filesystem") + log.G(ctx).WithError(err).WithField("path", renamed).Warnf("failed to remove root filesystem") } } @@ -215,7 +215,7 @@ func (o *snapshotter) Remove(ctx context.Context, key string) (err error) { } // Walk the committed snapshots. -func (o *snapshotter) Walk(ctx context.Context, fn func(context.Context, snapshot.Info) error) error { +func (o *snapshotter) Walk(ctx context.Context, fn func(context.Context, snapshots.Info) error) error { ctx, t, err := o.ms.TransactionContext(ctx, false) if err != nil { return err @@ -224,13 +224,13 @@ func (o *snapshotter) Walk(ctx context.Context, fn func(context.Context, snapsho return storage.WalkInfo(ctx, fn) } -func (o *snapshotter) createSnapshot(ctx context.Context, kind snapshot.Kind, key, parent string, opts []snapshot.Opt) ([]mount.Mount, error) { +func (o *snapshotter) createSnapshot(ctx context.Context, kind snapshots.Kind, key, parent string, opts []snapshots.Opt) ([]mount.Mount, error) { var ( err error path, td string ) - if kind == snapshot.KindActive || parent == "" { + if kind == snapshots.KindActive || parent == "" { td, err = ioutil.TempDir(filepath.Join(o.root, "snapshots"), "new-") if err != nil { return nil, errors.Wrap(err, "failed to create temp dir") @@ -259,7 +259,7 @@ func (o *snapshotter) createSnapshot(ctx context.Context, kind snapshot.Kind, ke s, err := storage.CreateSnapshot(ctx, kind, key, parent, opts...) if err != nil { if rerr := t.Rollback(); rerr != nil { - log.G(ctx).WithError(rerr).Warn("Failure rolling back transaction") + log.G(ctx).WithError(rerr).Warn("failed to rollback transaction") } return nil, errors.Wrap(err, "failed to create snapshot") } @@ -275,7 +275,7 @@ func (o *snapshotter) createSnapshot(ctx context.Context, kind snapshot.Kind, ke path = o.getSnapshotDir(s.ID) if err := os.Rename(td, path); err != nil { if rerr := t.Rollback(); rerr != nil { - log.G(ctx).WithError(rerr).Warn("Failure rolling back transaction") + log.G(ctx).WithError(rerr).Warn("failed to rollback transaction") } return nil, errors.Wrap(err, "failed to rename") } @@ -299,13 +299,13 @@ func (o *snapshotter) mounts(s storage.Snapshot) []mount.Mount { source string ) - if s.Kind == snapshot.KindView { + if s.Kind == snapshots.KindView { roFlag = "ro" } else { roFlag = "rw" } - if len(s.ParentIDs) == 0 || s.Kind == snapshot.KindActive { + if len(s.ParentIDs) == 0 || s.Kind == snapshots.KindActive { source = o.getSnapshotDir(s.ID) } else { source = o.getSnapshotDir(s.ParentIDs[0]) diff --git a/vendor/github.com/containerd/containerd/snapshot/overlay/overlay.go b/vendor/github.com/containerd/containerd/snapshots/overlay/overlay.go similarity index 80% rename from vendor/github.com/containerd/containerd/snapshot/overlay/overlay.go rename to vendor/github.com/containerd/containerd/snapshots/overlay/overlay.go index 33a1edd92..a37f412d1 100644 --- a/vendor/github.com/containerd/containerd/snapshot/overlay/overlay.go +++ b/vendor/github.com/containerd/containerd/snapshots/overlay/overlay.go @@ -16,8 +16,8 @@ import ( "github.com/containerd/containerd/mount" "github.com/containerd/containerd/platforms" "github.com/containerd/containerd/plugin" - "github.com/containerd/containerd/snapshot" - "github.com/containerd/containerd/snapshot/storage" + "github.com/containerd/containerd/snapshots" + "github.com/containerd/containerd/snapshots/storage" "github.com/pkg/errors" ) @@ -41,7 +41,7 @@ type snapshotter struct { // NewSnapshotter returns a Snapshotter which uses overlayfs. The overlayfs // diffs are stored under the provided root. A metadata file is stored under // the root. -func NewSnapshotter(root string) (snapshot.Snapshotter, error) { +func NewSnapshotter(root string) (snapshots.Snapshotter, error) { if err := os.MkdirAll(root, 0700); err != nil { return nil, err } @@ -72,34 +72,34 @@ func NewSnapshotter(root string) (snapshot.Snapshotter, error) { // // Should be used for parent resolution, existence checks and to discern // the kind of snapshot. -func (o *snapshotter) Stat(ctx context.Context, key string) (snapshot.Info, error) { +func (o *snapshotter) Stat(ctx context.Context, key string) (snapshots.Info, error) { ctx, t, err := o.ms.TransactionContext(ctx, false) if err != nil { - return snapshot.Info{}, err + return snapshots.Info{}, err } defer t.Rollback() _, info, _, err := storage.GetInfo(ctx, key) if err != nil { - return snapshot.Info{}, err + return snapshots.Info{}, err } return info, nil } -func (o *snapshotter) Update(ctx context.Context, info snapshot.Info, fieldpaths ...string) (snapshot.Info, error) { +func (o *snapshotter) Update(ctx context.Context, info snapshots.Info, fieldpaths ...string) (snapshots.Info, error) { ctx, t, err := o.ms.TransactionContext(ctx, true) if err != nil { - return snapshot.Info{}, err + return snapshots.Info{}, err } info, err = storage.UpdateInfo(ctx, info, fieldpaths...) if err != nil { t.Rollback() - return snapshot.Info{}, err + return snapshots.Info{}, err } if err := t.Commit(); err != nil { - return snapshot.Info{}, err + return snapshots.Info{}, err } return info, nil @@ -111,39 +111,39 @@ func (o *snapshotter) Update(ctx context.Context, info snapshot.Info, fieldpaths // "upper") directory and may take some time. // // For committed snapshots, the value is returned from the metadata database. -func (o *snapshotter) Usage(ctx context.Context, key string) (snapshot.Usage, error) { +func (o *snapshotter) Usage(ctx context.Context, key string) (snapshots.Usage, error) { ctx, t, err := o.ms.TransactionContext(ctx, false) if err != nil { - return snapshot.Usage{}, err + return snapshots.Usage{}, err } id, info, usage, err := storage.GetInfo(ctx, key) t.Rollback() // transaction no longer needed at this point. if err != nil { - return snapshot.Usage{}, err + return snapshots.Usage{}, err } upperPath := o.upperPath(id) - if info.Kind == snapshot.KindActive { + if info.Kind == snapshots.KindActive { du, err := fs.DiskUsage(upperPath) if err != nil { // TODO(stevvooe): Consider not reporting an error in this case. - return snapshot.Usage{}, err + return snapshots.Usage{}, err } - usage = snapshot.Usage(du) + usage = snapshots.Usage(du) } return usage, nil } -func (o *snapshotter) Prepare(ctx context.Context, key, parent string, opts ...snapshot.Opt) ([]mount.Mount, error) { - return o.createSnapshot(ctx, snapshot.KindActive, key, parent, opts) +func (o *snapshotter) Prepare(ctx context.Context, key, parent string, opts ...snapshots.Opt) ([]mount.Mount, error) { + return o.createSnapshot(ctx, snapshots.KindActive, key, parent, opts) } -func (o *snapshotter) View(ctx context.Context, key, parent string, opts ...snapshot.Opt) ([]mount.Mount, error) { - return o.createSnapshot(ctx, snapshot.KindView, key, parent, opts) +func (o *snapshotter) View(ctx context.Context, key, parent string, opts ...snapshots.Opt) ([]mount.Mount, error) { + return o.createSnapshot(ctx, snapshots.KindView, key, parent, opts) } // Mounts returns the mounts for the transaction identified by key. Can be @@ -163,7 +163,7 @@ func (o *snapshotter) Mounts(ctx context.Context, key string) ([]mount.Mount, er return o.mounts(s), nil } -func (o *snapshotter) Commit(ctx context.Context, name, key string, opts ...snapshot.Opt) error { +func (o *snapshotter) Commit(ctx context.Context, name, key string, opts ...snapshots.Opt) error { ctx, t, err := o.ms.TransactionContext(ctx, true) if err != nil { return err @@ -172,7 +172,7 @@ func (o *snapshotter) Commit(ctx context.Context, name, key string, opts ...snap defer func() { if err != nil { if rerr := t.Rollback(); rerr != nil { - log.G(ctx).WithError(rerr).Warn("Failure rolling back transaction") + log.G(ctx).WithError(rerr).Warn("failed to rollback transaction") } } }() @@ -188,7 +188,7 @@ func (o *snapshotter) Commit(ctx context.Context, name, key string, opts ...snap return err } - if _, err = storage.CommitActive(ctx, key, name, snapshot.Usage(usage), opts...); err != nil { + if _, err = storage.CommitActive(ctx, key, name, snapshots.Usage(usage), opts...); err != nil { return errors.Wrap(err, "failed to commit snapshot") } return t.Commit() @@ -204,7 +204,7 @@ func (o *snapshotter) Remove(ctx context.Context, key string) (err error) { defer func() { if err != nil && t != nil { if rerr := t.Rollback(); rerr != nil { - log.G(ctx).WithError(rerr).Warn("Failure rolling back transaction") + log.G(ctx).WithError(rerr).Warn("failed to rollback transaction") } } }() @@ -225,20 +225,20 @@ func (o *snapshotter) Remove(ctx context.Context, key string) (err error) { if err != nil { if err1 := os.Rename(renamed, path); err1 != nil { // May cause inconsistent data on disk - log.G(ctx).WithError(err1).WithField("path", renamed).Errorf("Failed to rename after failed commit") + log.G(ctx).WithError(err1).WithField("path", renamed).Errorf("failed to rename after failed commit") } return errors.Wrap(err, "failed to commit") } if err := os.RemoveAll(renamed); err != nil { // Must be cleaned up, any "rm-*" could be removed if no active transactions - log.G(ctx).WithError(err).WithField("path", renamed).Warnf("Failed to remove root filesystem") + log.G(ctx).WithError(err).WithField("path", renamed).Warnf("failed to remove root filesystem") } return nil } // Walk the committed snapshots. -func (o *snapshotter) Walk(ctx context.Context, fn func(context.Context, snapshot.Info) error) error { +func (o *snapshotter) Walk(ctx context.Context, fn func(context.Context, snapshots.Info) error) error { ctx, t, err := o.ms.TransactionContext(ctx, false) if err != nil { return err @@ -247,7 +247,7 @@ func (o *snapshotter) Walk(ctx context.Context, fn func(context.Context, snapsho return storage.WalkInfo(ctx, fn) } -func (o *snapshotter) createSnapshot(ctx context.Context, kind snapshot.Kind, key, parent string, opts []snapshot.Opt) ([]mount.Mount, error) { +func (o *snapshotter) createSnapshot(ctx context.Context, kind snapshots.Kind, key, parent string, opts []snapshots.Opt) ([]mount.Mount, error) { var ( path string snapshotDir = filepath.Join(o.root, "snapshots") @@ -277,7 +277,7 @@ func (o *snapshotter) createSnapshot(ctx context.Context, kind snapshot.Kind, ke return nil, err } - if kind == snapshot.KindActive { + if kind == snapshots.KindActive { if err = os.MkdirAll(filepath.Join(td, "work"), 0711); err != nil { return nil, err } @@ -287,43 +287,39 @@ func (o *snapshotter) createSnapshot(ctx context.Context, kind snapshot.Kind, ke if err != nil { return nil, err } + rollback := true + defer func() { + if rollback { + if rerr := t.Rollback(); rerr != nil { + log.G(ctx).WithError(rerr).Warn("failed to rollback transaction") + } + } + }() s, err := storage.CreateSnapshot(ctx, kind, key, parent, opts...) if err != nil { - if rerr := t.Rollback(); rerr != nil { - log.G(ctx).WithError(rerr).Warn("Failure rolling back transaction") - } - return nil, errors.Wrap(err, "failed to create active") + return nil, errors.Wrap(err, "failed to create snapshot") } if len(s.ParentIDs) > 0 { - st, err := os.Stat(filepath.Join(o.upperPath(s.ParentIDs[0]))) + st, err := os.Stat(o.upperPath(s.ParentIDs[0])) if err != nil { - if rerr := t.Rollback(); rerr != nil { - log.G(ctx).WithError(rerr).Warn("Failure rolling back transaction") - } return nil, errors.Wrap(err, "failed to stat parent") } stat := st.Sys().(*syscall.Stat_t) - if err := os.Lchown(fs, int(stat.Uid), int(stat.Gid)); err != nil { - if rerr := t.Rollback(); rerr != nil { - log.G(ctx).WithError(rerr).Warn("Failure rolling back transaction") - } return nil, errors.Wrap(err, "failed to chown") } } path = filepath.Join(snapshotDir, s.ID) if err = os.Rename(td, path); err != nil { - if rerr := t.Rollback(); rerr != nil { - log.G(ctx).WithError(rerr).Warn("Failure rolling back transaction") - } return nil, errors.Wrap(err, "failed to rename") } td = "" + rollback = false if err = t.Commit(); err != nil { return nil, errors.Wrap(err, "commit failed") } @@ -336,7 +332,7 @@ func (o *snapshotter) mounts(s storage.Snapshot) []mount.Mount { // if we only have one layer/no parents then just return a bind mount as overlay // will not work roFlag := "rw" - if s.Kind == snapshot.KindView { + if s.Kind == snapshots.KindView { roFlag = "ro" } @@ -353,7 +349,7 @@ func (o *snapshotter) mounts(s storage.Snapshot) []mount.Mount { } var options []string - if s.Kind == snapshot.KindActive { + if s.Kind == snapshots.KindActive { options = append(options, fmt.Sprintf("workdir=%s", o.workPath(s.ID)), fmt.Sprintf("upperdir=%s", o.upperPath(s.ID)), diff --git a/vendor/github.com/containerd/containerd/snapshot/snapshotter.go b/vendor/github.com/containerd/containerd/snapshots/snapshotter.go similarity index 99% rename from vendor/github.com/containerd/containerd/snapshot/snapshotter.go rename to vendor/github.com/containerd/containerd/snapshots/snapshotter.go index a7a33cc57..cde4c7261 100644 --- a/vendor/github.com/containerd/containerd/snapshot/snapshotter.go +++ b/vendor/github.com/containerd/containerd/snapshots/snapshotter.go @@ -1,4 +1,4 @@ -package snapshot +package snapshots import ( "context" diff --git a/vendor/github.com/containerd/containerd/snapshot/storage/bolt.go b/vendor/github.com/containerd/containerd/snapshots/storage/bolt.go similarity index 83% rename from vendor/github.com/containerd/containerd/snapshot/storage/bolt.go rename to vendor/github.com/containerd/containerd/snapshots/storage/bolt.go index 26c394c06..d690fd098 100644 --- a/vendor/github.com/containerd/containerd/snapshot/storage/bolt.go +++ b/vendor/github.com/containerd/containerd/snapshots/storage/bolt.go @@ -10,7 +10,7 @@ import ( "github.com/boltdb/bolt" "github.com/containerd/containerd/errdefs" "github.com/containerd/containerd/metadata/boltutil" - "github.com/containerd/containerd/snapshot" + "github.com/containerd/containerd/snapshots" "github.com/pkg/errors" ) @@ -56,11 +56,11 @@ func getParentPrefix(b []byte) uint64 { // GetInfo returns the snapshot Info directly from the metadata. Requires a // context with a storage transaction. -func GetInfo(ctx context.Context, key string) (string, snapshot.Info, snapshot.Usage, error) { +func GetInfo(ctx context.Context, key string) (string, snapshots.Info, snapshots.Usage, error) { var ( id uint64 - su snapshot.Usage - si = snapshot.Info{ + su snapshots.Usage + si = snapshots.Info{ Name: key, } ) @@ -69,15 +69,15 @@ func GetInfo(ctx context.Context, key string) (string, snapshot.Info, snapshot.U return readSnapshot(bkt, &id, &si) }) if err != nil { - return "", snapshot.Info{}, snapshot.Usage{}, err + return "", snapshots.Info{}, snapshots.Usage{}, err } return fmt.Sprintf("%d", id), si, su, nil } // UpdateInfo updates an existing snapshot info's data -func UpdateInfo(ctx context.Context, info snapshot.Info, fieldpaths ...string) (snapshot.Info, error) { - updated := snapshot.Info{ +func UpdateInfo(ctx context.Context, info snapshots.Info, fieldpaths ...string) (snapshots.Info, error) { + updated := snapshots.Info{ Name: info.Name, } err := withBucket(ctx, func(ctx context.Context, bkt, pbkt *bolt.Bucket) error { @@ -120,7 +120,7 @@ func UpdateInfo(ctx context.Context, info snapshot.Info, fieldpaths ...string) ( return boltutil.WriteLabels(sbkt, updated.Labels) }) if err != nil { - return snapshot.Info{}, err + return snapshots.Info{}, err } return updated, nil } @@ -128,7 +128,7 @@ func UpdateInfo(ctx context.Context, info snapshot.Info, fieldpaths ...string) ( // WalkInfo iterates through all metadata Info for the stored snapshots and // calls the provided function for each. Requires a context with a storage // transaction. -func WalkInfo(ctx context.Context, fn func(context.Context, snapshot.Info) error) error { +func WalkInfo(ctx context.Context, fn func(context.Context, snapshots.Info) error) error { return withBucket(ctx, func(ctx context.Context, bkt, pbkt *bolt.Bucket) error { return bkt.ForEach(func(k, v []byte) error { // skip non buckets @@ -137,7 +137,7 @@ func WalkInfo(ctx context.Context, fn func(context.Context, snapshot.Info) error } var ( sbkt = bkt.Bucket(k) - si = snapshot.Info{ + si = snapshots.Info{ Name: string(k), } ) @@ -162,7 +162,7 @@ func GetSnapshot(ctx context.Context, key string) (s Snapshot, err error) { s.ID = fmt.Sprintf("%d", readID(sbkt)) s.Kind = readKind(sbkt) - if s.Kind != snapshot.KindActive && s.Kind != snapshot.KindView { + if s.Kind != snapshots.KindActive && s.Kind != snapshots.KindView { return errors.Wrapf(errdefs.ErrFailedPrecondition, "requested snapshot %v not active or view", key) } @@ -187,13 +187,13 @@ func GetSnapshot(ctx context.Context, key string) (s Snapshot, err error) { } // CreateSnapshot inserts a record for an active or view snapshot with the provided parent. -func CreateSnapshot(ctx context.Context, kind snapshot.Kind, key, parent string, opts ...snapshot.Opt) (s Snapshot, err error) { +func CreateSnapshot(ctx context.Context, kind snapshots.Kind, key, parent string, opts ...snapshots.Opt) (s Snapshot, err error) { switch kind { - case snapshot.KindActive, snapshot.KindView: + case snapshots.KindActive, snapshots.KindView: default: return Snapshot{}, errors.Wrapf(errdefs.ErrInvalidArgument, "snapshot type %v invalid; only snapshots of type Active or View can be created", kind) } - var base snapshot.Info + var base snapshots.Info for _, opt := range opts { if err := opt(&base); err != nil { return Snapshot{}, err @@ -207,11 +207,11 @@ func CreateSnapshot(ctx context.Context, kind snapshot.Kind, key, parent string, if parent != "" { spbkt = bkt.Bucket([]byte(parent)) if spbkt == nil { - return errors.Wrap(errdefs.ErrNotFound, "missing parent bucket") + return errors.Wrapf(errdefs.ErrNotFound, "missing parent %q bucket", parent) } - if readKind(spbkt) != snapshot.KindCommitted { - return errors.Wrap(errdefs.ErrInvalidArgument, "parent is not committed snapshot") + if readKind(spbkt) != snapshots.KindCommitted { + return errors.Wrapf(errdefs.ErrInvalidArgument, "parent %q is not committed snapshot", parent) } } sbkt, err := bkt.CreateBucket([]byte(key)) @@ -224,11 +224,11 @@ func CreateSnapshot(ctx context.Context, kind snapshot.Kind, key, parent string, id, err := bkt.NextSequence() if err != nil { - return errors.Wrap(err, "unable to get identifier") + return errors.Wrapf(err, "unable to get identifier for snapshot %q", key) } t := time.Now().UTC() - si := snapshot.Info{ + si := snapshots.Info{ Parent: parent, Kind: kind, Labels: base.Labels, @@ -245,12 +245,12 @@ func CreateSnapshot(ctx context.Context, kind snapshot.Kind, key, parent string, // Store a backlink from the key to the parent. Store the snapshot name // as the value to allow following the backlink to the snapshot value. if err := pbkt.Put(parentKey(pid, id), []byte(key)); err != nil { - return errors.Wrap(err, "failed to write parent link") + return errors.Wrapf(err, "failed to write parent link for snapshot %q", key) } s.ParentIDs, err = parents(bkt, spbkt, pid) if err != nil { - return errors.Wrap(err, "failed to get parent chain") + return errors.Wrapf(err, "failed to get parent chain for snapshot %q", key) } } @@ -268,10 +268,10 @@ func CreateSnapshot(ctx context.Context, kind snapshot.Kind, key, parent string, // Remove removes a snapshot from the metastore. The string identifier for the // snapshot is returned as well as the kind. The provided context must contain a // writable transaction. -func Remove(ctx context.Context, key string) (string, snapshot.Kind, error) { +func Remove(ctx context.Context, key string) (string, snapshots.Kind, error) { var ( id uint64 - si snapshot.Info + si snapshots.Info ) if err := withBucket(ctx, func(ctx context.Context, bkt, pbkt *bolt.Bucket) error { @@ -320,10 +320,10 @@ func Remove(ctx context.Context, key string) (string, snapshot.Kind, error) { // lookup or removal. The returned string identifier for the committed snapshot // is the same identifier of the original active snapshot. The provided context // must contain a writable transaction. -func CommitActive(ctx context.Context, key, name string, usage snapshot.Usage, opts ...snapshot.Opt) (string, error) { +func CommitActive(ctx context.Context, key, name string, usage snapshots.Usage, opts ...snapshots.Opt) (string, error) { var ( id uint64 - base snapshot.Info + base snapshots.Info ) for _, opt := range opts { if err := opt(&base); err != nil { @@ -341,18 +341,18 @@ func CommitActive(ctx context.Context, key, name string, usage snapshot.Usage, o } sbkt := bkt.Bucket([]byte(key)) if sbkt == nil { - return errors.Wrap(errdefs.ErrNotFound, "failed to get active snapshot") + return errors.Wrapf(errdefs.ErrNotFound, "failed to get active snapshot %q", key) } - var si snapshot.Info + var si snapshots.Info if err := readSnapshot(sbkt, &id, &si); err != nil { - return errors.Wrap(err, "failed to read snapshot") + return errors.Wrapf(err, "failed to read active snapshot %q", key) } - if si.Kind != snapshot.KindActive { - return errors.Wrapf(errdefs.ErrFailedPrecondition, "snapshot %v is not active", name) + if si.Kind != snapshots.KindActive { + return errors.Wrapf(errdefs.ErrFailedPrecondition, "snapshot %q is not active", key) } - si.Kind = snapshot.KindCommitted + si.Kind = snapshots.KindCommitted si.Created = time.Now().UTC() si.Updated = si.Created @@ -366,18 +366,18 @@ func CommitActive(ctx context.Context, key, name string, usage snapshot.Usage, o return err } if err := bkt.DeleteBucket([]byte(key)); err != nil { - return errors.Wrap(err, "failed to delete active") + return errors.Wrapf(err, "failed to delete active snapshot %q", key) } if si.Parent != "" { spbkt := bkt.Bucket([]byte(si.Parent)) if spbkt == nil { - return errors.Wrap(errdefs.ErrNotFound, "missing parent") + return errors.Wrapf(errdefs.ErrNotFound, "missing parent %q of snapshot %q", si.Parent, key) } pid := readID(spbkt) // Updates parent back link to use new key if err := pbkt.Put(parentKey(pid, id), []byte(name)); err != nil { - return errors.Wrap(err, "failed to update parent link") + return errors.Wrapf(err, "failed to update parent link %q from %q to %q", pid, key, name) } } @@ -394,11 +394,11 @@ func withSnapshotBucket(ctx context.Context, key string, fn func(context.Context if !ok { return ErrNoTransaction } - bkt := tx.Bucket(bucketKeyStorageVersion) - if bkt == nil { + vbkt := tx.Bucket(bucketKeyStorageVersion) + if vbkt == nil { return errors.Wrap(errdefs.ErrNotFound, "bucket does not exist") } - bkt = bkt.Bucket(bucketKeySnapshot) + bkt := vbkt.Bucket(bucketKeySnapshot) if bkt == nil { return errors.Wrap(errdefs.ErrNotFound, "snapshots bucket does not exist") } @@ -407,7 +407,7 @@ func withSnapshotBucket(ctx context.Context, key string, fn func(context.Context return errors.Wrap(errdefs.ErrNotFound, "snapshot does not exist") } - return fn(ctx, bkt, bkt.Bucket(bucketKeyParents)) + return fn(ctx, bkt, vbkt.Bucket(bucketKeyParents)) } func withBucket(ctx context.Context, fn func(context.Context, *bolt.Bucket, *bolt.Bucket) error) error { @@ -438,7 +438,7 @@ func createBucketIfNotExists(ctx context.Context, fn func(context.Context, *bolt } pbkt, err := bkt.CreateBucketIfNotExists(bucketKeyParents) if err != nil { - return errors.Wrap(err, "failed to create snapshots bucket") + return errors.Wrap(err, "failed to create parents bucket") } return fn(ctx, sbkt, pbkt) } @@ -460,10 +460,10 @@ func parents(bkt, pbkt *bolt.Bucket, parent uint64) (parents []string, err error } } -func readKind(bkt *bolt.Bucket) (k snapshot.Kind) { +func readKind(bkt *bolt.Bucket) (k snapshots.Kind) { kind := bkt.Get(bucketKeyKind) if len(kind) == 1 { - k = snapshot.Kind(kind[0]) + k = snapshots.Kind(kind[0]) } return } @@ -473,7 +473,7 @@ func readID(bkt *bolt.Bucket) uint64 { return id } -func readSnapshot(bkt *bolt.Bucket, id *uint64, si *snapshot.Info) error { +func readSnapshot(bkt *bolt.Bucket, id *uint64, si *snapshots.Info) error { if id != nil { *id = readID(bkt) } @@ -495,7 +495,7 @@ func readSnapshot(bkt *bolt.Bucket, id *uint64, si *snapshot.Info) error { return nil } -func putSnapshot(bkt *bolt.Bucket, id uint64, si snapshot.Info) error { +func putSnapshot(bkt *bolt.Bucket, id uint64, si snapshots.Info) error { idEncoded, err := encodeID(id) if err != nil { return err @@ -519,12 +519,12 @@ func putSnapshot(bkt *bolt.Bucket, id uint64, si snapshot.Info) error { return boltutil.WriteLabels(bkt, si.Labels) } -func getUsage(bkt *bolt.Bucket, usage *snapshot.Usage) { +func getUsage(bkt *bolt.Bucket, usage *snapshots.Usage) { usage.Inodes, _ = binary.Varint(bkt.Get(bucketKeyInodes)) usage.Size, _ = binary.Varint(bkt.Get(bucketKeySize)) } -func putUsage(bkt *bolt.Bucket, usage snapshot.Usage) error { +func putUsage(bkt *bolt.Bucket, usage snapshots.Usage) error { for _, v := range []struct { key []byte value int64 diff --git a/vendor/github.com/containerd/containerd/snapshot/storage/metastore.go b/vendor/github.com/containerd/containerd/snapshots/storage/metastore.go similarity index 97% rename from vendor/github.com/containerd/containerd/snapshot/storage/metastore.go rename to vendor/github.com/containerd/containerd/snapshots/storage/metastore.go index a8d18868c..c6c49f263 100644 --- a/vendor/github.com/containerd/containerd/snapshot/storage/metastore.go +++ b/vendor/github.com/containerd/containerd/snapshots/storage/metastore.go @@ -10,7 +10,7 @@ import ( "sync" "github.com/boltdb/bolt" - "github.com/containerd/containerd/snapshot" + "github.com/containerd/containerd/snapshots" "github.com/pkg/errors" ) @@ -35,7 +35,7 @@ type Transactor interface { // the last index. The last index should always be considered the active // snapshots immediate parent. type Snapshot struct { - Kind snapshot.Kind + Kind snapshots.Kind ID string ParentIDs []string } diff --git a/vendor/github.com/containerd/containerd/spec_opts.go b/vendor/github.com/containerd/containerd/spec_opts.go deleted file mode 100644 index 2dbf8214a..000000000 --- a/vendor/github.com/containerd/containerd/spec_opts.go +++ /dev/null @@ -1,74 +0,0 @@ -package containerd - -import ( - "context" - - "github.com/containerd/containerd/containers" - "github.com/containerd/typeurl" - specs "github.com/opencontainers/runtime-spec/specs-go" -) - -// SpecOpts sets spec specific information to a newly generated OCI spec -type SpecOpts func(context.Context, *Client, *containers.Container, *specs.Spec) error - -// WithProcessArgs replaces the args on the generated spec -func WithProcessArgs(args ...string) SpecOpts { - return func(_ context.Context, _ *Client, _ *containers.Container, s *specs.Spec) error { - s.Process.Args = args - return nil - } -} - -// WithProcessCwd replaces the current working directory on the generated spec -func WithProcessCwd(cwd string) SpecOpts { - return func(_ context.Context, _ *Client, _ *containers.Container, s *specs.Spec) error { - s.Process.Cwd = cwd - return nil - } -} - -// WithHostname sets the container's hostname -func WithHostname(name string) SpecOpts { - return func(_ context.Context, _ *Client, _ *containers.Container, s *specs.Spec) error { - s.Hostname = name - return nil - } -} - -// WithNewSpec generates a new spec for a new container -func WithNewSpec(opts ...SpecOpts) NewContainerOpts { - return func(ctx context.Context, client *Client, c *containers.Container) error { - s, err := createDefaultSpec(ctx, c.ID) - if err != nil { - return err - } - for _, o := range opts { - if err := o(ctx, client, c, s); err != nil { - return err - } - } - any, err := typeurl.MarshalAny(s) - if err != nil { - return err - } - c.Spec = any - return nil - } -} - -// WithSpec sets the provided spec on the container -func WithSpec(s *specs.Spec, opts ...SpecOpts) NewContainerOpts { - return func(ctx context.Context, client *Client, c *containers.Container) error { - for _, o := range opts { - if err := o(ctx, client, c, s); err != nil { - return err - } - } - any, err := typeurl.MarshalAny(s) - if err != nil { - return err - } - c.Spec = any - return nil - } -} diff --git a/vendor/github.com/containerd/containerd/task.go b/vendor/github.com/containerd/containerd/task.go index c69579764..8d256834f 100644 --- a/vendor/github.com/containerd/containerd/task.go +++ b/vendor/github.com/containerd/containerd/task.go @@ -277,7 +277,7 @@ func (t *task) Delete(ctx context.Context, opts ...ProcessDeleteOpts) (*ExitStat return &ExitStatus{code: r.ExitStatus, exitedAt: r.ExitedAt}, nil } -func (t *task) Exec(ctx context.Context, id string, spec *specs.Process, ioCreate cio.Creation) (Process, error) { +func (t *task) Exec(ctx context.Context, id string, spec *specs.Process, ioCreate cio.Creation) (_ Process, err error) { if id == "" { return nil, errors.Wrapf(errdefs.ErrInvalidArgument, "exec id must not be empty") } @@ -285,6 +285,12 @@ func (t *task) Exec(ctx context.Context, id string, spec *specs.Process, ioCreat if err != nil { return nil, err } + defer func() { + if err != nil && i != nil { + i.Cancel() + i.Close() + } + }() any, err := typeurl.MarshalAny(spec) if err != nil { return nil, err @@ -356,7 +362,7 @@ func (t *task) Resize(ctx context.Context, w, h uint32) error { } func (t *task) Checkpoint(ctx context.Context, opts ...CheckpointTaskOpts) (Image, error) { - ctx, done, err := t.client.withLease(ctx) + ctx, done, err := t.client.WithLease(ctx) if err != nil { return nil, err } diff --git a/vendor/github.com/containerd/containerd/task_opts_linux.go b/vendor/github.com/containerd/containerd/task_opts_linux.go new file mode 100644 index 000000000..5b91cb548 --- /dev/null +++ b/vendor/github.com/containerd/containerd/task_opts_linux.go @@ -0,0 +1,15 @@ +package containerd + +import ( + "context" + + "github.com/opencontainers/runtime-spec/specs-go" +) + +// WithResources sets the provided resources for task updates +func WithResources(resources *specs.LinuxResources) UpdateTaskOpts { + return func(ctx context.Context, client *Client, r *UpdateTaskInfo) error { + r.Resources = resources + return nil + } +} diff --git a/vendor/github.com/containerd/containerd/task_opts_windows.go b/vendor/github.com/containerd/containerd/task_opts_windows.go new file mode 100644 index 000000000..d77402c67 --- /dev/null +++ b/vendor/github.com/containerd/containerd/task_opts_windows.go @@ -0,0 +1,15 @@ +package containerd + +import ( + "context" + + specs "github.com/opencontainers/runtime-spec/specs-go" +) + +// WithResources sets the provided resources on the spec for task updates +func WithResources(resources *specs.WindowsResources) UpdateTaskOpts { + return func(ctx context.Context, client *Client, r *UpdateTaskInfo) error { + r.Resources = resources + return nil + } +} diff --git a/vendor/github.com/containerd/containerd/vendor.conf b/vendor/github.com/containerd/containerd/vendor.conf index 6b8144421..fea47e480 100644 --- a/vendor/github.com/containerd/containerd/vendor.conf +++ b/vendor/github.com/containerd/containerd/vendor.conf @@ -1,7 +1,7 @@ github.com/coreos/go-systemd 48702e0da86bd25e76cfef347e2adeb434a0d0a6 github.com/containerd/go-runc ed1cbe1fc31f5fb2359d3a54b6330d1a097858b7 github.com/containerd/console 84eeaae905fa414d03e07bcd6c8d3f19e7cf180e -github.com/containerd/cgroups f7dd103d3e4e696aa67152f6b4ddd1779a3455a9 +github.com/containerd/cgroups 29da22c6171a4316169f9205ab6c49f59b5b852f github.com/containerd/typeurl f6943554a7e7e88b3c14aad190bf05932da84788 github.com/docker/go-metrics 8fd5772bf1584597834c6f7961a530f06cbfbb87 github.com/docker/go-events 9461782956ad83b30282bf90e31fa6a70c255ba9 @@ -35,9 +35,10 @@ golang.org/x/sync 450f422ab23cf9881c94e2db30cac0eb1b7cf80c github.com/BurntSushi/toml v0.2.0-21-g9906417 github.com/grpc-ecosystem/go-grpc-prometheus 6b7015e65d366bf3f19b2b2a000a831940f0f7e0 github.com/Microsoft/go-winio v0.4.4 -github.com/Microsoft/hcsshim v0.6.3 +github.com/Microsoft/hcsshim v0.6.7 github.com/Microsoft/opengcs v0.3.2 github.com/boltdb/bolt e9cf4fae01b5a8ff89d0ec6b32f0d9c9f79aefdd google.golang.org/genproto d80a6e20e776b0b17a324d0ba1ab50a39c8e8944 golang.org/x/text 19e51611da83d6be54ddafce4a4af510cb3e9ea4 -github.com/dmcgowan/go-tar 2e2c51242e8993c50445dab7c03c8e7febddd0cf +github.com/dmcgowan/go-tar go1.10 +github.com/stevvooe/ttrpc 76e68349ad9ab4d03d764c713826d31216715e4f diff --git a/vendor/github.com/dmcgowan/go-tar/common.go b/vendor/github.com/dmcgowan/go-tar/common.go index d2ae66d55..e3609536c 100644 --- a/vendor/github.com/dmcgowan/go-tar/common.go +++ b/vendor/github.com/dmcgowan/go-tar/common.go @@ -3,20 +3,23 @@ // license that can be found in the LICENSE file. // Package tar implements access to tar archives. -// It aims to cover most of the variations, including those produced -// by GNU and BSD tars. // -// References: -// http://www.freebsd.org/cgi/man.cgi?query=tar&sektion=5 -// http://www.gnu.org/software/tar/manual/html_node/Standard.html -// http://pubs.opengroup.org/onlinepubs/9699919799/utilities/pax.html +// Tape archives (tar) are a file format for storing a sequence of files that +// can be read and written in a streaming manner. +// This package aims to cover most variations of the format, +// including those produced by GNU and BSD tar tools. package tar import ( "errors" "fmt" + "io" + "math" "os" "path" + "reflect" + "strconv" + "strings" "time" ) @@ -24,42 +27,569 @@ import ( // architectures. If a large value is encountered when decoding, the result // stored in Header will be the truncated version. -// Header type flags. -const ( - TypeReg = '0' // regular file - TypeRegA = '\x00' // regular file - TypeLink = '1' // hard link - TypeSymlink = '2' // symbolic link - TypeChar = '3' // character device node - TypeBlock = '4' // block device node - TypeDir = '5' // directory - TypeFifo = '6' // fifo node - TypeCont = '7' // reserved - TypeXHeader = 'x' // extended header - TypeXGlobalHeader = 'g' // global extended header - TypeGNULongName = 'L' // Next file has a long name - TypeGNULongLink = 'K' // Next file symlinks to a file w/ a long name - TypeGNUSparse = 'S' // sparse file +var ( + ErrHeader = errors.New("tar: invalid tar header") + ErrWriteTooLong = errors.New("tar: write too long") + ErrFieldTooLong = errors.New("tar: header field too long") + ErrWriteAfterClose = errors.New("tar: write after close") + errMissData = errors.New("tar: sparse file references non-existent data") + errUnrefData = errors.New("tar: sparse file contains unreferenced data") + errWriteHole = errors.New("tar: write non-NUL byte in sparse hole") ) +type headerError []string + +func (he headerError) Error() string { + const prefix = "tar: cannot encode header" + var ss []string + for _, s := range he { + if s != "" { + ss = append(ss, s) + } + } + if len(ss) == 0 { + return prefix + } + return fmt.Sprintf("%s: %v", prefix, strings.Join(ss, "; and ")) +} + +// Type flags for Header.Typeflag. +const ( + // Type '0' indicates a regular file. + TypeReg = '0' + TypeRegA = '\x00' // For legacy support; use TypeReg instead + + // Type '1' to '6' are header-only flags and may not have a data body. + TypeLink = '1' // Hard link + TypeSymlink = '2' // Symbolic link + TypeChar = '3' // Character device node + TypeBlock = '4' // Block device node + TypeDir = '5' // Directory + TypeFifo = '6' // FIFO node + + // Type '7' is reserved. + TypeCont = '7' + + // Type 'x' is used by the PAX format to store key-value records that + // are only relevant to the next file. + // This package transparently handles these types. + TypeXHeader = 'x' + + // Type 'g' is used by the PAX format to store key-value records that + // are relevant to all subsequent files. + // This package only supports parsing and composing such headers, + // but does not currently support persisting the global state across files. + TypeXGlobalHeader = 'g' + + // Type 'S' indicates a sparse file in the GNU format. + // Header.SparseHoles should be populated when using this type. + TypeGNUSparse = 'S' + + // Types 'L' and 'K' are used by the GNU format for a meta file + // used to store the path or link name for the next file. + // This package transparently handles these types. + TypeGNULongName = 'L' + TypeGNULongLink = 'K' +) + +// Keywords for PAX extended header records. +const ( + paxNone = "" // Indicates that no PAX key is suitable + paxPath = "path" + paxLinkpath = "linkpath" + paxSize = "size" + paxUid = "uid" + paxGid = "gid" + paxUname = "uname" + paxGname = "gname" + paxMtime = "mtime" + paxAtime = "atime" + paxCtime = "ctime" // Removed from later revision of PAX spec, but was valid + paxCharset = "charset" // Currently unused + paxComment = "comment" // Currently unused + + paxSchilyXattr = "SCHILY.xattr." + + // Keywords for GNU sparse files in a PAX extended header. + paxGNUSparse = "GNU.sparse." + paxGNUSparseNumBlocks = "GNU.sparse.numblocks" + paxGNUSparseOffset = "GNU.sparse.offset" + paxGNUSparseNumBytes = "GNU.sparse.numbytes" + paxGNUSparseMap = "GNU.sparse.map" + paxGNUSparseName = "GNU.sparse.name" + paxGNUSparseMajor = "GNU.sparse.major" + paxGNUSparseMinor = "GNU.sparse.minor" + paxGNUSparseSize = "GNU.sparse.size" + paxGNUSparseRealSize = "GNU.sparse.realsize" +) + +// basicKeys is a set of the PAX keys for which we have built-in support. +// This does not contain "charset" or "comment", which are both PAX-specific, +// so adding them as first-class features of Header is unlikely. +// Users can use the PAXRecords field to set it themselves. +var basicKeys = map[string]bool{ + paxPath: true, paxLinkpath: true, paxSize: true, paxUid: true, paxGid: true, + paxUname: true, paxGname: true, paxMtime: true, paxAtime: true, paxCtime: true, +} + // A Header represents a single header in a tar archive. // Some fields may not be populated. +// +// For forward compatibility, users that retrieve a Header from Reader.Next, +// mutate it in some ways, and then pass it back to Writer.WriteHeader +// should do so by creating a new Header and copying the fields +// that they are interested in preserving. type Header struct { - Name string // name of header file entry - Mode int64 // permission and mode bits - Uid int // user id of owner - Gid int // group id of owner - Size int64 // length in bytes - ModTime time.Time // modified time - Typeflag byte // type of header entry - Linkname string // target name of link - Uname string // user name of owner - Gname string // group name of owner - Devmajor int64 // major number of character or block device - Devminor int64 // minor number of character or block device - AccessTime time.Time // access time - ChangeTime time.Time // status change time - Xattrs map[string]string + Typeflag byte // Type of header entry (should be TypeReg for most files) + + Name string // Name of file entry + Linkname string // Target name of link (valid for TypeLink or TypeSymlink) + + Size int64 // Logical file size in bytes + Mode int64 // Permission and mode bits + Uid int // User ID of owner + Gid int // Group ID of owner + Uname string // User name of owner + Gname string // Group name of owner + + // If the Format is unspecified, then Writer.WriteHeader rounds ModTime + // to the nearest second and ignores the AccessTime and ChangeTime fields. + // + // To use AccessTime or ChangeTime, specify the Format as PAX or GNU. + // To use sub-second resolution, specify the Format as PAX. + ModTime time.Time // Modification time + AccessTime time.Time // Access time (requires either PAX or GNU support) + ChangeTime time.Time // Change time (requires either PAX or GNU support) + + Devmajor int64 // Major device number (valid for TypeChar or TypeBlock) + Devminor int64 // Minor device number (valid for TypeChar or TypeBlock) + + // SparseHoles represents a sequence of holes in a sparse file. + // + // A file is sparse if len(SparseHoles) > 0 or Typeflag is TypeGNUSparse. + // If TypeGNUSparse is set, then the format is GNU, otherwise + // the format is PAX (by using GNU-specific PAX records). + // + // A sparse file consists of fragments of data, intermixed with holes + // (described by this field). A hole is semantically a block of NUL-bytes, + // but does not actually exist within the tar file. + // The holes must be sorted in ascending order, + // not overlap with each other, and not extend past the specified Size. + SparseHoles []SparseEntry + + // Xattrs stores extended attributes as PAX records under the + // "SCHILY.xattr." namespace. + // + // The following are semantically equivalent: + // h.Xattrs[key] = value + // h.PAXRecords["SCHILY.xattr."+key] = value + // + // When Writer.WriteHeader is called, the contents of Xattrs will take + // precedence over those in PAXRecords. + // + // Deprecated: Use PAXRecords instead. + Xattrs map[string]string + + // PAXRecords is a map of PAX extended header records. + // + // User-defined records should have keys of the following form: + // VENDOR.keyword + // Where VENDOR is some namespace in all uppercase, and keyword may + // not contain the '=' character (e.g., "GOLANG.pkg.version"). + // The key and value should be non-empty UTF-8 strings. + // + // When Writer.WriteHeader is called, PAX records derived from the + // the other fields in Header take precedence over PAXRecords. + PAXRecords map[string]string + + // Format specifies the format of the tar header. + // + // This is set by Reader.Next as a best-effort guess at the format. + // Since the Reader liberally reads some non-compliant files, + // it is possible for this to be FormatUnknown. + // + // If the format is unspecified when Writer.WriteHeader is called, + // then it uses the first format (in the order of USTAR, PAX, GNU) + // capable of encoding this Header (see Format). + Format Format +} + +// SparseEntry represents a Length-sized fragment at Offset in the file. +type SparseEntry struct{ Offset, Length int64 } + +func (s SparseEntry) endOffset() int64 { return s.Offset + s.Length } + +// A sparse file can be represented as either a sparseDatas or a sparseHoles. +// As long as the total size is known, they are equivalent and one can be +// converted to the other form and back. The various tar formats with sparse +// file support represent sparse files in the sparseDatas form. That is, they +// specify the fragments in the file that has data, and treat everything else as +// having zero bytes. As such, the encoding and decoding logic in this package +// deals with sparseDatas. +// +// However, the external API uses sparseHoles instead of sparseDatas because the +// zero value of sparseHoles logically represents a normal file (i.e., there are +// no holes in it). On the other hand, the zero value of sparseDatas implies +// that the file has no data in it, which is rather odd. +// +// As an example, if the underlying raw file contains the 10-byte data: +// var compactFile = "abcdefgh" +// +// And the sparse map has the following entries: +// var spd sparseDatas = []sparseEntry{ +// {Offset: 2, Length: 5}, // Data fragment for 2..6 +// {Offset: 18, Length: 3}, // Data fragment for 18..20 +// } +// var sph sparseHoles = []SparseEntry{ +// {Offset: 0, Length: 2}, // Hole fragment for 0..1 +// {Offset: 7, Length: 11}, // Hole fragment for 7..17 +// {Offset: 21, Length: 4}, // Hole fragment for 21..24 +// } +// +// Then the content of the resulting sparse file with a Header.Size of 25 is: +// var sparseFile = "\x00"*2 + "abcde" + "\x00"*11 + "fgh" + "\x00"*4 +type ( + sparseDatas []SparseEntry + sparseHoles []SparseEntry +) + +// validateSparseEntries reports whether sp is a valid sparse map. +// It does not matter whether sp represents data fragments or hole fragments. +func validateSparseEntries(sp []SparseEntry, size int64) bool { + // Validate all sparse entries. These are the same checks as performed by + // the BSD tar utility. + if size < 0 { + return false + } + var pre SparseEntry + for _, cur := range sp { + switch { + case cur.Offset < 0 || cur.Length < 0: + return false // Negative values are never okay + case cur.Offset > math.MaxInt64-cur.Length: + return false // Integer overflow with large length + case cur.endOffset() > size: + return false // Region extends beyond the actual size + case pre.endOffset() > cur.Offset: + return false // Regions cannot overlap and must be in order + } + pre = cur + } + return true +} + +// alignSparseEntries mutates src and returns dst where each fragment's +// starting offset is aligned up to the nearest block edge, and each +// ending offset is aligned down to the nearest block edge. +// +// Even though the Go tar Reader and the BSD tar utility can handle entries +// with arbitrary offsets and lengths, the GNU tar utility can only handle +// offsets and lengths that are multiples of blockSize. +func alignSparseEntries(src []SparseEntry, size int64) []SparseEntry { + dst := src[:0] + for _, s := range src { + pos, end := s.Offset, s.endOffset() + pos += blockPadding(+pos) // Round-up to nearest blockSize + if end != size { + end -= blockPadding(-end) // Round-down to nearest blockSize + } + if pos < end { + dst = append(dst, SparseEntry{Offset: pos, Length: end - pos}) + } + } + return dst +} + +// invertSparseEntries converts a sparse map from one form to the other. +// If the input is sparseHoles, then it will output sparseDatas and vice-versa. +// The input must have been already validated. +// +// This function mutates src and returns a normalized map where: +// * adjacent fragments are coalesced together +// * only the last fragment may be empty +// * the endOffset of the last fragment is the total size +func invertSparseEntries(src []SparseEntry, size int64) []SparseEntry { + dst := src[:0] + var pre SparseEntry + for _, cur := range src { + if cur.Length == 0 { + continue // Skip empty fragments + } + pre.Length = cur.Offset - pre.Offset + if pre.Length > 0 { + dst = append(dst, pre) // Only add non-empty fragments + } + pre.Offset = cur.endOffset() + } + pre.Length = size - pre.Offset // Possibly the only empty fragment + return append(dst, pre) +} + +// fileState tracks the number of logical (includes sparse holes) and physical +// (actual in tar archive) bytes remaining for the current file. +// +// Invariant: LogicalRemaining >= PhysicalRemaining +type fileState interface { + LogicalRemaining() int64 + PhysicalRemaining() int64 +} + +// allowedFormats determines which formats can be used. +// The value returned is the logical OR of multiple possible formats. +// If the value is FormatUnknown, then the input Header cannot be encoded +// and an error is returned explaining why. +// +// As a by-product of checking the fields, this function returns paxHdrs, which +// contain all fields that could not be directly encoded. +// A value receiver ensures that this method does not mutate the source Header. +func (h Header) allowedFormats() (format Format, paxHdrs map[string]string, err error) { + format = FormatUSTAR | FormatPAX | FormatGNU + paxHdrs = make(map[string]string) + + var whyNoUSTAR, whyNoPAX, whyNoGNU string + var preferPAX bool // Prefer PAX over USTAR + verifyString := func(s string, size int, name, paxKey string) { + // NUL-terminator is optional for path and linkpath. + // Technically, it is required for uname and gname, + // but neither GNU nor BSD tar checks for it. + tooLong := len(s) > size + allowLongGNU := paxKey == paxPath || paxKey == paxLinkpath + if hasNUL(s) || (tooLong && !allowLongGNU) { + whyNoGNU = fmt.Sprintf("GNU cannot encode %s=%q", name, s) + format.mustNotBe(FormatGNU) + } + if !isASCII(s) || tooLong { + canSplitUSTAR := paxKey == paxPath + if _, _, ok := splitUSTARPath(s); !canSplitUSTAR || !ok { + whyNoUSTAR = fmt.Sprintf("USTAR cannot encode %s=%q", name, s) + format.mustNotBe(FormatUSTAR) + } + if paxKey == paxNone { + whyNoPAX = fmt.Sprintf("PAX cannot encode %s=%q", name, s) + format.mustNotBe(FormatPAX) + } else { + paxHdrs[paxKey] = s + } + } + if v, ok := h.PAXRecords[paxKey]; ok && v == s { + paxHdrs[paxKey] = v + } + } + verifyNumeric := func(n int64, size int, name, paxKey string) { + if !fitsInBase256(size, n) { + whyNoGNU = fmt.Sprintf("GNU cannot encode %s=%d", name, n) + format.mustNotBe(FormatGNU) + } + if !fitsInOctal(size, n) { + whyNoUSTAR = fmt.Sprintf("USTAR cannot encode %s=%d", name, n) + format.mustNotBe(FormatUSTAR) + if paxKey == paxNone { + whyNoPAX = fmt.Sprintf("PAX cannot encode %s=%d", name, n) + format.mustNotBe(FormatPAX) + } else { + paxHdrs[paxKey] = strconv.FormatInt(n, 10) + } + } + if v, ok := h.PAXRecords[paxKey]; ok && v == strconv.FormatInt(n, 10) { + paxHdrs[paxKey] = v + } + } + verifyTime := func(ts time.Time, size int, name, paxKey string) { + if ts.IsZero() { + return // Always okay + } + if !fitsInBase256(size, ts.Unix()) { + whyNoGNU = fmt.Sprintf("GNU cannot encode %s=%v", name, ts) + format.mustNotBe(FormatGNU) + } + isMtime := paxKey == paxMtime + fitsOctal := fitsInOctal(size, ts.Unix()) + if (isMtime && !fitsOctal) || !isMtime { + whyNoUSTAR = fmt.Sprintf("USTAR cannot encode %s=%v", name, ts) + format.mustNotBe(FormatUSTAR) + } + needsNano := ts.Nanosecond() != 0 + if !isMtime || !fitsOctal || needsNano { + preferPAX = true // USTAR may truncate sub-second measurements + if paxKey == paxNone { + whyNoPAX = fmt.Sprintf("PAX cannot encode %s=%v", name, ts) + format.mustNotBe(FormatPAX) + } else { + paxHdrs[paxKey] = formatPAXTime(ts) + } + } + if v, ok := h.PAXRecords[paxKey]; ok && v == formatPAXTime(ts) { + paxHdrs[paxKey] = v + } + } + + // Check basic fields. + var blk block + v7 := blk.V7() + ustar := blk.USTAR() + gnu := blk.GNU() + verifyString(h.Name, len(v7.Name()), "Name", paxPath) + verifyString(h.Linkname, len(v7.LinkName()), "Linkname", paxLinkpath) + verifyString(h.Uname, len(ustar.UserName()), "Uname", paxUname) + verifyString(h.Gname, len(ustar.GroupName()), "Gname", paxGname) + verifyNumeric(h.Mode, len(v7.Mode()), "Mode", paxNone) + verifyNumeric(int64(h.Uid), len(v7.UID()), "Uid", paxUid) + verifyNumeric(int64(h.Gid), len(v7.GID()), "Gid", paxGid) + verifyNumeric(h.Size, len(v7.Size()), "Size", paxSize) + verifyNumeric(h.Devmajor, len(ustar.DevMajor()), "Devmajor", paxNone) + verifyNumeric(h.Devminor, len(ustar.DevMinor()), "Devminor", paxNone) + verifyTime(h.ModTime, len(v7.ModTime()), "ModTime", paxMtime) + verifyTime(h.AccessTime, len(gnu.AccessTime()), "AccessTime", paxAtime) + verifyTime(h.ChangeTime, len(gnu.ChangeTime()), "ChangeTime", paxCtime) + + // Check for header-only types. + var whyOnlyPAX, whyOnlyGNU string + switch h.Typeflag { + case TypeReg, TypeChar, TypeBlock, TypeFifo, TypeGNUSparse: + // Exclude TypeLink and TypeSymlink, since they may reference directories. + if strings.HasSuffix(h.Name, "/") { + return FormatUnknown, nil, headerError{"filename may not have trailing slash"} + } + case TypeXHeader, TypeGNULongName, TypeGNULongLink: + return FormatUnknown, nil, headerError{"cannot manually encode TypeXHeader, TypeGNULongName, or TypeGNULongLink headers"} + case TypeXGlobalHeader: + if !reflect.DeepEqual(h, Header{Typeflag: h.Typeflag, Xattrs: h.Xattrs, PAXRecords: h.PAXRecords, Format: h.Format}) { + return FormatUnknown, nil, headerError{"only PAXRecords may be set for TypeXGlobalHeader"} + } + whyOnlyPAX = "only PAX supports TypeXGlobalHeader" + format.mayOnlyBe(FormatPAX) + } + if !isHeaderOnlyType(h.Typeflag) && h.Size < 0 { + return FormatUnknown, nil, headerError{"negative size on header-only type"} + } + + // Check PAX records. + if len(h.Xattrs) > 0 { + for k, v := range h.Xattrs { + paxHdrs[paxSchilyXattr+k] = v + } + whyOnlyPAX = "only PAX supports Xattrs" + format.mayOnlyBe(FormatPAX) + } + if len(h.PAXRecords) > 0 { + for k, v := range h.PAXRecords { + switch _, exists := paxHdrs[k]; { + case exists: + continue // Do not overwrite existing records + case h.Typeflag == TypeXGlobalHeader: + paxHdrs[k] = v // Copy all records + case !basicKeys[k] && !strings.HasPrefix(k, paxGNUSparse): + paxHdrs[k] = v // Ignore local records that may conflict + } + } + whyOnlyPAX = "only PAX supports PAXRecords" + format.mayOnlyBe(FormatPAX) + } + for k, v := range paxHdrs { + if !validPAXRecord(k, v) { + return FormatUnknown, nil, headerError{fmt.Sprintf("invalid PAX record: %q", k+" = "+v)} + } + } + + // Check sparse files. + if len(h.SparseHoles) > 0 || h.Typeflag == TypeGNUSparse { + if isHeaderOnlyType(h.Typeflag) { + return FormatUnknown, nil, headerError{"header-only type cannot be sparse"} + } + if !validateSparseEntries(h.SparseHoles, h.Size) { + return FormatUnknown, nil, headerError{"invalid sparse holes"} + } + if h.Typeflag == TypeGNUSparse { + whyOnlyGNU = "only GNU supports TypeGNUSparse" + format.mayOnlyBe(FormatGNU) + } else { + whyNoGNU = "GNU supports sparse files only with TypeGNUSparse" + format.mustNotBe(FormatGNU) + } + whyNoUSTAR = "USTAR does not support sparse files" + format.mustNotBe(FormatUSTAR) + } + + // Check desired format. + if wantFormat := h.Format; wantFormat != FormatUnknown { + if wantFormat.has(FormatPAX) && !preferPAX { + wantFormat.mayBe(FormatUSTAR) // PAX implies USTAR allowed too + } + format.mayOnlyBe(wantFormat) // Set union of formats allowed and format wanted + } + if format == FormatUnknown { + switch h.Format { + case FormatUSTAR: + err = headerError{"Format specifies USTAR", whyNoUSTAR, whyOnlyPAX, whyOnlyGNU} + case FormatPAX: + err = headerError{"Format specifies PAX", whyNoPAX, whyOnlyGNU} + case FormatGNU: + err = headerError{"Format specifies GNU", whyNoGNU, whyOnlyPAX} + default: + err = headerError{whyNoUSTAR, whyNoPAX, whyNoGNU, whyOnlyPAX, whyOnlyGNU} + } + } + return format, paxHdrs, err +} + +var sysSparseDetect func(f *os.File) (sparseHoles, error) +var sysSparsePunch func(f *os.File, sph sparseHoles) error + +// DetectSparseHoles searches for holes within f to populate SparseHoles +// on supported operating systems and filesystems. +// The file offset is cleared to zero. +// +// When packing a sparse file, DetectSparseHoles should be called prior to +// serializing the header to the archive with Writer.WriteHeader. +func (h *Header) DetectSparseHoles(f *os.File) (err error) { + defer func() { + if _, serr := f.Seek(0, io.SeekStart); err == nil { + err = serr + } + }() + + h.SparseHoles = nil + if sysSparseDetect != nil { + sph, err := sysSparseDetect(f) + h.SparseHoles = sph + return err + } + return nil +} + +// PunchSparseHoles destroys the contents of f, and prepares a sparse file +// (on supported operating systems and filesystems) +// with holes punched according to SparseHoles. +// The file offset is cleared to zero. +// +// When extracting a sparse file, PunchSparseHoles should be called prior to +// populating the content of a file with Reader.WriteTo. +func (h *Header) PunchSparseHoles(f *os.File) (err error) { + defer func() { + if _, serr := f.Seek(0, io.SeekStart); err == nil { + err = serr + } + }() + + if err := f.Truncate(0); err != nil { + return err + } + + var size int64 + if len(h.SparseHoles) > 0 { + size = h.SparseHoles[len(h.SparseHoles)-1].endOffset() + } + if !validateSparseEntries(h.SparseHoles, size) { + return errors.New("tar: invalid sparse holes") + } + + if size == 0 { + return nil // For non-sparse files, do nothing (other than Truncate) + } + if sysSparsePunch != nil { + return sysSparsePunch(f, h.SparseHoles) + } + return f.Truncate(size) } // FileInfo returns an os.FileInfo for the Header. @@ -92,63 +622,43 @@ func (fi headerFileInfo) Mode() (mode os.FileMode) { // Set setuid, setgid and sticky bits. if fi.h.Mode&c_ISUID != 0 { - // setuid mode |= os.ModeSetuid } if fi.h.Mode&c_ISGID != 0 { - // setgid mode |= os.ModeSetgid } if fi.h.Mode&c_ISVTX != 0 { - // sticky mode |= os.ModeSticky } - // Set file mode bits. - // clear perm, setuid, setgid and sticky bits. - m := os.FileMode(fi.h.Mode) &^ 07777 - if m == c_ISDIR { - // directory + // Set file mode bits; clear perm, setuid, setgid, and sticky bits. + switch m := os.FileMode(fi.h.Mode) &^ 07777; m { + case c_ISDIR: mode |= os.ModeDir - } - if m == c_ISFIFO { - // named pipe (FIFO) + case c_ISFIFO: mode |= os.ModeNamedPipe - } - if m == c_ISLNK { - // symbolic link + case c_ISLNK: mode |= os.ModeSymlink - } - if m == c_ISBLK { - // device file + case c_ISBLK: mode |= os.ModeDevice - } - if m == c_ISCHR { - // Unix character device + case c_ISCHR: mode |= os.ModeDevice mode |= os.ModeCharDevice - } - if m == c_ISSOCK { - // Unix domain socket + case c_ISSOCK: mode |= os.ModeSocket } switch fi.h.Typeflag { case TypeSymlink: - // symbolic link mode |= os.ModeSymlink case TypeChar: - // character device node mode |= os.ModeDevice mode |= os.ModeCharDevice case TypeBlock: - // block device node mode |= os.ModeDevice case TypeDir: - // directory mode |= os.ModeDir case TypeFifo: - // fifo node mode |= os.ModeNamedPipe } @@ -158,11 +668,15 @@ func (fi headerFileInfo) Mode() (mode os.FileMode) { // sysStat, if non-nil, populates h from system-dependent fields of fi. var sysStat func(fi os.FileInfo, h *Header) error -// Mode constants from the tar spec. const ( - c_ISUID = 04000 // Set uid - c_ISGID = 02000 // Set gid - c_ISVTX = 01000 // Save text (sticky bit) + // Mode constants from the USTAR spec: + // See http://pubs.opengroup.org/onlinepubs/9699919799/utilities/pax.html#tag_20_92_13_06 + c_ISUID = 04000 // Set uid + c_ISGID = 02000 // Set gid + c_ISVTX = 01000 // Save text (sticky bit) + + // Common Unix mode constants; these are not defined in any common tar standard. + // Header.FileInfo understands these, but FileInfoHeader will never produce these. c_ISDIR = 040000 // Directory c_ISFIFO = 010000 // FIFO c_ISREG = 0100000 // Regular file @@ -172,30 +686,16 @@ const ( c_ISSOCK = 0140000 // Socket ) -// Keywords for the PAX Extended Header -const ( - paxAtime = "atime" - paxCharset = "charset" - paxComment = "comment" - paxCtime = "ctime" // please note that ctime is not a valid pax header. - paxGid = "gid" - paxGname = "gname" - paxLinkpath = "linkpath" - paxMtime = "mtime" - paxPath = "path" - paxSize = "size" - paxUid = "uid" - paxUname = "uname" - paxXattr = "SCHILY.xattr." - paxNone = "" -) - // FileInfoHeader creates a partially-populated Header from fi. // If fi describes a symlink, FileInfoHeader records link as the link target. // If fi describes a directory, a slash is appended to the name. -// Because os.FileInfo's Name method returns only the base name of -// the file it describes, it may be necessary to modify the Name field -// of the returned header to provide the full path name of the file. +// +// Since os.FileInfo's Name method only returns the base name of +// the file it describes, it may be necessary to modify Header.Name +// to provide the full path name of the file. +// +// This function does not populate Header.SparseHoles; +// for sparse file support, additionally call Header.DetectSparseHoles. func FileInfoHeader(fi os.FileInfo, link string) (*Header, error) { if fi == nil { return nil, errors.New("tar: FileInfo is nil") @@ -208,32 +708,26 @@ func FileInfoHeader(fi os.FileInfo, link string) (*Header, error) { } switch { case fm.IsRegular(): - h.Mode |= c_ISREG h.Typeflag = TypeReg h.Size = fi.Size() case fi.IsDir(): h.Typeflag = TypeDir - h.Mode |= c_ISDIR h.Name += "/" case fm&os.ModeSymlink != 0: h.Typeflag = TypeSymlink - h.Mode |= c_ISLNK h.Linkname = link case fm&os.ModeDevice != 0: if fm&os.ModeCharDevice != 0 { - h.Mode |= c_ISCHR h.Typeflag = TypeChar } else { - h.Mode |= c_ISBLK h.Typeflag = TypeBlock } case fm&os.ModeNamedPipe != 0: h.Typeflag = TypeFifo - h.Mode |= c_ISFIFO case fm&os.ModeSocket != 0: - h.Mode |= c_ISSOCK + return nil, fmt.Errorf("tar: sockets not supported") default: - return nil, fmt.Errorf("archive/tar: unknown file mode %v", fm) + return nil, fmt.Errorf("tar: unknown file mode %v", fm) } if fm&os.ModeSetuid != 0 { h.Mode |= c_ISUID @@ -267,6 +761,15 @@ func FileInfoHeader(fi os.FileInfo, link string) (*Header, error) { h.Size = 0 h.Linkname = sys.Linkname } + if sys.SparseHoles != nil { + h.SparseHoles = append([]SparseEntry{}, sys.SparseHoles...) + } + if sys.PAXRecords != nil { + h.PAXRecords = make(map[string]string) + for k, v := range sys.PAXRecords { + h.PAXRecords[k] = v + } + } } if sysStat != nil { return h, sysStat(fi, h) @@ -284,3 +787,10 @@ func isHeaderOnlyType(flag byte) bool { return false } } + +func min(a, b int64) int64 { + if a < b { + return a + } + return b +} diff --git a/vendor/github.com/dmcgowan/go-tar/format.go b/vendor/github.com/dmcgowan/go-tar/format.go index c2c9910d0..cf1289534 100644 --- a/vendor/github.com/dmcgowan/go-tar/format.go +++ b/vendor/github.com/dmcgowan/go-tar/format.go @@ -4,38 +4,131 @@ package tar +import "strings" + +// Format represents the tar archive format. +// +// The original tar format was introduced in Unix V7. +// Since then, there have been multiple competing formats attempting to +// standardize or extend the V7 format to overcome its limitations. +// The most common formats are the USTAR, PAX, and GNU formats, +// each with their own advantages and limitations. +// +// The following table captures the capabilities of each format: +// +// | USTAR | PAX | GNU +// ------------------+--------+-----------+---------- +// Name | 256B | unlimited | unlimited +// Linkname | 100B | unlimited | unlimited +// Size | uint33 | unlimited | uint89 +// Mode | uint21 | uint21 | uint57 +// Uid/Gid | uint21 | unlimited | uint57 +// Uname/Gname | 32B | unlimited | 32B +// ModTime | uint33 | unlimited | int89 +// AccessTime | n/a | unlimited | int89 +// ChangeTime | n/a | unlimited | int89 +// Devmajor/Devminor | uint21 | uint21 | uint57 +// ------------------+--------+-----------+---------- +// string encoding | ASCII | UTF-8 | binary +// sub-second times | no | yes | no +// sparse files | no | yes | yes +// +// The table's upper portion shows the Header fields, where each format reports +// the maximum number of bytes allowed for each string field and +// the integer type used to store each numeric field +// (where timestamps are stored as the number of seconds since the Unix epoch). +// +// The table's lower portion shows specialized features of each format, +// such as supported string encodings, support for sub-second timestamps, +// or support for sparse files. +type Format int + // Constants to identify various tar formats. const ( - // The format is unknown. - formatUnknown = (1 << iota) / 2 // Sequence of 0, 1, 2, 4, 8, etc... + // Deliberately hide the meaning of constants from public API. + _ Format = (1 << iota) / 4 // Sequence of 0, 0, 1, 2, 4, 8, etc... + + // FormatUnknown indicates that the format is unknown. + FormatUnknown // The format of the original Unix V7 tar tool prior to standardization. formatV7 - // The old and new GNU formats, which are incompatible with USTAR. - // This does cover the old GNU sparse extension. - // This does not cover the GNU sparse extensions using PAX headers, - // versions 0.0, 0.1, and 1.0; these fall under the PAX format. - formatGNU + // FormatUSTAR represents the USTAR header format defined in POSIX.1-1988. + // + // While this format is compatible with most tar readers, + // the format has several limitations making it unsuitable for some usages. + // Most notably, it cannot support sparse files, files larger than 8GiB, + // filenames larger than 256 characters, and non-ASCII filenames. + // + // Reference: + // http://pubs.opengroup.org/onlinepubs/9699919799/utilities/pax.html#tag_20_92_13_06 + FormatUSTAR + + // FormatPAX represents the PAX header format defined in POSIX.1-2001. + // + // PAX extends USTAR by writing a special file with Typeflag TypeXHeader + // preceding the original header. This file contains a set of key-value + // records, which are used to overcome USTAR's shortcomings, in addition to + // providing the ability to have sub-second resolution for timestamps. + // + // Some newer formats add their own extensions to PAX by defining their + // own keys and assigning certain semantic meaning to the associated values. + // For example, sparse file support in PAX is implemented using keys + // defined by the GNU manual (e.g., "GNU.sparse.map"). + // + // Reference: + // http://pubs.opengroup.org/onlinepubs/009695399/utilities/pax.html + FormatPAX + + // FormatGNU represents the GNU header format. + // + // The GNU header format is older than the USTAR and PAX standards and + // is not compatible with them. The GNU format supports + // arbitrary file sizes, filenames of arbitrary encoding and length, + // sparse files, and other features. + // + // It is recommended that PAX be chosen over GNU unless the target + // application can only parse GNU formatted archives. + // + // Reference: + // http://www.gnu.org/software/tar/manual/html_node/Standard.html + FormatGNU // Schily's tar format, which is incompatible with USTAR. // This does not cover STAR extensions to the PAX format; these fall under // the PAX format. formatSTAR - // USTAR is the former standardization of tar defined in POSIX.1-1988. - // This is incompatible with the GNU and STAR formats. - formatUSTAR - - // PAX is the latest standardization of tar defined in POSIX.1-2001. - // This is an extension of USTAR and is "backwards compatible" with it. - // - // Some newer formats add their own extensions to PAX, such as GNU sparse - // files and SCHILY extended attributes. Since they are backwards compatible - // with PAX, they will be labelled as "PAX". - formatPAX + formatMax ) +func (f Format) has(f2 Format) bool { return f&f2 != 0 } +func (f *Format) mayBe(f2 Format) { *f |= f2 } +func (f *Format) mayOnlyBe(f2 Format) { *f &= f2 } +func (f *Format) mustNotBe(f2 Format) { *f &^= f2 } + +var formatNames = map[Format]string{ + formatV7: "V7", FormatUSTAR: "USTAR", FormatPAX: "PAX", FormatGNU: "GNU", formatSTAR: "STAR", +} + +func (f Format) String() string { + var ss []string + for f2 := Format(1); f2 < formatMax; f2 <<= 1 { + if f.has(f2) { + ss = append(ss, formatNames[f2]) + } + } + switch len(ss) { + case 0: + return "" + case 1: + return ss[0] + default: + return "(" + strings.Join(ss, " | ") + ")" + } +} + // Magics used to identify various formats. const ( magicGNU, versionGNU = "ustar ", " \x00" @@ -50,6 +143,12 @@ const ( prefixSize = 155 // Max length of the prefix field in USTAR format ) +// blockPadding computes the number of bytes needed to pad offset up to the +// nearest block edge where 0 <= n < blockSize. +func blockPadding(offset int64) (n int64) { + return -offset & (blockSize - 1) +} + var zeroBlock block type block [blockSize]byte @@ -63,14 +162,14 @@ func (b *block) Sparse() sparseArray { return (sparseArray)(b[:]) } // GetFormat checks that the block is a valid tar header based on the checksum. // It then attempts to guess the specific format based on magic values. -// If the checksum fails, then formatUnknown is returned. -func (b *block) GetFormat() (format int) { +// If the checksum fails, then FormatUnknown is returned. +func (b *block) GetFormat() Format { // Verify checksum. var p parser value := p.parseOctal(b.V7().Chksum()) chksum1, chksum2 := b.ComputeChecksum() if p.err != nil || (value != chksum1 && value != chksum2) { - return formatUnknown + return FormatUnknown } // Guess the magic values. @@ -81,9 +180,9 @@ func (b *block) GetFormat() (format int) { case magic == magicUSTAR && trailer == trailerSTAR: return formatSTAR case magic == magicUSTAR: - return formatUSTAR + return FormatUSTAR | FormatPAX case magic == magicGNU && version == versionGNU: - return formatGNU + return FormatGNU default: return formatV7 } @@ -91,19 +190,19 @@ func (b *block) GetFormat() (format int) { // SetFormat writes the magic values necessary for specified format // and then updates the checksum accordingly. -func (b *block) SetFormat(format int) { +func (b *block) SetFormat(format Format) { // Set the magic values. - switch format { - case formatV7: + switch { + case format.has(formatV7): // Do nothing. - case formatGNU: + case format.has(FormatGNU): copy(b.GNU().Magic(), magicGNU) copy(b.GNU().Version(), versionGNU) - case formatSTAR: + case format.has(formatSTAR): copy(b.STAR().Magic(), magicUSTAR) copy(b.STAR().Version(), versionUSTAR) copy(b.STAR().Trailer(), trailerSTAR) - case formatUSTAR, formatPAX: + case format.has(FormatUSTAR | FormatPAX): copy(b.USTAR().Magic(), magicUSTAR) copy(b.USTAR().Version(), versionUSTAR) default: @@ -134,6 +233,11 @@ func (b *block) ComputeChecksum() (unsigned, signed int64) { return unsigned, signed } +// Reset clears the block with all zeros. +func (b *block) Reset() { + *b = block{} +} + type headerV7 [blockSize]byte func (h *headerV7) Name() []byte { return h[000:][:100] } @@ -187,11 +291,11 @@ func (h *headerUSTAR) Prefix() []byte { return h[345:][:155] } type sparseArray []byte -func (s sparseArray) Entry(i int) sparseNode { return (sparseNode)(s[i*24:]) } +func (s sparseArray) Entry(i int) sparseElem { return (sparseElem)(s[i*24:]) } func (s sparseArray) IsExtended() []byte { return s[24*s.MaxEntries():][:1] } func (s sparseArray) MaxEntries() int { return len(s) / 24 } -type sparseNode []byte +type sparseElem []byte -func (s sparseNode) Offset() []byte { return s[00:][:12] } -func (s sparseNode) NumBytes() []byte { return s[12:][:12] } +func (s sparseElem) Offset() []byte { return s[00:][:12] } +func (s sparseElem) Length() []byte { return s[12:][:12] } diff --git a/vendor/github.com/dmcgowan/go-tar/reader.go b/vendor/github.com/dmcgowan/go-tar/reader.go index a6142c6b8..1d0673020 100644 --- a/vendor/github.com/dmcgowan/go-tar/reader.go +++ b/vendor/github.com/dmcgowan/go-tar/reader.go @@ -4,33 +4,23 @@ package tar -// TODO(dsymonds): -// - pax extensions - import ( "bytes" - "errors" "io" "io/ioutil" - "math" "strconv" "strings" "time" ) -var ( - ErrHeader = errors.New("archive/tar: invalid tar header") -) - -// A Reader provides sequential access to the contents of a tar archive. -// A tar archive consists of a sequence of files. -// The Next method advances to the next file in the archive (including the first), -// and then it can be treated as an io.Reader to access the file's data. +// Reader provides sequential access to the contents of a tar archive. +// Reader.Next advances to the next file in the archive (including the first), +// and then Reader can be treated as an io.Reader to access the file's data. type Reader struct { r io.Reader - pad int64 // amount of padding (ignored) after current file entry - curr numBytesReader // reader for current file entry - blk block // buffer to use as temporary local storage + pad int64 // Amount of padding (ignored) after current file entry + curr fileReader // Reader for current file entry + blk block // Buffer to use as temporary local storage // err is a persistent error. // It is only the responsibility of every exported method of Reader to @@ -38,68 +28,21 @@ type Reader struct { err error } -// A numBytesReader is an io.Reader with a numBytes method, returning the number -// of bytes remaining in the underlying encoded data. -type numBytesReader interface { +type fileReader interface { io.Reader - numBytes() int64 -} + fileState -// A regFileReader is a numBytesReader for reading file data from a tar archive. -type regFileReader struct { - r io.Reader // underlying reader - nb int64 // number of unread bytes for current file entry + WriteTo(io.Writer) (int64, error) } -// A sparseFileReader is a numBytesReader for reading sparse file data from a -// tar archive. -type sparseFileReader struct { - rfr numBytesReader // Reads the sparse-encoded file data - sp []sparseEntry // The sparse map for the file - pos int64 // Keeps track of file position - total int64 // Total size of the file -} - -// A sparseEntry holds a single entry in a sparse file's sparse map. -// -// Sparse files are represented using a series of sparseEntrys. -// Despite the name, a sparseEntry represents an actual data fragment that -// references data found in the underlying archive stream. All regions not -// covered by a sparseEntry are logically filled with zeros. -// -// For example, if the underlying raw file contains the 10-byte data: -// var compactData = "abcdefgh" -// -// And the sparse map has the following entries: -// var sp = []sparseEntry{ -// {offset: 2, numBytes: 5} // Data fragment for [2..7] -// {offset: 18, numBytes: 3} // Data fragment for [18..21] -// } -// -// Then the content of the resulting sparse file with a "real" size of 25 is: -// var sparseData = "\x00"*2 + "abcde" + "\x00"*11 + "fgh" + "\x00"*4 -type sparseEntry struct { - offset int64 // Starting position of the fragment - numBytes int64 // Length of the fragment -} - -// Keywords for GNU sparse files in a PAX extended header -const ( - paxGNUSparseNumBlocks = "GNU.sparse.numblocks" - paxGNUSparseOffset = "GNU.sparse.offset" - paxGNUSparseNumBytes = "GNU.sparse.numbytes" - paxGNUSparseMap = "GNU.sparse.map" - paxGNUSparseName = "GNU.sparse.name" - paxGNUSparseMajor = "GNU.sparse.major" - paxGNUSparseMinor = "GNU.sparse.minor" - paxGNUSparseSize = "GNU.sparse.size" - paxGNUSparseRealSize = "GNU.sparse.realsize" -) - // NewReader creates a new Reader reading from r. -func NewReader(r io.Reader) *Reader { return &Reader{r: r} } +func NewReader(r io.Reader) *Reader { + return &Reader{r: r, curr: ®FileReader{r, 0}} +} // Next advances to the next entry in the tar archive. +// The Header.Size determines how many bytes can be read for the next file. +// Any remaining data in the current file is automatically discarded. // // io.EOF is returned at the end of the input. func (tr *Reader) Next() (*Header, error) { @@ -112,18 +55,26 @@ func (tr *Reader) Next() (*Header, error) { } func (tr *Reader) next() (*Header, error) { - var extHdrs map[string]string + var paxHdrs map[string]string + var gnuLongName, gnuLongLink string // Externally, Next iterates through the tar archive as if it is a series of // files. Internally, the tar format often uses fake "files" to add meta // data that describes the next file. These meta data "files" should not // normally be visible to the outside. As such, this loop iterates through // one or more "header files" until it finds a "normal file". + format := FormatUSTAR | FormatPAX | FormatGNU loop: for { - if err := tr.skipUnread(); err != nil { + // Discard the remainder of the file and any padding. + if err := discard(tr.r, tr.curr.PhysicalRemaining()); err != nil { return nil, err } + if _, err := tryReadFull(tr.r, tr.blk[:tr.pad]); err != nil { + return nil, err + } + tr.pad = 0 + hdr, rawHdr, err := tr.readHeader() if err != nil { return nil, err @@ -131,43 +82,57 @@ loop: if err := tr.handleRegularFile(hdr); err != nil { return nil, err } + format.mayOnlyBe(hdr.Format) // Check for PAX/GNU special headers and files. switch hdr.Typeflag { - case TypeXHeader: - extHdrs, err = parsePAX(tr) + case TypeXHeader, TypeXGlobalHeader: + format.mayOnlyBe(FormatPAX) + paxHdrs, err = parsePAX(tr) if err != nil { return nil, err } + if hdr.Typeflag == TypeXGlobalHeader { + mergePAX(hdr, paxHdrs) + return &Header{ + Typeflag: hdr.Typeflag, + Xattrs: hdr.Xattrs, + PAXRecords: hdr.PAXRecords, + Format: format, + }, nil + } continue loop // This is a meta header affecting the next header case TypeGNULongName, TypeGNULongLink: + format.mayOnlyBe(FormatGNU) realname, err := ioutil.ReadAll(tr) if err != nil { return nil, err } - // Convert GNU extensions to use PAX headers. - if extHdrs == nil { - extHdrs = make(map[string]string) - } var p parser switch hdr.Typeflag { case TypeGNULongName: - extHdrs[paxPath] = p.parseString(realname) + gnuLongName = p.parseString(realname) case TypeGNULongLink: - extHdrs[paxLinkpath] = p.parseString(realname) - } - if p.err != nil { - return nil, p.err + gnuLongLink = p.parseString(realname) } continue loop // This is a meta header affecting the next header default: // The old GNU sparse format is handled here since it is technically // just a regular file with additional attributes. - if err := mergePAX(hdr, extHdrs); err != nil { + if err := mergePAX(hdr, paxHdrs); err != nil { return nil, err } + if gnuLongName != "" { + hdr.Name = gnuLongName + } + if gnuLongLink != "" { + hdr.Linkname = gnuLongLink + } + if hdr.Typeflag == TypeRegA && strings.HasSuffix(hdr.Name, "/") { + hdr.Typeflag = TypeDir // Legacy archives use trailing slash for directories + } // The extended headers may have updated the size. // Thus, setup the regFileReader again after merging PAX headers. @@ -177,9 +142,15 @@ loop: // Sparse formats rely on being able to read from the logical data // section; there must be a preceding call to handleRegularFile. - if err := tr.handleSparseFile(hdr, rawHdr, extHdrs); err != nil { + if err := tr.handleSparseFile(hdr, rawHdr); err != nil { return nil, err } + + // Set the final guess at the format. + if format.has(FormatUSTAR) && format.has(FormatPAX) { + format.mayOnlyBe(FormatUSTAR) + } + hdr.Format = format return hdr, nil // This is a file, so stop } } @@ -197,105 +168,87 @@ func (tr *Reader) handleRegularFile(hdr *Header) error { return ErrHeader } - tr.pad = -nb & (blockSize - 1) // blockSize is a power of two + tr.pad = blockPadding(nb) tr.curr = ®FileReader{r: tr.r, nb: nb} return nil } // handleSparseFile checks if the current file is a sparse format of any type // and sets the curr reader appropriately. -func (tr *Reader) handleSparseFile(hdr *Header, rawHdr *block, extHdrs map[string]string) error { - var sp []sparseEntry +func (tr *Reader) handleSparseFile(hdr *Header, rawHdr *block) error { + var spd sparseDatas var err error if hdr.Typeflag == TypeGNUSparse { - sp, err = tr.readOldGNUSparseMap(hdr, rawHdr) - if err != nil { - return err - } + spd, err = tr.readOldGNUSparseMap(hdr, rawHdr) } else { - sp, err = tr.checkForGNUSparsePAXHeaders(hdr, extHdrs) - if err != nil { - return err - } + spd, err = tr.readGNUSparsePAXHeaders(hdr) } // If sp is non-nil, then this is a sparse file. - // Note that it is possible for len(sp) to be zero. - if sp != nil { - tr.curr, err = newSparseFileReader(tr.curr, sp, hdr.Size) + // Note that it is possible for len(sp) == 0. + if err == nil && spd != nil { + if isHeaderOnlyType(hdr.Typeflag) || !validateSparseEntries(spd, hdr.Size) { + return ErrHeader + } + sph := invertSparseEntries(spd, hdr.Size) + tr.curr = &sparseFileReader{tr.curr, sph, 0} + hdr.SparseHoles = append([]SparseEntry{}, sph...) } return err } -// checkForGNUSparsePAXHeaders checks the PAX headers for GNU sparse headers. If they are found, then -// this function reads the sparse map and returns it. Unknown sparse formats are ignored, causing the file to -// be treated as a regular file. -func (tr *Reader) checkForGNUSparsePAXHeaders(hdr *Header, headers map[string]string) ([]sparseEntry, error) { - var sparseFormat string - - // Check for sparse format indicators - major, majorOk := headers[paxGNUSparseMajor] - minor, minorOk := headers[paxGNUSparseMinor] - sparseName, sparseNameOk := headers[paxGNUSparseName] - _, sparseMapOk := headers[paxGNUSparseMap] - sparseSize, sparseSizeOk := headers[paxGNUSparseSize] - sparseRealSize, sparseRealSizeOk := headers[paxGNUSparseRealSize] - - // Identify which, if any, sparse format applies from which PAX headers are set - if majorOk && minorOk { - sparseFormat = major + "." + minor - } else if sparseNameOk && sparseMapOk { - sparseFormat = "0.1" - } else if sparseSizeOk { - sparseFormat = "0.0" - } else { - // Not a PAX format GNU sparse file. - return nil, nil +// readGNUSparsePAXHeaders checks the PAX headers for GNU sparse headers. +// If they are found, then this function reads the sparse map and returns it. +// This assumes that 0.0 headers have already been converted to 0.1 headers +// by the the PAX header parsing logic. +func (tr *Reader) readGNUSparsePAXHeaders(hdr *Header) (sparseDatas, error) { + // Identify the version of GNU headers. + var is1x0 bool + major, minor := hdr.PAXRecords[paxGNUSparseMajor], hdr.PAXRecords[paxGNUSparseMinor] + switch { + case major == "0" && (minor == "0" || minor == "1"): + is1x0 = false + case major == "1" && minor == "0": + is1x0 = true + case major != "" || minor != "": + return nil, nil // Unknown GNU sparse PAX version + case hdr.PAXRecords[paxGNUSparseMap] != "": + is1x0 = false // 0.0 and 0.1 did not have explicit version records, so guess + default: + return nil, nil // Not a PAX format GNU sparse file. } + hdr.Format.mayOnlyBe(FormatPAX) - // Check for unknown sparse format - if sparseFormat != "0.0" && sparseFormat != "0.1" && sparseFormat != "1.0" { - return nil, nil + // Update hdr from GNU sparse PAX headers. + if name := hdr.PAXRecords[paxGNUSparseName]; name != "" { + hdr.Name = name } - - // Update hdr from GNU sparse PAX headers - if sparseNameOk { - hdr.Name = sparseName + size := hdr.PAXRecords[paxGNUSparseSize] + if size == "" { + size = hdr.PAXRecords[paxGNUSparseRealSize] } - if sparseSizeOk { - realSize, err := strconv.ParseInt(sparseSize, 10, 64) + if size != "" { + n, err := strconv.ParseInt(size, 10, 64) if err != nil { return nil, ErrHeader } - hdr.Size = realSize - } else if sparseRealSizeOk { - realSize, err := strconv.ParseInt(sparseRealSize, 10, 64) - if err != nil { - return nil, ErrHeader - } - hdr.Size = realSize + hdr.Size = n } - // Set up the sparse map, according to the particular sparse format in use - var sp []sparseEntry - var err error - switch sparseFormat { - case "0.0", "0.1": - sp, err = readGNUSparseMap0x1(headers) - case "1.0": - sp, err = readGNUSparseMap1x0(tr.curr) + // Read the sparse map according to the appropriate format. + if is1x0 { + return readGNUSparseMap1x0(tr.curr) } - return sp, err + return readGNUSparseMap0x1(hdr.PAXRecords) } -// mergePAX merges well known headers according to PAX standard. -// In general headers with the same name as those found -// in the header struct overwrite those found in the header -// struct with higher precision or longer values. Esp. useful -// for name and linkname fields. -func mergePAX(hdr *Header, headers map[string]string) (err error) { - var id64 int64 - for k, v := range headers { +// mergePAX merges paxHdrs into hdr for all relevant fields of Header. +func mergePAX(hdr *Header, paxHdrs map[string]string) (err error) { + for k, v := range paxHdrs { + if v == "" { + continue // Keep the original USTAR value + } + var id64 int64 switch k { case paxPath: hdr.Name = v @@ -320,17 +273,18 @@ func mergePAX(hdr *Header, headers map[string]string) (err error) { case paxSize: hdr.Size, err = strconv.ParseInt(v, 10, 64) default: - if strings.HasPrefix(k, paxXattr) { + if strings.HasPrefix(k, paxSchilyXattr) { if hdr.Xattrs == nil { hdr.Xattrs = make(map[string]string) } - hdr.Xattrs[k[len(paxXattr):]] = v + hdr.Xattrs[k[len(paxSchilyXattr):]] = v } } if err != nil { return ErrHeader } } + hdr.PAXRecords = paxHdrs return nil } @@ -348,7 +302,7 @@ func parsePAX(r io.Reader) (map[string]string, error) { // headers since 0.0 headers were not PAX compliant. var sparseMap []string - extHdrs := make(map[string]string) + paxHdrs := make(map[string]string) for len(sbuf) > 0 { key, value, residual, err := parsePAXRecord(sbuf) if err != nil { @@ -366,58 +320,13 @@ func parsePAX(r io.Reader) (map[string]string, error) { } sparseMap = append(sparseMap, value) default: - // According to PAX specification, a value is stored only if it is - // non-empty. Otherwise, the key is deleted. - if len(value) > 0 { - extHdrs[key] = value - } else { - delete(extHdrs, key) - } + paxHdrs[key] = value } } if len(sparseMap) > 0 { - extHdrs[paxGNUSparseMap] = strings.Join(sparseMap, ",") + paxHdrs[paxGNUSparseMap] = strings.Join(sparseMap, ",") } - return extHdrs, nil -} - -// skipUnread skips any unread bytes in the existing file entry, as well as any -// alignment padding. It returns io.ErrUnexpectedEOF if any io.EOF is -// encountered in the data portion; it is okay to hit io.EOF in the padding. -// -// Note that this function still works properly even when sparse files are being -// used since numBytes returns the bytes remaining in the underlying io.Reader. -func (tr *Reader) skipUnread() error { - dataSkip := tr.numBytes() // Number of data bytes to skip - totalSkip := dataSkip + tr.pad // Total number of bytes to skip - tr.curr, tr.pad = nil, 0 - - // If possible, Seek to the last byte before the end of the data section. - // Do this because Seek is often lazy about reporting errors; this will mask - // the fact that the tar stream may be truncated. We can rely on the - // io.CopyN done shortly afterwards to trigger any IO errors. - var seekSkipped int64 // Number of bytes skipped via Seek - if sr, ok := tr.r.(io.Seeker); ok && dataSkip > 1 { - // Not all io.Seeker can actually Seek. For example, os.Stdin implements - // io.Seeker, but calling Seek always returns an error and performs - // no action. Thus, we try an innocent seek to the current position - // to see if Seek is really supported. - pos1, err := sr.Seek(0, io.SeekCurrent) - if err == nil { - // Seek seems supported, so perform the real Seek. - pos2, err := sr.Seek(dataSkip-1, io.SeekCurrent) - if err != nil { - return err - } - seekSkipped = pos2 - pos1 - } - } - - copySkipped, err := io.CopyN(ioutil.Discard, tr.r, totalSkip-seekSkipped) - if err == io.EOF && seekSkipped+copySkipped < dataSkip { - err = io.ErrUnexpectedEOF - } - return err + return paxHdrs, nil } // readHeader reads the next block header and assumes that the underlying reader @@ -445,7 +354,7 @@ func (tr *Reader) readHeader() (*Header, *block, error) { // Verify the header matches a known format. format := tr.blk.GetFormat() - if format == formatUnknown { + if format == FormatUnknown { return nil, nil, ErrHeader } @@ -454,37 +363,86 @@ func (tr *Reader) readHeader() (*Header, *block, error) { // Unpack the V7 header. v7 := tr.blk.V7() + hdr.Typeflag = v7.TypeFlag()[0] hdr.Name = p.parseString(v7.Name()) + hdr.Linkname = p.parseString(v7.LinkName()) + hdr.Size = p.parseNumeric(v7.Size()) hdr.Mode = p.parseNumeric(v7.Mode()) hdr.Uid = int(p.parseNumeric(v7.UID())) hdr.Gid = int(p.parseNumeric(v7.GID())) - hdr.Size = p.parseNumeric(v7.Size()) hdr.ModTime = time.Unix(p.parseNumeric(v7.ModTime()), 0) - hdr.Typeflag = v7.TypeFlag()[0] - hdr.Linkname = p.parseString(v7.LinkName()) // Unpack format specific fields. if format > formatV7 { ustar := tr.blk.USTAR() hdr.Uname = p.parseString(ustar.UserName()) hdr.Gname = p.parseString(ustar.GroupName()) - if hdr.Typeflag == TypeChar || hdr.Typeflag == TypeBlock { - hdr.Devmajor = p.parseNumeric(ustar.DevMajor()) - hdr.Devminor = p.parseNumeric(ustar.DevMinor()) - } + hdr.Devmajor = p.parseNumeric(ustar.DevMajor()) + hdr.Devminor = p.parseNumeric(ustar.DevMinor()) var prefix string - switch format { - case formatUSTAR, formatGNU: - // TODO(dsnet): Do not use the prefix field for the GNU format! - // See golang.org/issues/12594 + switch { + case format.has(FormatUSTAR | FormatPAX): + hdr.Format = format ustar := tr.blk.USTAR() prefix = p.parseString(ustar.Prefix()) - case formatSTAR: + + // For Format detection, check if block is properly formatted since + // the parser is more liberal than what USTAR actually permits. + notASCII := func(r rune) bool { return r >= 0x80 } + if bytes.IndexFunc(tr.blk[:], notASCII) >= 0 { + hdr.Format = FormatUnknown // Non-ASCII characters in block. + } + nul := func(b []byte) bool { return int(b[len(b)-1]) == 0 } + if !(nul(v7.Size()) && nul(v7.Mode()) && nul(v7.UID()) && nul(v7.GID()) && + nul(v7.ModTime()) && nul(ustar.DevMajor()) && nul(ustar.DevMinor())) { + hdr.Format = FormatUnknown // Numeric fields must end in NUL + } + case format.has(formatSTAR): star := tr.blk.STAR() prefix = p.parseString(star.Prefix()) hdr.AccessTime = time.Unix(p.parseNumeric(star.AccessTime()), 0) hdr.ChangeTime = time.Unix(p.parseNumeric(star.ChangeTime()), 0) + case format.has(FormatGNU): + hdr.Format = format + var p2 parser + gnu := tr.blk.GNU() + if b := gnu.AccessTime(); b[0] != 0 { + hdr.AccessTime = time.Unix(p2.parseNumeric(b), 0) + } + if b := gnu.ChangeTime(); b[0] != 0 { + hdr.ChangeTime = time.Unix(p2.parseNumeric(b), 0) + } + + // Prior to Go1.8, the Writer had a bug where it would output + // an invalid tar file in certain rare situations because the logic + // incorrectly believed that the old GNU format had a prefix field. + // This is wrong and leads to an output file that mangles the + // atime and ctime fields, which are often left unused. + // + // In order to continue reading tar files created by former, buggy + // versions of Go, we skeptically parse the atime and ctime fields. + // If we are unable to parse them and the prefix field looks like + // an ASCII string, then we fallback on the pre-Go1.8 behavior + // of treating these fields as the USTAR prefix field. + // + // Note that this will not use the fallback logic for all possible + // files generated by a pre-Go1.8 toolchain. If the generated file + // happened to have a prefix field that parses as valid + // atime and ctime fields (e.g., when they are valid octal strings), + // then it is impossible to distinguish between an valid GNU file + // and an invalid pre-Go1.8 file. + // + // See https://golang.org/issues/12594 + // See https://golang.org/issues/21005 + if p2.err != nil { + hdr.AccessTime, hdr.ChangeTime = time.Time{}, time.Time{} + ustar := tr.blk.USTAR() + if s := p.parseString(ustar.Prefix()); isASCII(s) { + prefix = s + } + hdr.Format = FormatUnknown // Buggy file is not GNU + } } if len(prefix) > 0 { hdr.Name = prefix + "/" + hdr.Name @@ -501,21 +459,22 @@ func (tr *Reader) readHeader() (*Header, *block, error) { // The Header.Size does not reflect the size of any extended headers used. // Thus, this function will read from the raw io.Reader to fetch extra headers. // This method mutates blk in the process. -func (tr *Reader) readOldGNUSparseMap(hdr *Header, blk *block) ([]sparseEntry, error) { +func (tr *Reader) readOldGNUSparseMap(hdr *Header, blk *block) (sparseDatas, error) { // Make sure that the input format is GNU. // Unfortunately, the STAR format also has a sparse header format that uses // the same type flag but has a completely different layout. - if blk.GetFormat() != formatGNU { + if blk.GetFormat() != FormatGNU { return nil, ErrHeader } + hdr.Format.mayOnlyBe(FormatGNU) var p parser hdr.Size = p.parseNumeric(blk.GNU().RealSize()) if p.err != nil { return nil, p.err } - var s sparseArray = blk.GNU().Sparse() - var sp = make([]sparseEntry, 0, s.MaxEntries()) + s := blk.GNU().Sparse() + spd := make(sparseDatas, 0, s.MaxEntries()) for { for i := 0; i < s.MaxEntries(); i++ { // This termination condition is identical to GNU and BSD tar. @@ -523,25 +482,22 @@ func (tr *Reader) readOldGNUSparseMap(hdr *Header, blk *block) ([]sparseEntry, e break // Don't return, need to process extended headers (even if empty) } offset := p.parseNumeric(s.Entry(i).Offset()) - numBytes := p.parseNumeric(s.Entry(i).NumBytes()) + length := p.parseNumeric(s.Entry(i).Length()) if p.err != nil { return nil, p.err } - sp = append(sp, sparseEntry{offset: offset, numBytes: numBytes}) + spd = append(spd, SparseEntry{Offset: offset, Length: length}) } if s.IsExtended()[0] > 0 { // There are more entries. Read an extension header and parse its entries. - if _, err := io.ReadFull(tr.r, blk[:]); err != nil { - if err == io.EOF { - err = io.ErrUnexpectedEOF - } + if _, err := mustReadFull(tr.r, blk[:]); err != nil { return nil, err } s = blk.Sparse() continue } - return sp, nil // Done + return spd, nil // Done } } @@ -549,28 +505,27 @@ func (tr *Reader) readOldGNUSparseMap(hdr *Header, blk *block) ([]sparseEntry, e // version 1.0. The format of the sparse map consists of a series of // newline-terminated numeric fields. The first field is the number of entries // and is always present. Following this are the entries, consisting of two -// fields (offset, numBytes). This function must stop reading at the end +// fields (offset, length). This function must stop reading at the end // boundary of the block containing the last newline. // // Note that the GNU manual says that numeric values should be encoded in octal // format. However, the GNU tar utility itself outputs these values in decimal. // As such, this library treats values as being encoded in decimal. -func readGNUSparseMap1x0(r io.Reader) ([]sparseEntry, error) { - var cntNewline int64 - var buf bytes.Buffer - var blk = make([]byte, blockSize) +func readGNUSparseMap1x0(r io.Reader) (sparseDatas, error) { + var ( + cntNewline int64 + buf bytes.Buffer + blk block + ) - // feedTokens copies data in numBlock chunks from r into buf until there are + // feedTokens copies data in blocks from r into buf until there are // at least cnt newlines in buf. It will not read more blocks than needed. - var feedTokens = func(cnt int64) error { - for cntNewline < cnt { - if _, err := io.ReadFull(r, blk); err != nil { - if err == io.EOF { - err = io.ErrUnexpectedEOF - } + feedTokens := func(n int64) error { + for cntNewline < n { + if _, err := mustReadFull(r, blk[:]); err != nil { return err } - buf.Write(blk) + buf.Write(blk[:]) for _, c := range blk { if c == '\n' { cntNewline++ @@ -582,10 +537,10 @@ func readGNUSparseMap1x0(r io.Reader) ([]sparseEntry, error) { // nextToken gets the next token delimited by a newline. This assumes that // at least one newline exists in the buffer. - var nextToken = func() string { + nextToken := func() string { cntNewline-- tok, _ := buf.ReadString('\n') - return tok[:len(tok)-1] // Cut off newline + return strings.TrimRight(tok, "\n") } // Parse for the number of entries. @@ -604,80 +559,67 @@ func readGNUSparseMap1x0(r io.Reader) ([]sparseEntry, error) { if err := feedTokens(2 * numEntries); err != nil { return nil, err } - sp := make([]sparseEntry, 0, numEntries) + spd := make(sparseDatas, 0, numEntries) for i := int64(0); i < numEntries; i++ { - offset, err := strconv.ParseInt(nextToken(), 10, 64) - if err != nil { + offset, err1 := strconv.ParseInt(nextToken(), 10, 64) + length, err2 := strconv.ParseInt(nextToken(), 10, 64) + if err1 != nil || err2 != nil { return nil, ErrHeader } - numBytes, err := strconv.ParseInt(nextToken(), 10, 64) - if err != nil { - return nil, ErrHeader - } - sp = append(sp, sparseEntry{offset: offset, numBytes: numBytes}) + spd = append(spd, SparseEntry{Offset: offset, Length: length}) } - return sp, nil + return spd, nil } // readGNUSparseMap0x1 reads the sparse map as stored in GNU's PAX sparse format // version 0.1. The sparse map is stored in the PAX headers. -func readGNUSparseMap0x1(extHdrs map[string]string) ([]sparseEntry, error) { +func readGNUSparseMap0x1(paxHdrs map[string]string) (sparseDatas, error) { // Get number of entries. // Use integer overflow resistant math to check this. - numEntriesStr := extHdrs[paxGNUSparseNumBlocks] + numEntriesStr := paxHdrs[paxGNUSparseNumBlocks] numEntries, err := strconv.ParseInt(numEntriesStr, 10, 0) // Intentionally parse as native int if err != nil || numEntries < 0 || int(2*numEntries) < int(numEntries) { return nil, ErrHeader } // There should be two numbers in sparseMap for each entry. - sparseMap := strings.Split(extHdrs[paxGNUSparseMap], ",") + sparseMap := strings.Split(paxHdrs[paxGNUSparseMap], ",") + if len(sparseMap) == 1 && sparseMap[0] == "" { + sparseMap = sparseMap[:0] + } if int64(len(sparseMap)) != 2*numEntries { return nil, ErrHeader } // Loop through the entries in the sparse map. // numEntries is trusted now. - sp := make([]sparseEntry, 0, numEntries) - for i := int64(0); i < numEntries; i++ { - offset, err := strconv.ParseInt(sparseMap[2*i], 10, 64) - if err != nil { + spd := make(sparseDatas, 0, numEntries) + for len(sparseMap) >= 2 { + offset, err1 := strconv.ParseInt(sparseMap[0], 10, 64) + length, err2 := strconv.ParseInt(sparseMap[1], 10, 64) + if err1 != nil || err2 != nil { return nil, ErrHeader } - numBytes, err := strconv.ParseInt(sparseMap[2*i+1], 10, 64) - if err != nil { - return nil, ErrHeader - } - sp = append(sp, sparseEntry{offset: offset, numBytes: numBytes}) + spd = append(spd, SparseEntry{Offset: offset, Length: length}) + sparseMap = sparseMap[2:] } - return sp, nil + return spd, nil } -// numBytes returns the number of bytes left to read in the current file's entry -// in the tar archive, or 0 if there is no current file. -func (tr *Reader) numBytes() int64 { - if tr.curr == nil { - // No current file, so no bytes - return 0 - } - return tr.curr.numBytes() -} - -// Read reads from the current entry in the tar archive. -// It returns 0, io.EOF when it reaches the end of that entry, -// until Next is called to advance to the next entry. +// Read reads from the current file in the tar archive. +// It returns (0, io.EOF) when it reaches the end of that file, +// until Next is called to advance to the next file. // -// Calling Read on special types like TypeLink, TypeSymLink, TypeChar, -// TypeBlock, TypeDir, and TypeFifo returns 0, io.EOF regardless of what +// If the current file is sparse, then the regions marked as a hole +// are read back as NUL-bytes. +// +// Calling Read on special types like TypeLink, TypeSymlink, TypeChar, +// TypeBlock, TypeDir, and TypeFifo returns (0, io.EOF) regardless of what // the Header.Size claims. func (tr *Reader) Read(b []byte) (int, error) { if tr.err != nil { return 0, tr.err } - if tr.curr == nil { - return 0, io.EOF - } - n, err := tr.curr.Read(b) if err != nil && err != io.EOF { tr.err = err @@ -685,116 +627,226 @@ func (tr *Reader) Read(b []byte) (int, error) { return n, err } -func (rfr *regFileReader) Read(b []byte) (n int, err error) { - if rfr.nb == 0 { - // file consumed - return 0, io.EOF +// WriteTo writes the content of the current file to w. +// The bytes written matches the number of remaining bytes in the current file. +// +// If the current file is sparse and w is an io.WriteSeeker, +// then WriteTo uses Seek to skip past holes defined in Header.SparseHoles, +// assuming that skipped regions are filled with NULs. +// This always writes the last byte to ensure w is the right size. +func (tr *Reader) WriteTo(w io.Writer) (int64, error) { + if tr.err != nil { + return 0, tr.err } - if int64(len(b)) > rfr.nb { - b = b[0:rfr.nb] - } - n, err = rfr.r.Read(b) - rfr.nb -= int64(n) - - if err == io.EOF && rfr.nb > 0 { - err = io.ErrUnexpectedEOF - } - return -} - -// numBytes returns the number of bytes left to read in the file's data in the tar archive. -func (rfr *regFileReader) numBytes() int64 { - return rfr.nb -} - -// newSparseFileReader creates a new sparseFileReader, but validates all of the -// sparse entries before doing so. -func newSparseFileReader(rfr numBytesReader, sp []sparseEntry, total int64) (*sparseFileReader, error) { - if total < 0 { - return nil, ErrHeader // Total size cannot be negative - } - - // Validate all sparse entries. These are the same checks as performed by - // the BSD tar utility. - for i, s := range sp { - switch { - case s.offset < 0 || s.numBytes < 0: - return nil, ErrHeader // Negative values are never okay - case s.offset > math.MaxInt64-s.numBytes: - return nil, ErrHeader // Integer overflow with large length - case s.offset+s.numBytes > total: - return nil, ErrHeader // Region extends beyond the "real" size - case i > 0 && sp[i-1].offset+sp[i-1].numBytes > s.offset: - return nil, ErrHeader // Regions can't overlap and must be in order - } - } - return &sparseFileReader{rfr: rfr, sp: sp, total: total}, nil -} - -// readHole reads a sparse hole ending at endOffset. -func (sfr *sparseFileReader) readHole(b []byte, endOffset int64) int { - n64 := endOffset - sfr.pos - if n64 > int64(len(b)) { - n64 = int64(len(b)) - } - n := int(n64) - for i := 0; i < n; i++ { - b[i] = 0 - } - sfr.pos += n64 - return n -} - -// Read reads the sparse file data in expanded form. -func (sfr *sparseFileReader) Read(b []byte) (n int, err error) { - // Skip past all empty fragments. - for len(sfr.sp) > 0 && sfr.sp[0].numBytes == 0 { - sfr.sp = sfr.sp[1:] - } - - // If there are no more fragments, then it is possible that there - // is one last sparse hole. - if len(sfr.sp) == 0 { - // This behavior matches the BSD tar utility. - // However, GNU tar stops returning data even if sfr.total is unmet. - if sfr.pos < sfr.total { - return sfr.readHole(b, sfr.total), nil - } - return 0, io.EOF - } - - // In front of a data fragment, so read a hole. - if sfr.pos < sfr.sp[0].offset { - return sfr.readHole(b, sfr.sp[0].offset), nil - } - - // In a data fragment, so read from it. - // This math is overflow free since we verify that offset and numBytes can - // be safely added when creating the sparseFileReader. - endPos := sfr.sp[0].offset + sfr.sp[0].numBytes // End offset of fragment - bytesLeft := endPos - sfr.pos // Bytes left in fragment - if int64(len(b)) > bytesLeft { - b = b[:bytesLeft] - } - - n, err = sfr.rfr.Read(b) - sfr.pos += int64(n) - if err == io.EOF { - if sfr.pos < endPos { - err = io.ErrUnexpectedEOF // There was supposed to be more data - } else if sfr.pos < sfr.total { - err = nil // There is still an implicit sparse hole at the end - } - } - - if sfr.pos == endPos { - sfr.sp = sfr.sp[1:] // We are done with this fragment, so pop it + n, err := tr.curr.WriteTo(w) + if err != nil { + tr.err = err } return n, err } -// numBytes returns the number of bytes left to read in the sparse file's -// sparse-encoded data in the tar archive. -func (sfr *sparseFileReader) numBytes() int64 { - return sfr.rfr.numBytes() +// regFileReader is a fileReader for reading data from a regular file entry. +type regFileReader struct { + r io.Reader // Underlying Reader + nb int64 // Number of remaining bytes to read +} + +func (fr *regFileReader) Read(b []byte) (n int, err error) { + if int64(len(b)) > fr.nb { + b = b[:fr.nb] + } + if len(b) > 0 { + n, err = fr.r.Read(b) + fr.nb -= int64(n) + } + switch { + case err == io.EOF && fr.nb > 0: + return n, io.ErrUnexpectedEOF + case err == nil && fr.nb == 0: + return n, io.EOF + default: + return n, err + } +} + +func (fr *regFileReader) WriteTo(w io.Writer) (int64, error) { + return io.Copy(w, struct{ io.Reader }{fr}) +} + +func (fr regFileReader) LogicalRemaining() int64 { + return fr.nb +} + +func (fr regFileReader) PhysicalRemaining() int64 { + return fr.nb +} + +// sparseFileReader is a fileReader for reading data from a sparse file entry. +type sparseFileReader struct { + fr fileReader // Underlying fileReader + sp sparseHoles // Normalized list of sparse holes + pos int64 // Current position in sparse file +} + +func (sr *sparseFileReader) Read(b []byte) (n int, err error) { + finished := int64(len(b)) >= sr.LogicalRemaining() + if finished { + b = b[:sr.LogicalRemaining()] + } + + b0 := b + endPos := sr.pos + int64(len(b)) + for endPos > sr.pos && err == nil { + var nf int // Bytes read in fragment + holeStart, holeEnd := sr.sp[0].Offset, sr.sp[0].endOffset() + if sr.pos < holeStart { // In a data fragment + bf := b[:min(int64(len(b)), holeStart-sr.pos)] + nf, err = tryReadFull(sr.fr, bf) + } else { // In a hole fragment + bf := b[:min(int64(len(b)), holeEnd-sr.pos)] + nf, err = tryReadFull(zeroReader{}, bf) + } + b = b[nf:] + sr.pos += int64(nf) + if sr.pos >= holeEnd && len(sr.sp) > 1 { + sr.sp = sr.sp[1:] // Ensure last fragment always remains + } + } + + n = len(b0) - len(b) + switch { + case err == io.EOF: + return n, errMissData // Less data in dense file than sparse file + case err != nil: + return n, err + case sr.LogicalRemaining() == 0 && sr.PhysicalRemaining() > 0: + return n, errUnrefData // More data in dense file than sparse file + case finished: + return n, io.EOF + default: + return n, nil + } +} + +func (sr *sparseFileReader) WriteTo(w io.Writer) (n int64, err error) { + ws, ok := w.(io.WriteSeeker) + if ok { + if _, err := ws.Seek(0, io.SeekCurrent); err != nil { + ok = false // Not all io.Seeker can really seek + } + } + if !ok { + return io.Copy(w, struct{ io.Reader }{sr}) + } + + var writeLastByte bool + pos0 := sr.pos + for sr.LogicalRemaining() > 0 && !writeLastByte && err == nil { + var nf int64 // Size of fragment + holeStart, holeEnd := sr.sp[0].Offset, sr.sp[0].endOffset() + if sr.pos < holeStart { // In a data fragment + nf = holeStart - sr.pos + nf, err = io.CopyN(ws, sr.fr, nf) + } else { // In a hole fragment + nf = holeEnd - sr.pos + if sr.PhysicalRemaining() == 0 { + writeLastByte = true + nf-- + } + _, err = ws.Seek(nf, io.SeekCurrent) + } + sr.pos += nf + if sr.pos >= holeEnd && len(sr.sp) > 1 { + sr.sp = sr.sp[1:] // Ensure last fragment always remains + } + } + + // If the last fragment is a hole, then seek to 1-byte before EOF, and + // write a single byte to ensure the file is the right size. + if writeLastByte && err == nil { + _, err = ws.Write([]byte{0}) + sr.pos++ + } + + n = sr.pos - pos0 + switch { + case err == io.EOF: + return n, errMissData // Less data in dense file than sparse file + case err != nil: + return n, err + case sr.LogicalRemaining() == 0 && sr.PhysicalRemaining() > 0: + return n, errUnrefData // More data in dense file than sparse file + default: + return n, nil + } +} + +func (sr sparseFileReader) LogicalRemaining() int64 { + return sr.sp[len(sr.sp)-1].endOffset() - sr.pos +} +func (sr sparseFileReader) PhysicalRemaining() int64 { + return sr.fr.PhysicalRemaining() +} + +type zeroReader struct{} + +func (zeroReader) Read(b []byte) (int, error) { + for i := range b { + b[i] = 0 + } + return len(b), nil +} + +// mustReadFull is like io.ReadFull except it returns +// io.ErrUnexpectedEOF when io.EOF is hit before len(b) bytes are read. +func mustReadFull(r io.Reader, b []byte) (int, error) { + n, err := tryReadFull(r, b) + if err == io.EOF { + err = io.ErrUnexpectedEOF + } + return n, err +} + +// tryReadFull is like io.ReadFull except it returns +// io.EOF when it is hit before len(b) bytes are read. +func tryReadFull(r io.Reader, b []byte) (n int, err error) { + for len(b) > n && err == nil { + var nn int + nn, err = r.Read(b[n:]) + n += nn + } + if len(b) == n && err == io.EOF { + err = nil + } + return n, err +} + +// discard skips n bytes in r, reporting an error if unable to do so. +func discard(r io.Reader, n int64) error { + // If possible, Seek to the last byte before the end of the data section. + // Do this because Seek is often lazy about reporting errors; this will mask + // the fact that the stream may be truncated. We can rely on the + // io.CopyN done shortly afterwards to trigger any IO errors. + var seekSkipped int64 // Number of bytes skipped via Seek + if sr, ok := r.(io.Seeker); ok && n > 1 { + // Not all io.Seeker can actually Seek. For example, os.Stdin implements + // io.Seeker, but calling Seek always returns an error and performs + // no action. Thus, we try an innocent seek to the current position + // to see if Seek is really supported. + pos1, err := sr.Seek(0, io.SeekCurrent) + if pos1 >= 0 && err == nil { + // Seek seems supported, so perform the real Seek. + pos2, err := sr.Seek(n-1, io.SeekCurrent) + if pos2 < 0 || err != nil { + return err + } + seekSkipped = pos2 - pos1 + } + } + + copySkipped, err := io.CopyN(ioutil.Discard, r, n-seekSkipped) + if err == io.EOF && seekSkipped+copySkipped < n { + err = io.ErrUnexpectedEOF + } + return err } diff --git a/vendor/github.com/dmcgowan/go-tar/sparse_unix.go b/vendor/github.com/dmcgowan/go-tar/sparse_unix.go new file mode 100644 index 000000000..c623c1ee4 --- /dev/null +++ b/vendor/github.com/dmcgowan/go-tar/sparse_unix.go @@ -0,0 +1,77 @@ +// Copyright 2017 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build linux darwin dragonfly freebsd openbsd netbsd solaris + +package tar + +import ( + "io" + "os" + "runtime" + "syscall" +) + +func init() { + sysSparseDetect = sparseDetectUnix +} + +func sparseDetectUnix(f *os.File) (sph sparseHoles, err error) { + // SEEK_DATA and SEEK_HOLE originated from Solaris and support for it + // has been added to most of the other major Unix systems. + var seekData, seekHole = 3, 4 // SEEK_DATA/SEEK_HOLE from unistd.h + + if runtime.GOOS == "darwin" { + // Darwin has the constants swapped, compared to all other UNIX. + seekData, seekHole = 4, 3 + } + + // Check for seekData/seekHole support. + // Different OS and FS may differ in the exact errno that is returned when + // there is no support. Rather than special-casing every possible errno + // representing "not supported", just assume that a non-nil error means + // that seekData/seekHole is not supported. + if _, err := f.Seek(0, seekHole); err != nil { + return nil, nil + } + + // Populate the SparseHoles. + var last, pos int64 = -1, 0 + for { + // Get the location of the next hole section. + if pos, err = fseek(f, pos, seekHole); pos == last || err != nil { + return sph, err + } + offset := pos + last = pos + + // Get the location of the next data section. + if pos, err = fseek(f, pos, seekData); pos == last || err != nil { + return sph, err + } + length := pos - offset + last = pos + + if length > 0 { + sph = append(sph, SparseEntry{offset, length}) + } + } +} + +func fseek(f *os.File, pos int64, whence int) (int64, error) { + pos, err := f.Seek(pos, whence) + if errno(err) == syscall.ENXIO { + // SEEK_DATA returns ENXIO when past the last data fragment, + // which makes determining the size of the last hole difficult. + pos, err = f.Seek(0, io.SeekEnd) + } + return pos, err +} + +func errno(err error) error { + if perr, ok := err.(*os.PathError); ok { + return perr.Err + } + return err +} diff --git a/vendor/github.com/dmcgowan/go-tar/sparse_windows.go b/vendor/github.com/dmcgowan/go-tar/sparse_windows.go new file mode 100644 index 000000000..05bf1a90b --- /dev/null +++ b/vendor/github.com/dmcgowan/go-tar/sparse_windows.go @@ -0,0 +1,129 @@ +// Copyright 2017 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build windows + +package tar + +import ( + "os" + "syscall" + "unsafe" +) + +var errInvalidFunc = syscall.Errno(1) // ERROR_INVALID_FUNCTION from WinError.h + +func init() { + sysSparseDetect = sparseDetectWindows + sysSparsePunch = sparsePunchWindows +} + +func sparseDetectWindows(f *os.File) (sph sparseHoles, err error) { + const queryAllocRanges = 0x000940CF // FSCTL_QUERY_ALLOCATED_RANGES from WinIoCtl.h + type allocRangeBuffer struct{ offset, length int64 } // FILE_ALLOCATED_RANGE_BUFFER from WinIoCtl.h + + s, err := f.Stat() + if err != nil { + return nil, err + } + + queryRange := allocRangeBuffer{0, s.Size()} + allocRanges := make([]allocRangeBuffer, 64) + + // Repeatedly query for ranges until the input buffer is large enough. + var bytesReturned uint32 + for { + err := syscall.DeviceIoControl( + syscall.Handle(f.Fd()), queryAllocRanges, + (*byte)(unsafe.Pointer(&queryRange)), uint32(unsafe.Sizeof(queryRange)), + (*byte)(unsafe.Pointer(&allocRanges[0])), uint32(len(allocRanges)*int(unsafe.Sizeof(allocRanges[0]))), + &bytesReturned, nil, + ) + if err == syscall.ERROR_MORE_DATA { + allocRanges = make([]allocRangeBuffer, 2*len(allocRanges)) + continue + } + if err == errInvalidFunc { + return nil, nil // Sparse file not supported on this FS + } + if err != nil { + return nil, err + } + break + } + n := bytesReturned / uint32(unsafe.Sizeof(allocRanges[0])) + allocRanges = append(allocRanges[:n], allocRangeBuffer{s.Size(), 0}) + + // Invert the data fragments into hole fragments. + var pos int64 + for _, r := range allocRanges { + if r.offset > pos { + sph = append(sph, SparseEntry{pos, r.offset - pos}) + } + pos = r.offset + r.length + } + return sph, nil +} + +func sparsePunchWindows(f *os.File, sph sparseHoles) error { + const setSparse = 0x000900C4 // FSCTL_SET_SPARSE from WinIoCtl.h + const setZeroData = 0x000980C8 // FSCTL_SET_ZERO_DATA from WinIoCtl.h + type zeroDataInfo struct{ start, end int64 } // FILE_ZERO_DATA_INFORMATION from WinIoCtl.h + + // Set the file as being sparse. + var bytesReturned uint32 + devErr := syscall.DeviceIoControl( + syscall.Handle(f.Fd()), setSparse, + nil, 0, nil, 0, + &bytesReturned, nil, + ) + if devErr != nil && devErr != errInvalidFunc { + return devErr + } + + // Set the file to the right size. + var size int64 + if len(sph) > 0 { + size = sph[len(sph)-1].endOffset() + } + if err := f.Truncate(size); err != nil { + return err + } + if devErr == errInvalidFunc { + // Sparse file not supported on this FS. + // Call sparsePunchManual since SetEndOfFile does not guarantee that + // the extended space is filled with zeros. + return sparsePunchManual(f, sph) + } + + // Punch holes for all relevant fragments. + for _, s := range sph { + zdi := zeroDataInfo{s.Offset, s.endOffset()} + err := syscall.DeviceIoControl( + syscall.Handle(f.Fd()), setZeroData, + (*byte)(unsafe.Pointer(&zdi)), uint32(unsafe.Sizeof(zdi)), + nil, 0, + &bytesReturned, nil, + ) + if err != nil { + return err + } + } + return nil +} + +// sparsePunchManual writes zeros into each hole. +func sparsePunchManual(f *os.File, sph sparseHoles) error { + const chunkSize = 32 << 10 + zbuf := make([]byte, chunkSize) + for _, s := range sph { + for pos := s.Offset; pos < s.endOffset(); pos += chunkSize { + n := min(chunkSize, s.endOffset()-pos) + if _, err := f.WriteAt(zbuf[:n], pos); err != nil { + return err + } + } + } + return nil +} diff --git a/vendor/github.com/dmcgowan/go-tar/stat_atim.go b/vendor/github.com/dmcgowan/go-tar/stat_actime1.go similarity index 100% rename from vendor/github.com/dmcgowan/go-tar/stat_atim.go rename to vendor/github.com/dmcgowan/go-tar/stat_actime1.go diff --git a/vendor/github.com/dmcgowan/go-tar/stat_atimespec.go b/vendor/github.com/dmcgowan/go-tar/stat_actime2.go similarity index 100% rename from vendor/github.com/dmcgowan/go-tar/stat_atimespec.go rename to vendor/github.com/dmcgowan/go-tar/stat_actime2.go diff --git a/vendor/github.com/dmcgowan/go-tar/stat_unix.go b/vendor/github.com/dmcgowan/go-tar/stat_unix.go index cb843db4c..868105f33 100644 --- a/vendor/github.com/dmcgowan/go-tar/stat_unix.go +++ b/vendor/github.com/dmcgowan/go-tar/stat_unix.go @@ -8,6 +8,10 @@ package tar import ( "os" + "os/user" + "runtime" + "strconv" + "sync" "syscall" ) @@ -15,6 +19,10 @@ func init() { sysStat = statUnix } +// userMap and groupMap caches UID and GID lookups for performance reasons. +// The downside is that renaming uname or gname by the OS never takes effect. +var userMap, groupMap sync.Map // map[int]string + func statUnix(fi os.FileInfo, h *Header) error { sys, ok := fi.Sys().(*syscall.Stat_t) if !ok { @@ -22,11 +30,67 @@ func statUnix(fi os.FileInfo, h *Header) error { } h.Uid = int(sys.Uid) h.Gid = int(sys.Gid) - // TODO(bradfitz): populate username & group. os/user - // doesn't cache LookupId lookups, and lacks group - // lookup functions. + + // Best effort at populating Uname and Gname. + // The os/user functions may fail for any number of reasons + // (not implemented on that platform, cgo not enabled, etc). + if u, ok := userMap.Load(h.Uid); ok { + h.Uname = u.(string) + } else if u, err := user.LookupId(strconv.Itoa(h.Uid)); err == nil { + h.Uname = u.Username + userMap.Store(h.Uid, h.Uname) + } + if g, ok := groupMap.Load(h.Gid); ok { + h.Gname = g.(string) + } else if g, err := user.LookupGroupId(strconv.Itoa(h.Gid)); err == nil { + h.Gname = g.Name + groupMap.Store(h.Gid, h.Gname) + } + h.AccessTime = statAtime(sys) h.ChangeTime = statCtime(sys) - // TODO(bradfitz): major/minor device numbers? + + // Best effort at populating Devmajor and Devminor. + if h.Typeflag == TypeChar || h.Typeflag == TypeBlock { + dev := uint64(sys.Rdev) // May be int32 or uint32 + switch runtime.GOOS { + case "linux": + // Copied from golang.org/x/sys/unix/dev_linux.go. + major := uint32((dev & 0x00000000000fff00) >> 8) + major |= uint32((dev & 0xfffff00000000000) >> 32) + minor := uint32((dev & 0x00000000000000ff) >> 0) + minor |= uint32((dev & 0x00000ffffff00000) >> 12) + h.Devmajor, h.Devminor = int64(major), int64(minor) + case "darwin": + // Copied from golang.org/x/sys/unix/dev_darwin.go. + major := uint32((dev >> 24) & 0xff) + minor := uint32(dev & 0xffffff) + h.Devmajor, h.Devminor = int64(major), int64(minor) + case "dragonfly": + // Copied from golang.org/x/sys/unix/dev_dragonfly.go. + major := uint32((dev >> 8) & 0xff) + minor := uint32(dev & 0xffff00ff) + h.Devmajor, h.Devminor = int64(major), int64(minor) + case "freebsd": + // Copied from golang.org/x/sys/unix/dev_freebsd.go. + major := uint32((dev >> 8) & 0xff) + minor := uint32(dev & 0xffff00ff) + h.Devmajor, h.Devminor = int64(major), int64(minor) + case "netbsd": + // Copied from golang.org/x/sys/unix/dev_netbsd.go. + major := uint32((dev & 0x000fff00) >> 8) + minor := uint32((dev & 0x000000ff) >> 0) + minor |= uint32((dev & 0xfff00000) >> 12) + h.Devmajor, h.Devminor = int64(major), int64(minor) + case "openbsd": + // Copied from golang.org/x/sys/unix/dev_openbsd.go. + major := uint32((dev & 0x0000ff00) >> 8) + minor := uint32((dev & 0x000000ff) >> 0) + minor |= uint32((dev & 0xffff0000) >> 8) + h.Devmajor, h.Devminor = int64(major), int64(minor) + default: + // TODO: Implement solaris (see https://golang.org/issue/8106) + } + } return nil } diff --git a/vendor/github.com/dmcgowan/go-tar/strconv.go b/vendor/github.com/dmcgowan/go-tar/strconv.go index bb5b51c02..8bbd65cd1 100644 --- a/vendor/github.com/dmcgowan/go-tar/strconv.go +++ b/vendor/github.com/dmcgowan/go-tar/strconv.go @@ -12,26 +12,34 @@ import ( "time" ) +// hasNUL reports whether the NUL character exists within s. +func hasNUL(s string) bool { + return strings.IndexByte(s, 0) >= 0 +} + +// isASCII reports whether the input is an ASCII C-style string. func isASCII(s string) bool { for _, c := range s { - if c >= 0x80 { + if c >= 0x80 || c == 0x00 { return false } } return true } +// toASCII converts the input to an ASCII C-style string. +// This a best effort conversion, so invalid characters are dropped. func toASCII(s string) string { if isASCII(s) { return s } - var buf bytes.Buffer + b := make([]byte, 0, len(s)) for _, c := range s { - if c < 0x80 { - buf.WriteByte(byte(c)) + if c < 0x80 && c != 0x00 { + b = append(b, byte(c)) } } - return buf.String() + return string(b) } type parser struct { @@ -45,23 +53,28 @@ type formatter struct { // parseString parses bytes as a NUL-terminated C-style string. // If a NUL byte is not found then the whole slice is returned as a string. func (*parser) parseString(b []byte) string { - n := 0 - for n < len(b) && b[n] != 0 { - n++ + if i := bytes.IndexByte(b, 0); i >= 0 { + return string(b[:i]) } - return string(b[0:n]) + return string(b) } -// Write s into b, terminating it with a NUL if there is room. +// formatString copies s into b, NUL-terminating if possible. func (f *formatter) formatString(b []byte, s string) { if len(s) > len(b) { f.err = ErrFieldTooLong - return } - ascii := toASCII(s) - copy(b, ascii) - if len(ascii) < len(b) { - b[len(ascii)] = 0 + copy(b, s) + if len(s) < len(b) { + b[len(s)] = 0 + } + + // Some buggy readers treat regular files with a trailing slash + // in the V7 path field as a directory even though the full path + // recorded elsewhere (e.g., via PAX record) contains no trailing slash. + if len(s) > len(b) && b[len(b)-1] == '/' { + n := len(strings.TrimRight(s[:len(b)], "/")) + b[n] = 0 // Replace trailing slash with NUL terminator } } @@ -73,7 +86,7 @@ func (f *formatter) formatString(b []byte, s string) { // that the first byte can only be either 0x80 or 0xff. Thus, the first byte is // equivalent to the sign bit in two's complement form. func fitsInBase256(n int, x int64) bool { - var binBits = uint(n-1) * 8 + binBits := uint(n-1) * 8 return n >= 9 || (x >= -1<= 0; i-- { b[i] = byte(x) @@ -155,6 +174,11 @@ func (p *parser) parseOctal(b []byte) int64 { } func (f *formatter) formatOctal(b []byte, x int64) { + if !fitsInOctal(len(b), x) { + x = 0 // Last resort, just write zero + f.err = ErrFieldTooLong + } + s := strconv.FormatInt(x, 8) // Add leading zeros, but leave room for a NUL. if n := len(b) - len(s) - 1; n > 0 { @@ -163,6 +187,13 @@ func (f *formatter) formatOctal(b []byte, x int64) { f.formatString(b, s) } +// fitsInOctal reports whether the integer x fits in a field n-bytes long +// using octal encoding with the appropriate NUL terminator. +func fitsInOctal(n int, x int64) bool { + octBits := uint(n-1) * 3 + return x >= 0 && (n >= 22 || x < 1<= 0 { + return false + } + switch k { + case paxPath, paxLinkpath, paxUname, paxGname: + return !hasNUL(v) + default: + return !hasNUL(k) } - return record } diff --git a/vendor/github.com/dmcgowan/go-tar/writer.go b/vendor/github.com/dmcgowan/go-tar/writer.go index 596fb8b9e..2eed61934 100644 --- a/vendor/github.com/dmcgowan/go-tar/writer.go +++ b/vendor/github.com/dmcgowan/go-tar/writer.go @@ -4,12 +4,8 @@ package tar -// TODO(dsymonds): -// - catch more errors (no first header, etc.) - import ( "bytes" - "errors" "fmt" "io" "path" @@ -19,234 +15,365 @@ import ( "time" ) -var ( - ErrWriteTooLong = errors.New("archive/tar: write too long") - ErrFieldTooLong = errors.New("archive/tar: header field too long") - ErrWriteAfterClose = errors.New("archive/tar: write after close") - errInvalidHeader = errors.New("archive/tar: header field too long or contains invalid values") -) - -// A Writer provides sequential writing of a tar archive in POSIX.1 format. -// A tar archive consists of a sequence of files. -// Call WriteHeader to begin a new file, and then call Write to supply that file's data, -// writing at most hdr.Size bytes in total. +// Writer provides sequential writing of a tar archive. +// Write.WriteHeader begins a new file with the provided Header, +// and then Writer can be treated as an io.Writer to supply that file's data. type Writer struct { - w io.Writer - err error - nb int64 // number of unwritten bytes for current file entry - pad int64 // amount of padding to write after current file entry - closed bool - usedBinary bool // whether the binary numeric field extension was used - preferPax bool // use PAX header instead of binary numeric header - hdrBuff block // buffer to use in writeHeader when writing a regular header - paxHdrBuff block // buffer to use in writeHeader when writing a PAX header + w io.Writer + pad int64 // Amount of padding to write after current file entry + curr fileWriter // Writer for current file entry + hdr Header // Shallow copy of Header that is safe for mutations + blk block // Buffer to use as temporary local storage + + // err is a persistent error. + // It is only the responsibility of every exported method of Writer to + // ensure that this error is sticky. + err error } // NewWriter creates a new Writer writing to w. -func NewWriter(w io.Writer) *Writer { return &Writer{w: w} } +func NewWriter(w io.Writer) *Writer { + return &Writer{w: w, curr: ®FileWriter{w, 0}} +} -// Flush finishes writing the current file (optional). +type fileWriter interface { + io.Writer + fileState + + ReadFrom(io.Reader) (int64, error) +} + +// Flush finishes writing the current file's block padding. +// The current file must be fully written before Flush can be called. +// +// Deprecated: This is unnecessary as the next call to WriteHeader or Close +// will implicitly flush out the file's padding. func (tw *Writer) Flush() error { - if tw.nb > 0 { - tw.err = fmt.Errorf("archive/tar: missed writing %d bytes", tw.nb) - return tw.err - } - - n := tw.nb + tw.pad - for n > 0 && tw.err == nil { - nr := n - if nr > blockSize { - nr = blockSize - } - var nw int - nw, tw.err = tw.w.Write(zeroBlock[0:nr]) - n -= int64(nw) - } - tw.nb = 0 - tw.pad = 0 - return tw.err -} - -var ( - minTime = time.Unix(0, 0) - // There is room for 11 octal digits (33 bits) of mtime. - maxTime = minTime.Add((1<<33 - 1) * time.Second) -) - -// WriteHeader writes hdr and prepares to accept the file's contents. -// WriteHeader calls Flush if it is not the first header. -// Calling after a Close will return ErrWriteAfterClose. -func (tw *Writer) WriteHeader(hdr *Header) error { - return tw.writeHeader(hdr, true) -} - -// WriteHeader writes hdr and prepares to accept the file's contents. -// WriteHeader calls Flush if it is not the first header. -// Calling after a Close will return ErrWriteAfterClose. -// As this method is called internally by writePax header to allow it to -// suppress writing the pax header. -func (tw *Writer) writeHeader(hdr *Header, allowPax bool) error { - if tw.closed { - return ErrWriteAfterClose - } - if tw.err == nil { - tw.Flush() - } if tw.err != nil { return tw.err } - - // a map to hold pax header records, if any are needed - paxHeaders := make(map[string]string) - - // TODO(dsnet): we might want to use PAX headers for - // subsecond time resolution, but for now let's just capture - // too long fields or non ascii characters - - // We need to select which scratch buffer to use carefully, - // since this method is called recursively to write PAX headers. - // If allowPax is true, this is the non-recursive call, and we will use hdrBuff. - // If allowPax is false, we are being called by writePAXHeader, and hdrBuff is - // already being used by the non-recursive call, so we must use paxHdrBuff. - header := &tw.hdrBuff - if !allowPax { - header = &tw.paxHdrBuff + if nb := tw.curr.LogicalRemaining(); nb > 0 { + return fmt.Errorf("tar: missed writing %d bytes", nb) } - copy(header[:], zeroBlock[:]) - - // Wrappers around formatter that automatically sets paxHeaders if the - // argument extends beyond the capacity of the input byte slice. - var f formatter - var formatString = func(b []byte, s string, paxKeyword string) { - needsPaxHeader := paxKeyword != paxNone && len(s) > len(b) || !isASCII(s) - if needsPaxHeader { - paxHeaders[paxKeyword] = s - return - } - f.formatString(b, s) - } - var formatNumeric = func(b []byte, x int64, paxKeyword string) { - // Try octal first. - s := strconv.FormatInt(x, 8) - if len(s) < len(b) { - f.formatOctal(b, x) - return - } - - // If it is too long for octal, and PAX is preferred, use a PAX header. - if paxKeyword != paxNone && tw.preferPax { - f.formatOctal(b, 0) - s := strconv.FormatInt(x, 10) - paxHeaders[paxKeyword] = s - return - } - - tw.usedBinary = true - f.formatNumeric(b, x) - } - - // Handle out of range ModTime carefully. - var modTime int64 - if !hdr.ModTime.Before(minTime) && !hdr.ModTime.After(maxTime) { - modTime = hdr.ModTime.Unix() - } - - v7 := header.V7() - formatString(v7.Name(), hdr.Name, paxPath) - // TODO(dsnet): The GNU format permits the mode field to be encoded in - // base-256 format. Thus, we can use formatNumeric instead of formatOctal. - f.formatOctal(v7.Mode(), hdr.Mode) - formatNumeric(v7.UID(), int64(hdr.Uid), paxUid) - formatNumeric(v7.GID(), int64(hdr.Gid), paxGid) - formatNumeric(v7.Size(), hdr.Size, paxSize) - // TODO(dsnet): Consider using PAX for finer time granularity. - formatNumeric(v7.ModTime(), modTime, paxNone) - v7.TypeFlag()[0] = hdr.Typeflag - formatString(v7.LinkName(), hdr.Linkname, paxLinkpath) - - ustar := header.USTAR() - formatString(ustar.UserName(), hdr.Uname, paxUname) - formatString(ustar.GroupName(), hdr.Gname, paxGname) - formatNumeric(ustar.DevMajor(), hdr.Devmajor, paxNone) - formatNumeric(ustar.DevMinor(), hdr.Devminor, paxNone) - - // TODO(dsnet): The logic surrounding the prefix field is broken when trying - // to encode the header as GNU format. The challenge with the current logic - // is that we are unsure what format we are using at any given moment until - // we have processed *all* of the fields. The problem is that by the time - // all fields have been processed, some work has already been done to handle - // each field under the assumption that it is for one given format or - // another. In some situations, this causes the Writer to be confused and - // encode a prefix field when the format being used is GNU. Thus, producing - // an invalid tar file. - // - // As a short-term fix, we disable the logic to use the prefix field, which - // will force the badly generated GNU files to become encoded as being - // the PAX format. - // - // As an alternative fix, we could hard-code preferPax to be true. However, - // this is problematic for the following reasons: - // * The preferPax functionality is not tested at all. - // * This can result in headers that try to use both the GNU and PAX - // features at the same time, which is also wrong. - // - // The proper fix for this is to use a two-pass method: - // * The first pass simply determines what set of formats can possibly - // encode the given header. - // * The second pass actually encodes the header as that given format - // without worrying about violating the format. - // - // See the following: - // https://golang.org/issue/12594 - // https://golang.org/issue/17630 - // https://golang.org/issue/9683 - const usePrefix = false - - // try to use a ustar header when only the name is too long - _, paxPathUsed := paxHeaders[paxPath] - if usePrefix && !tw.preferPax && len(paxHeaders) == 1 && paxPathUsed { - prefix, suffix, ok := splitUSTARPath(hdr.Name) - if ok { - // Since we can encode in USTAR format, disable PAX header. - delete(paxHeaders, paxPath) - - // Update the path fields - formatString(v7.Name(), suffix, paxNone) - formatString(ustar.Prefix(), prefix, paxNone) - } - } - - if tw.usedBinary { - header.SetFormat(formatGNU) - } else { - header.SetFormat(formatUSTAR) - } - - // Check if there were any formatting errors. - if f.err != nil { - tw.err = f.err + if _, tw.err = tw.w.Write(zeroBlock[:tw.pad]); tw.err != nil { return tw.err } + tw.pad = 0 + return nil +} - if allowPax { - for k, v := range hdr.Xattrs { - paxHeaders[paxXattr+k] = v +// WriteHeader writes hdr and prepares to accept the file's contents. +// The Header.Size determines how many bytes can be written for the next file. +// If the current file is not fully written, then this returns an error. +// This implicitly flushes any padding necessary before writing the header. +func (tw *Writer) WriteHeader(hdr *Header) error { + if err := tw.Flush(); err != nil { + return err + } + tw.hdr = *hdr // Shallow copy of Header + + // Round ModTime and ignore AccessTime and ChangeTime unless + // the format is explicitly chosen. + // This ensures nominal usage of WriteHeader (without specifying the format) + // does not always result in the PAX format being chosen, which + // causes a 1KiB increase to every header. + if tw.hdr.Format == FormatUnknown { + tw.hdr.ModTime = tw.hdr.ModTime.Round(time.Second) + tw.hdr.AccessTime = time.Time{} + tw.hdr.ChangeTime = time.Time{} + } + + allowedFormats, paxHdrs, err := tw.hdr.allowedFormats() + switch { + case allowedFormats.has(FormatUSTAR): + tw.err = tw.writeUSTARHeader(&tw.hdr) + return tw.err + case allowedFormats.has(FormatPAX): + tw.err = tw.writePAXHeader(&tw.hdr, paxHdrs) + return tw.err + case allowedFormats.has(FormatGNU): + tw.err = tw.writeGNUHeader(&tw.hdr) + return tw.err + default: + return err // Non-fatal error + } +} + +func (tw *Writer) writeUSTARHeader(hdr *Header) error { + // Check if we can use USTAR prefix/suffix splitting. + var namePrefix string + if prefix, suffix, ok := splitUSTARPath(hdr.Name); ok { + namePrefix, hdr.Name = prefix, suffix + } + + // Pack the main header. + var f formatter + blk := tw.templateV7Plus(hdr, f.formatString, f.formatOctal) + f.formatString(blk.USTAR().Prefix(), namePrefix) + blk.SetFormat(FormatUSTAR) + if f.err != nil { + return f.err // Should never happen since header is validated + } + return tw.writeRawHeader(blk, hdr.Size, hdr.Typeflag) +} + +func (tw *Writer) writePAXHeader(hdr *Header, paxHdrs map[string]string) error { + realName, realSize := hdr.Name, hdr.Size + + // Handle sparse files. + var spd sparseDatas + var spb []byte + if len(hdr.SparseHoles) > 0 { + sph := append([]SparseEntry{}, hdr.SparseHoles...) // Copy sparse map + sph = alignSparseEntries(sph, hdr.Size) + spd = invertSparseEntries(sph, hdr.Size) + + // Format the sparse map. + hdr.Size = 0 // Replace with encoded size + spb = append(strconv.AppendInt(spb, int64(len(spd)), 10), '\n') + for _, s := range spd { + hdr.Size += s.Length + spb = append(strconv.AppendInt(spb, s.Offset, 10), '\n') + spb = append(strconv.AppendInt(spb, s.Length, 10), '\n') + } + pad := blockPadding(int64(len(spb))) + spb = append(spb, zeroBlock[:pad]...) + hdr.Size += int64(len(spb)) // Accounts for encoded sparse map + + // Add and modify appropriate PAX records. + dir, file := path.Split(realName) + hdr.Name = path.Join(dir, "GNUSparseFile.0", file) + paxHdrs[paxGNUSparseMajor] = "1" + paxHdrs[paxGNUSparseMinor] = "0" + paxHdrs[paxGNUSparseName] = realName + paxHdrs[paxGNUSparseRealSize] = strconv.FormatInt(realSize, 10) + paxHdrs[paxSize] = strconv.FormatInt(hdr.Size, 10) + delete(paxHdrs, paxPath) // Recorded by paxGNUSparseName + } + + // Write PAX records to the output. + isGlobal := hdr.Typeflag == TypeXGlobalHeader + if len(paxHdrs) > 0 || isGlobal { + // Sort keys for deterministic ordering. + var keys []string + for k := range paxHdrs { + keys = append(keys, k) + } + sort.Strings(keys) + + // Write each record to a buffer. + var buf bytes.Buffer + for _, k := range keys { + rec, err := formatPAXRecord(k, paxHdrs[k]) + if err != nil { + return err + } + buf.WriteString(rec) + } + + // Write the extended header file. + var name string + var flag byte + if isGlobal { + name = "GlobalHead.0.0" + flag = TypeXGlobalHeader + } else { + dir, file := path.Split(realName) + name = path.Join(dir, "PaxHeaders.0", file) + flag = TypeXHeader + } + data := buf.String() + if err := tw.writeRawFile(name, data, flag, FormatPAX); err != nil || isGlobal { + return err // Global headers return here } } - if len(paxHeaders) > 0 { - if !allowPax { - return errInvalidHeader + // Pack the main header. + var f formatter // Ignore errors since they are expected + fmtStr := func(b []byte, s string) { f.formatString(b, toASCII(s)) } + blk := tw.templateV7Plus(hdr, fmtStr, f.formatOctal) + blk.SetFormat(FormatPAX) + if err := tw.writeRawHeader(blk, hdr.Size, hdr.Typeflag); err != nil { + return err + } + + // Write the sparse map and setup the sparse writer if necessary. + if len(spd) > 0 { + // Use tw.curr since the sparse map is accounted for in hdr.Size. + if _, err := tw.curr.Write(spb); err != nil { + return err } - if err := tw.writePAXHeader(hdr, paxHeaders); err != nil { + tw.curr = &sparseFileWriter{tw.curr, spd, 0} + } + return nil +} + +func (tw *Writer) writeGNUHeader(hdr *Header) error { + // Use long-link files if Name or Linkname exceeds the field size. + const longName = "././@LongLink" + if len(hdr.Name) > nameSize { + data := hdr.Name + "\x00" + if err := tw.writeRawFile(longName, data, TypeGNULongName, FormatGNU); err != nil { + return err + } + } + if len(hdr.Linkname) > nameSize { + data := hdr.Linkname + "\x00" + if err := tw.writeRawFile(longName, data, TypeGNULongLink, FormatGNU); err != nil { return err } } - tw.nb = hdr.Size - tw.pad = (blockSize - (tw.nb % blockSize)) % blockSize - _, tw.err = tw.w.Write(header[:]) - return tw.err + // Pack the main header. + var f formatter // Ignore errors since they are expected + var spd sparseDatas + var spb []byte + blk := tw.templateV7Plus(hdr, f.formatString, f.formatNumeric) + if !hdr.AccessTime.IsZero() { + f.formatNumeric(blk.GNU().AccessTime(), hdr.AccessTime.Unix()) + } + if !hdr.ChangeTime.IsZero() { + f.formatNumeric(blk.GNU().ChangeTime(), hdr.ChangeTime.Unix()) + } + if hdr.Typeflag == TypeGNUSparse { + sph := append([]SparseEntry{}, hdr.SparseHoles...) // Copy sparse map + sph = alignSparseEntries(sph, hdr.Size) + spd = invertSparseEntries(sph, hdr.Size) + + // Format the sparse map. + formatSPD := func(sp sparseDatas, sa sparseArray) sparseDatas { + for i := 0; len(sp) > 0 && i < sa.MaxEntries(); i++ { + f.formatNumeric(sa.Entry(i).Offset(), sp[0].Offset) + f.formatNumeric(sa.Entry(i).Length(), sp[0].Length) + sp = sp[1:] + } + if len(sp) > 0 { + sa.IsExtended()[0] = 1 + } + return sp + } + sp2 := formatSPD(spd, blk.GNU().Sparse()) + for len(sp2) > 0 { + var spHdr block + sp2 = formatSPD(sp2, spHdr.Sparse()) + spb = append(spb, spHdr[:]...) + } + + // Update size fields in the header block. + realSize := hdr.Size + hdr.Size = 0 // Encoded size; does not account for encoded sparse map + for _, s := range spd { + hdr.Size += s.Length + } + copy(blk.V7().Size(), zeroBlock[:]) // Reset field + f.formatNumeric(blk.V7().Size(), hdr.Size) + f.formatNumeric(blk.GNU().RealSize(), realSize) + } + blk.SetFormat(FormatGNU) + if err := tw.writeRawHeader(blk, hdr.Size, hdr.Typeflag); err != nil { + return err + } + + // Write the extended sparse map and setup the sparse writer if necessary. + if len(spd) > 0 { + // Use tw.w since the sparse map is not accounted for in hdr.Size. + if _, err := tw.w.Write(spb); err != nil { + return err + } + tw.curr = &sparseFileWriter{tw.curr, spd, 0} + } + return nil +} + +type ( + stringFormatter func([]byte, string) + numberFormatter func([]byte, int64) +) + +// templateV7Plus fills out the V7 fields of a block using values from hdr. +// It also fills out fields (uname, gname, devmajor, devminor) that are +// shared in the USTAR, PAX, and GNU formats using the provided formatters. +// +// The block returned is only valid until the next call to +// templateV7Plus or writeRawFile. +func (tw *Writer) templateV7Plus(hdr *Header, fmtStr stringFormatter, fmtNum numberFormatter) *block { + tw.blk.Reset() + + modTime := hdr.ModTime + if modTime.IsZero() { + modTime = time.Unix(0, 0) + } + + v7 := tw.blk.V7() + v7.TypeFlag()[0] = hdr.Typeflag + fmtStr(v7.Name(), hdr.Name) + fmtStr(v7.LinkName(), hdr.Linkname) + fmtNum(v7.Mode(), hdr.Mode) + fmtNum(v7.UID(), int64(hdr.Uid)) + fmtNum(v7.GID(), int64(hdr.Gid)) + fmtNum(v7.Size(), hdr.Size) + fmtNum(v7.ModTime(), modTime.Unix()) + + ustar := tw.blk.USTAR() + fmtStr(ustar.UserName(), hdr.Uname) + fmtStr(ustar.GroupName(), hdr.Gname) + fmtNum(ustar.DevMajor(), hdr.Devmajor) + fmtNum(ustar.DevMinor(), hdr.Devminor) + + return &tw.blk +} + +// writeRawFile writes a minimal file with the given name and flag type. +// It uses format to encode the header format and will write data as the body. +// It uses default values for all of the other fields (as BSD and GNU tar does). +func (tw *Writer) writeRawFile(name, data string, flag byte, format Format) error { + tw.blk.Reset() + + // Best effort for the filename. + name = toASCII(name) + if len(name) > nameSize { + name = name[:nameSize] + } + name = strings.TrimRight(name, "/") + + var f formatter + v7 := tw.blk.V7() + v7.TypeFlag()[0] = flag + f.formatString(v7.Name(), name) + f.formatOctal(v7.Mode(), 0) + f.formatOctal(v7.UID(), 0) + f.formatOctal(v7.GID(), 0) + f.formatOctal(v7.Size(), int64(len(data))) // Must be < 8GiB + f.formatOctal(v7.ModTime(), 0) + tw.blk.SetFormat(format) + if f.err != nil { + return f.err // Only occurs if size condition is violated + } + + // Write the header and data. + if err := tw.writeRawHeader(&tw.blk, int64(len(data)), flag); err != nil { + return err + } + _, err := io.WriteString(tw, data) + return err +} + +// writeRawHeader writes the value of blk, regardless of its value. +// It sets up the Writer such that it can accept a file of the given size. +// If the flag is a special header-only flag, then the size is treated as zero. +func (tw *Writer) writeRawHeader(blk *block, size int64, flag byte) error { + if err := tw.Flush(); err != nil { + return err + } + if _, err := tw.w.Write(blk[:]); err != nil { + return err + } + if isHeaderOnlyType(flag) { + size = 0 + } + tw.curr = ®FileWriter{tw.w, size} + tw.pad = blockPadding(size) + return nil } // splitUSTARPath splits a path according to USTAR prefix and suffix rules. @@ -270,95 +397,233 @@ func splitUSTARPath(name string) (prefix, suffix string, ok bool) { return name[:i], name[i+1:], true } -// writePaxHeader writes an extended pax header to the -// archive. -func (tw *Writer) writePAXHeader(hdr *Header, paxHeaders map[string]string) error { - // Prepare extended header - ext := new(Header) - ext.Typeflag = TypeXHeader - // Setting ModTime is required for reader parsing to - // succeed, and seems harmless enough. - ext.ModTime = hdr.ModTime - // The spec asks that we namespace our pseudo files - // with the current pid. However, this results in differing outputs - // for identical inputs. As such, the constant 0 is now used instead. - // golang.org/issue/12358 - dir, file := path.Split(hdr.Name) - fullName := path.Join(dir, "PaxHeaders.0", file) - - ascii := toASCII(fullName) - if len(ascii) > nameSize { - ascii = ascii[:nameSize] - } - ext.Name = ascii - // Construct the body - var buf bytes.Buffer - - // Keys are sorted before writing to body to allow deterministic output. - keys := make([]string, 0, len(paxHeaders)) - for k := range paxHeaders { - keys = append(keys, k) - } - sort.Strings(keys) - - for _, k := range keys { - fmt.Fprint(&buf, formatPAXRecord(k, paxHeaders[k])) - } - - ext.Size = int64(len(buf.Bytes())) - if err := tw.writeHeader(ext, false); err != nil { - return err - } - if _, err := tw.Write(buf.Bytes()); err != nil { - return err - } - if err := tw.Flush(); err != nil { - return err - } - return nil -} - -// Write writes to the current entry in the tar archive. +// Write writes to the current file in the tar archive. // Write returns the error ErrWriteTooLong if more than -// hdr.Size bytes are written after WriteHeader. -func (tw *Writer) Write(b []byte) (n int, err error) { - if tw.closed { - err = ErrWriteAfterClose - return +// Header.Size bytes are written after WriteHeader. +// +// If the current file is sparse, then the regions marked as a hole +// must be written as NUL-bytes. +// +// Calling Write on special types like TypeLink, TypeSymlink, TypeChar, +// TypeBlock, TypeDir, and TypeFifo returns (0, ErrWriteTooLong) regardless +// of what the Header.Size claims. +func (tw *Writer) Write(b []byte) (int, error) { + if tw.err != nil { + return 0, tw.err } - overwrite := false - if int64(len(b)) > tw.nb { - b = b[0:tw.nb] - overwrite = true + n, err := tw.curr.Write(b) + if err != nil && err != ErrWriteTooLong { + tw.err = err } - n, err = tw.w.Write(b) - tw.nb -= int64(n) - if err == nil && overwrite { - err = ErrWriteTooLong - return - } - tw.err = err - return + return n, err } -// Close closes the tar archive, flushing any unwritten -// data to the underlying writer. -func (tw *Writer) Close() error { - if tw.err != nil || tw.closed { - return tw.err +// ReadFrom populates the content of the current file by reading from r. +// The bytes read must match the number of remaining bytes in the current file. +// +// If the current file is sparse and r is an io.ReadSeeker, +// then ReadFrom uses Seek to skip past holes defined in Header.SparseHoles, +// assuming that skipped regions are all NULs. +// This always reads the last byte to ensure r is the right size. +func (tw *Writer) ReadFrom(r io.Reader) (int64, error) { + if tw.err != nil { + return 0, tw.err + } + n, err := tw.curr.ReadFrom(r) + if err != nil && err != ErrWriteTooLong { + tw.err = err + } + return n, err +} + +// Close closes the tar archive by flushing the padding, and writing the footer. +// If the current file (from a prior call to WriteHeader) is not fully written, +// then this returns an error. +func (tw *Writer) Close() error { + if tw.err == ErrWriteAfterClose { + return nil } - tw.Flush() - tw.closed = true if tw.err != nil { return tw.err } - // trailer: two zero blocks - for i := 0; i < 2; i++ { - _, tw.err = tw.w.Write(zeroBlock[:]) - if tw.err != nil { - break + // Trailer: two zero blocks. + err := tw.Flush() + for i := 0; i < 2 && err == nil; i++ { + _, err = tw.w.Write(zeroBlock[:]) + } + + // Ensure all future actions are invalid. + tw.err = ErrWriteAfterClose + return err // Report IO errors +} + +// regFileWriter is a fileWriter for writing data to a regular file entry. +type regFileWriter struct { + w io.Writer // Underlying Writer + nb int64 // Number of remaining bytes to write +} + +func (fw *regFileWriter) Write(b []byte) (n int, err error) { + overwrite := int64(len(b)) > fw.nb + if overwrite { + b = b[:fw.nb] + } + if len(b) > 0 { + n, err = fw.w.Write(b) + fw.nb -= int64(n) + } + switch { + case err != nil: + return n, err + case overwrite: + return n, ErrWriteTooLong + default: + return n, nil + } +} + +func (fw *regFileWriter) ReadFrom(r io.Reader) (int64, error) { + return io.Copy(struct{ io.Writer }{fw}, r) +} + +func (fw regFileWriter) LogicalRemaining() int64 { + return fw.nb +} +func (fw regFileWriter) PhysicalRemaining() int64 { + return fw.nb +} + +// sparseFileWriter is a fileWriter for writing data to a sparse file entry. +type sparseFileWriter struct { + fw fileWriter // Underlying fileWriter + sp sparseDatas // Normalized list of data fragments + pos int64 // Current position in sparse file +} + +func (sw *sparseFileWriter) Write(b []byte) (n int, err error) { + overwrite := int64(len(b)) > sw.LogicalRemaining() + if overwrite { + b = b[:sw.LogicalRemaining()] + } + + b0 := b + endPos := sw.pos + int64(len(b)) + for endPos > sw.pos && err == nil { + var nf int // Bytes written in fragment + dataStart, dataEnd := sw.sp[0].Offset, sw.sp[0].endOffset() + if sw.pos < dataStart { // In a hole fragment + bf := b[:min(int64(len(b)), dataStart-sw.pos)] + nf, err = zeroWriter{}.Write(bf) + } else { // In a data fragment + bf := b[:min(int64(len(b)), dataEnd-sw.pos)] + nf, err = sw.fw.Write(bf) + } + b = b[nf:] + sw.pos += int64(nf) + if sw.pos >= dataEnd && len(sw.sp) > 1 { + sw.sp = sw.sp[1:] // Ensure last fragment always remains } } - return tw.err + + n = len(b0) - len(b) + switch { + case err == ErrWriteTooLong: + return n, errMissData // Not possible; implies bug in validation logic + case err != nil: + return n, err + case sw.LogicalRemaining() == 0 && sw.PhysicalRemaining() > 0: + return n, errUnrefData // Not possible; implies bug in validation logic + case overwrite: + return n, ErrWriteTooLong + default: + return n, nil + } +} + +func (sw *sparseFileWriter) ReadFrom(r io.Reader) (n int64, err error) { + rs, ok := r.(io.ReadSeeker) + if ok { + if _, err := rs.Seek(0, io.SeekCurrent); err != nil { + ok = false // Not all io.Seeker can really seek + } + } + if !ok { + return io.Copy(struct{ io.Writer }{sw}, r) + } + + var readLastByte bool + pos0 := sw.pos + for sw.LogicalRemaining() > 0 && !readLastByte && err == nil { + var nf int64 // Size of fragment + dataStart, dataEnd := sw.sp[0].Offset, sw.sp[0].endOffset() + if sw.pos < dataStart { // In a hole fragment + nf = dataStart - sw.pos + if sw.PhysicalRemaining() == 0 { + readLastByte = true + nf-- + } + _, err = rs.Seek(nf, io.SeekCurrent) + } else { // In a data fragment + nf = dataEnd - sw.pos + nf, err = io.CopyN(sw.fw, rs, nf) + } + sw.pos += nf + if sw.pos >= dataEnd && len(sw.sp) > 1 { + sw.sp = sw.sp[1:] // Ensure last fragment always remains + } + } + + // If the last fragment is a hole, then seek to 1-byte before EOF, and + // read a single byte to ensure the file is the right size. + if readLastByte && err == nil { + _, err = mustReadFull(rs, []byte{0}) + sw.pos++ + } + + n = sw.pos - pos0 + switch { + case err == io.EOF: + return n, io.ErrUnexpectedEOF + case err == ErrWriteTooLong: + return n, errMissData // Not possible; implies bug in validation logic + case err != nil: + return n, err + case sw.LogicalRemaining() == 0 && sw.PhysicalRemaining() > 0: + return n, errUnrefData // Not possible; implies bug in validation logic + default: + return n, ensureEOF(rs) + } +} + +func (sw sparseFileWriter) LogicalRemaining() int64 { + return sw.sp[len(sw.sp)-1].endOffset() - sw.pos +} +func (sw sparseFileWriter) PhysicalRemaining() int64 { + return sw.fw.PhysicalRemaining() +} + +// zeroWriter may only be written with NULs, otherwise it returns errWriteHole. +type zeroWriter struct{} + +func (zeroWriter) Write(b []byte) (int, error) { + for i, c := range b { + if c != 0 { + return i, errWriteHole + } + } + return len(b), nil +} + +// ensureEOF checks whether r is at EOF, reporting ErrWriteTooLong if not so. +func ensureEOF(r io.Reader) error { + n, err := tryReadFull(r, []byte{0}) + switch { + case n > 0: + return ErrWriteTooLong + case err == io.EOF: + return nil + default: + return err + } } diff --git a/worker/oci/spec_unix.go b/worker/oci/spec_unix.go index bfb1b97ea..40e20f7b6 100644 --- a/worker/oci/spec_unix.go +++ b/worker/oci/spec_unix.go @@ -7,10 +7,10 @@ import ( "path" "sync" - "github.com/containerd/containerd" "github.com/containerd/containerd/containers" "github.com/containerd/containerd/mount" "github.com/containerd/containerd/namespaces" + "github.com/containerd/containerd/oci" "github.com/mitchellh/hashstructure" "github.com/moby/buildkit/snapshot" "github.com/moby/buildkit/worker" @@ -31,10 +31,10 @@ func GenerateSpec(ctx context.Context, meta worker.Meta, mounts []worker.Mount, } // Note that containerd.GenerateSpec is namespaced so as to make // specs.Linux.CgroupsPath namespaced - s, err := containerd.GenerateSpec(ctx, nil, c, - containerd.WithHostNamespace(specs.NetworkNamespace), - containerd.WithHostResolvconf, - containerd.WithHostHostsFile, + s, err := oci.GenerateSpec(ctx, nil, c, + oci.WithHostNamespace(specs.NetworkNamespace), + oci.WithHostResolvconf, + oci.WithHostHostsFile, ) if err != nil { return nil, nil, err