api/list: Expose manifests

Add `Manifests` field to `ImageSummary` which exposes all image
manifests (which includes other blobs using the image media type, like
buildkit attestations).

There's also a new `manifests` query field that needs to be set in order
for the response to contain the new information.

Signed-off-by: Paweł Gronowski <pawel.gronowski@docker.com>
(cherry picked from commit 050afe1e1a)
Signed-off-by: Paweł Gronowski <pawel.gronowski@docker.com>
This commit is contained in:
Paweł Gronowski
2024-03-07 20:56:00 +01:00
parent bb2fec6425
commit 89757f83ff
10 changed files with 482 additions and 29 deletions

View File

@@ -423,10 +423,16 @@ func (ir *imageRouter) getImagesJSON(ctx context.Context, w http.ResponseWriter,
sharedSize = httputils.BoolValue(r, "shared-size")
}
var manifests bool
if versions.GreaterThanOrEqualTo(version, "1.47") {
manifests = httputils.BoolValue(r, "manifests")
}
images, err := ir.backend.Images(ctx, imagetypes.ListOptions{
All: httputils.BoolValue(r, "all"),
Filters: imageFilters,
SharedSize: sharedSize,
Manifests: manifests,
})
if err != nil {
return err

View File

@@ -2265,6 +2265,19 @@ definitions:
x-nullable: false
type: "integer"
example: 2
Manifests:
description: |
Manifests is a list of manifests available in this image.
It provides a more detailed view of the platform-specific image manifests
or other image-attached data like build attestations.
WARNING: This is experimental and may change at any time without any backward
compatibility.
type: "array"
x-nullable: false
x-omitempty: true
items:
$ref: "#/definitions/ImageManifestSummary"
AuthConfig:
type: "object"
@@ -6644,6 +6657,120 @@ definitions:
additionalProperties:
type: "string"
ImageManifestSummary:
x-go-name: "ManifestSummary"
description: |
ImageManifestSummary represents a summary of an image manifest.
type: "object"
required: ["ID", "Descriptor", "Available", "Size", "Kind"]
properties:
ID:
description: |
ID is the content-addressable ID of an image and is the same as the
digest of the image manifest.
type: "string"
example: "sha256:95869fbcf224d947ace8d61d0e931d49e31bb7fc67fffbbe9c3198c33aa8e93f"
Descriptor:
$ref: "#/definitions/OCIDescriptor"
Available:
description: Indicates whether all the child content (image config, layers) is fully available locally.
type: "boolean"
example: true
Size:
type: "object"
x-nullable: false
required: ["Content", "Total"]
properties:
Total:
type: "integer"
format: "int64"
example: 8213251
description: |
Total is the total size (in bytes) of all the locally present
data (both distributable and non-distributable) that's related to
this manifest and its children.
This equal to the sum of [Content] size AND all the sizes in the
[Size] struct present in the Kind-specific data struct.
For example, for an image kind (Kind == "image")
this would include the size of the image content and unpacked
image snapshots ([Size.Content] + [ImageData.Size.Unpacked]).
Content:
description: |
Content is the size (in bytes) of all the locally present
content in the content store (e.g. image config, layers)
referenced by this manifest and its children.
This only includes blobs in the content store.
type: "integer"
format: "int64"
example: 3987495
Kind:
type: "string"
example: "image"
enum:
- "image"
- "attestation"
- "unknown"
description: |
The kind of the manifest.
kind | description
-------------|-----------------------------------------------------------
image | Image manifest that can be used to start a container.
attestation | Attestation manifest produced by the Buildkit builder for a specific image manifest.
ImageData:
description: |
The image data for the image manifest.
This field is only populated when Kind is "image".
type: "object"
x-nullable: true
x-omitempty: true
required: ["Platform", "Containers", "Size", "UnpackedSize"]
properties:
Platform:
$ref: "#/definitions/OCIPlatform"
description: |
OCI platform of the image. This will be the platform specified in the
manifest descriptor from the index/manifest list.
If it's not available, it will be obtained from the image config.
Containers:
description: |
The IDs of the containers that are using this image.
type: "array"
items:
type: "string"
example: ["ede54ee1fda366ab42f824e8a5ffd195155d853ceaec74a927f249ea270c7430", "abadbce344c096744d8d6071a90d474d28af8f1034b5ea9fb03c3f4bfc6d005e"]
Size:
type: "object"
x-nullable: false
required: ["Unpacked"]
properties:
Unpacked:
type: "integer"
format: "int64"
example: 3987495
description: |
Unpacked is the size (in bytes) of the locally unpacked
(uncompressed) image content that's directly usable by the containers
running this image.
It's independent of the distributable content - e.g.
the image might still have an unpacked data that's still used by
some container even when the distributable/compressed content is
already gone.
AttestationData:
description: |
The image data for the attestation manifest.
This field is only populated when Kind is "attestation".
type: "object"
x-nullable: true
x-omitempty: true
required: ["For"]
properties:
For:
description: |
The digest of the image manifest that this attestation is for.
type: "string"
example: "sha256:95869fbcf224d947ace8d61d0e931d49e31bb7fc67fffbbe9c3198c33aa8e93f"
paths:
/containers/json:
get:
@@ -8622,6 +8749,11 @@ paths:
description: "Show digest information as a `RepoDigests` field on each image."
type: "boolean"
default: false
- name: "manifests"
in: "query"
description: "Include `Manifests` in the image summary."
type: "boolean"
default: false
tags: ["Image"]
/build:
post:

View File

@@ -0,0 +1,99 @@
package image
import (
"github.com/opencontainers/go-digest"
ocispec "github.com/opencontainers/image-spec/specs-go/v1"
)
type ManifestKind string
const (
ManifestKindImage ManifestKind = "image"
ManifestKindAttestation ManifestKind = "attestation"
ManifestKindUnknown ManifestKind = "unknown"
)
type ManifestSummary struct {
// ID is the content-addressable ID of an image and is the same as the
// digest of the image manifest.
//
// Required: true
ID string `json:"ID"`
// Descriptor is the OCI descriptor of the image.
//
// Required: true
Descriptor ocispec.Descriptor `json:"Descriptor"`
// Indicates whether all the child content (image config, layers) is
// fully available locally
//
// Required: true
Available bool `json:"Available"`
// Size is the size information of the content related to this manifest.
// Note: These sizes only take the locally available content into account.
//
// Required: true
Size struct {
// Content is the size (in bytes) of all the locally present
// content in the content store (e.g. image config, layers)
// referenced by this manifest and its children.
// This only includes blobs in the content store.
Content int64 `json:"Content"`
// Total is the total size (in bytes) of all the locally present
// data (both distributable and non-distributable) that's related to
// this manifest and its children.
// This equal to the sum of [Content] size AND all the sizes in the
// [Size] struct present in the Kind-specific data struct.
// For example, for an image kind (Kind == ManifestKindImage),
// this would include the size of the image content and unpacked
// image snapshots ([Size.Content] + [ImageData.Size.Unpacked]).
Total int64 `json:"Total"`
} `json:"Size"`
// Kind is the kind of the image manifest.
//
// Required: true
Kind ManifestKind `json:"Kind"`
// Fields below are specific to the kind of the image manifest.
// Present only if Kind == ManifestKindImage.
ImageData *ImageProperties `json:"ImageData,omitempty"`
// Present only if Kind == ManifestKindAttestation.
AttestationData *AttestationProperties `json:"AttestationData,omitempty"`
}
type ImageProperties struct {
// Platform is the OCI platform object describing the platform of the image.
//
// Required: true
Platform ocispec.Platform `json:"Platform"`
Size struct {
// Unpacked is the size (in bytes) of the locally unpacked
// (uncompressed) image content that's directly usable by the containers
// running this image.
// It's independent of the distributable content - e.g.
// the image might still have an unpacked data that's still used by
// some container even when the distributable/compressed content is
// already gone.
//
// Required: true
Unpacked int64 `json:"Unpacked"`
}
// Containers is an array containing the IDs of the containers that are
// using this image.
//
// Required: true
Containers []string `json:"Containers"`
}
type AttestationProperties struct {
// For is the digest of the image manifest that this attestation is for.
For digest.Digest `json:"For"`
}

View File

@@ -76,6 +76,9 @@ type ListOptions struct {
// ContainerCount indicates whether container count should be computed.
ContainerCount bool
// Manifests indicates whether the image manifests should be returned.
Manifests bool
}
// RemoveOptions holds parameters to remove images.

View File

@@ -42,6 +42,14 @@ type Summary struct {
// Required: true
ParentID string `json:"ParentId"`
// Manifests is a list of image manifests available in this image. It
// provides a more detailed view of the platform-specific image manifests or
// other image-attached data like build attestations.
//
// WARNING: This is experimental and may change at any time without any backward
// compatibility.
Manifests []ManifestSummary `json:"Manifests,omitempty"`
// List of content-addressable digests of locally available image manifests
// that the image is referenced from. Multiple manifests can refer to the
// same image.

View File

@@ -11,6 +11,11 @@ import (
)
// ImageList returns a list of images in the docker host.
//
// Experimental: Setting the [options.Manifest] will populate
// [image.Summary.Manifests] with information about image manifests.
// This is experimental and might change in the future without any backward
// compatibility.
func (cli *Client) ImageList(ctx context.Context, options image.ListOptions) ([]image.Summary, error) {
var images []image.Summary
@@ -47,6 +52,9 @@ func (cli *Client) ImageList(ctx context.Context, options image.ListOptions) ([]
if options.SharedSize && versions.GreaterThanOrEqualTo(cli.version, "1.42") {
query.Set("shared-size", "1")
}
if options.Manifests && versions.GreaterThanOrEqualTo(cli.version, "1.47") {
query.Set("manifests", "1")
}
serverResp, err := cli.get(ctx, "/images/json", query, nil)
defer ensureReaderClosed(serverResp)

View File

@@ -23,6 +23,7 @@ import (
timetypes "github.com/docker/docker/api/types/time"
"github.com/docker/docker/container"
"github.com/docker/docker/errdefs"
"github.com/moby/buildkit/util/attestation"
dockerspec "github.com/moby/docker-image-spec/specs-go/v1"
"github.com/opencontainers/go-digest"
"github.com/opencontainers/image-spec/identity"
@@ -209,6 +210,8 @@ func (i *ImageService) Images(ctx context.Context, opts imagetypes.ListOptions)
func (i *ImageService) imageSummary(ctx context.Context, img images.Image, platformMatcher platforms.MatchComparer,
opts imagetypes.ListOptions, tagsByDigest map[digest.Digest][]string,
) (_ *imagetypes.Summary, allChainIDs []digest.Digest, _ error) {
var manifestSummaries []imagetypes.ManifestSummary
// Total size of the image including all its platform
var totalSize int64
@@ -222,67 +225,149 @@ func (i *ImageService) imageSummary(ctx context.Context, img images.Image, platf
var best *ImageManifest
var bestPlatform ocispec.Platform
err := i.walkImageManifests(ctx, img, func(img *ImageManifest) error {
if isPseudo, err := img.IsPseudoImage(ctx); isPseudo || err != nil {
err := i.walkReachableImageManifests(ctx, img, func(img *ImageManifest) error {
target := img.Target()
logger := log.G(ctx).WithFields(log.Fields{
"image": img.Name(),
"digest": target.Digest,
"manifest": target,
})
available, err := img.CheckContentAvailable(ctx)
if err != nil && !errdefs.IsNotFound(err) {
logger.WithError(err).Warn("checking availability of platform specific manifest failed")
return nil
}
available, err := img.CheckContentAvailable(ctx)
mfstSummary := imagetypes.ManifestSummary{
ID: target.Digest.String(),
Available: available,
Descriptor: target,
Kind: imagetypes.ManifestKindUnknown,
}
if opts.Manifests {
defer func() {
// If the platform is available, prepend it to the list of platforms
// otherwise append it at the end.
if available {
manifestSummaries = append([]imagetypes.ManifestSummary{mfstSummary}, manifestSummaries...)
} else {
manifestSummaries = append(manifestSummaries, mfstSummary)
}
}()
}
contentSize, err := img.Size(ctx)
if err != nil {
log.G(ctx).WithFields(log.Fields{
"error": err,
"manifest": img.Target(),
"image": img.Name(),
}).Warn("checking availability of platform specific manifest failed")
if !cerrdefs.IsNotFound(err) {
logger.WithError(err).Warn("failed to determine size")
}
} else {
mfstSummary.Size.Content = contentSize
totalSize += contentSize
mfstSummary.Size.Total = totalSize
}
isPseudo, err := img.IsPseudoImage(ctx)
// Ignore not found error as it's expected in case where the image is
// not fully available. Otherwise, just continue to the next manifest,
// so we don't error out the whole list in case the error is related to
// the content itself (e.g. corrupted data) or just manifest kind that
// we don't know about (yet).
if err != nil && !errdefs.IsNotFound(err) {
logger.WithError(err).Debug("pseudo image check failed")
return nil
}
logger = logger.WithField("isPseudo", isPseudo)
if isPseudo {
if img.IsAttestation() {
if s := target.Annotations[attestation.DockerAnnotationReferenceDigest]; s != "" {
dgst, err := digest.Parse(s)
if err != nil {
logger.WithError(err).Warn("failed to parse attestation digest")
return nil
}
mfstSummary.Kind = imagetypes.ManifestKindAttestation
mfstSummary.AttestationData = &imagetypes.AttestationProperties{For: dgst}
}
}
return nil
}
mfstSummary.Kind = imagetypes.ManifestKindImage
mfstSummary.ImageData = &imagetypes.ImageProperties{}
if target.Platform != nil {
mfstSummary.ImageData.Platform = *target.Platform
}
if !available {
return nil
}
conf, err := img.Config(ctx)
if err != nil {
return err
logger.WithError(err).Warn("failed to read image config")
return nil
}
var dockerImage dockerspec.DockerOCIImage
if err := readConfig(ctx, i.content, conf, &dockerImage); err != nil {
return err
logger.WithError(err).Warn("failed to read image config")
return nil
}
target := img.Target()
if target.Platform == nil {
mfstSummary.ImageData.Platform = dockerImage.Platform
}
diffIDs, err := img.RootFS(ctx)
if err != nil {
return err
logger.WithError(err).Warn("failed to read image config")
return nil
}
chainIDs := identity.ChainIDs(diffIDs)
ts, _, err := i.singlePlatformSize(ctx, img)
prevContentSize := contentSize
unpackedSize, contentSize, err := i.singlePlatformSize(ctx, img)
if err != nil {
return err
logger.WithError(err).Warn("failed to determine platform specific size")
return nil
}
totalSize += ts
// If the image-specific content size calculation produces different result
// than the "generic" one, adjust the total size with the difference.
if prevContentSize != contentSize {
logger.WithFields(log.Fields{
"prevSize": prevContentSize,
"contentSize": contentSize,
}).Debug("content size calculation mismatch")
totalSize += contentSize - prevContentSize
}
totalSize += unpackedSize
mfstSummary.Size.Total = totalSize
mfstSummary.ImageData.Size.Unpacked = unpackedSize
allChainsIDs = append(allChainsIDs, chainIDs...)
if opts.ContainerCount {
i.containers.ApplyAll(func(c *container.Container) {
if c.ImageManifest != nil && c.ImageManifest.Digest == target.Digest {
mfstSummary.ImageData.Containers = append(mfstSummary.ImageData.Containers, c.ID)
containersCount++
}
})
}
var platform ocispec.Platform
if target.Platform != nil {
platform = *target.Platform
} else {
platform = dockerImage.Platform
}
platform := mfstSummary.ImageData.Platform
// Filter out platforms that don't match the requested platform. Do it
// after the size, container count and chainIDs are summed up to have
// the single combined entry still represent the whole multi-platform
@@ -322,6 +407,7 @@ func (i *ImageService) imageSummary(ctx context.Context, img images.Image, platf
return nil, nil, err
}
image.Size = totalSize
image.Manifests = manifestSummaries
if opts.ContainerCount {
image.Containers = containersCount
@@ -329,7 +415,7 @@ func (i *ImageService) imageSummary(ctx context.Context, img images.Image, platf
return image, allChainsIDs, nil
}
func (i *ImageService) singlePlatformSize(ctx context.Context, imgMfst *ImageManifest) (totalSize int64, contentSize int64, _ error) {
func (i *ImageService) singlePlatformSize(ctx context.Context, imgMfst *ImageManifest) (unpackedSize int64, contentSize int64, _ error) {
// TODO(thaJeztah): do we need to take multiple snapshotters into account? See https://github.com/moby/moby/issues/45273
snapshotter := i.snapshotterService(i.snapshotter)
@@ -355,10 +441,7 @@ func (i *ImageService) singlePlatformSize(ctx context.Context, imgMfst *ImageMan
return -1, -1, err
}
// totalSize is the size of the image's packed layers and snapshots
// (unpacked layers) combined.
totalSize = contentSize + unpackedUsage.Size
return totalSize, contentSize, nil
return unpackedUsage.Size, contentSize, nil
}
func (i *ImageService) singlePlatformImage(ctx context.Context, contentStore content.Store, repoTags []string, imageManifest *ImageManifest) (*imagetypes.Summary, error) {
@@ -400,11 +483,15 @@ func (i *ImageService) singlePlatformImage(ctx context.Context, contentStore con
return nil, err
}
totalSize, _, err := i.singlePlatformSize(ctx, imageManifest)
unpackedSize, contentSize, err := i.singlePlatformSize(ctx, imageManifest)
if err != nil {
return nil, errors.Wrapf(err, "failed to calculate size of image %s", imageManifest.Name())
}
// totalSize is the size of the image's packed layers and snapshots
// (unpacked layers) combined.
totalSize := contentSize + unpackedSize
summary := &imagetypes.Summary{
ParentID: rawImg.Labels[imageLabelClassicBuilderParent],
ID: target.String(),

View File

@@ -123,6 +123,10 @@ func TestImageList(t *testing.T) {
assert.Check(t, is.Equal(all[0].ID, multilayer.Manifests[0].Digest.String()))
assert.Check(t, is.DeepEqual(all[0].RepoTags, []string{"multilayer:latest"}))
assert.Check(t, is.Len(all[0].Manifests, 1))
assert.Check(t, all[0].Manifests[0].Available)
assert.Check(t, is.Equal(all[0].Manifests[0].Kind, imagetypes.ManifestKindImage))
},
},
{
@@ -133,6 +137,18 @@ func TestImageList(t *testing.T) {
assert.Check(t, is.Equal(all[0].ID, twoplatform.Manifests[0].Digest.String()))
assert.Check(t, is.DeepEqual(all[0].RepoTags, []string{"twoplatform:latest"}))
i := all[0]
assert.Check(t, is.Len(i.Manifests, 2))
assert.Check(t, is.Equal(i.Manifests[0].Kind, imagetypes.ManifestKindImage))
if assert.Check(t, i.Manifests[0].ImageData != nil) {
assert.Check(t, is.Equal(i.Manifests[0].ImageData.Platform.Architecture, "arm64"))
}
assert.Check(t, is.Equal(i.Manifests[1].Kind, imagetypes.ManifestKindImage))
if assert.Check(t, i.Manifests[1].ImageData != nil) {
assert.Check(t, is.Equal(i.Manifests[1].ImageData.Platform.Architecture, "amd64"))
}
},
},
{
@@ -146,6 +162,14 @@ func TestImageList(t *testing.T) {
assert.Check(t, is.Equal(all[1].ID, twoplatform.Manifests[0].Digest.String()))
assert.Check(t, is.DeepEqual(all[1].RepoTags, []string{"twoplatform:latest"}))
assert.Check(t, is.Len(all[0].Manifests, 1))
assert.Check(t, is.Len(all[1].Manifests, 2))
assert.Check(t, is.Equal(all[0].Manifests[0].Kind, imagetypes.ManifestKindImage))
assert.Check(t, is.Equal(all[1].Manifests[0].Kind, imagetypes.ManifestKindImage))
assert.Check(t, is.Equal(all[1].Manifests[1].Kind, imagetypes.ManifestKindImage))
},
},
{
@@ -176,7 +200,9 @@ func TestImageList(t *testing.T) {
assert.NilError(t, err)
}
all, err := service.Images(ctx, tc.opts)
opts := tc.opts
opts.Manifests = true
all, err := service.Images(ctx, opts)
assert.NilError(t, err)
sort.Slice(all, func(i, j int) bool {

View File

@@ -13,6 +13,17 @@ keywords: "API, Docker, rcli, REST, documentation"
will be rejected.
-->
## v1.47 API changes
[Docker Engine API v1.47](https://docs.docker.com/engine/api/v1.47/) documentation
* `GET /images/json` response now includes `Manifests` field, which contains
information about the sub-manifests included in the image index. This
includes things like platform-specific manifests and build attestations.
The new field will only be populated if the request also sets the `manifests`
query parameter to `true`.
WARNING: This is experimental and may change at any time without any backward
compatibility.
## v1.46 API changes

View File

@@ -2,13 +2,17 @@ package image // import "github.com/docker/docker/integration/image"
import (
"fmt"
"slices"
"strings"
"testing"
"time"
"github.com/docker/docker/api"
containertypes "github.com/docker/docker/api/types/container"
"github.com/docker/docker/api/types/filters"
"github.com/docker/docker/api/types/image"
"github.com/docker/docker/api/types/versions"
"github.com/docker/docker/client"
"github.com/docker/docker/integration/internal/container"
"github.com/docker/docker/internal/testutils/specialimage"
"github.com/docker/docker/testutil"
@@ -227,3 +231,72 @@ func TestAPIImagesListSizeShared(t *testing.T) {
_, err := client.ImageList(ctx, image.ListOptions{SharedSize: true})
assert.NilError(t, err)
}
func TestAPIImagesListManifests(t *testing.T) {
skip.If(t, !testEnv.UsingSnapshotter())
// Sub-daemons not supported on Windows
skip.If(t, testEnv.DaemonInfo.OSType == "windows")
ctx := setupTest(t)
d := daemon.New(t)
d.Start(t)
defer d.Stop(t)
apiClient := d.NewClientT(t)
testPlatforms := []ocispec.Platform{
{OS: "windows", Architecture: "amd64"},
{OS: "linux", Architecture: "arm", Variant: "v7"},
{OS: "darwin", Architecture: "arm64"},
}
specialimage.Load(ctx, t, apiClient, func(dir string) (*ocispec.Index, error) {
return specialimage.MultiPlatform(dir, "multiplatform:latest", testPlatforms)
})
t.Run("unsupported before 1.47", func(t *testing.T) {
// TODO: Remove when MinSupportedAPIVersion >= 1.47
c := d.NewClientT(t, client.WithVersion(api.MinSupportedAPIVersion))
images, err := c.ImageList(ctx, image.ListOptions{Manifests: true})
assert.NilError(t, err)
assert.Assert(t, is.Len(images, 1))
assert.Check(t, is.Nil(images[0].Manifests))
})
skip.If(t, versions.LessThan(testEnv.DaemonAPIVersion(), "1.47"))
api147 := d.NewClientT(t, client.WithVersion("1.47"))
t.Run("no manifests if not requested", func(t *testing.T) {
images, err := api147.ImageList(ctx, image.ListOptions{})
assert.NilError(t, err)
assert.Assert(t, is.Len(images, 1))
assert.Check(t, is.Nil(images[0].Manifests))
})
images, err := api147.ImageList(ctx, image.ListOptions{Manifests: true})
assert.NilError(t, err)
assert.Check(t, is.Len(images, 1))
assert.Check(t, images[0].Manifests != nil)
assert.Check(t, is.Len(images[0].Manifests, 3))
for _, mfst := range images[0].Manifests {
// All manifests should be image manifests
assert.Check(t, is.Equal(mfst.Kind, image.ManifestKindImage))
// Full image was loaded so all manifests should be available
assert.Check(t, mfst.Available)
// The platform should be one of the test platforms
if assert.Check(t, is.Contains(testPlatforms, mfst.ImageData.Platform)) {
testPlatforms = slices.DeleteFunc(testPlatforms, func(p ocispec.Platform) bool {
op := mfst.ImageData.Platform
return p.OS == op.OS && p.Architecture == op.Architecture && p.Variant == op.Variant
})
}
}
}