From 9081ade948ac5cab80fb57688ad0368a24024fdd Mon Sep 17 00:00:00 2001 From: CrazyMax Date: Thu, 9 Feb 2023 06:03:11 +0100 Subject: [PATCH 1/4] remotecache: mutualize compression parsing attrs Signed-off-by: CrazyMax --- cache/remotecache/local/local.go | 42 ++-------------------- cache/remotecache/registry/registry.go | 48 +++----------------------- util/compression/attrs.go | 48 ++++++++++++++++++++++++++ 3 files changed, 55 insertions(+), 83 deletions(-) create mode 100644 util/compression/attrs.go diff --git a/cache/remotecache/local/local.go b/cache/remotecache/local/local.go index 7f3d83b70..4c06d717b 100644 --- a/cache/remotecache/local/local.go +++ b/cache/remotecache/local/local.go @@ -21,9 +21,6 @@ const ( attrDest = "dest" attrOCIMediatypes = "oci-mediatypes" contentStoreIDPrefix = "local:" - attrLayerCompression = "compression" - attrForceCompression = "force-compression" - attrCompressionLevel = "compression-level" ) // ResolveCacheExporterFunc for "local" cache exporter. @@ -33,7 +30,7 @@ func ResolveCacheExporterFunc(sm *session.Manager) remotecache.ResolveCacheExpor if store == "" { return nil, errors.New("local cache exporter requires dest") } - compressionConfig, err := attrsToCompression(attrs) + compressionConfig, err := compression.ParseAttributes(attrs) if err != nil { return nil, err } @@ -50,7 +47,7 @@ func ResolveCacheExporterFunc(sm *session.Manager) remotecache.ResolveCacheExpor if err != nil { return nil, err } - return remotecache.NewExporter(cs, "", ociMediatypes, *compressionConfig), nil + return remotecache.NewExporter(cs, "", ociMediatypes, compressionConfig), nil } } @@ -109,38 +106,3 @@ type unlazyProvider struct { func (p *unlazyProvider) UnlazySession(desc ocispecs.Descriptor) session.Group { return p.s } - -func attrsToCompression(attrs map[string]string) (*compression.Config, error) { - var compressionType compression.Type - if v, ok := attrs[attrLayerCompression]; ok { - c, err := compression.Parse(v) - if err != nil { - return nil, err - } - compressionType = c - } else { - compressionType = compression.Default - } - compressionConfig := compression.New(compressionType) - if v, ok := attrs[attrForceCompression]; ok { - var force bool - if v == "" { - force = true - } else { - b, err := strconv.ParseBool(v) - if err != nil { - return nil, errors.Wrapf(err, "non-bool value %s specified for %s", v, attrForceCompression) - } - force = b - } - compressionConfig = compressionConfig.SetForce(force) - } - if v, ok := attrs[attrCompressionLevel]; ok { - ii, err := strconv.ParseInt(v, 10, 64) - if err != nil { - return nil, errors.Wrapf(err, "non-integer value %s specified for %s", v, attrCompressionLevel) - } - compressionConfig = compressionConfig.SetLevel(int(ii)) - } - return &compressionConfig, nil -} diff --git a/cache/remotecache/registry/registry.go b/cache/remotecache/registry/registry.go index f5ce8f2a4..f282defc9 100644 --- a/cache/remotecache/registry/registry.go +++ b/cache/remotecache/registry/registry.go @@ -35,17 +35,14 @@ func canonicalizeRef(rawRef string) (reference.Named, error) { } const ( - attrRef = "ref" - attrOCIMediatypes = "oci-mediatypes" - attrLayerCompression = "compression" - attrForceCompression = "force-compression" - attrCompressionLevel = "compression-level" - attrInsecure = "registry.insecure" + attrRef = "ref" + attrOCIMediatypes = "oci-mediatypes" + attrInsecure = "registry.insecure" ) func ResolveCacheExporterFunc(sm *session.Manager, hosts docker.RegistryHosts) remotecache.ResolveCacheExporterFunc { return func(ctx context.Context, g session.Group, attrs map[string]string) (remotecache.Exporter, error) { - compressionConfig, err := attrsToCompression(attrs) + compressionConfig, err := compression.ParseAttributes(attrs) if err != nil { return nil, err } @@ -77,7 +74,7 @@ func ResolveCacheExporterFunc(sm *session.Manager, hosts docker.RegistryHosts) r if err != nil { return nil, err } - return remotecache.NewExporter(contentutil.FromPusher(pusher), refString, ociMediatypes, *compressionConfig), nil + return remotecache.NewExporter(contentutil.FromPusher(pusher), refString, ociMediatypes, compressionConfig), nil } } @@ -155,41 +152,6 @@ func (dsl *withDistributionSourceLabel) SnapshotLabels(descs []ocispecs.Descript return labels } -func attrsToCompression(attrs map[string]string) (*compression.Config, error) { - var compressionType compression.Type - if v, ok := attrs[attrLayerCompression]; ok { - c, err := compression.Parse(v) - if err != nil { - return nil, err - } - compressionType = c - } else { - compressionType = compression.Default - } - compressionConfig := compression.New(compressionType) - if v, ok := attrs[attrForceCompression]; ok { - var force bool - if v == "" { - force = true - } else { - b, err := strconv.ParseBool(v) - if err != nil { - return nil, errors.Wrapf(err, "non-bool value %s specified for %s", v, attrForceCompression) - } - force = b - } - compressionConfig = compressionConfig.SetForce(force) - } - if v, ok := attrs[attrCompressionLevel]; ok { - ii, err := strconv.ParseInt(v, 10, 64) - if err != nil { - return nil, errors.Wrapf(err, "non-integer value %s specified for %s", v, attrCompressionLevel) - } - compressionConfig = compressionConfig.SetLevel(int(ii)) - } - return &compressionConfig, nil -} - func registryConfig(hosts docker.RegistryHosts, ref reference.Named, scope string, insecure bool) (string, docker.RegistryHosts) { if insecure { insecureTrue := true diff --git a/util/compression/attrs.go b/util/compression/attrs.go new file mode 100644 index 000000000..1f986d371 --- /dev/null +++ b/util/compression/attrs.go @@ -0,0 +1,48 @@ +package compression + +import ( + "strconv" + + "github.com/pkg/errors" +) + +const ( + attrLayerCompression = "compression" + attrForceCompression = "force-compression" + attrCompressionLevel = "compression-level" +) + +func ParseAttributes(attrs map[string]string) (Config, error) { + var compressionType Type + if v, ok := attrs[attrLayerCompression]; ok { + c, err := Parse(v) + if err != nil { + return Config{}, err + } + compressionType = c + } else { + compressionType = Default + } + compressionConfig := New(compressionType) + if v, ok := attrs[attrForceCompression]; ok { + var force bool + if v == "" { + force = true + } else { + b, err := strconv.ParseBool(v) + if err != nil { + return Config{}, errors.Wrapf(err, "non-bool value %s specified for %s", v, attrForceCompression) + } + force = b + } + compressionConfig = compressionConfig.SetForce(force) + } + if v, ok := attrs[attrCompressionLevel]; ok { + ii, err := strconv.ParseInt(v, 10, 64) + if err != nil { + return Config{}, errors.Wrapf(err, "non-integer value %s specified for %s", v, attrCompressionLevel) + } + compressionConfig = compressionConfig.SetLevel(int(ii)) + } + return compressionConfig, nil +} From b3bb356b86281c1160fb6d720ae0451b7d0fa987 Mon Sep 17 00:00:00 2001 From: CrazyMax Date: Thu, 9 Feb 2023 07:14:56 +0100 Subject: [PATCH 2/4] exporter: use compression.ParseAttributes func Signed-off-by: CrazyMax --- exporter/containerimage/opts.go | 19 ++++--------------- 1 file changed, 4 insertions(+), 15 deletions(-) diff --git a/exporter/containerimage/opts.go b/exporter/containerimage/opts.go index c12d86127..96844aa42 100644 --- a/exporter/containerimage/opts.go +++ b/exporter/containerimage/opts.go @@ -13,9 +13,6 @@ import ( const ( keyImageName = "name" - keyLayerCompression = "compression" - keyCompressionLevel = "compression-level" - keyForceCompression = "force-compression" keyOCITypes = "oci-mediatypes" keyBuildInfo = "buildinfo" keyBuildInfoAttrs = "buildinfo-attrs" @@ -53,23 +50,15 @@ func (c *ImageCommitOpts) Load(opt map[string]string) (map[string]string, error) return nil, err } + if c.RefCfg.Compression, err = compression.ParseAttributes(opt); err != nil { + return nil, err + } + for k, v := range opt { var err error switch k { case keyImageName: c.ImageName = v - case keyLayerCompression: - c.RefCfg.Compression.Type, err = compression.Parse(v) - case keyCompressionLevel: - ii, err2 := strconv.ParseInt(v, 10, 64) - if err != nil { - err = errors.Wrapf(err2, "non-int value %s specified for %s", v, k) - break - } - v := int(ii) - c.RefCfg.Compression.Level = &v - case keyForceCompression: - err = parseBoolWithDefault(&c.RefCfg.Compression.Force, k, v, true) case keyOCITypes: err = parseBoolWithDefault(&c.OCITypes, k, v, true) case keyBuildInfo: From cf4e1bb7d29413bb7397af1a6882bc509852b3fd Mon Sep 17 00:00:00 2001 From: CrazyMax Date: Thu, 9 Feb 2023 18:17:46 +0100 Subject: [PATCH 3/4] remotecache: explicit names for registry and local Signed-off-by: CrazyMax --- cache/remotecache/local/local.go | 10 +++++++++- cache/remotecache/registry/registry.go | 10 +++++++++- 2 files changed, 18 insertions(+), 2 deletions(-) diff --git a/cache/remotecache/local/local.go b/cache/remotecache/local/local.go index 4c06d717b..9bff2cc79 100644 --- a/cache/remotecache/local/local.go +++ b/cache/remotecache/local/local.go @@ -23,6 +23,14 @@ const ( contentStoreIDPrefix = "local:" ) +type exporter struct { + remotecache.Exporter +} + +func (*exporter) Name() string { + return "exporting cache to client directory" +} + // ResolveCacheExporterFunc for "local" cache exporter. func ResolveCacheExporterFunc(sm *session.Manager) remotecache.ResolveCacheExporterFunc { return func(ctx context.Context, g session.Group, attrs map[string]string) (remotecache.Exporter, error) { @@ -47,7 +55,7 @@ func ResolveCacheExporterFunc(sm *session.Manager) remotecache.ResolveCacheExpor if err != nil { return nil, err } - return remotecache.NewExporter(cs, "", ociMediatypes, compressionConfig), nil + return &exporter{remotecache.NewExporter(cs, "", ociMediatypes, compressionConfig)}, nil } } diff --git a/cache/remotecache/registry/registry.go b/cache/remotecache/registry/registry.go index f282defc9..e82638ebf 100644 --- a/cache/remotecache/registry/registry.go +++ b/cache/remotecache/registry/registry.go @@ -40,6 +40,14 @@ const ( attrInsecure = "registry.insecure" ) +type exporter struct { + remotecache.Exporter +} + +func (*exporter) Name() string { + return "exporting cache to registry" +} + func ResolveCacheExporterFunc(sm *session.Manager, hosts docker.RegistryHosts) remotecache.ResolveCacheExporterFunc { return func(ctx context.Context, g session.Group, attrs map[string]string) (remotecache.Exporter, error) { compressionConfig, err := compression.ParseAttributes(attrs) @@ -74,7 +82,7 @@ func ResolveCacheExporterFunc(sm *session.Manager, hosts docker.RegistryHosts) r if err != nil { return nil, err } - return remotecache.NewExporter(contentutil.FromPusher(pusher), refString, ociMediatypes, compressionConfig), nil + return &exporter{remotecache.NewExporter(contentutil.FromPusher(pusher), refString, ociMediatypes, compressionConfig)}, nil } } From 1db81665fd7ff87c340c2077afa62adea5c5eea1 Mon Sep 17 00:00:00 2001 From: CrazyMax Date: Thu, 9 Feb 2023 06:55:21 +0100 Subject: [PATCH 4/4] remotecache: proper exporter naming for gha, s3 and azblob Signed-off-by: CrazyMax --- cache/remotecache/azblob/exporter.go | 2 +- cache/remotecache/gha/gha.go | 2 +- cache/remotecache/s3/s3.go | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/cache/remotecache/azblob/exporter.go b/cache/remotecache/azblob/exporter.go index 4d76770db..406b12f8f 100644 --- a/cache/remotecache/azblob/exporter.go +++ b/cache/remotecache/azblob/exporter.go @@ -54,7 +54,7 @@ type exporter struct { } func (ce *exporter) Name() string { - return "exporting cache to azure blob store" + return "exporting cache to Azure Blob Storage" } func (ce *exporter) Finalize(ctx context.Context) (map[string]string, error) { diff --git a/cache/remotecache/gha/gha.go b/cache/remotecache/gha/gha.go index 45cacb5b8..131076693 100644 --- a/cache/remotecache/gha/gha.go +++ b/cache/remotecache/gha/gha.go @@ -91,7 +91,7 @@ func NewExporter(c *Config) (remotecache.Exporter, error) { } func (*exporter) Name() string { - return "exporting to GitHub cache" + return "exporting to GitHub Actions Cache" } func (ce *exporter) Config() remotecache.Config { diff --git a/cache/remotecache/s3/s3.go b/cache/remotecache/s3/s3.go index f0e814b2f..2e900a616 100644 --- a/cache/remotecache/s3/s3.go +++ b/cache/remotecache/s3/s3.go @@ -162,7 +162,7 @@ type exporter struct { } func (*exporter) Name() string { - return "exporting cache to s3" + return "exporting cache to Amazon S3" } func (e *exporter) Config() remotecache.Config {