mirror of
https://github.com/moby/buildkit.git
synced 2026-08-09 17:18:11 +00:00
a9f1980ebb
There are a few goals with this refactor: 1. Remove external access to fields that no longer make sense and/or won't make sense soon due to other potential changes. For example, there can now be multiple blobs associated with a ref (for different compression types), so the fact that you could access the "Blob" field from the Info method on Ref incorrectly implied there was just a single blob for the ref. This is on top of the fact that there is no need for external access to blob digests. 2. Centralize use of cache metadata inside the cache package. Previously, many parts of the code outside the cache package could obtain the bolt storage item for any ref and read/write it directly. This made it hard to understand what fields are used and when. Now, the Metadata method has been removed from the Ref interface and replaced with getters+setters for metadata fields we want to expose outside the package, which makes it much easier to track and understand. Similar changes have been made to the metadata search interface. 3. Use a consistent getter+setter interface for metadata, replacing the mix of interfaces like Metadata(), Size(), Info() and other inconsistencies. Signed-off-by: Erik Sipsma <erik@sipsma.dev>
365 lines
8.8 KiB
Go
365 lines
8.8 KiB
Go
package http
|
|
|
|
import (
|
|
"context"
|
|
"io/ioutil"
|
|
"os"
|
|
"path/filepath"
|
|
"runtime"
|
|
"testing"
|
|
|
|
"github.com/containerd/containerd/content/local"
|
|
ctdmetadata "github.com/containerd/containerd/metadata"
|
|
"github.com/containerd/containerd/snapshots"
|
|
"github.com/containerd/containerd/snapshots/native"
|
|
"github.com/moby/buildkit/cache"
|
|
"github.com/moby/buildkit/cache/metadata"
|
|
"github.com/moby/buildkit/identity"
|
|
"github.com/moby/buildkit/snapshot"
|
|
containerdsnapshot "github.com/moby/buildkit/snapshot/containerd"
|
|
"github.com/moby/buildkit/source"
|
|
"github.com/moby/buildkit/util/leaseutil"
|
|
"github.com/moby/buildkit/util/testutil/httpserver"
|
|
digest "github.com/opencontainers/go-digest"
|
|
"github.com/stretchr/testify/require"
|
|
bolt "go.etcd.io/bbolt"
|
|
)
|
|
|
|
func TestHTTPSource(t *testing.T) {
|
|
if runtime.GOOS == "windows" {
|
|
t.Skip("Depends on unimplemented containerd bind-mount support on Windows")
|
|
}
|
|
|
|
t.Parallel()
|
|
ctx := context.TODO()
|
|
|
|
tmpdir, err := ioutil.TempDir("", "buildkit-state")
|
|
require.NoError(t, err)
|
|
defer os.RemoveAll(tmpdir)
|
|
|
|
hs, err := newHTTPSource(tmpdir)
|
|
require.NoError(t, err)
|
|
|
|
resp := httpserver.Response{
|
|
Etag: identity.NewID(),
|
|
Content: []byte("content1"),
|
|
}
|
|
server := httpserver.NewTestServer(map[string]httpserver.Response{
|
|
"/foo": resp,
|
|
})
|
|
defer server.Close()
|
|
|
|
id := &source.HTTPIdentifier{URL: server.URL + "/foo"}
|
|
|
|
h, err := hs.Resolve(ctx, id, nil, nil)
|
|
require.NoError(t, err)
|
|
|
|
k, _, _, err := h.CacheKey(ctx, nil, 0)
|
|
require.NoError(t, err)
|
|
|
|
expectedContent1 := "sha256:0b1a154faa3003c1fbe7fda9c8a42d55fde2df2a2c405c32038f8ac7ed6b044a"
|
|
|
|
require.Equal(t, expectedContent1, k)
|
|
require.Equal(t, server.Stats("/foo").AllRequests, 1)
|
|
require.Equal(t, server.Stats("/foo").CachedRequests, 0)
|
|
|
|
ref, err := h.Snapshot(ctx, nil)
|
|
require.NoError(t, err)
|
|
defer func() {
|
|
if ref != nil {
|
|
ref.Release(context.TODO())
|
|
ref = nil
|
|
}
|
|
}()
|
|
|
|
dt, err := readFile(ctx, ref, "foo")
|
|
require.NoError(t, err)
|
|
require.Equal(t, dt, []byte("content1"))
|
|
|
|
ref.Release(context.TODO())
|
|
ref = nil
|
|
|
|
// repeat, should use the etag
|
|
h, err = hs.Resolve(ctx, id, nil, nil)
|
|
require.NoError(t, err)
|
|
|
|
k, _, _, err = h.CacheKey(ctx, nil, 0)
|
|
require.NoError(t, err)
|
|
|
|
require.Equal(t, expectedContent1, k)
|
|
require.Equal(t, server.Stats("/foo").AllRequests, 2)
|
|
require.Equal(t, server.Stats("/foo").CachedRequests, 1)
|
|
|
|
ref, err = h.Snapshot(ctx, nil)
|
|
require.NoError(t, err)
|
|
defer func() {
|
|
if ref != nil {
|
|
ref.Release(context.TODO())
|
|
ref = nil
|
|
}
|
|
}()
|
|
|
|
dt, err = readFile(ctx, ref, "foo")
|
|
require.NoError(t, err)
|
|
require.Equal(t, dt, []byte("content1"))
|
|
|
|
ref.Release(context.TODO())
|
|
ref = nil
|
|
|
|
resp2 := httpserver.Response{
|
|
Etag: identity.NewID(),
|
|
Content: []byte("content2"),
|
|
}
|
|
|
|
expectedContent2 := "sha256:888722f299c02bfae173a747a0345bb2291cf6a076c36d8eb6fab442a8adddfa"
|
|
|
|
// update etag, downloads again
|
|
server.SetRoute("/foo", resp2)
|
|
|
|
h, err = hs.Resolve(ctx, id, nil, nil)
|
|
require.NoError(t, err)
|
|
|
|
k, _, _, err = h.CacheKey(ctx, nil, 0)
|
|
require.NoError(t, err)
|
|
|
|
require.Equal(t, expectedContent2, k)
|
|
require.Equal(t, server.Stats("/foo").AllRequests, 4)
|
|
require.Equal(t, server.Stats("/foo").CachedRequests, 1)
|
|
|
|
ref, err = h.Snapshot(ctx, nil)
|
|
require.NoError(t, err)
|
|
defer func() {
|
|
if ref != nil {
|
|
ref.Release(context.TODO())
|
|
ref = nil
|
|
}
|
|
}()
|
|
|
|
dt, err = readFile(ctx, ref, "foo")
|
|
require.NoError(t, err)
|
|
require.Equal(t, dt, []byte("content2"))
|
|
|
|
ref.Release(context.TODO())
|
|
ref = nil
|
|
}
|
|
|
|
func TestHTTPDefaultName(t *testing.T) {
|
|
if runtime.GOOS == "windows" {
|
|
t.Skip("Depends on unimplemented containerd bind-mount support on Windows")
|
|
}
|
|
|
|
t.Parallel()
|
|
ctx := context.TODO()
|
|
|
|
tmpdir, err := ioutil.TempDir("", "buildkit-state")
|
|
require.NoError(t, err)
|
|
defer os.RemoveAll(tmpdir)
|
|
|
|
hs, err := newHTTPSource(tmpdir)
|
|
require.NoError(t, err)
|
|
|
|
resp := httpserver.Response{
|
|
Etag: identity.NewID(),
|
|
Content: []byte("content1"),
|
|
}
|
|
server := httpserver.NewTestServer(map[string]httpserver.Response{
|
|
"/": resp,
|
|
})
|
|
defer server.Close()
|
|
|
|
id := &source.HTTPIdentifier{URL: server.URL}
|
|
|
|
h, err := hs.Resolve(ctx, id, nil, nil)
|
|
require.NoError(t, err)
|
|
|
|
k, _, _, err := h.CacheKey(ctx, nil, 0)
|
|
require.NoError(t, err)
|
|
|
|
require.Equal(t, "sha256:146f16ec8810a62a57ce314aba391f95f7eaaf41b8b1ebaf2ab65fd63b1ad437", k)
|
|
require.Equal(t, server.Stats("/").AllRequests, 1)
|
|
require.Equal(t, server.Stats("/").CachedRequests, 0)
|
|
|
|
ref, err := h.Snapshot(ctx, nil)
|
|
require.NoError(t, err)
|
|
defer func() {
|
|
if ref != nil {
|
|
ref.Release(context.TODO())
|
|
ref = nil
|
|
}
|
|
}()
|
|
|
|
dt, err := readFile(ctx, ref, "download")
|
|
require.NoError(t, err)
|
|
require.Equal(t, dt, []byte("content1"))
|
|
|
|
ref.Release(context.TODO())
|
|
ref = nil
|
|
}
|
|
|
|
func TestHTTPInvalidURL(t *testing.T) {
|
|
t.Parallel()
|
|
ctx := context.TODO()
|
|
|
|
tmpdir, err := ioutil.TempDir("", "buildkit-state")
|
|
require.NoError(t, err)
|
|
defer os.RemoveAll(tmpdir)
|
|
|
|
hs, err := newHTTPSource(tmpdir)
|
|
require.NoError(t, err)
|
|
|
|
server := httpserver.NewTestServer(map[string]httpserver.Response{})
|
|
defer server.Close()
|
|
|
|
id := &source.HTTPIdentifier{URL: server.URL + "/foo"}
|
|
|
|
h, err := hs.Resolve(ctx, id, nil, nil)
|
|
require.NoError(t, err)
|
|
|
|
_, _, _, err = h.CacheKey(ctx, nil, 0)
|
|
require.Error(t, err)
|
|
require.Contains(t, err.Error(), "invalid response")
|
|
}
|
|
|
|
func TestHTTPChecksum(t *testing.T) {
|
|
if runtime.GOOS == "windows" {
|
|
t.Skip("Depends on unimplemented containerd bind-mount support on Windows")
|
|
}
|
|
|
|
t.Parallel()
|
|
ctx := context.TODO()
|
|
|
|
tmpdir, err := ioutil.TempDir("", "buildkit-state")
|
|
require.NoError(t, err)
|
|
defer os.RemoveAll(tmpdir)
|
|
|
|
hs, err := newHTTPSource(tmpdir)
|
|
require.NoError(t, err)
|
|
|
|
resp := httpserver.Response{
|
|
Etag: identity.NewID(),
|
|
Content: []byte("content-correct"),
|
|
}
|
|
server := httpserver.NewTestServer(map[string]httpserver.Response{
|
|
"/foo": resp,
|
|
})
|
|
defer server.Close()
|
|
|
|
id := &source.HTTPIdentifier{URL: server.URL + "/foo", Checksum: digest.FromBytes([]byte("content-different"))}
|
|
|
|
h, err := hs.Resolve(ctx, id, nil, nil)
|
|
require.NoError(t, err)
|
|
|
|
k, _, _, err := h.CacheKey(ctx, nil, 0)
|
|
require.NoError(t, err)
|
|
|
|
expectedContentDifferent := "sha256:f25996f463dca69cffb580f8273ffacdda43332b5f0a8bea2ead33900616d44b"
|
|
expectedContentCorrect := "sha256:c6a440110a7757b9e1e47b52e413cba96c62377c37a474714b6b3c4f8b74e536"
|
|
|
|
require.Equal(t, expectedContentDifferent, k)
|
|
require.Equal(t, server.Stats("/foo").AllRequests, 0)
|
|
require.Equal(t, server.Stats("/foo").CachedRequests, 0)
|
|
|
|
_, err = h.Snapshot(ctx, nil)
|
|
require.Error(t, err)
|
|
|
|
require.Equal(t, expectedContentDifferent, k)
|
|
require.Equal(t, server.Stats("/foo").AllRequests, 1)
|
|
require.Equal(t, server.Stats("/foo").CachedRequests, 0)
|
|
|
|
id = &source.HTTPIdentifier{URL: server.URL + "/foo", Checksum: digest.FromBytes([]byte("content-correct"))}
|
|
|
|
h, err = hs.Resolve(ctx, id, nil, nil)
|
|
require.NoError(t, err)
|
|
|
|
k, _, _, err = h.CacheKey(ctx, nil, 0)
|
|
require.NoError(t, err)
|
|
|
|
require.Equal(t, expectedContentCorrect, k)
|
|
require.Equal(t, server.Stats("/foo").AllRequests, 1)
|
|
require.Equal(t, server.Stats("/foo").CachedRequests, 0)
|
|
|
|
ref, err := h.Snapshot(ctx, nil)
|
|
require.NoError(t, err)
|
|
defer func() {
|
|
if ref != nil {
|
|
ref.Release(context.TODO())
|
|
ref = nil
|
|
}
|
|
}()
|
|
|
|
dt, err := readFile(ctx, ref, "foo")
|
|
require.NoError(t, err)
|
|
require.Equal(t, dt, []byte("content-correct"))
|
|
|
|
require.Equal(t, expectedContentCorrect, k)
|
|
require.Equal(t, server.Stats("/foo").AllRequests, 2)
|
|
require.Equal(t, server.Stats("/foo").CachedRequests, 0)
|
|
|
|
ref.Release(context.TODO())
|
|
ref = nil
|
|
|
|
}
|
|
|
|
func readFile(ctx context.Context, ref cache.ImmutableRef, fp string) ([]byte, error) {
|
|
mount, err := ref.Mount(ctx, false, nil)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
lm := snapshot.LocalMounter(mount)
|
|
dir, err := lm.Mount()
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
defer lm.Unmount()
|
|
|
|
dt, err := ioutil.ReadFile(filepath.Join(dir, fp))
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
return dt, nil
|
|
}
|
|
|
|
func newHTTPSource(tmpdir string) (source.Source, error) {
|
|
snapshotter, err := native.NewSnapshotter(filepath.Join(tmpdir, "snapshots"))
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
store, err := local.NewStore(tmpdir)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
db, err := bolt.Open(filepath.Join(tmpdir, "containerdmeta.db"), 0644, nil)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
mdb := ctdmetadata.NewDB(db, store, map[string]snapshots.Snapshotter{
|
|
"native": snapshotter,
|
|
})
|
|
|
|
md, err := metadata.NewStore(filepath.Join(tmpdir, "metadata.db"))
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
cm, err := cache.NewManager(cache.ManagerOpt{
|
|
Snapshotter: snapshot.FromContainerdSnapshotter("native", containerdsnapshot.NSSnapshotter("buildkit", mdb.Snapshotter("native")), nil),
|
|
MetadataStore: md,
|
|
LeaseManager: leaseutil.WithNamespace(ctdmetadata.NewLeaseManager(mdb), "buildkit"),
|
|
ContentStore: mdb.ContentStore(),
|
|
GarbageCollect: mdb.GarbageCollect,
|
|
})
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
return NewSource(Opt{
|
|
CacheAccessor: cm,
|
|
})
|
|
}
|