mirror of
https://github.com/moby/buildkit.git
synced 2026-08-09 17:18:11 +00:00
9321ec2f82
This is mostly just preparation for merge-op. The existing Extract method is updated to be usable for unlazying any type of refs rather than just lazy blobs. The way views are created is simplified and centralized in one location. Signed-off-by: Erik Sipsma <erik@sipsma.dev>
378 lines
9.6 KiB
Go
378 lines
9.6 KiB
Go
package http
|
|
|
|
import (
|
|
"context"
|
|
"io/ioutil"
|
|
"os"
|
|
"path/filepath"
|
|
"runtime"
|
|
"testing"
|
|
|
|
"github.com/containerd/containerd/content/local"
|
|
ctdmetadata "github.com/containerd/containerd/metadata"
|
|
"github.com/containerd/containerd/snapshots"
|
|
"github.com/containerd/containerd/snapshots/native"
|
|
"github.com/moby/buildkit/cache"
|
|
"github.com/moby/buildkit/cache/metadata"
|
|
"github.com/moby/buildkit/identity"
|
|
"github.com/moby/buildkit/snapshot"
|
|
containerdsnapshot "github.com/moby/buildkit/snapshot/containerd"
|
|
"github.com/moby/buildkit/source"
|
|
"github.com/moby/buildkit/util/leaseutil"
|
|
"github.com/moby/buildkit/util/testutil/httpserver"
|
|
digest "github.com/opencontainers/go-digest"
|
|
"github.com/stretchr/testify/require"
|
|
bolt "go.etcd.io/bbolt"
|
|
)
|
|
|
|
func TestHTTPSource(t *testing.T) {
|
|
if runtime.GOOS == "windows" {
|
|
t.Skip("Depends on unimplemented containerd bind-mount support on Windows")
|
|
}
|
|
|
|
t.Parallel()
|
|
ctx := context.TODO()
|
|
|
|
tmpdir, err := ioutil.TempDir("", "buildkit-state")
|
|
require.NoError(t, err)
|
|
defer os.RemoveAll(tmpdir)
|
|
|
|
hs, err := newHTTPSource(tmpdir)
|
|
require.NoError(t, err)
|
|
|
|
resp := httpserver.Response{
|
|
Etag: identity.NewID(),
|
|
Content: []byte("content1"),
|
|
}
|
|
server := httpserver.NewTestServer(map[string]httpserver.Response{
|
|
"/foo": resp,
|
|
})
|
|
defer server.Close()
|
|
|
|
id := &source.HTTPIdentifier{URL: server.URL + "/foo"}
|
|
|
|
h, err := hs.Resolve(ctx, id, nil, nil)
|
|
require.NoError(t, err)
|
|
|
|
k, p, _, _, err := h.CacheKey(ctx, nil, 0)
|
|
require.NoError(t, err)
|
|
|
|
expectedContent1 := "sha256:0b1a154faa3003c1fbe7fda9c8a42d55fde2df2a2c405c32038f8ac7ed6b044a"
|
|
expectedPin1 := "sha256:d0b425e00e15a0d36b9b361f02bab63563aed6cb4665083905386c55d5b679fa"
|
|
|
|
require.Equal(t, expectedContent1, k)
|
|
require.Equal(t, expectedPin1, p)
|
|
require.Equal(t, server.Stats("/foo").AllRequests, 1)
|
|
require.Equal(t, server.Stats("/foo").CachedRequests, 0)
|
|
|
|
ref, err := h.Snapshot(ctx, nil)
|
|
require.NoError(t, err)
|
|
defer func() {
|
|
if ref != nil {
|
|
ref.Release(context.TODO())
|
|
ref = nil
|
|
}
|
|
}()
|
|
|
|
dt, err := readFile(ctx, ref, "foo")
|
|
require.NoError(t, err)
|
|
require.Equal(t, dt, []byte("content1"))
|
|
|
|
ref.Release(context.TODO())
|
|
ref = nil
|
|
|
|
// repeat, should use the etag
|
|
h, err = hs.Resolve(ctx, id, nil, nil)
|
|
require.NoError(t, err)
|
|
|
|
k, p, _, _, err = h.CacheKey(ctx, nil, 0)
|
|
require.NoError(t, err)
|
|
|
|
require.Equal(t, expectedContent1, k)
|
|
require.Equal(t, expectedPin1, p)
|
|
require.Equal(t, server.Stats("/foo").AllRequests, 2)
|
|
require.Equal(t, server.Stats("/foo").CachedRequests, 1)
|
|
|
|
ref, err = h.Snapshot(ctx, nil)
|
|
require.NoError(t, err)
|
|
defer func() {
|
|
if ref != nil {
|
|
ref.Release(context.TODO())
|
|
ref = nil
|
|
}
|
|
}()
|
|
|
|
dt, err = readFile(ctx, ref, "foo")
|
|
require.NoError(t, err)
|
|
require.Equal(t, dt, []byte("content1"))
|
|
|
|
ref.Release(context.TODO())
|
|
ref = nil
|
|
|
|
resp2 := httpserver.Response{
|
|
Etag: identity.NewID(),
|
|
Content: []byte("content2"),
|
|
}
|
|
|
|
expectedContent2 := "sha256:888722f299c02bfae173a747a0345bb2291cf6a076c36d8eb6fab442a8adddfa"
|
|
expectedPin2 := "sha256:dab741b6289e7dccc1ed42330cae1accc2b755ce8079c2cd5d4b5366c9f769a6"
|
|
|
|
// update etag, downloads again
|
|
server.SetRoute("/foo", resp2)
|
|
|
|
h, err = hs.Resolve(ctx, id, nil, nil)
|
|
require.NoError(t, err)
|
|
|
|
k, p, _, _, err = h.CacheKey(ctx, nil, 0)
|
|
require.NoError(t, err)
|
|
|
|
require.Equal(t, expectedContent2, k)
|
|
require.Equal(t, expectedPin2, p)
|
|
require.Equal(t, server.Stats("/foo").AllRequests, 4)
|
|
require.Equal(t, server.Stats("/foo").CachedRequests, 1)
|
|
|
|
ref, err = h.Snapshot(ctx, nil)
|
|
require.NoError(t, err)
|
|
defer func() {
|
|
if ref != nil {
|
|
ref.Release(context.TODO())
|
|
ref = nil
|
|
}
|
|
}()
|
|
|
|
dt, err = readFile(ctx, ref, "foo")
|
|
require.NoError(t, err)
|
|
require.Equal(t, dt, []byte("content2"))
|
|
|
|
ref.Release(context.TODO())
|
|
ref = nil
|
|
}
|
|
|
|
func TestHTTPDefaultName(t *testing.T) {
|
|
if runtime.GOOS == "windows" {
|
|
t.Skip("Depends on unimplemented containerd bind-mount support on Windows")
|
|
}
|
|
|
|
t.Parallel()
|
|
ctx := context.TODO()
|
|
|
|
tmpdir, err := ioutil.TempDir("", "buildkit-state")
|
|
require.NoError(t, err)
|
|
defer os.RemoveAll(tmpdir)
|
|
|
|
hs, err := newHTTPSource(tmpdir)
|
|
require.NoError(t, err)
|
|
|
|
resp := httpserver.Response{
|
|
Etag: identity.NewID(),
|
|
Content: []byte("content1"),
|
|
}
|
|
server := httpserver.NewTestServer(map[string]httpserver.Response{
|
|
"/": resp,
|
|
})
|
|
defer server.Close()
|
|
|
|
id := &source.HTTPIdentifier{URL: server.URL}
|
|
|
|
h, err := hs.Resolve(ctx, id, nil, nil)
|
|
require.NoError(t, err)
|
|
|
|
k, p, _, _, err := h.CacheKey(ctx, nil, 0)
|
|
require.NoError(t, err)
|
|
|
|
require.Equal(t, "sha256:146f16ec8810a62a57ce314aba391f95f7eaaf41b8b1ebaf2ab65fd63b1ad437", k)
|
|
require.Equal(t, "sha256:d0b425e00e15a0d36b9b361f02bab63563aed6cb4665083905386c55d5b679fa", p)
|
|
require.Equal(t, server.Stats("/").AllRequests, 1)
|
|
require.Equal(t, server.Stats("/").CachedRequests, 0)
|
|
|
|
ref, err := h.Snapshot(ctx, nil)
|
|
require.NoError(t, err)
|
|
defer func() {
|
|
if ref != nil {
|
|
ref.Release(context.TODO())
|
|
ref = nil
|
|
}
|
|
}()
|
|
|
|
dt, err := readFile(ctx, ref, "download")
|
|
require.NoError(t, err)
|
|
require.Equal(t, dt, []byte("content1"))
|
|
|
|
ref.Release(context.TODO())
|
|
ref = nil
|
|
}
|
|
|
|
func TestHTTPInvalidURL(t *testing.T) {
|
|
t.Parallel()
|
|
ctx := context.TODO()
|
|
|
|
tmpdir, err := ioutil.TempDir("", "buildkit-state")
|
|
require.NoError(t, err)
|
|
defer os.RemoveAll(tmpdir)
|
|
|
|
hs, err := newHTTPSource(tmpdir)
|
|
require.NoError(t, err)
|
|
|
|
server := httpserver.NewTestServer(map[string]httpserver.Response{})
|
|
defer server.Close()
|
|
|
|
id := &source.HTTPIdentifier{URL: server.URL + "/foo"}
|
|
|
|
h, err := hs.Resolve(ctx, id, nil, nil)
|
|
require.NoError(t, err)
|
|
|
|
_, _, _, _, err = h.CacheKey(ctx, nil, 0)
|
|
require.Error(t, err)
|
|
require.Contains(t, err.Error(), "invalid response")
|
|
}
|
|
|
|
func TestHTTPChecksum(t *testing.T) {
|
|
if runtime.GOOS == "windows" {
|
|
t.Skip("Depends on unimplemented containerd bind-mount support on Windows")
|
|
}
|
|
|
|
t.Parallel()
|
|
ctx := context.TODO()
|
|
|
|
tmpdir, err := ioutil.TempDir("", "buildkit-state")
|
|
require.NoError(t, err)
|
|
defer os.RemoveAll(tmpdir)
|
|
|
|
hs, err := newHTTPSource(tmpdir)
|
|
require.NoError(t, err)
|
|
|
|
resp := httpserver.Response{
|
|
Etag: identity.NewID(),
|
|
Content: []byte("content-correct"),
|
|
}
|
|
server := httpserver.NewTestServer(map[string]httpserver.Response{
|
|
"/foo": resp,
|
|
})
|
|
defer server.Close()
|
|
|
|
id := &source.HTTPIdentifier{URL: server.URL + "/foo", Checksum: digest.FromBytes([]byte("content-different"))}
|
|
|
|
h, err := hs.Resolve(ctx, id, nil, nil)
|
|
require.NoError(t, err)
|
|
|
|
k, p, _, _, err := h.CacheKey(ctx, nil, 0)
|
|
require.NoError(t, err)
|
|
|
|
expectedContentDifferent := "sha256:f25996f463dca69cffb580f8273ffacdda43332b5f0a8bea2ead33900616d44b"
|
|
expectedContentCorrect := "sha256:c6a440110a7757b9e1e47b52e413cba96c62377c37a474714b6b3c4f8b74e536"
|
|
expectedPinDifferent := "sha256:ab0d5a7aa55c1c95d59c302eb12c55368940e6f0a257646afd455cabe248edc4"
|
|
expectedPinCorrect := "sha256:f5fa14774044d2ec428ffe7efbfaa0a439db7bc8127d6b71aea21e1cd558d0f0"
|
|
|
|
require.Equal(t, expectedContentDifferent, k)
|
|
require.Equal(t, expectedPinDifferent, p)
|
|
require.Equal(t, server.Stats("/foo").AllRequests, 0)
|
|
require.Equal(t, server.Stats("/foo").CachedRequests, 0)
|
|
|
|
_, err = h.Snapshot(ctx, nil)
|
|
require.Error(t, err)
|
|
|
|
require.Equal(t, expectedContentDifferent, k)
|
|
require.Equal(t, expectedPinDifferent, p)
|
|
require.Equal(t, server.Stats("/foo").AllRequests, 1)
|
|
require.Equal(t, server.Stats("/foo").CachedRequests, 0)
|
|
|
|
id = &source.HTTPIdentifier{URL: server.URL + "/foo", Checksum: digest.FromBytes([]byte("content-correct"))}
|
|
|
|
h, err = hs.Resolve(ctx, id, nil, nil)
|
|
require.NoError(t, err)
|
|
|
|
k, p, _, _, err = h.CacheKey(ctx, nil, 0)
|
|
require.NoError(t, err)
|
|
|
|
require.Equal(t, expectedContentCorrect, k)
|
|
require.Equal(t, expectedPinCorrect, p)
|
|
require.Equal(t, server.Stats("/foo").AllRequests, 1)
|
|
require.Equal(t, server.Stats("/foo").CachedRequests, 0)
|
|
|
|
ref, err := h.Snapshot(ctx, nil)
|
|
require.NoError(t, err)
|
|
defer func() {
|
|
if ref != nil {
|
|
ref.Release(context.TODO())
|
|
ref = nil
|
|
}
|
|
}()
|
|
|
|
dt, err := readFile(ctx, ref, "foo")
|
|
require.NoError(t, err)
|
|
require.Equal(t, dt, []byte("content-correct"))
|
|
|
|
require.Equal(t, expectedContentCorrect, k)
|
|
require.Equal(t, expectedPinCorrect, p)
|
|
require.Equal(t, server.Stats("/foo").AllRequests, 2)
|
|
require.Equal(t, server.Stats("/foo").CachedRequests, 0)
|
|
|
|
ref.Release(context.TODO())
|
|
ref = nil
|
|
|
|
}
|
|
|
|
func readFile(ctx context.Context, ref cache.ImmutableRef, fp string) ([]byte, error) {
|
|
mount, err := ref.Mount(ctx, true, nil)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
lm := snapshot.LocalMounter(mount)
|
|
dir, err := lm.Mount()
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
defer lm.Unmount()
|
|
|
|
dt, err := ioutil.ReadFile(filepath.Join(dir, fp))
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
return dt, nil
|
|
}
|
|
|
|
func newHTTPSource(tmpdir string) (source.Source, error) {
|
|
snapshotter, err := native.NewSnapshotter(filepath.Join(tmpdir, "snapshots"))
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
store, err := local.NewStore(tmpdir)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
db, err := bolt.Open(filepath.Join(tmpdir, "containerdmeta.db"), 0644, nil)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
mdb := ctdmetadata.NewDB(db, store, map[string]snapshots.Snapshotter{
|
|
"native": snapshotter,
|
|
})
|
|
|
|
md, err := metadata.NewStore(filepath.Join(tmpdir, "metadata.db"))
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
lm := leaseutil.WithNamespace(ctdmetadata.NewLeaseManager(mdb), "buildkit")
|
|
|
|
cm, err := cache.NewManager(cache.ManagerOpt{
|
|
Snapshotter: snapshot.FromContainerdSnapshotter("native", containerdsnapshot.NSSnapshotter("buildkit", mdb.Snapshotter("native")), nil),
|
|
MetadataStore: md,
|
|
LeaseManager: lm,
|
|
ContentStore: mdb.ContentStore(),
|
|
GarbageCollect: mdb.GarbageCollect,
|
|
})
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
return NewSource(Opt{
|
|
CacheAccessor: cm,
|
|
})
|
|
}
|