add unit tests for http source

Signed-off-by: Tonis Tiigi <tonistiigi@gmail.com>

source/git/gitsource_test.go

@@ -27,7 +27,7 @@ func TestRepeatedFetchKeepGitDir(t *testing.T) {
 }
 
 func testRepeatedFetch(t *testing.T, keepGitDir bool) {
-	ctx := namespaces.WithNamespace(context.Background(), "buildkit-test")
+	ctx := context.TODO()
 
 	tmpdir, err := ioutil.TempDir("", "buildkit-state")
 	require.NoError(t, err)
@@ -278,10 +278,6 @@ func setupGitSource(t *testing.T, tmpdir string) source.Source {
 	})
 	assert.NoError(t, err)
 
-	repodir, err := ioutil.TempDir("", "buildkit-gitsource")
-	require.NoError(t, err)
-	defer os.RemoveAll(repodir)
-
 	gs, err := NewSource(Opt{
 		CacheAccessor: cm,
 		MetadataStore: md,

source/http/httpsource.go

@@ -57,7 +57,7 @@ type httpSourceHandler struct {
 func (hs *httpSource) Resolve(ctx context.Context, id source.Identifier) (source.SourceInstance, error) {
 	httpIdentifier, ok := id.(*source.HttpIdentifier)
 	if !ok {
-		return nil, errors.Errorf("invalid git identifier %v", id)
+		return nil, errors.Errorf("invalid http identifier %v", id)
 	}
 
 	return &httpSourceHandler{
@@ -105,7 +105,9 @@ func (hs *httpSourceHandler) CacheKey(ctx context.Context) (string, error) {
 	if err != nil {
 		return "", err
 	}
-
+	if resp.StatusCode < 200 || resp.StatusCode >= 400 {
+		return "", errors.Errorf("invalid response status %d", resp.StatusCode)
+	}
 	if resp.StatusCode == http.StatusNotModified {
 		respETag := resp.Header.Get("ETag")
 		si, ok := m[respETag]
@@ -164,7 +166,6 @@ func (hs *httpSourceHandler) save(ctx context.Context, resp *http.Response) (ref
 			lm.Unmount()
 		}
 	}()
-
 	f, err := os.Create(filepath.Join(dir, getFileName(hs.src.URL, resp)))
 	if err != nil {
 		return nil, "", err
@@ -227,7 +228,7 @@ func (hs *httpSourceHandler) Snapshot(ctx context.Context) (cache.ImmutableRef,
 		return nil, errors.Errorf("digest mismatch %s: %s", dgst, hs.cacheKey)
 	}
 
-	return ref, errors.Errorf("not-implemented")
+	return ref, nil
 }
 
 const keyETag = "etag"
@@ -296,7 +297,7 @@ func getFileName(urlStr string, resp *http.Response) string {
 	}
 	u, err := url.Parse(urlStr)
 	if err == nil {
-		if base := path.Base(u.Path); base != "" {
+		if base := path.Base(u.Path); base != "." {
 			return base
 		}
 	}

source/http/httpsource_test.go

@@ -0,0 +1,318 @@
+package http
+
+import (
+	"context"
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"testing"
+
+	"github.com/containerd/containerd/snapshot/naive"
+	"github.com/moby/buildkit/cache"
+	"github.com/moby/buildkit/cache/metadata"
+	"github.com/moby/buildkit/identity"
+	"github.com/moby/buildkit/snapshot"
+	"github.com/moby/buildkit/source"
+	"github.com/moby/buildkit/util/testutil/httpserver"
+	digest "github.com/opencontainers/go-digest"
+	"github.com/stretchr/testify/require"
+)
+
+func TestHTTPSource(t *testing.T) {
+	ctx := context.TODO()
+
+	tmpdir, err := ioutil.TempDir("", "buildkit-state")
+	require.NoError(t, err)
+	defer os.RemoveAll(tmpdir)
+
+	hs, err := newHTTPSource(tmpdir)
+	require.NoError(t, err)
+
+	resp := httpserver.Response{
+		Etag:    identity.NewID(),
+		Content: []byte("content1"),
+	}
+	server := httpserver.NewTestServer(map[string]httpserver.Response{
+		"/foo": resp,
+	})
+	defer server.Close()
+
+	id := &source.HttpIdentifier{URL: server.URL + "/foo"}
+
+	h, err := hs.Resolve(ctx, id)
+	require.NoError(t, err)
+
+	k, err := h.CacheKey(ctx)
+	require.NoError(t, err)
+
+	require.Equal(t, digest.FromBytes([]byte("content1")).String(), k)
+	require.Equal(t, server.Stats("/foo").AllRequests, 1)
+	require.Equal(t, server.Stats("/foo").CachedRequests, 0)
+
+	ref, err := h.Snapshot(ctx)
+	require.NoError(t, err)
+	defer func() {
+		if ref != nil {
+			ref.Release(context.TODO())
+			ref = nil
+		}
+	}()
+
+	dt, err := readFile(ctx, ref, "foo")
+	require.NoError(t, err)
+	require.Equal(t, dt, []byte("content1"))
+
+	ref.Release(context.TODO())
+	ref = nil
+
+	// repeat, should use the etag
+	h, err = hs.Resolve(ctx, id)
+	require.NoError(t, err)
+
+	k, err = h.CacheKey(ctx)
+	require.NoError(t, err)
+
+	require.Equal(t, digest.FromBytes([]byte("content1")).String(), k)
+	require.Equal(t, server.Stats("/foo").AllRequests, 2)
+	require.Equal(t, server.Stats("/foo").CachedRequests, 1)
+
+	ref, err = h.Snapshot(ctx)
+	require.NoError(t, err)
+	defer func() {
+		if ref != nil {
+			ref.Release(context.TODO())
+			ref = nil
+		}
+	}()
+
+	dt, err = readFile(ctx, ref, "foo")
+	require.NoError(t, err)
+	require.Equal(t, dt, []byte("content1"))
+
+	ref.Release(context.TODO())
+	ref = nil
+
+	resp2 := httpserver.Response{
+		Etag:    identity.NewID(),
+		Content: []byte("content2"),
+	}
+
+	// update etag, downloads again
+	server.SetRoute("/foo", resp2)
+
+	h, err = hs.Resolve(ctx, id)
+	require.NoError(t, err)
+
+	k, err = h.CacheKey(ctx)
+	require.NoError(t, err)
+
+	require.Equal(t, digest.FromBytes([]byte("content2")).String(), k)
+	require.Equal(t, server.Stats("/foo").AllRequests, 3)
+	require.Equal(t, server.Stats("/foo").CachedRequests, 1)
+
+	ref, err = h.Snapshot(ctx)
+	require.NoError(t, err)
+	defer func() {
+		if ref != nil {
+			ref.Release(context.TODO())
+			ref = nil
+		}
+	}()
+
+	dt, err = readFile(ctx, ref, "foo")
+	require.NoError(t, err)
+	require.Equal(t, dt, []byte("content2"))
+
+	ref.Release(context.TODO())
+	ref = nil
+}
+
+func TestHTTPDefaultName(t *testing.T) {
+	ctx := context.TODO()
+
+	tmpdir, err := ioutil.TempDir("", "buildkit-state")
+	require.NoError(t, err)
+	defer os.RemoveAll(tmpdir)
+
+	hs, err := newHTTPSource(tmpdir)
+	require.NoError(t, err)
+
+	resp := httpserver.Response{
+		Etag:    identity.NewID(),
+		Content: []byte("content1"),
+	}
+	server := httpserver.NewTestServer(map[string]httpserver.Response{
+		"/": resp,
+	})
+	defer server.Close()
+
+	id := &source.HttpIdentifier{URL: server.URL}
+
+	h, err := hs.Resolve(ctx, id)
+	require.NoError(t, err)
+
+	k, err := h.CacheKey(ctx)
+	require.NoError(t, err)
+
+	require.Equal(t, digest.FromBytes([]byte("content1")).String(), k)
+	require.Equal(t, server.Stats("/").AllRequests, 1)
+	require.Equal(t, server.Stats("/").CachedRequests, 0)
+
+	ref, err := h.Snapshot(ctx)
+	require.NoError(t, err)
+	defer func() {
+		if ref != nil {
+			ref.Release(context.TODO())
+			ref = nil
+		}
+	}()
+
+	dt, err := readFile(ctx, ref, "download")
+	require.NoError(t, err)
+	require.Equal(t, dt, []byte("content1"))
+
+	ref.Release(context.TODO())
+	ref = nil
+}
+
+func TestHTTPInvalidURL(t *testing.T) {
+	ctx := context.TODO()
+
+	tmpdir, err := ioutil.TempDir("", "buildkit-state")
+	require.NoError(t, err)
+	defer os.RemoveAll(tmpdir)
+
+	hs, err := newHTTPSource(tmpdir)
+	require.NoError(t, err)
+
+	server := httpserver.NewTestServer(map[string]httpserver.Response{})
+	defer server.Close()
+
+	id := &source.HttpIdentifier{URL: server.URL + "/foo"}
+
+	h, err := hs.Resolve(ctx, id)
+	require.NoError(t, err)
+
+	_, err = h.CacheKey(ctx)
+	require.Error(t, err)
+	require.Contains(t, err.Error(), "invalid response")
+}
+
+func TestHTTPChecksum(t *testing.T) {
+	ctx := context.TODO()
+
+	tmpdir, err := ioutil.TempDir("", "buildkit-state")
+	require.NoError(t, err)
+	defer os.RemoveAll(tmpdir)
+
+	hs, err := newHTTPSource(tmpdir)
+	require.NoError(t, err)
+
+	resp := httpserver.Response{
+		Etag:    identity.NewID(),
+		Content: []byte("content-correct"),
+	}
+	server := httpserver.NewTestServer(map[string]httpserver.Response{
+		"/foo": resp,
+	})
+	defer server.Close()
+
+	id := &source.HttpIdentifier{URL: server.URL + "/foo", Checksum: digest.FromBytes([]byte("content-different"))}
+
+	h, err := hs.Resolve(ctx, id)
+	require.NoError(t, err)
+
+	k, err := h.CacheKey(ctx)
+	require.NoError(t, err)
+
+	require.Equal(t, digest.FromBytes([]byte("content-different")).String(), k)
+	require.Equal(t, server.Stats("/foo").AllRequests, 0)
+	require.Equal(t, server.Stats("/foo").CachedRequests, 0)
+
+	_, err = h.Snapshot(ctx)
+	require.Error(t, err)
+
+	require.Equal(t, digest.FromBytes([]byte("content-different")).String(), k)
+	require.Equal(t, server.Stats("/foo").AllRequests, 1)
+	require.Equal(t, server.Stats("/foo").CachedRequests, 0)
+
+	id = &source.HttpIdentifier{URL: server.URL + "/foo", Checksum: digest.FromBytes([]byte("content-correct"))}
+
+	h, err = hs.Resolve(ctx, id)
+	require.NoError(t, err)
+
+	k, err = h.CacheKey(ctx)
+	require.NoError(t, err)
+
+	require.Equal(t, digest.FromBytes([]byte("content-correct")).String(), k)
+	require.Equal(t, server.Stats("/foo").AllRequests, 1)
+	require.Equal(t, server.Stats("/foo").CachedRequests, 0)
+
+	ref, err := h.Snapshot(ctx)
+	require.NoError(t, err)
+	defer func() {
+		if ref != nil {
+			ref.Release(context.TODO())
+			ref = nil
+		}
+	}()
+
+	dt, err := readFile(ctx, ref, "foo")
+	require.NoError(t, err)
+	require.Equal(t, dt, []byte("content-correct"))
+
+	require.Equal(t, digest.FromBytes([]byte("content-correct")).String(), k)
+	require.Equal(t, server.Stats("/foo").AllRequests, 2)
+	require.Equal(t, server.Stats("/foo").CachedRequests, 0)
+
+	ref.Release(context.TODO())
+	ref = nil
+
+}
+
+func readFile(ctx context.Context, ref cache.ImmutableRef, fp string) ([]byte, error) {
+	mount, err := ref.Mount(ctx, false)
+	if err != nil {
+		return nil, err
+	}
+
+	lm := snapshot.LocalMounter(mount)
+	dir, err := lm.Mount()
+	if err != nil {
+		return nil, err
+	}
+
+	defer lm.Unmount()
+
+	dt, err := ioutil.ReadFile(filepath.Join(dir, fp))
+	if err != nil {
+		return nil, err
+	}
+
+	return dt, nil
+}
+
+func newHTTPSource(tmpdir string) (source.Source, error) {
+	snapshotter, err := naive.NewSnapshotter(filepath.Join(tmpdir, "snapshots"))
+	if err != nil {
+		return nil, err
+	}
+
+	md, err := metadata.NewStore(filepath.Join(tmpdir, "metadata.db"))
+	if err != nil {
+		return nil, err
+	}
+
+	cm, err := cache.NewManager(cache.ManagerOpt{
+		Snapshotter:   snapshotter,
+		MetadataStore: md,
+	})
+	if err != nil {
+		return nil, err
+	}
+
+	return NewSource(Opt{
+		CacheAccessor: cm,
+		MetadataStore: md,
+	})
+}

util/testutil/httpserver/server.go

@@ -0,0 +1,78 @@
+package httpserver
+
+import (
+	"bytes"
+	"io"
+	"net/http"
+	"net/http/httptest"
+	"sync"
+)
+
+type TestServer struct {
+	*httptest.Server
+	mu     sync.Mutex
+	routes map[string]Response
+	stats  map[string]*Stat
+}
+
+func NewTestServer(routes map[string]Response) *TestServer {
+	ts := &TestServer{
+		routes: routes,
+		stats:  map[string]*Stat{},
+	}
+	ts.Server = httptest.NewServer(ts)
+	return ts
+}
+
+func (s *TestServer) SetRoute(name string, resp Response) {
+	s.mu.Lock()
+	defer s.mu.Unlock()
+	s.routes[name] = resp
+}
+
+func (s *TestServer) ServeHTTP(w http.ResponseWriter, r *http.Request) {
+	s.mu.Lock()
+	resp, ok := s.routes[r.URL.Path]
+	if !ok {
+		w.WriteHeader(http.StatusNotFound)
+		s.mu.Unlock()
+		return
+	}
+
+	if _, ok := s.stats[r.URL.Path]; !ok {
+		s.stats[r.URL.Path] = &Stat{}
+	}
+
+	s.stats[r.URL.Path].AllRequests += 1
+
+	if resp.Etag != "" {
+		w.Header().Set("ETag", resp.Etag)
+		if match := r.Header.Get("If-None-Match"); match == resp.Etag {
+			w.WriteHeader(http.StatusNotModified)
+			s.stats[r.URL.Path].CachedRequests++
+			s.mu.Unlock()
+			return
+		}
+	}
+
+	s.mu.Unlock()
+
+	w.WriteHeader(http.StatusOK)
+	io.Copy(w, bytes.NewReader(resp.Content))
+}
+
+func (s *TestServer) Stats(name string) (st Stat) {
+	if st, ok := s.stats[name]; ok {
+		return *st
+	}
+	return
+}
+
+type Response struct {
+	Content []byte
+	Etag    string
+}
+
+type Stat struct {
+	AllRequests, CachedRequests int
+}