From c9f45dbc22df16e7f2525cb6b1b42ca507a45cd5 Mon Sep 17 00:00:00 2001 From: Tonis Tiigi Date: Fri, 1 Dec 2017 11:44:27 -0800 Subject: [PATCH] add unit tests for http source Signed-off-by: Tonis Tiigi --- source/git/gitsource_test.go | 6 +- source/http/httpsource.go | 11 +- source/http/httpsource_test.go | 318 +++++++++++++++++++++++++++++ util/testutil/httpserver/server.go | 78 +++++++ 4 files changed, 403 insertions(+), 10 deletions(-) create mode 100644 source/http/httpsource_test.go create mode 100644 util/testutil/httpserver/server.go diff --git a/source/git/gitsource_test.go b/source/git/gitsource_test.go index 576e5509a..952162ed5 100644 --- a/source/git/gitsource_test.go +++ b/source/git/gitsource_test.go @@ -27,7 +27,7 @@ func TestRepeatedFetchKeepGitDir(t *testing.T) { } func testRepeatedFetch(t *testing.T, keepGitDir bool) { - ctx := namespaces.WithNamespace(context.Background(), "buildkit-test") + ctx := context.TODO() tmpdir, err := ioutil.TempDir("", "buildkit-state") require.NoError(t, err) @@ -278,10 +278,6 @@ func setupGitSource(t *testing.T, tmpdir string) source.Source { }) assert.NoError(t, err) - repodir, err := ioutil.TempDir("", "buildkit-gitsource") - require.NoError(t, err) - defer os.RemoveAll(repodir) - gs, err := NewSource(Opt{ CacheAccessor: cm, MetadataStore: md, diff --git a/source/http/httpsource.go b/source/http/httpsource.go index d256bd53b..921e03a09 100644 --- a/source/http/httpsource.go +++ b/source/http/httpsource.go @@ -57,7 +57,7 @@ type httpSourceHandler struct { func (hs *httpSource) Resolve(ctx context.Context, id source.Identifier) (source.SourceInstance, error) { httpIdentifier, ok := id.(*source.HttpIdentifier) if !ok { - return nil, errors.Errorf("invalid git identifier %v", id) + return nil, errors.Errorf("invalid http identifier %v", id) } return &httpSourceHandler{ @@ -105,7 +105,9 @@ func (hs *httpSourceHandler) CacheKey(ctx context.Context) (string, error) { if err != nil { return "", err } - + if resp.StatusCode < 200 || resp.StatusCode >= 400 { + return "", errors.Errorf("invalid response status %d", resp.StatusCode) + } if resp.StatusCode == http.StatusNotModified { respETag := resp.Header.Get("ETag") si, ok := m[respETag] @@ -164,7 +166,6 @@ func (hs *httpSourceHandler) save(ctx context.Context, resp *http.Response) (ref lm.Unmount() } }() - f, err := os.Create(filepath.Join(dir, getFileName(hs.src.URL, resp))) if err != nil { return nil, "", err @@ -227,7 +228,7 @@ func (hs *httpSourceHandler) Snapshot(ctx context.Context) (cache.ImmutableRef, return nil, errors.Errorf("digest mismatch %s: %s", dgst, hs.cacheKey) } - return ref, errors.Errorf("not-implemented") + return ref, nil } const keyETag = "etag" @@ -296,7 +297,7 @@ func getFileName(urlStr string, resp *http.Response) string { } u, err := url.Parse(urlStr) if err == nil { - if base := path.Base(u.Path); base != "" { + if base := path.Base(u.Path); base != "." { return base } } diff --git a/source/http/httpsource_test.go b/source/http/httpsource_test.go new file mode 100644 index 000000000..d4bb007de --- /dev/null +++ b/source/http/httpsource_test.go @@ -0,0 +1,318 @@ +package http + +import ( + "context" + "io/ioutil" + "os" + "path/filepath" + "testing" + + "github.com/containerd/containerd/snapshot/naive" + "github.com/moby/buildkit/cache" + "github.com/moby/buildkit/cache/metadata" + "github.com/moby/buildkit/identity" + "github.com/moby/buildkit/snapshot" + "github.com/moby/buildkit/source" + "github.com/moby/buildkit/util/testutil/httpserver" + digest "github.com/opencontainers/go-digest" + "github.com/stretchr/testify/require" +) + +func TestHTTPSource(t *testing.T) { + ctx := context.TODO() + + tmpdir, err := ioutil.TempDir("", "buildkit-state") + require.NoError(t, err) + defer os.RemoveAll(tmpdir) + + hs, err := newHTTPSource(tmpdir) + require.NoError(t, err) + + resp := httpserver.Response{ + Etag: identity.NewID(), + Content: []byte("content1"), + } + server := httpserver.NewTestServer(map[string]httpserver.Response{ + "/foo": resp, + }) + defer server.Close() + + id := &source.HttpIdentifier{URL: server.URL + "/foo"} + + h, err := hs.Resolve(ctx, id) + require.NoError(t, err) + + k, err := h.CacheKey(ctx) + require.NoError(t, err) + + require.Equal(t, digest.FromBytes([]byte("content1")).String(), k) + require.Equal(t, server.Stats("/foo").AllRequests, 1) + require.Equal(t, server.Stats("/foo").CachedRequests, 0) + + ref, err := h.Snapshot(ctx) + require.NoError(t, err) + defer func() { + if ref != nil { + ref.Release(context.TODO()) + ref = nil + } + }() + + dt, err := readFile(ctx, ref, "foo") + require.NoError(t, err) + require.Equal(t, dt, []byte("content1")) + + ref.Release(context.TODO()) + ref = nil + + // repeat, should use the etag + h, err = hs.Resolve(ctx, id) + require.NoError(t, err) + + k, err = h.CacheKey(ctx) + require.NoError(t, err) + + require.Equal(t, digest.FromBytes([]byte("content1")).String(), k) + require.Equal(t, server.Stats("/foo").AllRequests, 2) + require.Equal(t, server.Stats("/foo").CachedRequests, 1) + + ref, err = h.Snapshot(ctx) + require.NoError(t, err) + defer func() { + if ref != nil { + ref.Release(context.TODO()) + ref = nil + } + }() + + dt, err = readFile(ctx, ref, "foo") + require.NoError(t, err) + require.Equal(t, dt, []byte("content1")) + + ref.Release(context.TODO()) + ref = nil + + resp2 := httpserver.Response{ + Etag: identity.NewID(), + Content: []byte("content2"), + } + + // update etag, downloads again + server.SetRoute("/foo", resp2) + + h, err = hs.Resolve(ctx, id) + require.NoError(t, err) + + k, err = h.CacheKey(ctx) + require.NoError(t, err) + + require.Equal(t, digest.FromBytes([]byte("content2")).String(), k) + require.Equal(t, server.Stats("/foo").AllRequests, 3) + require.Equal(t, server.Stats("/foo").CachedRequests, 1) + + ref, err = h.Snapshot(ctx) + require.NoError(t, err) + defer func() { + if ref != nil { + ref.Release(context.TODO()) + ref = nil + } + }() + + dt, err = readFile(ctx, ref, "foo") + require.NoError(t, err) + require.Equal(t, dt, []byte("content2")) + + ref.Release(context.TODO()) + ref = nil +} + +func TestHTTPDefaultName(t *testing.T) { + ctx := context.TODO() + + tmpdir, err := ioutil.TempDir("", "buildkit-state") + require.NoError(t, err) + defer os.RemoveAll(tmpdir) + + hs, err := newHTTPSource(tmpdir) + require.NoError(t, err) + + resp := httpserver.Response{ + Etag: identity.NewID(), + Content: []byte("content1"), + } + server := httpserver.NewTestServer(map[string]httpserver.Response{ + "/": resp, + }) + defer server.Close() + + id := &source.HttpIdentifier{URL: server.URL} + + h, err := hs.Resolve(ctx, id) + require.NoError(t, err) + + k, err := h.CacheKey(ctx) + require.NoError(t, err) + + require.Equal(t, digest.FromBytes([]byte("content1")).String(), k) + require.Equal(t, server.Stats("/").AllRequests, 1) + require.Equal(t, server.Stats("/").CachedRequests, 0) + + ref, err := h.Snapshot(ctx) + require.NoError(t, err) + defer func() { + if ref != nil { + ref.Release(context.TODO()) + ref = nil + } + }() + + dt, err := readFile(ctx, ref, "download") + require.NoError(t, err) + require.Equal(t, dt, []byte("content1")) + + ref.Release(context.TODO()) + ref = nil +} + +func TestHTTPInvalidURL(t *testing.T) { + ctx := context.TODO() + + tmpdir, err := ioutil.TempDir("", "buildkit-state") + require.NoError(t, err) + defer os.RemoveAll(tmpdir) + + hs, err := newHTTPSource(tmpdir) + require.NoError(t, err) + + server := httpserver.NewTestServer(map[string]httpserver.Response{}) + defer server.Close() + + id := &source.HttpIdentifier{URL: server.URL + "/foo"} + + h, err := hs.Resolve(ctx, id) + require.NoError(t, err) + + _, err = h.CacheKey(ctx) + require.Error(t, err) + require.Contains(t, err.Error(), "invalid response") +} + +func TestHTTPChecksum(t *testing.T) { + ctx := context.TODO() + + tmpdir, err := ioutil.TempDir("", "buildkit-state") + require.NoError(t, err) + defer os.RemoveAll(tmpdir) + + hs, err := newHTTPSource(tmpdir) + require.NoError(t, err) + + resp := httpserver.Response{ + Etag: identity.NewID(), + Content: []byte("content-correct"), + } + server := httpserver.NewTestServer(map[string]httpserver.Response{ + "/foo": resp, + }) + defer server.Close() + + id := &source.HttpIdentifier{URL: server.URL + "/foo", Checksum: digest.FromBytes([]byte("content-different"))} + + h, err := hs.Resolve(ctx, id) + require.NoError(t, err) + + k, err := h.CacheKey(ctx) + require.NoError(t, err) + + require.Equal(t, digest.FromBytes([]byte("content-different")).String(), k) + require.Equal(t, server.Stats("/foo").AllRequests, 0) + require.Equal(t, server.Stats("/foo").CachedRequests, 0) + + _, err = h.Snapshot(ctx) + require.Error(t, err) + + require.Equal(t, digest.FromBytes([]byte("content-different")).String(), k) + require.Equal(t, server.Stats("/foo").AllRequests, 1) + require.Equal(t, server.Stats("/foo").CachedRequests, 0) + + id = &source.HttpIdentifier{URL: server.URL + "/foo", Checksum: digest.FromBytes([]byte("content-correct"))} + + h, err = hs.Resolve(ctx, id) + require.NoError(t, err) + + k, err = h.CacheKey(ctx) + require.NoError(t, err) + + require.Equal(t, digest.FromBytes([]byte("content-correct")).String(), k) + require.Equal(t, server.Stats("/foo").AllRequests, 1) + require.Equal(t, server.Stats("/foo").CachedRequests, 0) + + ref, err := h.Snapshot(ctx) + require.NoError(t, err) + defer func() { + if ref != nil { + ref.Release(context.TODO()) + ref = nil + } + }() + + dt, err := readFile(ctx, ref, "foo") + require.NoError(t, err) + require.Equal(t, dt, []byte("content-correct")) + + require.Equal(t, digest.FromBytes([]byte("content-correct")).String(), k) + require.Equal(t, server.Stats("/foo").AllRequests, 2) + require.Equal(t, server.Stats("/foo").CachedRequests, 0) + + ref.Release(context.TODO()) + ref = nil + +} + +func readFile(ctx context.Context, ref cache.ImmutableRef, fp string) ([]byte, error) { + mount, err := ref.Mount(ctx, false) + if err != nil { + return nil, err + } + + lm := snapshot.LocalMounter(mount) + dir, err := lm.Mount() + if err != nil { + return nil, err + } + + defer lm.Unmount() + + dt, err := ioutil.ReadFile(filepath.Join(dir, fp)) + if err != nil { + return nil, err + } + + return dt, nil +} + +func newHTTPSource(tmpdir string) (source.Source, error) { + snapshotter, err := naive.NewSnapshotter(filepath.Join(tmpdir, "snapshots")) + if err != nil { + return nil, err + } + + md, err := metadata.NewStore(filepath.Join(tmpdir, "metadata.db")) + if err != nil { + return nil, err + } + + cm, err := cache.NewManager(cache.ManagerOpt{ + Snapshotter: snapshotter, + MetadataStore: md, + }) + if err != nil { + return nil, err + } + + return NewSource(Opt{ + CacheAccessor: cm, + MetadataStore: md, + }) +} diff --git a/util/testutil/httpserver/server.go b/util/testutil/httpserver/server.go new file mode 100644 index 000000000..1ed0199a9 --- /dev/null +++ b/util/testutil/httpserver/server.go @@ -0,0 +1,78 @@ +package httpserver + +import ( + "bytes" + "io" + "net/http" + "net/http/httptest" + "sync" +) + +type TestServer struct { + *httptest.Server + mu sync.Mutex + routes map[string]Response + stats map[string]*Stat +} + +func NewTestServer(routes map[string]Response) *TestServer { + ts := &TestServer{ + routes: routes, + stats: map[string]*Stat{}, + } + ts.Server = httptest.NewServer(ts) + return ts +} + +func (s *TestServer) SetRoute(name string, resp Response) { + s.mu.Lock() + defer s.mu.Unlock() + s.routes[name] = resp +} + +func (s *TestServer) ServeHTTP(w http.ResponseWriter, r *http.Request) { + s.mu.Lock() + resp, ok := s.routes[r.URL.Path] + if !ok { + w.WriteHeader(http.StatusNotFound) + s.mu.Unlock() + return + } + + if _, ok := s.stats[r.URL.Path]; !ok { + s.stats[r.URL.Path] = &Stat{} + } + + s.stats[r.URL.Path].AllRequests += 1 + + if resp.Etag != "" { + w.Header().Set("ETag", resp.Etag) + if match := r.Header.Get("If-None-Match"); match == resp.Etag { + w.WriteHeader(http.StatusNotModified) + s.stats[r.URL.Path].CachedRequests++ + s.mu.Unlock() + return + } + } + + s.mu.Unlock() + + w.WriteHeader(http.StatusOK) + io.Copy(w, bytes.NewReader(resp.Content)) +} + +func (s *TestServer) Stats(name string) (st Stat) { + if st, ok := s.stats[name]; ok { + return *st + } + return +} + +type Response struct { + Content []byte + Etag string +} + +type Stat struct { + AllRequests, CachedRequests int +}