http: Support authentication

Support authentication for HTTP sources.

 - llb: Define general `llb.AuthOption` interface composed of
   `HTTPOption` and `GitOption`.
 - llb: Refactor `llb.AuthHeaderSecret` to return an `llb.AuthOption` so
   it may be used with both `llb.Git` and `llb.HTTP`.
 - llb: Define `HTTPInfo.AuthHeaderSecret`.
 - llb: Define and flag new `source.http.auth` capability when
   `HTTPInfo.AuthHeaderSecret` is set.
 - solver: Define new `http.auth` source attribute.
 - source/http: If an `http.auth` attribute is specified, resolve a
   secret named by its value and set the "Authorization" request header.

Signed-off-by: Dan Duvall <dduvall@wikimedia.org>

client/client_test.go

@@ -105,6 +105,7 @@ var allTests = []func(t *testing.T, sb integration.Sandbox){
 	testBuildMultiMount,
 	testBuildHTTPSource,
 	testBuildHTTPSourceEtagScope,
+	testBuildHTTPSourceAuthHeaderSecret,
 	testBuildPushAndValidate,
 	testBuildExportWithUncompressed,
 	testBuildExportScratch,
@@ -2987,6 +2988,47 @@ func testBuildHTTPSourceEtagScope(t *testing.T, sb integration.Sandbox) {
 	require.NoError(t, os.RemoveAll(filepath.Join(out2, "foo")))
 }
 
+func testBuildHTTPSourceAuthHeaderSecret(t *testing.T, sb integration.Sandbox) {
+	c, err := New(sb.Context(), sb.Address())
+	require.NoError(t, err)
+	defer c.Close()
+
+	modTime := time.Now().Add(-24 * time.Hour) // avoid false positive with current time
+
+	resp := httpserver.Response{
+		Etag:         identity.NewID(),
+		Content:      []byte("content1"),
+		LastModified: &modTime,
+	}
+
+	server := httpserver.NewTestServer(map[string]httpserver.Response{
+		"/foo": resp,
+	})
+	defer server.Close()
+
+	st := llb.HTTP(server.URL+"/foo", llb.AuthHeaderSecret("http-secret"))
+
+	def, err := st.Marshal(sb.Context())
+	require.NoError(t, err)
+
+	_, err = c.Solve(
+		sb.Context(),
+		def,
+		SolveOpt{
+			Session: []session.Attachable{secretsprovider.FromMap(map[string][]byte{
+				"http-secret": []byte("Bearer foo"),
+			})},
+		},
+		nil,
+	)
+	require.NoError(t, err)
+
+	allReqs := server.Stats("/foo").Requests
+	require.Equal(t, 1, len(allReqs))
+	require.Equal(t, http.MethodGet, allReqs[0].Method)
+	require.Equal(t, "Bearer foo", allReqs[0].Header.Get("Authorization"))
+}
+
 func testResolveAndHosts(t *testing.T, sb integration.Sandbox) {
 	requiresLinux(t)
 	c, err := New(sb.Context(), sb.Address())

client/llb/source.go

@@ -360,13 +360,6 @@ func AuthTokenSecret(v string) GitOption {
 	})
 }
 
-func AuthHeaderSecret(v string) GitOption {
-	return gitOptionFunc(func(gi *GitInfo) {
-		gi.AuthHeaderSecret = v
-		gi.addAuthCap = true
-	})
-}
-
 func KnownSSHHosts(key string) GitOption {
 	key = strings.TrimSuffix(key, "\n")
 	return gitOptionFunc(func(gi *GitInfo) {
@@ -380,6 +373,29 @@ func MountSSHSock(sshID string) GitOption {
 	})
 }
 
+// AuthOption can be used with either HTTP or Git sources.
+type AuthOption interface {
+	GitOption
+	HTTPOption
+}
+
+// AuthHeaderSecret returns an AuthOption that defines the name of a
+// secret to use for HTTP based authentication.
+func AuthHeaderSecret(secretName string) AuthOption {
+	return struct {
+		GitOption
+		HTTPOption
+	}{
+		GitOption: gitOptionFunc(func(gi *GitInfo) {
+			gi.AuthHeaderSecret = secretName
+			gi.addAuthCap = true
+		}),
+		HTTPOption: httpOptionFunc(func(hi *HTTPInfo) {
+			hi.AuthHeaderSecret = secretName
+		}),
+	}
+}
+
 // Scratch returns a state that represents an empty filesystem.
 func Scratch() State {
 	return NewState(nil)
@@ -595,6 +611,10 @@ func HTTP(url string, opts ...HTTPOption) State {
 		attrs[pb.AttrHTTPGID] = strconv.Itoa(hi.GID)
 		addCap(&hi.Constraints, pb.CapSourceHTTPUIDGID)
 	}
+	if hi.AuthHeaderSecret != "" {
+		attrs[pb.AttrHTTPAuthHeaderSecret] = hi.AuthHeaderSecret
+		addCap(&hi.Constraints, pb.CapSourceHTTPAuth)
+	}
 
 	addCap(&hi.Constraints, pb.CapSourceHTTP)
 	source := NewSource(url, attrs, hi.Constraints)
@@ -603,11 +623,12 @@ func HTTP(url string, opts ...HTTPOption) State {
 
 type HTTPInfo struct {
 	constraintsWrapper
-	Checksum digest.Digest
-	Filename string
-	Perm     int
-	UID      int
-	GID      int
+	Checksum         digest.Digest
+	Filename         string
+	Perm             int
+	UID              int
+	GID              int
+	AuthHeaderSecret string
 }
 
 type HTTPOption interface {

solver/pb/attr.go

@@ -20,6 +20,7 @@ const AttrHTTPFilename = "http.filename"
 const AttrHTTPPerm = "http.perm"
 const AttrHTTPUID = "http.uid"
 const AttrHTTPGID = "http.gid"
+const AttrHTTPAuthHeaderSecret = "http.authheadersecret"
 
 const AttrImageResolveMode = "image.resolvemode"
 const AttrImageResolveModeDefault = "default"

solver/pb/caps.go

@@ -31,9 +31,11 @@ const (
 	CapSourceGitSubdir        apicaps.CapID = "source.git.subdir"
 
 	CapSourceHTTP         apicaps.CapID = "source.http"
+	CapSourceHTTPAuth     apicaps.CapID = "source.http.auth"
 	CapSourceHTTPChecksum apicaps.CapID = "source.http.checksum"
 	CapSourceHTTPPerm     apicaps.CapID = "source.http.perm"
-	CapSourceHTTPUIDGID   apicaps.CapID = "soruce.http.uidgid"
+	// NOTE the historical typo
+	CapSourceHTTPUIDGID apicaps.CapID = "soruce.http.uidgid"
 
 	CapSourceOCILayout apicaps.CapID = "source.ocilayout"
 
@@ -229,6 +231,12 @@ func init() {
 		Status:  apicaps.CapStatusExperimental,
 	})
 
+	Caps.Init(apicaps.Cap{
+		ID:      CapSourceHTTPAuth,
+		Enabled: true,
+		Status:  apicaps.CapStatusExperimental,
+	})
+
 	Caps.Init(apicaps.Cap{
 		ID:      CapSourceOCILayout,
 		Enabled: true,

source/http/identifier.go

@@ -18,13 +18,14 @@ func NewHTTPIdentifier(str string, tls bool) (*HTTPIdentifier, error) {
 }
 
 type HTTPIdentifier struct {
-	TLS      bool
-	URL      string
-	Checksum digest.Digest
-	Filename string
-	Perm     int
-	UID      int
-	GID      int
+	TLS              bool
+	URL              string
+	Checksum         digest.Digest
+	Filename         string
+	Perm             int
+	UID              int
+	GID              int
+	AuthHeaderSecret string
 }
 
 var _ source.Identifier = (*HTTPIdentifier)(nil)

source/http/source.go

@@ -20,6 +20,7 @@ import (
 	"github.com/docker/docker/pkg/idtools"
 	"github.com/moby/buildkit/cache"
 	"github.com/moby/buildkit/session"
+	"github.com/moby/buildkit/session/secrets"
 	"github.com/moby/buildkit/snapshot"
 	"github.com/moby/buildkit/solver"
 	"github.com/moby/buildkit/solver/pb"
@@ -92,6 +93,8 @@ func (hs *httpSource) Identifier(scheme, ref string, attrs map[string]string, pl
 				return nil, err
 			}
 			id.GID = int(i)
+		case pb.AttrHTTPAuthHeaderSecret:
+			id.AuthHeaderSecret = v
 		}
 	}
 
@@ -127,16 +130,18 @@ func (hs *httpSourceHandler) client(g session.Group) *http.Client {
 // this package.
 func (hs *httpSourceHandler) urlHash() (digest.Digest, error) {
 	dt, err := json.Marshal(struct {
-		Filename       []byte
-		Perm, UID, GID int
+		Filename         []byte
+		Perm, UID, GID   int
+		AuthHeaderSecret string `json:",omitempty"`
 	}{
 		Filename: bytes.Join([][]byte{
 			[]byte(hs.src.URL),
 			[]byte(hs.src.Filename),
 		}, []byte{0}),
-		Perm: hs.src.Perm,
-		UID:  hs.src.UID,
-		GID:  hs.src.GID,
+		Perm:             hs.src.Perm,
+		UID:              hs.src.UID,
+		GID:              hs.src.GID,
+		AuthHeaderSecret: hs.src.AuthHeaderSecret,
 	})
 	if err != nil {
 		return "", err
@@ -146,17 +151,19 @@ func (hs *httpSourceHandler) urlHash() (digest.Digest, error) {
 
 func (hs *httpSourceHandler) formatCacheKey(filename string, dgst digest.Digest, lastModTime string) digest.Digest {
 	dt, err := json.Marshal(struct {
-		Filename       string
-		Perm, UID, GID int
-		Checksum       digest.Digest
-		LastModTime    string `json:",omitempty"`
+		Filename         string
+		Perm, UID, GID   int
+		Checksum         digest.Digest
+		LastModTime      string `json:",omitempty"`
+		AuthHeaderSecret string `json:",omitempty"`
 	}{
-		Filename:    filename,
-		Perm:        hs.src.Perm,
-		UID:         hs.src.UID,
-		GID:         hs.src.GID,
-		Checksum:    dgst,
-		LastModTime: lastModTime,
+		Filename:         filename,
+		Perm:             hs.src.Perm,
+		UID:              hs.src.UID,
+		GID:              hs.src.GID,
+		Checksum:         dgst,
+		LastModTime:      lastModTime,
+		AuthHeaderSecret: hs.src.AuthHeaderSecret,
 	})
 	if err != nil {
 		return dgst
@@ -181,7 +188,7 @@ func (hs *httpSourceHandler) CacheKey(ctx context.Context, g session.Group, inde
 		return "", "", nil, false, errors.Wrapf(err, "failed to search metadata for %s", uh)
 	}
 
-	req, err := hs.newHTTPRequest(ctx)
+	req, err := hs.newHTTPRequest(ctx, g)
 	if err != nil {
 		return "", "", nil, false, err
 	}
@@ -441,7 +448,7 @@ func (hs *httpSourceHandler) Snapshot(ctx context.Context, g session.Group) (cac
 		}
 	}
 
-	req, err := hs.newHTTPRequest(ctx)
+	req, err := hs.newHTTPRequest(ctx, g)
 	if err != nil {
 		return nil, err
 	}
@@ -468,7 +475,7 @@ func (hs *httpSourceHandler) Snapshot(ctx context.Context, g session.Group) (cac
 	return ref, nil
 }
 
-func (hs *httpSourceHandler) newHTTPRequest(ctx context.Context) (*http.Request, error) {
+func (hs *httpSourceHandler) newHTTPRequest(ctx context.Context, g session.Group) (*http.Request, error) {
 	req, err := http.NewRequest("GET", hs.src.URL, nil)
 	if err != nil {
 		return nil, err
@@ -476,6 +483,23 @@ func (hs *httpSourceHandler) newHTTPRequest(ctx context.Context) (*http.Request,
 
 	req.Header.Set("User-Agent", version.UserAgent())
 
+	if hs.src.AuthHeaderSecret != "" {
+		err := hs.sm.Any(ctx, g, func(ctx context.Context, _ string, caller session.Caller) error {
+			dt, err := secrets.GetSecret(ctx, caller, hs.src.AuthHeaderSecret)
+			if err != nil {
+				return err
+			}
+
+			req.Header.Set("Authorization", string(dt))
+
+			return nil
+		})
+
+		if err != nil {
+			return nil, errors.Wrapf(err, "failed to retrieve HTTP auth secret %s", hs.src.AuthHeaderSecret)
+		}
+	}
+
 	return req.WithContext(ctx), nil
 }