From 329e945c254cde7144b9300e00dee26d232657ad Mon Sep 17 00:00:00 2001 From: Tonis Tiigi Date: Mon, 9 Oct 2023 15:06:17 -0700 Subject: [PATCH 1/3] filesync: split stream data into 3MB chunks to avoid message limits Signed-off-by: Tonis Tiigi --- session/filesync/diffcopy.go | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/session/filesync/diffcopy.go b/session/filesync/diffcopy.go index 27bc5d541..56bffe535 100644 --- a/session/filesync/diffcopy.go +++ b/session/filesync/diffcopy.go @@ -47,6 +47,22 @@ type streamWriterCloser struct { } func (wc *streamWriterCloser) Write(dt []byte) (int, error) { + // grpc-go has a 4MB limit on messages by default. Split large messages + // so we don't get close to that limit. + const maxChunkSize = 3 * 1024 * 1024 + if len(dt) > maxChunkSize { + n1, err := wc.Write(dt[:maxChunkSize]) + if err != nil { + return n1, err + } + dt = dt[maxChunkSize:] + var n2 int + if n2, err = wc.Write(dt); err != nil { + return n1 + n2, err + } + return n1 + n2, nil + } + if err := wc.ClientStream.SendMsg(&BytesMessage{Data: dt}); err != nil { // SendMsg return EOF on remote errors if errors.Is(err, io.EOF) { From ac33aec24e86d374136bcc65e7cc59241bdf0a5e Mon Sep 17 00:00:00 2001 From: Tonis Tiigi Date: Mon, 9 Oct 2023 15:08:47 -0700 Subject: [PATCH 2/3] session: raise grpc message size limits for session endpoint These limits were already set for control API requests but not for session requests. Signed-off-by: Tonis Tiigi --- session/grpc.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/session/grpc.go b/session/grpc.go index bf8180722..0e475199a 100644 --- a/session/grpc.go +++ b/session/grpc.go @@ -7,6 +7,7 @@ import ( "sync/atomic" "time" + "github.com/containerd/containerd/defaults" grpc_middleware "github.com/grpc-ecosystem/go-grpc-middleware" "github.com/moby/buildkit/util/bklog" "github.com/moby/buildkit/util/grpcerrors" @@ -44,6 +45,8 @@ func grpcClientConn(ctx context.Context, conn net.Conn) (context.Context, *grpc. dialOpts := []grpc.DialOption{ dialer, grpc.WithTransportCredentials(insecure.NewCredentials()), + grpc.WithDefaultCallOptions(grpc.MaxCallRecvMsgSize(defaults.DefaultMaxRecvMsgSize)), + grpc.WithDefaultCallOptions(grpc.MaxCallSendMsgSize(defaults.DefaultMaxSendMsgSize)), } if span := trace.SpanFromContext(ctx); span.SpanContext().IsValid() { From 463bdab463884ac97c1cab8f4df3d121ee310ee3 Mon Sep 17 00:00:00 2001 From: Tonis Tiigi Date: Mon, 9 Oct 2023 15:11:22 -0700 Subject: [PATCH 3/3] uploadprovider: avoid too big grpc messages on tar upload Signed-off-by: Tonis Tiigi --- session/upload/uploadprovider/provider.go | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/session/upload/uploadprovider/provider.go b/session/upload/uploadprovider/provider.go index fe7b26a20..010959f43 100644 --- a/session/upload/uploadprovider/provider.go +++ b/session/upload/uploadprovider/provider.go @@ -59,6 +59,20 @@ type writer struct { } func (w *writer) Write(dt []byte) (int, error) { + // avoid sending too big messages on grpc stream + const maxChunkSize = 3 * 1024 * 1024 + if len(dt) > maxChunkSize { + n1, err := w.Write(dt[:maxChunkSize]) + if err != nil { + return n1, err + } + dt = dt[maxChunkSize:] + var n2 int + if n2, err := w.Write(dt); err != nil { + return n1 + n2, err + } + return n1 + n2, nil + } if err := w.SendMsg(&upload.BytesMessage{Data: dt}); err != nil { return 0, err }