diff --git a/builtin/unpack-objects.c b/builtin/unpack-objects.c index bc9b1e047e2..64e58e79fda 100644 --- a/builtin/unpack-objects.c +++ b/builtin/unpack-objects.c @@ -9,6 +9,7 @@ #include "hex.h" #include "object-file.h" #include "odb.h" +#include "odb/streaming.h" #include "odb/transaction.h" #include "object.h" #include "delta.h" @@ -360,24 +361,21 @@ static void unpack_non_delta_entry(enum object_type type, unsigned long size, struct input_zstream_data { git_zstream *zstream; - unsigned char buf[8192]; int status; }; -static const void *feed_input_zstream(struct odb_write_stream *in_stream, - unsigned long *readlen) +static ssize_t feed_input_zstream(struct odb_write_stream *in_stream, + unsigned char *buf, size_t buf_len) { struct input_zstream_data *data = in_stream->data; git_zstream *zstream = data->zstream; void *in = fill(1); - if (in_stream->is_finished) { - *readlen = 0; - return NULL; - } + if (in_stream->is_finished) + return 0; - zstream->next_out = data->buf; - zstream->avail_out = sizeof(data->buf); + zstream->next_out = buf; + zstream->avail_out = buf_len; zstream->next_in = in; zstream->avail_in = len; @@ -385,9 +383,7 @@ static const void *feed_input_zstream(struct odb_write_stream *in_stream, in_stream->is_finished = data->status != Z_OK; use(len - zstream->avail_in); - *readlen = sizeof(data->buf) - zstream->avail_out; - - return data->buf; + return buf_len - zstream->avail_out; } static void stream_blob(unsigned long size, unsigned nr) diff --git a/object-file.c b/object-file.c index bfbb632cf8b..a1afca23c5e 100644 --- a/object-file.c +++ b/object-file.c @@ -1066,6 +1066,7 @@ int odb_source_loose_write_stream(struct odb_source *source, struct git_hash_ctx c, compat_c; struct strbuf tmp_file = STRBUF_INIT; struct strbuf filename = STRBUF_INIT; + unsigned char buf[8192]; int dirlen; char hdr[MAX_HEADER_LEN]; int hdrlen; @@ -1098,9 +1099,17 @@ int odb_source_loose_write_stream(struct odb_source *source, unsigned char *in0 = stream.next_in; if (!stream.avail_in && !in_stream->is_finished) { - const void *in = in_stream->read(in_stream, &stream.avail_in); - stream.next_in = (void *)in; - in0 = (unsigned char *)in; + ssize_t read_len = odb_write_stream_read(in_stream, buf, + sizeof(buf)); + if (read_len < 0) { + close(fd); + err = -1; + goto cleanup; + } + + stream.avail_in = read_len; + stream.next_in = buf; + in0 = buf; /* All data has been read. */ if (in_stream->is_finished) flush = 1; diff --git a/odb.h b/odb.h index ec5367b13ed..6faeaa05891 100644 --- a/odb.h +++ b/odb.h @@ -529,11 +529,7 @@ static inline int odb_write_object(struct object_database *odb, return odb_write_object_ext(odb, buf, len, type, oid, NULL, 0); } -struct odb_write_stream { - const void *(*read)(struct odb_write_stream *, unsigned long *len); - void *data; - int is_finished; -}; +struct odb_write_stream; int odb_write_object_stream(struct object_database *odb, struct odb_write_stream *stream, size_t len, diff --git a/odb/streaming.c b/odb/streaming.c index 5927a12954b..a68dd2cbe37 100644 --- a/odb/streaming.c +++ b/odb/streaming.c @@ -232,6 +232,11 @@ struct odb_read_stream *odb_read_stream_open(struct object_database *odb, return st; } +ssize_t odb_write_stream_read(struct odb_write_stream *st, void *buf, size_t sz) +{ + return st->read(st, buf, sz); +} + int odb_stream_blob_to_fd(struct object_database *odb, int fd, const struct object_id *oid, diff --git a/odb/streaming.h b/odb/streaming.h index c7861f7e13c..65ced911fec 100644 --- a/odb/streaming.h +++ b/odb/streaming.h @@ -47,6 +47,24 @@ int odb_read_stream_close(struct odb_read_stream *stream); */ ssize_t odb_read_stream_read(struct odb_read_stream *stream, void *buf, size_t len); +/* + * A stream that provides an object to be written to the object database without + * loading all of it into memory. + */ +struct odb_write_stream { + ssize_t (*read)(struct odb_write_stream *, unsigned char *, size_t); + void *data; + int is_finished; +}; + +/* + * Read data from the stream into the buffer. Returns 0 when finished and the + * number of bytes read on success. Returns a negative error code in case + * reading from the stream fails. + */ +ssize_t odb_write_stream_read(struct odb_write_stream *stream, void *buf, + size_t len); + /* * Look up the object by its ID and write the full contents to the file * descriptor. The object must be a blob, or the function will fail. When