odb/streaming: support streaming arbitrary object types

The object database supports the ability to write object streams into
it. This functionality is used when we encounter a blob that is larger
than "core.bigFileThreshold" so that we don't have to soak large files
into memory.

As we only ever write large files, the infrastructure doesn't support
specifying any other object type than "blob". This limitation is quite
artificial though: there is no reason why we shouldn't support writing
arbitrary large objects with a stream. While it's very unlikely that we
encounter a huge object other than a blob, users are known to be
creative and sometimes like to inflict pain on themselves by creating
commits or trees that are huge.

Extend the infrastructure to support streaming arbitrary object types.
For now we don't use this functionality anywhere, but it brings us a bit
closer to unify `struct odb_read_stream` and `struct odb_write_stream`.

Signed-off-by: Patrick Steinhardt <ps@pks.im>
Signed-off-by: Junio C Hamano <gitster@pobox.com>

builtin/unpack-objects.c

@@ -393,6 +393,7 @@ static void stream_blob(unsigned long size, unsigned nr)
 		.read = feed_input_zstream,
 		.data = &data,
 		.size = size,
+		.type = OBJ_BLOB,
 	};
 	struct obj_info *info = &obj_list[nr];
 

object-file.c

@@ -702,9 +702,9 @@ static void prepare_packfile_transaction(struct odb_transaction_files *transacti
 		die_errno("unable to write pack header");
 }
 
-static int hash_blob_stream(struct odb_write_stream *stream,
-			    const struct git_hash_algo *hash_algo,
-			    struct object_id *result_oid)
+static int hash_stream(struct odb_write_stream *stream,
+		       const struct git_hash_algo *hash_algo,
+		       struct object_id *result_oid)
 {
 	unsigned char buf[16384];
 	struct git_hash_ctx ctx;
@@ -712,7 +712,7 @@ static int hash_blob_stream(struct odb_write_stream *stream,
 	size_t bytes_hashed = 0;
 
 	header_len = format_object_header((char *)buf, sizeof(buf),
-					  OBJ_BLOB, stream->size);
+					  stream->type, stream->size);
 	git_hash_init(&ctx, hash_algo);
 	git_hash_update(&ctx, buf, header_len);
 
@@ -740,9 +740,9 @@ static int hash_blob_stream(struct odb_write_stream *stream,
  * Read the contents from the stream provided, streaming it to the
  * packfile in state while updating the hash in ctx.
  */
-static void stream_blob_to_pack(struct transaction_packfile *state,
-				struct git_hash_ctx *ctx,
-				struct odb_write_stream *stream)
+static void stream_to_pack(struct transaction_packfile *state,
+			   struct git_hash_ctx *ctx,
+			   struct odb_write_stream *stream)
 {
 	git_zstream s;
 	unsigned char ibuf[16384];
@@ -755,7 +755,7 @@ static void stream_blob_to_pack(struct transaction_packfile *state,
 
 	git_deflate_init(&s, cfg->pack_compression_level);
 
-	hdrlen = encode_in_pack_object_header(obuf, sizeof(obuf), OBJ_BLOB, stream->size);
+	hdrlen = encode_in_pack_object_header(obuf, sizeof(obuf), stream->type, stream->size);
 	s.next_out = obuf + hdrlen;
 	s.avail_out = sizeof(obuf) - hdrlen;
 
@@ -764,7 +764,7 @@ static void stream_blob_to_pack(struct transaction_packfile *state,
 			ssize_t rsize = odb_write_stream_read(stream, ibuf,
 							      sizeof(ibuf));
 			if (rsize < 0)
-				die("failed to read blob data");
+				die("failed to read object data");
 			if (!rsize)
 				is_finished = true;
 
@@ -797,7 +797,7 @@ static void stream_blob_to_pack(struct transaction_packfile *state,
 	}
 
 	if (bytes_read != stream->size)
-		die("read %" PRIuMAX " bytes of blob data, but expected %" PRIuMAX " bytes",
+		die("read %" PRIuMAX " bytes of object data, but expected %" PRIuMAX " bytes",
 		    (uintmax_t)bytes_read, (uintmax_t)stream->size);
 
 	git_deflate_end(&s);
@@ -868,7 +868,7 @@ clear_exit:
  * result, which we need to know beforehand when writing a git object.
  * Since the primary motivation for trying to stream from the working
  * tree file and to avoid mmaping it in core is to deal with large
- * binary blobs, they generally do not want to get any conversion, and
+ * objects, they generally do not want to get any conversion, and
  * callers should avoid this code path when filters are requested.
  */
 static int odb_transaction_files_write_object_stream(struct odb_transaction *base,
@@ -886,7 +886,7 @@ static int odb_transaction_files_write_object_stream(struct odb_transaction *bas
 	struct pack_idx_entry *idx;
 
 	header_len = format_object_header((char *)obuf, sizeof(obuf),
-					  OBJ_BLOB, stream->size);
+					  stream->type, stream->size);
 	git_hash_init(&ctx, transaction->base.source->odb->repo->hash_algo);
 	git_hash_update(&ctx, obuf, header_len);
 
@@ -911,7 +911,7 @@ static int odb_transaction_files_write_object_stream(struct odb_transaction *bas
 	hashfile_checkpoint(state->f, &checkpoint);
 	idx->offset = state->offset;
 	crc32_begin(state->f);
-	stream_blob_to_pack(state, &ctx, stream);
+	stream_to_pack(state, &ctx, stream);
 	git_hash_final_oid(result_oid, &ctx);
 
 	idx->crc32 = crc32_end(state->f);
@@ -953,7 +953,7 @@ int index_fd(struct index_state *istate, struct object_id *oid,
 				 type, path, flags);
 	} else {
 		struct odb_write_stream stream;
-		odb_write_stream_from_fd(&stream, fd, xsize_t(st->st_size));
+		odb_write_stream_from_fd(&stream, fd, xsize_t(st->st_size), OBJ_BLOB);
 
 		if (flags & INDEX_WRITE_OBJECT) {
 			struct object_database *odb = the_repository->objects;
@@ -968,8 +968,7 @@ int index_fd(struct index_state *istate, struct object_id *oid,
 			if (!inflight)
 				odb_transaction_commit(transaction);
 		} else {
-			ret = hash_blob_stream(&stream,
-					       the_repository->hash_algo, oid);
+			ret = hash_stream(&stream, the_repository->hash_algo, oid);
 		}
 
 		odb_write_stream_release(&stream);

odb/source-inmemory.c

@@ -290,10 +290,11 @@ static int odb_source_inmemory_write_object_stream(struct odb_source *source,
 		goto out;
 	}
 
-	hash_object_file(source->odb->repo->hash_algo, data, total_read, OBJ_BLOB, oid);
+	hash_object_file(source->odb->repo->hash_algo, data, total_read,
+			 stream->type, oid);
 
 	ret = odb_source_inmemory_write_object(source, data, stream->size,
-					       OBJ_BLOB, oid, NULL, NULL, 0);
+					       stream->type, oid, NULL, NULL, 0);
 	if (ret < 0)
 		goto out;
 

odb/source-loose.c

@@ -868,7 +868,7 @@ static int odb_source_loose_write_object_stream(struct odb_source *source,
 
 	/* Since oid is not determined, save tmp file to odb path. */
 	strbuf_addf(&filename, "%s/", loose->base.path);
-	hdrlen = format_object_header(hdr, sizeof(hdr), OBJ_BLOB, in_stream->size);
+	hdrlen = format_object_header(hdr, sizeof(hdr), in_stream->type, in_stream->size);
 
 	/*
 	 * Common steps for write_loose_object and stream_loose_object to

odb/streaming.c

@@ -324,7 +324,7 @@ static ssize_t read_object_fd(struct odb_write_stream *stream,
 }
 
 void odb_write_stream_from_fd(struct odb_write_stream *stream, int fd,
-			      size_t size)
+			      size_t size, enum object_type type)
 {
 	struct read_object_fd_data *data;
 
@@ -335,4 +335,5 @@ void odb_write_stream_from_fd(struct odb_write_stream *stream, int fd,
 	stream->data = data;
 	stream->read = read_object_fd;
 	stream->size = size;
+	stream->type = type;
 }

odb/streaming.h

@@ -56,6 +56,7 @@ struct odb_write_stream {
 	ssize_t (*read)(struct odb_write_stream *, unsigned char *, size_t);
 	void *data;
 	size_t size;
+	enum object_type type;
 };
 
 /*
@@ -92,6 +93,6 @@ int odb_stream_blob_to_fd(struct object_database *odb,
  * Sets up an ODB write stream that reads from an fd.
  */
 void odb_write_stream_from_fd(struct odb_write_stream *stream, int fd,
-			      size_t size);
+			      size_t size, enum object_type type);
 
 #endif /* STREAMING_H */

odb/transaction.h

@@ -24,7 +24,7 @@ struct odb_transaction {
 
 	/*
 	 * This callback is expected to write the given object stream into
-	 * the ODB transaction. Note that for now, only blobs support streaming.
+	 * the ODB transaction.
 	 *
 	 * The resulting object ID shall be written into the out pointer. The
 	 * callback is expected to return 0 on success, a negative error code

t/unit-tests/u-odb-inmemory.c

@@ -297,8 +297,11 @@ void test_odb_inmemory__write_object_stream(void)
 	struct odb_source_inmemory *source = odb_source_inmemory_new(odb);
 	const char data[] = "foobar";
 	struct membuf_write_stream stream = {
-		.base.read = membuf_write_stream_read,
-		.base.size = strlen(data),
+		.base = {
+			.read = membuf_write_stream_read,
+			.size = strlen(data),
+			.type = OBJ_BLOB,
+		},
 		.buf = data,
 	};
 	struct object_id written_oid;