From c3b60bb31105955bfa1011e2bae321d83ef6bfe5 Mon Sep 17 00:00:00 2001 From: Patrick Steinhardt Date: Tue, 27 Oct 2026 07:59:26 +0100 Subject: [PATCH] setup: introduce object storage extension Introduce a new "extensions.objectStorage" extension that allows the user to configure the primary object database source of a repository. The value is a URI in the format "[://]", mirroring the format used by "extensions.refStorage": - The schema identifies the backend that shall be used. For now the only supported schema is "files", which is the existing object database backed by loose objects and packfiles. - The payload can be passed to configure the backend, and its interpretation is backend-specific. For the "files" backend, it allows you to configure a different location for the ".git/objects" directory. Making the location of the "files" backend configurable is not the most thrilling feature of this new extension. But it gives us a couple of advantages: - It allows us to start exercising the payload logic in our tests so that we know it's working properly for backends that will need it. - It allows us to easily detect tests that are broken with the new extension because they for example assume that an object is a file in ".git/objects/". Testing is not part of this series yet, but will be done in subsequent patch series. - It aligns with how we handle the ref storage extension, where we can also change its location to an arbitrary path. - It allows for some very esoteric and edge-case-y gimmicks. For example, you can initialize a single repository "skeleton" and then multiplex therein by setting both GIT_REF_STORAGE_FORMAT and GIT_OBJECT_STORAGE_FORMAT to point to different databases. These edge cases won't matter to many people out there, but we will use this for example as a building block for migrations. Eventually, this mechanism will be used to support alternative backends. Signed-off-by: Patrick Steinhardt Signed-off-by: Junio C Hamano --- Documentation/config/extensions.adoc | 23 ++++++++++++++ Documentation/object-storage-format.adoc | 6 ++++ odb.c | 39 +++++++++++++++++++++++- odb.h | 12 ++++++++ odb/source-files.c | 7 +++++ odb/source-files.h | 1 + odb/source.c | 14 ++++++++- odb/source.h | 5 ++- repository.c | 10 ++++++ repository.h | 18 +++++++++++ setup.c | 18 +++++++++++ setup.h | 3 ++ t/t0001-init.sh | 30 ++++++++++++++++++ t/unit-tests/u-odb-inmemory.c | 1 + 14 files changed, 184 insertions(+), 3 deletions(-) create mode 100644 Documentation/object-storage-format.adoc diff --git a/Documentation/config/extensions.adoc b/Documentation/config/extensions.adoc index be6678bb5b..4f3a66a454 100644 --- a/Documentation/config/extensions.adoc +++ b/Documentation/config/extensions.adoc @@ -56,6 +56,29 @@ preciousObjects::: For historical reasons, this extension is respected regardless of the `core.repositoryFormatVersion` setting. +objectStorage::: + Specify the object storage format and a corresponding payload. The + value can be either a format name or a URI: ++ +-- +* A format name alone (e.g., `files`). + +* A URI format `://` explicitly specifies both the + format and payload (e.g., `files:///foo/bar`). + +Supported formats are: + +include::../object-storage-format.adoc[] + +-- ++ +Note that this setting should only be set by linkgit:git-init[1] or +linkgit:git-clone[1]. Trying to change it after initialization will make +preexisting objects inaccessible. ++ +NOTE: This extension is still considered EXPERIMENTAL and may be subject to +change. + refStorage::: Specify the ref storage format and a corresponding payload. The value can be either a format name or a URI: diff --git a/Documentation/object-storage-format.adoc b/Documentation/object-storage-format.adoc new file mode 100644 index 0000000000..4c4aef6dda --- /dev/null +++ b/Documentation/object-storage-format.adoc @@ -0,0 +1,6 @@ +`files[://]`;; for loose objects and packfiles. This is the default. ++ +The optional payload can be passed to specify an alternative filesystem path +where the objects will be stored. Relative paths are resolved relative to the +`$GIT_COMMON_DIR`. Defaults to `$GIT_COMMON_DIR/objects` when no payload is +provided. The payload is ignored when `GIT_OBJECT_DIRECTORY` is set. diff --git a/odb.c b/odb.c index 98c5c862f5..0649622bc6 100644 --- a/odb.c +++ b/odb.c @@ -27,6 +27,42 @@ #include "trace2.h" #include "write-or-die.h" +static const char * const object_storage_name_by_format[] = { + [OBJECT_STORAGE_FORMAT_UNKNOWN] = "unknown", + [OBJECT_STORAGE_FORMAT_FILES] = "files", +}; + +enum object_storage_format object_storage_format_by_name(const char *name) +{ + for (size_t i = 0; i < ARRAY_SIZE(object_storage_name_by_format); i++) + if (!strcmp(object_storage_name_by_format[i], name)) + return i; + return OBJECT_STORAGE_FORMAT_UNKNOWN; +} + +enum object_storage_format object_storage_format_by_uri(const char *uri, char **payload) +{ + enum object_storage_format format; + const char *schema_end; + char *name; + + schema_end = strstr(uri, "://"); + if (!schema_end) { + name = xstrdup(uri); + if (payload) + *payload = NULL; + } else { + name = xstrndup(uri, schema_end - uri); + if (payload) + *payload = *(schema_end + 3) ? xstrdup(schema_end + 3) : NULL; + } + + format = object_storage_format_by_name(name); + + free(name); + return format; +} + int odb_mkstemp(struct object_database *odb, struct strbuf *temp_filename, const char *pattern) { @@ -757,7 +793,8 @@ struct object_database *odb_new(struct repository *repo, o->repo = repo; pthread_mutex_init(&o->replace_mutex, NULL); - o->source = odb_source_new(o, flags); + o->source = odb_source_new(o, repo->object_storage_format, + repo->object_storage_payload, flags); o->inmemory_objects = &odb_source_inmemory_new(o)->base; return o; diff --git a/odb.h b/odb.h index 0ccc47ee73..cb0505334a 100644 --- a/odb.h +++ b/odb.h @@ -6,6 +6,7 @@ #include "oid-array.h" #include "oidset.h" #include "oidmap.h" +#include "repository.h" #include "string-list.h" #include "thread-utils.h" @@ -26,6 +27,17 @@ struct strvec; */ char *compute_alternate_path(const char *path, struct strbuf *err); +/* + * Parse the given URI in the format "[://]" and return the + * object storage format identified by the schema. If `payload` is non-NULL it + * will be set to a newly allocated copy of the payload, or NULL if the URI has + * no or an empty payload. Returns `OBJECT_STORAGE_FORMAT_UNKNOWN` for unknown + * schemas. + */ +enum object_storage_format object_storage_format_by_uri(const char *uri, char **payload); + +enum object_storage_format object_storage_format_by_name(const char *name); + /* * The object database encapsulates access to objects in a repository. It * manages the object source as well as auxiliary data structures required to diff --git a/odb/source-files.c b/odb/source-files.c index 455f409db0..4a7cbbfd9b 100644 --- a/odb/source-files.c +++ b/odb/source-files.c @@ -1235,6 +1235,7 @@ struct odb_files_dir *odb_source_files_find_dir(struct object_database *odb, con } struct odb_source_files *odb_source_files_new(struct object_database *odb, + const char *payload, enum odb_new_flags flags) { struct odb_source_files *files; @@ -1245,6 +1246,12 @@ struct odb_source_files *odb_source_files_new(struct object_database *odb, object_dir = xstrdup_or_null(getenv(DB_ENVIRONMENT)); alternates = xstrdup_or_null(getenv(ALTERNATE_DB_ENVIRONMENT)); } + if (!object_dir && payload) { + if (is_absolute_path(payload)) + object_dir = xstrdup(payload); + else + object_dir = xstrfmt("%s/%s", odb->repo->commondir, payload); + } if (!object_dir) object_dir = xstrfmt("%s/objects", odb->repo->commondir); diff --git a/odb/source-files.h b/odb/source-files.h index 837e890304..71e8db84ff 100644 --- a/odb/source-files.h +++ b/odb/source-files.h @@ -70,6 +70,7 @@ struct odb_source_files { /* Allocate and initialize a new object source. */ struct odb_source_files *odb_source_files_new(struct object_database *odb, + const char *payload, enum odb_new_flags flags); /* diff --git a/odb/source.c b/odb/source.c index b25ef14df8..a2a6d97ccd 100644 --- a/odb/source.c +++ b/odb/source.c @@ -24,9 +24,21 @@ const char *odb_source_type_to_name(enum odb_source_type type) } struct odb_source *odb_source_new(struct object_database *odb, + enum object_storage_format format, + const char *payload, enum odb_new_flags flags) { - return &odb_source_files_new(odb, flags)->base; + struct odb_source *source; + + switch (format) { + case OBJECT_STORAGE_FORMAT_FILES: + source = &odb_source_files_new(odb, payload, flags)->base; + break; + default: + BUG("unknown object storage format"); + } + + return source; } void odb_source_init(struct odb_source *source, diff --git a/odb/source.h b/odb/source.h index 1867347dfc..1ab3208888 100644 --- a/odb/source.h +++ b/odb/source.h @@ -304,9 +304,12 @@ struct odb_source { /* * Allocate and initialize a new source for the given object database. The path - * of the source is derived from repository paths. + * of the source is derived from repository paths or from the object storage + * format. */ struct odb_source *odb_source_new(struct object_database *odb, + enum object_storage_format format, + const char *payload, enum odb_new_flags flags); /* diff --git a/repository.c b/repository.c index 970d181828..77ecf6b604 100644 --- a/repository.c +++ b/repository.c @@ -217,6 +217,15 @@ void repo_set_ref_storage_format(struct repository *repo, repo->ref_storage_payload = xstrdup_or_null(payload); } +void repo_set_object_storage_format(struct repository *repo, + enum object_storage_format format, + const char *payload) +{ + repo->object_storage_format = format; + free(repo->object_storage_payload); + repo->object_storage_payload = xstrdup_or_null(payload); +} + /* * Attempt to resolve and set the provided 'gitdir' for repository 'repo'. * Return 0 upon success and a non-zero value upon failure. @@ -382,6 +391,7 @@ void repo_clear(struct repository *repo) free(repo->worktree); free(repo->submodule_prefix); free(repo->ref_storage_payload); + free(repo->object_storage_payload); odb_free(repo->objects); diff --git a/repository.h b/repository.h index 2a348012e8..8c4437762e 100644 --- a/repository.h +++ b/repository.h @@ -22,6 +22,11 @@ enum ref_storage_format { REF_STORAGE_FORMAT_REFTABLE, }; +enum object_storage_format { + OBJECT_STORAGE_FORMAT_UNKNOWN, + OBJECT_STORAGE_FORMAT_FILES, +}; + #ifdef WITH_BREAKING_CHANGES /* Git 3.0 */ # define REF_STORAGE_FORMAT_DEFAULT REF_STORAGE_FORMAT_REFTABLE #else @@ -178,6 +183,16 @@ struct repository { */ char *ref_storage_payload; + /* Repository's object storage format, as serialized on disk. */ + enum object_storage_format object_storage_format; + + /* + * Object storage information as needed for the backend. This contains + * only the payload from the object storage format URI without the + * schema. + */ + char *object_storage_payload; + /* A unique-id for tracing purposes. */ int trace2_repo_id; @@ -255,6 +270,9 @@ void repo_set_compat_hash_algo(struct repository *repo, uint32_t compat_algo); void repo_set_ref_storage_format(struct repository *repo, enum ref_storage_format format, const char *payload); +void repo_set_object_storage_format(struct repository *repo, + enum object_storage_format format, + const char *payload); void initialize_repository(struct repository *repo); RESULT_MUST_BE_USED int repo_init(struct repository *r, const char *gitdir, const char *worktree); diff --git a/setup.c b/setup.c index 3ee19f3e3c..99ead7d90c 100644 --- a/setup.c +++ b/setup.c @@ -686,6 +686,20 @@ static enum extension_result handle_extension(const char *var, return error(_("invalid value for '%s': '%s'"), "extensions.refstorage", value); data->ref_storage_format = format; + return EXTENSION_OK; + } else if (!strcmp(ext, "objectstorage")) { + enum object_storage_format format; + + if (!value) + return config_error_nonbool(var); + + FREE_AND_NULL(data->object_storage_payload); + format = object_storage_format_by_uri(value, &data->object_storage_payload); + if (format == OBJECT_STORAGE_FORMAT_UNKNOWN) + return error(_("invalid value for '%s': '%s'"), + "extensions.objectstorage", value); + data->object_storage_format = format; + return EXTENSION_OK; } else if (!strcmp(ext, "relativeworktrees")) { data->relative_worktrees = git_config_bool(var, value); @@ -864,6 +878,7 @@ void clear_repository_format(struct repository_format *format) free(format->work_tree); free(format->partial_clone); free(format->ref_storage_payload); + free(format->object_storage_payload); init_repository_format(format); } @@ -1786,6 +1801,9 @@ int apply_repository_format(struct repository *repo, repo_set_ref_storage_format(repo, format->ref_storage_format, format->ref_storage_payload); + repo_set_object_storage_format(repo, + format->object_storage_format, + format->object_storage_payload); repo->repository_format_worktree_config = format->worktree_config; repo->repository_format_submodule_path_cfg = diff --git a/setup.h b/setup.h index 7a3efe5e48..a4afa8910e 100644 --- a/setup.h +++ b/setup.h @@ -190,6 +190,8 @@ struct repository_format { int compat_hash_algo; enum ref_storage_format ref_storage_format; char *ref_storage_payload; + enum object_storage_format object_storage_format; + char *object_storage_payload; int sparse_index; char *work_tree; struct string_list unknown_extensions; @@ -207,6 +209,7 @@ struct repository_format { .is_bare = -1, \ .hash_algo = GIT_HASH_DEFAULT, \ .ref_storage_format = REF_STORAGE_FORMAT_FILES, \ + .object_storage_format = OBJECT_STORAGE_FORMAT_FILES, \ .unknown_extensions = STRING_LIST_INIT_DUP, \ .v1_only_extensions = STRING_LIST_INIT_DUP, \ } diff --git a/t/t0001-init.sh b/t/t0001-init.sh index f1d227f271..c7c464d3a2 100755 --- a/t/t0001-init.sh +++ b/t/t0001-init.sh @@ -839,6 +839,36 @@ test_expect_success 'init with --ref-storage-format=garbage' ' test_cmp expect err ' +test_expect_success DEFAULT_REPO_FORMAT 'extensions.objectStorage is not allowed with repo version 0' ' + test_when_finished "rm -rf objstorage" && + git init objstorage && + git -C objstorage config set extensions.objectStorage files && + test_must_fail git -C objstorage rev-parse 2>err && + test_grep "repo version is 0, but v1-only extension found" err +' + +test_expect_success DEFAULT_REPO_FORMAT 'extensions.objectStorage with files backend' ' + test_when_finished "rm -rf objstorage" && + git init objstorage && + git -C objstorage config set core.repositoryformatversion 1 && + git -C objstorage config set extensions.objectStorage files && + test_commit -C objstorage A && + git -C objstorage rev-parse --verify HEAD +' + +test_expect_success DEFAULT_REPO_FORMAT 'extensions.objectStorage with unknown backend' ' + test_when_finished "rm -rf objstorage" && + git init objstorage && + git -C objstorage config set core.repositoryformatversion 1 && + git -C objstorage config set extensions.objectStorage garbage && + test_must_fail git -C objstorage rev-parse 2>err && + test_grep "invalid value for ${SQ}extensions.objectstorage${SQ}: ${SQ}garbage${SQ}" err && + + git config --file objstorage/.git/config extensions.objectStorage garbage://payload && + test_must_fail git -C objstorage rev-parse 2>err && + test_grep "invalid value for ${SQ}extensions.objectstorage${SQ}: ${SQ}garbage://payload${SQ}" err +' + test_expect_success MINGW 'core.hidedotfiles = false' ' git config --global core.hidedotfiles false && rm -rf newdir && diff --git a/t/unit-tests/u-odb-inmemory.c b/t/unit-tests/u-odb-inmemory.c index a8da278c68..c4412949c1 100644 --- a/t/unit-tests/u-odb-inmemory.c +++ b/t/unit-tests/u-odb-inmemory.c @@ -12,6 +12,7 @@ static struct repository repo = { .hash_algo = &hash_algos[GIT_HASH_SHA1], + .object_storage_format = OBJECT_STORAGE_FORMAT_FILES, }; static struct object_database *odb;