Files
git/odb.h
T
Junio C Hamano f2d8f4a906 Merge branch 'ps/odb-object-storage-extension' into jch
The object storage format has been made configurable via the
'extensions.objectStorage' configuration variable, allowing users
to specify the storage backend and an optional payload, mirroring
the design of 'extensions.refStorage'.

* ps/odb-object-storage-extension:
  builtin/submodule: propagate "--object-storage-format="
  builtin/clone: introduce "--object-storage-format="
  builtin/init: introduce handling of object storage format
  builtin: expose the object storage format
  setup: allow overriding object storage format via environment
  setup: introduce object storage extension
  loose: drop unused `repo_write_loose_object_map()`
2026-11-05 20:44:35 -08:00

809 lines
25 KiB
C

#ifndef ODB_H
#define ODB_H
#include "hashmap.h"
#include "object.h"
#include "oid-array.h"
#include "oidset.h"
#include "oidmap.h"
#include "repository.h"
#include "string-list.h"
#include "thread-utils.h"
struct cached_object_entry;
struct list_objects_filter_options;
struct odb_source_inmemory;
struct odb_files_dir;
struct packed_git;
struct repository;
struct strbuf;
struct strvec;
/*
* Compute the exact path an alternate is at and returns it. In case of
* error NULL is returned and the human readable error is added to `err`
* `path` may be relative and should point to $GIT_DIR.
* `err` must not be null.
*/
char *compute_alternate_path(const char *path, struct strbuf *err);
/*
* Parse the given URI in the format "<schema>[://<payload>]" and return the
* object storage format identified by the schema. If `payload` is non-NULL it
* will be set to a newly allocated copy of the payload, or NULL if the URI has
* no or an empty payload. Returns `OBJECT_STORAGE_FORMAT_UNKNOWN` for unknown
* schemas.
*/
enum object_storage_format object_storage_format_by_uri(const char *uri, char **payload);
enum object_storage_format object_storage_format_by_name(const char *name);
const char *object_storage_format_to_name(enum object_storage_format format);
/*
* The object database encapsulates access to objects in a repository. It
* manages the object source as well as auxiliary data structures required to
* manage objects.
*/
struct object_database {
/* Repository that owns this database. */
struct repository *repo;
/* The source backing this object database. */
struct odb_source *source;
/*
* State of current object database transaction. Only one
* transaction may be pending at a time. Is NULL when no transaction is
* configured.
*/
struct odb_transaction *transaction;
/*
* Objects that should be substituted by other objects
* (see git-replace(1)).
*/
struct oidmap replace_map;
unsigned replace_map_initialized : 1;
pthread_mutex_t replace_mutex; /* protect object replace functions */
struct commit_graph *commit_graph;
unsigned commit_graph_attempted : 1; /* if loading has been attempted */
/*
* This is meant to hold a *small* number of objects that you would
* want odb_read_object() to be able to return, but yet you do not want
* to write them into the object store (e.g. a browse-only
* application).
*/
struct odb_source *inmemory_objects;
/*
* A fast, rough count of the number of objects in the repository.
* These two fields are not meant for direct access. Use
* odb_count_objects() instead.
*/
unsigned long object_count;
unsigned object_count_flags;
unsigned object_count_valid : 1;
};
enum odb_new_flags {
/*
* Honor environment variables when constructing the object database
* sources. This makes us respect the following environment variables:
*
* - GIT_OBJECT_DIRECTORY to override the primary object directory.
*
* - GIT_ALTERNATE_OBJECT_DIRECTORIES to override alternates.
*
* Environment variables may be backend-specific.
*/
ODB_NEW_HONOR_ENV = (1 << 0),
};
/*
* Create a new object database for the given repository.
*
* If the primary source parameter is set it will override the usual primary
* object directory derived from the repository's common directory. The
* alternate sources are expected to be a PATH_SEP-separated list of secondary
* sources. Note that these alternate sources will be added in addition to, not
* instead of, the alternates identified by the primary source.
*
* Returns the newly created object database.
*/
struct object_database *odb_new(struct repository *repo,
enum odb_new_flags flags);
/* Free the object database and release all resources. */
void odb_free(struct object_database *o);
enum odb_optimize_strategy {
ODB_OPTIMIZE_INCREMENTAL,
ODB_OPTIMIZE_GEOMETRIC,
};
enum odb_optimize_flags {
/* Enable verbose logging and progress reporting. */
ODB_OPTIMIZE_VERBOSE = (1 << 0),
/* Perform auto-maintenance, only optimizing objects as required. */
ODB_OPTIMIZE_AUTO = (1 << 1),
/* Recompute existing deltas. */
ODB_OPTIMIZE_NO_REUSE_DELTAS = (1 << 2),
};
struct odb_optimize_options {
enum odb_optimize_strategy strategy;
enum odb_optimize_flags flags;
const char *prune_expire;
const char *expire_to;
int depth;
int window;
/* Backend-specific options. */
int keep_largest_pack;
int cruft_packs;
unsigned long max_cruft_size;
};
/*
* Optimize the object database. Returns 0 on success, a negative error code
* otherwise.
*/
int odb_optimize(struct object_database *odb,
const struct odb_optimize_options *opts);
/*
* Check whether optimization of the object database is required given the
* provided options. Returns true if optimization should be performed, false
* otherwise.
*/
bool odb_optimize_required(struct object_database *odb,
const struct odb_optimize_options *opts);
/*
* Close the object database and all of its sources so that any held resources
* will be released. The database can still be used after closing it, in which
* case these resources may be reallocated.
*/
void odb_close(struct object_database *o);
enum odb_prepare_flags {
/*
* Flush caches, reload alternates and then re-prepare each object
* source so that new objects may become accessible.
*/
ODB_PREPARE_FLUSH_CACHES = (1 << 0),
};
/*
* Prepare the object database for use. Calling this function is generally not
* needed, but can be useful in case the caller wants to pre-open individual
* sources.
*/
void odb_prepare(struct object_database *o, enum odb_prepare_flags flags);
/* Equivalent to `odb_prepare(o, ODB_PREPARE_FLUSH_CACHES)`. */
void odb_reprepare(struct object_database *o);
enum odb_fsck_flags {
/*
* If set, perform a full consistency check for the full object
* database, including all of its sources and the contents of their
* optimized formats. Otherwise, only check the local source, and
* restrict checks of its optimized formats to cheap structural
* verification of their metadata.
*/
ODB_FSCK_FULL = (1 << 0),
/* Display a progress meter, if sensible. */
ODB_FSCK_PROGRESS = (1 << 1),
/* Be extra verbose when checking the database. */
ODB_FSCK_VERBOSE = (1 << 2),
};
/* Options that shall be passed to `odb_fsck()`. */
struct odb_fsck_options {
enum odb_fsck_flags flags;
int (*object_cb)(const struct object_id *oid, enum object_type type,
unsigned long size, void *buffer, int *eaten, void *cb_data);
void *object_payload;
};
/*
* Run backend-specific integrity checks on all object sources. Each source
* performs the checks appropriate to its type. Returns 0 on success, a
* negative error code otherwise.
*/
int odb_fsck(struct object_database *odb, struct odb_fsck_options *opts);
/*
* Iterate through all alternates of the database and execute the provided
* callback function for each of them. Stop iterating once the callback
* function returns a non-zero value, in which case the value is bubbled up
* from the callback.
*/
typedef int odb_for_each_alternate_fn(struct odb_files_dir *, void *);
int odb_for_each_alternate(struct object_database *odb,
odb_for_each_alternate_fn cb, void *payload);
/*
* Iterate through all alternates of the database and yield their respective
* references.
*/
typedef void odb_for_each_alternate_ref_fn(const struct object_id *oid, void *);
void odb_for_each_alternate_ref(struct object_database *odb,
odb_for_each_alternate_ref_fn cb, void *payload);
/*
* Create a temporary file rooted in the primary alternate's directory, or die
* on failure. The filename is taken from "pattern", which should have the
* usual "XXXXXX" trailer, and the resulting filename is written into the
* "template" buffer. Returns the open descriptor.
*/
int odb_mkstemp(struct object_database *odb,
struct strbuf *temp_filename, const char *pattern);
/*
* Check whether the object database has any alternates. The primary object
* source does not count as alternate.
*/
int odb_has_alternates(struct object_database *odb);
/*
* Read an object from the database. Returns the object data and assigns object
* type and size to the `type` and `size` pointers, if these pointers are
* non-NULL. Returns a `NULL` pointer in case the object does not exist.
*
* This function dies on corrupt objects; the callers who want to deal with
* them should arrange to call odb_read_object_info_extended() and give error
* messages themselves.
*/
void *odb_read_object(struct object_database *odb,
const struct object_id *oid,
enum object_type *type,
size_t *size);
void *odb_read_object_peeled(struct object_database *odb,
const struct object_id *oid,
enum object_type required_type,
size_t *size,
struct object_id *oid_ret);
/*
* Add an object file to the in-memory object store, without writing it
* to disk.
*
* Callers are responsible for calling write_object_file to record the
* object in persistent storage before writing any other new objects
* that reference it.
*/
int odb_pretend_object(struct object_database *odb,
void *buf, size_t len, enum object_type type,
struct object_id *oid);
/*
* Object database source information that can be used to uniquely identify an
* object and learn more about how exactly it is stored.
*/
struct odb_source_info {
/* The source that this object has been looked up from. */
struct odb_source *source;
/*
* Backend-specific information about the specific object. This can be
* used for example to uniquely identify a given object in case it
* exists multiple times.
*/
union {
/*
* struct {
* ... Nothing to expose in this case
* } cached;
* struct {
* ... Nothing to expose in this case
* } loose;
*/
struct {
struct packed_git *pack;
off_t offset;
enum packed_object_type {
PACKED_OBJECT_TYPE_UNKNOWN,
PACKED_OBJECT_TYPE_FULL,
PACKED_OBJECT_TYPE_OFS_DELTA,
PACKED_OBJECT_TYPE_REF_DELTA,
} type;
} packed;
} u;
};
/*
* The object info contains the query and response that is to be used for
* functions that end up reading object information. Callers are expected to
* populate pointers whose information they want to request.
*/
struct object_info {
/* The object type. */
enum object_type *typep;
/* The inflated object size in bytes. */
size_t *sizep;
/* The object size as stored on disk. */
off_t *disk_sizep;
/*
* The base the object is deltified against, in case it is stored as a
* delta.
*/
struct object_id *delta_base_oid;
/* The object contents. Ownership of memory goes over to the caller. */
void **contentp;
/*
* The time the given looked-up object has been last modified.
*
* Note: the mtime may be ambiguous in case the object exists multiple
* times in the object database. It is thus _not_ recommended to use
* this field outside of contexts where you would read every instance
* of the object, like for example with `odb_for_each_object()`. As it
* is impossible to say at the ODB level what the intent of the caller
* is (e.g. whether to find the oldest or newest object), it is the
* responsibility of the caller to disambiguate the mtimes.
*/
time_t *mtimep;
/*
* Backend-specific information that tells the caller where exactly an
* object was looked up from. This information should help disambiguate
* object lookups in case the same object exists in multiple sources,
* or multiple times in the same source.
*/
struct odb_source_info *source_infop;
/*
* object-info protocol specific. Set by the protocol when the remote
* does not recognize the requested object.
*/
unsigned int unrecognized:1;
};
/*
* Initializer for a "struct object_info" that wants no items. You may
* also memset() the memory to all-zeroes.
*/
#define OBJECT_INFO_INIT { 0 }
/* Flags that can be passed to `odb_read_object_info_extended()`. */
enum object_info_flags {
/* Invoke lookup_replace_object() on the given hash. */
OBJECT_INFO_LOOKUP_REPLACE = (1 << 0),
/* Do not reprepare object sources when the first lookup has failed. */
OBJECT_INFO_QUICK = (1 << 1),
/*
* Do not attempt to fetch the object if missing (even if fetch_is_missing is
* nonzero).
*/
OBJECT_INFO_SKIP_FETCH_OBJECT = (1 << 2),
/* Die if object corruption (not just an object being missing) was detected. */
OBJECT_INFO_DIE_IF_CORRUPT = (1 << 3),
/*
* This is meant for bulk prefetching of missing blobs in a partial
* clone. Implies OBJECT_INFO_SKIP_FETCH_OBJECT and OBJECT_INFO_QUICK.
*/
OBJECT_INFO_FOR_PREFETCH = (OBJECT_INFO_SKIP_FETCH_OBJECT | OBJECT_INFO_QUICK),
};
enum odb_read_status {
/* The read was successful. */
ODB_READ_OK = 0,
/* The read resulted in a generic error. */
ODB_READ_ERROR = -1,
/* The object could not be found. */
ODB_READ_NOT_FOUND = -2,
};
/*
* Read object info from the object database and populate the `object_info`
* structure. Returns 0 on success, a negative error code otherwise.
*/
enum odb_read_status odb_read_object_info_extended(struct object_database *odb,
const struct object_id *oid,
struct object_info *oi,
enum object_info_flags flags);
/*
* Read a subset of object info for the given object ID. Returns an `enum
* object_type` on success, a negative error code otherwise. If successful and
* `sizep` is non-NULL, then the size of the object will be written to the
* pointer.
*/
int odb_read_object_info(struct object_database *odb,
const struct object_id *oid,
size_t *sizep);
enum odb_has_object_flags {
/* Retry packed storage after checking packed and loose storage */
ODB_HAS_OBJECT_RECHECK_PACKED = (1 << 0),
/* Allow fetching the object in case the repository has a promisor remote. */
ODB_HAS_OBJECT_FETCH_PROMISOR = (1 << 1),
};
/*
* Returns 1 if the object exists. This function will not lazily fetch objects
* in a partial clone by default.
*/
int odb_has_object(struct object_database *odb,
const struct object_id *oid,
enum odb_has_object_flags flags);
int odb_freshen_object(struct object_database *odb,
const struct object_id *oid);
void odb_assert_oid_type(struct object_database *odb,
const struct object_id *oid, enum object_type expect);
/*
* Enabling the object read lock allows multiple threads to safely call the
* following functions in parallel: odb_read_object(),
* odb_read_object_peeled(), odb_read_object_info() and odb().
*
* obj_read_lock() and obj_read_unlock() may also be used to protect other
* section which cannot execute in parallel with object reading. Since the used
* lock is a recursive mutex, these sections can even contain calls to object
* reading functions. However, beware that in these cases zlib inflation won't
* be performed in parallel, losing performance.
*
* TODO: odb_read_object_info_extended()'s call stack has a recursive behavior. If
* any of its callees end up calling it, this recursive call won't benefit from
* parallel inflation.
*/
void enable_obj_read_lock(void);
void disable_obj_read_lock(void);
extern int obj_read_use_lock;
extern pthread_mutex_t obj_read_mutex;
static inline void obj_read_lock(void)
{
if(obj_read_use_lock)
pthread_mutex_lock(&obj_read_mutex);
}
static inline void obj_read_unlock(void)
{
if(obj_read_use_lock)
pthread_mutex_unlock(&obj_read_mutex);
}
/* Flags for for_each_*_object(). */
enum odb_for_each_object_flags {
/* Iterate only over local objects, not alternates. */
ODB_FOR_EACH_OBJECT_LOCAL_ONLY = (1<<0),
/* Only iterate over packs obtained from the promisor remote. */
ODB_FOR_EACH_OBJECT_PROMISOR_ONLY = (1<<1),
/*
* Visit objects within a pack in packfile order rather than .idx order
*/
ODB_FOR_EACH_OBJECT_PACK_ORDER = (1<<2),
/* Only iterate over packs that are not marked as kept in-core. */
ODB_FOR_EACH_OBJECT_SKIP_IN_CORE_KEPT_PACKS = (1<<3),
/* Only iterate over packs that do not have .keep files. */
ODB_FOR_EACH_OBJECT_SKIP_ON_DISK_KEPT_PACKS = (1<<4),
};
/*
* A callback function that can be used to iterate through objects. If given,
* the optional `oi` parameter will be populated the same as if you would call
* `odb_read_object_info()`.
*
* Returning a non-zero error code will cause iteration to abort. The error
* code will be propagated.
*/
typedef int (*odb_for_each_object_cb)(const struct object_id *oid,
struct object_info *oi,
void *cb_data);
/*
* Options that can be passed to `odb_for_each_object()` and its
* backend-specific implementations.
*/
struct odb_for_each_object_options {
/* A bitfield of `odb_for_each_object_flags`. */
enum odb_for_each_object_flags flags;
/*
* If set, only iterate through objects whose first `prefix_hex_len`
* hex characters matches the given prefix.
*/
const struct object_id *prefix;
size_t prefix_hex_len;
/*
* Optional object filter that allows backends to skip yielding
* objects that are excluded by the filter as an optimization. The
* filter is a best-effort hint: backends may use it to skip
* excluded objects (e.g. by consulting a reachability bitmap), but
* are also free to ignore it entirely and yield every object. As a
* consequence, callers must re-apply the filter on yielded objects
* if they require strict filtering semantics.
*/
const struct list_objects_filter_options *filter;
};
/*
* Iterate through all objects contained in the object database. Note that
* objects may be iterated over multiple times in case they are either stored
* in different backends or in case they are stored in multiple sources.
* If an object info request is given, then the object info will be read and
* passed to the callback as if `odb_read_object_info()` was called for the
* object.
*
* Returning a non-zero error code from the callback function will cause
* iteration to abort. The error code will be propagated.
*
* Returns 0 on success, a negative error code in case a failure occurred, or
* an arbitrary non-zero error code returned by the callback itself.
*/
int odb_for_each_object_ext(struct object_database *odb,
const struct object_info *request,
odb_for_each_object_cb cb,
void *cb_data,
const struct odb_for_each_object_options *opts);
/* Same as `odb_for_each_object_ext()` with `opts.flags` set to the given flags. */
int odb_for_each_object(struct object_database *odb,
const struct object_info *request,
odb_for_each_object_cb cb,
void *cb_data,
enum odb_for_each_object_flags flags);
enum odb_count_objects_flags {
/*
* Instead of providing an accurate count, allow the number of objects
* to be approximated. Details of how this approximation works are
* subject to the specific source's implementation.
*/
ODB_COUNT_OBJECTS_APPROXIMATE = (1 << 0),
};
/*
* Count the number of objects in the given object database. This object count
* may double-count objects that are stored in multiple backends, or which are
* stored multiple times in a single backend.
*
* Returns 0 on success, a negative error code otherwise. The number of objects
* will be assigned to the `out` pointer on success.
*/
int odb_count_objects(struct object_database *odb,
enum odb_count_objects_flags flags,
unsigned long *out);
/*
* Given an object ID, find the minimum required length required to make the
* object ID unique across the whole object database.
*
* The `min_len` determines the minimum abbreviated length that'll be returned
* by this function. If `min_len < 0`, then the function will set a sensible
* default minimum abbreviation length.
*
* Returns 0 on success, a negative error code otherwise. The computed length
* will be assigned to `*out`.
*/
int odb_find_abbrev_len(struct object_database *odb,
const struct object_id *oid,
int min_len,
unsigned *out);
enum odb_write_object_flags {
/*
* By default, `odb_write_object()` does not actually write anything
* into the object store, but only computes the object ID. This flag
* changes that so that the object will be written as a loose object
* and persisted.
*/
ODB_WRITE_OBJECT_PERSIST = (1 << 0),
/*
* Do not print an error in case something goes wrong.
*/
ODB_WRITE_OBJECT_SILENT = (1 << 1),
};
/*
* Write an object into the object database. The object is being written into
* the local alternate of the repository. If provided, the object ID of the
* final object is written into `oid`.
*
* If the caller provides a `compat_oid`, then this compatibility object hash
* will be stored instead of computing the compatibility hash ad-hoc.
*
* Returns 0 on success, a negative error code otherwise.
*/
int odb_write_object_ext(struct object_database *odb,
const void *buf, unsigned long len,
enum object_type type,
struct object_id *oid,
const struct object_id *compat_oid,
enum odb_write_object_flags flags);
static inline int odb_write_object(struct object_database *odb,
const void *buf, unsigned long len,
enum object_type type,
struct object_id *oid)
{
return odb_write_object_ext(odb, buf, len, type, oid, NULL, 0);
}
struct odb_stream;
int odb_write_object_stream(struct object_database *odb,
struct odb_stream *stream,
struct object_id *oid);
/*
* Options for generating a packfile via `odb_generate_pack()`.
*/
struct odb_generate_pack_options {
/* Tips of the object graph that shall be packed. */
struct oid_array wants;
/*
* Boundary of the object graph. Objects reachable from any of these
* tips are expected to already be available to whoever consumes the
* pack and shall thus not be packed.
*/
struct oid_array haves;
/*
* The shallow boundary that shall be used when computing object
* reachability. When set, any shallow information of the repository
* itself shall be ignored in favor of these objects.
*/
struct oid_array shallows;
/*
* Pre-expanded object filter specification that limits the set of
* objects that shall be packed. May be `NULL` in case no filter shall
* be applied.
*/
const char *filter_spec;
/*
* Protocols that may be used to offload objects via packfile URIs.
* May be `NULL` in case packfile URIs shall not be used.
*/
const struct string_list *uri_protocols;
/*
* Hook command that shall be executed instead of the internal
* machinery to generate the pack. It is up to the specific backend
* whether or not this hook is supported. May be `NULL` in case no
* hook shall be executed.
*/
const char *pack_objects_hook;
/*
* File descriptor that the generated pack shall be written to. If set
* to `-1`, a pipe will be created and exposed via the pack generator's
* `out` field. If set to `0`, the pack will be written to the standard
* output stream. Otherwise, the provided descriptor will be written to
* and is consumed by the generator.
*/
int pack_fd;
/*
* File descriptor that progress output shall be written to. The same
* semantics as for `pack_fd` apply, except that `0` will cause the
* generator to write to stderr instead of stdout.
*/
int progress_fd;
/* Whether to print progress or not. */
enum {
/* Don't print progress output. */
ODB_GENERATE_PACK_PROGRESS_NONE,
/*
* Print progress while computing the packfile, but stop
* printing progress once starting to write it.
*/
ODB_GENERATE_PACK_PROGRESS_STANDARD,
/*
* Similar to STANDARD, but also print progress when writing
* the packfile.
*/
ODB_GENERATE_PACK_PROGRESS_VERBOSE,
} progress;
/* Allow the pack to contain deltas against unpacked objects. */
unsigned thin:1;
/* Use offset deltas instead of reference deltas. */
unsigned ofs_delta:1;
/* Include unasked-for annotated tags of packed objects. */
unsigned include_tag:1;
/* The generated pack is destined for a shallow consumer. */
unsigned shallow:1;
/* Allow objects that may be missing due to a promisor remote. */
unsigned missing_allow_promisor:1;
/* Do not use bitmap indices when computing reachability. */
unsigned disable_bitmaps:1;
};
#define ODB_GENERATE_PACK_OPTIONS_INIT { \
.wants = OID_ARRAY_INIT, \
.haves = OID_ARRAY_INIT, \
.shallows = OID_ARRAY_INIT, \
.pack_fd = -1, \
}
/* Release resources associated with the options. */
void odb_generate_pack_options_release(struct odb_generate_pack_options *opts);
/*
* A handle for an ongoing packfile generation as started via
* `odb_generate_pack()`.
*/
struct odb_pack_generator {
/*
* File descriptor from which the generated pack can be read. Only set
* when the pack generation was started with `pack_fd == -1`. The
* caller is responsible for closing the descriptor.
*/
int out;
/*
* File descriptor from which progress output can be read. Only set
* when the pack generation was started with `progress_fd == -1`. The
* caller is responsible for closing the descriptor.
*/
int err;
/*
* Callback function to finish this generator. This callback is
* expected to wait for the packfile generation to complete and to then
* free the generator itself.
*/
int (*finish)(struct odb_pack_generator *);
};
/*
* Start generating a packfile from the object database with the given
* options. The pack is generated asynchronously; the caller is expected to
* consume the file descriptors exposed via the pack generator and to then
* wait for completion via `odb_pack_generator_finish()`.
*
* Returns 0 on success and populates the `out` pointer with the pack
* generator. Returns a negative error code otherwise.
*/
int odb_generate_pack(struct object_database *odb,
struct odb_pack_generator **out,
const struct odb_generate_pack_options *opts);
/*
* Wait for the packfile generation to complete and free the pack generator.
* Returns 0 on success, a negative error code otherwise.
*/
int odb_pack_generator_finish(struct odb_pack_generator *generator);
#endif /* ODB_H */