Patrick Steinhardt
5a817d648b
When looking up an object fails we perform a second read in all sources with the `OBJECT_INFO_SECOND_READ` flag set. This flag tells sources that the first lookup has failed and that they should reload any on-disk state that may have changed in the meantime so that the object may be surfaced after all. The "packed" source uses this to reprepare its packfiles, whereas the "loose" source short-circuits the second read because there are no caches to flush. This logic lives in the generic object database layer even though it is highly specific to how the "files" backend handles its reads. But now that alternates are an implementation detail of that source, we can easily move that logic into the backend, as well. Move the logic into the "files" source and drop the flag. Note that the "packed" source also used this flag to figure out whether it should try to surface an object via a packfile that should have the object as determined by a multi-pack index, but where the packfile has vanished. The multi-pack index is stale in that case, and that scenario should really only happen when the process is racing with a concurrent maintenance task. This code is adapted to instead use `OBJECT_INFO_QUICK`, which means that the fallback may trigger on the first read already, but not in a case where callers know to handle false negatives already. In the worst case this could cause another scan through all packfiles covered by the multi-pack index. But that scan should be fast enough anyway, and it only triggers when we hit the race. So the effect thereof should be limited. Signed-off-by: Patrick Steinhardt <ps@pks.im> Signed-off-by: Junio C Hamano <gitster@pobox.com>
552 lines
18 KiB
C
552 lines
18 KiB
C
#ifndef ODB_SOURCE_H
|
|
#define ODB_SOURCE_H
|
|
|
|
#include "hashmap.h"
|
|
#include "object.h"
|
|
#include "odb.h"
|
|
#include "odb/transaction.h"
|
|
|
|
enum odb_source_type {
|
|
/*
|
|
* The "unknown" type, which should never be in use. This type mostly
|
|
* exists to catch cases where the type field remains zeroed out.
|
|
*/
|
|
ODB_SOURCE_UNKNOWN,
|
|
|
|
/* The "files" backend that uses loose objects and packfiles. */
|
|
ODB_SOURCE_FILES,
|
|
|
|
/* The "loose" backend that uses loose objects, only. */
|
|
ODB_SOURCE_LOOSE,
|
|
|
|
/* The "packed" backend that uses packfiles. */
|
|
ODB_SOURCE_PACKED,
|
|
|
|
/* The "in-memory" backend that stores objects in memory. */
|
|
ODB_SOURCE_INMEMORY,
|
|
};
|
|
|
|
/*
|
|
* Convert between the enum and its name. Returns the equivalent of "unknown"
|
|
* for unknown types.
|
|
*/
|
|
const char *odb_source_type_to_name(enum odb_source_type type);
|
|
|
|
struct object_id;
|
|
struct odb_stream;
|
|
struct strbuf;
|
|
struct strvec;
|
|
|
|
struct odb_create_on_disk_options {
|
|
/*
|
|
* Alternates that shall be written into the newly created object
|
|
* database. Whether or not this option can be handled is specific to
|
|
* the backend.
|
|
*/
|
|
const struct strvec *alternates;
|
|
};
|
|
|
|
/*
|
|
* The source is the part of the object database that stores the actual
|
|
* objects. It thus encapsulates the logic to read and write the specific
|
|
* on-disk format.
|
|
*/
|
|
struct odb_source {
|
|
/* Object database that owns this object source. */
|
|
struct object_database *odb;
|
|
|
|
/* The type used by this source. */
|
|
enum odb_source_type type;
|
|
|
|
/*
|
|
* Figure out whether this is the local source of the owning
|
|
* repository, which would typically be its ".git/objects" directory.
|
|
* This local object directory is usually where objects would be
|
|
* written to.
|
|
*/
|
|
bool local;
|
|
|
|
/*
|
|
* Path to the source. If this is a relative path, it is relative to
|
|
* the current working directory.
|
|
*/
|
|
char *path;
|
|
|
|
/*
|
|
* This callback is expected to free the underlying object database source and
|
|
* all associated resources. The function will never be called with a NULL pointer.
|
|
*/
|
|
void (*free)(struct odb_source *source);
|
|
|
|
/*
|
|
* This callback is expected to close any open resources, like for
|
|
* example file descriptors or connections. The source is expected to
|
|
* still be usable after it has been closed. Closed resources may need
|
|
* to be reopened in that case.
|
|
*/
|
|
void (*close)(struct odb_source *source);
|
|
|
|
/*
|
|
* This callback is expected to create on-disk data structures that are
|
|
* required for this source to operate.
|
|
*
|
|
* The callback is expected to return 0 on success, a negative error
|
|
* code otherwise.
|
|
*
|
|
* This callback may be NULL in case the source does not need any
|
|
* on-disk setup.
|
|
*/
|
|
int (*create_on_disk)(struct odb_source *source,
|
|
const struct odb_create_on_disk_options *opts);
|
|
|
|
/*
|
|
* This callback is expected to prepare the source so that it becomes
|
|
* ready for use. It optionally clears underlying caches of the object
|
|
* database source.
|
|
*/
|
|
void (*prepare)(struct odb_source *source,
|
|
enum odb_prepare_flags flags);
|
|
|
|
/*
|
|
* This callback is expected to read object information from the object
|
|
* database source. The object info will be partially populated with
|
|
* pointers for each bit of information that was requested by the
|
|
* caller.
|
|
*
|
|
* The flags field is a combination of `OBJECT_INFO` flags. Only the
|
|
* following fields need to be handled by the backend:
|
|
*
|
|
* - `OBJECT_INFO_QUICK` indicates it is fine to use caches without
|
|
* re-verifying the data.
|
|
*
|
|
* The callback is expected to return an `enum odb_read_status`. Please
|
|
* refer to the individual values that can be returned. In case reading
|
|
* the object has failed with a generic error and `errmsg` is non-NULL,
|
|
* the callback is expected to populate it with a human-readable
|
|
* message that describes the failure.
|
|
*/
|
|
enum odb_read_status (*read_object_info)(struct odb_source *source,
|
|
const struct object_id *oid,
|
|
struct object_info *oi,
|
|
enum object_info_flags flags,
|
|
struct strbuf *errmsg);
|
|
|
|
/*
|
|
* This callback is expected to create a new read stream that can be
|
|
* used to stream the object identified by the given ID.
|
|
*
|
|
* The callback is expected to return a negative error code in case
|
|
* creating the object stream has failed, 0 otherwise.
|
|
*/
|
|
int (*read_object_stream)(struct odb_stream **out,
|
|
struct odb_source *source,
|
|
const struct object_id *oid);
|
|
|
|
/*
|
|
* This callback is expected to iterate over all objects stored in this
|
|
* source and invoke the callback function for each of them. It is
|
|
* valid to yield the same object multiple time. A non-zero exit code
|
|
* from the object callback shall abort iteration.
|
|
*
|
|
* The optional `request` structure should serve as a template for
|
|
* looking up object info for every individual iterated object. It
|
|
* should not be modified directly and should instead be copied into a
|
|
* separate `struct object_info` that gets passed to the callback. If
|
|
* the caller passes a `NULL` pointer then the object itself shall not
|
|
* be read.
|
|
*
|
|
* The callback is expected to return a negative error code in case the
|
|
* iteration has failed to read all objects, 0 otherwise. When the
|
|
* callback function returns a non-zero error code then that error code
|
|
* should be returned.
|
|
*/
|
|
int (*for_each_object)(struct odb_source *source,
|
|
const struct object_info *request,
|
|
odb_for_each_object_cb cb,
|
|
void *cb_data,
|
|
const struct odb_for_each_object_options *opts);
|
|
|
|
/*
|
|
* This callback is expected to count objects in the given object
|
|
* database source. The callback function does not have to guarantee
|
|
* that only unique objects are counted. The result shall be assigned
|
|
* to the `out` pointer.
|
|
*
|
|
* Accepts `enum odb_count_objects_flag` flags to alter the behaviour.
|
|
*
|
|
* The callback is expected to return 0 on success, or a negative error
|
|
* code otherwise.
|
|
*/
|
|
int (*count_objects)(struct odb_source *source,
|
|
enum odb_count_objects_flags flags,
|
|
unsigned long *out);
|
|
|
|
/*
|
|
* This callback is expected to find the minimum required length to
|
|
* make the given object ID unique.
|
|
*
|
|
* The callback is expected to return a negative error code in case it
|
|
* failed, 0 otherwise.
|
|
*/
|
|
int (*find_abbrev_len)(struct odb_source *source,
|
|
const struct object_id *oid,
|
|
unsigned min_length,
|
|
unsigned *out);
|
|
|
|
/*
|
|
* This callback is expected to freshen the given object so that its
|
|
* last access time is set to the current time. This is used to ensure
|
|
* that objects that are recent will not get garbage collected even if
|
|
* they were unreachable.
|
|
*
|
|
* Returns 0 in case the object does not exist, 1 in case the object
|
|
* has been freshened.
|
|
*/
|
|
int (*freshen_object)(struct odb_source *source,
|
|
const struct object_id *oid,
|
|
const time_t *mtime);
|
|
|
|
/*
|
|
* This callback is expected to persist the given object into the
|
|
* object source. In case the object already exists it shall be
|
|
* freshened.
|
|
*
|
|
* The flags field is a combination of `WRITE_OBJECT` flags.
|
|
*
|
|
* The resulting object ID (and optionally the compatibility object ID)
|
|
* shall be written into the out pointers. The callback is expected to
|
|
* return 0 on success, a negative error code otherwise.
|
|
*/
|
|
int (*write_object)(struct odb_source *source,
|
|
const void *buf, size_t len,
|
|
enum object_type type,
|
|
const struct object_id *oid,
|
|
const struct object_id *compat_oid,
|
|
const time_t *mtime,
|
|
enum odb_write_object_flags flags);
|
|
|
|
/*
|
|
* This callback is expected to persist the given object stream into
|
|
* the object source.
|
|
*
|
|
* The resulting object ID shall be written into the out pointer. The
|
|
* callback is expected to return 0 on success, a negative error code
|
|
* otherwise.
|
|
*/
|
|
int (*write_object_stream)(struct odb_source *source,
|
|
struct odb_stream *stream,
|
|
struct object_id *oid);
|
|
|
|
/*
|
|
* This callback is expected to create a new transaction that can be
|
|
* used to write objects to. The objects shall only be persisted into
|
|
* the object database when the transcation's commit function is
|
|
* called. Otherwise, the objects shall be discarded.
|
|
*
|
|
* Returns 0 on success, in which case the `*out` pointer will have
|
|
* been populated with the object database transaction. Returns a
|
|
* negative error code otherwise.
|
|
*/
|
|
int (*begin_transaction)(struct odb_source *source,
|
|
struct odb_transaction **out,
|
|
enum odb_transaction_flags flags);
|
|
|
|
/*
|
|
* This callback is expected to optimize the object database source.
|
|
* Returns 0 on success, a negative error code otherwise.
|
|
*/
|
|
int (*optimize)(struct odb_source *source,
|
|
const struct odb_optimize_options *opts);
|
|
|
|
/*
|
|
* This callback is expected to check whether optimization of the
|
|
* object database source is required given the provided options.
|
|
* Returns true if optimization should be performed, false otherwise.
|
|
*/
|
|
bool (*optimize_required)(struct odb_source *source,
|
|
const struct odb_optimize_options *opts);
|
|
|
|
/*
|
|
* This callback is expected to start generating a packfile with the
|
|
* given options. The pack shall be generated asynchronously so that
|
|
* the caller can consume the pack data and progress output while the
|
|
* pack is being generated.
|
|
*
|
|
* This callback is optional. Sources that cannot generate packfiles
|
|
* shall leave it unset.
|
|
*
|
|
* The callback is expected to return 0 on success and populate the
|
|
* `out` pointer with the pack generator, a negative error code
|
|
* otherwise.
|
|
*/
|
|
int (*generate_pack)(struct odb_source *source,
|
|
struct odb_pack_generator **out,
|
|
const struct odb_generate_pack_options *opts);
|
|
|
|
/*
|
|
* This callback is expected to check the integrity of the object source
|
|
* and report any errors found via the fsck options. The checks performed
|
|
* are backend-specific.
|
|
*
|
|
* The callback is expected to return 0 on success, a negative error
|
|
* code otherwise.
|
|
*/
|
|
int (*fsck)(struct odb_source *source,
|
|
struct odb_fsck_options *options);
|
|
};
|
|
|
|
/*
|
|
* Allocate and initialize a new source for the given object database. The path
|
|
* of the source is derived from repository paths.
|
|
*/
|
|
struct odb_source *odb_source_new(struct object_database *odb,
|
|
enum odb_new_flags flags);
|
|
|
|
/*
|
|
* Initialize the source for the given object database located at `path`.
|
|
* `local` indicates whether or not the source is the local and thus primary
|
|
* object source of the object database.
|
|
*
|
|
* This function is only supposed to be called by specific object source
|
|
* implementations.
|
|
*/
|
|
void odb_source_init(struct odb_source *source,
|
|
struct object_database *odb,
|
|
enum odb_source_type type,
|
|
const char *path,
|
|
bool local);
|
|
|
|
/*
|
|
* Free the object database source, releasing all associated resources and
|
|
* freeing the structure itself.
|
|
*/
|
|
void odb_source_free(struct odb_source *source);
|
|
|
|
/*
|
|
* Release the object database source, releasing all associated resources.
|
|
*
|
|
* This function is only supposed to be called by specific object source
|
|
* implementations.
|
|
*/
|
|
void odb_source_release(struct odb_source *source);
|
|
|
|
/*
|
|
* Close the object database source without releasing he underlying data. The
|
|
* source can still be used going forward, but it first needs to be reopened.
|
|
* This can be useful to reduce resource usage.
|
|
*/
|
|
static inline void odb_source_close(struct odb_source *source)
|
|
{
|
|
source->close(source);
|
|
}
|
|
|
|
/*
|
|
* Create on-disk data structures that are required for this source to operate
|
|
* correctly. Returns 0 on success, a negative error code otherwise.
|
|
*/
|
|
static inline int odb_source_create_on_disk(struct odb_source *source,
|
|
const struct odb_create_on_disk_options *opts)
|
|
{
|
|
if (!source->create_on_disk)
|
|
return 0;
|
|
return source->create_on_disk(source, opts);
|
|
}
|
|
|
|
/*
|
|
* Prepare the object database source and clear any caches. Depending on the
|
|
* backend used this may have the effect that concurrently-written objects
|
|
* become visible.
|
|
*/
|
|
static inline void odb_source_prepare(struct odb_source *source,
|
|
enum odb_prepare_flags flags)
|
|
{
|
|
source->prepare(source, flags);
|
|
}
|
|
|
|
/*
|
|
* Read an object from the object database source identified by its object ID.
|
|
* Please refer to `enum odb_read_status` for the individual error codes.
|
|
*
|
|
* In case reading the object has failed with a generic error and `errmsg` is
|
|
* non-NULL it will be populated with a human-readable message that describes
|
|
* the failure.
|
|
*/
|
|
static inline enum odb_read_status odb_source_read_object_info(struct odb_source *source,
|
|
const struct object_id *oid,
|
|
struct object_info *oi,
|
|
enum object_info_flags flags,
|
|
struct strbuf *errmsg)
|
|
{
|
|
return source->read_object_info(source, oid, oi, flags, errmsg);
|
|
}
|
|
|
|
/*
|
|
* Create a new read stream for the given object ID. Returns 0 on success, a
|
|
* negative error code otherwise.
|
|
*/
|
|
static inline int odb_source_read_object_stream(struct odb_stream **out,
|
|
struct odb_source *source,
|
|
const struct object_id *oid)
|
|
{
|
|
return source->read_object_stream(out, source, oid);
|
|
}
|
|
|
|
/*
|
|
* Iterate through all objects contained in the given source and invoke the
|
|
* callback function for each of them. Returning a non-zero code from the
|
|
* callback function aborts iteration. There is no guarantee that objects
|
|
* are only iterated over once.
|
|
*
|
|
* The optional `request` structure serves as a template for retrieving the
|
|
* object info for each individual iterated object and will be populated as if
|
|
* `odb_source_read_object_info()` was called on the object. It will not be
|
|
* modified, the callback will instead be invoked with a separate `struct
|
|
* object_info` for every object. Object info will not be read when passing a
|
|
* `NULL` pointer.
|
|
*
|
|
* The flags is a bitfield of `ODB_FOR_EACH_OBJECT_*` flags. Not all flags may
|
|
* apply to a specific backend, so whether or not they are honored is defined
|
|
* by the implementation.
|
|
*
|
|
* Returns 0 when all objects have been iterated over, a negative error code in
|
|
* case iteration has failed, or a non-zero value returned from the callback.
|
|
*/
|
|
static inline int odb_source_for_each_object(struct odb_source *source,
|
|
const struct object_info *request,
|
|
odb_for_each_object_cb cb,
|
|
void *cb_data,
|
|
const struct odb_for_each_object_options *opts)
|
|
{
|
|
return source->for_each_object(source, request, cb, cb_data, opts);
|
|
}
|
|
|
|
/*
|
|
* Count the number of objects in the given object database source.
|
|
*
|
|
* Returns 0 on success, a negative error code otherwise.
|
|
*/
|
|
static inline int odb_source_count_objects(struct odb_source *source,
|
|
enum odb_count_objects_flags flags,
|
|
unsigned long *out)
|
|
{
|
|
return source->count_objects(source, flags, out);
|
|
}
|
|
|
|
/*
|
|
* Determine the minimum required length to make the given object ID unique in
|
|
* the given source. Returns 0 on success, a negative error code otherwise.
|
|
*/
|
|
static inline int odb_source_find_abbrev_len(struct odb_source *source,
|
|
const struct object_id *oid,
|
|
unsigned min_len,
|
|
unsigned *out)
|
|
{
|
|
return source->find_abbrev_len(source, oid, min_len, out);
|
|
}
|
|
|
|
/*
|
|
* Freshen an object in the object database by updating its timestamp.
|
|
* Returns 1 in case the object has been freshened, 0 in case the object does
|
|
* not exist.
|
|
*/
|
|
static inline int odb_source_freshen_object(struct odb_source *source,
|
|
const struct object_id *oid,
|
|
const time_t *mtime)
|
|
{
|
|
return source->freshen_object(source, oid, mtime);
|
|
}
|
|
|
|
/*
|
|
* Write an object into the object database source. Returns 0 on success, a
|
|
* negative error code otherwise. Populates the given out pointers for the
|
|
* object ID and the compatibility object ID, if non-NULL.
|
|
*/
|
|
static inline int odb_source_write_object(struct odb_source *source,
|
|
const void *buf, unsigned long len,
|
|
enum object_type type,
|
|
const struct object_id *oid,
|
|
const struct object_id *compat_oid,
|
|
const time_t *mtime,
|
|
enum odb_write_object_flags flags)
|
|
{
|
|
return source->write_object(source, buf, len, type, oid,
|
|
compat_oid, mtime, flags);
|
|
}
|
|
|
|
/*
|
|
* Write an object into the object database source via a stream. The overall
|
|
* length of the object must be known in advance.
|
|
*
|
|
* Return 0 on success, a negative error code otherwise. Populates the given
|
|
* out pointer for the object ID.
|
|
*/
|
|
static inline int odb_source_write_object_stream(struct odb_source *source,
|
|
struct odb_stream *stream,
|
|
struct object_id *oid)
|
|
{
|
|
return source->write_object_stream(source, stream, oid);
|
|
}
|
|
|
|
/*
|
|
* Create a new transaction that can be used to write objects into a temporary
|
|
* staging area. The objects will only be persisted when the transaction is
|
|
* committed.
|
|
*
|
|
* Returns 0 on success, a negative error code otherwise.
|
|
*/
|
|
static inline int odb_source_begin_transaction(struct odb_source *source,
|
|
struct odb_transaction **out,
|
|
enum odb_transaction_flags flags)
|
|
{
|
|
return source->begin_transaction(source, out, flags);
|
|
}
|
|
|
|
/*
|
|
* Optimize the object database source. Returns 0 on success, a negative error
|
|
* code otherwise.
|
|
*/
|
|
static inline int odb_source_optimize(struct odb_source *source,
|
|
const struct odb_optimize_options *opts)
|
|
{
|
|
return source->optimize(source, opts);
|
|
}
|
|
|
|
/*
|
|
* Check whether optimization of the object database source is required given
|
|
* the provided options. Returns true if optimization should be performed,
|
|
* false otherwise.
|
|
*/
|
|
static inline bool odb_source_optimize_required(struct odb_source *source,
|
|
const struct odb_optimize_options *opts)
|
|
{
|
|
return source->optimize_required(source, opts);
|
|
}
|
|
|
|
/*
|
|
* Start generating a packfile from the given source with the given options.
|
|
* The pack is generated asynchronously; the caller is expected to consume the
|
|
* file descriptors exposed via the pack generator and to then wait for
|
|
* completion via `odb_pack_generator_finish()`.
|
|
*
|
|
* Returns 0 on success and populates the `out` pointer with the pack
|
|
* generator, a negative error code otherwise.
|
|
*/
|
|
static inline int odb_source_generate_pack(struct odb_source *source,
|
|
struct odb_pack_generator **out,
|
|
const struct odb_generate_pack_options *opts)
|
|
{
|
|
return source->generate_pack(source, out, opts);
|
|
}
|
|
|
|
/*
|
|
* Check the integrity of the object database source. The checks performed
|
|
* are backend-specific. Returns 0 on success, a negative error code otherwise.
|
|
*/
|
|
static inline int odb_source_fsck(struct odb_source *source,
|
|
struct odb_fsck_options *opts)
|
|
{
|
|
return source->fsck(source, opts);
|
|
}
|
|
|
|
#endif
|