Files
git/commit-graph.h
T
Kristofer Karlsson cef2d8c4cb fetch: write commit-graph using updated refs only
When fetch.writeCommitGraph was introduced in 50f26bd035 (fetch: add
fetch.writeCommitGraph config setting, 2019-09-02), the stated goal
was to stay updated with the latest commits after fetching new
objects.  The implementation used write_commit_graph_reachable()
because it was the only API available, but two things have changed
since then:

 1. write_commit_graph() was added, and it accepts an explicit set of
    commits as seeds, enabling more targeted commit-graph updates.

 2. The ref-scanning callback add_ref_to_set() became more expensive
    in 630cd5194e (commit-graph.c: peel refs in 'add_ref_to_set',
    2020-07-22) when it started to validate the refs against the odb
    for correctness.  On a repository with many refs, this makes the
    full reachable scan unnecessarily costly for a targeted fetch.

Optimize the commit-graph write by using only the newly updated refs
as seeds instead of scanning all refs after every fetch.  To keep
this change small, skip the optimization for multi-remote fetches
(since that would require propagating the set of refs across process
boundaries).

This relies on the commit-graph write being additive, keeping the
commits that are already in the graph.  fetch already operates in
this mode (COMMIT_GRAPH_WRITE_SPLIT) and now that becomes
required for correctness.  Without that mode, the write would
replace the commit-graph and lose other commits.

After fetch_one() returns, call prepare_commit_graph() (which is
made non-static by this commit) to determine the graph-write mode:

 - If no commit-graph exists yet, fall back to the full reachable
   scan so the first graph creation covers all refs.

 - If a commit-graph exists and the fetch updated at least one ref,
   write incrementally using only the new refs as seeds.

 - If a commit-graph exists but the fetch is a no-op, skip the
   commit-graph write entirely.

 - For the multi-remote path (fetch --all), where child processes
   do the actual fetching, fall back to the full reachable scan.

Full commit-graph coverage of all refs remains the responsibility
of "git maintenance", "git gc" and "git commit-graph write".
Regular Git operations may trigger "git maintenance run --auto",
which periodically rebuilds the commit-graph from all reachable
refs.

The effect was measured on a synthetic setup: git.git with 200K
extra packed refs (~206K total), a local file:// remote, an existing
split commit-graph and a warm page cache.  The times below are the
median of 9 runs of the trace2 region fetch/write-commit-graph:

    scenario          before    after
    no-op fetch       380 ms    (skipped)
    1 ref updated     357 ms    9.3 ms
    10 refs updated   359 ms    8.9 ms

Signed-off-by: Kristofer Karlsson <krka@spotify.com>
Signed-off-by: Junio C Hamano <gitster@pobox.com>
2026-10-07 09:50:48 -07:00

217 lines
7.0 KiB
C

#ifndef COMMIT_GRAPH_H
#define COMMIT_GRAPH_H
#include "odb.h"
#include "oidset.h"
#define GIT_TEST_COMMIT_GRAPH "GIT_TEST_COMMIT_GRAPH"
#define GIT_TEST_COMMIT_GRAPH_DIE_ON_PARSE "GIT_TEST_COMMIT_GRAPH_DIE_ON_PARSE"
#define GIT_TEST_COMMIT_GRAPH_CHANGED_PATHS "GIT_TEST_COMMIT_GRAPH_CHANGED_PATHS"
/*
* This environment variable controls whether commits looked up via the
* commit graph will be double checked to exist in the object database.
*/
#define GIT_COMMIT_GRAPH_PARANOIA "GIT_COMMIT_GRAPH_PARANOIA"
/*
* This method is only used to enhance coverage of the commit-graph
* feature in the test suite with the GIT_TEST_COMMIT_GRAPH and
* GIT_TEST_COMMIT_GRAPH_CHANGED_PATHS environment variables. Do not
* call this method outside of a builtin, and only if you know what
* you are doing!
*/
void git_test_write_commit_graph_or_die(struct odb_source *source);
struct commit;
struct bloom_filter_settings;
struct repository;
struct object_database;
struct string_list;
char *get_commit_graph_filename(struct odb_source *source);
char *get_commit_graph_chain_filename(struct odb_source *source);
struct commit_graph *prepare_commit_graph(struct repository *r);
int open_commit_graph(const char *graph_file, int *fd, struct stat *st);
int open_commit_graph_chain(const char *chain_file, int *fd, struct stat *st,
const struct git_hash_algo *hash_algo);
/*
* Given a commit struct, try to fill the commit struct info, including:
* 1. tree object
* 2. date
* 3. parents.
*
* Returns 1 if and only if the commit was found in the packed graph.
*
* See parse_commit_buffer() for the fallback after this call.
*/
int parse_commit_in_graph(struct repository *r, struct commit *item);
/*
* Fills `*pos` with the graph position of `c`, and returns the graph `c` is
* found in, or NULL otherwise. Initializes the commit-graphs belonging to
* `r` if it hasn't been already.
*
* Note: this is a low-level helper that does not alter any slab data
* associated with `c`. Useful in circumstances where the slab data is
* already being modified (e.g., writing the commit-graph itself).
*
* In most cases, callers should use `parse_commit_in_graph()` instead.
*/
struct commit_graph *repo_find_commit_pos_in_graph(struct repository *r,
struct commit *c,
uint32_t *pos);
/*
* Look up the given commit ID in the commit-graph. This will only return a
* commit if the ID exists both in the graph and in the object database such
* that we don't return commits whose object has been pruned. Otherwise, this
* function returns `NULL`.
*/
struct commit *lookup_commit_in_graph(struct repository *repo, const struct object_id *id);
/*
* It is possible that we loaded commit contents from the commit buffer,
* but we also want to ensure the commit-graph content is correctly
* checked and filled. Fill the graph_pos and generation members of
* the given commit.
*/
void load_commit_graph_info(struct repository *r, struct commit *item);
struct tree *get_commit_tree_in_graph(struct repository *r,
const struct commit *c);
struct commit_graph {
const unsigned char *data;
size_t data_len;
const struct git_hash_algo *hash_algo;
unsigned char num_chunks;
uint32_t num_commits;
struct object_id oid;
char *filename;
struct odb_source *odb_source;
uint32_t num_commits_in_base;
unsigned int read_generation_data;
struct commit_graph *base_graph;
const uint32_t *chunk_oid_fanout;
const unsigned char *chunk_oid_lookup;
const unsigned char *chunk_commit_data;
const unsigned char *chunk_generation_data;
const unsigned char *chunk_generation_data_overflow;
size_t chunk_generation_data_overflow_size;
const unsigned char *chunk_extra_edges;
size_t chunk_extra_edges_size;
const unsigned char *chunk_base_graphs;
size_t chunk_base_graphs_size;
const unsigned char *chunk_bloom_indexes;
const unsigned char *chunk_bloom_data;
size_t chunk_bloom_data_size;
struct topo_level_slab *topo_levels;
struct bloom_filter_settings *bloom_filter_settings;
};
struct commit_graph *load_commit_graph_one_fd_st(struct odb_source *source,
int fd, struct stat *st);
struct commit_graph *load_commit_graph_chain_fd_st(struct object_database *odb,
int fd, struct stat *st,
int *incomplete_chain);
struct commit_graph *read_commit_graph_one(struct odb_source *source);
struct repo_settings;
/*
* Callers should initialize the repo_settings with prepare_repo_settings()
* prior to calling parse_commit_graph().
*/
struct commit_graph *parse_commit_graph(struct repository *r,
void *graph_map, size_t graph_size);
/*
* Return 1 if and only if the repository has a commit-graph
* file and generation numbers are computed in that file.
*/
int generation_numbers_enabled(struct repository *r);
/*
* Return 1 if and only if the repository has a commit-graph
* file and generation data chunk has been written for the file.
*/
int corrected_commit_dates_enabled(struct repository *r);
struct bloom_filter_settings *get_bloom_filter_settings(struct repository *r);
enum commit_graph_write_flags {
COMMIT_GRAPH_WRITE_APPEND = (1 << 0),
COMMIT_GRAPH_WRITE_PROGRESS = (1 << 1),
COMMIT_GRAPH_WRITE_SPLIT = (1 << 2),
COMMIT_GRAPH_WRITE_BLOOM_FILTERS = (1 << 3),
COMMIT_GRAPH_NO_WRITE_BLOOM_FILTERS = (1 << 4),
};
enum commit_graph_split_flags {
COMMIT_GRAPH_SPLIT_UNSPECIFIED = 0,
COMMIT_GRAPH_SPLIT_MERGE_PROHIBITED = 1,
COMMIT_GRAPH_SPLIT_REPLACE = 2
};
struct commit_graph_opts {
int size_multiple;
int max_commits;
timestamp_t expire_time;
enum commit_graph_split_flags split_flags;
int max_new_filters;
};
/*
* The write_commit_graph* methods return zero on success
* and a negative value on failure. Note that if the repository
* is not compatible with the commit-graph feature, then the
* methods will return 0 without writing a commit-graph.
*/
int write_commit_graph_reachable(struct odb_source *source,
enum commit_graph_write_flags flags,
const struct commit_graph_opts *opts);
int write_commit_graph(struct odb_source *source,
const struct string_list *pack_indexes,
struct oidset *commits,
enum commit_graph_write_flags flags,
const struct commit_graph_opts *opts);
#define COMMIT_GRAPH_VERIFY_SHALLOW (1 << 0)
int verify_commit_graph(struct commit_graph *g, int flags);
void close_commit_graph(struct object_database *);
void free_commit_graph(struct commit_graph *);
/*
* Disable further use of the commit graph in this process when parsing a
* "struct commit".
*/
void disable_commit_graph(struct repository *r);
struct commit_graph_data {
uint32_t graph_pos;
timestamp_t generation;
};
/*
* Commits should be parsed before accessing generation, graph positions.
*/
timestamp_t commit_graph_generation(const struct commit *);
uint32_t commit_graph_position(const struct commit *);
/*
* After this method, all commits reachable from those in the given
* list will have non-zero, non-infinite generation numbers.
*/
void ensure_generations_valid(struct repository *r,
struct commit **commits, size_t nr);
#endif