Skip to content

Commit a69b8d3

Browse files
MayCXCMayCXC
committed
odb: add write_packfile, for_each_unique_abbrev, convert_object_id
Add three vtable methods to odb_source that were not part of the recent ps/odb-sources and ps/object-counting series: - write_packfile: ingest a pack from a file descriptor. The files backend chooses between index-pack (large packs) and unpack-objects (small packs below fetch.unpackLimit). Options cover thin-pack fixing, promisor marking, fsck, lockfile capture, and shallow file passing. - for_each_unique_abbrev: iterate objects matching a hex prefix for disambiguation. Searches loose objects via oidtree, then multi-pack indices, then non-MIDX packs. - convert_object_id: translate between hash algorithms using the loose object map. Used during SHA-1 to SHA-256 migration. Also add ODB_SOURCE_HELPER to the source type enum, preparing for the helper backend in the next commit. The write_packfile vtable method replaces the pattern where callers spawn index-pack/unpack-objects directly. fast-import already uses odb_write_packfile() and this allows non-files backends to handle pack ingestion through their own mechanism. Signed-off-by: Aaron Paterson <apaterson@pm.me>
1 parent 41688c1 commit a69b8d3

10 files changed

Lines changed: 584 additions & 119 deletions

File tree

‎builtin/pack-objects.c‎

Lines changed: 28 additions & 80 deletions
Original file line numberDiff line numberDiff line change
@@ -32,6 +32,7 @@
3232
#include "packfile.h"
3333
#include "object-file.h"
3434
#include "odb.h"
35+
#include "odb/source.h"
3536
#include "odb/streaming.h"
3637
#include "replace-object.h"
3738
#include "dir.h"
@@ -1541,57 +1542,15 @@ static int have_duplicate_entry(const struct object_id *oid,
15411542

15421543
static int want_cruft_object_mtime(struct repository *r,
15431544
const struct object_id *oid,
1544-
unsigned flags, uint32_t mtime)
1545+
unsigned flags UNUSED, uint32_t mtime UNUSED)
15451546
{
1546-
struct odb_source *source;
1547-
1548-
for (source = r->objects->sources; source; source = source->next) {
1549-
struct odb_source_files *files = odb_source_files_downcast(source);
1550-
struct packed_git **cache = packfile_store_get_kept_pack_cache(files->packed, flags);
1551-
1552-
for (; *cache; cache++) {
1553-
struct packed_git *p = *cache;
1554-
off_t ofs;
1555-
uint32_t candidate_mtime;
1556-
1557-
ofs = find_pack_entry_one(oid, p);
1558-
if (!ofs)
1559-
continue;
1560-
1561-
/*
1562-
* We have a copy of the object 'oid' in a non-cruft
1563-
* pack. We can avoid packing an additional copy
1564-
* regardless of what the existing copy's mtime is since
1565-
* it is outside of a cruft pack.
1566-
*/
1567-
if (!p->is_cruft)
1568-
return 0;
1569-
1570-
/*
1571-
* If we have a copy of the object 'oid' in a cruft
1572-
* pack, then either read the cruft pack's mtime for
1573-
* that object, or, if that can't be loaded, assume the
1574-
* pack's mtime itself.
1575-
*/
1576-
if (!load_pack_mtimes(p)) {
1577-
uint32_t pos;
1578-
if (offset_to_pack_pos(p, ofs, &pos) < 0)
1579-
continue;
1580-
candidate_mtime = nth_packed_mtime(p, pos);
1581-
} else {
1582-
candidate_mtime = p->mtime;
1583-
}
1584-
1585-
/*
1586-
* We have a surviving copy of the object in a cruft
1587-
* pack whose mtime is greater than or equal to the one
1588-
* we are considering. We can thus avoid packing an
1589-
* additional copy of that object.
1590-
*/
1591-
if (mtime <= candidate_mtime)
1592-
return 0;
1593-
}
1594-
}
1547+
/*
1548+
* Check if the object exists in a kept source. Dispatches through
1549+
* the vtable: files backends check kept packs, non-files backends
1550+
* check their own kept tracking via OBJECT_INFO_KEPT_ONLY.
1551+
*/
1552+
if (odb_has_object_kept(r->objects, oid))
1553+
return 0;
15951554

15961555
return -1;
15971556
}
@@ -1657,7 +1616,7 @@ static int want_found_object(const struct object_id *oid, int exclude,
16571616
return 0;
16581617
if (ignore_packed_keep_in_core && p->pack_keep_in_core)
16591618
return 0;
1660-
if (has_object_kept_pack(p->repo, oid, flags))
1619+
if (odb_has_object_kept(p->repo->objects, oid))
16611620
return 0;
16621621
} else {
16631622
/*
@@ -1726,8 +1685,6 @@ static int want_object_in_pack_mtime(const struct object_id *oid,
17261685
uint32_t found_mtime)
17271686
{
17281687
int want;
1729-
struct packfile_list_entry *e;
1730-
struct odb_source *source;
17311688

17321689
if (!exclude && local) {
17331690
/*
@@ -1757,25 +1714,18 @@ static int want_object_in_pack_mtime(const struct object_id *oid,
17571714

17581715
odb_prepare_alternates(the_repository->objects);
17591716

1760-
for (source = the_repository->objects->sources; source; source = source->next) {
1761-
struct multi_pack_index *m = get_multi_pack_index(source);
1762-
struct pack_entry e;
1763-
1764-
if (m && fill_midx_entry(m, oid, &e)) {
1765-
want = want_object_in_pack_one(e.p, oid, exclude, found_pack, found_offset, found_mtime);
1766-
if (want != -1)
1767-
return want;
1768-
}
1769-
}
1770-
1771-
for (source = the_repository->objects->sources; source; source = source->next) {
1772-
struct odb_source_files *files = odb_source_files_downcast(source);
1717+
{
1718+
struct object_info oi = OBJECT_INFO_INIT;
17731719

1774-
for (e = files->packed->packs.head; e; e = e->next) {
1775-
struct packed_git *p = e->pack;
1776-
want = want_object_in_pack_one(p, oid, exclude, found_pack, found_offset, found_mtime);
1777-
if (!exclude && want > 0)
1778-
packfile_list_prepend(&files->packed->packs, p);
1720+
if (!odb_read_object_info_extended(the_repository->objects,
1721+
oid, &oi,
1722+
OBJECT_INFO_QUICK) &&
1723+
oi.whence == OI_PACKED) {
1724+
struct packed_git *p = oi.u.packed.pack;
1725+
want = want_object_in_pack_one(p, oid, exclude,
1726+
found_pack,
1727+
found_offset,
1728+
found_mtime);
17791729
if (want != -1)
17801730
return want;
17811731
}
@@ -4065,7 +4015,7 @@ static void show_cruft_commit(struct commit *commit, void *data)
40654015

40664016
static int cruft_include_check_obj(struct object *obj, void *data UNUSED)
40674017
{
4068-
return !has_object_kept_pack(to_pack.repo, &obj->oid, KEPT_PACK_IN_CORE);
4018+
return !odb_has_object_kept(to_pack.repo->objects, &obj->oid);
40694019
}
40704020

40714021
static int cruft_include_check(struct commit *commit, void *data)
@@ -4365,17 +4315,15 @@ static void add_objects_in_unpacked_packs(void)
43654315

43664316
odb_prepare_alternates(to_pack.repo->objects);
43674317
for (source = to_pack.repo->objects->sources; source; source = source->next) {
4368-
struct odb_source_files *files = odb_source_files_downcast(source);
4369-
43704318
if (!source->local)
43714319
continue;
43724320

4373-
if (packfile_store_for_each_object(files->packed, &oi,
4374-
add_object_in_unpacked_pack, NULL,
4375-
ODB_FOR_EACH_OBJECT_PACK_ORDER |
4376-
ODB_FOR_EACH_OBJECT_LOCAL_ONLY |
4377-
ODB_FOR_EACH_OBJECT_SKIP_IN_CORE_KEPT_PACKS |
4378-
ODB_FOR_EACH_OBJECT_SKIP_ON_DISK_KEPT_PACKS))
4321+
if (odb_source_for_each_object(source, &oi,
4322+
add_object_in_unpacked_pack, NULL,
4323+
ODB_FOR_EACH_OBJECT_PACK_ORDER |
4324+
ODB_FOR_EACH_OBJECT_LOCAL_ONLY |
4325+
ODB_FOR_EACH_OBJECT_SKIP_IN_CORE_KEPT_PACKS |
4326+
ODB_FOR_EACH_OBJECT_SKIP_ON_DISK_KEPT_PACKS))
43794327
die(_("cannot open pack index"));
43804328
}
43814329
}

‎loose.c‎

Lines changed: 10 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -128,13 +128,17 @@ int repo_read_loose_object_map(struct repository *repo)
128128

129129
int repo_write_loose_object_map(struct repository *repo)
130130
{
131-
struct odb_source_files *files = odb_source_files_downcast(repo->objects->sources);
132-
kh_oid_map_t *map = files->loose->map->to_compat;
131+
struct odb_source *source = repo->objects->sources;
132+
kh_oid_map_t *map;
133133
struct lock_file lock;
134134
int fd;
135135
khiter_t iter;
136136
struct strbuf buf = STRBUF_INIT, path = STRBUF_INIT;
137137

138+
if (!source->loose || !source->loose->map)
139+
return 0;
140+
map = source->loose->map->to_compat;
141+
138142
if (!should_use_loose_object_map(repo))
139143
return 0;
140144

@@ -235,8 +239,10 @@ int repo_loose_object_map_oid(struct repository *repo,
235239
khiter_t pos;
236240

237241
for (source = repo->objects->sources; source; source = source->next) {
238-
struct odb_source_files *files = odb_source_files_downcast(source);
239-
struct loose_object_map *loose_map = files->loose->map;
242+
struct loose_object_map *loose_map;
243+
if (!source->loose)
244+
continue;
245+
loose_map = source->loose->map;
240246
if (!loose_map)
241247
continue;
242248
map = (to == repo->compat_hash_algo) ?

‎midx.c‎

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -101,9 +101,10 @@ static int midx_read_object_offsets(const unsigned char *chunk_start,
101101

102102
struct multi_pack_index *get_multi_pack_index(struct odb_source *source)
103103
{
104-
struct odb_source_files *files = odb_source_files_downcast(source);
105-
packfile_store_prepare(files->packed);
106-
return files->packed->midx;
104+
if (!source->packed)
105+
return NULL;
106+
packfile_store_prepare(source->packed);
107+
return source->packed->midx;
107108
}
108109

109110
static struct multi_pack_index *load_multi_pack_index_one(struct odb_source *source,

‎object-name.c‎

Lines changed: 61 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,7 @@
2020
#include "packfile.h"
2121
#include "pretty.h"
2222
#include "object-file.h"
23+
#include "odb/source.h"
2324
#include "read-cache-ll.h"
2425
#include "repo-settings.h"
2526
#include "repository.h"
@@ -111,13 +112,28 @@ static enum cb_next match_prefix(const struct object_id *oid, void *arg)
111112
return ds->ambiguous ? CB_BREAK : CB_CONTINUE;
112113
}
113114

115+
static int disambiguate_cb(const struct object_id *oid,
116+
struct object_info *oi UNUSED, void *data)
117+
{
118+
struct disambiguate_state *ds = data;
119+
update_candidates(ds, oid);
120+
return ds->ambiguous ? 1 : 0;
121+
}
122+
114123
static void find_short_object_filename(struct disambiguate_state *ds)
115124
{
116125
struct odb_source *source;
117126

118-
for (source = ds->repo->objects->sources; source && !ds->ambiguous; source = source->next)
119-
oidtree_each(odb_source_loose_cache(source, &ds->bin_pfx),
120-
&ds->bin_pfx, ds->len, match_prefix, ds);
127+
for (source = ds->repo->objects->sources; source && !ds->ambiguous; source = source->next) {
128+
if (source->for_each_unique_abbrev) {
129+
odb_source_for_each_unique_abbrev(
130+
source, &ds->bin_pfx, ds->len,
131+
disambiguate_cb, ds);
132+
} else {
133+
oidtree_each(odb_source_loose_cache(source, &ds->bin_pfx),
134+
&ds->bin_pfx, ds->len, match_prefix, ds);
135+
}
136+
}
121137
}
122138

123139
static int match_hash(unsigned len, const unsigned char *a, const unsigned char *b)
@@ -208,15 +224,23 @@ static void find_short_packed_object(struct disambiguate_state *ds)
208224

209225
odb_prepare_alternates(ds->repo->objects);
210226
for (source = ds->repo->objects->sources; source && !ds->ambiguous; source = source->next) {
211-
struct multi_pack_index *m = get_multi_pack_index(source);
212-
if (m)
213-
unique_in_midx(m, ds);
227+
if (source->for_each_unique_abbrev) {
228+
odb_source_for_each_unique_abbrev(
229+
source, &ds->bin_pfx, ds->len,
230+
disambiguate_cb, ds);
231+
} else {
232+
struct multi_pack_index *m = get_multi_pack_index(source);
233+
if (m)
234+
unique_in_midx(m, ds);
235+
}
214236
}
215237

216-
repo_for_each_pack(ds->repo, p) {
217-
if (ds->ambiguous)
218-
break;
219-
unique_in_pack(p, ds);
238+
if (!ds->repo->objects->sources->for_each_unique_abbrev) {
239+
repo_for_each_pack(ds->repo, p) {
240+
if (ds->ambiguous)
241+
break;
242+
unique_in_pack(p, ds);
243+
}
220244
}
221245
}
222246

@@ -796,19 +820,38 @@ static void find_abbrev_len_for_pack(struct packed_git *p,
796820
mad->init_len = mad->cur_len;
797821
}
798822

799-
static void find_abbrev_len_packed(struct min_abbrev_data *mad)
823+
static int abbrev_len_cb(const struct object_id *oid,
824+
struct object_info *oi UNUSED, void *data)
800825
{
801-
struct packed_git *p;
826+
struct min_abbrev_data *mad = data;
827+
extend_abbrev_len(oid, mad);
828+
return 0;
829+
}
802830

831+
static void find_abbrev_len_packed(struct min_abbrev_data *mad)
832+
{
803833
odb_prepare_alternates(mad->repo->objects);
804-
for (struct odb_source *source = mad->repo->objects->sources; source; source = source->next) {
805-
struct multi_pack_index *m = get_multi_pack_index(source);
806-
if (m)
807-
find_abbrev_len_for_midx(m, mad);
834+
835+
for (struct odb_source *source = mad->repo->objects->sources;
836+
source; source = source->next) {
837+
if (source->for_each_unique_abbrev) {
838+
mad->init_len = 0;
839+
odb_source_for_each_unique_abbrev(
840+
source, mad->oid, mad->cur_len,
841+
abbrev_len_cb, mad);
842+
mad->init_len = mad->cur_len;
843+
} else {
844+
struct multi_pack_index *m = get_multi_pack_index(source);
845+
if (m)
846+
find_abbrev_len_for_midx(m, mad);
847+
}
808848
}
809849

810-
repo_for_each_pack(mad->repo, p)
811-
find_abbrev_len_for_pack(p, mad);
850+
if (!mad->repo->objects->sources->for_each_unique_abbrev) {
851+
struct packed_git *p;
852+
repo_for_each_pack(mad->repo, p)
853+
find_abbrev_len_for_pack(p, mad);
854+
}
812855
}
813856

814857
void strbuf_repo_add_unique_abbrev(struct strbuf *sb, struct repository *repo,

‎odb.c‎

Lines changed: 39 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -981,6 +981,45 @@ int odb_write_object_stream(struct object_database *odb,
981981
return odb_source_write_object_stream(odb->sources, stream, len, oid);
982982
}
983983

984+
int odb_has_object_kept(struct object_database *odb,
985+
const struct object_id *oid)
986+
{
987+
struct odb_source *source;
988+
989+
odb_prepare_alternates(odb);
990+
for (source = odb->sources; source; source = source->next)
991+
if (!odb_source_read_object_info(source, oid, NULL,
992+
OBJECT_INFO_KEPT_ONLY))
993+
return 1;
994+
return 0;
995+
}
996+
997+
int odb_write_packfile(struct object_database *odb,
998+
int pack_fd,
999+
struct odb_write_packfile_options *opts)
1000+
{
1001+
return odb_source_write_packfile(odb->sources, pack_fd, opts);
1002+
}
1003+
1004+
int odb_for_each_unique_abbrev(struct object_database *odb,
1005+
const struct object_id *oid_prefix,
1006+
unsigned int prefix_len,
1007+
odb_for_each_object_cb cb,
1008+
void *cb_data)
1009+
{
1010+
int ret;
1011+
1012+
odb_prepare_alternates(odb);
1013+
for (struct odb_source *source = odb->sources; source; source = source->next) {
1014+
ret = odb_source_for_each_unique_abbrev(source, oid_prefix,
1015+
prefix_len, cb, cb_data);
1016+
if (ret)
1017+
return ret;
1018+
}
1019+
1020+
return 0;
1021+
}
1022+
9841023
struct object_database *odb_new(struct repository *repo,
9851024
const char *primary_source,
9861025
const char *secondary_sources)

0 commit comments

Comments
 (0)