Merge changes I70d0e59d,I10ed8daf,Icdf7a34f * changes: Add missing LF at the end of GarbageCollectCommandTest Add GarbageCollectCommand#setGcConfig GC: add flag to control whether gc should pack all refs
diff --git a/.bazelrc b/.bazelrc index 5bf3ef8..385a71d 100644 --- a/.bazelrc +++ b/.bazelrc
@@ -1,6 +1,6 @@ # TODO(davido): Migrate all dependencies from WORKSPACE to MODULE.bazel # https://issues.gerritcodereview.com/issues/303819949 -common --enable_bzlmod +common --enable_bzlmod --lockfile_mode=error common --enable_workspace build --workspace_status_command="python3 ./tools/workspace_status.py"
diff --git a/org.eclipse.jgit.test/src/org/eclipse/jgit/internal/storage/dfs/MidxTestUtils.java b/org.eclipse.jgit.test/src/org/eclipse/jgit/internal/storage/dfs/MidxTestUtils.java index b34b822..c5210c8 100644 --- a/org.eclipse.jgit.test/src/org/eclipse/jgit/internal/storage/dfs/MidxTestUtils.java +++ b/org.eclipse.jgit.test/src/org/eclipse/jgit/internal/storage/dfs/MidxTestUtils.java
@@ -14,13 +14,20 @@ import static org.junit.Assert.assertEquals; import java.io.IOException; +import java.util.ArrayList; import java.util.Arrays; import java.util.List; import java.util.zip.Deflater; import org.eclipse.jgit.annotations.Nullable; +import org.eclipse.jgit.junit.TestRepository; import org.eclipse.jgit.lib.NullProgressMonitor; import org.eclipse.jgit.lib.ObjectId; +import org.eclipse.jgit.lib.Ref; +import org.eclipse.jgit.revwalk.RevBlob; +import org.eclipse.jgit.revwalk.RevCommit; +import org.eclipse.jgit.revwalk.RevTree; +import org.eclipse.jgit.storage.pack.PackConfig; /** * Helpers to write multipack indexes @@ -136,12 +143,44 @@ static DfsPackFileMidx writeSinglePackMidx(DfsRepository db, */ static DfsPackFileMidx writeMultipackIndex(DfsRepository db, DfsPackFile[] packs, DfsPackFileMidx base) throws IOException { + PackConfig packConfig = new PackConfig(db); + packConfig.setBitmapRecentCommitSpan(1); // bitmap every commit! DfsPackDescription desc = DfsMidxWriter.writeMidx( NullProgressMonitor.INSTANCE, db.getObjectDatabase(), Arrays.asList(packs), - base != null ? base.getPackDescription() : null); + base != null ? base.getPackDescription() : null, packConfig); db.getObjectDatabase().commitPack(List.of(desc), null); return DfsPackFileMidx.create(DfsBlockCache.getInstance(), desc, Arrays.asList(packs), base); } + + record CommitObjects(RevCommit commit, RevTree tree, RevBlob blob) { + } + + private static int commitCounter = 1; + + static List<CommitObjects> writeCommitChain(DfsRepository db, + String refname, int length) throws Exception { + List<CommitObjects> co = new ArrayList<>(length); + RevCommit tip = null; + Ref ref = db.getRefDatabase().findRef(refname); + if (ref != null) { + tip = db.parseCommit(ref.getObjectId()); + } + + try (TestRepository<InMemoryRepository> repository = new TestRepository<>( + (InMemoryRepository) db); + DfsInserter ins = (DfsInserter) db.getObjectDatabase() + .newInserter()) { + for (int i = 0; i < length; i++) { + RevBlob blob = repository.blob("blob" + commitCounter); + + tip = repository.branch(refname).commit().parent(tip) + .add("blob" + commitCounter, blob).create(); + commitCounter++; + co.add(new CommitObjects(tip, tip.getTree(), blob)); + } + } + return co; + } }
diff --git a/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/dfs/DfsMidxWriterTest.java b/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/dfs/DfsMidxWriterTest.java new file mode 100644 index 0000000..ac89baa --- /dev/null +++ b/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/dfs/DfsMidxWriterTest.java
@@ -0,0 +1,172 @@ +/* + * Copyright (C) 2026, Google LLC. + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Distribution License v. 1.0 which is available at + * https://www.eclipse.org/org/documents/edl-v10.php. + * + * SPDX-License-Identifier: BSD-3-Clause + */ +package org.eclipse.jgit.internal.storage.dfs; + +import static org.eclipse.jgit.internal.storage.dfs.MidxTestUtils.writeMultipackIndex; +import static org.eclipse.jgit.internal.storage.dfs.MidxTestUtils.writeSinglePackMidx; +import static org.eclipse.jgit.lib.ConfigConstants.CONFIG_KEY_BITMAP_DISTANT_COMMIT_SPAN; +import static org.eclipse.jgit.lib.ConfigConstants.CONFIG_PACK_SECTION; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import org.eclipse.jgit.internal.storage.dfs.MidxTestUtils.CommitObjects; +import org.eclipse.jgit.internal.storage.file.PackBitmapIndex; +import org.eclipse.jgit.internal.storage.file.PackReverseIndex; +import org.eclipse.jgit.lib.NullProgressMonitor; +import org.eclipse.jgit.lib.ObjectId; +import org.eclipse.jgit.revwalk.RevCommit; +import org.eclipse.jgit.revwalk.RevWalk; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.Parameterized; + +import com.googlecode.javaewah.EWAHCompressedBitmap; + +@RunWith(Parameterized.class) +public class DfsMidxWriterTest { + + @Parameterized.Parameters(name = "{0}") + public static Iterable<TestInput> data() throws Exception { + return List.of(setupOneMidxOverOnePack(), setupOneMidxOverNPacks()); + } + + private record TestInput(String testDesc, DfsRepository db, + DfsPackFileMidx midx, List<CommitObjects> commitObjects, + Map<String, CommitObjects> tips, int expectedBitmaps) { + @Override + public String toString() { + return testDesc; + } + } + + private TestInput ti; + + public DfsMidxWriterTest(TestInput ti) { + this.ti = ti; + } + + @Test + public void bitmapIndex_allObjectsHaveBitmapPosition() throws IOException { + try (DfsReader ctx = ti.db().getObjectDatabase().newReader()) { + PackBitmapIndex bi = ti.midx().getBitmapIndex(ctx); + for (int i = 0; i < ti.midx().getObjectCount(ctx); i++) { + PackReverseIndex reverseIdx = ti.midx().getReverseIdx(ctx); + // All objects in the bitmap + ObjectId oidByOffset = reverseIdx.findObjectByPosition(i); + assertEquals(i, bi.findPosition(oidByOffset)); + } + } + } + + @Test + public void bitmapIndex_bitmapHasRightObjects() throws IOException { + try (DfsReader ctx = ti.db().getObjectDatabase().newReader()) { + PackBitmapIndex bi = ti.midx().getBitmapIndex(ctx); + + ObjectId mainTip = ti.tips().get("refs/heads/main").commit(); + ObjectId devTip = ti.tips().get("refs/heads/dev").commit(); + EWAHCompressedBitmap mainBitmap = bi.getBitmap(mainTip); + EWAHCompressedBitmap devBitmap = bi.getBitmap(devTip); + + // main and dev commit chains do not have any commit in common + assertTrue(mainBitmap.and(devBitmap).isEmpty()); + assertEquals(420, ti.midx().getObjectCount(ctx)); + + RevWalk rw = new RevWalk(ti.db()); + rw.markStart(rw.parseCommit(mainTip)); + for (RevCommit c; (c = rw.next()) != null;) { + int bitmapPos = bi.findPosition(c); + assertTrue(mainBitmap.get(bitmapPos)); + assertFalse(devBitmap.get(bitmapPos)); + } + rw.reset(); + + // dev is an independent chain of commits. None of them + // should be in the bitmap of "main" + rw.markStart(rw.parseCommit(devTip)); + for (RevCommit c; (c = rw.next()) != null;) { + int bitmapPos = bi.findPosition(c); + assertTrue(devBitmap.get(bitmapPos)); + assertFalse(mainBitmap.get(bitmapPos)); + } + } + } + + static TestInput setupOneMidxOverNPacks() throws Exception { + InMemoryRepository db = new InMemoryRepository( + new DfsRepositoryDescription("one_midx_n_packs")); + db.getObjectDatabase().getReaderOptions().setUseMidxBitmaps(true); + + List<CommitObjects> mainObjs = MidxTestUtils.writeCommitChain(db, + "refs/heads/main", 100); + List<CommitObjects> devObjs = MidxTestUtils.writeCommitChain(db, + "refs/heads/dev", 40); + + Map<String, CommitObjects> tips = new HashMap<>(); + tips.put("refs/heads/main", last(mainObjs)); + tips.put("refs/heads/dev", last(devObjs)); + + List<CommitObjects> commitObjects = new ArrayList<>(160); + commitObjects.addAll(mainObjs); + commitObjects.addAll(devObjs); + + DfsPackFileMidx midx1 = writeMultipackIndex(db, + db.getObjectDatabase().getPacks(), null); + return new TestInput("one midx - n packs", db, midx1, commitObjects, + tips, tips.size()); + } + + static TestInput setupOneMidxOverOnePack() throws Exception { + InMemoryRepository db = new InMemoryRepository( + new DfsRepositoryDescription("one_midx_n_packs")); + // Mo midx bitmaps in midx over one pack. No need to set useMidxBitmaps. + enableMidxBitmaps(db); + + List<CommitObjects> mainObjs = MidxTestUtils.writeCommitChain(db, + "refs/heads/main", 100); + List<CommitObjects> devObjs = MidxTestUtils.writeCommitChain(db, + "refs/heads/dev", 40); + runGc(db); + + Map<String, CommitObjects> tips = new HashMap<>(); + tips.put("refs/heads/main", last(mainObjs)); + tips.put("refs/heads/dev", last(devObjs)); + + List<CommitObjects> commitObjects = new ArrayList<>(160); + commitObjects.addAll(mainObjs); + commitObjects.addAll(devObjs); + + DfsPackFileMidx midx1 = writeSinglePackMidx(db); + return new TestInput("one midx - one pack", db, midx1, commitObjects, + tips, 0); + } + + private static void enableMidxBitmaps(DfsRepository repo) { + repo.getConfig().setInt(CONFIG_PACK_SECTION, null, + CONFIG_KEY_BITMAP_DISTANT_COMMIT_SPAN, 1); + } + + private static void runGc(DfsRepository db) throws IOException { + DfsGarbageCollector garbageCollector = new DfsGarbageCollector(db); + garbageCollector.pack(NullProgressMonitor.INSTANCE); + assertEquals(1, garbageCollector.getNewPacks().size()); + } + + private static CommitObjects last(List<CommitObjects> l) { + return l.get(l.size() - 1); + } +}
diff --git a/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/dfs/DfsPackFileMidxNPacksTest.java b/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/dfs/DfsPackFileMidxNPacksTest.java index 706bc91..0f0e113 100644 --- a/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/dfs/DfsPackFileMidxNPacksTest.java +++ b/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/dfs/DfsPackFileMidxNPacksTest.java
@@ -12,6 +12,8 @@ import static java.nio.charset.StandardCharsets.UTF_8; import static org.eclipse.jgit.internal.storage.dfs.DfsObjDatabase.PackSource.GC; import static org.eclipse.jgit.internal.storage.pack.PackExt.PACK; +import static org.eclipse.jgit.lib.ConfigConstants.CONFIG_KEY_BITMAP_DISTANT_COMMIT_SPAN; +import static org.eclipse.jgit.lib.ConfigConstants.CONFIG_PACK_SECTION; import static org.eclipse.jgit.lib.Constants.OBJ_BLOB; import static org.eclipse.jgit.lib.Constants.OBJ_COMMIT; import static org.eclipse.jgit.lib.Constants.OBJ_TREE; @@ -50,6 +52,7 @@ import org.eclipse.jgit.lib.ObjectInserter; import org.eclipse.jgit.lib.ObjectLoader; import org.eclipse.jgit.lib.ProgressMonitor; +import org.eclipse.jgit.lib.Ref; import org.eclipse.jgit.revwalk.RevBlob; import org.eclipse.jgit.revwalk.RevCommit; import org.eclipse.jgit.revwalk.RevTree; @@ -58,6 +61,8 @@ import org.junit.Before; import org.junit.Test; +import com.googlecode.javaewah.EWAHCompressedBitmap; + public class DfsPackFileMidxNPacksTest { private static final ObjectId NOT_IN_PACK = ObjectId @@ -664,7 +669,6 @@ public void midx_getObjectType_withBase() throws Exception { ObjectId commit = writePackWithCommit(); ObjectId blob = writePackWithRandomBlob(200); writePackWithCommit(); - writePackWithCommit(); writePackWithRandomBlob(300); ObjectId newCommit = writePackWithCommit(); @@ -725,8 +729,6 @@ public void midx_getObjectSize_byId_withBase() throws Exception { writePackWithRandomBlob(300); ObjectId blobTwo = writePackWithRandomBlob(100); - System.out.println( - "pack count: " + db.getObjectDatabase().getPacks().length); DfsPackFile[] packs = db.getObjectDatabase().getPacks(); // Packs are in reverse insertion order @@ -865,7 +867,7 @@ public void midx_fillRepresentation_withBase() throws Exception { assertEquals(midxTip.getPackDescription(), rep.pack.getPackDescription()); assertEquals(midxTip.findOffset(ctx, commitInTip), rep.offset); - assertEquals(150, rep.length); + assertEquals(151, rep.length); // Commit in base midx rep = fillRepresentation(midxTip, commitInBase, OBJ_COMMIT); @@ -884,7 +886,7 @@ public void midx_fillRepresentation_withBase() throws Exception { } @Test - public void midx_getBitmapIndex() throws Exception { + public void midx_getBitmapIndex_gc() throws Exception { RevCommit c1 = writePackWithCommit(); RevCommit c2 = writePackWithCommit(); gcWithBitmaps(); @@ -903,6 +905,32 @@ public void midx_getBitmapIndex() throws Exception { } @Test + public void midx_getBitmapIndex_midx() throws Exception { + RevCommit c1 = writePackWithCommit(); + RevCommit c2 = writePackWithCommit(); + gcWithBitmaps(); + + RevCommit c3 = writePackWithCommit(); + DfsPackFileMidx dfsPackFileMidx = writeMultipackIndexWithBitmaps(); + try (DfsReader ctx = db.getObjectDatabase().newReader()) { + ctx.getOptions().setUseMidxBitmaps(true); + PackBitmapIndex bitmapIndex = dfsPackFileMidx.getBitmapIndex(ctx); + assertNotNull(bitmapIndex); + assertEquals(3, bitmapIndex.getBitmapCount()); + // Both commits have same tree and blob + assertEquals(5, bitmapIndex.getObjectCount()); + + assertNotNull(bitmapIndex.getBitmap(c3)); + assertNotNull(bitmapIndex.getBitmap(c2)); + assertNotNull(bitmapIndex.getBitmap(c1)); + + EWAHCompressedBitmap bitmapC3 = bitmapIndex.getBitmap(c3); + EWAHCompressedBitmap bitmapC2 = bitmapIndex.getBitmap(c2); + assertEquals(1, bitmapC3.andNot(bitmapC2).cardinality()); + } + } + + @Test public void midx_getAllCoveredPacks() throws Exception { writePackWithCommit(); writePackWithRandomBlob(300); @@ -915,7 +943,7 @@ public void midx_getAllCoveredPacks() throws Exception { assertEquals(4, midx.getAllCoveredPacks().size()); List<DfsPackDescription> expected = Arrays.stream(packs) - .map(p -> p.getPackDescription()).toList(); + .map(DfsPackFile::getPackDescription).toList(); List<DfsPackDescription> actual = midx.getAllCoveredPacks().stream() .map(DfsPackFile::getPackDescription).toList(); assertEquals(expected, actual); @@ -1215,6 +1243,19 @@ private DfsPackFileMidx writeMultipackIndex() throws IOException { return MidxTestUtils.writeMultipackIndex(db, packs, null); } + private DfsPackFileMidx writeMultipackIndexWithBitmaps() + throws IOException { + enableMidxBitmaps(db); + DfsPackFile[] packs = db.getObjectDatabase().getPacks(); + return MidxTestUtils.writeMultipackIndex(db, packs, + null); + } + + private static void enableMidxBitmaps(DfsRepository repo) { + repo.getConfig().setInt(CONFIG_PACK_SECTION, null, + CONFIG_KEY_BITMAP_DISTANT_COMMIT_SPAN, 1); + } + private void gcWithBitmaps() throws IOException { DfsGarbageCollector garbageCollector = new DfsGarbageCollector(db); garbageCollector.pack(NullProgressMonitor.INSTANCE); @@ -1223,7 +1264,12 @@ private void gcWithBitmaps() throws IOException { private RevCommit writePackWithCommit() throws Exception { try (TestRepository<InMemoryRepository> repository = new TestRepository<>( db)) { - return repository.branch("/refs/heads/main").commit() + Ref ref = repository.getRepository().getRefDatabase() + .findRef("refs/heads/main"); + RevWalk rw = repository.getRevWalk(); + RevCommit parent = ref != null ? rw.parseCommit(ref.getObjectId()) + : null; + return repository.branch("refs/heads/main").commit().parent(parent) .add("blob1", "blob1").create(); } }
diff --git a/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/dfs/RefAdvancerWalkTest.java b/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/dfs/RefAdvancerWalkTest.java new file mode 100644 index 0000000..321ea01 --- /dev/null +++ b/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/dfs/RefAdvancerWalkTest.java
@@ -0,0 +1,155 @@ +/* + * Copyright (C) 2026, Google LLC. + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Distribution License v. 1.0 which is available at + * https://www.eclipse.org/org/documents/edl-v10.php. + * + * SPDX-License-Identifier: BSD-3-Clause + */ +package org.eclipse.jgit.internal.storage.dfs; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +import java.io.IOException; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.stream.Collectors; + +import org.eclipse.jgit.junit.TestRepository; +import org.eclipse.jgit.lib.ObjectId; +import org.eclipse.jgit.revwalk.RevCommit; +import org.junit.Before; +import org.junit.Test; + +public class RefAdvancerWalkTest { + private static final String MAIN = "refs/heads/main"; + + private InMemoryRepository db; + + private TestRepository<InMemoryRepository> git; + + private Set<RevCommit> commitsInMidx; + + private Map<String, RevCommit> commitByLetter; + + @Before + public void setUp() throws Exception { + db = new InMemoryRepository( + new DfsRepositoryDescription("ref advance")); + git = new TestRepository<>(db); + setupRepo(); + } + + /** + * <pre> + * tipMergeBeforeMidx -> H + * | + * | + * F G <- tipStraight + * |\ | + * | \| + * tipMergeMidxCommits -> D E + * |\ | + * +--------+ + * | B C <-- tipIn + * | | / | + * | |/ | + * | A | + * | midx| + * +--------+ + * </pre> + */ + private void setupRepo() throws Exception { + RevCommit a = commitToMain(); + RevCommit b = commitToMain(); + RevCommit c = commit(a); + RevCommit d = commitToMain(c); + RevCommit e = commit(c); + /* unused */ commitToMain(); + RevCommit g = commit(e); + RevCommit h = commitToMain(); + + commitsInMidx = new HashSet<>(); + commitsInMidx.add(a); + commitsInMidx.add(b); + commitsInMidx.add(c); + + commitByLetter = Map.of("a", a, "b", b, "c", c, "d", d, "e", e, "g", g, + "h", h); + + } + + @Test + public void singleWant_linearHistory() throws Exception { + runTest("g", Set.of("c")); + } + + @Test + public void singleWant_alreadyInMidx() throws Exception { + runTest("c", Set.of("c")); + } + + @Test + public void singleWant_mergeCommitsInMidx() throws Exception { + runTest("d", Set.of("b", "c")); + } + + @Test + public void singleWant_mergeBeforeMidx() throws Exception { + runTest("h", Set.of("b", "c")); + } + + @Test + public void manyWant_mergeBeforeMidx() throws Exception { + runTest(Set.of("h", "c", "d", "g"), Set.of("b", "c")); + } + + private void runTest(String want, Set<String> expectedTips) + throws IOException { + runTest(Set.of(want), expectedTips); + } + + private void runTest(Set<String> want, Set<String> expectedTips) + throws IOException { + RefAdvancerWalk advancer = new RefAdvancerWalk(db, + commitsInMidx::contains); + List<ObjectId> wants = want.stream().map(commitByLetter::get) + .collect(Collectors.toUnmodifiableList()); + Set<RevCommit> tipsInMidx = advancer.advance(wants); + + Set<RevCommit> expected = expectedTips.stream().map(commitByLetter::get) + .collect(Collectors.toUnmodifiableSet()); + assertEquals(expected.size(), tipsInMidx.size()); + assertTrue(tipsInMidx.containsAll(expected)); + } + + private static int commitCounter = 0; + + private RevCommit commitToMain() throws Exception { + int i = commitCounter++; + return git.branch(MAIN).commit() + .add("xx" + i, git.blob("content #" + i)).create(); + } + + private RevCommit commitToMain(RevCommit... extraParent) throws Exception { + int i = commitCounter++; + TestRepository<InMemoryRepository>.CommitBuilder commit = git + .branch(MAIN).commit(); + for (RevCommit p : extraParent) { + commit.parent(p); + } + + return commit.add("xx" + i, git.blob("content #" + i)).create(); + } + + private RevCommit commit(RevCommit parent) throws Exception { + int i = commitCounter++; + return git.commit().parent(parent) + .add("cc" + i, git.blob("out of main content #" + i)).create(); + } + +}
diff --git a/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/RefDirectoryTest.java b/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/RefDirectoryTest.java index baa0182..6552bac 100644 --- a/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/RefDirectoryTest.java +++ b/org.eclipse.jgit.test/tst/org/eclipse/jgit/internal/storage/file/RefDirectoryTest.java
@@ -37,6 +37,7 @@ import java.util.concurrent.atomic.AtomicInteger; import java.util.concurrent.atomic.AtomicReference; +import org.eclipse.jgit.api.PackRefsCommand; import org.eclipse.jgit.errors.LockFailedException; import org.eclipse.jgit.events.ListenerHandle; import org.eclipse.jgit.events.RefsChangedEvent; @@ -44,6 +45,7 @@ import org.eclipse.jgit.junit.Repeat; import org.eclipse.jgit.junit.TestRepository; import org.eclipse.jgit.lib.AnyObjectId; +import org.eclipse.jgit.lib.Constants; import org.eclipse.jgit.lib.Ref; import org.eclipse.jgit.lib.Ref.Storage; import org.eclipse.jgit.lib.RefDatabase; @@ -1062,6 +1064,60 @@ public void test_repack() throws Exception { } @Test + public void testPackedRefsHeaderWithSorted() throws Exception { + writeLooseRef("refs/heads/master", A); + writeLooseRef("refs/heads/other", B); + writeLooseRef("refs/tags/v1.0", v1_0); + + PackRefsCommand packRefsCommand = new PackRefsCommand(diskRepo); + packRefsCommand.setAll(true); + packRefsCommand.call(); + + File packedRefsFile = new File(diskRepo.getCommonDirectory(), Constants.PACKED_REFS); + assertTrue("packed-refs should exist", packedRefsFile.exists()); + + String content = read(packedRefsFile); + String firstLine = content.split("\n")[0]; + assertTrue("packed-refs should have header with sorted", + firstLine.contains(" sorted")); + + int masterIndex = content.indexOf(A.name() + " refs/heads/master"); + int otherIndex = content.indexOf(B.name() + " refs/heads/other"); + int tagIndex = content.indexOf(v1_0.name() + " refs/tags/v1.0"); + assertTrue("packed-refs should be sorted", + masterIndex < otherIndex && otherIndex < tagIndex); + } + + @Test + public void testPackedRefsUnsortedGetsSorted() throws Exception { + writePackedRefs("# pack-refs with: peeled \n" + // + B.name() + " refs/heads/other\n" + // + v1_0.name() + " refs/tags/v1.0\n" + // + "^" + v1_0.getObject().name() + "\n" + // + A.name() + " refs/heads/master\n"); + + // extra loose-ref to trigger packing + writeLooseRef("refs/heads/loose", A); + + PackRefsCommand packRefsCommand = new PackRefsCommand(diskRepo); + packRefsCommand.setAll(true); + packRefsCommand.call(); + + File packedRefsFile = new File(diskRepo.getCommonDirectory(), Constants.PACKED_REFS); + String content = read(packedRefsFile); + int looseIndex = content.indexOf(v1_0.name() + " refs/tags/loose"); + int masterIndex = content.indexOf(A.name() + " refs/heads/master"); + int otherIndex = content.indexOf(B.name() + " refs/heads/other"); + int tagIndex = content.indexOf(v1_0.name() + " refs/tags/v1.0"); + assertTrue( + "packed-refs should be sorted", + looseIndex < masterIndex && + masterIndex < otherIndex && + otherIndex < tagIndex + ); + } + + @Test public void testFindRef_EmptyDatabase() throws IOException { Ref r;
diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsMidxWriter.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsMidxWriter.java index 8302694..654bc30 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsMidxWriter.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsMidxWriter.java
@@ -15,15 +15,27 @@ import static org.eclipse.jgit.internal.storage.pack.PackExt.MULTI_PACK_INDEX; import java.io.IOException; +import java.util.ArrayList; +import java.util.HashSet; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; +import java.util.Objects; +import java.util.Set; import java.util.function.Function; import org.eclipse.jgit.annotations.Nullable; +import org.eclipse.jgit.internal.storage.file.PackBitmapIndexBuilder; import org.eclipse.jgit.internal.storage.file.PackIndex; import org.eclipse.jgit.internal.storage.midx.MultiPackIndexWriter; +import org.eclipse.jgit.internal.storage.pack.PackBitmapCalculator; +import org.eclipse.jgit.internal.storage.pack.PackBitmapIndexWriter; +import org.eclipse.jgit.lib.Constants; +import org.eclipse.jgit.lib.NullProgressMonitor; +import org.eclipse.jgit.lib.ObjectId; import org.eclipse.jgit.lib.ProgressMonitor; +import org.eclipse.jgit.revwalk.RevCommit; +import org.eclipse.jgit.storage.pack.PackConfig; /** * Create a pack with a multipack index, setting the required fields in the @@ -35,6 +47,30 @@ private DfsMidxWriter() { } /** + * Create a pack with the multipack index (without bitmaps). + * + * @param pm + * a progress monitor + * @param objdb + * an object database + * @param packs + * the packs to cover + * @param base + * parent of this midx in the chain (if any). + * + * @return a pack (uncommitted) with the multipack index of the packs passed + * as parameter. + * @throws IOException + * an error opening the packs or writing the stream. + */ + public static DfsPackDescription writeMidx(ProgressMonitor pm, + DfsObjDatabase objdb, List<DfsPackFile> packs, + @Nullable DfsPackDescription base) throws IOException { + return writeMidx(pm, objdb, packs, base, + new PackConfig(objdb.getRepository())); + } + + /** * Create a pack with the multipack index * * @param pm @@ -45,6 +81,8 @@ private DfsMidxWriter() { * the packs to cover * @param base * parent of this midx in the chain (if any). + * @param packConfig + * pack config with the parameters to write bitmaps. * @return a pack (uncommitted) with the multipack index of the packs passed * as parameter. * @throws IOException @@ -52,7 +90,8 @@ private DfsMidxWriter() { */ public static DfsPackDescription writeMidx(ProgressMonitor pm, DfsObjDatabase objdb, List<DfsPackFile> packs, - @Nullable DfsPackDescription base) throws IOException { + @Nullable DfsPackDescription base, PackConfig packConfig) + throws IOException { LinkedHashMap<String, PackIndex> inputs = new LinkedHashMap<>( packs.size()); try (DfsReader ctx = objdb.newReader()) { @@ -83,6 +122,52 @@ public static DfsPackDescription writeMidx(ProgressMonitor pm, } } + // TODO(ifrade): At the moment write bitmaps only in the bottom midx. + // A single-pack midx in the base should be covering only GC. No + // need to write midx bitmaps (we will use GC bitmaps). + if (base == null && midxPackDesc.getCoveredPacks().size() > 1) { + createAndAttachBitmaps(objdb.getRepository(), midxPackDesc, + packConfig); + } + return midxPackDesc; } + + private static void createAndAttachBitmaps(DfsRepository db, + DfsPackDescription desc, PackConfig cfg) throws IOException { + + DfsObjDatabase objdb = db.getObjectDatabase(); + // We need a DfsPackFile to reread the contents + DfsPackFileMidx midxPack = db.getObjectDatabase().createDfsPackFileMidx( + DfsBlockCache.getInstance(), desc, new ArrayList<>()); + + // TODO(ifrade): Verify we duplicate the behaviour about tags of regular + // bitmapping + List<ObjectId> allHeads = db.getRefDatabase() + .getRefsByPrefix(Constants.R_HEADS).stream() + .map(r -> r.getObjectId()).filter(Objects::nonNull).toList(); + if (allHeads.isEmpty()) { + return; + } + + try (DfsReader ctx = objdb.newReader()) { + RefAdvancerWalk adv = new RefAdvancerWalk(db, + c -> midxPack.hasObject(ctx, c)); + Set<RevCommit> inPack = adv.advance(allHeads); + + byte[] checksum = midxPack.getChecksum(ctx); + PackBitmapIndexBuilder writeBitmaps = new PackBitmapIndexBuilder( + midxPack.getLocalObjects(ctx)); + int commitCount = writeBitmaps.getCommits().cardinality(); + + PackBitmapCalculator calculator = new PackBitmapCalculator(cfg); + // This will do ctx.getBitmapIndex() to reuse/copy previous bitmaps + calculator.calculate(ctx, NullProgressMonitor.INSTANCE, commitCount, + inPack, new HashSet<>(), writeBitmaps); + PackBitmapIndexWriter pbiWriter = db.getObjectDatabase() + .getPackBitmapIndexWriter(desc); + pbiWriter.write(writeBitmaps, checksum); + } + } + }
diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsPackFileMidxSingle.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsPackFileMidxSingle.java index 188cb81..71ff884 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsPackFileMidxSingle.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/DfsPackFileMidxSingle.java
@@ -407,11 +407,6 @@ public long getMaxOffset() { */ private static class LocalPackOffset extends PackOffset { - LocalPackOffset() { - super(); - setValues(0, 0); - } - void setOffset(long offset) { super.setValues(0, offset); }
diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/MidxPackFilter.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/MidxPackFilter.java index 1183306..670950a 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/MidxPackFilter.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/MidxPackFilter.java
@@ -82,10 +82,6 @@ public static List<DfsPackDescription> useMidx( List<DfsPackDescription> midxs = packs.stream() .filter(desc -> desc.hasFileExt(PackExt.MULTI_PACK_INDEX)) .sorted(midxComparator).toList(); - for (DfsPackDescription d : midxs) { - System.out.println(String.format(" %s - %d - %d", d.getPackName(), - d.getLastModified(), getTotalCoveredObjects(d))); - } if (midxs.isEmpty()) { return packs; }
diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/RefAdvancerWalk.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/RefAdvancerWalk.java new file mode 100644 index 0000000..d0d8056 --- /dev/null +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/dfs/RefAdvancerWalk.java
@@ -0,0 +1,112 @@ +/* + * Copyright (C) 2026, Google LLC. + * + * This program and the accompanying materials are made available under the + * terms of the Eclipse Distribution License v. 1.0 which is available at + * https://www.eclipse.org/org/documents/edl-v10.php. + * + * SPDX-License-Identifier: BSD-3-Clause + */ +package org.eclipse.jgit.internal.storage.dfs; + +import java.io.IOException; +import java.util.HashSet; +import java.util.List; +import java.util.Set; + +import org.eclipse.jgit.errors.StopWalkException; +import org.eclipse.jgit.lib.ObjectId; +import org.eclipse.jgit.revwalk.RevCommit; +import org.eclipse.jgit.revwalk.RevObject; +import org.eclipse.jgit.revwalk.RevSort; +import org.eclipse.jgit.revwalk.RevWalk; +import org.eclipse.jgit.revwalk.filter.RevFilter; + +/** + * Walk from some commits and find where to they enter a pack + */ +class RefAdvancerWalk { + + private final DfsRepository db; + + private final InPackPredicate includeP; + + /** + * True when the commit is in the pack + */ + @FunctionalInterface + interface InPackPredicate { + boolean test(RevCommit c) throws IOException; + } + + RefAdvancerWalk(DfsRepository db, InPackPredicate include) { + this.db = db; + this.includeP = include; + } + + private RevWalk createRevWalk() { + RevWalk rw = new RevWalk(db); + rw.sort(RevSort.COMMIT_TIME_DESC); + rw.setRevFilter(new FirstInPack(includeP)); + return rw; + } + + /** + * Advance the tips to their first commit inside the pack + * + * @param allTips + * tips of interesting refs + * @return first commit(s) where the tips enter the pack. A tips may + * translate into 0 commits (it doesn't enter the pack in its + * history), 1 commit (a linear history) or n commits (merges lead + * to multiple histories into the pack). A tip already inside the + * pack is returned as it is. + * @throws IOException + * error browsing history + */ + Set<RevCommit> advance(List<ObjectId> allTips) throws IOException { + Set<RevCommit> tipsInMidx = new HashSet<>(allTips.size()); + try (RevWalk rw = createRevWalk()) { + for (ObjectId tip : allTips) { + RevObject tipObject = rw.parseAny(tip); + if (!(tipObject instanceof RevCommit tipCommit)) { + continue; + } + + rw.markStart(tipCommit); + RevCommit inPack; + while ((inPack = rw.next()) != null) { + tipsInMidx.add(inPack); + } + } + } + return tipsInMidx; + } + + private static class FirstInPack extends RevFilter { + + private final InPackPredicate isInPack; + + FirstInPack(InPackPredicate isInPack) { + this.isInPack = isInPack; + } + + @Override + public boolean include(RevWalk walker, RevCommit cmit) + throws StopWalkException, IOException { + if (!isInPack.test(cmit)) { + return false; + } + + for (RevCommit p : cmit.getParents()) { + walker.markUninteresting(p); + } + return true; + } + + @Override + public RevFilter clone() { + return this; + } + } +}
diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/RefDirectory.java b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/RefDirectory.java index 9c262e9..9fa3ff3 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/RefDirectory.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/internal/storage/file/RefDirectory.java
@@ -121,6 +121,9 @@ public class RefDirectory extends RefDatabase { /** If in the header, denotes the file has peeled data. */ public static final String PACKED_REFS_PEELED = " peeled"; //$NON-NLS-1$ + /** If in the header, denotes the file has sorted data. */ + public static final String PACKED_REFS_SORTED = " sorted"; //$NON-NLS-1$ + @SuppressWarnings("boxing") private static final List<Integer> RETRY_SLEEP_MS = Collections.unmodifiableList(Arrays.asList(0, 100, 200, 400, 800, 1600));
diff --git a/org.eclipse.jgit/src/org/eclipse/jgit/lib/RefWriter.java b/org.eclipse.jgit/src/org/eclipse/jgit/lib/RefWriter.java index 41917f8..58aed82 100644 --- a/org.eclipse.jgit/src/org/eclipse/jgit/lib/RefWriter.java +++ b/org.eclipse.jgit/src/org/eclipse/jgit/lib/RefWriter.java
@@ -139,6 +139,7 @@ public void writePackedRefs() throws IOException { if (peeled) { w.write(RefDirectory.PACKED_REFS_HEADER); w.write(RefDirectory.PACKED_REFS_PEELED); + w.write(RefDirectory.PACKED_REFS_SORTED); w.write('\n'); }