From 37947ad9352b40dd0a44927ab774f346c841b069 Mon Sep 17 00:00:00 2001 From: Bartosz Fidrysiak <300359074+bartoszfidrysiak@users.noreply.github.com> Date: Mon, 6 Jul 2026 19:52:11 +0200 Subject: [PATCH 1/5] bugfix: lazy string loading while initializing new collapse group & ordinal fast path comparison for string sorted doc values from the same segment --- .../solr/search/CollapsingQParserPlugin.java | 52 ++++++- .../apache/solr/search/LazyStringValue.java | 41 +++++ .../search/TestCollapseQParserPlugin.java | 146 ++++++++++++++++++ 3 files changed, 236 insertions(+), 3 deletions(-) create mode 100644 solr/core/src/java/org/apache/solr/search/LazyStringValue.java diff --git a/solr/core/src/java/org/apache/solr/search/CollapsingQParserPlugin.java b/solr/core/src/java/org/apache/solr/search/CollapsingQParserPlugin.java index ae3102e41ced..8e616e49d397 100644 --- a/solr/core/src/java/org/apache/solr/search/CollapsingQParserPlugin.java +++ b/solr/core/src/java/org/apache/solr/search/CollapsingQParserPlugin.java @@ -3565,6 +3565,9 @@ private static class SortFieldsCompare { private Object[][] groupHeadValues; // growable private final Object[] nullGroupValues; + private final SortedDocValues[] stringSortDVs; + private final int[] stringMissingOrd; + /** * Constructs an instance based on the (raw, un-rewritten) SortFields to be used, and an initial * number of expected groups (will grow as needed). @@ -3576,6 +3579,7 @@ public SortFieldsCompare(SortField[] sorts, int initNumGroups) { fieldComparators = new FieldComparator[numClauses]; leafFieldComparators = new LeafFieldComparator[numClauses]; reverseMul = new int[numClauses]; + stringMissingOrd = new int[numClauses]; for (int clause = 0; clause < numClauses; clause++) { SortField sf = sorts[clause]; // we only need one slot for every comparator @@ -3587,14 +3591,28 @@ public SortFieldsCompare(SortField[] sorts, int initNumGroups) { : Pruning.NONE); reverseMul[clause] = sf.getReverse() ? -1 : 1; + if (sf.getType() == SortField.Type.STRING) { + stringMissingOrd[clause] = + (sf.getMissingValue() == SortField.STRING_LAST) ? Integer.MAX_VALUE : -1; + } } groupHeadValues = new Object[initNumGroups][]; nullGroupValues = new Object[numClauses]; + stringSortDVs = new SortedDocValues[numClauses]; // populated in setNextReader } public void setNextReader(LeafReaderContext context) throws IOException { for (int clause = 0; clause < numClauses; clause++) { leafFieldComparators[clause] = fieldComparators[clause].getLeafComparator(context); + if (sorts[clause].getType() == SortField.Type.STRING) { + String field = sorts[clause].getField(); + FieldInfo fi = context.reader().getFieldInfos().fieldInfo(field); + if (fi != null && fi.getDocValuesType() == DocValuesType.SORTED) { + stringSortDVs[clause] = DocValues.getSorted(context.reader(), field); + } else { + stringSortDVs[clause] = null; + } + } } } @@ -3652,12 +3670,20 @@ public void setNullGroupValues(int contextDoc) throws IOException { /** * Records the SortField values for the specified contextDoc into the values array provided by - * the caller. + * the caller. STRING clauses with SORTED DocValues are stored as {@link LazyStringValue} to + * defer {@code lookupOrd()} until a second document actually competes for the group. */ private void setGroupValues(Object[] values, int contextDoc) throws IOException { for (int clause = 0; clause < numClauses; clause++) { - leafFieldComparators[clause].copy(0, contextDoc); - values[clause] = cloneIfBytesRef(fieldComparators[clause].value(0)); + if (stringSortDVs[clause] != null) { + SortedDocValues dv = stringSortDVs[clause]; + int missingOrd = stringMissingOrd[clause]; + int ord = dv.advanceExact(contextDoc) ? dv.ordValue() : missingOrd; + values[clause] = new LazyStringValue(dv, ord, missingOrd); + } else { + leafFieldComparators[clause].copy(0, contextDoc); + values[clause] = cloneIfBytesRef(fieldComparators[clause].value(0)); + } } } @@ -3696,6 +3722,19 @@ private boolean testAndSetGroupValues(Object[] values, int contextDoc) throws IO int testClause = 0; for ( /* testClause */ ; testClause < numClauses; testClause++) { + if (values[testClause] instanceof LazyStringValue) { + LazyStringValue headVal = (LazyStringValue) values[testClause]; + SortedDocValues segDV = stringSortDVs[testClause]; + if (segDV != null && segDV == headVal.dv) { + int missingOrd = headVal.missingOrd; + int candidateOrd = segDV.advanceExact(contextDoc) ? segDV.ordValue() : missingOrd; + lastCompare = reverseMul[testClause] * Integer.compare(candidateOrd, headVal.ord); + stash[testClause] = new LazyStringValue(segDV, candidateOrd, missingOrd); + if (0 != lastCompare) break; + continue; + } + values[testClause] = headVal.materialize(); + } leafFieldComparators[testClause].copy(0, contextDoc); FieldComparator fcomp = fieldComparators[testClause]; stash[testClause] = cloneIfBytesRef(fcomp.value(0)); @@ -3719,6 +3758,13 @@ private boolean testAndSetGroupValues(Object[] values, int contextDoc) throws IO System.arraycopy(stash, 0, values, 0, testClause); // read the remaining values we didn't need to test for (int copyClause = testClause; copyClause < numClauses; copyClause++) { + SortedDocValues segDV = stringSortDVs[copyClause]; + if (segDV != null) { + int missingOrd = stringMissingOrd[copyClause]; + int candidateOrd = segDV.advanceExact(contextDoc) ? segDV.ordValue() : missingOrd; + values[copyClause] = new LazyStringValue(segDV, candidateOrd, missingOrd); + continue; + } leafFieldComparators[copyClause].copy(0, contextDoc); values[copyClause] = cloneIfBytesRef(fieldComparators[copyClause].value(0)); } diff --git a/solr/core/src/java/org/apache/solr/search/LazyStringValue.java b/solr/core/src/java/org/apache/solr/search/LazyStringValue.java new file mode 100644 index 000000000000..3b4c7aae18a8 --- /dev/null +++ b/solr/core/src/java/org/apache/solr/search/LazyStringValue.java @@ -0,0 +1,41 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.solr.search; + +import java.io.IOException; +import org.apache.lucene.index.SortedDocValues; +import org.apache.lucene.util.BytesRef; + +final class LazyStringValue { + + final SortedDocValues dv; + final int ord; + final int missingOrd; + + LazyStringValue(SortedDocValues dv, int ord, int missingOrd) { + this.dv = dv; + this.ord = ord; + this.missingOrd = missingOrd; + } + + BytesRef materialize() throws IOException { + if (ord == missingOrd || ord < 0) { + return null; + } + return dv.lookupOrd(ord); + } +} diff --git a/solr/core/src/test/org/apache/solr/search/TestCollapseQParserPlugin.java b/solr/core/src/test/org/apache/solr/search/TestCollapseQParserPlugin.java index 716a4a129bc4..7c193f87a735 100644 --- a/solr/core/src/test/org/apache/solr/search/TestCollapseQParserPlugin.java +++ b/solr/core/src/test/org/apache/solr/search/TestCollapseQParserPlugin.java @@ -1578,4 +1578,150 @@ public void testNullGroupNumericVsStringCollapse() { } } } + + @Test + public void testCollapseStringSortLazyLoadingTieDoesNotEvictGroupHead() { + // Group A: id=1 and id=2 both have term_s_dv=AAA → tie on the sort field. + // Group B: id=3 (BBB) and id=4 (CCC) → no tie, id=3 is the winner on asc. + assertU(adoc("id", "1", "group_s_dv", "A", "term_s_dv", "AAA")); + assertU(adoc("id", "2", "group_s_dv", "A", "term_s_dv", "AAA")); + assertU(adoc("id", "3", "group_s_dv", "B", "term_s_dv", "BBB")); + assertU(adoc("id", "4", "group_s_dv", "B", "term_s_dv", "CCC")); + assertU(commit()); + + // Single segment: ordinal fast path, tie → first doc (id=1) stays as group head. + assertQ( + req( + "q", "*:*", + "fq", "{!collapse field=group_s_dv sort='term_s_dv asc'}", + "sort", "id_i asc", + "fl", "id"), + "*[count(//doc)=2]", + "//result/doc[1]/str[@name='id'][.='1']", + "//result/doc[2]/str[@name='id'][.='3']"); + + // Multi-segment: slow path, same tie expectation. + assertU(adoc("id", "5", "group_s_dv", "A", "term_s_dv", "AAA")); + assertU(commit()); + + assertQ( + req( + "q", "*:*", + "fq", "{!collapse field=group_s_dv sort='term_s_dv asc'}", + "sort", "id_i asc", + "fl", "id"), + "*[count(//doc)=2]", + "//result/doc[1]/str[@name='id'][.='1']", + "//result/doc[2]/str[@name='id'][.='3']"); + } + + @Test + public void testCollapseStringSortOrdinalFastPathMultiClauseTieBreaking() { + // Group A: id=1 (AAA, ZZZ) vs id=2 (AAA, MMM) → clause-1 ties (both AAA), + // clause-2 decides: MMM < ZZZ → id=2 wins on 'term_s_dv asc, term2_s_dv asc'. + // Group B: id=3 (BBB, X) vs id=4 (CCC, X) → clause-1 decides, no tie. + assertU(adoc("id", "1", "group_s_dv", "A", "term_s_dv", "AAA", "term2_s_dv", "ZZZ")); + assertU(adoc("id", "2", "group_s_dv", "A", "term_s_dv", "AAA", "term2_s_dv", "MMM")); + assertU(adoc("id", "3", "group_s_dv", "B", "term_s_dv", "BBB", "term2_s_dv", "X")); + assertU(adoc("id", "4", "group_s_dv", "B", "term_s_dv", "CCC", "term2_s_dv", "X")); + assertU(commit()); + + // Single segment: ordinal fast path on clause-1 (tie), then clause-2 decides. + assertQ( + req( + "q", "*:*", + "fq", "{!collapse field=group_s_dv sort='term_s_dv asc,term2_s_dv asc'}", + "sort", "id_i asc", + "fl", "id"), + "*[count(//doc)=2]", + "//result/doc[1]/str[@name='id'][.='2']", + "//result/doc[2]/str[@name='id'][.='3']"); + + // Add a cross-segment competitor for group A to exercise the slow path too. + assertU(adoc("id", "5", "group_s_dv", "A", "term_s_dv", "AAA", "term2_s_dv", "AAA")); + assertU(commit()); + + // id=5 (AAA, AAA) beats id=2 (AAA, MMM) because AAA < MMM on clause-2. + assertQ( + req( + "q", "*:*", + "fq", "{!collapse field=group_s_dv sort='term_s_dv asc,term2_s_dv asc'}", + "sort", "id_i asc", + "fl", "id"), + "*[count(//doc)=2]", + "//result/doc[1]/str[@name='id'][.='3']", + "//result/doc[2]/str[@name='id'][.='5']"); + } + + @Test + public void testCollapseStringSortWithoutDocValuesSkipsLazyLoadingAndOrdinalFastPath() { + // term_s is a *_s field: indexed but no docValues → stringSortDVs[0] will be null. + // Group A: id=1 (ZZZ) vs id=2 (AAA) → id=2 wins on asc. + // Group B: id=3 (MMM) vs id=4 (BBB) → id=4 wins on asc. + assertU(adoc("id", "1", "group_s_dv", "A", "term_s", "ZZZ")); + assertU(adoc("id", "2", "group_s_dv", "A", "term_s", "AAA")); + assertU(adoc("id", "3", "group_s_dv", "B", "term_s", "MMM")); + assertU(adoc("id", "4", "group_s_dv", "B", "term_s", "BBB")); + assertU(commit()); + assertU(adoc("id", "5", "group_s_dv", "A", "term_s", "BBB")); + assertU(commit()); + + assertQ( + req( + "q", "*:*", + "fq", "{!collapse field=group_s_dv sort='term_s asc'}", + "sort", "id_i asc", + "fl", "id"), + "*[count(//doc)=2]", + "//result/doc[1]/str[@name='id'][.='2']", + "//result/doc[2]/str[@name='id'][.='4']"); + + assertQ( + req( + "q", "*:*", + "fq", "{!collapse field=group_s_dv sort='term_s desc'}", + "sort", "id_i asc", + "fl", "id"), + "*[count(//doc)=2]", + "//result/doc[1]/str[@name='id'][.='1']", + "//result/doc[2]/str[@name='id'][.='3']"); + } + + @Test + public void testCollapseStringSortOrdinalFastPathDescendingWithMissingValues() { + // Group A: id=1 (ZZZ), id=2 (AAA), id=3 (no term_s_dv → missing, sorts last). + // On 'term_s_dv desc': ZZZ > AAA > missing → winner is id=1. + // Group B: id=4 (MMM), id=5 (no term_s_dv → missing). + // On 'term_s_dv desc': MMM > missing → winner is id=4. + assertU(adoc("id", "1", "group_s_dv", "A", "term_s_dv", "ZZZ")); + assertU(adoc("id", "4", "group_s_dv", "B", "term_s_dv", "MMM")); + assertU(commit()); + assertU(adoc("id", "2", "group_s_dv", "A", "term_s_dv", "AAA")); + assertU(adoc("id", "5", "group_s_dv", "B")); + assertU(commit()); + assertU(adoc("id", "3", "group_s_dv", "A")); + assertU(commit()); + + assertQ( + req( + "q", "*:*", + "fq", "{!collapse field=group_s_dv sort='term_s_dv desc'}", + "sort", "id_i asc", + "fl", "id"), + "*[count(//doc)=2]", + "//result/doc[1]/str[@name='id'][.='1']", + "//result/doc[2]/str[@name='id'][.='4']"); + + assertU(optimize("maxSegments", "1")); + + assertQ( + req( + "q", "*:*", + "fq", "{!collapse field=group_s_dv sort='term_s_dv desc'}", + "sort", "id_i asc", + "fl", "id"), + "*[count(//doc)=2]", + "//result/doc[1]/str[@name='id'][.='1']", + "//result/doc[2]/str[@name='id'][.='4']"); + } } From bc12e530b81f9e88a114fdb1af59702e052c431b Mon Sep 17 00:00:00 2001 From: Bartosz Fidrysiak <300359074+bartoszfidrysiak@users.noreply.github.com> Date: Mon, 6 Jul 2026 19:52:11 +0200 Subject: [PATCH 2/5] Added CollapsingSearch benchmark --- .../solr/bench/search/CollapsingSearch.java | 229 ++++++++++++++++++ .../configs/cloud-minimal/conf/schema.xml | 1 + 2 files changed, 230 insertions(+) create mode 100644 solr/benchmark/src/java/org/apache/solr/bench/search/CollapsingSearch.java diff --git a/solr/benchmark/src/java/org/apache/solr/bench/search/CollapsingSearch.java b/solr/benchmark/src/java/org/apache/solr/bench/search/CollapsingSearch.java new file mode 100644 index 000000000000..edbee0a8ecb6 --- /dev/null +++ b/solr/benchmark/src/java/org/apache/solr/bench/search/CollapsingSearch.java @@ -0,0 +1,229 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.solr.bench.search; + +import org.apache.solr.bench.BaseBenchState; +import org.apache.solr.bench.MiniClusterState; +import org.apache.solr.client.solrj.SolrQuery; +import org.apache.solr.client.solrj.SolrServerException; +import org.apache.solr.client.solrj.request.CollectionAdminRequest; +import org.apache.solr.client.solrj.request.QueryRequest; +import org.apache.solr.client.solrj.request.UpdateRequest; +import org.apache.solr.common.SolrInputDocument; +import org.apache.solr.common.params.CommonParams; +import org.openjdk.jmh.annotations.Benchmark; +import org.openjdk.jmh.annotations.BenchmarkMode; +import org.openjdk.jmh.annotations.Fork; +import org.openjdk.jmh.annotations.Level; +import org.openjdk.jmh.annotations.Measurement; +import org.openjdk.jmh.annotations.Mode; +import org.openjdk.jmh.annotations.OutputTimeUnit; +import org.openjdk.jmh.annotations.Param; +import org.openjdk.jmh.annotations.Scope; +import org.openjdk.jmh.annotations.Setup; +import org.openjdk.jmh.annotations.State; +import org.openjdk.jmh.annotations.TearDown; +import org.openjdk.jmh.annotations.Threads; +import org.openjdk.jmh.annotations.Warmup; + +import java.io.IOException; +import java.util.UUID; +import java.util.concurrent.TimeUnit; + +@Fork(value = 1) +@Warmup(time = 5, iterations = 5) +@Measurement(time = 15, iterations = 5) +@BenchmarkMode(Mode.AverageTime) +@OutputTimeUnit(TimeUnit.MILLISECONDS) +@Threads(value = 1) +public class CollapsingSearch { + + static final String COLLECTION = "collapse_bench"; + + @State(Scope.Benchmark) + public static class BenchState { + + @Param({"2000000"}) + int numDocs; + + @Param({"100000"}) + int numGroups; + + @Param({"1", "10"}) + int numSegments; + + final QueryRequest qSimple = + new QueryRequest(new SolrQuery("q", "*:*", "rows", "10", "cache", "false")); + + final QueryRequest qCollapseWithoutSort = + new QueryRequest( + new SolrQuery( + "q", "*:*", + "fq", "{!collapse field=group_s_dv cache=false}", + "rows", "10", + "cache", "false")); + + final QueryRequest qCollapseByStr = + new QueryRequest( + new SolrQuery( + "q", "*:*", + "fq", "{!collapse field=group_s_dv sort='str_s_dv asc' cache=false}", + "rows", "10", + "cache", "false")); + + final QueryRequest qCollapseByDate = + new QueryRequest( + new SolrQuery( + "q", "*:*", + "fq", "{!collapse field=group_s_dv sort='date_dt_dv asc' cache=false}", + "rows", "10", + "cache", "false")); + + final QueryRequest qCollapseByLong = + new QueryRequest( + new SolrQuery( + "q", "*:*", + "fq", "{!collapse field=group_s_dv sort='long_l_dv asc' cache=false}", + "rows", "10", + "cache", "false")); + + final QueryRequest qCollapseByDateAndStr = + new QueryRequest( + new SolrQuery( + "q", "*:*", + "fq", "{!collapse field=group_s_dv sort='date_dt_dv asc, str_s_dv asc' cache=false}", + "rows", "10", + "cache", "false")); + + @Setup(Level.Trial) + public void setupTrial(MiniClusterState.MiniClusterBenchState miniClusterState) + throws Exception { + System.setProperty("commitwithin.softcommit", "false"); + miniClusterState.startMiniCluster(1); + miniClusterState.createCollection(COLLECTION, 1, 1); + + indexDocs(miniClusterState); + miniClusterState.forceMerge(COLLECTION, numSegments); + + int actualSegments = getActualSegmentCount(miniClusterState); + BaseBenchState.log( + "CollapsingSearch ready: numDocs=" + + numDocs + + " numGroups=" + + numGroups + + " numSegments(requested)=" + + numSegments + + " numSegments(actual)=" + + actualSegments); + } + + @Setup(Level.Iteration) + public void setupIteration(MiniClusterState.MiniClusterBenchState miniClusterState) + throws SolrServerException, IOException { + CollectionAdminRequest.Reload reload = CollectionAdminRequest.reloadCollection(COLLECTION); + miniClusterState.client.request(reload); + } + + @TearDown(Level.Trial) + public void teardown(MiniClusterState.MiniClusterBenchState miniClusterState) + throws Exception { + CollectionAdminRequest.deleteCollection(COLLECTION) + .process(miniClusterState.client); + } + + private int pickGroup(int docId) { + return docId % numGroups; + } + + private int getActualSegmentCount(MiniClusterState.MiniClusterBenchState state) + throws SolrServerException, IOException { + SolrQuery lukeQuery = new SolrQuery(); + lukeQuery.set(CommonParams.QT, "/admin/luke"); + lukeQuery.set("show", "index"); + var resp = state.client.query(COLLECTION, lukeQuery); + Object segCount = resp.getResponse().findRecursive("index", "segmentCount"); + return segCount instanceof Number ? ((Number) segCount).intValue() : -1; + } + + private void indexDocs(MiniClusterState.MiniClusterBenchState state) throws Exception { + int docsPerSegment = numDocs / numSegments; + String[] groupIds = new String[numGroups]; + for (int i = 0; i < numGroups; i++) { + groupIds[i] = String.format("group_%06d", i); + } + java.util.Random rng = new java.util.Random(42); + java.time.Instant baseDate = java.time.Instant.parse("2020-01-01T00:00:00Z"); + + int docId = 0; + for (int seg = 0; seg < numSegments; seg++) { + int segDocs = (seg < numSegments - 1) ? docsPerSegment : (numDocs - docId); + UpdateRequest req = new UpdateRequest(); + for (int i = 0; i < segDocs; i++) { + SolrInputDocument doc = new SolrInputDocument(); + doc.addField("id", String.valueOf(docId)); + doc.addField("group_s_dv", groupIds[pickGroup(docId)]); + docId++; + doc.addField("str_s_dv", UUID.randomUUID().toString()); + doc.addField( + "date_dt_dv", + java.time.format.DateTimeFormatter.ISO_INSTANT.format( + baseDate.plusSeconds(rng.nextInt(10_000_000)))); + doc.addField("long_l_dv", rng.nextInt(10_000_000)); + req.add(doc); + } + req.commit(state.client, COLLECTION); + } + } + } + + @Benchmark + public Object simple(BenchState b, MiniClusterState.MiniClusterBenchState cluster) + throws SolrServerException, IOException { + return cluster.client.request(b.qSimple, COLLECTION); + } + + @Benchmark + public Object collapseWithoutSort(BenchState b, MiniClusterState.MiniClusterBenchState cluster) + throws SolrServerException, IOException { + return cluster.client.request(b.qCollapseWithoutSort, COLLECTION); + } + + @Benchmark + public Object collapseByStr(BenchState b, MiniClusterState.MiniClusterBenchState cluster) + throws SolrServerException, IOException { + return cluster.client.request(b.qCollapseByStr, COLLECTION); + } + + @Benchmark + public Object collapseByDate(BenchState b, MiniClusterState.MiniClusterBenchState cluster) + throws SolrServerException, IOException { + return cluster.client.request(b.qCollapseByDate, COLLECTION); + } + + @Benchmark + public Object collapseByLong(BenchState b, MiniClusterState.MiniClusterBenchState cluster) + throws SolrServerException, IOException { + return cluster.client.request(b.qCollapseByLong, COLLECTION); + } + + @Benchmark + public Object collapseByDateAndStr(BenchState b, MiniClusterState.MiniClusterBenchState cluster) + throws SolrServerException, IOException { + return cluster.client.request(b.qCollapseByDateAndStr, COLLECTION); + } + +} diff --git a/solr/benchmark/src/resources/configs/cloud-minimal/conf/schema.xml b/solr/benchmark/src/resources/configs/cloud-minimal/conf/schema.xml index b1851c3c2906..e3f77a826047 100644 --- a/solr/benchmark/src/resources/configs/cloud-minimal/conf/schema.xml +++ b/solr/benchmark/src/resources/configs/cloud-minimal/conf/schema.xml @@ -56,6 +56,7 @@ + id From 45457c664843ca5897c748f618b7ab516c88415c Mon Sep 17 00:00:00 2001 From: Bartosz Fidrysiak <300359074+bartoszfidrysiak@users.noreply.github.com> Date: Mon, 6 Jul 2026 19:52:12 +0200 Subject: [PATCH 3/5] Fixed issues in CollapsingSearch: forbidden APIs & code reformatting --- .../solr/bench/search/CollapsingSearch.java | 334 +++++++++--------- 1 file changed, 167 insertions(+), 167 deletions(-) diff --git a/solr/benchmark/src/java/org/apache/solr/bench/search/CollapsingSearch.java b/solr/benchmark/src/java/org/apache/solr/bench/search/CollapsingSearch.java index edbee0a8ecb6..2b2dd40e56b8 100644 --- a/solr/benchmark/src/java/org/apache/solr/bench/search/CollapsingSearch.java +++ b/solr/benchmark/src/java/org/apache/solr/bench/search/CollapsingSearch.java @@ -16,6 +16,13 @@ */ package org.apache.solr.bench.search; +import java.io.IOException; +import java.time.Instant; +import java.time.format.DateTimeFormatter; +import java.util.Locale; +import java.util.Random; +import java.util.UUID; +import java.util.concurrent.TimeUnit; import org.apache.solr.bench.BaseBenchState; import org.apache.solr.bench.MiniClusterState; import org.apache.solr.client.solrj.SolrQuery; @@ -40,10 +47,6 @@ import org.openjdk.jmh.annotations.Threads; import org.openjdk.jmh.annotations.Warmup; -import java.io.IOException; -import java.util.UUID; -import java.util.concurrent.TimeUnit; - @Fork(value = 1) @Warmup(time = 5, iterations = 5) @Measurement(time = 15, iterations = 5) @@ -52,178 +55,175 @@ @Threads(value = 1) public class CollapsingSearch { - static final String COLLECTION = "collapse_bench"; - - @State(Scope.Benchmark) - public static class BenchState { - - @Param({"2000000"}) - int numDocs; - - @Param({"100000"}) - int numGroups; - - @Param({"1", "10"}) - int numSegments; - - final QueryRequest qSimple = - new QueryRequest(new SolrQuery("q", "*:*", "rows", "10", "cache", "false")); - - final QueryRequest qCollapseWithoutSort = - new QueryRequest( - new SolrQuery( - "q", "*:*", - "fq", "{!collapse field=group_s_dv cache=false}", - "rows", "10", - "cache", "false")); - - final QueryRequest qCollapseByStr = - new QueryRequest( - new SolrQuery( - "q", "*:*", - "fq", "{!collapse field=group_s_dv sort='str_s_dv asc' cache=false}", - "rows", "10", - "cache", "false")); - - final QueryRequest qCollapseByDate = - new QueryRequest( - new SolrQuery( - "q", "*:*", - "fq", "{!collapse field=group_s_dv sort='date_dt_dv asc' cache=false}", - "rows", "10", - "cache", "false")); - - final QueryRequest qCollapseByLong = - new QueryRequest( - new SolrQuery( - "q", "*:*", - "fq", "{!collapse field=group_s_dv sort='long_l_dv asc' cache=false}", - "rows", "10", - "cache", "false")); - - final QueryRequest qCollapseByDateAndStr = - new QueryRequest( - new SolrQuery( - "q", "*:*", - "fq", "{!collapse field=group_s_dv sort='date_dt_dv asc, str_s_dv asc' cache=false}", - "rows", "10", - "cache", "false")); - - @Setup(Level.Trial) - public void setupTrial(MiniClusterState.MiniClusterBenchState miniClusterState) - throws Exception { - System.setProperty("commitwithin.softcommit", "false"); - miniClusterState.startMiniCluster(1); - miniClusterState.createCollection(COLLECTION, 1, 1); - - indexDocs(miniClusterState); - miniClusterState.forceMerge(COLLECTION, numSegments); - - int actualSegments = getActualSegmentCount(miniClusterState); - BaseBenchState.log( - "CollapsingSearch ready: numDocs=" - + numDocs - + " numGroups=" - + numGroups - + " numSegments(requested)=" - + numSegments - + " numSegments(actual)=" - + actualSegments); - } - - @Setup(Level.Iteration) - public void setupIteration(MiniClusterState.MiniClusterBenchState miniClusterState) - throws SolrServerException, IOException { - CollectionAdminRequest.Reload reload = CollectionAdminRequest.reloadCollection(COLLECTION); - miniClusterState.client.request(reload); - } - - @TearDown(Level.Trial) - public void teardown(MiniClusterState.MiniClusterBenchState miniClusterState) - throws Exception { - CollectionAdminRequest.deleteCollection(COLLECTION) - .process(miniClusterState.client); - } - - private int pickGroup(int docId) { - return docId % numGroups; - } - - private int getActualSegmentCount(MiniClusterState.MiniClusterBenchState state) - throws SolrServerException, IOException { - SolrQuery lukeQuery = new SolrQuery(); - lukeQuery.set(CommonParams.QT, "/admin/luke"); - lukeQuery.set("show", "index"); - var resp = state.client.query(COLLECTION, lukeQuery); - Object segCount = resp.getResponse().findRecursive("index", "segmentCount"); - return segCount instanceof Number ? ((Number) segCount).intValue() : -1; - } - - private void indexDocs(MiniClusterState.MiniClusterBenchState state) throws Exception { - int docsPerSegment = numDocs / numSegments; - String[] groupIds = new String[numGroups]; - for (int i = 0; i < numGroups; i++) { - groupIds[i] = String.format("group_%06d", i); - } - java.util.Random rng = new java.util.Random(42); - java.time.Instant baseDate = java.time.Instant.parse("2020-01-01T00:00:00Z"); - - int docId = 0; - for (int seg = 0; seg < numSegments; seg++) { - int segDocs = (seg < numSegments - 1) ? docsPerSegment : (numDocs - docId); - UpdateRequest req = new UpdateRequest(); - for (int i = 0; i < segDocs; i++) { - SolrInputDocument doc = new SolrInputDocument(); - doc.addField("id", String.valueOf(docId)); - doc.addField("group_s_dv", groupIds[pickGroup(docId)]); - docId++; - doc.addField("str_s_dv", UUID.randomUUID().toString()); - doc.addField( - "date_dt_dv", - java.time.format.DateTimeFormatter.ISO_INSTANT.format( - baseDate.plusSeconds(rng.nextInt(10_000_000)))); - doc.addField("long_l_dv", rng.nextInt(10_000_000)); - req.add(doc); - } - req.commit(state.client, COLLECTION); - } - } - } - - @Benchmark - public Object simple(BenchState b, MiniClusterState.MiniClusterBenchState cluster) - throws SolrServerException, IOException { - return cluster.client.request(b.qSimple, COLLECTION); + static final String COLLECTION = "collapse_bench"; + + @State(Scope.Benchmark) + public static class BenchState { + + @Param({"2000000"}) + int numDocs; + + @Param({"100000"}) + int numGroups; + + @Param({"1", "10"}) + int numSegments; + + final QueryRequest qSimple = + new QueryRequest(new SolrQuery("q", "*:*", "rows", "10", "cache", "false")); + + final QueryRequest qCollapseWithoutSort = + new QueryRequest( + new SolrQuery( + "q", "*:*", + "fq", "{!collapse field=group_s_dv cache=false}", + "rows", "10", + "cache", "false")); + + final QueryRequest qCollapseByStr = + new QueryRequest( + new SolrQuery( + "q", "*:*", + "fq", "{!collapse field=group_s_dv sort='str_s_dv asc' cache=false}", + "rows", "10", + "cache", "false")); + + final QueryRequest qCollapseByDate = + new QueryRequest( + new SolrQuery( + "q", "*:*", + "fq", "{!collapse field=group_s_dv sort='date_dt_dv asc' cache=false}", + "rows", "10", + "cache", "false")); + + final QueryRequest qCollapseByLong = + new QueryRequest( + new SolrQuery( + "q", "*:*", + "fq", "{!collapse field=group_s_dv sort='long_l_dv asc' cache=false}", + "rows", "10", + "cache", "false")); + + final QueryRequest qCollapseByDateAndStr = + new QueryRequest( + new SolrQuery( + "q", "*:*", + "fq", + "{!collapse field=group_s_dv sort='date_dt_dv asc, str_s_dv asc' cache=false}", + "rows", "10", + "cache", "false")); + + @Setup(Level.Trial) + public void setupTrial(MiniClusterState.MiniClusterBenchState miniClusterState) + throws Exception { + System.setProperty("commitwithin.softcommit", "false"); + miniClusterState.startMiniCluster(1); + miniClusterState.createCollection(COLLECTION, 1, 1); + + indexDocs(miniClusterState); + miniClusterState.forceMerge(COLLECTION, numSegments); + + int actualSegments = getActualSegmentCount(miniClusterState); + BaseBenchState.log( + "CollapsingSearch ready: numDocs=" + + numDocs + + " numGroups=" + + numGroups + + " numSegments(requested)=" + + numSegments + + " numSegments(actual)=" + + actualSegments); } - @Benchmark - public Object collapseWithoutSort(BenchState b, MiniClusterState.MiniClusterBenchState cluster) - throws SolrServerException, IOException { - return cluster.client.request(b.qCollapseWithoutSort, COLLECTION); + @Setup(Level.Iteration) + public void setupIteration(MiniClusterState.MiniClusterBenchState miniClusterState) + throws SolrServerException, IOException { + CollectionAdminRequest.Reload reload = CollectionAdminRequest.reloadCollection(COLLECTION); + miniClusterState.client.request(reload); } - @Benchmark - public Object collapseByStr(BenchState b, MiniClusterState.MiniClusterBenchState cluster) - throws SolrServerException, IOException { - return cluster.client.request(b.qCollapseByStr, COLLECTION); + @TearDown(Level.Trial) + public void teardown(MiniClusterState.MiniClusterBenchState miniClusterState) throws Exception { + CollectionAdminRequest.deleteCollection(COLLECTION).process(miniClusterState.client); } - @Benchmark - public Object collapseByDate(BenchState b, MiniClusterState.MiniClusterBenchState cluster) - throws SolrServerException, IOException { - return cluster.client.request(b.qCollapseByDate, COLLECTION); + private int pickGroup(int docId) { + return docId % numGroups; } - @Benchmark - public Object collapseByLong(BenchState b, MiniClusterState.MiniClusterBenchState cluster) - throws SolrServerException, IOException { - return cluster.client.request(b.qCollapseByLong, COLLECTION); + private int getActualSegmentCount(MiniClusterState.MiniClusterBenchState state) + throws SolrServerException, IOException { + SolrQuery lukeQuery = new SolrQuery(); + lukeQuery.set(CommonParams.QT, "/admin/luke"); + lukeQuery.set("show", "index"); + var resp = state.client.query(COLLECTION, lukeQuery); + Object segCount = resp.getResponse().findRecursive("index", "segmentCount"); + return segCount instanceof Number ? ((Number) segCount).intValue() : -1; } - @Benchmark - public Object collapseByDateAndStr(BenchState b, MiniClusterState.MiniClusterBenchState cluster) - throws SolrServerException, IOException { - return cluster.client.request(b.qCollapseByDateAndStr, COLLECTION); + private void indexDocs(MiniClusterState.MiniClusterBenchState state) throws Exception { + int docsPerSegment = numDocs / numSegments; + String[] groupIds = new String[numGroups]; + for (int i = 0; i < numGroups; i++) { + groupIds[i] = String.format(Locale.ROOT, "group_%06d", i); + } + Random rng = new Random(42); + Instant baseDate = Instant.parse("2020-01-01T00:00:00Z"); + + int docId = 0; + for (int seg = 0; seg < numSegments; seg++) { + int segDocs = (seg < numSegments - 1) ? docsPerSegment : (numDocs - docId); + UpdateRequest req = new UpdateRequest(); + for (int i = 0; i < segDocs; i++) { + SolrInputDocument doc = new SolrInputDocument(); + doc.addField("id", String.valueOf(docId)); + doc.addField("group_s_dv", groupIds[pickGroup(docId)]); + docId++; + doc.addField("str_s_dv", UUID.randomUUID().toString()); + doc.addField( + "date_dt_dv", + DateTimeFormatter.ISO_INSTANT.format(baseDate.plusSeconds(rng.nextInt(10_000_000)))); + doc.addField("long_l_dv", rng.nextInt(10_000_000)); + req.add(doc); + } + req.commit(state.client, COLLECTION); + } } - + } + + @Benchmark + public Object simple(BenchState b, MiniClusterState.MiniClusterBenchState cluster) + throws SolrServerException, IOException { + return cluster.client.request(b.qSimple, COLLECTION); + } + + @Benchmark + public Object collapseWithoutSort(BenchState b, MiniClusterState.MiniClusterBenchState cluster) + throws SolrServerException, IOException { + return cluster.client.request(b.qCollapseWithoutSort, COLLECTION); + } + + @Benchmark + public Object collapseByStr(BenchState b, MiniClusterState.MiniClusterBenchState cluster) + throws SolrServerException, IOException { + return cluster.client.request(b.qCollapseByStr, COLLECTION); + } + + @Benchmark + public Object collapseByDate(BenchState b, MiniClusterState.MiniClusterBenchState cluster) + throws SolrServerException, IOException { + return cluster.client.request(b.qCollapseByDate, COLLECTION); + } + + @Benchmark + public Object collapseByLong(BenchState b, MiniClusterState.MiniClusterBenchState cluster) + throws SolrServerException, IOException { + return cluster.client.request(b.qCollapseByLong, COLLECTION); + } + + @Benchmark + public Object collapseByDateAndStr(BenchState b, MiniClusterState.MiniClusterBenchState cluster) + throws SolrServerException, IOException { + return cluster.client.request(b.qCollapseByDateAndStr, COLLECTION); + } } From 552b414067f90f5f24ffeb1d43cbd25222a14179 Mon Sep 17 00:00:00 2001 From: Bartosz Fidrysiak <300359074+bartoszfidrysiak@users.noreply.github.com> Date: Mon, 6 Jul 2026 19:52:12 +0200 Subject: [PATCH 4/5] fix: deep copy BytesRef from lookupOrd() in LazyStringValue.materialize() --- solr/core/src/java/org/apache/solr/search/LazyStringValue.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/solr/core/src/java/org/apache/solr/search/LazyStringValue.java b/solr/core/src/java/org/apache/solr/search/LazyStringValue.java index 3b4c7aae18a8..675c13337e59 100644 --- a/solr/core/src/java/org/apache/solr/search/LazyStringValue.java +++ b/solr/core/src/java/org/apache/solr/search/LazyStringValue.java @@ -36,6 +36,6 @@ BytesRef materialize() throws IOException { if (ord == missingOrd || ord < 0) { return null; } - return dv.lookupOrd(ord); + return BytesRef.deepCopyOf(dv.lookupOrd(ord)); } } From b75e6e9539a5e5e23b1f7d5f248c7199bfa59360 Mon Sep 17 00:00:00 2001 From: Bartosz Fidrysiak <300359074+bartoszfidrysiak@users.noreply.github.com> Date: Tue, 7 Jul 2026 15:06:17 +0200 Subject: [PATCH 5/5] fix: CollapsingSearch adjusted to use SolrBenchState in Solr 10 --- .../solr/bench/search/CollapsingSearch.java | 33 ++++++++++--------- 1 file changed, 17 insertions(+), 16 deletions(-) diff --git a/solr/benchmark/src/java/org/apache/solr/bench/search/CollapsingSearch.java b/solr/benchmark/src/java/org/apache/solr/bench/search/CollapsingSearch.java index 2b2dd40e56b8..a271925926ee 100644 --- a/solr/benchmark/src/java/org/apache/solr/bench/search/CollapsingSearch.java +++ b/solr/benchmark/src/java/org/apache/solr/bench/search/CollapsingSearch.java @@ -24,14 +24,15 @@ import java.util.UUID; import java.util.concurrent.TimeUnit; import org.apache.solr.bench.BaseBenchState; -import org.apache.solr.bench.MiniClusterState; -import org.apache.solr.client.solrj.SolrQuery; +import org.apache.solr.bench.SolrBenchState; import org.apache.solr.client.solrj.SolrServerException; import org.apache.solr.client.solrj.request.CollectionAdminRequest; import org.apache.solr.client.solrj.request.QueryRequest; +import org.apache.solr.client.solrj.request.SolrQuery; import org.apache.solr.client.solrj.request.UpdateRequest; import org.apache.solr.common.SolrInputDocument; import org.apache.solr.common.params.CommonParams; +import org.apache.solr.common.util.NamedList; import org.openjdk.jmh.annotations.Benchmark; import org.openjdk.jmh.annotations.BenchmarkMode; import org.openjdk.jmh.annotations.Fork; @@ -114,10 +115,9 @@ public static class BenchState { "cache", "false")); @Setup(Level.Trial) - public void setupTrial(MiniClusterState.MiniClusterBenchState miniClusterState) - throws Exception { + public void setupTrial(SolrBenchState miniClusterState) throws Exception { System.setProperty("commitwithin.softcommit", "false"); - miniClusterState.startMiniCluster(1); + miniClusterState.startSolr(1); miniClusterState.createCollection(COLLECTION, 1, 1); indexDocs(miniClusterState); @@ -136,14 +136,14 @@ public void setupTrial(MiniClusterState.MiniClusterBenchState miniClusterState) } @Setup(Level.Iteration) - public void setupIteration(MiniClusterState.MiniClusterBenchState miniClusterState) + public void setupIteration(SolrBenchState miniClusterState) throws SolrServerException, IOException { CollectionAdminRequest.Reload reload = CollectionAdminRequest.reloadCollection(COLLECTION); miniClusterState.client.request(reload); } @TearDown(Level.Trial) - public void teardown(MiniClusterState.MiniClusterBenchState miniClusterState) throws Exception { + public void teardown(SolrBenchState miniClusterState) throws Exception { CollectionAdminRequest.deleteCollection(COLLECTION).process(miniClusterState.client); } @@ -151,17 +151,18 @@ private int pickGroup(int docId) { return docId % numGroups; } - private int getActualSegmentCount(MiniClusterState.MiniClusterBenchState state) + private int getActualSegmentCount(SolrBenchState state) throws SolrServerException, IOException { SolrQuery lukeQuery = new SolrQuery(); lukeQuery.set(CommonParams.QT, "/admin/luke"); lukeQuery.set("show", "index"); var resp = state.client.query(COLLECTION, lukeQuery); - Object segCount = resp.getResponse().findRecursive("index", "segmentCount"); + NamedList indexInfo = (NamedList) resp.getResponse().get("index"); + Object segCount = indexInfo != null ? indexInfo.get("segmentCount") : null; return segCount instanceof Number ? ((Number) segCount).intValue() : -1; } - private void indexDocs(MiniClusterState.MiniClusterBenchState state) throws Exception { + private void indexDocs(SolrBenchState state) throws Exception { int docsPerSegment = numDocs / numSegments; String[] groupIds = new String[numGroups]; for (int i = 0; i < numGroups; i++) { @@ -192,37 +193,37 @@ private void indexDocs(MiniClusterState.MiniClusterBenchState state) throws Exce } @Benchmark - public Object simple(BenchState b, MiniClusterState.MiniClusterBenchState cluster) + public Object simple(BenchState b, SolrBenchState cluster) throws SolrServerException, IOException { return cluster.client.request(b.qSimple, COLLECTION); } @Benchmark - public Object collapseWithoutSort(BenchState b, MiniClusterState.MiniClusterBenchState cluster) + public Object collapseWithoutSort(BenchState b, SolrBenchState cluster) throws SolrServerException, IOException { return cluster.client.request(b.qCollapseWithoutSort, COLLECTION); } @Benchmark - public Object collapseByStr(BenchState b, MiniClusterState.MiniClusterBenchState cluster) + public Object collapseByStr(BenchState b, SolrBenchState cluster) throws SolrServerException, IOException { return cluster.client.request(b.qCollapseByStr, COLLECTION); } @Benchmark - public Object collapseByDate(BenchState b, MiniClusterState.MiniClusterBenchState cluster) + public Object collapseByDate(BenchState b, SolrBenchState cluster) throws SolrServerException, IOException { return cluster.client.request(b.qCollapseByDate, COLLECTION); } @Benchmark - public Object collapseByLong(BenchState b, MiniClusterState.MiniClusterBenchState cluster) + public Object collapseByLong(BenchState b, SolrBenchState cluster) throws SolrServerException, IOException { return cluster.client.request(b.qCollapseByLong, COLLECTION); } @Benchmark - public Object collapseByDateAndStr(BenchState b, MiniClusterState.MiniClusterBenchState cluster) + public Object collapseByDateAndStr(BenchState b, SolrBenchState cluster) throws SolrServerException, IOException { return cluster.client.request(b.qCollapseByDateAndStr, COLLECTION); }