From 964f60b8d04a07d9044f0546bd2b08e7a721c2f5 Mon Sep 17 00:00:00 2001 From: Eric Pugh Date: Sun, 26 Jul 2026 13:31:54 -0300 Subject: [PATCH 01/11] SOLR-18320: Use Configsets V2 API instead of direct ZooKeeper access when uploading a configset in bin/solr create. createCollection/createCore now just take a SolrClient; the only remaining ZK-specific step is resolving a live node's URL when only --zk-host is given. Co-Authored-By: Claude Sonnet 5 --- ...OLR-18320-skip-zk-for-configset-upload.yml | 7 ++ .../java/org/apache/solr/cli/CreateTool.java | 96 ++++++++----------- .../org/apache/solr/cli/CreateToolTest.java | 20 ++++ 3 files changed, 67 insertions(+), 56 deletions(-) create mode 100644 changelog/unreleased/SOLR-18320-skip-zk-for-configset-upload.yml diff --git a/changelog/unreleased/SOLR-18320-skip-zk-for-configset-upload.yml b/changelog/unreleased/SOLR-18320-skip-zk-for-configset-upload.yml new file mode 100644 index 000000000000..fa9d69fa2f78 --- /dev/null +++ b/changelog/unreleased/SOLR-18320-skip-zk-for-configset-upload.yml @@ -0,0 +1,7 @@ +title: bin/solr create no longer requires a direct ZooKeeper connection to upload a configset; it now uses the Configsets V2 API. +type: changed +authors: + - name: Eric Pugh +links: + - name: SOLR-18320 + url: https://issues.apache.org/jira/browse/SOLR-18320 diff --git a/solr/core/src/java/org/apache/solr/cli/CreateTool.java b/solr/core/src/java/org/apache/solr/cli/CreateTool.java index 6b7c6549503f..4ca4d8733454 100644 --- a/solr/core/src/java/org/apache/solr/cli/CreateTool.java +++ b/solr/core/src/java/org/apache/solr/cli/CreateTool.java @@ -16,13 +16,18 @@ */ package org.apache.solr.cli; +import java.io.ByteArrayInputStream; +import java.io.ByteArrayOutputStream; import java.io.IOException; +import java.nio.file.FileVisitResult; import java.nio.file.Files; import java.nio.file.Path; +import java.nio.file.SimpleFileVisitor; import java.nio.file.StandardCopyOption; +import java.nio.file.attribute.BasicFileAttributes; import java.util.Locale; -import java.util.Set; -import java.util.concurrent.TimeUnit; +import java.util.zip.ZipEntry; +import java.util.zip.ZipOutputStream; import org.apache.commons.cli.CommandLine; import org.apache.commons.cli.Option; import org.apache.commons.cli.Options; @@ -30,14 +35,11 @@ import org.apache.solr.cli.CommonCLIOptions.DefaultValues; import org.apache.solr.client.solrj.SolrClient; import org.apache.solr.client.solrj.SolrServerException; -import org.apache.solr.client.solrj.impl.CloudSolrClient; -import org.apache.solr.client.solrj.jetty.HttpJettySolrClient; import org.apache.solr.client.solrj.request.CollectionsApi; +import org.apache.solr.client.solrj.request.ConfigsetsApi; import org.apache.solr.client.solrj.request.CoresApi; import org.apache.solr.client.solrj.request.SystemInfoRequest; import org.apache.solr.client.solrj.response.SystemInfoResponse; -import org.apache.solr.cloud.ZkConfigSetService; -import org.apache.solr.common.cloud.ZkStateReader; import org.apache.solr.common.util.EnvUtils; import org.apache.solr.core.ConfigSetService; @@ -125,7 +127,7 @@ public Options getOptions() { public void runImpl(CommandLine cli) throws Exception { try (var solrClient = CLIUtils.getSolrClient(cli)) { if (CLIUtils.isCloudMode(solrClient)) { - createCollection(cli); + createCollection(cli, solrClient); } else { createCore(cli, solrClient); } @@ -193,29 +195,7 @@ protected void createCore(CommandLine cli, SolrClient solrClient) throws Excepti } } - protected void createCollection(CommandLine cli) throws Exception { - var builder = - new HttpJettySolrClient.Builder() - .withIdleTimeout(30, TimeUnit.SECONDS) - .withConnectionTimeout(15, TimeUnit.SECONDS) - .withKeyStoreReloadInterval(-1, TimeUnit.SECONDS) - .withOptionalBasicAuthCredentials( - cli.getOptionValue(CommonCLIOptions.CREDENTIALS_OPTION)); - String zkHost = CLIUtils.getZkHost(cli); - echoIfVerbose("Connecting to ZooKeeper at " + zkHost); - var zkSolrConnection = CloudSolrClient.CloudSolrClientConnection.parse(zkHost); - if (!zkSolrConnection.isZookeeper()) { - throw new IOException( - String.format( - Locale.ROOT, "Expected ZooKeeper connection string, but got: '%s'.", zkHost)); - } - try (var cloudSolrClient = CLIUtils.getCloudSolrClient(zkSolrConnection, builder)) { - createCollection(cloudSolrClient, cli); - } - } - - protected void createCollection(CloudSolrClient cloudSolrClient, CommandLine cli) - throws Exception { + protected void createCollection(CommandLine cli, SolrClient solrClient) throws Exception { String collectionName = cli.getOptionValue(COLLECTION_NAME_OPTION); final String solrInstallDir = EnvUtils.getProperty("solr.install.dir"); @@ -226,42 +206,25 @@ protected void createCollection(CloudSolrClient cloudSolrClient, CommandLine cli ensureConfDirExists(solrInstallDirPath, confDirPath); printDefaultConfigsetWarningIfNecessary(cli); - Set liveNodes = cloudSolrClient.getClusterState().getLiveNodes(); - if (liveNodes.isEmpty()) - throw new IllegalStateException( - "No live nodes found! Cannot create a collection until " - + "there is at least 1 live node in the cluster."); - - String solrUrl; - if (CLIUtils.hasConnectionOption(cli)) { - solrUrl = CLIUtils.normalizeSolrUrl(cli); - } else { - String firstLiveNode = liveNodes.iterator().next(); - solrUrl = ZkStateReader.from(cloudSolrClient).getBaseUrlForNodeName(firstLiveNode); - } + String solrUrl = CLIUtils.normalizeSolrUrl(cli); // build a URL to create the collection int numShards = cli.getParsedOptionValue(SHARDS_OPTION, 1); int replicationFactor = cli.getParsedOptionValue(REPLICATION_FACTOR_OPTION, 1); - boolean configExistsInZk = + boolean configExists = confName != null && !confName.trim().isEmpty() - && ZkStateReader.from(cloudSolrClient).getZkClient().exists("/configs/" + confName); + && new ConfigsetsApi.ListConfigSet().process(solrClient).configSets.contains(confName); - if (configExistsInZk) { + if (configExists) { echo("Re-using existing configuration directory " + confName); } else { // if (confdir != null && !confdir.trim().isEmpty()) { if (confName == null || confName.trim().isEmpty()) { confName = collectionName; } - // TODO: This should be done using the configSet API. This would let us remove - // the direct dependency on ZooKeeper APIs. Unlike the bin/solr zk comamnds that - // work directly with ZooKeeper. final Path configsetsDirPath = CLIUtils.getConfigSetsDir(solrInstallDirPath); - ConfigSetService configSetService = - new ZkConfigSetService(ZkStateReader.from(cloudSolrClient).getZkClient()); Path confPath = ConfigSetService.getConfigsetPath(confDir, configsetsDirPath.toString()); echoIfVerbose( @@ -269,10 +232,11 @@ protected void createCollection(CloudSolrClient cloudSolrClient, CommandLine cli + confPath.toAbsolutePath() + " for config " + confName - + " to ZooKeeper at " - + cloudSolrClient.getClusterStateProvider().getQuorumHosts()); - // We will trust the config since we have the Zookeeper Address - configSetService.uploadConfig(confName, confPath); + + " using the Configsets V2 API"); + var uploadReq = + new ConfigsetsApi.UploadConfigSet( + confName, new ByteArrayInputStream(zipConfigSet(confPath))); + uploadReq.process(solrClient); } // since creating a collection is a heavy-weight operation, check for existence first @@ -293,7 +257,7 @@ protected void createCollection(CloudSolrClient cloudSolrClient, CommandLine cli req.setConfig(confName); req.setNumShards(numShards); req.setReplicationFactor(replicationFactor); - var response = req.process(cloudSolrClient); + var response = req.process(solrClient); echoIfVerbose(response); } catch (SolrServerException sse) { throw new Exception( @@ -314,6 +278,26 @@ protected void createCollection(CloudSolrClient cloudSolrClient, CommandLine cli echo(endMessage); } + /** Zips the contents of a configset directory for upload via the Configsets V2 API. */ + private static byte[] zipConfigSet(Path confPath) throws IOException { + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + try (ZipOutputStream zipOut = new ZipOutputStream(baos)) { + Files.walkFileTree( + confPath, + new SimpleFileVisitor<>() { + @Override + public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) + throws IOException { + zipOut.putNextEntry(new ZipEntry(confPath.relativize(file).toString())); + Files.copy(file, zipOut); + zipOut.closeEntry(); + return FileVisitResult.CONTINUE; + } + }); + } + return baos.toByteArray(); + } + private Path getFullConfDir(Path solrInstallDir, Path confDirName) { return CLIUtils.getConfigSetsDir(solrInstallDir).resolve(confDirName); } diff --git a/solr/core/src/test/org/apache/solr/cli/CreateToolTest.java b/solr/core/src/test/org/apache/solr/cli/CreateToolTest.java index 74c9a6411668..b95692da2444 100644 --- a/solr/core/src/test/org/apache/solr/cli/CreateToolTest.java +++ b/solr/core/src/test/org/apache/solr/cli/CreateToolTest.java @@ -52,4 +52,24 @@ public void testCreateCollectionWithBasicAuth() throws Exception { assertEquals(0, CLITestHelper.runTool(args, CreateTool.class)); } + + @Test + public void testCreateCollectionUploadsNewConfigSet() throws Exception { + String[] args = { + "create", + "-c", + "testCreateCollectionUploadsNewConfigSet", + "-d", + configset("cloud-minimal").toString(), + "-n", + "cloud-minimal-uploaded", + "-z", + cluster.getZkClient().getZkServerAddress(), + "--credentials", + SecurityJson.USER_PASS, + "--verbose" + }; + + assertEquals(0, CLITestHelper.runTool(args, CreateTool.class)); + } } From 20f76414b70324868dc4106fd6ee393984c71e9e Mon Sep 17 00:00:00 2001 From: Eric Pugh Date: Tue, 28 Jul 2026 13:40:03 -0300 Subject: [PATCH 02/11] don't tie commment to specific api, but still leave in why we have this here, its for uploading the configset. --- solr/core/src/java/org/apache/solr/cli/CreateTool.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/solr/core/src/java/org/apache/solr/cli/CreateTool.java b/solr/core/src/java/org/apache/solr/cli/CreateTool.java index 4ca4d8733454..e9f4c82cb099 100644 --- a/solr/core/src/java/org/apache/solr/cli/CreateTool.java +++ b/solr/core/src/java/org/apache/solr/cli/CreateTool.java @@ -278,7 +278,7 @@ protected void createCollection(CommandLine cli, SolrClient solrClient) throws E echo(endMessage); } - /** Zips the contents of a configset directory for upload via the Configsets V2 API. */ + /** Zips the contents of a configset directory for upload. */ private static byte[] zipConfigSet(Path confPath) throws IOException { ByteArrayOutputStream baos = new ByteArrayOutputStream(); try (ZipOutputStream zipOut = new ZipOutputStream(baos)) { From 7fc5b08b43e0ab40406cf4244d4f2c0859d6b678 Mon Sep 17 00:00:00 2001 From: Eric Pugh Date: Tue, 28 Jul 2026 13:50:52 -0300 Subject: [PATCH 03/11] Tighten up logic on what gets put into a configset. --- .../java/org/apache/solr/cli/CreateTool.java | 39 ++++++++++++++-- .../org/apache/solr/cli/CreateToolTest.java | 45 +++++++++++++++++++ 2 files changed, 81 insertions(+), 3 deletions(-) diff --git a/solr/core/src/java/org/apache/solr/cli/CreateTool.java b/solr/core/src/java/org/apache/solr/cli/CreateTool.java index e9f4c82cb099..2737a1d2a3ed 100644 --- a/solr/core/src/java/org/apache/solr/cli/CreateTool.java +++ b/solr/core/src/java/org/apache/solr/cli/CreateTool.java @@ -278,17 +278,50 @@ protected void createCollection(CommandLine cli, SolrClient solrClient) throws E echo(endMessage); } - /** Zips the contents of a configset directory for upload. */ - private static byte[] zipConfigSet(Path confPath) throws IOException { + /** + * Zips the contents of a configset directory for upload. + * + *

Mirrors the hidden-file skipping, directory-entry, and forbidden-file-type logic in {@link + * org.apache.solr.handler.configsets.DownloadConfigSet#zipConfigSet}. + */ + static byte[] zipConfigSet(Path confPath) throws IOException { ByteArrayOutputStream baos = new ByteArrayOutputStream(); try (ZipOutputStream zipOut = new ZipOutputStream(baos)) { Files.walkFileTree( confPath, new SimpleFileVisitor<>() { + @Override + public FileVisitResult preVisitDirectory(Path dir, BasicFileAttributes attrs) + throws IOException { + if (Files.isHidden(dir)) { + return FileVisitResult.SKIP_SUBTREE; + } + String dirName = confPath.relativize(dir).toString().replace('\\', '/'); + if (!dirName.isEmpty()) { + if (!dirName.endsWith("/")) { + dirName += "/"; + } + zipOut.putNextEntry(new ZipEntry(dirName)); + zipOut.closeEntry(); + } + return FileVisitResult.CONTINUE; + } + @Override public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException { - zipOut.putNextEntry(new ZipEntry(confPath.relativize(file).toString())); + if (Files.isHidden(file)) { + return FileVisitResult.CONTINUE; + } + String filename = file.getFileName().toString(); + if (ConfigSetService.isFileForbiddenInConfigSets(filename)) { + throw new IOException( + "The file type provided for upload, '" + + filename + + "', is forbidden for use in uploading configsets."); + } + String entryName = confPath.relativize(file).toString().replace('\\', '/'); + zipOut.putNextEntry(new ZipEntry(entryName)); Files.copy(file, zipOut); zipOut.closeEntry(); return FileVisitResult.CONTINUE; diff --git a/solr/core/src/test/org/apache/solr/cli/CreateToolTest.java b/solr/core/src/test/org/apache/solr/cli/CreateToolTest.java index b95692da2444..13206286145f 100644 --- a/solr/core/src/test/org/apache/solr/cli/CreateToolTest.java +++ b/solr/core/src/test/org/apache/solr/cli/CreateToolTest.java @@ -17,6 +17,14 @@ package org.apache.solr.cli; +import java.io.ByteArrayInputStream; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.HashSet; +import java.util.Set; +import java.util.zip.ZipEntry; +import java.util.zip.ZipInputStream; import org.apache.solr.cloud.SolrCloudTestCase; import org.apache.solr.util.SecurityJson; import org.junit.BeforeClass; @@ -71,5 +79,42 @@ public void testCreateCollectionUploadsNewConfigSet() throws Exception { }; assertEquals(0, CLITestHelper.runTool(args, CreateTool.class)); + assertTrue(cluster.getZkClient().exists("/configs/cloud-minimal-uploaded")); + } + + @Test + public void testZipConfigSetSkipsHiddenFilesAndIncludesDirectoryEntries() throws Exception { + Path confDir = createTempDir("zipConfigSetTest"); + Files.writeString(confDir.resolve("solrconfig.xml"), ""); + Files.writeString(confDir.resolve(".hidden-file"), "should not be zipped"); + Path langDir = Files.createDirectory(confDir.resolve("lang")); + Files.writeString(langDir.resolve("stopwords.txt"), "the\na\n"); + Path hiddenDir = Files.createDirectory(confDir.resolve(".hiddenDir")); + Files.writeString(hiddenDir.resolve("nope.txt"), "should not be zipped either"); + + byte[] zipBytes = CreateTool.zipConfigSet(confDir); + + Set entryNames = new HashSet<>(); + try (ZipInputStream zipIn = new ZipInputStream(new ByteArrayInputStream(zipBytes))) { + ZipEntry entry; + while ((entry = zipIn.getNextEntry()) != null) { + entryNames.add(entry.getName()); + } + } + + assertTrue(entryNames.contains("solrconfig.xml")); + assertTrue(entryNames.contains("lang/")); + assertTrue(entryNames.contains("lang/stopwords.txt")); + assertFalse(entryNames.contains(".hidden-file")); + assertTrue(entryNames.stream().noneMatch(name -> name.startsWith(".hiddenDir"))); + } + + @Test + public void testZipConfigSetRejectsForbiddenFileType() throws Exception { + Path confDir = createTempDir("zipConfigSetForbiddenTest"); + Files.writeString(confDir.resolve("evil.jar"), "not really a jar"); + + IOException thrown = expectThrows(IOException.class, () -> CreateTool.zipConfigSet(confDir)); + assertTrue(thrown.getMessage().contains("forbidden")); } } From adda596e0acd341e910f51aa20b8d6df33b484d8 Mon Sep 17 00:00:00 2001 From: Eric Pugh Date: Tue, 28 Jul 2026 14:00:40 -0300 Subject: [PATCH 04/11] Pull all the various configset zipping methods into one helper. --- .../java/org/apache/solr/cli/CreateTool.java | 56 ++-------------- .../apache/solr/core/ConfigSetService.java | 65 +++++++++++++++++++ .../handler/configsets/DownloadConfigSet.java | 45 +------------ .../SchemaDesignerConfigSetHelper.java | 44 +------------ 4 files changed, 72 insertions(+), 138 deletions(-) diff --git a/solr/core/src/java/org/apache/solr/cli/CreateTool.java b/solr/core/src/java/org/apache/solr/cli/CreateTool.java index 2737a1d2a3ed..b65ba8b0d0d2 100644 --- a/solr/core/src/java/org/apache/solr/cli/CreateTool.java +++ b/solr/core/src/java/org/apache/solr/cli/CreateTool.java @@ -17,17 +17,11 @@ package org.apache.solr.cli; import java.io.ByteArrayInputStream; -import java.io.ByteArrayOutputStream; import java.io.IOException; -import java.nio.file.FileVisitResult; import java.nio.file.Files; import java.nio.file.Path; -import java.nio.file.SimpleFileVisitor; import java.nio.file.StandardCopyOption; -import java.nio.file.attribute.BasicFileAttributes; import java.util.Locale; -import java.util.zip.ZipEntry; -import java.util.zip.ZipOutputStream; import org.apache.commons.cli.CommandLine; import org.apache.commons.cli.Option; import org.apache.commons.cli.Options; @@ -281,54 +275,12 @@ protected void createCollection(CommandLine cli, SolrClient solrClient) throws E /** * Zips the contents of a configset directory for upload. * - *

Mirrors the hidden-file skipping, directory-entry, and forbidden-file-type logic in {@link - * org.apache.solr.handler.configsets.DownloadConfigSet#zipConfigSet}. + *

Delegates to {@link ConfigSetService#zipDirectory}, which is shared with {@link + * org.apache.solr.handler.configsets.DownloadConfigSet#zipConfigSet} and {@link + * org.apache.solr.handler.designer.SchemaDesignerConfigSetHelper#downloadAndZipConfigSet}. */ static byte[] zipConfigSet(Path confPath) throws IOException { - ByteArrayOutputStream baos = new ByteArrayOutputStream(); - try (ZipOutputStream zipOut = new ZipOutputStream(baos)) { - Files.walkFileTree( - confPath, - new SimpleFileVisitor<>() { - @Override - public FileVisitResult preVisitDirectory(Path dir, BasicFileAttributes attrs) - throws IOException { - if (Files.isHidden(dir)) { - return FileVisitResult.SKIP_SUBTREE; - } - String dirName = confPath.relativize(dir).toString().replace('\\', '/'); - if (!dirName.isEmpty()) { - if (!dirName.endsWith("/")) { - dirName += "/"; - } - zipOut.putNextEntry(new ZipEntry(dirName)); - zipOut.closeEntry(); - } - return FileVisitResult.CONTINUE; - } - - @Override - public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) - throws IOException { - if (Files.isHidden(file)) { - return FileVisitResult.CONTINUE; - } - String filename = file.getFileName().toString(); - if (ConfigSetService.isFileForbiddenInConfigSets(filename)) { - throw new IOException( - "The file type provided for upload, '" - + filename - + "', is forbidden for use in uploading configsets."); - } - String entryName = confPath.relativize(file).toString().replace('\\', '/'); - zipOut.putNextEntry(new ZipEntry(entryName)); - Files.copy(file, zipOut); - zipOut.closeEntry(); - return FileVisitResult.CONTINUE; - } - }); - } - return baos.toByteArray(); + return ConfigSetService.zipDirectory(confPath, true); } private Path getFullConfDir(Path solrInstallDir, Path confDirName) { diff --git a/solr/core/src/java/org/apache/solr/core/ConfigSetService.java b/solr/core/src/java/org/apache/solr/core/ConfigSetService.java index 6976e65c84ba..9d0dc7da6518 100644 --- a/solr/core/src/java/org/apache/solr/core/ConfigSetService.java +++ b/solr/core/src/java/org/apache/solr/core/ConfigSetService.java @@ -18,16 +18,22 @@ import com.github.benmanes.caffeine.cache.Cache; import com.github.benmanes.caffeine.cache.Caffeine; +import java.io.ByteArrayOutputStream; import java.io.IOException; import java.lang.invoke.MethodHandles; import java.lang.reflect.Constructor; +import java.nio.file.FileVisitResult; import java.nio.file.Files; import java.nio.file.Path; +import java.nio.file.SimpleFileVisitor; +import java.nio.file.attribute.BasicFileAttributes; import java.util.List; import java.util.Locale; import java.util.Map; import java.util.Set; import java.util.regex.Pattern; +import java.util.zip.ZipEntry; +import java.util.zip.ZipOutputStream; import org.apache.solr.cloud.ZkConfigSetService; import org.apache.solr.cloud.ZkController; import org.apache.solr.common.ConfigNode; @@ -75,6 +81,65 @@ public static boolean isFileForbiddenInConfigSets(String filePath) { return lastDot >= 0 && USE_FORBIDDEN_FILE_TYPES.contains(filePath.substring(lastDot + 1)); } + /** + * Zips the contents of {@code rootPath} into an in-memory archive. Hidden files and directories + * (as determined by {@link Files#isHidden}) are skipped, directory entries are written for + * non-empty subdirectories, and zip entry names are normalized to use {@code /} separators + * regardless of platform. + * + * @param rootPath the directory to zip + * @param validateFileTypes if true, a file with a forbidden extension (see {@link + * #isFileForbiddenInConfigSets}) causes an {@link IOException} instead of being silently + * included + * @return the zipped bytes + */ + public static byte[] zipDirectory(Path rootPath, boolean validateFileTypes) throws IOException { + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + try (ZipOutputStream zipOut = new ZipOutputStream(baos)) { + Files.walkFileTree( + rootPath, + new SimpleFileVisitor<>() { + @Override + public FileVisitResult preVisitDirectory(Path dir, BasicFileAttributes attrs) + throws IOException { + if (Files.isHidden(dir)) { + return FileVisitResult.SKIP_SUBTREE; + } + String dirName = rootPath.relativize(dir).toString().replace('\\', '/'); + if (!dirName.isEmpty()) { + if (!dirName.endsWith("/")) { + dirName += "/"; + } + zipOut.putNextEntry(new ZipEntry(dirName)); + zipOut.closeEntry(); + } + return FileVisitResult.CONTINUE; + } + + @Override + public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) + throws IOException { + if (Files.isHidden(file)) { + return FileVisitResult.CONTINUE; + } + String filename = file.getFileName().toString(); + if (validateFileTypes && isFileForbiddenInConfigSets(filename)) { + throw new IOException( + "The file type provided for upload, '" + + filename + + "', is forbidden for use in uploading configsets."); + } + String entryName = rootPath.relativize(file).toString().replace('\\', '/'); + zipOut.putNextEntry(new ZipEntry(entryName)); + Files.copy(file, zipOut); + zipOut.closeEntry(); + return FileVisitResult.CONTINUE; + } + }); + } + return baos.toByteArray(); + } + private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); public static ConfigSetService createConfigSetService(CoreContainer coreContainer) { diff --git a/solr/core/src/java/org/apache/solr/handler/configsets/DownloadConfigSet.java b/solr/core/src/java/org/apache/solr/handler/configsets/DownloadConfigSet.java index 729aaf00d914..cdd695d9975c 100644 --- a/solr/core/src/java/org/apache/solr/handler/configsets/DownloadConfigSet.java +++ b/solr/core/src/java/org/apache/solr/handler/configsets/DownloadConfigSet.java @@ -21,16 +21,9 @@ import jakarta.inject.Inject; import jakarta.ws.rs.core.Response; import jakarta.ws.rs.core.StreamingOutput; -import java.io.ByteArrayOutputStream; import java.io.IOException; -import java.io.InputStream; -import java.nio.file.FileVisitResult; import java.nio.file.Files; import java.nio.file.Path; -import java.nio.file.SimpleFileVisitor; -import java.nio.file.attribute.BasicFileAttributes; -import java.util.zip.ZipEntry; -import java.util.zip.ZipOutputStream; import org.apache.commons.io.file.PathUtils; import org.apache.solr.client.api.endpoint.ConfigsetsApi; import org.apache.solr.common.SolrException; @@ -86,48 +79,12 @@ public static Response buildZipResponse(ConfigSetService configSetService, Strin */ public static byte[] zipConfigSet(ConfigSetService configSetService, String configSetName) throws IOException { - ByteArrayOutputStream baos = new ByteArrayOutputStream(); Path tmpDirectory = Files.createTempDirectory("configset-download-"); try { configSetService.downloadConfig(configSetName, tmpDirectory); - try (ZipOutputStream zipOut = new ZipOutputStream(baos)) { - Files.walkFileTree( - tmpDirectory, - new SimpleFileVisitor<>() { - @Override - public FileVisitResult preVisitDirectory(Path dir, BasicFileAttributes attrs) - throws IOException { - if (Files.isHidden(dir)) { - return FileVisitResult.SKIP_SUBTREE; - } - String dirName = tmpDirectory.relativize(dir).toString(); - if (!dirName.isEmpty()) { - if (!dirName.endsWith("/")) { - dirName += "/"; - } - zipOut.putNextEntry(new ZipEntry(dirName)); - zipOut.closeEntry(); - } - return FileVisitResult.CONTINUE; - } - - @Override - public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) - throws IOException { - if (!Files.isHidden(file)) { - try (InputStream fis = Files.newInputStream(file)) { - ZipEntry zipEntry = new ZipEntry(tmpDirectory.relativize(file).toString()); - zipOut.putNextEntry(zipEntry); - fis.transferTo(zipOut); - } - } - return FileVisitResult.CONTINUE; - } - }); - } + return ConfigSetService.zipDirectory(tmpDirectory, false); } finally { PathUtils.deleteDirectory(tmpDirectory); } - return baos.toByteArray(); } } diff --git a/solr/core/src/java/org/apache/solr/handler/designer/SchemaDesignerConfigSetHelper.java b/solr/core/src/java/org/apache/solr/handler/designer/SchemaDesignerConfigSetHelper.java index 1af273cc9d09..a7c979d61541 100644 --- a/solr/core/src/java/org/apache/solr/handler/designer/SchemaDesignerConfigSetHelper.java +++ b/solr/core/src/java/org/apache/solr/handler/designer/SchemaDesignerConfigSetHelper.java @@ -26,18 +26,14 @@ import static org.apache.solr.schema.IndexSchema.ROOT_FIELD_NAME; import static org.apache.solr.schema.ManagedIndexSchemaFactory.DEFAULT_MANAGED_SCHEMA_RESOURCE_NAME; -import java.io.ByteArrayOutputStream; import java.io.FileNotFoundException; import java.io.IOException; import java.io.InputStream; import java.lang.invoke.MethodHandles; import java.nio.ByteBuffer; import java.nio.charset.StandardCharsets; -import java.nio.file.FileVisitResult; import java.nio.file.Files; import java.nio.file.Path; -import java.nio.file.SimpleFileVisitor; -import java.nio.file.attribute.BasicFileAttributes; import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; @@ -52,8 +48,6 @@ import java.util.concurrent.TimeoutException; import java.util.concurrent.atomic.AtomicReference; import java.util.stream.Collectors; -import java.util.zip.ZipEntry; -import java.util.zip.ZipOutputStream; import org.apache.commons.io.FilenameUtils; import org.apache.commons.io.file.PathUtils; import org.apache.lucene.util.IOSupplier; @@ -81,6 +75,7 @@ import org.apache.solr.common.util.NamedList; import org.apache.solr.common.util.SimpleOrderedMap; import org.apache.solr.common.util.Utils; +import org.apache.solr.core.ConfigSetService; import org.apache.solr.core.CoreContainer; import org.apache.solr.core.SolrConfig; import org.apache.solr.core.SolrResourceLoader; @@ -1100,49 +1095,14 @@ List listConfigsInZk() throws IOException { } byte[] downloadAndZipConfigSet(String configId) throws IOException { - ByteArrayOutputStream baos = new ByteArrayOutputStream(); Path tmpDirectory = Files.createTempDirectory("schema-designer-" + FilenameUtils.getName(configId)); try { cc.getConfigSetService().downloadConfig(configId, tmpDirectory); - try (ZipOutputStream zipOut = new ZipOutputStream(baos)) { - Files.walkFileTree( - tmpDirectory, - new SimpleFileVisitor<>() { - @Override - public FileVisitResult preVisitDirectory(Path dir, BasicFileAttributes attrs) - throws IOException { - if (Files.isHidden(dir)) { - return FileVisitResult.SKIP_SUBTREE; - } - - String dirName = tmpDirectory.relativize(dir).toString(); - if (!dirName.endsWith("/")) { - dirName += "/"; - } - zipOut.putNextEntry(new ZipEntry(dirName)); - zipOut.closeEntry(); - return FileVisitResult.CONTINUE; - } - - @Override - public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) - throws IOException { - if (!Files.isHidden(file)) { - try (InputStream fis = Files.newInputStream(file)) { - ZipEntry zipEntry = new ZipEntry(tmpDirectory.relativize(file).toString()); - zipOut.putNextEntry(zipEntry); - fis.transferTo(zipOut); - } - } - return FileVisitResult.CONTINUE; - } - }); - } + return ConfigSetService.zipDirectory(tmpDirectory, false); } finally { PathUtils.deleteDirectory(tmpDirectory); } - return baos.toByteArray(); } protected ZkSolrResourceLoader zkLoaderForConfigSet(final String configSet) { From 9b5d91d82f9b841420aeed059d5cc252221703b4 Mon Sep 17 00:00:00 2001 From: Eric Pugh Date: Sun, 27 Sep 2026 11:11:21 -0700 Subject: [PATCH 05/11] It's not an IO error it's a bad argument error! --- solr/core/src/java/org/apache/solr/core/ConfigSetService.java | 4 ++-- solr/core/src/test/org/apache/solr/cli/CreateToolTest.java | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/solr/core/src/java/org/apache/solr/core/ConfigSetService.java b/solr/core/src/java/org/apache/solr/core/ConfigSetService.java index 9d0dc7da6518..eaf479f02c8c 100644 --- a/solr/core/src/java/org/apache/solr/core/ConfigSetService.java +++ b/solr/core/src/java/org/apache/solr/core/ConfigSetService.java @@ -89,7 +89,7 @@ public static boolean isFileForbiddenInConfigSets(String filePath) { * * @param rootPath the directory to zip * @param validateFileTypes if true, a file with a forbidden extension (see {@link - * #isFileForbiddenInConfigSets}) causes an {@link IOException} instead of being silently + * #isFileForbiddenInConfigSets}) causes an {@link IllegalArgumentException} instead of being * included * @return the zipped bytes */ @@ -124,7 +124,7 @@ public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) } String filename = file.getFileName().toString(); if (validateFileTypes && isFileForbiddenInConfigSets(filename)) { - throw new IOException( + throw new IllegalArgumentException( "The file type provided for upload, '" + filename + "', is forbidden for use in uploading configsets."); diff --git a/solr/core/src/test/org/apache/solr/cli/CreateToolTest.java b/solr/core/src/test/org/apache/solr/cli/CreateToolTest.java index 13206286145f..d288391a358b 100644 --- a/solr/core/src/test/org/apache/solr/cli/CreateToolTest.java +++ b/solr/core/src/test/org/apache/solr/cli/CreateToolTest.java @@ -18,7 +18,6 @@ package org.apache.solr.cli; import java.io.ByteArrayInputStream; -import java.io.IOException; import java.nio.file.Files; import java.nio.file.Path; import java.util.HashSet; @@ -114,7 +113,8 @@ public void testZipConfigSetRejectsForbiddenFileType() throws Exception { Path confDir = createTempDir("zipConfigSetForbiddenTest"); Files.writeString(confDir.resolve("evil.jar"), "not really a jar"); - IOException thrown = expectThrows(IOException.class, () -> CreateTool.zipConfigSet(confDir)); + IllegalArgumentException thrown = + expectThrows(IllegalArgumentException.class, () -> CreateTool.zipConfigSet(confDir)); assertTrue(thrown.getMessage().contains("forbidden")); } } From 538e93b75921fa4a78e1243422a14c547bc5f97d Mon Sep 17 00:00:00 2001 From: Eric Pugh Date: Sun, 27 Sep 2026 16:31:01 -0700 Subject: [PATCH 06/11] dead code --- .../designer/SchemaDesignerConfigSetHelper.java | 16 ---------------- 1 file changed, 16 deletions(-) diff --git a/solr/core/src/java/org/apache/solr/handler/designer/SchemaDesignerConfigSetHelper.java b/solr/core/src/java/org/apache/solr/handler/designer/SchemaDesignerConfigSetHelper.java index ccedc63ab184..72d8c3bb99be 100644 --- a/solr/core/src/java/org/apache/solr/handler/designer/SchemaDesignerConfigSetHelper.java +++ b/solr/core/src/java/org/apache/solr/handler/designer/SchemaDesignerConfigSetHelper.java @@ -32,8 +32,6 @@ import java.lang.invoke.MethodHandles; import java.nio.ByteBuffer; import java.nio.charset.StandardCharsets; -import java.nio.file.Files; -import java.nio.file.Path; import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; @@ -47,8 +45,6 @@ import java.util.concurrent.TimeoutException; import java.util.concurrent.atomic.AtomicReference; import java.util.stream.Collectors; -import org.apache.commons.io.FilenameUtils; -import org.apache.commons.io.file.PathUtils; import org.apache.lucene.util.IOSupplier; import org.apache.solr.client.solrj.SolrRequest; import org.apache.solr.client.solrj.SolrResponse; @@ -73,7 +69,6 @@ import org.apache.solr.common.util.NamedList; import org.apache.solr.common.util.SimpleOrderedMap; import org.apache.solr.common.util.Utils; -import org.apache.solr.core.ConfigSetService; import org.apache.solr.core.CoreContainer; import org.apache.solr.core.SolrConfig; import org.apache.solr.core.SolrResourceLoader; @@ -1069,17 +1064,6 @@ List listConfigsInZk() throws IOException { return cc.getConfigSetService().listConfigs(); } - byte[] downloadAndZipConfigSet(String configId) throws IOException { - Path tmpDirectory = - Files.createTempDirectory("schema-designer-" + FilenameUtils.getName(configId)); - try { - cc.getConfigSetService().downloadConfig(configId, tmpDirectory); - return ConfigSetService.zipDirectory(tmpDirectory, false); - } finally { - PathUtils.deleteDirectory(tmpDirectory); - } - } - protected ZkSolrResourceLoader zkLoaderForConfigSet(final String configSet) { SolrResourceLoader loader = cc.getResourceLoader(); return new ZkSolrResourceLoader( From 31b7432a4b42d3a684e556c9955e76b54f58f107 Mon Sep 17 00:00:00 2001 From: Eric Pugh Date: Sun, 27 Sep 2026 16:31:17 -0700 Subject: [PATCH 07/11] pass url around so we don't recrate client twice --- .../java/org/apache/solr/cli/CreateTool.java | 24 +++++++++---------- 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/solr/core/src/java/org/apache/solr/cli/CreateTool.java b/solr/core/src/java/org/apache/solr/cli/CreateTool.java index b65ba8b0d0d2..5700e25589e9 100644 --- a/solr/core/src/java/org/apache/solr/cli/CreateTool.java +++ b/solr/core/src/java/org/apache/solr/cli/CreateTool.java @@ -119,18 +119,20 @@ public Options getOptions() { @Override public void runImpl(CommandLine cli) throws Exception { - try (var solrClient = CLIUtils.getSolrClient(cli)) { + String solrUrl = CLIUtils.normalizeSolrUrl(cli); + String credentials = cli.getOptionValue(CommonCLIOptions.CREDENTIALS_OPTION); + try (var solrClient = CLIUtils.getSolrClient(solrUrl, credentials)) { if (CLIUtils.isCloudMode(solrClient)) { - createCollection(cli, solrClient); + createCollection(cli, solrClient, solrUrl); } else { - createCore(cli, solrClient); + createCore(cli, solrClient, solrUrl); } } } - protected void createCore(CommandLine cli, SolrClient solrClient) throws Exception { + protected void createCore(CommandLine cli, SolrClient solrClient, String solrUrl) + throws Exception { String coreName = cli.getOptionValue(COLLECTION_NAME_OPTION); - String solrUrl = CLIUtils.normalizeSolrUrl(cli); final String solrInstallDir = EnvUtils.getProperty("solr.install.dir"); final String confDirName = @@ -189,7 +191,8 @@ protected void createCore(CommandLine cli, SolrClient solrClient) throws Excepti } } - protected void createCollection(CommandLine cli, SolrClient solrClient) throws Exception { + protected void createCollection(CommandLine cli, SolrClient solrClient, String solrUrl) + throws Exception { String collectionName = cli.getOptionValue(COLLECTION_NAME_OPTION); final String solrInstallDir = EnvUtils.getProperty("solr.install.dir"); @@ -200,8 +203,6 @@ protected void createCollection(CommandLine cli, SolrClient solrClient) throws E ensureConfDirExists(solrInstallDirPath, confDirPath); printDefaultConfigsetWarningIfNecessary(cli); - String solrUrl = CLIUtils.normalizeSolrUrl(cli); - // build a URL to create the collection int numShards = cli.getParsedOptionValue(SHARDS_OPTION, 1); int replicationFactor = cli.getParsedOptionValue(REPLICATION_FACTOR_OPTION, 1); @@ -213,7 +214,7 @@ protected void createCollection(CommandLine cli, SolrClient solrClient) throws E if (configExists) { echo("Re-using existing configuration directory " + confName); - } else { // if (confdir != null && !confdir.trim().isEmpty()) { + } else { if (confName == null || confName.trim().isEmpty()) { confName = collectionName; } @@ -275,9 +276,8 @@ protected void createCollection(CommandLine cli, SolrClient solrClient) throws E /** * Zips the contents of a configset directory for upload. * - *

Delegates to {@link ConfigSetService#zipDirectory}, which is shared with {@link - * org.apache.solr.handler.configsets.DownloadConfigSet#zipConfigSet} and {@link - * org.apache.solr.handler.designer.SchemaDesignerConfigSetHelper#downloadAndZipConfigSet}. + *

Delegates to {@link ConfigSetService#zipDirectory}, which is also used to zip a configset + * for download. */ static byte[] zipConfigSet(Path confPath) throws IOException { return ConfigSetService.zipDirectory(confPath, true); From 8372c52e2461669e61dd0eacad423232b2ab382d Mon Sep 17 00:00:00 2001 From: Eric Pugh Date: Sun, 27 Sep 2026 20:46:03 -0700 Subject: [PATCH 08/11] Move configset zip-directory logic from ConfigSetService into SolrJ MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Review feedback: CreateTool (a client) shouldn't reach into solr/core's ConfigSetService to zip a directory, and it's not the client's place to enforce which file types are forbidden in configsets anyway — that's a server concern. Move zipDirectory into SolrJ's Utils without the validateFileTypes/isFileForbiddenInConfigSets check, and update CreateTool and DownloadConfigSet to call it there. Server-side forbidden-file-type enforcement is unaffected and still covered by TestConfigSetsAPI. Co-Authored-By: Claude Sonnet 5 Claude-Session: https://claude.ai/code/session_01HaP4sXv7KQBw9ZEqtvwug2 --- .../java/org/apache/solr/cli/CreateTool.java | 10 +-- .../apache/solr/core/ConfigSetService.java | 65 ------------------- .../handler/configsets/DownloadConfigSet.java | 3 +- .../org/apache/solr/cli/CreateToolTest.java | 10 --- .../org/apache/solr/common/util/Utils.java | 56 ++++++++++++++++ 5 files changed, 61 insertions(+), 83 deletions(-) diff --git a/solr/core/src/java/org/apache/solr/cli/CreateTool.java b/solr/core/src/java/org/apache/solr/cli/CreateTool.java index 5700e25589e9..273f0fb91e3b 100644 --- a/solr/core/src/java/org/apache/solr/cli/CreateTool.java +++ b/solr/core/src/java/org/apache/solr/cli/CreateTool.java @@ -35,6 +35,7 @@ import org.apache.solr.client.solrj.request.SystemInfoRequest; import org.apache.solr.client.solrj.response.SystemInfoResponse; import org.apache.solr.common.util.EnvUtils; +import org.apache.solr.common.util.Utils; import org.apache.solr.core.ConfigSetService; /** Supports create command in the bin/solr script. */ @@ -273,14 +274,9 @@ protected void createCollection(CommandLine cli, SolrClient solrClient, String s echo(endMessage); } - /** - * Zips the contents of a configset directory for upload. - * - *

Delegates to {@link ConfigSetService#zipDirectory}, which is also used to zip a configset - * for download. - */ + /** Zips the contents of a configset directory for upload. */ static byte[] zipConfigSet(Path confPath) throws IOException { - return ConfigSetService.zipDirectory(confPath, true); + return Utils.zipDirectory(confPath); } private Path getFullConfDir(Path solrInstallDir, Path confDirName) { diff --git a/solr/core/src/java/org/apache/solr/core/ConfigSetService.java b/solr/core/src/java/org/apache/solr/core/ConfigSetService.java index eaf479f02c8c..6976e65c84ba 100644 --- a/solr/core/src/java/org/apache/solr/core/ConfigSetService.java +++ b/solr/core/src/java/org/apache/solr/core/ConfigSetService.java @@ -18,22 +18,16 @@ import com.github.benmanes.caffeine.cache.Cache; import com.github.benmanes.caffeine.cache.Caffeine; -import java.io.ByteArrayOutputStream; import java.io.IOException; import java.lang.invoke.MethodHandles; import java.lang.reflect.Constructor; -import java.nio.file.FileVisitResult; import java.nio.file.Files; import java.nio.file.Path; -import java.nio.file.SimpleFileVisitor; -import java.nio.file.attribute.BasicFileAttributes; import java.util.List; import java.util.Locale; import java.util.Map; import java.util.Set; import java.util.regex.Pattern; -import java.util.zip.ZipEntry; -import java.util.zip.ZipOutputStream; import org.apache.solr.cloud.ZkConfigSetService; import org.apache.solr.cloud.ZkController; import org.apache.solr.common.ConfigNode; @@ -81,65 +75,6 @@ public static boolean isFileForbiddenInConfigSets(String filePath) { return lastDot >= 0 && USE_FORBIDDEN_FILE_TYPES.contains(filePath.substring(lastDot + 1)); } - /** - * Zips the contents of {@code rootPath} into an in-memory archive. Hidden files and directories - * (as determined by {@link Files#isHidden}) are skipped, directory entries are written for - * non-empty subdirectories, and zip entry names are normalized to use {@code /} separators - * regardless of platform. - * - * @param rootPath the directory to zip - * @param validateFileTypes if true, a file with a forbidden extension (see {@link - * #isFileForbiddenInConfigSets}) causes an {@link IllegalArgumentException} instead of being - * included - * @return the zipped bytes - */ - public static byte[] zipDirectory(Path rootPath, boolean validateFileTypes) throws IOException { - ByteArrayOutputStream baos = new ByteArrayOutputStream(); - try (ZipOutputStream zipOut = new ZipOutputStream(baos)) { - Files.walkFileTree( - rootPath, - new SimpleFileVisitor<>() { - @Override - public FileVisitResult preVisitDirectory(Path dir, BasicFileAttributes attrs) - throws IOException { - if (Files.isHidden(dir)) { - return FileVisitResult.SKIP_SUBTREE; - } - String dirName = rootPath.relativize(dir).toString().replace('\\', '/'); - if (!dirName.isEmpty()) { - if (!dirName.endsWith("/")) { - dirName += "/"; - } - zipOut.putNextEntry(new ZipEntry(dirName)); - zipOut.closeEntry(); - } - return FileVisitResult.CONTINUE; - } - - @Override - public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) - throws IOException { - if (Files.isHidden(file)) { - return FileVisitResult.CONTINUE; - } - String filename = file.getFileName().toString(); - if (validateFileTypes && isFileForbiddenInConfigSets(filename)) { - throw new IllegalArgumentException( - "The file type provided for upload, '" - + filename - + "', is forbidden for use in uploading configsets."); - } - String entryName = rootPath.relativize(file).toString().replace('\\', '/'); - zipOut.putNextEntry(new ZipEntry(entryName)); - Files.copy(file, zipOut); - zipOut.closeEntry(); - return FileVisitResult.CONTINUE; - } - }); - } - return baos.toByteArray(); - } - private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); public static ConfigSetService createConfigSetService(CoreContainer coreContainer) { diff --git a/solr/core/src/java/org/apache/solr/handler/configsets/DownloadConfigSet.java b/solr/core/src/java/org/apache/solr/handler/configsets/DownloadConfigSet.java index cdd695d9975c..76d12758db01 100644 --- a/solr/core/src/java/org/apache/solr/handler/configsets/DownloadConfigSet.java +++ b/solr/core/src/java/org/apache/solr/handler/configsets/DownloadConfigSet.java @@ -28,6 +28,7 @@ import org.apache.solr.client.api.endpoint.ConfigsetsApi; import org.apache.solr.common.SolrException; import org.apache.solr.common.util.StrUtils; +import org.apache.solr.common.util.Utils; import org.apache.solr.core.ConfigSetService; import org.apache.solr.core.CoreContainer; import org.apache.solr.jersey.PermissionName; @@ -82,7 +83,7 @@ public static byte[] zipConfigSet(ConfigSetService configSetService, String conf Path tmpDirectory = Files.createTempDirectory("configset-download-"); try { configSetService.downloadConfig(configSetName, tmpDirectory); - return ConfigSetService.zipDirectory(tmpDirectory, false); + return Utils.zipDirectory(tmpDirectory); } finally { PathUtils.deleteDirectory(tmpDirectory); } diff --git a/solr/core/src/test/org/apache/solr/cli/CreateToolTest.java b/solr/core/src/test/org/apache/solr/cli/CreateToolTest.java index d288391a358b..5d58b6933f4f 100644 --- a/solr/core/src/test/org/apache/solr/cli/CreateToolTest.java +++ b/solr/core/src/test/org/apache/solr/cli/CreateToolTest.java @@ -107,14 +107,4 @@ public void testZipConfigSetSkipsHiddenFilesAndIncludesDirectoryEntries() throws assertFalse(entryNames.contains(".hidden-file")); assertTrue(entryNames.stream().noneMatch(name -> name.startsWith(".hiddenDir"))); } - - @Test - public void testZipConfigSetRejectsForbiddenFileType() throws Exception { - Path confDir = createTempDir("zipConfigSetForbiddenTest"); - Files.writeString(confDir.resolve("evil.jar"), "not really a jar"); - - IllegalArgumentException thrown = - expectThrows(IllegalArgumentException.class, () -> CreateTool.zipConfigSet(confDir)); - assertTrue(thrown.getMessage().contains("forbidden")); - } } diff --git a/solr/solrj/src/java/org/apache/solr/common/util/Utils.java b/solr/solrj/src/java/org/apache/solr/common/util/Utils.java index 71cdbb59c23c..810cf0861328 100644 --- a/solr/solrj/src/java/org/apache/solr/common/util/Utils.java +++ b/solr/solrj/src/java/org/apache/solr/common/util/Utils.java @@ -43,6 +43,11 @@ import java.nio.BufferOverflowException; import java.nio.ByteBuffer; import java.nio.charset.StandardCharsets; +import java.nio.file.FileVisitResult; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.SimpleFileVisitor; +import java.nio.file.attribute.BasicFileAttributes; import java.security.MessageDigest; import java.security.NoSuchAlgorithmException; import java.util.AbstractMap; @@ -69,6 +74,8 @@ import java.util.function.Predicate; import java.util.regex.Matcher; import java.util.regex.Pattern; +import java.util.zip.ZipEntry; +import java.util.zip.ZipOutputStream; import org.apache.solr.common.IteratorWriter; import org.apache.solr.common.LinkedHashMapWriter; import org.apache.solr.common.MapWriter; @@ -1045,4 +1052,53 @@ public static ByteBuffer toByteArray(InputStream is) throws IOException { return bos.getByteBuffer(); } } + + /** + * Zips the contents of {@code rootPath} into an in-memory archive. Hidden files and directories + * (as determined by {@link Files#isHidden}) are skipped, directory entries are written for + * non-empty subdirectories, and zip entry names are normalized to use {@code /} separators + * regardless of platform. + * + * @param rootPath the directory to zip + * @return the zipped bytes + */ + public static byte[] zipDirectory(Path rootPath) throws IOException { + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + try (ZipOutputStream zipOut = new ZipOutputStream(baos)) { + Files.walkFileTree( + rootPath, + new SimpleFileVisitor<>() { + @Override + public FileVisitResult preVisitDirectory(Path dir, BasicFileAttributes attrs) + throws IOException { + if (Files.isHidden(dir)) { + return FileVisitResult.SKIP_SUBTREE; + } + String dirName = rootPath.relativize(dir).toString().replace('\\', '/'); + if (!dirName.isEmpty()) { + if (!dirName.endsWith("/")) { + dirName += "/"; + } + zipOut.putNextEntry(new ZipEntry(dirName)); + zipOut.closeEntry(); + } + return FileVisitResult.CONTINUE; + } + + @Override + public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) + throws IOException { + if (Files.isHidden(file)) { + return FileVisitResult.CONTINUE; + } + String entryName = rootPath.relativize(file).toString().replace('\\', '/'); + zipOut.putNextEntry(new ZipEntry(entryName)); + Files.copy(file, zipOut); + zipOut.closeEntry(); + return FileVisitResult.CONTINUE; + } + }); + } + return baos.toByteArray(); + } } From 4eb4281a02236c6a741f1198f72926d13c9b665b Mon Sep 17 00:00:00 2001 From: Eric Pugh Date: Sun, 27 Sep 2026 20:46:09 -0700 Subject: [PATCH 09/11] Have RemoteSolrBackend reuse the shared Utils.zipDirectory RemoteSolrBackend had its own bespoke directory-to-zip-file helper duplicating what just moved to SolrJ's Utils.zipDirectory. Delegate to it instead of maintaining a second implementation. Co-Authored-By: Claude Sonnet 5 Claude-Session: https://claude.ai/code/session_01HaP4sXv7KQBw9ZEqtvwug2 --- .../apache/solr/cloud/RemoteSolrBackend.java | 23 ++----------------- 1 file changed, 2 insertions(+), 21 deletions(-) diff --git a/solr/test-framework/src/java/org/apache/solr/cloud/RemoteSolrBackend.java b/solr/test-framework/src/java/org/apache/solr/cloud/RemoteSolrBackend.java index 78cb06ecee0a..3ef526a01762 100644 --- a/solr/test-framework/src/java/org/apache/solr/cloud/RemoteSolrBackend.java +++ b/solr/test-framework/src/java/org/apache/solr/cloud/RemoteSolrBackend.java @@ -20,8 +20,6 @@ import java.nio.file.Files; import java.nio.file.Path; import java.util.List; -import java.util.zip.ZipEntry; -import java.util.zip.ZipOutputStream; import org.apache.solr.SolrBackend; import org.apache.solr.client.solrj.SolrClient; import org.apache.solr.client.solrj.SolrServerException; @@ -30,6 +28,7 @@ import org.apache.solr.client.solrj.request.ConfigSetAdminRequest; import org.apache.solr.common.util.IOUtils; import org.apache.solr.common.util.URLUtil; +import org.apache.solr.common.util.Utils; import org.apache.solr.core.CoreContainer; /** @@ -68,7 +67,7 @@ public SolrClient getSolrClient() { public void uploadConfigSet(Path configDir, String name) throws SolrServerException, IOException { Path tempZip = Files.createTempFile("configset-", ".zip"); try { - zipDirectory(configDir, tempZip); + Files.write(tempZip, Utils.zipDirectory(configDir)); new ConfigSetAdminRequest.Upload() .setConfigSetName(name) .setUploadFile(tempZip, "application/zip") @@ -83,24 +82,6 @@ public boolean hasConfigSet(String name) throws SolrServerException, IOException return new ConfigSetAdminRequest.List().process(adminClient).getConfigSets().contains(name); } - /** Packages {@code sourceDir} contents into a zip file. */ - private static void zipDirectory(Path sourceDir, Path targetZip) throws IOException { - try (ZipOutputStream zos = new ZipOutputStream(Files.newOutputStream(targetZip))) { - Files.walk(sourceDir) - .filter(Files::isRegularFile) - .forEach( - file -> { - try { - zos.putNextEntry(new ZipEntry(sourceDir.relativize(file).toString())); - Files.copy(file, zos); - zos.closeEntry(); - } catch (IOException e) { - throw new RuntimeException(e); - } - }); - } - } - @Override public void createCollection(CollectionAdminRequest.Create create) throws SolrServerException, IOException { From 134d2f635434bf9bee64ab6d577b785a7047b0ff Mon Sep 17 00:00:00 2001 From: Eric Pugh Date: Sun, 27 Sep 2026 20:56:06 -0700 Subject: [PATCH 10/11] Document why zk upconfig doesn't use the Configsets V2 API Explored migrating this onto the same V2 API path CreateTool now uses (SOLR-18320), but ZkSubcommandsTest.testUpConfigDownConfigClearZk exposed why that's wrong: this command must work against a bare ZooKeeper ensemble with no Solr node running yet, and the V2 API requires an HTTP endpoint on an already-running node. Leaving the direct-ZK path in place and documenting why, so the divergence from CreateTool isn't mistaken for an oversight. Co-Authored-By: Claude Sonnet 5 Claude-Session: https://claude.ai/code/session_01HaP4sXv7KQBw9ZEqtvwug2 --- .../java/org/apache/solr/cli/ConfigSetUploadTool.java | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/solr/core/src/java/org/apache/solr/cli/ConfigSetUploadTool.java b/solr/core/src/java/org/apache/solr/cli/ConfigSetUploadTool.java index 71daf15d3386..08b971dddc1c 100644 --- a/solr/core/src/java/org/apache/solr/cli/ConfigSetUploadTool.java +++ b/solr/core/src/java/org/apache/solr/cli/ConfigSetUploadTool.java @@ -29,7 +29,14 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; -/** Supports zk upconfig command in the bin/solr script. */ +/** + * Supports zk upconfig command in the bin/solr script. + * + *

Unlike {@link CreateTool}, this uploads directly to ZooKeeper rather than going through the + * Configsets V2 API. That's intentional: this command is meant to work against a bare ZooKeeper + * ensemble with no Solr node running yet (e.g. provisioning configs before the cluster starts), + * and the V2 API requires an HTTP endpoint on an already-running node to PUT the zip to. + */ public class ConfigSetUploadTool extends ToolBase { private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); From e9c78b4aaf4731fa534a7206405abd251ee3f07f Mon Sep 17 00:00:00 2001 From: Eric Pugh Date: Tue, 27 Oct 2026 21:22:26 -0400 Subject: [PATCH 11/11] Apply tidy formatting to merge-resolved ConfigSetUploadTool javadoc --- .../src/java/org/apache/solr/cli/ConfigSetUploadTool.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/solr/core/src/java/org/apache/solr/cli/ConfigSetUploadTool.java b/solr/core/src/java/org/apache/solr/cli/ConfigSetUploadTool.java index 9ba3441977e3..20cf3b5820e5 100644 --- a/solr/core/src/java/org/apache/solr/cli/ConfigSetUploadTool.java +++ b/solr/core/src/java/org/apache/solr/cli/ConfigSetUploadTool.java @@ -36,8 +36,8 @@ * *

Unlike {@link CreateTool}, this uploads directly to ZooKeeper rather than going through the * Configsets V2 API. That's intentional: this command is meant to work against a bare ZooKeeper - * ensemble with no Solr node running yet (e.g. provisioning configs before the cluster starts), - * and the V2 API requires an HTTP endpoint on an already-running node to PUT the zip to. + * ensemble with no Solr node running yet (e.g. provisioning configs before the cluster starts), and + * the V2 API requires an HTTP endpoint on an already-running node to PUT the zip to. */ @SuppressWarnings("UnnecessarilyFullyQualified") @picocli.CommandLine.Command(