From 129aa4a84440b0e9e33874e9d98d545f25621305 Mon Sep 17 00:00:00 2001 From: Zihan Dai Date: Mon, 31 Aug 2026 02:58:59 +1000 Subject: [PATCH] [lumina] Release the vector file when FileBackedDataset fails to construct FileBackedDataset opens a RandomAccessFile and then validates the record size and allocates the read buffer, either of which can throw. A caller using try-with-resources never receives an object whose constructor threw, so nothing closes that handle. The enclosing writer already guards its own file this way; this brings the nested reader in line. --- .../index/LuminaVectorGlobalIndexWriter.java | 28 +++++--- .../LuminaFileBackedDatasetCloseTest.java | 69 +++++++++++++++++++ 2 files changed, 87 insertions(+), 10 deletions(-) create mode 100644 paimon-lumina/src/test/java/org/apache/paimon/lumina/index/LuminaFileBackedDatasetCloseTest.java diff --git a/paimon-lumina/src/main/java/org/apache/paimon/lumina/index/LuminaVectorGlobalIndexWriter.java b/paimon-lumina/src/main/java/org/apache/paimon/lumina/index/LuminaVectorGlobalIndexWriter.java index 9d9ebeaad8d8..53920ac04458 100644 --- a/paimon-lumina/src/main/java/org/apache/paimon/lumina/index/LuminaVectorGlobalIndexWriter.java +++ b/paimon-lumina/src/main/java/org/apache/paimon/lumina/index/LuminaVectorGlobalIndexWriter.java @@ -28,6 +28,7 @@ import org.apache.paimon.types.DataType; import org.apache.paimon.types.FloatType; import org.apache.paimon.types.VectorType; +import org.apache.paimon.utils.IOUtils; import org.aliyun.lumina.LuminaDataset; import org.aliyun.lumina.LuminaFileOutput; @@ -427,16 +428,23 @@ static class FileBackedDataset implements LuminaDataset, Closeable { FileBackedDataset(File file, int dim, long totalCount, String phase, int bufferSize) throws IOException { this.raf = new RandomAccessFile(file, "r"); - this.channel = raf.getChannel(); - this.dim = dim; - this.totalCount = totalCount; - this.recordSizeInBytes = checkedRecordSize(dim, bufferSize); - this.cursor = 0; - this.readBuf = ByteBuffer.allocateDirect(bufferSize); - this.readBuf.order(ByteOrder.nativeOrder()); - this.readBuf.limit(0); // empty initially - this.phase = phase; - this.lastLoggedPercent = -1; + try { + this.channel = raf.getChannel(); + this.dim = dim; + this.totalCount = totalCount; + this.recordSizeInBytes = checkedRecordSize(dim, bufferSize); + this.cursor = 0; + this.readBuf = ByteBuffer.allocateDirect(bufferSize); + this.readBuf.order(ByteOrder.nativeOrder()); + this.readBuf.limit(0); // empty initially + this.phase = phase; + this.lastLoggedPercent = -1; + } catch (RuntimeException e) { + // a caller's try-with-resources never sees an object whose construction failed, + // so this is the only chance to release the file handle + IOUtils.closeQuietly(raf); + throw e; + } } @Override diff --git a/paimon-lumina/src/test/java/org/apache/paimon/lumina/index/LuminaFileBackedDatasetCloseTest.java b/paimon-lumina/src/test/java/org/apache/paimon/lumina/index/LuminaFileBackedDatasetCloseTest.java new file mode 100644 index 000000000000..a51874fc7c31 --- /dev/null +++ b/paimon-lumina/src/test/java/org/apache/paimon/lumina/index/LuminaFileBackedDatasetCloseTest.java @@ -0,0 +1,69 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.lumina.index; + +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +import java.io.File; +import java.io.FileOutputStream; +import java.nio.file.Path; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; +import static org.junit.jupiter.api.Assumptions.assumeTrue; + +/** + * Tests that {@link LuminaVectorGlobalIndexWriter.FileBackedDataset} releases the file it opened + * when the rest of its construction fails. + */ +public class LuminaFileBackedDatasetCloseTest { + + @TempDir Path tempDir; + + /** + * checkedRecordSize rejects a dimension whose record does not fit the read buffer, and it runs + * after the file is open. A caller's try-with-resources never sees an object whose constructor + * threw, so nothing would close that handle. + */ + @Test + public void testFailedConstructionReleasesTheFile() throws Exception { + File fdDir = new File("/dev/fd"); + assumeTrue(fdDir.isDirectory() && fdDir.list() != null, "needs /dev/fd"); + + File file = new File(tempDir.toFile(), "vectors.bin"); + try (FileOutputStream out = new FileOutputStream(file)) { + out.write(new byte[64]); + } + + int before = fdDir.list().length; + for (int i = 0; i < 200; i++) { + assertThatThrownBy( + () -> + new LuminaVectorGlobalIndexWriter.FileBackedDataset( + file, Integer.MAX_VALUE / 4, 1L, "test", 4096)) + .isInstanceOf(IllegalStateException.class); + } + int after = fdDir.list().length; + + assertThat(after - before) + .as("200 failed constructions must not strand 200 descriptors") + .isLessThan(50); + } +}