diff --git a/paimon-benchmark/paimon-micro-benchmarks/src/test/java/org/apache/paimon/benchmark/ManifestFileSorterBenchmark.java b/paimon-benchmark/paimon-micro-benchmarks/src/test/java/org/apache/paimon/benchmark/ManifestFileSorterBenchmark.java index 937906490c6c..5a5f31d875d0 100644 --- a/paimon-benchmark/paimon-micro-benchmarks/src/test/java/org/apache/paimon/benchmark/ManifestFileSorterBenchmark.java +++ b/paimon-benchmark/paimon-micro-benchmarks/src/test/java/org/apache/paimon/benchmark/ManifestFileSorterBenchmark.java @@ -33,7 +33,7 @@ import org.apache.paimon.manifest.ManifestFileMeta; import org.apache.paimon.operation.ManifestFileMerger; import org.apache.paimon.options.Options; -import org.apache.paimon.schema.SchemaManager; +import org.apache.paimon.schema.FileSystemSchemaManager; import org.apache.paimon.stats.SimpleStats; import org.apache.paimon.types.IntType; import org.apache.paimon.types.RowType; @@ -294,7 +294,7 @@ private ManifestFile createManifestFile() { null); return new ManifestFile.Factory( fileIO, - new SchemaManager(fileIO, tablePath), + new FileSystemSchemaManager(fileIO, tablePath), PARTITION_TYPE, FileFormat.fromIdentifier("avro", new Options()), "zstd", diff --git a/paimon-core/src/main/java/org/apache/paimon/catalog/AbstractCatalog.java b/paimon-core/src/main/java/org/apache/paimon/catalog/AbstractCatalog.java index 5784d56b0ed4..e0f02de21ce8 100644 --- a/paimon-core/src/main/java/org/apache/paimon/catalog/AbstractCatalog.java +++ b/paimon-core/src/main/java/org/apache/paimon/catalog/AbstractCatalog.java @@ -34,6 +34,7 @@ import org.apache.paimon.partition.Partition; import org.apache.paimon.partition.PartitionStatistics; import org.apache.paimon.rest.responses.GetTagResponse; +import org.apache.paimon.schema.FileSystemSchemaManager; import org.apache.paimon.schema.Schema; import org.apache.paimon.schema.SchemaChange; import org.apache.paimon.schema.SchemaManager; @@ -886,7 +887,8 @@ protected List listTablesInFileSystem(Path databasePath) throws IOExcept } protected boolean tableExistsInFileSystem(Path tablePath, String branchName) { - SchemaManager schemaManager = new SchemaManager(fileIO(tablePath), tablePath, branchName); + SchemaManager schemaManager = + new FileSystemSchemaManager(fileIO(tablePath), tablePath, branchName); // in order to improve the performance, check the schema-0 firstly. boolean schemaZeroExists = schemaManager.schemaExists(0); @@ -900,7 +902,7 @@ protected boolean tableExistsInFileSystem(Path tablePath, String branchName) { public Optional tableSchemaInFileSystem(Path tablePath, String branchName) { Optional schema = - new SchemaManager(fileIO(tablePath), tablePath, branchName).latest(); + new FileSystemSchemaManager(fileIO(tablePath), tablePath, branchName).latest(); if (!DEFAULT_MAIN_BRANCH.equals(branchName)) { schema = schema.map( diff --git a/paimon-core/src/main/java/org/apache/paimon/catalog/FileSystemCatalog.java b/paimon-core/src/main/java/org/apache/paimon/catalog/FileSystemCatalog.java index b0b8b1ac7f30..cb68850a1920 100644 --- a/paimon-core/src/main/java/org/apache/paimon/catalog/FileSystemCatalog.java +++ b/paimon-core/src/main/java/org/apache/paimon/catalog/FileSystemCatalog.java @@ -21,6 +21,7 @@ import org.apache.paimon.fs.FileIO; import org.apache.paimon.fs.Path; import org.apache.paimon.operation.Lock; +import org.apache.paimon.schema.FileSystemSchemaManager; import org.apache.paimon.schema.Schema; import org.apache.paimon.schema.SchemaChange; import org.apache.paimon.schema.SchemaManager; @@ -151,7 +152,7 @@ public T runWithLock(Identifier identifier, Callable callable) throws Exc private SchemaManager schemaManager(Identifier identifier) { Path path = getTableLocation(identifier); - return new SchemaManager(fileIO, path, identifier.getBranchNameOrDefault()); + return new FileSystemSchemaManager(fileIO, path, identifier.getBranchNameOrDefault()); } @Override diff --git a/paimon-core/src/main/java/org/apache/paimon/iceberg/IcebergCommitCallback.java b/paimon-core/src/main/java/org/apache/paimon/iceberg/IcebergCommitCallback.java index cfeb6acb1c7d..5f2f5ff0e5f6 100644 --- a/paimon-core/src/main/java/org/apache/paimon/iceberg/IcebergCommitCallback.java +++ b/paimon-core/src/main/java/org/apache/paimon/iceberg/IcebergCommitCallback.java @@ -52,6 +52,7 @@ import org.apache.paimon.manifest.ManifestEntry; import org.apache.paimon.options.Options; import org.apache.paimon.partition.PartitionPredicate; +import org.apache.paimon.schema.FileSystemSchemaManager; import org.apache.paimon.schema.SchemaManager; import org.apache.paimon.schema.TableSchema; import org.apache.paimon.table.FileStoreTable; @@ -1957,7 +1958,7 @@ private boolean isSameFormatVersion(int baseFormatVersion) { private class SchemaCache { - SchemaManager schemaManager = new SchemaManager(table.fileIO(), table.location()); + SchemaManager schemaManager = new FileSystemSchemaManager(table.fileIO(), table.location()); Map schemas = new HashMap<>(); private IcebergSchema get(long schemaId) { diff --git a/paimon-core/src/main/java/org/apache/paimon/index/FileIndexProcessor.java b/paimon-core/src/main/java/org/apache/paimon/index/FileIndexProcessor.java index 84e48c553869..e1714ee77fab 100644 --- a/paimon-core/src/main/java/org/apache/paimon/index/FileIndexProcessor.java +++ b/paimon-core/src/main/java/org/apache/paimon/index/FileIndexProcessor.java @@ -31,6 +31,7 @@ import org.apache.paimon.manifest.ManifestEntry; import org.apache.paimon.options.Options; import org.apache.paimon.reader.RecordReader; +import org.apache.paimon.schema.FileSystemSchemaManager; import org.apache.paimon.schema.SchemaManager; import org.apache.paimon.schema.TableSchema; import org.apache.paimon.table.FileStoreTable; @@ -75,7 +76,8 @@ public FileIndexProcessor(FileStoreTable table) { this.pathFactory = table.store().pathFactory(); this.pathFactories = new DataFilePathFactories(pathFactory); this.schemaInfoCache = - new SchemaCache(fileIndexOptions, new SchemaManager(fileIO, table.location())); + new SchemaCache( + fileIndexOptions, new FileSystemSchemaManager(fileIO, table.location())); this.sizeInMeta = table.coreOptions().fileIndexInManifestThreshold(); } diff --git a/paimon-core/src/main/java/org/apache/paimon/jdbc/JdbcCatalog.java b/paimon-core/src/main/java/org/apache/paimon/jdbc/JdbcCatalog.java index 484ac3803bca..e73db8a5c007 100644 --- a/paimon-core/src/main/java/org/apache/paimon/jdbc/JdbcCatalog.java +++ b/paimon-core/src/main/java/org/apache/paimon/jdbc/JdbcCatalog.java @@ -38,6 +38,7 @@ import org.apache.paimon.operation.Lock; import org.apache.paimon.options.CatalogOptions; import org.apache.paimon.options.Options; +import org.apache.paimon.schema.FileSystemSchemaManager; import org.apache.paimon.schema.Schema; import org.apache.paimon.schema.SchemaChange; import org.apache.paimon.schema.SchemaManager; @@ -668,7 +669,7 @@ protected void renameTableImpl(Identifier fromTable, Identifier toTable) { } Path fromPath = getTableLocation(fromTable); - if (!new SchemaManager(fileIO, fromPath).listAllIds().isEmpty()) { + if (!new FileSystemSchemaManager(fileIO, fromPath).listAllIds().isEmpty()) { // Rename the file system's table directory. Maintain consistency between tables in // the file system and tables in the Hive Metastore. Path toPath = getTableLocation(toTable); @@ -975,7 +976,7 @@ private Map collectTableProperties(TableSchema tableSchema) { } private SchemaManager getSchemaManager(Identifier identifier) { - return new SchemaManager(fileIO, getTableLocation(identifier)); + return new FileSystemSchemaManager(fileIO, getTableLocation(identifier)); } private Map fetchProperties(String databaseName) { diff --git a/paimon-core/src/main/java/org/apache/paimon/rest/RESTCatalog.java b/paimon-core/src/main/java/org/apache/paimon/rest/RESTCatalog.java index f1b5d94cc1a5..754c63996fe6 100644 --- a/paimon-core/src/main/java/org/apache/paimon/rest/RESTCatalog.java +++ b/paimon-core/src/main/java/org/apache/paimon/rest/RESTCatalog.java @@ -56,6 +56,7 @@ import org.apache.paimon.rest.responses.GetTableResponse; import org.apache.paimon.rest.responses.GetTagResponse; import org.apache.paimon.rest.responses.GetViewResponse; +import org.apache.paimon.schema.FileSystemSchemaManager; import org.apache.paimon.schema.Schema; import org.apache.paimon.schema.SchemaChange; import org.apache.paimon.schema.SchemaManager; @@ -1411,7 +1412,7 @@ private Schema inferSchemaIfExternalPaimonTable(Schema schema) throws Exception if (TableType.TABLE.equals(tableType) && Objects.nonNull(externalLocation)) { Path externalPath = new Path(externalLocation); SchemaManager schemaManager = - new SchemaManager(fileIOFromOptions(externalPath), externalPath); + new FileSystemSchemaManager(fileIOFromOptions(externalPath), externalPath); Optional latest = schemaManager.latest(); if (latest.isPresent()) { // Note we just validate schema here, will not create a new table diff --git a/paimon-core/src/main/java/org/apache/paimon/schema/FileSystemSchemaManager.java b/paimon-core/src/main/java/org/apache/paimon/schema/FileSystemSchemaManager.java new file mode 100644 index 000000000000..2665ef0427a7 --- /dev/null +++ b/paimon-core/src/main/java/org/apache/paimon/schema/FileSystemSchemaManager.java @@ -0,0 +1,1516 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.paimon.schema; + +import org.apache.paimon.CoreOptions; +import org.apache.paimon.annotation.VisibleForTesting; +import org.apache.paimon.casting.CastExecutors; +import org.apache.paimon.catalog.Catalog; +import org.apache.paimon.catalog.Identifier; +import org.apache.paimon.fs.FileIO; +import org.apache.paimon.fs.Path; +import org.apache.paimon.schema.ColumnDirectiveUtils.ConvertedColumn; +import org.apache.paimon.schema.SchemaChange.AddColumn; +import org.apache.paimon.schema.SchemaChange.DropColumn; +import org.apache.paimon.schema.SchemaChange.RemoveOption; +import org.apache.paimon.schema.SchemaChange.RenameColumn; +import org.apache.paimon.schema.SchemaChange.SetOption; +import org.apache.paimon.schema.SchemaChange.UpdateColumnComment; +import org.apache.paimon.schema.SchemaChange.UpdateColumnDefaultValue; +import org.apache.paimon.schema.SchemaChange.UpdateColumnNullability; +import org.apache.paimon.schema.SchemaChange.UpdateColumnPosition; +import org.apache.paimon.schema.SchemaChange.UpdateColumnType; +import org.apache.paimon.schema.SchemaChange.UpdateComment; +import org.apache.paimon.table.FileStoreTableFactory; +import org.apache.paimon.table.SchemaModification; +import org.apache.paimon.types.ArrayType; +import org.apache.paimon.types.DataField; +import org.apache.paimon.types.DataType; +import org.apache.paimon.types.DataTypeCasts; +import org.apache.paimon.types.MapType; +import org.apache.paimon.types.ReassignFieldId; +import org.apache.paimon.types.RowType; +import org.apache.paimon.utils.BranchManager; +import org.apache.paimon.utils.ChangelogManager; +import org.apache.paimon.utils.LazyField; +import org.apache.paimon.utils.Preconditions; +import org.apache.paimon.utils.SnapshotManager; +import org.apache.paimon.utils.StringUtils; +import org.apache.paimon.utils.TagManager; + +import org.apache.paimon.shade.guava30.com.google.common.collect.FluentIterable; +import org.apache.paimon.shade.guava30.com.google.common.collect.ImmutableList; +import org.apache.paimon.shade.guava30.com.google.common.collect.Iterables; +import org.apache.paimon.shade.guava30.com.google.common.collect.Maps; +import org.apache.paimon.shade.guava30.com.google.common.collect.Streams; + +import javax.annotation.Nullable; +import javax.annotation.concurrent.ThreadSafe; + +import java.io.FileNotFoundException; +import java.io.IOException; +import java.io.UncheckedIOException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Optional; +import java.util.Set; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.function.BiFunction; +import java.util.function.Function; +import java.util.function.Predicate; +import java.util.stream.Collectors; + +import static org.apache.paimon.CoreOptions.AGG_FUNCTION; +import static org.apache.paimon.CoreOptions.BUCKET_KEY; +import static org.apache.paimon.CoreOptions.CLUSTERING_COLUMNS; +import static org.apache.paimon.CoreOptions.DELETION_VECTORS_ENABLED; +import static org.apache.paimon.CoreOptions.DELETION_VECTORS_MODIFIABLE; +import static org.apache.paimon.CoreOptions.DISTINCT; +import static org.apache.paimon.CoreOptions.FIELDS_PREFIX; +import static org.apache.paimon.CoreOptions.IGNORE_DELETE; +import static org.apache.paimon.CoreOptions.IGNORE_RETRACT; +import static org.apache.paimon.CoreOptions.IGNORE_UPDATE_BEFORE; +import static org.apache.paimon.CoreOptions.LIST_AGG_DELIMITER; +import static org.apache.paimon.CoreOptions.MAP_SHARED_SHREDDING_COLUMN_PLACEMENT_POLICY; +import static org.apache.paimon.CoreOptions.MAP_SHARED_SHREDDING_MAX_COLUMNS; +import static org.apache.paimon.CoreOptions.MAP_STORAGE_LAYOUT; +import static org.apache.paimon.CoreOptions.NESTED_KEY; +import static org.apache.paimon.CoreOptions.PK_CLUSTERING_OVERRIDE; +import static org.apache.paimon.CoreOptions.SEQUENCE_FIELD; +import static org.apache.paimon.catalog.AbstractCatalog.DB_SUFFIX; +import static org.apache.paimon.catalog.Identifier.DEFAULT_MAIN_BRANCH; +import static org.apache.paimon.catalog.Identifier.UNKNOWN_DATABASE; +import static org.apache.paimon.mergetree.compact.PartialUpdateMergeFunction.SEQUENCE_GROUP; +import static org.apache.paimon.schema.ColumnDirectiveUtils.applyAddColumnDirective; +import static org.apache.paimon.schema.ColumnDirectiveUtils.applyDirectives; +import static org.apache.paimon.types.BlobType.isBlobFileField; +import static org.apache.paimon.utils.DefaultValueUtils.validateDefaultValue; +import static org.apache.paimon.utils.FileUtils.listVersionedFiles; +import static org.apache.paimon.utils.Preconditions.checkArgument; +import static org.apache.paimon.utils.Preconditions.checkState; + +/** File system implementation of {@link SchemaManager}. */ +@ThreadSafe +public class FileSystemSchemaManager implements SchemaManager { + + public static final String SCHEMA_PREFIX = "schema-"; + + private final FileIO fileIO; + private final Path tableRoot; + + private final String branch; + + public FileSystemSchemaManager(FileIO fileIO, Path tableRoot) { + this(fileIO, tableRoot, DEFAULT_MAIN_BRANCH); + } + + /** Specify the default branch for data writing. */ + public FileSystemSchemaManager(FileIO fileIO, Path tableRoot, String branch) { + this.fileIO = fileIO; + this.tableRoot = tableRoot; + this.branch = BranchManager.normalizeBranch(branch); + } + + public FileSystemSchemaManager copyWithBranch(String branchName) { + return new FileSystemSchemaManager(fileIO, tableRoot, branchName); + } + + public Optional latest() { + try { + return listVersionedFiles(fileIO, schemaDirectory(), SCHEMA_PREFIX) + .reduce(Math::max) + .map(this::schema); + } catch (IOException e) { + throw new UncheckedIOException(e); + } + } + + public TableSchema latestOrThrow(String message) { + return latest().orElseThrow(() -> new RuntimeException(message)); + } + + public long earliestCreationTime() { + try { + long earliest = 0; + if (!schemaExists(0)) { + Optional min = + listVersionedFiles(fileIO, schemaDirectory(), SCHEMA_PREFIX) + .reduce(Math::min); + checkArgument(min.isPresent()); + earliest = min.get(); + } + + Path schemaPath = toSchemaPath(earliest); + return fileIO.getFileStatus(schemaPath).getModificationTime(); + } catch (IOException e) { + throw new UncheckedIOException(e); + } + } + + public List listAll() { + return listAllIds().stream().map(this::schema).collect(Collectors.toList()); + } + + /** List all schema IDs. */ + public List listAllIds() { + try { + return listVersionedFiles(fileIO, schemaDirectory(), SCHEMA_PREFIX) + .collect(Collectors.toList()); + } catch (IOException e) { + throw new UncheckedIOException(e); + } + } + + public TableSchema createTable(Schema schema) throws Exception { + return createTable(schema, false); + } + + public TableSchema createTable(Schema schema, boolean externalTable) throws Exception { + while (true) { + Optional latest = latest(); + if (latest.isPresent()) { + TableSchema latestSchema = latest.get(); + if (externalTable) { + checkSchemaForExternalTable(latestSchema.toSchema(), schema); + return latestSchema; + } else { + throw new IllegalStateException( + "Schema in filesystem exists, creation is not allowed."); + } + } + + schema = applyDirectives(schema); + TableSchema newSchema = TableSchema.create(0, schema); + + // validate table from creating table + FileStoreTableFactory.create(fileIO, tableRoot, newSchema).store(); + + boolean success = commit(newSchema); + if (success) { + return newSchema; + } + } + } + + private void checkSchemaForExternalTable(Schema existsSchema, Schema newSchema) { + // When creating an external table, if the table already exists in the location, we can + // choose not to specify the fields. + if ((newSchema.fields().isEmpty() + || newSchema.rowType().equalsIgnoreFieldId(existsSchema.rowType())) + && (newSchema.partitionKeys().isEmpty() + || Objects.equals(newSchema.partitionKeys(), existsSchema.partitionKeys())) + && (newSchema.primaryKeys().isEmpty() + || Objects.equals(newSchema.primaryKeys(), existsSchema.primaryKeys()))) { + // check for options + Map existsOptions = existsSchema.options(); + Map newOptions = newSchema.options(); + newOptions.forEach( + (key, value) -> { + // ignore `owner` and `path` + if (!key.equals(Catalog.OWNER_PROP) + && !key.equals(CoreOptions.PATH.key()) + && (!existsOptions.containsKey(key) + || !existsOptions.get(key).equals(value))) { + throw new RuntimeException( + "New schema's options are not equal to the exists schema's, new schema: " + + newOptions + + ", exists schema: " + + existsOptions); + } + }); + } else { + throw new RuntimeException( + "New schema is not equal to exists schema, new schema: " + + newSchema + + ", exists schema: " + + existsSchema); + } + } + + /** Update {@link SchemaChange}s. */ + public TableSchema commitChanges(SchemaChange... changes) throws Exception { + return commitChanges(Arrays.asList(changes)); + } + + /** Update {@link SchemaChange}s. */ + public TableSchema commitChanges(List changes) + throws Catalog.TableNotExistException, Catalog.ColumnAlreadyExistException, + Catalog.ColumnNotExistException { + SnapshotManager snapshotManager = + new SnapshotManager(fileIO, tableRoot, branch, null, null); + LazyField hasSnapshots = + new LazyField<>(() -> snapshotManager.latestSnapshot() != null); + + while (true) { + TableSchema oldTableSchema = + latest().orElseThrow( + () -> + new Catalog.TableNotExistException( + identifierFromPath( + tableRoot.toString(), true, branch))); + LazyField lazyIdentifier = + new LazyField<>(() -> identifierFromPath(tableRoot.toString(), true, branch)); + TableSchema newTableSchema = + generateTableSchema(oldTableSchema, changes, hasSnapshots, lazyIdentifier); + try { + boolean success = commit(newTableSchema); + if (success) { + return newTableSchema; + } + } catch (Exception e) { + throw new RuntimeException(e); + } + } + } + + public static TableSchema generateTableSchema( + TableSchema oldTableSchema, + List changes, + LazyField hasSnapshots, + LazyField lazyIdentifier) + throws Catalog.ColumnAlreadyExistException, Catalog.ColumnNotExistException { + Map oldOptions = new HashMap<>(oldTableSchema.options()); + Map newOptions = new HashMap<>(oldTableSchema.options()); + boolean disableNullToNotNull = + Boolean.parseBoolean( + oldOptions.getOrDefault( + CoreOptions.DISABLE_ALTER_COLUMN_NULL_TO_NOT_NULL.key(), + CoreOptions.DISABLE_ALTER_COLUMN_NULL_TO_NOT_NULL + .defaultValue() + .toString())); + + boolean disableExplicitTypeCasting = + Boolean.parseBoolean( + oldOptions.getOrDefault( + CoreOptions.DISABLE_EXPLICIT_TYPE_CASTING.key(), + CoreOptions.DISABLE_EXPLICIT_TYPE_CASTING + .defaultValue() + .toString())); + + boolean addColumnBeforePartition = + Boolean.parseBoolean( + oldOptions.getOrDefault( + CoreOptions.ADD_COLUMN_BEFORE_PARTITION.key(), + CoreOptions.ADD_COLUMN_BEFORE_PARTITION.defaultValue().toString())); + List partitionKeys = oldTableSchema.partitionKeys(); + + List newFields = new ArrayList<>(oldTableSchema.fields()); + AtomicInteger highestFieldId = new AtomicInteger(oldTableSchema.highestFieldId()); + String newComment = oldTableSchema.comment(); + List newPrimaryKeys = oldTableSchema.primaryKeys(); + for (SchemaChange change : changes) { + if (change instanceof SetOption) { + SetOption setOption = (SetOption) change; + String oldValue = oldOptions.get(setOption.key()); + String newValue = setOption.value(); + boolean unchanged = + Objects.equals(oldValue, newValue) + || isUnchangedNormalizedKey( + setOption.key(), oldValue, newValue, oldTableSchema); + // reject 'type' even without snapshots: format tables hold data but + // create no snapshots, so the snapshot check would not catch them + if (!unchanged && CoreOptions.TYPE.key().equals(setOption.key())) { + throw new UnsupportedOperationException("Change 'type' is not supported yet."); + } + if (hasSnapshots.get() && !unchanged) { + checkAlterTableOption(oldOptions, setOption.key(), oldValue, newValue); + } + if (!unchanged) { + newOptions.put(setOption.key(), setOption.value()); + } + } else if (change instanceof RemoveOption) { + RemoveOption removeOption = (RemoveOption) change; + if (CoreOptions.TYPE.key().equals(removeOption.key())) { + throw new UnsupportedOperationException("Change 'type' is not supported yet."); + } + if (hasSnapshots.get()) { + checkResetTableOption(oldOptions, removeOption.key()); + } + newOptions.remove(removeOption.key()); + } else if (change instanceof UpdateComment) { + UpdateComment updateComment = (UpdateComment) change; + newComment = updateComment.comment(); + } else if (change instanceof AddColumn) { + AddColumn addColumn = (AddColumn) change; + SchemaChange.Move move = addColumn.move(); + Preconditions.checkArgument( + addColumn.dataType().isNullable(), + "Column %s cannot specify NOT NULL in the %s table.", + String.join(".", addColumn.fieldNames()), + lazyIdentifier.get().getFullName()); + + ConvertedColumn converted = + applyAddColumnDirective( + addColumn.description(), + addColumn.fieldNames()[0], + addColumn.dataType(), + newOptions); + DataType requestedDataType = addColumn.dataType(); + String effectiveComment = addColumn.description(); + if (converted != null) { + Preconditions.checkArgument( + addColumn.fieldNames().length == 1, + "Comment directive cannot be used on a nested column %s.", + String.join(".", addColumn.fieldNames())); + requestedDataType = converted.type(); + effectiveComment = converted.comment(); + } + + int id = highestFieldId.incrementAndGet(); + DataType dataType = ReassignFieldId.reassign(requestedDataType, highestFieldId); + String storedComment = effectiveComment; + new NestedColumnModifier(addColumn.fieldNames(), lazyIdentifier) { + @Override + protected void updateLastColumn( + int depth, List newFields, String fieldName) + throws Catalog.ColumnAlreadyExistException, + Catalog.ColumnNotExistException { + assertColumnNotExists(newFields, fieldName, lazyIdentifier); + + DataField dataField = new DataField(id, fieldName, dataType, storedComment); + + // key: name ; value : index + Map map = new HashMap<>(); + for (int i = 0; i < newFields.size(); i++) { + map.put(newFields.get(i).name(), i); + } + + if (null != move) { + if (move.type().equals(SchemaChange.Move.MoveType.FIRST)) { + newFields.add(0, dataField); + } else if (move.type().equals(SchemaChange.Move.MoveType.AFTER)) { + if (map.containsKey(move.referenceFieldName())) { + int fieldIndex = map.get(move.referenceFieldName()); + newFields.add(fieldIndex + 1, dataField); + } else { + throw new Catalog.ColumnNotExistException( + lazyIdentifier.get(), move.referenceFieldName()); + } + } else if (move.type().equals(SchemaChange.Move.MoveType.BEFORE)) { + if (map.containsKey(move.referenceFieldName())) { + int fieldIndex = map.get(move.referenceFieldName()); + newFields.add(fieldIndex, dataField); + } else { + throw new Catalog.ColumnNotExistException( + lazyIdentifier.get(), move.referenceFieldName()); + } + } else if (move.type().equals(SchemaChange.Move.MoveType.LAST)) { + newFields.add(dataField); + } else { + throw new UnsupportedOperationException( + "Unsupported move type: " + move.type()); + } + } else if (addColumnBeforePartition + && !partitionKeys.isEmpty() + && addColumn.fieldNames().length == 1) { + int insertIndex = newFields.size(); + for (int i = 0; i < newFields.size(); i++) { + if (partitionKeys.contains(newFields.get(i).name())) { + insertIndex = i; + break; + } + } + newFields.add(insertIndex, dataField); + } else { + newFields.add(dataField); + } + } + }.updateIntermediateColumn(newFields, 0); + } else if (change instanceof RenameColumn) { + RenameColumn rename = (RenameColumn) change; + assertNotUpdatingPartitionKeys(oldTableSchema, rename.fieldNames(), "rename"); + assertNotUpdatingPrimaryKeyIndexColumn( + oldTableSchema, rename.fieldNames(), "rename"); + assertNotRenamingBlobColumn(newFields, rename.fieldNames()); + new NestedColumnModifier(rename.fieldNames(), lazyIdentifier) { + @Override + protected void updateLastColumn( + int depth, List newFields, String fieldName) + throws Catalog.ColumnNotExistException, + Catalog.ColumnAlreadyExistException { + assertColumnExists(newFields, fieldName, lazyIdentifier); + assertColumnNotExists(newFields, rename.newName(), lazyIdentifier); + for (int i = 0; i < newFields.size(); i++) { + DataField field = newFields.get(i); + if (!field.name().equals(fieldName)) { + continue; + } + + DataField newField = + new DataField( + field.id(), + rename.newName(), + field.type(), + field.description(), + field.defaultValue()); + newFields.set(i, newField); + return; + } + } + }.updateIntermediateColumn(newFields, 0); + } else if (change instanceof DropColumn) { + DropColumn drop = (DropColumn) change; + dropColumnValidation(oldTableSchema, drop); + if (drop.fieldNames().length == 1) { + String dropName = drop.fieldNames()[0]; + newFields.stream() + .filter(f -> f.name().equals(dropName)) + .findFirst() + .ifPresent( + f -> + ColumnDirectiveUtils.removeDroppedDirectiveOptions( + dropName, f.type(), newOptions)); + } + new NestedColumnModifier(drop.fieldNames(), lazyIdentifier) { + @Override + protected void updateLastColumn( + int depth, List newFields, String fieldName) + throws Catalog.ColumnNotExistException { + assertColumnExists(newFields, fieldName, lazyIdentifier); + newFields.removeIf(f -> f.name().equals(fieldName)); + if (newFields.isEmpty()) { + throw new IllegalArgumentException("Cannot drop all fields in table"); + } + } + }.updateIntermediateColumn(newFields, 0); + } else if (change instanceof UpdateColumnType) { + UpdateColumnType update = (UpdateColumnType) change; + assertNotUpdatingPartitionKeys(oldTableSchema, update.fieldNames(), "update"); + assertNotUpdatingPrimaryKeys(oldTableSchema, update.fieldNames(), "update"); + assertNotUpdatingPrimaryKeyIndexColumn( + oldTableSchema, update.fieldNames(), "update type of"); + assertNotChangingBlobColumnType( + newFields, update.fieldNames(), update.newDataType()); + updateNestedColumn( + newFields, + update.fieldNames(), + (field, depth) -> { + // find the dataType at depth and update the type for it + DataType sourceRootType = + getRootType(field.type(), depth, update.fieldNames().length); + DataType targetRootType = update.newDataType(); + if (update.keepNullability()) { + targetRootType = targetRootType.copy(sourceRootType.isNullable()); + } else { + assertNullabilityChange( + sourceRootType.isNullable(), + targetRootType.isNullable(), + StringUtils.join(Arrays.asList(update.fieldNames()), "."), + disableNullToNotNull); + } + checkState( + DataTypeCasts.supportsCast( + sourceRootType, + targetRootType, + !disableExplicitTypeCasting) + && CastExecutors.resolve(sourceRootType, targetRootType) + != null, + String.format( + "Column type %s[%s] cannot be converted to %s without losing information.", + field.name(), sourceRootType, targetRootType)); + return new DataField( + field.id(), + field.name(), + getArrayMapTypeWithTargetTypeRoot( + field.type(), + targetRootType, + depth, + update.fieldNames().length), + field.description(), + field.defaultValue()); + }, + lazyIdentifier); + } else if (change instanceof UpdateColumnNullability) { + UpdateColumnNullability update = (UpdateColumnNullability) change; + if (update.newNullability()) { + assertNotUpdatingPrimaryKeys( + oldTableSchema, update.fieldNames(), "change nullability of"); + } + updateNestedColumn( + newFields, + update.fieldNames(), + (field, depth) -> { + // find the DataType at depth and update that DataTypes nullability + DataType sourceRootType = + getRootType(field.type(), depth, update.fieldNames().length); + assertNullabilityChange( + sourceRootType.isNullable(), + update.newNullability(), + StringUtils.join(Arrays.asList(update.fieldNames()), "."), + disableNullToNotNull); + sourceRootType = sourceRootType.copy(update.newNullability()); + return new DataField( + field.id(), + field.name(), + getArrayMapTypeWithTargetTypeRoot( + field.type(), + sourceRootType, + depth, + update.fieldNames().length), + field.description(), + field.defaultValue()); + }, + lazyIdentifier); + } else if (change instanceof UpdateColumnComment) { + UpdateColumnComment update = (UpdateColumnComment) change; + updateNestedColumn( + newFields, + update.fieldNames(), + (field, depth) -> + new DataField( + field.id(), + field.name(), + field.type(), + update.newDescription(), + field.defaultValue()), + lazyIdentifier); + } else if (change instanceof UpdateColumnPosition) { + UpdateColumnPosition update = (UpdateColumnPosition) change; + SchemaChange.Move move = update.move(); + applyMove(newFields, move); + } else if (change instanceof UpdateColumnDefaultValue) { + UpdateColumnDefaultValue update = (UpdateColumnDefaultValue) change; + updateNestedColumn( + newFields, + update.fieldNames(), + (field, depth) -> { + validateDefaultValue(field.type(), update.newDefaultValue()); + return new DataField( + field.id(), + field.name(), + field.type(), + field.description(), + update.newDefaultValue()); + }, + lazyIdentifier); + } else if (change instanceof SchemaChange.DropPrimaryKey) { + if (hasSnapshots.get()) { + throw new UnsupportedOperationException( + "Cannot drop primary keys on a non-empty table."); + } + newPrimaryKeys = Collections.emptyList(); + } else { + throw new UnsupportedOperationException("Unsupported change: " + change.getClass()); + } + } + + // We change TableSchema to Schema, because we want to deal with primary-key and + // partition in options. + Schema newSchema = + new Schema( + newFields, + oldTableSchema.partitionKeys(), + applyNotNestedColumnRename( + newPrimaryKeys, Iterables.filter(changes, RenameColumn.class)), + applyRenameColumnsToOptions(newOptions, changes), + newComment); + + TableSchema newTableSchema = + new TableSchema( + oldTableSchema.id() + 1, + newSchema.fields(), + highestFieldId.get(), + newSchema.partitionKeys(), + newSchema.primaryKeys(), + newSchema.options(), + newSchema.comment()); + SchemaValidation.validateTableSchema(newTableSchema); + return newTableSchema; + } + + // gets the rootType at the defined depth + // ex: ARRAY>> + // if we want to update ARRAY -> ARRAY + // the maxDepth will be based on updateFieldNames + // which in the case will be [v, element, value, element], + // so maxDepth is 4 and return DataType will be INT + private static DataType getRootType(DataType type, int currDepth, int maxDepth) { + if (currDepth == maxDepth - 1) { + return type; + } + switch (type.getTypeRoot()) { + case ARRAY: + return getRootType(((ArrayType) type).getElementType(), currDepth + 1, maxDepth); + case MAP: + return getRootType(((MapType) type).getValueType(), currDepth + 1, maxDepth); + default: + return type; + } + } + + // builds the targetType from source type based on the maxDepth which needs to be updated + // ex: ARRAY>> -> ARRAY>> + // here we only need to update type of ARRAY to ARRAY and rest of the type + // remains same. This function achieves this. + private static DataType getArrayMapTypeWithTargetTypeRoot( + DataType source, DataType target, int currDepth, int maxDepth) { + if (currDepth == maxDepth - 1) { + return target; + } + switch (source.getTypeRoot()) { + case ARRAY: + return new ArrayType( + source.isNullable(), + getArrayMapTypeWithTargetTypeRoot( + ((ArrayType) source).getElementType(), + target, + currDepth + 1, + maxDepth)); + case MAP: + return new MapType( + source.isNullable(), + ((MapType) source).getKeyType(), + getArrayMapTypeWithTargetTypeRoot( + ((MapType) source).getValueType(), + target, + currDepth + 1, + maxDepth)); + default: + return target; + } + } + + private static void assertNullabilityChange( + boolean oldNullability, + boolean newNullability, + String fieldName, + boolean disableNullToNotNull) { + if (disableNullToNotNull && oldNullability && !newNullability) { + throw new UnsupportedOperationException( + String.format( + "Cannot update column type from nullable to non nullable for %s. " + + "You can set table configuration option 'alter-column-null-to-not-null.disabled' = 'false' " + + "to allow converting null columns to not null", + fieldName)); + } + } + + public static void applyMove(List newFields, SchemaChange.Move move) { + Map map = new HashMap<>(); + for (int i = 0; i < newFields.size(); i++) { + map.put(newFields.get(i).name(), i); + } + + int fieldIndex = map.getOrDefault(move.fieldName(), -1); + if (fieldIndex == -1) { + throw new IllegalArgumentException("Field name not found: " + move.fieldName()); + } + + // Handling FIRST and LAST cases directly since they don't need refIndex + switch (move.type()) { + case FIRST: + checkMoveIndexEqual(move, fieldIndex, 0); + moveField(newFields, fieldIndex, 0); + return; + case LAST: + checkMoveIndexEqual(move, fieldIndex, newFields.size() - 1); + moveField(newFields, fieldIndex, newFields.size() - 1); + return; + } + + Integer refIndex = map.getOrDefault(move.referenceFieldName(), -1); + if (refIndex == -1) { + throw new IllegalArgumentException( + "Reference field name not found: " + move.referenceFieldName()); + } + + checkMoveIndexEqual(move, fieldIndex, refIndex); + + // For AFTER and BEFORE, adjust the target index based on current and reference positions + int targetIndex = refIndex; + if (move.type() == SchemaChange.Move.MoveType.AFTER && fieldIndex > refIndex) { + targetIndex++; + } + // Ensure adjustments for moving element forwards or backwards + if (move.type() == SchemaChange.Move.MoveType.BEFORE && fieldIndex < refIndex) { + targetIndex--; + } + + if (targetIndex > (newFields.size() - 1)) { + targetIndex = newFields.size() - 1; + } + + moveField(newFields, fieldIndex, targetIndex); + } + + // Utility method to move a field within the list, handling range checks + private static void moveField(List newFields, int fromIndex, int toIndex) { + if (fromIndex < 0 || fromIndex >= newFields.size() || toIndex < 0) { + return; + } + DataField fieldToMove = newFields.remove(fromIndex); + newFields.add(toIndex, fieldToMove); + } + + private static void checkMoveIndexEqual(SchemaChange.Move move, int fieldIndex, int refIndex) { + if (refIndex == fieldIndex) { + throw new UnsupportedOperationException( + String.format("Cannot move itself for column %s", move.fieldName())); + } + } + + /** + * Merge {@code rowType} into the current schema (via {@link SchemaMergingUtils#mergeSchemas}) + * and persist the result. Returns {@code true} if the schema changed and was committed, {@code + * false} if the merge was a no-op. See {@code SchemaMergingUtils} for how {@code typeWidening} + * / {@code allowExplicitCast} drive existing-column type evolution. + */ + public boolean mergeSchema( + RowType rowType, + boolean typeWidening, + boolean allowExplicitCast, + boolean caseSensitive, + @Nullable SchemaModification schemaModification) { + TableSchema current = + latest().orElseThrow( + () -> + new RuntimeException( + "It requires that the current schema to exist when calling 'mergeSchema'")); + TableSchema update = + SchemaMergingUtils.mergeSchemas( + current, rowType, typeWidening, allowExplicitCast, caseSensitive); + if (current.equals(update)) { + return false; + } + try { + if (schemaModification != null) { + List changes = + SchemaMergingUtils.diffSchemaChanges(current, update, caseSensitive); + schemaModification.alterSchema(changes); + return true; + } else { + return commit(update); + } + } catch (Exception e) { + throw new RuntimeException("Failed to commit the schema.", e); + } + } + + private static Map applyRenameColumnsToOptions( + Map options, Iterable changes) { + Iterable renameColumns = + FluentIterable.from(changes).filter(RenameColumn.class); + + if (Iterables.isEmpty(renameColumns)) { + return options; + } + + Map newOptions = Maps.newHashMap(options); + + Map renameMappings = + Streams.stream(renameColumns) + .collect( + Collectors.toMap( + // currently only non-nested columns are supported + rename -> rename.fieldNames()[0], + RenameColumn::newName)); + + // case 1: the option key is fixed and only value may contain field names + + // bucket key rename + String bucketKeysStr = options.get(BUCKET_KEY.key()); + if (!StringUtils.isNullOrWhitespaceOnly(bucketKeysStr)) { + List bucketColumns = Arrays.asList(bucketKeysStr.split(",")); + List newBucketColumns = + applyNotNestedColumnRename(bucketColumns, renameMappings); + newOptions.put(BUCKET_KEY.key(), String.join(",", newBucketColumns)); + } + + // sequence field rename + String sequenceFieldsStr = options.get(SEQUENCE_FIELD.key()); + if (!StringUtils.isNullOrWhitespaceOnly(sequenceFieldsStr)) { + List sequenceFields = Arrays.asList(sequenceFieldsStr.split(",")); + List newSequenceFields = + applyNotNestedColumnRename(sequenceFields, renameMappings); + newOptions.put(SEQUENCE_FIELD.key(), String.join(",", newSequenceFields)); + } + + // case 2: the option key is composed of certain fixed prefixes, suffixes, and the field + // name, while the option value doesn't contain field names. + List> fieldNameToOptionKeys = + ImmutableList.of( + // NESTED_KEY is not added since renaming nested columns is not supported + // currently + fieldName -> FIELDS_PREFIX + "." + fieldName + "." + AGG_FUNCTION, + fieldName -> FIELDS_PREFIX + "." + fieldName + "." + IGNORE_RETRACT, + fieldName -> FIELDS_PREFIX + "." + fieldName + "." + DISTINCT, + fieldName -> FIELDS_PREFIX + "." + fieldName + "." + LIST_AGG_DELIMITER, + fieldName -> FIELDS_PREFIX + "." + fieldName + "." + MAP_STORAGE_LAYOUT, + fieldName -> + FIELDS_PREFIX + + "." + + fieldName + + "." + + MAP_SHARED_SHREDDING_MAX_COLUMNS, + fieldName -> + FIELDS_PREFIX + + "." + + fieldName + + "." + + MAP_SHARED_SHREDDING_COLUMN_PLACEMENT_POLICY); + + for (RenameColumn rename : renameColumns) { + String fieldName = rename.fieldNames()[0]; + String newFieldName = rename.newName(); + + for (Function fieldNameToKey : fieldNameToOptionKeys) { + String key = fieldNameToKey.apply(fieldName); + if (newOptions.containsKey(key)) { + String value = newOptions.remove(key); + newOptions.put(fieldNameToKey.apply(newFieldName), value); + } + } + } + + // case 3: both option key and option value may contain field names + for (String key : options.keySet()) { + if (key.startsWith(FIELDS_PREFIX)) { + String matchedSuffix = null; + if (key.endsWith(SEQUENCE_GROUP)) { + matchedSuffix = SEQUENCE_GROUP; + } else if (key.endsWith(NESTED_KEY)) { + matchedSuffix = NESTED_KEY; + } + + if (matchedSuffix != null) { + // Both the key and value may contain field names. If we were to perform a + // "match then replace" operation, the conditions would become quite complex. + // Instead, we directly make a replacement across all instances + String keyFieldsStr = + key.substring( + FIELDS_PREFIX.length() + 1, + key.length() - matchedSuffix.length() - 1); + List keyFields = Arrays.asList(keyFieldsStr.split(",")); + List newKeyFields = + applyNotNestedColumnRename(keyFields, renameMappings); + + String valueFieldsStr = newOptions.remove(key); + List valueFields = Arrays.asList(valueFieldsStr.split(",")); + List newValueFields = + applyNotNestedColumnRename(valueFields, renameMappings); + newOptions.put( + FIELDS_PREFIX + + "." + + String.join(",", newKeyFields) + + "." + + matchedSuffix, + String.join(",", newValueFields)); + } + } + } + + return newOptions; + } + + // Apply column rename changes on not nested columns to the list of column names, this will not + // change the order of the column names + private static List applyNotNestedColumnRename( + List columns, Iterable renames) { + if (Iterables.isEmpty(renames)) { + return columns; + } + + Map columnNames = Maps.newHashMap(); + for (RenameColumn renameColumn : renames) { + if (renameColumn.fieldNames().length == 1) { + columnNames.put(renameColumn.fieldNames()[0], renameColumn.newName()); + } + } + return applyNotNestedColumnRename(columns, columnNames); + } + + private static List applyNotNestedColumnRename( + List columns, Map renameMapping) { + + // The order of the column names will be preserved, as a non-parallel stream is used here. + return columns.stream() + .map(column -> renameMapping.getOrDefault(column, column)) + .collect(Collectors.toList()); + } + + private static void dropColumnValidation(TableSchema schema, DropColumn change) { + // primary keys and partition keys can't be nested columns + if (change.fieldNames().length > 1) { + return; + } + String columnToDrop = change.fieldNames()[0]; + if (schema.partitionKeys().contains(columnToDrop) + || schema.primaryKeys().contains(columnToDrop)) { + throw new UnsupportedOperationException( + String.format("Cannot drop partition key or primary key: [%s]", columnToDrop)); + } + assertNotUpdatingPrimaryKeyIndexColumn(schema, change.fieldNames(), "drop"); + } + + private static void assertNotUpdatingPartitionKeys( + TableSchema schema, String[] fieldNames, String operation) { + // partition keys can't be nested columns + if (fieldNames.length > 1) { + return; + } + String fieldName = fieldNames[0]; + if (schema.partitionKeys().contains(fieldName)) { + throw new UnsupportedOperationException( + String.format("Cannot %s partition column: [%s]", operation, fieldName)); + } + } + + private static void assertNotUpdatingPrimaryKeys( + TableSchema schema, String[] fieldNames, String operation) { + // primary keys can't be nested columns + if (fieldNames.length > 1) { + return; + } + String fieldName = fieldNames[0]; + if (schema.primaryKeys().contains(fieldName)) { + throw new UnsupportedOperationException( + String.format("Cannot %s primary key", operation)); + } + } + + private static void assertNotRenamingBlobColumn(List fields, String[] fieldNames) { + if (fieldNames.length > 1) { + return; + } + String fieldName = fieldNames[0]; + for (DataField field : fields) { + if (field.name().equals(fieldName) && isBlobFileField(field.type())) { + throw new UnsupportedOperationException( + String.format("Cannot rename BLOB column: [%s]", fieldName)); + } + } + } + + private static void assertNotUpdatingPrimaryKeyIndexColumn( + TableSchema schema, String[] fieldNames, String operation) { + if (fieldNames.length > 1) { + return; + } + String fieldName = fieldNames[0]; + CoreOptions options = new CoreOptions(schema.options()); + if (options.primaryKeyVectorIndexColumns().contains(fieldName) + || options.primaryKeyBTreeIndexColumns().contains(fieldName) + || options.primaryKeyBitmapIndexColumns().contains(fieldName) + || options.primaryKeyFullTextIndexColumns().contains(fieldName)) { + throw new UnsupportedOperationException( + String.format( + "Cannot %s primary-key index column: [%s]", operation, fieldName)); + } + } + + private static void assertNotChangingBlobColumnType( + List fields, String[] fieldNames, DataType newType) { + if (fieldNames.length > 1) { + return; + } + String fieldName = fieldNames[0]; + for (DataField field : fields) { + if (!field.name().equals(fieldName)) { + continue; + } + boolean wasBlob = isBlobFileField(field.type()); + boolean willBeBlob = isBlobFileField(newType); + if (wasBlob || willBeBlob) { + throw new UnsupportedOperationException( + String.format( + "Cannot change column type involving BLOB: [%s] %s -> %s", + fieldName, field.type(), newType)); + } + return; + } + } + + private abstract static class NestedColumnModifier { + + private final String[] updateFieldNames; + private final LazyField identifier; + + private NestedColumnModifier(String[] updateFieldNames, LazyField identifier) { + this.updateFieldNames = updateFieldNames; + this.identifier = identifier; + } + + private void updateIntermediateColumn( + List newFields, List previousFields, int depth, int prevDepth) + throws Catalog.ColumnNotExistException, Catalog.ColumnAlreadyExistException { + if (depth == updateFieldNames.length - 1) { + updateLastColumn(depth, newFields, updateFieldNames[depth]); + return; + } else if (depth >= updateFieldNames.length) { + // to handle the case of ARRAY or MAP type evolution + // for instance : ARRAY -> ARRAY + // the updateFieldNames in this case is [v, element] where v is array field name + // the depth returned by extractRowDataFields is 2 which will overflow. + // So the logic is to go to previous depth and update the column using previous + // fields which will have DataFields from prevDepth + // The reason for this handling is the addition of element and value for array + // and map type in FlinkCatalog as dummy column name + updateLastColumn(prevDepth, previousFields, updateFieldNames[prevDepth]); + return; + } + + for (int i = 0; i < newFields.size(); i++) { + DataField field = newFields.get(i); + if (!field.name().equals(updateFieldNames[depth])) { + continue; + } + List nestedFields = new ArrayList<>(); + int newDepth = depth + extractRowDataFields(field.type(), nestedFields); + updateIntermediateColumn(nestedFields, newFields, newDepth, depth); + field = newFields.get(i); + newFields.set( + i, + new DataField( + field.id(), + field.name(), + wrapNewRowType(field.type(), nestedFields), + field.description(), + field.defaultValue())); + return; + } + + throw new Catalog.ColumnNotExistException( + identifier.get(), + String.join(".", Arrays.asList(updateFieldNames).subList(0, depth + 1))); + } + + public void updateIntermediateColumn(List newFields, int depth) + throws Catalog.ColumnNotExistException, Catalog.ColumnAlreadyExistException { + updateIntermediateColumn(newFields, newFields, depth, depth); + } + + private int extractRowDataFields(DataType type, List nestedFields) { + switch (type.getTypeRoot()) { + case ROW: + nestedFields.addAll(((RowType) type).getFields()); + return 1; + case ARRAY: + return extractRowDataFields(((ArrayType) type).getElementType(), nestedFields) + + 1; + case MAP: + return extractRowDataFields(((MapType) type).getValueType(), nestedFields) + 1; + default: + return 1; + } + } + + private DataType wrapNewRowType(DataType type, List nestedFields) { + switch (type.getTypeRoot()) { + case ROW: + return new RowType(type.isNullable(), nestedFields); + case ARRAY: + return new ArrayType( + type.isNullable(), + wrapNewRowType(((ArrayType) type).getElementType(), nestedFields)); + case MAP: + MapType mapType = (MapType) type; + return new MapType( + type.isNullable(), + mapType.getKeyType(), + wrapNewRowType(mapType.getValueType(), nestedFields)); + default: + return type; + } + } + + protected abstract void updateLastColumn( + int depth, List newFields, String fieldName) + throws Catalog.ColumnNotExistException, Catalog.ColumnAlreadyExistException; + + protected void assertColumnExists( + List newFields, String fieldName, LazyField lazyIdentifier) + throws Catalog.ColumnNotExistException { + for (DataField field : newFields) { + if (field.name().equals(fieldName)) { + return; + } + } + throw new Catalog.ColumnNotExistException( + lazyIdentifier.get(), getLastFieldName(fieldName)); + } + + protected void assertColumnNotExists( + List newFields, String fieldName, LazyField lazyIdentifier) + throws Catalog.ColumnAlreadyExistException { + for (DataField field : newFields) { + if (field.name().equals(fieldName)) { + throw new Catalog.ColumnAlreadyExistException( + lazyIdentifier.get(), getLastFieldName(fieldName)); + } + } + } + + private String getLastFieldName(String fieldName) { + List fieldNames = new ArrayList<>(); + for (int i = 0; i + 1 < updateFieldNames.length; i++) { + fieldNames.add(updateFieldNames[i]); + } + fieldNames.add(fieldName); + return String.join(".", fieldNames); + } + } + + private static void updateNestedColumn( + List newFields, + String[] updateFieldNames, + BiFunction updateFunc, + LazyField lazyIdentifier) + throws Catalog.ColumnNotExistException, Catalog.ColumnAlreadyExistException { + new NestedColumnModifier(updateFieldNames, lazyIdentifier) { + @Override + protected void updateLastColumn(int depth, List newFields, String fieldName) + throws Catalog.ColumnNotExistException { + for (int i = 0; i < newFields.size(); i++) { + DataField field = newFields.get(i); + if (!field.name().equals(fieldName)) { + continue; + } + + newFields.set(i, updateFunc.apply(field, depth)); + return; + } + + throw new Catalog.ColumnNotExistException( + lazyIdentifier.get(), String.join(".", updateFieldNames)); + } + }.updateIntermediateColumn(newFields, 0); + } + + @VisibleForTesting + public boolean commit(TableSchema newSchema) throws Exception { + SchemaValidation.validateTableSchema(newSchema); + SchemaValidation.validateFallbackBranch(this, newSchema); + Path schemaPath = toSchemaPath(newSchema.id()); + return fileIO.tryToWriteAtomic(schemaPath, newSchema.toString()); + } + + /** Read schema for schema id. */ + public TableSchema schema(long id) { + return fromPath(fileIO, toSchemaPath(id)); + } + + /** Check if a schema exists. */ + public boolean schemaExists(long id) { + Path path = toSchemaPath(id); + try { + return fileIO.exists(path); + } catch (IOException e) { + throw new RuntimeException( + String.format( + "Failed to determine if schema '%s' exists in path %s.", id, path), + e); + } + } + + private String branchPath() { + return BranchManager.branchPath(tableRoot, branch); + } + + public Path schemaDirectory() { + return new Path(branchPath() + "/schema"); + } + + @VisibleForTesting + public Path toSchemaPath(long schemaId) { + return new Path(branchPath() + "/schema/" + SCHEMA_PREFIX + schemaId); + } + + public List schemaPaths(Predicate predicate) throws IOException { + return listVersionedFiles(fileIO, schemaDirectory(), SCHEMA_PREFIX) + .filter(predicate) + .map(this::toSchemaPath) + .collect(Collectors.toList()); + } + + /** + * Delete schema with specific id. + * + * @param schemaId the schema id to delete. + */ + public void deleteSchema(long schemaId) { + fileIO.deleteQuietly(toSchemaPath(schemaId)); + } + + /** + * Rollback to a specific schema version. All schema versions greater than the target will be + * deleted. This operation will fail if any snapshot, tag, or changelog references a schema + * version greater than the target. + * + * @param targetSchemaId the schema version to rollback to. + * @param snapshotManager the snapshot manager to check snapshot references. + * @param tagManager the tag manager to check tag references. + * @param changelogManager the changelog manager to check changelog references. + */ + public void rollbackTo( + long targetSchemaId, + SnapshotManager snapshotManager, + TagManager tagManager, + ChangelogManager changelogManager) + throws IOException { + checkArgument(schemaExists(targetSchemaId), "Schema %s does not exist.", targetSchemaId); + + // Collect all schemaIds referenced by snapshots, tags, and changelogs + Set usedSchemaIds = new HashSet<>(); + + snapshotManager.pickOrLatest( + snapshot -> { + usedSchemaIds.add(snapshot.schemaId()); + return false; + }); + tagManager.taggedSnapshots().forEach(s -> usedSchemaIds.add(s.schemaId())); + changelogManager.changelogs().forEachRemaining(c -> usedSchemaIds.add(c.schemaId())); + + // Check if any referenced schema is newer than the target + Optional conflict = + usedSchemaIds.stream().filter(id -> id > targetSchemaId).min(Long::compareTo); + if (conflict.isPresent()) { + throw new RuntimeException( + String.format( + "Cannot rollback to schema %d, schema %d is still referenced by snapshots/tags/changelogs.", + targetSchemaId, conflict.get())); + } + + // Delete all schemas newer than the target + List toBeDeleted = + listAllIds().stream() + .filter(id -> id > targetSchemaId) + .collect(Collectors.toList()); + toBeDeleted.sort((o1, o2) -> Long.compare(o2, o1)); + for (Long id : toBeDeleted) { + fileIO.delete(toSchemaPath(id), false); + } + } + + /** + * Whether an option change is a semantic no-op: 'primary-key'/'partition' are normalized into + * dedicated schema fields (old value null), and 'type' compares case-insensitively, defaulting + * when unset. + */ + public static boolean isUnchangedNormalizedKey( + String key, + @Nullable String oldValue, + @Nullable String newValue, + TableSchema tableSchema) { + return isUnchangedNormalizedKey( + key, oldValue, newValue, tableSchema.primaryKeys(), tableSchema.partitionKeys()); + } + + /** + * Overload for callers holding the primary/partition keys rather than a {@link TableSchema}. + */ + public static boolean isUnchangedNormalizedKey( + String key, + @Nullable String oldValue, + @Nullable String newValue, + List primaryKeys, + List partitionKeys) { + if (newValue == null) { + return false; + } + if (CoreOptions.TYPE.key().equals(key)) { + // 'type' compares case-insensitively (like convertToEnum); an unset type is the default + String effectiveOld = + oldValue == null ? CoreOptions.TYPE.defaultValue().toString() : oldValue; + return newValue.equalsIgnoreCase(effectiveOld); + } + if (oldValue != null) { + return false; + } + if (CoreOptions.PRIMARY_KEY.key().equals(key)) { + return normalizeKeyList(newValue).equals(primaryKeys); + } + if (CoreOptions.PARTITION.key().equals(key)) { + return normalizeKeyList(newValue).equals(partitionKeys); + } + return false; + } + + private static List normalizeKeyList(String value) { + return Arrays.stream(value.split(",")) + .map(String::trim) + .filter(s -> !s.isEmpty()) + .collect(Collectors.toList()); + } + + public static void checkAlterTableOption( + Map options, String key, @Nullable String oldValue, String newValue) { + if (CoreOptions.IMMUTABLE_OPTIONS.contains(key)) { + throw new UnsupportedOperationException( + String.format("Change '%s' is not supported yet.", key)); + } + + if (CoreOptions.BUCKET.key().equals(key)) { + int oldBucket = + oldValue == null + ? CoreOptions.BUCKET.defaultValue() + : Integer.parseInt(oldValue); + int newBucket = Integer.parseInt(newValue); + + if (oldBucket == -1) { + throw new UnsupportedOperationException("Cannot change bucket when it is -1."); + } + if (newBucket == -1) { + throw new UnsupportedOperationException("Cannot change bucket to -1."); + } + } + + if (DELETION_VECTORS_ENABLED.key().equals(key)) { + boolean dvModifiable = + Boolean.parseBoolean( + options.getOrDefault( + DELETION_VECTORS_MODIFIABLE.key(), + DELETION_VECTORS_MODIFIABLE.defaultValue().toString())); + if (!dvModifiable) { + boolean oldDv = + oldValue == null + ? DELETION_VECTORS_ENABLED.defaultValue() + : Boolean.parseBoolean(oldValue); + boolean newDv = Boolean.parseBoolean(newValue); + + if (oldDv != newDv) { + throw new UnsupportedOperationException( + String.format( + "Cannot change deletion vectors mode from %s to %s. If modifying table deletion-vectors mode without full-compaction, this may result in data duplication. " + + "If you are confident, you can set table option '%s' = 'true' to allow deletion vectors modification.", + oldDv, newDv, DELETION_VECTORS_MODIFIABLE.key())); + } + } + } + + if (IGNORE_DELETE.key().equals(key)) { + boolean oldIgnoreDelete = + oldValue == null + ? IGNORE_DELETE.defaultValue() + : Boolean.parseBoolean(oldValue); + boolean newIgnoreDelete = Boolean.parseBoolean(newValue); + if (oldIgnoreDelete && !newIgnoreDelete) { + throw new UnsupportedOperationException( + String.format("Cannot change %s from true to false.", IGNORE_DELETE.key())); + } + } + + if (IGNORE_UPDATE_BEFORE.key().equals(key)) { + boolean oldIgnoreUpdateBefore = + oldValue == null + ? IGNORE_UPDATE_BEFORE.defaultValue() + : Boolean.parseBoolean(oldValue); + boolean newIgnoreUpdateBefore = Boolean.parseBoolean(newValue); + if (oldIgnoreUpdateBefore && !newIgnoreUpdateBefore) { + throw new UnsupportedOperationException( + String.format( + "Cannot change %s from true to false.", + IGNORE_UPDATE_BEFORE.key())); + } + } + + if (CLUSTERING_COLUMNS.key().equals(key)) { + if (options.containsKey(PK_CLUSTERING_OVERRIDE.key())) { + throw new UnsupportedOperationException( + String.format( + "Cannot change %s when %s enabled.", + CLUSTERING_COLUMNS.key(), PK_CLUSTERING_OVERRIDE.key())); + } + } + } + + public static void checkResetTableOption(Map options, String key) { + if (CoreOptions.IMMUTABLE_OPTIONS.contains(key)) { + throw new UnsupportedOperationException( + String.format("Change '%s' is not supported yet.", key)); + } + + if (CoreOptions.BUCKET.key().equals(key)) { + throw new UnsupportedOperationException(String.format("Cannot reset %s.", key)); + } + + if (options.containsKey(PK_CLUSTERING_OVERRIDE.key()) + && CLUSTERING_COLUMNS.key().equals(key)) { + throw new UnsupportedOperationException( + String.format( + "Cannot reset %s when %s enabled.", key, PK_CLUSTERING_OVERRIDE.key())); + } + } + + public static void checkAlterTablePath(String key) { + if (CoreOptions.PATH.key().equalsIgnoreCase(key)) { + throw new UnsupportedOperationException("Change path is not supported yet."); + } + } + + public static Identifier identifierFromPath(String tablePath, boolean ignoreIfUnknownDatabase) { + return identifierFromPath(tablePath, ignoreIfUnknownDatabase, null); + } + + public static Identifier identifierFromPath( + String tablePath, boolean ignoreIfUnknownDatabase, @Nullable String branchName) { + if (DEFAULT_MAIN_BRANCH.equals(branchName)) { + branchName = null; + } + + String[] paths = tablePath.split("/"); + if (paths.length < 2) { + if (!ignoreIfUnknownDatabase) { + throw new IllegalArgumentException( + String.format( + "Path '%s' is not a valid path, please use catalog table path instead: 'warehouse_path/your_database.db/your_table'.", + tablePath)); + } + return new Identifier(UNKNOWN_DATABASE, paths[0]); + } + + String database = paths[paths.length - 2]; + int index = database.lastIndexOf(DB_SUFFIX); + if (index == -1) { + if (!ignoreIfUnknownDatabase) { + throw new IllegalArgumentException( + String.format( + "Path '%s' is not a valid path, please use catalog table path instead: 'warehouse_path/your_database.db/your_table'.", + tablePath)); + } + return new Identifier(UNKNOWN_DATABASE, paths[paths.length - 1], branchName, null); + } + database = database.substring(0, index); + + return new Identifier(database, paths[paths.length - 1], branchName, null); + } + + public static TableSchema fromPath(FileIO fileIO, Path path) { + try { + return tryFromPath(fileIO, path); + } catch (FileNotFoundException e) { + throw new RuntimeException(e.getMessage(), e); + } + } + + public static TableSchema tryFromPath(FileIO fileIO, Path path) throws FileNotFoundException { + try { + return TableSchema.fromJson(fileIO.readFileUtf8(path)); + } catch (FileNotFoundException e) { + throw e; + } catch (IOException e) { + throw new UncheckedIOException(e); + } + } +} diff --git a/paimon-core/src/main/java/org/apache/paimon/schema/SchemaManager.java b/paimon-core/src/main/java/org/apache/paimon/schema/SchemaManager.java index 4c238781b9b1..bf0086793de6 100644 --- a/paimon-core/src/main/java/org/apache/paimon/schema/SchemaManager.java +++ b/paimon-core/src/main/java/org/apache/paimon/schema/SchemaManager.java @@ -18,1500 +18,151 @@ package org.apache.paimon.schema; -import org.apache.paimon.CoreOptions; -import org.apache.paimon.annotation.VisibleForTesting; -import org.apache.paimon.casting.CastExecutors; import org.apache.paimon.catalog.Catalog; import org.apache.paimon.catalog.Identifier; import org.apache.paimon.fs.FileIO; import org.apache.paimon.fs.Path; -import org.apache.paimon.schema.ColumnDirectiveUtils.ConvertedColumn; -import org.apache.paimon.schema.SchemaChange.AddColumn; -import org.apache.paimon.schema.SchemaChange.DropColumn; -import org.apache.paimon.schema.SchemaChange.RemoveOption; -import org.apache.paimon.schema.SchemaChange.RenameColumn; -import org.apache.paimon.schema.SchemaChange.SetOption; -import org.apache.paimon.schema.SchemaChange.UpdateColumnComment; -import org.apache.paimon.schema.SchemaChange.UpdateColumnDefaultValue; -import org.apache.paimon.schema.SchemaChange.UpdateColumnNullability; -import org.apache.paimon.schema.SchemaChange.UpdateColumnPosition; -import org.apache.paimon.schema.SchemaChange.UpdateColumnType; -import org.apache.paimon.schema.SchemaChange.UpdateComment; -import org.apache.paimon.table.FileStoreTableFactory; import org.apache.paimon.table.SchemaModification; -import org.apache.paimon.types.ArrayType; import org.apache.paimon.types.DataField; -import org.apache.paimon.types.DataType; -import org.apache.paimon.types.DataTypeCasts; -import org.apache.paimon.types.MapType; -import org.apache.paimon.types.ReassignFieldId; import org.apache.paimon.types.RowType; -import org.apache.paimon.utils.BranchManager; import org.apache.paimon.utils.ChangelogManager; import org.apache.paimon.utils.LazyField; -import org.apache.paimon.utils.Preconditions; import org.apache.paimon.utils.SnapshotManager; -import org.apache.paimon.utils.StringUtils; import org.apache.paimon.utils.TagManager; -import org.apache.paimon.shade.guava30.com.google.common.collect.FluentIterable; -import org.apache.paimon.shade.guava30.com.google.common.collect.ImmutableList; -import org.apache.paimon.shade.guava30.com.google.common.collect.Iterables; -import org.apache.paimon.shade.guava30.com.google.common.collect.Maps; -import org.apache.paimon.shade.guava30.com.google.common.collect.Streams; - import javax.annotation.Nullable; import javax.annotation.concurrent.ThreadSafe; import java.io.FileNotFoundException; import java.io.IOException; import java.io.Serializable; -import java.io.UncheckedIOException; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collections; -import java.util.HashMap; -import java.util.HashSet; import java.util.List; import java.util.Map; -import java.util.Objects; import java.util.Optional; -import java.util.Set; -import java.util.concurrent.atomic.AtomicInteger; -import java.util.function.BiFunction; -import java.util.function.Function; import java.util.function.Predicate; -import java.util.stream.Collectors; - -import static org.apache.paimon.CoreOptions.AGG_FUNCTION; -import static org.apache.paimon.CoreOptions.BUCKET_KEY; -import static org.apache.paimon.CoreOptions.CLUSTERING_COLUMNS; -import static org.apache.paimon.CoreOptions.DELETION_VECTORS_ENABLED; -import static org.apache.paimon.CoreOptions.DELETION_VECTORS_MODIFIABLE; -import static org.apache.paimon.CoreOptions.DISTINCT; -import static org.apache.paimon.CoreOptions.FIELDS_PREFIX; -import static org.apache.paimon.CoreOptions.IGNORE_DELETE; -import static org.apache.paimon.CoreOptions.IGNORE_RETRACT; -import static org.apache.paimon.CoreOptions.IGNORE_UPDATE_BEFORE; -import static org.apache.paimon.CoreOptions.LIST_AGG_DELIMITER; -import static org.apache.paimon.CoreOptions.MAP_SHARED_SHREDDING_COLUMN_PLACEMENT_POLICY; -import static org.apache.paimon.CoreOptions.MAP_SHARED_SHREDDING_MAX_COLUMNS; -import static org.apache.paimon.CoreOptions.MAP_STORAGE_LAYOUT; -import static org.apache.paimon.CoreOptions.NESTED_KEY; -import static org.apache.paimon.CoreOptions.PK_CLUSTERING_OVERRIDE; -import static org.apache.paimon.CoreOptions.SEQUENCE_FIELD; -import static org.apache.paimon.catalog.AbstractCatalog.DB_SUFFIX; -import static org.apache.paimon.catalog.Identifier.DEFAULT_MAIN_BRANCH; -import static org.apache.paimon.catalog.Identifier.UNKNOWN_DATABASE; -import static org.apache.paimon.mergetree.compact.PartialUpdateMergeFunction.SEQUENCE_GROUP; -import static org.apache.paimon.schema.ColumnDirectiveUtils.applyAddColumnDirective; -import static org.apache.paimon.schema.ColumnDirectiveUtils.applyDirectives; -import static org.apache.paimon.types.BlobType.isBlobFileField; -import static org.apache.paimon.utils.DefaultValueUtils.validateDefaultValue; -import static org.apache.paimon.utils.FileUtils.listVersionedFiles; -import static org.apache.paimon.utils.Preconditions.checkArgument; -import static org.apache.paimon.utils.Preconditions.checkState; -/** Schema Manager to manage schema versions. */ +/** + * Schema manager abstraction. + * + *

The default implementation is {@link FileSystemSchemaManager}. Catalogs can provide another + * implementation while table components continue to depend on this interface. + */ @ThreadSafe -public class SchemaManager implements Serializable { - - public static final String SCHEMA_PREFIX = "schema-"; +public interface SchemaManager extends Serializable { - private final FileIO fileIO; - private final Path tableRoot; + String SCHEMA_PREFIX = "schema-"; - private final String branch; + SchemaManager copyWithBranch(String branchName); - public SchemaManager(FileIO fileIO, Path tableRoot) { - this(fileIO, tableRoot, DEFAULT_MAIN_BRANCH); - } + Optional latest(); - /** Specify the default branch for data writing. */ - public SchemaManager(FileIO fileIO, Path tableRoot, String branch) { - this.fileIO = fileIO; - this.tableRoot = tableRoot; - this.branch = BranchManager.normalizeBranch(branch); - } + TableSchema latestOrThrow(String message); - public SchemaManager copyWithBranch(String branchName) { - return new SchemaManager(fileIO, tableRoot, branchName); - } + long earliestCreationTime(); - public Optional latest() { - try { - return listVersionedFiles(fileIO, schemaDirectory(), SCHEMA_PREFIX) - .reduce(Math::max) - .map(this::schema); - } catch (IOException e) { - throw new UncheckedIOException(e); - } - } + List listAll(); - public TableSchema latestOrThrow(String message) { - return latest().orElseThrow(() -> new RuntimeException(message)); - } - - public long earliestCreationTime() { - try { - long earliest = 0; - if (!schemaExists(0)) { - Optional min = - listVersionedFiles(fileIO, schemaDirectory(), SCHEMA_PREFIX) - .reduce(Math::min); - checkArgument(min.isPresent()); - earliest = min.get(); - } - - Path schemaPath = toSchemaPath(earliest); - return fileIO.getFileStatus(schemaPath).getModificationTime(); - } catch (IOException e) { - throw new UncheckedIOException(e); - } - } - - public List listAll() { - return listAllIds().stream().map(this::schema).collect(Collectors.toList()); - } - - /** List all schema IDs. */ - public List listAllIds() { - try { - return listVersionedFiles(fileIO, schemaDirectory(), SCHEMA_PREFIX) - .collect(Collectors.toList()); - } catch (IOException e) { - throw new UncheckedIOException(e); - } - } - - public TableSchema createTable(Schema schema) throws Exception { - return createTable(schema, false); - } + List listAllIds(); - public TableSchema createTable(Schema schema, boolean externalTable) throws Exception { - while (true) { - Optional latest = latest(); - if (latest.isPresent()) { - TableSchema latestSchema = latest.get(); - if (externalTable) { - checkSchemaForExternalTable(latestSchema.toSchema(), schema); - return latestSchema; - } else { - throw new IllegalStateException( - "Schema in filesystem exists, creation is not allowed."); - } - } + TableSchema createTable(Schema schema) throws Exception; - schema = applyDirectives(schema); - TableSchema newSchema = TableSchema.create(0, schema); + TableSchema createTable(Schema schema, boolean externalTable) throws Exception; - // validate table from creating table - FileStoreTableFactory.create(fileIO, tableRoot, newSchema).store(); + TableSchema commitChanges(SchemaChange... changes) throws Exception; - boolean success = commit(newSchema); - if (success) { - return newSchema; - } - } - } - - private void checkSchemaForExternalTable(Schema existsSchema, Schema newSchema) { - // When creating an external table, if the table already exists in the location, we can - // choose not to specify the fields. - if ((newSchema.fields().isEmpty() - || newSchema.rowType().equalsIgnoreFieldId(existsSchema.rowType())) - && (newSchema.partitionKeys().isEmpty() - || Objects.equals(newSchema.partitionKeys(), existsSchema.partitionKeys())) - && (newSchema.primaryKeys().isEmpty() - || Objects.equals(newSchema.primaryKeys(), existsSchema.primaryKeys()))) { - // check for options - Map existsOptions = existsSchema.options(); - Map newOptions = newSchema.options(); - newOptions.forEach( - (key, value) -> { - // ignore `owner` and `path` - if (!key.equals(Catalog.OWNER_PROP) - && !key.equals(CoreOptions.PATH.key()) - && (!existsOptions.containsKey(key) - || !existsOptions.get(key).equals(value))) { - throw new RuntimeException( - "New schema's options are not equal to the exists schema's, new schema: " - + newOptions - + ", exists schema: " - + existsOptions); - } - }); - } else { - throw new RuntimeException( - "New schema is not equal to exists schema, new schema: " - + newSchema - + ", exists schema: " - + existsSchema); - } - } - - /** Update {@link SchemaChange}s. */ - public TableSchema commitChanges(SchemaChange... changes) throws Exception { - return commitChanges(Arrays.asList(changes)); - } - - /** Update {@link SchemaChange}s. */ - public TableSchema commitChanges(List changes) + TableSchema commitChanges(List changes) throws Catalog.TableNotExistException, Catalog.ColumnAlreadyExistException, - Catalog.ColumnNotExistException { - SnapshotManager snapshotManager = - new SnapshotManager(fileIO, tableRoot, branch, null, null); - LazyField hasSnapshots = - new LazyField<>(() -> snapshotManager.latestSnapshot() != null); - - while (true) { - TableSchema oldTableSchema = - latest().orElseThrow( - () -> - new Catalog.TableNotExistException( - identifierFromPath( - tableRoot.toString(), true, branch))); - LazyField lazyIdentifier = - new LazyField<>(() -> identifierFromPath(tableRoot.toString(), true, branch)); - TableSchema newTableSchema = - generateTableSchema(oldTableSchema, changes, hasSnapshots, lazyIdentifier); - try { - boolean success = commit(newTableSchema); - if (success) { - return newTableSchema; - } - } catch (Exception e) { - throw new RuntimeException(e); - } - } - } - - public static TableSchema generateTableSchema( - TableSchema oldTableSchema, - List changes, - LazyField hasSnapshots, - LazyField lazyIdentifier) - throws Catalog.ColumnAlreadyExistException, Catalog.ColumnNotExistException { - Map oldOptions = new HashMap<>(oldTableSchema.options()); - Map newOptions = new HashMap<>(oldTableSchema.options()); - boolean disableNullToNotNull = - Boolean.parseBoolean( - oldOptions.getOrDefault( - CoreOptions.DISABLE_ALTER_COLUMN_NULL_TO_NOT_NULL.key(), - CoreOptions.DISABLE_ALTER_COLUMN_NULL_TO_NOT_NULL - .defaultValue() - .toString())); - - boolean disableExplicitTypeCasting = - Boolean.parseBoolean( - oldOptions.getOrDefault( - CoreOptions.DISABLE_EXPLICIT_TYPE_CASTING.key(), - CoreOptions.DISABLE_EXPLICIT_TYPE_CASTING - .defaultValue() - .toString())); - - boolean addColumnBeforePartition = - Boolean.parseBoolean( - oldOptions.getOrDefault( - CoreOptions.ADD_COLUMN_BEFORE_PARTITION.key(), - CoreOptions.ADD_COLUMN_BEFORE_PARTITION.defaultValue().toString())); - List partitionKeys = oldTableSchema.partitionKeys(); - - List newFields = new ArrayList<>(oldTableSchema.fields()); - AtomicInteger highestFieldId = new AtomicInteger(oldTableSchema.highestFieldId()); - String newComment = oldTableSchema.comment(); - List newPrimaryKeys = oldTableSchema.primaryKeys(); - for (SchemaChange change : changes) { - if (change instanceof SetOption) { - SetOption setOption = (SetOption) change; - String oldValue = oldOptions.get(setOption.key()); - String newValue = setOption.value(); - boolean unchanged = - Objects.equals(oldValue, newValue) - || isUnchangedNormalizedKey( - setOption.key(), oldValue, newValue, oldTableSchema); - // reject 'type' even without snapshots: format tables hold data but - // create no snapshots, so the snapshot check would not catch them - if (!unchanged && CoreOptions.TYPE.key().equals(setOption.key())) { - throw new UnsupportedOperationException("Change 'type' is not supported yet."); - } - if (hasSnapshots.get() && !unchanged) { - checkAlterTableOption(oldOptions, setOption.key(), oldValue, newValue); - } - if (!unchanged) { - newOptions.put(setOption.key(), setOption.value()); - } - } else if (change instanceof RemoveOption) { - RemoveOption removeOption = (RemoveOption) change; - if (CoreOptions.TYPE.key().equals(removeOption.key())) { - throw new UnsupportedOperationException("Change 'type' is not supported yet."); - } - if (hasSnapshots.get()) { - checkResetTableOption(oldOptions, removeOption.key()); - } - newOptions.remove(removeOption.key()); - } else if (change instanceof UpdateComment) { - UpdateComment updateComment = (UpdateComment) change; - newComment = updateComment.comment(); - } else if (change instanceof AddColumn) { - AddColumn addColumn = (AddColumn) change; - SchemaChange.Move move = addColumn.move(); - Preconditions.checkArgument( - addColumn.dataType().isNullable(), - "Column %s cannot specify NOT NULL in the %s table.", - String.join(".", addColumn.fieldNames()), - lazyIdentifier.get().getFullName()); - - ConvertedColumn converted = - applyAddColumnDirective( - addColumn.description(), - addColumn.fieldNames()[0], - addColumn.dataType(), - newOptions); - DataType requestedDataType = addColumn.dataType(); - String effectiveComment = addColumn.description(); - if (converted != null) { - Preconditions.checkArgument( - addColumn.fieldNames().length == 1, - "Comment directive cannot be used on a nested column %s.", - String.join(".", addColumn.fieldNames())); - requestedDataType = converted.type(); - effectiveComment = converted.comment(); - } + Catalog.ColumnNotExistException; - int id = highestFieldId.incrementAndGet(); - DataType dataType = ReassignFieldId.reassign(requestedDataType, highestFieldId); - String storedComment = effectiveComment; - new NestedColumnModifier(addColumn.fieldNames(), lazyIdentifier) { - @Override - protected void updateLastColumn( - int depth, List newFields, String fieldName) - throws Catalog.ColumnAlreadyExistException, - Catalog.ColumnNotExistException { - assertColumnNotExists(newFields, fieldName, lazyIdentifier); - - DataField dataField = new DataField(id, fieldName, dataType, storedComment); - - // key: name ; value : index - Map map = new HashMap<>(); - for (int i = 0; i < newFields.size(); i++) { - map.put(newFields.get(i).name(), i); - } - - if (null != move) { - if (move.type().equals(SchemaChange.Move.MoveType.FIRST)) { - newFields.add(0, dataField); - } else if (move.type().equals(SchemaChange.Move.MoveType.AFTER)) { - if (map.containsKey(move.referenceFieldName())) { - int fieldIndex = map.get(move.referenceFieldName()); - newFields.add(fieldIndex + 1, dataField); - } else { - throw new Catalog.ColumnNotExistException( - lazyIdentifier.get(), move.referenceFieldName()); - } - } else if (move.type().equals(SchemaChange.Move.MoveType.BEFORE)) { - if (map.containsKey(move.referenceFieldName())) { - int fieldIndex = map.get(move.referenceFieldName()); - newFields.add(fieldIndex, dataField); - } else { - throw new Catalog.ColumnNotExistException( - lazyIdentifier.get(), move.referenceFieldName()); - } - } else if (move.type().equals(SchemaChange.Move.MoveType.LAST)) { - newFields.add(dataField); - } else { - throw new UnsupportedOperationException( - "Unsupported move type: " + move.type()); - } - } else if (addColumnBeforePartition - && !partitionKeys.isEmpty() - && addColumn.fieldNames().length == 1) { - int insertIndex = newFields.size(); - for (int i = 0; i < newFields.size(); i++) { - if (partitionKeys.contains(newFields.get(i).name())) { - insertIndex = i; - break; - } - } - newFields.add(insertIndex, dataField); - } else { - newFields.add(dataField); - } - } - }.updateIntermediateColumn(newFields, 0); - } else if (change instanceof RenameColumn) { - RenameColumn rename = (RenameColumn) change; - assertNotUpdatingPartitionKeys(oldTableSchema, rename.fieldNames(), "rename"); - assertNotUpdatingPrimaryKeyIndexColumn( - oldTableSchema, rename.fieldNames(), "rename"); - assertNotRenamingBlobColumn(newFields, rename.fieldNames()); - new NestedColumnModifier(rename.fieldNames(), lazyIdentifier) { - @Override - protected void updateLastColumn( - int depth, List newFields, String fieldName) - throws Catalog.ColumnNotExistException, - Catalog.ColumnAlreadyExistException { - assertColumnExists(newFields, fieldName, lazyIdentifier); - assertColumnNotExists(newFields, rename.newName(), lazyIdentifier); - for (int i = 0; i < newFields.size(); i++) { - DataField field = newFields.get(i); - if (!field.name().equals(fieldName)) { - continue; - } - - DataField newField = - new DataField( - field.id(), - rename.newName(), - field.type(), - field.description(), - field.defaultValue()); - newFields.set(i, newField); - return; - } - } - }.updateIntermediateColumn(newFields, 0); - } else if (change instanceof DropColumn) { - DropColumn drop = (DropColumn) change; - dropColumnValidation(oldTableSchema, drop); - if (drop.fieldNames().length == 1) { - String dropName = drop.fieldNames()[0]; - newFields.stream() - .filter(f -> f.name().equals(dropName)) - .findFirst() - .ifPresent( - f -> - ColumnDirectiveUtils.removeDroppedDirectiveOptions( - dropName, f.type(), newOptions)); - } - new NestedColumnModifier(drop.fieldNames(), lazyIdentifier) { - @Override - protected void updateLastColumn( - int depth, List newFields, String fieldName) - throws Catalog.ColumnNotExistException { - assertColumnExists(newFields, fieldName, lazyIdentifier); - newFields.removeIf(f -> f.name().equals(fieldName)); - if (newFields.isEmpty()) { - throw new IllegalArgumentException("Cannot drop all fields in table"); - } - } - }.updateIntermediateColumn(newFields, 0); - } else if (change instanceof UpdateColumnType) { - UpdateColumnType update = (UpdateColumnType) change; - assertNotUpdatingPartitionKeys(oldTableSchema, update.fieldNames(), "update"); - assertNotUpdatingPrimaryKeys(oldTableSchema, update.fieldNames(), "update"); - assertNotUpdatingPrimaryKeyIndexColumn( - oldTableSchema, update.fieldNames(), "update type of"); - assertNotChangingBlobColumnType( - newFields, update.fieldNames(), update.newDataType()); - updateNestedColumn( - newFields, - update.fieldNames(), - (field, depth) -> { - // find the dataType at depth and update the type for it - DataType sourceRootType = - getRootType(field.type(), depth, update.fieldNames().length); - DataType targetRootType = update.newDataType(); - if (update.keepNullability()) { - targetRootType = targetRootType.copy(sourceRootType.isNullable()); - } else { - assertNullabilityChange( - sourceRootType.isNullable(), - targetRootType.isNullable(), - StringUtils.join(Arrays.asList(update.fieldNames()), "."), - disableNullToNotNull); - } - checkState( - DataTypeCasts.supportsCast( - sourceRootType, - targetRootType, - !disableExplicitTypeCasting) - && CastExecutors.resolve(sourceRootType, targetRootType) - != null, - String.format( - "Column type %s[%s] cannot be converted to %s without losing information.", - field.name(), sourceRootType, targetRootType)); - return new DataField( - field.id(), - field.name(), - getArrayMapTypeWithTargetTypeRoot( - field.type(), - targetRootType, - depth, - update.fieldNames().length), - field.description(), - field.defaultValue()); - }, - lazyIdentifier); - } else if (change instanceof UpdateColumnNullability) { - UpdateColumnNullability update = (UpdateColumnNullability) change; - if (update.newNullability()) { - assertNotUpdatingPrimaryKeys( - oldTableSchema, update.fieldNames(), "change nullability of"); - } - updateNestedColumn( - newFields, - update.fieldNames(), - (field, depth) -> { - // find the DataType at depth and update that DataTypes nullability - DataType sourceRootType = - getRootType(field.type(), depth, update.fieldNames().length); - assertNullabilityChange( - sourceRootType.isNullable(), - update.newNullability(), - StringUtils.join(Arrays.asList(update.fieldNames()), "."), - disableNullToNotNull); - sourceRootType = sourceRootType.copy(update.newNullability()); - return new DataField( - field.id(), - field.name(), - getArrayMapTypeWithTargetTypeRoot( - field.type(), - sourceRootType, - depth, - update.fieldNames().length), - field.description(), - field.defaultValue()); - }, - lazyIdentifier); - } else if (change instanceof UpdateColumnComment) { - UpdateColumnComment update = (UpdateColumnComment) change; - updateNestedColumn( - newFields, - update.fieldNames(), - (field, depth) -> - new DataField( - field.id(), - field.name(), - field.type(), - update.newDescription(), - field.defaultValue()), - lazyIdentifier); - } else if (change instanceof UpdateColumnPosition) { - UpdateColumnPosition update = (UpdateColumnPosition) change; - SchemaChange.Move move = update.move(); - applyMove(newFields, move); - } else if (change instanceof UpdateColumnDefaultValue) { - UpdateColumnDefaultValue update = (UpdateColumnDefaultValue) change; - updateNestedColumn( - newFields, - update.fieldNames(), - (field, depth) -> { - validateDefaultValue(field.type(), update.newDefaultValue()); - return new DataField( - field.id(), - field.name(), - field.type(), - field.description(), - update.newDefaultValue()); - }, - lazyIdentifier); - } else if (change instanceof SchemaChange.DropPrimaryKey) { - if (hasSnapshots.get()) { - throw new UnsupportedOperationException( - "Cannot drop primary keys on a non-empty table."); - } - newPrimaryKeys = Collections.emptyList(); - } else { - throw new UnsupportedOperationException("Unsupported change: " + change.getClass()); - } - } - - // We change TableSchema to Schema, because we want to deal with primary-key and - // partition in options. - Schema newSchema = - new Schema( - newFields, - oldTableSchema.partitionKeys(), - applyNotNestedColumnRename( - newPrimaryKeys, Iterables.filter(changes, RenameColumn.class)), - applyRenameColumnsToOptions(newOptions, changes), - newComment); - - TableSchema newTableSchema = - new TableSchema( - oldTableSchema.id() + 1, - newSchema.fields(), - highestFieldId.get(), - newSchema.partitionKeys(), - newSchema.primaryKeys(), - newSchema.options(), - newSchema.comment()); - SchemaValidation.validateTableSchema(newTableSchema); - return newTableSchema; - } - - // gets the rootType at the defined depth - // ex: ARRAY>> - // if we want to update ARRAY -> ARRAY - // the maxDepth will be based on updateFieldNames - // which in the case will be [v, element, value, element], - // so maxDepth is 4 and return DataType will be INT - private static DataType getRootType(DataType type, int currDepth, int maxDepth) { - if (currDepth == maxDepth - 1) { - return type; - } - switch (type.getTypeRoot()) { - case ARRAY: - return getRootType(((ArrayType) type).getElementType(), currDepth + 1, maxDepth); - case MAP: - return getRootType(((MapType) type).getValueType(), currDepth + 1, maxDepth); - default: - return type; - } - } - - // builds the targetType from source type based on the maxDepth which needs to be updated - // ex: ARRAY>> -> ARRAY>> - // here we only need to update type of ARRAY to ARRAY and rest of the type - // remains same. This function achieves this. - private static DataType getArrayMapTypeWithTargetTypeRoot( - DataType source, DataType target, int currDepth, int maxDepth) { - if (currDepth == maxDepth - 1) { - return target; - } - switch (source.getTypeRoot()) { - case ARRAY: - return new ArrayType( - source.isNullable(), - getArrayMapTypeWithTargetTypeRoot( - ((ArrayType) source).getElementType(), - target, - currDepth + 1, - maxDepth)); - case MAP: - return new MapType( - source.isNullable(), - ((MapType) source).getKeyType(), - getArrayMapTypeWithTargetTypeRoot( - ((MapType) source).getValueType(), - target, - currDepth + 1, - maxDepth)); - default: - return target; - } - } - - private static void assertNullabilityChange( - boolean oldNullability, - boolean newNullability, - String fieldName, - boolean disableNullToNotNull) { - if (disableNullToNotNull && oldNullability && !newNullability) { - throw new UnsupportedOperationException( - String.format( - "Cannot update column type from nullable to non nullable for %s. " - + "You can set table configuration option 'alter-column-null-to-not-null.disabled' = 'false' " - + "to allow converting null columns to not null", - fieldName)); - } - } - - public static void applyMove(List newFields, SchemaChange.Move move) { - Map map = new HashMap<>(); - for (int i = 0; i < newFields.size(); i++) { - map.put(newFields.get(i).name(), i); - } - - int fieldIndex = map.getOrDefault(move.fieldName(), -1); - if (fieldIndex == -1) { - throw new IllegalArgumentException("Field name not found: " + move.fieldName()); - } - - // Handling FIRST and LAST cases directly since they don't need refIndex - switch (move.type()) { - case FIRST: - checkMoveIndexEqual(move, fieldIndex, 0); - moveField(newFields, fieldIndex, 0); - return; - case LAST: - checkMoveIndexEqual(move, fieldIndex, newFields.size() - 1); - moveField(newFields, fieldIndex, newFields.size() - 1); - return; - } - - Integer refIndex = map.getOrDefault(move.referenceFieldName(), -1); - if (refIndex == -1) { - throw new IllegalArgumentException( - "Reference field name not found: " + move.referenceFieldName()); - } - - checkMoveIndexEqual(move, fieldIndex, refIndex); - - // For AFTER and BEFORE, adjust the target index based on current and reference positions - int targetIndex = refIndex; - if (move.type() == SchemaChange.Move.MoveType.AFTER && fieldIndex > refIndex) { - targetIndex++; - } - // Ensure adjustments for moving element forwards or backwards - if (move.type() == SchemaChange.Move.MoveType.BEFORE && fieldIndex < refIndex) { - targetIndex--; - } - - if (targetIndex > (newFields.size() - 1)) { - targetIndex = newFields.size() - 1; - } - - moveField(newFields, fieldIndex, targetIndex); - } - - // Utility method to move a field within the list, handling range checks - private static void moveField(List newFields, int fromIndex, int toIndex) { - if (fromIndex < 0 || fromIndex >= newFields.size() || toIndex < 0) { - return; - } - DataField fieldToMove = newFields.remove(fromIndex); - newFields.add(toIndex, fieldToMove); - } - - private static void checkMoveIndexEqual(SchemaChange.Move move, int fieldIndex, int refIndex) { - if (refIndex == fieldIndex) { - throw new UnsupportedOperationException( - String.format("Cannot move itself for column %s", move.fieldName())); - } - } - - /** - * Merge {@code rowType} into the current schema (via {@link SchemaMergingUtils#mergeSchemas}) - * and persist the result. Returns {@code true} if the schema changed and was committed, {@code - * false} if the merge was a no-op. See {@code SchemaMergingUtils} for how {@code typeWidening} - * / {@code allowExplicitCast} drive existing-column type evolution. - */ - public boolean mergeSchema( + boolean mergeSchema( RowType rowType, boolean typeWidening, boolean allowExplicitCast, boolean caseSensitive, - @Nullable SchemaModification schemaModification) { - TableSchema current = - latest().orElseThrow( - () -> - new RuntimeException( - "It requires that the current schema to exist when calling 'mergeSchema'")); - TableSchema update = - SchemaMergingUtils.mergeSchemas( - current, rowType, typeWidening, allowExplicitCast, caseSensitive); - if (current.equals(update)) { - return false; - } - try { - if (schemaModification != null) { - List changes = - SchemaMergingUtils.diffSchemaChanges(current, update, caseSensitive); - schemaModification.alterSchema(changes); - return true; - } else { - return commit(update); - } - } catch (Exception e) { - throw new RuntimeException("Failed to commit the schema.", e); - } - } - - private static Map applyRenameColumnsToOptions( - Map options, Iterable changes) { - Iterable renameColumns = - FluentIterable.from(changes).filter(RenameColumn.class); - - if (Iterables.isEmpty(renameColumns)) { - return options; - } - - Map newOptions = Maps.newHashMap(options); - - Map renameMappings = - Streams.stream(renameColumns) - .collect( - Collectors.toMap( - // currently only non-nested columns are supported - rename -> rename.fieldNames()[0], - RenameColumn::newName)); - - // case 1: the option key is fixed and only value may contain field names - - // bucket key rename - String bucketKeysStr = options.get(BUCKET_KEY.key()); - if (!StringUtils.isNullOrWhitespaceOnly(bucketKeysStr)) { - List bucketColumns = Arrays.asList(bucketKeysStr.split(",")); - List newBucketColumns = - applyNotNestedColumnRename(bucketColumns, renameMappings); - newOptions.put(BUCKET_KEY.key(), String.join(",", newBucketColumns)); - } - - // sequence field rename - String sequenceFieldsStr = options.get(SEQUENCE_FIELD.key()); - if (!StringUtils.isNullOrWhitespaceOnly(sequenceFieldsStr)) { - List sequenceFields = Arrays.asList(sequenceFieldsStr.split(",")); - List newSequenceFields = - applyNotNestedColumnRename(sequenceFields, renameMappings); - newOptions.put(SEQUENCE_FIELD.key(), String.join(",", newSequenceFields)); - } - - // case 2: the option key is composed of certain fixed prefixes, suffixes, and the field - // name, while the option value doesn't contain field names. - List> fieldNameToOptionKeys = - ImmutableList.of( - // NESTED_KEY is not added since renaming nested columns is not supported - // currently - fieldName -> FIELDS_PREFIX + "." + fieldName + "." + AGG_FUNCTION, - fieldName -> FIELDS_PREFIX + "." + fieldName + "." + IGNORE_RETRACT, - fieldName -> FIELDS_PREFIX + "." + fieldName + "." + DISTINCT, - fieldName -> FIELDS_PREFIX + "." + fieldName + "." + LIST_AGG_DELIMITER, - fieldName -> FIELDS_PREFIX + "." + fieldName + "." + MAP_STORAGE_LAYOUT, - fieldName -> - FIELDS_PREFIX - + "." - + fieldName - + "." - + MAP_SHARED_SHREDDING_MAX_COLUMNS, - fieldName -> - FIELDS_PREFIX - + "." - + fieldName - + "." - + MAP_SHARED_SHREDDING_COLUMN_PLACEMENT_POLICY); - - for (RenameColumn rename : renameColumns) { - String fieldName = rename.fieldNames()[0]; - String newFieldName = rename.newName(); + @Nullable SchemaModification schemaModification); - for (Function fieldNameToKey : fieldNameToOptionKeys) { - String key = fieldNameToKey.apply(fieldName); - if (newOptions.containsKey(key)) { - String value = newOptions.remove(key); - newOptions.put(fieldNameToKey.apply(newFieldName), value); - } - } - } + boolean commit(TableSchema newSchema) throws Exception; - // case 3: both option key and option value may contain field names - for (String key : options.keySet()) { - if (key.startsWith(FIELDS_PREFIX)) { - String matchedSuffix = null; - if (key.endsWith(SEQUENCE_GROUP)) { - matchedSuffix = SEQUENCE_GROUP; - } else if (key.endsWith(NESTED_KEY)) { - matchedSuffix = NESTED_KEY; - } + TableSchema schema(long id); - if (matchedSuffix != null) { - // Both the key and value may contain field names. If we were to perform a - // "match then replace" operation, the conditions would become quite complex. - // Instead, we directly make a replacement across all instances - String keyFieldsStr = - key.substring( - FIELDS_PREFIX.length() + 1, - key.length() - matchedSuffix.length() - 1); - List keyFields = Arrays.asList(keyFieldsStr.split(",")); - List newKeyFields = - applyNotNestedColumnRename(keyFields, renameMappings); + boolean schemaExists(long id); - String valueFieldsStr = newOptions.remove(key); - List valueFields = Arrays.asList(valueFieldsStr.split(",")); - List newValueFields = - applyNotNestedColumnRename(valueFields, renameMappings); - newOptions.put( - FIELDS_PREFIX - + "." - + String.join(",", newKeyFields) - + "." - + matchedSuffix, - String.join(",", newValueFields)); - } - } - } + Path schemaDirectory(); - return newOptions; - } - - // Apply column rename changes on not nested columns to the list of column names, this will not - // change the order of the column names - private static List applyNotNestedColumnRename( - List columns, Iterable renames) { - if (Iterables.isEmpty(renames)) { - return columns; - } - - Map columnNames = Maps.newHashMap(); - for (RenameColumn renameColumn : renames) { - if (renameColumn.fieldNames().length == 1) { - columnNames.put(renameColumn.fieldNames()[0], renameColumn.newName()); - } - } - return applyNotNestedColumnRename(columns, columnNames); - } - - private static List applyNotNestedColumnRename( - List columns, Map renameMapping) { - - // The order of the column names will be preserved, as a non-parallel stream is used here. - return columns.stream() - .map(column -> renameMapping.getOrDefault(column, column)) - .collect(Collectors.toList()); - } - - private static void dropColumnValidation(TableSchema schema, DropColumn change) { - // primary keys and partition keys can't be nested columns - if (change.fieldNames().length > 1) { - return; - } - String columnToDrop = change.fieldNames()[0]; - if (schema.partitionKeys().contains(columnToDrop) - || schema.primaryKeys().contains(columnToDrop)) { - throw new UnsupportedOperationException( - String.format("Cannot drop partition key or primary key: [%s]", columnToDrop)); - } - assertNotUpdatingPrimaryKeyIndexColumn(schema, change.fieldNames(), "drop"); - } - - private static void assertNotUpdatingPartitionKeys( - TableSchema schema, String[] fieldNames, String operation) { - // partition keys can't be nested columns - if (fieldNames.length > 1) { - return; - } - String fieldName = fieldNames[0]; - if (schema.partitionKeys().contains(fieldName)) { - throw new UnsupportedOperationException( - String.format("Cannot %s partition column: [%s]", operation, fieldName)); - } - } - - private static void assertNotUpdatingPrimaryKeys( - TableSchema schema, String[] fieldNames, String operation) { - // primary keys can't be nested columns - if (fieldNames.length > 1) { - return; - } - String fieldName = fieldNames[0]; - if (schema.primaryKeys().contains(fieldName)) { - throw new UnsupportedOperationException( - String.format("Cannot %s primary key", operation)); - } - } - - private static void assertNotRenamingBlobColumn(List fields, String[] fieldNames) { - if (fieldNames.length > 1) { - return; - } - String fieldName = fieldNames[0]; - for (DataField field : fields) { - if (field.name().equals(fieldName) && isBlobFileField(field.type())) { - throw new UnsupportedOperationException( - String.format("Cannot rename BLOB column: [%s]", fieldName)); - } - } - } - - private static void assertNotUpdatingPrimaryKeyIndexColumn( - TableSchema schema, String[] fieldNames, String operation) { - if (fieldNames.length > 1) { - return; - } - String fieldName = fieldNames[0]; - CoreOptions options = new CoreOptions(schema.options()); - if (options.primaryKeyVectorIndexColumns().contains(fieldName) - || options.primaryKeyBTreeIndexColumns().contains(fieldName) - || options.primaryKeyBitmapIndexColumns().contains(fieldName) - || options.primaryKeyFullTextIndexColumns().contains(fieldName)) { - throw new UnsupportedOperationException( - String.format( - "Cannot %s primary-key index column: [%s]", operation, fieldName)); - } - } - - private static void assertNotChangingBlobColumnType( - List fields, String[] fieldNames, DataType newType) { - if (fieldNames.length > 1) { - return; - } - String fieldName = fieldNames[0]; - for (DataField field : fields) { - if (!field.name().equals(fieldName)) { - continue; - } - boolean wasBlob = isBlobFileField(field.type()); - boolean willBeBlob = isBlobFileField(newType); - if (wasBlob || willBeBlob) { - throw new UnsupportedOperationException( - String.format( - "Cannot change column type involving BLOB: [%s] %s -> %s", - fieldName, field.type(), newType)); - } - return; - } - } - - private abstract static class NestedColumnModifier { - - private final String[] updateFieldNames; - private final LazyField identifier; - - private NestedColumnModifier(String[] updateFieldNames, LazyField identifier) { - this.updateFieldNames = updateFieldNames; - this.identifier = identifier; - } - - private void updateIntermediateColumn( - List newFields, List previousFields, int depth, int prevDepth) - throws Catalog.ColumnNotExistException, Catalog.ColumnAlreadyExistException { - if (depth == updateFieldNames.length - 1) { - updateLastColumn(depth, newFields, updateFieldNames[depth]); - return; - } else if (depth >= updateFieldNames.length) { - // to handle the case of ARRAY or MAP type evolution - // for instance : ARRAY -> ARRAY - // the updateFieldNames in this case is [v, element] where v is array field name - // the depth returned by extractRowDataFields is 2 which will overflow. - // So the logic is to go to previous depth and update the column using previous - // fields which will have DataFields from prevDepth - // The reason for this handling is the addition of element and value for array - // and map type in FlinkCatalog as dummy column name - updateLastColumn(prevDepth, previousFields, updateFieldNames[prevDepth]); - return; - } - - for (int i = 0; i < newFields.size(); i++) { - DataField field = newFields.get(i); - if (!field.name().equals(updateFieldNames[depth])) { - continue; - } - List nestedFields = new ArrayList<>(); - int newDepth = depth + extractRowDataFields(field.type(), nestedFields); - updateIntermediateColumn(nestedFields, newFields, newDepth, depth); - field = newFields.get(i); - newFields.set( - i, - new DataField( - field.id(), - field.name(), - wrapNewRowType(field.type(), nestedFields), - field.description(), - field.defaultValue())); - return; - } - - throw new Catalog.ColumnNotExistException( - identifier.get(), - String.join(".", Arrays.asList(updateFieldNames).subList(0, depth + 1))); - } - - public void updateIntermediateColumn(List newFields, int depth) - throws Catalog.ColumnNotExistException, Catalog.ColumnAlreadyExistException { - updateIntermediateColumn(newFields, newFields, depth, depth); - } - - private int extractRowDataFields(DataType type, List nestedFields) { - switch (type.getTypeRoot()) { - case ROW: - nestedFields.addAll(((RowType) type).getFields()); - return 1; - case ARRAY: - return extractRowDataFields(((ArrayType) type).getElementType(), nestedFields) - + 1; - case MAP: - return extractRowDataFields(((MapType) type).getValueType(), nestedFields) + 1; - default: - return 1; - } - } - - private DataType wrapNewRowType(DataType type, List nestedFields) { - switch (type.getTypeRoot()) { - case ROW: - return new RowType(type.isNullable(), nestedFields); - case ARRAY: - return new ArrayType( - type.isNullable(), - wrapNewRowType(((ArrayType) type).getElementType(), nestedFields)); - case MAP: - MapType mapType = (MapType) type; - return new MapType( - type.isNullable(), - mapType.getKeyType(), - wrapNewRowType(mapType.getValueType(), nestedFields)); - default: - return type; - } - } - - protected abstract void updateLastColumn( - int depth, List newFields, String fieldName) - throws Catalog.ColumnNotExistException, Catalog.ColumnAlreadyExistException; - - protected void assertColumnExists( - List newFields, String fieldName, LazyField lazyIdentifier) - throws Catalog.ColumnNotExistException { - for (DataField field : newFields) { - if (field.name().equals(fieldName)) { - return; - } - } - throw new Catalog.ColumnNotExistException( - lazyIdentifier.get(), getLastFieldName(fieldName)); - } - - protected void assertColumnNotExists( - List newFields, String fieldName, LazyField lazyIdentifier) - throws Catalog.ColumnAlreadyExistException { - for (DataField field : newFields) { - if (field.name().equals(fieldName)) { - throw new Catalog.ColumnAlreadyExistException( - lazyIdentifier.get(), getLastFieldName(fieldName)); - } - } - } - - private String getLastFieldName(String fieldName) { - List fieldNames = new ArrayList<>(); - for (int i = 0; i + 1 < updateFieldNames.length; i++) { - fieldNames.add(updateFieldNames[i]); - } - fieldNames.add(fieldName); - return String.join(".", fieldNames); - } - } - - private static void updateNestedColumn( - List newFields, - String[] updateFieldNames, - BiFunction updateFunc, - LazyField lazyIdentifier) - throws Catalog.ColumnNotExistException, Catalog.ColumnAlreadyExistException { - new NestedColumnModifier(updateFieldNames, lazyIdentifier) { - @Override - protected void updateLastColumn(int depth, List newFields, String fieldName) - throws Catalog.ColumnNotExistException { - for (int i = 0; i < newFields.size(); i++) { - DataField field = newFields.get(i); - if (!field.name().equals(fieldName)) { - continue; - } - - newFields.set(i, updateFunc.apply(field, depth)); - return; - } - - throw new Catalog.ColumnNotExistException( - lazyIdentifier.get(), String.join(".", updateFieldNames)); - } - }.updateIntermediateColumn(newFields, 0); - } - - @VisibleForTesting - public boolean commit(TableSchema newSchema) throws Exception { - SchemaValidation.validateTableSchema(newSchema); - SchemaValidation.validateFallbackBranch(this, newSchema); - Path schemaPath = toSchemaPath(newSchema.id()); - return fileIO.tryToWriteAtomic(schemaPath, newSchema.toString()); - } - - /** Read schema for schema id. */ - public TableSchema schema(long id) { - return fromPath(fileIO, toSchemaPath(id)); - } + Path toSchemaPath(long schemaId); - /** Check if a schema exists. */ - public boolean schemaExists(long id) { - Path path = toSchemaPath(id); - try { - return fileIO.exists(path); - } catch (IOException e) { - throw new RuntimeException( - String.format( - "Failed to determine if schema '%s' exists in path %s.", id, path), - e); - } - } + List schemaPaths(Predicate predicate) throws IOException; - private String branchPath() { - return BranchManager.branchPath(tableRoot, branch); - } + void deleteSchema(long schemaId); - public Path schemaDirectory() { - return new Path(branchPath() + "/schema"); - } - - @VisibleForTesting - public Path toSchemaPath(long schemaId) { - return new Path(branchPath() + "/schema/" + SCHEMA_PREFIX + schemaId); - } - - public List schemaPaths(Predicate predicate) throws IOException { - return listVersionedFiles(fileIO, schemaDirectory(), SCHEMA_PREFIX) - .filter(predicate) - .map(this::toSchemaPath) - .collect(Collectors.toList()); - } - - /** - * Delete schema with specific id. - * - * @param schemaId the schema id to delete. - */ - public void deleteSchema(long schemaId) { - fileIO.deleteQuietly(toSchemaPath(schemaId)); - } - - /** - * Rollback to a specific schema version. All schema versions greater than the target will be - * deleted. This operation will fail if any snapshot, tag, or changelog references a schema - * version greater than the target. - * - * @param targetSchemaId the schema version to rollback to. - * @param snapshotManager the snapshot manager to check snapshot references. - * @param tagManager the tag manager to check tag references. - * @param changelogManager the changelog manager to check changelog references. - */ - public void rollbackTo( + void rollbackTo( long targetSchemaId, SnapshotManager snapshotManager, TagManager tagManager, ChangelogManager changelogManager) - throws IOException { - checkArgument(schemaExists(targetSchemaId), "Schema %s does not exist.", targetSchemaId); + throws IOException; - // Collect all schemaIds referenced by snapshots, tags, and changelogs - Set usedSchemaIds = new HashSet<>(); - - snapshotManager.pickOrLatest( - snapshot -> { - usedSchemaIds.add(snapshot.schemaId()); - return false; - }); - tagManager.taggedSnapshots().forEach(s -> usedSchemaIds.add(s.schemaId())); - changelogManager.changelogs().forEachRemaining(c -> usedSchemaIds.add(c.schemaId())); - - // Check if any referenced schema is newer than the target - Optional conflict = - usedSchemaIds.stream().filter(id -> id > targetSchemaId).min(Long::compareTo); - if (conflict.isPresent()) { - throw new RuntimeException( - String.format( - "Cannot rollback to schema %d, schema %d is still referenced by snapshots/tags/changelogs.", - targetSchemaId, conflict.get())); - } + static TableSchema generateTableSchema( + TableSchema oldTableSchema, + List changes, + LazyField hasSnapshots, + LazyField lazyIdentifier) + throws Catalog.ColumnAlreadyExistException, Catalog.ColumnNotExistException { + return FileSystemSchemaManager.generateTableSchema( + oldTableSchema, changes, hasSnapshots, lazyIdentifier); + } - // Delete all schemas newer than the target - List toBeDeleted = - listAllIds().stream() - .filter(id -> id > targetSchemaId) - .collect(Collectors.toList()); - toBeDeleted.sort((o1, o2) -> Long.compare(o2, o1)); - for (Long id : toBeDeleted) { - fileIO.delete(toSchemaPath(id), false); - } + static void applyMove(List newFields, SchemaChange.Move move) { + FileSystemSchemaManager.applyMove(newFields, move); } - /** - * Whether an option change is a semantic no-op: 'primary-key'/'partition' are normalized into - * dedicated schema fields (old value null), and 'type' compares case-insensitively, defaulting - * when unset. - */ - public static boolean isUnchangedNormalizedKey( + static boolean isUnchangedNormalizedKey( String key, @Nullable String oldValue, @Nullable String newValue, TableSchema tableSchema) { - return isUnchangedNormalizedKey( - key, oldValue, newValue, tableSchema.primaryKeys(), tableSchema.partitionKeys()); + return FileSystemSchemaManager.isUnchangedNormalizedKey( + key, oldValue, newValue, tableSchema); } - /** - * Overload for callers holding the primary/partition keys rather than a {@link TableSchema}. - */ - public static boolean isUnchangedNormalizedKey( + static boolean isUnchangedNormalizedKey( String key, @Nullable String oldValue, @Nullable String newValue, List primaryKeys, List partitionKeys) { - if (newValue == null) { - return false; - } - if (CoreOptions.TYPE.key().equals(key)) { - // 'type' compares case-insensitively (like convertToEnum); an unset type is the default - String effectiveOld = - oldValue == null ? CoreOptions.TYPE.defaultValue().toString() : oldValue; - return newValue.equalsIgnoreCase(effectiveOld); - } - if (oldValue != null) { - return false; - } - if (CoreOptions.PRIMARY_KEY.key().equals(key)) { - return normalizeKeyList(newValue).equals(primaryKeys); - } - if (CoreOptions.PARTITION.key().equals(key)) { - return normalizeKeyList(newValue).equals(partitionKeys); - } - return false; + return FileSystemSchemaManager.isUnchangedNormalizedKey( + key, oldValue, newValue, primaryKeys, partitionKeys); } - private static List normalizeKeyList(String value) { - return Arrays.stream(value.split(",")) - .map(String::trim) - .filter(s -> !s.isEmpty()) - .collect(Collectors.toList()); - } - - public static void checkAlterTableOption( + static void checkAlterTableOption( Map options, String key, @Nullable String oldValue, String newValue) { - if (CoreOptions.IMMUTABLE_OPTIONS.contains(key)) { - throw new UnsupportedOperationException( - String.format("Change '%s' is not supported yet.", key)); - } - - if (CoreOptions.BUCKET.key().equals(key)) { - int oldBucket = - oldValue == null - ? CoreOptions.BUCKET.defaultValue() - : Integer.parseInt(oldValue); - int newBucket = Integer.parseInt(newValue); - - if (oldBucket == -1) { - throw new UnsupportedOperationException("Cannot change bucket when it is -1."); - } - if (newBucket == -1) { - throw new UnsupportedOperationException("Cannot change bucket to -1."); - } - } - - if (DELETION_VECTORS_ENABLED.key().equals(key)) { - boolean dvModifiable = - Boolean.parseBoolean( - options.getOrDefault( - DELETION_VECTORS_MODIFIABLE.key(), - DELETION_VECTORS_MODIFIABLE.defaultValue().toString())); - if (!dvModifiable) { - boolean oldDv = - oldValue == null - ? DELETION_VECTORS_ENABLED.defaultValue() - : Boolean.parseBoolean(oldValue); - boolean newDv = Boolean.parseBoolean(newValue); - - if (oldDv != newDv) { - throw new UnsupportedOperationException( - String.format( - "Cannot change deletion vectors mode from %s to %s. If modifying table deletion-vectors mode without full-compaction, this may result in data duplication. " - + "If you are confident, you can set table option '%s' = 'true' to allow deletion vectors modification.", - oldDv, newDv, DELETION_VECTORS_MODIFIABLE.key())); - } - } - } - - if (IGNORE_DELETE.key().equals(key)) { - boolean oldIgnoreDelete = - oldValue == null - ? IGNORE_DELETE.defaultValue() - : Boolean.parseBoolean(oldValue); - boolean newIgnoreDelete = Boolean.parseBoolean(newValue); - if (oldIgnoreDelete && !newIgnoreDelete) { - throw new UnsupportedOperationException( - String.format("Cannot change %s from true to false.", IGNORE_DELETE.key())); - } - } - - if (IGNORE_UPDATE_BEFORE.key().equals(key)) { - boolean oldIgnoreUpdateBefore = - oldValue == null - ? IGNORE_UPDATE_BEFORE.defaultValue() - : Boolean.parseBoolean(oldValue); - boolean newIgnoreUpdateBefore = Boolean.parseBoolean(newValue); - if (oldIgnoreUpdateBefore && !newIgnoreUpdateBefore) { - throw new UnsupportedOperationException( - String.format( - "Cannot change %s from true to false.", - IGNORE_UPDATE_BEFORE.key())); - } - } - - if (CLUSTERING_COLUMNS.key().equals(key)) { - if (options.containsKey(PK_CLUSTERING_OVERRIDE.key())) { - throw new UnsupportedOperationException( - String.format( - "Cannot change %s when %s enabled.", - CLUSTERING_COLUMNS.key(), PK_CLUSTERING_OVERRIDE.key())); - } - } + FileSystemSchemaManager.checkAlterTableOption(options, key, oldValue, newValue); } - public static void checkResetTableOption(Map options, String key) { - if (CoreOptions.IMMUTABLE_OPTIONS.contains(key)) { - throw new UnsupportedOperationException( - String.format("Change '%s' is not supported yet.", key)); - } - - if (CoreOptions.BUCKET.key().equals(key)) { - throw new UnsupportedOperationException(String.format("Cannot reset %s.", key)); - } - - if (options.containsKey(PK_CLUSTERING_OVERRIDE.key()) - && CLUSTERING_COLUMNS.key().equals(key)) { - throw new UnsupportedOperationException( - String.format( - "Cannot reset %s when %s enabled.", key, PK_CLUSTERING_OVERRIDE.key())); - } + static void checkResetTableOption(Map options, String key) { + FileSystemSchemaManager.checkResetTableOption(options, key); } - public static void checkAlterTablePath(String key) { - if (CoreOptions.PATH.key().equalsIgnoreCase(key)) { - throw new UnsupportedOperationException("Change path is not supported yet."); - } + static void checkAlterTablePath(String key) { + FileSystemSchemaManager.checkAlterTablePath(key); } - public static Identifier identifierFromPath(String tablePath, boolean ignoreIfUnknownDatabase) { - return identifierFromPath(tablePath, ignoreIfUnknownDatabase, null); + static Identifier identifierFromPath(String tablePath, boolean ignoreIfUnknownDatabase) { + return FileSystemSchemaManager.identifierFromPath(tablePath, ignoreIfUnknownDatabase); } - public static Identifier identifierFromPath( + static Identifier identifierFromPath( String tablePath, boolean ignoreIfUnknownDatabase, @Nullable String branchName) { - if (DEFAULT_MAIN_BRANCH.equals(branchName)) { - branchName = null; - } - - String[] paths = tablePath.split("/"); - if (paths.length < 2) { - if (!ignoreIfUnknownDatabase) { - throw new IllegalArgumentException( - String.format( - "Path '%s' is not a valid path, please use catalog table path instead: 'warehouse_path/your_database.db/your_table'.", - tablePath)); - } - return new Identifier(UNKNOWN_DATABASE, paths[0]); - } - - String database = paths[paths.length - 2]; - int index = database.lastIndexOf(DB_SUFFIX); - if (index == -1) { - if (!ignoreIfUnknownDatabase) { - throw new IllegalArgumentException( - String.format( - "Path '%s' is not a valid path, please use catalog table path instead: 'warehouse_path/your_database.db/your_table'.", - tablePath)); - } - return new Identifier(UNKNOWN_DATABASE, paths[paths.length - 1], branchName, null); - } - database = database.substring(0, index); - - return new Identifier(database, paths[paths.length - 1], branchName, null); + return FileSystemSchemaManager.identifierFromPath( + tablePath, ignoreIfUnknownDatabase, branchName); } - public static TableSchema fromPath(FileIO fileIO, Path path) { - try { - return tryFromPath(fileIO, path); - } catch (FileNotFoundException e) { - throw new RuntimeException(e.getMessage(), e); - } + static TableSchema fromPath(FileIO fileIO, Path path) { + return FileSystemSchemaManager.fromPath(fileIO, path); } - public static TableSchema tryFromPath(FileIO fileIO, Path path) throws FileNotFoundException { - try { - return TableSchema.fromJson(fileIO.readFileUtf8(path)); - } catch (FileNotFoundException e) { - throw e; - } catch (IOException e) { - throw new UncheckedIOException(e); - } + static TableSchema tryFromPath(FileIO fileIO, Path path) throws FileNotFoundException { + return FileSystemSchemaManager.tryFromPath(fileIO, path); } } diff --git a/paimon-core/src/main/java/org/apache/paimon/table/AbstractFileStoreTable.java b/paimon-core/src/main/java/org/apache/paimon/table/AbstractFileStoreTable.java index 2f893d4e3b84..f6d06c50099e 100644 --- a/paimon-core/src/main/java/org/apache/paimon/table/AbstractFileStoreTable.java +++ b/paimon-core/src/main/java/org/apache/paimon/table/AbstractFileStoreTable.java @@ -32,6 +32,7 @@ import org.apache.paimon.options.ExpireConfig; import org.apache.paimon.options.Options; import org.apache.paimon.predicate.Predicate; +import org.apache.paimon.schema.FileSystemSchemaManager; import org.apache.paimon.schema.SchemaManager; import org.apache.paimon.schema.SchemaValidation; import org.apache.paimon.schema.TableSchema; @@ -408,7 +409,7 @@ public FileStoreTable copy(TableSchema newTableSchema) { @Override public SchemaManager schemaManager() { - return new SchemaManager(fileIO(), path, currentBranch()); + return new FileSystemSchemaManager(fileIO(), path, currentBranch()); } @Override @@ -767,7 +768,7 @@ public FileStoreTable switchToBranch(String branchName) { } Optional optionalSchema = - new SchemaManager(fileIO(), location(), targetBranch).latest(); + new FileSystemSchemaManager(fileIO(), location(), targetBranch).latest(); Preconditions.checkArgument( optionalSchema.isPresent(), "Branch " + targetBranch + " does not exist"); diff --git a/paimon-core/src/main/java/org/apache/paimon/table/FileStoreTableFactory.java b/paimon-core/src/main/java/org/apache/paimon/table/FileStoreTableFactory.java index 53be2803ce70..e58fb03e41c5 100644 --- a/paimon-core/src/main/java/org/apache/paimon/table/FileStoreTableFactory.java +++ b/paimon-core/src/main/java/org/apache/paimon/table/FileStoreTableFactory.java @@ -24,7 +24,7 @@ import org.apache.paimon.fs.FileIO; import org.apache.paimon.fs.Path; import org.apache.paimon.options.Options; -import org.apache.paimon.schema.SchemaManager; +import org.apache.paimon.schema.FileSystemSchemaManager; import org.apache.paimon.schema.TableSchema; import org.apache.paimon.utils.ChainTableUtils; import org.apache.paimon.utils.StringUtils; @@ -63,7 +63,7 @@ public static FileStoreTable create(FileIO fileIO, Options options) { Path tablePath = CoreOptions.path(options); String branchName = CoreOptions.branch(options.toMap()); TableSchema tableSchema = - new SchemaManager(fileIO, tablePath, branchName) + new FileSystemSchemaManager(fileIO, tablePath, branchName) .latest() .orElseThrow( () -> @@ -158,7 +158,7 @@ public static FileStoreTable createChainTable( Options snapshotBranchOptions = new Options(dynamicOptions.toMap()); snapshotBranchOptions.set(CoreOptions.BRANCH, scanFallbackSnapshotBranch); Optional snapshotSchema = - new SchemaManager(fileIO, tablePath, scanFallbackSnapshotBranch).latest(); + new FileSystemSchemaManager(fileIO, tablePath, scanFallbackSnapshotBranch).latest(); AbstractFileStoreTable snapshotTable = (AbstractFileStoreTable) createWithoutFallbackBranch( @@ -170,7 +170,7 @@ public static FileStoreTable createChainTable( Options deltaBranchOptions = new Options(dynamicOptions.toMap()); deltaBranchOptions.set(CoreOptions.BRANCH, scanFallbackDeltaBranch); Optional deltaSchema = - new SchemaManager(fileIO, tablePath, scanFallbackDeltaBranch).latest(); + new FileSystemSchemaManager(fileIO, tablePath, scanFallbackDeltaBranch).latest(); AbstractFileStoreTable deltaTable = (AbstractFileStoreTable) createWithoutFallbackBranch( @@ -192,7 +192,8 @@ private static FileStoreTable createOtherBranchTable( CatalogEnvironment catalogEnvironment) { Options branchOptions = new Options(dynamicOptions.toMap()); branchOptions.set(CoreOptions.BRANCH, branchName); - Optional schema = new SchemaManager(fileIO, tablePath, branchName).latest(); + Optional schema = + new FileSystemSchemaManager(fileIO, tablePath, branchName).latest(); if (schema.isPresent()) { Identifier identifier = catalogEnvironment.identifier(); CatalogEnvironment branchCatalogEnvironment = catalogEnvironment; diff --git a/paimon-core/src/main/java/org/apache/paimon/utils/FileSystemBranchManager.java b/paimon-core/src/main/java/org/apache/paimon/utils/FileSystemBranchManager.java index 9466ccd851cc..5975f3f07620 100644 --- a/paimon-core/src/main/java/org/apache/paimon/utils/FileSystemBranchManager.java +++ b/paimon-core/src/main/java/org/apache/paimon/utils/FileSystemBranchManager.java @@ -25,6 +25,7 @@ import org.apache.paimon.manifest.FileEntry; import org.apache.paimon.manifest.FileKind; import org.apache.paimon.manifest.ManifestEntry; +import org.apache.paimon.schema.FileSystemSchemaManager; import org.apache.paimon.schema.SchemaManager; import org.apache.paimon.schema.TableSchema; import org.apache.paimon.tag.Tag; @@ -267,8 +268,10 @@ private void validateMergeBranches(String sourceBranch, String targetBranch) { } private void validateLatestSchema(String sourceBranch, String targetBranch) { - SchemaManager sourceSchemaMgr = new SchemaManager(fileIO, tablePath, sourceBranch); - SchemaManager targetSchemaMgr = new SchemaManager(fileIO, tablePath, targetBranch); + SchemaManager sourceSchemaMgr = + new FileSystemSchemaManager(fileIO, tablePath, sourceBranch); + SchemaManager targetSchemaMgr = + new FileSystemSchemaManager(fileIO, tablePath, targetBranch); TableSchema sourceSchema = sourceSchemaMgr.latest().get(); TableSchema targetSchema = targetSchemaMgr.latest().get(); checkArgument( @@ -280,8 +283,10 @@ private void validateLatestSchema(String sourceBranch, String targetBranch) { private void validateMergeFileSchemas( String sourceBranch, String targetBranch, List filesToMerge) { - SchemaManager sourceSchemaMgr = new SchemaManager(fileIO, tablePath, sourceBranch); - SchemaManager targetSchemaMgr = new SchemaManager(fileIO, tablePath, targetBranch); + SchemaManager sourceSchemaMgr = + new FileSystemSchemaManager(fileIO, tablePath, sourceBranch); + SchemaManager targetSchemaMgr = + new FileSystemSchemaManager(fileIO, tablePath, targetBranch); for (Long schemaId : filesToMerge.stream() @@ -307,7 +312,8 @@ private void validateMergeFileSchemas( } private void validateAppendOnly(String sourceBranch, String targetBranch) { - SchemaManager sourceSchemaMgr = new SchemaManager(fileIO, tablePath, sourceBranch); + SchemaManager sourceSchemaMgr = + new FileSystemSchemaManager(fileIO, tablePath, sourceBranch); TableSchema sourceSchema = sourceSchemaMgr.latest().get(); checkArgument( sourceSchema.primaryKeys().isEmpty(), @@ -315,7 +321,8 @@ private void validateAppendOnly(String sourceBranch, String targetBranch) { + "but branch '%s' has primary keys.", sourceBranch); - SchemaManager targetSchemaMgr = new SchemaManager(fileIO, tablePath, targetBranch); + SchemaManager targetSchemaMgr = + new FileSystemSchemaManager(fileIO, tablePath, targetBranch); TableSchema targetSchema = targetSchemaMgr.latest().get(); checkArgument( targetSchema.primaryKeys().isEmpty(), @@ -369,7 +376,7 @@ private void validateCompleteAppendOnly(SnapshotManager sm, String branch) { private void validateNoDataEvolution(String sourceBranch, String targetBranch) { for (String branch : new String[] {sourceBranch, targetBranch}) { - SchemaManager sm = new SchemaManager(fileIO, tablePath, branch); + SchemaManager sm = new FileSystemSchemaManager(fileIO, tablePath, branch); TableSchema schema = sm.latest().get(); CoreOptions opts = new CoreOptions(schema.options()); checkArgument( @@ -393,7 +400,7 @@ private void validateRowTrackingConsistent(String sourceBranch, String targetBra } private boolean isRowTrackingEnabled(String branch) { - SchemaManager schemaManager = new SchemaManager(fileIO, tablePath, branch); + SchemaManager schemaManager = new FileSystemSchemaManager(fileIO, tablePath, branch); TableSchema schema = schemaManager.latest().get(); return new CoreOptions(schema.options()).rowTrackingEnabled(); } diff --git a/paimon-core/src/test/java/org/apache/paimon/JavaPyE2ETest.java b/paimon-core/src/test/java/org/apache/paimon/JavaPyE2ETest.java index b6801b4a7bca..d49d20b4910c 100644 --- a/paimon-core/src/test/java/org/apache/paimon/JavaPyE2ETest.java +++ b/paimon-core/src/test/java/org/apache/paimon/JavaPyE2ETest.java @@ -61,8 +61,8 @@ import org.apache.paimon.options.Options; import org.apache.paimon.predicate.Predicate; import org.apache.paimon.predicate.PredicateBuilder; +import org.apache.paimon.schema.FileSystemSchemaManager; import org.apache.paimon.schema.Schema; -import org.apache.paimon.schema.SchemaManager; import org.apache.paimon.schema.SchemaUtils; import org.apache.paimon.schema.TableSchema; import org.apache.paimon.table.AppendOnlyFileStoreTable; @@ -636,7 +636,7 @@ public void testBtreeRawFallbackWrite() throws Exception { options.set(GLOBAL_INDEX_ENABLED, true); TableSchema tableSchema = SchemaUtils.forceCommit( - new SchemaManager(LocalFileIO.create(), tablePath), + new FileSystemSchemaManager(LocalFileIO.create(), tablePath), new Schema( rowType.getFields(), Collections.emptyList(), @@ -697,7 +697,7 @@ public void testBitmapIndexWrite() throws Exception { options.set(GLOBAL_INDEX_ENABLED, true); TableSchema tableSchema = SchemaUtils.forceCommit( - new SchemaManager(LocalFileIO.create(), tablePath), + new FileSystemSchemaManager(LocalFileIO.create(), tablePath), new Schema( rowType.getFields(), Collections.emptyList(), @@ -795,7 +795,7 @@ private void testBtreeIndexWriteGeneric( options.set(GLOBAL_INDEX_ENABLED, true); TableSchema tableSchema = SchemaUtils.forceCommit( - new SchemaManager(LocalFileIO.create(), tablePath), + new FileSystemSchemaManager(LocalFileIO.create(), tablePath), new Schema( rowType.getFields(), Collections.emptyList(), @@ -864,7 +864,7 @@ private void writeCompressedGlobalIndexTable( TableSchema tableSchema = SchemaUtils.forceCommit( - new SchemaManager(LocalFileIO.create(), tablePath), + new FileSystemSchemaManager(LocalFileIO.create(), tablePath), new Schema( rowType.getFields(), Collections.emptyList(), @@ -931,7 +931,7 @@ private void testBtreeIndexWriteLarge() throws Exception { options.set(BTREE_INDEX_COMPRESSION, "zstd"); TableSchema tableSchema = SchemaUtils.forceCommit( - new SchemaManager(LocalFileIO.create(), tablePath), + new FileSystemSchemaManager(LocalFileIO.create(), tablePath), new Schema( rowType.getFields(), Collections.emptyList(), @@ -997,7 +997,7 @@ private void testBtreeIndexWriteNull() throws Exception { options.set(GLOBAL_INDEX_ENABLED, true); TableSchema tableSchema = SchemaUtils.forceCommit( - new SchemaManager(LocalFileIO.create(), tablePath), + new FileSystemSchemaManager(LocalFileIO.create(), tablePath), new Schema( rowType.getFields(), Collections.emptyList(), @@ -1239,7 +1239,7 @@ protected FileStoreTable createFileStoreTable(Consumer configure, Path configure.accept(options); TableSchema tableSchema = SchemaUtils.forceCommit( - new SchemaManager(LocalFileIO.create(), tablePath), + new FileSystemSchemaManager(LocalFileIO.create(), tablePath), new Schema( rowType.getFields(), Collections.singletonList("pt"), diff --git a/paimon-core/src/test/java/org/apache/paimon/TestAppendFileStore.java b/paimon-core/src/test/java/org/apache/paimon/TestAppendFileStore.java index 2754022dd77a..827ec9d499c3 100644 --- a/paimon-core/src/test/java/org/apache/paimon/TestAppendFileStore.java +++ b/paimon-core/src/test/java/org/apache/paimon/TestAppendFileStore.java @@ -33,6 +33,7 @@ import org.apache.paimon.io.DataIncrement; import org.apache.paimon.manifest.ManifestCommittable; import org.apache.paimon.operation.FileStoreCommitImpl; +import org.apache.paimon.schema.FileSystemSchemaManager; import org.apache.paimon.schema.Schema; import org.apache.paimon.schema.SchemaManager; import org.apache.paimon.schema.SchemaUtils; @@ -205,7 +206,7 @@ public static TestAppendFileStore createAppendStore( String root = TraceableFileIO.SCHEME + "://" + tempDir.toString(); Path path = new Path(tempDir.toUri()); FileIO fileIO = FileIOFinder.find(new Path(root)); - SchemaManager schemaManage = new SchemaManager(new LocalFileIO(), path); + SchemaManager schemaManage = new FileSystemSchemaManager(new LocalFileIO(), path); options.put(CoreOptions.PATH.key(), root); TableSchema tableSchema = diff --git a/paimon-core/src/test/java/org/apache/paimon/TestFileStore.java b/paimon-core/src/test/java/org/apache/paimon/TestFileStore.java index b670ffa48f0b..2daf59c833da 100644 --- a/paimon-core/src/test/java/org/apache/paimon/TestFileStore.java +++ b/paimon-core/src/test/java/org/apache/paimon/TestFileStore.java @@ -46,6 +46,7 @@ import org.apache.paimon.options.MemorySize; import org.apache.paimon.options.Options; import org.apache.paimon.reader.RecordReaderIterator; +import org.apache.paimon.schema.FileSystemSchemaManager; import org.apache.paimon.schema.KeyValueFieldsExtractor; import org.apache.paimon.schema.SchemaManager; import org.apache.paimon.schema.TableSchema; @@ -155,7 +156,7 @@ private static List cleanPrimaryKeys(List primaryKeys) { } private static SchemaManager schemaManager(String root, CoreOptions options) { - return new SchemaManager(FileIOFinder.find(new Path(root)), options.path()); + return new FileSystemSchemaManager(FileIOFinder.find(new Path(root)), options.path()); } public FileIO fileIO() { @@ -538,7 +539,7 @@ public void assertCleaned() throws IOException { private Set getFilesInUse() { Set result = new HashSet<>(); - SchemaManager schemaManager = new SchemaManager(fileIO, options.path()); + SchemaManager schemaManager = new FileSystemSchemaManager(fileIO, options.path()); schemaManager.listAllIds().forEach(id -> result.add(schemaManager.toSchemaPath(id))); SnapshotManager snapshotManager = snapshotManager(); @@ -625,7 +626,8 @@ private static Set getSnapshotFileInUse( ManifestList manifestList, ManifestFile manifestFile) { Set result = new HashSet<>(); - SchemaManager schemaManager = new SchemaManager(fileIO, snapshotManager.tablePath()); + SchemaManager schemaManager = + new FileSystemSchemaManager(fileIO, snapshotManager.tablePath()); CoreOptions options = new CoreOptions(schemaManager.latest().get().options()); boolean produceChangelog = options.changelogProducer() != CoreOptions.ChangelogProducer.NONE; @@ -698,7 +700,8 @@ private static Set getChangelogFileInUse( ManifestList manifestList, ManifestFile manifestFile) { Set result = new HashSet<>(); - SchemaManager schemaManager = new SchemaManager(fileIO, snapshotManager.tablePath()); + SchemaManager schemaManager = + new FileSystemSchemaManager(fileIO, snapshotManager.tablePath()); CoreOptions options = new CoreOptions(schemaManager.latest().get().options()); Path changelogPath = changelogManager.longLivedChangelogPath(changelogId); diff --git a/paimon-core/src/test/java/org/apache/paimon/append/AppendCompactCoordinatorTest.java b/paimon-core/src/test/java/org/apache/paimon/append/AppendCompactCoordinatorTest.java index 35bfa201d94f..703bcd8814c0 100644 --- a/paimon-core/src/test/java/org/apache/paimon/append/AppendCompactCoordinatorTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/append/AppendCompactCoordinatorTest.java @@ -24,6 +24,7 @@ import org.apache.paimon.fs.local.LocalFileIO; import org.apache.paimon.io.DataFileMeta; import org.apache.paimon.manifest.FileSource; +import org.apache.paimon.schema.FileSystemSchemaManager; import org.apache.paimon.schema.Schema; import org.apache.paimon.schema.SchemaManager; import org.apache.paimon.schema.TableSchema; @@ -227,7 +228,7 @@ private static Schema schema() { public void createCoordinator() throws Exception { FileIO fileIO = new LocalFileIO(); org.apache.paimon.fs.Path path = new org.apache.paimon.fs.Path(tempDir.toString()); - SchemaManager schemaManager = new SchemaManager(fileIO, path); + SchemaManager schemaManager = new FileSystemSchemaManager(fileIO, path); TableSchema tableSchema = schemaManager.createTable(schema()); appendOnlyFileStoreTable = diff --git a/paimon-core/src/test/java/org/apache/paimon/append/AppendCompactTaskTest.java b/paimon-core/src/test/java/org/apache/paimon/append/AppendCompactTaskTest.java index 0e95135873b0..19fc3875931d 100644 --- a/paimon-core/src/test/java/org/apache/paimon/append/AppendCompactTaskTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/append/AppendCompactTaskTest.java @@ -34,6 +34,7 @@ import org.apache.paimon.operation.BaseAppendFileStoreWrite; import org.apache.paimon.operation.FileStoreScan; import org.apache.paimon.operation.RawFileSplitRead; +import org.apache.paimon.schema.FileSystemSchemaManager; import org.apache.paimon.schema.Schema; import org.apache.paimon.schema.SchemaManager; import org.apache.paimon.schema.SchemaUtils; @@ -186,7 +187,7 @@ private TestAppendFileStore createAppendStore( String root = TraceableFileIO.SCHEME + "://" + tempDir.toString(); Path path = new Path(tempDir.toUri()); FileIO fileIO = FileIOFinder.find(new Path(root)); - SchemaManager schemaManage = new SchemaManager(new LocalFileIO(), path); + SchemaManager schemaManage = new FileSystemSchemaManager(new LocalFileIO(), path); Map options = new HashMap<>(dynamicOptions); options.put(CoreOptions.PATH.key(), root); diff --git a/paimon-core/src/test/java/org/apache/paimon/append/AppendOnlyTableCompactionTest.java b/paimon-core/src/test/java/org/apache/paimon/append/AppendOnlyTableCompactionTest.java index e4d1c43ed29e..6c3246ae0b12 100644 --- a/paimon-core/src/test/java/org/apache/paimon/append/AppendOnlyTableCompactionTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/append/AppendOnlyTableCompactionTest.java @@ -25,8 +25,8 @@ import org.apache.paimon.fs.local.LocalFileIO; import org.apache.paimon.io.DataFileMeta; import org.apache.paimon.operation.BaseAppendFileStoreWrite; +import org.apache.paimon.schema.FileSystemSchemaManager; import org.apache.paimon.schema.Schema; -import org.apache.paimon.schema.SchemaManager; import org.apache.paimon.schema.TableSchema; import org.apache.paimon.table.FileStoreTable; import org.apache.paimon.table.FileStoreTableFactory; @@ -75,7 +75,7 @@ public class AppendOnlyTableCompactionTest { public void createNegativeAppendOnlyTable() throws Exception { FileIO fileIO = new LocalFileIO(); path = new org.apache.paimon.fs.Path(tempDir.toString()); - tableSchema = new SchemaManager(fileIO, path).createTable(schema()); + tableSchema = new FileSystemSchemaManager(fileIO, path).createTable(schema()); snapshotManager = newSnapshotManager(fileIO, path); recreate(); } diff --git a/paimon-core/src/test/java/org/apache/paimon/append/cluster/HistoryPartitionClusterTest.java b/paimon-core/src/test/java/org/apache/paimon/append/cluster/HistoryPartitionClusterTest.java index f1f93ca989b2..2d64663f9afd 100644 --- a/paimon-core/src/test/java/org/apache/paimon/append/cluster/HistoryPartitionClusterTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/append/cluster/HistoryPartitionClusterTest.java @@ -26,6 +26,7 @@ import org.apache.paimon.fs.local.LocalFileIO; import org.apache.paimon.mergetree.LevelSortedRun; import org.apache.paimon.partition.PartitionPredicate; +import org.apache.paimon.schema.FileSystemSchemaManager; import org.apache.paimon.schema.Schema; import org.apache.paimon.schema.SchemaManager; import org.apache.paimon.table.FileStoreTable; @@ -126,7 +127,7 @@ protected FileStoreTable createTable( ""); SchemaManager schemaManager = - new SchemaManager(LocalFileIO.create(), new Path(tempDir.toString())); + new FileSystemSchemaManager(LocalFileIO.create(), new Path(tempDir.toString())); return FileStoreTableFactory.create( LocalFileIO.create(), new Path(tempDir.toString()), diff --git a/paimon-core/src/test/java/org/apache/paimon/append/cluster/IncrementalClusterManagerTest.java b/paimon-core/src/test/java/org/apache/paimon/append/cluster/IncrementalClusterManagerTest.java index b38cf3af997a..464c3c77038c 100644 --- a/paimon-core/src/test/java/org/apache/paimon/append/cluster/IncrementalClusterManagerTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/append/cluster/IncrementalClusterManagerTest.java @@ -29,6 +29,7 @@ import org.apache.paimon.manifest.FileSource; import org.apache.paimon.mergetree.LevelSortedRun; import org.apache.paimon.partition.PartitionPredicate; +import org.apache.paimon.schema.FileSystemSchemaManager; import org.apache.paimon.schema.Schema; import org.apache.paimon.schema.SchemaChange; import org.apache.paimon.schema.SchemaManager; @@ -260,7 +261,7 @@ protected FileStoreTable createTable( ""); SchemaManager schemaManager = - new SchemaManager(LocalFileIO.create(), new Path(tempDir.toString())); + new FileSystemSchemaManager(LocalFileIO.create(), new Path(tempDir.toString())); return FileStoreTableFactory.create( LocalFileIO.create(), new Path(tempDir.toString()), diff --git a/paimon-core/src/test/java/org/apache/paimon/append/cluster/IncrementalClusterStrategyTest.java b/paimon-core/src/test/java/org/apache/paimon/append/cluster/IncrementalClusterStrategyTest.java index a00f2442ae48..9f87b5c276c9 100644 --- a/paimon-core/src/test/java/org/apache/paimon/append/cluster/IncrementalClusterStrategyTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/append/cluster/IncrementalClusterStrategyTest.java @@ -26,6 +26,7 @@ import org.apache.paimon.manifest.FileSource; import org.apache.paimon.mergetree.LevelSortedRun; import org.apache.paimon.mergetree.SortedRun; +import org.apache.paimon.schema.FileSystemSchemaManager; import org.apache.paimon.schema.Schema; import org.apache.paimon.schema.SchemaChange; import org.apache.paimon.schema.SchemaManager; @@ -56,7 +57,8 @@ public class IncrementalClusterStrategyTest { @BeforeAll public static void setUp() throws Exception { - schemaManager = new SchemaManager(LocalFileIO.create(), new Path(tempDir.toString())); + schemaManager = + new FileSystemSchemaManager(LocalFileIO.create(), new Path(tempDir.toString())); prepareSchema(); incrementalClusterStrategy = new IncrementalClusterStrategy(schemaManager, Arrays.asList("f0", "f1"), 25, 1, 3); diff --git a/paimon-core/src/test/java/org/apache/paimon/append/cluster/SorterTest.java b/paimon-core/src/test/java/org/apache/paimon/append/cluster/SorterTest.java index 31fbeb5c28ee..8b315310325b 100644 --- a/paimon-core/src/test/java/org/apache/paimon/append/cluster/SorterTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/append/cluster/SorterTest.java @@ -27,6 +27,7 @@ import org.apache.paimon.fs.Path; import org.apache.paimon.fs.local.LocalFileIO; import org.apache.paimon.reader.RecordReaderIterator; +import org.apache.paimon.schema.FileSystemSchemaManager; import org.apache.paimon.schema.Schema; import org.apache.paimon.schema.SchemaManager; import org.apache.paimon.table.FileStoreTable; @@ -130,7 +131,8 @@ protected FileStoreTable createTable(Map customOptions, String c ""); SchemaManager schemaManager = - new SchemaManager(LocalFileIO.create(), new Path(tableTempDir.toString())); + new FileSystemSchemaManager( + LocalFileIO.create(), new Path(tableTempDir.toString())); return FileStoreTableFactory.create( LocalFileIO.create(), new Path(tableTempDir.toString()), diff --git a/paimon-core/src/test/java/org/apache/paimon/catalog/FileSystemCatalogTest.java b/paimon-core/src/test/java/org/apache/paimon/catalog/FileSystemCatalogTest.java index 5e186cdf10df..d6706b4bdaf6 100644 --- a/paimon-core/src/test/java/org/apache/paimon/catalog/FileSystemCatalogTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/catalog/FileSystemCatalogTest.java @@ -22,8 +22,8 @@ import org.apache.paimon.TableType; import org.apache.paimon.fs.Path; import org.apache.paimon.options.Options; +import org.apache.paimon.schema.FileSystemSchemaManager; import org.apache.paimon.schema.Schema; -import org.apache.paimon.schema.SchemaManager; import org.apache.paimon.types.DataTypes; import org.apache.paimon.shade.guava30.com.google.common.collect.Lists; @@ -142,7 +142,7 @@ public void testPartitionsFromCatalogAreRejectedOutsideRestCatalog() throws Exce Path tablePath = ((FileSystemCatalog) DelegateCatalog.rootCatalog(catalog)) .getTableLocation(identifier); - new SchemaManager(fileIO, tablePath).createTable(schema); + new FileSystemSchemaManager(fileIO, tablePath).createTable(schema); // The option names one thing only, so a catalog that cannot serve it says so rather than // reading the table's partitions from somewhere the option did not ask for. diff --git a/paimon-core/src/test/java/org/apache/paimon/iceberg/IcebergCompatibilityTest.java b/paimon-core/src/test/java/org/apache/paimon/iceberg/IcebergCompatibilityTest.java index 6c6ceed190db..5f6dae1cb349 100644 --- a/paimon-core/src/test/java/org/apache/paimon/iceberg/IcebergCompatibilityTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/iceberg/IcebergCompatibilityTest.java @@ -46,6 +46,7 @@ import org.apache.paimon.options.ExpireConfig; import org.apache.paimon.options.MemorySize; import org.apache.paimon.options.Options; +import org.apache.paimon.schema.FileSystemSchemaManager; import org.apache.paimon.schema.Schema; import org.apache.paimon.schema.SchemaChange; import org.apache.paimon.schema.SchemaManager; @@ -479,7 +480,7 @@ public void testCommitAfterRollbackDoesNotDuplicateSchemas() throws Exception { write.close(); commit.close(); - SchemaManager schemaManager = new SchemaManager(table.fileIO(), table.location()); + SchemaManager schemaManager = new FileSystemSchemaManager(table.fileIO(), table.location()); schemaManager.commitChanges(SchemaChange.addColumn("w", DataTypes.INT())); table = table.copyWithLatestSchema(); write = table.newWrite(commitUser); @@ -531,7 +532,7 @@ public void testRollbackSnapshotRecordsItsOwnSchema() throws Exception { write.close(); commit.close(); - SchemaManager schemaManager = new SchemaManager(table.fileIO(), table.location()); + SchemaManager schemaManager = new FileSystemSchemaManager(table.fileIO(), table.location()); schemaManager.commitChanges(SchemaChange.addColumn("w", DataTypes.INT())); table = table.copyWithLatestSchema(); write = table.newWrite(commitUser); @@ -581,7 +582,7 @@ public void testBaseLessRebuildRecordsRollbackSnapshotSchema() throws Exception write.close(); commit.close(); - SchemaManager schemaManager = new SchemaManager(table.fileIO(), table.location()); + SchemaManager schemaManager = new FileSystemSchemaManager(table.fileIO(), table.location()); schemaManager.commitChanges(SchemaChange.addColumn("w", DataTypes.INT())); FileStoreTable evolved = table.copyWithLatestSchema(); TableWriteImpl write2 = evolved.newWrite(commitUser); @@ -635,7 +636,7 @@ public void testBaseLessRebuildKeepsLastColumnIdAboveDroppedFields() throws Exce write.close(); commit.close(); - SchemaManager schemaManager = new SchemaManager(table.fileIO(), table.location()); + SchemaManager schemaManager = new FileSystemSchemaManager(table.fileIO(), table.location()); schemaManager.commitChanges(SchemaChange.addColumn("w", DataTypes.INT())); schemaManager.commitChanges(SchemaChange.dropColumn("w")); @@ -681,7 +682,7 @@ public void testSchemaPointerRollbackKeepsHistory() throws Exception { write.close(); commit.close(); - SchemaManager schemaManager = new SchemaManager(table.fileIO(), table.location()); + SchemaManager schemaManager = new FileSystemSchemaManager(table.fileIO(), table.location()); schemaManager.commitChanges(SchemaChange.addColumn("w", DataTypes.INT())); FileStoreTable evolved = table.copyWithLatestSchema(); TableWriteImpl write2 = evolved.newWrite(commitUser); @@ -740,7 +741,7 @@ public void testSchemaRollbackWithAbandonedBaseRebuildsMetadata() throws Excepti write.close(); commit.close(); - SchemaManager schemaManager = new SchemaManager(table.fileIO(), table.location()); + SchemaManager schemaManager = new FileSystemSchemaManager(table.fileIO(), table.location()); schemaManager.commitChanges(SchemaChange.addColumn("w", DataTypes.INT())); FileStoreTable evolved = table.copyWithLatestSchema(); TableWriteImpl write2 = evolved.newWrite(commitUser); @@ -783,7 +784,7 @@ public void testReusedSchemaIdAfterSchemaRollbackRebuildsMetadata() throws Excep write.close(); commit.close(); - SchemaManager schemaManager = new SchemaManager(table.fileIO(), table.location()); + SchemaManager schemaManager = new FileSystemSchemaManager(table.fileIO(), table.location()); schemaManager.commitChanges(SchemaChange.addColumn("w", DataTypes.INT())); FileStoreTable evolved = table.copyWithLatestSchema(); TableWriteImpl write2 = evolved.newWrite(commitUser); @@ -838,7 +839,7 @@ public void testRebuildAfterSchemaRollbackKeepsLastColumnIdHighWaterMark() throw write.close(); commit.close(); - SchemaManager schemaManager = new SchemaManager(table.fileIO(), table.location()); + SchemaManager schemaManager = new FileSystemSchemaManager(table.fileIO(), table.location()); schemaManager.commitChanges(SchemaChange.addColumn("w", DataTypes.INT())); schemaManager.commitChanges(SchemaChange.addColumn("y", DataTypes.INT())); FileStoreTable evolved = table.copyWithLatestSchema(); @@ -894,7 +895,7 @@ public void testSchemaChange() throws Exception { commit.commit(1, write.prepareCommit(false, 1)); assertThat(getIcebergResult()).containsExactlyInAnyOrder("Record(1, 10)", "Record(2, 20)"); - SchemaManager schemaManager = new SchemaManager(table.fileIO(), table.location()); + SchemaManager schemaManager = new FileSystemSchemaManager(table.fileIO(), table.location()); schemaManager.commitChanges(SchemaChange.addColumn("v2", DataTypes.STRING())); table = table.copyWithLatestSchema(); write.close(); diff --git a/paimon-core/src/test/java/org/apache/paimon/index/pkfulltext/PkFullTextDataFileReaderTest.java b/paimon-core/src/test/java/org/apache/paimon/index/pkfulltext/PkFullTextDataFileReaderTest.java index 47a6303302fe..56a181152ab5 100644 --- a/paimon-core/src/test/java/org/apache/paimon/index/pkfulltext/PkFullTextDataFileReaderTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/index/pkfulltext/PkFullTextDataFileReaderTest.java @@ -32,6 +32,7 @@ import org.apache.paimon.io.RollingFileWriter; import org.apache.paimon.manifest.FileSource; import org.apache.paimon.options.Options; +import org.apache.paimon.schema.FileSystemSchemaManager; import org.apache.paimon.schema.KeyValueFieldsExtractor; import org.apache.paimon.schema.SchemaManager; import org.apache.paimon.schema.TableSchema; @@ -77,7 +78,7 @@ void testReadsProjectedTextAndPreservesNullPositions() throws Exception { Collections.singletonList("id"), Collections.emptyMap(), ""); - SchemaManager schemaManager = new SchemaManager(fileIO, tablePath); + SchemaManager schemaManager = new FileSystemSchemaManager(fileIO, tablePath); assertThat(schemaManager.commit(schema)).isTrue(); FileStorePathFactory pathFactory = createNonPartFactory(tablePath); FlushingFileFormat format = new FlushingFileFormat("avro"); diff --git a/paimon-core/src/test/java/org/apache/paimon/index/pksorted/PkSortedDataFileReaderTest.java b/paimon-core/src/test/java/org/apache/paimon/index/pksorted/PkSortedDataFileReaderTest.java index 424689c2e818..c6ee34018cfc 100644 --- a/paimon-core/src/test/java/org/apache/paimon/index/pksorted/PkSortedDataFileReaderTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/index/pksorted/PkSortedDataFileReaderTest.java @@ -32,6 +32,7 @@ import org.apache.paimon.io.RollingFileWriter; import org.apache.paimon.manifest.FileSource; import org.apache.paimon.options.Options; +import org.apache.paimon.schema.FileSystemSchemaManager; import org.apache.paimon.schema.KeyValueFieldsExtractor; import org.apache.paimon.schema.SchemaManager; import org.apache.paimon.schema.TableSchema; @@ -75,7 +76,7 @@ void testReadsNullWithItsPhysicalPosition() throws Exception { Collections.singletonList("id"), Collections.emptyMap(), ""); - SchemaManager schemaManager = new SchemaManager(fileIO, tablePath); + SchemaManager schemaManager = new FileSystemSchemaManager(fileIO, tablePath); assertThat(schemaManager.commit(schema)).isTrue(); FileStorePathFactory pathFactory = createNonPartFactory(tablePath); FlushingFileFormat format = new FlushingFileFormat("avro"); diff --git a/paimon-core/src/test/java/org/apache/paimon/index/pksorted/PrimaryKeySortedIndexMaintenanceTest.java b/paimon-core/src/test/java/org/apache/paimon/index/pksorted/PrimaryKeySortedIndexMaintenanceTest.java index 473ffe79633f..8c320c2054a6 100644 --- a/paimon-core/src/test/java/org/apache/paimon/index/pksorted/PrimaryKeySortedIndexMaintenanceTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/index/pksorted/PrimaryKeySortedIndexMaintenanceTest.java @@ -43,6 +43,7 @@ import org.apache.paimon.mergetree.compact.LookupMergeFunction; import org.apache.paimon.operation.AbstractFileStoreWrite; import org.apache.paimon.operation.FileStoreCommit; +import org.apache.paimon.schema.FileSystemSchemaManager; import org.apache.paimon.schema.Schema; import org.apache.paimon.schema.SchemaManager; import org.apache.paimon.schema.TableSchema; @@ -324,7 +325,7 @@ private TestFileStore createStore() throws Exception { options.put(CoreOptions.PK_BTREE_INDEX_COLUMNS.key(), "itemId"); options.put(CoreOptions.PK_BITMAP_INDEX_COLUMNS.key(), "comment"); SchemaManager schemaManager = - new SchemaManager(LocalFileIO.create(), new Path(tempDir.toUri())); + new FileSystemSchemaManager(LocalFileIO.create(), new Path(tempDir.toUri())); TableSchema schema = schemaManager.createTable( new Schema( @@ -367,7 +368,7 @@ private TestFileStore createMixedStore() throws Exception { fields.add(new DataField(7, "embedding", DataTypes.VECTOR(2, DataTypes.FLOAT()))); RowType rowType = new RowType(fields); SchemaManager schemaManager = - new SchemaManager(LocalFileIO.create(), new Path(tempDir.toUri())); + new FileSystemSchemaManager(LocalFileIO.create(), new Path(tempDir.toUri())); TableSchema schema = schemaManager.createTable( new Schema( diff --git a/paimon-core/src/test/java/org/apache/paimon/index/pkvector/PkVectorDataFileReaderTest.java b/paimon-core/src/test/java/org/apache/paimon/index/pkvector/PkVectorDataFileReaderTest.java index 420e0d5ed5c0..685d5d4dfee0 100644 --- a/paimon-core/src/test/java/org/apache/paimon/index/pkvector/PkVectorDataFileReaderTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/index/pkvector/PkVectorDataFileReaderTest.java @@ -33,6 +33,7 @@ import org.apache.paimon.io.RollingFileWriter; import org.apache.paimon.manifest.FileSource; import org.apache.paimon.options.Options; +import org.apache.paimon.schema.FileSystemSchemaManager; import org.apache.paimon.schema.KeyValueFieldsExtractor; import org.apache.paimon.schema.SchemaManager; import org.apache.paimon.schema.TableSchema; @@ -79,7 +80,7 @@ void testReadsProjectedVectorsAndPreservesNullPositions() throws Exception { Collections.singletonList("id"), Collections.emptyMap(), ""); - SchemaManager schemaManager = new SchemaManager(fileIO, tablePath); + SchemaManager schemaManager = new FileSystemSchemaManager(fileIO, tablePath); assertThat(schemaManager.commit(schema)).isTrue(); FileStorePathFactory pathFactory = createNonPartFactory(tablePath); FlushingFileFormat format = new FlushingFileFormat("avro"); diff --git a/paimon-core/src/test/java/org/apache/paimon/io/DataFileIndexWriterTest.java b/paimon-core/src/test/java/org/apache/paimon/io/DataFileIndexWriterTest.java index e02a04b06391..df892f344758 100644 --- a/paimon-core/src/test/java/org/apache/paimon/io/DataFileIndexWriterTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/io/DataFileIndexWriterTest.java @@ -36,6 +36,7 @@ import org.apache.paimon.options.CatalogOptions; import org.apache.paimon.options.Options; import org.apache.paimon.predicate.FieldRef; +import org.apache.paimon.schema.FileSystemSchemaManager; import org.apache.paimon.schema.Schema; import org.apache.paimon.schema.SchemaManager; import org.apache.paimon.schema.TableSchema; @@ -158,7 +159,7 @@ protected void foreachIndexReader( throws Catalog.TableNotExistException { Path tableRoot = fileSystemCatalog.getTableLocation(Identifier.create(tableName, tableName)); - SchemaManager schemaManager = new SchemaManager(fileIO, tableRoot); + SchemaManager schemaManager = new FileSystemSchemaManager(fileIO, tableRoot); FileStorePathFactory pathFactory = new FileStorePathFactory( tableRoot, diff --git a/paimon-core/src/test/java/org/apache/paimon/manifest/ManifestFileMetaTest.java b/paimon-core/src/test/java/org/apache/paimon/manifest/ManifestFileMetaTest.java index 7dff697e8fc6..24a327c373be 100644 --- a/paimon-core/src/test/java/org/apache/paimon/manifest/ManifestFileMetaTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/manifest/ManifestFileMetaTest.java @@ -33,7 +33,7 @@ import org.apache.paimon.operation.ManifestFileMerger; import org.apache.paimon.options.Options; import org.apache.paimon.partition.PartitionPredicate; -import org.apache.paimon.schema.SchemaManager; +import org.apache.paimon.schema.FileSystemSchemaManager; import org.apache.paimon.stats.StatsTestUtils; import org.apache.paimon.types.IntType; import org.apache.paimon.types.RowType; @@ -1423,7 +1423,7 @@ public void testManifestSortWithMultiplePartitions() { ManifestFile multiPartManifestFile = new ManifestFile.Factory( fileIO, - new SchemaManager(fileIO, path), + new FileSystemSchemaManager(fileIO, path), multiPartitionType, avro, "zstd", @@ -1834,7 +1834,7 @@ private ManifestFile createManifestFileForPartitionType(RowType partitionType) { FileIO fileIO = FileIOFinder.find(path); return new ManifestFile.Factory( fileIO, - new SchemaManager(fileIO, path), + new FileSystemSchemaManager(fileIO, path), partitionType, avro, "zstd", diff --git a/paimon-core/src/test/java/org/apache/paimon/manifest/ManifestFileMetaTestBase.java b/paimon-core/src/test/java/org/apache/paimon/manifest/ManifestFileMetaTestBase.java index b336bd8dd505..06efb33fa33a 100644 --- a/paimon-core/src/test/java/org/apache/paimon/manifest/ManifestFileMetaTestBase.java +++ b/paimon-core/src/test/java/org/apache/paimon/manifest/ManifestFileMetaTestBase.java @@ -28,7 +28,7 @@ import org.apache.paimon.fs.Path; import org.apache.paimon.io.DataFileMeta; import org.apache.paimon.options.Options; -import org.apache.paimon.schema.SchemaManager; +import org.apache.paimon.schema.FileSystemSchemaManager; import org.apache.paimon.stats.StatsTestUtils; import org.apache.paimon.types.RowType; import org.apache.paimon.utils.FileStorePathFactory; @@ -144,7 +144,7 @@ protected ManifestFile createManifestFile(String pathStr, FileIO fileIO) { Path path = new Path(pathStr); return new ManifestFile.Factory( fileIO, - new SchemaManager(fileIO, path), + new FileSystemSchemaManager(fileIO, path), getPartitionType(), avro, "zstd", diff --git a/paimon-core/src/test/java/org/apache/paimon/manifest/ManifestFileTest.java b/paimon-core/src/test/java/org/apache/paimon/manifest/ManifestFileTest.java index 0846cb2697c3..d2e593ea79e7 100644 --- a/paimon-core/src/test/java/org/apache/paimon/manifest/ManifestFileTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/manifest/ManifestFileTest.java @@ -32,7 +32,7 @@ import org.apache.paimon.io.DataFileMeta; import org.apache.paimon.options.Options; import org.apache.paimon.partition.PartitionPredicate; -import org.apache.paimon.schema.SchemaManager; +import org.apache.paimon.schema.FileSystemSchemaManager; import org.apache.paimon.stats.StatsTestUtils; import org.apache.paimon.types.DataField; import org.apache.paimon.types.RowType; @@ -825,7 +825,7 @@ private ManifestFile createManifestFile(String pathStr, long suggestedFileSize) FileIO fileIO = FileIOFinder.find(path); return new ManifestFile.Factory( fileIO, - new SchemaManager(fileIO, path), + new FileSystemSchemaManager(fileIO, path), DEFAULT_PART_TYPE, avro, "zstd", diff --git a/paimon-core/src/test/java/org/apache/paimon/mergetree/ChangelogMergeTreeRewriterTest.java b/paimon-core/src/test/java/org/apache/paimon/mergetree/ChangelogMergeTreeRewriterTest.java index 02ea7309a7ff..7c926eeec94b 100644 --- a/paimon-core/src/test/java/org/apache/paimon/mergetree/ChangelogMergeTreeRewriterTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/mergetree/ChangelogMergeTreeRewriterTest.java @@ -39,6 +39,7 @@ import org.apache.paimon.mergetree.compact.MergeFunctionWrapper; import org.apache.paimon.options.Options; import org.apache.paimon.reader.RecordReader; +import org.apache.paimon.schema.FileSystemSchemaManager; import org.apache.paimon.schema.KeyValueFieldsExtractor; import org.apache.paimon.schema.Schema; import org.apache.paimon.schema.SchemaManager; @@ -88,7 +89,7 @@ public class ChangelogMergeTreeRewriterTest { public void beforeEach() throws Exception { path = new Path(tempDir.toString()); comparator = Comparator.comparingInt(o -> o.getInt(0)); - schemaManager = new SchemaManager(LocalFileIO.create(), path); + schemaManager = new FileSystemSchemaManager(LocalFileIO.create(), path); RowType recordType = DataTypes.ROW( DataTypes.FIELD(0, "key", DataTypes.INT()), diff --git a/paimon-core/src/test/java/org/apache/paimon/mergetree/compact/MergeTreeCompactManagerTest.java b/paimon-core/src/test/java/org/apache/paimon/mergetree/compact/MergeTreeCompactManagerTest.java index 05c041521b4d..5dd63465a302 100644 --- a/paimon-core/src/test/java/org/apache/paimon/mergetree/compact/MergeTreeCompactManagerTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/mergetree/compact/MergeTreeCompactManagerTest.java @@ -47,8 +47,8 @@ import org.apache.paimon.operation.SplitRead; import org.apache.paimon.reader.RecordReader; import org.apache.paimon.reader.RecordReaderIterator; +import org.apache.paimon.schema.FileSystemSchemaManager; import org.apache.paimon.schema.Schema; -import org.apache.paimon.schema.SchemaManager; import org.apache.paimon.schema.SchemaUtils; import org.apache.paimon.schema.TableSchema; import org.apache.paimon.table.source.DataSplit; @@ -273,7 +273,7 @@ public void testCompactWithKeepDelete() throws Exception { // 1) Build a minimal TestFileStore with primary key and one bucket. TableSchema tableSchema = SchemaUtils.forceCommit( - new SchemaManager(new LocalFileIO(), new Path(tempDir.toUri())), + new FileSystemSchemaManager(new LocalFileIO(), new Path(tempDir.toUri())), new Schema( TestKeyValueGenerator.DEFAULT_ROW_TYPE.getFields(), TestKeyValueGenerator.DEFAULT_PART_TYPE.getFieldNames(), diff --git a/paimon-core/src/test/java/org/apache/paimon/operation/ChainTablePartitionExpireTest.java b/paimon-core/src/test/java/org/apache/paimon/operation/ChainTablePartitionExpireTest.java index c8440a3506f7..c06b09066202 100644 --- a/paimon-core/src/test/java/org/apache/paimon/operation/ChainTablePartitionExpireTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/operation/ChainTablePartitionExpireTest.java @@ -30,6 +30,7 @@ import org.apache.paimon.options.Options; import org.apache.paimon.partition.PartitionStatistics; import org.apache.paimon.partition.PartitionUpdateTimeExpireStrategy; +import org.apache.paimon.schema.FileSystemSchemaManager; import org.apache.paimon.schema.Schema; import org.apache.paimon.schema.SchemaChange; import org.apache.paimon.schema.SchemaManager; @@ -642,7 +643,7 @@ private Path tablePath(String tableName) { private void createChainTable(Path tablePath, boolean withGroupPartition) throws Exception { LocalFileIO fileIO = LocalFileIO.create(); - SchemaManager schemaManager = new SchemaManager(fileIO, tablePath); + SchemaManager schemaManager = new FileSystemSchemaManager(fileIO, tablePath); Map options = new HashMap<>(); options.put(CoreOptions.BUCKET.key(), "1"); @@ -700,8 +701,8 @@ private void createChainTable(Path tablePath, boolean withGroupPartition) throws chainTableOptions.add(SchemaChange.setOption("chain-table.chain-partition-keys", "dt")); } schemaManager.commitChanges(chainTableOptions); - new SchemaManager(fileIO, tablePath, "snapshot").commitChanges(chainTableOptions); - new SchemaManager(fileIO, tablePath, "delta").commitChanges(chainTableOptions); + new FileSystemSchemaManager(fileIO, tablePath, "snapshot").commitChanges(chainTableOptions); + new FileSystemSchemaManager(fileIO, tablePath, "delta").commitChanges(chainTableOptions); } private FileStoreTable loadTable(Path tablePath) { @@ -709,7 +710,8 @@ private FileStoreTable loadTable(Path tablePath) { Options options = new Options(); options.set(CoreOptions.PATH, tablePath.toString()); String branchName = CoreOptions.branch(options.toMap()); - TableSchema tableSchema = new SchemaManager(fileIO, tablePath, branchName).latest().get(); + TableSchema tableSchema = + new FileSystemSchemaManager(fileIO, tablePath, branchName).latest().get(); return FileStoreTableFactory.create( fileIO, tablePath, tableSchema, CatalogEnvironment.empty()); } diff --git a/paimon-core/src/test/java/org/apache/paimon/operation/DataEvolutionSplitReadTest.java b/paimon-core/src/test/java/org/apache/paimon/operation/DataEvolutionSplitReadTest.java index ac865ba2ab93..cd60d981643a 100644 --- a/paimon-core/src/test/java/org/apache/paimon/operation/DataEvolutionSplitReadTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/operation/DataEvolutionSplitReadTest.java @@ -33,6 +33,7 @@ import org.apache.paimon.manifest.FileSource; import org.apache.paimon.options.Options; import org.apache.paimon.reader.RecordReader; +import org.apache.paimon.schema.FileSystemSchemaManager; import org.apache.paimon.schema.Schema; import org.apache.paimon.schema.SchemaManager; import org.apache.paimon.schema.TableSchema; @@ -285,7 +286,7 @@ public void testSparseRowIdReadUsesRowSidecar() throws Exception { .column("f0", DataTypes.INT()) .column("f1", DataTypes.STRING()) .build(); - SchemaManager schemaManager = new SchemaManager(fileIO, tableRoot); + SchemaManager schemaManager = new FileSystemSchemaManager(fileIO, tableRoot); TableSchema tableSchema = schemaManager.createTable(schema); RowType rowType = tableSchema.logicalRowType(); @@ -353,7 +354,7 @@ public void testSparseRowIdReadUsesParquetRowsAtSelectedPositions() throws Excep .option(CoreOptions.ROW_TRACKING_ENABLED.key(), "true") .option(CoreOptions.DATA_EVOLUTION_ENABLED.key(), "true") .build(); - SchemaManager schemaManager = new SchemaManager(fileIO, tableRoot); + SchemaManager schemaManager = new FileSystemSchemaManager(fileIO, tableRoot); TableSchema tableSchema = schemaManager.createTable(schema); RowType rowType = tableSchema.logicalRowType(); diff --git a/paimon-core/src/test/java/org/apache/paimon/operation/ExpireSnapshotsTest.java b/paimon-core/src/test/java/org/apache/paimon/operation/ExpireSnapshotsTest.java index c4de0f44e1a3..30b94ef617cc 100644 --- a/paimon-core/src/test/java/org/apache/paimon/operation/ExpireSnapshotsTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/operation/ExpireSnapshotsTest.java @@ -40,6 +40,7 @@ import org.apache.paimon.manifest.ManifestFileMeta; import org.apache.paimon.mergetree.compact.DeduplicateMergeFunction; import org.apache.paimon.options.ExpireConfig; +import org.apache.paimon.schema.FileSystemSchemaManager; import org.apache.paimon.schema.Schema; import org.apache.paimon.schema.SchemaManager; import org.apache.paimon.stats.SimpleStats; @@ -102,7 +103,8 @@ public void beforeEach() throws Exception { store = createStore(); snapshotManager = store.snapshotManager(); changelogManager = store.changelogManager(); - SchemaManager schemaManager = new SchemaManager(fileIO, new Path(tempDir.toUri())); + SchemaManager schemaManager = + new FileSystemSchemaManager(fileIO, new Path(tempDir.toUri())); schemaManager.createTable( new Schema( TestKeyValueGenerator.DEFAULT_ROW_TYPE.getFields(), diff --git a/paimon-core/src/test/java/org/apache/paimon/operation/FileDeletionTest.java b/paimon-core/src/test/java/org/apache/paimon/operation/FileDeletionTest.java index ee4eccd1d545..8102602d4707 100644 --- a/paimon-core/src/test/java/org/apache/paimon/operation/FileDeletionTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/operation/FileDeletionTest.java @@ -37,6 +37,7 @@ import org.apache.paimon.manifest.ManifestList; import org.apache.paimon.mergetree.compact.DeduplicateMergeFunction; import org.apache.paimon.options.ExpireConfig; +import org.apache.paimon.schema.FileSystemSchemaManager; import org.apache.paimon.schema.Schema; import org.apache.paimon.schema.SchemaManager; import org.apache.paimon.schema.TableSchema; @@ -863,7 +864,7 @@ private TestFileStore createStore(TestKeyValueGenerator.GeneratorMode mode, int throw new UnsupportedOperationException("Unsupported generator mode: " + mode); } - SchemaManager schemaManager = new SchemaManager(fileIO, new Path(root)); + SchemaManager schemaManager = new FileSystemSchemaManager(fileIO, new Path(root)); TableSchema tableSchema = schemaManager.createTable( diff --git a/paimon-core/src/test/java/org/apache/paimon/operation/FileStoreCommitTest.java b/paimon-core/src/test/java/org/apache/paimon/operation/FileStoreCommitTest.java index f326be18dcab..0dfaf9f1d64d 100644 --- a/paimon-core/src/test/java/org/apache/paimon/operation/FileStoreCommitTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/operation/FileStoreCommitTest.java @@ -51,8 +51,8 @@ import org.apache.paimon.operation.commit.ManifestEntryChanges; import org.apache.paimon.operation.commit.RetryCommitResult; import org.apache.paimon.predicate.PredicateBuilder; +import org.apache.paimon.schema.FileSystemSchemaManager; import org.apache.paimon.schema.Schema; -import org.apache.paimon.schema.SchemaManager; import org.apache.paimon.schema.SchemaUtils; import org.apache.paimon.schema.TableSchema; import org.apache.paimon.stats.ColStats; @@ -1982,7 +1982,7 @@ private FileStoreCommitImpl newCommitWithSnapshotCommit( return new FileStoreCommitImpl( snapshotCommit, store.fileIO(), - new SchemaManager(store.fileIO(), store.options().path()), + new FileSystemSchemaManager(store.fileIO(), store.options().path()), tableName, commitUser, store.partitionType(), @@ -2313,7 +2313,7 @@ private TestFileStore createStore( TestKeyValueGenerator.GeneratorMode.MULTI_PARTITIONED); TableSchema tableSchema = SchemaUtils.forceCommit( - new SchemaManager(new LocalFileIO(), path), + new FileSystemSchemaManager(new LocalFileIO(), path), new Schema( TestKeyValueGenerator.DEFAULT_ROW_TYPE.getFields(), TestKeyValueGenerator.DEFAULT_PART_TYPE.getFieldNames(), diff --git a/paimon-core/src/test/java/org/apache/paimon/operation/KeyValueFileStoreScanTest.java b/paimon-core/src/test/java/org/apache/paimon/operation/KeyValueFileStoreScanTest.java index ac6825ed8cbe..e0fa947860d5 100644 --- a/paimon-core/src/test/java/org/apache/paimon/operation/KeyValueFileStoreScanTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/operation/KeyValueFileStoreScanTest.java @@ -30,6 +30,7 @@ import org.apache.paimon.mergetree.compact.DeduplicateMergeFunction; import org.apache.paimon.predicate.Predicate; import org.apache.paimon.predicate.PredicateBuilder; +import org.apache.paimon.schema.FileSystemSchemaManager; import org.apache.paimon.schema.Schema; import org.apache.paimon.schema.SchemaManager; import org.apache.paimon.schema.SchemaUtils; @@ -80,7 +81,7 @@ public void beforeEach() throws Exception { .build(); SchemaManager schemaManager = - new SchemaManager(LocalFileIO.create(), new Path(tempDir.toUri())); + new FileSystemSchemaManager(LocalFileIO.create(), new Path(tempDir.toUri())); schemaManager.createTable( new Schema( TestKeyValueGenerator.DEFAULT_ROW_TYPE.getFields(), @@ -423,7 +424,7 @@ public void testLimitPushdownWithPartialUpdateMergeEngine() throws Exception { // Test that limit pushdown is disabled for PARTIAL_UPDATE merge engine // Create a store with PARTIAL_UPDATE merge engine by setting it in schema options SchemaManager schemaManager = - new SchemaManager(LocalFileIO.create(), new Path(tempDir.toUri())); + new FileSystemSchemaManager(LocalFileIO.create(), new Path(tempDir.toUri())); Schema schema = new Schema( TestKeyValueGenerator.DEFAULT_ROW_TYPE.getFields(), @@ -471,7 +472,7 @@ public void testLimitPushdownWithPartialUpdateMergeEngine() throws Exception { public void testLimitPushdownWithAggregateMergeEngine() throws Exception { // Test that limit pushdown is disabled for AGGREGATE merge engine SchemaManager schemaManager = - new SchemaManager(LocalFileIO.create(), new Path(tempDir.toUri())); + new FileSystemSchemaManager(LocalFileIO.create(), new Path(tempDir.toUri())); Schema schema = new Schema( TestKeyValueGenerator.DEFAULT_ROW_TYPE.getFields(), @@ -519,7 +520,7 @@ public void testLimitPushdownWithAggregateMergeEngine() throws Exception { public void testLimitPushdownWithDeletionVectors() throws Exception { // Test that limit pushdown is disabled when deletion vectors are enabled SchemaManager schemaManager = - new SchemaManager(LocalFileIO.create(), new Path(tempDir.toUri())); + new FileSystemSchemaManager(LocalFileIO.create(), new Path(tempDir.toUri())); Schema schema = new Schema( TestKeyValueGenerator.DEFAULT_ROW_TYPE.getFields(), diff --git a/paimon-core/src/test/java/org/apache/paimon/operation/KeyValueFileStoreWriteTest.java b/paimon-core/src/test/java/org/apache/paimon/operation/KeyValueFileStoreWriteTest.java index 81beed117e34..e121264eee97 100644 --- a/paimon-core/src/test/java/org/apache/paimon/operation/KeyValueFileStoreWriteTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/operation/KeyValueFileStoreWriteTest.java @@ -32,6 +32,7 @@ import org.apache.paimon.mergetree.compact.DeduplicateMergeFunction; import org.apache.paimon.mergetree.compact.ForceUpLevel0Compaction; import org.apache.paimon.mergetree.compact.MergeTreeCompactManager; +import org.apache.paimon.schema.FileSystemSchemaManager; import org.apache.paimon.schema.Schema; import org.apache.paimon.schema.SchemaChange; import org.apache.paimon.schema.SchemaManager; @@ -143,7 +144,7 @@ public void testSnapshotSequenceNumberInitBootstrapsGlobalMaxSequenceNumber() th .isEmpty(); SchemaManager schemaManager = - new SchemaManager(LocalFileIO.create(), new Path(tempDir.toUri())); + new FileSystemSchemaManager(LocalFileIO.create(), new Path(tempDir.toUri())); TableSchema snapshotSchema = schemaManager.commitChanges( SchemaChange.setOption( @@ -184,7 +185,7 @@ public void testSnapshotSequenceNumberInitUsesWriterCreationSnapshot() throws Ex .containsEntry(SequenceSnapshotProperties.MAX_SEQUENCE_NUMBER, "0"); SchemaManager schemaManager = - new SchemaManager(LocalFileIO.create(), new Path(tempDir.toUri())); + new FileSystemSchemaManager(LocalFileIO.create(), new Path(tempDir.toUri())); TableSchema scanSchema = schemaManager.commitChanges( SchemaChange.setOption( @@ -227,7 +228,7 @@ private KeyValueFileStoreWrite createWriteWithOptions(Map option private TestFileStore createStoreWithOptions(Map options) throws Exception { SchemaManager schemaManager = - new SchemaManager(LocalFileIO.create(), new Path(tempDir.toUri())); + new FileSystemSchemaManager(LocalFileIO.create(), new Path(tempDir.toUri())); TableSchema schema = schemaManager.createTable( diff --git a/paimon-core/src/test/java/org/apache/paimon/operation/LocalOrphanFilesCleanTest.java b/paimon-core/src/test/java/org/apache/paimon/operation/LocalOrphanFilesCleanTest.java index cd6480d7e74c..bfe3f6d76ca8 100644 --- a/paimon-core/src/test/java/org/apache/paimon/operation/LocalOrphanFilesCleanTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/operation/LocalOrphanFilesCleanTest.java @@ -37,8 +37,8 @@ import org.apache.paimon.reader.ReaderSupplier; import org.apache.paimon.reader.RecordReader; import org.apache.paimon.reader.RecordReaderIterator; +import org.apache.paimon.schema.FileSystemSchemaManager; import org.apache.paimon.schema.Schema; -import org.apache.paimon.schema.SchemaManager; import org.apache.paimon.schema.SchemaUtils; import org.apache.paimon.schema.TableSchema; import org.apache.paimon.table.FileStoreTable; @@ -994,7 +994,7 @@ private FileStoreTable createFileStoreTable(RowType rowType, Options conf) throw conf.set(CoreOptions.BUCKET, RANDOM.nextInt(3) + 1); TableSchema tableSchema = SchemaUtils.forceCommit( - new SchemaManager(fileIO, tablePath), + new FileSystemSchemaManager(fileIO, tablePath), new Schema( rowType.getFields(), Arrays.asList("part1", "part2"), diff --git a/paimon-core/src/test/java/org/apache/paimon/operation/MergeFileSplitReadTest.java b/paimon-core/src/test/java/org/apache/paimon/operation/MergeFileSplitReadTest.java index d573b5bd905a..b2901e9f1440 100644 --- a/paimon-core/src/test/java/org/apache/paimon/operation/MergeFileSplitReadTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/operation/MergeFileSplitReadTest.java @@ -36,6 +36,7 @@ import org.apache.paimon.mergetree.compact.MergeFunctionFactory; import org.apache.paimon.reader.RecordReader; import org.apache.paimon.reader.RecordReaderIterator; +import org.apache.paimon.schema.FileSystemSchemaManager; import org.apache.paimon.schema.KeyValueFieldsExtractor; import org.apache.paimon.schema.Schema; import org.apache.paimon.schema.SchemaManager; @@ -458,7 +459,7 @@ private TestFileStore createStore( MergeFunctionFactory mfFactory) throws Exception { Path path = new Path(tempDir.toUri()); - SchemaManager schemaManager = new SchemaManager(FileIOFinder.find(path), path); + SchemaManager schemaManager = new FileSystemSchemaManager(FileIOFinder.find(path), path); boolean valueCountMode = mfFactory.create() instanceof TestValueCountMergeFunction; Schema schema = new Schema( diff --git a/paimon-core/src/test/java/org/apache/paimon/operation/OrphanFilesCleanTest.java b/paimon-core/src/test/java/org/apache/paimon/operation/OrphanFilesCleanTest.java index beead97cc746..75648f519bc3 100644 --- a/paimon-core/src/test/java/org/apache/paimon/operation/OrphanFilesCleanTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/operation/OrphanFilesCleanTest.java @@ -25,8 +25,8 @@ import org.apache.paimon.fs.Path; import org.apache.paimon.fs.local.LocalFileIO; import org.apache.paimon.options.Options; +import org.apache.paimon.schema.FileSystemSchemaManager; import org.apache.paimon.schema.Schema; -import org.apache.paimon.schema.SchemaManager; import org.apache.paimon.schema.SchemaUtils; import org.apache.paimon.schema.TableSchema; import org.apache.paimon.table.FileStoreTable; @@ -134,7 +134,7 @@ private FileStoreTable createFileStoreTable( conf.set(CoreOptions.BUCKET, 2); TableSchema tableSchema = SchemaUtils.forceCommit( - new SchemaManager(fileIO, tablePath), + new FileSystemSchemaManager(fileIO, tablePath), new Schema( rowType.getFields(), Arrays.asList("part1", "part2"), diff --git a/paimon-core/src/test/java/org/apache/paimon/operation/PartitionExpireTest.java b/paimon-core/src/test/java/org/apache/paimon/operation/PartitionExpireTest.java index e6abe773ebfa..1b9b375c553b 100644 --- a/paimon-core/src/test/java/org/apache/paimon/operation/PartitionExpireTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/operation/PartitionExpireTest.java @@ -31,6 +31,7 @@ import org.apache.paimon.options.Options; import org.apache.paimon.partition.PartitionStatistics; import org.apache.paimon.partition.actions.AddDonePartitionAction; +import org.apache.paimon.schema.FileSystemSchemaManager; import org.apache.paimon.schema.Schema; import org.apache.paimon.schema.SchemaManager; import org.apache.paimon.schema.TableSchema; @@ -107,7 +108,8 @@ private void newTable() { options.set(PATH, path.toString()); Path tablePath = CoreOptions.path(options); String branchName = CoreOptions.branch(options.toMap()); - TableSchema tableSchema = new SchemaManager(fileIO, tablePath, branchName).latest().get(); + TableSchema tableSchema = + new FileSystemSchemaManager(fileIO, tablePath, branchName).latest().get(); createdPartitions = new HashSet<>(); deletedPartitions = new ArrayList<>(); PartitionModification partitionModification = @@ -158,7 +160,7 @@ public PartitionModification partitionModification() { @Test public void testNonPartitionedTable() { - SchemaManager schemaManager = new SchemaManager(LocalFileIO.create(), path); + SchemaManager schemaManager = new FileSystemSchemaManager(LocalFileIO.create(), path); assertThatCode( () -> schemaManager.createTable( @@ -174,7 +176,7 @@ public void testNonPartitionedTable() { @Test public void testIllegalPartition() throws Exception { - SchemaManager schemaManager = new SchemaManager(LocalFileIO.create(), path); + SchemaManager schemaManager = new FileSystemSchemaManager(LocalFileIO.create(), path); schemaManager.createTable( new Schema( RowType.of(VarCharType.STRING_TYPE, VarCharType.STRING_TYPE).getFields(), @@ -197,7 +199,7 @@ public void testIllegalPartition() throws Exception { @Test public void testBatchExpire() throws Exception { - SchemaManager schemaManager = new SchemaManager(LocalFileIO.create(), path); + SchemaManager schemaManager = new FileSystemSchemaManager(LocalFileIO.create(), path); schemaManager.createTable( new Schema( RowType.of(VarCharType.STRING_TYPE, VarCharType.STRING_TYPE).getFields(), @@ -228,7 +230,7 @@ public void testBatchExpire() throws Exception { @Test public void testExpireWithNullOrEmptyPartition() throws Exception { - SchemaManager schemaManager = new SchemaManager(LocalFileIO.create(), path); + SchemaManager schemaManager = new FileSystemSchemaManager(LocalFileIO.create(), path); schemaManager.createTable( new Schema( RowType.of(VarCharType.STRING_TYPE, VarCharType.STRING_TYPE).getFields(), @@ -256,7 +258,7 @@ public void testExpireWithNullOrEmptyPartition() throws Exception { @Test public void test() throws Exception { - SchemaManager schemaManager = new SchemaManager(LocalFileIO.create(), path); + SchemaManager schemaManager = new FileSystemSchemaManager(LocalFileIO.create(), path); schemaManager.createTable( new Schema( RowType.of(VarCharType.STRING_TYPE, VarCharType.STRING_TYPE).getFields(), @@ -299,7 +301,7 @@ public void test() throws Exception { @Test public void testDonePartitionExpire() throws Exception { - SchemaManager schemaManager = new SchemaManager(LocalFileIO.create(), path); + SchemaManager schemaManager = new FileSystemSchemaManager(LocalFileIO.create(), path); schemaManager.createTable( new Schema( RowType.of(VarCharType.STRING_TYPE, VarCharType.STRING_TYPE).getFields(), @@ -334,7 +336,7 @@ public void testDonePartitionExpire() throws Exception { @Test public void testFilterCommittedAfterExpiring() throws Exception { - SchemaManager schemaManager = new SchemaManager(LocalFileIO.create(), path); + SchemaManager schemaManager = new FileSystemSchemaManager(LocalFileIO.create(), path); schemaManager.createTable( new Schema( RowType.of(VarCharType.STRING_TYPE, VarCharType.STRING_TYPE).getFields(), @@ -409,7 +411,7 @@ public void testFilterCommittedAfterExpiring() throws Exception { @Test public void testDeleteExpiredPartition() throws Exception { - SchemaManager schemaManager = new SchemaManager(LocalFileIO.create(), path); + SchemaManager schemaManager = new FileSystemSchemaManager(LocalFileIO.create(), path); schemaManager.createTable( new Schema( RowType.of(VarCharType.STRING_TYPE, VarCharType.STRING_TYPE).getFields(), diff --git a/paimon-core/src/test/java/org/apache/paimon/operation/PrimaryKeyIndexWriteTest.java b/paimon-core/src/test/java/org/apache/paimon/operation/PrimaryKeyIndexWriteTest.java index 4d2110931026..8c975ca14cb5 100644 --- a/paimon-core/src/test/java/org/apache/paimon/operation/PrimaryKeyIndexWriteTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/operation/PrimaryKeyIndexWriteTest.java @@ -30,6 +30,7 @@ import org.apache.paimon.fs.local.LocalFileIO; import org.apache.paimon.mergetree.compact.DeduplicateMergeFunction; import org.apache.paimon.postpone.PostponeBucketFileStoreWrite; +import org.apache.paimon.schema.FileSystemSchemaManager; import org.apache.paimon.schema.Schema; import org.apache.paimon.schema.SchemaManager; import org.apache.paimon.schema.TableSchema; @@ -198,7 +199,7 @@ private TestFileStore createStore(Map options, RowType rowType) private TableSchema createSchema(Map options, RowType rowType) throws Exception { SchemaManager schemaManager = - new SchemaManager(LocalFileIO.create(), new Path(tempDir.toUri())); + new FileSystemSchemaManager(LocalFileIO.create(), new Path(tempDir.toUri())); return schemaManager.createTable( new Schema( rowType.getFields(), diff --git a/paimon-core/src/test/java/org/apache/paimon/operation/PrimaryKeyManagedBlobStoreTest.java b/paimon-core/src/test/java/org/apache/paimon/operation/PrimaryKeyManagedBlobStoreTest.java index 01a0455e0fbb..f6098f5f325e 100644 --- a/paimon-core/src/test/java/org/apache/paimon/operation/PrimaryKeyManagedBlobStoreTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/operation/PrimaryKeyManagedBlobStoreTest.java @@ -40,9 +40,9 @@ import org.apache.paimon.manifest.ManifestEntry; import org.apache.paimon.mergetree.compact.DeduplicateMergeFunction; import org.apache.paimon.postpone.PostponeBucketWriter; +import org.apache.paimon.schema.FileSystemSchemaManager; import org.apache.paimon.schema.KeyValueFieldsExtractor; import org.apache.paimon.schema.Schema; -import org.apache.paimon.schema.SchemaManager; import org.apache.paimon.schema.TableSchema; import org.apache.paimon.table.BucketMode; import org.apache.paimon.table.SpecialFields; @@ -505,7 +505,7 @@ private TestFileStore createStore( options.put(CoreOptions.BLOB_FIELD.key(), payloadName); options.put(CoreOptions.BLOB_TARGET_FILE_SIZE.key(), "1 b"); TableSchema schema = - new SchemaManager(fileIO, tablePath) + new FileSystemSchemaManager(fileIO, tablePath) .createTable( new Schema( valueFields, diff --git a/paimon-core/src/test/java/org/apache/paimon/operation/RawFileSplitReadTest.java b/paimon-core/src/test/java/org/apache/paimon/operation/RawFileSplitReadTest.java index fd4fcb2d4853..774f2f1155ac 100644 --- a/paimon-core/src/test/java/org/apache/paimon/operation/RawFileSplitReadTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/operation/RawFileSplitReadTest.java @@ -30,8 +30,8 @@ import org.apache.paimon.options.Options; import org.apache.paimon.predicate.Predicate; import org.apache.paimon.reader.RecordReader; +import org.apache.paimon.schema.FileSystemSchemaManager; import org.apache.paimon.schema.Schema; -import org.apache.paimon.schema.SchemaManager; import org.apache.paimon.schema.SchemaUtils; import org.apache.paimon.schema.TableSchema; import org.apache.paimon.table.FileStoreTable; @@ -146,7 +146,8 @@ private FileStoreTable createTable(String directory) throws Exception { .options(options.toMap()) .build(); TableSchema tableSchema = - SchemaUtils.forceCommit(new SchemaManager(LocalFileIO.create(), tablePath), schema); + SchemaUtils.forceCommit( + new FileSystemSchemaManager(LocalFileIO.create(), tablePath), schema); FileStoreTable table = FileStoreTableFactory.create(LocalFileIO.create(), tablePath, tableSchema); diff --git a/paimon-core/src/test/java/org/apache/paimon/operation/commit/OverwriteChangesProviderTest.java b/paimon-core/src/test/java/org/apache/paimon/operation/commit/OverwriteChangesProviderTest.java index 7aa8a16023f1..938dbf0dddb6 100644 --- a/paimon-core/src/test/java/org/apache/paimon/operation/commit/OverwriteChangesProviderTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/operation/commit/OverwriteChangesProviderTest.java @@ -33,8 +33,8 @@ import org.apache.paimon.mergetree.compact.DeduplicateMergeFunction; import org.apache.paimon.operation.FileStoreCommitImpl; import org.apache.paimon.partition.PartitionPredicate; +import org.apache.paimon.schema.FileSystemSchemaManager; import org.apache.paimon.schema.Schema; -import org.apache.paimon.schema.SchemaManager; import org.apache.paimon.schema.SchemaUtils; import org.apache.paimon.schema.TableSchema; import org.apache.paimon.types.RowKind; @@ -299,7 +299,7 @@ private TestFileStore createStore(int numBucket, Map options) th Path path = new Path(tempDir.toUri()); TableSchema tableSchema = SchemaUtils.forceCommit( - new SchemaManager(new LocalFileIO(), path), + new FileSystemSchemaManager(new LocalFileIO(), path), new Schema( TestKeyValueGenerator.DEFAULT_ROW_TYPE.getFields(), TestKeyValueGenerator.DEFAULT_PART_TYPE.getFieldNames(), diff --git a/paimon-core/src/test/java/org/apache/paimon/rest/RESTCatalogServer.java b/paimon-core/src/test/java/org/apache/paimon/rest/RESTCatalogServer.java index c571b19c3891..419cf7e732c3 100644 --- a/paimon-core/src/test/java/org/apache/paimon/rest/RESTCatalogServer.java +++ b/paimon-core/src/test/java/org/apache/paimon/rest/RESTCatalogServer.java @@ -101,6 +101,7 @@ import org.apache.paimon.rest.responses.ListViewDetailsResponse; import org.apache.paimon.rest.responses.ListViewsGloballyResponse; import org.apache.paimon.rest.responses.ListViewsResponse; +import org.apache.paimon.schema.FileSystemSchemaManager; import org.apache.paimon.schema.Schema; import org.apache.paimon.schema.SchemaChange; import org.apache.paimon.schema.SchemaManager; @@ -1122,7 +1123,7 @@ private MockResponse rollbackSchemaHandle(Identifier identifier, String data) th } FileStoreTable table = getFileTable(identifier); long schemaId = requestBody.getSchemaId(); - SchemaManager schemaManager = new SchemaManager(table.fileIO(), table.location()); + SchemaManager schemaManager = new FileSystemSchemaManager(table.fileIO(), table.location()); try { schemaManager.rollbackTo( schemaId, diff --git a/paimon-core/src/test/java/org/apache/paimon/rest/RESTCatalogTest.java b/paimon-core/src/test/java/org/apache/paimon/rest/RESTCatalogTest.java index 8fc5e38a25e3..3870509a4a47 100644 --- a/paimon-core/src/test/java/org/apache/paimon/rest/RESTCatalogTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/rest/RESTCatalogTest.java @@ -72,6 +72,7 @@ import org.apache.paimon.rest.responses.CreatePartitionsResponse; import org.apache.paimon.rest.responses.DropPartitionsResponse; import org.apache.paimon.rest.responses.GetTagResponse; +import org.apache.paimon.schema.FileSystemSchemaManager; import org.apache.paimon.schema.Schema; import org.apache.paimon.schema.SchemaChange; import org.apache.paimon.schema.SchemaManager; @@ -2365,7 +2366,7 @@ public void testRollbackSchema() throws Exception { // get initial schema id FileStoreTable table = (FileStoreTable) catalog.getTable(identifier); - SchemaManager schemaManager = new SchemaManager(table.fileIO(), table.location()); + SchemaManager schemaManager = new FileSystemSchemaManager(table.fileIO(), table.location()); long firstSchemaId = schemaManager.latest().get().id(); // evolve schema @@ -2391,7 +2392,7 @@ public void testRollbackSchemaFromTable() throws Exception { // get initial schema id FileStoreTable table = (FileStoreTable) catalog.getTable(identifier); - SchemaManager schemaManager = new SchemaManager(table.fileIO(), table.location()); + SchemaManager schemaManager = new FileSystemSchemaManager(table.fileIO(), table.location()); long firstSchemaId = schemaManager.latest().get().id(); // evolve schema @@ -2416,7 +2417,7 @@ public void testRollbackSchemaFailedWithSnapshotReference() throws Exception { createTable(identifier, Maps.newHashMap(), Lists.newArrayList("col1")); FileStoreTable table = (FileStoreTable) catalog.getTable(identifier); - SchemaManager schemaManager = new SchemaManager(table.fileIO(), table.location()); + SchemaManager schemaManager = new FileSystemSchemaManager(table.fileIO(), table.location()); long firstSchemaId = schemaManager.latest().get().id(); // write data to create a snapshot referencing firstSchemaId @@ -4641,7 +4642,7 @@ private void createExternalTableDirectory(Path externalTablePath, Schema schema) } // Create schema file in the external table directory - SchemaManager schemaManager = new SchemaManager(fileIO, externalTablePath); + SchemaManager schemaManager = new FileSystemSchemaManager(fileIO, externalTablePath); schemaManager.createTable(schema, true); // true indicates external table } } diff --git a/paimon-core/src/test/java/org/apache/paimon/rest/RESTSimpleTableTest.java b/paimon-core/src/test/java/org/apache/paimon/rest/RESTSimpleTableTest.java index 93c8437955c5..1c52a0f7f215 100644 --- a/paimon-core/src/test/java/org/apache/paimon/rest/RESTSimpleTableTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/rest/RESTSimpleTableTest.java @@ -21,9 +21,9 @@ import org.apache.paimon.catalog.Identifier; import org.apache.paimon.fs.local.LocalFileIO; import org.apache.paimon.options.Options; +import org.apache.paimon.schema.FileSystemSchemaManager; import org.apache.paimon.schema.Schema; import org.apache.paimon.schema.SchemaChange; -import org.apache.paimon.schema.SchemaManager; import org.apache.paimon.schema.SchemaUtils; import org.apache.paimon.schema.TableSchema; import org.apache.paimon.table.FileStoreTable; @@ -75,7 +75,7 @@ protected FileStoreTable createFileStoreTable(Consumer configure, RowTy } TableSchema tableSchema = SchemaUtils.forceCommit( - new SchemaManager(LocalFileIO.create(), tablePath), + new FileSystemSchemaManager(LocalFileIO.create(), tablePath), new Schema( rowType.getFields(), Collections.singletonList("pt"), diff --git a/paimon-core/src/test/java/org/apache/paimon/schema/SchemaManagerTest.java b/paimon-core/src/test/java/org/apache/paimon/schema/SchemaManagerTest.java index c080d0eef1e3..d51637122750 100644 --- a/paimon-core/src/test/java/org/apache/paimon/schema/SchemaManagerTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/schema/SchemaManagerTest.java @@ -106,7 +106,7 @@ public void beforeEach() throws IOException { FailingFileIO.reset(failingName, 100, 100); String root = FailingFileIO.getFailingPath(failingName, tempDir.toString()); path = new Path(root); - manager = new SchemaManager(FileIOFinder.find(path), path); + manager = new FileSystemSchemaManager(FileIOFinder.find(path), path); } @AfterEach @@ -291,7 +291,7 @@ public void testRejectRenamePrimaryKeyVectorIndexColumn() throws Exception { Collections.singletonList("id"), options, ""); - SchemaManager manager = new SchemaManager(LocalFileIO.create(), path); + SchemaManager manager = new FileSystemSchemaManager(LocalFileIO.create(), path); manager.createTable(schema); assertThatThrownBy( @@ -318,7 +318,7 @@ public void testRejectRenamePrimaryKeyBTreeIndexColumn() throws Exception { Collections.singletonList("id"), options, ""); - SchemaManager manager = new SchemaManager(LocalFileIO.create(), path); + SchemaManager manager = new FileSystemSchemaManager(LocalFileIO.create(), path); manager.createTable(schema); assertThatThrownBy( @@ -345,7 +345,7 @@ public void testRejectDestructivePrimaryKeyFullTextIndexColumnChanges() throws E Collections.singletonList("id"), options, ""); - SchemaManager manager = new SchemaManager(LocalFileIO.create(), path); + SchemaManager manager = new FileSystemSchemaManager(LocalFileIO.create(), path); manager.createTable(schema); assertThatThrownBy( @@ -381,7 +381,7 @@ public void testRejectDropPrimaryKeyBitmapIndexColumn() throws Exception { Collections.singletonList("id"), options, ""); - SchemaManager manager = new SchemaManager(LocalFileIO.create(), path); + SchemaManager manager = new FileSystemSchemaManager(LocalFileIO.create(), path); manager.createTable(schema); assertThatThrownBy(() -> manager.commitChanges(SchemaChange.dropColumn("status"))) @@ -404,7 +404,7 @@ public void testRejectTypeChangeOfPrimaryKeyBitmapIndexColumn() throws Exception Collections.singletonList("id"), options, ""); - SchemaManager manager = new SchemaManager(LocalFileIO.create(), path); + SchemaManager manager = new FileSystemSchemaManager(LocalFileIO.create(), path); manager.createTable(schema); assertThatThrownBy( @@ -640,7 +640,7 @@ public void testDeleteSchemaWithSchemaId() throws Exception { options, "append-only table with primary key"); // use non-failing manager - SchemaManager manager = new SchemaManager(LocalFileIO.create(), path); + SchemaManager manager = new FileSystemSchemaManager(LocalFileIO.create(), path); manager.createTable(schema); String schemaContent = manager.latest().get().toString(); @@ -660,7 +660,7 @@ void testApplyMoveOperations( List expectedOrder) { List fields = new LinkedList<>(initialFields); - manager.applyMove(fields, moveOperation); + SchemaManager.applyMove(fields, moveOperation); for (int i = 0; i < expectedOrder.size(); i++) { assertEquals( @@ -722,7 +722,7 @@ public void testAlterImmutableOptionsOnEmptyTable() throws Exception { options, ""); Path tableRoot = new Path(tempDir.toString(), "table"); - SchemaManager manager = new SchemaManager(LocalFileIO.create(), tableRoot); + SchemaManager manager = new FileSystemSchemaManager(LocalFileIO.create(), tableRoot); manager.createTable(schema); // 'type' is rejected even without snapshots (format tables hold data but create none) @@ -825,7 +825,7 @@ public void testCopyWithPrimaryKeyInOptions() throws Exception { tableOptions, ""); Path tableRoot = new Path(tempDir.toString(), "table"); - SchemaManager manager = new SchemaManager(LocalFileIO.create(), tableRoot); + SchemaManager manager = new FileSystemSchemaManager(LocalFileIO.create(), tableRoot); manager.createTable(schema); FileStoreTable table = FileStoreTableFactory.create(LocalFileIO.create(), tableRoot); @@ -905,7 +905,7 @@ public void testAlterUnchangedNormalizedOptionsOnNonEmptyTable() throws Exceptio tableOptions, ""); Path tableRoot = new Path(tempDir.toString(), "table"); - SchemaManager manager = new SchemaManager(LocalFileIO.create(), tableRoot); + SchemaManager manager = new FileSystemSchemaManager(LocalFileIO.create(), tableRoot); manager.createTable(schema); FileStoreTable table = FileStoreTableFactory.create(LocalFileIO.create(), tableRoot); @@ -931,7 +931,7 @@ public void testAlterUnchangedNormalizedOptionsOnNonEmptyTable() throws Exceptio @Test public void testDropPrimaryKeyOnEmptyTable() throws Exception { Path tableRoot = new Path(tempDir.toString(), "table"); - SchemaManager manager = new SchemaManager(LocalFileIO.create(), tableRoot); + SchemaManager manager = new FileSystemSchemaManager(LocalFileIO.create(), tableRoot); manager.createTable(schema); // drop primary keys on empty table should succeed @@ -953,7 +953,7 @@ public void testDropPrimaryKeyOnNonEmptyTable() throws Exception { tableOptions, ""); Path tableRoot = new Path(tempDir.toString(), "table"); - SchemaManager manager = new SchemaManager(LocalFileIO.create(), tableRoot); + SchemaManager manager = new FileSystemSchemaManager(LocalFileIO.create(), tableRoot); manager.createTable(pkSchema); // write data to create a snapshot @@ -994,7 +994,7 @@ public void testAddAndDropNestedColumns() throws Exception { Collections.emptyList(), new HashMap<>(), ""); - SchemaManager manager = new SchemaManager(LocalFileIO.create(), path); + SchemaManager manager = new FileSystemSchemaManager(LocalFileIO.create(), path); manager.createTable(schema); SchemaChange addColumn = @@ -1135,7 +1135,7 @@ public void testRenameNestedColumns() throws Exception { Collections.emptyList(), new HashMap<>(), ""); - SchemaManager manager = new SchemaManager(LocalFileIO.create(), path); + SchemaManager manager = new FileSystemSchemaManager(LocalFileIO.create(), path); manager.createTable(schema); SchemaChange renameColumn = @@ -1192,7 +1192,7 @@ public void testUpdateNestedColumnType() throws Exception { Collections.emptyList(), new HashMap<>(), ""); - SchemaManager manager = new SchemaManager(LocalFileIO.create(), path); + SchemaManager manager = new FileSystemSchemaManager(LocalFileIO.create(), path); manager.createTable(schema); SchemaChange updateColumnType = @@ -1239,7 +1239,7 @@ public void testUpdateRowTypeInArrayAndMap() throws Exception { Collections.emptyList(), new HashMap<>(), ""); - SchemaManager manager = new SchemaManager(LocalFileIO.create(), path); + SchemaManager manager = new FileSystemSchemaManager(LocalFileIO.create(), path); manager.createTable(schema); SchemaChange addColumn = @@ -1278,7 +1278,7 @@ public void testAlterDeletionVectorsMode() throws Exception { options, ""); Path tableRoot = new Path(tempDir.toString(), "table"); - SchemaManager manager = new SchemaManager(LocalFileIO.create(), tableRoot); + SchemaManager manager = new FileSystemSchemaManager(LocalFileIO.create(), tableRoot); manager.createTable(schema); // write table @@ -1310,7 +1310,7 @@ public void testAlterDeletionVectorsMode() throws Exception { @Test public void testRollbackSchemaSuccess() throws Exception { - SchemaManager manager = new SchemaManager(LocalFileIO.create(), path); + SchemaManager manager = new FileSystemSchemaManager(LocalFileIO.create(), path); manager.createTable(schema); long firstSchemaId = manager.latest().get().id(); @@ -1344,7 +1344,7 @@ public void testRollbackSchemaFailedDueToSnapshotReference() throws Exception { options, ""); Path tableRoot = new Path(tempDir.toString(), "table"); - SchemaManager manager = new SchemaManager(LocalFileIO.create(), tableRoot); + SchemaManager manager = new FileSystemSchemaManager(LocalFileIO.create(), tableRoot); manager.createTable(appendOnlySchema); long firstSchemaId = manager.latest().get().id(); @@ -1394,7 +1394,7 @@ public void testRollbackSchemaFailedDueToSnapshotReference() throws Exception { @Test public void testRollbackSchemaNotExist() throws Exception { - SchemaManager manager = new SchemaManager(LocalFileIO.create(), path); + SchemaManager manager = new FileSystemSchemaManager(LocalFileIO.create(), path); manager.createTable(schema); assertThatThrownBy( diff --git a/paimon-core/src/test/java/org/apache/paimon/table/AppendOnlySimpleTableTest.java b/paimon-core/src/test/java/org/apache/paimon/table/AppendOnlySimpleTableTest.java index c0c3f5e738d7..a18a2e87f011 100644 --- a/paimon-core/src/test/java/org/apache/paimon/table/AppendOnlySimpleTableTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/table/AppendOnlySimpleTableTest.java @@ -54,6 +54,7 @@ import org.apache.paimon.predicate.PredicateBuilder; import org.apache.paimon.predicate.TopN; import org.apache.paimon.reader.RecordReader; +import org.apache.paimon.schema.FileSystemSchemaManager; import org.apache.paimon.schema.Schema; import org.apache.paimon.schema.SchemaChange; import org.apache.paimon.schema.SchemaManager; @@ -1714,7 +1715,7 @@ protected FileStoreTable createFileStoreTable(Consumer configure, RowTy } TableSchema tableSchema = SchemaUtils.forceCommit( - new SchemaManager(LocalFileIO.create(), tablePath), + new FileSystemSchemaManager(LocalFileIO.create(), tablePath), new Schema( rowType.getFields(), Collections.singletonList("pt"), @@ -1732,7 +1733,7 @@ protected FileStoreTable createUnawareBucketFileStoreTable(Consumer con configure.accept(conf); TableSchema tableSchema = SchemaUtils.forceCommit( - new SchemaManager(LocalFileIO.create(), tablePath), + new FileSystemSchemaManager(LocalFileIO.create(), tablePath), new Schema( ROW_TYPE.getFields(), Collections.emptyList(), @@ -1750,7 +1751,7 @@ protected FileStoreTable createUnawareBucketFileStoreTable( configure.accept(conf); TableSchema tableSchema = SchemaUtils.forceCommit( - new SchemaManager(LocalFileIO.create(), tablePath), + new FileSystemSchemaManager(LocalFileIO.create(), tablePath), new Schema( rowType.getFields(), Collections.emptyList(), @@ -1990,7 +1991,7 @@ public void testMergeBranchSchemaConflict() throws Exception { table.createBranch(BRANCH_NAME, "tag1"); // Modify schema on main (add a column) - SchemaManager schemaManager = new SchemaManager(table.fileIO(), table.location()); + SchemaManager schemaManager = new FileSystemSchemaManager(table.fileIO(), table.location()); schemaManager.commitChanges(SchemaChange.addColumn("new_col", DataTypes.INT())); // Merge should fail due to schema mismatch @@ -2015,7 +2016,7 @@ public void testMergeBranchSchemaHistoryConflict() throws Exception { table.createBranch(BRANCH_NAME, "tag1"); SchemaManager branchSchemaManager = - new SchemaManager(table.fileIO(), table.location(), BRANCH_NAME); + new FileSystemSchemaManager(table.fileIO(), table.location(), BRANCH_NAME); branchSchemaManager.commitChanges(SchemaChange.addColumn("source_col", DataTypes.INT())); FileStoreTable tableBranch = table.switchToBranch(BRANCH_NAME); try (BatchTableWrite write = @@ -2027,7 +2028,8 @@ public void testMergeBranchSchemaHistoryConflict() throws Exception { branchSchemaManager.commitChanges( Collections.singletonList(SchemaChange.dropColumn("source_col"))); - SchemaManager mainSchemaManager = new SchemaManager(table.fileIO(), table.location()); + SchemaManager mainSchemaManager = + new FileSystemSchemaManager(table.fileIO(), table.location()); mainSchemaManager.commitChanges(SchemaChange.addColumn("target_col", DataTypes.INT())); mainSchemaManager.commitChanges( Collections.singletonList(SchemaChange.dropColumn("target_col"))); @@ -2250,7 +2252,7 @@ public void testMergeBranchRowTrackingMismatch() throws Exception { // Directly write a new schema to the branch with row-tracking disabled SchemaManager branchSchemaManager = - new SchemaManager(table.fileIO(), table.location(), BRANCH_NAME); + new FileSystemSchemaManager(table.fileIO(), table.location(), BRANCH_NAME); TableSchema branchSchema = branchSchemaManager.latest().get(); Map newOptions = new HashMap<>(branchSchema.options()); newOptions.remove("row-tracking.enabled"); diff --git a/paimon-core/src/test/java/org/apache/paimon/table/ChainTableDeletionVectorReadTest.java b/paimon-core/src/test/java/org/apache/paimon/table/ChainTableDeletionVectorReadTest.java index 4fb8eee379f0..a0aad820f154 100644 --- a/paimon-core/src/test/java/org/apache/paimon/table/ChainTableDeletionVectorReadTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/table/ChainTableDeletionVectorReadTest.java @@ -28,6 +28,7 @@ import org.apache.paimon.fs.local.LocalFileIO; import org.apache.paimon.options.Options; import org.apache.paimon.reader.RecordReader; +import org.apache.paimon.schema.FileSystemSchemaManager; import org.apache.paimon.schema.Schema; import org.apache.paimon.schema.SchemaChange; import org.apache.paimon.schema.SchemaManager; @@ -88,7 +89,7 @@ public void testChainReadMatchesBranchReadWithLateLowerSequenceInsert(boolean de private void createTable(boolean deletionVectors) throws Exception { tablePath = new Path("file://" + tempDir + "/chain_dv_" + deletionVectors); - SchemaManager schemaManager = new SchemaManager(fileIO, tablePath); + SchemaManager schemaManager = new FileSystemSchemaManager(fileIO, tablePath); Options options = new Options(); options.set(CoreOptions.BUCKET, 1); options.set(CoreOptions.BUCKET_KEY, "k"); @@ -128,7 +129,7 @@ private void createTable(boolean deletionVectors) throws Exception { } private void configureBranch(String branchName) throws Exception { - SchemaManager sm = new SchemaManager(fileIO, tablePath, branchName); + SchemaManager sm = new FileSystemSchemaManager(fileIO, tablePath, branchName); sm.commitChanges( SchemaChange.setOption( CoreOptions.SCAN_FALLBACK_SNAPSHOT_BRANCH.key(), SNAPSHOT_BRANCH), @@ -139,7 +140,8 @@ private FileStoreTable load() { Options options = new Options(); options.set(CoreOptions.PATH, tablePath.toString()); String branchName = CoreOptions.branch(options.toMap()); - TableSchema schema = new SchemaManager(fileIO, tablePath, branchName).latest().get(); + TableSchema schema = + new FileSystemSchemaManager(fileIO, tablePath, branchName).latest().get(); return FileStoreTableFactory.create(fileIO, tablePath, schema, CatalogEnvironment.empty()); } diff --git a/paimon-core/src/test/java/org/apache/paimon/table/ChainTableFileStoreTableTest.java b/paimon-core/src/test/java/org/apache/paimon/table/ChainTableFileStoreTableTest.java index f0027ab44066..66b82195f0c8 100644 --- a/paimon-core/src/test/java/org/apache/paimon/table/ChainTableFileStoreTableTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/table/ChainTableFileStoreTableTest.java @@ -28,6 +28,7 @@ import org.apache.paimon.fs.local.LocalFileIO; import org.apache.paimon.options.Options; import org.apache.paimon.reader.RecordReader; +import org.apache.paimon.schema.FileSystemSchemaManager; import org.apache.paimon.schema.Schema; import org.apache.paimon.schema.SchemaChange; import org.apache.paimon.schema.SchemaManager; @@ -355,7 +356,8 @@ private FileStoreTable loadTable() { Options options = new Options(); options.set(CoreOptions.PATH, tablePath.toString()); String branchName = CoreOptions.branch(options.toMap()); - Optional schemaOpt = new SchemaManager(fileIO, tablePath, branchName).latest(); + Optional schemaOpt = + new FileSystemSchemaManager(fileIO, tablePath, branchName).latest(); assertThat(schemaOpt.isPresent()).isTrue(); return FileStoreTableFactory.create( fileIO, tablePath, schemaOpt.get(), CatalogEnvironment.empty()); @@ -364,7 +366,7 @@ private FileStoreTable loadTable() { private void createChainTable(Consumer optionCustomizer) throws Exception { Path tablePath = new Path(tempDir.toUri().toString(), tableName); LocalFileIO fileIO = LocalFileIO.create(); - SchemaManager mainSchemaManager = new SchemaManager(fileIO, tablePath); + SchemaManager mainSchemaManager = new FileSystemSchemaManager(fileIO, tablePath); Options options = new Options(); options.set(BUCKET, 1); @@ -415,7 +417,7 @@ private void createPartialUpdateChainTable(Consumer optionCustomizer) throws Exception { Path tablePath = new Path(tempDir.toUri().toString(), tableName); LocalFileIO fileIO = LocalFileIO.create(); - SchemaManager mainSchemaManager = new SchemaManager(fileIO, tablePath); + SchemaManager mainSchemaManager = new FileSystemSchemaManager(fileIO, tablePath); Options options = new Options(); options.set(BUCKET, 1); @@ -462,7 +464,8 @@ private void createPartialUpdateChainTable(Consumer optionCustomizer) private void configureBranchOptions(LocalFileIO fileIO, Path tablePath, String branchName) throws Exception { - SchemaManager branchSchemaManager = new SchemaManager(fileIO, tablePath, branchName); + SchemaManager branchSchemaManager = + new FileSystemSchemaManager(fileIO, tablePath, branchName); branchSchemaManager.commitChanges( SchemaChange.setOption( CoreOptions.SCAN_FALLBACK_SNAPSHOT_BRANCH.key(), SNAPSHOT_BRANCH), diff --git a/paimon-core/src/test/java/org/apache/paimon/table/DeletionVectorIndexCleanupTest.java b/paimon-core/src/test/java/org/apache/paimon/table/DeletionVectorIndexCleanupTest.java index 0c42d809201b..08c3267275fa 100644 --- a/paimon-core/src/test/java/org/apache/paimon/table/DeletionVectorIndexCleanupTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/table/DeletionVectorIndexCleanupTest.java @@ -29,8 +29,8 @@ import org.apache.paimon.io.DataFileMeta; import org.apache.paimon.manifest.IndexManifestEntry; import org.apache.paimon.options.Options; +import org.apache.paimon.schema.FileSystemSchemaManager; import org.apache.paimon.schema.Schema; -import org.apache.paimon.schema.SchemaManager; import org.apache.paimon.schema.SchemaUtils; import org.apache.paimon.schema.TableSchema; import org.apache.paimon.table.sink.StreamTableCommit; @@ -184,7 +184,7 @@ private FileStoreTable createTable(Consumer configure) throws Exception configure.accept(options); TableSchema schema = SchemaUtils.forceCommit( - new SchemaManager(LocalFileIO.create(), tablePath), + new FileSystemSchemaManager(LocalFileIO.create(), tablePath), new Schema( ROW_TYPE.getFields(), Collections.singletonList("pt"), diff --git a/paimon-core/src/test/java/org/apache/paimon/table/FallbackReadFileStoreTableTest.java b/paimon-core/src/test/java/org/apache/paimon/table/FallbackReadFileStoreTableTest.java index 6716dc56a7ca..708cadd6b6c8 100644 --- a/paimon-core/src/test/java/org/apache/paimon/table/FallbackReadFileStoreTableTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/table/FallbackReadFileStoreTableTest.java @@ -35,8 +35,8 @@ import org.apache.paimon.predicate.Predicate; import org.apache.paimon.predicate.PredicateBuilder; import org.apache.paimon.reader.RecordReader; +import org.apache.paimon.schema.FileSystemSchemaManager; import org.apache.paimon.schema.Schema; -import org.apache.paimon.schema.SchemaManager; import org.apache.paimon.schema.SchemaUtils; import org.apache.paimon.schema.TableSchema; import org.apache.paimon.table.sink.StreamTableCommit; @@ -494,7 +494,7 @@ void testSwitchToBranch() throws Exception { TableSchema tableSchema = SchemaUtils.forceCommit( - new SchemaManager(LocalFileIO.create(), tablePath), + new FileSystemSchemaManager(LocalFileIO.create(), tablePath), new Schema( ROW_TYPE.getFields(), Collections.singletonList("pt"), @@ -539,7 +539,7 @@ private void writeDataIntoTable( private AppendOnlyFileStoreTable createTable() throws Exception { TableSchema tableSchema = SchemaUtils.forceCommit( - new SchemaManager(LocalFileIO.create(), tablePath), + new FileSystemSchemaManager(LocalFileIO.create(), tablePath), new Schema( ROW_TYPE.getFields(), Collections.singletonList("pt"), @@ -555,7 +555,7 @@ private FileStoreTable createTableFromBranch(FileStoreTable baseTable, String br return new AppendOnlyFileStoreTable( fileIO, tablePath, - new SchemaManager(fileIO, tablePath, branchName).latest().get()) + new FileSystemSchemaManager(fileIO, tablePath, branchName).latest().get()) .copy(options.toMap()); } } diff --git a/paimon-core/src/test/java/org/apache/paimon/table/PrimaryKeySimpleTableTest.java b/paimon-core/src/test/java/org/apache/paimon/table/PrimaryKeySimpleTableTest.java index 0fa383dd4d94..60f80218754b 100644 --- a/paimon-core/src/test/java/org/apache/paimon/table/PrimaryKeySimpleTableTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/table/PrimaryKeySimpleTableTest.java @@ -49,8 +49,8 @@ import org.apache.paimon.predicate.PredicateBuilder; import org.apache.paimon.predicate.TopN; import org.apache.paimon.reader.RecordReader; +import org.apache.paimon.schema.FileSystemSchemaManager; import org.apache.paimon.schema.Schema; -import org.apache.paimon.schema.SchemaManager; import org.apache.paimon.schema.SchemaUtils; import org.apache.paimon.schema.TableSchema; import org.apache.paimon.table.query.LocalTableQuery; @@ -2985,7 +2985,7 @@ protected FileStoreTable createFileStoreTable(Consumer configure, RowTy configure.accept(options); TableSchema tableSchema = SchemaUtils.forceCommit( - new SchemaManager(LocalFileIO.create(), tablePath), + new FileSystemSchemaManager(LocalFileIO.create(), tablePath), new Schema( rowType.getFields(), Collections.singletonList("pt"), diff --git a/paimon-core/src/test/java/org/apache/paimon/table/SchemaEvolutionTableTestBase.java b/paimon-core/src/test/java/org/apache/paimon/table/SchemaEvolutionTableTestBase.java index 27789241037b..5ba7d29de9f5 100644 --- a/paimon-core/src/test/java/org/apache/paimon/table/SchemaEvolutionTableTestBase.java +++ b/paimon-core/src/test/java/org/apache/paimon/table/SchemaEvolutionTableTestBase.java @@ -33,6 +33,7 @@ import org.apache.paimon.reader.ReaderSupplier; import org.apache.paimon.reader.RecordReader; import org.apache.paimon.reader.RecordReaderIterator; +import org.apache.paimon.schema.FileSystemSchemaManager; import org.apache.paimon.schema.Schema; import org.apache.paimon.schema.SchemaChange; import org.apache.paimon.schema.SchemaManager; @@ -486,7 +487,7 @@ protected List getResult( } /** {@link SchemaManager} subclass for testing. */ - public static class TestingSchemaManager extends SchemaManager { + public static class TestingSchemaManager extends FileSystemSchemaManager { private final Map tableSchemas; public TestingSchemaManager(Path tableRoot, Map tableSchemas) { diff --git a/paimon-core/src/test/java/org/apache/paimon/table/SchemaEvolutionTest.java b/paimon-core/src/test/java/org/apache/paimon/table/SchemaEvolutionTest.java index 33531d8720b7..65d4c67eefe7 100644 --- a/paimon-core/src/test/java/org/apache/paimon/table/SchemaEvolutionTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/table/SchemaEvolutionTest.java @@ -30,6 +30,7 @@ import org.apache.paimon.fs.local.LocalFileIO; import org.apache.paimon.predicate.Predicate; import org.apache.paimon.predicate.PredicateBuilder; +import org.apache.paimon.schema.FileSystemSchemaManager; import org.apache.paimon.schema.Schema; import org.apache.paimon.schema.SchemaChange; import org.apache.paimon.schema.SchemaManager; @@ -93,7 +94,7 @@ public class SchemaEvolutionTest { public void beforeEach() { tablePath = new Path(tempDir.toUri()); identifier = SchemaManager.identifierFromPath(tablePath.toString(), true); - schemaManager = new SchemaManager(LocalFileIO.create(), tablePath); + schemaManager = new FileSystemSchemaManager(LocalFileIO.create(), tablePath); commitUser = UUID.randomUUID().toString(); } diff --git a/paimon-core/src/test/java/org/apache/paimon/table/SimpleTableTestBase.java b/paimon-core/src/test/java/org/apache/paimon/table/SimpleTableTestBase.java index 8c7266939f89..f768753d7fe7 100644 --- a/paimon-core/src/test/java/org/apache/paimon/table/SimpleTableTestBase.java +++ b/paimon-core/src/test/java/org/apache/paimon/table/SimpleTableTestBase.java @@ -43,6 +43,7 @@ import org.apache.paimon.reader.ReaderSupplier; import org.apache.paimon.reader.RecordReader; import org.apache.paimon.reader.RecordReaderIterator; +import org.apache.paimon.schema.FileSystemSchemaManager; import org.apache.paimon.schema.SchemaChange; import org.apache.paimon.schema.SchemaManager; import org.apache.paimon.schema.TableSchema; @@ -587,7 +588,7 @@ public void testCopyWithLatestSchema() throws Exception { write.write(rowData(1, 20, 200L)); commit.commit(0, write.prepareCommit(true, 0)); - SchemaManager schemaManager = new SchemaManager(table.fileIO(), table.location()); + SchemaManager schemaManager = new FileSystemSchemaManager(table.fileIO(), table.location()); schemaManager.commitChanges(SchemaChange.addColumn("added", DataTypes.INT())); table = table.copyWithLatestSchema(); assertThat(table.coreOptions().snapshotNumRetainMax()).isEqualTo(100); @@ -618,7 +619,7 @@ public void testCopyWithLatestSchema() throws Exception { @Test public void testCopyWithLatestSchemaPicksUpAlteredOptions() throws Exception { FileStoreTable table = createFileStoreTable(); - SchemaManager schemaManager = new SchemaManager(table.fileIO(), table.location()); + SchemaManager schemaManager = new FileSystemSchemaManager(table.fileIO(), table.location()); schemaManager.commitChanges(SchemaChange.setOption("my-custom-key", "my-custom-value")); @@ -1098,7 +1099,7 @@ public void testCreateBranch() throws Exception { // verify schema in test-branch is equal to schema 0 SchemaManager schemaManager = - new SchemaManager(table.fileIO(), table.location(), "test-branch"); + new FileSystemSchemaManager(table.fileIO(), table.location(), "test-branch"); TableSchema branchSchema = SchemaManager.fromPath(table.fileIO(), schemaManager.toSchemaPath(0)); TableSchema schema0 = schemaManager.schema(0); @@ -1350,7 +1351,7 @@ public void testFastForward() throws Exception { assertThat(branchSnapshot.equals(snapshot)).isTrue(); // verify schema in branch1 and main branch is same - SchemaManager schemaManager = new SchemaManager(table.fileIO(), table.location()); + SchemaManager schemaManager = new FileSystemSchemaManager(table.fileIO(), table.location()); TableSchema branchSchema = SchemaManager.fromPath( table.fileIO(), schemaManager.copyWithBranch(BRANCH_NAME).toSchemaPath(0)); diff --git a/paimon-core/src/test/java/org/apache/paimon/table/sink/TableCommitTest.java b/paimon-core/src/test/java/org/apache/paimon/table/sink/TableCommitTest.java index d5a4237cc5ef..4c1a3a94e910 100644 --- a/paimon-core/src/test/java/org/apache/paimon/table/sink/TableCommitTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/table/sink/TableCommitTest.java @@ -34,8 +34,8 @@ import org.apache.paimon.io.DataIncrement; import org.apache.paimon.manifest.ManifestCommittable; import org.apache.paimon.options.Options; +import org.apache.paimon.schema.FileSystemSchemaManager; import org.apache.paimon.schema.Schema; -import org.apache.paimon.schema.SchemaManager; import org.apache.paimon.schema.SchemaUtils; import org.apache.paimon.schema.TableSchema; import org.apache.paimon.stats.SimpleStats; @@ -127,7 +127,7 @@ private void testCommitCallbackWithFailureImpl(int bucket, int numIdentifiers, S testId); TableSchema tableSchema = SchemaUtils.forceCommit( - new SchemaManager(LocalFileIO.create(), new Path(path)), + new FileSystemSchemaManager(LocalFileIO.create(), new Path(path)), new Schema( rowType.getFields(), Collections.emptyList(), @@ -219,7 +219,7 @@ public void testRecoverDeletedFiles() throws Exception { options.set(CoreOptions.BUCKET, 1); TableSchema tableSchema = SchemaUtils.forceCommit( - new SchemaManager(LocalFileIO.create(), new Path(path)), + new FileSystemSchemaManager(LocalFileIO.create(), new Path(path)), new Schema( rowType.getFields(), Collections.emptyList(), @@ -282,7 +282,7 @@ public void testGiveUpCommitWhenTotalBucketsChanged() throws Exception { options.set(CoreOptions.BUCKET, 1); TableSchema tableSchema = SchemaUtils.forceCommit( - new SchemaManager(LocalFileIO.create(), new Path(path)), + new FileSystemSchemaManager(LocalFileIO.create(), new Path(path)), new Schema( rowType.getFields(), Collections.emptyList(), @@ -338,7 +338,7 @@ public void testGiveUpCommitWhenAppendFoundTotalBucketsChanged(boolean checkAppe options.set(CoreOptions.BUCKET, 1); TableSchema tableSchema = SchemaUtils.forceCommit( - new SchemaManager(LocalFileIO.create(), new Path(path)), + new FileSystemSchemaManager(LocalFileIO.create(), new Path(path)), new Schema( rowType.getFields(), Collections.emptyList(), @@ -402,7 +402,7 @@ public void testStrictModeForCompact() throws Exception { options.set(CoreOptions.NUM_SORTED_RUNS_COMPACTION_TRIGGER, 10); TableSchema tableSchema = SchemaUtils.forceCommit( - new SchemaManager(LocalFileIO.create(), new Path(path)), + new FileSystemSchemaManager(LocalFileIO.create(), new Path(path)), new Schema( rowType.getFields(), Collections.singletonList("pt"), @@ -475,7 +475,7 @@ public void testStrictModeForOverwrite() throws Exception { options.set(CoreOptions.NUM_SORTED_RUNS_COMPACTION_TRIGGER, 10); TableSchema tableSchema = SchemaUtils.forceCommit( - new SchemaManager(LocalFileIO.create(), new Path(path)), + new FileSystemSchemaManager(LocalFileIO.create(), new Path(path)), new Schema( rowType.getFields(), Collections.singletonList("pt"), @@ -562,7 +562,7 @@ public void testStrictModeForIndexOnlyCompact(boolean dataEvolutionEnabled) thro options.set(CoreOptions.DATA_EVOLUTION_ENABLED, dataEvolutionEnabled); TableSchema tableSchema = SchemaUtils.forceCommit( - new SchemaManager(LocalFileIO.create(), new Path(path)), + new FileSystemSchemaManager(LocalFileIO.create(), new Path(path)), new Schema( rowType.getFields(), Collections.singletonList("pt"), @@ -648,7 +648,7 @@ public void testStrictModeForDvOnlyOverwrite() throws Exception { options.set(CoreOptions.DELETION_VECTORS_ENABLED, true); TableSchema tableSchema = SchemaUtils.forceCommit( - new SchemaManager(LocalFileIO.create(), new Path(path)), + new FileSystemSchemaManager(LocalFileIO.create(), new Path(path)), new Schema( rowType.getFields(), Collections.singletonList("pt"), @@ -723,7 +723,7 @@ public void testStrictModeShortCircuitsWhenIndexManifestUnchanged() throws Excep options.set(CoreOptions.DELETION_VECTORS_ENABLED, true); TableSchema tableSchema = SchemaUtils.forceCommit( - new SchemaManager(LocalFileIO.create(), new Path(path)), + new FileSystemSchemaManager(LocalFileIO.create(), new Path(path)), new Schema( rowType.getFields(), Collections.singletonList("pt"), @@ -825,7 +825,7 @@ public void testStrictModeForOverwriteCheckAppend() throws Exception { options.set(CoreOptions.NUM_SORTED_RUNS_COMPACTION_TRIGGER, 10); TableSchema tableSchema = SchemaUtils.forceCommit( - new SchemaManager(LocalFileIO.create(), new Path(path)), + new FileSystemSchemaManager(LocalFileIO.create(), new Path(path)), new Schema( rowType.getFields(), Collections.singletonList("pt"), @@ -934,7 +934,7 @@ public void testStrictModeScanStateNotLeakedAcrossSnapshots() throws Exception { options.set(CoreOptions.NUM_SORTED_RUNS_COMPACTION_TRIGGER, 10); TableSchema tableSchema = SchemaUtils.forceCommit( - new SchemaManager(LocalFileIO.create(), new Path(path)), + new FileSystemSchemaManager(LocalFileIO.create(), new Path(path)), new Schema( rowType.getFields(), Collections.singletonList("pt"), @@ -1011,7 +1011,7 @@ public void testStrictModeForNonOverwriteNoCheckAppend() throws Exception { options.set(CoreOptions.NUM_SORTED_RUNS_COMPACTION_TRIGGER, 10); TableSchema tableSchema = SchemaUtils.forceCommit( - new SchemaManager(LocalFileIO.create(), new Path(path)), + new FileSystemSchemaManager(LocalFileIO.create(), new Path(path)), new Schema( rowType.getFields(), Collections.emptyList(), @@ -1067,7 +1067,7 @@ public void testStrictModeForNonPartitionedTable() throws Exception { options.set(CoreOptions.NUM_SORTED_RUNS_COMPACTION_TRIGGER, 10); TableSchema tableSchema = SchemaUtils.forceCommit( - new SchemaManager(LocalFileIO.create(), new Path(path)), + new FileSystemSchemaManager(LocalFileIO.create(), new Path(path)), new Schema( rowType.getFields(), Collections.emptyList(), @@ -1155,7 +1155,7 @@ public void testExpireForEmptyCommit() throws Exception { options.set(CoreOptions.SNAPSHOT_NUM_RETAINED_MIN, 2); TableSchema tableSchema = SchemaUtils.forceCommit( - new SchemaManager(LocalFileIO.create(), new Path(path)), + new FileSystemSchemaManager(LocalFileIO.create(), new Path(path)), new Schema( rowType.getFields(), Collections.emptyList(), @@ -1206,7 +1206,7 @@ public void testRecoverCompactedChangelogFiles() throws Exception { options.set(CoreOptions.BUCKET, 3); TableSchema tableSchema = SchemaUtils.forceCommit( - new SchemaManager(LocalFileIO.create(), new Path(path)), + new FileSystemSchemaManager(LocalFileIO.create(), new Path(path)), new Schema( rowType.getFields(), Collections.emptyList(), @@ -1379,7 +1379,7 @@ public void testCompactConflictWithUpgrade() throws Exception { options.set(CoreOptions.BUCKET, 1); TableSchema tableSchema = SchemaUtils.forceCommit( - new SchemaManager(LocalFileIO.create(), new Path(path)), + new FileSystemSchemaManager(LocalFileIO.create(), new Path(path)), new Schema( rowType.getFields(), Collections.emptyList(), diff --git a/paimon-core/src/test/java/org/apache/paimon/table/sink/TableWriteTest.java b/paimon-core/src/test/java/org/apache/paimon/table/sink/TableWriteTest.java index 659c010d529a..afef43538009 100644 --- a/paimon-core/src/test/java/org/apache/paimon/table/sink/TableWriteTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/table/sink/TableWriteTest.java @@ -31,8 +31,8 @@ import org.apache.paimon.options.Options; import org.apache.paimon.reader.RecordReader; import org.apache.paimon.reader.RecordReaderIterator; +import org.apache.paimon.schema.FileSystemSchemaManager; import org.apache.paimon.schema.Schema; -import org.apache.paimon.schema.SchemaManager; import org.apache.paimon.schema.SchemaUtils; import org.apache.paimon.schema.TableSchema; import org.apache.paimon.table.FileStoreTable; @@ -338,7 +338,7 @@ List streamingRead( private FileStoreTable createFileStoreTable(Options conf) throws Exception { TableSchema tableSchema = SchemaUtils.forceCommit( - new SchemaManager(LocalFileIO.create(), tablePath), + new FileSystemSchemaManager(LocalFileIO.create(), tablePath), new Schema( ROW_TYPE.getFields(), Collections.singletonList("pt"), diff --git a/paimon-core/src/test/java/org/apache/paimon/table/source/PrimaryKeyVectorRawFileSplitReadTest.java b/paimon-core/src/test/java/org/apache/paimon/table/source/PrimaryKeyVectorRawFileSplitReadTest.java index 73b75cfec576..4eb8b61db61e 100644 --- a/paimon-core/src/test/java/org/apache/paimon/table/source/PrimaryKeyVectorRawFileSplitReadTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/table/source/PrimaryKeyVectorRawFileSplitReadTest.java @@ -27,8 +27,8 @@ import org.apache.paimon.options.Options; import org.apache.paimon.reader.RecordReader; import org.apache.paimon.reader.ScoreRecordIterator; +import org.apache.paimon.schema.FileSystemSchemaManager; import org.apache.paimon.schema.Schema; -import org.apache.paimon.schema.SchemaManager; import org.apache.paimon.schema.SchemaUtils; import org.apache.paimon.schema.TableSchema; import org.apache.paimon.table.FileStoreTable; @@ -65,7 +65,8 @@ void testRegularLimitDoesNotDropSelectedPhysicalPositions() throws Exception { .options(options.toMap()) .build(); TableSchema tableSchema = - SchemaUtils.forceCommit(new SchemaManager(LocalFileIO.create(), tablePath), schema); + SchemaUtils.forceCommit( + new FileSystemSchemaManager(LocalFileIO.create(), tablePath), schema); FileStoreTable table = FileStoreTableFactory.create(LocalFileIO.create(), tablePath, tableSchema); diff --git a/paimon-core/src/test/java/org/apache/paimon/table/source/snapshot/ScannerTestBase.java b/paimon-core/src/test/java/org/apache/paimon/table/source/snapshot/ScannerTestBase.java index 2f92f905654b..a7fac3321a66 100644 --- a/paimon-core/src/test/java/org/apache/paimon/table/source/snapshot/ScannerTestBase.java +++ b/paimon-core/src/test/java/org/apache/paimon/table/source/snapshot/ScannerTestBase.java @@ -31,6 +31,7 @@ import org.apache.paimon.reader.ReaderSupplier; import org.apache.paimon.reader.RecordReader; import org.apache.paimon.reader.RecordReaderIterator; +import org.apache.paimon.schema.FileSystemSchemaManager; import org.apache.paimon.schema.Schema; import org.apache.paimon.schema.SchemaChange; import org.apache.paimon.schema.SchemaManager; @@ -153,7 +154,7 @@ protected FileStoreTable createFileStoreTable(boolean withPrimaryKeys) throws Ex protected FileStoreTable createFileStoreTable( boolean withPrimaryKeys, Options conf, Path tablePath) throws Exception { - SchemaManager schemaManager = new SchemaManager(fileIO, tablePath); + SchemaManager schemaManager = new FileSystemSchemaManager(fileIO, tablePath); List primaryKeys = new ArrayList<>(); if (withPrimaryKeys) { primaryKeys = Arrays.asList("pt", "a"); diff --git a/paimon-core/src/test/java/org/apache/paimon/table/source/snapshot/SnapshotReaderTest.java b/paimon-core/src/test/java/org/apache/paimon/table/source/snapshot/SnapshotReaderTest.java index 02a170c362ef..e0bfd5fd918e 100644 --- a/paimon-core/src/test/java/org/apache/paimon/table/source/snapshot/SnapshotReaderTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/table/source/snapshot/SnapshotReaderTest.java @@ -28,6 +28,7 @@ import org.apache.paimon.fs.Path; import org.apache.paimon.io.DataFileMeta; import org.apache.paimon.options.Options; +import org.apache.paimon.schema.FileSystemSchemaManager; import org.apache.paimon.schema.Schema; import org.apache.paimon.schema.SchemaChange; import org.apache.paimon.schema.SchemaManager; @@ -215,7 +216,7 @@ public void testGetAppendOnlyRawFiles() throws Exception { // change file schema write.close(); - SchemaManager schemaManager = new SchemaManager(fileIO, tablePath); + SchemaManager schemaManager = new FileSystemSchemaManager(fileIO, tablePath); schemaManager.commitChanges(SchemaChange.addColumn("v2", DataTypes.STRING())); table = table.copyWithLatestSchema(); write = table.newWrite(commitUser); @@ -299,7 +300,7 @@ public void testGetAppendOnlyIndexFiles() throws Exception { // change file schema write.close(); - SchemaManager schemaManager = new SchemaManager(fileIO, tablePath); + SchemaManager schemaManager = new FileSystemSchemaManager(fileIO, tablePath); schemaManager.commitChanges(SchemaChange.addColumn("v2", DataTypes.STRING())); table = table.copyWithLatestSchema(); write = table.newWrite(commitUser); @@ -358,7 +359,7 @@ private FileStoreTable createFileStoreTable( + CoreOptions.COLUMNS, rowType.getFieldNames().get(0)); - SchemaManager schemaManager = new SchemaManager(fileIO, tablePath); + SchemaManager schemaManager = new FileSystemSchemaManager(fileIO, tablePath); TableSchema tableSchema = schemaManager.createTable( new Schema( diff --git a/paimon-core/src/test/java/org/apache/paimon/table/system/AggregationFieldsTableTest.java b/paimon-core/src/test/java/org/apache/paimon/table/system/AggregationFieldsTableTest.java index a633f015e80d..2cb19ac07f4b 100644 --- a/paimon-core/src/test/java/org/apache/paimon/table/system/AggregationFieldsTableTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/table/system/AggregationFieldsTableTest.java @@ -25,6 +25,7 @@ import org.apache.paimon.fs.FileIO; import org.apache.paimon.fs.Path; import org.apache.paimon.fs.local.LocalFileIO; +import org.apache.paimon.schema.FileSystemSchemaManager; import org.apache.paimon.schema.Schema; import org.apache.paimon.schema.SchemaManager; import org.apache.paimon.schema.SchemaUtils; @@ -76,7 +77,7 @@ public void before() throws Exception { FileIO fileIO = LocalFileIO.create(); Path tablePath = new Path(String.format("%s/%s.db/%s", warehouse, database, tableName)); - schemaManager = new SchemaManager(fileIO, tablePath); + schemaManager = new FileSystemSchemaManager(fileIO, tablePath); } @Test diff --git a/paimon-core/src/test/java/org/apache/paimon/table/system/AuditLogTableTest.java b/paimon-core/src/test/java/org/apache/paimon/table/system/AuditLogTableTest.java index 00b42c2841b1..1b88d2b9b9bc 100644 --- a/paimon-core/src/test/java/org/apache/paimon/table/system/AuditLogTableTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/table/system/AuditLogTableTest.java @@ -32,6 +32,7 @@ import org.apache.paimon.options.Options; import org.apache.paimon.predicate.PredicateBuilder; import org.apache.paimon.reader.RecordReader; +import org.apache.paimon.schema.FileSystemSchemaManager; import org.apache.paimon.schema.Schema; import org.apache.paimon.schema.SchemaChange; import org.apache.paimon.schema.SchemaManager; @@ -84,7 +85,7 @@ public void testReadAuditLogWithPrimaryKeySortedIndex() throws Exception { FileIO fileIO = LocalFileIO.create(); TableSchema tableSchema = SchemaUtils.forceCommit( - new SchemaManager(fileIO, tablePath), + new FileSystemSchemaManager(fileIO, tablePath), Schema.newBuilder() .column("pk", DataTypes.INT()) .column("score", DataTypes.INT()) @@ -127,7 +128,7 @@ public void testReadAuditLogWithPrimaryKeySortedIndex() throws Exception { public void testChainTableAuditLogPreservesChainScan() throws Exception { Path tablePath = new Path(String.format("%s/%s.db/chain_audit_table", warehouse, database)); FileIO fileIO = LocalFileIO.create(); - SchemaManager schemaManager = new SchemaManager(fileIO, tablePath); + SchemaManager schemaManager = new FileSystemSchemaManager(fileIO, tablePath); schemaManager.createTable( Schema.newBuilder() .column("dt", DataTypes.STRING()) @@ -157,8 +158,8 @@ public void testChainTableAuditLogPreservesChainScan() throws Exception { SchemaChange.setOption("partition.timestamp-pattern", "$dt"), SchemaChange.setOption("partition.timestamp-formatter", "yyyyMMdd")); schemaManager.commitChanges(chainOptions); - new SchemaManager(fileIO, tablePath, "snapshot").commitChanges(chainOptions); - new SchemaManager(fileIO, tablePath, "delta").commitChanges(chainOptions); + new FileSystemSchemaManager(fileIO, tablePath, "snapshot").commitChanges(chainOptions); + new FileSystemSchemaManager(fileIO, tablePath, "delta").commitChanges(chainOptions); FileStoreTable snapshotTable = branchTable(fileIO, tablePath, "snapshot"); FileStoreTable deltaTable = branchTable(fileIO, tablePath, "delta"); @@ -221,7 +222,7 @@ public void testReadBaseTableAfterAuditLogDoesNotIncludeSequenceNumber() throws TableSchema tableSchema = SchemaUtils.forceCommit( - new SchemaManager(fileIO, tablePath), + new FileSystemSchemaManager(fileIO, tablePath), Schema.newBuilder() .column("pk", DataTypes.INT()) .column("pt", DataTypes.INT()) @@ -298,7 +299,7 @@ private AuditLogTable createAuditLogTable(String tableName, boolean enableSequen TableSchema tableSchema = SchemaUtils.forceCommit( - new SchemaManager(fileIO, tablePath), schemaBuilder.build()); + new FileSystemSchemaManager(fileIO, tablePath), schemaBuilder.build()); FileStoreTable table = FileStoreTableFactory.create(LocalFileIO.create(), tablePath, tableSchema); @@ -311,7 +312,7 @@ private AuditLogTable createAuditLogTable(String tableName, boolean enableSequen private FileStoreTable branchTable(FileIO fileIO, Path tablePath, String branch) { TableSchema branchSchema = - new SchemaManager(fileIO, tablePath, branch) + new FileSystemSchemaManager(fileIO, tablePath, branch) .latest() .orElseThrow(AssertionError::new); Options dynamicOptions = new Options(); diff --git a/paimon-core/src/test/java/org/apache/paimon/table/system/BinlogTableTest.java b/paimon-core/src/test/java/org/apache/paimon/table/system/BinlogTableTest.java index 6f187fbffd98..f698302b3940 100644 --- a/paimon-core/src/test/java/org/apache/paimon/table/system/BinlogTableTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/table/system/BinlogTableTest.java @@ -27,9 +27,9 @@ import org.apache.paimon.fs.FileIO; import org.apache.paimon.fs.Path; import org.apache.paimon.fs.local.LocalFileIO; +import org.apache.paimon.schema.FileSystemSchemaManager; import org.apache.paimon.schema.Schema; import org.apache.paimon.schema.SchemaChange; -import org.apache.paimon.schema.SchemaManager; import org.apache.paimon.schema.SchemaUtils; import org.apache.paimon.schema.TableSchema; import org.apache.paimon.table.FileStoreTable; @@ -120,7 +120,7 @@ private BinlogTable createBinlogTable(String tableName, boolean enableSequenceNu TableSchema tableSchema = SchemaUtils.forceCommit( - new SchemaManager(fileIO, tablePath), schemaBuilder.build()); + new FileSystemSchemaManager(fileIO, tablePath), schemaBuilder.build()); FileStoreTable table = FileStoreTableFactory.create(LocalFileIO.create(), tablePath, tableSchema); diff --git a/paimon-core/src/test/java/org/apache/paimon/table/system/BucketsTableTest.java b/paimon-core/src/test/java/org/apache/paimon/table/system/BucketsTableTest.java index 03431cb3dd7b..777f3e3c5806 100644 --- a/paimon-core/src/test/java/org/apache/paimon/table/system/BucketsTableTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/table/system/BucketsTableTest.java @@ -28,8 +28,8 @@ import org.apache.paimon.predicate.Predicate; import org.apache.paimon.predicate.PredicateBuilder; import org.apache.paimon.reader.RecordReader; +import org.apache.paimon.schema.FileSystemSchemaManager; import org.apache.paimon.schema.Schema; -import org.apache.paimon.schema.SchemaManager; import org.apache.paimon.schema.SchemaUtils; import org.apache.paimon.schema.TableSchema; import org.apache.paimon.table.FileStoreTable; @@ -71,7 +71,7 @@ public void before() throws Exception { .option("bucket", "2") .build(); TableSchema tableSchema = - SchemaUtils.forceCommit(new SchemaManager(fileIO, tablePath), schema); + SchemaUtils.forceCommit(new FileSystemSchemaManager(fileIO, tablePath), schema); FileStoreTable table = FileStoreTableFactory.create(LocalFileIO.create(), tablePath, tableSchema); diff --git a/paimon-core/src/test/java/org/apache/paimon/table/system/FilesTableTest.java b/paimon-core/src/test/java/org/apache/paimon/table/system/FilesTableTest.java index 4c4e3c88598a..1c8975ffef3a 100644 --- a/paimon-core/src/test/java/org/apache/paimon/table/system/FilesTableTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/table/system/FilesTableTest.java @@ -34,9 +34,9 @@ import org.apache.paimon.predicate.LeafPredicate; import org.apache.paimon.predicate.Predicate; import org.apache.paimon.predicate.PredicateBuilder; +import org.apache.paimon.schema.FileSystemSchemaManager; import org.apache.paimon.schema.Schema; import org.apache.paimon.schema.SchemaChange; -import org.apache.paimon.schema.SchemaManager; import org.apache.paimon.schema.SchemaUtils; import org.apache.paimon.schema.TableSchema; import org.apache.paimon.table.FileStoreTable; @@ -95,7 +95,7 @@ public void before() throws Exception { .option(CoreOptions.SEQUENCE_FIELD.key(), "col1") .build(); TableSchema tableSchema = - SchemaUtils.forceCommit(new SchemaManager(fileIO, tablePath), schema); + SchemaUtils.forceCommit(new FileSystemSchemaManager(fileIO, tablePath), schema); table = FileStoreTableFactory.create(LocalFileIO.create(), tablePath, tableSchema); scan = table.store().newScan(); diff --git a/paimon-core/src/test/java/org/apache/paimon/table/system/OptionsTableTest.java b/paimon-core/src/test/java/org/apache/paimon/table/system/OptionsTableTest.java index a2f458fc4fb6..1c2b02c8eedd 100644 --- a/paimon-core/src/test/java/org/apache/paimon/table/system/OptionsTableTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/table/system/OptionsTableTest.java @@ -26,6 +26,7 @@ import org.apache.paimon.fs.FileIO; import org.apache.paimon.fs.Path; import org.apache.paimon.fs.local.LocalFileIO; +import org.apache.paimon.schema.FileSystemSchemaManager; import org.apache.paimon.schema.Schema; import org.apache.paimon.schema.SchemaManager; import org.apache.paimon.schema.SchemaUtils; @@ -71,7 +72,7 @@ public void before() throws Exception { FileIO fileIO = LocalFileIO.create(); Path tablePath = new Path(String.format("%s/%s.db/%s", warehouse, database, tableName)); - schemaManager = new SchemaManager(fileIO, tablePath); + schemaManager = new FileSystemSchemaManager(fileIO, tablePath); } @Test diff --git a/paimon-core/src/test/java/org/apache/paimon/table/system/PartitionsTableTest.java b/paimon-core/src/test/java/org/apache/paimon/table/system/PartitionsTableTest.java index ef18580dc6c0..740913ed098c 100644 --- a/paimon-core/src/test/java/org/apache/paimon/table/system/PartitionsTableTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/table/system/PartitionsTableTest.java @@ -29,8 +29,8 @@ import org.apache.paimon.predicate.Predicate; import org.apache.paimon.predicate.PredicateBuilder; import org.apache.paimon.reader.RecordReader; +import org.apache.paimon.schema.FileSystemSchemaManager; import org.apache.paimon.schema.Schema; -import org.apache.paimon.schema.SchemaManager; import org.apache.paimon.schema.SchemaUtils; import org.apache.paimon.schema.TableSchema; import org.apache.paimon.table.FileStoreTable; @@ -76,7 +76,7 @@ public void before() throws Exception { .option("bucket", "1") .build(); TableSchema tableSchema = - SchemaUtils.forceCommit(new SchemaManager(fileIO, tablePath), schema); + SchemaUtils.forceCommit(new FileSystemSchemaManager(fileIO, tablePath), schema); table = FileStoreTableFactory.create(LocalFileIO.create(), tablePath, tableSchema); Identifier filesTableId = diff --git a/paimon-core/src/test/java/org/apache/paimon/table/system/SchemasTableTest.java b/paimon-core/src/test/java/org/apache/paimon/table/system/SchemasTableTest.java index c7c0ca5cf872..14de5a573dc7 100644 --- a/paimon-core/src/test/java/org/apache/paimon/table/system/SchemasTableTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/table/system/SchemasTableTest.java @@ -27,6 +27,7 @@ import org.apache.paimon.fs.FileIO; import org.apache.paimon.fs.Path; import org.apache.paimon.fs.local.LocalFileIO; +import org.apache.paimon.schema.FileSystemSchemaManager; import org.apache.paimon.schema.Schema; import org.apache.paimon.schema.SchemaManager; import org.apache.paimon.schema.TableSchema; @@ -68,7 +69,7 @@ public void before() throws Exception { FileIO fileIO = LocalFileIO.create(); Path tablePath = new Path(String.format("%s/%s.db/%s", warehouse, database, "T")); - schemaManager = new SchemaManager(fileIO, tablePath); + schemaManager = new FileSystemSchemaManager(fileIO, tablePath); } @Test diff --git a/paimon-core/src/test/java/org/apache/paimon/table/system/SnapshotsTableTest.java b/paimon-core/src/test/java/org/apache/paimon/table/system/SnapshotsTableTest.java index c1a6b6f7140a..9bb72743bfb4 100644 --- a/paimon-core/src/test/java/org/apache/paimon/table/system/SnapshotsTableTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/table/system/SnapshotsTableTest.java @@ -28,8 +28,8 @@ import org.apache.paimon.fs.FileIO; import org.apache.paimon.fs.Path; import org.apache.paimon.fs.local.LocalFileIO; +import org.apache.paimon.schema.FileSystemSchemaManager; import org.apache.paimon.schema.Schema; -import org.apache.paimon.schema.SchemaManager; import org.apache.paimon.schema.SchemaUtils; import org.apache.paimon.schema.TableSchema; import org.apache.paimon.table.FileStoreTable; @@ -74,7 +74,7 @@ public void before() throws Exception { .build(); snapshotManager = newSnapshotManager(fileIO, tablePath); TableSchema tableSchema = - SchemaUtils.forceCommit(new SchemaManager(fileIO, tablePath), schema); + SchemaUtils.forceCommit(new FileSystemSchemaManager(fileIO, tablePath), schema); FileStoreTable table = FileStoreTableFactory.create(LocalFileIO.create(), tablePath, tableSchema); diff --git a/paimon-core/src/test/java/org/apache/paimon/utils/FileSystemBranchManagerTest.java b/paimon-core/src/test/java/org/apache/paimon/utils/FileSystemBranchManagerTest.java index 109303a9288d..b2aba70b2b0c 100644 --- a/paimon-core/src/test/java/org/apache/paimon/utils/FileSystemBranchManagerTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/utils/FileSystemBranchManagerTest.java @@ -21,6 +21,7 @@ import org.apache.paimon.fs.FileIO; import org.apache.paimon.fs.FileIOFinder; import org.apache.paimon.fs.Path; +import org.apache.paimon.schema.FileSystemSchemaManager; import org.apache.paimon.schema.Schema; import org.apache.paimon.schema.SchemaManager; import org.apache.paimon.types.DataTypes; @@ -61,7 +62,7 @@ void before() throws Exception { .build(); // Create schema manager and initialize schema - schemaManager = new SchemaManager(fileIO, tablePath); + schemaManager = new FileSystemSchemaManager(fileIO, tablePath); schemaManager.createTable(schema); // Create snapshot, tag, and branch managers diff --git a/paimon-core/src/test/java/org/apache/paimon/utils/PartitionStatisticsReporterTest.java b/paimon-core/src/test/java/org/apache/paimon/utils/PartitionStatisticsReporterTest.java index 53b5f3759271..2f99fc85ca10 100644 --- a/paimon-core/src/test/java/org/apache/paimon/utils/PartitionStatisticsReporterTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/utils/PartitionStatisticsReporterTest.java @@ -23,6 +23,7 @@ import org.apache.paimon.fs.Path; import org.apache.paimon.fs.local.LocalFileIO; import org.apache.paimon.partition.PartitionStatistics; +import org.apache.paimon.schema.FileSystemSchemaManager; import org.apache.paimon.schema.Schema; import org.apache.paimon.schema.SchemaManager; import org.apache.paimon.table.FileStoreTable; @@ -55,7 +56,7 @@ public class PartitionStatisticsReporterTest { @Test public void testReportAction() throws Exception { Path tablePath = new Path(tempDir.toString(), "table"); - SchemaManager schemaManager = new SchemaManager(LocalFileIO.create(), tablePath); + SchemaManager schemaManager = new FileSystemSchemaManager(LocalFileIO.create(), tablePath); Schema schema = new Schema( Lists.newArrayList( diff --git a/paimon-core/src/test/java/org/apache/paimon/utils/TagManagerTest.java b/paimon-core/src/test/java/org/apache/paimon/utils/TagManagerTest.java index eaf11bc4e756..ede52e6ef3ab 100644 --- a/paimon-core/src/test/java/org/apache/paimon/utils/TagManagerTest.java +++ b/paimon-core/src/test/java/org/apache/paimon/utils/TagManagerTest.java @@ -29,6 +29,7 @@ import org.apache.paimon.fs.local.LocalFileIO; import org.apache.paimon.mergetree.compact.DeduplicateMergeFunction; import org.apache.paimon.operation.FileStoreTestUtils; +import org.apache.paimon.schema.FileSystemSchemaManager; import org.apache.paimon.schema.Schema; import org.apache.paimon.schema.SchemaManager; import org.apache.paimon.schema.TableSchema; @@ -201,7 +202,7 @@ private TestFileStore createStore(TestKeyValueGenerator.GeneratorMode mode, int throw new UnsupportedOperationException("Unsupported generator mode: " + mode); } - SchemaManager schemaManager = new SchemaManager(fileIO, new Path(root)); + SchemaManager schemaManager = new FileSystemSchemaManager(fileIO, new Path(root)); TableSchema tableSchema = schemaManager.createTable( new Schema( diff --git a/paimon-flink/paimon-flink-cdc/src/main/java/org/apache/paimon/flink/sink/cdc/CdcSinkBuilder.java b/paimon-flink/paimon-flink-cdc/src/main/java/org/apache/paimon/flink/sink/cdc/CdcSinkBuilder.java index 3ae0deb5481c..ac6b5a156f41 100644 --- a/paimon-flink/paimon-flink-cdc/src/main/java/org/apache/paimon/flink/sink/cdc/CdcSinkBuilder.java +++ b/paimon-flink/paimon-flink-cdc/src/main/java/org/apache/paimon/flink/sink/cdc/CdcSinkBuilder.java @@ -23,7 +23,7 @@ import org.apache.paimon.catalog.Identifier; import org.apache.paimon.flink.action.cdc.TypeMapping; import org.apache.paimon.flink.utils.SingleOutputStreamOperatorUtils; -import org.apache.paimon.schema.SchemaManager; +import org.apache.paimon.schema.FileSystemSchemaManager; import org.apache.paimon.table.BucketMode; import org.apache.paimon.table.FileStoreTable; import org.apache.paimon.table.Table; @@ -121,7 +121,8 @@ public DataStreamSink build() { parsed, CdcParsingProcessFunction.SCHEMA_CHANGE_OUTPUT_TAG) .process( new UpdatedDataFieldsProcessFunction( - new SchemaManager(dataTable.fileIO(), dataTable.location()), + new FileSystemSchemaManager( + dataTable.fileIO(), dataTable.location()), identifier, catalogLoader, typeMapping)) diff --git a/paimon-flink/paimon-flink-cdc/src/main/java/org/apache/paimon/flink/sink/cdc/FlinkCdcSyncDatabaseSinkBuilder.java b/paimon-flink/paimon-flink-cdc/src/main/java/org/apache/paimon/flink/sink/cdc/FlinkCdcSyncDatabaseSinkBuilder.java index f5cd33f01929..0480db51f6a3 100644 --- a/paimon-flink/paimon-flink-cdc/src/main/java/org/apache/paimon/flink/sink/cdc/FlinkCdcSyncDatabaseSinkBuilder.java +++ b/paimon-flink/paimon-flink-cdc/src/main/java/org/apache/paimon/flink/sink/cdc/FlinkCdcSyncDatabaseSinkBuilder.java @@ -28,7 +28,7 @@ import org.apache.paimon.flink.utils.SingleOutputStreamOperatorUtils; import org.apache.paimon.options.MemorySize; import org.apache.paimon.options.Options; -import org.apache.paimon.schema.SchemaManager; +import org.apache.paimon.schema.FileSystemSchemaManager; import org.apache.paimon.table.BucketMode; import org.apache.paimon.table.FileStoreTable; import org.apache.paimon.utils.Preconditions; @@ -239,7 +239,8 @@ private void buildDividedCdcSink() { table.name())) .process( new UpdatedDataFieldsProcessFunction( - new SchemaManager(table.fileIO(), table.location()), + new FileSystemSchemaManager( + table.fileIO(), table.location()), Identifier.create(database, table.name()), catalogLoader, typeMapping)) diff --git a/paimon-flink/paimon-flink-cdc/src/main/java/org/apache/paimon/flink/sink/cdc/MultiTableUpdatedDataFieldsProcessFunction.java b/paimon-flink/paimon-flink-cdc/src/main/java/org/apache/paimon/flink/sink/cdc/MultiTableUpdatedDataFieldsProcessFunction.java index d10f6d9f5e1e..3e7e3df827de 100644 --- a/paimon-flink/paimon-flink-cdc/src/main/java/org/apache/paimon/flink/sink/cdc/MultiTableUpdatedDataFieldsProcessFunction.java +++ b/paimon-flink/paimon-flink-cdc/src/main/java/org/apache/paimon/flink/sink/cdc/MultiTableUpdatedDataFieldsProcessFunction.java @@ -22,6 +22,7 @@ import org.apache.paimon.catalog.CatalogLoader; import org.apache.paimon.catalog.Identifier; import org.apache.paimon.flink.action.cdc.TypeMapping; +import org.apache.paimon.schema.FileSystemSchemaManager; import org.apache.paimon.schema.SchemaChange; import org.apache.paimon.schema.SchemaManager; import org.apache.paimon.table.FileStoreTable; @@ -78,7 +79,7 @@ public void processElement( } catch (Catalog.TableNotExistException e) { return null; } - return new SchemaManager(table.fileIO(), table.location()); + return new FileSystemSchemaManager(table.fileIO(), table.location()); }); if (Objects.isNull(schemaManager)) { LOG.error("Failed to get schema manager for table " + tableId); diff --git a/paimon-flink/paimon-flink-cdc/src/test/java/org/apache/paimon/flink/action/cdc/SchemaEvolutionTest.java b/paimon-flink/paimon-flink-cdc/src/test/java/org/apache/paimon/flink/action/cdc/SchemaEvolutionTest.java index eca731cb0069..38fa344e073c 100644 --- a/paimon-flink/paimon-flink-cdc/src/test/java/org/apache/paimon/flink/action/cdc/SchemaEvolutionTest.java +++ b/paimon-flink/paimon-flink-cdc/src/test/java/org/apache/paimon/flink/action/cdc/SchemaEvolutionTest.java @@ -28,8 +28,8 @@ import org.apache.paimon.fs.Path; import org.apache.paimon.fs.local.LocalFileIO; import org.apache.paimon.options.Options; +import org.apache.paimon.schema.FileSystemSchemaManager; import org.apache.paimon.schema.Schema; -import org.apache.paimon.schema.SchemaManager; import org.apache.paimon.schema.SchemaUtils; import org.apache.paimon.schema.TableSchema; import org.apache.paimon.table.FileStoreTable; @@ -201,7 +201,7 @@ public void before() throws Exception { .option(CoreOptions.SEQUENCE_FIELD.key(), "col1") .build(); TableSchema tableSchema = - SchemaUtils.forceCommit(new SchemaManager(fileIO, tablePath), schema); + SchemaUtils.forceCommit(new FileSystemSchemaManager(fileIO, tablePath), schema); table = FileStoreTableFactory.create(LocalFileIO.create(), tablePath, tableSchema); } @@ -219,7 +219,7 @@ public void testSchemaCompatibleTypeWidening() throws Exception { .primaryKey("id") .build(); TableSchema tableSchema = - SchemaUtils.forceCommit(new SchemaManager(fileIO, tablePath), baseSchema); + SchemaUtils.forceCommit(new FileSystemSchemaManager(fileIO, tablePath), baseSchema); // Paimon INT, source BIGINT: Paimon can evolve INT -> BIGINT, compatible. List bigintFields = @@ -262,7 +262,8 @@ public void testSchemaEvolution() throws Exception { upDataFieldStream .process( new UpdatedDataFieldsProcessFunction( - new SchemaManager(table.fileIO(), table.location()), + new FileSystemSchemaManager( + table.fileIO(), table.location()), identifier, catalogLoader, TypeMapping.defaultMapping())) diff --git a/paimon-flink/paimon-flink-cdc/src/test/java/org/apache/paimon/flink/action/cdc/mysql/MySqlSyncTableActionITCase.java b/paimon-flink/paimon-flink-cdc/src/test/java/org/apache/paimon/flink/action/cdc/mysql/MySqlSyncTableActionITCase.java index 065b9262049b..be2513241dd4 100644 --- a/paimon-flink/paimon-flink-cdc/src/test/java/org/apache/paimon/flink/action/cdc/mysql/MySqlSyncTableActionITCase.java +++ b/paimon-flink/paimon-flink-cdc/src/test/java/org/apache/paimon/flink/action/cdc/mysql/MySqlSyncTableActionITCase.java @@ -21,6 +21,7 @@ import org.apache.paimon.CoreOptions; import org.apache.paimon.flink.action.cdc.TypeMapping; import org.apache.paimon.options.CatalogOptions; +import org.apache.paimon.schema.FileSystemSchemaManager; import org.apache.paimon.schema.Schema; import org.apache.paimon.schema.SchemaChange; import org.apache.paimon.schema.SchemaManager; @@ -732,7 +733,8 @@ private void testAllTypesImpl(Statement statement) throws Exception { waitForResult(newExpected, table, newRowType, Arrays.asList("pt", "_id")); } finally { statement.executeUpdate("ALTER TABLE all_types_table DROP COLUMN v"); - SchemaManager schemaManager = new SchemaManager(table.fileIO(), table.location()); + SchemaManager schemaManager = + new FileSystemSchemaManager(table.fileIO(), table.location()); schemaManager.commitChanges(SchemaChange.dropColumn("v")); } } diff --git a/paimon-flink/paimon-flink-cdc/src/test/java/org/apache/paimon/flink/sink/cdc/CdcRecordChannelComputerTest.java b/paimon-flink/paimon-flink-cdc/src/test/java/org/apache/paimon/flink/sink/cdc/CdcRecordChannelComputerTest.java index 8271ad18751c..92647016e8a1 100644 --- a/paimon-flink/paimon-flink-cdc/src/test/java/org/apache/paimon/flink/sink/cdc/CdcRecordChannelComputerTest.java +++ b/paimon-flink/paimon-flink-cdc/src/test/java/org/apache/paimon/flink/sink/cdc/CdcRecordChannelComputerTest.java @@ -23,6 +23,7 @@ import org.apache.paimon.fs.Path; import org.apache.paimon.fs.local.LocalFileIO; import org.apache.paimon.options.Options; +import org.apache.paimon.schema.FileSystemSchemaManager; import org.apache.paimon.schema.Schema; import org.apache.paimon.schema.SchemaManager; import org.apache.paimon.schema.TableSchema; @@ -61,7 +62,7 @@ public void testSchemaWithPartition() throws Exception { options.set(CoreOptions.BUCKET, random.nextInt(1, 5)); SchemaManager schemaManager = - new SchemaManager(LocalFileIO.create(), new Path(tempDir.toString())); + new FileSystemSchemaManager(LocalFileIO.create(), new Path(tempDir.toString())); TableSchema schema = schemaManager.createTable( new Schema( @@ -96,7 +97,7 @@ public void testSchemaNoPartition() throws Exception { options.set(CoreOptions.BUCKET, random.nextInt(1, 5)); SchemaManager schemaManager = - new SchemaManager(LocalFileIO.create(), new Path(tempDir.toString())); + new FileSystemSchemaManager(LocalFileIO.create(), new Path(tempDir.toString())); TableSchema schema = schemaManager.createTable( new Schema( diff --git a/paimon-flink/paimon-flink-cdc/src/test/java/org/apache/paimon/flink/sink/cdc/CdcRecordKeyAndBucketExtractorTest.java b/paimon-flink/paimon-flink-cdc/src/test/java/org/apache/paimon/flink/sink/cdc/CdcRecordKeyAndBucketExtractorTest.java index 802a3ea9d4cf..f61fa5c611e7 100644 --- a/paimon-flink/paimon-flink-cdc/src/test/java/org/apache/paimon/flink/sink/cdc/CdcRecordKeyAndBucketExtractorTest.java +++ b/paimon-flink/paimon-flink-cdc/src/test/java/org/apache/paimon/flink/sink/cdc/CdcRecordKeyAndBucketExtractorTest.java @@ -21,8 +21,8 @@ import org.apache.paimon.flink.sink.RowDataKeyAndBucketExtractor; import org.apache.paimon.fs.Path; import org.apache.paimon.fs.local.LocalFileIO; +import org.apache.paimon.schema.FileSystemSchemaManager; import org.apache.paimon.schema.Schema; -import org.apache.paimon.schema.SchemaManager; import org.apache.paimon.schema.SchemaUtils; import org.apache.paimon.schema.TableSchema; import org.apache.paimon.types.DataType; @@ -180,7 +180,7 @@ public void testEmptyPartition() throws Exception { private TableSchema createTableSchema() throws Exception { return SchemaUtils.forceCommit( - new SchemaManager(LocalFileIO.create(), new Path(tempDir.toString())), + new FileSystemSchemaManager(LocalFileIO.create(), new Path(tempDir.toString())), new Schema( ROW_TYPE.getFields(), Arrays.asList("pt1", "pt2"), diff --git a/paimon-flink/paimon-flink-cdc/src/test/java/org/apache/paimon/flink/sink/cdc/CdcRecordStoreMultiWriteOperatorTest.java b/paimon-flink/paimon-flink-cdc/src/test/java/org/apache/paimon/flink/sink/cdc/CdcRecordStoreMultiWriteOperatorTest.java index ee64162ad910..940d1559a21d 100644 --- a/paimon-flink/paimon-flink-cdc/src/test/java/org/apache/paimon/flink/sink/cdc/CdcRecordStoreMultiWriteOperatorTest.java +++ b/paimon-flink/paimon-flink-cdc/src/test/java/org/apache/paimon/flink/sink/cdc/CdcRecordStoreMultiWriteOperatorTest.java @@ -32,6 +32,7 @@ import org.apache.paimon.operation.AbstractFileStoreWrite; import org.apache.paimon.options.CatalogOptions; import org.apache.paimon.options.Options; +import org.apache.paimon.schema.FileSystemSchemaManager; import org.apache.paimon.schema.Schema; import org.apache.paimon.schema.SchemaChange; import org.apache.paimon.schema.SchemaManager; @@ -331,7 +332,7 @@ public void testSingleTableAddColumn() throws Exception { actual = runner.poll(1); assertThat(actual).isNull(); - SchemaManager schemaManager = new SchemaManager(table.fileIO(), table.location()); + SchemaManager schemaManager = new FileSystemSchemaManager(table.fileIO(), table.location()); schemaManager.commitChanges(SchemaChange.addColumn("v2", DataTypes.INT())); actual = runner.take(); assertThat(actual).isEqualTo(expected); @@ -398,7 +399,7 @@ public void testSingleTableUpdateColumnType() throws Exception { actual = runner.poll(1); assertThat(actual).isNull(); - SchemaManager schemaManager = new SchemaManager(table.fileIO(), table.location()); + SchemaManager schemaManager = new FileSystemSchemaManager(table.fileIO(), table.location()); schemaManager.commitChanges(SchemaChange.updateColumnType("v1", DataTypes.BIGINT())); actual = runner.take(); assertThat(actual).isEqualTo(expected); @@ -528,7 +529,7 @@ public void testMultiTableUpdateColumnType() throws Exception { actual = runner.poll(1); assertThat(actual).isNull(); - schemaManager = new SchemaManager(table1.fileIO(), table1.location()); + schemaManager = new FileSystemSchemaManager(table1.fileIO(), table1.location()); schemaManager.commitChanges(SchemaChange.addColumn("v2", DataTypes.STRING())); actual = runner.take(); assertThat(actual).isEqualTo(expected); @@ -547,7 +548,7 @@ public void testMultiTableUpdateColumnType() throws Exception { actual = runner.poll(1); assertThat(actual).isNull(); - schemaManager = new SchemaManager(table2.fileIO(), table2.location()); + schemaManager = new FileSystemSchemaManager(table2.fileIO(), table2.location()); schemaManager.commitChanges(SchemaChange.updateColumnType("v1", DataTypes.BIGINT())); actual = runner.take(); assertThat(actual).isEqualTo(expected); @@ -568,7 +569,7 @@ public void testMultiTableUpdateColumnType() throws Exception { actual = runner.poll(1); assertThat(actual).isNull(); - schemaManager = new SchemaManager(table2.fileIO(), table2.location()); + schemaManager = new FileSystemSchemaManager(table2.fileIO(), table2.location()); schemaManager.commitChanges(SchemaChange.updateColumnType("v2", DataTypes.DOUBLE())); actual = runner.take(); assertThat(actual).isEqualTo(expected); @@ -588,7 +589,7 @@ public void testMultiTableUpdateColumnType() throws Exception { actual = runner.poll(1); assertThat(actual).isNull(); - schemaManager = new SchemaManager(table2.fileIO(), table2.location()); + schemaManager = new FileSystemSchemaManager(table2.fileIO(), table2.location()); schemaManager.commitChanges(SchemaChange.updateColumnType("v3", DataTypes.VARCHAR(10))); actual = runner.take(); assertThat(actual).isEqualTo(expected); diff --git a/paimon-flink/paimon-flink-cdc/src/test/java/org/apache/paimon/flink/sink/cdc/CdcRecordStoreWriteOperatorTest.java b/paimon-flink/paimon-flink-cdc/src/test/java/org/apache/paimon/flink/sink/cdc/CdcRecordStoreWriteOperatorTest.java index 7fa0b09de362..3b10d4ef8ae5 100644 --- a/paimon-flink/paimon-flink-cdc/src/test/java/org/apache/paimon/flink/sink/cdc/CdcRecordStoreWriteOperatorTest.java +++ b/paimon-flink/paimon-flink-cdc/src/test/java/org/apache/paimon/flink/sink/cdc/CdcRecordStoreWriteOperatorTest.java @@ -25,6 +25,7 @@ import org.apache.paimon.fs.Path; import org.apache.paimon.fs.local.LocalFileIO; import org.apache.paimon.options.Options; +import org.apache.paimon.schema.FileSystemSchemaManager; import org.apache.paimon.schema.Schema; import org.apache.paimon.schema.SchemaChange; import org.apache.paimon.schema.SchemaManager; @@ -138,7 +139,7 @@ public void testAddColumn() throws Exception { actual = runner.poll(1); assertThat(actual).isNull(); - SchemaManager schemaManager = new SchemaManager(table.fileIO(), table.location()); + SchemaManager schemaManager = new FileSystemSchemaManager(table.fileIO(), table.location()); schemaManager.commitChanges(SchemaChange.addColumn("v2", DataTypes.INT())); actual = runner.take(); assertThat(actual).isEqualTo(expected); @@ -199,7 +200,7 @@ public void testUpdateColumnType() throws Exception { actual = runner.poll(1); assertThat(actual).isNull(); - SchemaManager schemaManager = new SchemaManager(table.fileIO(), table.location()); + SchemaManager schemaManager = new FileSystemSchemaManager(table.fileIO(), table.location()); schemaManager.commitChanges(SchemaChange.updateColumnType("v1", DataTypes.BIGINT())); actual = runner.take(); assertThat(actual).isEqualTo(expected); @@ -401,7 +402,7 @@ private FileStoreTable createFileStoreTable( TableSchema tableSchema = SchemaUtils.forceCommit( - new SchemaManager(LocalFileIO.create(), tablePath), + new FileSystemSchemaManager(LocalFileIO.create(), tablePath), new Schema(rowType.getFields(), partitions, primaryKeys, conf.toMap(), "")); return FileStoreTableFactory.create(LocalFileIO.create(), tablePath, tableSchema); } diff --git a/paimon-flink/paimon-flink-cdc/src/test/java/org/apache/paimon/flink/sink/cdc/FlinkCdcSyncDatabaseSinkITCase.java b/paimon-flink/paimon-flink-cdc/src/test/java/org/apache/paimon/flink/sink/cdc/FlinkCdcSyncDatabaseSinkITCase.java index 4db53f7f8007..db5126b670bd 100644 --- a/paimon-flink/paimon-flink-cdc/src/test/java/org/apache/paimon/flink/sink/cdc/FlinkCdcSyncDatabaseSinkITCase.java +++ b/paimon-flink/paimon-flink-cdc/src/test/java/org/apache/paimon/flink/sink/cdc/FlinkCdcSyncDatabaseSinkITCase.java @@ -32,6 +32,7 @@ import org.apache.paimon.options.MemorySize; import org.apache.paimon.options.Options; import org.apache.paimon.reader.RecordReaderIterator; +import org.apache.paimon.schema.FileSystemSchemaManager; import org.apache.paimon.schema.Schema; import org.apache.paimon.schema.SchemaManager; import org.apache.paimon.schema.SchemaUtils; @@ -212,7 +213,8 @@ private void innerTestRandomCdcEvents(Supplier bucket, boolean unawareB for (int i = 0; i < numTables; i++) { FileStoreTable table = fileStoreTables.get(i).copyWithLatestSchema(); - SchemaManager schemaManager = new SchemaManager(table.fileIO(), table.location()); + SchemaManager schemaManager = + new FileSystemSchemaManager(table.fileIO(), table.location()); TableSchema schema = schemaManager.latest().get(); ReadBuilder readBuilder = table.newReadBuilder(); @@ -244,7 +246,7 @@ private FileStoreTable createFileStoreTable( TableSchema tableSchema = SchemaUtils.forceCommit( - new SchemaManager(fileIO, tablePath), + new FileSystemSchemaManager(fileIO, tablePath), new Schema(rowType.getFields(), partitions, primaryKeys, conf.toMap(), "")); return FileStoreTableFactory.create(fileIO, tablePath, tableSchema); } diff --git a/paimon-flink/paimon-flink-cdc/src/test/java/org/apache/paimon/flink/sink/cdc/FlinkCdcSyncTableSinkITCase.java b/paimon-flink/paimon-flink-cdc/src/test/java/org/apache/paimon/flink/sink/cdc/FlinkCdcSyncTableSinkITCase.java index c7e66452a95d..7ac7eeed6ed4 100644 --- a/paimon-flink/paimon-flink-cdc/src/test/java/org/apache/paimon/flink/sink/cdc/FlinkCdcSyncTableSinkITCase.java +++ b/paimon-flink/paimon-flink-cdc/src/test/java/org/apache/paimon/flink/sink/cdc/FlinkCdcSyncTableSinkITCase.java @@ -32,6 +32,7 @@ import org.apache.paimon.options.MemorySize; import org.apache.paimon.options.Options; import org.apache.paimon.reader.RecordReaderIterator; +import org.apache.paimon.schema.FileSystemSchemaManager; import org.apache.paimon.schema.Schema; import org.apache.paimon.schema.SchemaManager; import org.apache.paimon.schema.SchemaUtils; @@ -206,7 +207,7 @@ private void innerTestRandomCdcEvents( } table = table.copyWithLatestSchema(); - SchemaManager schemaManager = new SchemaManager(table.fileIO(), table.location()); + SchemaManager schemaManager = new FileSystemSchemaManager(table.fileIO(), table.location()); TableSchema schema = schemaManager.latest().get(); ReadBuilder readBuilder = table.newReadBuilder(); @@ -237,7 +238,7 @@ private FileStoreTable createFileStoreTable( TableSchema tableSchema = SchemaUtils.forceCommit( - new SchemaManager(fileIO, tablePath), + new FileSystemSchemaManager(fileIO, tablePath), new Schema(rowType.getFields(), partitions, primaryKeys, conf.toMap(), "")); return FileStoreTableFactory.create(fileIO, tablePath, tableSchema); } diff --git a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/FlinkTableFactory.java b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/FlinkTableFactory.java index 3415dbaa3f23..e75c48aeb7c0 100644 --- a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/FlinkTableFactory.java +++ b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/FlinkTableFactory.java @@ -22,6 +22,7 @@ import org.apache.paimon.fs.FileIO; import org.apache.paimon.fs.Path; import org.apache.paimon.options.Options; +import org.apache.paimon.schema.FileSystemSchemaManager; import org.apache.paimon.schema.SchemaManager; import org.apache.flink.table.catalog.ResolvedCatalogTable; @@ -69,7 +70,7 @@ private void createTableIfNeeded(Context context) { Path tablePath = CoreOptions.path(table.getOptions()); String branch = CoreOptions.branch(table.getOptions()); SchemaManager schemaManager = - new SchemaManager( + new FileSystemSchemaManager( FileIO.get(tablePath, createCatalogContext(context)), tablePath, branch); diff --git a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/copy/CopyFilesUtil.java b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/copy/CopyFilesUtil.java index a3adec13b8ef..34acde04e84c 100644 --- a/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/copy/CopyFilesUtil.java +++ b/paimon-flink/paimon-flink-common/src/main/java/org/apache/paimon/flink/copy/CopyFilesUtil.java @@ -30,6 +30,7 @@ import org.apache.paimon.manifest.ManifestFileMeta; import org.apache.paimon.manifest.ManifestList; import org.apache.paimon.manifest.SimpleFileEntry; +import org.apache.paimon.schema.FileSystemSchemaManager; import org.apache.paimon.schema.SchemaManager; import org.apache.paimon.table.FileStoreTable; import org.apache.paimon.table.Table; @@ -71,7 +72,7 @@ public static List getSchemaUsedFilesForSnapshot(FileStoreTable table, lon FileStore store = table.store(); SnapshotManager snapshotManager = store.snapshotManager(); Snapshot snapshot = snapshotManager.tryGetSnapshot(snapshotId); - SchemaManager schemaManager = new SchemaManager(table.fileIO(), table.location()); + SchemaManager schemaManager = new FileSystemSchemaManager(table.fileIO(), table.location()); IndexFileHandler indexFileHandler = store.newIndexFileHandler(); List fileList = new ArrayList<>(); if (snapshot != null) { diff --git a/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/BranchSqlITCase.java b/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/BranchSqlITCase.java index e2e52ae3a379..557e05ecbe9a 100644 --- a/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/BranchSqlITCase.java +++ b/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/BranchSqlITCase.java @@ -18,6 +18,7 @@ package org.apache.paimon.flink; +import org.apache.paimon.schema.FileSystemSchemaManager; import org.apache.paimon.schema.SchemaManager; import org.apache.paimon.table.FileStoreTable; import org.apache.paimon.utils.BlockingIterator; @@ -823,7 +824,8 @@ public void testReadBranchTableWithMultiSchemaIds() throws Exception { sql("CALL sys.create_branch('default.T', 'test', 'tag1')"); FileStoreTable table = paimonTable("T"); - SchemaManager schemaManager = new SchemaManager(table.fileIO(), table.location(), "test"); + SchemaManager schemaManager = + new FileSystemSchemaManager(table.fileIO(), table.location(), "test"); List schemaIds = schemaManager.listAllIds(); assertThat(schemaIds.size()).isEqualTo(2); diff --git a/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/ChangelogModeTest.java b/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/ChangelogModeTest.java index eb7536ae12ab..a652efa63b1d 100644 --- a/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/ChangelogModeTest.java +++ b/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/ChangelogModeTest.java @@ -24,8 +24,8 @@ import org.apache.paimon.fs.Path; import org.apache.paimon.fs.local.LocalFileIO; import org.apache.paimon.options.Options; +import org.apache.paimon.schema.FileSystemSchemaManager; import org.apache.paimon.schema.Schema; -import org.apache.paimon.schema.SchemaManager; import org.apache.paimon.table.FileStoreTable; import org.apache.paimon.table.FileStoreTableFactory; import org.apache.paimon.types.IntType; @@ -62,7 +62,7 @@ public void beforeEach() { } private FileStoreTable createTable(Options options) throws Exception { - new SchemaManager(LocalFileIO.create(), path) + new FileSystemSchemaManager(LocalFileIO.create(), path) .createTable( new Schema( RowType.of(new IntType(), new IntType()).getFields(), diff --git a/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/FileStoreITCase.java b/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/FileStoreITCase.java index 0d9f512fbee9..1c7015168b89 100644 --- a/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/FileStoreITCase.java +++ b/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/FileStoreITCase.java @@ -28,8 +28,8 @@ import org.apache.paimon.fs.Path; import org.apache.paimon.fs.local.LocalFileIO; import org.apache.paimon.options.Options; +import org.apache.paimon.schema.FileSystemSchemaManager; import org.apache.paimon.schema.Schema; -import org.apache.paimon.schema.SchemaManager; import org.apache.paimon.table.FileStoreTable; import org.apache.paimon.table.FileStoreTableFactory; import org.apache.paimon.utils.BlockingIterator; @@ -503,7 +503,8 @@ public static FileStoreTable buildFileStoreTable( ""); return retryArtificialException( () -> { - new SchemaManager(LocalFileIO.create(), tablePath, branch).createTable(schema); + new FileSystemSchemaManager(LocalFileIO.create(), tablePath, branch) + .createTable(schema); return FileStoreTableFactory.create(LocalFileIO.create(), options); }); } diff --git a/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/FlinkBatchJobPartitionMarkdoneITCase.java b/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/FlinkBatchJobPartitionMarkdoneITCase.java index 340213aeae1b..e89f5ba84c82 100644 --- a/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/FlinkBatchJobPartitionMarkdoneITCase.java +++ b/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/FlinkBatchJobPartitionMarkdoneITCase.java @@ -24,8 +24,8 @@ import org.apache.paimon.fs.local.LocalFileIO; import org.apache.paimon.options.Options; import org.apache.paimon.partition.file.SuccessFile; +import org.apache.paimon.schema.FileSystemSchemaManager; import org.apache.paimon.schema.Schema; -import org.apache.paimon.schema.SchemaManager; import org.apache.paimon.table.FileStoreTable; import org.apache.paimon.table.FileStoreTableFactory; @@ -180,7 +180,8 @@ private FileStoreTable buildFileStoreTable(int[] partitions, int[] primaryKey) ""); return retryArtificialException( () -> { - new SchemaManager(LocalFileIO.create(), tablePath).createTable(schema); + new FileSystemSchemaManager(LocalFileIO.create(), tablePath) + .createTable(schema); return FileStoreTableFactory.create(LocalFileIO.create(), options); }); } diff --git a/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/FlinkEndInputPartitionExpireITCase.java b/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/FlinkEndInputPartitionExpireITCase.java index 63eebcd7d1f2..e800a9282004 100644 --- a/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/FlinkEndInputPartitionExpireITCase.java +++ b/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/FlinkEndInputPartitionExpireITCase.java @@ -24,8 +24,8 @@ import org.apache.paimon.fs.Path; import org.apache.paimon.fs.local.LocalFileIO; import org.apache.paimon.options.Options; +import org.apache.paimon.schema.FileSystemSchemaManager; import org.apache.paimon.schema.Schema; -import org.apache.paimon.schema.SchemaManager; import org.apache.paimon.table.FileStoreTable; import org.apache.paimon.table.FileStoreTableFactory; @@ -158,7 +158,8 @@ private FileStoreTable buildFileStoreTable(int[] partitions, int[] primaryKey) ""); return retryArtificialException( () -> { - new SchemaManager(LocalFileIO.create(), tablePath).createTable(schema); + new FileSystemSchemaManager(LocalFileIO.create(), tablePath) + .createTable(schema); return FileStoreTableFactory.create(LocalFileIO.create(), options); }); } diff --git a/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/FlinkEndInputWatermarkITCase.java b/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/FlinkEndInputWatermarkITCase.java index c85b570dd53a..deb8b8a7c98e 100644 --- a/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/FlinkEndInputWatermarkITCase.java +++ b/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/FlinkEndInputWatermarkITCase.java @@ -23,8 +23,8 @@ import org.apache.paimon.fs.Path; import org.apache.paimon.fs.local.LocalFileIO; import org.apache.paimon.options.Options; +import org.apache.paimon.schema.FileSystemSchemaManager; import org.apache.paimon.schema.Schema; -import org.apache.paimon.schema.SchemaManager; import org.apache.paimon.table.FileStoreTable; import org.apache.paimon.table.FileStoreTableFactory; @@ -175,7 +175,8 @@ private FileStoreTable buildFileStoreTable(int[] partitions, int[] primaryKey) ""); return retryArtificialException( () -> { - new SchemaManager(LocalFileIO.create(), tablePath).createTable(schema); + new FileSystemSchemaManager(LocalFileIO.create(), tablePath) + .createTable(schema); return FileStoreTableFactory.create(LocalFileIO.create(), options); }); } diff --git a/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/ReadWriteTableITCase.java b/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/ReadWriteTableITCase.java index 940dd95af0b0..def5be553ae8 100644 --- a/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/ReadWriteTableITCase.java +++ b/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/ReadWriteTableITCase.java @@ -23,6 +23,7 @@ import org.apache.paimon.flink.util.AbstractTestBase; import org.apache.paimon.fs.Path; import org.apache.paimon.fs.local.LocalFileIO; +import org.apache.paimon.schema.FileSystemSchemaManager; import org.apache.paimon.schema.Schema; import org.apache.paimon.schema.SchemaManager; import org.apache.paimon.schema.TableSchema; @@ -1468,7 +1469,8 @@ public void testCleanedSchemaOptions() { // validate schema options SchemaManager schemaManager = - new SchemaManager(LocalFileIO.create(), new Path(warehouse, "default.db/T")); + new FileSystemSchemaManager( + LocalFileIO.create(), new Path(warehouse, "default.db/T")); TableSchema schema = schemaManager.latest().get(); Map expected = new HashMap<>(); // metadata column @@ -1530,7 +1532,8 @@ public void testReadFromOldStyleSchemaOptions() throws Exception { .build(); SchemaManager schemaManager = - new SchemaManager(LocalFileIO.create(), new Path(warehouse, "default.db/T")); + new FileSystemSchemaManager( + LocalFileIO.create(), new Path(warehouse, "default.db/T")); schemaManager.createTable(schema); validateSchemaOptionResult(); @@ -1920,7 +1923,7 @@ private void testSinkParallelism( Path path = CoreOptions.path(context.getCatalogTable().getOptions()); LocalFileIO.create().mkdirs(path); // update schema - new SchemaManager(LocalFileIO.create(), path) + new FileSystemSchemaManager(LocalFileIO.create(), path) .createTable(FlinkCatalog.fromCatalogTable(context.getCatalogTable())); DynamicTableSink tableSink = diff --git a/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/RescaleBucketITCase.java b/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/RescaleBucketITCase.java index 9b981102ec56..e3e8656fa981 100644 --- a/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/RescaleBucketITCase.java +++ b/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/RescaleBucketITCase.java @@ -20,6 +20,7 @@ import org.apache.paimon.Snapshot; import org.apache.paimon.fs.local.LocalFileIO; +import org.apache.paimon.schema.FileSystemSchemaManager; import org.apache.paimon.schema.SchemaManager; import org.apache.paimon.schema.TableSchema; import org.apache.paimon.utils.SnapshotManager; @@ -74,7 +75,7 @@ public void testSuspendAndRecoverAfterRescaleOverwrite() throws Exception { "CREATE TABLE IF NOT EXISTS `T3` (f0 INT) WITH ('bucket' = '2', 'bucket-key' = 'f0')", "CREATE TABLE IF NOT EXISTS `T4` (f0 INT)")); SchemaManager schemaManager = - new SchemaManager(LocalFileIO.create(), getTableDirectory("T3")); + new FileSystemSchemaManager(LocalFileIO.create(), getTableDirectory("T3")); assertLatestSchema(schemaManager, 0L, 2); String streamSql = @@ -187,7 +188,7 @@ private void innerTest(String catalogName, String tableName) { assertThat(snapshot).isNotNull(); SchemaManager schemaManager = - new SchemaManager(LocalFileIO.create(), getTableDirectory(tableName)); + new FileSystemSchemaManager(LocalFileIO.create(), getTableDirectory(tableName)); assertSnapshotSchema(schemaManager, snapshot.schemaId(), 0L, 2); // for managed table schema id remains unchanged, for catalog table id increase from 0 to 1 diff --git a/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/action/RemoveOrphanFilesActionITCaseBase.java b/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/action/RemoveOrphanFilesActionITCaseBase.java index e54fd5c66205..d65a2713a026 100644 --- a/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/action/RemoveOrphanFilesActionITCaseBase.java +++ b/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/action/RemoveOrphanFilesActionITCaseBase.java @@ -25,6 +25,7 @@ import org.apache.paimon.fs.FileStatus; import org.apache.paimon.fs.Path; import org.apache.paimon.options.Options; +import org.apache.paimon.schema.FileSystemSchemaManager; import org.apache.paimon.schema.SchemaChange; import org.apache.paimon.schema.SchemaManager; import org.apache.paimon.schema.TableSchema; @@ -274,7 +275,8 @@ public void testCleanWithBranch(boolean isNamedArgument) throws Exception { // create first branch and write some data table.createBranch("br"); - SchemaManager schemaManager = new SchemaManager(table.fileIO(), table.location(), "br"); + SchemaManager schemaManager = + new FileSystemSchemaManager(table.fileIO(), table.location(), "br"); TableSchema branchSchema = schemaManager.commitChanges(SchemaChange.addColumn("v2", DataTypes.INT())); Options branchOptions = new Options(branchSchema.options()); diff --git a/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/lineage/LineageUtilsTest.java b/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/lineage/LineageUtilsTest.java index b8ab16ec8116..8bdcefdb140a 100644 --- a/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/lineage/LineageUtilsTest.java +++ b/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/lineage/LineageUtilsTest.java @@ -28,8 +28,8 @@ import org.apache.paimon.fs.Path; import org.apache.paimon.fs.local.LocalFileIO; import org.apache.paimon.options.Options; +import org.apache.paimon.schema.FileSystemSchemaManager; import org.apache.paimon.schema.Schema; -import org.apache.paimon.schema.SchemaManager; import org.apache.paimon.table.CatalogEnvironment; import org.apache.paimon.table.FileStoreTable; import org.apache.paimon.table.FileStoreTableFactory; @@ -73,7 +73,7 @@ private FileStoreTable createTable( java.util.List partitionKeys, java.util.List primaryKeys) throws Exception { - new SchemaManager(LocalFileIO.create(), tablePath) + new FileSystemSchemaManager(LocalFileIO.create(), tablePath) .createTable( new Schema( RowType.of(new IntType(), new VarCharType(100), new IntType()) diff --git a/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/lookup/DynamicPartitionLevelLoaderTest.java b/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/lookup/DynamicPartitionLevelLoaderTest.java index 915efe3d4020..1a245755f012 100644 --- a/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/lookup/DynamicPartitionLevelLoaderTest.java +++ b/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/lookup/DynamicPartitionLevelLoaderTest.java @@ -24,6 +24,7 @@ import org.apache.paimon.data.GenericRow; import org.apache.paimon.flink.FlinkConnectorOptions; import org.apache.paimon.options.Options; +import org.apache.paimon.schema.FileSystemSchemaManager; import org.apache.paimon.schema.Schema; import org.apache.paimon.schema.SchemaManager; import org.apache.paimon.schema.TableSchema; @@ -217,7 +218,7 @@ public void testScanPartitionsOnNonPartitionedTable() throws Exception { private FileStoreTable createFileStoreTable( List partitionKeys, List primaryKeys, Map customOptions) throws Exception { - SchemaManager schemaManager = new SchemaManager(fileIO, tablePath); + SchemaManager schemaManager = new FileSystemSchemaManager(fileIO, tablePath); Options conf = new Options(customOptions); conf.set(CoreOptions.BUCKET, 2); conf.set(CoreOptions.LOOKUP_CONTINUOUS_DISCOVERY_INTERVAL, Duration.ofSeconds(1)); diff --git a/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/lookup/DynamicPartitionNumberLoaderTest.java b/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/lookup/DynamicPartitionNumberLoaderTest.java index 6c84cca6efec..405b6b9ad4f4 100644 --- a/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/lookup/DynamicPartitionNumberLoaderTest.java +++ b/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/lookup/DynamicPartitionNumberLoaderTest.java @@ -23,6 +23,7 @@ import org.apache.paimon.data.GenericRow; import org.apache.paimon.flink.FlinkConnectorOptions; import org.apache.paimon.options.Options; +import org.apache.paimon.schema.FileSystemSchemaManager; import org.apache.paimon.schema.Schema; import org.apache.paimon.schema.SchemaManager; import org.apache.paimon.schema.TableSchema; @@ -128,7 +129,7 @@ private List partitions(DynamicPartitionLoader loader) { private FileStoreTable createFileStoreTable() throws Exception { org.apache.paimon.fs.Path tablePath = new org.apache.paimon.fs.Path(tempDir.toString()); - SchemaManager schemaManager = new SchemaManager(fileIO, tablePath); + SchemaManager schemaManager = new FileSystemSchemaManager(fileIO, tablePath); Options options = new Options(); options.set(CoreOptions.BUCKET, 1); options.set(FlinkConnectorOptions.LOOKUP_DYNAMIC_PARTITION_REFRESH_INTERVAL, Duration.ZERO); diff --git a/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/lookup/FileStoreLookupFunctionTest.java b/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/lookup/FileStoreLookupFunctionTest.java index 2db4d2f3bea8..5296cd488d27 100644 --- a/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/lookup/FileStoreLookupFunctionTest.java +++ b/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/lookup/FileStoreLookupFunctionTest.java @@ -28,6 +28,7 @@ import org.apache.paimon.flink.lookup.PrimaryKeyPartialLookupTable.QueryExecutor; import org.apache.paimon.flink.lookup.PrimaryKeyPartialLookupTable.RemoteQueryExecutor; import org.apache.paimon.options.Options; +import org.apache.paimon.schema.FileSystemSchemaManager; import org.apache.paimon.schema.Schema; import org.apache.paimon.schema.SchemaManager; import org.apache.paimon.schema.TableSchema; @@ -139,7 +140,7 @@ private FileStoreTable createFileStoreTable( boolean refreshAsync, Integer fullLoadThreshold) throws Exception { - SchemaManager schemaManager = new SchemaManager(fileIO, tablePath); + SchemaManager schemaManager = new FileSystemSchemaManager(fileIO, tablePath); Options conf = new Options(); conf.set(FlinkConnectorOptions.LOOKUP_REFRESH_ASYNC, refreshAsync); conf.set(CoreOptions.BUCKET, 2); @@ -462,7 +463,7 @@ private InternalRow randomRow() { } private FileStoreTable createStringFileStoreTable() throws Exception { - SchemaManager schemaManager = new SchemaManager(fileIO, tablePath); + SchemaManager schemaManager = new FileSystemSchemaManager(fileIO, tablePath); Options conf = new Options(); conf.set(CoreOptions.BUCKET, 2); conf.set(CoreOptions.LOOKUP_CONTINUOUS_DISCOVERY_INTERVAL, Duration.ofSeconds(1)); diff --git a/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/lookup/LookupTableTest.java b/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/lookup/LookupTableTest.java index f3b291b9768d..9a34eb0e1d97 100644 --- a/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/lookup/LookupTableTest.java +++ b/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/lookup/LookupTableTest.java @@ -31,8 +31,8 @@ import org.apache.paimon.fs.local.LocalFileIO; import org.apache.paimon.options.Options; import org.apache.paimon.predicate.PredicateBuilder; +import org.apache.paimon.schema.FileSystemSchemaManager; import org.apache.paimon.schema.Schema; -import org.apache.paimon.schema.SchemaManager; import org.apache.paimon.schema.SchemaUtils; import org.apache.paimon.schema.TableSchema; import org.apache.paimon.table.FileStoreTable; @@ -1106,7 +1106,7 @@ private FileStoreTable createDimTable() throws Exception { .option(CoreOptions.BUCKET_KEY.key(), "pk2") .build(); TableSchema tableSchema = - SchemaUtils.forceCommit(new SchemaManager(fileIO, tablePath), schema); + SchemaUtils.forceCommit(new FileSystemSchemaManager(fileIO, tablePath), schema); return FileStoreTableFactory.create(LocalFileIO.create(), tablePath, tableSchema); } diff --git a/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/sink/CommitterOperatorTestBase.java b/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/sink/CommitterOperatorTestBase.java index a69f8dbd3a35..79fa79ff16c0 100644 --- a/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/sink/CommitterOperatorTestBase.java +++ b/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/sink/CommitterOperatorTestBase.java @@ -25,6 +25,7 @@ import org.apache.paimon.options.Options; import org.apache.paimon.reader.RecordReader; import org.apache.paimon.reader.RecordReaderIterator; +import org.apache.paimon.schema.FileSystemSchemaManager; import org.apache.paimon.schema.Schema; import org.apache.paimon.schema.SchemaManager; import org.apache.paimon.table.FileStoreTable; @@ -102,7 +103,7 @@ protected FileStoreTable createFileStoreTable( conf.setString("bucket", "1"); conf.setString("bucket-key", "a"); setOptions.accept(conf); - SchemaManager schemaManager = new SchemaManager(LocalFileIO.create(), tablePath); + SchemaManager schemaManager = new FileSystemSchemaManager(LocalFileIO.create(), tablePath); schemaManager.createTable( new Schema( ROW_TYPE.getFields(), diff --git a/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/sink/CompactorSinkITCase.java b/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/sink/CompactorSinkITCase.java index 2152a518e852..bebf412f4276 100644 --- a/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/sink/CompactorSinkITCase.java +++ b/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/sink/CompactorSinkITCase.java @@ -27,6 +27,7 @@ import org.apache.paimon.fs.Path; import org.apache.paimon.fs.local.LocalFileIO; import org.apache.paimon.partition.PartitionPredicate; +import org.apache.paimon.schema.FileSystemSchemaManager; import org.apache.paimon.schema.Schema; import org.apache.paimon.schema.SchemaManager; import org.apache.paimon.schema.TableSchema; @@ -203,7 +204,7 @@ private GenericRow rowData(Object... values) { } private FileStoreTable createFileStoreTable() throws Exception { - SchemaManager schemaManager = new SchemaManager(LocalFileIO.create(), tablePath); + SchemaManager schemaManager = new FileSystemSchemaManager(LocalFileIO.create(), tablePath); TableSchema tableSchema = schemaManager.createTable( new Schema( diff --git a/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/sink/FlinkSinkBuilderLineageTest.java b/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/sink/FlinkSinkBuilderLineageTest.java index a7ba7f687985..1b1c872e077a 100644 --- a/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/sink/FlinkSinkBuilderLineageTest.java +++ b/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/sink/FlinkSinkBuilderLineageTest.java @@ -21,8 +21,8 @@ import org.apache.paimon.CoreOptions; import org.apache.paimon.fs.Path; import org.apache.paimon.fs.local.LocalFileIO; +import org.apache.paimon.schema.FileSystemSchemaManager; import org.apache.paimon.schema.Schema; -import org.apache.paimon.schema.SchemaManager; import org.apache.paimon.table.FileStoreTable; import org.apache.paimon.table.FileStoreTableFactory; import org.apache.paimon.types.IntType; @@ -71,7 +71,7 @@ private FileStoreTable createTable() throws Exception { Path tablePath = new Path(temp.toUri().toString()); Map options = new HashMap<>(); options.put(CoreOptions.BUCKET.key(), "-1"); - new SchemaManager(LocalFileIO.create(), tablePath) + new FileSystemSchemaManager(LocalFileIO.create(), tablePath) .createTable( new Schema( RowType.of(new IntType()).getFields(), diff --git a/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/sink/FlinkSinkTest.java b/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/sink/FlinkSinkTest.java index 91110d996218..2bf72fffb561 100644 --- a/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/sink/FlinkSinkTest.java +++ b/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/sink/FlinkSinkTest.java @@ -22,6 +22,7 @@ import org.apache.paimon.flink.FlinkConnectorOptions; import org.apache.paimon.fs.local.LocalFileIO; import org.apache.paimon.options.Options; +import org.apache.paimon.schema.FileSystemSchemaManager; import org.apache.paimon.schema.Schema; import org.apache.paimon.schema.SchemaManager; import org.apache.paimon.table.FileStoreTable; @@ -142,7 +143,7 @@ private FileStoreTable createPrimaryKeyTable() throws Exception { Options conf = new Options(); conf.set(CoreOptions.PATH, tablePath.toString()); conf.setString("bucket", "1"); - SchemaManager schemaManager = new SchemaManager(LocalFileIO.create(), tablePath); + SchemaManager schemaManager = new FileSystemSchemaManager(LocalFileIO.create(), tablePath); schemaManager.createTable( new Schema( ROW_TYPE.getFields(), diff --git a/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/sink/ParallelismConfiguredITCase.java b/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/sink/ParallelismConfiguredITCase.java index fb485ea3b6ce..6ac8f9bdefb2 100644 --- a/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/sink/ParallelismConfiguredITCase.java +++ b/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/sink/ParallelismConfiguredITCase.java @@ -23,8 +23,8 @@ import org.apache.paimon.fs.Path; import org.apache.paimon.fs.local.LocalFileIO; import org.apache.paimon.options.Options; +import org.apache.paimon.schema.FileSystemSchemaManager; import org.apache.paimon.schema.Schema; -import org.apache.paimon.schema.SchemaManager; import org.apache.paimon.table.BucketMode; import org.apache.paimon.table.FileStoreTable; import org.apache.paimon.table.FileStoreTableFactory; @@ -197,7 +197,8 @@ private static FileStoreTable buildFileStoreTable( ""); return retryArtificialException( () -> { - new SchemaManager(LocalFileIO.create(), tablePath).createTable(schema); + new FileSystemSchemaManager(LocalFileIO.create(), tablePath) + .createTable(schema); return FileStoreTableFactory.create(LocalFileIO.create(), options); }); } diff --git a/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/sink/PostponeFixedBucketChannelComputerTest.java b/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/sink/PostponeFixedBucketChannelComputerTest.java index be0525593864..db49618ab630 100644 --- a/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/sink/PostponeFixedBucketChannelComputerTest.java +++ b/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/sink/PostponeFixedBucketChannelComputerTest.java @@ -23,6 +23,7 @@ import org.apache.paimon.data.InternalRow; import org.apache.paimon.fs.Path; import org.apache.paimon.fs.local.LocalFileIO; +import org.apache.paimon.schema.FileSystemSchemaManager; import org.apache.paimon.schema.Schema; import org.apache.paimon.schema.SchemaManager; import org.apache.paimon.schema.TableSchema; @@ -55,7 +56,7 @@ public void testRecordsDistributedAcrossChannels() throws Exception { new String[] {"pt", "k", "v"}); SchemaManager schemaManager = - new SchemaManager(LocalFileIO.create(), new Path(tempDir.toString())); + new FileSystemSchemaManager(LocalFileIO.create(), new Path(tempDir.toString())); TableSchema schema = schemaManager.createTable( new Schema( @@ -96,7 +97,7 @@ public void testNoPartitionDistribution() throws Exception { new String[] {"k", "v"}); SchemaManager schemaManager = - new SchemaManager(LocalFileIO.create(), new Path(tempDir.toString())); + new FileSystemSchemaManager(LocalFileIO.create(), new Path(tempDir.toString())); TableSchema schema = schemaManager.createTable( new Schema( @@ -138,7 +139,7 @@ public void testSameKeyGoesToSameChannel() throws Exception { new String[] {"k", "v"}); SchemaManager schemaManager = - new SchemaManager(LocalFileIO.create(), new Path(tempDir.toString())); + new FileSystemSchemaManager(LocalFileIO.create(), new Path(tempDir.toString())); TableSchema schema = schemaManager.createTable( new Schema( @@ -175,7 +176,7 @@ public void testDefaultBucketNumberRespectsMaxParallelism() throws Exception { new String[] {"pt", "k"}); SchemaManager schemaManager = - new SchemaManager(LocalFileIO.create(), new Path(tempDir.toString())); + new FileSystemSchemaManager(LocalFileIO.create(), new Path(tempDir.toString())); Map options = new HashMap<>(); options.put("bucket", "-2"); options.put("postpone.batch-write-fixed-bucket.max-parallelism", "2"); diff --git a/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/sink/RowDataChannelComputerTest.java b/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/sink/RowDataChannelComputerTest.java index 687407141bf1..f13a89d4a191 100644 --- a/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/sink/RowDataChannelComputerTest.java +++ b/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/sink/RowDataChannelComputerTest.java @@ -23,6 +23,7 @@ import org.apache.paimon.data.InternalRow; import org.apache.paimon.fs.Path; import org.apache.paimon.fs.local.LocalFileIO; +import org.apache.paimon.schema.FileSystemSchemaManager; import org.apache.paimon.schema.Schema; import org.apache.paimon.schema.SchemaManager; import org.apache.paimon.schema.TableSchema; @@ -57,7 +58,7 @@ public void testSchemaWithPartition() throws Exception { new String[] {"pt", "k", "v"}); SchemaManager schemaManager = - new SchemaManager(LocalFileIO.create(), new Path(tempDir.toString())); + new FileSystemSchemaManager(LocalFileIO.create(), new Path(tempDir.toString())); TableSchema schema = schemaManager.createTable( new Schema( @@ -86,7 +87,7 @@ public void testSchemaNoPartition() throws Exception { new String[] {"k", "v"}); SchemaManager schemaManager = - new SchemaManager(LocalFileIO.create(), new Path(tempDir.toString())); + new FileSystemSchemaManager(LocalFileIO.create(), new Path(tempDir.toString())); TableSchema schema = schemaManager.createTable( new Schema( diff --git a/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/sink/WriterOperatorTest.java b/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/sink/WriterOperatorTest.java index 6c48b9379b0a..b7bfcfd5278c 100644 --- a/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/sink/WriterOperatorTest.java +++ b/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/sink/WriterOperatorTest.java @@ -29,6 +29,7 @@ import org.apache.paimon.io.CompactIncrement; import org.apache.paimon.options.Options; import org.apache.paimon.reader.RecordReader; +import org.apache.paimon.schema.FileSystemSchemaManager; import org.apache.paimon.schema.Schema; import org.apache.paimon.schema.SchemaManager; import org.apache.paimon.table.FileStoreTable; @@ -601,7 +602,7 @@ private FileStoreTable createFileStoreTable( RowType rowType, List primaryKeys, List partitionKeys, Options conf) throws Exception { conf.set(CoreOptions.PATH, tablePath.toString()); - SchemaManager schemaManager = new SchemaManager(LocalFileIO.create(), tablePath); + SchemaManager schemaManager = new FileSystemSchemaManager(LocalFileIO.create(), tablePath); schemaManager.createTable( new Schema(rowType.getFields(), partitionKeys, primaryKeys, conf.toMap(), "")); return FileStoreTableFactory.create(LocalFileIO.create(), conf); diff --git a/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/sink/listener/CustomCommitListenerTest.java b/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/sink/listener/CustomCommitListenerTest.java index 7d6441cc7a9b..d133bd36d272 100644 --- a/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/sink/listener/CustomCommitListenerTest.java +++ b/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/sink/listener/CustomCommitListenerTest.java @@ -26,6 +26,7 @@ import org.apache.paimon.fs.Path; import org.apache.paimon.fs.local.LocalFileIO; import org.apache.paimon.manifest.ManifestCommittable; +import org.apache.paimon.schema.FileSystemSchemaManager; import org.apache.paimon.schema.Schema; import org.apache.paimon.schema.SchemaManager; import org.apache.paimon.table.FileStoreTable; @@ -60,7 +61,7 @@ public class CustomCommitListenerTest { @Test public void testCustomCommitListener() throws Exception { Path tablePath = new Path(tempDir.toString()); - SchemaManager schemaManager = new SchemaManager(LocalFileIO.create(), tablePath); + SchemaManager schemaManager = new FileSystemSchemaManager(LocalFileIO.create(), tablePath); String testId = UUID.randomUUID().toString(); Schema schema = Schema.newBuilder() diff --git a/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/sink/listener/HttpReportMarkDoneActionTest.java b/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/sink/listener/HttpReportMarkDoneActionTest.java index 8114711ba614..1bd25d690fa7 100644 --- a/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/sink/listener/HttpReportMarkDoneActionTest.java +++ b/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/sink/listener/HttpReportMarkDoneActionTest.java @@ -25,8 +25,8 @@ import org.apache.paimon.partition.actions.HttpReportMarkDoneAction; import org.apache.paimon.partition.actions.HttpReportMarkDoneAction.HttpReportMarkDoneRequest; import org.apache.paimon.rest.TestHttpWebServer; +import org.apache.paimon.schema.FileSystemSchemaManager; import org.apache.paimon.schema.Schema; -import org.apache.paimon.schema.SchemaManager; import org.apache.paimon.schema.SchemaUtils; import org.apache.paimon.schema.TableSchema; import org.apache.paimon.table.CatalogEnvironment; @@ -166,7 +166,7 @@ public FileStoreTable createFileStoreTable() throws Exception { options.set(CoreOptions.PATH, tablePath.toString()); TableSchema tableSchema = SchemaUtils.forceCommit( - new SchemaManager(LocalFileIO.create(), tablePath), + new FileSystemSchemaManager(LocalFileIO.create(), tablePath), new Schema( ROW_TYPE.getFields(), Collections.emptyList(), diff --git a/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/source/CompactorSourceITCase.java b/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/source/CompactorSourceITCase.java index 8f7686dbff9f..294ee6ffc08c 100644 --- a/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/source/CompactorSourceITCase.java +++ b/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/source/CompactorSourceITCase.java @@ -29,6 +29,7 @@ import org.apache.paimon.fs.local.LocalFileIO; import org.apache.paimon.io.DataFileMetaSerializer; import org.apache.paimon.partition.PartitionPredicate; +import org.apache.paimon.schema.FileSystemSchemaManager; import org.apache.paimon.schema.Schema; import org.apache.paimon.schema.SchemaManager; import org.apache.paimon.schema.TableSchema; @@ -429,7 +430,7 @@ private BinaryRow binaryRow(String dt, int hh) { } private FileStoreTable createFileStoreTable() throws Exception { - SchemaManager schemaManager = new SchemaManager(LocalFileIO.create(), tablePath); + SchemaManager schemaManager = new FileSystemSchemaManager(LocalFileIO.create(), tablePath); TableSchema tableSchema = schemaManager.createTable( new Schema( diff --git a/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/source/DataTableSourceTest.java b/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/source/DataTableSourceTest.java index 296d825a29f7..23614f13b3fc 100644 --- a/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/source/DataTableSourceTest.java +++ b/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/source/DataTableSourceTest.java @@ -24,6 +24,7 @@ import org.apache.paimon.fs.FileIO; import org.apache.paimon.fs.Path; import org.apache.paimon.fs.local.LocalFileIO; +import org.apache.paimon.schema.FileSystemSchemaManager; import org.apache.paimon.schema.Schema; import org.apache.paimon.schema.SchemaManager; import org.apache.paimon.schema.TableSchema; @@ -253,7 +254,7 @@ public TypeInformation createTypeInformation( private FileStoreTable createTable(Map options) throws Exception { FileIO fileIO = LocalFileIO.create(); Path tablePath = new Path(path.toString()); - SchemaManager schemaManager = new SchemaManager(fileIO, tablePath); + SchemaManager schemaManager = new FileSystemSchemaManager(fileIO, tablePath); TableSchema tableSchema = schemaManager.createTable( Schema.newBuilder() @@ -267,7 +268,7 @@ private FileStoreTable createTable(Map options) throws Exception private FileStoreTable createPostponeTable(boolean partitioned) throws Exception { FileIO fileIO = LocalFileIO.create(); Path tablePath = new Path(path.toString()); - SchemaManager schemaManager = new SchemaManager(fileIO, tablePath); + SchemaManager schemaManager = new FileSystemSchemaManager(fileIO, tablePath); Schema.Builder schemaBuilder = Schema.newBuilder() .column("pt", DataTypes.INT()) diff --git a/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/source/FileStoreSourceMetricsTest.java b/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/source/FileStoreSourceMetricsTest.java index 2e42401a3c82..bd9d4a6cb9ec 100644 --- a/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/source/FileStoreSourceMetricsTest.java +++ b/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/source/FileStoreSourceMetricsTest.java @@ -24,6 +24,7 @@ import org.apache.paimon.fs.FileIO; import org.apache.paimon.fs.Path; import org.apache.paimon.fs.local.LocalFileIO; +import org.apache.paimon.schema.FileSystemSchemaManager; import org.apache.paimon.schema.Schema; import org.apache.paimon.schema.SchemaManager; import org.apache.paimon.schema.TableSchema; @@ -64,8 +65,9 @@ public void before(@TempDir java.nio.file.Path path) throws Exception { FileIO fileIO = LocalFileIO.create(); Path tablePath = new Path(path.toString()); Path fixBucketTablePath = new Path(path.toString(), "fix_bucket"); - SchemaManager schemaManager = new SchemaManager(fileIO, tablePath); - SchemaManager fixBucketSchemaManager = new SchemaManager(fileIO, fixBucketTablePath); + SchemaManager schemaManager = new FileSystemSchemaManager(fileIO, tablePath); + SchemaManager fixBucketSchemaManager = + new FileSystemSchemaManager(fileIO, fixBucketTablePath); TableSchema tableSchema = schemaManager.createTable( Schema.newBuilder() diff --git a/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/source/FileStoreSourceReaderTest.java b/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/source/FileStoreSourceReaderTest.java index 06e160091799..77119b583db8 100644 --- a/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/source/FileStoreSourceReaderTest.java +++ b/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/source/FileStoreSourceReaderTest.java @@ -22,6 +22,7 @@ import org.apache.paimon.flink.source.metrics.FileStoreSourceReaderMetrics; import org.apache.paimon.fs.Path; import org.apache.paimon.fs.local.LocalFileIO; +import org.apache.paimon.schema.FileSystemSchemaManager; import org.apache.paimon.schema.Schema; import org.apache.paimon.schema.SchemaManager; import org.apache.paimon.table.source.KeyValueTableRead; @@ -62,7 +63,7 @@ public class FileStoreSourceReaderTest { @BeforeEach public void beforeEach() throws Exception { SchemaManager schemaManager = - new SchemaManager(LocalFileIO.create(), new Path(tempDir.toUri())); + new FileSystemSchemaManager(LocalFileIO.create(), new Path(tempDir.toUri())); schemaManager.createTable( new Schema( new RowType( diff --git a/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/source/FileStoreSourceSplitReaderTest.java b/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/source/FileStoreSourceSplitReaderTest.java index 135cb38dab0a..b3a941409346 100644 --- a/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/source/FileStoreSourceSplitReaderTest.java +++ b/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/source/FileStoreSourceSplitReaderTest.java @@ -29,6 +29,7 @@ import org.apache.paimon.io.DataFileMeta; import org.apache.paimon.metrics.MetricRegistry; import org.apache.paimon.reader.RecordReader; +import org.apache.paimon.schema.FileSystemSchemaManager; import org.apache.paimon.schema.Schema; import org.apache.paimon.schema.SchemaManager; import org.apache.paimon.table.source.Split; @@ -81,7 +82,7 @@ public class FileStoreSourceSplitReaderTest { @BeforeEach public void beforeEach() throws Exception { SchemaManager schemaManager = - new SchemaManager(LocalFileIO.create(), new Path(tempDir.toUri())); + new FileSystemSchemaManager(LocalFileIO.create(), new Path(tempDir.toUri())); schemaManager.createTable( new Schema( toDataType( diff --git a/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/source/FlinkTableSourceTest.java b/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/source/FlinkTableSourceTest.java index ba7dbd08938e..74183a40e28b 100644 --- a/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/source/FlinkTableSourceTest.java +++ b/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/source/FlinkTableSourceTest.java @@ -23,8 +23,8 @@ import org.apache.paimon.fs.local.LocalFileIO; import org.apache.paimon.predicate.CompoundPredicate; import org.apache.paimon.predicate.Or; +import org.apache.paimon.schema.FileSystemSchemaManager; import org.apache.paimon.schema.Schema; -import org.apache.paimon.schema.SchemaManager; import org.apache.paimon.schema.TableSchema; import org.apache.paimon.table.FileStoreTableFactory; import org.apache.paimon.table.Table; @@ -53,7 +53,8 @@ public void testApplyFilterNonPartitionTable() throws Exception { FileIO fileIO = LocalFileIO.create(); Path tablePath = new Path(String.format("%s/%s.db/%s", warehouse, database, "T")); Schema schema = Schema.newBuilder().column("col1", DataTypes.INT()).build(); - TableSchema tableSchema = new SchemaManager(fileIO, tablePath).createTable(schema); + TableSchema tableSchema = + new FileSystemSchemaManager(fileIO, tablePath).createTable(schema); Table table = FileStoreTableFactory.create(LocalFileIO.create(), tablePath, tableSchema); DataTableSource tableSource = new DataTableSource( @@ -77,7 +78,8 @@ public void testApplyPartitionTable() throws Exception { .column("p2", DataTypes.STRING()) .partitionKeys("p1", "p2") .build(); - TableSchema tableSchema = new SchemaManager(fileIO, tablePath).createTable(schema); + TableSchema tableSchema = + new FileSystemSchemaManager(fileIO, tablePath).createTable(schema); Table table = FileStoreTableFactory.create(LocalFileIO.create(), tablePath, tableSchema); FlinkTableSource tableSource = new DataTableSource( @@ -231,7 +233,8 @@ private Table createStringTable() throws Exception { FileIO fileIO = LocalFileIO.create(); Path tablePath = new Path(String.format("%s/%s.db/%s", warehouse, database, "T")); Schema schema = Schema.newBuilder().column("contract_address", DataTypes.STRING()).build(); - TableSchema tableSchema = new SchemaManager(fileIO, tablePath).createTable(schema); + TableSchema tableSchema = + new FileSystemSchemaManager(fileIO, tablePath).createTable(schema); return FileStoreTableFactory.create(LocalFileIO.create(), tablePath, tableSchema); } diff --git a/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/source/TestChangelogDataReadWrite.java b/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/source/TestChangelogDataReadWrite.java index 58324fe0160f..920f2970286f 100644 --- a/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/source/TestChangelogDataReadWrite.java +++ b/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/source/TestChangelogDataReadWrite.java @@ -37,6 +37,7 @@ import org.apache.paimon.operation.MergeFileSplitRead; import org.apache.paimon.operation.RawFileSplitRead; import org.apache.paimon.options.Options; +import org.apache.paimon.schema.FileSystemSchemaManager; import org.apache.paimon.schema.KeyValueFieldsExtractor; import org.apache.paimon.schema.SchemaManager; import org.apache.paimon.schema.TableSchema; @@ -121,7 +122,7 @@ public TestChangelogDataReadWrite(String root) { } public KeyValueTableRead createReadWithKey() { - SchemaManager schemaManager = new SchemaManager(LocalFileIO.create(), tablePath); + SchemaManager schemaManager = new FileSystemSchemaManager(LocalFileIO.create(), tablePath); CoreOptions options = new CoreOptions(new HashMap<>()); TableSchema schema = schemaManager.schema(0); MergeFileSplitRead read = @@ -175,7 +176,7 @@ public RecordWriter createMergeTreeWriter(BinaryRow partition, int buc CoreOptions options = new CoreOptions(Collections.singletonMap(CoreOptions.FILE_FORMAT.key(), "avro")); - SchemaManager schemaManager = new SchemaManager(LocalFileIO.create(), tablePath); + SchemaManager schemaManager = new FileSystemSchemaManager(LocalFileIO.create(), tablePath); KeyValueFileStoreWrite write = new KeyValueFileStoreWrite( LocalFileIO.create(), diff --git a/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/source/align/AlignedContinuousFileSplitEnumeratorTest.java b/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/source/align/AlignedContinuousFileSplitEnumeratorTest.java index ad71fa81ca12..7031560e2c78 100644 --- a/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/source/align/AlignedContinuousFileSplitEnumeratorTest.java +++ b/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/source/align/AlignedContinuousFileSplitEnumeratorTest.java @@ -28,6 +28,7 @@ import org.apache.paimon.fs.Path; import org.apache.paimon.fs.local.LocalFileIO; import org.apache.paimon.io.DataFileMeta; +import org.apache.paimon.schema.FileSystemSchemaManager; import org.apache.paimon.schema.Schema; import org.apache.paimon.schema.SchemaManager; import org.apache.paimon.schema.TableSchema; @@ -78,7 +79,7 @@ public class AlignedContinuousFileSplitEnumeratorTest public void before() throws Exception { Path tablePath = new Path(TraceableFileIO.SCHEME + "://" + tempDir.toString()); FileIO fileIO = FileIOFinder.find(tablePath); - SchemaManager schemaManager = new SchemaManager(fileIO, tablePath); + SchemaManager schemaManager = new FileSystemSchemaManager(fileIO, tablePath); TableSchema tableSchema = schemaManager.createTable( new Schema( diff --git a/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/source/align/AlignedSourceReaderTest.java b/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/source/align/AlignedSourceReaderTest.java index a7afa1a2e3ce..2b5f2aeb0b52 100644 --- a/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/source/align/AlignedSourceReaderTest.java +++ b/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/source/align/AlignedSourceReaderTest.java @@ -29,6 +29,7 @@ import org.apache.paimon.fs.FileIO; import org.apache.paimon.fs.Path; import org.apache.paimon.fs.local.LocalFileIO; +import org.apache.paimon.schema.FileSystemSchemaManager; import org.apache.paimon.schema.SchemaManager; import org.apache.paimon.schema.TableSchema; import org.apache.paimon.table.FileStoreTable; @@ -88,7 +89,7 @@ public void beforeEach() throws Exception { super.beforeEach(); FileIO fileIO = LocalFileIO.create(); Path tablePath = new Path(tempDir.toUri()); - SchemaManager schemaManager = new SchemaManager(fileIO, tablePath); + SchemaManager schemaManager = new FileSystemSchemaManager(fileIO, tablePath); TableSchema tableSchema = schemaManager.latest().get(); this.table = FileStoreTableFactory.create(fileIO, tablePath, tableSchema); } diff --git a/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/source/statistics/AppendOnlyTableStatisticsTest.java b/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/source/statistics/AppendOnlyTableStatisticsTest.java index 6b6ae8c3a515..5148e41fb384 100644 --- a/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/source/statistics/AppendOnlyTableStatisticsTest.java +++ b/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/source/statistics/AppendOnlyTableStatisticsTest.java @@ -21,7 +21,7 @@ import org.apache.paimon.CoreOptions; import org.apache.paimon.fs.local.LocalFileIO; import org.apache.paimon.options.Options; -import org.apache.paimon.schema.SchemaManager; +import org.apache.paimon.schema.FileSystemSchemaManager; import org.apache.paimon.schema.TableSchema; import org.apache.paimon.table.FileStoreTable; import org.apache.paimon.table.FileStoreTableFactory; @@ -34,7 +34,7 @@ protected FileStoreTable createStoreTable() throws Exception { conf.set(CoreOptions.PATH, tablePath.toString()); conf.set(CoreOptions.BUCKET, 1); TableSchema tableSchema = - new SchemaManager(LocalFileIO.create(), tablePath) + new FileSystemSchemaManager(LocalFileIO.create(), tablePath) .createTable(schemaBuilder().partitionKeys("pt").build()); return FileStoreTableFactory.create(LocalFileIO.create(), tablePath, tableSchema); } diff --git a/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/source/statistics/PrimaryKeyTableStatisticsTest.java b/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/source/statistics/PrimaryKeyTableStatisticsTest.java index f8ec891b3eca..1737745f3d8e 100644 --- a/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/source/statistics/PrimaryKeyTableStatisticsTest.java +++ b/paimon-flink/paimon-flink-common/src/test/java/org/apache/paimon/flink/source/statistics/PrimaryKeyTableStatisticsTest.java @@ -23,8 +23,8 @@ import org.apache.paimon.fs.local.LocalFileIO; import org.apache.paimon.options.Options; import org.apache.paimon.predicate.PredicateBuilder; +import org.apache.paimon.schema.FileSystemSchemaManager; import org.apache.paimon.schema.Schema; -import org.apache.paimon.schema.SchemaManager; import org.apache.paimon.schema.TableSchema; import org.apache.paimon.table.FileStoreTable; import org.apache.paimon.table.FileStoreTableFactory; @@ -63,7 +63,7 @@ FileStoreTable createStoreTable() throws Exception { Schema.Builder builder = schemaBuilder(); builder.options(options.toMap()); TableSchema tableSchema = - new SchemaManager(LocalFileIO.create(), tablePath) + new FileSystemSchemaManager(LocalFileIO.create(), tablePath) .createTable(builder.partitionKeys("pt").primaryKey("pt", "a").build()); return FileStoreTableFactory.create(LocalFileIO.create(), tablePath, tableSchema); } diff --git a/paimon-full-text/src/test/java/org/apache/paimon/fulltext/index/JavaPyNativeFullTextE2ETest.java b/paimon-full-text/src/test/java/org/apache/paimon/fulltext/index/JavaPyNativeFullTextE2ETest.java index 7056909d5bf3..7883f0a58454 100644 --- a/paimon-full-text/src/test/java/org/apache/paimon/fulltext/index/JavaPyNativeFullTextE2ETest.java +++ b/paimon-full-text/src/test/java/org/apache/paimon/fulltext/index/JavaPyNativeFullTextE2ETest.java @@ -31,8 +31,8 @@ import org.apache.paimon.io.CompactIncrement; import org.apache.paimon.io.DataIncrement; import org.apache.paimon.options.Options; +import org.apache.paimon.schema.FileSystemSchemaManager; import org.apache.paimon.schema.Schema; -import org.apache.paimon.schema.SchemaManager; import org.apache.paimon.schema.SchemaUtils; import org.apache.paimon.schema.TableSchema; import org.apache.paimon.table.AppendOnlyFileStoreTable; @@ -149,7 +149,7 @@ private void writeTableWithNativeFullTextIndex( TableSchema tableSchema = SchemaUtils.forceCommit( - new SchemaManager(fileIO, tablePath), + new FileSystemSchemaManager(fileIO, tablePath), new Schema( rowType.getFields(), Collections.emptyList(), diff --git a/paimon-hive/paimon-hive-catalog/src/main/java/org/apache/paimon/hive/HiveCatalog.java b/paimon-hive/paimon-hive-catalog/src/main/java/org/apache/paimon/hive/HiveCatalog.java index 7b57c6bcea07..12eb7ef06ed6 100644 --- a/paimon-hive/paimon-hive-catalog/src/main/java/org/apache/paimon/hive/HiveCatalog.java +++ b/paimon-hive/paimon-hive-catalog/src/main/java/org/apache/paimon/hive/HiveCatalog.java @@ -40,6 +40,7 @@ import org.apache.paimon.options.Options; import org.apache.paimon.options.OptionsUtils; import org.apache.paimon.partition.PartitionStatistics; +import org.apache.paimon.schema.FileSystemSchemaManager; import org.apache.paimon.schema.Schema; import org.apache.paimon.schema.SchemaChange; import org.apache.paimon.schema.SchemaManager; @@ -1319,7 +1320,7 @@ protected void renameTableImpl(Identifier fromTable, Identifier toTable) { FileIO fileIO = fileIO(fromPath); if (!isExternalTable(table) && !fromPath.equals(toPath) - && !new SchemaManager(fileIO, fromPath).listAllIds().isEmpty()) { + && !new FileSystemSchemaManager(fileIO, fromPath).listAllIds().isEmpty()) { // Rename the file system's table directory. Maintain consistency between tables in // the file system and tables in the Hive Metastore. try { @@ -1891,7 +1892,8 @@ private FieldSchema convertToFieldSchema(DataField dataField) { } private SchemaManager schemaManager(Identifier identifier, Path location) { - return new SchemaManager(fileIO(location), location, identifier.getBranchNameOrDefault()); + return new FileSystemSchemaManager( + fileIO(location), location, identifier.getBranchNameOrDefault()); } public T runWithLock(Identifier identifier, Callable callable) throws Exception { diff --git a/paimon-hive/paimon-hive-connector-2.3/src/test/java/org/apache/paimon/hive/Hive23CatalogITCase.java b/paimon-hive/paimon-hive-connector-2.3/src/test/java/org/apache/paimon/hive/Hive23CatalogITCase.java index 8a4745a09022..783002d14f89 100644 --- a/paimon-hive/paimon-hive-connector-2.3/src/test/java/org/apache/paimon/hive/Hive23CatalogITCase.java +++ b/paimon-hive/paimon-hive-connector-2.3/src/test/java/org/apache/paimon/hive/Hive23CatalogITCase.java @@ -19,7 +19,7 @@ package org.apache.paimon.hive; import org.apache.paimon.fs.local.LocalFileIO; -import org.apache.paimon.schema.SchemaManager; +import org.apache.paimon.schema.FileSystemSchemaManager; import com.klarna.hiverunner.annotations.HiveRunnerSetup; import com.klarna.hiverunner.config.HiveRunnerConfig; @@ -136,7 +136,7 @@ public void testCreateExistTableInHive() { .hasMessage( "Could not execute CreateTable in path `my_hive_custom_client`.`test_db`.`hive_table`"); assertThat( - new SchemaManager( + new FileSystemSchemaManager( LocalFileIO.create(), new org.apache.paimon.fs.Path( path, "test_db.db/hive_table")) @@ -170,7 +170,7 @@ public void testAlterTableFailedInHive() throws Exception { "Could not execute AlterTable in path `my_alter_hive`.`test_db`.`alter_failed_table`")); assertThat( - new SchemaManager( + new FileSystemSchemaManager( LocalFileIO.create(), new org.apache.paimon.fs.Path( path, "test_db.db/alter_failed_table")) diff --git a/paimon-hive/paimon-hive-connector-3.1/src/test/java/org/apache/paimon/hive/Hive31CatalogITCase.java b/paimon-hive/paimon-hive-connector-3.1/src/test/java/org/apache/paimon/hive/Hive31CatalogITCase.java index 48d41d27e8d8..e406750a8244 100644 --- a/paimon-hive/paimon-hive-connector-3.1/src/test/java/org/apache/paimon/hive/Hive31CatalogITCase.java +++ b/paimon-hive/paimon-hive-connector-3.1/src/test/java/org/apache/paimon/hive/Hive31CatalogITCase.java @@ -19,7 +19,7 @@ package org.apache.paimon.hive; import org.apache.paimon.fs.local.LocalFileIO; -import org.apache.paimon.schema.SchemaManager; +import org.apache.paimon.schema.FileSystemSchemaManager; import com.klarna.hiverunner.annotations.HiveRunnerSetup; import com.klarna.hiverunner.config.HiveRunnerConfig; @@ -136,7 +136,7 @@ public void testCreateExistTableInHive() throws Exception { .hasMessage( "Could not execute CreateTable in path `my_hive_custom_client`.`test_db`.`hive_table`"); assertThat( - new SchemaManager( + new FileSystemSchemaManager( LocalFileIO.create(), new org.apache.paimon.fs.Path( path, "test_db.db/hive_table")) @@ -170,7 +170,7 @@ public void testAlterTableFailedInHive() throws Exception { "Could not execute AlterTable in path `my_alter_hive`.`test_db`.`alter_failed_table`")); assertThat( - new SchemaManager( + new FileSystemSchemaManager( LocalFileIO.create(), new org.apache.paimon.fs.Path( path, "test_db.db/alter_failed_table")) diff --git a/paimon-hive/paimon-hive-connector-common/src/main/java/org/apache/paimon/hive/HiveSchema.java b/paimon-hive/paimon-hive-connector-common/src/main/java/org/apache/paimon/hive/HiveSchema.java index 382119a25ef3..aac33db8700b 100644 --- a/paimon-hive/paimon-hive-connector-common/src/main/java/org/apache/paimon/hive/HiveSchema.java +++ b/paimon-hive/paimon-hive-connector-common/src/main/java/org/apache/paimon/hive/HiveSchema.java @@ -24,7 +24,7 @@ import org.apache.paimon.fs.Path; import org.apache.paimon.hive.utils.HiveUtils; import org.apache.paimon.options.Options; -import org.apache.paimon.schema.SchemaManager; +import org.apache.paimon.schema.FileSystemSchemaManager; import org.apache.paimon.schema.TableSchema; import org.apache.paimon.types.DataField; import org.apache.paimon.types.DataType; @@ -202,7 +202,7 @@ private static Optional getExistingSchema( options.set(CoreOptions.PATH, location); CatalogContext context = CatalogContext.create(options, configuration); try { - return new SchemaManager(FileIO.get(path, context), path).latest(); + return new FileSystemSchemaManager(FileIO.get(path, context), path).latest(); } catch (IOException e) { LOG.warn( "Failed to fetch Paimon table schema from path " diff --git a/paimon-hive/paimon-hive-connector-common/src/main/java/org/apache/paimon/hive/PaimonMetaHook.java b/paimon-hive/paimon-hive-connector-common/src/main/java/org/apache/paimon/hive/PaimonMetaHook.java index 003f72ce3bf8..19a2cc3ebbeb 100644 --- a/paimon-hive/paimon-hive-connector-common/src/main/java/org/apache/paimon/hive/PaimonMetaHook.java +++ b/paimon-hive/paimon-hive-connector-common/src/main/java/org/apache/paimon/hive/PaimonMetaHook.java @@ -28,6 +28,7 @@ import org.apache.paimon.hive.mapred.PaimonOutputFormat; import org.apache.paimon.hive.utils.HiveUtils; import org.apache.paimon.options.Options; +import org.apache.paimon.schema.FileSystemSchemaManager; import org.apache.paimon.schema.Schema; import org.apache.paimon.schema.SchemaManager; import org.apache.paimon.schema.TableSchema; @@ -100,7 +101,7 @@ public void preCreateTable(Table table) throws MetaException { throw new RuntimeException(e); } - SchemaManager schemaManager = new SchemaManager(fileIO, path); + SchemaManager schemaManager = new FileSystemSchemaManager(fileIO, path); Optional tableSchema = schemaManager.latest(); if (tableSchema.isPresent()) { existingPaimonTable.add(identifier); diff --git a/paimon-hive/paimon-hive-connector-common/src/test/java/org/apache/paimon/hive/CreateTableITCase.java b/paimon-hive/paimon-hive-connector-common/src/test/java/org/apache/paimon/hive/CreateTableITCase.java index 7819f7b44c05..75f873c10765 100644 --- a/paimon-hive/paimon-hive-connector-common/src/test/java/org/apache/paimon/hive/CreateTableITCase.java +++ b/paimon-hive/paimon-hive-connector-common/src/test/java/org/apache/paimon/hive/CreateTableITCase.java @@ -26,8 +26,8 @@ import org.apache.paimon.fs.local.LocalFileIO; import org.apache.paimon.options.CatalogOptions; import org.apache.paimon.options.Options; +import org.apache.paimon.schema.FileSystemSchemaManager; import org.apache.paimon.schema.Schema; -import org.apache.paimon.schema.SchemaManager; import org.apache.paimon.schema.TableSchema; import org.apache.paimon.table.Table; import org.apache.paimon.types.DataField; @@ -95,7 +95,7 @@ public void testCreateExternalTableWithPaimonTable() throws Exception { ""); Identifier identifier = Identifier.create(DATABASE_TEST, tableName); Path tablePath = newTableLocation(path, identifier); - new SchemaManager(LocalFileIO.create(), tablePath).createTable(schema); + new FileSystemSchemaManager(LocalFileIO.create(), tablePath).createTable(schema); // Create hive external table String hiveSql = @@ -191,7 +191,7 @@ public void testCreateTableUsePartitionedBy() { Identifier identifier = Identifier.create(DATABASE_TEST, tableName); Path tablePath = newTableLocation(path, identifier); Optional tableSchema = - new SchemaManager(LocalFileIO.create(), tablePath).latest(); + new FileSystemSchemaManager(LocalFileIO.create(), tablePath).latest(); assertThat(tableSchema).isPresent(); assertThat(tableSchema.get().primaryKeys()).contains("dt", "hh", "user_id"); assertThat(tableSchema.get().partitionKeys()).contains("dt", "hh"); @@ -253,7 +253,7 @@ public void testLowerTableName() throws Catalog.TableNotExistException { Table table = catalog.getTable(identifier); assertThat(table.name()).isEqualTo(tableName.toLowerCase()); Optional tableSchema = - new SchemaManager(LocalFileIO.create(), tablePath).latest(); + new FileSystemSchemaManager(LocalFileIO.create(), tablePath).latest(); assertThat(tableSchema).isPresent(); } @@ -318,7 +318,7 @@ public void testLowerDBName() throws Catalog.TableNotExistException { Table table = catalog.getTable(identifier); assertThat(table.name()).isEqualTo(tableName.toLowerCase()); Optional tableSchema = - new SchemaManager(LocalFileIO.create(), tablePath).latest(); + new FileSystemSchemaManager(LocalFileIO.create(), tablePath).latest(); assertThat(tableSchema).isPresent(); } @@ -357,7 +357,7 @@ public void testCreateTableWithPrimaryKey() { Identifier identifier = Identifier.create(DATABASE_TEST, tableName); Path tablePath = newTableLocation(path, identifier); Optional tableSchema = - new SchemaManager(LocalFileIO.create(), tablePath).latest(); + new FileSystemSchemaManager(LocalFileIO.create(), tablePath).latest(); assertThat(tableSchema).isPresent(); assertThat(tableSchema.get().primaryKeys()).contains("dt", "hh", "user_id"); assertThat(tableSchema.get().partitionKeys()).isEmpty(); @@ -399,7 +399,7 @@ public void testCreateTableWithPartition() { Identifier identifier = Identifier.create(DATABASE_TEST, tableName); Path tablePath = newTableLocation(path, identifier); Optional tableSchema = - new SchemaManager(LocalFileIO.create(), tablePath).latest(); + new FileSystemSchemaManager(LocalFileIO.create(), tablePath).latest(); assertThat(tableSchema).isPresent(); assertThat(tableSchema.get().primaryKeys()).contains("dt", "hh", "user_id"); assertThat(tableSchema.get().partitionKeys()).contains("dt", "hh"); @@ -443,7 +443,7 @@ public void testCreateTableSpecifyProperties() { Identifier identifier = Identifier.create(DATABASE_TEST, tableName); Path tablePath = newTableLocation(path, identifier); Optional tableSchema = - new SchemaManager(LocalFileIO.create(), tablePath).latest(); + new FileSystemSchemaManager(LocalFileIO.create(), tablePath).latest(); assertThat(tableSchema).isPresent(); assertThat(tableSchema.get().options()).containsEntry("bucket", "2"); assertThat(tableSchema.get().options()).containsEntry("bucket-key", "user_id"); @@ -490,7 +490,7 @@ public void testCreateTableFailing() throws Exception { ""); Identifier identifier = Identifier.create(DATABASE_TEST, tableName); Path tablePath = newTableLocation(path, identifier); - new SchemaManager(LocalFileIO.create(), tablePath).createTable(schema); + new FileSystemSchemaManager(LocalFileIO.create(), tablePath).createTable(schema); String hiveSql = String.join( @@ -504,7 +504,9 @@ public void testCreateTableFailing() throws Exception { } catch (Throwable ignore) { } finally { boolean isPresent = - new SchemaManager(LocalFileIO.create(), tablePath).latest().isPresent(); + new FileSystemSchemaManager(LocalFileIO.create(), tablePath) + .latest() + .isPresent(); Assertions.assertThat(isPresent).isTrue(); } } @@ -535,7 +537,9 @@ public void testCreateTableFailing() throws Exception { Identifier identifier = Identifier.create(DATABASE_TEST, tableName); Path tablePath = newTableLocation(path, identifier); boolean isPresent = - new SchemaManager(LocalFileIO.create(), tablePath).latest().isPresent(); + new FileSystemSchemaManager(LocalFileIO.create(), tablePath) + .latest() + .isPresent(); Assertions.assertThat(isPresent).isFalse(); } } diff --git a/paimon-hive/paimon-hive-connector-common/src/test/java/org/apache/paimon/hive/HiveLocationTest.java b/paimon-hive/paimon-hive-connector-common/src/test/java/org/apache/paimon/hive/HiveLocationTest.java index da75d05a2864..78a0333a5147 100644 --- a/paimon-hive/paimon-hive-connector-common/src/test/java/org/apache/paimon/hive/HiveLocationTest.java +++ b/paimon-hive/paimon-hive-connector-common/src/test/java/org/apache/paimon/hive/HiveLocationTest.java @@ -26,6 +26,7 @@ import org.apache.paimon.hive.runner.PaimonEmbeddedHiveRunner; import org.apache.paimon.options.CatalogOptions; import org.apache.paimon.options.Options; +import org.apache.paimon.schema.FileSystemSchemaManager; import org.apache.paimon.schema.Schema; import org.apache.paimon.schema.SchemaManager; import org.apache.paimon.schema.TableSchema; @@ -273,7 +274,7 @@ private void testRWinHive(String createTableStr, String location, String tableNa hiveShell.execute(insertStr); FileIO fIO = getFileIO(catalogContext, new Path(location)); - SchemaManager schemaManager = new SchemaManager(fIO, new Path(location)); + SchemaManager schemaManager = new FileSystemSchemaManager(fIO, new Path(location)); Optional tableSchema = schemaManager.latest(); assertThat(tableSchema).isPresent(); diff --git a/paimon-hive/paimon-hive-connector-common/src/test/java/org/apache/paimon/hive/HiveReadITCaseBase.java b/paimon-hive/paimon-hive-connector-common/src/test/java/org/apache/paimon/hive/HiveReadITCaseBase.java index 4b16788ee716..2a15ffae344b 100644 --- a/paimon-hive/paimon-hive-connector-common/src/test/java/org/apache/paimon/hive/HiveReadITCaseBase.java +++ b/paimon-hive/paimon-hive-connector-common/src/test/java/org/apache/paimon/hive/HiveReadITCaseBase.java @@ -34,6 +34,7 @@ import org.apache.paimon.hive.objectinspector.PaimonObjectInspectorFactory; import org.apache.paimon.options.CatalogOptions; import org.apache.paimon.options.Options; +import org.apache.paimon.schema.FileSystemSchemaManager; import org.apache.paimon.schema.Schema; import org.apache.paimon.schema.SchemaChange; import org.apache.paimon.schema.SchemaManager; @@ -976,7 +977,7 @@ public void testReadExternalTableWithEmptyDataAndIgnoreCase() throws Exception { ""); Identifier identifier = Identifier.create(DATABASE_TEST, tableName); Path tablePath = AbstractCatalog.newTableLocation(path, identifier); - new SchemaManager(LocalFileIO.create(), tablePath).createTable(schema); + new FileSystemSchemaManager(LocalFileIO.create(), tablePath).createTable(schema); // Create hive external table String hiveSql = @@ -1058,7 +1059,7 @@ public void testReadExternalTableWithDataAndIgnoreCase() throws Exception { // add column, do some ddl which will generate a new version schema-n file. Path tablePath = AbstractCatalog.newTableLocation(path, identifier); - SchemaManager schemaManager = new SchemaManager(LocalFileIO.create(), tablePath); + SchemaManager schemaManager = new FileSystemSchemaManager(LocalFileIO.create(), tablePath); schemaManager.commitChanges(SchemaChange.addColumn("N1", DataTypes.STRING())); // Create hive external table diff --git a/paimon-hive/paimon-hive-connector-common/src/test/java/org/apache/paimon/hive/HiveSplitGeneratorTest.java b/paimon-hive/paimon-hive-connector-common/src/test/java/org/apache/paimon/hive/HiveSplitGeneratorTest.java index 16aa824c1b7b..d41751b126c7 100644 --- a/paimon-hive/paimon-hive-connector-common/src/test/java/org/apache/paimon/hive/HiveSplitGeneratorTest.java +++ b/paimon-hive/paimon-hive-connector-common/src/test/java/org/apache/paimon/hive/HiveSplitGeneratorTest.java @@ -30,6 +30,7 @@ import org.apache.paimon.io.DataFileMeta; import org.apache.paimon.manifest.FileSource; import org.apache.paimon.options.Options; +import org.apache.paimon.schema.FileSystemSchemaManager; import org.apache.paimon.schema.SchemaManager; import org.apache.paimon.schema.TableSchema; import org.apache.paimon.table.AppendOnlyFileStoreTable; @@ -208,7 +209,7 @@ private FileStoreTable createPartitionedTableWithTwoPartitions() throws Exceptio } private FileStoreTable createFileStoreTable(TableSchema tableSchema) throws Exception { - SchemaManager schemaManager = new SchemaManager(fileIO, tablePath); + SchemaManager schemaManager = new FileSystemSchemaManager(fileIO, tablePath); schemaManager.commit(tableSchema); return new AppendOnlyFileStoreTable( diff --git a/paimon-hive/paimon-hive-connector-common/src/test/java/org/apache/paimon/hive/HiveTableSchemaTest.java b/paimon-hive/paimon-hive-connector-common/src/test/java/org/apache/paimon/hive/HiveTableSchemaTest.java index a15360fac79f..a32337dbe1c2 100644 --- a/paimon-hive/paimon-hive-connector-common/src/test/java/org/apache/paimon/hive/HiveTableSchemaTest.java +++ b/paimon-hive/paimon-hive-connector-common/src/test/java/org/apache/paimon/hive/HiveTableSchemaTest.java @@ -20,8 +20,8 @@ import org.apache.paimon.fs.Path; import org.apache.paimon.fs.local.LocalFileIO; +import org.apache.paimon.schema.FileSystemSchemaManager; import org.apache.paimon.schema.Schema; -import org.apache.paimon.schema.SchemaManager; import org.apache.paimon.types.DataField; import org.apache.paimon.types.DataTypes; import org.apache.paimon.types.RowType; @@ -268,7 +268,7 @@ public void testTooManyColumns() throws Exception { } private void createSchema() throws Exception { - new SchemaManager(LocalFileIO.create(), new Path(tempDir.toString())) + new FileSystemSchemaManager(LocalFileIO.create(), new Path(tempDir.toString())) .createTable( new Schema( ROW_TYPE.getFields(), @@ -373,7 +373,7 @@ public void testTooManyPartitionKeys() throws Exception { } private void createSchemaWithPartition() throws Exception { - new SchemaManager(LocalFileIO.create(), new Path(tempDir.toString())) + new FileSystemSchemaManager(LocalFileIO.create(), new Path(tempDir.toString())) .createTable( new Schema( ROW_TYPE.getFields(), diff --git a/paimon-hive/paimon-hive-connector-common/src/test/java/org/apache/paimon/hive/PaimonSerDeTest.java b/paimon-hive/paimon-hive-connector-common/src/test/java/org/apache/paimon/hive/PaimonSerDeTest.java index 449c4ce213e5..c61d91b4f223 100644 --- a/paimon-hive/paimon-hive-connector-common/src/test/java/org/apache/paimon/hive/PaimonSerDeTest.java +++ b/paimon-hive/paimon-hive-connector-common/src/test/java/org/apache/paimon/hive/PaimonSerDeTest.java @@ -24,8 +24,8 @@ import org.apache.paimon.fs.Path; import org.apache.paimon.fs.local.LocalFileIO; import org.apache.paimon.hive.objectinspector.PaimonInternalRowObjectInspector; +import org.apache.paimon.schema.FileSystemSchemaManager; import org.apache.paimon.schema.Schema; -import org.apache.paimon.schema.SchemaManager; import org.apache.paimon.types.DataType; import org.apache.paimon.types.DataTypes; import org.apache.paimon.types.RowType; @@ -123,7 +123,7 @@ private PaimonSerDe createInitializedSerDe() throws Exception { } private PaimonSerDe createInitializedSerDe(RowType rowType) throws Exception { - new SchemaManager(LocalFileIO.create(), new Path(tempDir.toString())) + new FileSystemSchemaManager(LocalFileIO.create(), new Path(tempDir.toString())) .createTable( new Schema( rowType.getFields(), diff --git a/paimon-hive/paimon-hive-connector-common/src/test/java/org/apache/paimon/hive/mapred/PaimonInputSplitTest.java b/paimon-hive/paimon-hive-connector-common/src/test/java/org/apache/paimon/hive/mapred/PaimonInputSplitTest.java index 755b2482f335..79c25e8b90e5 100644 --- a/paimon-hive/paimon-hive-connector-common/src/test/java/org/apache/paimon/hive/mapred/PaimonInputSplitTest.java +++ b/paimon-hive/paimon-hive-connector-common/src/test/java/org/apache/paimon/hive/mapred/PaimonInputSplitTest.java @@ -24,6 +24,7 @@ import org.apache.paimon.fs.Path; import org.apache.paimon.fs.local.LocalFileIO; import org.apache.paimon.io.DataFileTestDataGenerator; +import org.apache.paimon.schema.FileSystemSchemaManager; import org.apache.paimon.schema.Schema; import org.apache.paimon.schema.SchemaManager; import org.apache.paimon.table.FileStoreTable; @@ -100,7 +101,7 @@ private void assertPaimonInputSplitSerialization(PaimonInputSplit split) throws @Test public void testWriteAndReadWithTable() throws Exception { Path path = new Path(tempDir.toString()); - SchemaManager schemaManager = new SchemaManager(LocalFileIO.create(), path); + SchemaManager schemaManager = new FileSystemSchemaManager(LocalFileIO.create(), path); schemaManager.createTable( new Schema( RowType.of(VarCharType.STRING_TYPE).getFields(), diff --git a/paimon-iceberg/src/test/java/org/apache/paimon/iceberg/IcebergRestMetadataCommitterTest.java b/paimon-iceberg/src/test/java/org/apache/paimon/iceberg/IcebergRestMetadataCommitterTest.java index 052f76bda7ca..b7af09c34204 100644 --- a/paimon-iceberg/src/test/java/org/apache/paimon/iceberg/IcebergRestMetadataCommitterTest.java +++ b/paimon-iceberg/src/test/java/org/apache/paimon/iceberg/IcebergRestMetadataCommitterTest.java @@ -32,6 +32,7 @@ import org.apache.paimon.iceberg.metadata.IcebergSnapshot; import org.apache.paimon.options.MemorySize; import org.apache.paimon.options.Options; +import org.apache.paimon.schema.FileSystemSchemaManager; import org.apache.paimon.schema.SchemaChange; import org.apache.paimon.schema.SchemaManager; import org.apache.paimon.table.FileStoreTable; @@ -376,7 +377,7 @@ public void testSchemaAndPropertiesChange() throws Exception { commit.commit(1, write.prepareCommit(false, 1)); assertThat(getIcebergResult()).containsExactlyInAnyOrder("Record(1, 10)", "Record(2, 20)"); - SchemaManager schemaManager = new SchemaManager(table.fileIO(), table.location()); + SchemaManager schemaManager = new FileSystemSchemaManager(table.fileIO(), table.location()); // change1: add a column // change2: change 'metadata.iceberg.delete-after-commit.enabled' to false // change3: change 'metadata.iceberg.previous-versions-max' to 10 @@ -436,7 +437,7 @@ public void testCommitAfterSchemaRollback() throws Exception { commit.commit(1, write.prepareCommit(false, 1)); table.createTag("before-evolution", 1); - SchemaManager schemaManager = new SchemaManager(table.fileIO(), table.location()); + SchemaManager schemaManager = new FileSystemSchemaManager(table.fileIO(), table.location()); schemaManager.commitChanges(SchemaChange.addColumn("v2", DataTypes.STRING())); table = table.copy(table.schemaManager().latest().get()); write.close(); @@ -478,7 +479,10 @@ public void testCommitAfterSchemaRollback() throws Exception { + localMeta.currentSnapshot().schemaId()); System.out.println( "PROBE schemaLatest=" - + new SchemaManager(table.fileIO(), table.location()).latest().get().id()); + + new FileSystemSchemaManager(table.fileIO(), table.location()) + .latest() + .get() + .id()); for (org.apache.paimon.fs.FileStatus st : table.fileIO() .listStatus( @@ -522,7 +526,7 @@ public void testOptionOnlyAlterTableDoesNotCrashIcebergSync() throws Exception { // Perform 3 option-only schema changes — each increments Paimon schema ID // but does NOT change columns, creating the dedup scenario - SchemaManager schemaManager = new SchemaManager(table.fileIO(), table.location()); + SchemaManager schemaManager = new FileSystemSchemaManager(table.fileIO(), table.location()); schemaManager.commitChanges(SchemaChange.setOption("my.custom.option.1", "value1")); schemaManager.commitChanges(SchemaChange.setOption("my.custom.option.2", "value2")); schemaManager.commitChanges(SchemaChange.setOption("my.custom.option.3", "value3")); @@ -654,7 +658,7 @@ private class EvolveContext implements AutoCloseable { } void evolve(SchemaChange change) throws Exception { - new SchemaManager(table.fileIO(), table.location()).commitChanges(change); + new FileSystemSchemaManager(table.fileIO(), table.location()).commitChanges(change); table = table.copy(table.schemaManager().latest().get()); write.close(); write = table.newWrite(commitUser); @@ -704,7 +708,7 @@ public void testSchemaChangeBeforeSync() throws Exception { commit.commit(1, write.prepareCommit(false, 1)); // schema change - SchemaManager schemaManager = new SchemaManager(table.fileIO(), table.location()); + SchemaManager schemaManager = new FileSystemSchemaManager(table.fileIO(), table.location()); schemaManager.commitChanges(SchemaChange.addColumn("v2", DataTypes.STRING())); table = table.copyWithLatestSchema(); write.close(); diff --git a/paimon-lumina/src/test/java/org/apache/paimon/lumina/index/JavaPyLuminaE2ETest.java b/paimon-lumina/src/test/java/org/apache/paimon/lumina/index/JavaPyLuminaE2ETest.java index d402a40ac942..3a311b5a58e2 100644 --- a/paimon-lumina/src/test/java/org/apache/paimon/lumina/index/JavaPyLuminaE2ETest.java +++ b/paimon-lumina/src/test/java/org/apache/paimon/lumina/index/JavaPyLuminaE2ETest.java @@ -32,8 +32,8 @@ import org.apache.paimon.io.CompactIncrement; import org.apache.paimon.io.DataIncrement; import org.apache.paimon.options.Options; +import org.apache.paimon.schema.FileSystemSchemaManager; import org.apache.paimon.schema.Schema; -import org.apache.paimon.schema.SchemaManager; import org.apache.paimon.schema.SchemaUtils; import org.apache.paimon.schema.TableSchema; import org.apache.paimon.table.AppendOnlyFileStoreTable; @@ -132,7 +132,7 @@ public void testLuminaVectorIndexWrite(String fileFormat) throws Exception { TableSchema tableSchema = SchemaUtils.forceCommit( - new SchemaManager(LocalFileIO.create(), tablePath), + new FileSystemSchemaManager(LocalFileIO.create(), tablePath), new Schema( rowType.getFields(), Collections.emptyList(), @@ -239,7 +239,7 @@ public void testLuminaVectorWithBTreeIndexWrite() throws Exception { TableSchema tableSchema = SchemaUtils.forceCommit( - new SchemaManager(LocalFileIO.create(), tablePath), + new FileSystemSchemaManager(LocalFileIO.create(), tablePath), new Schema( rowType.getFields(), Collections.emptyList(), diff --git a/paimon-lumina/src/test/java/org/apache/paimon/lumina/index/LuminaVectorGlobalIndexScanTest.java b/paimon-lumina/src/test/java/org/apache/paimon/lumina/index/LuminaVectorGlobalIndexScanTest.java index 85d4ebe6be98..cd7919096555 100644 --- a/paimon-lumina/src/test/java/org/apache/paimon/lumina/index/LuminaVectorGlobalIndexScanTest.java +++ b/paimon-lumina/src/test/java/org/apache/paimon/lumina/index/LuminaVectorGlobalIndexScanTest.java @@ -34,6 +34,7 @@ import org.apache.paimon.io.CompactIncrement; import org.apache.paimon.io.DataIncrement; import org.apache.paimon.options.Options; +import org.apache.paimon.schema.FileSystemSchemaManager; import org.apache.paimon.schema.Schema; import org.apache.paimon.schema.SchemaManager; import org.apache.paimon.schema.TableSchema; @@ -96,7 +97,7 @@ public void before() throws Exception { Path tablePath = new Path(tempDir.toString()); fileIO = new LocalFileIO(); - SchemaManager schemaManager = new SchemaManager(fileIO, tablePath); + SchemaManager schemaManager = new FileSystemSchemaManager(fileIO, tablePath); String similarityMetric = "l2"; Schema schema = @@ -155,7 +156,7 @@ public void testVectorIndexScanEndToEnd() throws Exception { public void testVectorIndexScanWithDifferentMetrics() throws Exception { Path tablePath = new Path(tempDir.toString(), "inner_product"); fileIO.mkdirs(tablePath); - SchemaManager schemaManager = new SchemaManager(fileIO, tablePath); + SchemaManager schemaManager = new FileSystemSchemaManager(fileIO, tablePath); Schema schema = Schema.newBuilder() diff --git a/paimon-spark/paimon-spark-common/src/test/java/org/apache/paimon/spark/read/SparkDataEvolutionVectorReadTest.java b/paimon-spark/paimon-spark-common/src/test/java/org/apache/paimon/spark/read/SparkDataEvolutionVectorReadTest.java index dd13cd365b2c..4855ea0addcb 100644 --- a/paimon-spark/paimon-spark-common/src/test/java/org/apache/paimon/spark/read/SparkDataEvolutionVectorReadTest.java +++ b/paimon-spark/paimon-spark-common/src/test/java/org/apache/paimon/spark/read/SparkDataEvolutionVectorReadTest.java @@ -36,8 +36,8 @@ import org.apache.paimon.index.IndexFileMeta; import org.apache.paimon.index.IndexPathFactory; import org.apache.paimon.options.Options; +import org.apache.paimon.schema.FileSystemSchemaManager; import org.apache.paimon.schema.Schema; -import org.apache.paimon.schema.SchemaManager; import org.apache.paimon.schema.SchemaUtils; import org.apache.paimon.schema.TableSchema; import org.apache.paimon.table.FileStoreTable; @@ -191,7 +191,8 @@ private FileStoreTable createTable() throws Exception { .options(options.toMap()) .build(); TableSchema tableSchema = - SchemaUtils.forceCommit(new SchemaManager(LocalFileIO.create(), tablePath), schema); + SchemaUtils.forceCommit( + new FileSystemSchemaManager(LocalFileIO.create(), tablePath), schema); return FileStoreTableFactory.create(LocalFileIO.create(), tablePath, tableSchema); } diff --git a/paimon-spark/paimon-spark-ut/src/test/java/org/apache/paimon/spark/SparkFileIndexITCase.java b/paimon-spark/paimon-spark-ut/src/test/java/org/apache/paimon/spark/SparkFileIndexITCase.java index ed3c53683bb4..9e9d839eddfb 100644 --- a/paimon-spark/paimon-spark-ut/src/test/java/org/apache/paimon/spark/SparkFileIndexITCase.java +++ b/paimon-spark/paimon-spark-ut/src/test/java/org/apache/paimon/spark/SparkFileIndexITCase.java @@ -35,6 +35,7 @@ import org.apache.paimon.io.DataFilePathFactory; import org.apache.paimon.options.Options; import org.apache.paimon.predicate.FieldRef; +import org.apache.paimon.schema.FileSystemSchemaManager; import org.apache.paimon.schema.SchemaManager; import org.apache.paimon.schema.TableSchema; import org.apache.paimon.spark.extensions.PaimonSparkSessionExtensions; @@ -196,7 +197,7 @@ public void testReadWriteTableWithRangeBitmapIndex() throws Catalog.TableNotExis protected void foreachIndexReader(String tableName, Consumer consumer) throws Catalog.TableNotExistException { Path tableRoot = fileSystemCatalog.getTableLocation(Identifier.create("db", tableName)); - SchemaManager schemaManager = new SchemaManager(fileIO, tableRoot); + SchemaManager schemaManager = new FileSystemSchemaManager(fileIO, tableRoot); FileStorePathFactory pathFactory = new FileStorePathFactory( tableRoot, diff --git a/paimon-spark/paimon-spark-ut/src/test/java/org/apache/paimon/spark/catalog/functions/BucketFunctionTest.java b/paimon-spark/paimon-spark-ut/src/test/java/org/apache/paimon/spark/catalog/functions/BucketFunctionTest.java index 43425486c84d..7c7aa98b4dec 100644 --- a/paimon-spark/paimon-spark-ut/src/test/java/org/apache/paimon/spark/catalog/functions/BucketFunctionTest.java +++ b/paimon-spark/paimon-spark-ut/src/test/java/org/apache/paimon/spark/catalog/functions/BucketFunctionTest.java @@ -27,8 +27,8 @@ import org.apache.paimon.disk.IOManagerImpl; import org.apache.paimon.fs.Path; import org.apache.paimon.fs.local.LocalFileIO; +import org.apache.paimon.schema.FileSystemSchemaManager; import org.apache.paimon.schema.Schema; -import org.apache.paimon.schema.SchemaManager; import org.apache.paimon.schema.SchemaUtils; import org.apache.paimon.schema.TableSchema; import org.apache.paimon.spark.SparkCatalog; @@ -212,7 +212,7 @@ public static void setupTable(String... bucketColumns) { try { TableSchema tableSchema = SchemaUtils.forceCommit( - new SchemaManager(LocalFileIO.create(), tablePath), + new FileSystemSchemaManager(LocalFileIO.create(), tablePath), new Schema( ROW_TYPE.getFields(), Collections.emptyList(), diff --git a/paimon-spark/paimon-spark-ut/src/test/scala/org/apache/paimon/spark/table/PaimonExternalTableTest.scala b/paimon-spark/paimon-spark-ut/src/test/scala/org/apache/paimon/spark/table/PaimonExternalTableTest.scala index 1ea9b2ee81d1..0c80b5158a22 100644 --- a/paimon-spark/paimon-spark-ut/src/test/scala/org/apache/paimon/spark/table/PaimonExternalTableTest.scala +++ b/paimon-spark/paimon-spark-ut/src/test/scala/org/apache/paimon/spark/table/PaimonExternalTableTest.scala @@ -21,7 +21,7 @@ package org.apache.paimon.spark.table import org.apache.paimon.catalog.Identifier import org.apache.paimon.fs.Path import org.apache.paimon.fs.local.LocalFileIO -import org.apache.paimon.schema.{Schema, SchemaManager} +import org.apache.paimon.schema.{FileSystemSchemaManager, Schema} import org.apache.paimon.spark.PaimonSparkTestWithRestCatalogBase import org.apache.paimon.table.FileStoreTable import org.apache.paimon.types.DataTypes @@ -62,7 +62,7 @@ class PaimonExternalTableTest extends PaimonSparkTestWithRestCatalogBase { .option("path", externalTbLocation) .option("type", "table") .build() - new SchemaManager(schemaFileIO, schemaTablePath).createTable(schema, true) + new FileSystemSchemaManager(schemaFileIO, schemaTablePath).createTable(schema, true) // create external table sql( @@ -146,7 +146,7 @@ class PaimonExternalTableTest extends PaimonSparkTestWithRestCatalogBase { .option("path", externalTbLocation) .option("type", "table") .build() - new SchemaManager(schemaFileIO, schemaTablePath).createTable(schema, true) + new FileSystemSchemaManager(schemaFileIO, schemaTablePath).createTable(schema, true) sql(s""" |CREATE TABLE external_tbl (id INT, name STRING, value DOUBLE) USING paimon @@ -219,7 +219,7 @@ class PaimonExternalTableTest extends PaimonSparkTestWithRestCatalogBase { .option("path", externalTbLocation) .option("type", "table") .build() - new SchemaManager(schemaFileIO, schemaTablePath).createTable(schema, true) + new FileSystemSchemaManager(schemaFileIO, schemaTablePath).createTable(schema, true) // create external table sql( @@ -267,7 +267,7 @@ class PaimonExternalTableTest extends PaimonSparkTestWithRestCatalogBase { .option("path", externalTbLocation) .option("type", "table") .build() - new SchemaManager(schemaFileIO, schemaTablePath).createTable(schema, true) + new FileSystemSchemaManager(schemaFileIO, schemaTablePath).createTable(schema, true) // First create a table with schema and data sql(s""" @@ -378,7 +378,7 @@ class PaimonExternalTableTest extends PaimonSparkTestWithRestCatalogBase { .option("path", externalTbLocation) .option("type", "table") .build() - new SchemaManager(schemaFileIO, schemaTablePath).createTable(schema, true) + new FileSystemSchemaManager(schemaFileIO, schemaTablePath).createTable(schema, true) sql( s"CREATE TABLE external_tbl (age INT, name STRING) USING paimon LOCATION '$externalTbLocation'") @@ -416,7 +416,7 @@ class PaimonExternalTableTest extends PaimonSparkTestWithRestCatalogBase { .option("path", externalTbLocation) .option("type", "table") .build() - new SchemaManager(schemaFileIO, schemaTablePath).createTable(schema, true) + new FileSystemSchemaManager(schemaFileIO, schemaTablePath).createTable(schema, true) sql(s""" |CREATE TABLE external_tbl (age INT, name STRING) USING paimon @@ -462,7 +462,7 @@ class PaimonExternalTableTest extends PaimonSparkTestWithRestCatalogBase { .option("path", externalTbLocation) .option("type", "table") .build() - new SchemaManager(schemaFileIO, schemaTablePath).createTable(schema, true) + new FileSystemSchemaManager(schemaFileIO, schemaTablePath).createTable(schema, true) sql(s""" |CREATE TABLE external_tbl (id INT, p1 INT, p2 STRING) USING paimon diff --git a/paimon-vector/src/test/java/org/apache/paimon/JavaPyE2ETest.java b/paimon-vector/src/test/java/org/apache/paimon/JavaPyE2ETest.java index 4c8cbfd59cc1..8b87bd2bb5cd 100644 --- a/paimon-vector/src/test/java/org/apache/paimon/JavaPyE2ETest.java +++ b/paimon-vector/src/test/java/org/apache/paimon/JavaPyE2ETest.java @@ -31,8 +31,8 @@ import org.apache.paimon.io.CompactIncrement; import org.apache.paimon.io.DataIncrement; import org.apache.paimon.options.Options; +import org.apache.paimon.schema.FileSystemSchemaManager; import org.apache.paimon.schema.Schema; -import org.apache.paimon.schema.SchemaManager; import org.apache.paimon.schema.SchemaUtils; import org.apache.paimon.schema.TableSchema; import org.apache.paimon.table.AppendOnlyFileStoreTable; @@ -115,7 +115,7 @@ public void testVindexVectorIndexWrite() throws Exception { TableSchema tableSchema = SchemaUtils.forceCommit( - new SchemaManager(fileIO, tablePath), + new FileSystemSchemaManager(fileIO, tablePath), new Schema( rowType.getFields(), Collections.emptyList(), @@ -228,7 +228,7 @@ public void testVindexVectorRawFallbackWrite() throws Exception { TableSchema tableSchema = SchemaUtils.forceCommit( - new SchemaManager(fileIO, tablePath), + new FileSystemSchemaManager(fileIO, tablePath), new Schema( rowType.getFields(), Collections.emptyList(),