diff --git a/integration-test/src/test/java/org/apache/iotdb/tools/it/SchemaTagsRoundTripIT.java b/integration-test/src/test/java/org/apache/iotdb/tools/it/SchemaTagsRoundTripIT.java new file mode 100644 index 0000000000000..a8f33bc59f454 --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/tools/it/SchemaTagsRoundTripIT.java @@ -0,0 +1,655 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iotdb.tools.it; + +import org.apache.iotdb.cli.it.AbstractScriptIT; +import org.apache.iotdb.isession.ISession; +import org.apache.iotdb.isession.SessionConfig; +import org.apache.iotdb.isession.SessionDataSet; +import org.apache.iotdb.it.env.EnvFactory; +import org.apache.iotdb.it.framework.IoTDBTestRunner; +import org.apache.iotdb.itbase.category.ClusterIT; +import org.apache.iotdb.itbase.category.LocalStandaloneIT; +import org.apache.iotdb.rpc.IoTDBConnectionException; +import org.apache.iotdb.rpc.StatementExecutionException; +import org.apache.iotdb.tool.common.Constants; + +import com.google.gson.Gson; +import com.google.gson.reflect.TypeToken; +import org.apache.commons.csv.CSVFormat; +import org.apache.commons.csv.CSVParser; +import org.apache.commons.csv.CSVRecord; +import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.file.metadata.enums.CompressionType; +import org.apache.tsfile.file.metadata.enums.TSEncoding; +import org.apache.tsfile.read.common.Field; +import org.awaitility.Awaitility; +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.Rule; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.rules.TemporaryFolder; +import org.junit.runner.RunWith; + +import java.io.File; +import java.io.IOException; +import java.lang.reflect.Type; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.EnumSet; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.TreeMap; +import java.util.concurrent.TimeUnit; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertTrue; + +/** + * Proves that the tree model schema tools restore the tags and attributes of every time series: + * {@code export-schema} writes them as RFC4180 quoted JSON, {@code import-schema} reads that format + * back, and awkward characters survive the round trip unchanged. + * + *

The schema of one database and one exported file mixes everything the tools have to keep + * apart: every data type/encoding/compression combination, all four tag/attribute combinations on + * different devices, and several combinations across the measurements of one device. + */ +@RunWith(IoTDBTestRunner.class) +@Category({LocalStandaloneIT.class, ClusterIT.class}) +public class SchemaTagsRoundTripIT extends AbstractScriptIT { + + private static final String DATABASE = "root.rt"; + private static final String EXPORT_PREFIX = "schema_tags"; + private static final String HEADER = + "Timeseries,Alias,DataType,Encoding,Compression,Tags,Attributes"; + private static final String ALIAS = "aliased"; + + /** The row whose tag value carries a double quote, i.e. the row that must be RFC4180 quoted. */ + private static final String QUOTE_ROW = DATABASE + ".quote.s1"; + + /** The JSON object the export writes for the tag value {@code a"b}. */ + private static final String QUOTE_ROW_TAGS_JSON = "{\"k\":\"a\\\"b\"}"; + + private static final String NO_ATTRIBUTES_ROW = DATABASE + ".none.s1"; + + /** How long the metadata drop may take to become visible on every node. */ + private static final int SCHEMA_CLEANUP_TIMEOUT_SECONDS = 60; + + private static final Type JSON_MAP_TYPE = new TypeToken>() {}.getType(); + private static final Gson GSON = new Gson(); + private static final Map NO_VALUES = Collections.emptyMap(); + + /** Time series path to the tags, attributes and alias the test creates for it. */ + private static final Map CREATED = new LinkedHashMap<>(); + + static { + // Awkward tag values, one device per value because tags belong to the device. + singleTag("v1", "v1", Format.BOOLEAN_PLAIN_SNAPPY); + singleTag("comma", "a,b", Format.INT32_TS_2DIFF_SNAPPY); + singleTag("quote", "a\"b", Format.DOUBLE_GORILLA_LZ4); + singleTag("backslash", "a\\b", Format.INT64_GORILLA_LZ4); + singleTag("backslashn", "a\\nb", Format.FLOAT_GORILLA_GZIP); + singleTag("tab", "a\tb", Format.DOUBLE_GORILLA_LZ4); + singleTag("newline", "a\nb", Format.TEXT_DICTIONARY_SNAPPY); + singleTag("cjk", "中文😀", Format.INT64_GORILLA_LZ4); + singleTag("nulllit", "null", Format.BOOLEAN_PLAIN_SNAPPY); + singleTag("empty", "", Format.INT32_TS_2DIFF_SNAPPY); + singleTag("backtick", "`x`", Format.FLOAT_GORILLA_GZIP); + tagged("spacedkey", "k e y", "v", Format.TEXT_DICTIONARY_SNAPPY); + + // The four tag/attribute combinations, each on a device of its own. + series( + "both", + "s1", + Format.BOOLEAN_PLAIN_SNAPPY, + mapOf("t1", "v1", "t2", "a,b"), + mapOf("a1", "one", "a2", "he said \"hi\""), + null); + series( + "multi", + "s1", + Format.TEXT_DICTIONARY_SNAPPY, + mapOf("a", "1", "b", "a,b", "c", "a\"b"), + NO_VALUES, + null); + series( + "attr", + "s1", + Format.TEXT_DICTIONARY_SNAPPY, + NO_VALUES, + mapOf("note", "he said \"hi\"", "plain", "v"), + null); + series("none", "s1", Format.TEXT_DICTIONARY_SNAPPY, NO_VALUES, NO_VALUES, null); + series("alias", "s1", Format.FLOAT_GORILLA_GZIP, mapOf("aliasTag", "v"), NO_VALUES, ALIAS); + + // One device mixing three of the combinations across its measurements. + series( + "mixed", + "tagsOnly", + Format.INT32_TS_2DIFF_SNAPPY, + mapOf("m1", "x", "m2", "y"), + NO_VALUES, + null); + series( + "mixed", + "attrsOnly", + Format.INT64_GORILLA_LZ4, + NO_VALUES, + mapOf("m1", "x", "m2", "y"), + null); + series("mixed", "neither", Format.DOUBLE_GORILLA_LZ4, NO_VALUES, NO_VALUES, null); + } + + @Rule public TemporaryFolder folder = new TemporaryFolder(); + + private static String ip; + private static String port; + private static String toolsPath; + private static String homePath; + + @BeforeClass + public static void setUp() { + EnvFactory.getEnv().initClusterEnvironment(); + ip = EnvFactory.getEnv().getIP(); + port = EnvFactory.getEnv().getPort(); + toolsPath = EnvFactory.getEnv().getToolsPath(); + final String libPath = EnvFactory.getEnv().getLibPath(); + homePath = + libPath.substring(0, libPath.lastIndexOf(File.separator + "lib" + File.separator + "*")); + } + + @AfterClass + public static void tearDown() { + EnvFactory.getEnv().cleanClusterEnvironment(); + } + + @Test + public void test() throws IOException { + String os = System.getProperty("os.name").toLowerCase(); + if (os.startsWith("windows")) { + testOnWindows(); + } else { + testOnUnix(); + } + } + + @Override + protected void testOnWindows() throws IOException { + try { + roundTrip(true); + } catch (IOException e) { + throw e; + } catch (Exception e) { + throw new IOException(e); + } + } + + @Override + protected void testOnUnix() throws IOException { + try { + roundTrip(false); + } catch (IOException e) { + throw e; + } catch (Exception e) { + throw new IOException(e); + } + } + + /** + * Exports the schema, drops it, imports the exported file again and compares the tags and + * attributes read back from the server with the ones that were created. + */ + private void roundTrip(boolean windows) throws Exception { + final Path outputDirectory = folder.newFolder("export").toPath(); + + createSchema(); + + testOutput( + exportProcessBuilder(windows, outputDirectory), + new String[] {Constants.EXPORT_COMPLETELY}, + 0); + + final Path csv = singleExportFile(outputDirectory); + assertExportFormat(csv); + + dropSchema(); + + testOutput(importProcessBuilder(windows, csv), new String[] {Constants.IMPORT_COMPLETELY}, 0); + + assertSchemaRestored(); + } + + private static void createSchema() throws IoTDBConnectionException, StatementExecutionException { + try (ISession session = EnvFactory.getEnv().getSessionConnection()) { + session.createDatabase(DATABASE); + for (Map.Entry entry : CREATED.entrySet()) { + final Series series = entry.getValue(); + // Set the tags and attributes through the session API, so that no SQL escaping can hide a + // character that the export or the import tool mangles. + session.createTimeseries( + entry.getKey(), + series.format.dataType, + series.format.encoding, + series.format.compression, + null, + series.tags.isEmpty() ? null : series.tags, + series.attributes.isEmpty() ? null : series.attributes, + series.alias); + } + } + } + + private static void dropSchema() throws IoTDBConnectionException, StatementExecutionException { + try (ISession session = EnvFactory.getEnv().getSessionConnection()) { + session.executeNonQueryStatement("DROP DATABASE " + DATABASE); + } + // The drop reaches the DataNodes asynchronously. Waiting until it is observable keeps a + // leftover device from making the import pass without restoring anything. + Awaitility.await() + .atMost(SCHEMA_CLEANUP_TIMEOUT_SECONDS, TimeUnit.SECONDS) + .pollInterval(500, TimeUnit.MILLISECONDS) + .until(() -> !databaseExists() && readSchema().isEmpty()); + try (ISession session = EnvFactory.getEnv().getSessionConnection()) { + // Importing a file only creates time series, so the database has to exist beforehand. + session.createDatabase(DATABASE); + } + } + + private static boolean databaseExists() + throws IoTDBConnectionException, StatementExecutionException { + try (ISession session = EnvFactory.getEnv().getSessionConnection(); + SessionDataSet dataSet = session.executeQueryStatement("SHOW DATABASES")) { + while (dataSet.hasNext()) { + if (DATABASE.equals(cell(dataSet.next().getFields(), 0))) { + return true; + } + } + } + return false; + } + + private Path singleExportFile(Path outputDirectory) throws IOException { + try (Stream files = Files.list(outputDirectory)) { + final List csvFiles = + files + .filter(file -> file.getFileName().toString().startsWith(EXPORT_PREFIX)) + .filter(file -> file.getFileName().toString().endsWith(".csv")) + .collect(Collectors.toList()); + assertEquals("expected exactly one exported CSV file, found " + csvFiles, 1, csvFiles.size()); + return csvFiles.get(0); + } + } + + private static void assertExportFormat(Path csv) throws IOException { + final List lines = Files.readAllLines(csv, StandardCharsets.UTF_8); + assertTrue("the exported schema file is empty", lines.size() > 1); + assertEquals(HEADER, lines.get(0)); + + final String content = Files.readString(csv, StandardCharsets.UTF_8); + // RFC4180 uses LF as the record separator; a CR would come from a platform dependent writer. + assertFalse("the export must use LF line endings", content.contains("\r")); + // RFC4180: the JSON object of the value containing a double quote is wrapped in a quoted cell + // and every inner quote is doubled, instead of being escaped with a backslash. + final String quotedCell = "\"" + QUOTE_ROW_TAGS_JSON.replace("\"", "\"\"") + "\""; + assertTrue( + "the Tags cell of " + + QUOTE_ROW + + " is not RFC4180 quoted: " + + quotedCell + + " in\n" + + content, + content.contains(quotedCell)); + + try (CSVParser parser = CSVParser.parse(csv, StandardCharsets.UTF_8, csvFormat())) { + final Map records = new TreeMap<>(); + for (CSVRecord record : parser) { + records.put(record.get("Timeseries"), record); + } + assertEquals( + "unexpected number of exported rows: " + records, CREATED.size(), records.size()); + + final Set formats = EnumSet.noneOf(Format.class); + for (Map.Entry entry : CREATED.entrySet()) { + final String path = entry.getKey(); + final Series series = entry.getValue(); + final CSVRecord record = records.get(path); + assertNotNull("the export is missing " + path, record); + assertEquals(path + " DataType", series.format.dataType.name(), record.get("DataType")); + assertEquals(path + " Encoding", series.format.encoding.name(), record.get("Encoding")); + assertEquals( + path + " Compression", series.format.compression.name(), record.get("Compression")); + assertEquals( + path + " Alias", series.alias == null ? "" : series.alias, record.get("Alias")); + assertEquals(path + " Tags", series.tags, parseJsonObject(record.get("Tags"))); + assertEquals( + path + " Attributes", series.attributes, parseJsonObject(record.get("Attributes"))); + // The three assertions above proved that the file carries this combination. + formats.add(series.format); + } + assertEquals( + "the file must carry every data type, encoding and compression combination", + EnumSet.allOf(Format.class), + formats); + + // Nothing to write is written as an empty cell, not as {} and not as null. + assertEquals("", records.get(NO_ATTRIBUTES_ROW).get("Tags")); + assertEquals("", records.get(NO_ATTRIBUTES_ROW).get("Attributes")); + + assertPresenceCombinations(records); + } + } + + /** + * The one exported file must carry all four tag/attribute combinations, spread over several + * devices, and at least one device must mix several of them across its measurements. + */ + private static void assertPresenceCombinations(Map records) { + final Set presences = EnumSet.noneOf(Presence.class); + final Map> presencesByDevice = new TreeMap<>(); + for (Map.Entry entry : records.entrySet()) { + final String path = entry.getKey(); + final CSVRecord record = entry.getValue(); + final Presence presence = + Presence.of( + parseJsonObject(record.get("Tags")), parseJsonObject(record.get("Attributes"))); + presences.add(presence); + presencesByDevice + .computeIfAbsent( + path.substring(0, path.lastIndexOf('.')), device -> EnumSet.noneOf(Presence.class)) + .add(presence); + } + assertEquals( + "the file must carry every tag/attribute combination", + EnumSet.allOf(Presence.class), + presences); + assertTrue( + "the combinations must be spread over at least four devices: " + presencesByDevice, + presencesByDevice.size() >= 4); + assertTrue( + "one device must mix several combinations across its measurements: " + presencesByDevice, + presencesByDevice.values().stream().mapToInt(Set::size).max().orElse(0) >= 3); + } + + private static void assertSchemaRestored() + throws IoTDBConnectionException, StatementExecutionException { + final Map restored = readSchema(); + assertEquals("the import must restore every time series", CREATED.keySet(), restored.keySet()); + for (Map.Entry entry : CREATED.entrySet()) { + final String path = entry.getKey(); + final Series expected = entry.getValue(); + final Series actual = restored.get(path); + assertEquals(path + " data type/encoding/compression", expected.format, actual.format); + assertEquals(path + " tags", expected.tags, actual.tags); + assertEquals(path + " attributes", expected.attributes, actual.attributes); + assertEquals(path + " alias", expected.alias, actual.alias); + } + } + + /** Reads the tags, attributes and alias of every time series of the test database. */ + private static Map readSchema() + throws IoTDBConnectionException, StatementExecutionException { + final Map schema = new TreeMap<>(); + try (ISession session = EnvFactory.getEnv().getSessionConnection(); + SessionDataSet dataSet = + session.executeQueryStatement("SHOW TIMESERIES " + DATABASE + ".**")) { + final List columns = dataSet.getColumnNames(); + final int timeseriesIndex = columns.indexOf("Timeseries"); + final int aliasIndex = columns.indexOf("Alias"); + final int dataTypeIndex = columns.indexOf("DataType"); + final int encodingIndex = columns.indexOf("Encoding"); + final int compressionIndex = columns.indexOf("Compression"); + final int tagsIndex = columns.indexOf("Tags"); + final int attributesIndex = columns.indexOf("Attributes"); + assertTrue( + "SHOW TIMESERIES must expose the Timeseries, Alias, DataType, Encoding, Compression, " + + "Tags and Attributes columns: " + + columns, + timeseriesIndex >= 0 + && aliasIndex >= 0 + && dataTypeIndex >= 0 + && encodingIndex >= 0 + && compressionIndex >= 0 + && tagsIndex >= 0 + && attributesIndex >= 0); + while (dataSet.hasNext()) { + final List fields = dataSet.next().getFields(); + final String path = cell(fields, timeseriesIndex); + schema.put( + path, + new Series( + Format.of( + cell(fields, dataTypeIndex), + cell(fields, encodingIndex), + cell(fields, compressionIndex)), + parseJsonObject(cell(fields, tagsIndex)), + parseJsonObject(cell(fields, attributesIndex)), + cell(fields, aliasIndex))); + } + } + return schema; + } + + /** Returns a cell of a schema row, or {@code null} when the server sent a NULL there. */ + private static String cell(List fields, int index) { + if (index < 0 || index >= fields.size()) { + return null; + } + final Field field = fields.get(index); + if (field == null) { + return null; + } + // A NULL field is rendered as the text "null" by getStringValue(), which is how the export + // tool distinguishes a missing value as well. + final String value = field.getStringValue(); + return "null".equals(value) ? null : value; + } + + /** Parses a JSON object cell into a map; an empty cell carries no tags and no attributes. */ + private static Map parseJsonObject(String cell) { + if (cell == null || cell.trim().isEmpty()) { + return new TreeMap<>(); + } + final Map parsed = GSON.fromJson(cell, JSON_MAP_TYPE); + return parsed == null ? new TreeMap<>() : new TreeMap<>(parsed); + } + + private static CSVFormat csvFormat() { + return CSVFormat.Builder.create(CSVFormat.DEFAULT) + .setHeader() + .setSkipHeaderRecord(true) + .build(); + } + + private ProcessBuilder exportProcessBuilder(boolean windows, Path outputDirectory) { + final List command = new ArrayList<>(); + addScriptInvocation(command, windows, "export-schema"); + addConnectionArguments(command); + command.addAll( + Arrays.asList( + "-t", + outputDirectory.toAbsolutePath().toString(), + "-pfn", + EXPORT_PREFIX, + "-path", + DATABASE + ".**")); + return processBuilder(command, windows); + } + + private ProcessBuilder importProcessBuilder(boolean windows, Path csv) { + final List command = new ArrayList<>(); + addScriptInvocation(command, windows, "import-schema"); + addConnectionArguments(command); + command.addAll(Arrays.asList("-db", DATABASE, "-s", csv.toAbsolutePath().toString())); + return processBuilder(command, windows); + } + + private static void addConnectionArguments(List command) { + command.addAll( + Arrays.asList( + "-h", + ip, + "-p", + port, + "-u", + SessionConfig.DEFAULT_USER, + "-pw", + SessionConfig.DEFAULT_PASSWORD)); + } + + private static void addScriptInvocation(List command, boolean windows, String script) { + if (windows) { + command.add("cmd.exe"); + command.add("/c"); + command.add( + toolsPath + + File.separator + + "windows" + + File.separator + + "schema" + + File.separator + + script + + ".bat"); + } else { + command.add("bash"); + command.add(toolsPath + File.separator + "schema" + File.separator + script + ".sh"); + } + } + + private static ProcessBuilder processBuilder(List command, boolean windows) { + if (windows) { + // cmd.exe exits with the error level of the script it ran. + command.add("&"); + command.add("exit"); + command.add("%^errorlevel%"); + } + final ProcessBuilder builder = new ProcessBuilder(command); + builder.environment().put("IOTDB_HOME", homePath); + return builder; + } + + private static void singleTag(String device, String value, Format format) { + tagged(device, "k", value, format); + } + + private static void tagged(String device, String key, String value, Format format) { + series(device, "s1", format, mapOf(key, value), NO_VALUES, null); + } + + private static void series( + String device, + String measurement, + Format format, + Map tags, + Map attributes, + String alias) { + CREATED.put( + DATABASE + "." + device + "." + measurement, new Series(format, tags, attributes, alias)); + } + + private static Map mapOf(String... keysAndValues) { + final Map map = new LinkedHashMap<>(); + for (int i = 0; i < keysAndValues.length; i += 2) { + map.put(keysAndValues[i], keysAndValues[i + 1]); + } + return map; + } + + /** + * A data type, encoding and compression combination of the test schema. Every combination has to + * appear in the exported file, and the server renders them with these names. + */ + private enum Format { + BOOLEAN_PLAIN_SNAPPY(TSDataType.BOOLEAN, TSEncoding.PLAIN, CompressionType.SNAPPY), + INT32_TS_2DIFF_SNAPPY(TSDataType.INT32, TSEncoding.TS_2DIFF, CompressionType.SNAPPY), + INT64_GORILLA_LZ4(TSDataType.INT64, TSEncoding.GORILLA, CompressionType.LZ4), + DOUBLE_GORILLA_LZ4(TSDataType.DOUBLE, TSEncoding.GORILLA, CompressionType.LZ4), + FLOAT_GORILLA_GZIP(TSDataType.FLOAT, TSEncoding.GORILLA, CompressionType.GZIP), + TEXT_DICTIONARY_SNAPPY(TSDataType.TEXT, TSEncoding.DICTIONARY, CompressionType.SNAPPY); + + private final TSDataType dataType; + private final TSEncoding encoding; + private final CompressionType compression; + + Format(TSDataType dataType, TSEncoding encoding, CompressionType compression) { + this.dataType = dataType; + this.encoding = encoding; + this.compression = compression; + } + + /** Resolves the combination rendered by {@code SHOW TIMESERIES}, i.e. by the export. */ + private static Format of(String dataType, String encoding, String compression) { + for (Format format : values()) { + if (format.dataType.name().equals(dataType) + && format.encoding.name().equals(encoding) + && format.compression.name().equals(compression)) { + return format; + } + } + throw new AssertionError( + "unexpected data type/encoding/compression combination: " + + dataType + + "/" + + encoding + + "/" + + compression); + } + } + + /** Whether a time series carries tags and attributes; all four must coexist in one file. */ + private enum Presence { + BOTH, + TAGS_ONLY, + ATTRIBUTES_ONLY, + NEITHER; + + private static Presence of(Map tags, Map attributes) { + if (!tags.isEmpty()) { + return attributes.isEmpty() ? TAGS_ONLY : BOTH; + } + return attributes.isEmpty() ? NEITHER : ATTRIBUTES_ONLY; + } + } + + /** One time series of the test schema, i.e. one row of the exported file. */ + private static final class Series { + private final Format format; + private final Map tags; + private final Map attributes; + private final String alias; + + private Series( + Format format, Map tags, Map attributes, String alias) { + this.format = format; + this.tags = tags; + this.attributes = attributes; + this.alias = alias; + } + } +} diff --git a/iotdb-client/cli/pom.xml b/iotdb-client/cli/pom.xml index a1e00508740bd..5302c86b8fd81 100644 --- a/iotdb-client/cli/pom.xml +++ b/iotdb-client/cli/pom.xml @@ -144,6 +144,10 @@ org.apache.commons commons-csv + + com.google.code.gson + gson + junit junit diff --git a/iotdb-client/cli/src/main/i18n/en/org/apache/iotdb/cli/i18n/CliMessages.java b/iotdb-client/cli/src/main/i18n/en/org/apache/iotdb/cli/i18n/CliMessages.java index 8cd97e9b07884..5ac63fff12c65 100644 --- a/iotdb-client/cli/src/main/i18n/en/org/apache/iotdb/cli/i18n/CliMessages.java +++ b/iotdb-client/cli/src/main/i18n/en/org/apache/iotdb/cli/i18n/CliMessages.java @@ -157,6 +157,13 @@ public final class CliMessages { MESSAGE_THE_HEADER_OF_THE_CSV_FILE_TO_BE_IMPORTED_IS_ILLEGAL_THE_CORRECT_FORMAT_IS_TIMESERIES_ALIBABA_DATATYPE_ENCODING_COMPRESSION_C429E7C6 = "The header of the CSV file to be imported is illegal. The correct format is" + " \"Timeseries, Alibaba, DataType, Encoding, Compression\"!"; + public static final String + MESSAGE_THE_HEADER_OF_THE_CSV_FILE_TO_BE_IMPORTED_IS_ILLEGAL_THE_CORRECT_FORMAT_IS_TIMESERIES_ALIAS_DATATYPE_ENCODING_COMPRESSION_TAGS_ATTRIBUTES_14EDBFB9 = + "The header of the CSV file to be imported is illegal. The correct format is" + + " \"Timeseries, Alias, DataType, Encoding, Compression, Tags, Attributes\"!"; + public static final String + MESSAGE_FAILED_TO_PARSE_THE_ARG_COLUMN_OF_TIMESERIES_ARG_AS_JSON_THE_RECORD_IS_SKIPPED_ARG_35D898C8 = + "Failed to parse the %s column of timeseries %s as JSON, the record is skipped: %s"; private CliMessages() {} diff --git a/iotdb-client/cli/src/main/i18n/zh/org/apache/iotdb/cli/i18n/CliMessages.java b/iotdb-client/cli/src/main/i18n/zh/org/apache/iotdb/cli/i18n/CliMessages.java index 6123237db8d74..7efdc8d83ec7e 100644 --- a/iotdb-client/cli/src/main/i18n/zh/org/apache/iotdb/cli/i18n/CliMessages.java +++ b/iotdb-client/cli/src/main/i18n/zh/org/apache/iotdb/cli/i18n/CliMessages.java @@ -145,6 +145,13 @@ public final class CliMessages { public static final String MESSAGE_THE_HEADER_OF_THE_CSV_FILE_TO_BE_IMPORTED_IS_ILLEGAL_THE_CORRECT_FORMAT_IS_TIMESERIES_ALIBABA_DATATYPE_ENCODING_COMPRESSION_C429E7C6 = "待导入的 CSV 文件表头不合法,正确格式为 \"Timeseries, Alibaba, DataType, Encoding, Compression\"!"; + public static final String + MESSAGE_THE_HEADER_OF_THE_CSV_FILE_TO_BE_IMPORTED_IS_ILLEGAL_THE_CORRECT_FORMAT_IS_TIMESERIES_ALIAS_DATATYPE_ENCODING_COMPRESSION_TAGS_ATTRIBUTES_14EDBFB9 = + "待导入的 CSV 文件表头不合法,正确格式为" + + " \"Timeseries, Alias, DataType, Encoding, Compression, Tags, Attributes\"!"; + public static final String + MESSAGE_FAILED_TO_PARSE_THE_ARG_COLUMN_OF_TIMESERIES_ARG_AS_JSON_THE_RECORD_IS_SKIPPED_ARG_35D898C8 = + "时序 JSON 解析失败,列:%s,时序:%s,已跳过该记录,原因:%s"; private CliMessages() {} diff --git a/iotdb-client/cli/src/main/java/org/apache/iotdb/tool/common/Constants.java b/iotdb-client/cli/src/main/java/org/apache/iotdb/tool/common/Constants.java index ec4d2c7632cbc..e2c96f4d2e744 100644 --- a/iotdb-client/cli/src/main/java/org/apache/iotdb/tool/common/Constants.java +++ b/iotdb-client/cli/src/main/java/org/apache/iotdb/tool/common/Constants.java @@ -204,8 +204,31 @@ public class Constants { public static final String COLON = ": "; public static final String MINUS = "-"; + public static final String HEADER_TAGS = "Tags"; + public static final String HEADER_ATTRIBUTES = "Attributes"; + public static final String HEADER_ALIAS = "Alias"; + public static final String HEADER_DATATYPE = "DataType"; + public static final String HEADER_ENCODING = "Encoding"; + public static final String HEADER_COMPRESSION = "Compression"; + public static final List HEAD_COLUMNS = - Arrays.asList("Timeseries", "Alias", "DataType", "Encoding", "Compression"); + Arrays.asList( + "Timeseries", HEADER_ALIAS, HEADER_DATATYPE, HEADER_ENCODING, HEADER_COMPRESSION); + + /** + * Column layout written by the tree model schema export. The Tags and Attributes cells hold a + * JSON object, so files carrying them are parsed with the RFC4180 settings instead of the legacy + * ones. + */ + public static final List HEAD_COLUMNS_V2 = + Arrays.asList( + "Timeseries", + HEADER_ALIAS, + HEADER_DATATYPE, + HEADER_ENCODING, + HEADER_COMPRESSION, + HEADER_TAGS, + HEADER_ATTRIBUTES); // export constants public static final String EXPORT_CLI_PREFIX = CliMessages.EXCEPTION_EXPORT_DATA_2F1FB3D9; diff --git a/iotdb-client/cli/src/main/java/org/apache/iotdb/tool/schema/AbstractSchemaTool.java b/iotdb-client/cli/src/main/java/org/apache/iotdb/tool/schema/AbstractSchemaTool.java index b88cfda2c0a32..f9998491a1f41 100644 --- a/iotdb-client/cli/src/main/java/org/apache/iotdb/tool/schema/AbstractSchemaTool.java +++ b/iotdb-client/cli/src/main/java/org/apache/iotdb/tool/schema/AbstractSchemaTool.java @@ -30,6 +30,9 @@ import org.apache.iotdb.session.pool.TableSessionPoolBuilder; import org.apache.iotdb.tool.common.Constants; +import com.google.gson.Gson; +import com.google.gson.GsonBuilder; +import com.google.gson.reflect.TypeToken; import org.apache.commons.cli.CommandLine; import org.apache.commons.csv.CSVFormat; import org.apache.commons.csv.CSVPrinter; @@ -41,8 +44,11 @@ import java.io.IOException; import java.io.PrintWriter; +import java.lang.reflect.Type; import java.util.Iterator; import java.util.List; +import java.util.Map; +import java.util.TreeMap; import java.util.concurrent.atomic.LongAdder; public abstract class AbstractSchemaTool { @@ -77,6 +83,9 @@ public abstract class AbstractSchemaTool { private static final IoTPrinter ioTPrinter = new IoTPrinter(System.out); private static final Logger LOGGER = LoggerFactory.getLogger(AbstractSchemaTool.class); + private static final Gson JSON_GSON = new GsonBuilder().disableHtmlEscaping().create(); + private static final Type JSON_CELL_TYPE = new TypeToken>() {}.getType(); + protected AbstractSchemaTool() {} protected static Session.Builder configureSsl(Session.Builder builder) { @@ -218,6 +227,51 @@ public static Boolean writeCsvFile(List> records, String filePath) } } + /** + * The RFC4180 CSV layout of the schema files, i.e. the delimiter is {@code ','}, the quote char + * is {@code '"'} and no escape char is configured. A JSON object cell is therefore always quoted + * and its inner quotes are doubled, which is exactly what RFC4180 asks for. + */ + static CSVFormat.Builder createRfc4180CsvFormat() { + return CSVFormat.Builder.create(CSVFormat.DEFAULT) + .setQuote('"') + .setQuoteMode(QuoteMode.MINIMAL) + .setRecordSeparator('\n'); + } + + /** + * Serializes tags or attributes into the compact JSON object cell of the schema file. Keys are + * sorted so that repeated exports of the same schema are identical, and HTML escaping is disabled + * so that printable characters such as {@code '}, {@code =}, {@code <} and {@code &} stay + * readable. + * + * @return an empty string when there is nothing to write, never {@code {}} or {@code null} + */ + static String toJsonCell(Map values) { + if (values == null || values.isEmpty()) { + return ""; + } + return JSON_GSON.toJson(new TreeMap<>(values)); + } + + /** + * Parses a Tags / Attributes cell written by {@link #toJsonCell(Map)}. + * + * @return {@code null} when the cell carries no value, never an empty map + * @throws com.google.gson.JsonParseException when the cell is not a JSON object + */ + static Map parseJsonCell(String cell) { + if (cell == null) { + return null; + } + String trimmed = cell.trim(); + if (trimmed.isEmpty() || "null".equals(trimmed)) { + return null; + } + Map parsed = JSON_GSON.fromJson(trimmed, JSON_CELL_TYPE); + return parsed == null || parsed.isEmpty() ? null : new TreeMap<>(parsed); + } + static class CSVPrinterWrapper { private final String filePath; private final CSVFormat csvFormat; @@ -225,13 +279,7 @@ static class CSVPrinterWrapper { public CSVPrinterWrapper(String filePath) { this.filePath = filePath; - this.csvFormat = - CSVFormat.Builder.create(CSVFormat.DEFAULT) - .setHeader() - .setSkipHeaderRecord(true) - .setEscape('\\') - .setQuoteMode(QuoteMode.NONE) - .build(); + this.csvFormat = createRfc4180CsvFormat().setHeader().setSkipHeaderRecord(true).build(); } public void printRecord(final Iterable values) throws IOException { diff --git a/iotdb-client/cli/src/main/java/org/apache/iotdb/tool/schema/ExportSchemaTree.java b/iotdb-client/cli/src/main/java/org/apache/iotdb/tool/schema/ExportSchemaTree.java index b8cd60323e22c..12e3b5066485a 100644 --- a/iotdb-client/cli/src/main/java/org/apache/iotdb/tool/schema/ExportSchemaTree.java +++ b/iotdb-client/cli/src/main/java/org/apache/iotdb/tool/schema/ExportSchemaTree.java @@ -32,7 +32,9 @@ import java.io.File; import java.io.IOException; +import java.util.ArrayList; import java.util.List; +import java.util.Map; import static org.apache.iotdb.commons.schema.SchemaConstant.AUDIT_DATABASE; import static org.apache.iotdb.commons.schema.SchemaConstant.SYSTEM_DATABASE; @@ -98,25 +100,19 @@ private static void writeCsvFile( while (i++ < linesPerFile) { if (sessionDataSet.hasNext()) { if (i == 1) { - csvPrinterWrapper.printRecord(Constants.HEAD_COLUMNS); + csvPrinterWrapper.printRecord(Constants.HEAD_COLUMNS_V2); } RowRecord rowRecord = sessionDataSet.next(); List fields = rowRecord.getFields(); - if (fields.get(timeseriesIndex).getStringValue().startsWith(SYSTEM_DATABASE + ".") - || fields.get(timeseriesIndex).getStringValue().startsWith(AUDIT_DATABASE + ".") - || !fields.get(viewTypeIndex).getStringValue().equals(Constants.BASE_VIEW_TYPE)) { + String timeseries = fields.get(timeseriesIndex).getStringValue(); + if (timeseries.startsWith(SYSTEM_DATABASE + ".") + || timeseries.startsWith(AUDIT_DATABASE + ".") + || !Constants.BASE_VIEW_TYPE.equals(fields.get(viewTypeIndex).getStringValue())) { + continue; + } + if (!printRecord(csvPrinterWrapper, headers, fields, timeseries)) { continue; } - Constants.HEAD_COLUMNS.forEach( - column -> { - Field field = fields.get(headers.indexOf(column)); - String fieldStringValue = field.getStringValue(); - if (!"null".equals(field.getStringValue())) { - csvPrinterWrapper.print(fieldStringValue); - } else { - csvPrinterWrapper.print(""); - } - }); csvPrinterWrapper.println(); } else { hasNext = false; @@ -128,4 +124,44 @@ private static void writeCsvFile( csvPrinterWrapper.close(); } } + + /** + * Writes the cells of one {@code SHOW TIMESERIES} record. + * + * @return {@code false} when the record must be skipped because a Tags or Attributes cell could + * not be parsed, in which case nothing has been written for it + */ + static boolean printRecord( + CSVPrinterWrapper csvPrinterWrapper, + List headers, + List fields, + String timeseries) { + List cells = new ArrayList<>(Constants.HEAD_COLUMNS_V2.size()); + for (String column : Constants.HEAD_COLUMNS_V2) { + int index = headers.indexOf(column); + Field field = index < 0 ? null : fields.get(index); + String fieldStringValue = field == null ? null : field.getStringValue(); + if (Constants.HEADER_TAGS.equals(column) || Constants.HEADER_ATTRIBUTES.equals(column)) { + Map tags; + try { + tags = parseJsonCell(fieldStringValue); + } catch (RuntimeException e) { + ioTPrinter.println( + String.format( + CliMessages + .MESSAGE_FAILED_TO_PARSE_THE_ARG_COLUMN_OF_TIMESERIES_ARG_AS_JSON_THE_RECORD_IS_SKIPPED_ARG_35D898C8, + column, + timeseries, + String.valueOf(e.getMessage()))); + return false; + } + cells.add(toJsonCell(tags)); + } else { + cells.add( + "null".equals(fieldStringValue) || fieldStringValue == null ? "" : fieldStringValue); + } + } + cells.forEach(csvPrinterWrapper::print); + return true; + } } diff --git a/iotdb-client/cli/src/main/java/org/apache/iotdb/tool/schema/ImportSchemaTree.java b/iotdb-client/cli/src/main/java/org/apache/iotdb/tool/schema/ImportSchemaTree.java index 231467b33fadc..9e0471a80d605 100644 --- a/iotdb-client/cli/src/main/java/org/apache/iotdb/tool/schema/ImportSchemaTree.java +++ b/iotdb-client/cli/src/main/java/org/apache/iotdb/tool/schema/ImportSchemaTree.java @@ -38,11 +38,13 @@ import org.apache.tsfile.file.metadata.enums.CompressionType; import org.apache.tsfile.file.metadata.enums.TSEncoding; +import java.io.BufferedReader; import java.io.File; import java.io.FileInputStream; import java.io.IOException; import java.io.InputStreamReader; import java.util.ArrayList; +import java.util.Arrays; import java.util.Collections; import java.util.HashSet; import java.util.List; @@ -58,7 +60,7 @@ public class ImportSchemaTree extends AbstractImportSchema { private static final IoTPrinter ioTPrinter = new IoTPrinter(System.out); - private static SessionPool sessionPool; + static SessionPool sessionPool; public void init() throws InterruptedException, IoTDBConnectionException, StatementExecutionException { @@ -131,18 +133,65 @@ protected void importSchemaFromCsvFile(File file) { } } - private static CSVParser readCsvFile(String path) throws IOException { - return CSVFormat.Builder.create(CSVFormat.DEFAULT) - .setHeader() - .setSkipHeaderRecord(true) - .setQuote('`') - .setEscape('\\') - .setIgnoreEmptyLines(true) - .build() - .parse(new InputStreamReader(new FileInputStream(path))); + static CSVParser readCsvFile(String path) throws IOException { + CSVFormat.Builder builder = + CSVFormat.Builder.create(CSVFormat.DEFAULT) + .setHeader() + .setSkipHeaderRecord(true) + .setIgnoreEmptyLines(true); + if (isV2Format(readHeaderNames(path))) { + builder.setQuote('"'); + } else { + builder.setQuote('`').setEscape('\\'); + } + return builder.build().parse(new InputStreamReader(new FileInputStream(path))); + } + + /** + * Reads the column names of a schema file without parsing it, so that the CSV settings of the + * file can be selected before it is read. + */ + static List readHeaderNames(String path) throws IOException { + try (BufferedReader reader = + new BufferedReader(new InputStreamReader(new FileInputStream(path)))) { + String headerLine = reader.readLine(); + if (headerLine == null) { + return Collections.emptyList(); + } + return Arrays.stream(headerLine.split(",")) + .map(ImportSchemaTree::unquoteColumnName) + .collect(Collectors.toList()); + } } - private static boolean checkHeader(List headerNames) { + /** Accepts a header written by a spreadsheet tool, which quotes every column name. */ + private static String unquoteColumnName(String columnName) { + String name = columnName.trim(); + if (name.length() > 1 && name.startsWith("\"") && name.endsWith("\"")) { + return name.substring(1, name.length() - 1).trim(); + } + return name; + } + + /** + * A file whose header carries a Tags or an Attributes column is a v2 file, which is RFC4180 and + * holds JSON cells. Everything else is a legacy file and keeps the settings it was written with. + */ + static boolean isV2Format(List headerNames) { + return headerNames.contains(Constants.HEADER_TAGS) + || headerNames.contains(Constants.HEADER_ATTRIBUTES); + } + + static boolean checkHeader(List headerNames) { + if (isV2Format(headerNames)) { + if (headerNames.containsAll(Constants.HEAD_COLUMNS_V2)) { + return true; + } + ioTPrinter.println( + CliMessages + .MESSAGE_THE_HEADER_OF_THE_CSV_FILE_TO_BE_IMPORTED_IS_ILLEGAL_THE_CORRECT_FORMAT_IS_TIMESERIES_ALIAS_DATATYPE_ENCODING_COMPRESSION_TAGS_ATTRIBUTES_14EDBFB9); + return false; + } if (CollectionUtils.isNotEmpty(headerNames) && new HashSet<>(headerNames).size() == Constants.HEAD_COLUMNS.size()) { List strangers = @@ -159,18 +208,25 @@ private static boolean checkHeader(List headerNames) { return true; } - private static void writeScheme( + static void writeScheme( String fileName, List headerNames, Stream records, String failedFilePath) throws IllegalPathException { + boolean v2 = isV2Format(headerNames); + int tagsIndex = headerNames.indexOf(Constants.HEADER_TAGS); + int attributesIndex = headerNames.indexOf(Constants.HEADER_ATTRIBUTES); List paths = new ArrayList<>(); List dataTypes = new ArrayList<>(); List encodings = new ArrayList<>(); List compressors = new ArrayList<>(); + List> tags = v2 ? new ArrayList<>() : null; + List> attributes = v2 ? new ArrayList<>() : null; List pathsWithAlias = new ArrayList<>(); List dataTypesWithAlias = new ArrayList<>(); List encodingsWithAlias = new ArrayList<>(); List compressorsWithAlias = new ArrayList<>(); List measurementAlias = new ArrayList<>(); + List> tagsWithAlias = v2 ? new ArrayList<>() : null; + List> attributesWithAlias = v2 ? new ArrayList<>() : null; AtomicReference hasStarted = new AtomicReference<>(false); AtomicInteger pointSize = new AtomicInteger(0); @@ -185,7 +241,7 @@ private static void writeScheme( try { if (CollectionUtils.isNotEmpty(paths)) { writeAndEmptyDataSet( - paths, dataTypes, encodings, compressors, null, null, null, null, 3); + paths, dataTypes, encodings, compressors, null, tags, attributes, null, 3); } } catch (Exception e) { paths.forEach(t -> failedRecords.add(Collections.singletonList(t))); @@ -198,8 +254,8 @@ private static void writeScheme( encodingsWithAlias, compressorsWithAlias, null, - null, - null, + tagsWithAlias, + attributesWithAlias, measurementAlias, 3); } @@ -210,7 +266,17 @@ private static void writeScheme( dataTypes.clear(); encodings.clear(); compressors.clear(); + pathsWithAlias.clear(); + dataTypesWithAlias.clear(); + encodingsWithAlias.clear(); + compressorsWithAlias.clear(); measurementAlias.clear(); + if (v2) { + tags.clear(); + attributes.clear(); + tagsWithAlias.clear(); + attributesWithAlias.clear(); + } pointSize.set(0); } } else { @@ -218,7 +284,17 @@ private static void writeScheme( dataTypes.clear(); encodings.clear(); compressors.clear(); + pathsWithAlias.clear(); + dataTypesWithAlias.clear(); + encodingsWithAlias.clear(); + compressorsWithAlias.clear(); measurementAlias.clear(); + if (v2) { + tags.clear(); + attributes.clear(); + tagsWithAlias.clear(); + attributesWithAlias.clear(); + } } String path = recordObj.get(headerNames.indexOf(Constants.HEAD_COLUMNS.get(0))); String alias = recordObj.get(headerNames.indexOf(Constants.HEAD_COLUMNS.get(1))); @@ -230,6 +306,23 @@ private static void writeScheme( String compressionTypeRaw = recordObj.get(headerNames.indexOf(Constants.HEAD_COLUMNS.get(4))); CompressionType compressionType = compressInfer(compressionTypeRaw); + Map recordTags = null; + Map recordAttributes = null; + boolean tagsParsed = true; + if (v2) { + try { + recordTags = parseJsonCell(recordObj.get(tagsIndex)); + } catch (RuntimeException e) { + ioTPrinter.println(invalidJsonMessage(Constants.HEADER_TAGS, path, e)); + tagsParsed = false; + } + try { + recordAttributes = parseJsonCell(recordObj.get(attributesIndex)); + } catch (RuntimeException e) { + ioTPrinter.println(invalidJsonMessage(Constants.HEADER_ATTRIBUTES, path, e)); + tagsParsed = false; + } + } if (StringUtils.isBlank(path) || path.trim().startsWith(SYSTEM_DATABASE) || path.trim().startsWith(AUDIT_DATABASE)) { @@ -268,31 +361,56 @@ private static void writeScheme( compressionTypeRaw)); failedRecords.add(recordObj.stream().collect(Collectors.toList())); failed = true; + } else if (!tagsParsed) { + failedRecords.add(recordObj.stream().collect(Collectors.toList())); + failed = true; } else { if (StringUtils.isBlank(alias)) { paths.add(path); dataTypes.add(dataType); encodings.add(encodingType); compressors.add(compressionType); + if (v2) { + tags.add(recordTags); + attributes.add(recordAttributes); + } } else { pathsWithAlias.add(path); dataTypesWithAlias.add(dataType); encodingsWithAlias.add(encodingType); compressorsWithAlias.add(compressionType); measurementAlias.add(alias); + if (v2) { + tagsWithAlias.add(recordTags); + attributesWithAlias.add(recordAttributes); + } } pointSize.getAndIncrement(); } if (!failed && aligned) { String deviceId = path.substring(0, path.lastIndexOf(".")); paths.add(0, path.substring(deviceId.length() + 1)); + if (v2) { + // keep the maps aligned with the measurements list, whose head now holds this one + tags.add(0, recordTags); + attributes.add(0, recordAttributes); + } writeAndEmptyDataSetAligned( - deviceId, paths, dataTypes, encodings, compressors, measurementAlias, 3); + deviceId, + paths, + dataTypes, + encodings, + compressors, + measurementAlias, + tags, + attributes, + 3); } }); try { if (CollectionUtils.isNotEmpty(paths)) { - writeAndEmptyDataSet(paths, dataTypes, encodings, compressors, null, null, null, null, 3); + writeAndEmptyDataSet( + paths, dataTypes, encodings, compressors, null, tags, attributes, null, 3); } } catch (Exception e) { paths.forEach(t -> failedRecords.add(Collections.singletonList(t))); @@ -305,8 +423,8 @@ private static void writeScheme( encodingsWithAlias, compressorsWithAlias, null, - null, - null, + tagsWithAlias, + attributesWithAlias, measurementAlias, 3); } @@ -380,10 +498,24 @@ private static void writeAndEmptyDataSetAligned( List encodings, List compressors, List measurementAliasList, + List> tagsList, + List> attributesList, int retryTime) { try { - sessionPool.createAlignedTimeseries( - deviceId, measurements, dataTypes, encodings, compressors, measurementAliasList); + if (tagsList == null && attributesList == null) { + sessionPool.createAlignedTimeseries( + deviceId, measurements, dataTypes, encodings, compressors, measurementAliasList); + } else { + sessionPool.createAlignedTimeseries( + deviceId, + measurements, + dataTypes, + encodings, + compressors, + measurementAliasList, + tagsList, + attributesList); + } } catch (IoTDBConnectionException e) { if (retryTime > 0) { writeAndEmptyDataSetAligned( @@ -393,6 +525,8 @@ private static void writeAndEmptyDataSetAligned( encodings, compressors, measurementAliasList, + tagsList, + attributesList, --retryTime); } } catch (StatementExecutionException e) { @@ -405,9 +539,24 @@ private static void writeAndEmptyDataSetAligned( encodings.clear(); compressors.clear(); measurementAliasList.clear(); + if (tagsList != null) { + tagsList.clear(); + } + if (attributesList != null) { + attributesList.clear(); + } } } + private static String invalidJsonMessage(String column, String path, Exception e) { + return String.format( + CliMessages + .MESSAGE_FAILED_TO_PARSE_THE_ARG_COLUMN_OF_TIMESERIES_ARG_AS_JSON_THE_RECORD_IS_SKIPPED_ARG_35D898C8, + column, + path, + String.valueOf(e.getMessage())); + } + private static TSDataType typeInfer(String typeStr) { try { if (StringUtils.isNotBlank(typeStr)) { diff --git a/iotdb-client/cli/src/test/java/org/apache/iotdb/tool/schema/SchemaTagsFidelityTest.java b/iotdb-client/cli/src/test/java/org/apache/iotdb/tool/schema/SchemaTagsFidelityTest.java new file mode 100644 index 0000000000000..85fd629ddbbbb --- /dev/null +++ b/iotdb-client/cli/src/test/java/org/apache/iotdb/tool/schema/SchemaTagsFidelityTest.java @@ -0,0 +1,318 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.tool.schema; + +import org.apache.iotdb.tool.common.Constants; + +import org.apache.commons.csv.CSVParser; +import org.apache.commons.csv.CSVRecord; +import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.read.common.Field; +import org.apache.tsfile.utils.Binary; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.TemporaryFolder; + +import java.io.File; +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.util.Arrays; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertTrue; + +/** Guards the lossless Tags / Attributes round trip of the tree model schema tools. */ +public class SchemaTagsFidelityTest { + + @Rule public TemporaryFolder temporaryFolder = new TemporaryFolder(); + + @Test + public void jsonCellSortsKeysAndKeepsPrintableCharactersReadable() { + Map tags = new LinkedHashMap<>(); + tags.put("type", "sensor"); + tags.put("room", "101"); + tags.put("a-b", "it's = & w"); + + assertEquals( + "{\"a-b\":\"it's = & w\",\"room\":\"101\",\"type\":\"sensor\"}", + AbstractSchemaTool.toJsonCell(tags)); + } + + @Test + public void jsonCellEscapesOnlyWhatJsonRequires() { + Map tags = new LinkedHashMap<>(); + tags.put("quote", "a\"b"); + tags.put("backslash", "a\\b"); + tags.put("tab", "a\tb"); + tags.put("newline", "a\nb"); + tags.put("comma", "a,b"); + tags.put("cjk", "中文😀"); + tags.put("backtick", "`x`"); + tags.put("null", "null"); + tags.put("empty", ""); + + assertEquals( + "{\"backslash\":\"a\\\\b\",\"backtick\":\"`x`\",\"cjk\":\"中文😀\",\"comma\":\"a,b\"," + + "\"empty\":\"\",\"newline\":\"a\\nb\",\"null\":\"null\",\"quote\":\"a\\\"b\"," + + "\"tab\":\"a\\tb\"}", + AbstractSchemaTool.toJsonCell(tags)); + } + + @Test + public void jsonCellOfMissingValuesIsAnEmptyCell() { + assertEquals("", AbstractSchemaTool.toJsonCell(null)); + assertEquals("", AbstractSchemaTool.toJsonCell(new LinkedHashMap<>())); + } + + @Test + public void jsonCellRoundTripsEveryCharacter() { + Map tags = new LinkedHashMap<>(); + tags.put("quote", "a\"b"); + tags.put("backslash", "a\\b"); + tags.put("tab", "a\tb"); + tags.put("newline", "a\nb"); + tags.put("comma", "a,b"); + tags.put("cjk", "中文😀"); + tags.put("backtick", "`x`"); + tags.put("null", "null"); + tags.put("empty", ""); + tags.put("key with space", "[ C ]"); + + assertEquals(tags, AbstractSchemaTool.parseJsonCell(AbstractSchemaTool.toJsonCell(tags))); + } + + @Test + public void jsonCellOfNothingParsesToNoValue() { + assertNull(AbstractSchemaTool.parseJsonCell(null)); + assertNull(AbstractSchemaTool.parseJsonCell("")); + assertNull(AbstractSchemaTool.parseJsonCell(" ")); + assertNull(AbstractSchemaTool.parseJsonCell("null")); + assertNull(AbstractSchemaTool.parseJsonCell("{}")); + } + + @Test + public void invalidJsonCellIsRejected() { + for (String cell : Arrays.asList("not json", "[1,2]", "\"abc\"", "{\"k\":\"v\"} trailing")) { + try { + AbstractSchemaTool.parseJsonCell(cell); + throw new AssertionError("expected a parse failure for " + cell); + } catch (RuntimeException expected) { + // the export reports this and skips the record + } + } + } + + @Test + public void csvRoundTripKeepsEveryFieldByteIdentical() throws IOException { + Map tags = new LinkedHashMap<>(); + tags.put("quote", "a\"b"); + tags.put("backslash", "a\\b"); + tags.put("comma", "a,b"); + tags.put("tab", "a\tb"); + tags.put("newline", "a\nb"); + Map attributes = new LinkedHashMap<>(); + attributes.put("owner", "team a"); + attributes.put("note", "q\"w\\e,r\ty\nz"); + + List row = + Arrays.asList( + "root.t.d1.s1", + "aliasA", + "INT64", + "TS_2DIFF", + "SNAPPY", + AbstractSchemaTool.toJsonCell(tags), + AbstractSchemaTool.toJsonCell(attributes)); + File csv = writeV2File("roundtrip.csv", row); + + CSVParser parser = ImportSchemaTree.readCsvFile(csv.getAbsolutePath()); + List records = parser.getRecords(); + assertEquals(1, records.size()); + CSVRecord record = records.get(0); + for (int i = 0; i < row.size(); i++) { + assertEquals(row.get(i), record.get(Constants.HEAD_COLUMNS_V2.get(i))); + } + assertEquals(tags, AbstractSchemaTool.parseJsonCell(record.get(Constants.HEADER_TAGS))); + assertEquals( + attributes, AbstractSchemaTool.parseJsonCell(record.get(Constants.HEADER_ATTRIBUTES))); + } + + @Test + public void emptyTagsAndAttributesAreWrittenAsEmptyCells() throws IOException { + File csv = + writeV2File( + "empty.csv", Arrays.asList("root.t.d1.s1", "", "INT64", "TS_2DIFF", "SNAPPY", "", "")); + + assertEquals( + "Timeseries,Alias,DataType,Encoding,Compression,Tags,Attributes\n" + + "root.t.d1.s1,,INT64,TS_2DIFF,SNAPPY,,\n", + new String(Files.readAllBytes(csv.toPath()), StandardCharsets.UTF_8)); + CSVRecord record = ImportSchemaTree.readCsvFile(csv.getAbsolutePath()).getRecords().get(0); + assertNull(AbstractSchemaTool.parseJsonCell(record.get(Constants.HEADER_TAGS))); + assertNull(AbstractSchemaTool.parseJsonCell(record.get(Constants.HEADER_ATTRIBUTES))); + } + + @Test + public void jsonCellsAreRfc4180Quoted() throws IOException { + Map tags = new LinkedHashMap<>(); + tags.put("room", "101"); + tags.put("type", "sensor"); + File csv = + writeV2File( + "quoted.csv", + Arrays.asList( + "root.t.d1.s1", + "", + "INT64", + "TS_2DIFF", + "SNAPPY", + AbstractSchemaTool.toJsonCell(tags), + "")); + + assertEquals( + "Timeseries,Alias,DataType,Encoding,Compression,Tags,Attributes\n" + + "root.t.d1.s1,,INT64,TS_2DIFF,SNAPPY,\"{\"\"room\"\":\"\"101\"\"," + + "\"\"type\"\":\"\"sensor\"\"}\",\n", + new String(Files.readAllBytes(csv.toPath()), StandardCharsets.UTF_8)); + } + + @Test + public void legacyFiveColumnHeaderSelectsTheLegacyPath() { + assertFalse(ImportSchemaTree.isV2Format(Constants.HEAD_COLUMNS)); + assertTrue(ImportSchemaTree.isV2Format(Constants.HEAD_COLUMNS_V2)); + } + + @Test + public void quotedHeaderStillSelectsTheV2Path() throws IOException { + File csv = temporaryFolder.newFile("quoted_header.csv"); + Files.write( + csv.toPath(), + ("\"Timeseries\",\"Alias\",\"DataType\",\"Encoding\",\"Compression\",\"Tags\"," + + "\"Attributes\"\nroot.t.d1.s1,,INT64,TS_2DIFF,SNAPPY,,\n") + .getBytes(StandardCharsets.UTF_8)); + + assertTrue( + ImportSchemaTree.isV2Format(ImportSchemaTree.readHeaderNames(csv.getAbsolutePath()))); + } + + @Test + public void legacyFileKeepsItsBackslashEscapes() throws IOException { + File csv = temporaryFolder.newFile("legacy.csv"); + Files.write( + csv.toPath(), + ("Timeseries,Alias,DataType,Encoding,Compression\n" + + "root.sg.d.s1,a\\,b,INT64,TS_2DIFF,SNAPPY\n") + .getBytes(StandardCharsets.UTF_8)); + + CSVRecord record = ImportSchemaTree.readCsvFile(csv.getAbsolutePath()).getRecords().get(0); + assertEquals(5, record.size()); + assertEquals("a,b", record.get(Constants.HEADER_ALIAS)); + } + + @Test + public void exportedRecordCarriesTagsAndAttributes() throws IOException { + File csv = temporaryFolder.newFile("export.csv"); + AbstractSchemaTool.CSVPrinterWrapper printer = + new AbstractSchemaTool.CSVPrinterWrapper(csv.getAbsolutePath()); + printer.printRecord(Constants.HEAD_COLUMNS_V2); + + List headers = + Arrays.asList( + Constants.HEADER_TIMESERIES, + Constants.HEADER_ALIAS, + "Database", + Constants.HEADER_DATATYPE, + Constants.HEADER_ENCODING, + Constants.HEADER_COMPRESSION, + Constants.HEADER_TAGS, + Constants.HEADER_ATTRIBUTES, + Constants.HEADER_VIEW_TYPE); + List fields = + Arrays.asList( + text("root.t.d1.s1"), + emptyField(), + text("root.t"), + text("INT64"), + text("TS_2DIFF"), + text("SNAPPY"), + text("{\"room\": \"101\", \"type\": \"sensor\"}"), + emptyField(), + text(Constants.BASE_VIEW_TYPE)); + assertTrue(ExportSchemaTree.printRecord(printer, headers, fields, "root.t.d1.s1")); + printer.println(); + printer.close(); + + CSVRecord record = ImportSchemaTree.readCsvFile(csv.getAbsolutePath()).getRecords().get(0); + assertEquals("root.t.d1.s1", record.get(Constants.HEADER_TIMESERIES)); + assertEquals("", record.get(Constants.HEADER_ALIAS)); + assertEquals("INT64", record.get(Constants.HEADER_DATATYPE)); + assertEquals("TS_2DIFF", record.get(Constants.HEADER_ENCODING)); + assertEquals("SNAPPY", record.get(Constants.HEADER_COMPRESSION)); + assertEquals("{\"room\":\"101\",\"type\":\"sensor\"}", record.get(Constants.HEADER_TAGS)); + assertEquals("", record.get(Constants.HEADER_ATTRIBUTES)); + } + + @Test + public void exportedRecordWithBrokenJsonIsSkipped() throws IOException { + File csv = temporaryFolder.newFile("broken.csv"); + AbstractSchemaTool.CSVPrinterWrapper printer = + new AbstractSchemaTool.CSVPrinterWrapper(csv.getAbsolutePath()); + List headers = + Arrays.asList( + Constants.HEADER_TIMESERIES, Constants.HEADER_TAGS, Constants.HEADER_ATTRIBUTES); + List fields = Arrays.asList(text("root.t.d1.s1"), text("not json"), emptyField()); + + assertFalse(ExportSchemaTree.printRecord(printer, headers, fields, "root.t.d1.s1")); + printer.close(); + assertEquals(0, csv.length()); + } + + /** + * Writes a v2 file with the writer of the schema tools so that the reader is exercised against + * the bytes the export really produces. + */ + private File writeV2File(String name, List row) throws IOException { + File csv = temporaryFolder.newFile(name); + AbstractSchemaTool.CSVPrinterWrapper printer = + new AbstractSchemaTool.CSVPrinterWrapper(csv.getAbsolutePath()); + printer.printRecord(Constants.HEAD_COLUMNS_V2); + row.forEach(printer::print); + printer.println(); + printer.close(); + return csv; + } + + private static Field text(String value) { + Field field = new Field(TSDataType.TEXT); + field.setBinaryV(new Binary(value, StandardCharsets.UTF_8)); + return field; + } + + /** The shape SHOW TIMESERIES uses for a cell without a value. */ + private static Field emptyField() { + return new Field(null); + } +} diff --git a/iotdb-client/cli/src/test/java/org/apache/iotdb/tool/schema/SchemaTagsImportTest.java b/iotdb-client/cli/src/test/java/org/apache/iotdb/tool/schema/SchemaTagsImportTest.java new file mode 100644 index 0000000000000..4b4065c8ccf02 --- /dev/null +++ b/iotdb-client/cli/src/test/java/org/apache/iotdb/tool/schema/SchemaTagsImportTest.java @@ -0,0 +1,288 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.tool.schema; + +import org.apache.iotdb.session.pool.SessionPool; +import org.apache.iotdb.tool.common.Constants; + +import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.external.commons.collections4.CollectionUtils; +import org.apache.tsfile.file.metadata.enums.CompressionType; +import org.apache.tsfile.file.metadata.enums.TSEncoding; +import org.junit.After; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.TemporaryFolder; + +import java.io.File; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.doAnswer; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.verify; + +/** Checks that the import passes the parsed tags and attributes on to the client API. */ +public class SchemaTagsImportTest { + + @Rule public TemporaryFolder temporaryFolder = new TemporaryFolder(); + + @After + public void reset() { + ImportSchemaTree.sessionPool = null; + ImportSchemaTree.aligned = false; + ImportSchemaTree.batchPointSize = Constants.BATCH_POINT_SIZE; + } + + @Test + public void legacyFileKeepsTheLegacyCreateCall() throws Exception { + SessionPool pool = mock(SessionPool.class); + ImportSchemaTree.sessionPool = pool; + + runImport( + "Timeseries,Alias,DataType,Encoding,Compression\n" + + "root.sg.d.s1,,INT64,TS_2DIFF,SNAPPY\n" + + "root.sg.d.s2,aliasB,INT64,TS_2DIFF,SNAPPY\n"); + + verify(pool) + .createMultiTimeseries( + Arrays.asList("root.sg.d.s1"), + Arrays.asList(TSDataType.INT64), + Arrays.asList(TSEncoding.TS_2DIFF), + Arrays.asList(CompressionType.SNAPPY), + null, + null, + null, + null); + verify(pool) + .createMultiTimeseries( + Arrays.asList("root.sg.d.s2"), + Arrays.asList(TSDataType.INT64), + Arrays.asList(TSEncoding.TS_2DIFF), + Arrays.asList(CompressionType.SNAPPY), + null, + null, + null, + Arrays.asList("aliasB")); + } + + @Test + public void v2FilePassesTagsAndAttributesToCreateMultiTimeseries() throws Exception { + SessionPool pool = mock(SessionPool.class); + ImportSchemaTree.sessionPool = pool; + Map tags = new LinkedHashMap<>(); + tags.put("room", "101"); + tags.put("type", "sensor"); + Map attributes = new LinkedHashMap<>(); + attributes.put("owner", "team a"); + + runImport( + Constants.HEAD_COLUMNS_V2.stream().collect(Collectors.joining(",")) + + "\nroot.sg.d.s1,,INT64,TS_2DIFF,SNAPPY,\"" + + AbstractSchemaTool.toJsonCell(tags).replace("\"", "\"\"") + + "\",\"" + + AbstractSchemaTool.toJsonCell(attributes).replace("\"", "\"\"") + + "\"\n" + + "root.sg.d.s2,,INT64,TS_2DIFF,SNAPPY,,\n"); + + verify(pool) + .createMultiTimeseries( + Arrays.asList("root.sg.d.s1", "root.sg.d.s2"), + Arrays.asList(TSDataType.INT64, TSDataType.INT64), + Arrays.asList(TSEncoding.TS_2DIFF, TSEncoding.TS_2DIFF), + Arrays.asList(CompressionType.SNAPPY, CompressionType.SNAPPY), + null, + Arrays.asList(new LinkedHashMap<>(tags), null), + Arrays.asList(new LinkedHashMap<>(attributes), null), + null); + } + + @Test + public void v2AlignedFilePassesTagsAndAttributesToCreateAlignedTimeseries() throws Exception { + SessionPool pool = mock(SessionPool.class); + ImportSchemaTree.sessionPool = pool; + ImportSchemaTree.aligned = true; + Map tags = new LinkedHashMap<>(); + tags.put("room", "101"); + Map attributes = new LinkedHashMap<>(); + attributes.put("owner", "team a"); + // the aligned writer empties its lists in a finally block, so snapshot the arguments + List alignedCall = new ArrayList<>(); + doAnswer( + call -> { + for (Object argument : call.getArguments()) { + alignedCall.add( + argument instanceof List ? new ArrayList<>((List) argument) : argument); + } + return null; + }) + .when(pool) + .createAlignedTimeseries(any(), any(), any(), any(), any(), any(), any(), any()); + + runImport( + Constants.HEAD_COLUMNS_V2.stream().collect(Collectors.joining(",")) + + "\nroot.sg.d.s1,,INT64,TS_2DIFF,SNAPPY,\"" + + AbstractSchemaTool.toJsonCell(tags).replace("\"", "\"\"") + + "\",\"" + + AbstractSchemaTool.toJsonCell(attributes).replace("\"", "\"\"") + + "\"\n"); + + // the pre-existing aligned path hands over both the measurement and the full path, so the maps + // must be built with the same shape to stay positionally aligned + assertEquals( + Arrays.asList( + "root.sg.d", + Arrays.asList("s1", "root.sg.d.s1"), + Arrays.asList(TSDataType.INT64), + Arrays.asList(TSEncoding.TS_2DIFF), + Arrays.asList(CompressionType.SNAPPY), + new ArrayList(), + Arrays.asList(new LinkedHashMap<>(tags), new LinkedHashMap<>(tags)), + Arrays.asList(new LinkedHashMap<>(attributes), new LinkedHashMap<>(attributes))), + alignedCall); + verify(pool, never()) + .createMultiTimeseries(any(), any(), any(), any(), any(), any(), any(), any()); + } + + @Test + public void v2HeaderWithoutAllColumnsCreatesNothing() throws Exception { + SessionPool pool = mock(SessionPool.class); + ImportSchemaTree.sessionPool = pool; + assertFalse( + ImportSchemaTree.checkHeader( + Arrays.asList("Timeseries", "Alias", "DataType", "Encoding", "Compression", "Tags"))); + + verify(pool, never()) + .createMultiTimeseries(any(), any(), any(), any(), any(), any(), any(), any()); + } + + @Test + public void brokenJsonCellSkipsTheRecord() throws Exception { + SessionPool pool = mock(SessionPool.class); + ImportSchemaTree.sessionPool = pool; + + runImport( + Constants.HEAD_COLUMNS_V2.stream().collect(Collectors.joining(",")) + + "\nroot.sg.d.s1,,INT64,TS_2DIFF,SNAPPY,\"{not json}\",\n"); + + verify(pool, never()) + .createMultiTimeseries(any(), any(), any(), any(), any(), any(), any(), any()); + } + + /** + * Guards the batch-flush boundary: every aliased list must be emptied together, otherwise the + * next batch pairs the aliases still buffered with the wrong paths. + */ + @Test + public void batchedAliasedRecordsKeepAliasTagsAndPathsPaired() throws Exception { + SessionPool pool = mock(SessionPool.class); + ImportSchemaTree.sessionPool = pool; + ImportSchemaTree.batchPointSize = 2; + List>> calls = new ArrayList<>(); + doAnswer( + call -> { + List> snapshot = new ArrayList<>(); + for (int i = 0; i < call.getArguments().length; i++) { + List argument = call.getArgument(i); + snapshot.add(argument == null ? null : new ArrayList<>(argument)); + } + calls.add(snapshot); + return null; + }) + .when(pool) + .createMultiTimeseries(any(), any(), any(), any(), any(), any(), any(), any()); + + File csv = temporaryFolder.newFile("batch.csv"); + AbstractSchemaTool.CSVPrinterWrapper printer = + new AbstractSchemaTool.CSVPrinterWrapper(csv.getAbsolutePath()); + printer.printRecord(Constants.HEAD_COLUMNS_V2); + for (int i = 1; i <= 5; i++) { + printer.print("root.sg.d.s" + i); + printer.print("alias" + i); + printer.print("INT64"); + printer.print("TS_2DIFF"); + printer.print("SNAPPY"); + printer.print(AbstractSchemaTool.toJsonCell(Collections.singletonMap("k", "v" + i))); + printer.print(""); + printer.println(); + } + printer.close(); + runImportCsv(csv); + + // 3 flushes with batchPointSize 2: [s1,s2], [s3,s4], [s5] + assertEquals(3, calls.size()); + List sentPaths = new ArrayList<>(); + for (List> call : calls) { + List paths = call.get(0); + List dataTypes = call.get(1); + List encodings = call.get(2); + List compressors = call.get(3); + List tags = call.get(5); + List attributes = call.get(6); + List measurementAlias = call.get(7); + assertTrue(CollectionUtils.isNotEmpty(paths)); + // every list that is indexed by the path position must have the same length + for (List parallel : + Arrays.asList(dataTypes, encodings, compressors, tags, attributes, measurementAlias)) { + assertEquals("lists must stay positionally aligned", paths.size(), parallel.size()); + } + for (int i = 0; i < paths.size(); i++) { + String path = (String) paths.get(i); + String seq = path.substring(path.length() - 1); + sentPaths.add(path); + // s must travel with alias and with the tag built from the same n + assertEquals(path, "alias" + seq, measurementAlias.get(i)); + assertEquals("v" + seq, ((Map) tags.get(i)).get("k")); + } + } + assertEquals( + Arrays.asList( + "root.sg.d.s1", "root.sg.d.s2", "root.sg.d.s3", "root.sg.d.s4", "root.sg.d.s5"), + sentPaths); + } + + private void runImport(String csvContent) throws Exception { + File csv = temporaryFolder.newFile("schema_" + System.nanoTime() + ".csv"); + Files.write(csv.toPath(), csvContent.getBytes(StandardCharsets.UTF_8)); + runImportCsv(csv); + } + + private void runImportCsv(File csv) throws Exception { + String failedFilePath = temporaryFolder.getRoot().getAbsolutePath() + "/failed"; + try (Stream records = + ImportSchemaTree.readCsvFile(csv.getAbsolutePath()).stream()) { + List headerNames = ImportSchemaTree.readHeaderNames(csv.getAbsolutePath()); + ImportSchemaTree.writeScheme(csv.getName(), headerNames, records, failedFilePath); + } + } +} diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/schema/source/LogicalViewSchemaSource.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/schema/source/LogicalViewSchemaSource.java index 144ec16d4a37e..6d2a6f061b864 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/schema/source/LogicalViewSchemaSource.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/schema/source/LogicalViewSchemaSource.java @@ -38,8 +38,8 @@ import java.util.Collections; import java.util.List; -import java.util.Map; -import java.util.stream.Collectors; + +import static org.apache.iotdb.db.queryengine.execution.operator.schema.source.TimeSeriesSchemaSource.mapToString; public class LogicalViewSchemaSource implements ISchemaSource { @@ -118,15 +118,4 @@ public boolean hasSchemaStatistic(ISchemaRegion schemaRegion) { public long getSchemaStatistic(ISchemaRegion schemaRegion) { return 0; } - - private String mapToString(Map map) { - if (map == null || map.isEmpty()) { - return null; - } - String content = - map.entrySet().stream() - .map(e -> "\"" + e.getKey() + "\"" + ":" + "\"" + e.getValue() + "\"") - .collect(Collectors.joining(",")); - return "{" + content + "}"; - } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/schema/source/TimeSeriesSchemaSource.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/schema/source/TimeSeriesSchemaSource.java index 0b89998052b00..832cf46100def 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/schema/source/TimeSeriesSchemaSource.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/schema/source/TimeSeriesSchemaSource.java @@ -42,7 +42,6 @@ import java.util.List; import java.util.Map; -import java.util.stream.Collectors; import static org.apache.iotdb.commons.schema.SchemaConstant.ALL_MATCH_PATTERN; @@ -161,14 +160,68 @@ public boolean shouldSkipSchemaRegion(final ISchemaRegion schemaRegion) { return false; } + /** + * Renders a tag/attribute map as a JSON object (RFC 8259). Keys and values are escaped as JSON + * strings, while non-ASCII characters are kept raw for readability. + * + * @return the JSON object text, or {@code null} when the map is null or empty + */ public static String mapToString(Map map) { if (map == null || map.isEmpty()) { return null; } - String content = - map.entrySet().stream() - .map(e -> "\"" + e.getKey() + "\"" + ":" + "\"" + e.getValue() + "\"") - .collect(Collectors.joining(",")); - return "{" + content + "}"; + final StringBuilder builder = new StringBuilder("{"); + boolean first = true; + for (final Map.Entry entry : map.entrySet()) { + if (!first) { + builder.append(','); + } + first = false; + appendJsonString(builder, entry.getKey()); + builder.append(':'); + appendJsonString(builder, entry.getValue()); + } + return builder.append('}').toString(); + } + + private static final char[] HEX_DIGITS = "0123456789abcdef".toCharArray(); + + private static void appendJsonString(final StringBuilder builder, final String raw) { + // a null key or value is rendered as the literal text "null", as before + final String value = raw == null ? "null" : raw; + builder.append('"'); + for (int i = 0, length = value.length(); i < length; i++) { + final char c = value.charAt(i); + switch (c) { + case '"': + builder.append("\\\""); + break; + case '\\': + builder.append("\\\\"); + break; + case '\b': + builder.append("\\b"); + break; + case '\f': + builder.append("\\f"); + break; + case '\n': + builder.append("\\n"); + break; + case '\r': + builder.append("\\r"); + break; + case '\t': + builder.append("\\t"); + break; + default: + if (c < 0x20) { + builder.append("\\u00").append(HEX_DIGITS[(c >> 4) & 0xF]).append(HEX_DIGITS[c & 0xF]); + } else { + builder.append(c); + } + } + } + builder.append('"'); } } diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/operator/schema/source/TimeSeriesSchemaSourceMapToStringTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/operator/schema/source/TimeSeriesSchemaSourceMapToStringTest.java new file mode 100644 index 0000000000000..1fa137d5d5a8c --- /dev/null +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/operator/schema/source/TimeSeriesSchemaSourceMapToStringTest.java @@ -0,0 +1,221 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.queryengine.execution.operator.schema.source; + +import com.google.gson.JsonElement; +import com.google.gson.JsonObject; +import com.google.gson.JsonParser; +import org.junit.Test; + +import java.util.Collections; +import java.util.LinkedHashMap; +import java.util.Map; +import java.util.stream.Collectors; + +import static org.apache.iotdb.db.queryengine.execution.operator.schema.source.TimeSeriesSchemaSource.mapToString; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNull; + +public class TimeSeriesSchemaSourceMapToStringTest { + + /** The pre-escaping implementation, kept here to prove backward compatibility. */ + private static String legacyMapToString(final Map map) { + if (map == null || map.isEmpty()) { + return null; + } + final String content = + map.entrySet().stream() + .map(e -> "\"" + e.getKey() + "\"" + ":" + "\"" + e.getValue() + "\"") + .collect(Collectors.joining(",")); + return "{" + content + "}"; + } + + private static Map mapOf(final String key, final String value) { + final Map map = new LinkedHashMap<>(); + map.put(key, value); + return map; + } + + private static JsonObject parse(final String json) { + final JsonElement element = JsonParser.parseString(json); + assertEquals("not a JSON object: " + json, true, element.isJsonObject()); + return element.getAsJsonObject(); + } + + /** Renders, parses with a standard JSON parser, and checks the round-trip to the source map. */ + private static void assertJsonRoundTrip(final Map map) { + final JsonObject json = parse(mapToString(map)); + assertEquals(map.size(), json.size()); + for (final Map.Entry entry : map.entrySet()) { + assertEquals(true, json.has(entry.getKey())); + assertEquals(entry.getValue(), json.get(entry.getKey()).getAsString()); + } + } + + @Test + public void testPlainValue() { + final Map map = mapOf("room", "101"); + assertEquals("{\"room\":\"101\"}", mapToString(map)); + assertJsonRoundTrip(map); + } + + @Test + public void testCommaIsNotEscaped() { + final Map map = mapOf("k", "a,b"); + assertEquals("{\"k\":\"a,b\"}", mapToString(map)); + assertJsonRoundTrip(map); + } + + @Test + public void testDoubleQuote() { + final Map map = mapOf("k", "a\"b"); + assertEquals("{\"k\":\"a\\\"b\"}", mapToString(map)); + assertJsonRoundTrip(map); + // the escaped quote must not split the document into extra keys + assertEquals(1, parse(mapToString(map)).size()); + } + + @Test + public void testBackslash() { + final Map map = mapOf("k", "a\\b"); + assertEquals("{\"k\":\"a\\\\b\"}", mapToString(map)); + assertJsonRoundTrip(map); + } + + @Test + public void testLiteralBackslashAndN() { + final Map map = mapOf("k", "a\\nb"); + assertEquals("{\"k\":\"a\\\\nb\"}", mapToString(map)); + assertJsonRoundTrip(map); + assertEquals("a\\nb", parse(mapToString(map)).get("k").getAsString()); + } + + @Test + public void testDoubleBackslash() { + final Map map = mapOf("k", "a\\\\b"); + assertEquals("{\"k\":\"a\\\\\\\\b\"}", mapToString(map)); + assertJsonRoundTrip(map); + assertEquals("a\\\\b", parse(mapToString(map)).get("k").getAsString()); + } + + @Test + public void testTab() { + final Map map = mapOf("k", "\t"); + assertEquals("{\"k\":\"\\t\"}", mapToString(map)); + assertJsonRoundTrip(map); + } + + @Test + public void testLineFeed() { + final Map map = mapOf("k", "a\nb"); + assertEquals("{\"k\":\"a\\nb\"}", mapToString(map)); + assertJsonRoundTrip(map); + } + + @Test + public void testCarriageReturnAndLineFeed() { + final Map map = mapOf("k", "a\r\nb"); + assertEquals("{\"k\":\"a\\r\\nb\"}", mapToString(map)); + assertJsonRoundTrip(map); + } + + @Test + public void testBackspaceAndFormFeed() { + final Map map = mapOf("k", "a\bb\fc"); + assertEquals("{\"k\":\"a\\bb\\fc\"}", mapToString(map)); + assertJsonRoundTrip(map); + } + + @Test + public void testOtherControlCharacterUsesUnicodeEscape() { + final Map map = mapOf("k", "a\u0001b"); + assertEquals("{\"k\":\"a\\u0001b\"}", mapToString(map)); + assertJsonRoundTrip(map); + assertEquals("a\u0001b", parse(mapToString(map)).get("k").getAsString()); + } + + @Test + public void testNonAsciiIsKeptRaw() { + final Map map = mapOf("cn", "中文\uD83D\uDE00"); + assertEquals("{\"cn\":\"中文\uD83D\uDE00\"}", mapToString(map)); + assertJsonRoundTrip(map); + // no \\uXXXX escaping of non-ASCII characters + assertEquals(false, mapToString(map).contains("\\u")); + } + + @Test + public void testEmptyStringValue() { + final Map map = mapOf("k", ""); + assertEquals("{\"k\":\"\"}", mapToString(map)); + assertJsonRoundTrip(map); + } + + @Test + public void testEscapingInKey() { + final Map map = mapOf("a\"b\\c\td", "v"); + assertEquals("{\"a\\\"b\\\\c\\td\":\"v\"}", mapToString(map)); + assertJsonRoundTrip(map); + } + + @Test + public void testMultipleKeys() { + final Map map = new LinkedHashMap<>(); + map.put("root.sg", "101"); + map.put("unit", "celsius"); + map.put("note", "a\"b\\c"); + assertEquals( + "{\"root.sg\":\"101\",\"unit\":\"celsius\",\"note\":\"a\\\"b\\\\c\"}", mapToString(map)); + assertJsonRoundTrip(map); + } + + @Test + public void testNullAndEmptyMap() { + assertNull(mapToString(null)); + assertNull(mapToString(Collections.emptyMap())); + } + + @Test + public void testIdenticalToLegacyOutputWhenNothingNeedsEscaping() { + final Map plain = new LinkedHashMap<>(); + plain.put("room", "101"); + plain.put("sensor", "temperature"); + plain.put("path", "root.sg.d1.s1"); + plain.put("empty", ""); + plain.put("punctuation", "a,b;c=d (e) [f] {g} /h@i#j$k%l^m&n*o-p+q_r"); + plain.put("cjk", "中文😀"); + plain.put("single", "x"); + + assertEquals(legacyMapToString(plain), mapToString(plain)); + assertJsonRoundTrip(plain); + + final Map single = mapOf("room", "101"); + assertEquals(legacyMapToString(single), mapToString(single)); + } + + @Test + public void testLegacyComparisonForMultiplePlainMaps() { + final Map map = new LinkedHashMap<>(); + map.put("a", "1"); + map.put("b", "2"); + map.put("c", "3"); + assertEquals(legacyMapToString(map), mapToString(map)); + assertEquals("{\"a\":\"1\",\"b\":\"2\",\"c\":\"3\"}", mapToString(map)); + } +}