From 98a788e9210d50377567f557e22d7d5af6dd2412 Mon Sep 17 00:00:00 2001 From: CritasWang Date: Thu, 5 Nov 2026 18:37:49 +0800 Subject: [PATCH 1/6] [Server] Render tags and attributes as valid JSON in SHOW TIMESERIES mapToString concatenated keys and values without any escaping, so a tag value containing a double quote produced invalid JSON, a backslash was silently reinterpreted by JSON parsers, and control characters such as tab or newline were emitted raw. Escape keys and values per RFC 8259 while keeping non-ASCII characters readable, and drop the duplicate implementation in LogicalViewSchemaSource in favour of the shared one. The output is unchanged for values that need no escaping. --- .../source/LogicalViewSchemaSource.java | 15 +- .../schema/source/TimeSeriesSchemaSource.java | 65 +++++- ...TimeSeriesSchemaSourceMapToStringTest.java | 221 ++++++++++++++++++ 3 files changed, 282 insertions(+), 19 deletions(-) create mode 100644 iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/operator/schema/source/TimeSeriesSchemaSourceMapToStringTest.java diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/schema/source/LogicalViewSchemaSource.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/schema/source/LogicalViewSchemaSource.java index 144ec16d4a37e..6d2a6f061b864 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/schema/source/LogicalViewSchemaSource.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/schema/source/LogicalViewSchemaSource.java @@ -38,8 +38,8 @@ import java.util.Collections; import java.util.List; -import java.util.Map; -import java.util.stream.Collectors; + +import static org.apache.iotdb.db.queryengine.execution.operator.schema.source.TimeSeriesSchemaSource.mapToString; public class LogicalViewSchemaSource implements ISchemaSource { @@ -118,15 +118,4 @@ public boolean hasSchemaStatistic(ISchemaRegion schemaRegion) { public long getSchemaStatistic(ISchemaRegion schemaRegion) { return 0; } - - private String mapToString(Map map) { - if (map == null || map.isEmpty()) { - return null; - } - String content = - map.entrySet().stream() - .map(e -> "\"" + e.getKey() + "\"" + ":" + "\"" + e.getValue() + "\"") - .collect(Collectors.joining(",")); - return "{" + content + "}"; - } } diff --git a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/schema/source/TimeSeriesSchemaSource.java b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/schema/source/TimeSeriesSchemaSource.java index 0b89998052b00..832cf46100def 100644 --- a/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/schema/source/TimeSeriesSchemaSource.java +++ b/iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/execution/operator/schema/source/TimeSeriesSchemaSource.java @@ -42,7 +42,6 @@ import java.util.List; import java.util.Map; -import java.util.stream.Collectors; import static org.apache.iotdb.commons.schema.SchemaConstant.ALL_MATCH_PATTERN; @@ -161,14 +160,68 @@ public boolean shouldSkipSchemaRegion(final ISchemaRegion schemaRegion) { return false; } + /** + * Renders a tag/attribute map as a JSON object (RFC 8259). Keys and values are escaped as JSON + * strings, while non-ASCII characters are kept raw for readability. + * + * @return the JSON object text, or {@code null} when the map is null or empty + */ public static String mapToString(Map map) { if (map == null || map.isEmpty()) { return null; } - String content = - map.entrySet().stream() - .map(e -> "\"" + e.getKey() + "\"" + ":" + "\"" + e.getValue() + "\"") - .collect(Collectors.joining(",")); - return "{" + content + "}"; + final StringBuilder builder = new StringBuilder("{"); + boolean first = true; + for (final Map.Entry entry : map.entrySet()) { + if (!first) { + builder.append(','); + } + first = false; + appendJsonString(builder, entry.getKey()); + builder.append(':'); + appendJsonString(builder, entry.getValue()); + } + return builder.append('}').toString(); + } + + private static final char[] HEX_DIGITS = "0123456789abcdef".toCharArray(); + + private static void appendJsonString(final StringBuilder builder, final String raw) { + // a null key or value is rendered as the literal text "null", as before + final String value = raw == null ? "null" : raw; + builder.append('"'); + for (int i = 0, length = value.length(); i < length; i++) { + final char c = value.charAt(i); + switch (c) { + case '"': + builder.append("\\\""); + break; + case '\\': + builder.append("\\\\"); + break; + case '\b': + builder.append("\\b"); + break; + case '\f': + builder.append("\\f"); + break; + case '\n': + builder.append("\\n"); + break; + case '\r': + builder.append("\\r"); + break; + case '\t': + builder.append("\\t"); + break; + default: + if (c < 0x20) { + builder.append("\\u00").append(HEX_DIGITS[(c >> 4) & 0xF]).append(HEX_DIGITS[c & 0xF]); + } else { + builder.append(c); + } + } + } + builder.append('"'); } } diff --git a/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/operator/schema/source/TimeSeriesSchemaSourceMapToStringTest.java b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/operator/schema/source/TimeSeriesSchemaSourceMapToStringTest.java new file mode 100644 index 0000000000000..1fa137d5d5a8c --- /dev/null +++ b/iotdb-core/datanode/src/test/java/org/apache/iotdb/db/queryengine/execution/operator/schema/source/TimeSeriesSchemaSourceMapToStringTest.java @@ -0,0 +1,221 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.db.queryengine.execution.operator.schema.source; + +import com.google.gson.JsonElement; +import com.google.gson.JsonObject; +import com.google.gson.JsonParser; +import org.junit.Test; + +import java.util.Collections; +import java.util.LinkedHashMap; +import java.util.Map; +import java.util.stream.Collectors; + +import static org.apache.iotdb.db.queryengine.execution.operator.schema.source.TimeSeriesSchemaSource.mapToString; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNull; + +public class TimeSeriesSchemaSourceMapToStringTest { + + /** The pre-escaping implementation, kept here to prove backward compatibility. */ + private static String legacyMapToString(final Map map) { + if (map == null || map.isEmpty()) { + return null; + } + final String content = + map.entrySet().stream() + .map(e -> "\"" + e.getKey() + "\"" + ":" + "\"" + e.getValue() + "\"") + .collect(Collectors.joining(",")); + return "{" + content + "}"; + } + + private static Map mapOf(final String key, final String value) { + final Map map = new LinkedHashMap<>(); + map.put(key, value); + return map; + } + + private static JsonObject parse(final String json) { + final JsonElement element = JsonParser.parseString(json); + assertEquals("not a JSON object: " + json, true, element.isJsonObject()); + return element.getAsJsonObject(); + } + + /** Renders, parses with a standard JSON parser, and checks the round-trip to the source map. */ + private static void assertJsonRoundTrip(final Map map) { + final JsonObject json = parse(mapToString(map)); + assertEquals(map.size(), json.size()); + for (final Map.Entry entry : map.entrySet()) { + assertEquals(true, json.has(entry.getKey())); + assertEquals(entry.getValue(), json.get(entry.getKey()).getAsString()); + } + } + + @Test + public void testPlainValue() { + final Map map = mapOf("room", "101"); + assertEquals("{\"room\":\"101\"}", mapToString(map)); + assertJsonRoundTrip(map); + } + + @Test + public void testCommaIsNotEscaped() { + final Map map = mapOf("k", "a,b"); + assertEquals("{\"k\":\"a,b\"}", mapToString(map)); + assertJsonRoundTrip(map); + } + + @Test + public void testDoubleQuote() { + final Map map = mapOf("k", "a\"b"); + assertEquals("{\"k\":\"a\\\"b\"}", mapToString(map)); + assertJsonRoundTrip(map); + // the escaped quote must not split the document into extra keys + assertEquals(1, parse(mapToString(map)).size()); + } + + @Test + public void testBackslash() { + final Map map = mapOf("k", "a\\b"); + assertEquals("{\"k\":\"a\\\\b\"}", mapToString(map)); + assertJsonRoundTrip(map); + } + + @Test + public void testLiteralBackslashAndN() { + final Map map = mapOf("k", "a\\nb"); + assertEquals("{\"k\":\"a\\\\nb\"}", mapToString(map)); + assertJsonRoundTrip(map); + assertEquals("a\\nb", parse(mapToString(map)).get("k").getAsString()); + } + + @Test + public void testDoubleBackslash() { + final Map map = mapOf("k", "a\\\\b"); + assertEquals("{\"k\":\"a\\\\\\\\b\"}", mapToString(map)); + assertJsonRoundTrip(map); + assertEquals("a\\\\b", parse(mapToString(map)).get("k").getAsString()); + } + + @Test + public void testTab() { + final Map map = mapOf("k", "\t"); + assertEquals("{\"k\":\"\\t\"}", mapToString(map)); + assertJsonRoundTrip(map); + } + + @Test + public void testLineFeed() { + final Map map = mapOf("k", "a\nb"); + assertEquals("{\"k\":\"a\\nb\"}", mapToString(map)); + assertJsonRoundTrip(map); + } + + @Test + public void testCarriageReturnAndLineFeed() { + final Map map = mapOf("k", "a\r\nb"); + assertEquals("{\"k\":\"a\\r\\nb\"}", mapToString(map)); + assertJsonRoundTrip(map); + } + + @Test + public void testBackspaceAndFormFeed() { + final Map map = mapOf("k", "a\bb\fc"); + assertEquals("{\"k\":\"a\\bb\\fc\"}", mapToString(map)); + assertJsonRoundTrip(map); + } + + @Test + public void testOtherControlCharacterUsesUnicodeEscape() { + final Map map = mapOf("k", "a\u0001b"); + assertEquals("{\"k\":\"a\\u0001b\"}", mapToString(map)); + assertJsonRoundTrip(map); + assertEquals("a\u0001b", parse(mapToString(map)).get("k").getAsString()); + } + + @Test + public void testNonAsciiIsKeptRaw() { + final Map map = mapOf("cn", "中文\uD83D\uDE00"); + assertEquals("{\"cn\":\"中文\uD83D\uDE00\"}", mapToString(map)); + assertJsonRoundTrip(map); + // no \\uXXXX escaping of non-ASCII characters + assertEquals(false, mapToString(map).contains("\\u")); + } + + @Test + public void testEmptyStringValue() { + final Map map = mapOf("k", ""); + assertEquals("{\"k\":\"\"}", mapToString(map)); + assertJsonRoundTrip(map); + } + + @Test + public void testEscapingInKey() { + final Map map = mapOf("a\"b\\c\td", "v"); + assertEquals("{\"a\\\"b\\\\c\\td\":\"v\"}", mapToString(map)); + assertJsonRoundTrip(map); + } + + @Test + public void testMultipleKeys() { + final Map map = new LinkedHashMap<>(); + map.put("root.sg", "101"); + map.put("unit", "celsius"); + map.put("note", "a\"b\\c"); + assertEquals( + "{\"root.sg\":\"101\",\"unit\":\"celsius\",\"note\":\"a\\\"b\\\\c\"}", mapToString(map)); + assertJsonRoundTrip(map); + } + + @Test + public void testNullAndEmptyMap() { + assertNull(mapToString(null)); + assertNull(mapToString(Collections.emptyMap())); + } + + @Test + public void testIdenticalToLegacyOutputWhenNothingNeedsEscaping() { + final Map plain = new LinkedHashMap<>(); + plain.put("room", "101"); + plain.put("sensor", "temperature"); + plain.put("path", "root.sg.d1.s1"); + plain.put("empty", ""); + plain.put("punctuation", "a,b;c=d (e) [f] {g} /h@i#j$k%l^m&n*o-p+q_r"); + plain.put("cjk", "中文😀"); + plain.put("single", "x"); + + assertEquals(legacyMapToString(plain), mapToString(plain)); + assertJsonRoundTrip(plain); + + final Map single = mapOf("room", "101"); + assertEquals(legacyMapToString(single), mapToString(single)); + } + + @Test + public void testLegacyComparisonForMultiplePlainMaps() { + final Map map = new LinkedHashMap<>(); + map.put("a", "1"); + map.put("b", "2"); + map.put("c", "3"); + assertEquals(legacyMapToString(map), mapToString(map)); + assertEquals("{\"a\":\"1\",\"b\":\"2\",\"c\":\"3\"}", mapToString(map)); + } +} From 0f1fdb5b68b86cf3915d057e49f121c58d4f82d9 Mon Sep 17 00:00:00 2001 From: CritasWang Date: Thu, 5 Nov 2026 18:37:52 +0800 Subject: [PATCH 2/6] [Tools] Export and import tags and attributes in tree-model schema files export-schema wrote only Timeseries, Alias, DataType, Encoding and Compression, so tags and attributes were always lost; import-schema had no way to restore them even though createMultiTimeseries and createAlignedTimeseries accept them. Write Tags and Attributes as compact JSON columns (sorted keys, HTML escaping disabled, empty cell when absent) and restore them on import for both the normal and the aligned path. The schema CSV moves to RFC4180 quoting with no escape character, and files carrying the previous five-column header are still read with the legacy settings. A header that looks like the new format is rejected unless all expected columns are present. --- .../iotdb/tools/it/SchemaTagsRoundTripIT.java | 483 ++++++++++++++++++ .../apache/iotdb/cli/i18n/CliMessages.java | 7 + .../apache/iotdb/cli/i18n/CliMessages.java | 7 + .../apache/iotdb/tool/common/Constants.java | 25 +- .../iotdb/tool/schema/AbstractSchemaTool.java | 62 ++- .../iotdb/tool/schema/ExportSchemaTree.java | 64 ++- .../iotdb/tool/schema/ImportSchemaTree.java | 179 ++++++- .../tool/schema/SchemaTagsFidelityTest.java | 318 ++++++++++++ .../tool/schema/SchemaTagsImportTest.java | 208 ++++++++ 9 files changed, 1310 insertions(+), 43 deletions(-) create mode 100644 integration-test/src/test/java/org/apache/iotdb/tools/it/SchemaTagsRoundTripIT.java create mode 100644 iotdb-client/cli/src/test/java/org/apache/iotdb/tool/schema/SchemaTagsFidelityTest.java create mode 100644 iotdb-client/cli/src/test/java/org/apache/iotdb/tool/schema/SchemaTagsImportTest.java diff --git a/integration-test/src/test/java/org/apache/iotdb/tools/it/SchemaTagsRoundTripIT.java b/integration-test/src/test/java/org/apache/iotdb/tools/it/SchemaTagsRoundTripIT.java new file mode 100644 index 0000000000000..2df7ff0511bc7 --- /dev/null +++ b/integration-test/src/test/java/org/apache/iotdb/tools/it/SchemaTagsRoundTripIT.java @@ -0,0 +1,483 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.iotdb.tools.it; + +import org.apache.iotdb.cli.it.AbstractScriptIT; +import org.apache.iotdb.isession.ISession; +import org.apache.iotdb.isession.SessionConfig; +import org.apache.iotdb.isession.SessionDataSet; +import org.apache.iotdb.it.env.EnvFactory; +import org.apache.iotdb.it.framework.IoTDBTestRunner; +import org.apache.iotdb.itbase.category.ClusterIT; +import org.apache.iotdb.itbase.category.LocalStandaloneIT; +import org.apache.iotdb.rpc.IoTDBConnectionException; +import org.apache.iotdb.rpc.StatementExecutionException; +import org.apache.iotdb.tool.common.Constants; + +import com.google.gson.Gson; +import com.google.gson.reflect.TypeToken; +import org.apache.commons.csv.CSVFormat; +import org.apache.commons.csv.CSVParser; +import org.apache.commons.csv.CSVRecord; +import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.file.metadata.enums.CompressionType; +import org.apache.tsfile.file.metadata.enums.TSEncoding; +import org.apache.tsfile.read.common.Field; +import org.awaitility.Awaitility; +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.Rule; +import org.junit.Test; +import org.junit.experimental.categories.Category; +import org.junit.rules.TemporaryFolder; +import org.junit.runner.RunWith; + +import java.io.File; +import java.io.IOException; +import java.lang.reflect.Type; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.TreeMap; +import java.util.concurrent.TimeUnit; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertTrue; + +/** + * Proves that the tree model schema tools restore the tags and attributes of every time series: + * {@code export-schema} writes them as RFC4180 quoted JSON, {@code import-schema} reads that format + * back, and awkward characters survive the round trip unchanged. + */ +@RunWith(IoTDBTestRunner.class) +@Category({LocalStandaloneIT.class, ClusterIT.class}) +public class SchemaTagsRoundTripIT extends AbstractScriptIT { + + private static final String DATABASE = "root.rt"; + private static final String EXPORT_PREFIX = "schema_tags"; + private static final String HEADER = + "Timeseries,Alias,DataType,Encoding,Compression,Tags,Attributes"; + private static final String ALIAS = "aliased"; + + /** The row whose tag value carries a double quote, i.e. the row that must be RFC4180 quoted. */ + private static final String QUOTE_ROW = DATABASE + ".quote.s1"; + + /** The JSON object the export writes for the tag value {@code a"b}. */ + private static final String QUOTE_ROW_TAGS_JSON = "{\"k\":\"a\\\"b\"}"; + + private static final String NO_ATTRIBUTES_ROW = DATABASE + ".none.s1"; + + /** How long the metadata drop may take to become visible on every node. */ + private static final int SCHEMA_CLEANUP_TIMEOUT_SECONDS = 60; + + private static final Type JSON_MAP_TYPE = new TypeToken>() {}.getType(); + private static final Gson GSON = new Gson(); + private static final Map NO_VALUES = Collections.emptyMap(); + + /** Time series path to the tags, attributes and alias the test creates for it. */ + private static final Map CREATED = new LinkedHashMap<>(); + + static { + // one tag value per device, because tags belong to the device rather than to the measurement + singleTag("v1", "v1"); + singleTag("comma", "a,b"); + singleTag("quote", "a\"b"); + singleTag("backslash", "a\\b"); + singleTag("backslashn", "a\\nb"); + singleTag("tab", "a\tb"); + singleTag("newline", "a\nb"); + singleTag("cjk", "中文😀"); + singleTag("nulllit", "null"); + singleTag("empty", ""); + singleTag("backtick", "`x`"); + tagged("spacedkey", "k e y", "v"); + CREATED.put( + DATABASE + ".multi.s1", + new Series(mapOf("a", "1", "b", "a,b", "c", "a\"b"), NO_VALUES, null)); + CREATED.put( + DATABASE + ".attr.s1", + new Series(NO_VALUES, mapOf("note", "he said \"hi\"", "plain", "v"), null)); + CREATED.put(NO_ATTRIBUTES_ROW, new Series(NO_VALUES, NO_VALUES, null)); + CREATED.put(DATABASE + ".alias.s1", new Series(mapOf("aliasTag", "v"), NO_VALUES, ALIAS)); + } + + @Rule public TemporaryFolder folder = new TemporaryFolder(); + + private static String ip; + private static String port; + private static String toolsPath; + private static String homePath; + + @BeforeClass + public static void setUp() { + EnvFactory.getEnv().initClusterEnvironment(); + ip = EnvFactory.getEnv().getIP(); + port = EnvFactory.getEnv().getPort(); + toolsPath = EnvFactory.getEnv().getToolsPath(); + final String libPath = EnvFactory.getEnv().getLibPath(); + homePath = + libPath.substring(0, libPath.lastIndexOf(File.separator + "lib" + File.separator + "*")); + } + + @AfterClass + public static void tearDown() { + EnvFactory.getEnv().cleanClusterEnvironment(); + } + + @Test + public void test() throws IOException { + String os = System.getProperty("os.name").toLowerCase(); + if (os.startsWith("windows")) { + testOnWindows(); + } else { + testOnUnix(); + } + } + + @Override + protected void testOnWindows() throws IOException { + try { + roundTrip(true); + } catch (IOException e) { + throw e; + } catch (Exception e) { + throw new IOException(e); + } + } + + @Override + protected void testOnUnix() throws IOException { + try { + roundTrip(false); + } catch (IOException e) { + throw e; + } catch (Exception e) { + throw new IOException(e); + } + } + + /** + * Exports the schema, drops it, imports the exported file again and compares the tags and + * attributes read back from the server with the ones that were created. + */ + private void roundTrip(boolean windows) throws Exception { + final Path outputDirectory = folder.newFolder("export").toPath(); + + createSchema(); + + testOutput( + exportProcessBuilder(windows, outputDirectory), + new String[] {Constants.EXPORT_COMPLETELY}, + 0); + + final Path csv = singleExportFile(outputDirectory); + assertExportFormat(csv); + + dropSchema(); + + testOutput(importProcessBuilder(windows, csv), new String[] {Constants.IMPORT_COMPLETELY}, 0); + + assertSchemaRestored(); + } + + private static void createSchema() throws IoTDBConnectionException, StatementExecutionException { + try (ISession session = EnvFactory.getEnv().getSessionConnection()) { + session.createDatabase(DATABASE); + for (Map.Entry entry : CREATED.entrySet()) { + final Series series = entry.getValue(); + // Set the tags and attributes through the session API, so that no SQL escaping can hide a + // character that the export or the import tool mangles. + session.createTimeseries( + entry.getKey(), + TSDataType.DOUBLE, + TSEncoding.GORILLA, + CompressionType.LZ4, + null, + series.tags.isEmpty() ? null : series.tags, + series.attributes.isEmpty() ? null : series.attributes, + series.alias); + } + } + } + + private static void dropSchema() throws IoTDBConnectionException, StatementExecutionException { + try (ISession session = EnvFactory.getEnv().getSessionConnection()) { + session.executeNonQueryStatement("DROP DATABASE " + DATABASE); + } + // The drop reaches the DataNodes asynchronously. Waiting until it is observable keeps a + // leftover device from making the import pass without restoring anything. + Awaitility.await() + .atMost(SCHEMA_CLEANUP_TIMEOUT_SECONDS, TimeUnit.SECONDS) + .pollInterval(500, TimeUnit.MILLISECONDS) + .until(() -> !databaseExists() && readSchema().isEmpty()); + try (ISession session = EnvFactory.getEnv().getSessionConnection()) { + // Importing a file only creates time series, so the database has to exist beforehand. + session.createDatabase(DATABASE); + } + } + + private static boolean databaseExists() + throws IoTDBConnectionException, StatementExecutionException { + try (ISession session = EnvFactory.getEnv().getSessionConnection(); + SessionDataSet dataSet = session.executeQueryStatement("SHOW DATABASES")) { + while (dataSet.hasNext()) { + if (DATABASE.equals(cell(dataSet.next().getFields(), 0))) { + return true; + } + } + } + return false; + } + + private Path singleExportFile(Path outputDirectory) throws IOException { + try (Stream files = Files.list(outputDirectory)) { + final List csvFiles = + files + .filter(file -> file.getFileName().toString().startsWith(EXPORT_PREFIX)) + .filter(file -> file.getFileName().toString().endsWith(".csv")) + .collect(Collectors.toList()); + assertEquals("expected exactly one exported CSV file, found " + csvFiles, 1, csvFiles.size()); + return csvFiles.get(0); + } + } + + private static void assertExportFormat(Path csv) throws IOException { + final List lines = Files.readAllLines(csv, StandardCharsets.UTF_8); + assertTrue("the exported schema file is empty", lines.size() > 1); + assertEquals(HEADER, lines.get(0)); + + final String content = Files.readString(csv, StandardCharsets.UTF_8); + // RFC4180 uses LF as the record separator; a CR would come from a platform dependent writer. + assertFalse("the export must use LF line endings", content.contains("\r")); + // RFC4180: the JSON object of the value containing a double quote is wrapped in a quoted cell + // and every inner quote is doubled, instead of being escaped with a backslash. + final String quotedCell = "\"" + QUOTE_ROW_TAGS_JSON.replace("\"", "\"\"") + "\""; + assertTrue( + "the Tags cell of " + + QUOTE_ROW + + " is not RFC4180 quoted: " + + quotedCell + + " in\n" + + content, + content.contains(quotedCell)); + + try (CSVParser parser = CSVParser.parse(csv, StandardCharsets.UTF_8, csvFormat())) { + final Map records = new TreeMap<>(); + for (CSVRecord record : parser) { + records.put(record.get("Timeseries"), record); + } + assertEquals( + "unexpected number of exported rows: " + records, CREATED.size(), records.size()); + + for (Map.Entry entry : CREATED.entrySet()) { + final String path = entry.getKey(); + final Series series = entry.getValue(); + final CSVRecord record = records.get(path); + assertNotNull("the export is missing " + path, record); + assertEquals(path + " DataType", "DOUBLE", record.get("DataType")); + assertEquals(path + " Encoding", "GORILLA", record.get("Encoding")); + assertEquals(path + " Compression", "LZ4", record.get("Compression")); + assertEquals( + path + " Alias", series.alias == null ? "" : series.alias, record.get("Alias")); + assertEquals(path + " Tags", series.tags, parseJsonObject(record.get("Tags"))); + assertEquals( + path + " Attributes", series.attributes, parseJsonObject(record.get("Attributes"))); + } + + // Nothing to write is written as an empty cell, not as {} and not as null. + assertEquals("", records.get(NO_ATTRIBUTES_ROW).get("Tags")); + assertEquals("", records.get(NO_ATTRIBUTES_ROW).get("Attributes")); + } + } + + private static void assertSchemaRestored() + throws IoTDBConnectionException, StatementExecutionException { + final Map restored = readSchema(); + assertEquals("the import must restore every time series", CREATED.keySet(), restored.keySet()); + for (Map.Entry entry : CREATED.entrySet()) { + final String path = entry.getKey(); + final Series expected = entry.getValue(); + final Series actual = restored.get(path); + assertEquals(path + " tags", expected.tags, actual.tags); + assertEquals(path + " attributes", expected.attributes, actual.attributes); + assertEquals(path + " alias", expected.alias, actual.alias); + } + } + + /** Reads the tags, attributes and alias of every time series of the test database. */ + private static Map readSchema() + throws IoTDBConnectionException, StatementExecutionException { + final Map schema = new TreeMap<>(); + try (ISession session = EnvFactory.getEnv().getSessionConnection(); + SessionDataSet dataSet = + session.executeQueryStatement("SHOW TIMESERIES " + DATABASE + ".**")) { + final List columns = dataSet.getColumnNames(); + final int timeseriesIndex = columns.indexOf("Timeseries"); + final int aliasIndex = columns.indexOf("Alias"); + final int tagsIndex = columns.indexOf("Tags"); + final int attributesIndex = columns.indexOf("Attributes"); + assertTrue( + "SHOW TIMESERIES must expose the Timeseries, Alias, Tags and Attributes columns: " + + columns, + timeseriesIndex >= 0 && aliasIndex >= 0 && tagsIndex >= 0 && attributesIndex >= 0); + while (dataSet.hasNext()) { + final List fields = dataSet.next().getFields(); + final String path = cell(fields, timeseriesIndex); + schema.put( + path, + new Series( + parseJsonObject(cell(fields, tagsIndex)), + parseJsonObject(cell(fields, attributesIndex)), + cell(fields, aliasIndex))); + } + } + return schema; + } + + /** Returns a cell of a schema row, or {@code null} when the server sent a NULL there. */ + private static String cell(List fields, int index) { + if (index < 0 || index >= fields.size()) { + return null; + } + final Field field = fields.get(index); + return field == null ? null : field.getStringValue(); + } + + /** Parses a JSON object cell into a map; an empty cell carries no tags and no attributes. */ + private static Map parseJsonObject(String cell) { + if (cell == null || cell.trim().isEmpty()) { + return new TreeMap<>(); + } + final Map parsed = GSON.fromJson(cell, JSON_MAP_TYPE); + return parsed == null ? new TreeMap<>() : new TreeMap<>(parsed); + } + + private static CSVFormat csvFormat() { + return CSVFormat.Builder.create(CSVFormat.DEFAULT) + .setHeader() + .setSkipHeaderRecord(true) + .build(); + } + + private ProcessBuilder exportProcessBuilder(boolean windows, Path outputDirectory) { + final List command = new ArrayList<>(); + addScriptInvocation(command, windows, "export-schema"); + addConnectionArguments(command); + command.addAll( + Arrays.asList( + "-t", + outputDirectory.toAbsolutePath().toString(), + "-pfn", + EXPORT_PREFIX, + "-path", + DATABASE + ".**")); + return processBuilder(command, windows); + } + + private ProcessBuilder importProcessBuilder(boolean windows, Path csv) { + final List command = new ArrayList<>(); + addScriptInvocation(command, windows, "import-schema"); + addConnectionArguments(command); + command.addAll(Arrays.asList("-db", DATABASE, "-s", csv.toAbsolutePath().toString())); + return processBuilder(command, windows); + } + + private static void addConnectionArguments(List command) { + command.addAll( + Arrays.asList( + "-h", + ip, + "-p", + port, + "-u", + SessionConfig.DEFAULT_USER, + "-pw", + SessionConfig.DEFAULT_PASSWORD)); + } + + private static void addScriptInvocation(List command, boolean windows, String script) { + if (windows) { + command.add("cmd.exe"); + command.add("/c"); + command.add( + toolsPath + + File.separator + + "windows" + + File.separator + + "schema" + + File.separator + + script + + ".bat"); + } else { + command.add("bash"); + command.add(toolsPath + File.separator + "schema" + File.separator + script + ".sh"); + } + } + + private static ProcessBuilder processBuilder(List command, boolean windows) { + if (windows) { + // cmd.exe exits with the error level of the script it ran. + command.add("&"); + command.add("exit"); + command.add("%^errorlevel%"); + } + final ProcessBuilder builder = new ProcessBuilder(command); + builder.environment().put("IOTDB_HOME", homePath); + return builder; + } + + private static void singleTag(String device, String value) { + tagged(device, "k", value); + } + + private static void tagged(String device, String key, String value) { + CREATED.put(DATABASE + "." + device + ".s1", new Series(mapOf(key, value), NO_VALUES, null)); + } + + private static Map mapOf(String... keysAndValues) { + final Map map = new LinkedHashMap<>(); + for (int i = 0; i < keysAndValues.length; i += 2) { + map.put(keysAndValues[i], keysAndValues[i + 1]); + } + return map; + } + + /** The tags, attributes and alias created for one time series. */ + private static final class Series { + private final Map tags; + private final Map attributes; + private final String alias; + + private Series(Map tags, Map attributes, String alias) { + this.tags = tags; + this.attributes = attributes; + this.alias = alias; + } + } +} diff --git a/iotdb-client/cli/src/main/i18n/en/org/apache/iotdb/cli/i18n/CliMessages.java b/iotdb-client/cli/src/main/i18n/en/org/apache/iotdb/cli/i18n/CliMessages.java index 8cd97e9b07884..5ac63fff12c65 100644 --- a/iotdb-client/cli/src/main/i18n/en/org/apache/iotdb/cli/i18n/CliMessages.java +++ b/iotdb-client/cli/src/main/i18n/en/org/apache/iotdb/cli/i18n/CliMessages.java @@ -157,6 +157,13 @@ public final class CliMessages { MESSAGE_THE_HEADER_OF_THE_CSV_FILE_TO_BE_IMPORTED_IS_ILLEGAL_THE_CORRECT_FORMAT_IS_TIMESERIES_ALIBABA_DATATYPE_ENCODING_COMPRESSION_C429E7C6 = "The header of the CSV file to be imported is illegal. The correct format is" + " \"Timeseries, Alibaba, DataType, Encoding, Compression\"!"; + public static final String + MESSAGE_THE_HEADER_OF_THE_CSV_FILE_TO_BE_IMPORTED_IS_ILLEGAL_THE_CORRECT_FORMAT_IS_TIMESERIES_ALIAS_DATATYPE_ENCODING_COMPRESSION_TAGS_ATTRIBUTES_14EDBFB9 = + "The header of the CSV file to be imported is illegal. The correct format is" + + " \"Timeseries, Alias, DataType, Encoding, Compression, Tags, Attributes\"!"; + public static final String + MESSAGE_FAILED_TO_PARSE_THE_ARG_COLUMN_OF_TIMESERIES_ARG_AS_JSON_THE_RECORD_IS_SKIPPED_ARG_35D898C8 = + "Failed to parse the %s column of timeseries %s as JSON, the record is skipped: %s"; private CliMessages() {} diff --git a/iotdb-client/cli/src/main/i18n/zh/org/apache/iotdb/cli/i18n/CliMessages.java b/iotdb-client/cli/src/main/i18n/zh/org/apache/iotdb/cli/i18n/CliMessages.java index 6123237db8d74..7efdc8d83ec7e 100644 --- a/iotdb-client/cli/src/main/i18n/zh/org/apache/iotdb/cli/i18n/CliMessages.java +++ b/iotdb-client/cli/src/main/i18n/zh/org/apache/iotdb/cli/i18n/CliMessages.java @@ -145,6 +145,13 @@ public final class CliMessages { public static final String MESSAGE_THE_HEADER_OF_THE_CSV_FILE_TO_BE_IMPORTED_IS_ILLEGAL_THE_CORRECT_FORMAT_IS_TIMESERIES_ALIBABA_DATATYPE_ENCODING_COMPRESSION_C429E7C6 = "待导入的 CSV 文件表头不合法,正确格式为 \"Timeseries, Alibaba, DataType, Encoding, Compression\"!"; + public static final String + MESSAGE_THE_HEADER_OF_THE_CSV_FILE_TO_BE_IMPORTED_IS_ILLEGAL_THE_CORRECT_FORMAT_IS_TIMESERIES_ALIAS_DATATYPE_ENCODING_COMPRESSION_TAGS_ATTRIBUTES_14EDBFB9 = + "待导入的 CSV 文件表头不合法,正确格式为" + + " \"Timeseries, Alias, DataType, Encoding, Compression, Tags, Attributes\"!"; + public static final String + MESSAGE_FAILED_TO_PARSE_THE_ARG_COLUMN_OF_TIMESERIES_ARG_AS_JSON_THE_RECORD_IS_SKIPPED_ARG_35D898C8 = + "时序 JSON 解析失败,列:%s,时序:%s,已跳过该记录,原因:%s"; private CliMessages() {} diff --git a/iotdb-client/cli/src/main/java/org/apache/iotdb/tool/common/Constants.java b/iotdb-client/cli/src/main/java/org/apache/iotdb/tool/common/Constants.java index ec4d2c7632cbc..e2c96f4d2e744 100644 --- a/iotdb-client/cli/src/main/java/org/apache/iotdb/tool/common/Constants.java +++ b/iotdb-client/cli/src/main/java/org/apache/iotdb/tool/common/Constants.java @@ -204,8 +204,31 @@ public class Constants { public static final String COLON = ": "; public static final String MINUS = "-"; + public static final String HEADER_TAGS = "Tags"; + public static final String HEADER_ATTRIBUTES = "Attributes"; + public static final String HEADER_ALIAS = "Alias"; + public static final String HEADER_DATATYPE = "DataType"; + public static final String HEADER_ENCODING = "Encoding"; + public static final String HEADER_COMPRESSION = "Compression"; + public static final List HEAD_COLUMNS = - Arrays.asList("Timeseries", "Alias", "DataType", "Encoding", "Compression"); + Arrays.asList( + "Timeseries", HEADER_ALIAS, HEADER_DATATYPE, HEADER_ENCODING, HEADER_COMPRESSION); + + /** + * Column layout written by the tree model schema export. The Tags and Attributes cells hold a + * JSON object, so files carrying them are parsed with the RFC4180 settings instead of the legacy + * ones. + */ + public static final List HEAD_COLUMNS_V2 = + Arrays.asList( + "Timeseries", + HEADER_ALIAS, + HEADER_DATATYPE, + HEADER_ENCODING, + HEADER_COMPRESSION, + HEADER_TAGS, + HEADER_ATTRIBUTES); // export constants public static final String EXPORT_CLI_PREFIX = CliMessages.EXCEPTION_EXPORT_DATA_2F1FB3D9; diff --git a/iotdb-client/cli/src/main/java/org/apache/iotdb/tool/schema/AbstractSchemaTool.java b/iotdb-client/cli/src/main/java/org/apache/iotdb/tool/schema/AbstractSchemaTool.java index b88cfda2c0a32..f9998491a1f41 100644 --- a/iotdb-client/cli/src/main/java/org/apache/iotdb/tool/schema/AbstractSchemaTool.java +++ b/iotdb-client/cli/src/main/java/org/apache/iotdb/tool/schema/AbstractSchemaTool.java @@ -30,6 +30,9 @@ import org.apache.iotdb.session.pool.TableSessionPoolBuilder; import org.apache.iotdb.tool.common.Constants; +import com.google.gson.Gson; +import com.google.gson.GsonBuilder; +import com.google.gson.reflect.TypeToken; import org.apache.commons.cli.CommandLine; import org.apache.commons.csv.CSVFormat; import org.apache.commons.csv.CSVPrinter; @@ -41,8 +44,11 @@ import java.io.IOException; import java.io.PrintWriter; +import java.lang.reflect.Type; import java.util.Iterator; import java.util.List; +import java.util.Map; +import java.util.TreeMap; import java.util.concurrent.atomic.LongAdder; public abstract class AbstractSchemaTool { @@ -77,6 +83,9 @@ public abstract class AbstractSchemaTool { private static final IoTPrinter ioTPrinter = new IoTPrinter(System.out); private static final Logger LOGGER = LoggerFactory.getLogger(AbstractSchemaTool.class); + private static final Gson JSON_GSON = new GsonBuilder().disableHtmlEscaping().create(); + private static final Type JSON_CELL_TYPE = new TypeToken>() {}.getType(); + protected AbstractSchemaTool() {} protected static Session.Builder configureSsl(Session.Builder builder) { @@ -218,6 +227,51 @@ public static Boolean writeCsvFile(List> records, String filePath) } } + /** + * The RFC4180 CSV layout of the schema files, i.e. the delimiter is {@code ','}, the quote char + * is {@code '"'} and no escape char is configured. A JSON object cell is therefore always quoted + * and its inner quotes are doubled, which is exactly what RFC4180 asks for. + */ + static CSVFormat.Builder createRfc4180CsvFormat() { + return CSVFormat.Builder.create(CSVFormat.DEFAULT) + .setQuote('"') + .setQuoteMode(QuoteMode.MINIMAL) + .setRecordSeparator('\n'); + } + + /** + * Serializes tags or attributes into the compact JSON object cell of the schema file. Keys are + * sorted so that repeated exports of the same schema are identical, and HTML escaping is disabled + * so that printable characters such as {@code '}, {@code =}, {@code <} and {@code &} stay + * readable. + * + * @return an empty string when there is nothing to write, never {@code {}} or {@code null} + */ + static String toJsonCell(Map values) { + if (values == null || values.isEmpty()) { + return ""; + } + return JSON_GSON.toJson(new TreeMap<>(values)); + } + + /** + * Parses a Tags / Attributes cell written by {@link #toJsonCell(Map)}. + * + * @return {@code null} when the cell carries no value, never an empty map + * @throws com.google.gson.JsonParseException when the cell is not a JSON object + */ + static Map parseJsonCell(String cell) { + if (cell == null) { + return null; + } + String trimmed = cell.trim(); + if (trimmed.isEmpty() || "null".equals(trimmed)) { + return null; + } + Map parsed = JSON_GSON.fromJson(trimmed, JSON_CELL_TYPE); + return parsed == null || parsed.isEmpty() ? null : new TreeMap<>(parsed); + } + static class CSVPrinterWrapper { private final String filePath; private final CSVFormat csvFormat; @@ -225,13 +279,7 @@ static class CSVPrinterWrapper { public CSVPrinterWrapper(String filePath) { this.filePath = filePath; - this.csvFormat = - CSVFormat.Builder.create(CSVFormat.DEFAULT) - .setHeader() - .setSkipHeaderRecord(true) - .setEscape('\\') - .setQuoteMode(QuoteMode.NONE) - .build(); + this.csvFormat = createRfc4180CsvFormat().setHeader().setSkipHeaderRecord(true).build(); } public void printRecord(final Iterable values) throws IOException { diff --git a/iotdb-client/cli/src/main/java/org/apache/iotdb/tool/schema/ExportSchemaTree.java b/iotdb-client/cli/src/main/java/org/apache/iotdb/tool/schema/ExportSchemaTree.java index b8cd60323e22c..12e3b5066485a 100644 --- a/iotdb-client/cli/src/main/java/org/apache/iotdb/tool/schema/ExportSchemaTree.java +++ b/iotdb-client/cli/src/main/java/org/apache/iotdb/tool/schema/ExportSchemaTree.java @@ -32,7 +32,9 @@ import java.io.File; import java.io.IOException; +import java.util.ArrayList; import java.util.List; +import java.util.Map; import static org.apache.iotdb.commons.schema.SchemaConstant.AUDIT_DATABASE; import static org.apache.iotdb.commons.schema.SchemaConstant.SYSTEM_DATABASE; @@ -98,25 +100,19 @@ private static void writeCsvFile( while (i++ < linesPerFile) { if (sessionDataSet.hasNext()) { if (i == 1) { - csvPrinterWrapper.printRecord(Constants.HEAD_COLUMNS); + csvPrinterWrapper.printRecord(Constants.HEAD_COLUMNS_V2); } RowRecord rowRecord = sessionDataSet.next(); List fields = rowRecord.getFields(); - if (fields.get(timeseriesIndex).getStringValue().startsWith(SYSTEM_DATABASE + ".") - || fields.get(timeseriesIndex).getStringValue().startsWith(AUDIT_DATABASE + ".") - || !fields.get(viewTypeIndex).getStringValue().equals(Constants.BASE_VIEW_TYPE)) { + String timeseries = fields.get(timeseriesIndex).getStringValue(); + if (timeseries.startsWith(SYSTEM_DATABASE + ".") + || timeseries.startsWith(AUDIT_DATABASE + ".") + || !Constants.BASE_VIEW_TYPE.equals(fields.get(viewTypeIndex).getStringValue())) { + continue; + } + if (!printRecord(csvPrinterWrapper, headers, fields, timeseries)) { continue; } - Constants.HEAD_COLUMNS.forEach( - column -> { - Field field = fields.get(headers.indexOf(column)); - String fieldStringValue = field.getStringValue(); - if (!"null".equals(field.getStringValue())) { - csvPrinterWrapper.print(fieldStringValue); - } else { - csvPrinterWrapper.print(""); - } - }); csvPrinterWrapper.println(); } else { hasNext = false; @@ -128,4 +124,44 @@ private static void writeCsvFile( csvPrinterWrapper.close(); } } + + /** + * Writes the cells of one {@code SHOW TIMESERIES} record. + * + * @return {@code false} when the record must be skipped because a Tags or Attributes cell could + * not be parsed, in which case nothing has been written for it + */ + static boolean printRecord( + CSVPrinterWrapper csvPrinterWrapper, + List headers, + List fields, + String timeseries) { + List cells = new ArrayList<>(Constants.HEAD_COLUMNS_V2.size()); + for (String column : Constants.HEAD_COLUMNS_V2) { + int index = headers.indexOf(column); + Field field = index < 0 ? null : fields.get(index); + String fieldStringValue = field == null ? null : field.getStringValue(); + if (Constants.HEADER_TAGS.equals(column) || Constants.HEADER_ATTRIBUTES.equals(column)) { + Map tags; + try { + tags = parseJsonCell(fieldStringValue); + } catch (RuntimeException e) { + ioTPrinter.println( + String.format( + CliMessages + .MESSAGE_FAILED_TO_PARSE_THE_ARG_COLUMN_OF_TIMESERIES_ARG_AS_JSON_THE_RECORD_IS_SKIPPED_ARG_35D898C8, + column, + timeseries, + String.valueOf(e.getMessage()))); + return false; + } + cells.add(toJsonCell(tags)); + } else { + cells.add( + "null".equals(fieldStringValue) || fieldStringValue == null ? "" : fieldStringValue); + } + } + cells.forEach(csvPrinterWrapper::print); + return true; + } } diff --git a/iotdb-client/cli/src/main/java/org/apache/iotdb/tool/schema/ImportSchemaTree.java b/iotdb-client/cli/src/main/java/org/apache/iotdb/tool/schema/ImportSchemaTree.java index 231467b33fadc..604e0ad17bb89 100644 --- a/iotdb-client/cli/src/main/java/org/apache/iotdb/tool/schema/ImportSchemaTree.java +++ b/iotdb-client/cli/src/main/java/org/apache/iotdb/tool/schema/ImportSchemaTree.java @@ -38,11 +38,13 @@ import org.apache.tsfile.file.metadata.enums.CompressionType; import org.apache.tsfile.file.metadata.enums.TSEncoding; +import java.io.BufferedReader; import java.io.File; import java.io.FileInputStream; import java.io.IOException; import java.io.InputStreamReader; import java.util.ArrayList; +import java.util.Arrays; import java.util.Collections; import java.util.HashSet; import java.util.List; @@ -58,7 +60,7 @@ public class ImportSchemaTree extends AbstractImportSchema { private static final IoTPrinter ioTPrinter = new IoTPrinter(System.out); - private static SessionPool sessionPool; + static SessionPool sessionPool; public void init() throws InterruptedException, IoTDBConnectionException, StatementExecutionException { @@ -131,18 +133,65 @@ protected void importSchemaFromCsvFile(File file) { } } - private static CSVParser readCsvFile(String path) throws IOException { - return CSVFormat.Builder.create(CSVFormat.DEFAULT) - .setHeader() - .setSkipHeaderRecord(true) - .setQuote('`') - .setEscape('\\') - .setIgnoreEmptyLines(true) - .build() - .parse(new InputStreamReader(new FileInputStream(path))); + static CSVParser readCsvFile(String path) throws IOException { + CSVFormat.Builder builder = + CSVFormat.Builder.create(CSVFormat.DEFAULT) + .setHeader() + .setSkipHeaderRecord(true) + .setIgnoreEmptyLines(true); + if (isV2Format(readHeaderNames(path))) { + builder.setQuote('"'); + } else { + builder.setQuote('`').setEscape('\\'); + } + return builder.build().parse(new InputStreamReader(new FileInputStream(path))); + } + + /** + * Reads the column names of a schema file without parsing it, so that the CSV settings of the + * file can be selected before it is read. + */ + static List readHeaderNames(String path) throws IOException { + try (BufferedReader reader = + new BufferedReader(new InputStreamReader(new FileInputStream(path)))) { + String headerLine = reader.readLine(); + if (headerLine == null) { + return Collections.emptyList(); + } + return Arrays.stream(headerLine.split(",")) + .map(ImportSchemaTree::unquoteColumnName) + .collect(Collectors.toList()); + } } - private static boolean checkHeader(List headerNames) { + /** Accepts a header written by a spreadsheet tool, which quotes every column name. */ + private static String unquoteColumnName(String columnName) { + String name = columnName.trim(); + if (name.length() > 1 && name.startsWith("\"") && name.endsWith("\"")) { + return name.substring(1, name.length() - 1).trim(); + } + return name; + } + + /** + * A file whose header carries a Tags or an Attributes column is a v2 file, which is RFC4180 and + * holds JSON cells. Everything else is a legacy file and keeps the settings it was written with. + */ + static boolean isV2Format(List headerNames) { + return headerNames.contains(Constants.HEADER_TAGS) + || headerNames.contains(Constants.HEADER_ATTRIBUTES); + } + + static boolean checkHeader(List headerNames) { + if (isV2Format(headerNames)) { + if (headerNames.containsAll(Constants.HEAD_COLUMNS_V2)) { + return true; + } + ioTPrinter.println( + CliMessages + .MESSAGE_THE_HEADER_OF_THE_CSV_FILE_TO_BE_IMPORTED_IS_ILLEGAL_THE_CORRECT_FORMAT_IS_TIMESERIES_ALIAS_DATATYPE_ENCODING_COMPRESSION_TAGS_ATTRIBUTES_14EDBFB9); + return false; + } if (CollectionUtils.isNotEmpty(headerNames) && new HashSet<>(headerNames).size() == Constants.HEAD_COLUMNS.size()) { List strangers = @@ -159,18 +208,25 @@ private static boolean checkHeader(List headerNames) { return true; } - private static void writeScheme( + static void writeScheme( String fileName, List headerNames, Stream records, String failedFilePath) throws IllegalPathException { + boolean v2 = isV2Format(headerNames); + int tagsIndex = headerNames.indexOf(Constants.HEADER_TAGS); + int attributesIndex = headerNames.indexOf(Constants.HEADER_ATTRIBUTES); List paths = new ArrayList<>(); List dataTypes = new ArrayList<>(); List encodings = new ArrayList<>(); List compressors = new ArrayList<>(); + List> tags = v2 ? new ArrayList<>() : null; + List> attributes = v2 ? new ArrayList<>() : null; List pathsWithAlias = new ArrayList<>(); List dataTypesWithAlias = new ArrayList<>(); List encodingsWithAlias = new ArrayList<>(); List compressorsWithAlias = new ArrayList<>(); List measurementAlias = new ArrayList<>(); + List> tagsWithAlias = v2 ? new ArrayList<>() : null; + List> attributesWithAlias = v2 ? new ArrayList<>() : null; AtomicReference hasStarted = new AtomicReference<>(false); AtomicInteger pointSize = new AtomicInteger(0); @@ -185,7 +241,7 @@ private static void writeScheme( try { if (CollectionUtils.isNotEmpty(paths)) { writeAndEmptyDataSet( - paths, dataTypes, encodings, compressors, null, null, null, null, 3); + paths, dataTypes, encodings, compressors, null, tags, attributes, null, 3); } } catch (Exception e) { paths.forEach(t -> failedRecords.add(Collections.singletonList(t))); @@ -198,8 +254,8 @@ private static void writeScheme( encodingsWithAlias, compressorsWithAlias, null, - null, - null, + tagsWithAlias, + attributesWithAlias, measurementAlias, 3); } @@ -211,6 +267,10 @@ private static void writeScheme( encodings.clear(); compressors.clear(); measurementAlias.clear(); + if (v2) { + tags.clear(); + attributes.clear(); + } pointSize.set(0); } } else { @@ -219,6 +279,10 @@ private static void writeScheme( encodings.clear(); compressors.clear(); measurementAlias.clear(); + if (v2) { + tags.clear(); + attributes.clear(); + } } String path = recordObj.get(headerNames.indexOf(Constants.HEAD_COLUMNS.get(0))); String alias = recordObj.get(headerNames.indexOf(Constants.HEAD_COLUMNS.get(1))); @@ -230,6 +294,23 @@ private static void writeScheme( String compressionTypeRaw = recordObj.get(headerNames.indexOf(Constants.HEAD_COLUMNS.get(4))); CompressionType compressionType = compressInfer(compressionTypeRaw); + Map recordTags = null; + Map recordAttributes = null; + boolean tagsParsed = true; + if (v2) { + try { + recordTags = parseJsonCell(recordObj.get(tagsIndex)); + } catch (RuntimeException e) { + ioTPrinter.println(invalidJsonMessage(Constants.HEADER_TAGS, path, e)); + tagsParsed = false; + } + try { + recordAttributes = parseJsonCell(recordObj.get(attributesIndex)); + } catch (RuntimeException e) { + ioTPrinter.println(invalidJsonMessage(Constants.HEADER_ATTRIBUTES, path, e)); + tagsParsed = false; + } + } if (StringUtils.isBlank(path) || path.trim().startsWith(SYSTEM_DATABASE) || path.trim().startsWith(AUDIT_DATABASE)) { @@ -268,31 +349,56 @@ private static void writeScheme( compressionTypeRaw)); failedRecords.add(recordObj.stream().collect(Collectors.toList())); failed = true; + } else if (!tagsParsed) { + failedRecords.add(recordObj.stream().collect(Collectors.toList())); + failed = true; } else { if (StringUtils.isBlank(alias)) { paths.add(path); dataTypes.add(dataType); encodings.add(encodingType); compressors.add(compressionType); + if (v2) { + tags.add(recordTags); + attributes.add(recordAttributes); + } } else { pathsWithAlias.add(path); dataTypesWithAlias.add(dataType); encodingsWithAlias.add(encodingType); compressorsWithAlias.add(compressionType); measurementAlias.add(alias); + if (v2) { + tagsWithAlias.add(recordTags); + attributesWithAlias.add(recordAttributes); + } } pointSize.getAndIncrement(); } if (!failed && aligned) { String deviceId = path.substring(0, path.lastIndexOf(".")); paths.add(0, path.substring(deviceId.length() + 1)); + if (v2) { + // keep the maps aligned with the measurements list, whose head now holds this one + tags.add(0, recordTags); + attributes.add(0, recordAttributes); + } writeAndEmptyDataSetAligned( - deviceId, paths, dataTypes, encodings, compressors, measurementAlias, 3); + deviceId, + paths, + dataTypes, + encodings, + compressors, + measurementAlias, + tags, + attributes, + 3); } }); try { if (CollectionUtils.isNotEmpty(paths)) { - writeAndEmptyDataSet(paths, dataTypes, encodings, compressors, null, null, null, null, 3); + writeAndEmptyDataSet( + paths, dataTypes, encodings, compressors, null, tags, attributes, null, 3); } } catch (Exception e) { paths.forEach(t -> failedRecords.add(Collections.singletonList(t))); @@ -305,8 +411,8 @@ private static void writeScheme( encodingsWithAlias, compressorsWithAlias, null, - null, - null, + tagsWithAlias, + attributesWithAlias, measurementAlias, 3); } @@ -380,10 +486,24 @@ private static void writeAndEmptyDataSetAligned( List encodings, List compressors, List measurementAliasList, + List> tagsList, + List> attributesList, int retryTime) { try { - sessionPool.createAlignedTimeseries( - deviceId, measurements, dataTypes, encodings, compressors, measurementAliasList); + if (tagsList == null && attributesList == null) { + sessionPool.createAlignedTimeseries( + deviceId, measurements, dataTypes, encodings, compressors, measurementAliasList); + } else { + sessionPool.createAlignedTimeseries( + deviceId, + measurements, + dataTypes, + encodings, + compressors, + measurementAliasList, + tagsList, + attributesList); + } } catch (IoTDBConnectionException e) { if (retryTime > 0) { writeAndEmptyDataSetAligned( @@ -393,6 +513,8 @@ private static void writeAndEmptyDataSetAligned( encodings, compressors, measurementAliasList, + tagsList, + attributesList, --retryTime); } } catch (StatementExecutionException e) { @@ -405,9 +527,24 @@ private static void writeAndEmptyDataSetAligned( encodings.clear(); compressors.clear(); measurementAliasList.clear(); + if (tagsList != null) { + tagsList.clear(); + } + if (attributesList != null) { + attributesList.clear(); + } } } + private static String invalidJsonMessage(String column, String path, Exception e) { + return String.format( + CliMessages + .MESSAGE_FAILED_TO_PARSE_THE_ARG_COLUMN_OF_TIMESERIES_ARG_AS_JSON_THE_RECORD_IS_SKIPPED_ARG_35D898C8, + column, + path, + String.valueOf(e.getMessage())); + } + private static TSDataType typeInfer(String typeStr) { try { if (StringUtils.isNotBlank(typeStr)) { diff --git a/iotdb-client/cli/src/test/java/org/apache/iotdb/tool/schema/SchemaTagsFidelityTest.java b/iotdb-client/cli/src/test/java/org/apache/iotdb/tool/schema/SchemaTagsFidelityTest.java new file mode 100644 index 0000000000000..85fd629ddbbbb --- /dev/null +++ b/iotdb-client/cli/src/test/java/org/apache/iotdb/tool/schema/SchemaTagsFidelityTest.java @@ -0,0 +1,318 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.tool.schema; + +import org.apache.iotdb.tool.common.Constants; + +import org.apache.commons.csv.CSVParser; +import org.apache.commons.csv.CSVRecord; +import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.read.common.Field; +import org.apache.tsfile.utils.Binary; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.TemporaryFolder; + +import java.io.File; +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.util.Arrays; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertTrue; + +/** Guards the lossless Tags / Attributes round trip of the tree model schema tools. */ +public class SchemaTagsFidelityTest { + + @Rule public TemporaryFolder temporaryFolder = new TemporaryFolder(); + + @Test + public void jsonCellSortsKeysAndKeepsPrintableCharactersReadable() { + Map tags = new LinkedHashMap<>(); + tags.put("type", "sensor"); + tags.put("room", "101"); + tags.put("a-b", "it's = & w"); + + assertEquals( + "{\"a-b\":\"it's = & w\",\"room\":\"101\",\"type\":\"sensor\"}", + AbstractSchemaTool.toJsonCell(tags)); + } + + @Test + public void jsonCellEscapesOnlyWhatJsonRequires() { + Map tags = new LinkedHashMap<>(); + tags.put("quote", "a\"b"); + tags.put("backslash", "a\\b"); + tags.put("tab", "a\tb"); + tags.put("newline", "a\nb"); + tags.put("comma", "a,b"); + tags.put("cjk", "中文😀"); + tags.put("backtick", "`x`"); + tags.put("null", "null"); + tags.put("empty", ""); + + assertEquals( + "{\"backslash\":\"a\\\\b\",\"backtick\":\"`x`\",\"cjk\":\"中文😀\",\"comma\":\"a,b\"," + + "\"empty\":\"\",\"newline\":\"a\\nb\",\"null\":\"null\",\"quote\":\"a\\\"b\"," + + "\"tab\":\"a\\tb\"}", + AbstractSchemaTool.toJsonCell(tags)); + } + + @Test + public void jsonCellOfMissingValuesIsAnEmptyCell() { + assertEquals("", AbstractSchemaTool.toJsonCell(null)); + assertEquals("", AbstractSchemaTool.toJsonCell(new LinkedHashMap<>())); + } + + @Test + public void jsonCellRoundTripsEveryCharacter() { + Map tags = new LinkedHashMap<>(); + tags.put("quote", "a\"b"); + tags.put("backslash", "a\\b"); + tags.put("tab", "a\tb"); + tags.put("newline", "a\nb"); + tags.put("comma", "a,b"); + tags.put("cjk", "中文😀"); + tags.put("backtick", "`x`"); + tags.put("null", "null"); + tags.put("empty", ""); + tags.put("key with space", "[ C ]"); + + assertEquals(tags, AbstractSchemaTool.parseJsonCell(AbstractSchemaTool.toJsonCell(tags))); + } + + @Test + public void jsonCellOfNothingParsesToNoValue() { + assertNull(AbstractSchemaTool.parseJsonCell(null)); + assertNull(AbstractSchemaTool.parseJsonCell("")); + assertNull(AbstractSchemaTool.parseJsonCell(" ")); + assertNull(AbstractSchemaTool.parseJsonCell("null")); + assertNull(AbstractSchemaTool.parseJsonCell("{}")); + } + + @Test + public void invalidJsonCellIsRejected() { + for (String cell : Arrays.asList("not json", "[1,2]", "\"abc\"", "{\"k\":\"v\"} trailing")) { + try { + AbstractSchemaTool.parseJsonCell(cell); + throw new AssertionError("expected a parse failure for " + cell); + } catch (RuntimeException expected) { + // the export reports this and skips the record + } + } + } + + @Test + public void csvRoundTripKeepsEveryFieldByteIdentical() throws IOException { + Map tags = new LinkedHashMap<>(); + tags.put("quote", "a\"b"); + tags.put("backslash", "a\\b"); + tags.put("comma", "a,b"); + tags.put("tab", "a\tb"); + tags.put("newline", "a\nb"); + Map attributes = new LinkedHashMap<>(); + attributes.put("owner", "team a"); + attributes.put("note", "q\"w\\e,r\ty\nz"); + + List row = + Arrays.asList( + "root.t.d1.s1", + "aliasA", + "INT64", + "TS_2DIFF", + "SNAPPY", + AbstractSchemaTool.toJsonCell(tags), + AbstractSchemaTool.toJsonCell(attributes)); + File csv = writeV2File("roundtrip.csv", row); + + CSVParser parser = ImportSchemaTree.readCsvFile(csv.getAbsolutePath()); + List records = parser.getRecords(); + assertEquals(1, records.size()); + CSVRecord record = records.get(0); + for (int i = 0; i < row.size(); i++) { + assertEquals(row.get(i), record.get(Constants.HEAD_COLUMNS_V2.get(i))); + } + assertEquals(tags, AbstractSchemaTool.parseJsonCell(record.get(Constants.HEADER_TAGS))); + assertEquals( + attributes, AbstractSchemaTool.parseJsonCell(record.get(Constants.HEADER_ATTRIBUTES))); + } + + @Test + public void emptyTagsAndAttributesAreWrittenAsEmptyCells() throws IOException { + File csv = + writeV2File( + "empty.csv", Arrays.asList("root.t.d1.s1", "", "INT64", "TS_2DIFF", "SNAPPY", "", "")); + + assertEquals( + "Timeseries,Alias,DataType,Encoding,Compression,Tags,Attributes\n" + + "root.t.d1.s1,,INT64,TS_2DIFF,SNAPPY,,\n", + new String(Files.readAllBytes(csv.toPath()), StandardCharsets.UTF_8)); + CSVRecord record = ImportSchemaTree.readCsvFile(csv.getAbsolutePath()).getRecords().get(0); + assertNull(AbstractSchemaTool.parseJsonCell(record.get(Constants.HEADER_TAGS))); + assertNull(AbstractSchemaTool.parseJsonCell(record.get(Constants.HEADER_ATTRIBUTES))); + } + + @Test + public void jsonCellsAreRfc4180Quoted() throws IOException { + Map tags = new LinkedHashMap<>(); + tags.put("room", "101"); + tags.put("type", "sensor"); + File csv = + writeV2File( + "quoted.csv", + Arrays.asList( + "root.t.d1.s1", + "", + "INT64", + "TS_2DIFF", + "SNAPPY", + AbstractSchemaTool.toJsonCell(tags), + "")); + + assertEquals( + "Timeseries,Alias,DataType,Encoding,Compression,Tags,Attributes\n" + + "root.t.d1.s1,,INT64,TS_2DIFF,SNAPPY,\"{\"\"room\"\":\"\"101\"\"," + + "\"\"type\"\":\"\"sensor\"\"}\",\n", + new String(Files.readAllBytes(csv.toPath()), StandardCharsets.UTF_8)); + } + + @Test + public void legacyFiveColumnHeaderSelectsTheLegacyPath() { + assertFalse(ImportSchemaTree.isV2Format(Constants.HEAD_COLUMNS)); + assertTrue(ImportSchemaTree.isV2Format(Constants.HEAD_COLUMNS_V2)); + } + + @Test + public void quotedHeaderStillSelectsTheV2Path() throws IOException { + File csv = temporaryFolder.newFile("quoted_header.csv"); + Files.write( + csv.toPath(), + ("\"Timeseries\",\"Alias\",\"DataType\",\"Encoding\",\"Compression\",\"Tags\"," + + "\"Attributes\"\nroot.t.d1.s1,,INT64,TS_2DIFF,SNAPPY,,\n") + .getBytes(StandardCharsets.UTF_8)); + + assertTrue( + ImportSchemaTree.isV2Format(ImportSchemaTree.readHeaderNames(csv.getAbsolutePath()))); + } + + @Test + public void legacyFileKeepsItsBackslashEscapes() throws IOException { + File csv = temporaryFolder.newFile("legacy.csv"); + Files.write( + csv.toPath(), + ("Timeseries,Alias,DataType,Encoding,Compression\n" + + "root.sg.d.s1,a\\,b,INT64,TS_2DIFF,SNAPPY\n") + .getBytes(StandardCharsets.UTF_8)); + + CSVRecord record = ImportSchemaTree.readCsvFile(csv.getAbsolutePath()).getRecords().get(0); + assertEquals(5, record.size()); + assertEquals("a,b", record.get(Constants.HEADER_ALIAS)); + } + + @Test + public void exportedRecordCarriesTagsAndAttributes() throws IOException { + File csv = temporaryFolder.newFile("export.csv"); + AbstractSchemaTool.CSVPrinterWrapper printer = + new AbstractSchemaTool.CSVPrinterWrapper(csv.getAbsolutePath()); + printer.printRecord(Constants.HEAD_COLUMNS_V2); + + List headers = + Arrays.asList( + Constants.HEADER_TIMESERIES, + Constants.HEADER_ALIAS, + "Database", + Constants.HEADER_DATATYPE, + Constants.HEADER_ENCODING, + Constants.HEADER_COMPRESSION, + Constants.HEADER_TAGS, + Constants.HEADER_ATTRIBUTES, + Constants.HEADER_VIEW_TYPE); + List fields = + Arrays.asList( + text("root.t.d1.s1"), + emptyField(), + text("root.t"), + text("INT64"), + text("TS_2DIFF"), + text("SNAPPY"), + text("{\"room\": \"101\", \"type\": \"sensor\"}"), + emptyField(), + text(Constants.BASE_VIEW_TYPE)); + assertTrue(ExportSchemaTree.printRecord(printer, headers, fields, "root.t.d1.s1")); + printer.println(); + printer.close(); + + CSVRecord record = ImportSchemaTree.readCsvFile(csv.getAbsolutePath()).getRecords().get(0); + assertEquals("root.t.d1.s1", record.get(Constants.HEADER_TIMESERIES)); + assertEquals("", record.get(Constants.HEADER_ALIAS)); + assertEquals("INT64", record.get(Constants.HEADER_DATATYPE)); + assertEquals("TS_2DIFF", record.get(Constants.HEADER_ENCODING)); + assertEquals("SNAPPY", record.get(Constants.HEADER_COMPRESSION)); + assertEquals("{\"room\":\"101\",\"type\":\"sensor\"}", record.get(Constants.HEADER_TAGS)); + assertEquals("", record.get(Constants.HEADER_ATTRIBUTES)); + } + + @Test + public void exportedRecordWithBrokenJsonIsSkipped() throws IOException { + File csv = temporaryFolder.newFile("broken.csv"); + AbstractSchemaTool.CSVPrinterWrapper printer = + new AbstractSchemaTool.CSVPrinterWrapper(csv.getAbsolutePath()); + List headers = + Arrays.asList( + Constants.HEADER_TIMESERIES, Constants.HEADER_TAGS, Constants.HEADER_ATTRIBUTES); + List fields = Arrays.asList(text("root.t.d1.s1"), text("not json"), emptyField()); + + assertFalse(ExportSchemaTree.printRecord(printer, headers, fields, "root.t.d1.s1")); + printer.close(); + assertEquals(0, csv.length()); + } + + /** + * Writes a v2 file with the writer of the schema tools so that the reader is exercised against + * the bytes the export really produces. + */ + private File writeV2File(String name, List row) throws IOException { + File csv = temporaryFolder.newFile(name); + AbstractSchemaTool.CSVPrinterWrapper printer = + new AbstractSchemaTool.CSVPrinterWrapper(csv.getAbsolutePath()); + printer.printRecord(Constants.HEAD_COLUMNS_V2); + row.forEach(printer::print); + printer.println(); + printer.close(); + return csv; + } + + private static Field text(String value) { + Field field = new Field(TSDataType.TEXT); + field.setBinaryV(new Binary(value, StandardCharsets.UTF_8)); + return field; + } + + /** The shape SHOW TIMESERIES uses for a cell without a value. */ + private static Field emptyField() { + return new Field(null); + } +} diff --git a/iotdb-client/cli/src/test/java/org/apache/iotdb/tool/schema/SchemaTagsImportTest.java b/iotdb-client/cli/src/test/java/org/apache/iotdb/tool/schema/SchemaTagsImportTest.java new file mode 100644 index 0000000000000..c3531bac976fc --- /dev/null +++ b/iotdb-client/cli/src/test/java/org/apache/iotdb/tool/schema/SchemaTagsImportTest.java @@ -0,0 +1,208 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.iotdb.tool.schema; + +import org.apache.iotdb.session.pool.SessionPool; +import org.apache.iotdb.tool.common.Constants; + +import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.file.metadata.enums.CompressionType; +import org.apache.tsfile.file.metadata.enums.TSEncoding; +import org.junit.After; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.TemporaryFolder; + +import java.io.File; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.Mockito.doAnswer; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.verify; + +/** Checks that the import passes the parsed tags and attributes on to the client API. */ +public class SchemaTagsImportTest { + + @Rule public TemporaryFolder temporaryFolder = new TemporaryFolder(); + + @After + public void reset() { + ImportSchemaTree.sessionPool = null; + ImportSchemaTree.aligned = false; + } + + @Test + public void legacyFileKeepsTheLegacyCreateCall() throws Exception { + SessionPool pool = mock(SessionPool.class); + ImportSchemaTree.sessionPool = pool; + + runImport( + "Timeseries,Alias,DataType,Encoding,Compression\n" + + "root.sg.d.s1,,INT64,TS_2DIFF,SNAPPY\n" + + "root.sg.d.s2,aliasB,INT64,TS_2DIFF,SNAPPY\n"); + + verify(pool) + .createMultiTimeseries( + Arrays.asList("root.sg.d.s1"), + Arrays.asList(TSDataType.INT64), + Arrays.asList(TSEncoding.TS_2DIFF), + Arrays.asList(CompressionType.SNAPPY), + null, + null, + null, + null); + verify(pool) + .createMultiTimeseries( + Arrays.asList("root.sg.d.s2"), + Arrays.asList(TSDataType.INT64), + Arrays.asList(TSEncoding.TS_2DIFF), + Arrays.asList(CompressionType.SNAPPY), + null, + null, + null, + Arrays.asList("aliasB")); + } + + @Test + public void v2FilePassesTagsAndAttributesToCreateMultiTimeseries() throws Exception { + SessionPool pool = mock(SessionPool.class); + ImportSchemaTree.sessionPool = pool; + Map tags = new LinkedHashMap<>(); + tags.put("room", "101"); + tags.put("type", "sensor"); + Map attributes = new LinkedHashMap<>(); + attributes.put("owner", "team a"); + + runImport( + Constants.HEAD_COLUMNS_V2.stream().collect(Collectors.joining(",")) + + "\nroot.sg.d.s1,,INT64,TS_2DIFF,SNAPPY,\"" + + AbstractSchemaTool.toJsonCell(tags).replace("\"", "\"\"") + + "\",\"" + + AbstractSchemaTool.toJsonCell(attributes).replace("\"", "\"\"") + + "\"\n" + + "root.sg.d.s2,,INT64,TS_2DIFF,SNAPPY,,\n"); + + verify(pool) + .createMultiTimeseries( + Arrays.asList("root.sg.d.s1", "root.sg.d.s2"), + Arrays.asList(TSDataType.INT64, TSDataType.INT64), + Arrays.asList(TSEncoding.TS_2DIFF, TSEncoding.TS_2DIFF), + Arrays.asList(CompressionType.SNAPPY, CompressionType.SNAPPY), + null, + Arrays.asList(new LinkedHashMap<>(tags), null), + Arrays.asList(new LinkedHashMap<>(attributes), null), + null); + } + + @Test + public void v2AlignedFilePassesTagsAndAttributesToCreateAlignedTimeseries() throws Exception { + SessionPool pool = mock(SessionPool.class); + ImportSchemaTree.sessionPool = pool; + ImportSchemaTree.aligned = true; + Map tags = new LinkedHashMap<>(); + tags.put("room", "101"); + Map attributes = new LinkedHashMap<>(); + attributes.put("owner", "team a"); + // the aligned writer empties its lists in a finally block, so snapshot the arguments + List alignedCall = new ArrayList<>(); + doAnswer( + call -> { + for (Object argument : call.getArguments()) { + alignedCall.add( + argument instanceof List ? new ArrayList<>((List) argument) : argument); + } + return null; + }) + .when(pool) + .createAlignedTimeseries(any(), any(), any(), any(), any(), any(), any(), any()); + + runImport( + Constants.HEAD_COLUMNS_V2.stream().collect(Collectors.joining(",")) + + "\nroot.sg.d.s1,,INT64,TS_2DIFF,SNAPPY,\"" + + AbstractSchemaTool.toJsonCell(tags).replace("\"", "\"\"") + + "\",\"" + + AbstractSchemaTool.toJsonCell(attributes).replace("\"", "\"\"") + + "\"\n"); + + // the pre-existing aligned path hands over both the measurement and the full path, so the maps + // must be built with the same shape to stay positionally aligned + assertEquals( + Arrays.asList( + "root.sg.d", + Arrays.asList("s1", "root.sg.d.s1"), + Arrays.asList(TSDataType.INT64), + Arrays.asList(TSEncoding.TS_2DIFF), + Arrays.asList(CompressionType.SNAPPY), + new ArrayList(), + Arrays.asList(new LinkedHashMap<>(tags), new LinkedHashMap<>(tags)), + Arrays.asList(new LinkedHashMap<>(attributes), new LinkedHashMap<>(attributes))), + alignedCall); + verify(pool, never()) + .createMultiTimeseries(any(), any(), any(), any(), any(), any(), any(), any()); + } + + @Test + public void v2HeaderWithoutAllColumnsCreatesNothing() throws Exception { + SessionPool pool = mock(SessionPool.class); + ImportSchemaTree.sessionPool = pool; + assertFalse( + ImportSchemaTree.checkHeader( + Arrays.asList("Timeseries", "Alias", "DataType", "Encoding", "Compression", "Tags"))); + + verify(pool, never()) + .createMultiTimeseries(any(), any(), any(), any(), any(), any(), any(), any()); + } + + @Test + public void brokenJsonCellSkipsTheRecord() throws Exception { + SessionPool pool = mock(SessionPool.class); + ImportSchemaTree.sessionPool = pool; + + runImport( + Constants.HEAD_COLUMNS_V2.stream().collect(Collectors.joining(",")) + + "\nroot.sg.d.s1,,INT64,TS_2DIFF,SNAPPY,\"{not json}\",\n"); + + verify(pool, never()) + .createMultiTimeseries(any(), any(), any(), any(), any(), any(), any(), any()); + } + + private void runImport(String csvContent) throws Exception { + File csv = temporaryFolder.newFile("schema_" + System.nanoTime() + ".csv"); + Files.write(csv.toPath(), csvContent.getBytes(StandardCharsets.UTF_8)); + String failedFilePath = temporaryFolder.getRoot().getAbsolutePath() + "/failed"; + try (Stream records = + ImportSchemaTree.readCsvFile(csv.getAbsolutePath()).stream()) { + List headerNames = ImportSchemaTree.readHeaderNames(csv.getAbsolutePath()); + ImportSchemaTree.writeScheme(csv.getName(), headerNames, records, failedFilePath); + } + } +} From e923f62ce3a468549b159f7895858e584a9d185b Mon Sep 17 00:00:00 2001 From: CritasWang Date: Thu, 5 Nov 2026 19:07:39 +0800 Subject: [PATCH 3/6] [Tools] Clear the aliased lists at every import batch boundary writeScheme cleared only paths, dataTypes, encodings, compressors and measurementAlias when a batch was flushed, while pathsWithAlias and its parallel lists kept their entries. A file with more aliased records than the batch size therefore paired aliases (and tags) with the wrong paths and re-sent time series that had already been created. --- .../iotdb/tool/schema/ImportSchemaTree.java | 12 +++ .../tool/schema/SchemaTagsImportTest.java | 80 +++++++++++++++++++ 2 files changed, 92 insertions(+) diff --git a/iotdb-client/cli/src/main/java/org/apache/iotdb/tool/schema/ImportSchemaTree.java b/iotdb-client/cli/src/main/java/org/apache/iotdb/tool/schema/ImportSchemaTree.java index 604e0ad17bb89..9e0471a80d605 100644 --- a/iotdb-client/cli/src/main/java/org/apache/iotdb/tool/schema/ImportSchemaTree.java +++ b/iotdb-client/cli/src/main/java/org/apache/iotdb/tool/schema/ImportSchemaTree.java @@ -266,10 +266,16 @@ static void writeScheme( dataTypes.clear(); encodings.clear(); compressors.clear(); + pathsWithAlias.clear(); + dataTypesWithAlias.clear(); + encodingsWithAlias.clear(); + compressorsWithAlias.clear(); measurementAlias.clear(); if (v2) { tags.clear(); attributes.clear(); + tagsWithAlias.clear(); + attributesWithAlias.clear(); } pointSize.set(0); } @@ -278,10 +284,16 @@ static void writeScheme( dataTypes.clear(); encodings.clear(); compressors.clear(); + pathsWithAlias.clear(); + dataTypesWithAlias.clear(); + encodingsWithAlias.clear(); + compressorsWithAlias.clear(); measurementAlias.clear(); if (v2) { tags.clear(); attributes.clear(); + tagsWithAlias.clear(); + attributesWithAlias.clear(); } } String path = recordObj.get(headerNames.indexOf(Constants.HEAD_COLUMNS.get(0))); diff --git a/iotdb-client/cli/src/test/java/org/apache/iotdb/tool/schema/SchemaTagsImportTest.java b/iotdb-client/cli/src/test/java/org/apache/iotdb/tool/schema/SchemaTagsImportTest.java index c3531bac976fc..4b4065c8ccf02 100644 --- a/iotdb-client/cli/src/test/java/org/apache/iotdb/tool/schema/SchemaTagsImportTest.java +++ b/iotdb-client/cli/src/test/java/org/apache/iotdb/tool/schema/SchemaTagsImportTest.java @@ -23,6 +23,7 @@ import org.apache.iotdb.tool.common.Constants; import org.apache.tsfile.enums.TSDataType; +import org.apache.tsfile.external.commons.collections4.CollectionUtils; import org.apache.tsfile.file.metadata.enums.CompressionType; import org.apache.tsfile.file.metadata.enums.TSEncoding; import org.junit.After; @@ -35,6 +36,7 @@ import java.nio.file.Files; import java.util.ArrayList; import java.util.Arrays; +import java.util.Collections; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; @@ -43,6 +45,7 @@ import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; import static org.mockito.ArgumentMatchers.any; import static org.mockito.Mockito.doAnswer; import static org.mockito.Mockito.mock; @@ -58,6 +61,7 @@ public class SchemaTagsImportTest { public void reset() { ImportSchemaTree.sessionPool = null; ImportSchemaTree.aligned = false; + ImportSchemaTree.batchPointSize = Constants.BATCH_POINT_SIZE; } @Test @@ -195,9 +199,85 @@ public void brokenJsonCellSkipsTheRecord() throws Exception { .createMultiTimeseries(any(), any(), any(), any(), any(), any(), any(), any()); } + /** + * Guards the batch-flush boundary: every aliased list must be emptied together, otherwise the + * next batch pairs the aliases still buffered with the wrong paths. + */ + @Test + public void batchedAliasedRecordsKeepAliasTagsAndPathsPaired() throws Exception { + SessionPool pool = mock(SessionPool.class); + ImportSchemaTree.sessionPool = pool; + ImportSchemaTree.batchPointSize = 2; + List>> calls = new ArrayList<>(); + doAnswer( + call -> { + List> snapshot = new ArrayList<>(); + for (int i = 0; i < call.getArguments().length; i++) { + List argument = call.getArgument(i); + snapshot.add(argument == null ? null : new ArrayList<>(argument)); + } + calls.add(snapshot); + return null; + }) + .when(pool) + .createMultiTimeseries(any(), any(), any(), any(), any(), any(), any(), any()); + + File csv = temporaryFolder.newFile("batch.csv"); + AbstractSchemaTool.CSVPrinterWrapper printer = + new AbstractSchemaTool.CSVPrinterWrapper(csv.getAbsolutePath()); + printer.printRecord(Constants.HEAD_COLUMNS_V2); + for (int i = 1; i <= 5; i++) { + printer.print("root.sg.d.s" + i); + printer.print("alias" + i); + printer.print("INT64"); + printer.print("TS_2DIFF"); + printer.print("SNAPPY"); + printer.print(AbstractSchemaTool.toJsonCell(Collections.singletonMap("k", "v" + i))); + printer.print(""); + printer.println(); + } + printer.close(); + runImportCsv(csv); + + // 3 flushes with batchPointSize 2: [s1,s2], [s3,s4], [s5] + assertEquals(3, calls.size()); + List sentPaths = new ArrayList<>(); + for (List> call : calls) { + List paths = call.get(0); + List dataTypes = call.get(1); + List encodings = call.get(2); + List compressors = call.get(3); + List tags = call.get(5); + List attributes = call.get(6); + List measurementAlias = call.get(7); + assertTrue(CollectionUtils.isNotEmpty(paths)); + // every list that is indexed by the path position must have the same length + for (List parallel : + Arrays.asList(dataTypes, encodings, compressors, tags, attributes, measurementAlias)) { + assertEquals("lists must stay positionally aligned", paths.size(), parallel.size()); + } + for (int i = 0; i < paths.size(); i++) { + String path = (String) paths.get(i); + String seq = path.substring(path.length() - 1); + sentPaths.add(path); + // s must travel with alias and with the tag built from the same n + assertEquals(path, "alias" + seq, measurementAlias.get(i)); + assertEquals("v" + seq, ((Map) tags.get(i)).get("k")); + } + } + assertEquals( + Arrays.asList( + "root.sg.d.s1", "root.sg.d.s2", "root.sg.d.s3", "root.sg.d.s4", "root.sg.d.s5"), + sentPaths); + } + private void runImport(String csvContent) throws Exception { File csv = temporaryFolder.newFile("schema_" + System.nanoTime() + ".csv"); Files.write(csv.toPath(), csvContent.getBytes(StandardCharsets.UTF_8)); + runImportCsv(csv); + } + + private void runImportCsv(File csv) throws Exception { String failedFilePath = temporaryFolder.getRoot().getAbsolutePath() + "/failed"; try (Stream records = ImportSchemaTree.readCsvFile(csv.getAbsolutePath()).stream()) { From 5967f290561dce2ee6a633936611afb9ae501c40 Mon Sep 17 00:00:00 2001 From: CritasWang Date: Thu, 5 Nov 2026 19:15:12 +0800 Subject: [PATCH 4/6] [Tools] Declare the gson dependency used by the schema tools The schema export/import tools parse the tags and attributes cells with gson, which was only available transitively; the dependency analysis in mvn verify rejects used-but-undeclared dependencies. --- iotdb-client/cli/pom.xml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/iotdb-client/cli/pom.xml b/iotdb-client/cli/pom.xml index a1e00508740bd..5302c86b8fd81 100644 --- a/iotdb-client/cli/pom.xml +++ b/iotdb-client/cli/pom.xml @@ -144,6 +144,10 @@ org.apache.commons commons-csv + + com.google.code.gson + gson + junit junit From 12cf1b6a8d09a6ed900ec4e550b2d607a1e76311 Mon Sep 17 00:00:00 2001 From: CritasWang Date: Thu, 5 Nov 2026 22:40:56 +0800 Subject: [PATCH 5/6] [IT] Read a NULL alias back as null in the schema tags round-trip test Field.getStringValue() renders a NULL cell as the text "null", so comparing the restored alias against null failed for every time series without an alias. Normalise the value in the cell helper, the same way the export tool distinguishes a missing value. --- .../org/apache/iotdb/tools/it/SchemaTagsRoundTripIT.java | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/integration-test/src/test/java/org/apache/iotdb/tools/it/SchemaTagsRoundTripIT.java b/integration-test/src/test/java/org/apache/iotdb/tools/it/SchemaTagsRoundTripIT.java index 2df7ff0511bc7..248643eefd46d 100644 --- a/integration-test/src/test/java/org/apache/iotdb/tools/it/SchemaTagsRoundTripIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/tools/it/SchemaTagsRoundTripIT.java @@ -366,7 +366,13 @@ private static String cell(List fields, int index) { return null; } final Field field = fields.get(index); - return field == null ? null : field.getStringValue(); + if (field == null) { + return null; + } + // A NULL field is rendered as the text "null" by getStringValue(), which is how the export + // tool distinguishes a missing value as well. + final String value = field.getStringValue(); + return "null".equals(value) ? null : value; } /** Parses a JSON object cell into a map; an empty cell carries no tags and no attributes. */ From 3ffe671c99c5d12763a9fafaf0181be7fb78a71f Mon Sep 17 00:00:00 2001 From: CritasWang Date: Fri, 6 Nov 2026 10:32:08 +0800 Subject: [PATCH 6/6] [IT] Cover mixed devices, types and tag/attribute combinations in the round trip The test only created DOUBLE/GORILLA/LZ4 series, so a regression that lost a per-series type, encoding or compression, or one of the four tag/attribute presence combinations, could pass unnoticed. Cover six type/encoding/compression combinations over twenty series spread over twenty devices, keep all four presence combinations (both, tags only, attributes only, neither) in the single exported file, assert that one device mixes several of them across its measurements, and check the format of every row after the import as well. --- .../iotdb/tools/it/SchemaTagsRoundTripIT.java | 236 +++++++++++++++--- 1 file changed, 201 insertions(+), 35 deletions(-) diff --git a/integration-test/src/test/java/org/apache/iotdb/tools/it/SchemaTagsRoundTripIT.java b/integration-test/src/test/java/org/apache/iotdb/tools/it/SchemaTagsRoundTripIT.java index 248643eefd46d..a8f33bc59f454 100644 --- a/integration-test/src/test/java/org/apache/iotdb/tools/it/SchemaTagsRoundTripIT.java +++ b/integration-test/src/test/java/org/apache/iotdb/tools/it/SchemaTagsRoundTripIT.java @@ -57,9 +57,11 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; +import java.util.EnumSet; import java.util.LinkedHashMap; import java.util.List; import java.util.Map; +import java.util.Set; import java.util.TreeMap; import java.util.concurrent.TimeUnit; import java.util.stream.Collectors; @@ -74,6 +76,10 @@ * Proves that the tree model schema tools restore the tags and attributes of every time series: * {@code export-schema} writes them as RFC4180 quoted JSON, {@code import-schema} reads that format * back, and awkward characters survive the round trip unchanged. + * + *

The schema of one database and one exported file mixes everything the tools have to keep + * apart: every data type/encoding/compression combination, all four tag/attribute combinations on + * different devices, and several combinations across the measurements of one device. */ @RunWith(IoTDBTestRunner.class) @Category({LocalStandaloneIT.class, ClusterIT.class}) @@ -104,27 +110,61 @@ public class SchemaTagsRoundTripIT extends AbstractScriptIT { private static final Map CREATED = new LinkedHashMap<>(); static { - // one tag value per device, because tags belong to the device rather than to the measurement - singleTag("v1", "v1"); - singleTag("comma", "a,b"); - singleTag("quote", "a\"b"); - singleTag("backslash", "a\\b"); - singleTag("backslashn", "a\\nb"); - singleTag("tab", "a\tb"); - singleTag("newline", "a\nb"); - singleTag("cjk", "中文😀"); - singleTag("nulllit", "null"); - singleTag("empty", ""); - singleTag("backtick", "`x`"); - tagged("spacedkey", "k e y", "v"); - CREATED.put( - DATABASE + ".multi.s1", - new Series(mapOf("a", "1", "b", "a,b", "c", "a\"b"), NO_VALUES, null)); - CREATED.put( - DATABASE + ".attr.s1", - new Series(NO_VALUES, mapOf("note", "he said \"hi\"", "plain", "v"), null)); - CREATED.put(NO_ATTRIBUTES_ROW, new Series(NO_VALUES, NO_VALUES, null)); - CREATED.put(DATABASE + ".alias.s1", new Series(mapOf("aliasTag", "v"), NO_VALUES, ALIAS)); + // Awkward tag values, one device per value because tags belong to the device. + singleTag("v1", "v1", Format.BOOLEAN_PLAIN_SNAPPY); + singleTag("comma", "a,b", Format.INT32_TS_2DIFF_SNAPPY); + singleTag("quote", "a\"b", Format.DOUBLE_GORILLA_LZ4); + singleTag("backslash", "a\\b", Format.INT64_GORILLA_LZ4); + singleTag("backslashn", "a\\nb", Format.FLOAT_GORILLA_GZIP); + singleTag("tab", "a\tb", Format.DOUBLE_GORILLA_LZ4); + singleTag("newline", "a\nb", Format.TEXT_DICTIONARY_SNAPPY); + singleTag("cjk", "中文😀", Format.INT64_GORILLA_LZ4); + singleTag("nulllit", "null", Format.BOOLEAN_PLAIN_SNAPPY); + singleTag("empty", "", Format.INT32_TS_2DIFF_SNAPPY); + singleTag("backtick", "`x`", Format.FLOAT_GORILLA_GZIP); + tagged("spacedkey", "k e y", "v", Format.TEXT_DICTIONARY_SNAPPY); + + // The four tag/attribute combinations, each on a device of its own. + series( + "both", + "s1", + Format.BOOLEAN_PLAIN_SNAPPY, + mapOf("t1", "v1", "t2", "a,b"), + mapOf("a1", "one", "a2", "he said \"hi\""), + null); + series( + "multi", + "s1", + Format.TEXT_DICTIONARY_SNAPPY, + mapOf("a", "1", "b", "a,b", "c", "a\"b"), + NO_VALUES, + null); + series( + "attr", + "s1", + Format.TEXT_DICTIONARY_SNAPPY, + NO_VALUES, + mapOf("note", "he said \"hi\"", "plain", "v"), + null); + series("none", "s1", Format.TEXT_DICTIONARY_SNAPPY, NO_VALUES, NO_VALUES, null); + series("alias", "s1", Format.FLOAT_GORILLA_GZIP, mapOf("aliasTag", "v"), NO_VALUES, ALIAS); + + // One device mixing three of the combinations across its measurements. + series( + "mixed", + "tagsOnly", + Format.INT32_TS_2DIFF_SNAPPY, + mapOf("m1", "x", "m2", "y"), + NO_VALUES, + null); + series( + "mixed", + "attrsOnly", + Format.INT64_GORILLA_LZ4, + NO_VALUES, + mapOf("m1", "x", "m2", "y"), + null); + series("mixed", "neither", Format.DOUBLE_GORILLA_LZ4, NO_VALUES, NO_VALUES, null); } @Rule public TemporaryFolder folder = new TemporaryFolder(); @@ -215,9 +255,9 @@ private static void createSchema() throws IoTDBConnectionException, StatementExe // character that the export or the import tool mangles. session.createTimeseries( entry.getKey(), - TSDataType.DOUBLE, - TSEncoding.GORILLA, - CompressionType.LZ4, + series.format.dataType, + series.format.encoding, + series.format.compression, null, series.tags.isEmpty() ? null : series.tags, series.attributes.isEmpty() ? null : series.attributes, @@ -295,25 +335,66 @@ private static void assertExportFormat(Path csv) throws IOException { assertEquals( "unexpected number of exported rows: " + records, CREATED.size(), records.size()); + final Set formats = EnumSet.noneOf(Format.class); for (Map.Entry entry : CREATED.entrySet()) { final String path = entry.getKey(); final Series series = entry.getValue(); final CSVRecord record = records.get(path); assertNotNull("the export is missing " + path, record); - assertEquals(path + " DataType", "DOUBLE", record.get("DataType")); - assertEquals(path + " Encoding", "GORILLA", record.get("Encoding")); - assertEquals(path + " Compression", "LZ4", record.get("Compression")); + assertEquals(path + " DataType", series.format.dataType.name(), record.get("DataType")); + assertEquals(path + " Encoding", series.format.encoding.name(), record.get("Encoding")); + assertEquals( + path + " Compression", series.format.compression.name(), record.get("Compression")); assertEquals( path + " Alias", series.alias == null ? "" : series.alias, record.get("Alias")); assertEquals(path + " Tags", series.tags, parseJsonObject(record.get("Tags"))); assertEquals( path + " Attributes", series.attributes, parseJsonObject(record.get("Attributes"))); + // The three assertions above proved that the file carries this combination. + formats.add(series.format); } + assertEquals( + "the file must carry every data type, encoding and compression combination", + EnumSet.allOf(Format.class), + formats); // Nothing to write is written as an empty cell, not as {} and not as null. assertEquals("", records.get(NO_ATTRIBUTES_ROW).get("Tags")); assertEquals("", records.get(NO_ATTRIBUTES_ROW).get("Attributes")); + + assertPresenceCombinations(records); + } + } + + /** + * The one exported file must carry all four tag/attribute combinations, spread over several + * devices, and at least one device must mix several of them across its measurements. + */ + private static void assertPresenceCombinations(Map records) { + final Set presences = EnumSet.noneOf(Presence.class); + final Map> presencesByDevice = new TreeMap<>(); + for (Map.Entry entry : records.entrySet()) { + final String path = entry.getKey(); + final CSVRecord record = entry.getValue(); + final Presence presence = + Presence.of( + parseJsonObject(record.get("Tags")), parseJsonObject(record.get("Attributes"))); + presences.add(presence); + presencesByDevice + .computeIfAbsent( + path.substring(0, path.lastIndexOf('.')), device -> EnumSet.noneOf(Presence.class)) + .add(presence); } + assertEquals( + "the file must carry every tag/attribute combination", + EnumSet.allOf(Presence.class), + presences); + assertTrue( + "the combinations must be spread over at least four devices: " + presencesByDevice, + presencesByDevice.size() >= 4); + assertTrue( + "one device must mix several combinations across its measurements: " + presencesByDevice, + presencesByDevice.values().stream().mapToInt(Set::size).max().orElse(0) >= 3); } private static void assertSchemaRestored() @@ -324,6 +405,7 @@ private static void assertSchemaRestored() final String path = entry.getKey(); final Series expected = entry.getValue(); final Series actual = restored.get(path); + assertEquals(path + " data type/encoding/compression", expected.format, actual.format); assertEquals(path + " tags", expected.tags, actual.tags); assertEquals(path + " attributes", expected.attributes, actual.attributes); assertEquals(path + " alias", expected.alias, actual.alias); @@ -340,18 +422,32 @@ private static Map readSchema() final List columns = dataSet.getColumnNames(); final int timeseriesIndex = columns.indexOf("Timeseries"); final int aliasIndex = columns.indexOf("Alias"); + final int dataTypeIndex = columns.indexOf("DataType"); + final int encodingIndex = columns.indexOf("Encoding"); + final int compressionIndex = columns.indexOf("Compression"); final int tagsIndex = columns.indexOf("Tags"); final int attributesIndex = columns.indexOf("Attributes"); assertTrue( - "SHOW TIMESERIES must expose the Timeseries, Alias, Tags and Attributes columns: " + "SHOW TIMESERIES must expose the Timeseries, Alias, DataType, Encoding, Compression, " + + "Tags and Attributes columns: " + columns, - timeseriesIndex >= 0 && aliasIndex >= 0 && tagsIndex >= 0 && attributesIndex >= 0); + timeseriesIndex >= 0 + && aliasIndex >= 0 + && dataTypeIndex >= 0 + && encodingIndex >= 0 + && compressionIndex >= 0 + && tagsIndex >= 0 + && attributesIndex >= 0); while (dataSet.hasNext()) { final List fields = dataSet.next().getFields(); final String path = cell(fields, timeseriesIndex); schema.put( path, new Series( + Format.of( + cell(fields, dataTypeIndex), + cell(fields, encodingIndex), + cell(fields, compressionIndex)), parseJsonObject(cell(fields, tagsIndex)), parseJsonObject(cell(fields, attributesIndex)), cell(fields, aliasIndex))); @@ -458,12 +554,23 @@ private static ProcessBuilder processBuilder(List command, boolean windo return builder; } - private static void singleTag(String device, String value) { - tagged(device, "k", value); + private static void singleTag(String device, String value, Format format) { + tagged(device, "k", value, format); } - private static void tagged(String device, String key, String value) { - CREATED.put(DATABASE + "." + device + ".s1", new Series(mapOf(key, value), NO_VALUES, null)); + private static void tagged(String device, String key, String value, Format format) { + series(device, "s1", format, mapOf(key, value), NO_VALUES, null); + } + + private static void series( + String device, + String measurement, + Format format, + Map tags, + Map attributes, + String alias) { + CREATED.put( + DATABASE + "." + device + "." + measurement, new Series(format, tags, attributes, alias)); } private static Map mapOf(String... keysAndValues) { @@ -474,13 +581,72 @@ private static Map mapOf(String... keysAndValues) { return map; } - /** The tags, attributes and alias created for one time series. */ + /** + * A data type, encoding and compression combination of the test schema. Every combination has to + * appear in the exported file, and the server renders them with these names. + */ + private enum Format { + BOOLEAN_PLAIN_SNAPPY(TSDataType.BOOLEAN, TSEncoding.PLAIN, CompressionType.SNAPPY), + INT32_TS_2DIFF_SNAPPY(TSDataType.INT32, TSEncoding.TS_2DIFF, CompressionType.SNAPPY), + INT64_GORILLA_LZ4(TSDataType.INT64, TSEncoding.GORILLA, CompressionType.LZ4), + DOUBLE_GORILLA_LZ4(TSDataType.DOUBLE, TSEncoding.GORILLA, CompressionType.LZ4), + FLOAT_GORILLA_GZIP(TSDataType.FLOAT, TSEncoding.GORILLA, CompressionType.GZIP), + TEXT_DICTIONARY_SNAPPY(TSDataType.TEXT, TSEncoding.DICTIONARY, CompressionType.SNAPPY); + + private final TSDataType dataType; + private final TSEncoding encoding; + private final CompressionType compression; + + Format(TSDataType dataType, TSEncoding encoding, CompressionType compression) { + this.dataType = dataType; + this.encoding = encoding; + this.compression = compression; + } + + /** Resolves the combination rendered by {@code SHOW TIMESERIES}, i.e. by the export. */ + private static Format of(String dataType, String encoding, String compression) { + for (Format format : values()) { + if (format.dataType.name().equals(dataType) + && format.encoding.name().equals(encoding) + && format.compression.name().equals(compression)) { + return format; + } + } + throw new AssertionError( + "unexpected data type/encoding/compression combination: " + + dataType + + "/" + + encoding + + "/" + + compression); + } + } + + /** Whether a time series carries tags and attributes; all four must coexist in one file. */ + private enum Presence { + BOTH, + TAGS_ONLY, + ATTRIBUTES_ONLY, + NEITHER; + + private static Presence of(Map tags, Map attributes) { + if (!tags.isEmpty()) { + return attributes.isEmpty() ? TAGS_ONLY : BOTH; + } + return attributes.isEmpty() ? NEITHER : ATTRIBUTES_ONLY; + } + } + + /** One time series of the test schema, i.e. one row of the exported file. */ private static final class Series { + private final Format format; private final Map tags; private final Map attributes; private final String alias; - private Series(Map tags, Map attributes, String alias) { + private Series( + Format format, Map tags, Map attributes, String alias) { + this.format = format; this.tags = tags; this.attributes = attributes; this.alias = alias;