Support tags and attributes in tree-model schema export/import

iotdb-client/cli/pom.xml

@@ -144,6 +144,10 @@
             <groupId>org.apache.commons</groupId>
             <artifactId>commons-csv</artifactId>
         </dependency>
+        <dependency>
+            <groupId>com.google.code.gson</groupId>
+            <artifactId>gson</artifactId>
+        </dependency>
         <dependency>
             <groupId>junit</groupId>
             <artifactId>junit</artifactId>

iotdb-client/cli/src/main/i18n/en/org/apache/iotdb/cli/i18n/CliMessages.java

@@ -157,6 +157,13 @@ public final class CliMessages {
       MESSAGE_THE_HEADER_OF_THE_CSV_FILE_TO_BE_IMPORTED_IS_ILLEGAL_THE_CORRECT_FORMAT_IS_TIMESERIES_ALIBABA_DATATYPE_ENCODING_COMPRESSION_C429E7C6 =
           "The header of the CSV file to be imported is illegal. The correct format is"
               + " \"Timeseries, Alibaba, DataType, Encoding, Compression\"!";
+  public static final String
+      MESSAGE_THE_HEADER_OF_THE_CSV_FILE_TO_BE_IMPORTED_IS_ILLEGAL_THE_CORRECT_FORMAT_IS_TIMESERIES_ALIAS_DATATYPE_ENCODING_COMPRESSION_TAGS_ATTRIBUTES_14EDBFB9 =
+          "The header of the CSV file to be imported is illegal. The correct format is"
+              + " \"Timeseries, Alias, DataType, Encoding, Compression, Tags, Attributes\"!";
+  public static final String
+      MESSAGE_FAILED_TO_PARSE_THE_ARG_COLUMN_OF_TIMESERIES_ARG_AS_JSON_THE_RECORD_IS_SKIPPED_ARG_35D898C8 =
+          "Failed to parse the %s column of timeseries %s as JSON, the record is skipped: %s";
 
   private CliMessages() {}
 

iotdb-client/cli/src/main/i18n/zh/org/apache/iotdb/cli/i18n/CliMessages.java

@@ -145,6 +145,13 @@ public final class CliMessages {
   public static final String
       MESSAGE_THE_HEADER_OF_THE_CSV_FILE_TO_BE_IMPORTED_IS_ILLEGAL_THE_CORRECT_FORMAT_IS_TIMESERIES_ALIBABA_DATATYPE_ENCODING_COMPRESSION_C429E7C6 =
           "待导入的 CSV 文件表头不合法，正确格式为 \"Timeseries, Alibaba, DataType, Encoding, Compression\"！";
+  public static final String
+      MESSAGE_THE_HEADER_OF_THE_CSV_FILE_TO_BE_IMPORTED_IS_ILLEGAL_THE_CORRECT_FORMAT_IS_TIMESERIES_ALIAS_DATATYPE_ENCODING_COMPRESSION_TAGS_ATTRIBUTES_14EDBFB9 =
+          "待导入的 CSV 文件表头不合法，正确格式为"
+              + " \"Timeseries, Alias, DataType, Encoding, Compression, Tags, Attributes\"！";
+  public static final String
+      MESSAGE_FAILED_TO_PARSE_THE_ARG_COLUMN_OF_TIMESERIES_ARG_AS_JSON_THE_RECORD_IS_SKIPPED_ARG_35D898C8 =
+          "时序 JSON 解析失败，列：%s，时序：%s，已跳过该记录，原因：%s";
 
   private CliMessages() {}
 

iotdb-client/cli/src/main/java/org/apache/iotdb/tool/common/Constants.java

@@ -204,8 +204,31 @@ public class Constants {
   public static final String COLON = ": ";
   public static final String MINUS = "-";
 
+  public static final String HEADER_TAGS = "Tags";
+  public static final String HEADER_ATTRIBUTES = "Attributes";
+  public static final String HEADER_ALIAS = "Alias";
+  public static final String HEADER_DATATYPE = "DataType";
+  public static final String HEADER_ENCODING = "Encoding";
+  public static final String HEADER_COMPRESSION = "Compression";
+
   public static final List<String> HEAD_COLUMNS =
-      Arrays.asList("Timeseries", "Alias", "DataType", "Encoding", "Compression");
+      Arrays.asList(
+          "Timeseries", HEADER_ALIAS, HEADER_DATATYPE, HEADER_ENCODING, HEADER_COMPRESSION);
+
+  /**
+   * Column layout written by the tree model schema export. The Tags and Attributes cells hold a
+   * JSON object, so files carrying them are parsed with the RFC4180 settings instead of the legacy
+   * ones.
+   */
+  public static final List<String> HEAD_COLUMNS_V2 =
+      Arrays.asList(
+          "Timeseries",
+          HEADER_ALIAS,
+          HEADER_DATATYPE,
+          HEADER_ENCODING,
+          HEADER_COMPRESSION,
+          HEADER_TAGS,
+          HEADER_ATTRIBUTES);
 
   // export constants
   public static final String EXPORT_CLI_PREFIX = CliMessages.EXCEPTION_EXPORT_DATA_2F1FB3D9;

iotdb-client/cli/src/main/java/org/apache/iotdb/tool/schema/AbstractSchemaTool.java

@@ -30,6 +30,9 @@
 import org.apache.iotdb.session.pool.TableSessionPoolBuilder;
 import org.apache.iotdb.tool.common.Constants;
 
+import com.google.gson.Gson;
+import com.google.gson.GsonBuilder;
+import com.google.gson.reflect.TypeToken;
 import org.apache.commons.cli.CommandLine;
 import org.apache.commons.csv.CSVFormat;
 import org.apache.commons.csv.CSVPrinter;
@@ -41,8 +44,11 @@
 
 import java.io.IOException;
 import java.io.PrintWriter;
+import java.lang.reflect.Type;
 import java.util.Iterator;
 import java.util.List;
+import java.util.Map;
+import java.util.TreeMap;
 import java.util.concurrent.atomic.LongAdder;
 
 public abstract class AbstractSchemaTool {
@@ -77,6 +83,9 @@ public abstract class AbstractSchemaTool {
   private static final IoTPrinter ioTPrinter = new IoTPrinter(System.out);
   private static final Logger LOGGER = LoggerFactory.getLogger(AbstractSchemaTool.class);
 
+  private static final Gson JSON_GSON = new GsonBuilder().disableHtmlEscaping().create();
+  private static final Type JSON_CELL_TYPE = new TypeToken<Map<String, String>>() {}.getType();
+
   protected AbstractSchemaTool() {}
 
   protected static Session.Builder configureSsl(Session.Builder builder) {
@@ -218,20 +227,59 @@ public static Boolean writeCsvFile(List<List<Object>> records, String filePath)
     }
   }
 
+  /**
+   * The RFC4180 CSV layout of the schema files, i.e. the delimiter is {@code ','}, the quote char
+   * is {@code '"'} and no escape char is configured. A JSON object cell is therefore always quoted
+   * and its inner quotes are doubled, which is exactly what RFC4180 asks for.
+   */
+  static CSVFormat.Builder createRfc4180CsvFormat() {
+    return CSVFormat.Builder.create(CSVFormat.DEFAULT)
+        .setQuote('"')
+        .setQuoteMode(QuoteMode.MINIMAL)
+        .setRecordSeparator('\n');
+  }
+
+  /**
+   * Serializes tags or attributes into the compact JSON object cell of the schema file. Keys are
+   * sorted so that repeated exports of the same schema are identical, and HTML escaping is disabled
+   * so that printable characters such as {@code '}, {@code =}, {@code <} and {@code &} stay
+   * readable.
+   *
+   * @return an empty string when there is nothing to write, never {@code {}} or {@code null}
+   */
+  static String toJsonCell(Map<String, String> values) {
+    if (values == null || values.isEmpty()) {
+      return "";
+    }
+    return JSON_GSON.toJson(new TreeMap<>(values));
+  }
+
+  /**
+   * Parses a Tags / Attributes cell written by {@link #toJsonCell(Map)}.
+   *
+   * @return {@code null} when the cell carries no value, never an empty map
+   * @throws com.google.gson.JsonParseException when the cell is not a JSON object
+   */
+  static Map<String, String> parseJsonCell(String cell) {
+    if (cell == null) {
+      return null;
+    }
+    String trimmed = cell.trim();
+    if (trimmed.isEmpty() || "null".equals(trimmed)) {
+      return null;
+    }
+    Map<String, String> parsed = JSON_GSON.fromJson(trimmed, JSON_CELL_TYPE);
+    return parsed == null || parsed.isEmpty() ? null : new TreeMap<>(parsed);
+  }
+
   static class CSVPrinterWrapper {
     private final String filePath;
     private final CSVFormat csvFormat;
     private CSVPrinter csvPrinter;
 
     public CSVPrinterWrapper(String filePath) {
       this.filePath = filePath;
-      this.csvFormat =
-          CSVFormat.Builder.create(CSVFormat.DEFAULT)
-              .setHeader()
-              .setSkipHeaderRecord(true)
-              .setEscape('\\')
-              .setQuoteMode(QuoteMode.NONE)
-              .build();
+      this.csvFormat = createRfc4180CsvFormat().setHeader().setSkipHeaderRecord(true).build();
     }
 
     public void printRecord(final Iterable<?> values) throws IOException {

iotdb-client/cli/src/main/java/org/apache/iotdb/tool/schema/ExportSchemaTree.java

@@ -32,7 +32,9 @@
 
 import java.io.File;
 import java.io.IOException;
+import java.util.ArrayList;
 import java.util.List;
+import java.util.Map;
 
 import static org.apache.iotdb.commons.schema.SchemaConstant.AUDIT_DATABASE;
 import static org.apache.iotdb.commons.schema.SchemaConstant.SYSTEM_DATABASE;
@@ -98,25 +100,19 @@ private static void writeCsvFile(
       while (i++ < linesPerFile) {
         if (sessionDataSet.hasNext()) {
           if (i == 1) {
-            csvPrinterWrapper.printRecord(Constants.HEAD_COLUMNS);
+            csvPrinterWrapper.printRecord(Constants.HEAD_COLUMNS_V2);
           }
           RowRecord rowRecord = sessionDataSet.next();
           List<Field> fields = rowRecord.getFields();
-          if (fields.get(timeseriesIndex).getStringValue().startsWith(SYSTEM_DATABASE + ".")
-              || fields.get(timeseriesIndex).getStringValue().startsWith(AUDIT_DATABASE + ".")
-              || !fields.get(viewTypeIndex).getStringValue().equals(Constants.BASE_VIEW_TYPE)) {
+          String timeseries = fields.get(timeseriesIndex).getStringValue();
+          if (timeseries.startsWith(SYSTEM_DATABASE + ".")
+              || timeseries.startsWith(AUDIT_DATABASE + ".")
+              || !Constants.BASE_VIEW_TYPE.equals(fields.get(viewTypeIndex).getStringValue())) {
+            continue;
+          }
+          if (!printRecord(csvPrinterWrapper, headers, fields, timeseries)) {
             continue;
           }
-          Constants.HEAD_COLUMNS.forEach(
-              column -> {
-                Field field = fields.get(headers.indexOf(column));
-                String fieldStringValue = field.getStringValue();
-                if (!"null".equals(field.getStringValue())) {
-                  csvPrinterWrapper.print(fieldStringValue);
-                } else {
-                  csvPrinterWrapper.print("");
-                }
-              });
           csvPrinterWrapper.println();
         } else {
           hasNext = false;
@@ -128,4 +124,44 @@ private static void writeCsvFile(
       csvPrinterWrapper.close();
     }
   }
+
+  /**
+   * Writes the cells of one {@code SHOW TIMESERIES} record.
+   *
+   * @return {@code false} when the record must be skipped because a Tags or Attributes cell could
+   *     not be parsed, in which case nothing has been written for it
+   */
+  static boolean printRecord(
+      CSVPrinterWrapper csvPrinterWrapper,
+      List<String> headers,
+      List<Field> fields,
+      String timeseries) {
+    List<String> cells = new ArrayList<>(Constants.HEAD_COLUMNS_V2.size());
+    for (String column : Constants.HEAD_COLUMNS_V2) {
+      int index = headers.indexOf(column);
+      Field field = index < 0 ? null : fields.get(index);
+      String fieldStringValue = field == null ? null : field.getStringValue();
+      if (Constants.HEADER_TAGS.equals(column) || Constants.HEADER_ATTRIBUTES.equals(column)) {
+        Map<String, String> tags;
+        try {
+          tags = parseJsonCell(fieldStringValue);
+        } catch (RuntimeException e) {
+          ioTPrinter.println(
+              String.format(
+                  CliMessages
+                      .MESSAGE_FAILED_TO_PARSE_THE_ARG_COLUMN_OF_TIMESERIES_ARG_AS_JSON_THE_RECORD_IS_SKIPPED_ARG_35D898C8,
+                  column,
+                  timeseries,
+                  String.valueOf(e.getMessage())));
+          return false;
+        }
+        cells.add(toJsonCell(tags));
+      } else {
+        cells.add(
+            "null".equals(fieldStringValue) || fieldStringValue == null ? "" : fieldStringValue);
+      }
+    }
+    cells.forEach(csvPrinterWrapper::print);
+    return true;
+  }
 }