diff --git a/.github/workflows/build-security.yaml b/.github/workflows/build-security.yaml index 2960c397..21587784 100644 --- a/.github/workflows/build-security.yaml +++ b/.github/workflows/build-security.yaml @@ -6,6 +6,9 @@ on: - '.github/workflows/build-security.yaml' - '.github/workflows/reloader-dockerhub.yaml' - 'pkg/common/**' + - 'cmd/**' + - 'platform-monitoring/operator/init.sh' + - 'monitor-snapshot/master/operator/init.sh' - 'scripts/**' - 'go.mod' - 'go.sum' @@ -25,4 +28,5 @@ jobs: with: go-version-file: go.mod - run: go test ./pkg/common + - run: go test ./cmd/monitoring.go ./cmd/dm_rules_test.go ./cmd/persisted_rules_test.go - run: python3 -m unittest discover -s scripts -p 'test_build_security.py' -v diff --git a/cmd/PERSISTED_RULES.md b/cmd/PERSISTED_RULES.md new file mode 100644 index 00000000..220eb58a --- /dev/null +++ b/cmd/PERSISTED_RULES.md @@ -0,0 +1,36 @@ +# Restoring persisted alert rules + +Current initializer entrypoints (`cmd/init.sh`, +`platform-monitoring/operator/init.sh` and +`monitor-snapshot/master/operator/init.sh`) install the bundled rules on every +start. They clear the dedicated active `rules` directory first for both DM and +TiDB. Rules previously mirrored to `PROM_PERSISTENT_DIR/latest-rules/` +are **not restored by default**, because those files may have been written before +the editing API was authenticated. The initializer does not delete the PVC copy. + +This deliberately changes restart behavior for deployments that relied on +persisted custom rules. Before upgrading, back up the PVC, compare the stored +rules with the expected configuration and validate them with the matching +Prometheus `promtool check rules`. If an administrator trusts the volume and +wants to restore the reviewed customizations, set `PROM_RESTORE_RULES=true` in +the initializer environment. TiDB Operator v1 supports this through: + +```yaml +spec: + initializer: + envs: + PROM_RESTORE_RULES: "true" +``` + +Keep the initializer's existing image configuration alongside this setting. +Unset or `false` uses bundled defaults. Other values fail initialization. +Opt-in restoration accepts only regular `*.rules.yml` files with names already +installed from the bundle for the selected DM/TiDB version; unknown rule names +and symlink rule files/directories fail initialization. Custom additional files +must be packaged into a reviewed bundle instead. Copy errors also fail startup. + +Opting in is an administrator trust decision, not cryptographic provenance. +Restrict writes to the PVC and active rules directory to trusted components; +any local process with write access remains able to change those files. These +changes apply to current initializer sources, not historical version snapshots +or already-built images. Deploy a rebuilt image to obtain the protection. diff --git a/cmd/init.sh b/cmd/init.sh index b15e7f47..65e5739c 100644 --- a/cmd/init.sh +++ b/cmd/init.sh @@ -6,6 +6,50 @@ replace_template() { sed -i "s|$1|$replacement|g" -- "$3" } +# Historical PVC rules may predate API authentication. Restore only after an +# administrator has reviewed the volume and explicitly opted in. +case "${PROM_RESTORE_RULES:-false}" in + true|false) ;; + *) echo "PROM_RESTORE_RULES must be true or false" >&2; exit 1;; +esac + +reset_rules() { + rules_dir="$PROM_CONFIG_PATH/rules" + if [ -L "$rules_dir" ]; then + echo "Rules directory must not be a symbolic link" >&2 + return 1 + fi + mkdir -p -- "$rules_dir" || return 1 + for entry in "$rules_dir"/* "$rules_dir"/.[!.]* "$rules_dir"/..?*; do + [ -e "$entry" ] || [ -L "$entry" ] || continue + rm -rf -- "$entry" || return 1 + done +} + +restore_rules() { + [ "${PROM_RESTORE_RULES:-false}" = true ] || return 0 + [ -n "$PROM_PERSISTENT_DIR" ] || return 0 + case "$1" in + ''|.|..) echo "A valid rule version is required for restore" >&2; return 1;; + esac + persisted_root="$PROM_PERSISTENT_DIR/latest-rules" + persisted_version="$persisted_root/$1" + if [ -L "$persisted_root" ] || [ -L "$persisted_version" ]; then + echo "Persisted rule directories must not be symbolic links" >&2 + return 1 + fi + [ -d "$persisted_version" ] || return 0 + for rule in "$persisted_version"/*.rules.yml; do + [ -e "$rule" ] || [ -L "$rule" ] || continue + name=${rule##*/} + if [ -L "$rule" ] || [ ! -f "$rule" ] || [ ! -f "$PROM_CONFIG_PATH/rules/$name" ]; then + echo "Refusing unrecognized or non-regular persisted rule: $name" >&2 + return 1 + fi + cp -f -- "$rule" "$PROM_CONFIG_PATH/rules/$name" || return 1 + done +} + # Multiline replacements would become new sed commands or malformed configuration. for value in "$DM_CLUSTER_NAME" "$TIDB_CLUSTER_NAME" "$TIDB_CLUSTER_NAMESPACE" \ "$GF_DM_PROMETHEUS_URL" "$GF_K8S_PROMETHEUS_URL" "$GF_TIDB_PROMETHEUS_URL"; do @@ -34,9 +78,7 @@ if [ -n "$DM_CLUSTER_NAME" ]; then replace_template 'Test-Cluster-DM' "${DM_CLUSTER_NAME}-DM-Professional" "$GF_PROVISIONING_PATH/dashboards"/DM-Monitor-Professional.json # Rules - if [ ! -d "$PROM_CONFIG_PATH/rules" ]; then - mkdir -p -- "$PROM_CONFIG_PATH/rules" - fi + reset_rules || exit 1 echo $META_TYPE echo $META_INSTANCE @@ -47,12 +89,7 @@ if [ -n "$DM_CLUSTER_NAME" ]; then fi replace_template 'ENV_LABELS_ENV' "${DM_CLUSTER_NAME}" "$PROM_CONFIG_PATH/rules/dm_worker.rules.yml" || exit 1 - # Copy Persistent rules to override raw files - if [ -n "$PROM_PERSISTENT_DIR" ]; then - if [ -d "$PROM_PERSISTENT_DIR/latest-rules/${DM_VERSION##*/}" ]; then - cp -f -- "$PROM_PERSISTENT_DIR/latest-rules/${DM_VERSION##*/}"/dm_worker.rules.yml "$PROM_CONFIG_PATH/rules" - fi - fi + restore_rules "${DM_VERSION##*/}" || exit 1 # Data sources if [ -n "$GF_DATASOURCE_PATH" ]; then @@ -153,27 +190,18 @@ for f in "$GF_PROVISIONING_PATH/dashboards"/*.json; do done # Rules -if [ ! -d "$PROM_CONFIG_PATH/rules" ]; then - mkdir -p -- "$PROM_CONFIG_PATH/rules" -else - rm -rf -- "$PROM_CONFIG_PATH/rules"/* -fi +reset_rules || exit 1 echo $META_TYPE echo $META_INSTANCE echo $META_VALUE -cp -- /tmp/*.rules.yml "$PROM_CONFIG_PATH/rules" +cp -- /tmp/*.rules.yml "$PROM_CONFIG_PATH/rules" || exit 1 for file in "$PROM_CONFIG_PATH/rules"/*; do - replace_template 'ENV_LABELS_ENV' "${TIDB_CLUSTER_NAME}" "$file" + replace_template 'ENV_LABELS_ENV' "${TIDB_CLUSTER_NAME}" "$file" || exit 1 sed -i 's%job=%component=%g' "$file" sed -i -e 's%\(by\s(\)job\(,.*)\)%\1component\2%g' -e 's%\(by\s(.*\),job,\(.*)\)%\1,component,\2%g' -e 's%\(by\s(.*,\)job)%\1component)%g' -e 's%\(by\s(\)job)%\1component)%g' "$file" done -# Copy Persistent rules to override raw files -if [ -n "$PROM_PERSISTENT_DIR" ]; then - if [ -d "$PROM_PERSISTENT_DIR/latest-rules/${TIDB_VERSION##*/}" ]; then - cp -f -- "$PROM_PERSISTENT_DIR/latest-rules/${TIDB_VERSION##*/}"/*.rules.yml "$PROM_CONFIG_PATH/rules" - fi -fi +restore_rules "${TIDB_VERSION##*/}" || exit 1 # Datasources if [ -n "$GF_DATASOURCE_PATH" ]; then diff --git a/cmd/persisted_rules_test.go b/cmd/persisted_rules_test.go new file mode 100644 index 00000000..cee7ca0e --- /dev/null +++ b/cmd/persisted_rules_test.go @@ -0,0 +1,138 @@ +package main + +import ( + "os" + "os/exec" + "path/filepath" + "regexp" + "runtime" + "strings" + "testing" +) + +func TestInitializerPersistedRulesRequireExplicitTrust(t *testing.T) { + if runtime.GOOS == "windows" { + t.Skip("initializer requires POSIX shell") + } + for _, source := range []string{"init.sh", "../platform-monitoring/operator/init.sh", "../monitor-snapshot/master/operator/init.sh"} { + for _, dm := range []bool{false, true} { + for _, scenario := range []string{"default", "trusted", "unknown", "symlink", "directory-link", "invalid"} { + t.Run(source+"/"+map[bool]string{false: "tidb", true: "dm"}[dm]+"/"+scenario, func(t *testing.T) { + root := t.TempDir() + assets, output, persistent := filepath.Join(root, "assets"), filepath.Join(root, "prometheus"), filepath.Join(root, "persistent") + for _, dir := range []string{assets, filepath.Join(output, "rules"), filepath.Join(persistent, "latest-rules", "v1")} { + if err := os.MkdirAll(dir, 0755); err != nil { + t.Fatal(err) + } + } + extraPath := "" + if runtime.GOOS == "darwin" { + gsed, err := exec.LookPath("gsed") + if err != nil { + t.Skip("GNU sed required on macOS") + } + bin := filepath.Join(root, "bin") + if err := os.Mkdir(bin, 0755); err != nil { + t.Fatal(err) + } + if err := os.Symlink(gsed, filepath.Join(bin, "sed")); err != nil { + t.Fatal(err) + } + extraPath = bin + string(os.PathListSeparator) + } + script, err := os.ReadFile(source) + if err != nil { + t.Fatal(err) + } + for _, match := range regexp.MustCompile(`/tmp/([A-Za-z0-9_.-]+)`).FindAllStringSubmatch(string(script), -1) { + name := match[1] + if filepath.Ext(name) != ".json" && filepath.Ext(name) != ".yml" && filepath.Ext(name) != ".yaml" { + continue + } + if err := os.WriteFile(filepath.Join(assets, name), []byte("{}"), 0644); err != nil { + t.Fatal(err) + } + } + name := "test.rules.yml" + if dm { + name = "dm_worker.rules.yml" + } + bundled, restored := "groups: []\n# bundled\n", "groups: []\n# reviewed custom rules\n" + if err := os.WriteFile(filepath.Join(assets, name), []byte(bundled), 0644); err != nil { + t.Fatal(err) + } + old := filepath.Join(output, "rules", "foreign.rules.yml") + if err := os.WriteFile(old, []byte("groups: []"), 0644); err != nil { + t.Fatal(err) + } + saved := filepath.Join(persistent, "latest-rules", "v1", name) + if err := os.WriteFile(saved, []byte(restored), 0644); err != nil { + t.Fatal(err) + } + trust := "true" + if scenario == "default" { + trust = "" + } + switch scenario { + case "unknown": + if err := os.WriteFile(filepath.Join(filepath.Dir(saved), "foreign.rules.yml"), []byte(restored), 0644); err != nil { + t.Fatal(err) + } + case "symlink": + if err := os.Remove(saved); err != nil { + t.Fatal(err) + } + if err := os.Symlink(filepath.Join(assets, name), saved); err != nil { + t.Fatal(err) + } + case "directory-link": + if err := os.Rename(filepath.Dir(saved), filepath.Join(root, "untrusted")); err != nil { + t.Fatal(err) + } + if err := os.Symlink(filepath.Join(root, "untrusted"), filepath.Dir(saved)); err != nil { + t.Fatal(err) + } + case "invalid": + trust = "yes" + } + scriptPath := filepath.Join(root, "init.sh") + if err := os.WriteFile(scriptPath, []byte(strings.ReplaceAll(string(script), "/tmp/", assets+"/")), 0644); err != nil { + t.Fatal(err) + } + cluster := "" + if dm { + cluster = "test" + } + cmd := exec.Command("sh", scriptPath) + cmd.Env = append(os.Environ(), "DM_CLUSTER_NAME="+cluster, "TIDB_CLUSTER_NAME=test", "DM_VERSION=v1", "TIDB_VERSION=v1", "GF_PROVISIONING_PATH="+filepath.Join(root, "grafana"), "PROM_CONFIG_PATH="+output, "PROM_PERSISTENT_DIR="+persistent, "PROM_RESTORE_RULES="+trust, "GF_DATASOURCE_PATH=", "PATH="+extraPath+os.Getenv("PATH")) + out, err := cmd.CombinedOutput() + success := scenario == "default" || scenario == "trusted" + if success && err != nil { + t.Fatalf("initializer failed: %v %s", err, out) + } + if !success && err == nil { + t.Fatalf("unsafe restore succeeded: %s", out) + } + if !success { + return + } + want := bundled + if scenario == "trusted" { + want = restored + } + got, err := os.ReadFile(filepath.Join(output, "rules", name)) + if err != nil || string(got) != want { + t.Fatalf("rules = %q, want %q: %v", got, want, err) + } + if _, err := os.Lstat(old); !os.IsNotExist(err) { + t.Fatalf("foreign active rule survived: %v", err) + } + got, err = os.ReadFile(saved) + if err != nil || string(got) != restored { + t.Fatalf("PVC data was modified: %q %v", got, err) + } + }) + } + } + } +} diff --git a/monitor-snapshot/master/operator/init.sh b/monitor-snapshot/master/operator/init.sh index a9774e9f..a9925263 100755 --- a/monitor-snapshot/master/operator/init.sh +++ b/monitor-snapshot/master/operator/init.sh @@ -6,6 +6,50 @@ replace_template() { sed -i "s|$1|$replacement|g" -- "$3" } +# Historical PVC rules may predate API authentication. Restore only after an +# administrator has reviewed the volume and explicitly opted in. +case "${PROM_RESTORE_RULES:-false}" in + true|false) ;; + *) echo "PROM_RESTORE_RULES must be true or false" >&2; exit 1;; +esac + +reset_rules() { + rules_dir="$PROM_CONFIG_PATH/rules" + if [ -L "$rules_dir" ]; then + echo "Rules directory must not be a symbolic link" >&2 + return 1 + fi + mkdir -p -- "$rules_dir" || return 1 + for entry in "$rules_dir"/* "$rules_dir"/.[!.]* "$rules_dir"/..?*; do + [ -e "$entry" ] || [ -L "$entry" ] || continue + rm -rf -- "$entry" || return 1 + done +} + +restore_rules() { + [ "${PROM_RESTORE_RULES:-false}" = true ] || return 0 + [ -n "$PROM_PERSISTENT_DIR" ] || return 0 + case "$1" in + ''|.|..) echo "A valid rule version is required for restore" >&2; return 1;; + esac + persisted_root="$PROM_PERSISTENT_DIR/latest-rules" + persisted_version="$persisted_root/$1" + if [ -L "$persisted_root" ] || [ -L "$persisted_version" ]; then + echo "Persisted rule directories must not be symbolic links" >&2 + return 1 + fi + [ -d "$persisted_version" ] || return 0 + for rule in "$persisted_version"/*.rules.yml; do + [ -e "$rule" ] || [ -L "$rule" ] || continue + name=${rule##*/} + if [ -L "$rule" ] || [ ! -f "$rule" ] || [ ! -f "$PROM_CONFIG_PATH/rules/$name" ]; then + echo "Refusing unrecognized or non-regular persisted rule: $name" >&2 + return 1 + fi + cp -f -- "$rule" "$PROM_CONFIG_PATH/rules/$name" || return 1 + done +} + # Multiline replacements would become new sed commands or malformed configuration. for value in "$DM_CLUSTER_NAME" "$TIDB_CLUSTER_NAME" "$TIDB_CLUSTER_NAMESPACE" \ "$GF_DM_PROMETHEUS_URL" "$GF_K8S_PROMETHEUS_URL" "$GF_TIDB_PROMETHEUS_URL"; do @@ -34,9 +78,7 @@ if [ -n "$DM_CLUSTER_NAME" ]; then replace_template 'Test-Cluster-DM' "${DM_CLUSTER_NAME}-DM-Professional" "$GF_PROVISIONING_PATH/dashboards"/DM-Monitor-Professional.json # Rules - if [ ! -d "$PROM_CONFIG_PATH/rules" ]; then - mkdir -p -- "$PROM_CONFIG_PATH/rules" - fi + reset_rules || exit 1 echo $META_TYPE echo $META_INSTANCE @@ -47,12 +89,7 @@ if [ -n "$DM_CLUSTER_NAME" ]; then fi replace_template 'ENV_LABELS_ENV' "${DM_CLUSTER_NAME}" "$PROM_CONFIG_PATH/rules/dm_worker.rules.yml" || exit 1 - # Copy Persistent rules to override raw files - if [ -n "$PROM_PERSISTENT_DIR" ]; then - if [ -d "$PROM_PERSISTENT_DIR/latest-rules/${DM_VERSION##*/}" ]; then - cp -f -- "$PROM_PERSISTENT_DIR/latest-rules/${DM_VERSION##*/}"/dm_worker.rules.yml "$PROM_CONFIG_PATH/rules" - fi - fi + restore_rules "${DM_VERSION##*/}" || exit 1 # Data sources if [ -n "$GF_DATASOURCE_PATH" ]; then @@ -161,27 +198,18 @@ for f in "$GF_PROVISIONING_PATH/dashboards"/*.json; do done # Rules -if [ ! -d "$PROM_CONFIG_PATH/rules" ]; then - mkdir -p -- "$PROM_CONFIG_PATH/rules" -else - rm -rf -- "$PROM_CONFIG_PATH/rules"/* -fi +reset_rules || exit 1 echo $META_TYPE echo $META_INSTANCE echo $META_VALUE -cp -- /tmp/*.rules.yml "$PROM_CONFIG_PATH/rules" +cp -- /tmp/*.rules.yml "$PROM_CONFIG_PATH/rules" || exit 1 for file in "$PROM_CONFIG_PATH/rules"/*; do sed -i 's/ENV_LABELS_ENV/Cluster/g' "$file" sed -i 's%job=%component=%g' "$file" sed -i -e 's%\(by\s(\)job\(,.*)\)%\1component\2%g' -e 's%\(by\s(.*\),job,\(.*)\)%\1,component,\2%g' -e 's%\(by\s(.*,\)job)%\1component)%g' -e 's%\(by\s(\)job)%\1component)%g' "$file" done -# Copy Persistent rules to override raw files -if [ -n "$PROM_PERSISTENT_DIR" ]; then - if [ -d "$PROM_PERSISTENT_DIR/latest-rules/${TIDB_VERSION##*/}" ]; then - cp -f -- "$PROM_PERSISTENT_DIR/latest-rules/${TIDB_VERSION##*/}"/*.rules.yml "$PROM_CONFIG_PATH/rules" - fi -fi +restore_rules "${TIDB_VERSION##*/}" || exit 1 # Datasources if [ -n "$GF_DATASOURCE_PATH" ]; then diff --git a/platform-monitoring/operator/init.sh b/platform-monitoring/operator/init.sh index a9774e9f..a9925263 100755 --- a/platform-monitoring/operator/init.sh +++ b/platform-monitoring/operator/init.sh @@ -6,6 +6,50 @@ replace_template() { sed -i "s|$1|$replacement|g" -- "$3" } +# Historical PVC rules may predate API authentication. Restore only after an +# administrator has reviewed the volume and explicitly opted in. +case "${PROM_RESTORE_RULES:-false}" in + true|false) ;; + *) echo "PROM_RESTORE_RULES must be true or false" >&2; exit 1;; +esac + +reset_rules() { + rules_dir="$PROM_CONFIG_PATH/rules" + if [ -L "$rules_dir" ]; then + echo "Rules directory must not be a symbolic link" >&2 + return 1 + fi + mkdir -p -- "$rules_dir" || return 1 + for entry in "$rules_dir"/* "$rules_dir"/.[!.]* "$rules_dir"/..?*; do + [ -e "$entry" ] || [ -L "$entry" ] || continue + rm -rf -- "$entry" || return 1 + done +} + +restore_rules() { + [ "${PROM_RESTORE_RULES:-false}" = true ] || return 0 + [ -n "$PROM_PERSISTENT_DIR" ] || return 0 + case "$1" in + ''|.|..) echo "A valid rule version is required for restore" >&2; return 1;; + esac + persisted_root="$PROM_PERSISTENT_DIR/latest-rules" + persisted_version="$persisted_root/$1" + if [ -L "$persisted_root" ] || [ -L "$persisted_version" ]; then + echo "Persisted rule directories must not be symbolic links" >&2 + return 1 + fi + [ -d "$persisted_version" ] || return 0 + for rule in "$persisted_version"/*.rules.yml; do + [ -e "$rule" ] || [ -L "$rule" ] || continue + name=${rule##*/} + if [ -L "$rule" ] || [ ! -f "$rule" ] || [ ! -f "$PROM_CONFIG_PATH/rules/$name" ]; then + echo "Refusing unrecognized or non-regular persisted rule: $name" >&2 + return 1 + fi + cp -f -- "$rule" "$PROM_CONFIG_PATH/rules/$name" || return 1 + done +} + # Multiline replacements would become new sed commands or malformed configuration. for value in "$DM_CLUSTER_NAME" "$TIDB_CLUSTER_NAME" "$TIDB_CLUSTER_NAMESPACE" \ "$GF_DM_PROMETHEUS_URL" "$GF_K8S_PROMETHEUS_URL" "$GF_TIDB_PROMETHEUS_URL"; do @@ -34,9 +78,7 @@ if [ -n "$DM_CLUSTER_NAME" ]; then replace_template 'Test-Cluster-DM' "${DM_CLUSTER_NAME}-DM-Professional" "$GF_PROVISIONING_PATH/dashboards"/DM-Monitor-Professional.json # Rules - if [ ! -d "$PROM_CONFIG_PATH/rules" ]; then - mkdir -p -- "$PROM_CONFIG_PATH/rules" - fi + reset_rules || exit 1 echo $META_TYPE echo $META_INSTANCE @@ -47,12 +89,7 @@ if [ -n "$DM_CLUSTER_NAME" ]; then fi replace_template 'ENV_LABELS_ENV' "${DM_CLUSTER_NAME}" "$PROM_CONFIG_PATH/rules/dm_worker.rules.yml" || exit 1 - # Copy Persistent rules to override raw files - if [ -n "$PROM_PERSISTENT_DIR" ]; then - if [ -d "$PROM_PERSISTENT_DIR/latest-rules/${DM_VERSION##*/}" ]; then - cp -f -- "$PROM_PERSISTENT_DIR/latest-rules/${DM_VERSION##*/}"/dm_worker.rules.yml "$PROM_CONFIG_PATH/rules" - fi - fi + restore_rules "${DM_VERSION##*/}" || exit 1 # Data sources if [ -n "$GF_DATASOURCE_PATH" ]; then @@ -161,27 +198,18 @@ for f in "$GF_PROVISIONING_PATH/dashboards"/*.json; do done # Rules -if [ ! -d "$PROM_CONFIG_PATH/rules" ]; then - mkdir -p -- "$PROM_CONFIG_PATH/rules" -else - rm -rf -- "$PROM_CONFIG_PATH/rules"/* -fi +reset_rules || exit 1 echo $META_TYPE echo $META_INSTANCE echo $META_VALUE -cp -- /tmp/*.rules.yml "$PROM_CONFIG_PATH/rules" +cp -- /tmp/*.rules.yml "$PROM_CONFIG_PATH/rules" || exit 1 for file in "$PROM_CONFIG_PATH/rules"/*; do sed -i 's/ENV_LABELS_ENV/Cluster/g' "$file" sed -i 's%job=%component=%g' "$file" sed -i -e 's%\(by\s(\)job\(,.*)\)%\1component\2%g' -e 's%\(by\s(.*\),job,\(.*)\)%\1,component,\2%g' -e 's%\(by\s(.*,\)job)%\1component)%g' -e 's%\(by\s(\)job)%\1component)%g' "$file" done -# Copy Persistent rules to override raw files -if [ -n "$PROM_PERSISTENT_DIR" ]; then - if [ -d "$PROM_PERSISTENT_DIR/latest-rules/${TIDB_VERSION##*/}" ]; then - cp -f -- "$PROM_PERSISTENT_DIR/latest-rules/${TIDB_VERSION##*/}"/*.rules.yml "$PROM_CONFIG_PATH/rules" - fi -fi +restore_rules "${TIDB_VERSION##*/}" || exit 1 # Datasources if [ -n "$GF_DATASOURCE_PATH" ]; then