initializer: require explicit trust before restoring persisted rules

.github/workflows/build-security.yaml

@@ -6,6 +6,9 @@ on:
       - '.github/workflows/build-security.yaml'
       - '.github/workflows/reloader-dockerhub.yaml'
       - 'pkg/common/**'
+      - 'cmd/**'
+      - 'platform-monitoring/operator/init.sh'
+      - 'monitor-snapshot/master/operator/init.sh'
       - 'scripts/**'
       - 'go.mod'
       - 'go.sum'
@@ -25,4 +28,5 @@ jobs:
         with:
           go-version-file: go.mod
       - run: go test ./pkg/common
+      - run: go test ./cmd/monitoring.go ./cmd/dm_rules_test.go ./cmd/persisted_rules_test.go
       - run: python3 -m unittest discover -s scripts -p 'test_build_security.py' -v

cmd/PERSISTED_RULES.md

@@ -0,0 +1,36 @@
+# Restoring persisted alert rules
+
+Current initializer entrypoints (`cmd/init.sh`,
+`platform-monitoring/operator/init.sh` and
+`monitor-snapshot/master/operator/init.sh`) install the bundled rules on every
+start. They clear the dedicated active `rules` directory first for both DM and
+TiDB. Rules previously mirrored to `PROM_PERSISTENT_DIR/latest-rules/<version>`
+are **not restored by default**, because those files may have been written before
+the editing API was authenticated. The initializer does not delete the PVC copy.
+
+This deliberately changes restart behavior for deployments that relied on
+persisted custom rules. Before upgrading, back up the PVC, compare the stored
+rules with the expected configuration and validate them with the matching
+Prometheus `promtool check rules`. If an administrator trusts the volume and
+wants to restore the reviewed customizations, set `PROM_RESTORE_RULES=true` in
+the initializer environment. TiDB Operator v1 supports this through:
+
+```yaml
+spec:
+  initializer:
+    envs:
+      PROM_RESTORE_RULES: "true"
+```
+
+Keep the initializer's existing image configuration alongside this setting.
+Unset or `false` uses bundled defaults. Other values fail initialization.
+Opt-in restoration accepts only regular `*.rules.yml` files with names already
+installed from the bundle for the selected DM/TiDB version; unknown rule names
+and symlink rule files/directories fail initialization. Custom additional files
+must be packaged into a reviewed bundle instead. Copy errors also fail startup.
+
+Opting in is an administrator trust decision, not cryptographic provenance.
+Restrict writes to the PVC and active rules directory to trusted components;
+any local process with write access remains able to change those files. These
+changes apply to current initializer sources, not historical version snapshots
+or already-built images. Deploy a rebuilt image to obtain the protection.

cmd/init.sh

@@ -6,6 +6,50 @@ replace_template() {
     sed -i "s|$1|$replacement|g" -- "$3"
 }
 
+# Historical PVC rules may predate API authentication. Restore only after an
+# administrator has reviewed the volume and explicitly opted in.
+case "${PROM_RESTORE_RULES:-false}" in
+    true|false) ;;
+    *) echo "PROM_RESTORE_RULES must be true or false" >&2; exit 1;;
+esac
+
+reset_rules() {
+    rules_dir="$PROM_CONFIG_PATH/rules"
+    if [ -L "$rules_dir" ]; then
+        echo "Rules directory must not be a symbolic link" >&2
+        return 1
+    fi
+    mkdir -p -- "$rules_dir" || return 1
+    for entry in "$rules_dir"/* "$rules_dir"/.[!.]* "$rules_dir"/..?*; do
+        [ -e "$entry" ] || [ -L "$entry" ] || continue
+        rm -rf -- "$entry" || return 1
+    done
+}
+
+restore_rules() {
+    [ "${PROM_RESTORE_RULES:-false}" = true ] || return 0
+    [ -n "$PROM_PERSISTENT_DIR" ] || return 0
+    case "$1" in
+        ''|.|..) echo "A valid rule version is required for restore" >&2; return 1;;
+    esac
+    persisted_root="$PROM_PERSISTENT_DIR/latest-rules"
+    persisted_version="$persisted_root/$1"
+    if [ -L "$persisted_root" ] || [ -L "$persisted_version" ]; then
+        echo "Persisted rule directories must not be symbolic links" >&2
+        return 1
+    fi
+    [ -d "$persisted_version" ] || return 0
+    for rule in "$persisted_version"/*.rules.yml; do
+        [ -e "$rule" ] || [ -L "$rule" ] || continue
+        name=${rule##*/}
+        if [ -L "$rule" ] || [ ! -f "$rule" ] || [ ! -f "$PROM_CONFIG_PATH/rules/$name" ]; then
+            echo "Refusing unrecognized or non-regular persisted rule: $name" >&2
+            return 1
+        fi
+        cp -f -- "$rule" "$PROM_CONFIG_PATH/rules/$name" || return 1
+    done
+}
+
 # Multiline replacements would become new sed commands or malformed configuration.
 for value in "$DM_CLUSTER_NAME" "$TIDB_CLUSTER_NAME" "$TIDB_CLUSTER_NAMESPACE" \
     "$GF_DM_PROMETHEUS_URL" "$GF_K8S_PROMETHEUS_URL" "$GF_TIDB_PROMETHEUS_URL"; do
@@ -34,9 +78,7 @@ if [ -n "$DM_CLUSTER_NAME" ]; then
     replace_template 'Test-Cluster-DM' "${DM_CLUSTER_NAME}-DM-Professional" "$GF_PROVISIONING_PATH/dashboards"/DM-Monitor-Professional.json
 
     # Rules
-    if [ ! -d "$PROM_CONFIG_PATH/rules" ]; then
-        mkdir -p -- "$PROM_CONFIG_PATH/rules"
-    fi
+    reset_rules || exit 1
 
     echo $META_TYPE
     echo $META_INSTANCE
@@ -47,12 +89,7 @@ if [ -n "$DM_CLUSTER_NAME" ]; then
     fi
     replace_template 'ENV_LABELS_ENV' "${DM_CLUSTER_NAME}" "$PROM_CONFIG_PATH/rules/dm_worker.rules.yml" || exit 1
 
-    # Copy Persistent rules to override raw files
-    if [ -n "$PROM_PERSISTENT_DIR" ]; then
-        if [ -d "$PROM_PERSISTENT_DIR/latest-rules/${DM_VERSION##*/}" ]; then
-            cp -f -- "$PROM_PERSISTENT_DIR/latest-rules/${DM_VERSION##*/}"/dm_worker.rules.yml "$PROM_CONFIG_PATH/rules"
-        fi
-    fi
+    restore_rules "${DM_VERSION##*/}" || exit 1
 
     # Data sources
     if [ -n "$GF_DATASOURCE_PATH" ]; then
@@ -153,27 +190,18 @@ for f in "$GF_PROVISIONING_PATH/dashboards"/*.json; do
 done
 
 # Rules
-if [ ! -d "$PROM_CONFIG_PATH/rules" ]; then
-    mkdir -p -- "$PROM_CONFIG_PATH/rules"
-else
-    rm -rf -- "$PROM_CONFIG_PATH/rules"/*
-fi
+reset_rules || exit 1
 echo $META_TYPE
 echo $META_INSTANCE
 echo $META_VALUE
-cp -- /tmp/*.rules.yml "$PROM_CONFIG_PATH/rules"
+cp -- /tmp/*.rules.yml "$PROM_CONFIG_PATH/rules" || exit 1
 for file in "$PROM_CONFIG_PATH/rules"/*; do
-    replace_template 'ENV_LABELS_ENV' "${TIDB_CLUSTER_NAME}" "$file"
+    replace_template 'ENV_LABELS_ENV' "${TIDB_CLUSTER_NAME}" "$file" || exit 1
     sed -i 's%job=%component=%g' "$file"
     sed -i -e 's%\(by\s(\)job\(,.*)\)%\1component\2%g' -e 's%\(by\s(.*\),job,\(.*)\)%\1,component,\2%g' -e 's%\(by\s(.*,\)job)%\1component)%g' -e 's%\(by\s(\)job)%\1component)%g' "$file"
 done
 
-# Copy Persistent rules to override raw files
-if [ -n "$PROM_PERSISTENT_DIR" ]; then
-    if [ -d "$PROM_PERSISTENT_DIR/latest-rules/${TIDB_VERSION##*/}" ]; then
-        cp -f -- "$PROM_PERSISTENT_DIR/latest-rules/${TIDB_VERSION##*/}"/*.rules.yml "$PROM_CONFIG_PATH/rules"
-    fi
-fi
+restore_rules "${TIDB_VERSION##*/}" || exit 1
 
 # Datasources
 if [ -n "$GF_DATASOURCE_PATH" ]; then

cmd/persisted_rules_test.go

@@ -0,0 +1,138 @@
+package main
+
+import (
+	"os"
+	"os/exec"
+	"path/filepath"
+	"regexp"
+	"runtime"
+	"strings"
+	"testing"
+)
+
+func TestInitializerPersistedRulesRequireExplicitTrust(t *testing.T) {
+	if runtime.GOOS == "windows" {
+		t.Skip("initializer requires POSIX shell")
+	}
+	for _, source := range []string{"init.sh", "../platform-monitoring/operator/init.sh", "../monitor-snapshot/master/operator/init.sh"} {
+		for _, dm := range []bool{false, true} {
+			for _, scenario := range []string{"default", "trusted", "unknown", "symlink", "directory-link", "invalid"} {
+				t.Run(source+"/"+map[bool]string{false: "tidb", true: "dm"}[dm]+"/"+scenario, func(t *testing.T) {
+					root := t.TempDir()
+					assets, output, persistent := filepath.Join(root, "assets"), filepath.Join(root, "prometheus"), filepath.Join(root, "persistent")
+					for _, dir := range []string{assets, filepath.Join(output, "rules"), filepath.Join(persistent, "latest-rules", "v1")} {
+						if err := os.MkdirAll(dir, 0755); err != nil {
+							t.Fatal(err)
+						}
+					}
+					extraPath := ""
+					if runtime.GOOS == "darwin" {
+						gsed, err := exec.LookPath("gsed")
+						if err != nil {
+							t.Skip("GNU sed required on macOS")
+						}
+						bin := filepath.Join(root, "bin")
+						if err := os.Mkdir(bin, 0755); err != nil {
+							t.Fatal(err)
+						}
+						if err := os.Symlink(gsed, filepath.Join(bin, "sed")); err != nil {
+							t.Fatal(err)
+						}
+						extraPath = bin + string(os.PathListSeparator)
+					}
+					script, err := os.ReadFile(source)
+					if err != nil {
+						t.Fatal(err)
+					}
+					for _, match := range regexp.MustCompile(`/tmp/([A-Za-z0-9_.-]+)`).FindAllStringSubmatch(string(script), -1) {
+						name := match[1]
+						if filepath.Ext(name) != ".json" && filepath.Ext(name) != ".yml" && filepath.Ext(name) != ".yaml" {
+							continue
+						}
+						if err := os.WriteFile(filepath.Join(assets, name), []byte("{}"), 0644); err != nil {
+							t.Fatal(err)
+						}
+					}
+					name := "test.rules.yml"
+					if dm {
+						name = "dm_worker.rules.yml"
+					}
+					bundled, restored := "groups: []\n# bundled\n", "groups: []\n# reviewed custom rules\n"
+					if err := os.WriteFile(filepath.Join(assets, name), []byte(bundled), 0644); err != nil {
+						t.Fatal(err)
+					}
+					old := filepath.Join(output, "rules", "foreign.rules.yml")
+					if err := os.WriteFile(old, []byte("groups: []"), 0644); err != nil {
+						t.Fatal(err)
+					}
+					saved := filepath.Join(persistent, "latest-rules", "v1", name)
+					if err := os.WriteFile(saved, []byte(restored), 0644); err != nil {
+						t.Fatal(err)
+					}
+					trust := "true"
+					if scenario == "default" {
+						trust = ""
+					}
+					switch scenario {
+					case "unknown":
+						if err := os.WriteFile(filepath.Join(filepath.Dir(saved), "foreign.rules.yml"), []byte(restored), 0644); err != nil {
+							t.Fatal(err)
+						}
+					case "symlink":
+						if err := os.Remove(saved); err != nil {
+							t.Fatal(err)
+						}
+						if err := os.Symlink(filepath.Join(assets, name), saved); err != nil {
+							t.Fatal(err)
+						}
+					case "directory-link":
+						if err := os.Rename(filepath.Dir(saved), filepath.Join(root, "untrusted")); err != nil {
+							t.Fatal(err)
+						}
+						if err := os.Symlink(filepath.Join(root, "untrusted"), filepath.Dir(saved)); err != nil {
+							t.Fatal(err)
+						}
+					case "invalid":
+						trust = "yes"
+					}
+					scriptPath := filepath.Join(root, "init.sh")
+					if err := os.WriteFile(scriptPath, []byte(strings.ReplaceAll(string(script), "/tmp/", assets+"/")), 0644); err != nil {
+						t.Fatal(err)
+					}
+					cluster := ""
+					if dm {
+						cluster = "test"
+					}
+					cmd := exec.Command("sh", scriptPath)
+					cmd.Env = append(os.Environ(), "DM_CLUSTER_NAME="+cluster, "TIDB_CLUSTER_NAME=test", "DM_VERSION=v1", "TIDB_VERSION=v1", "GF_PROVISIONING_PATH="+filepath.Join(root, "grafana"), "PROM_CONFIG_PATH="+output, "PROM_PERSISTENT_DIR="+persistent, "PROM_RESTORE_RULES="+trust, "GF_DATASOURCE_PATH=", "PATH="+extraPath+os.Getenv("PATH"))
+					out, err := cmd.CombinedOutput()
+					success := scenario == "default" || scenario == "trusted"
+					if success && err != nil {
+						t.Fatalf("initializer failed: %v %s", err, out)
+					}
+					if !success && err == nil {
+						t.Fatalf("unsafe restore succeeded: %s", out)
+					}
+					if !success {
+						return
+					}
+					want := bundled
+					if scenario == "trusted" {
+						want = restored
+					}
+					got, err := os.ReadFile(filepath.Join(output, "rules", name))
+					if err != nil || string(got) != want {
+						t.Fatalf("rules = %q, want %q: %v", got, want, err)
+					}
+					if _, err := os.Lstat(old); !os.IsNotExist(err) {
+						t.Fatalf("foreign active rule survived: %v", err)
+					}
+					got, err = os.ReadFile(saved)
+					if err != nil || string(got) != restored {
+						t.Fatalf("PVC data was modified: %q %v", got, err)
+					}
+				})
+			}
+		}
+	}
+}