From 971715ef22fff8a1c65d49a1776348edb18bfb1f Mon Sep 17 00:00:00 2001 From: csuzhangxc Date: Wed, 11 Nov 2026 17:29:06 +0800 Subject: [PATCH 1/2] initializer: keep DM rules out of TiDB-only monitors The shared initializer image contains dm_worker.rules.yml for DM monitoring, but the TiDB branch copied every /tmp/*.rules.yml and restored every persisted *.rules.yml. TiDB-only monitors therefore loaded DM alert rules. Skip the DM-only rule when installing bundled rules and when restoring persistent rules in the TiDB path. The DM initializer still installs and restores it in the shared rule directory. Cover the operator template and the master snapshot with a regression test. --- cmd/dm_rule_scope_test.go | 140 +++++++++++++++++++++++ monitor-snapshot/master/operator/init.sh | 21 +++- platform-monitoring/operator/init.sh | 21 +++- 3 files changed, 178 insertions(+), 4 deletions(-) create mode 100644 cmd/dm_rule_scope_test.go diff --git a/cmd/dm_rule_scope_test.go b/cmd/dm_rule_scope_test.go new file mode 100644 index 00000000..262959b4 --- /dev/null +++ b/cmd/dm_rule_scope_test.go @@ -0,0 +1,140 @@ +package main + +import ( + "fmt" + "os" + "os/exec" + "path/filepath" + "regexp" + "runtime" + "strings" + "testing" +) + +// The operator initializer is shared by TiDB and DM monitors. DM rules must only +// be installed or restored by the DM initializer; a TiDB-only monitor must never +// end up loading them. +var operatorInitializerSources = []string{ + "../platform-monitoring/operator/init.sh", + "../monitor-snapshot/master/operator/init.sh", +} + +func TestTiDBInitializerDoesNotInstallOrRestoreDMRules(t *testing.T) { + if runtime.GOOS == "windows" { + t.Skip("initializer requires a POSIX shell") + } + for _, source := range operatorInitializerSources { + t.Run(source, func(t *testing.T) { + root := initializerTempDir(t) + assets := filepath.Join(root, "assets") + if err := os.MkdirAll(assets, 0755); err != nil { + t.Fatal(err) + } + script, err := os.ReadFile(source) + if err != nil { + t.Fatal(err) + } + // Create every dashboard/datasource referenced by the selected + // initializer so the test exercises the real script, not a subset. + for _, match := range regexp.MustCompile(`/tmp/([A-Za-z0-9_.-]+)`).FindAllStringSubmatch(string(script), -1) { + name := match[1] + if filepath.Ext(name) != ".json" && filepath.Ext(name) != ".yaml" { + continue + } + if err := os.WriteFile(filepath.Join(assets, name), []byte("{}\n"), 0644); err != nil { + t.Fatal(err) + } + } + for name, body := range map[string]string{ + "tidb.rules.yml": "groups: []\n# tidb bundled\n", + "dm_worker.rules.yml": "groups: []\n# dm bundled\n", + } { + if err := os.WriteFile(filepath.Join(assets, name), []byte(body), 0644); err != nil { + t.Fatal(err) + } + } + + output := filepath.Join(root, "prometheus") + rules := filepath.Join(output, "rules") + persistent := filepath.Join(root, "persistent") + store := filepath.Join(persistent, "latest-rules", "tidb:v1") + for _, dir := range []string{rules, store} { + if err := os.MkdirAll(dir, 0755); err != nil { + t.Fatal(err) + } + } + // Simulate both a previous buggy install and a persisted DM rule. + for _, path := range []string{ + filepath.Join(rules, "dm_worker.rules.yml"), + filepath.Join(store, "dm_worker.rules.yml"), + } { + if err := os.WriteFile(path, []byte("groups: []\n# dm legacy\n"), 0644); err != nil { + t.Fatal(err) + } + } + if err := os.WriteFile(filepath.Join(store, "tidb.rules.yml"), []byte("groups: []\n# tidb saved\n"), 0644); err != nil { + t.Fatal(err) + } + + path := os.Getenv("PATH") + if runtime.GOOS == "darwin" { + gsed, err := exec.LookPath("gsed") + if err != nil { + t.Skip("GNU sed is required to exercise the BusyBox initializer on macOS") + } + bin := filepath.Join(root, "bin") + if err := os.MkdirAll(bin, 0755); err != nil { + t.Fatal(err) + } + if err := os.Symlink(gsed, filepath.Join(bin, "sed")); err != nil { + t.Fatal(err) + } + path = bin + string(os.PathListSeparator) + path + } + scriptPath := filepath.Join(root, "init.sh") + if err := os.WriteFile(scriptPath, []byte(strings.ReplaceAll(string(script), "/tmp/", assets+"/")), 0644); err != nil { + t.Fatal(err) + } + baseEnv := []string{ + "PATH=" + path, + "GF_PROVISIONING_PATH=" + filepath.Join(root, "grafana"), + "PROM_CONFIG_PATH=" + output, + "PROM_PERSISTENT_DIR=" + persistent, + "TIDB_CLUSTER_NAME=test", + "TIDB_VERSION=tidb:v1", + "DM_VERSION=tidb:v1", + "GF_DATASOURCE_PATH=", + fmt.Sprintf("RELOADER_UID=%d", os.Geteuid()), + fmt.Sprintf("RELOADER_GID=%d", os.Getegid()), + } + + run := func(dmCluster string) { + t.Helper() + cmd := exec.Command("sh", scriptPath) + cmd.Env = append(os.Environ(), baseEnv...) + cmd.Env = append(cmd.Env, "DM_CLUSTER_NAME="+dmCluster) + if out, err := cmd.CombinedOutput(); err != nil { + t.Fatalf("initializer failed: %v %s", err, out) + } + } + + run("") + if _, err := os.Stat(filepath.Join(rules, "dm_worker.rules.yml")); !os.IsNotExist(err) { + t.Fatalf("TiDB-only initializer installed/restored DM rules: %v", err) + } + if _, err := os.Stat(filepath.Join(rules, "tidb.rules.yml")); err != nil { + t.Fatalf("TiDB rules were not restored: %v", err) + } + + // The DM initializer runs after the TiDB initializer against the + // same directory and must still install/restore the DM rules. + run("test") + if _, err := os.Stat(filepath.Join(rules, "dm_worker.rules.yml")); err != nil { + t.Fatalf("DM initializer did not install DM rules: %v", err) + } + if _, err := os.Stat(filepath.Join(rules, "tidb.rules.yml")); err != nil { + t.Fatalf("DM initializer removed TiDB rules: %v", err) + } + }) + } +} diff --git a/monitor-snapshot/master/operator/init.sh b/monitor-snapshot/master/operator/init.sh index a166ca4c..dc9dfa16 100755 --- a/monitor-snapshot/master/operator/init.sh +++ b/monitor-snapshot/master/operator/init.sh @@ -6,6 +6,10 @@ replace_template() { sed -i "s|$1|$replacement|g" -- "$3" } +# Rules installed only by the DM initializer. The initializer image and its +# /tmp rule bundle are shared by TiDB and DM monitors. +DM_ONLY_RULES="dm_worker.rules.yml" + # Reject symlink ancestors before any rule deletion, copy or ownership changes. validate_rule_path() { case "$1" in @@ -103,6 +107,7 @@ restore_rules() { case "$1" in ''|.|..|*/*) echo "Invalid persisted rules version" >&2; return 1;; esac + exclude=${2:-} saved_root="$PROM_PERSISTENT_DIR/latest-rules" saved_version="$saved_root/$1" if [ -L "$PROM_PERSISTENT_DIR" ] || [ -L "$saved_root" ] || [ -L "$saved_version" ]; then @@ -114,6 +119,9 @@ restore_rules() { [ -d "$saved_version" ] || return 0 for rule in "$saved_version"/*.rules.yml "$saved_version"/.rules.yml "$saved_version"/.*.rules.yml; do [ -e "$rule" ] || [ -L "$rule" ] || continue + if [ -n "$exclude" ] && [ "${rule##*/}" = "$exclude" ]; then + continue + fi validate_rule_file "$rule" || return 1 destination="$PROM_CONFIG_PATH/rules/${rule##*/}" if [ -e "$destination" ] || [ -L "$destination" ]; then @@ -285,14 +293,23 @@ reset_rules || exit 1 echo $META_TYPE echo $META_INSTANCE echo $META_VALUE -cp -- /tmp/*.rules.yml "$PROM_CONFIG_PATH/rules" || exit 1 +for rule in /tmp/*.rules.yml; do + [ -e "$rule" ] || continue + case "${rule##*/}" in + "$DM_ONLY_RULES") + # DM rules are installed by the DM initializer, not this one. + continue + ;; + esac + cp -f -- "$rule" "$PROM_CONFIG_PATH/rules" || exit 1 +done for file in "$PROM_CONFIG_PATH/rules"/*; do sed -i 's/ENV_LABELS_ENV/Cluster/g' "$file" sed -i 's%job=%component=%g' "$file" sed -i -e 's%\(by\s(\)job\(,.*)\)%\1component\2%g' -e 's%\(by\s(.*\),job,\(.*)\)%\1,component,\2%g' -e 's%\(by\s(.*,\)job)%\1component)%g' -e 's%\(by\s(\)job)%\1component)%g' "$file" done -restore_rules "${TIDB_VERSION##*/}" || exit 1 +restore_rules "${TIDB_VERSION##*/}" "$DM_ONLY_RULES" || exit 1 # Datasources prepare_reloader_rules "${TIDB_VERSION##*/}" || exit 1 diff --git a/platform-monitoring/operator/init.sh b/platform-monitoring/operator/init.sh index a166ca4c..dc9dfa16 100755 --- a/platform-monitoring/operator/init.sh +++ b/platform-monitoring/operator/init.sh @@ -6,6 +6,10 @@ replace_template() { sed -i "s|$1|$replacement|g" -- "$3" } +# Rules installed only by the DM initializer. The initializer image and its +# /tmp rule bundle are shared by TiDB and DM monitors. +DM_ONLY_RULES="dm_worker.rules.yml" + # Reject symlink ancestors before any rule deletion, copy or ownership changes. validate_rule_path() { case "$1" in @@ -103,6 +107,7 @@ restore_rules() { case "$1" in ''|.|..|*/*) echo "Invalid persisted rules version" >&2; return 1;; esac + exclude=${2:-} saved_root="$PROM_PERSISTENT_DIR/latest-rules" saved_version="$saved_root/$1" if [ -L "$PROM_PERSISTENT_DIR" ] || [ -L "$saved_root" ] || [ -L "$saved_version" ]; then @@ -114,6 +119,9 @@ restore_rules() { [ -d "$saved_version" ] || return 0 for rule in "$saved_version"/*.rules.yml "$saved_version"/.rules.yml "$saved_version"/.*.rules.yml; do [ -e "$rule" ] || [ -L "$rule" ] || continue + if [ -n "$exclude" ] && [ "${rule##*/}" = "$exclude" ]; then + continue + fi validate_rule_file "$rule" || return 1 destination="$PROM_CONFIG_PATH/rules/${rule##*/}" if [ -e "$destination" ] || [ -L "$destination" ]; then @@ -285,14 +293,23 @@ reset_rules || exit 1 echo $META_TYPE echo $META_INSTANCE echo $META_VALUE -cp -- /tmp/*.rules.yml "$PROM_CONFIG_PATH/rules" || exit 1 +for rule in /tmp/*.rules.yml; do + [ -e "$rule" ] || continue + case "${rule##*/}" in + "$DM_ONLY_RULES") + # DM rules are installed by the DM initializer, not this one. + continue + ;; + esac + cp -f -- "$rule" "$PROM_CONFIG_PATH/rules" || exit 1 +done for file in "$PROM_CONFIG_PATH/rules"/*; do sed -i 's/ENV_LABELS_ENV/Cluster/g' "$file" sed -i 's%job=%component=%g' "$file" sed -i -e 's%\(by\s(\)job\(,.*)\)%\1component\2%g' -e 's%\(by\s(.*\),job,\(.*)\)%\1,component,\2%g' -e 's%\(by\s(.*,\)job)%\1component)%g' -e 's%\(by\s(\)job)%\1component)%g' "$file" done -restore_rules "${TIDB_VERSION##*/}" || exit 1 +restore_rules "${TIDB_VERSION##*/}" "$DM_ONLY_RULES" || exit 1 # Datasources prepare_reloader_rules "${TIDB_VERSION##*/}" || exit 1 From c7698180d3270122ff5cec058ea8b2ecbbcc1847 Mon Sep 17 00:00:00 2001 From: csuzhangxc Date: Wed, 11 Nov 2026 17:38:30 +0800 Subject: [PATCH 2/2] test: drop standalone DM rule scope test --- cmd/dm_rule_scope_test.go | 140 -------------------------------------- 1 file changed, 140 deletions(-) delete mode 100644 cmd/dm_rule_scope_test.go diff --git a/cmd/dm_rule_scope_test.go b/cmd/dm_rule_scope_test.go deleted file mode 100644 index 262959b4..00000000 --- a/cmd/dm_rule_scope_test.go +++ /dev/null @@ -1,140 +0,0 @@ -package main - -import ( - "fmt" - "os" - "os/exec" - "path/filepath" - "regexp" - "runtime" - "strings" - "testing" -) - -// The operator initializer is shared by TiDB and DM monitors. DM rules must only -// be installed or restored by the DM initializer; a TiDB-only monitor must never -// end up loading them. -var operatorInitializerSources = []string{ - "../platform-monitoring/operator/init.sh", - "../monitor-snapshot/master/operator/init.sh", -} - -func TestTiDBInitializerDoesNotInstallOrRestoreDMRules(t *testing.T) { - if runtime.GOOS == "windows" { - t.Skip("initializer requires a POSIX shell") - } - for _, source := range operatorInitializerSources { - t.Run(source, func(t *testing.T) { - root := initializerTempDir(t) - assets := filepath.Join(root, "assets") - if err := os.MkdirAll(assets, 0755); err != nil { - t.Fatal(err) - } - script, err := os.ReadFile(source) - if err != nil { - t.Fatal(err) - } - // Create every dashboard/datasource referenced by the selected - // initializer so the test exercises the real script, not a subset. - for _, match := range regexp.MustCompile(`/tmp/([A-Za-z0-9_.-]+)`).FindAllStringSubmatch(string(script), -1) { - name := match[1] - if filepath.Ext(name) != ".json" && filepath.Ext(name) != ".yaml" { - continue - } - if err := os.WriteFile(filepath.Join(assets, name), []byte("{}\n"), 0644); err != nil { - t.Fatal(err) - } - } - for name, body := range map[string]string{ - "tidb.rules.yml": "groups: []\n# tidb bundled\n", - "dm_worker.rules.yml": "groups: []\n# dm bundled\n", - } { - if err := os.WriteFile(filepath.Join(assets, name), []byte(body), 0644); err != nil { - t.Fatal(err) - } - } - - output := filepath.Join(root, "prometheus") - rules := filepath.Join(output, "rules") - persistent := filepath.Join(root, "persistent") - store := filepath.Join(persistent, "latest-rules", "tidb:v1") - for _, dir := range []string{rules, store} { - if err := os.MkdirAll(dir, 0755); err != nil { - t.Fatal(err) - } - } - // Simulate both a previous buggy install and a persisted DM rule. - for _, path := range []string{ - filepath.Join(rules, "dm_worker.rules.yml"), - filepath.Join(store, "dm_worker.rules.yml"), - } { - if err := os.WriteFile(path, []byte("groups: []\n# dm legacy\n"), 0644); err != nil { - t.Fatal(err) - } - } - if err := os.WriteFile(filepath.Join(store, "tidb.rules.yml"), []byte("groups: []\n# tidb saved\n"), 0644); err != nil { - t.Fatal(err) - } - - path := os.Getenv("PATH") - if runtime.GOOS == "darwin" { - gsed, err := exec.LookPath("gsed") - if err != nil { - t.Skip("GNU sed is required to exercise the BusyBox initializer on macOS") - } - bin := filepath.Join(root, "bin") - if err := os.MkdirAll(bin, 0755); err != nil { - t.Fatal(err) - } - if err := os.Symlink(gsed, filepath.Join(bin, "sed")); err != nil { - t.Fatal(err) - } - path = bin + string(os.PathListSeparator) + path - } - scriptPath := filepath.Join(root, "init.sh") - if err := os.WriteFile(scriptPath, []byte(strings.ReplaceAll(string(script), "/tmp/", assets+"/")), 0644); err != nil { - t.Fatal(err) - } - baseEnv := []string{ - "PATH=" + path, - "GF_PROVISIONING_PATH=" + filepath.Join(root, "grafana"), - "PROM_CONFIG_PATH=" + output, - "PROM_PERSISTENT_DIR=" + persistent, - "TIDB_CLUSTER_NAME=test", - "TIDB_VERSION=tidb:v1", - "DM_VERSION=tidb:v1", - "GF_DATASOURCE_PATH=", - fmt.Sprintf("RELOADER_UID=%d", os.Geteuid()), - fmt.Sprintf("RELOADER_GID=%d", os.Getegid()), - } - - run := func(dmCluster string) { - t.Helper() - cmd := exec.Command("sh", scriptPath) - cmd.Env = append(os.Environ(), baseEnv...) - cmd.Env = append(cmd.Env, "DM_CLUSTER_NAME="+dmCluster) - if out, err := cmd.CombinedOutput(); err != nil { - t.Fatalf("initializer failed: %v %s", err, out) - } - } - - run("") - if _, err := os.Stat(filepath.Join(rules, "dm_worker.rules.yml")); !os.IsNotExist(err) { - t.Fatalf("TiDB-only initializer installed/restored DM rules: %v", err) - } - if _, err := os.Stat(filepath.Join(rules, "tidb.rules.yml")); err != nil { - t.Fatalf("TiDB rules were not restored: %v", err) - } - - // The DM initializer runs after the TiDB initializer against the - // same directory and must still install/restore the DM rules. - run("test") - if _, err := os.Stat(filepath.Join(rules, "dm_worker.rules.yml")); err != nil { - t.Fatalf("DM initializer did not install DM rules: %v", err) - } - if _, err := os.Stat(filepath.Join(rules, "tidb.rules.yml")); err != nil { - t.Fatalf("DM initializer removed TiDB rules: %v", err) - } - }) - } -}