Regenerate all explain files with ClickHouse 25.8 (#77)

Author: kyleconroy

cmd/regenerate-explain/main.go

@@ -12,8 +12,10 @@ import (
 	"os"
 	"os/exec"
 	"path/filepath"
+	"runtime"
 	"strconv"
 	"strings"
+	"sync"
 	"syscall"
 	"time"
 )
@@ -40,6 +42,7 @@ func main() {
 	dryRun := flag.Bool("dry-run", false, "Print statements without executing")
 	serverOnly := flag.Bool("server", false, "Only ensure server is running, don't regenerate")
 	stopServer := flag.Bool("stop", false, "Stop the ClickHouse server")
+	parallel := flag.Int("j", runtime.NumCPU(), "Number of parallel workers (default: number of CPUs)")
 	flag.Parse()
 
 	// Handle stop command
@@ -85,19 +88,68 @@ func main() {
 		os.Exit(1)
 	}
 
-	var errors []string
-	var processed, skipped int
+	// Collect test directories
+	var testDirs []string
 	for _, entry := range entries {
 		if !entry.IsDir() {
 			continue
 		}
-		testDir := filepath.Join(testdataDir, entry.Name())
-		if err := processTest(testDir, *dryRun); err != nil {
-			if strings.Contains(err.Error(), "no statements found") {
-				skipped++
-				continue
+		testDirs = append(testDirs, filepath.Join(testdataDir, entry.Name()))
+	}
+
+	// Process tests in parallel
+	type result struct {
+		name    string
+		err     error
+		skipped bool
+	}
+
+	numWorkers := *parallel
+	if numWorkers < 1 {
+		numWorkers = 1
+	}
+	fmt.Printf("Processing %d tests with %d workers...\n", len(testDirs), numWorkers)
+
+	jobs := make(chan string, len(testDirs))
+	results := make(chan result, len(testDirs))
+
+	// Start workers
+	var wg sync.WaitGroup
+	for w := 0; w < numWorkers; w++ {
+		wg.Add(1)
+		go func() {
+			defer wg.Done()
+			for testDir := range jobs {
+				err := processTest(testDir, *dryRun)
+				skipped := err != nil && strings.Contains(err.Error(), "no statements found")
+				if skipped {
+					err = nil
+				}
+				results <- result{name: filepath.Base(testDir), err: err, skipped: skipped}
 			}
-			errors = append(errors, fmt.Sprintf("%s: %v", entry.Name(), err))
+		}()
+	}
+
+	// Send jobs
+	for _, testDir := range testDirs {
+		jobs <- testDir
+	}
+	close(jobs)
+
+	// Wait for workers and close results
+	go func() {
+		wg.Wait()
+		close(results)
+	}()
+
+	// Collect results
+	var errors []string
+	var processed, skipped int
+	for r := range results {
+		if r.skipped {
+			skipped++
+		} else if r.err != nil {
+			errors = append(errors, fmt.Sprintf("%s: %v", r.name, r.err))
 		} else {
 			processed++
 		}
@@ -524,21 +576,23 @@ func processTest(testDir string, dryRun bool) error {
 		return fmt.Errorf("no statements found")
 	}
 
-	fmt.Printf("Processing %s (%d statements)\n", filepath.Base(testDir), len(statements))
+	testName := filepath.Base(testDir)
 
-	// Generate version header comment
-	versionHeader := fmt.Sprintf("-- Generated by ClickHouse %s\n", requiredVersion)
+	if dryRun {
+		fmt.Printf("Processing %s (%d statements)\n", testName, len(statements))
+		for i, stmt := range statements {
+			fmt.Printf("  [%d] %s\n", i+1, truncate(stmt, 80))
+		}
+		return nil
+	}
 
+	var stmtErrors []string
 	for i, stmt := range statements {
 		stmtNum := i + 1 // 1-indexed
-		if dryRun {
-			fmt.Printf("  [%d] %s\n", stmtNum, truncate(stmt, 80))
-			continue
-		}
 
 		explain, err := explainAST(stmt)
 		if err != nil {
-			fmt.Printf("  [%d] ERROR: %v\n", stmtNum, err)
+			stmtErrors = append(stmtErrors, fmt.Sprintf("stmt %d: %v", stmtNum, err))
 			// Skip statements that fail - they might be intentionally invalid
 			continue
 		}
@@ -551,12 +605,17 @@ func processTest(testDir string, dryRun bool) error {
 			outputPath = filepath.Join(testDir, fmt.Sprintf("explain_%d.txt", stmtNum))
 		}
 
-		// Write with version header
-		content := versionHeader + explain + "\n"
+		content := explain + "\n"
 		if err := os.WriteFile(outputPath, []byte(content), 0644); err != nil {
 			return fmt.Errorf("writing %s: %w", outputPath, err)
 		}
-		fmt.Printf("  [%d] -> %s\n", stmtNum, filepath.Base(outputPath))
+	}
+
+	// Print summary
+	if len(stmtErrors) > 0 {
+		fmt.Printf("%s: %d stmts, %d errors\n", testName, len(statements), len(stmtErrors))
+	} else {
+		fmt.Printf("%s: %d stmts OK\n", testName, len(statements))
 	}
 
 	return nil

parser/parser_test.go

@@ -268,98 +268,6 @@ func TestParser(t *testing.T) {
 	}
 }
 
-// TestExplainVersionConsistency ensures all explain*.txt files have the same ClickHouse version header.
-// This test:
-// - Scans all explain*.txt files in testdata/
-// - Checks each file for a "-- Generated by ClickHouse X.X.X.X" header
-// - Reports files without version headers
-// - Reports if multiple different versions are found
-// - Passes only when all files have the same version header
-func TestExplainVersionConsistency(t *testing.T) {
-	testdataDir := "testdata"
-
-	// Find all explain*.txt files
-	var explainFiles []string
-	err := filepath.Walk(testdataDir, func(path string, info os.FileInfo, err error) error {
-		if err != nil {
-			return err
-		}
-		if !info.IsDir() && strings.HasPrefix(info.Name(), "explain") && strings.HasSuffix(info.Name(), ".txt") {
-			explainFiles = append(explainFiles, path)
-		}
-		return nil
-	})
-	if err != nil {
-		t.Fatalf("Failed to walk testdata directory: %v", err)
-	}
-
-	if len(explainFiles) == 0 {
-		t.Skip("No explain*.txt files found")
-	}
-
-	// Track versions and files without headers
-	versionToFiles := make(map[string][]string)
-	var filesWithoutHeaders []string
-
-	versionPattern := "-- Generated by ClickHouse "
-
-	for _, path := range explainFiles {
-		content, err := os.ReadFile(path)
-		if err != nil {
-			t.Errorf("Failed to read %s: %v", path, err)
-			continue
-		}
-
-		lines := strings.Split(string(content), "\n")
-		if len(lines) == 0 {
-			filesWithoutHeaders = append(filesWithoutHeaders, path)
-			continue
-		}
-
-		firstLine := lines[0]
-		if !strings.HasPrefix(firstLine, versionPattern) {
-			filesWithoutHeaders = append(filesWithoutHeaders, path)
-			continue
-		}
-
-		// Extract version from "-- Generated by ClickHouse X.X.X.X"
-		version := strings.TrimPrefix(firstLine, versionPattern)
-		versionToFiles[version] = append(versionToFiles[version], path)
-	}
-
-	// Report findings
-	if len(filesWithoutHeaders) > 0 {
-		t.Errorf("Found %d files without version headers:", len(filesWithoutHeaders))
-		// Show first 10 files as examples
-		limit := 10
-		if len(filesWithoutHeaders) < limit {
-			limit = len(filesWithoutHeaders)
-		}
-		for _, path := range filesWithoutHeaders[:limit] {
-			t.Errorf("  - %s", path)
-		}
-		if len(filesWithoutHeaders) > 10 {
-			t.Errorf("  ... and %d more", len(filesWithoutHeaders)-10)
-		}
-	}
-
-	if len(versionToFiles) > 1 {
-		t.Errorf("Found %d different ClickHouse versions:", len(versionToFiles))
-		for version, files := range versionToFiles {
-			t.Errorf("  Version %q: %d files", version, len(files))
-		}
-	}
-
-	if len(filesWithoutHeaders) > 0 || len(versionToFiles) != 1 {
-		t.FailNow()
-	}
-
-	// Log the consistent version
-	for version := range versionToFiles {
-		t.Logf("All %d explain files have consistent version: %s", len(explainFiles), version)
-	}
-}
-
 // BenchmarkParser benchmarks the parser performance using a complex query
 func BenchmarkParser(b *testing.B) {
 	query := `

parser/testdata/00001_select_1/explain.txt

@@ -1,4 +1,3 @@
--- Generated by ClickHouse 25.8.13.73
 SelectWithUnionQuery (children 1)
  ExpressionList (children 1)
   SelectQuery (children 1)

parser/testdata/00011_array_join_alias/explain.txt

@@ -19,4 +19,3 @@ SelectWithUnionQuery (children 1)
     TablesInSelectQueryElement (children 1)
      ArrayJoin (children 1)
       ExpressionList
-The query succeeded but the server error '42' was expected (query: EXPLAIN AST SELECT x, a FROM (SELECT arrayJoin(['Hello', 'Goodbye']) AS x, [1, 2, 3] AS arr) ARRAY JOIN; -- { serverError NUMBER_OF_ARGUMENTS_DOESNT_MATCH }).

parser/testdata/00013_create_table_with_arrays/explain_3.txt

@@ -0,0 +1,2 @@
+InsertQuery   (children 1)
+ Identifier arrays_test

parser/testdata/00014_select_from_table_with_nested/explain_3.txt

@@ -0,0 +1,2 @@
+InsertQuery   (children 1)
+ Identifier nested_test

parser/testdata/00030_alter_table/explain_4.txt

@@ -0,0 +1,2 @@
+InsertQuery   (children 1)
+ Identifier alter_test

parser/testdata/00036_array_element/explain_11.txt

@@ -0,0 +1,2 @@
+InsertQuery   (children 1)
+ Identifier array_element_test

parser/testdata/00036_array_element/explain_15.txt

@@ -0,0 +1,2 @@
+InsertQuery   (children 1)
+ Identifier array_element_test

parser/testdata/00036_array_element/explain_19.txt

@@ -0,0 +1,2 @@
+InsertQuery   (children 1)
+ Identifier array_element_test