LLM benchmark tool updates (#4413)

# Description of Changes

LLM benchmark updates for local development:

- **Local SDK paths**: Templates use relative paths to workspace crates
(`crates/bindings`, `crates/bindings-csharp`,
`crates/bindings-typescript`) instead of published packages, so the
bench runs against local SDK changes.
- **NODEJS_DIR support**: On Windows (e.g. nvm4w), if `pnpm` is not on
PATH, the bench uses `NODEJS_DIR` to locate `pnpm` and prepends it to
PATH for subprocesses.
- **Refactor**: Extracted `relative_to_workspace()` in `templates.rs`
and removed noisy `NODEJS_DIR` logging in `publishers.rs`.
- **Benchmark results**: Updated `docs/llms/llm-comparison-details.json`
and `docs/llms/llm-comparison-summary.json`.

# API and ABI breaking changes

None.

# Expected complexity level and risk

**2** — Local-only changes to the benchmark tool. Templates now require
local SDKs to be built (especially TypeScript: `pnpm build` in
`crates/bindings-typescript`). No impact on published SDKs or runtime.

# Testing

- [ ] Run `cargo llm run --lang rust --modes docs --providers openai`
from repo root
- [ ] Run TypeScript benchmarks with `pnpm build` in
`crates/bindings-typescript` first
- [ ] On Windows with nvm4w, set `NODEJS_DIR` if `pnpm` is not on PATH
and run TypeScript benchmarks

---------

Co-authored-by: Cursor <cursoragent@cursor.com>
Co-authored-by: clockwork-labs-bot <bot@clockworklabs.com>
Co-authored-by: clockwork-labs-bot <clockwork-labs-bot@users.noreply.github.com>

Author: 4 people

Cargo.lock

@@ -1,16 +1,25 @@
-# DEVELOP.md
+# DEVELOP.md
 
 This document explains how to configure the environment, run the LLM benchmark tool, and work with the benchmark suite.
 
 ---
 
 ## Table of Contents
 
-1. [Quick Checks & Fixes](#quick-checks-fixes)
-2. [Environment Variables](#environment-variables)
-3. [Benchmark Suite](#benchmark-suite)
-4. [Context Construction](#context-construction)
-5. [Troubleshooting](#troubleshooting)
+1. [Prerequisites](#prerequisites)
+2. [Quick Checks & Fixes](#quick-checks-fixes)
+3. [Environment Variables](#environment-variables)
+4. [Benchmark Suite](#benchmark-suite)
+5. [Context Construction](#context-construction)
+6. [Troubleshooting](#troubleshooting)
+---
+
+## Prerequisites
+
+- **Run from repo root** — `cargo llm` and related commands must be run from the workspace root (this repo).
+- **TypeScript benchmarks** — Run `pnpm build` in `crates/bindings-typescript` first. Rust and C# use local crates that are built as part of the workspace.
+- **Windows (nvm4w)** — If `pnpm` is not found when running TypeScript benchmarks, set `NODEJS_DIR` to your Node.js bin directory (e.g. `C:\nvm\v20.10.0`).
+
 ---
 
 ## Quick Checks & Fixes

docs/DEVELOP.md

@@ -19,6 +19,7 @@ serde_json.workspace = true
 blake3.workspace = true
 clap.workspace = true
 chrono = { version = "0.4", features = ["clock", "serde"] }
+dotenvy = "0.15"
 async-trait = "0.1.89"
 tokio = { version = "1", features = ["rt-multi-thread", "macros"] }
 urlencoding = "2.1.3"

tools/xtask-llm-benchmark/Cargo.toml

@@ -2,8 +2,9 @@ use crate::bench::utils::sanitize_db_name;
 use anyhow::{bail, Result};
 use regex::Regex;
 use std::borrow::Cow;
+use std::env;
 use std::fs;
-use std::path::Path;
+use std::path::{Path, PathBuf};
 use std::process::Command;
 use std::sync::LazyLock;
 
@@ -248,14 +249,63 @@ impl Publisher for TypeScriptPublisher {
         Self::ensure_package_json(source)?;
         let db = sanitize_db_name(module_name);
 
-        // Install dependencies (--ignore-workspace to avoid parent workspace interference)
-        run(
-            Command::new("pnpm")
-                .arg("install")
-                .arg("--ignore-workspace")
-                .current_dir(source),
-            "pnpm install (typescript)",
-        )?;
+        // Install dependencies (--ignore-workspace to avoid parent workspace interference).
+        // If NODEJS_DIR is set (e.g. nvm4w on Windows), use full path to pnpm so spawn finds it.
+        let pnpm_exe = env::var("NODEJS_DIR")
+            .ok()
+            .map(|s| s.trim().trim_matches('"').trim().to_string())
+            .filter(|s| !s.is_empty())
+            .map(PathBuf::from)
+            .and_then(|dir| {
+                #[cfg(windows)]
+                {
+                    let pnpm_cmd = dir.join("pnpm.cmd");
+                    let pnpm_exe_path = dir.join("pnpm.exe");
+                    if pnpm_cmd.is_file() {
+                        eprintln!("[pnpm] using NODEJS_DIR: {} (pnpm.cmd)", dir.display());
+                        Some(pnpm_cmd)
+                    } else if pnpm_exe_path.is_file() {
+                        eprintln!("[pnpm] using NODEJS_DIR: {} (pnpm.exe)", dir.display());
+                        Some(pnpm_exe_path)
+                    } else {
+                        eprintln!(
+                            "[pnpm] NODEJS_DIR set to {} but pnpm.cmd/pnpm.exe not found there, using PATH",
+                            dir.display()
+                        );
+                        None
+                    }
+                }
+                #[cfg(not(windows))]
+                {
+                    let pnpm = dir.join("pnpm");
+                    if pnpm.is_file() {
+                        eprintln!("[pnpm] using NODEJS_DIR: {} (pnpm)", dir.display());
+                        Some(pnpm)
+                    } else {
+                        eprintln!(
+                            "[pnpm] NODEJS_DIR set to {} but pnpm not found there, using PATH",
+                            dir.display()
+                        );
+                        None
+                    }
+                }
+            });
+        let mut pnpm_cmd = match &pnpm_exe {
+            Some(p) => Command::new(p),
+            None => Command::new("pnpm"),
+        };
+        pnpm_cmd.arg("install").arg("--ignore-workspace").current_dir(source);
+        // When using NODEJS_DIR, prepend it to PATH so pnpm.cmd can find node.
+        if let Some(ref dir) = pnpm_exe {
+            if let Some(parent) = dir.parent() {
+                let mut paths: Vec<PathBuf> = env::split_paths(&env::var("PATH").unwrap_or_default()).collect();
+                paths.insert(0, parent.to_path_buf());
+                if let Ok(new_path) = env::join_paths(paths) {
+                    pnpm_cmd.env("PATH", new_path);
+                }
+            }
+        }
+        run(&mut pnpm_cmd, "pnpm install (typescript)")?;
 
         // Publish (spacetime CLI handles TypeScript compilation internally)
         run(

tools/xtask-llm-benchmark/src/bench/publishers.rs

@@ -690,6 +690,7 @@ fn load_golden_source(task: &TaskPaths, lang: Lang) -> Result<String> {
 }
 
 // "1" | "01" | "001" | "t_001" -> "t_001"
+// "t_000_empty_reducers" | "t_001_basic_tables" -> accepted as-is (full task dir name)
 fn normalize_task_selector(raw: &str) -> Result<String> {
     let s = raw.trim().to_ascii_lowercase();
     if s.is_empty() {
@@ -700,6 +701,12 @@ fn normalize_task_selector(raw: &str) -> Result<String> {
             let n: u32 = rest.parse()?;
             return Ok(format!("t_{:03}", n));
         }
+        // Full task dir name: t_000_empty_reducers, t_001_basic_tables, etc.
+        if rest.chars().next().is_some_and(|c| c.is_ascii_digit())
+            && rest.chars().all(|c| c.is_ascii_alphanumeric() || c == '_')
+        {
+            return Ok(s);
+        }
         bail!("invalid task selector: {raw}");
     }
     if s.chars().all(|c| c.is_ascii_digit()) {

tools/xtask-llm-benchmark/src/bench/runner.rs

@@ -43,6 +43,31 @@ fn tmpl_root() -> PathBuf {
     PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("src").join("templates")
 }
 
+/// Workspace root (public/) for local SDK paths.
+fn workspace_root() -> PathBuf {
+    PathBuf::from(env!("CARGO_MANIFEST_DIR"))
+        .ancestors()
+        .nth(2)
+        .expect("xtask-llm-benchmark is under public/tools/xtask-llm-benchmark")
+        .to_path_buf()
+}
+
+/// Relative path from materialized root to a workspace subpath (e.g. "crates/bindings").
+/// Avoids Windows canonical paths (//?/D:/...) which can break Cargo/MSBuild/pnpm.
+fn relative_to_workspace(root: &Path, ws_subpath: &str) -> Result<String> {
+    let ws = workspace_root()
+        .canonicalize()
+        .with_context(|| "workspace root not found")?;
+    let root_canon = root
+        .canonicalize()
+        .with_context(|| format!("materialized root not found: {}", root.display()))?;
+    let root_rel = root_canon
+        .strip_prefix(&ws)
+        .with_context(|| format!("materialized dir {:?} not under workspace {:?}", root_canon, ws))?;
+    let ups = root_rel.components().count();
+    Ok(std::iter::repeat_n("..", ups).collect::<Vec<_>>().join("/") + "/" + ws_subpath)
+}
+
 fn copy_tree_with_templates(src: &Path, dst: &Path) -> Result<()> {
     fn recurse(from: &Path, to: &Path) -> Result<()> {
         fs::create_dir_all(to)?;
@@ -98,7 +123,19 @@ fn inject_rust(root: &Path, llm_code: &str) -> anyhow::Result<()> {
         }
         contents.push_str(&cleaned);
     }
-    fs::write(&lib, contents).with_context(|| format!("write {}", lib.display()))
+    fs::write(&lib, contents).with_context(|| format!("write {}", lib.display()))?;
+
+    let relative = relative_to_workspace(root, "crates/bindings")?;
+    let sdk_path = workspace_root().join("crates/bindings");
+    if !sdk_path.is_dir() {
+        bail!("local Rust SDK not found at {}", sdk_path.display());
+    }
+    let replacement = format!(r#"{{ path = "{}" }}"#, relative);
+    let cargo_toml = root.join("Cargo.toml");
+    let mut toml = fs::read_to_string(&cargo_toml).with_context(|| format!("read {}", cargo_toml.display()))?;
+    toml = toml.replace("{SPACETIME_RUST_SDK_PATH}", &replacement);
+    fs::write(&cargo_toml, toml).with_context(|| format!("write {}", cargo_toml.display()))?;
+    Ok(())
 }
 
 fn inject_csharp(root: &Path, llm_code: &str) -> anyhow::Result<()> {
@@ -116,7 +153,23 @@ fn inject_csharp(root: &Path, llm_code: &str) -> anyhow::Result<()> {
         }
         contents.push_str(&cleaned);
     }
-    fs::write(&prog, contents).with_context(|| format!("write {}", prog.display()))
+    fs::write(&prog, contents).with_context(|| format!("write {}", prog.display()))?;
+
+    let base_rel = relative_to_workspace(root, "crates/bindings-csharp")?;
+    let runtime_csproj = workspace_root().join("crates/bindings-csharp/Runtime/Runtime.csproj");
+    if !runtime_csproj.is_file() {
+        bail!("local C# Runtime not found at {}", runtime_csproj.display());
+    }
+    let runtime_ref = format!("{}/Runtime/Runtime.csproj", base_rel);
+    let runtime_dir = format!("{}/Runtime", base_rel);
+    let codegen_ref = format!("{}/Codegen/Codegen.csproj", base_rel);
+    let csproj_path = root.join("StdbModule.csproj");
+    let mut csproj = fs::read_to_string(&csproj_path).with_context(|| format!("read {}", csproj_path.display()))?;
+    csproj = csproj.replace("{SPACETIME_CSHARP_RUNTIME_DIR}", &runtime_dir);
+    csproj = csproj.replace("{SPACETIME_CSHARP_RUNTIME_REF}", &runtime_ref);
+    csproj = csproj.replace("{SPACETIME_CSHARP_CODEGEN_REF}", &codegen_ref);
+    fs::write(&csproj_path, csproj).with_context(|| format!("write {}", csproj_path.display()))?;
+    Ok(())
 }
 
 fn inject_typescript(root: &Path, llm_code: &str) -> anyhow::Result<()> {
@@ -134,7 +187,26 @@ fn inject_typescript(root: &Path, llm_code: &str) -> anyhow::Result<()> {
         }
         contents.push_str(&cleaned);
     }
-    fs::write(&lib, contents).with_context(|| format!("write {}", lib.display()))
+    fs::write(&lib, contents).with_context(|| format!("write {}", lib.display()))?;
+
+    let relative = relative_to_workspace(root, "crates/bindings-typescript")?;
+    let sdk_path = workspace_root().join("crates/bindings-typescript");
+    if !sdk_path.is_dir() {
+        bail!("local TypeScript SDK not found at {}", sdk_path.display());
+    }
+    let dist_server = sdk_path.join("dist/server/index.mjs");
+    if !dist_server.is_file() {
+        bail!(
+            "local TypeScript SDK at {} is not built (missing dist/server). Run: pnpm build (in crates/bindings-typescript)",
+            sdk_path.display()
+        );
+    }
+    let replacement = format!("file:{}", relative);
+    let package_json = root.join("package.json");
+    let mut pkg = fs::read_to_string(&package_json).with_context(|| format!("read {}", package_json.display()))?;
+    pkg = pkg.replace("{SPACETIME_TS_SDK_REF}", &replacement);
+    fs::write(&package_json, pkg).with_context(|| format!("write {}", package_json.display()))?;
+    Ok(())
 }
 
 /// Remove leading/trailing Markdown fences like ```rust ... ``` or ~~~

tools/xtask-llm-benchmark/src/bench/templates.rs

@@ -2,6 +2,12 @@
 
 public static partial class Module
 {
+    [Table(Accessor = "EmptyTable")]
+    public partial struct EmptyTable
+    {
+        [PrimaryKey] public int Id;
+    }
+
     [Reducer]
     public static void EmptyReducer_NoArgs(ReducerContext ctx) { }
 

tools/xtask-llm-benchmark/src/benchmarks/basics/t_000_empty_reducers/answers/csharp.cs

@@ -1,4 +1,10 @@
-use spacetimedb::{reducer, ReducerContext};
+use spacetimedb::{reducer, table, ReducerContext};
+
+#[table(accessor = empty_table)]
+pub struct EmptyTable {
+    #[primary_key]
+    pub id: i32,
+}
 
 #[reducer]
 pub fn empty_reducer_no_args(ctx: &ReducerContext) -> Result<(), String> {

tools/xtask-llm-benchmark/src/benchmarks/basics/t_000_empty_reducers/answers/rust.rs

@@ -1,6 +1,8 @@
-import { schema, t } from 'spacetimedb/server';
+import { schema, table, t } from 'spacetimedb/server';
 
-const spacetimedb = schema({});
+const emptyTable = table({ name: 'empty_table' }, { id: t.i32().primaryKey() });
+
+const spacetimedb = schema({ emptyTable });
 export default spacetimedb;
 
 export const emptyReducerNoArgs = spacetimedb.reducer({}, ctx => {

tools/xtask-llm-benchmark/src/benchmarks/basics/t_000_empty_reducers/answers/typescript.ts

@@ -1,4 +1,10 @@
-Write a SpacetimeDB backend module in C# that defines only these five empty reducers.
+Write a SpacetimeDB backend module in C# that defines a table and these five empty reducers.
+
+TABLES
+- EmptyTable
+  - Struct: EmptyTable
+  - Fields:
+    - Id: int (primary key)
 
 REDUCERS
 - EmptyReducer_NoArgs: no arguments, returns void, empty body