From f9883592891c41ab61b90e46625e2778a740f13a Mon Sep 17 00:00:00 2001 From: 0z5a <192209249+0z5a@users.noreply.github.com> Date: Sun, 13 Sep 2026 03:14:59 +0000 Subject: [PATCH 1/2] Add SmolLM2 tile128 schedule preset and generation evidence Signed-off-by: 0z5a <192209249+0z5a@users.noreply.github.com> Signed-off-by: 0z5a --- docs/smollm2_tile128.md | 75 +++++++++++ docs/smollm2_tile128_results.json | 194 +++++++++++++++++++++++++++++ examples/smollm2_gemv_tile128.json | 7 ++ 3 files changed, 276 insertions(+) create mode 100644 docs/smollm2_tile128.md create mode 100644 docs/smollm2_tile128_results.json create mode 100644 examples/smollm2_gemv_tile128.json diff --git a/docs/smollm2_tile128.md b/docs/smollm2_tile128.md new file mode 100644 index 0000000..6419c5d --- /dev/null +++ b/docs/smollm2_tile128.md @@ -0,0 +1,75 @@ +# SmolLM2 GEMV tile128 schedule experiment + +`examples/smollm2_gemv_tile128.json` selects the existing `N_tile=128` schedule +knob. The configuration is opt-in; the default lowerer, CUDA VM, ABI, oracle and +kernel knobs are unchanged. + +On the measured SmolLM2 workload, default tiling produced 5,286 tasks per step +and tile128 produced 1,716, with 848 buffers and 38,912 bytes of dynamic shared +memory in both arms. A wider tile can reduce schedule/table setup work while +reducing GPU parallelism. The result below measures complete generation; it +is not a GPU-only GEMV claim. + +## Using the configuration + +The existing evaluator accepts the preset: + +```bash +uv run amk eval HuggingFaceTB/SmolLM2-135M-Instruct --gpu rtx5090 \ + --config examples/smollm2_gemv_tile128.json +``` + +That command evaluates one forward using the registered target. It does not +reproduce the complete-generation experiment or substitute an RTX 5090 target +for a PRO 4000. The archived generation runner reads the live SM120 device +limits and passes the same `ScheduleConfig` to the unmodified lowerer at each +step. `generate` currently passes `config=None`, so reproducing the full request +requires that external adapter; this PR adds no new generation API. + +## Recorded complete-generation experiment + +Frozen upstream: `a514bbc20a03bbf698a17443f8f14a27a617fc10`. +Model: `HuggingFaceTB/SmolLM2-135M-Instruct`, revision +`12fd25f77366fa6b3b4b768ec3050bf629380bac`, all 30 layers and 134,515,008 +parameters, FP32 weights/activations, no quantization. Six checkpoint file hashes +are included in the evidence. Runtime: Python 3.12, Torch 2.13.0+cu130, +Transformers 5.16.1, CUDA 13.0, one RTX PRO 4000 Blackwell 24 GB (SM120). + +Four independent processes ran default/tile128/tile128/default, with independent +first-build directories. Context lengths were 8 and 64 tokens; each request +produced 8 new tokens. All 344 step-level logits comparisons passed the existing +FP32 `atol=rtol=1e-4` oracle against both CPU ReferenceVM and full eager. +All 64 correctness-stage generated tokens matched exactly. Only after both +workloads passed did each process perform one warmup and three timed requests +per context; timed and warmup sequences also matched. + +| Context | Process pair | Default median seconds | Tile128 median seconds | Latency reduction | +|---|---|---:|---:|---:| +| 8 | A0/P1 | 3.486624 | 1.778946 | 48.98% | +| 8 | A3/P2 | 3.515794 | 1.755405 | 50.07% | +| 64 | A0/P1 | 16.170751 | 7.911105 | 51.08% | +| 64 | A3/P2 | 15.953324 | 7.895794 | 50.51% | + +Allocator peak: 1,320,648,704 bytes in each process. The two process-pair +geometric means are 49.53% and 50.79% latency reduction for contexts 8 and 64. +The baseline last/first median changes are +0.84% and -1.34% respectively. +These are exploratory observations from two process pairs, not a confidence +interval or independently confirmed stable speedup. + +Timing includes per-step lowering/validation, VM and table construction, weight +H2D, KV transfer and sampling from a CPU-resident model. It excludes disk model +loading, first CUDA compilation and correctness instrumentation. The existing +path reconstructs the VM each step; this is not GPU-resident steady-state +decode, concurrent service throughput, CUDA Graph replay, or a comparison with +cuBLAS/vLLM. + +The preset passed the experiment's advancement rule: every context/process +pair improved by at least 1%. Further independent validation is required before +changing a default. A separate `cpa_stages=2` candidate regressed at context64 +and is not included. + +[All timings, per-step comparisons, checkpoint hashes and the original runner](https://gist.github.com/0z5a/7b6a65fb1899b27411706bb402201016) +are published together. The archive documents the exact directory layout and +commands for replaying all four processes. The compact checked-in +[results](smollm2_tile128_results.json) retain every recorded request time and +original receipt hashes. No GPU test was rerun for publication. diff --git a/docs/smollm2_tile128_results.json b/docs/smollm2_tile128_results.json new file mode 100644 index 0000000..fa18af9 --- /dev/null +++ b/docs/smollm2_tile128_results.json @@ -0,0 +1,194 @@ +{ + "upstream_commit": "a514bbc20a03bbf698a17443f8f14a27a617fc10", + "claim_level": "EXPLORATORY_2_PROCESS_PAIRS", + "metric": "synchronized full generate wall seconds from resident CPU model, including lowering/validation/H2D/table/KV/sampling; excludes disk checkpoint loading and completed compilation", + "model": { + "repo_id": "HuggingFaceTB/SmolLM2-135M-Instruct", + "revision": "12fd25f77366fa6b3b4b768ec3050bf629380bac", + "files": { + "config.json": { + "bytes": 861, + "sha256": "8eb740e8bbe4cff95ea7b4588d17a2432deb16e8075bc5828ff7ba9be94d982a" + }, + "generation_config.json": { + "bytes": 132, + "sha256": "87b916edaaab66b3899b9d0dd0752727dff6666686da0504d89ae0a6e055a013" + }, + "model.safetensors": { + "bytes": 269060552, + "sha256": "5af571cbf074e6d21a03528d2330792e532ca608f24ac70a143f6b369968ab8c" + }, + "tokenizer_config.json": { + "bytes": 3764, + "sha256": "4ec77d44f62efeb38d7e044a1db318f6a939438425312dfa333b8382dbad98df" + }, + "special_tokens_map.json": { + "bytes": 655, + "sha256": "2b7379f3ae813529281a5c602bc5a11c1d4e0a99107aaa597fe936c1e813ca52" + }, + "tokenizer.json": { + "bytes": 2104556, + "sha256": "9ca9acddb6525a194ec8ac7a87f24fbba7232a9a15ffa1af0c1224fcd888e47c" + } + } + }, + "order": [ + "A", + "P", + "P", + "A" + ], + "precision": { + "weights": "float32", + "activations": "float32", + "quantization": null, + "atol": 0.0001, + "rtol": 0.0001 + }, + "gpu": "NVIDIA RTX PRO 4000 Blackwell", + "torch": "2.13.0+cu130", + "cuda": "13.0", + "arms": [ + { + "index": 0, + "tile": 0, + "performance": [ + { + "context": 8, + "new_tokens": 8, + "warmup_requests": 1, + "recorded_requests": 3, + "seconds": [ + 3.4639218139927834, + 3.533638576976955, + 3.4866243698634207 + ], + "metric": "complete generate wall seconds, synchronized; disk model load excluded; lowering, validation, table setup, H2D, KV and sampling included", + "median_seconds": 3.4866243698634207 + }, + { + "context": 64, + "new_tokens": 8, + "warmup_requests": 1, + "recorded_requests": 3, + "seconds": [ + 16.039617849048227, + 16.17075055697933, + 16.200428175041452 + ], + "metric": "complete generate wall seconds, synchronized; disk model load excluded; lowering, validation, table setup, H2D, KV and sampling included", + "median_seconds": 16.17075055697933 + } + ], + "allocator_peak_bytes": 1320648704, + "source_receipt_sha256": "948604e4d6f107582494ee3f358477ba9f55fae0e9dd076dae9f71a221be5c15" + }, + { + "index": 1, + "tile": 128, + "performance": [ + { + "context": 8, + "new_tokens": 8, + "warmup_requests": 1, + "recorded_requests": 3, + "seconds": [ + 1.778946174075827, + 1.8109971762169152, + 1.765465283067897 + ], + "metric": "complete generate wall seconds, synchronized; disk model load excluded; lowering, validation, table setup, H2D, KV and sampling included", + "median_seconds": 1.778946174075827 + }, + { + "context": 64, + "new_tokens": 8, + "warmup_requests": 1, + "recorded_requests": 3, + "seconds": [ + 7.911105460021645, + 8.040637442842126, + 7.805065699853003 + ], + "metric": "complete generate wall seconds, synchronized; disk model load excluded; lowering, validation, table setup, H2D, KV and sampling included", + "median_seconds": 7.911105460021645 + } + ], + "allocator_peak_bytes": 1319820288, + "source_receipt_sha256": "6b3a39c0c1b28d22c70e2c97c4c63f026556222935a8eec332605bf584fff8ca" + }, + { + "index": 2, + "tile": 128, + "performance": [ + { + "context": 8, + "new_tokens": 8, + "warmup_requests": 1, + "recorded_requests": 3, + "seconds": [ + 1.7565167911816388, + 1.7502881770487875, + 1.7554047601297498 + ], + "metric": "complete generate wall seconds, synchronized; disk model load excluded; lowering, validation, table setup, H2D, KV and sampling included", + "median_seconds": 1.7554047601297498 + }, + { + "context": 64, + "new_tokens": 8, + "warmup_requests": 1, + "recorded_requests": 3, + "seconds": [ + 7.8609512688126415, + 8.023758386960253, + 7.8957938810344785 + ], + "metric": "complete generate wall seconds, synchronized; disk model load excluded; lowering, validation, table setup, H2D, KV and sampling included", + "median_seconds": 7.8957938810344785 + } + ], + "allocator_peak_bytes": 1319820288, + "source_receipt_sha256": "f2e1f950386da0a8de66000b226ea64847e6c6ffb4a395cbe1880c66149363ad" + }, + { + "index": 3, + "tile": 0, + "performance": [ + { + "context": 8, + "new_tokens": 8, + "warmup_requests": 1, + "recorded_requests": 3, + "seconds": [ + 3.5770872849971056, + 3.515794370090589, + 3.4626008081249893 + ], + "metric": "complete generate wall seconds, synchronized; disk model load excluded; lowering, validation, table setup, H2D, KV and sampling included", + "median_seconds": 3.515794370090589 + }, + { + "context": 64, + "new_tokens": 8, + "warmup_requests": 1, + "recorded_requests": 3, + "seconds": [ + 15.95332440501079, + 16.115130202146247, + 15.89765602699481 + ], + "metric": "complete generate wall seconds, synchronized; disk model load excluded; lowering, validation, table setup, H2D, KV and sampling included", + "median_seconds": 15.95332440501079 + } + ], + "allocator_peak_bytes": 1320648704, + "source_receipt_sha256": "45b11107487573ed07ae7fade072ed2f96686dd77e2b4bc9a19bcaf47db75da2" + } + ], + "validation": { + "step_comparisons": 344, + "all_cases_pass": true, + "all_generated_tokens_match": true + } +} diff --git a/examples/smollm2_gemv_tile128.json b/examples/smollm2_gemv_tile128.json new file mode 100644 index 0000000..9da9f77 --- /dev/null +++ b/examples/smollm2_gemv_tile128.json @@ -0,0 +1,7 @@ +{ + "tiling": { + "gemv": { + "N_tile": 128 + } + } +} From cfd53cbdc20992c5d3f8297500d74ca686fec870 Mon Sep 17 00:00:00 2001 From: 0z5a Date: Tue, 22 Sep 2026 10:09:19 +0000 Subject: [PATCH 2/2] Keep generated validation artifacts local Signed-off-by: 0z5a --- docs/smollm2_tile128.md | 2 +- docs/smollm2_tile128_results.json | 194 ------------------------------ 2 files changed, 1 insertion(+), 195 deletions(-) delete mode 100644 docs/smollm2_tile128_results.json diff --git a/docs/smollm2_tile128.md b/docs/smollm2_tile128.md index 6419c5d..2eff8c4 100644 --- a/docs/smollm2_tile128.md +++ b/docs/smollm2_tile128.md @@ -71,5 +71,5 @@ and is not included. [All timings, per-step comparisons, checkpoint hashes and the original runner](https://gist.github.com/0z5a/7b6a65fb1899b27411706bb402201016) are published together. The archive documents the exact directory layout and commands for replaying all four processes. The compact checked-in -[results](smollm2_tile128_results.json) retain every recorded request time and +results (record retained locally) retain every recorded request time and original receipt hashes. No GPU test was rerun for publication. diff --git a/docs/smollm2_tile128_results.json b/docs/smollm2_tile128_results.json deleted file mode 100644 index fa18af9..0000000 --- a/docs/smollm2_tile128_results.json +++ /dev/null @@ -1,194 +0,0 @@ -{ - "upstream_commit": "a514bbc20a03bbf698a17443f8f14a27a617fc10", - "claim_level": "EXPLORATORY_2_PROCESS_PAIRS", - "metric": "synchronized full generate wall seconds from resident CPU model, including lowering/validation/H2D/table/KV/sampling; excludes disk checkpoint loading and completed compilation", - "model": { - "repo_id": "HuggingFaceTB/SmolLM2-135M-Instruct", - "revision": "12fd25f77366fa6b3b4b768ec3050bf629380bac", - "files": { - "config.json": { - "bytes": 861, - "sha256": "8eb740e8bbe4cff95ea7b4588d17a2432deb16e8075bc5828ff7ba9be94d982a" - }, - "generation_config.json": { - "bytes": 132, - "sha256": "87b916edaaab66b3899b9d0dd0752727dff6666686da0504d89ae0a6e055a013" - }, - "model.safetensors": { - "bytes": 269060552, - "sha256": "5af571cbf074e6d21a03528d2330792e532ca608f24ac70a143f6b369968ab8c" - }, - "tokenizer_config.json": { - "bytes": 3764, - "sha256": "4ec77d44f62efeb38d7e044a1db318f6a939438425312dfa333b8382dbad98df" - }, - "special_tokens_map.json": { - "bytes": 655, - "sha256": "2b7379f3ae813529281a5c602bc5a11c1d4e0a99107aaa597fe936c1e813ca52" - }, - "tokenizer.json": { - "bytes": 2104556, - "sha256": "9ca9acddb6525a194ec8ac7a87f24fbba7232a9a15ffa1af0c1224fcd888e47c" - } - } - }, - "order": [ - "A", - "P", - "P", - "A" - ], - "precision": { - "weights": "float32", - "activations": "float32", - "quantization": null, - "atol": 0.0001, - "rtol": 0.0001 - }, - "gpu": "NVIDIA RTX PRO 4000 Blackwell", - "torch": "2.13.0+cu130", - "cuda": "13.0", - "arms": [ - { - "index": 0, - "tile": 0, - "performance": [ - { - "context": 8, - "new_tokens": 8, - "warmup_requests": 1, - "recorded_requests": 3, - "seconds": [ - 3.4639218139927834, - 3.533638576976955, - 3.4866243698634207 - ], - "metric": "complete generate wall seconds, synchronized; disk model load excluded; lowering, validation, table setup, H2D, KV and sampling included", - "median_seconds": 3.4866243698634207 - }, - { - "context": 64, - "new_tokens": 8, - "warmup_requests": 1, - "recorded_requests": 3, - "seconds": [ - 16.039617849048227, - 16.17075055697933, - 16.200428175041452 - ], - "metric": "complete generate wall seconds, synchronized; disk model load excluded; lowering, validation, table setup, H2D, KV and sampling included", - "median_seconds": 16.17075055697933 - } - ], - "allocator_peak_bytes": 1320648704, - "source_receipt_sha256": "948604e4d6f107582494ee3f358477ba9f55fae0e9dd076dae9f71a221be5c15" - }, - { - "index": 1, - "tile": 128, - "performance": [ - { - "context": 8, - "new_tokens": 8, - "warmup_requests": 1, - "recorded_requests": 3, - "seconds": [ - 1.778946174075827, - 1.8109971762169152, - 1.765465283067897 - ], - "metric": "complete generate wall seconds, synchronized; disk model load excluded; lowering, validation, table setup, H2D, KV and sampling included", - "median_seconds": 1.778946174075827 - }, - { - "context": 64, - "new_tokens": 8, - "warmup_requests": 1, - "recorded_requests": 3, - "seconds": [ - 7.911105460021645, - 8.040637442842126, - 7.805065699853003 - ], - "metric": "complete generate wall seconds, synchronized; disk model load excluded; lowering, validation, table setup, H2D, KV and sampling included", - "median_seconds": 7.911105460021645 - } - ], - "allocator_peak_bytes": 1319820288, - "source_receipt_sha256": "6b3a39c0c1b28d22c70e2c97c4c63f026556222935a8eec332605bf584fff8ca" - }, - { - "index": 2, - "tile": 128, - "performance": [ - { - "context": 8, - "new_tokens": 8, - "warmup_requests": 1, - "recorded_requests": 3, - "seconds": [ - 1.7565167911816388, - 1.7502881770487875, - 1.7554047601297498 - ], - "metric": "complete generate wall seconds, synchronized; disk model load excluded; lowering, validation, table setup, H2D, KV and sampling included", - "median_seconds": 1.7554047601297498 - }, - { - "context": 64, - "new_tokens": 8, - "warmup_requests": 1, - "recorded_requests": 3, - "seconds": [ - 7.8609512688126415, - 8.023758386960253, - 7.8957938810344785 - ], - "metric": "complete generate wall seconds, synchronized; disk model load excluded; lowering, validation, table setup, H2D, KV and sampling included", - "median_seconds": 7.8957938810344785 - } - ], - "allocator_peak_bytes": 1319820288, - "source_receipt_sha256": "f2e1f950386da0a8de66000b226ea64847e6c6ffb4a395cbe1880c66149363ad" - }, - { - "index": 3, - "tile": 0, - "performance": [ - { - "context": 8, - "new_tokens": 8, - "warmup_requests": 1, - "recorded_requests": 3, - "seconds": [ - 3.5770872849971056, - 3.515794370090589, - 3.4626008081249893 - ], - "metric": "complete generate wall seconds, synchronized; disk model load excluded; lowering, validation, table setup, H2D, KV and sampling included", - "median_seconds": 3.515794370090589 - }, - { - "context": 64, - "new_tokens": 8, - "warmup_requests": 1, - "recorded_requests": 3, - "seconds": [ - 15.95332440501079, - 16.115130202146247, - 15.89765602699481 - ], - "metric": "complete generate wall seconds, synchronized; disk model load excluded; lowering, validation, table setup, H2D, KV and sampling included", - "median_seconds": 15.95332440501079 - } - ], - "allocator_peak_bytes": 1320648704, - "source_receipt_sha256": "45b11107487573ed07ae7fade072ed2f96686dd77e2b4bc9a19bcaf47db75da2" - } - ], - "validation": { - "step_comparisons": 344, - "all_cases_pass": true, - "all_generated_tokens_match": true - } -}