From 2df7c85559346322de7ffa9d47d8526cdc187dc5 Mon Sep 17 00:00:00 2001 From: Thorsten Fuchs <76904008+ThorFuchs@users.noreply.github.com> Date: Thu, 15 Oct 2026 20:33:50 +0200 Subject: [PATCH 01/11] Add borrowed native composition scope and independent ownership proof --- Documentation/Architecture/API.md | 21 ++ .../Architecture/ExtractionManifest.json | 3 +- Documentation/Architecture/ResourceLimits.md | 31 ++ Documentation/CI/README.md | 16 +- .../Qualification/Composition/README.md | 44 +++ .../Qualification/TestDiscovery.json | 6 +- Scripts/ci.py | 6 +- Scripts/qualify_composition.py | 76 ++++ Sources/JSON/Document/JSONDocument.swift | 3 + .../JSON/Documentation.docc/Composition.md | 87 +++++ Sources/JSON/Documentation.docc/JSON.md | 2 + .../JSON/Limits/JSONCompositionScope.swift | 196 ++++++++++ .../JSON/Limits/JSONOperationContext.swift | 4 +- Sources/JSON/Limits/JSONWorkMeter.swift | 29 ++ Sources/JSON/Pointer/JSONPointer.swift | 8 +- Sources/JSONCLIEngine/CLIBuildIdentity.swift | 2 +- .../Document/JSONCompositionScope+URI.swift | 38 ++ .../JSONSchema/Document/JSONSchemaURI.swift | 4 +- .../CompositionOwnedProofTests.swift | 72 ++++ .../Negative/Async.swift.txt | 6 + .../Negative/Construct.swift.txt | 4 + .../Negative/Copy.swift.txt | 4 + .../Negative/Escape.swift.txt | 5 + .../Negative/Fork.swift.txt | 4 + .../Negative/PackageMeter.swift.txt | 4 + .../Negative/RawMeter.swift.txt | 4 + .../Negative/Refund.swift.txt | 4 + .../Negative/Reset.swift.txt | 4 + .../Negative/Return.swift.txt | 4 + .../Negative/Sendable.swift.txt | 5 + .../Negative/Store.swift.txt | 5 + .../Negative/Substitute.swift.txt | 6 + Tools/CompositionQualification/Package.swift | 20 ++ Tools/CompositionQualification/README.md | 27 ++ .../Sources/CompositionBenchmark/main.swift | 52 +++ .../Sources/CompositionProbe/Probe.swift | 21 ++ .../CompositionTests.swift | 340 ++++++++++++++++++ 37 files changed, 1154 insertions(+), 13 deletions(-) create mode 100644 Documentation/Qualification/Composition/README.md create mode 100644 Scripts/qualify_composition.py create mode 100644 Sources/JSON/Documentation.docc/Composition.md create mode 100644 Sources/JSON/Limits/JSONCompositionScope.swift create mode 100644 Sources/JSONSchema/Document/JSONCompositionScope+URI.swift create mode 100644 Tests/JSONIntegrationTests/CompositionOwnedProofTests.swift create mode 100644 Tools/CompositionQualification/Negative/Async.swift.txt create mode 100644 Tools/CompositionQualification/Negative/Construct.swift.txt create mode 100644 Tools/CompositionQualification/Negative/Copy.swift.txt create mode 100644 Tools/CompositionQualification/Negative/Escape.swift.txt create mode 100644 Tools/CompositionQualification/Negative/Fork.swift.txt create mode 100644 Tools/CompositionQualification/Negative/PackageMeter.swift.txt create mode 100644 Tools/CompositionQualification/Negative/RawMeter.swift.txt create mode 100644 Tools/CompositionQualification/Negative/Refund.swift.txt create mode 100644 Tools/CompositionQualification/Negative/Reset.swift.txt create mode 100644 Tools/CompositionQualification/Negative/Return.swift.txt create mode 100644 Tools/CompositionQualification/Negative/Sendable.swift.txt create mode 100644 Tools/CompositionQualification/Negative/Store.swift.txt create mode 100644 Tools/CompositionQualification/Negative/Substitute.swift.txt create mode 100644 Tools/CompositionQualification/Package.swift create mode 100644 Tools/CompositionQualification/README.md create mode 100644 Tools/CompositionQualification/Sources/CompositionBenchmark/main.swift create mode 100644 Tools/CompositionQualification/Sources/CompositionProbe/Probe.swift create mode 100644 Tools/CompositionQualification/Tests/CompositionPublicTests/CompositionTests.swift diff --git a/Documentation/Architecture/API.md b/Documentation/Architecture/API.md index 0acd3c2..1a37899 100644 --- a/Documentation/Architecture/API.md +++ b/Documentation/Architecture/API.md @@ -244,3 +244,24 @@ includes `revisionKind: "source-sha256"` and `revision: "sha256:"`. This is SHA-256 of the ordered Package.swift plus Sources content manifest, excluding only the generated identity file itself. It is not a Git commit, binary digest or claim of publication. Generation/checking needs no Git, time, environment or network. + + +## Borrowed native composition + +`JSON.withComposition(limits:_:)` synchronously borrows a noncopyable, +non-Sendable `JSONCompositionScope` and returns `JSONOperationReport` +only after healthy finalization. The scope has no public initializer. Public +methods parse a source-backed `Data` document, admit an existing value, construct, +parse, append and resolve native Pointers, charge external work, reserve checked +external logical bytes, checkpoint and tighten limits. Importing JSONSchema adds +`uri(_:relativeTo:)` and `removingFragment(_:)`; JSON has no sibling dependency. +URI operations use the core domain, with no compiled-plan or compilation-work +charge. This slice does not compose schema compilation/evaluation or Codable. + +All owned errors latch the original failure. Neither a caught failure, a nested +independent invocation nor later callback failure permits a completed outer +report. Scope handles cannot escape the synchronous borrow or cross suspension. +Native values may be returned; adapter-owned containers must be prepaid. Reports +are ordinary publicly constructible values, not certification of arbitrary +callback work. See [composition qualification](../Qualification/Composition/README.md) +and the JSON DocC composition guide for the exact accounting/ownership contract. diff --git a/Documentation/Architecture/ExtractionManifest.json b/Documentation/Architecture/ExtractionManifest.json index 10849c5..f95cd0c 100644 --- a/Documentation/Architecture/ExtractionManifest.json +++ b/Documentation/Architecture/ExtractionManifest.json @@ -269,7 +269,8 @@ "1.4 C4 immutable IR/function/pattern metadata, invocation closures/thunks, continuations, scalar string and format helpers, root-provenance paths and transactional mutation are original Apache-2.0 code. No eager upstream evaluator, Foundation regex or approximate arithmetic was copied.", "1.4 C5 stream/offline-schema adapters, checked sink handoff, command help and consumers are original Apache-2.0 code. Five NewInputs fixtures are copied byte-for-byte from the maintainer CLI briefing pack 43f8819c57e3d93bc6f7cf561d2d111442e089f2973cb7c62e16e3bd3fd73d0d; these authored data fixtures introduce no runtime dependency. Missing provider helper prerequisites remain blocked rather than adapted or silently rewritten.", "1.4 checked schema resource lookup and its missing/quota/cancellation/install regressions are original Apache-2.0 code. Supported app/framework/command layouts retain all nine official resource pins; no unbounded directory scan, fallback engine or external dependency is introduced.", - "1.5.1 restores 126 original Apache-2.0 test functions and qualification tools from Swift-JSON 1.2.1 (e8b687f822c577311156b13bf1d15c2d5620b84e), with file hashes and stable function identities in Qualification/Release1.5.1/Restoration.json. The diagnostic omission and Git-free provenance repairs are restored original code; current seven-product harness reconciliation and deterministic CLI identity are original contributions. No new upstream adaptation or dependency." + "1.5.1 restores 126 original Apache-2.0 test functions and qualification tools from Swift-JSON 1.2.1 (e8b687f822c577311156b13bf1d15c2d5620b84e), with file hashes and stable function identities in Qualification/Release1.5.1/Restoration.json. The diagnostic omission and Git-free provenance repairs are restored original code; current seven-product harness reconciliation and deterministic CLI identity are original contributions. No new upstream adaptation or dependency.", + "Borrowed composition scope, checked external reservations, final-report overlap accounting and independent public ownership/accounting probes are original Apache-2.0 contributions. Native-only helpers were reviewed and reused from Swift-JSON composition prototype 720da59648804c6716c0ee2f8bb431ebcb47b82b on fresh main 037e735dc6224789e4d04d2c18d15b8d27440198; no downstream proposal source was adapted and no dependency was added. Existing public entry points retain their meter semantics." ], "jsonhtml_export_original_contributions": { "license": "Apache-2.0", diff --git a/Documentation/Architecture/ResourceLimits.md b/Documentation/Architecture/ResourceLimits.md index 5a5c9c0..2720350 100644 --- a/Documentation/Architecture/ResourceLimits.md +++ b/Documentation/Architecture/ResourceLimits.md @@ -121,3 +121,34 @@ so retained reusable scratch is zero. Live owned capacities still charge retainedBytes. Scalar scanning and immutable plan-local regex sharing remain the qualified choices; no SIMD or prefilter optimization is claimed. Native performance observations are separate from the logical policy in [performance notes](../Performance/README.md). + + +## Composition scope accounting, version 1 + +The additive borrowed scope uses one original core ledger and the existing 49 +metrics/default profiles. It does not introduce adjustable counters or widen any +cap. `chargeExternalWork` charges aggregate work before adapter work; +`reserveExternalLogicalBytes(bytes,count:overhead:)` checks multiplication then +addition and live retention before growth. Zero is valid but never establishes +that opaque caller storage is free. All reservations are conservative until scope +exit; repeated admission and physical aliases count again. Caller instructions, +allocator capacity and process RSS remain outside this contract. + +`parseDocument` checks the full Data length against input and retained limits +before allocating its copied byte buffer, charges each copied byte, then invokes +the existing strict parser/source mapper. Caller Data is excluded. Input capacity +is per document: the preceding input count is checked and retired into the same +cumulative report before the next document. Work, retention, accepted-capacity +history and first-failure identity never restart. Remaining core counts are +conservative across the invocation; full source bytes/root/map and re-admitted +Pointer inputs/outputs remain charged. URI identity is core work/storage, never +schema compilation or compiledPlanBytes. + +Completion admits an additional 424 logical bytes (32 header + 49 eight-byte +counter slots) and 50 work units (49 counter copies and one final work update). +All result reservations overlap this report buffer. Overflow, quota failure and +cancellation prevent a completed result/report even when caught by the adapter. +No report is emitted after an error. Native accounting-version-1 weights and the +ordinary entry points' accounting stay unchanged. This proof slice has no public +release/promote tokens, refunds or partial reports. Independent adapter events +and failing omitted-charge mutants qualify the authored adapter only. diff --git a/Documentation/CI/README.md b/Documentation/CI/README.md index c30e5da..2831666 100644 --- a/Documentation/CI/README.md +++ b/Documentation/CI/README.md @@ -1,8 +1,8 @@ # Native CI This main-based forward-port retains seven products and restores the entire -1.2.1 regression inventory. Current native root discovery is 546 functions and -325 parameter invocations; Linux is 530/325. Full ASan uses the same roots; +1.2.1 regression inventory. Current native root discovery is 548 functions and +325 parameter invocations; Linux is 532/325. Full ASan uses the same roots; TSan checks 41 native or 40 Linux functions. QA requires the original 145-function maintenance subset. Exact per-module counts are in [the discovery contract](../Qualification/TestDiscovery.json); parameter-function @@ -132,3 +132,15 @@ regex 12, streaming 14, patch 11 and portable HTML 61. Its public consumer requi remain required. The first failed inherited expectations are retained; no blanket nonzero/exit-zero replacement is used. Real receiving-app and physical-device UI observations remain separate from automated SDK/API and portable execution. + + +## Independent composition proof + +`python3 Scripts/ci.py --lane composition` builds an unrelated SwiftPM test +package against normal public JSON/JSONSchema products. It requires 25 functions +in both Debug and Release, the independent four-owner event ledger, two actually +failing omitted-charge mutants, and 13 separate intended compiler failures. +The two supplementary native functions exercise in-loop cancellation and the +original continuation's health/identity. Root counts increased by exactly two; +all 126 restored identities, 325 parameter invocations and 49 cap outcomes remain +required. These expectations do not assert an unexecuted platform pass. diff --git a/Documentation/Qualification/Composition/README.md b/Documentation/Qualification/Composition/README.md new file mode 100644 index 0000000..57020b3 --- /dev/null +++ b/Documentation/Qualification/Composition/README.md @@ -0,0 +1,44 @@ +# Borrowed composition proof + +This additive slice is confined to one synchronous borrowed native scope and +explicitly instrumented adapter work. It is not a full external importer, schema +compiler/evaluator composition, general callback instruction quota or RSS bound. +No package product or runtime dependency is added. + +The exact compiled public declarations live in JSONCompositionScope.swift and +JSONCompositionScope+URI.swift. The native grammar, scalar key identity and exact +number authority are reused. URI work uses core accounting; parsing temporarily +uses the parsing domain and Pointers the pointer domain, then returns to core. +No schema plan ownership is manufactured for URI identity. + +Run `python3 Scripts/ci.py --lane composition`, followed by affected native, +safety and docs lanes. The ordinary public consumer package is +`Tools/CompositionQualification`; it has no package access, SPI, @testable or +shim. The root supplementary tests use existing package-only observers solely +to locate cancellation and substitution boundaries. They are not the oracle for +undercharging. + +The independent authored adapter's golden event trace is queue 65 work/96 +logical bytes, scratch 33/64, result 49/80 and final receipt 50/424: totals +197 work and 664 retained bytes. Three buffers remain returned and live. Exact +and one-below caps and an inserted work unit are tested. Omitting one byte's work +or reservation makes the unchanged positive trace assertion fail in a separate +expected-failing test process. This does not prove an arbitrary adapter is honest; +each downstream adapter needs its own ownership ledger and audited charges. + +All owned failures latch the first cause. Finalization verifies the original +ledger identity and health and prepays result/report overlap. Scope access is +borrowed and not Sendable; copying, storing, escaping, suspension, construction, +substitution, reset/fork/refund and private ledger access are separately rejected +by the compiler. A new invocation has new allowances and cannot restore the old +invocation's health. There is no public resource-release token in this slice. + +The 6.4 compiler crashed when a local nested helper captured the borrowed scope. +Use an explicit `borrowing JSONCompositionScope` helper parameter. Evidence must +retain that failure separately from the intended negative compiler probes. + +Qualification receipts belong outside the tracked public source distribution. +They must bind the source manifest, input pins, environment, exact counts, actual +commands/exits, mutant failures and per-platform status. A blocked Linux runtime +or SDK is not a native pass. There is no downstream pin/release authorization +in this proof. The user-facing return supplies materialized receipts and patches. diff --git a/Documentation/Qualification/TestDiscovery.json b/Documentation/Qualification/TestDiscovery.json index 6fd5f94..857fc75 100644 --- a/Documentation/Qualification/TestDiscovery.json +++ b/Documentation/Qualification/TestDiscovery.json @@ -6,7 +6,7 @@ "JSONRegexRuntimeTests": 16, "JSONStreamingTests": 30, "JSONPatchTests": 16, - "JSONIntegrationTests": 20, + "JSONIntegrationTests": 22, "JSONHTMLTests": 97, "JSONBridgesTests": 10, "JSONCLITests": 88 @@ -60,7 +60,7 @@ "JSONRegexRuntimeTests": 16, "JSONStreamingTests": 30, "JSONPatchTests": 16, - "JSONIntegrationTests": 20, + "JSONIntegrationTests": 22, "JSONHTMLTests": 81, "JSONBridgesTests": 10, "JSONCLITests": 88 @@ -71,7 +71,7 @@ "JSONRegexRuntimeTests": 16, "JSONStreamingTests": 30, "JSONPatchTests": 16, - "JSONIntegrationTests": 20, + "JSONIntegrationTests": 22, "JSONHTMLTests": 81, "JSONBridgesTests": 10 }, diff --git a/Scripts/ci.py b/Scripts/ci.py index e0c0dfe..2f26ac8 100644 --- a/Scripts/ci.py +++ b/Scripts/ci.py @@ -18,6 +18,8 @@ def snapshot(): def commands(lane, output): # The qualification executable resolves output paths from its repository root. output = output.relative_to(ROOT) + if lane == 'composition': + return [['python3', '-B', 'Scripts/qualify_composition.py', '--output', str(output / 'public')]] if lane == 'archive': return [['python3', '-B', 'Scripts/public_check.py'], ['python3', '-B', 'Scripts/qualify_native_reproduction.py', '--output', str(output / 'reproduction')]] @@ -200,11 +202,13 @@ def check_result(command, code, log, events=None): passed = passed and 'JSONHTML_BROWSER passed engines=3 cases=76 skips=0' in log.splitlines() if 'Scripts/test_qualification_harness.py' in command: passed = passed and 'HARNESS_TESTS passed cases=14' in log.splitlines() + if command[:3] == ['python3', '-B', 'Scripts/qualify_composition.py']: + passed = passed and 'COMPOSITION_PROOF passed public_tests=25 configurations=2 mutants_rejected=2 compiler_negative=13' in log.splitlines() return passed, counts def main(): parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument('--lane', choices=['native', 'safety', 'docs', 'r0', 'qa', 'linux', 'linux-safety', 'wasm', 'browser', 'export', 'cli', 'coverage', 'linux-coverage', 'linux-qa', 'wasm-qa', 'archive'], required=True) + parser.add_argument('--lane', choices=['composition', 'native', 'safety', 'docs', 'r0', 'qa', 'linux', 'linux-safety', 'wasm', 'browser', 'export', 'cli', 'coverage', 'linux-coverage', 'linux-qa', 'wasm-qa', 'archive'], required=True) parser.add_argument('--output') args = parser.parse_args() output = ROOT / (args.output or '.build/ci/' + args.lane) diff --git a/Scripts/qualify_composition.py b/Scripts/qualify_composition.py new file mode 100644 index 0000000..5ef3b1b --- /dev/null +++ b/Scripts/qualify_composition.py @@ -0,0 +1,76 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: Apache-2.0 +"""Independent public SwiftPM tests, real mutant failures and ownership builds.""" +import argparse, datetime, hashlib, json, os, pathlib, subprocess, tempfile +from testing_events import read_events + +ROOT = pathlib.Path(__file__).resolve().parents[1] +PACKAGE = 'Tools/CompositionQualification' +EXPECTED_TESTS = 25 +NEGATIVE = { + 'Copy': ["'copy' cannot be applied to noncopyable types"], 'Construct': ['inaccessible'], 'Escape': ['borrowed'], + 'Return': ['Copyable'], 'Sendable': ['Sendable'], 'Async': ['async', 'synchronous'], + 'Reset': ["no member 'reset'"], 'Fork': ["no member 'fork'"], + 'Refund': ["no member 'release'"], 'PackageMeter': ['inaccessible'], + 'RawMeter': ['JSONWorkMeter', 'cannot find'], 'Substitute': ['immutable'], + 'Store': ['borrowed'] +} + +def main(): + p = argparse.ArgumentParser(description=__doc__) + p.add_argument('--output', required=True) + a = p.parse_args(); output = (ROOT / a.output).resolve(); output.mkdir(parents=True, exist_ok=True) + receipt = dict(status='running', commands=[], public_tests_per_configuration=EXPECTED_TESTS, + compiler_negative_cases=len(NEGATIVE), expected_failing_mutants=2) + def save(): (output/'receipt.json').write_text(json.dumps(receipt, indent=2)+'\n') + def run(name, command, env=None, expected_failure=False): + log = output/(name+'.log'); started = datetime.datetime.now(datetime.timezone.utc).isoformat() + with log.open('w') as stream: + result = subprocess.run(command, cwd=ROOT, env=env, stdout=stream, stderr=subprocess.STDOUT, timeout=1800) + record = dict(name=name, command=command, exit_code=result.returncode, + started_utc=started, log=log.name, log_sha256=hashlib.sha256(log.read_bytes()).hexdigest(), + expected_failure=expected_failure) + receipt['commands'].append(record); save() + print(name, 'exit='+str(result.returncode), flush=True) + return record, log.read_text() + save() + for config in ['debug', 'release']: + directory=output/config; directory.mkdir(exist_ok=True) + events=directory/'events.jsonl'; env=os.environ.copy(); env['JCOMP_TRACE_DIRECTORY']=str(directory) + command=['swift','test','--package-path',PACKAGE,'-c',config,'--event-stream-output-path',str(events)] + record, log=run(config,command,env) + assert record['exit_code']==0,log[-6000:] + parsed=read_events(events.read_text()) + assert parsed['module_function_counts']=={'CompositionPublicTests':EXPECTED_TESTS} and parsed['parameter_case_invocations']==0 + record['swift_testing_events']=dict(parsed,path=events.relative_to(output).as_posix(),sha256=hashlib.sha256(events.read_bytes()).hexdigest());save() + assert (directory/'owned-events.tsv').read_text()=='owner\twork\tlogical_bytes\nqueue\t65\t96\nscratch\t33\t64\nresult\t49\t80\nreceipt\t50\t424\n' + # Mutants run the unchanged positive test as an actually failing test process. + for mutation in ['work','reservation']: + events=output/('mutant-'+mutation+'.events.jsonl') + env=os.environ.copy();env['JCOMP_TRACE_MUTATION']=mutation + command=['swift','test','--package-path',PACKAGE,'--skip-build','--filter','externalTraceMatchesIndependentLedger','--event-stream-output-path',str(events)] + record,log=run('mutant-'+mutation,command,env,expected_failure=True) + assert record['exit_code']==1 and ('workTrace' if mutation=='work' else 'retentionTrace') in log + rows=[json.loads(line) for line in events.read_text().splitlines() if line] + issues=[r for r in rows if r['kind']=='event' and r['payload']['kind']=='issueRecorded'] + completed=[r for r in rows if r['kind']=='event' and r['payload']['kind']=='testEnded'] + assert len(issues)==1 and len(completed)==1 + record.update(expected_issue=mutation,raw_issue_events=1,completed_test_functions=1,event_stream=events.name,event_sha256=hashlib.sha256(events.read_bytes()).hexdigest());save() + # Each fixture is a separate unrelated SwiftPM package/build, using exactly + # normal public products. No package-name override, @testable, SPI or shim. + with tempfile.TemporaryDirectory(prefix='swift-json-composition-negative-') as temp: + for name,diagnostics in NEGATIVE.items(): + directory=pathlib.Path(temp)/name; sources=directory/'Sources/NegativeConsumer';sources.mkdir(parents=True) + (directory/'Package.swift').write_text('// swift-tools-version: 6.4\nimport PackageDescription\nlet package = Package(name: "IndependentNegative'+name+'", platforms: [.macOS("27.0")], dependencies: [.package(name: "Swift-JSON", path: '+json.dumps(str(ROOT))+')], targets: [.executableTarget(name: "NegativeConsumer", dependencies: [.product(name: "JSON", package: "Swift-JSON"), .product(name: "JSONSchema", package: "Swift-JSON")])], swiftLanguageModes: [.v6])\n') + fixture=ROOT/PACKAGE/'Negative'/(name+'.swift.txt') + (sources/'main.swift').write_bytes(fixture.read_bytes()) + command=['swift','build','--build-system','native','--package-path',str(directory),'--scratch-path',str(output/'negative-build')] + record,log=run('negative-'+name,command,expected_failure=True) + assert record['exit_code']==1 and all(d in log for d in diagnostics), log[-6000:] + assert not any(d in log for d in ['no such module','Please submit a bug report','Stack dump','SIL verification failed']) + record.update(expected_diagnostics=diagnostics,fixture_sha256=hashlib.sha256(fixture.read_bytes()).hexdigest());save() + receipt['status']='passed'; save() + print('COMPOSITION_PROOF passed public_tests=25 configurations=2 mutants_rejected=2 compiler_negative=13') + return 0 + +if __name__=='__main__': raise SystemExit(main()) diff --git a/Sources/JSON/Document/JSONDocument.swift b/Sources/JSON/Document/JSONDocument.swift index f3afd6f..43d7707 100644 --- a/Sources/JSON/Document/JSONDocument.swift +++ b/Sources/JSON/Document/JSONDocument.swift @@ -41,6 +41,9 @@ extension JSONParser { if let source { try JSONAdmission.text(source, meter: &meter, owned: true) } var bytes: [UInt8] = [] for byte in data { try meter.work(); try meter.require(.inputBytes, UInt64(bytes.count) + 1); try meter.retain(1); bytes.append(byte) } + return try parseDocument(bytes, source: source, meter: &meter) + } + package func parseDocument(_ bytes: [UInt8], source: String?, meter: inout JSONWorkMeter) throws -> JSONDocument { var machine = JSONTokenMachine(), assembler = JSONValueAssembler(), mapping = JSONSourceMapBuilder(), index = 0 var result: JSON? while index < bytes.count { diff --git a/Sources/JSON/Documentation.docc/Composition.md b/Sources/JSON/Documentation.docc/Composition.md new file mode 100644 index 0000000..34ac4ba --- /dev/null +++ b/Sources/JSON/Documentation.docc/Composition.md @@ -0,0 +1,87 @@ +# Compose native operations + +Run synchronous native operations and explicitly instrumented adapter work with +one finite ledger. The composition facade does not meter arbitrary +callback instructions, allocator sizes, physical RSS, or Foundation instructions. + +```swift +import Foundation +import JSON +import JSONSchema + +let report = try JSON.withComposition { scope in + let document = try scope.parseDocument(Data(#"{"count":9007199254740993}"#.utf8)) + try scope.reserveExternalLogicalBytes(64) // Adapter-declared output frame weight. + try scope.chargeExternalWork(1) // Before the adapter's corresponding work. + let pointer = try scope.pointer(tokens: ["count"]) + _ = try scope.uri("https://example.invalid/schema.json") + return try scope.resolve(pointer, in: document.root) +} +``` + +The scope is borrowed, noncopyable, synchronous and not Sendable. Its methods +mutate invocation-owned private state. The closure cannot replace or swap the +borrowed facade. It has no public +initializer, continuation, refund or ownership token. Every reservation lasts +until completion. Re-admitting a value reserves its graph again; resolving +reserves its input graph and selected graph again. This intentionally overcounts +aliases and dead temporaries rather than assuming copy-on-write releases memory. +Reserve each external result, queue, diagnostic and copied slot before growth; +choose and audit its logical weights in the adapter. A zero reservation makes no +claim about opaque storage. External loops must charge/checkpoint regularly. + +`parseDocument` pre-admits the complete Data-to-array buffer before allocation, +charges each copied byte and runs the native source-map parser on the same meter. +The original caller Data is excluded; the array copy remains conservatively +reserved alongside decoded values, source bytes, spans and final output. +Each document applies `inputBytes` independently. Completed input counts remain +in the cumulative report; work, logical retention, accepted-capacity history and +terminal health never restart. Other native capacities remain conservative across +the scope. Version 1 native weights and internal scratch releases follow the resource +contract. Reported peak is this ledger's conservative logical peak, not a sum of +independent operation peaks. Final retained bytes include all unreleased +reservations, even when the callback returns a smaller result. + +Completion reserves 424 logical bytes (32 header bytes plus 49 UInt64 slots) +before allocating the independent report counter buffer. It charges 49 counter +copies plus one final aggregate-work update: 50 work units. The returned result +and all conservative reservations remain live through this final admission. +An adapter must also reserve its result containers before growth; the generic +result type does not reveal arbitrary storage to the library. + +All owned helper errors are terminal, including syntax and missing-target errors. +Catching one cannot produce a successful report. Scope exit rethrows the original +owned failure ahead of any later callback error. Restriction intersects limits +and preserves admitted capacities, cumulative work, live bytes and health. +A fresh invocation has a fresh budget; nesting does not certify inner work as +part of an outer report. Use a single borrowed scope throughout the adapter. + +URI operations in JSONSchema use the core domain: URI identity and adapter models +are not compiled schema plans. They do not charge compiledPlanBytes or +compileWorkUnits. This facade does not compose schema compilation or evaluation. +No circular JSON-to-JSONSchema dependency is introduced. + +`JSONOperationReport` has a public initializer. Its type is not a cryptographic or +unforgeable receipt, and it cannot establish that an arbitrary callback charged +all its work. An adopting adapter must own its final accounting boundary and audit +all external work and reservations. Independent reports cannot retroactively +certify one enforced invocation. + + +## Compiler and portability boundary + +Pass the borrowed scope explicitly to adapter helper functions: + +```swift +func visit(_ scope: borrowing JSONCompositionScope) throws { + try scope.chargeExternalWork(1) +} +``` + +Apple Swift 6.4.0.34.1 crashed during SIL generation for a nested local function +that captured the borrowed scope. An explicit borrowed helper parameter compiles +without changing the public ownership model. Avoid that compiler pattern until +it is fixed. Separate expected-failing SwiftPM builds qualify the prohibited +copy, escape, async, Sendable, construction, replacement and ledger-access forms. +See the source qualification package for the exact +executed scope; platform build results are evidence for their observed SDK only. diff --git a/Sources/JSON/Documentation.docc/JSON.md b/Sources/JSON/Documentation.docc/JSON.md index 8197bd6..4539c16 100644 --- a/Sources/JSON/Documentation.docc/JSON.md +++ b/Sources/JSON/Documentation.docc/JSON.md @@ -57,6 +57,7 @@ belongs exclusively to each invocation. - - - +- ### Exact model - ``JSON/JSON`` @@ -88,6 +89,7 @@ belongs exclusively to each invocation. - ``JSONResourceMetric`` - ``JSONOperationStatistics`` - ``JSONOperationReport`` +- ``JSONCompositionScope`` ### Scoped reads diff --git a/Sources/JSON/Limits/JSONCompositionScope.swift b/Sources/JSON/Limits/JSONCompositionScope.swift new file mode 100644 index 0000000..b194cee --- /dev/null +++ b/Sources/JSON/Limits/JSONCompositionScope.swift @@ -0,0 +1,196 @@ +// SPDX-License-Identifier: Apache-2.0 +import Foundation + +private final class JSONCompositionState { + var context: JSONOperationContext + init(limits: JSONLimits) throws { + context = try JSONOperationContext(limits: limits, domain: .core, trackAcceptedCapacities: true) + } +} + +/// One synchronous, noncopyable composition of native and explicitly instrumented work. +/// +/// Create a scope with ``JSON/withComposition(limits:_:)``. Native results and +/// external reservations remain logically owned until completion, even if a local +/// value dies. There is no public refund or continuation. This conservative model +/// covers aliases and final output overlap, not arbitrary callback instructions or RSS. +public struct JSONCompositionScope: ~Copyable { + private let state: JSONCompositionState + + private init(limits: JSONLimits) throws { + state = try JSONCompositionState(limits: limits) + } + + /// Executes a native helper on the sole ledger; all owned errors are terminal. + package func withMeter(domain: JSONOperationDomain = .core, + _ body: (inout JSONWorkMeter) throws -> T) throws -> T { + do { + try state.context.meter.operationDomain(domain) + let result = try body(&state.context.meter) + try state.context.meter.operationDomain(.core) + return result + } catch { + state.context.meter.recordFailure(error) + throw error + } + } + + /// Parses strict Data, retaining the exact root, source copy and source map. + /// + /// The whole byte-buffer reservation precedes allocation. The input buffer is + /// conservatively retained in the ledger after parsing; caller Data is excluded. + /// - Parameter data: Caller-owned UTF-8 input, never fetched or repaired. + /// - Returns: The exact source-backed document. + /// - Throws: Original native failure or cancellation; caught failures stay terminal. + public func parseDocument(_ data: Data) throws -> JSONDocument { + try withMeter(domain: .parsing) { meter in + try meter.beginCompositionDocument() + try meter.require(.inputBytes, UInt64(data.count)) + try meter.retain(UInt64(data.count)) + var bytes: [UInt8] = [] + bytes.reserveCapacity(data.count) + for byte in data { try meter.work(); bytes.append(byte) } + return try JSONParser().parseDocument(bytes, source: nil, meter: &meter) + } + } + + /// Admits and reserves an existing exact graph without conversion. + /// - Parameter value: Trusted caller value; its prior construction is caller work. + /// - Returns: The admitted value. Each call reserves a complete logical graph. + /// - Throws: Native admission failure or cancellation. + public func admitTrusted(_ value: JSON) throws -> JSON { + try withMeter { meter in + try JSONAdmission.value(value, meter: &meter, owned: true) + return value + } + } + + /// Constructs an exact Pointer using this invocation's remaining budget. + /// - Parameter tokens: Caller-owned decoded tokens. + /// - Returns: The admitted Pointer. + /// - Throws: Original native Pointer/admission failure or cancellation. + public func pointer(tokens: [String]) throws -> JSONPointer { + try withMeter(domain: .pointer) { try JSONPointer.fromStrings(tokens, meter: &$0) } + } + + /// Parses RFC 6901 text using the same ledger. + /// - Parameter text: Encoded Pointer text; empty selects the root. + /// - Returns: The admitted Pointer. + /// - Throws: Original native Pointer/admission failure or cancellation. + public func pointer(parsing text: String) throws -> JSONPointer { + try withMeter(domain: .pointer) { try JSONPointer.parse(text, meter: &$0) } + } + + /// Decodes a URI fragment exactly once, then parses its Pointer escapes. + /// - Parameter fragment: Encoded fragment including its leading #. + /// - Returns: The admitted Pointer. + /// - Throws: Original native Pointer/admission failure or cancellation. + public func pointer(uriFragment fragment: String) throws -> JSONPointer { + try withMeter(domain: .pointer) { try JSONPointer.uriFragment(fragment, meter: &$0) } + } + + /// Appends an exact decoded key, reserving the complete resulting Pointer. + /// - Parameters: + /// - pointer: A caller or scope-created Pointer. + /// - key: The decoded token, also usable as a strict array index spelling. + /// - Returns: The admitted appended Pointer. + /// - Throws: Original native Pointer/admission failure or cancellation. + public func appending(_ pointer: JSONPointer, key: String) throws -> JSONPointer { + try withMeter(domain: .pointer) { try pointer.appending(key: key, meter: &$0) } + } + + /// Resolves exact keys and strict array indices after admitting both inputs. + /// + /// Each input graph and selected output graph is conservatively reserved anew; + /// aliases never refund prior ownership. Native typed errors are preserved. + /// - Parameters: + /// - pointer: The exact path to resolve. + /// - value: A caller or scope-created root. + /// - Returns: The selected exact value. + /// - Throws: Original native Pointer/admission failure or cancellation. + public func resolve(_ pointer: JSONPointer, in value: JSON) throws -> JSON { + try withMeter(domain: .pointer) { meter in + try JSONAdmission.value(value, meter: &meter, owned: true) + let admitted = try JSONPointer.decodedTokens(pointer.tokens, meter: &meter) + let result = try admitted.resolve(in: value, meter: &meter) + try JSONAdmission.value(result, meter: &meter, owned: true) + return result + } + } + + /// Charges instrumented adapter work once to aggregate core work. + /// - Parameter units: Declared primitive work admitted before performing it. + /// - Throws: Quota/overflow failure or cancellation, permanently terminal. + public func chargeExternalWork(_ units: UInt64) throws { + try withMeter { try $0.work(units) } + } + + /// Reserves adapter-owned logical bytes until scope completion, without refund. + /// + /// Call before growth. Separate calls count separately even for physical aliases. + /// Units are adapter-declared logical bytes, not measured heap allocation. This + /// never charges compiledPlanBytes. Zero does not certify opaque storage as free. + /// - Parameters: + /// - bytes: Logical weight per unit. + /// - count: Number of units, multiplied with checked UInt64 arithmetic. + /// - overhead: Additional logical header or bookkeeping bytes, added before growth. + /// - Throws: Quota/overflow failure or cancellation, permanently terminal. + public func reserveExternalLogicalBytes(_ bytes: UInt64, count: UInt64 = 1, overhead: UInt64 = 0) throws { + try withMeter { meter in + let (total, overflow) = bytes.multipliedReportingOverflow(by: count) + guard !overflow else { + throw JSONFailure(.resource, code: .counterOverflow, resource: .init( + domain: .core, metric: .retainedBytes, limit: meter.limits[.retainedBytes], + attemptedUsage: nil, overflow: true)) + } + let complete = try meter.adding(.retainedBytes, total, overhead) + try meter.retain(complete) + } + } + + /// Checks cancellation and prior terminal failure before adapter work. + /// - Throws: The original terminal failure or cancellation. + public func checkpoint() throws { try withMeter { try $0.checkpoint() } } + + /// Tightens finite limits without resetting work, live reservations or admissions. + /// - Parameter limits: Requested ceilings, intersected with existing ceilings. + /// - Throws: Invalid configuration, prior admission/work violation or cancellation. + public func restrict(to limits: JSONLimits) throws { + try withMeter { try $0.restrict(to: limits) } + } + + fileprivate static func run(limits: JSONLimits, + _ body: (borrowing Self) throws -> T) throws -> JSONOperationReport { + let scope = try Self(limits: limits) + let original = scope.state.context.meter.continuation + do { + let result = try body(scope) + try original.check(scope.state.context.meter) + let statistics = try scope.withMeter { try $0.compositionStatistics() } + try original.check(scope.state.context.meter) + return .init(result: result, statistics: statistics) + } catch { + // A caller error cannot conceal an earlier native terminal failure or substitution. + try original.check(scope.state.context.meter) + throw error + } + } +} + +extension JSON { + /// Composes native helpers and instrumented adapter work on one finite ledger. + /// + /// The nonescaping synchronous scope is borrowed, noncopyable and not Sendable. Results + /// may own exact immutable values. Reserve adapter output before allocation; + /// arbitrary callback instructions remain caller work. Reports are ordinary + /// values with a public initializer, not unforgeable certification tokens. + /// - Parameters: + /// - limits: Finite inclusive limits for the entire invocation. + /// - body: Synchronous work borrowing the invocation's sole scope. + /// - Returns: The result and one cumulative successful report, including final overlap. + /// - Throws: Original owned terminal failure, cancellation or caller error; no report on failure. + public static func withComposition(limits: JSONLimits = .balanced, + _ body: (borrowing JSONCompositionScope) throws -> Result) throws -> JSONOperationReport { + try JSONCompositionScope.run(limits: limits, body) + } +} diff --git a/Sources/JSON/Limits/JSONOperationContext.swift b/Sources/JSON/Limits/JSONOperationContext.swift index d06ab98..7a16f9c 100644 --- a/Sources/JSON/Limits/JSONOperationContext.swift +++ b/Sources/JSON/Limits/JSONOperationContext.swift @@ -2,7 +2,7 @@ /// All stages of a composed operation borrow this single invocation-owned context. package struct JSONOperationContext: ~Copyable { package var meter: JSONWorkMeter - package init(limits: JSONLimits, domain: JSONOperationDomain) throws { - meter = try JSONWorkMeter(limits: limits, domain: domain) + package init(limits: JSONLimits, domain: JSONOperationDomain, trackAcceptedCapacities: Bool = false) throws { + meter = try JSONWorkMeter(limits: limits, domain: domain, trackAcceptedCapacities: trackAcceptedCapacities) } } diff --git a/Sources/JSON/Limits/JSONWorkMeter.swift b/Sources/JSON/Limits/JSONWorkMeter.swift index 0f5a1f2..73b19cd 100644 --- a/Sources/JSON/Limits/JSONWorkMeter.swift +++ b/Sources/JSON/Limits/JSONWorkMeter.swift @@ -206,6 +206,35 @@ package struct JSONWorkMeter: ~Copyable { guard bytes <= used(.annotationBytes) else { throw JSONFailure(.configuration, code: .invalidLimits) } counters[JSONResourceMetric.annotationBytes.index] -= bytes } + /// Input is a per-document capacity. Retire its charged count into the same + /// cumulative report; never reset work, retention, accepted history or health. + package mutating func beginCompositionDocument() throws { + try checkpoint() + try retire(.inputBytes) + } + /// Composition-only completion owns a separate 49-slot report through handoff. + /// Existing report entry points retain their original accounting semantics. + package mutating func compositionStatistics() throws -> JSONOperationStatistics { + // Version 1 composition receipt: 32 logical header bytes + 49 UInt64 slots. + try retain(424) + var report: [UInt64] = [] + report.reserveCapacity(JSONResourceMetric.allCases.count) + for metric in JSONResourceMetric.allCases { + try work() + let (total, overflow) = counters[metric.index].addingReportingOverflow(retired[metric.index]) + guard !overflow else { throw violation(metric, nil, overflow: true) } + report.append(total) + } + // The aggregate-work slot may precede the last copied slot. Charge and + // publish the final count only after every copied counter is accounted. + try work() + let (work, overflow) = counters[JSONResourceMetric.workUnits.index] + .addingReportingOverflow(retired[JSONResourceMetric.workUnits.index]) + guard !overflow else { throw violation(.workUnits, nil, overflow: true) } + report[JSONResourceMetric.workUnits.index] = work + try checkpoint() + return .init(domain: domain, counters: report, peak: peak) + } package func statistics() -> JSONOperationStatistics { var report = counters for metric in JSONResourceMetric.allCases { report[metric.index] += retired[metric.index] } diff --git a/Sources/JSON/Pointer/JSONPointer.swift b/Sources/JSON/Pointer/JSONPointer.swift index 7b1c2a3..e1d3b37 100644 --- a/Sources/JSON/Pointer/JSONPointer.swift +++ b/Sources/JSON/Pointer/JSONPointer.swift @@ -27,6 +27,9 @@ public struct JSONPointer: Sendable, Hashable { /// - limits: Finite inclusive capacities and work quotas for this call. public init(tokens: [String], limits: JSONLimits = .balanced) throws { var meter = try JSONWorkMeter(limits: limits, domain: .pointer) + self = try Self.fromStrings(tokens, meter: &meter) + } + package static func fromStrings(_ tokens: [String], meter: inout JSONWorkMeter) throws -> Self { var keys: [JSONKey] = [] var encodedBytes: UInt64 = 0 for token in tokens { @@ -35,7 +38,7 @@ public struct JSONPointer: Sendable, Hashable { for byte in token.utf8 { try meter.work(); encodedBytes = try meter.adding(.pointerBytes, encodedBytes, byte == 47 || byte == 126 ? 2 : 1) } try meter.retain(16); keys.append(try JSONKey(token, meter: &meter)) } - self = try Self.decodedTokens(keys, meter: &meter) + return try Self.decodedTokens(keys, meter: &meter) } package static func decodedTokens(_ tokens: S, meter: inout JSONWorkMeter) throws -> Self where S.Element == JSONKey { var keys: [JSONKey] = [], bytes: UInt64 = 0 @@ -152,6 +155,9 @@ public struct JSONPointer: Sendable, Hashable { /// - limits: Finite inclusive capacities and work quotas for this call. public func appending(key: String, limits: JSONLimits = .balanced) throws -> Self { var meter = try JSONWorkMeter(limits: limits, domain: .pointer) + return try appending(key: key, meter: &meter) + } + package func appending(key: String, meter: inout JSONWorkMeter) throws -> Self { try meter.require(.pointerTokens, UInt64(tokens.count) + 1) var encoded: UInt64 = 1 for byte in key.utf8 { try meter.work(); encoded = try meter.adding(.pointerBytes, encoded, byte == 47 || byte == 126 ? 2 : 1) } diff --git a/Sources/JSONCLIEngine/CLIBuildIdentity.swift b/Sources/JSONCLIEngine/CLIBuildIdentity.swift index 9641f33..9bff73e 100644 --- a/Sources/JSONCLIEngine/CLIBuildIdentity.swift +++ b/Sources/JSONCLIEngine/CLIBuildIdentity.swift @@ -2,6 +2,6 @@ // Generated by Scripts/generate_cli_identity.py; excludes this file. // SHA-256 of the ordered Package.swift + Sources content manifest, not a Git commit. package enum CLIBuildIdentity { - package static let revision = "sha256:b3147f90fc72f25bd4507ddb859ed94f9f1dde3ea7b365d8a15212532b802ff4" + package static let revision = "sha256:cc8ae93aaf5e28f15bd13224949920c4b8881f3892674664405d46a5f730d8b1" package static let kind = "source-sha256" } diff --git a/Sources/JSONSchema/Document/JSONCompositionScope+URI.swift b/Sources/JSONSchema/Document/JSONCompositionScope+URI.swift new file mode 100644 index 0000000..60a44ae --- /dev/null +++ b/Sources/JSONSchema/Document/JSONCompositionScope+URI.swift @@ -0,0 +1,38 @@ +// SPDX-License-Identifier: Apache-2.0 +import JSON + +extension JSONCompositionScope { + /// Normalizes an offline URI using this composition's core ledger. + /// + /// URI identity is not a compiled plan: compiledPlanBytes and compileWorkUnits + /// remain untouched. A supplied base is admitted and retained conservatively. + /// - Parameters: + /// - text: Absolute URI or relative reference. + /// - base: Explicit normalized base for a relative reference; never fetched. + /// - Returns: Native normalized URI identity. + /// - Throws: Native identifier/resource failure or cancellation, permanently terminal. + public func uri(_ text: String, relativeTo base: JSONSchemaURI? = nil) throws -> JSONSchemaURI { + try withMeter { meter in + if let base { try JSONAdmission.text(base.string, meter: &meter, owned: true) } + return try JSONSchemaURI.parse(text, relativeTo: base, meter: &meter) + } + } + + /// Returns the native resource identity without its fragment on the same ledger. + /// - Parameter uri: Caller or scope-created identity, re-admitted before use. + /// - Returns: Normalized URI without its fragment. + /// - Throws: Native identifier/resource failure or cancellation, permanently terminal. + public func removingFragment(_ uri: JSONSchemaURI) throws -> JSONSchemaURI { + try withMeter { meter in + let admitted = try JSONSchemaURI.parse(uri.string, relativeTo: nil, meter: &meter) + // Avoid the ordinary unmetered fragment convenience (including its + // substring allocation). Charge and checkpoint this discovery scan. + var fragmentPresent = false + for byte in admitted.string.utf8 { + try meter.work() + if byte == 35 { fragmentPresent = true; break } + } + return try admitted.withoutFragment(meter: &meter, fragmentPresent: fragmentPresent) + } + } +} diff --git a/Sources/JSONSchema/Document/JSONSchemaURI.swift b/Sources/JSONSchema/Document/JSONSchemaURI.swift index ca3cb14..b833dd1 100644 --- a/Sources/JSONSchema/Document/JSONSchemaURI.swift +++ b/Sources/JSONSchema/Document/JSONSchemaURI.swift @@ -61,8 +61,8 @@ public struct JSONSchemaURI: Sendable, Hashable { public func removingFragment(limits: JSONLimits = .balanced) throws -> Self { try Self(string.split(separator: "#", maxSplits: 1, omittingEmptySubsequences: false).first.map(String.init) ?? string, limits: limits) } - package func withoutFragment(meter: inout JSONWorkMeter) throws -> Self { - if fragment == nil { try meter.work(); return self } + package func withoutFragment(meter: inout JSONWorkMeter, fragmentPresent: Bool? = nil) throws -> Self { + if fragmentPresent == false || (fragmentPresent == nil && fragment == nil) { try meter.work(); return self } var bytes: [UInt8] = [] for byte in string.utf8 { try meter.work(); if byte == 35 { break }; try meter.retain(1); bytes.append(byte) diff --git a/Tests/JSONIntegrationTests/CompositionOwnedProofTests.swift b/Tests/JSONIntegrationTests/CompositionOwnedProofTests.swift new file mode 100644 index 0000000..176f348 --- /dev/null +++ b/Tests/JSONIntegrationTests/CompositionOwnedProofTests.swift @@ -0,0 +1,72 @@ +// SPDX-License-Identifier: Apache-2.0 +import Foundation +import Synchronization +import Testing +import JSON +import JSONSchema + +// Native-only rendezvous is supplementary to the unrelated public package. It +// uses the existing invocation-owned observer; it is not an accounting oracle. +@Test func jcompOwnedCancellationOccursInsideNativeWork() async throws { + for stage in 0..<7 { + let seen = Mutex(nil) + let task = Task.detached { + var limits = JSONLimits.offline; limits[.cancellationQuantum] = 1 + let threshold: UInt64 = stage == 1 ? 4200 : stage == 4 ? 6 : 32 + return try JSON.withComposition(limits: limits) { scope in + try scope.withMeter { meter in + meter.checkpointObserver = { work in + if work >= threshold { + seen.withLock { if $0 == nil { $0 = work } } + withUnsafeCurrentTask { $0?.cancel() } + } + } + } + do { + switch stage { + case 0, 1: + _ = try scope.parseDocument(Data(("\"" + String(repeating: "a", count: 4096) + "\"").utf8)) + case 2: _ = try scope.pointer(parsing: "/" + String(repeating: "a", count: 4096)) + case 3: _ = try scope.uri("https://example.invalid/" + String(repeating: "a", count: 4096)) + case 4: try scope.chargeExternalWork(5) // Cancellation must occur in report copying. + case 5: + for _ in 0..<128 { try scope.chargeExternalWork(1) } + default: + let path = try scope.pointer(tokens: []) + _ = try scope.resolve(path, in: .array(Array(repeating: .null, count: 256))) + } + } catch is CancellationError { } + return 1 + } + } + do { _ = try await task.value; Issue.record("caught cancellation completed at stage \(stage)") } + catch is CancellationError { } + #expect(seen.withLock { $0 != nil }) + if stage == 1 { #expect(seen.withLock { ($0 ?? 0) > 4098 }) } + } +} + +@Test func jcompOriginalHealthAndIdentitySurvivePackageSubstitution() throws { + for stage in 0..<3 { + do { + _ = try JSON.withComposition { scope in + try scope.withMeter { meter in + if stage == 1 { + do { try meter.work(UInt64.max) } catch { } + } else if stage == 2 { + do { try meter.retain(UInt64.max) } catch { } + } + // Impossible through public API; exercise the original + // continuation's identity and first-cause check directly. + meter = try JSONWorkMeter(limits: .balanced, domain: .core) + } + return 1 + } + Issue.record("substituted native ledger completed") + } catch let failure as JSONFailure { + if stage == 0 { #expect(failure.category == .configuration && failure.code == .invalidOperation) } + if stage == 1 { #expect(failure.resource?.metric == .workUnits) } + if stage == 2 { #expect(failure.resource?.metric == .retainedBytes) } + } + } +} diff --git a/Tools/CompositionQualification/Negative/Async.swift.txt b/Tools/CompositionQualification/Negative/Async.swift.txt new file mode 100644 index 0000000..10a764e --- /dev/null +++ b/Tools/CompositionQualification/Negative/Async.swift.txt @@ -0,0 +1,6 @@ +// SPDX-License-Identifier: Apache-2.0 +import JSON +import JSONSchema +func illegal() async throws { + _ = try await JSON.withComposition { scope in await Task.yield(); try scope.checkpoint() } +} diff --git a/Tools/CompositionQualification/Negative/Construct.swift.txt b/Tools/CompositionQualification/Negative/Construct.swift.txt new file mode 100644 index 0000000..36ec7cf --- /dev/null +++ b/Tools/CompositionQualification/Negative/Construct.swift.txt @@ -0,0 +1,4 @@ +// SPDX-License-Identifier: Apache-2.0 +import JSON +import JSONSchema +let scope = try JSONCompositionScope(limits: .balanced) diff --git a/Tools/CompositionQualification/Negative/Copy.swift.txt b/Tools/CompositionQualification/Negative/Copy.swift.txt new file mode 100644 index 0000000..45f818f --- /dev/null +++ b/Tools/CompositionQualification/Negative/Copy.swift.txt @@ -0,0 +1,4 @@ +// SPDX-License-Identifier: Apache-2.0 +import JSON +import JSONSchema +_ = try JSON.withComposition { scope in let duplicate = copy scope; try duplicate.checkpoint() } diff --git a/Tools/CompositionQualification/Negative/Escape.swift.txt b/Tools/CompositionQualification/Negative/Escape.swift.txt new file mode 100644 index 0000000..d51543f --- /dev/null +++ b/Tools/CompositionQualification/Negative/Escape.swift.txt @@ -0,0 +1,5 @@ +// SPDX-License-Identifier: Apache-2.0 +import JSON +import JSONSchema +var escaped: (() throws -> Void)? +_ = try JSON.withComposition { scope in escaped = { try scope.checkpoint() } } diff --git a/Tools/CompositionQualification/Negative/Fork.swift.txt b/Tools/CompositionQualification/Negative/Fork.swift.txt new file mode 100644 index 0000000..ecb8efe --- /dev/null +++ b/Tools/CompositionQualification/Negative/Fork.swift.txt @@ -0,0 +1,4 @@ +// SPDX-License-Identifier: Apache-2.0 +import JSON +import JSONSchema +_ = try JSON.withComposition { scope in try scope.fork() } diff --git a/Tools/CompositionQualification/Negative/PackageMeter.swift.txt b/Tools/CompositionQualification/Negative/PackageMeter.swift.txt new file mode 100644 index 0000000..fa89bd9 --- /dev/null +++ b/Tools/CompositionQualification/Negative/PackageMeter.swift.txt @@ -0,0 +1,4 @@ +// SPDX-License-Identifier: Apache-2.0 +import JSON +import JSONSchema +_ = try JSON.withComposition { scope in try scope.withMeter { _ in 0 } } diff --git a/Tools/CompositionQualification/Negative/RawMeter.swift.txt b/Tools/CompositionQualification/Negative/RawMeter.swift.txt new file mode 100644 index 0000000..75b2279 --- /dev/null +++ b/Tools/CompositionQualification/Negative/RawMeter.swift.txt @@ -0,0 +1,4 @@ +// SPDX-License-Identifier: Apache-2.0 +import JSON +import JSONSchema +let meter = try JSONWorkMeter(limits: .balanced, domain: .core) diff --git a/Tools/CompositionQualification/Negative/Refund.swift.txt b/Tools/CompositionQualification/Negative/Refund.swift.txt new file mode 100644 index 0000000..b940bf7 --- /dev/null +++ b/Tools/CompositionQualification/Negative/Refund.swift.txt @@ -0,0 +1,4 @@ +// SPDX-License-Identifier: Apache-2.0 +import JSON +import JSONSchema +_ = try JSON.withComposition { scope in try scope.release(1) } diff --git a/Tools/CompositionQualification/Negative/Reset.swift.txt b/Tools/CompositionQualification/Negative/Reset.swift.txt new file mode 100644 index 0000000..c50d1c7 --- /dev/null +++ b/Tools/CompositionQualification/Negative/Reset.swift.txt @@ -0,0 +1,4 @@ +// SPDX-License-Identifier: Apache-2.0 +import JSON +import JSONSchema +_ = try JSON.withComposition { scope in try scope.reset() } diff --git a/Tools/CompositionQualification/Negative/Return.swift.txt b/Tools/CompositionQualification/Negative/Return.swift.txt new file mode 100644 index 0000000..ad13f6a --- /dev/null +++ b/Tools/CompositionQualification/Negative/Return.swift.txt @@ -0,0 +1,4 @@ +// SPDX-License-Identifier: Apache-2.0 +import JSON +import JSONSchema +let scope = try JSON.withComposition { scope in scope } diff --git a/Tools/CompositionQualification/Negative/Sendable.swift.txt b/Tools/CompositionQualification/Negative/Sendable.swift.txt new file mode 100644 index 0000000..be9a833 --- /dev/null +++ b/Tools/CompositionQualification/Negative/Sendable.swift.txt @@ -0,0 +1,5 @@ +// SPDX-License-Identifier: Apache-2.0 +import JSON +import JSONSchema +func needsSendable(_ value: borrowing T) {} +_ = try JSON.withComposition { scope in needsSendable(scope) } diff --git a/Tools/CompositionQualification/Negative/Store.swift.txt b/Tools/CompositionQualification/Negative/Store.swift.txt new file mode 100644 index 0000000..8e2d0c8 --- /dev/null +++ b/Tools/CompositionQualification/Negative/Store.swift.txt @@ -0,0 +1,5 @@ +// SPDX-License-Identifier: Apache-2.0 +import JSON +import JSONSchema +var saved: JSONCompositionScope? +_ = try JSON.withComposition { scope in saved = scope } diff --git a/Tools/CompositionQualification/Negative/Substitute.swift.txt b/Tools/CompositionQualification/Negative/Substitute.swift.txt new file mode 100644 index 0000000..a679266 --- /dev/null +++ b/Tools/CompositionQualification/Negative/Substitute.swift.txt @@ -0,0 +1,6 @@ +// SPDX-License-Identifier: Apache-2.0 +import JSON +import JSONSchema +_ = try JSON.withComposition { outer in + _ = try JSON.withComposition { inner in swap(&outer, &inner) } +} diff --git a/Tools/CompositionQualification/Package.swift b/Tools/CompositionQualification/Package.swift new file mode 100644 index 0000000..c5b9b61 --- /dev/null +++ b/Tools/CompositionQualification/Package.swift @@ -0,0 +1,20 @@ +// swift-tools-version: 6.4 +// SPDX-License-Identifier: Apache-2.0 +import PackageDescription + +let package = Package( + name: "IndependentCompositionQualification", + platforms: [.macOS("27.0"), .iOS("27.0")], + dependencies: [.package(name: "Swift-JSON", path: "../..")], + targets: [.executableTarget(name: "CompositionBenchmark", dependencies: [ + .product(name: "JSON", package: "Swift-JSON"), + .product(name: "JSONSchema", package: "Swift-JSON") + ]), .target(name: "CompositionProbe", dependencies: [ + .product(name: "JSON", package: "Swift-JSON"), + .product(name: "JSONSchema", package: "Swift-JSON") + ]), .testTarget(name: "CompositionPublicTests", dependencies: [ + .product(name: "JSON", package: "Swift-JSON"), + .product(name: "JSONSchema", package: "Swift-JSON") + ])], + swiftLanguageModes: [.v6] +) diff --git a/Tools/CompositionQualification/README.md b/Tools/CompositionQualification/README.md new file mode 100644 index 0000000..2a31a0b --- /dev/null +++ b/Tools/CompositionQualification/README.md @@ -0,0 +1,27 @@ +# Independent public composition consumer + +Swift 6.4 / language 6; imports only normal public JSON and JSONSchema products. +The root package dependency graph stays empty. Tests do not use @testable, SPI, +private meter shims or package-name impersonation. The dependency's explicit +SwiftPM display name is unrelated to package-access identity. + +Run from the repository root: + +```sh +python3 Scripts/ci.py --lane composition +``` + +This executes 24 Swift Testing functions in Debug and Release, checks exact raw +Swift Testing events and the golden owned-event TSV, runs the same positive test +with two missing-charge mutations expecting real assertion failures, then builds +13 unrelated negative consumer packages. A wrong import/dependency or compiler +crash is not an accepted negative outcome. Logs preserve diagnostics and hashes. + +`CompositionProbe` is a compile-only ordinary public consumer suitable for Apple +SDK and Linux cross-build probes. `Negative/*.swift.txt` are fixtures compiled as +separate ordinary SwiftPM executable packages, not included in passing targets. + +Apple Swift 6.4.0.34.1 has a SIL-generation crash for a nested local function +capturing the borrowed scope. Explicit borrowed helper parameters compile; the +positive consumer uses that shape. No unsafe annotations or compiler flags are +needed. Platform receipts and the captured crash are reported with the handoff. diff --git a/Tools/CompositionQualification/Sources/CompositionBenchmark/main.swift b/Tools/CompositionQualification/Sources/CompositionBenchmark/main.swift new file mode 100644 index 0000000..1241054 --- /dev/null +++ b/Tools/CompositionQualification/Sources/CompositionBenchmark/main.swift @@ -0,0 +1,52 @@ +// SPDX-License-Identifier: Apache-2.0 +import Foundation +import JSON +import JSONSchema + +struct Sample: Codable { + let name: String + let inputBytes: Int + let iterations: Int + let seconds: [Double] + let work: UInt64? + let peak: UInt64? +} +// Deterministic 512-field exact/scalar workload, not an external conformance corpus. +let text = "{" + (0..<512).map { "\"field\($0)\":" + #"{"n":9007199254740993,"s":"é","a":[true,null,0.01]}"# }.joined(separator: ",") + "}" +let bytes = Array(text.utf8), data = Data(text.utf8) +let root = try JSONParser().parse(bytes) +let schema = try JSONSchemaDocument(JSONParser().parse(#"{"type":"object","properties":{"n":{"type":"integer"},"s":{"type":"string"}},"required":["n"]}"#)) +let parserProbe = try JSONParser().parseWithReport(bytes) +let schemaProbe = try JSONSchemaCompiler().compileWithReport(schema) +let clock = ContinuousClock() +func seconds(_ duration: Duration) -> Double { Double(duration.components.seconds) + Double(duration.components.attoseconds) / 1e18 } +func measure(_ name: String, iterations: Int, work: UInt64? = nil, peak: UInt64? = nil, _ operation: () throws -> Void) throws -> Sample { + for _ in 0..<2 { for _ in 0.. JSONOperationReport { + try JSON.withComposition { scope in + let document = try scope.parseDocument(data) + try adapterStep(scope) + let path = try scope.pointer(tokens: ["count"]) + let uri = try scope.uri("https://example.invalid/schema.json#/$defs/count") + _ = try scope.removingFragment(uri) + return try scope.resolve(path, in: document.root) + } +} + +private func adapterStep(_ scope: borrowing JSONCompositionScope) throws { + try scope.reserveExternalLogicalBytes(1, count: 32, overhead: 32) + try scope.chargeExternalWork(1) +} diff --git a/Tools/CompositionQualification/Tests/CompositionPublicTests/CompositionTests.swift b/Tools/CompositionQualification/Tests/CompositionPublicTests/CompositionTests.swift new file mode 100644 index 0000000..3f6569d --- /dev/null +++ b/Tools/CompositionQualification/Tests/CompositionPublicTests/CompositionTests.swift @@ -0,0 +1,340 @@ +// SPDX-License-Identifier: Apache-2.0 +import Foundation +import Testing +import JSON +import JSONSchema + +private enum AuditError: Error { case workTrace, retentionTrace, caller } +private let source = Data(##"{"n":9007199254740993,"adjacent":9007199254740992,"long":123456789012345678901234567890.01234567890123456789,"fraction":0.01,"negative":-0,"é":true,"e\u0301":false,"a/b":{"~":null},"items":[false],"type":["string","null"],"$dynamicRef":"#node","$schema":"https://unknown.invalid/dialect"}"##.utf8) + +// Independent audited-adapter ledger, not derived from operation statistics. +// Three bounded buffer fills: one frame visit and one unit per written byte; +// 32 logical header bytes per buffer. Final native receipt copies 49 slots and +// settles its aggregate slot (50 work), retaining 32 + 49*8 logical bytes. +private let goldenWork: UInt64 = 197 +private let goldenBytes: UInt64 = 664 +private func fill(_ scope: borrowing JSONCompositionScope, count: Int, first: Bool, mutation: String?) throws -> [UInt8] { + let reservation = count - (first && mutation == "reservation" ? 1 : 0) + try scope.reserveExternalLogicalBytes(1, count: UInt64(reservation), overhead: 32) + try scope.chargeExternalWork(1) + var bytes: [UInt8] = [] + bytes.reserveCapacity(count) + for i in 0.. JSONOperationReport<([UInt8], [UInt8], [UInt8])> { + let mutation = ProcessInfo.processInfo.environment["JCOMP_TRACE_MUTATION"] + return try JSON.withComposition(limits: limits) { scope in + let queue = try fill(scope, count: 64, first: true, mutation: mutation) + let scratch = try fill(scope, count: 32, first: false, mutation: mutation) + let output = try fill(scope, count: 48, first: false, mutation: mutation) + try scope.chargeExternalWork(extra) + return (queue, scratch, output) + } +} +private func audit(_ report: JSONOperationReport<([UInt8], [UInt8], [UInt8])>) throws { + guard report.statistics[.workUnits] == goldenWork else { throw AuditError.workTrace } + guard report.statistics[.retainedBytes] == goldenBytes, + report.statistics.peakLogicalRetainedBytes == goldenBytes else { throw AuditError.retentionTrace } +} +private func capture(_ body: () throws -> Void) -> (any Error)? { + do { try body(); Issue.record("expected an error"); return nil } catch { return error } +} + +@Test func externalTraceMatchesIndependentLedger() throws { + let report = try traceAdapter() + #expect(report.result.0.count == 64 && report.result.1.count == 32 && report.result.2.count == 48) + try audit(report) + if let directory = ProcessInfo.processInfo.environment["JCOMP_TRACE_DIRECTORY"] { + let trace = "owner\twork\tlogical_bytes\nqueue\t65\t96\nscratch\t33\t64\nresult\t49\t80\nreceipt\t50\t424\n" + try trace.write(toFile: directory + "/owned-events.tsv", atomically: true, encoding: .utf8) + } +} + +@Test func exactWorkAndRetentionLimitsAreActive() throws { + var exact = JSONLimits.balanced + exact[.workUnits] = goldenWork; exact[.retainedBytes] = goldenBytes + try audit(traceAdapter(exact)) + for metric in [JSONResourceMetric.workUnits, .retainedBytes] { + var below = exact; below[metric] -= 1 + let failure = capture { _ = try traceAdapter(below) } as? JSONFailure + #expect(failure?.category == .resource && failure?.resource?.metric == metric) + } + #expect(throws: JSONFailure.self) { _ = try traceAdapter(exact, extra: 1) } +} + +@Test func reportOverlapIsAdmittedByFinalizer() throws { + var limits = JSONLimits.balanced + limits[.retainedBytes] = 424; limits[.workUnits] = 50 + let report = try JSON.withComposition(limits: limits) { _ in 7 } + #expect(report.result == 7 && report.statistics[.retainedBytes] == 424) + #expect(report.statistics[.workUnits] == 50 && report.statistics.peakLogicalRetainedBytes == 424) + limits[.retainedBytes] = 423 + #expect(throws: JSONFailure.self) { try JSON.withComposition(limits: limits) { _ in 7 } } + limits[.retainedBytes] = 424; limits[.workUnits] = 49 + #expect(throws: JSONFailure.self) { try JSON.withComposition(limits: limits) { _ in 7 } } +} + +@Test func finiteMaximumWorkDoesNotOverflowCompletion() throws { + var limits = JSONLimits.balanced; limits[.workUnits] = UInt64.max - 1 + let report = try JSON.withComposition(limits: limits) { scope in try scope.chargeExternalWork(UInt64.max - 51) } + #expect(report.statistics[.workUnits] == UInt64.max - 1) + #expect(throws: JSONFailure.self) { + try JSON.withComposition(limits: limits) { scope in try scope.chargeExternalWork(UInt64.max - 50) } + } +} + +@Test func checkedProductAndOverheadFailBeforeGrowth() throws { + for mode in 0..<2 { + var growth = false + let error = capture { + _ = try JSON.withComposition { scope in + do { + if mode == 0 { try scope.reserveExternalLogicalBytes(UInt64.max, count: 2) } + else { try scope.reserveExternalLogicalBytes(1, overhead: UInt64.max) } + growth = true + } catch let failure as JSONFailure { #expect(failure.code == .counterOverflow) } + return 0 + } + } as? JSONFailure + #expect(!growth && error?.code == .counterOverflow) + } +} + +@Test func zeroReservationAndZeroQuotaHaveDistinctMeaning() throws { + let report = try JSON.withComposition { try $0.reserveExternalLogicalBytes(0, count: 0) } + #expect(report.statistics[.retainedBytes] == 424) + var none = JSONLimits.balanced; none[.workUnits] = 0 + #expect(throws: JSONFailure.self) { + try JSON.withComposition(limits: none) { scope in do { try scope.chargeExternalWork(1) } catch { } } + } +} + +@Test func firstQuotaFailureSurvivesCatchCleanupAndCallerError() throws { + var limits = JSONLimits.balanced; limits[.workUnits] = 1 + let error = capture { + _ = try JSON.withComposition(limits: limits) { scope in + do { try scope.chargeExternalWork(2) } catch { } + do { try scope.reserveExternalLogicalBytes(UInt64.max, count: 2) } catch { } + throw AuditError.caller + } + } as? JSONFailure + #expect(error?.code == .limitExceeded && error?.resource?.metric == .workUnits) + #expect(error?.resource?.attemptedUsage == 2) +} + +@Test func firstOverflowSurvivesLaterCalls() throws { + let error = capture { + _ = try JSON.withComposition { scope in + try scope.chargeExternalWork(1) + do { try scope.chargeExternalWork(UInt64.max) } catch { } + do { _ = try scope.parseDocument(Data("{".utf8)) } catch { } + return 7 + } + } as? JSONFailure + #expect(error?.code == .counterOverflow && error?.resource?.metric == .workUnits) +} + +@Test func freshInvocationHasFreshLedgerAfterFailure() throws { + #expect(throws: JSONFailure.self) { + try JSON.withComposition { try $0.reserveExternalLogicalBytes(UInt64.max, count: 2) } + } + try audit(traceAdapter()) +} + +@Test func preCopyCapacityFailurePrecedesAnyCopyWork() throws { + var limits = JSONLimits.balanced; limits[.workUnits] = 0 + limits[.inputBytes] = UInt64(source.count - 1) + let input = capture { _ = try JSON.withComposition(limits: limits) { try $0.parseDocument(source) } } as? JSONFailure + #expect(input?.resource?.metric == .inputBytes && input?.resource?.attemptedUsage == UInt64(source.count)) + limits[.inputBytes] = UInt64(source.count); limits[.retainedBytes] = UInt64(source.count - 1) + let bytes = capture { _ = try JSON.withComposition(limits: limits) { try $0.parseDocument(source) } } as? JSONFailure + #expect(bytes?.resource?.metric == .retainedBytes && bytes?.resource?.attemptedUsage == UInt64(source.count)) +} + +@Test func sourceLexemesScalarKeysAndSpansArePreserved() throws { + let report = try JSON.withComposition { try $0.parseDocument(source) } + let doc = report.result; let ordinary = try JSONParser().parseDocument(source) + #expect(doc.root == ordinary.root && doc.sourceBytes == source) + #expect(doc.root.objectValue?["n"]?.numberValue?.originalLexeme == "9007199254740993") + #expect(doc.root.objectValue?["adjacent"]?.numberValue?.originalLexeme == "9007199254740992") + #expect(doc.root.objectValue?["long"]?.numberValue?.originalLexeme == "123456789012345678901234567890.01234567890123456789") + #expect(doc.root.objectValue?["fraction"]?.numberValue?.originalLexeme == "0.01") + #expect(doc.root.objectValue?["negative"]?.numberValue?.originalLexeme == "-0") + #expect(doc.root.objectValue?["é"] == true && doc.root.objectValue?["e\u{301}"] == false) + #expect(doc.sourceMap?.entries.count == ordinary.sourceMap?.entries.count) + #expect(report.statistics[.numericWorkUnits] > 0 && report.statistics[.sourceMapEntries] > 0) +} + +@Test func perDocumentInputBoundAndCumulativeResources() throws { + let bytes = Data("null".utf8) + var limits = JSONLimits.balanced; limits[.inputBytes] = 4 + let report = try JSON.withComposition(limits: limits) { scope in + let entry = try scope.parseDocument(bytes) + let a = try scope.parseDocument(bytes) + let b = try scope.parseDocument(bytes) + return (entry, a, b) + } + #expect(report.result.0.sourceBytes == bytes && report.result.1.sourceBytes == bytes && report.result.2.sourceBytes == bytes) + #expect(report.statistics[.inputBytes] == 12) + // Independent ownership lower bound: 3*(source+copy+node+map) + receipt. + #expect(report.statistics[.retainedBytes] >= 3 * (8 + 32 + 32) + 424) + limits[.inputBytes] = 3 + #expect(throws: JSONFailure.self) { try JSON.withComposition(limits: limits) { try $0.parseDocument(bytes) } } +} + +@Test func emptyAndLongSourceRemainStrictAndBounded() throws { + for bytes in [Data(), Data(" ".utf8), Data("null true".utf8)] { + #expect(throws: JSONFailure.self) { try JSON.withComposition { try $0.parseDocument(bytes) } } + } + let bytes = Data(("\"" + String(repeating: "x", count: 8192) + "\"").utf8) + let doc = try JSON.withComposition { try $0.parseDocument(bytes) }.result + #expect(doc.sourceBytes == bytes && doc.root.stringValue?.utf8.count == 8192) +} + +@Test func decodedDuplicateKeysStillFail() throws { + let bytes = Data(#"{"é":0,"\u00e9":1}"#.utf8) + #expect(throws: JSONFailure.self) { try JSON.withComposition { try $0.parseDocument(bytes) } } +} + +@Test func nativePointerEscapesAndScalarIdentity() throws { + try JSON.withComposition { scope in + let doc = try scope.parseDocument(source) + let root = try scope.pointer(parsing: "") + #expect(try scope.resolve(root, in: doc.root) == doc.root) + let path = try scope.pointer(tokens: ["a/b", "~"]) + #expect(try scope.resolve(path, in: doc.root) == .null) + let scalar = try scope.pointer(uriFragment: "#/%C3%A9") + #expect(try scope.resolve(scalar, in: doc.root) == true) + let decomposed = try scope.appending(root, key: "e\u{301}") + #expect(try scope.resolve(decomposed, in: doc.root) == false) + } +} + +@Test func nativePointerIndexAndPercentGrammarAreNotPermissive() throws { + for text in ["/01", "/-", "/-1", "/999999999999999999999999"] { + #expect(throws: JSONFailure.self) { + try JSON.withComposition { scope in try scope.resolve(scope.pointer(parsing: text), in: .array([.null])) } + } + } + for text in ["#/%", "#/%FF", "#/%7E2"] { + #expect(throws: JSONFailure.self) { try JSON.withComposition { try $0.pointer(uriFragment: text) } } + } + let pointer = try JSON.withComposition { try $0.pointer(uriFragment: "#/%252F") }.result + #expect(pointer.tokens.first?.string == "%2F") +} + +@Test func nativeURICompositionUsesCoreRatherThanCompiledPlanQuota() throws { + var limits = JSONLimits.balanced; limits[.compiledPlanBytes] = 0; limits[.compileWorkUnits] = 0 + let report = try JSON.withComposition(limits: limits) { scope in + let base = try scope.uri("https://EXAMPLE.invalid/a/../api.json#part") + let relative = try scope.uri("schemas.json#/n", relativeTo: base) + let resource = try scope.removingFragment(relative) + #expect(resource.string == "https://example.invalid/schemas.json") + return relative + } + let ordinary = try JSONSchemaURI("schemas.json#/n", relativeTo: JSONSchemaURI("https://EXAMPLE.invalid/a/../api.json#part")) + #expect(report.result == ordinary) + #expect(report.statistics[.compiledPlanBytes] == 0 && report.statistics[.compileWorkUnits] == 0) + #expect(report.statistics.domain == .core) +} + +@Test func URIDataFailureIsStickyAndNeverFetches() throws { + #expect(throws: (any Error).self) { + try JSON.withComposition { scope in do { _ = try scope.uri("https://example.invalid/%ZZ") } catch { }; return 0 } + } +} + +@Test func acceptedInputHistorySurvivesRestriction() throws { + #expect(throws: JSONFailure.self) { + try JSON.withComposition { scope in + _ = try scope.parseDocument(source) + var lower = JSONLimits.balanced; lower[.inputBytes] = UInt64(source.count - 1) + do { try scope.restrict(to: lower) } catch { } + return 0 + } + } +} + +@Test func independentNestedInvocationCannotRestoreOuterHealth() throws { + var limits = JSONLimits.balanced; limits[.workUnits] = 0 + #expect(throws: JSONFailure.self) { + try JSON.withComposition(limits: limits) { outer in + do { try outer.chargeExternalWork(1) } catch { } + _ = try JSON.withComposition { _ in 0 } + return 0 + } + } +} + +@Test func publicScopeCancellationCannotBeCaughtIntoSuccess() async throws { + let task = Task.detached { + try JSON.withComposition { scope in + withUnsafeCurrentTask { $0?.cancel() } + do { try scope.checkpoint() } catch is CancellationError { } + return 9 + } + } + do { _ = try await task.value; Issue.record("cancelled scope finalized") } catch is CancellationError { } +} + +@Test func firstQuotaCauseWinsOverLaterCancellation() async throws { + let task = Task.detached { + var limits = JSONLimits.balanced; limits[.workUnits] = 0 + return try JSON.withComposition(limits: limits) { scope in + do { try scope.chargeExternalWork(1) } catch { } + withUnsafeCurrentTask { $0?.cancel() } + do { try scope.checkpoint() } catch { } + return 9 + } + } + do { _ = try await task.value; Issue.record("terminal scope finalized") } + catch let failure as JSONFailure { #expect(failure.resource?.metric == .workUnits && failure.resource?.attemptedUsage == 1) } +} + +@Test func independentConcurrentSessionsHaveDeterministicReports() async throws { + try await withThrowingTaskGroup(of: JSONOperationStatistics.self) { group in + for _ in 0..<16 { group.addTask { try traceAdapter().statistics } } + for try await report in group { #expect(report[.workUnits] == goldenWork && report.peakLogicalRetainedBytes == goldenBytes) } + } +} + +@Test func callerErrorsAndForgedReportsAreNotCertificates() throws { + #expect(throws: AuditError.self) { try JSON.withComposition { _ in throw AuditError.caller } } + let genuine = try traceAdapter() + let forged = JSONOperationReport(result: "not the audited result", statistics: genuine.statistics) + #expect(forged.result == "not the audited result") + #expect(forged.statistics[.workUnits] == goldenWork) +} + + +@Test func multiResourceSourceAndFinalOutputOverlap() throws { + // Independent native weight ledger for each scalar document: + // 4 copied bytes + 4 preserved source bytes + 32 value + 32 source-map entry. + // All three documents remain in the returned tuple, alongside two buffers. + // 3*72 + (64+32) + (48+32) + 424 report = 816 logical bytes. + let expected: UInt64 = 816 + for ceiling in [expected, expected - 1] { + var limits = JSONLimits.balanced; limits[.inputBytes] = 4; limits[.retainedBytes] = ceiling + do { + let report = try JSON.withComposition(limits: limits) { scope in + let entry = try scope.parseDocument(Data("null".utf8)) + let resourceA = try scope.parseDocument(Data("null".utf8)) + let resourceB = try scope.parseDocument(Data("null".utf8)) + let queue = try fill(scope, count: 64, first: false, mutation: nil) + let output = try fill(scope, count: 48, first: false, mutation: nil) + return (entry, resourceA, resourceB, queue, output) + } + #expect(ceiling == expected) + #expect(report.statistics[.retainedBytes] == expected) + #expect(report.statistics.peakLogicalRetainedBytes == expected) + #expect(report.result.0.sourceBytes == Data("null".utf8) && report.result.4.count == 48) + } catch let failure as JSONFailure { + #expect(ceiling == expected - 1 && failure.resource?.metric == .retainedBytes) + #expect(failure.resource?.attemptedUsage == expected) + } + } +} From 8028fefcd9b8940bcd098f78c01cd748b06ef78e Mon Sep 17 00:00:00 2001 From: Thorsten Fuchs <76904008+ThorFuchs@users.noreply.github.com> Date: Thu, 15 Oct 2026 20:47:30 +0200 Subject: [PATCH 02/11] Qualify explicit ownership probes and preserve Swift compiler crash reproducers --- Documentation/Qualification/Composition/README.md | 15 +++++++++++++++ Sources/JSON/Documentation.docc/Composition.md | 6 +++++- Sources/JSONCLIEngine/CLIBuildIdentity.swift | 2 +- .../CompilerBugs/BoundDirect.swift.txt | 4 ++++ .../CompilerBugs/BoundHelper.swift.txt | 3 +++ .../CompilerBugs/DirectEscape.swift.txt | 5 +++++ .../CompilerBugs/NestedHelper.swift.txt | 8 ++++++++ .../Negative/Escape.swift.txt | 5 +++-- .../Negative/Return.swift.txt | 2 +- .../Negative/Store.swift.txt | 2 +- Tools/CompositionQualification/README.md | 15 +++++++++++++++ 11 files changed, 61 insertions(+), 6 deletions(-) create mode 100644 Tools/CompositionQualification/CompilerBugs/BoundDirect.swift.txt create mode 100644 Tools/CompositionQualification/CompilerBugs/BoundHelper.swift.txt create mode 100644 Tools/CompositionQualification/CompilerBugs/DirectEscape.swift.txt create mode 100644 Tools/CompositionQualification/CompilerBugs/NestedHelper.swift.txt diff --git a/Documentation/Qualification/Composition/README.md b/Documentation/Qualification/Composition/README.md index 57020b3..2b7bac3 100644 --- a/Documentation/Qualification/Composition/README.md +++ b/Documentation/Qualification/Composition/README.md @@ -42,3 +42,18 @@ They must bind the source manifest, input pins, environment, exact counts, actua commands/exits, mutant failures and per-platform status. A blocked Linux runtime or SDK is not a native pass. There is no downstream pin/release authorization in this proof. The user-facing return supplies materialized receipts and patches. + +The direct escaping-closure capture shape also crashes the observed 6.4 compiler +during SIL ownership verification (`Have operand with incompatible ownership?!`, +`copy_addr`). It is preserved under `CompilerBugs` and in failed exploratory +receipts; it is not counted as an intended compiler-negative pass. The negative +escape probe instead attempts to return an escaping closure from an explicit +borrowed-parameter helper and requires a normal ownership diagnostic. These +compiler defects remain usability/qualification limitations, not runtime passes. + +Forming a bound method reference also triggers the same observed SIL verifier +crash, both directly and through a borrowed helper. Those two reproducers are +retained separately as compiler limitations. Call an explicit borrowed helper +instead of forming a method value. The return probe uses explicit `return`; an +unused noncopyable expression can infer Void and does not attempt escape. +The storage probe explicitly consumes to test the borrowed-value restriction. diff --git a/Sources/JSON/Documentation.docc/Composition.md b/Sources/JSON/Documentation.docc/Composition.md index 34ac4ba..be25b85 100644 --- a/Sources/JSON/Documentation.docc/Composition.md +++ b/Sources/JSON/Documentation.docc/Composition.md @@ -79,7 +79,11 @@ func visit(_ scope: borrowing JSONCompositionScope) throws { ``` Apple Swift 6.4.0.34.1 crashed during SIL generation for a nested local function -that captured the borrowed scope. An explicit borrowed helper parameter compiles +that captured the borrowed scope, and for direct escaping-closure capture. +Bound method references also trigger that verifier defect. These crashes are +recorded compiler defects, not intended diagnostic passes. Use explicit helper +calls rather than forming a method value. +An explicit borrowed helper parameter compiles without changing the public ownership model. Avoid that compiler pattern until it is fixed. Separate expected-failing SwiftPM builds qualify the prohibited copy, escape, async, Sendable, construction, replacement and ledger-access forms. diff --git a/Sources/JSONCLIEngine/CLIBuildIdentity.swift b/Sources/JSONCLIEngine/CLIBuildIdentity.swift index 9bff73e..e87360a 100644 --- a/Sources/JSONCLIEngine/CLIBuildIdentity.swift +++ b/Sources/JSONCLIEngine/CLIBuildIdentity.swift @@ -2,6 +2,6 @@ // Generated by Scripts/generate_cli_identity.py; excludes this file. // SHA-256 of the ordered Package.swift + Sources content manifest, not a Git commit. package enum CLIBuildIdentity { - package static let revision = "sha256:cc8ae93aaf5e28f15bd13224949920c4b8881f3892674664405d46a5f730d8b1" + package static let revision = "sha256:6b12c60904ed056c0834743b5a113f9a5c1383620951ad53033c78fa6e9fb63b" package static let kind = "source-sha256" } diff --git a/Tools/CompositionQualification/CompilerBugs/BoundDirect.swift.txt b/Tools/CompositionQualification/CompilerBugs/BoundDirect.swift.txt new file mode 100644 index 0000000..ca8c4f3 --- /dev/null +++ b/Tools/CompositionQualification/CompilerBugs/BoundDirect.swift.txt @@ -0,0 +1,4 @@ +// SPDX-License-Identifier: Apache-2.0 +import JSON +let report = try JSON.withComposition { scope in return scope.checkpoint } +try report.result() diff --git a/Tools/CompositionQualification/CompilerBugs/BoundHelper.swift.txt b/Tools/CompositionQualification/CompilerBugs/BoundHelper.swift.txt new file mode 100644 index 0000000..182a48d --- /dev/null +++ b/Tools/CompositionQualification/CompilerBugs/BoundHelper.swift.txt @@ -0,0 +1,3 @@ +// SPDX-License-Identifier: Apache-2.0 +import JSON +func rejected(_ scope: borrowing JSONCompositionScope) -> () throws -> Void { scope.checkpoint } diff --git a/Tools/CompositionQualification/CompilerBugs/DirectEscape.swift.txt b/Tools/CompositionQualification/CompilerBugs/DirectEscape.swift.txt new file mode 100644 index 0000000..d51543f --- /dev/null +++ b/Tools/CompositionQualification/CompilerBugs/DirectEscape.swift.txt @@ -0,0 +1,5 @@ +// SPDX-License-Identifier: Apache-2.0 +import JSON +import JSONSchema +var escaped: (() throws -> Void)? +_ = try JSON.withComposition { scope in escaped = { try scope.checkpoint() } } diff --git a/Tools/CompositionQualification/CompilerBugs/NestedHelper.swift.txt b/Tools/CompositionQualification/CompilerBugs/NestedHelper.swift.txt new file mode 100644 index 0000000..13786e4 --- /dev/null +++ b/Tools/CompositionQualification/CompilerBugs/NestedHelper.swift.txt @@ -0,0 +1,8 @@ +// SPDX-License-Identifier: Apache-2.0 +import JSON +func reproducer() throws { + _ = try JSON.withComposition { scope in + func local() throws { try scope.chargeExternalWork(1) } + try local() + } +} diff --git a/Tools/CompositionQualification/Negative/Escape.swift.txt b/Tools/CompositionQualification/Negative/Escape.swift.txt index d51543f..9e5c604 100644 --- a/Tools/CompositionQualification/Negative/Escape.swift.txt +++ b/Tools/CompositionQualification/Negative/Escape.swift.txt @@ -1,5 +1,6 @@ // SPDX-License-Identifier: Apache-2.0 import JSON import JSONSchema -var escaped: (() throws -> Void)? -_ = try JSON.withComposition { scope in escaped = { try scope.checkpoint() } } +func rejected(_ scope: borrowing JSONCompositionScope) -> () throws -> Void { + { try scope.checkpoint() } +} diff --git a/Tools/CompositionQualification/Negative/Return.swift.txt b/Tools/CompositionQualification/Negative/Return.swift.txt index ad13f6a..2ebe84e 100644 --- a/Tools/CompositionQualification/Negative/Return.swift.txt +++ b/Tools/CompositionQualification/Negative/Return.swift.txt @@ -1,4 +1,4 @@ // SPDX-License-Identifier: Apache-2.0 import JSON import JSONSchema -let scope = try JSON.withComposition { scope in scope } +let scope = try JSON.withComposition { scope in return scope } diff --git a/Tools/CompositionQualification/Negative/Store.swift.txt b/Tools/CompositionQualification/Negative/Store.swift.txt index 8e2d0c8..37fd253 100644 --- a/Tools/CompositionQualification/Negative/Store.swift.txt +++ b/Tools/CompositionQualification/Negative/Store.swift.txt @@ -2,4 +2,4 @@ import JSON import JSONSchema var saved: JSONCompositionScope? -_ = try JSON.withComposition { scope in saved = scope } +_ = try JSON.withComposition { scope in saved = consume scope } diff --git a/Tools/CompositionQualification/README.md b/Tools/CompositionQualification/README.md index 2a31a0b..4ebf293 100644 --- a/Tools/CompositionQualification/README.md +++ b/Tools/CompositionQualification/README.md @@ -25,3 +25,18 @@ Apple Swift 6.4.0.34.1 has a SIL-generation crash for a nested local function capturing the borrowed scope. Explicit borrowed helper parameters compile; the positive consumer uses that shape. No unsafe annotations or compiler flags are needed. Platform receipts and the captured crash are reported with the handoff. + +The direct escaping-closure capture shape also crashes the observed 6.4 compiler +during SIL ownership verification (`Have operand with incompatible ownership?!`, +`copy_addr`). It is preserved under `CompilerBugs` and in failed exploratory +receipts; it is not counted as an intended compiler-negative pass. The negative +escape probe instead attempts to return an escaping closure from an explicit +borrowed-parameter helper and requires a normal ownership diagnostic. These +compiler defects remain usability/qualification limitations, not runtime passes. + +Forming a bound method reference also triggers the same observed SIL verifier +crash, both directly and through a borrowed helper. Those two reproducers are +retained separately as compiler limitations. Call an explicit borrowed helper +instead of forming a method value. The return probe uses explicit `return`; an +unused noncopyable expression can infer Void and does not attempt escape. +The storage probe explicitly consumes to test the borrowed-value restriction. From 584cc9280523d57fcee47584ab519b78469e6456 Mon Sep 17 00:00:00 2001 From: Thorsten Fuchs <76904008+ThorFuchs@users.noreply.github.com> Date: Thu, 15 Oct 2026 20:59:48 +0200 Subject: [PATCH 03/11] Prove retained admission history after a smaller subsequent document --- .../CompositionPublicTests/CompositionTests.swift | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/Tools/CompositionQualification/Tests/CompositionPublicTests/CompositionTests.swift b/Tools/CompositionQualification/Tests/CompositionPublicTests/CompositionTests.swift index 3f6569d..123001c 100644 --- a/Tools/CompositionQualification/Tests/CompositionPublicTests/CompositionTests.swift +++ b/Tools/CompositionQualification/Tests/CompositionPublicTests/CompositionTests.swift @@ -249,14 +249,19 @@ private func capture(_ body: () throws -> Void) -> (any Error)? { } @Test func acceptedInputHistorySurvivesRestriction() throws { - #expect(throws: JSONFailure.self) { - try JSON.withComposition { scope in + let error = capture { + _ = try JSON.withComposition { scope in _ = try scope.parseDocument(source) + // Retire the larger input count. The current input is now only four + // bytes; restriction must still consult accepted-capacity history. + _ = try scope.parseDocument(Data("null".utf8)) var lower = JSONLimits.balanced; lower[.inputBytes] = UInt64(source.count - 1) do { try scope.restrict(to: lower) } catch { } return 0 } - } + } as? JSONFailure + #expect(error?.code == .limitExceeded && error?.resource?.metric == .inputBytes) + #expect(error?.resource?.attemptedUsage == UInt64(source.count)) } @Test func independentNestedInvocationCannotRestoreOuterHealth() throws { From 33ac4f7512025510acd32db9f2a3c05857e07371 Mon Sep 17 00:00:00 2001 From: Thorsten Fuchs <76904008+ThorFuchs@users.noreply.github.com> Date: Thu, 15 Oct 2026 22:20:13 +0200 Subject: [PATCH 04/11] Qualify explicit borrowed captures and add matching platform consumer gates --- .github/workflows/linux.yml | 3 ++ .github/workflows/native.yml | 2 +- Documentation/CI/README.md | 9 ++++ .../Qualification/Composition/README.md | 17 ++++++++ Scripts/ci.py | 6 +-- Scripts/probe_composition_compiler.py | 43 +++++++++++++++++++ Scripts/qualify_composition.py | 6 +-- .../JSON/Documentation.docc/Composition.md | 30 +++++++------ Sources/JSONCLIEngine/CLIBuildIdentity.swift | 2 +- .../CompilerBugs/README.md | 42 ++++++++++++++++++ .../Reduced/bound-direct.swift.txt | 11 +++++ .../Reduced/bound-explicit.swift.txt | 11 +++++ .../Reduced/bound-helper.swift.txt | 11 +++++ .../Reduced/escape-explicit.swift.txt | 11 +++++ .../Reduced/escape-inferred.swift.txt | 11 +++++ .../explicit-helper-negative.swift.txt | 11 +++++ .../explicit-helper-positive.swift.txt | 11 +++++ .../Reduced/nested-explicit.swift.txt | 13 ++++++ .../Reduced/nested-inferred.swift.txt | 13 ++++++ .../Negative/ExplicitEscape.swift.txt | 9 ++++ .../Sources/CompositionProbe/Probe.swift | 2 +- .../CompositionTests.swift | 25 +++++++++++ 22 files changed, 278 insertions(+), 21 deletions(-) create mode 100644 Scripts/probe_composition_compiler.py create mode 100644 Tools/CompositionQualification/CompilerBugs/README.md create mode 100644 Tools/CompositionQualification/CompilerBugs/Reduced/bound-direct.swift.txt create mode 100644 Tools/CompositionQualification/CompilerBugs/Reduced/bound-explicit.swift.txt create mode 100644 Tools/CompositionQualification/CompilerBugs/Reduced/bound-helper.swift.txt create mode 100644 Tools/CompositionQualification/CompilerBugs/Reduced/escape-explicit.swift.txt create mode 100644 Tools/CompositionQualification/CompilerBugs/Reduced/escape-inferred.swift.txt create mode 100644 Tools/CompositionQualification/CompilerBugs/Reduced/explicit-helper-negative.swift.txt create mode 100644 Tools/CompositionQualification/CompilerBugs/Reduced/explicit-helper-positive.swift.txt create mode 100644 Tools/CompositionQualification/CompilerBugs/Reduced/nested-explicit.swift.txt create mode 100644 Tools/CompositionQualification/CompilerBugs/Reduced/nested-inferred.swift.txt create mode 100644 Tools/CompositionQualification/Negative/ExplicitEscape.swift.txt diff --git a/.github/workflows/linux.yml b/.github/workflows/linux.yml index 5324728..c3c04d6 100644 --- a/.github/workflows/linux.yml +++ b/.github/workflows/linux.yml @@ -41,6 +41,9 @@ jobs: persist-credentials: false fetch-depth: 0 - run: python3 -B Scripts/ci.py --lane '${{ matrix.lane }}' + - name: Qualify independent public composition consumer + if: matrix.lane == 'linux' + run: python3 -B Scripts/ci.py --lane linux-composition --output .build/ci/linux/composition - name: Preserve Linux receipts if: always() uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6 diff --git a/.github/workflows/native.yml b/.github/workflows/native.yml index 81c5f50..beff84d 100644 --- a/.github/workflows/native.yml +++ b/.github/workflows/native.yml @@ -17,7 +17,7 @@ jobs: strategy: fail-fast: false matrix: - lane: [native, safety, docs, archive] + lane: [native, safety, docs, archive, composition] env: DEVELOPER_DIR: /Applications/Xcode_27.2.app/Contents/Developer PYTHONDONTWRITEBYTECODE: '1' diff --git a/Documentation/CI/README.md b/Documentation/CI/README.md index 2831666..5bf1067 100644 --- a/Documentation/CI/README.md +++ b/Documentation/CI/README.md @@ -144,3 +144,12 @@ The two supplementary native functions exercise in-loop cancellation and the original continuation's health/identity. Root counts increased by exactly two; all 126 restored identities, 325 parameter invocations and 49 cap outcomes remain required. These expectations do not assert an unexecuted platform pass. + +The additive borrowed-composition public consumer runs as `--lane composition` +on the supported Mac and `--lane linux-composition` on actual qualified Linux. +Both enforce 26 discovered functions in Debug/Release, two expected-failing +missing-charge mutants and 14 intended compiler diagnostics. Reduced compiler +crashes remain a separate investigation, not diagnostic successes. Existing +native/Linux lane names, cardinalities, 49 cap probes and protection requirements +are preserved. Native CI adds a composition job; each Linux behavior architecture +runs the consumer after its complete platform lane. diff --git a/Documentation/Qualification/Composition/README.md b/Documentation/Qualification/Composition/README.md index 2b7bac3..44e6363 100644 --- a/Documentation/Qualification/Composition/README.md +++ b/Documentation/Qualification/Composition/README.md @@ -57,3 +57,20 @@ retained separately as compiler limitations. Call an explicit borrowed helper instead of forming a method value. The return probe uses explicit `return`; an unused noncopyable expression can infer Void and does not attempt escape. The storage probe explicitly consumes to test the borrowed-value restriction. + +## Compiler investigation follow-up + +Reduced no-import reproducers reproduce the same SIL ownership verifier failures +on Apple Swift 6.4.0.34.1 and the installed Swift.org 6.4 release. An explicit +`(scope: borrowing JSONCompositionScope)` closure parameter fixes the nested local +helper and changes escaping capture into the intended diagnostic. The public +consumer now executes that nested form with an independent 55-work/432-byte ledger, +exact and one-below work ceilings. A separate explicit-capture negative build joins +the existing helper-escape probe: 26 public functions, two configurations, two +rejected mutants and 14 intended diagnostic builds. + +Bound method references remain compiler defects even with explicit borrowing. +The reduced compiler investigation is separate from intended negative builds; +crashes never count as diagnostic passes. See the CompilerBugs catalog and its +replay command. The final local report retains the prior frozen proof unchanged +and binds successor evidence to this branch's final source identity. diff --git a/Scripts/ci.py b/Scripts/ci.py index 2f26ac8..ba5e15e 100644 --- a/Scripts/ci.py +++ b/Scripts/ci.py @@ -18,7 +18,7 @@ def snapshot(): def commands(lane, output): # The qualification executable resolves output paths from its repository root. output = output.relative_to(ROOT) - if lane == 'composition': + if lane in ('composition', 'linux-composition'): return [['python3', '-B', 'Scripts/qualify_composition.py', '--output', str(output / 'public')]] if lane == 'archive': return [['python3', '-B', 'Scripts/public_check.py'], @@ -203,12 +203,12 @@ def check_result(command, code, log, events=None): if 'Scripts/test_qualification_harness.py' in command: passed = passed and 'HARNESS_TESTS passed cases=14' in log.splitlines() if command[:3] == ['python3', '-B', 'Scripts/qualify_composition.py']: - passed = passed and 'COMPOSITION_PROOF passed public_tests=25 configurations=2 mutants_rejected=2 compiler_negative=13' in log.splitlines() + passed = passed and 'COMPOSITION_PROOF passed public_tests=26 configurations=2 mutants_rejected=2 compiler_negative=14' in log.splitlines() return passed, counts def main(): parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument('--lane', choices=['composition', 'native', 'safety', 'docs', 'r0', 'qa', 'linux', 'linux-safety', 'wasm', 'browser', 'export', 'cli', 'coverage', 'linux-coverage', 'linux-qa', 'wasm-qa', 'archive'], required=True) + parser.add_argument('--lane', choices=['composition', 'linux-composition', 'native', 'safety', 'docs', 'r0', 'qa', 'linux', 'linux-safety', 'wasm', 'browser', 'export', 'cli', 'coverage', 'linux-coverage', 'linux-qa', 'wasm-qa', 'archive'], required=True) parser.add_argument('--output') args = parser.parse_args() output = ROOT / (args.output or '.build/ci/' + args.lane) diff --git a/Scripts/probe_composition_compiler.py b/Scripts/probe_composition_compiler.py new file mode 100644 index 0000000..f667107 --- /dev/null +++ b/Scripts/probe_composition_compiler.py @@ -0,0 +1,43 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: Apache-2.0 +"""Record standalone compiler outcomes; observed compiler crashes return exit 2.""" +import argparse, hashlib, json, os, pathlib, subprocess, time + +ROOT = pathlib.Path(__file__).resolve().parents[1] + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('--compiler', default='swiftc') + parser.add_argument('--output', required=True) + args = parser.parse_args() + output = pathlib.Path(args.output).resolve(); output.mkdir(parents=True, exist_ok=True) + version = subprocess.run([args.compiler, '--version'], capture_output=True, text=True, check=True).stdout + receipt = {'compiler': args.compiler, 'version': version, 'sdkroot': os.environ.get('SDKROOT'), + 'status': 'running', 'commands': []} + for fixture in sorted((ROOT/'Tools/CompositionQualification/CompilerBugs/Reduced').glob('*.swift.txt')): + name = fixture.name.removesuffix('.swift.txt') + source = output/(name+'.swift'); source.write_bytes(fixture.read_bytes()) + command = [args.compiler, '-swift-version', '6', '-module-cache-path', str(output/'cache'), + '-c', str(source), '-o', str(output/(name+'.o'))] + started = time.monotonic() + try: + run = subprocess.run(command, capture_output=True, text=True, timeout=120) + code, text = run.returncode, run.stdout+run.stderr + except subprocess.TimeoutExpired as error: + code, text = None, str(error) + log = output/(name+'.log'); log.write_text(text) + crash = code is not None and (code < 0 or any(s in text for s in ['Stack dump', 'Please submit a bug report', 'SIL verification failed'])) + outcome = 'compiler_crash' if crash else 'compiled' if code == 0 else 'timeout' if code is None else 'diagnostic' + receipt['commands'].append({'fixture': fixture.relative_to(ROOT).as_posix(), + 'source_sha256': hashlib.sha256(source.read_bytes()).hexdigest(), 'command': command, + 'exit_code': code, 'outcome': outcome, 'seconds': time.monotonic()-started, + 'log': log.name, 'log_sha256': hashlib.sha256(log.read_bytes()).hexdigest()}) + (output/'receipt.json').write_text(json.dumps(receipt, indent=2)+'\n') + print(name, outcome, 'exit='+str(code), flush=True) + receipt['counts'] = {key:sum(r['outcome']==key for r in receipt['commands']) + for key in ['compiled', 'diagnostic', 'compiler_crash', 'timeout']} + receipt['status'] = 'compiler_defects_observed' if receipt['counts']['compiler_crash'] else 'recorded' + (output/'receipt.json').write_text(json.dumps(receipt, indent=2)+'\n') + return 2 if receipt['counts']['compiler_crash'] else 1 if receipt['counts']['timeout'] else 0 + +if __name__ == '__main__': raise SystemExit(main()) diff --git a/Scripts/qualify_composition.py b/Scripts/qualify_composition.py index 5ef3b1b..460fa5f 100644 --- a/Scripts/qualify_composition.py +++ b/Scripts/qualify_composition.py @@ -6,10 +6,10 @@ ROOT = pathlib.Path(__file__).resolve().parents[1] PACKAGE = 'Tools/CompositionQualification' -EXPECTED_TESTS = 25 +EXPECTED_TESTS = 26 NEGATIVE = { 'Copy': ["'copy' cannot be applied to noncopyable types"], 'Construct': ['inaccessible'], 'Escape': ['borrowed'], - 'Return': ['Copyable'], 'Sendable': ['Sendable'], 'Async': ['async', 'synchronous'], + 'ExplicitEscape': ['borrowed'], 'Return': ['Copyable'], 'Sendable': ['Sendable'], 'Async': ['async', 'synchronous'], 'Reset': ["no member 'reset'"], 'Fork': ["no member 'fork'"], 'Refund': ["no member 'release'"], 'PackageMeter': ['inaccessible'], 'RawMeter': ['JSONWorkMeter', 'cannot find'], 'Substitute': ['immutable'], @@ -70,7 +70,7 @@ def run(name, command, env=None, expected_failure=False): assert not any(d in log for d in ['no such module','Please submit a bug report','Stack dump','SIL verification failed']) record.update(expected_diagnostics=diagnostics,fixture_sha256=hashlib.sha256(fixture.read_bytes()).hexdigest());save() receipt['status']='passed'; save() - print('COMPOSITION_PROOF passed public_tests=25 configurations=2 mutants_rejected=2 compiler_negative=13') + print('COMPOSITION_PROOF passed public_tests=26 configurations=2 mutants_rejected=2 compiler_negative=14') return 0 if __name__=='__main__': raise SystemExit(main()) diff --git a/Sources/JSON/Documentation.docc/Composition.md b/Sources/JSON/Documentation.docc/Composition.md index be25b85..5114642 100644 --- a/Sources/JSON/Documentation.docc/Composition.md +++ b/Sources/JSON/Documentation.docc/Composition.md @@ -9,7 +9,7 @@ import Foundation import JSON import JSONSchema -let report = try JSON.withComposition { scope in +let report = try JSON.withComposition { (scope: borrowing JSONCompositionScope) in let document = try scope.parseDocument(Data(#"{"count":9007199254740993}"#.utf8)) try scope.reserveExternalLogicalBytes(64) // Adapter-declared output frame weight. try scope.chargeExternalWork(1) // Before the adapter's corresponding work. @@ -78,14 +78,20 @@ func visit(_ scope: borrowing JSONCompositionScope) throws { } ``` -Apple Swift 6.4.0.34.1 crashed during SIL generation for a nested local function -that captured the borrowed scope, and for direct escaping-closure capture. -Bound method references also trigger that verifier defect. These crashes are -recorded compiler defects, not intended diagnostic passes. Use explicit helper -calls rather than forming a method value. -An explicit borrowed helper parameter compiles -without changing the public ownership model. Avoid that compiler pattern until -it is fixed. Separate expected-failing SwiftPM builds qualify the prohibited -copy, escape, async, Sendable, construction, replacement and ledger-access forms. -See the source qualification package for the exact -executed scope; platform build results are evidence for their observed SDK only. +Both Apple Swift 6.4.0.34.1 and Swift.org 6.4 reproduce an ownership-verifier +crash when a nested helper captures an inferred closure parameter. Explicitly +annotate `(scope: borrowing JSONCompositionScope)` as above: the nested synchronous +helper compiles, shares the original ledger, and escaping capture receives the +normal borrowed-parameter diagnostic. Ordinary calls also work through explicit +borrowed helper parameters. + +Forming a bound method reference such as `scope.checkpoint` still crashes both +observed compilers, including through an explicit borrowed helper. Call the method +or a helper directly. Reduced reproducers with no JSON imports are retained in the +qualification package. These are compiler defects, not intended negative-test +passes; no experimental flags or weaker ownership model are used. + +Separate expected-failing SwiftPM builds qualify copying, explicit capture escape, +async suspension, Sendable, construction, replacement and ledger-access attempts. +See the qualification package for the exact executed scope. Cross-compilation is +evidence for the observed SDK only and does not establish Linux runtime safety. diff --git a/Sources/JSONCLIEngine/CLIBuildIdentity.swift b/Sources/JSONCLIEngine/CLIBuildIdentity.swift index e87360a..dee174f 100644 --- a/Sources/JSONCLIEngine/CLIBuildIdentity.swift +++ b/Sources/JSONCLIEngine/CLIBuildIdentity.swift @@ -2,6 +2,6 @@ // Generated by Scripts/generate_cli_identity.py; excludes this file. // SHA-256 of the ordered Package.swift + Sources content manifest, not a Git commit. package enum CLIBuildIdentity { - package static let revision = "sha256:6b12c60904ed056c0834743b5a113f9a5c1383620951ad53033c78fa6e9fb63b" + package static let revision = "sha256:3996e4caf9adebb9c7eab6f37aeb499eae8ae9fc24a112b6d0a7094f1e74fb9f" package static let kind = "source-sha256" } diff --git a/Tools/CompositionQualification/CompilerBugs/README.md b/Tools/CompositionQualification/CompilerBugs/README.md new file mode 100644 index 0000000..47ce497 --- /dev/null +++ b/Tools/CompositionQualification/CompilerBugs/README.md @@ -0,0 +1,42 @@ +# Swift 6.4 ownership compiler investigation + +Original public-product reproducers are in this directory. `Reduced` contains +original Apache-2.0 reductions with no imports, private native meter or package +code. The class holds one mutable integer, the facade is noncopyable and the +synchronous entry borrows it. The defect persists without Foundation or JSON. + +Replay with a selected installed compiler and the supported macOS SDK: + +```sh +python3 -B Scripts/probe_composition_compiler.py --compiler /path/to/swiftc --output .build/composition-compiler +``` + +This investigation deliberately returns **2 when a compiler crash is observed**. +It is separate from the composition lane's intended diagnostic builds. Never +interpret a signal, stack dump or verifier failure as a passing negative test. +The receipt preserves the command, source/log hashes, version, exit and outcome. + +Observed on Apple Swift 6.4.0.34.1 and Swift.org `swift-6.4-RELEASE`: + +| Form | Outcome | +| --- | --- | +| Inferred borrowed closure parameter captured by nested synchronous helper | SIL ownership verifier crash | +| Explicit borrowing closure parameter, nested synchronous helper | Compiles | +| Inferred parameter captured by escaping closure | SIL ownership verifier crash | +| Explicit borrowing closure parameter, escaping capture | Intended borrowed-parameter diagnostic | +| Direct method reference, inferred or explicit borrowing parameter | SIL ownership verifier crash | +| Method reference from explicit borrowed helper | SIL ownership verifier crash | +| Explicit helper parameter with direct method call | Compiles | +| Escaping closure returned from explicit borrowed helper | Intended borrowed-parameter diagnostic | + +The normal diagnostic is `'scope' cannot be captured by an escaping closure since +it is a borrowed parameter`. Crash diagnostics include `Have operand with +incompatible ownership?!` and `copy_addr`. These are observations on two specific +installed compilers; no claim is made about unexecuted newer releases. + +Use an explicit borrowing closure parameter and direct calls, or pass the scope +as an explicit borrowing parameter to ordinary helpers. The independent consumer +executes the nested form, checks exact aggregate work/retention, and fails one unit +below completion. No lifetime experiment flag, copyable facade, unchecked +Sendable or arbitrary callback instruction claim is needed. The method-value forms +remain unqualified. No compiler issue has been sent on the maintainer's behalf. diff --git a/Tools/CompositionQualification/CompilerBugs/Reduced/bound-direct.swift.txt b/Tools/CompositionQualification/CompilerBugs/Reduced/bound-direct.swift.txt new file mode 100644 index 0000000..a94d039 --- /dev/null +++ b/Tools/CompositionQualification/CompilerBugs/Reduced/bound-direct.swift.txt @@ -0,0 +1,11 @@ +// SPDX-License-Identifier: Apache-2.0 +private final class Ledger { var units = 0 } +struct Scope: ~Copyable { + private let ledger = Ledger() + func checkpoint() throws { ledger.units += 1 } +} +func compose(_ body: (borrowing Scope) throws -> R) rethrows -> R { + let scope = Scope() + return try body(scope) +} +func test() throws { let result = try compose { scope in return scope.checkpoint }; try result() } diff --git a/Tools/CompositionQualification/CompilerBugs/Reduced/bound-explicit.swift.txt b/Tools/CompositionQualification/CompilerBugs/Reduced/bound-explicit.swift.txt new file mode 100644 index 0000000..63a5e52 --- /dev/null +++ b/Tools/CompositionQualification/CompilerBugs/Reduced/bound-explicit.swift.txt @@ -0,0 +1,11 @@ +// SPDX-License-Identifier: Apache-2.0 +private final class Ledger { var units = 0 } +struct Scope: ~Copyable { + private let ledger = Ledger() + func checkpoint() throws { ledger.units += 1 } +} +func compose(_ body: (borrowing Scope) throws -> R) rethrows -> R { + let scope = Scope() + return try body(scope) +} +func test() throws { let result = try compose { (scope: borrowing Scope) in return scope.checkpoint }; try result() } diff --git a/Tools/CompositionQualification/CompilerBugs/Reduced/bound-helper.swift.txt b/Tools/CompositionQualification/CompilerBugs/Reduced/bound-helper.swift.txt new file mode 100644 index 0000000..aed3927 --- /dev/null +++ b/Tools/CompositionQualification/CompilerBugs/Reduced/bound-helper.swift.txt @@ -0,0 +1,11 @@ +// SPDX-License-Identifier: Apache-2.0 +private final class Ledger { var units = 0 } +struct Scope: ~Copyable { + private let ledger = Ledger() + func checkpoint() throws { ledger.units += 1 } +} +func compose(_ body: (borrowing Scope) throws -> R) rethrows -> R { + let scope = Scope() + return try body(scope) +} +func test(_ scope: borrowing Scope) -> () throws -> Void { scope.checkpoint } diff --git a/Tools/CompositionQualification/CompilerBugs/Reduced/escape-explicit.swift.txt b/Tools/CompositionQualification/CompilerBugs/Reduced/escape-explicit.swift.txt new file mode 100644 index 0000000..9ba93fc --- /dev/null +++ b/Tools/CompositionQualification/CompilerBugs/Reduced/escape-explicit.swift.txt @@ -0,0 +1,11 @@ +// SPDX-License-Identifier: Apache-2.0 +private final class Ledger { var units = 0 } +struct Scope: ~Copyable { + private let ledger = Ledger() + func checkpoint() throws { ledger.units += 1 } +} +func compose(_ body: (borrowing Scope) throws -> R) rethrows -> R { + let scope = Scope() + return try body(scope) +} +func test() throws { var escaped: (() throws -> Void)?; _ = try compose { (scope: borrowing Scope) in escaped = { try scope.checkpoint() } }; try escaped?() } diff --git a/Tools/CompositionQualification/CompilerBugs/Reduced/escape-inferred.swift.txt b/Tools/CompositionQualification/CompilerBugs/Reduced/escape-inferred.swift.txt new file mode 100644 index 0000000..0b8bb39 --- /dev/null +++ b/Tools/CompositionQualification/CompilerBugs/Reduced/escape-inferred.swift.txt @@ -0,0 +1,11 @@ +// SPDX-License-Identifier: Apache-2.0 +private final class Ledger { var units = 0 } +struct Scope: ~Copyable { + private let ledger = Ledger() + func checkpoint() throws { ledger.units += 1 } +} +func compose(_ body: (borrowing Scope) throws -> R) rethrows -> R { + let scope = Scope() + return try body(scope) +} +func test() throws { var escaped: (() throws -> Void)?; _ = try compose { scope in escaped = { try scope.checkpoint() } }; try escaped?() } diff --git a/Tools/CompositionQualification/CompilerBugs/Reduced/explicit-helper-negative.swift.txt b/Tools/CompositionQualification/CompilerBugs/Reduced/explicit-helper-negative.swift.txt new file mode 100644 index 0000000..37dce69 --- /dev/null +++ b/Tools/CompositionQualification/CompilerBugs/Reduced/explicit-helper-negative.swift.txt @@ -0,0 +1,11 @@ +// SPDX-License-Identifier: Apache-2.0 +private final class Ledger { var units = 0 } +struct Scope: ~Copyable { + private let ledger = Ledger() + func checkpoint() throws { ledger.units += 1 } +} +func compose(_ body: (borrowing Scope) throws -> R) rethrows -> R { + let scope = Scope() + return try body(scope) +} +func test(_ scope: borrowing Scope) -> () throws -> Void { { try scope.checkpoint() } } diff --git a/Tools/CompositionQualification/CompilerBugs/Reduced/explicit-helper-positive.swift.txt b/Tools/CompositionQualification/CompilerBugs/Reduced/explicit-helper-positive.swift.txt new file mode 100644 index 0000000..279f6c5 --- /dev/null +++ b/Tools/CompositionQualification/CompilerBugs/Reduced/explicit-helper-positive.swift.txt @@ -0,0 +1,11 @@ +// SPDX-License-Identifier: Apache-2.0 +private final class Ledger { var units = 0 } +struct Scope: ~Copyable { + private let ledger = Ledger() + func checkpoint() throws { ledger.units += 1 } +} +func compose(_ body: (borrowing Scope) throws -> R) rethrows -> R { + let scope = Scope() + return try body(scope) +} +func visit(_ scope: borrowing Scope) throws { try scope.checkpoint() }; func test() throws { try compose { scope in try visit(scope) } } diff --git a/Tools/CompositionQualification/CompilerBugs/Reduced/nested-explicit.swift.txt b/Tools/CompositionQualification/CompilerBugs/Reduced/nested-explicit.swift.txt new file mode 100644 index 0000000..5bfee16 --- /dev/null +++ b/Tools/CompositionQualification/CompilerBugs/Reduced/nested-explicit.swift.txt @@ -0,0 +1,13 @@ +// SPDX-License-Identifier: Apache-2.0 +private final class Ledger { var units = 0 } +struct Scope: ~Copyable { + private let ledger = Ledger() + func checkpoint() throws { ledger.units += 1 } +} +func compose(_ body: (borrowing Scope) throws -> R) rethrows -> R { + let scope = Scope() + return try body(scope) +} +func test() throws { _ = try compose { (scope: borrowing Scope) in + func local() throws { try scope.checkpoint() }; try local() +} } diff --git a/Tools/CompositionQualification/CompilerBugs/Reduced/nested-inferred.swift.txt b/Tools/CompositionQualification/CompilerBugs/Reduced/nested-inferred.swift.txt new file mode 100644 index 0000000..af781fa --- /dev/null +++ b/Tools/CompositionQualification/CompilerBugs/Reduced/nested-inferred.swift.txt @@ -0,0 +1,13 @@ +// SPDX-License-Identifier: Apache-2.0 +private final class Ledger { var units = 0 } +struct Scope: ~Copyable { + private let ledger = Ledger() + func checkpoint() throws { ledger.units += 1 } +} +func compose(_ body: (borrowing Scope) throws -> R) rethrows -> R { + let scope = Scope() + return try body(scope) +} +func test() throws { _ = try compose { scope in + func local() throws { try scope.checkpoint() }; try local() +} } diff --git a/Tools/CompositionQualification/Negative/ExplicitEscape.swift.txt b/Tools/CompositionQualification/Negative/ExplicitEscape.swift.txt new file mode 100644 index 0000000..08a5b76 --- /dev/null +++ b/Tools/CompositionQualification/Negative/ExplicitEscape.swift.txt @@ -0,0 +1,9 @@ +// SPDX-License-Identifier: Apache-2.0 +import JSON +func rejected() throws { + var escaped: (() throws -> Void)? + _ = try JSON.withComposition { (scope: borrowing JSONCompositionScope) in + escaped = { try scope.checkpoint() } + } + try escaped?() +} diff --git a/Tools/CompositionQualification/Sources/CompositionProbe/Probe.swift b/Tools/CompositionQualification/Sources/CompositionProbe/Probe.swift index 5a292d5..f8929f7 100644 --- a/Tools/CompositionQualification/Sources/CompositionProbe/Probe.swift +++ b/Tools/CompositionQualification/Sources/CompositionProbe/Probe.swift @@ -5,7 +5,7 @@ import JSONSchema /// Minimal ordinary public consumer, also compiled for supported Apple SDKs. public func compositionProbe(_ data: Data) throws -> JSONOperationReport { - try JSON.withComposition { scope in + try JSON.withComposition { (scope: borrowing JSONCompositionScope) in let document = try scope.parseDocument(data) try adapterStep(scope) let path = try scope.pointer(tokens: ["count"]) diff --git a/Tools/CompositionQualification/Tests/CompositionPublicTests/CompositionTests.swift b/Tools/CompositionQualification/Tests/CompositionPublicTests/CompositionTests.swift index 123001c..72ad6b1 100644 --- a/Tools/CompositionQualification/Tests/CompositionPublicTests/CompositionTests.swift +++ b/Tools/CompositionQualification/Tests/CompositionPublicTests/CompositionTests.swift @@ -343,3 +343,28 @@ private func capture(_ body: () throws -> Void) -> (any Error)? { } } } + +@Test func explicitBorrowAllowsNestedSynchronousHelperOnTheSameLedger() throws { + // Independently counted: 2 entry units, 3 helper units, then 50 finalizer + // units; the helper's 8-byte frame overlaps the 424-byte report. + var limits = JSONLimits.balanced + limits[.workUnits] = 55; limits[.retainedBytes] = 432 + let report = try JSON.withComposition(limits: limits) { (scope: borrowing JSONCompositionScope) in + func local() throws { + try scope.reserveExternalLogicalBytes(8) + try scope.chargeExternalWork(3) + } + try scope.chargeExternalWork(2) + try local() + return 7 + } + #expect(report.result == 7 && report.statistics[.workUnits] == 55) + #expect(report.statistics[.retainedBytes] == 432) + limits[.workUnits] = 54 + #expect(throws: JSONFailure.self) { + try JSON.withComposition(limits: limits) { (scope: borrowing JSONCompositionScope) in + func local() throws { try scope.chargeExternalWork(5) } + try local() + } + } +} From cddac5c94a768fbf45e2bc87e0d183eaf09b9898 Mon Sep 17 00:00:00 2001 From: Thorsten Fuchs <76904008+ThorFuchs@users.noreply.github.com> Date: Thu, 15 Oct 2026 22:23:52 +0200 Subject: [PATCH 05/11] Align public qualification instructions with verified compiler forms --- Documentation/CI/README.md | 4 ++-- Tools/CompositionQualification/README.md | 18 +++++++++++++----- 2 files changed, 15 insertions(+), 7 deletions(-) diff --git a/Documentation/CI/README.md b/Documentation/CI/README.md index 5bf1067..d86dfc6 100644 --- a/Documentation/CI/README.md +++ b/Documentation/CI/README.md @@ -137,9 +137,9 @@ observations remain separate from automated SDK/API and portable execution. ## Independent composition proof `python3 Scripts/ci.py --lane composition` builds an unrelated SwiftPM test -package against normal public JSON/JSONSchema products. It requires 25 functions +package against normal public JSON/JSONSchema products. It requires 26 functions in both Debug and Release, the independent four-owner event ledger, two actually -failing omitted-charge mutants, and 13 separate intended compiler failures. +failing omitted-charge mutants, and 14 separate intended compiler failures. The two supplementary native functions exercise in-loop cancellation and the original continuation's health/identity. Root counts increased by exactly two; all 126 restored identities, 325 parameter invocations and 49 cap outcomes remain diff --git a/Tools/CompositionQualification/README.md b/Tools/CompositionQualification/README.md index 4ebf293..ebb2dbc 100644 --- a/Tools/CompositionQualification/README.md +++ b/Tools/CompositionQualification/README.md @@ -11,19 +11,20 @@ Run from the repository root: python3 Scripts/ci.py --lane composition ``` -This executes 24 Swift Testing functions in Debug and Release, checks exact raw +This executes 26 Swift Testing functions in Debug and Release, checks exact raw Swift Testing events and the golden owned-event TSV, runs the same positive test with two missing-charge mutations expecting real assertion failures, then builds -13 unrelated negative consumer packages. A wrong import/dependency or compiler +14 unrelated negative consumer packages. A wrong import/dependency or compiler crash is not an accepted negative outcome. Logs preserve diagnostics and hashes. `CompositionProbe` is a compile-only ordinary public consumer suitable for Apple SDK and Linux cross-build probes. `Negative/*.swift.txt` are fixtures compiled as separate ordinary SwiftPM executable packages, not included in passing targets. -Apple Swift 6.4.0.34.1 has a SIL-generation crash for a nested local function -capturing the borrowed scope. Explicit borrowed helper parameters compile; the -positive consumer uses that shape. No unsafe annotations or compiler flags are +Apple Swift 6.4.0.34.1 and Swift.org 6.4 have a SIL-generation crash for a nested +local function capturing an inferred borrowed scope parameter. Explicitly annotate +`(scope: borrowing JSONCompositionScope)` to compile the nested synchronous helper. +The public tests qualify this form; explicit borrowed helper parameters also compile. No unsafe annotations or compiler flags are needed. Platform receipts and the captured crash are reported with the handoff. The direct escaping-closure capture shape also crashes the observed 6.4 compiler @@ -40,3 +41,10 @@ retained separately as compiler limitations. Call an explicit borrowed helper instead of forming a method value. The return probe uses explicit `return`; an unused noncopyable expression can infer Void and does not attempt escape. The storage probe explicitly consumes to test the borrowed-value restriction. + +Run the same proof on the supported Ubuntu/glibc runtime with +`python3 Scripts/ci.py --lane linux-composition`. The matching Swift.org 6.4.0 +static Linux SDK can compile `CompositionProbe` and link `CompositionBenchmark` +for ARM64 and x86_64. That SDK lacks the Testing module; these cross-builds do not +replace actual Linux behavior/sanitizer execution. See CompilerBugs/README.md for +standalone reductions and the replay command returning 2 on observed crashes. From bab6316d31e9c1aca30ba3f9040b20edf1e6ae85 Mon Sep 17 00:00:00 2001 From: Thorsten Fuchs <76904008+ThorFuchs@users.noreply.github.com> Date: Thu, 15 Oct 2026 22:34:21 +0200 Subject: [PATCH 06/11] Run matched composition performance proof on an independent quiet native runner --- .github/workflows/composition-performance.yml | 39 +++++ .../Qualification/Composition/README.md | 21 +++ Scripts/qualify_composition_performance.py | 143 ++++++++++++++++++ .../Sources/CompositionBenchmark/main.swift | 12 +- 4 files changed, 211 insertions(+), 4 deletions(-) create mode 100644 .github/workflows/composition-performance.yml create mode 100644 Scripts/qualify_composition_performance.py diff --git a/.github/workflows/composition-performance.yml b/.github/workflows/composition-performance.yml new file mode 100644 index 0000000..f1e217d --- /dev/null +++ b/.github/workflows/composition-performance.yml @@ -0,0 +1,39 @@ +name: Composition performance +on: + pull_request: + paths: + - '.github/workflows/composition-performance.yml' + - 'Scripts/qualify_composition_performance.py' + - 'Tools/CompositionQualification/Sources/CompositionBenchmark/**' + workflow_dispatch: +permissions: + contents: read +concurrency: + group: composition-performance-${{ github.ref }} + cancel-in-progress: false +jobs: + compare: + name: Composition / quiet native Release + runs-on: xcode-27 + timeout-minutes: 30 + env: + DEVELOPER_DIR: /Applications/Xcode_27.2.app/Contents/Developer + SDKROOT: /Applications/Xcode_27.2.app/Contents/Developer/Platforms/MacOSX.platform/Developer/SDKs/MacOSX.sdk + PYTHONDONTWRITEBYTECODE: '1' + steps: + - uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6 + with: + persist-credentials: false + fetch-depth: 0 + - name: Compare exact baseline and candidate on a quiet host + run: python3 -B Scripts/qualify_composition_performance.py --output .build/ci/composition-performance + - name: Preserve actual measurements and eligibility receipts + if: always() + uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6 + with: + name: composition-performance + if-no-files-found: error + retention-days: 30 + path: | + .build/ci/composition-performance/receipt.json + .build/ci/composition-performance/*.log diff --git a/Documentation/Qualification/Composition/README.md b/Documentation/Qualification/Composition/README.md index 44e6363..b5df194 100644 --- a/Documentation/Qualification/Composition/README.md +++ b/Documentation/Qualification/Composition/README.md @@ -74,3 +74,24 @@ The reduced compiler investigation is separate from intended negative builds; crashes never count as diagnostic passes. See the CompilerBugs catalog and its replay command. The final local report retains the prior frozen proof unchanged and binds successor evidence to this branch's final source identity. + +## Controlled native performance replay + +`python3 Scripts/qualify_composition_performance.py --output .build/composition-performance` +compares the approved full main base 037e735dc6224789e4d04d2c18d15b8d27440198 +and the current source. Both use the same Swift 6.4+ compiler, native Release +build engine, benchmark source, runtime path and identical schema resource bytes. +Two warmup sets precede nine samples per process; ABBA ordering yields 18 samples +per source/row. Four operations run for wide512 and the pinned frozen JSON input. +`schema-only` always compiles the same small schema, not the frozen definitions. +Standalone Data/document and Pointer reports are unavailable and remain null. + +Every process requires at least 90% idle before and immediately after timing. +A bounded settling period precedes runs; a failed post-run probe invalidates the +controlled result. The 10% review trigger is unchanged. Exit 2 means blocked and +exit 3 requires review; neither is a performance pass or automatic exception. +Logical retention and owned work remain distinct from RSS and caller instructions. +The Composition performance workflow executes this exact harness when its benchmark +files change in a PR, or through optional manual dispatch after default-branch +availability, on an independent native runner, retaining source/fixture/binary/log identities. +It changes no existing required check name, protection or release workflow. diff --git a/Scripts/qualify_composition_performance.py b/Scripts/qualify_composition_performance.py new file mode 100644 index 0000000..783c56c --- /dev/null +++ b/Scripts/qualify_composition_performance.py @@ -0,0 +1,143 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: Apache-2.0 +"""Matched native Release ABBA comparison with mandatory quiet-host admission.""" +import argparse, datetime, hashlib, json, os, pathlib, re, shutil, statistics, subprocess, tarfile, time +from preflight import collect +from archive_candidate import files + +ROOT = pathlib.Path(__file__).resolve().parents[1] +BASE = '037e735dc6224789e4d04d2c18d15b8d27440198' +FIXTURE_SHA = 'ef70b61f99b6d2e5e3b46863822eab08dff6a45bedc7a08914e0e5b133f40203' +NAMES = {'parser-bytes-wide512': 8, 'parser-document-data-wide512': 6, + 'pointer-wide512': 30, 'schema-only': 3} + +def digest(path): return hashlib.sha256(path.read_bytes()).hexdigest() +def inventory(directory): + return {p.relative_to(directory).as_posix(): digest(p) for p in directory.rglob('*') if p.is_file()} +def source_identity(): + return [{'path':p.relative_to(ROOT).as_posix(), 'sha256':digest(p)} for p in files()] + +class Blocked(Exception): pass + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('--output', required=True) + args = parser.parse_args(); output = pathlib.Path(args.output).resolve(); output.mkdir(parents=True, exist_ok=True) + inputs = source_identity() + receipt = {'status':'running', 'baseline':BASE, 'candidate':None, 'environment':collect(), + 'source_inputs':inputs, 'source_manifest_sha256':hashlib.sha256(json.dumps(inputs, sort_keys=True).encode()).hexdigest(), + 'configuration':'release', 'build_system':'native', 'review_trigger_percent':10, + 'minimum_pre_and_post_idle_percent':90, 'commands':[], 'host_checks':[], 'runs':[], + 'started_utc':datetime.datetime.now(datetime.timezone.utc).isoformat()} + def save(): (output/'receipt.json').write_text(json.dumps(receipt, indent=2)+'\n') + def run(name, command, env=None): + log = output/(name+'.log'); start = time.monotonic() + with log.open('w') as stream: + result = subprocess.run(command, cwd=ROOT, env=env, stdout=stream, stderr=subprocess.STDOUT, timeout=1200) + receipt['commands'].append({'name':name, 'command':command, 'exit_code':result.returncode, + 'seconds':time.monotonic()-start, 'log':log.name, 'log_sha256':digest(log)}) + save() + if result.returncode: raise ValueError('Command failed: '+name) + return log.read_text() + def host(name): + result = subprocess.run(['top','-l','2','-n','0','-s','1'], capture_output=True, text=True, timeout=15) + log = output/(name+'.log'); log.write_text(result.stdout+result.stderr) + matches = re.findall(r'([0-9.]+)% idle', result.stdout) + idle = float(matches[-1]) if result.returncode == 0 and matches else None + receipt['host_checks'].append({'name':name, 'command':['top','-l','2','-n','0','-s','1'], + 'exit_code':result.returncode, 'idle_percent':idle, 'log':log.name, 'log_sha256':digest(log)}) + save(); print(name, 'idle='+str(idle), flush=True) + return idle is not None and idle >= 90 + try: + save() + if receipt['environment']['status'] != 'passed': raise Blocked('Supported native Mac/toolchain/SDK required') + run('cpu', ['sysctl','-n','machdep.cpu.brand_string']) + run('memory', ['sysctl','-n','hw.memsize']) + run('os-build', ['sw_vers','-buildVersion']) + run('compiler', ['swift','--version']) + receipt['candidate'] = run('candidate', ['git','rev-parse','HEAD']).strip() + if run('baseline', ['git','rev-parse',BASE]).strip() != BASE: raise ValueError('Unexpected baseline') + baseline = output/'baseline-source'; baseline.mkdir(exist_ok=True) + archive = output/'baseline.tar' + with archive.open('wb') as stream: subprocess.run(['git','archive',BASE], cwd=ROOT, stdout=stream, check=True) + with tarfile.open(archive) as tar: + if any(m.issym() or m.islnk() or pathlib.PurePosixPath(m.name).is_absolute() or '..' in pathlib.PurePosixPath(m.name).parts for m in tar): + raise ValueError('Unsafe baseline archive') + tar.extractall(baseline) + source = ROOT/'Tools/CompositionQualification/Sources/CompositionBenchmark/main.swift' + receipt['benchmark_source_sha256'] = digest(source) + fixture = ROOT/'Tools/MCPReadiness/Sources/MCPReadinessKit/Resources/schema-2026-07-28.json' + if digest(fixture) != FIXTURE_SHA: raise ValueError('Changed frozen fixture') + receipt['fixture'] = {'path':fixture.relative_to(ROOT).as_posix(), 'bytes':fixture.stat().st_size, 'sha256':FIXTURE_SHA, + 'scope':'JSON bytes/Pointer only; schema-only always compiles the fixed small schema, not this definitions-only resource'} + binaries = {} + for name, dependency in [('baseline',baseline), ('candidate',ROOT)]: + package = output/(name+'-package'); target = package/'Sources/CompositionBenchmark'; target.mkdir(parents=True, exist_ok=True) + shutil.copy2(source,target/'main.swift') + (package/'Package.swift').write_text('// swift-tools-version: 6.4\nimport PackageDescription\nlet package = Package(name: "MatchedBenchmark", platforms: [.macOS("27.0")], dependencies: [.package(name: "Swift-JSON", path: '+json.dumps(str(dependency))+')], targets: [.executableTarget(name: "CompositionBenchmark", dependencies: [.product(name: "JSON", package: "Swift-JSON"), .product(name: "JSONSchema", package: "Swift-JSON")])], swiftLanguageModes: [.v6])\n') + command = ['swift','build','--build-system','native','--package-path',str(package),'-c','release','--jobs','2'] + run(name+'-build',command+['--product','CompositionBenchmark']) + location = run(name+'-bin-path',command+['--show-bin-path']) + paths = [pathlib.Path(line) for line in location.splitlines() + if line.startswith('/') and (pathlib.Path(line)/'CompositionBenchmark').is_file()] + if len(paths)!=1: raise ValueError('Ambiguous/missing native binary directory') + binaries[name] = paths[0]/'CompositionBenchmark' + bundle = 'Swift-JSON_JSONSchema.bundle' + resources = inventory(binaries['baseline'].parent/bundle) + if not resources or resources != inventory(binaries['candidate'].parent/bundle): raise ValueError('Resource bundles differ') + runtime = output/'Runtime'; runtime.mkdir(exist_ok=True) + if (runtime/bundle).exists(): shutil.rmtree(runtime/bundle) + shutil.copytree(binaries['baseline'].parent/bundle, runtime/bundle) + receipt['resource_files'] = resources + receipt['binaries'] = {name:{'sha256':digest(path), 'bytes':path.stat().st_size} for name,path in binaries.items()} + save() + for workload in ['wide512','frozen-schema']: + env = os.environ.copy(); env.pop('JCOMP_BENCHMARK_INPUT',None) + if workload == 'frozen-schema': env['JCOMP_BENCHMARK_INPUT'] = str(fixture) + for index, name in enumerate(['baseline','candidate','candidate','baseline']): + label = workload+'-'+str(index)+'-'+name + # A bounded settling period is allowed before a measurement. + # Post-run eligibility is checked immediately, with no waiting to hide noise. + for attempt in range(12): + if host(label+'-before-'+str(attempt)): break + time.sleep(3) + else: raise Blocked('No 90% pre-run idle window: '+label) + shutil.copy2(binaries[name],runtime/'CompositionBenchmark') + text = run(label,[str(runtime/'CompositionBenchmark')],env) + rows = json.loads(text) + if len(rows)!=4 or {row['name']:row['iterations'] for row in rows} != NAMES: + raise ValueError('Changed benchmark discovery/iterations') + if any(len(row['seconds'])!=9 or any(v<=0 for v in row['seconds']) for row in rows): + raise ValueError('Missing/nonpositive timing samples') + receipt['runs'].append({'workload':workload, 'name':name, 'log':label+'.log', 'samples':rows}) + save() + if not host(label+'-after'): raise Blocked('Post-run idle below 90%: '+label) + comparison = [] + for workload in ['wide512','frozen-schema']: + for operation in NAMES: + rows = {name:[] for name in binaries}; counters = {name:set() for name in binaries} + for sample in receipt['runs']: + if sample['workload'] != workload: continue + row = next(r for r in sample['samples'] if r['name']==operation) + rows[sample['name']].extend(row['seconds']) + counters[sample['name']].add((row.get('work'),row.get('peak'),row['inputBytes'])) + if len(counters['baseline'])!=1 or counters['baseline'] != counters['candidate']: + raise ValueError('Changed old-API work/retention/input identity: '+operation) + before, after = [statistics.median(rows[name]) for name in ['baseline','candidate']] + if any(len(v)!=18 for v in rows.values()): raise ValueError('Missing ABBA samples') + comparison.append({'workload':workload,'operation':operation,'baseline_median_seconds':before, + 'candidate_median_seconds':after,'change_percent':100*(after/before-1), + 'samples_per_build':18,'unchanged_work_peak_input':list(next(iter(counters['baseline'])))}) + receipt['comparison'] = comparison + receipt['status'] = 'review_required' if any(row['change_percent']>10 for row in comparison) else 'passed' + receipt['disposition'] = '10% review trigger unchanged; no automatic exception, speedup or RSS claim.' + except Blocked as error: + receipt.update(status='blocked', error=str(error)) + except (OSError,ValueError,subprocess.SubprocessError) as error: + receipt.update(status='failed', error=str(error)) + receipt['source_inputs_unchanged'] = source_identity() == inputs + if not receipt['source_inputs_unchanged']: receipt.update(status='failed',error='Source changed during measurement') + receipt['finished_utc'] = datetime.datetime.now(datetime.timezone.utc).isoformat(); save() + return {'passed':0,'blocked':2,'review_required':3}.get(receipt['status'],1) + +if __name__ == '__main__': raise SystemExit(main()) diff --git a/Tools/CompositionQualification/Sources/CompositionBenchmark/main.swift b/Tools/CompositionQualification/Sources/CompositionBenchmark/main.swift index 1241054..0825d17 100644 --- a/Tools/CompositionQualification/Sources/CompositionBenchmark/main.swift +++ b/Tools/CompositionQualification/Sources/CompositionBenchmark/main.swift @@ -13,7 +13,10 @@ struct Sample: Codable { } // Deterministic 512-field exact/scalar workload, not an external conformance corpus. let text = "{" + (0..<512).map { "\"field\($0)\":" + #"{"n":9007199254740993,"s":"é","a":[true,null,0.01]}"# }.joined(separator: ",") + "}" -let bytes = Array(text.utf8), data = Data(text.utf8) +let realPath = ProcessInfo.processInfo.environment["JCOMP_BENCHMARK_INPUT"] +let data = try realPath.map { try Data(contentsOf: URL(fileURLWithPath: $0)) } ?? Data(text.utf8) +let bytes = Array(data) +let expectedMembers = realPath == nil ? 512 : 2 let root = try JSONParser().parse(bytes) let schema = try JSONSchemaDocument(JSONParser().parse(#"{"type":"object","properties":{"n":{"type":"integer"},"s":{"type":"string"}},"required":["n"]}"#)) let parserProbe = try JSONParser().parseWithReport(bytes) @@ -33,16 +36,17 @@ func measure(_ name: String, iterations: Int, work: UInt64? = nil, peak: UInt64? let samples = try [ measure("parser-bytes-wide512", iterations: 8, work: parserProbe.statistics[.workUnits], peak: parserProbe.statistics.peakLogicalRetainedBytes) { let parsed = try JSONParser().parse(bytes) - precondition(parsed.objectValue?.count == 512) + precondition(parsed.objectValue?.count == expectedMembers) }, measure("parser-document-data-wide512", iterations: 6) { let value = try JSONParser().parseDocument(data) precondition(value.sourceBytes == data) }, measure("pointer-wide512", iterations: 30) { - let pointer = try JSONPointer(tokens: ["field511"]).appending(key: "n") + let pointer = try JSONPointer(tokens: realPath == nil ? ["field511", "n"] : ["$defs", "Request"]) let value = try pointer.resolve(in: root) - precondition(value.numberValue?.originalLexeme == "9007199254740993") + if realPath == nil { precondition(value.numberValue?.originalLexeme == "9007199254740993") } + else { precondition(value.objectValue != nil) } }, measure("schema-only", iterations: 3, work: schemaProbe.statistics[.workUnits], peak: schemaProbe.statistics.peakLogicalRetainedBytes) { _ = try JSONSchemaCompiler().compile(schema) From c348cd468f52980e159724818409a731b64ca28b Mon Sep 17 00:00:00 2001 From: Thorsten Fuchs <76904008+ThorFuchs@users.noreply.github.com> Date: Thu, 15 Oct 2026 23:26:14 +0200 Subject: [PATCH 07/11] Reconcile composition accounting with independent ownership proofs --- .github/workflows/composition-performance.yml | 41 ++ .github/workflows/linux.yml | 4 +- .github/workflows/native.yml | 2 +- Documentation/Architecture/API.md | 21 + .../Architecture/CompositionQualification.md | 9 +- .../Architecture/ExtractionManifest.json | 3 +- Documentation/Architecture/ResourceLimits.md | 31 ++ Documentation/CI/README.md | 9 + .../Qualification/Composition/README.md | 97 +++++ .../Composition/Reconciliation.md | 70 ++++ Documentation/Qualification/README.md | 2 +- .../Qualification/TestDiscovery.json | 9 +- Scripts/ci.py | 19 +- Scripts/probe_composition_compiler.py | 43 ++ Scripts/qualify_composition.py | 2 +- Scripts/qualify_composition_performance.py | 151 +++++++ Scripts/qualify_composition_proof.py | 78 ++++ Scripts/test_qualification_harness.py | 6 + .../JSON/Documentation.docc/Composition.md | 45 ++- .../JSON/Limits/JSONCompositionScope.swift | 20 +- Sources/JSON/Limits/JSONWorkMeter.swift | 29 ++ Sources/JSONCLIEngine/CLIBuildIdentity.swift | 2 +- .../CompositionContractTests.swift | 16 +- .../CompositionOwnedProofTests.swift | 72 ++++ .../CompilerBugs/BoundDirect.swift.txt | 4 + .../CompilerBugs/BoundHelper.swift.txt | 3 + .../CompilerBugs/DirectEscape.swift.txt | 5 + .../CompilerBugs/NestedHelper.swift.txt | 8 + .../CompilerBugs/README.md | 42 ++ .../Reduced/bound-direct.swift.txt | 11 + .../Reduced/bound-explicit.swift.txt | 11 + .../Reduced/bound-helper.swift.txt | 11 + .../Reduced/escape-explicit.swift.txt | 11 + .../Reduced/escape-inferred.swift.txt | 11 + .../explicit-helper-negative.swift.txt | 11 + .../explicit-helper-positive.swift.txt | 11 + .../Reduced/nested-explicit.swift.txt | 13 + .../Reduced/nested-inferred.swift.txt | 13 + Tools/CompositionQualification/Package.swift | 10 +- .../ProofNegative/Async.swift.txt | 6 + .../ProofNegative/Construct.swift.txt | 4 + .../ProofNegative/Copy.swift.txt | 4 + .../ProofNegative/Escape.swift.txt | 6 + .../ProofNegative/ExplicitEscape.swift.txt | 9 + .../ProofNegative/Fork.swift.txt | 4 + .../ProofNegative/PackageMeter.swift.txt | 4 + .../ProofNegative/RawMeter.swift.txt | 4 + .../ProofNegative/Refund.swift.txt | 4 + .../ProofNegative/Reset.swift.txt | 4 + .../ProofNegative/Return.swift.txt | 4 + .../ProofNegative/Sendable.swift.txt | 5 + .../ProofNegative/Store.swift.txt | 5 + .../ProofNegative/Substitute.swift.txt | 6 + Tools/CompositionQualification/README.md | 10 + .../Sources/CompositionBenchmark/main.swift | 61 +++ .../Sources/CompositionProbe/Probe.swift | 21 + .../CompositionTests.swift | 370 ++++++++++++++++++ 57 files changed, 1454 insertions(+), 33 deletions(-) create mode 100644 .github/workflows/composition-performance.yml create mode 100644 Documentation/Qualification/Composition/README.md create mode 100644 Documentation/Qualification/Composition/Reconciliation.md create mode 100644 Scripts/probe_composition_compiler.py create mode 100644 Scripts/qualify_composition_performance.py create mode 100644 Scripts/qualify_composition_proof.py create mode 100644 Tests/JSONIntegrationTests/CompositionOwnedProofTests.swift create mode 100644 Tools/CompositionQualification/CompilerBugs/BoundDirect.swift.txt create mode 100644 Tools/CompositionQualification/CompilerBugs/BoundHelper.swift.txt create mode 100644 Tools/CompositionQualification/CompilerBugs/DirectEscape.swift.txt create mode 100644 Tools/CompositionQualification/CompilerBugs/NestedHelper.swift.txt create mode 100644 Tools/CompositionQualification/CompilerBugs/README.md create mode 100644 Tools/CompositionQualification/CompilerBugs/Reduced/bound-direct.swift.txt create mode 100644 Tools/CompositionQualification/CompilerBugs/Reduced/bound-explicit.swift.txt create mode 100644 Tools/CompositionQualification/CompilerBugs/Reduced/bound-helper.swift.txt create mode 100644 Tools/CompositionQualification/CompilerBugs/Reduced/escape-explicit.swift.txt create mode 100644 Tools/CompositionQualification/CompilerBugs/Reduced/escape-inferred.swift.txt create mode 100644 Tools/CompositionQualification/CompilerBugs/Reduced/explicit-helper-negative.swift.txt create mode 100644 Tools/CompositionQualification/CompilerBugs/Reduced/explicit-helper-positive.swift.txt create mode 100644 Tools/CompositionQualification/CompilerBugs/Reduced/nested-explicit.swift.txt create mode 100644 Tools/CompositionQualification/CompilerBugs/Reduced/nested-inferred.swift.txt create mode 100644 Tools/CompositionQualification/ProofNegative/Async.swift.txt create mode 100644 Tools/CompositionQualification/ProofNegative/Construct.swift.txt create mode 100644 Tools/CompositionQualification/ProofNegative/Copy.swift.txt create mode 100644 Tools/CompositionQualification/ProofNegative/Escape.swift.txt create mode 100644 Tools/CompositionQualification/ProofNegative/ExplicitEscape.swift.txt create mode 100644 Tools/CompositionQualification/ProofNegative/Fork.swift.txt create mode 100644 Tools/CompositionQualification/ProofNegative/PackageMeter.swift.txt create mode 100644 Tools/CompositionQualification/ProofNegative/RawMeter.swift.txt create mode 100644 Tools/CompositionQualification/ProofNegative/Refund.swift.txt create mode 100644 Tools/CompositionQualification/ProofNegative/Reset.swift.txt create mode 100644 Tools/CompositionQualification/ProofNegative/Return.swift.txt create mode 100644 Tools/CompositionQualification/ProofNegative/Sendable.swift.txt create mode 100644 Tools/CompositionQualification/ProofNegative/Store.swift.txt create mode 100644 Tools/CompositionQualification/ProofNegative/Substitute.swift.txt create mode 100644 Tools/CompositionQualification/Sources/CompositionBenchmark/main.swift create mode 100644 Tools/CompositionQualification/Sources/CompositionProbe/Probe.swift create mode 100644 Tools/CompositionQualification/Tests/CompositionPublicTests/CompositionTests.swift diff --git a/.github/workflows/composition-performance.yml b/.github/workflows/composition-performance.yml new file mode 100644 index 0000000..8d06730 --- /dev/null +++ b/.github/workflows/composition-performance.yml @@ -0,0 +1,41 @@ +name: Composition performance +on: + pull_request: + paths: + - '.github/workflows/composition-performance.yml' + - 'Scripts/qualify_composition_performance.py' + - 'Sources/JSON/Limits/JSONCompositionScope.swift' + - 'Sources/JSON/Limits/JSONWorkMeter.swift' + - 'Tools/CompositionQualification/Sources/CompositionBenchmark/**' + workflow_dispatch: +permissions: + contents: read +concurrency: + group: composition-performance-${{ github.ref }} + cancel-in-progress: false +jobs: + compare: + name: Composition / quiet native Release + runs-on: xcode-27 + timeout-minutes: 30 + env: + DEVELOPER_DIR: /Applications/Xcode_27.2.app/Contents/Developer + SDKROOT: /Applications/Xcode_27.2.app/Contents/Developer/Platforms/MacOSX.platform/Developer/SDKs/MacOSX.sdk + PYTHONDONTWRITEBYTECODE: '1' + steps: + - uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6 + with: + persist-credentials: false + fetch-depth: 0 + - name: Compare exact baseline and candidate on a quiet host + run: python3 -B Scripts/qualify_composition_performance.py --output .build/ci/composition-performance + - name: Preserve actual measurements and eligibility receipts + if: always() + uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6 + with: + name: composition-performance + if-no-files-found: error + retention-days: 30 + path: | + .build/ci/composition-performance/receipt.json + .build/ci/composition-performance/*.log diff --git a/.github/workflows/linux.yml b/.github/workflows/linux.yml index 3ff7f60..9fd6431 100644 --- a/.github/workflows/linux.yml +++ b/.github/workflows/linux.yml @@ -43,7 +43,9 @@ jobs: - run: python3 -B Scripts/ci.py --lane '${{ matrix.lane }}' - name: Public composition and reference imports if: matrix.lane == 'linux' - run: python3 -B Scripts/ci.py --lane composition + run: | + python3 -B Scripts/ci.py --lane composition + python3 -B Scripts/ci.py --lane composition-proof - name: Preserve Linux receipts if: always() uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6 diff --git a/.github/workflows/native.yml b/.github/workflows/native.yml index 14515c1..049f381 100644 --- a/.github/workflows/native.yml +++ b/.github/workflows/native.yml @@ -17,7 +17,7 @@ jobs: strategy: fail-fast: false matrix: - lane: [native, safety, docs, archive] + lane: [native, safety, docs, archive, composition-proof] env: DEVELOPER_DIR: /Applications/Xcode_27.2.app/Contents/Developer PYTHONDONTWRITEBYTECODE: '1' diff --git a/Documentation/Architecture/API.md b/Documentation/Architecture/API.md index 0acd3c2..1a37899 100644 --- a/Documentation/Architecture/API.md +++ b/Documentation/Architecture/API.md @@ -244,3 +244,24 @@ includes `revisionKind: "source-sha256"` and `revision: "sha256:"`. This is SHA-256 of the ordered Package.swift plus Sources content manifest, excluding only the generated identity file itself. It is not a Git commit, binary digest or claim of publication. Generation/checking needs no Git, time, environment or network. + + +## Borrowed native composition + +`JSON.withComposition(limits:_:)` synchronously borrows a noncopyable, +non-Sendable `JSONCompositionScope` and returns `JSONOperationReport` +only after healthy finalization. The scope has no public initializer. Public +methods parse a source-backed `Data` document, admit an existing value, construct, +parse, append and resolve native Pointers, charge external work, reserve checked +external logical bytes, checkpoint and tighten limits. Importing JSONSchema adds +`uri(_:relativeTo:)` and `removingFragment(_:)`; JSON has no sibling dependency. +URI operations use the core domain, with no compiled-plan or compilation-work +charge. This slice does not compose schema compilation/evaluation or Codable. + +All owned errors latch the original failure. Neither a caught failure, a nested +independent invocation nor later callback failure permits a completed outer +report. Scope handles cannot escape the synchronous borrow or cross suspension. +Native values may be returned; adapter-owned containers must be prepaid. Reports +are ordinary publicly constructible values, not certification of arbitrary +callback work. See [composition qualification](../Qualification/Composition/README.md) +and the JSON DocC composition guide for the exact accounting/ownership contract. diff --git a/Documentation/Architecture/CompositionQualification.md b/Documentation/Architecture/CompositionQualification.md index 20b83de..413cbac 100644 --- a/Documentation/Architecture/CompositionQualification.md +++ b/Documentation/Architecture/CompositionQualification.md @@ -62,7 +62,7 @@ Run python3 Scripts/ci.py --lane composition on the supported native or Linux runner. The lane requires 12 core functions in each build mode, separate public consumers, five external Swift Testing functions in Debug/Release/ASan, all eight scale cells and 11 compiler rejection cases. Exact module/cardinality checks -reject missing tests and misclassified resource failures. The 558 native / 542 +reject missing tests and misclassified resource failures. The 560 native / 544 Linux root functions, 325 parameter invocations, restored identities and 49 public metric probes retain their existing contracts. @@ -72,6 +72,13 @@ late missing-target/cancellation failures before attempting normal completion. Finalization must still throw the original native failure. Each report retains its actual source, environment, command exits, counts and input hashes. +Completion additionally owns a 424-byte counter report and charges 50 work units. +The retained threshold assertions include that independent buffer, and source input +capacity is per document while reported bytes accumulate across all documents. +The original two-argument reservation method is retained; a checked `overhead:` +overload is additive. See [reconciliation](../Qualification/Composition/Reconciliation.md) +for the independent ledger/mutant and compiler proof carried from PR #36. + CI runs the focused lane alongside the existing Native / native and both Linux architecture jobs. Their names and protection rules remain unchanged. Sanitizer and full-root lanes continue independently. Linux uses the existing Swift diff --git a/Documentation/Architecture/ExtractionManifest.json b/Documentation/Architecture/ExtractionManifest.json index 0dd8127..5d799b0 100644 --- a/Documentation/Architecture/ExtractionManifest.json +++ b/Documentation/Architecture/ExtractionManifest.json @@ -271,7 +271,8 @@ "1.4 C5 stream/offline-schema adapters, checked sink handoff, command help and consumers are original Apache-2.0 code. Five NewInputs fixtures are copied byte-for-byte from the maintainer CLI briefing pack 43f8819c57e3d93bc6f7cf561d2d111442e089f2973cb7c62e16e3bd3fd73d0d; these authored data fixtures introduce no runtime dependency. Missing provider helper prerequisites remain blocked rather than adapted or silently rewritten.", "1.4 checked schema resource lookup and its missing/quota/cancellation/install regressions are original Apache-2.0 code. Supported app/framework/command layouts retain all nine official resource pins; no unbounded directory scan, fallback engine or external dependency is introduced.", "1.5.1 restores 126 original Apache-2.0 test functions and qualification tools from Swift-JSON 1.2.1 (e8b687f822c577311156b13bf1d15c2d5620b84e), with file hashes and stable function identities in Qualification/Release1.5.1/Restoration.json. The diagnostic omission and Git-free provenance repairs are restored original code; current seven-product harness reconciliation and deterministic CLI identity are original contributions. No new upstream adaptation or dependency.", - "Composition reference-load fixtures, serial descriptor staging, public-only adapter, capacity/terminal tests and strict CI receipt checks are original Apache-2.0 code. Generated JSON graphs introduce no upstream protocol fixtures, engine or dependency. Core public APIs, finite profiles and accounting weights are unchanged." + "Composition reference-load fixtures, serial descriptor staging, public-only adapter, capacity/terminal tests and strict CI receipt checks are original Apache-2.0 code. Generated JSON graphs introduce no upstream protocol fixtures, engine or dependency. Core public APIs, finite profiles and accounting weights are unchanged.", + "Composition reconciliation retains PR35 native routes/reference fixtures and adds PR36 original Apache-2.0 report-overlap, per-document capacity/history, independent owned-event/mutant and compiler proofs. No third-party source or root dependency is added." ], "jsonhtml_export_original_contributions": { "license": "Apache-2.0", diff --git a/Documentation/Architecture/ResourceLimits.md b/Documentation/Architecture/ResourceLimits.md index 30cefad..3ef9887 100644 --- a/Documentation/Architecture/ResourceLimits.md +++ b/Documentation/Architecture/ResourceLimits.md @@ -143,3 +143,34 @@ uninstrumented adapter allocations and process RSS are outside the claim. A publ JSONOperationReport initializer permits pairing statistics with another result; its type alone does not establish provenance or adapter completeness. See the [composition guide](../../Sources/JSON/Documentation.docc/Composition.md). + +## Composition completion accounting + + +The additive borrowed scope uses one original core ledger and the existing 49 +metrics/default profiles. It does not introduce adjustable counters or widen any +cap. `chargeExternalWork` charges aggregate work before adapter work; +`reserveExternalLogicalBytes(bytes,count:overhead:)` checks multiplication then +addition and live retention before growth. Zero is valid but never establishes +that opaque caller storage is free. All reservations are conservative until scope +exit; repeated admission and physical aliases count again. Caller instructions, +allocator capacity and process RSS remain outside this contract. + +`parseDocument` checks the full Data length against input and retained limits +before allocating its copied byte buffer, charges each copied byte, then invokes +the existing strict parser/source mapper. Caller Data is excluded. Input capacity +is per document: the preceding input count is checked and retired into the same +cumulative report before the next document. Work, retention, accepted-capacity +history and first-failure identity never restart. Remaining core counts are +conservative across the invocation; full source bytes/root/map and re-admitted +Pointer inputs/outputs remain charged. URI identity is core work/storage, never +schema compilation or compiledPlanBytes. + +Completion admits an additional 424 logical bytes (32 header + 49 eight-byte +counter slots) and 50 work units (49 counter copies and one final work update). +All result reservations overlap this report buffer. Overflow, quota failure and +cancellation prevent a completed result/report even when caught by the adapter. +No report is emitted after an error. Native accounting-version-1 weights and the +ordinary entry points' accounting stay unchanged. This proof slice has no public +release/promote tokens, refunds or partial reports. Independent adapter events +and failing omitted-charge mutants qualify the authored adapter only. diff --git a/Documentation/CI/README.md b/Documentation/CI/README.md index 04cb04d..0ec6e56 100644 --- a/Documentation/CI/README.md +++ b/Documentation/CI/README.md @@ -148,3 +148,12 @@ The focused composition lane also runs on the Linux architecture jobs and Native Its separate-package discovery is additive; all root and 49-cap expectations remain intact. The qualification harness now has 15 regression functions, retaining its original 14. See [composition qualification](../Architecture/CompositionQualification.md) for finite capacity outcomes. + +The additive `composition-proof` lane runs two native boundary functions and the +independent 26-function public suite in both configurations, two omitted-charge +mutants and 14 separate diagnostic builds. Both focused lanes run on Linux; +Native / composition-proof is separate from the protected names. The original +reference-load lane and its exact counts remain required. The controlled +composition performance job compares pinned main 30631e8 with this successor; +[the reconciliation record](../Qualification/Composition/Reconciliation.md) states +the expected accounting delta and unchanged idle/review thresholds. diff --git a/Documentation/Qualification/Composition/README.md b/Documentation/Qualification/Composition/README.md new file mode 100644 index 0000000..aafc9e4 --- /dev/null +++ b/Documentation/Qualification/Composition/README.md @@ -0,0 +1,97 @@ +# Borrowed composition proof + +This additive slice is confined to one synchronous borrowed native scope and +explicitly instrumented adapter work. It is not a full external importer, schema +compiler/evaluator composition, general callback instruction quota or RSS bound. +No package product or runtime dependency is added. + +The exact compiled public declarations live in JSONCompositionScope.swift and +JSONCompositionScope+URI.swift. The native grammar, scalar key identity and exact +number authority are reused. URI work uses core accounting; parsing temporarily +uses the parsing domain and Pointers the pointer domain, then returns to core. +No schema plan ownership is manufactured for URI identity. + +Run `python3 Scripts/ci.py --lane composition` and `--lane composition-proof`, followed by affected native, +safety and docs lanes. The ordinary public consumer package is +`Tools/CompositionQualification`; it has no package access, SPI, @testable or +shim. The root supplementary tests use existing package-only observers solely +to locate cancellation and substitution boundaries. They are not the oracle for +undercharging. + +The independent authored adapter's golden event trace is queue 65 work/96 +logical bytes, scratch 33/64, result 49/80 and final receipt 50/424: totals +197 work and 664 retained bytes. Three buffers remain returned and live. Exact +and one-below caps and an inserted work unit are tested. Omitting one byte's work +or reservation makes the unchanged positive trace assertion fail in a separate +expected-failing test process. This does not prove an arbitrary adapter is honest; +each downstream adapter needs its own ownership ledger and audited charges. + +All owned failures latch the first cause. Finalization verifies the original +ledger identity and health and prepays result/report overlap. Scope access is +borrowed and not Sendable; copying, storing, escaping, suspension, construction, +substitution, reset/fork/refund and private ledger access are separately rejected +by the compiler. A new invocation has new allowances and cannot restore the old +invocation's health. There is no public resource-release token in this slice. + +The 6.4 compiler crashed when a local nested helper captured the borrowed scope. +Use an explicit `borrowing JSONCompositionScope` helper parameter. Evidence must +retain that failure separately from the intended negative compiler probes. + +Qualification receipts belong outside the tracked public source distribution. +They must bind the source manifest, input pins, environment, exact counts, actual +commands/exits, mutant failures and per-platform status. A blocked Linux runtime +or SDK is not a native pass. There is no downstream pin/release authorization +in this proof. The user-facing return supplies materialized receipts and patches. + +The direct escaping-closure capture shape also crashes the observed 6.4 compiler +during SIL ownership verification (`Have operand with incompatible ownership?!`, +`copy_addr`). It is preserved under `CompilerBugs` and in failed exploratory +receipts; it is not counted as an intended compiler-negative pass. The negative +escape probe instead attempts to return an escaping closure from an explicit +borrowed-parameter helper and requires a normal ownership diagnostic. These +compiler defects remain usability/qualification limitations, not runtime passes. + +Forming a bound method reference also triggers the same observed SIL verifier +crash, both directly and through a borrowed helper. Those two reproducers are +retained separately as compiler limitations. Call an explicit borrowed helper +instead of forming a method value. The return probe uses explicit `return`; an +unused noncopyable expression can infer Void and does not attempt escape. +The storage probe explicitly consumes to test the borrowed-value restriction. + +## Compiler investigation follow-up + +Reduced no-import reproducers reproduce the same SIL ownership verifier failures +on Apple Swift 6.4.0.34.1 and the installed Swift.org 6.4 release. An explicit +`(scope: borrowing JSONCompositionScope)` closure parameter fixes the nested local +helper and changes escaping capture into the intended diagnostic. The public +consumer now executes that nested form with an independent 55-work/432-byte ledger, +exact and one-below work ceilings. A separate explicit-capture negative build joins +the existing helper-escape probe: 26 public functions, two configurations, two +rejected mutants and 14 intended diagnostic builds. + +Bound method references remain compiler defects even with explicit borrowing. +The reduced compiler investigation is separate from intended negative builds; +crashes never count as diagnostic passes. See the CompilerBugs catalog and its +replay command. The final local report retains the prior frozen proof unchanged +and binds successor evidence to this branch's final source identity. + +## Controlled native performance replay + +`python3 Scripts/qualify_composition_performance.py --output .build/composition-performance` +compares the approved full main base 30631e8fe60201518c00544c172340ec6576e0d5 +and the current source. Both use the same Swift 6.4+ compiler, native Release +build engine, benchmark source, runtime path and identical schema resource bytes. +Two warmup sets precede nine samples per process; ABBA ordering yields 18 samples +per source/row. Five operations run for wide512 and the pinned frozen JSON input. +`schema-only` always compiles the same small schema, not the frozen definitions. +Standalone Data/document and Pointer reports are unavailable and remain null. + +Every process requires at least 90% idle before and immediately after timing. +A bounded settling period precedes runs; a failed post-run probe invalidates the +controlled result. The 10% review trigger is unchanged. Exit 2 means blocked and +exit 3 requires review; neither is a performance pass or automatic exception. +Logical retention and owned work remain distinct from RSS and caller instructions. +The Composition performance workflow executes this exact harness when its benchmark +files change in a PR, or through optional manual dispatch after default-branch +availability, on an independent native runner, retaining source/fixture/binary/log identities. +It changes no existing required check name, protection or release workflow. diff --git a/Documentation/Qualification/Composition/Reconciliation.md b/Documentation/Qualification/Composition/Reconciliation.md new file mode 100644 index 0000000..ddcfc1f --- /dev/null +++ b/Documentation/Qualification/Composition/Reconciliation.md @@ -0,0 +1,70 @@ +# Composition reconciliation: PR35 and PR36 + +PR35 landed as main 30631e8fe60201518c00544c172340ec6576e0d5. This follow-up +reconciles that implementation with proof head +bab6316d31e9c1aca30ba3f9040b20edf1e6ae85 from PR36. The independent original +proofs and source archives remain historical evidence, not receipts for this tree. +No downstream importer, release or dependency adoption is included. + +## Selected implementation + +There is one noncopyable borrowed `JSONCompositionScope`, one private state and +one native ledger. PR35's native parse, Pointer, trusted admission, URI and +terminal-failure routes are retained. JSON remains independent of JSONSchema. +The original `reserveExternalLogicalBytes(_:count:)` declaration is retained; +the explicit `overhead:` overload checks multiplication and addition before growth. + +PR36 adds per-document input capacity retirement with cumulative reported counts +and persistent accepted-capacity history. It also prepays completion's independent +49-slot report: 424 logical bytes and 50 work units. These are intentional +composition accounting corrections. Existing standalone API counters, all 49 +metric meanings, finite profiles and native accounting weights are unchanged. +Original continuation health/identity is checked before and after completion. +A caught owned quota, arithmetic, syntax or cancellation failure cannot finalize. + +The 12 original root behavior functions and five external reference-import +functions remain, including the failed direct 128/128 capacity hypothesis and +eight characterized cells. Their assertions now include report/result overlap; +the same native work still refuses the direct adapter after 98 references. +The two supplementary native functions and 26 independent public functions from +PR36 add cancellation boundaries, original-ledger identity, per-source capacity, +exact values, event-ledger thresholds and deliberate omitted-charge mutants. +There are 560 native / 544 Linux root functions and 325 parameter invocations. +The 126 restored identities and 49 exact public cap probes remain required. + +## Reproducible checks + +Run Scripts/ci.py lanes `composition`, `composition-proof`, `native`, `safety`, +`docs` and `archive` on the supported native toolchain. Linux architecture jobs +retain `linux` and `linux-safety` plus both focused composition lanes. Existing +protected check names/settings are preserved. The focused proof is a separate +job so root discovery guards and reference-import cardinalities stay explicit. +The original lane requires 12 root / five external functions, eight scale cells, +11 intended compiler diagnostics and distinct consumers. The proof lane requires +two root / 26 public functions, two rejected mutants and 14 intended diagnostics. +Crashes never count as normal diagnostic rejection. Exact discovery guards retain +15 Python harness functions; additional rejection assertions live in that harness. + +Compiler reductions and qualified explicit-borrow helper forms are preserved in +`Tools/CompositionQualification/CompilerBugs`. Known Swift 6.4 inferred-capture and +bound-method-value crashes remain usability limits. No unsafe annotation or +experimental ownership flag is added. Public products remain seven and root +external dependencies remain empty. Physical devices are not requalified here; +portable compilation/runtime evidence must identify its actual host and SDK. + +## Performance and evidence policy + +The controlled Release comparison uses main 30631e8 as its full pinned baseline. +Both existing PRs' comparisons against 037e735 remain historical. Identical public +harnesses, runtime/resource paths and fixtures compare unchanged APIs and the +composition document route. Existing API counters must match; composition's +expected completion delta is exactly 50 work / 424 logical bytes. Five operations +on two inputs, nine samples per process and ABBA ordering yield 18 samples per +revision/row. Each pre/post admission requires three interval samples >=90% idle. +The review trigger stays 10%; operational refusals and blocked/noisy timing are +never evaluated successes. No RSS, throughput or arbitrary adapter honesty claim. + +Final commands, exits, source hashes, manifest-backed archives, materialized hosted +receipts, performance disposition and review findings belong in the maintainer's +return evidence. Source documents here specify replay and acceptance, not a +fabricated pass. Private evidence and build caches are excluded from source archives. diff --git a/Documentation/Qualification/README.md b/Documentation/Qualification/README.md index e6c1394..4e4dae3 100644 --- a/Documentation/Qualification/README.md +++ b/Documentation/Qualification/README.md @@ -2,7 +2,7 @@ The 1.5.1 repair candidate restores all 126 functions from the separate 1.2.1 maintenance line without removing the later HTML, bridge, provider or CLI suites. -The experimental composition increment adds 12 functions. Expected full discovery is 558 native / 542 Linux functions, with 325 parameter +The reconciled experimental composition increment retains 12 functions and adds two owned-boundary functions. Expected full discovery is 560 native / 544 Linux functions, with 325 parameter invocations. The restored QA subset is 145 functions; shared-state safety is 42 native / 41 Linux. [TestDiscovery.json](TestDiscovery.json) supplies exact module cardinalities, while the event reader checks identities and each parameter count. diff --git a/Documentation/Qualification/TestDiscovery.json b/Documentation/Qualification/TestDiscovery.json index 9ff271b..12fd1cc 100644 --- a/Documentation/Qualification/TestDiscovery.json +++ b/Documentation/Qualification/TestDiscovery.json @@ -6,7 +6,7 @@ "JSONRegexRuntimeTests": 16, "JSONStreamingTests": 30, "JSONPatchTests": 16, - "JSONIntegrationTests": 32, + "JSONIntegrationTests": 34, "JSONHTMLTests": 97, "JSONBridgesTests": 10, "JSONCLITests": 88 @@ -61,7 +61,7 @@ "JSONRegexRuntimeTests": 16, "JSONStreamingTests": 30, "JSONPatchTests": 16, - "JSONIntegrationTests": 32, + "JSONIntegrationTests": 34, "JSONHTMLTests": 81, "JSONBridgesTests": 10, "JSONCLITests": 88 @@ -72,7 +72,7 @@ "JSONRegexRuntimeTests": 16, "JSONStreamingTests": 30, "JSONPatchTests": 16, - "JSONIntegrationTests": 32, + "JSONIntegrationTests": 34, "JSONHTMLTests": 81, "JSONBridgesTests": 10 }, @@ -90,5 +90,6 @@ "export_functions": { "JSONSchemaTests": 1, "JSONHTMLTests": 39 - } + }, + "composition_public_functions": 26 } diff --git a/Scripts/ci.py b/Scripts/ci.py index b6a6478..9ca611b 100644 --- a/Scripts/ci.py +++ b/Scripts/ci.py @@ -19,15 +19,20 @@ def snapshot(): def commands(lane, output): # The qualification executable resolves output paths from its repository root. output = output.relative_to(ROOT) + if lane == 'composition-proof': + result = [['swift', 'test', '--filter', 'jcompOwned|jcompOriginal'], + ['swift', 'test', '-c', 'release', '--filter', 'jcompOwned|jcompOriginal'], + ['python3', '-B', 'Scripts/qualify_composition_proof.py', '--output', str(output / 'public')]] + return [c + ['--disable-xctest'] if sys.platform == 'linux' and c[:2] == ['swift', 'test'] else c for c in result] if lane == 'composition': result = [ ['swift', 'test', '--filter', 'composition|composed|caughtOperationalFailureCannotComplete'], ['swift', 'test', '-c', 'release', '--filter', 'composition|composed|caughtOperationalFailureCannotComplete'], ['swift', 'run', '--package-path', 'Tools/CompositionQualification', 'CompositionConsumer'], ['swift', 'run', '-c', 'release', '--package-path', 'Tools/CompositionQualification', 'CompositionConsumer'], - ['swift', 'test', '--package-path', 'Tools/CompositionQualification'], - ['swift', 'test', '-c', 'release', '--package-path', 'Tools/CompositionQualification'], - ['swift', 'test', '-c', 'release', '--sanitize', 'address', '--scratch-path', '.build/composition-address', '--package-path', 'Tools/CompositionQualification'], + ['swift', 'test', '--package-path', 'Tools/CompositionQualification', '--filter', 'CompositionScaleTests'], + ['swift', 'test', '-c', 'release', '--package-path', 'Tools/CompositionQualification', '--filter', 'CompositionScaleTests'], + ['swift', 'test', '-c', 'release', '--sanitize', 'address', '--scratch-path', '.build/composition-address', '--package-path', 'Tools/CompositionQualification', '--filter', 'CompositionScaleTests'], ['python3', '-B', 'Scripts/qualify_composition_scale.py', '--output', str(output / 'scale')], ['python3', '-B', 'Scripts/qualify_composition.py', '--output', str(output / 'compiler-negative')], ] @@ -159,7 +164,7 @@ def check_result(command, code, log, events=None): passed = code == 0 if events is not None: counts = events['bundle_counts'] if command[:2] == ['swift', 'test']: - expected = [DISCOVERY['composition_external_functions']] if 'Tools/CompositionQualification' in command else [12] if 'composition|composed|caughtOperationalFailureCannotComplete' in command else [88] if 'cli|sharedFilterPlan' in command else sorted(DISCOVERY['export_functions'].values()) if 'export|Export|apple' in command else [14] if 'Tools/WasmAudit' in command else sorted(DISCOVERY['qa_functions'].values()) if 'qa' in command else [7] if R0_TESTS in command else [3] if 'Tools/MCPReadiness' in command or 'Tools/MobileQualification' in command else [1] if 'Tools/Qualification' in command else sorted(SHARED_COUNTS.values()) if 'thread' in command else sorted(ROOT_COUNTS.values()) + expected = [2] if 'jcompOwned|jcompOriginal' in command else [DISCOVERY['composition_external_functions']] if 'Tools/CompositionQualification' in command else [12] if 'composition|composed|caughtOperationalFailureCannotComplete' in command else [88] if 'cli|sharedFilterPlan' in command else sorted(DISCOVERY['export_functions'].values()) if 'export|Export|apple' in command else [14] if 'Tools/WasmAudit' in command else sorted(DISCOVERY['qa_functions'].values()) if 'qa' in command else [7] if R0_TESTS in command else [3] if 'Tools/MCPReadiness' in command or 'Tools/MobileQualification' in command else [1] if 'Tools/Qualification' in command else sorted(SHARED_COUNTS.values()) if 'thread' in command else sorted(ROOT_COUNTS.values()) passed = passed and sorted(counts) == expected if 'Tools/CompositionQualification' in command and events is not None: passed = passed and events['module_function_counts'] == {'CompositionScaleTests': DISCOVERY['composition_external_functions']} and events['parameter_case_invocations'] == 0 @@ -179,6 +184,8 @@ def check_result(command, code, log, events=None): if command[:3] == ['python3', '-B', 'Scripts/qualify_cli.py'] and passed: data = json.loads(log.strip().splitlines()[-1]) passed = data['observed'] == data['passed'] == 53 and data['failed'] == 0 + if command[:3] == ['python3', '-B', 'Scripts/qualify_composition_proof.py'] and passed: + passed = 'COMPOSITION_PROOF passed public_tests=26 configurations=2 mutants_rejected=2 compiler_negative=14' in log.splitlines() if command[:3] == ['python3', '-B', 'Scripts/qualify_composition_scale.py'] and passed: data = json.loads(log.strip().splitlines()[-1]) passed = data['status'] == 'passed' and (data['cells'], data['completed_imports'], data['operational_refusals']) == (8, 5, 3) and data['adoption_status'] == 'bounded_fixtures_only' and data['simple_strategy_capacity_status'] == 'blocked' and data['unmet_capacity_targets'] == ['balanced-128-128'] @@ -225,7 +232,7 @@ def check_result(command, code, log, events=None): def main(): parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument('--lane', choices=['composition', 'native', 'safety', 'docs', 'r0', 'qa', 'linux', 'linux-safety', 'wasm', 'browser', 'export', 'cli', 'coverage', 'linux-coverage', 'linux-qa', 'wasm-qa', 'archive'], required=True) + parser.add_argument('--lane', choices=['composition-proof', 'composition', 'native', 'safety', 'docs', 'r0', 'qa', 'linux', 'linux-safety', 'wasm', 'browser', 'export', 'cli', 'coverage', 'linux-coverage', 'linux-qa', 'wasm-qa', 'archive'], required=True) parser.add_argument('--output') args = parser.parse_args() output = ROOT / (args.output or '.build/ci/' + args.lane) @@ -234,7 +241,7 @@ def main(): repository = subprocess.run(['git', 'rev-parse', '--show-toplevel'], cwd=ROOT, capture_output=True, text=True) is_checkout = repository.returncode == 0 and pathlib.Path(repository.stdout.strip()).resolve() == ROOT.resolve() commit = subprocess.run(['git', 'rev-parse', 'HEAD'], cwd=ROOT, capture_output=True, text=True) if is_checkout else None - environment = collect_linux() if args.lane.startswith('linux') or (args.lane == 'composition' and sys.platform == 'linux') else collect() + environment = collect_linux() if args.lane.startswith('linux') or (args.lane.startswith('composition') and sys.platform == 'linux') else collect() receipt = {'schema_version': 1, 'lane': args.lane, 'status': 'blocked', 'environment': environment, 'started_utc': datetime.datetime.now(datetime.timezone.utc).isoformat(), 'git_commit': commit.stdout.strip() if commit is not None and commit.returncode == 0 else None, 'github_run_id': os.environ.get('GITHUB_RUN_ID'), 'source_inputs': inputs, 'source_manifest_sha256': hashlib.sha256(json.dumps(inputs, sort_keys=True).encode()).hexdigest(), 'commands': []} def save(): (output / 'receipt.json').write_text(json.dumps(receipt, indent=2) + '\n') diff --git a/Scripts/probe_composition_compiler.py b/Scripts/probe_composition_compiler.py new file mode 100644 index 0000000..f667107 --- /dev/null +++ b/Scripts/probe_composition_compiler.py @@ -0,0 +1,43 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: Apache-2.0 +"""Record standalone compiler outcomes; observed compiler crashes return exit 2.""" +import argparse, hashlib, json, os, pathlib, subprocess, time + +ROOT = pathlib.Path(__file__).resolve().parents[1] + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('--compiler', default='swiftc') + parser.add_argument('--output', required=True) + args = parser.parse_args() + output = pathlib.Path(args.output).resolve(); output.mkdir(parents=True, exist_ok=True) + version = subprocess.run([args.compiler, '--version'], capture_output=True, text=True, check=True).stdout + receipt = {'compiler': args.compiler, 'version': version, 'sdkroot': os.environ.get('SDKROOT'), + 'status': 'running', 'commands': []} + for fixture in sorted((ROOT/'Tools/CompositionQualification/CompilerBugs/Reduced').glob('*.swift.txt')): + name = fixture.name.removesuffix('.swift.txt') + source = output/(name+'.swift'); source.write_bytes(fixture.read_bytes()) + command = [args.compiler, '-swift-version', '6', '-module-cache-path', str(output/'cache'), + '-c', str(source), '-o', str(output/(name+'.o'))] + started = time.monotonic() + try: + run = subprocess.run(command, capture_output=True, text=True, timeout=120) + code, text = run.returncode, run.stdout+run.stderr + except subprocess.TimeoutExpired as error: + code, text = None, str(error) + log = output/(name+'.log'); log.write_text(text) + crash = code is not None and (code < 0 or any(s in text for s in ['Stack dump', 'Please submit a bug report', 'SIL verification failed'])) + outcome = 'compiler_crash' if crash else 'compiled' if code == 0 else 'timeout' if code is None else 'diagnostic' + receipt['commands'].append({'fixture': fixture.relative_to(ROOT).as_posix(), + 'source_sha256': hashlib.sha256(source.read_bytes()).hexdigest(), 'command': command, + 'exit_code': code, 'outcome': outcome, 'seconds': time.monotonic()-started, + 'log': log.name, 'log_sha256': hashlib.sha256(log.read_bytes()).hexdigest()}) + (output/'receipt.json').write_text(json.dumps(receipt, indent=2)+'\n') + print(name, outcome, 'exit='+str(code), flush=True) + receipt['counts'] = {key:sum(r['outcome']==key for r in receipt['commands']) + for key in ['compiled', 'diagnostic', 'compiler_crash', 'timeout']} + receipt['status'] = 'compiler_defects_observed' if receipt['counts']['compiler_crash'] else 'recorded' + (output/'receipt.json').write_text(json.dumps(receipt, indent=2)+'\n') + return 2 if receipt['counts']['compiler_crash'] else 1 if receipt['counts']['timeout'] else 0 + +if __name__ == '__main__': raise SystemExit(main()) diff --git a/Scripts/qualify_composition.py b/Scripts/qualify_composition.py index b88f0de..68cb17d 100644 --- a/Scripts/qualify_composition.py +++ b/Scripts/qualify_composition.py @@ -29,7 +29,7 @@ def main(): log = run.stdout + run.stderr (output / (name + '.log')).write_text(log) records.append(dict(name=name, command=command, exit_code=run.returncode, - expected_diagnostic=diagnostic, status='passed' if run.returncode != 0 and diagnostic in log and 'no such module' not in log else 'failed')) + expected_diagnostic=diagnostic, status='passed' if run.returncode == 1 and diagnostic in log and not any(s in log for s in ['no such module', 'Stack dump', 'Please submit a bug report', 'SIL verification failed']) else 'failed')) receipt = dict(status='passed' if all(r['status']=='passed' for r in records) else 'failed', compiler_negative_cases=len(records), records=records) (output / 'receipt.json').write_text(json.dumps(receipt, indent=2)+'\n') diff --git a/Scripts/qualify_composition_performance.py b/Scripts/qualify_composition_performance.py new file mode 100644 index 0000000..aa1cc69 --- /dev/null +++ b/Scripts/qualify_composition_performance.py @@ -0,0 +1,151 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: Apache-2.0 +"""Matched native Release ABBA comparison with mandatory quiet-host admission.""" +import argparse, datetime, hashlib, json, os, pathlib, re, shutil, statistics, subprocess, tarfile, time +from preflight import collect +from archive_candidate import files + +ROOT = pathlib.Path(__file__).resolve().parents[1] +BASE = '30631e8fe60201518c00544c172340ec6576e0d5' +FIXTURE_SHA = 'ef70b61f99b6d2e5e3b46863822eab08dff6a45bedc7a08914e0e5b133f40203' +NAMES = {'parser-bytes-wide512': 8, 'parser-document-data-wide512': 6, + 'pointer-wide512': 30, 'schema-only': 3, 'composition-document': 4} + +def digest(path): return hashlib.sha256(path.read_bytes()).hexdigest() +def inventory(directory): + return {p.relative_to(directory).as_posix(): digest(p) for p in directory.rglob('*') if p.is_file()} +def source_identity(): + return [{'path':p.relative_to(ROOT).as_posix(), 'sha256':digest(p)} for p in files()] + +class Blocked(Exception): pass + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('--output', required=True) + args = parser.parse_args(); output = pathlib.Path(args.output).resolve(); output.mkdir(parents=True, exist_ok=True) + inputs = source_identity() + receipt = {'status':'running', 'baseline':BASE, 'candidate':None, 'environment':collect(), + 'source_inputs':inputs, 'source_manifest_sha256':hashlib.sha256(json.dumps(inputs, sort_keys=True).encode()).hexdigest(), + 'configuration':'release', 'build_system':'native', 'review_trigger_percent':10, + 'minimum_pre_and_post_idle_percent':90, 'commands':[], 'host_checks':[], 'runs':[], + 'started_utc':datetime.datetime.now(datetime.timezone.utc).isoformat()} + def save(): (output/'receipt.json').write_text(json.dumps(receipt, indent=2)+'\n') + def run(name, command, env=None): + log = output/(name+'.log'); start = time.monotonic() + with log.open('w') as stream: + result = subprocess.run(command, cwd=ROOT, env=env, stdout=stream, stderr=subprocess.STDOUT, timeout=1200) + receipt['commands'].append({'name':name, 'command':command, 'exit_code':result.returncode, + 'seconds':time.monotonic()-start, 'log':log.name, 'log_sha256':digest(log)}) + save() + if result.returncode: raise ValueError('Command failed: '+name) + return log.read_text() + def host(name): + result = subprocess.run(['top','-l','4','-n','0','-s','1'], capture_output=True, text=True, timeout=15) + log = output/(name+'.log'); log.write_text(result.stdout+result.stderr) + matches = re.findall(r'([0-9.]+)% idle', result.stdout) + intervals = [float(v) for v in matches[-3:]] if result.returncode == 0 and len(matches) == 4 else [] + idle = min(intervals) if intervals else None + receipt['host_checks'].append({'name':name, 'command':['top','-l','4','-n','0','-s','1'], 'interval_idle_percent':intervals, + 'exit_code':result.returncode, 'idle_percent':idle, 'log':log.name, 'log_sha256':digest(log)}) + save(); print(name, 'idle='+str(idle), flush=True) + return idle is not None and idle >= 90 + try: + save() + if receipt['environment']['status'] != 'passed': raise Blocked('Supported native Mac/toolchain/SDK required') + run('cpu', ['sysctl','-n','machdep.cpu.brand_string']) + run('memory', ['sysctl','-n','hw.memsize']) + run('os-build', ['sw_vers','-buildVersion']) + run('compiler', ['swift','--version']) + receipt['candidate'] = run('candidate', ['git','rev-parse','HEAD']).strip() + if run('baseline', ['git','rev-parse',BASE]).strip() != BASE: raise ValueError('Unexpected baseline') + baseline = output/'baseline-source'; baseline.mkdir(exist_ok=True) + archive = output/'baseline.tar' + with archive.open('wb') as stream: subprocess.run(['git','archive',BASE], cwd=ROOT, stdout=stream, check=True) + with tarfile.open(archive) as tar: + if any(m.issym() or m.islnk() or pathlib.PurePosixPath(m.name).is_absolute() or '..' in pathlib.PurePosixPath(m.name).parts for m in tar): + raise ValueError('Unsafe baseline archive') + tar.extractall(baseline) + source = ROOT/'Tools/CompositionQualification/Sources/CompositionBenchmark/main.swift' + receipt['benchmark_source_sha256'] = digest(source) + fixture = ROOT/'Tools/MCPReadiness/Sources/MCPReadinessKit/Resources/schema-2026-07-28.json' + if digest(fixture) != FIXTURE_SHA: raise ValueError('Changed frozen fixture') + receipt['fixture'] = {'path':fixture.relative_to(ROOT).as_posix(), 'bytes':fixture.stat().st_size, 'sha256':FIXTURE_SHA, + 'scope':'JSON bytes/Pointer only; schema-only always compiles the fixed small schema, not this definitions-only resource'} + binaries = {} + for name, dependency in [('baseline',baseline), ('candidate',ROOT)]: + package = output/(name+'-package'); target = package/'Sources/CompositionBenchmark'; target.mkdir(parents=True, exist_ok=True) + shutil.copy2(source,target/'main.swift') + (package/'Package.swift').write_text('// swift-tools-version: 6.4\nimport PackageDescription\nlet package = Package(name: "MatchedBenchmark", platforms: [.macOS("27.0")], dependencies: [.package(name: "Swift-JSON", path: '+json.dumps(str(dependency))+')], targets: [.executableTarget(name: "CompositionBenchmark", dependencies: [.product(name: "JSON", package: "Swift-JSON"), .product(name: "JSONSchema", package: "Swift-JSON")])], swiftLanguageModes: [.v6])\n') + command = ['swift','build','--build-system','native','--package-path',str(package),'-c','release','--jobs','2'] + run(name+'-build',command+['--product','CompositionBenchmark']) + location = run(name+'-bin-path',command+['--show-bin-path']) + paths = [pathlib.Path(line) for line in location.splitlines() + if line.startswith('/') and (pathlib.Path(line)/'CompositionBenchmark').is_file()] + if len(paths)!=1: raise ValueError('Ambiguous/missing native binary directory') + binaries[name] = paths[0]/'CompositionBenchmark' + bundle = 'Swift-JSON_JSONSchema.bundle' + resources = inventory(binaries['baseline'].parent/bundle) + if not resources or resources != inventory(binaries['candidate'].parent/bundle): raise ValueError('Resource bundles differ') + runtime = output/'Runtime'; runtime.mkdir(exist_ok=True) + if (runtime/bundle).exists(): shutil.rmtree(runtime/bundle) + shutil.copytree(binaries['baseline'].parent/bundle, runtime/bundle) + receipt['resource_files'] = resources + receipt['binaries'] = {name:{'sha256':digest(path), 'bytes':path.stat().st_size} for name,path in binaries.items()} + save() + for workload in ['wide512','frozen-schema']: + env = os.environ.copy(); env.pop('JCOMP_BENCHMARK_INPUT',None) + if workload == 'frozen-schema': env['JCOMP_BENCHMARK_INPUT'] = str(fixture) + for index, name in enumerate(['baseline','candidate','candidate','baseline']): + label = workload+'-'+str(index)+'-'+name + # A bounded settling period is allowed before a measurement. + # Post-run eligibility is checked immediately, with no waiting to hide noise. + for attempt in range(12): + if host(label+'-before-'+str(attempt)): break + time.sleep(3) + else: raise Blocked('No 90% pre-run idle window: '+label) + shutil.copy2(binaries[name],runtime/'CompositionBenchmark') + text = run(label,[str(runtime/'CompositionBenchmark')],env) + rows = json.loads(text) + if len(rows)!=5 or {row['name']:row['iterations'] for row in rows} != NAMES: + raise ValueError('Changed benchmark discovery/iterations') + if any(len(row['seconds'])!=9 or any(v<=0 for v in row['seconds']) for row in rows): + raise ValueError('Missing/nonpositive timing samples') + receipt['runs'].append({'workload':workload, 'name':name, 'log':label+'.log', 'samples':rows}) + save() + if not host(label+'-after'): raise Blocked('Post-run idle below 90%: '+label) + comparison = [] + for workload in ['wide512','frozen-schema']: + for operation in NAMES: + rows = {name:[] for name in binaries}; counters = {name:set() for name in binaries} + for sample in receipt['runs']: + if sample['workload'] != workload: continue + row = next(r for r in sample['samples'] if r['name']==operation) + rows[sample['name']].extend(row['seconds']) + counters[sample['name']].add((row.get('work'),row.get('peak'),row['inputBytes'])) + if len(counters['baseline'])!=1 or len(counters['candidate'])!=1: + raise ValueError('Nondeterministic accounting: '+operation) + base_count = next(iter(counters['baseline'])); candidate_count = next(iter(counters['candidate'])) + if operation == 'composition-document': + if candidate_count != (base_count[0]+50, base_count[1]+424, base_count[2]): + raise ValueError('Unexpected composition receipt delta') + elif counters['baseline'] != counters['candidate']: + raise ValueError('Changed old-API work/retention/input identity: '+operation) + before, after = [statistics.median(rows[name]) for name in ['baseline','candidate']] + if any(len(v)!=18 for v in rows.values()): raise ValueError('Missing ABBA samples') + comparison.append({'workload':workload,'operation':operation,'baseline_median_seconds':before, + 'candidate_median_seconds':after,'change_percent':100*(after/before-1), + 'samples_per_build':18,'baseline_work_peak_input':list(base_count), 'candidate_work_peak_input':list(candidate_count), + 'accounting_disposition':'completion +50 work/+424 logical bytes' if operation=='composition-document' else 'unchanged'}) + receipt['comparison'] = comparison + receipt['status'] = 'review_required' if any(row['change_percent']>10 for row in comparison) else 'passed' + receipt['disposition'] = '10% review trigger unchanged; no automatic exception, speedup or RSS claim.' + except Blocked as error: + receipt.update(status='blocked', error=str(error)) + except (OSError,ValueError,subprocess.SubprocessError) as error: + receipt.update(status='failed', error=str(error)) + receipt['source_inputs_unchanged'] = source_identity() == inputs + if not receipt['source_inputs_unchanged']: receipt.update(status='failed',error='Source changed during measurement') + receipt['finished_utc'] = datetime.datetime.now(datetime.timezone.utc).isoformat(); save() + return {'passed':0,'blocked':2,'review_required':3}.get(receipt['status'],1) + +if __name__ == '__main__': raise SystemExit(main()) diff --git a/Scripts/qualify_composition_proof.py b/Scripts/qualify_composition_proof.py new file mode 100644 index 0000000..08cc29f --- /dev/null +++ b/Scripts/qualify_composition_proof.py @@ -0,0 +1,78 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: Apache-2.0 +"""Independent public SwiftPM tests, real mutant failures and ownership builds.""" +import argparse, datetime, hashlib, json, os, pathlib, subprocess, sys, tempfile +from testing_events import read_events + +ROOT = pathlib.Path(__file__).resolve().parents[1] +PACKAGE = 'Tools/CompositionQualification' +EXPECTED_TESTS = 26 +NEGATIVE = { + 'Copy': ["'copy' cannot be applied to noncopyable types"], 'Construct': ['inaccessible'], 'Escape': ['borrowed'], + 'ExplicitEscape': ['borrowed'], 'Return': ['Copyable'], 'Sendable': ['Sendable'], 'Async': ['async', 'synchronous'], + 'Reset': ["no member 'reset'"], 'Fork': ["no member 'fork'"], + 'Refund': ["no member 'release'"], 'PackageMeter': ['inaccessible'], + 'RawMeter': ['JSONWorkMeter', 'cannot find'], 'Substitute': ['immutable'], + 'Store': ['borrowed'] +} + +def main(): + p = argparse.ArgumentParser(description=__doc__) + p.add_argument('--output', required=True) + a = p.parse_args(); output = (ROOT / a.output).resolve(); output.mkdir(parents=True, exist_ok=True) + receipt = dict(status='running', commands=[], public_tests_per_configuration=EXPECTED_TESTS, + compiler_negative_cases=len(NEGATIVE), expected_failing_mutants=2) + def save(): (output/'receipt.json').write_text(json.dumps(receipt, indent=2)+'\n') + def run(name, command, env=None, expected_failure=False): + log = output/(name+'.log'); started = datetime.datetime.now(datetime.timezone.utc).isoformat() + with log.open('w') as stream: + result = subprocess.run(command, cwd=ROOT, env=env, stdout=stream, stderr=subprocess.STDOUT, timeout=1800) + record = dict(name=name, command=command, exit_code=result.returncode, + started_utc=started, log=log.name, log_sha256=hashlib.sha256(log.read_bytes()).hexdigest(), + expected_failure=expected_failure) + receipt['commands'].append(record); save() + print(name, 'exit='+str(result.returncode), flush=True) + return record, log.read_text() + save() + for config in ['debug', 'release']: + directory=output/config; directory.mkdir(exist_ok=True) + events=directory/'events.jsonl'; env=os.environ.copy(); env['JCOMP_TRACE_DIRECTORY']=str(directory) + command=['swift','test','--package-path',PACKAGE,'-c',config,'--filter','CompositionPublicTests','--event-stream-output-path',str(events)] + if sys.platform == 'linux': command.append('--disable-xctest') + record, log=run(config,command,env) + assert record['exit_code']==0,log[-6000:] + parsed=read_events(events.read_text()) + assert parsed['module_function_counts']=={'CompositionPublicTests':EXPECTED_TESTS} and parsed['parameter_case_invocations']==0 + record['swift_testing_events']=dict(parsed,path=events.relative_to(output).as_posix(),sha256=hashlib.sha256(events.read_bytes()).hexdigest());save() + assert (directory/'owned-events.tsv').read_text()=='owner\twork\tlogical_bytes\nqueue\t65\t96\nscratch\t33\t64\nresult\t49\t80\nreceipt\t50\t424\n' + # Mutants run the unchanged positive test as an actually failing test process. + for mutation in ['work','reservation']: + events=output/('mutant-'+mutation+'.events.jsonl') + env=os.environ.copy();env['JCOMP_TRACE_MUTATION']=mutation + command=['swift','test','--package-path',PACKAGE,'--skip-build','--filter','externalTraceMatchesIndependentLedger','--event-stream-output-path',str(events)] + if sys.platform == 'linux': command.append('--disable-xctest') + record,log=run('mutant-'+mutation,command,env,expected_failure=True) + assert record['exit_code']==1 and ('workTrace' if mutation=='work' else 'retentionTrace') in log + rows=[json.loads(line) for line in events.read_text().splitlines() if line] + issues=[r for r in rows if r['kind']=='event' and r['payload']['kind']=='issueRecorded'] + completed=[r for r in rows if r['kind']=='event' and r['payload']['kind']=='testEnded'] + assert len(issues)==1 and len(completed)==1 + record.update(expected_issue=mutation,raw_issue_events=1,completed_test_functions=1,event_stream=events.name,event_sha256=hashlib.sha256(events.read_bytes()).hexdigest());save() + # Each fixture is a separate unrelated SwiftPM package/build, using exactly + # normal public products. No package-name override, @testable, SPI or shim. + with tempfile.TemporaryDirectory(prefix='swift-json-composition-negative-') as temp: + for name,diagnostics in NEGATIVE.items(): + directory=pathlib.Path(temp)/name; sources=directory/'Sources/NegativeConsumer';sources.mkdir(parents=True) + (directory/'Package.swift').write_text('// swift-tools-version: 6.4\nimport PackageDescription\nlet package = Package(name: "IndependentNegative'+name+'", platforms: [.macOS("27.0")], dependencies: [.package(name: "Swift-JSON", path: '+json.dumps(str(ROOT))+')], targets: [.executableTarget(name: "NegativeConsumer", dependencies: [.product(name: "JSON", package: "Swift-JSON"), .product(name: "JSONSchema", package: "Swift-JSON")])], swiftLanguageModes: [.v6])\n') + fixture=ROOT/PACKAGE/'ProofNegative'/(name+'.swift.txt') + (sources/'main.swift').write_bytes(fixture.read_bytes()) + command=['swift','build','--build-system','native','--package-path',str(directory),'--scratch-path',str(output/'negative-build')] + record,log=run('negative-'+name,command,expected_failure=True) + assert record['exit_code']==1 and all(d in log for d in diagnostics), log[-6000:] + assert not any(d in log for d in ['no such module','Please submit a bug report','Stack dump','SIL verification failed']) + record.update(expected_diagnostics=diagnostics,fixture_sha256=hashlib.sha256(fixture.read_bytes()).hexdigest());save() + receipt['status']='passed'; save() + print('COMPOSITION_PROOF passed public_tests=26 configurations=2 mutants_rejected=2 compiler_negative=14') + return 0 + +if __name__=='__main__': raise SystemExit(main()) diff --git a/Scripts/test_qualification_harness.py b/Scripts/test_qualification_harness.py index cbf9ad7..60dd00e 100644 --- a/Scripts/test_qualification_harness.py +++ b/Scripts/test_qualification_harness.py @@ -126,6 +126,12 @@ def test_exact_discovery_rejects_missing_duplicate_and_changed_parameters(self): self.assertTrue(check_result(harness, 0, 'HARNESS_TESTS passed cases=15')[0]) for stale in ['HARNESS_TESTS passed cases=10', 'HARNESS_TESTS passed cases=11', 'HARNESS_TESTS passed cases=12', 'HARNESS_TESTS passed cases=13', 'HARNESS_TESTS passed cases=14', '']: self.assertFalse(check_result(harness, 0, stale)[0]) + proof = ['python3', '-B', 'Scripts/qualify_composition_proof.py'] + summary = 'COMPOSITION_PROOF passed public_tests=26 configurations=2 mutants_rejected=2 compiler_negative=14' + self.assertTrue(check_result(proof, 0, summary)[0]) + for broken in ['', summary.replace('26', '25'), summary.replace('rejected=2', 'rejected=0'), summary.replace('negative=14', 'negative=13')]: + self.assertFalse(check_result(proof, 0, broken)[0]) + self.assertFalse(check_result(proof, 1, summary)[0]) # Scoped portability lanes retain exact discovery and safety rather # than describing an exit-zero subset as full qualification. output = pathlib.Path(__file__).resolve().parents[1] / 'Artifacts/HarnessProbe' diff --git a/Sources/JSON/Documentation.docc/Composition.md b/Sources/JSON/Documentation.docc/Composition.md index 6de53b5..5114642 100644 --- a/Sources/JSON/Documentation.docc/Composition.md +++ b/Sources/JSON/Documentation.docc/Composition.md @@ -1,7 +1,7 @@ # Compose native operations Run synchronous native operations and explicitly instrumented adapter work with -one finite ledger. The experimental composition facade does not meter arbitrary +one finite ledger. The composition facade does not meter arbitrary callback instructions, allocator sizes, physical RSS, or Foundation instructions. ```swift @@ -9,7 +9,7 @@ import Foundation import JSON import JSONSchema -let report = try JSON.withComposition { scope in +let report = try JSON.withComposition { (scope: borrowing JSONCompositionScope) in let document = try scope.parseDocument(Data(#"{"count":9007199254740993}"#.utf8)) try scope.reserveExternalLogicalBytes(64) // Adapter-declared output frame weight. try scope.chargeExternalWork(1) // Before the adapter's corresponding work. @@ -34,11 +34,21 @@ claim about opaque storage. External loops must charge/checkpoint regularly. charges each copied byte and runs the native source-map parser on the same meter. The original caller Data is excluded; the array copy remains conservatively reserved alongside decoded values, source bytes, spans and final output. -Version 1 native weights and internal scratch releases follow the resource +Each document applies `inputBytes` independently. Completed input counts remain +in the cumulative report; work, logical retention, accepted-capacity history and +terminal health never restart. Other native capacities remain conservative across +the scope. Version 1 native weights and internal scratch releases follow the resource contract. Reported peak is this ledger's conservative logical peak, not a sum of independent operation peaks. Final retained bytes include all unreleased reservations, even when the callback returns a smaller result. +Completion reserves 424 logical bytes (32 header bytes plus 49 UInt64 slots) +before allocating the independent report counter buffer. It charges 49 counter +copies plus one final aggregate-work update: 50 work units. The returned result +and all conservative reservations remain live through this final admission. +An adapter must also reserve its result containers before growth; the generic +result type does not reveal arbitrary storage to the library. + All owned helper errors are terminal, including syntax and missing-target errors. Catching one cannot produce a successful report. Scope exit rethrows the original owned failure ahead of any later callback error. Restriction intersects limits @@ -56,3 +66,32 @@ unforgeable receipt, and it cannot establish that an arbitrary callback charged all its work. An adopting adapter must own its final accounting boundary and audit all external work and reservations. Independent reports cannot retroactively certify one enforced invocation. + + +## Compiler and portability boundary + +Pass the borrowed scope explicitly to adapter helper functions: + +```swift +func visit(_ scope: borrowing JSONCompositionScope) throws { + try scope.chargeExternalWork(1) +} +``` + +Both Apple Swift 6.4.0.34.1 and Swift.org 6.4 reproduce an ownership-verifier +crash when a nested helper captures an inferred closure parameter. Explicitly +annotate `(scope: borrowing JSONCompositionScope)` as above: the nested synchronous +helper compiles, shares the original ledger, and escaping capture receives the +normal borrowed-parameter diagnostic. Ordinary calls also work through explicit +borrowed helper parameters. + +Forming a bound method reference such as `scope.checkpoint` still crashes both +observed compilers, including through an explicit borrowed helper. Call the method +or a helper directly. Reduced reproducers with no JSON imports are retained in the +qualification package. These are compiler defects, not intended negative-test +passes; no experimental flags or weaker ownership model are used. + +Separate expected-failing SwiftPM builds qualify copying, explicit capture escape, +async suspension, Sendable, construction, replacement and ledger-access attempts. +See the qualification package for the exact executed scope. Cross-compilation is +evidence for the observed SDK only and does not establish Linux runtime safety. diff --git a/Sources/JSON/Limits/JSONCompositionScope.swift b/Sources/JSON/Limits/JSONCompositionScope.swift index 3ebf9b3..5c4a3e8 100644 --- a/Sources/JSON/Limits/JSONCompositionScope.swift +++ b/Sources/JSON/Limits/JSONCompositionScope.swift @@ -44,6 +44,7 @@ public struct JSONCompositionScope: ~Copyable { /// - Throws: Original native failure or cancellation; caught failures stay terminal. public func parseDocument(_ data: Data) throws -> JSONDocument { try withMeter(domain: .parsing) { meter in + try meter.beginCompositionDocument() try meter.require(.inputBytes, UInt64(data.count)) try meter.retain(UInt64(data.count)) var bytes: [UInt8] = [] @@ -124,6 +125,15 @@ public struct JSONCompositionScope: ~Copyable { try withMeter { try $0.work(units) } } + /// Reserves checked adapter bytes before growth, retaining the original API shape. + /// - Parameters: + /// - bytes: Logical weight per unit. + /// - count: Number of units, multiplied with checked arithmetic. + /// - Throws: Terminal quota/overflow failure or cancellation. + public func reserveExternalLogicalBytes(_ bytes: UInt64, count: UInt64 = 1) throws { + try reserveExternalLogicalBytes(bytes, count: count, overhead: 0) + } + /// Reserves adapter-owned logical bytes until scope completion, without refund. /// /// Call before growth. Separate calls count separately even for physical aliases. @@ -132,8 +142,9 @@ public struct JSONCompositionScope: ~Copyable { /// - Parameters: /// - bytes: Logical weight per unit. /// - count: Number of units, multiplied with checked UInt64 arithmetic. + /// - overhead: Additional logical header or bookkeeping bytes, added before growth. /// - Throws: Quota/overflow failure or cancellation, permanently terminal. - public func reserveExternalLogicalBytes(_ bytes: UInt64, count: UInt64 = 1) throws { + public func reserveExternalLogicalBytes(_ bytes: UInt64, count: UInt64 = 1, overhead: UInt64) throws { try withMeter { meter in let (total, overflow) = bytes.multipliedReportingOverflow(by: count) guard !overflow else { @@ -141,7 +152,8 @@ public struct JSONCompositionScope: ~Copyable { domain: .core, metric: .retainedBytes, limit: meter.limits[.retainedBytes], attemptedUsage: nil, overflow: true)) } - try meter.retain(total) + let complete = try meter.adding(.retainedBytes, total, overhead) + try meter.retain(complete) } } @@ -163,7 +175,9 @@ public struct JSONCompositionScope: ~Copyable { do { let result = try body(scope) try original.check(scope.state.context.meter) - return .init(result: result, statistics: scope.state.context.meter.statistics()) + let statistics = try scope.withMeter { try $0.compositionStatistics() } + try original.check(scope.state.context.meter) + return .init(result: result, statistics: statistics) } catch { // A caller error cannot conceal an earlier native terminal failure or substitution. try original.check(scope.state.context.meter) diff --git a/Sources/JSON/Limits/JSONWorkMeter.swift b/Sources/JSON/Limits/JSONWorkMeter.swift index 0f5a1f2..73b19cd 100644 --- a/Sources/JSON/Limits/JSONWorkMeter.swift +++ b/Sources/JSON/Limits/JSONWorkMeter.swift @@ -206,6 +206,35 @@ package struct JSONWorkMeter: ~Copyable { guard bytes <= used(.annotationBytes) else { throw JSONFailure(.configuration, code: .invalidLimits) } counters[JSONResourceMetric.annotationBytes.index] -= bytes } + /// Input is a per-document capacity. Retire its charged count into the same + /// cumulative report; never reset work, retention, accepted history or health. + package mutating func beginCompositionDocument() throws { + try checkpoint() + try retire(.inputBytes) + } + /// Composition-only completion owns a separate 49-slot report through handoff. + /// Existing report entry points retain their original accounting semantics. + package mutating func compositionStatistics() throws -> JSONOperationStatistics { + // Version 1 composition receipt: 32 logical header bytes + 49 UInt64 slots. + try retain(424) + var report: [UInt64] = [] + report.reserveCapacity(JSONResourceMetric.allCases.count) + for metric in JSONResourceMetric.allCases { + try work() + let (total, overflow) = counters[metric.index].addingReportingOverflow(retired[metric.index]) + guard !overflow else { throw violation(metric, nil, overflow: true) } + report.append(total) + } + // The aggregate-work slot may precede the last copied slot. Charge and + // publish the final count only after every copied counter is accounted. + try work() + let (work, overflow) = counters[JSONResourceMetric.workUnits.index] + .addingReportingOverflow(retired[JSONResourceMetric.workUnits.index]) + guard !overflow else { throw violation(.workUnits, nil, overflow: true) } + report[JSONResourceMetric.workUnits.index] = work + try checkpoint() + return .init(domain: domain, counters: report, peak: peak) + } package func statistics() -> JSONOperationStatistics { var report = counters for metric in JSONResourceMetric.allCases { report[metric.index] += retired[metric.index] } diff --git a/Sources/JSONCLIEngine/CLIBuildIdentity.swift b/Sources/JSONCLIEngine/CLIBuildIdentity.swift index a8cb76f..d7546c2 100644 --- a/Sources/JSONCLIEngine/CLIBuildIdentity.swift +++ b/Sources/JSONCLIEngine/CLIBuildIdentity.swift @@ -2,6 +2,6 @@ // Generated by Scripts/generate_cli_identity.py; excludes this file. // SHA-256 of the ordered Package.swift + Sources content manifest, not a Git commit. package enum CLIBuildIdentity { - package static let revision = "sha256:6da72a4288423d431dd46acd076fc9ee2f4a5cc68a5ea8bf55e31bd9b0b7b1ea" + package static let revision = "sha256:a0890c08b95b4eb2620018832f2145737e2fa5ac39e687f1e8aba2af41cae752" package static let kind = "source-sha256" } diff --git a/Tests/JSONIntegrationTests/CompositionContractTests.swift b/Tests/JSONIntegrationTests/CompositionContractTests.swift index c650339..57c04a5 100644 --- a/Tests/JSONIntegrationTests/CompositionContractTests.swift +++ b/Tests/JSONIntegrationTests/CompositionContractTests.swift @@ -91,11 +91,11 @@ private func composedImport(limits: JSONLimits = .balanced, extra: UInt64 = 0) t try scope.reserveExternalLogicalBytes(64) return 0 } - #expect(two.statistics.peakLogicalRetainedBytes == 128) - #expect(two.statistics[.retainedBytes] == 128) - #expect(one.statistics.peakLogicalRetainedBytes == 64) + #expect(two.statistics.peakLogicalRetainedBytes == 128 + 424) + #expect(two.statistics[.retainedBytes] == 128 + 424) + #expect(one.statistics.peakLogicalRetainedBytes == 64 + 424) var belowOverlap = JSONLimits.balanced - belowOverlap[.retainedBytes] = 127 + belowOverlap[.retainedBytes] = 127 + 424 #expect(throws: JSONFailure.self) { try JSON.withComposition(limits: belowOverlap) { scope in try scope.reserveExternalLogicalBytes(64) @@ -103,7 +103,7 @@ private func composedImport(limits: JSONLimits = .balanced, extra: UInt64 = 0) t return 0 } } - #expect(try JSON.withComposition { try $0.admitTrusted(.null) }.statistics[.retainedBytes] == 32) + #expect(try JSON.withComposition { try $0.admitTrusted(.null) }.statistics[.retainedBytes] == 32 + 424) // Independent stage peaks are not a composition oracle: the parser's scratch // peak and a later URI peak need not be simultaneously live. let parse = try JSON.withComposition { try $0.parseDocument(compositionInput) } @@ -178,7 +178,7 @@ private func composedImport(limits: JSONLimits = .balanced, extra: UInt64 = 0) t _ = try scope.admitTrusted(alias) return alias } - #expect(two.statistics[.retainedBytes] == one.statistics[.retainedBytes] * 2) + #expect(two.statistics[.retainedBytes] == (one.statistics[.retainedBytes] - 424) * 2 + 424) var limits = JSONLimits.balanced limits[.compiledPlanBytes] = 0 limits[.compileWorkUnits] = 0 @@ -303,10 +303,10 @@ private func composedImport(limits: JSONLimits = .balanced, extra: UInt64 = 0) t try inner.chargeExternalWork(5) return 0 } - #expect(inner.statistics[.workUnits] == 5) + #expect(inner.statistics[.workUnits] == 5 + 50) return 0 } - #expect(report.statistics[.workUnits] == 3) + #expect(report.statistics[.workUnits] == 3 + 50) // Substitution of the borrowed facade is rejected by the external compiler // fixture Substitute.swift.txt, including swap/use/restore. Independent // nested invocations are explicitly outside the outer report's claim. diff --git a/Tests/JSONIntegrationTests/CompositionOwnedProofTests.swift b/Tests/JSONIntegrationTests/CompositionOwnedProofTests.swift new file mode 100644 index 0000000..176f348 --- /dev/null +++ b/Tests/JSONIntegrationTests/CompositionOwnedProofTests.swift @@ -0,0 +1,72 @@ +// SPDX-License-Identifier: Apache-2.0 +import Foundation +import Synchronization +import Testing +import JSON +import JSONSchema + +// Native-only rendezvous is supplementary to the unrelated public package. It +// uses the existing invocation-owned observer; it is not an accounting oracle. +@Test func jcompOwnedCancellationOccursInsideNativeWork() async throws { + for stage in 0..<7 { + let seen = Mutex(nil) + let task = Task.detached { + var limits = JSONLimits.offline; limits[.cancellationQuantum] = 1 + let threshold: UInt64 = stage == 1 ? 4200 : stage == 4 ? 6 : 32 + return try JSON.withComposition(limits: limits) { scope in + try scope.withMeter { meter in + meter.checkpointObserver = { work in + if work >= threshold { + seen.withLock { if $0 == nil { $0 = work } } + withUnsafeCurrentTask { $0?.cancel() } + } + } + } + do { + switch stage { + case 0, 1: + _ = try scope.parseDocument(Data(("\"" + String(repeating: "a", count: 4096) + "\"").utf8)) + case 2: _ = try scope.pointer(parsing: "/" + String(repeating: "a", count: 4096)) + case 3: _ = try scope.uri("https://example.invalid/" + String(repeating: "a", count: 4096)) + case 4: try scope.chargeExternalWork(5) // Cancellation must occur in report copying. + case 5: + for _ in 0..<128 { try scope.chargeExternalWork(1) } + default: + let path = try scope.pointer(tokens: []) + _ = try scope.resolve(path, in: .array(Array(repeating: .null, count: 256))) + } + } catch is CancellationError { } + return 1 + } + } + do { _ = try await task.value; Issue.record("caught cancellation completed at stage \(stage)") } + catch is CancellationError { } + #expect(seen.withLock { $0 != nil }) + if stage == 1 { #expect(seen.withLock { ($0 ?? 0) > 4098 }) } + } +} + +@Test func jcompOriginalHealthAndIdentitySurvivePackageSubstitution() throws { + for stage in 0..<3 { + do { + _ = try JSON.withComposition { scope in + try scope.withMeter { meter in + if stage == 1 { + do { try meter.work(UInt64.max) } catch { } + } else if stage == 2 { + do { try meter.retain(UInt64.max) } catch { } + } + // Impossible through public API; exercise the original + // continuation's identity and first-cause check directly. + meter = try JSONWorkMeter(limits: .balanced, domain: .core) + } + return 1 + } + Issue.record("substituted native ledger completed") + } catch let failure as JSONFailure { + if stage == 0 { #expect(failure.category == .configuration && failure.code == .invalidOperation) } + if stage == 1 { #expect(failure.resource?.metric == .workUnits) } + if stage == 2 { #expect(failure.resource?.metric == .retainedBytes) } + } + } +} diff --git a/Tools/CompositionQualification/CompilerBugs/BoundDirect.swift.txt b/Tools/CompositionQualification/CompilerBugs/BoundDirect.swift.txt new file mode 100644 index 0000000..ca8c4f3 --- /dev/null +++ b/Tools/CompositionQualification/CompilerBugs/BoundDirect.swift.txt @@ -0,0 +1,4 @@ +// SPDX-License-Identifier: Apache-2.0 +import JSON +let report = try JSON.withComposition { scope in return scope.checkpoint } +try report.result() diff --git a/Tools/CompositionQualification/CompilerBugs/BoundHelper.swift.txt b/Tools/CompositionQualification/CompilerBugs/BoundHelper.swift.txt new file mode 100644 index 0000000..182a48d --- /dev/null +++ b/Tools/CompositionQualification/CompilerBugs/BoundHelper.swift.txt @@ -0,0 +1,3 @@ +// SPDX-License-Identifier: Apache-2.0 +import JSON +func rejected(_ scope: borrowing JSONCompositionScope) -> () throws -> Void { scope.checkpoint } diff --git a/Tools/CompositionQualification/CompilerBugs/DirectEscape.swift.txt b/Tools/CompositionQualification/CompilerBugs/DirectEscape.swift.txt new file mode 100644 index 0000000..d51543f --- /dev/null +++ b/Tools/CompositionQualification/CompilerBugs/DirectEscape.swift.txt @@ -0,0 +1,5 @@ +// SPDX-License-Identifier: Apache-2.0 +import JSON +import JSONSchema +var escaped: (() throws -> Void)? +_ = try JSON.withComposition { scope in escaped = { try scope.checkpoint() } } diff --git a/Tools/CompositionQualification/CompilerBugs/NestedHelper.swift.txt b/Tools/CompositionQualification/CompilerBugs/NestedHelper.swift.txt new file mode 100644 index 0000000..13786e4 --- /dev/null +++ b/Tools/CompositionQualification/CompilerBugs/NestedHelper.swift.txt @@ -0,0 +1,8 @@ +// SPDX-License-Identifier: Apache-2.0 +import JSON +func reproducer() throws { + _ = try JSON.withComposition { scope in + func local() throws { try scope.chargeExternalWork(1) } + try local() + } +} diff --git a/Tools/CompositionQualification/CompilerBugs/README.md b/Tools/CompositionQualification/CompilerBugs/README.md new file mode 100644 index 0000000..47ce497 --- /dev/null +++ b/Tools/CompositionQualification/CompilerBugs/README.md @@ -0,0 +1,42 @@ +# Swift 6.4 ownership compiler investigation + +Original public-product reproducers are in this directory. `Reduced` contains +original Apache-2.0 reductions with no imports, private native meter or package +code. The class holds one mutable integer, the facade is noncopyable and the +synchronous entry borrows it. The defect persists without Foundation or JSON. + +Replay with a selected installed compiler and the supported macOS SDK: + +```sh +python3 -B Scripts/probe_composition_compiler.py --compiler /path/to/swiftc --output .build/composition-compiler +``` + +This investigation deliberately returns **2 when a compiler crash is observed**. +It is separate from the composition lane's intended diagnostic builds. Never +interpret a signal, stack dump or verifier failure as a passing negative test. +The receipt preserves the command, source/log hashes, version, exit and outcome. + +Observed on Apple Swift 6.4.0.34.1 and Swift.org `swift-6.4-RELEASE`: + +| Form | Outcome | +| --- | --- | +| Inferred borrowed closure parameter captured by nested synchronous helper | SIL ownership verifier crash | +| Explicit borrowing closure parameter, nested synchronous helper | Compiles | +| Inferred parameter captured by escaping closure | SIL ownership verifier crash | +| Explicit borrowing closure parameter, escaping capture | Intended borrowed-parameter diagnostic | +| Direct method reference, inferred or explicit borrowing parameter | SIL ownership verifier crash | +| Method reference from explicit borrowed helper | SIL ownership verifier crash | +| Explicit helper parameter with direct method call | Compiles | +| Escaping closure returned from explicit borrowed helper | Intended borrowed-parameter diagnostic | + +The normal diagnostic is `'scope' cannot be captured by an escaping closure since +it is a borrowed parameter`. Crash diagnostics include `Have operand with +incompatible ownership?!` and `copy_addr`. These are observations on two specific +installed compilers; no claim is made about unexecuted newer releases. + +Use an explicit borrowing closure parameter and direct calls, or pass the scope +as an explicit borrowing parameter to ordinary helpers. The independent consumer +executes the nested form, checks exact aggregate work/retention, and fails one unit +below completion. No lifetime experiment flag, copyable facade, unchecked +Sendable or arbitrary callback instruction claim is needed. The method-value forms +remain unqualified. No compiler issue has been sent on the maintainer's behalf. diff --git a/Tools/CompositionQualification/CompilerBugs/Reduced/bound-direct.swift.txt b/Tools/CompositionQualification/CompilerBugs/Reduced/bound-direct.swift.txt new file mode 100644 index 0000000..a94d039 --- /dev/null +++ b/Tools/CompositionQualification/CompilerBugs/Reduced/bound-direct.swift.txt @@ -0,0 +1,11 @@ +// SPDX-License-Identifier: Apache-2.0 +private final class Ledger { var units = 0 } +struct Scope: ~Copyable { + private let ledger = Ledger() + func checkpoint() throws { ledger.units += 1 } +} +func compose(_ body: (borrowing Scope) throws -> R) rethrows -> R { + let scope = Scope() + return try body(scope) +} +func test() throws { let result = try compose { scope in return scope.checkpoint }; try result() } diff --git a/Tools/CompositionQualification/CompilerBugs/Reduced/bound-explicit.swift.txt b/Tools/CompositionQualification/CompilerBugs/Reduced/bound-explicit.swift.txt new file mode 100644 index 0000000..63a5e52 --- /dev/null +++ b/Tools/CompositionQualification/CompilerBugs/Reduced/bound-explicit.swift.txt @@ -0,0 +1,11 @@ +// SPDX-License-Identifier: Apache-2.0 +private final class Ledger { var units = 0 } +struct Scope: ~Copyable { + private let ledger = Ledger() + func checkpoint() throws { ledger.units += 1 } +} +func compose(_ body: (borrowing Scope) throws -> R) rethrows -> R { + let scope = Scope() + return try body(scope) +} +func test() throws { let result = try compose { (scope: borrowing Scope) in return scope.checkpoint }; try result() } diff --git a/Tools/CompositionQualification/CompilerBugs/Reduced/bound-helper.swift.txt b/Tools/CompositionQualification/CompilerBugs/Reduced/bound-helper.swift.txt new file mode 100644 index 0000000..aed3927 --- /dev/null +++ b/Tools/CompositionQualification/CompilerBugs/Reduced/bound-helper.swift.txt @@ -0,0 +1,11 @@ +// SPDX-License-Identifier: Apache-2.0 +private final class Ledger { var units = 0 } +struct Scope: ~Copyable { + private let ledger = Ledger() + func checkpoint() throws { ledger.units += 1 } +} +func compose(_ body: (borrowing Scope) throws -> R) rethrows -> R { + let scope = Scope() + return try body(scope) +} +func test(_ scope: borrowing Scope) -> () throws -> Void { scope.checkpoint } diff --git a/Tools/CompositionQualification/CompilerBugs/Reduced/escape-explicit.swift.txt b/Tools/CompositionQualification/CompilerBugs/Reduced/escape-explicit.swift.txt new file mode 100644 index 0000000..9ba93fc --- /dev/null +++ b/Tools/CompositionQualification/CompilerBugs/Reduced/escape-explicit.swift.txt @@ -0,0 +1,11 @@ +// SPDX-License-Identifier: Apache-2.0 +private final class Ledger { var units = 0 } +struct Scope: ~Copyable { + private let ledger = Ledger() + func checkpoint() throws { ledger.units += 1 } +} +func compose(_ body: (borrowing Scope) throws -> R) rethrows -> R { + let scope = Scope() + return try body(scope) +} +func test() throws { var escaped: (() throws -> Void)?; _ = try compose { (scope: borrowing Scope) in escaped = { try scope.checkpoint() } }; try escaped?() } diff --git a/Tools/CompositionQualification/CompilerBugs/Reduced/escape-inferred.swift.txt b/Tools/CompositionQualification/CompilerBugs/Reduced/escape-inferred.swift.txt new file mode 100644 index 0000000..0b8bb39 --- /dev/null +++ b/Tools/CompositionQualification/CompilerBugs/Reduced/escape-inferred.swift.txt @@ -0,0 +1,11 @@ +// SPDX-License-Identifier: Apache-2.0 +private final class Ledger { var units = 0 } +struct Scope: ~Copyable { + private let ledger = Ledger() + func checkpoint() throws { ledger.units += 1 } +} +func compose(_ body: (borrowing Scope) throws -> R) rethrows -> R { + let scope = Scope() + return try body(scope) +} +func test() throws { var escaped: (() throws -> Void)?; _ = try compose { scope in escaped = { try scope.checkpoint() } }; try escaped?() } diff --git a/Tools/CompositionQualification/CompilerBugs/Reduced/explicit-helper-negative.swift.txt b/Tools/CompositionQualification/CompilerBugs/Reduced/explicit-helper-negative.swift.txt new file mode 100644 index 0000000..37dce69 --- /dev/null +++ b/Tools/CompositionQualification/CompilerBugs/Reduced/explicit-helper-negative.swift.txt @@ -0,0 +1,11 @@ +// SPDX-License-Identifier: Apache-2.0 +private final class Ledger { var units = 0 } +struct Scope: ~Copyable { + private let ledger = Ledger() + func checkpoint() throws { ledger.units += 1 } +} +func compose(_ body: (borrowing Scope) throws -> R) rethrows -> R { + let scope = Scope() + return try body(scope) +} +func test(_ scope: borrowing Scope) -> () throws -> Void { { try scope.checkpoint() } } diff --git a/Tools/CompositionQualification/CompilerBugs/Reduced/explicit-helper-positive.swift.txt b/Tools/CompositionQualification/CompilerBugs/Reduced/explicit-helper-positive.swift.txt new file mode 100644 index 0000000..279f6c5 --- /dev/null +++ b/Tools/CompositionQualification/CompilerBugs/Reduced/explicit-helper-positive.swift.txt @@ -0,0 +1,11 @@ +// SPDX-License-Identifier: Apache-2.0 +private final class Ledger { var units = 0 } +struct Scope: ~Copyable { + private let ledger = Ledger() + func checkpoint() throws { ledger.units += 1 } +} +func compose(_ body: (borrowing Scope) throws -> R) rethrows -> R { + let scope = Scope() + return try body(scope) +} +func visit(_ scope: borrowing Scope) throws { try scope.checkpoint() }; func test() throws { try compose { scope in try visit(scope) } } diff --git a/Tools/CompositionQualification/CompilerBugs/Reduced/nested-explicit.swift.txt b/Tools/CompositionQualification/CompilerBugs/Reduced/nested-explicit.swift.txt new file mode 100644 index 0000000..5bfee16 --- /dev/null +++ b/Tools/CompositionQualification/CompilerBugs/Reduced/nested-explicit.swift.txt @@ -0,0 +1,13 @@ +// SPDX-License-Identifier: Apache-2.0 +private final class Ledger { var units = 0 } +struct Scope: ~Copyable { + private let ledger = Ledger() + func checkpoint() throws { ledger.units += 1 } +} +func compose(_ body: (borrowing Scope) throws -> R) rethrows -> R { + let scope = Scope() + return try body(scope) +} +func test() throws { _ = try compose { (scope: borrowing Scope) in + func local() throws { try scope.checkpoint() }; try local() +} } diff --git a/Tools/CompositionQualification/CompilerBugs/Reduced/nested-inferred.swift.txt b/Tools/CompositionQualification/CompilerBugs/Reduced/nested-inferred.swift.txt new file mode 100644 index 0000000..af781fa --- /dev/null +++ b/Tools/CompositionQualification/CompilerBugs/Reduced/nested-inferred.swift.txt @@ -0,0 +1,13 @@ +// SPDX-License-Identifier: Apache-2.0 +private final class Ledger { var units = 0 } +struct Scope: ~Copyable { + private let ledger = Ledger() + func checkpoint() throws { ledger.units += 1 } +} +func compose(_ body: (borrowing Scope) throws -> R) rethrows -> R { + let scope = Scope() + return try body(scope) +} +func test() throws { _ = try compose { scope in + func local() throws { try scope.checkpoint() }; try local() +} } diff --git a/Tools/CompositionQualification/Package.swift b/Tools/CompositionQualification/Package.swift index 47d1933..436e811 100644 --- a/Tools/CompositionQualification/Package.swift +++ b/Tools/CompositionQualification/Package.swift @@ -3,9 +3,15 @@ import PackageDescription let package = Package( name: "CompositionQualification", - platforms: [.macOS("27.0")], + platforms: [.macOS("27.0"), .iOS("27.0")], dependencies: [.package(name: "Swift-JSON", path: "../..")], - targets: [.target(name: "CompositionFixtures", dependencies: [ + targets: [.executableTarget(name: "CompositionBenchmark", dependencies: [ + .product(name: "JSON", package: "Swift-JSON"), .product(name: "JSONSchema", package: "Swift-JSON") + ]), .target(name: "CompositionProbe", dependencies: [ + .product(name: "JSON", package: "Swift-JSON"), .product(name: "JSONSchema", package: "Swift-JSON") + ]), .testTarget(name: "CompositionPublicTests", dependencies: [ + .product(name: "JSON", package: "Swift-JSON"), .product(name: "JSONSchema", package: "Swift-JSON") + ]), .target(name: "CompositionFixtures", dependencies: [ .product(name: "JSON", package: "Swift-JSON"), .product(name: "JSONSchema", package: "Swift-JSON") ]), .testTarget(name: "CompositionScaleTests", dependencies: [ diff --git a/Tools/CompositionQualification/ProofNegative/Async.swift.txt b/Tools/CompositionQualification/ProofNegative/Async.swift.txt new file mode 100644 index 0000000..10a764e --- /dev/null +++ b/Tools/CompositionQualification/ProofNegative/Async.swift.txt @@ -0,0 +1,6 @@ +// SPDX-License-Identifier: Apache-2.0 +import JSON +import JSONSchema +func illegal() async throws { + _ = try await JSON.withComposition { scope in await Task.yield(); try scope.checkpoint() } +} diff --git a/Tools/CompositionQualification/ProofNegative/Construct.swift.txt b/Tools/CompositionQualification/ProofNegative/Construct.swift.txt new file mode 100644 index 0000000..36ec7cf --- /dev/null +++ b/Tools/CompositionQualification/ProofNegative/Construct.swift.txt @@ -0,0 +1,4 @@ +// SPDX-License-Identifier: Apache-2.0 +import JSON +import JSONSchema +let scope = try JSONCompositionScope(limits: .balanced) diff --git a/Tools/CompositionQualification/ProofNegative/Copy.swift.txt b/Tools/CompositionQualification/ProofNegative/Copy.swift.txt new file mode 100644 index 0000000..45f818f --- /dev/null +++ b/Tools/CompositionQualification/ProofNegative/Copy.swift.txt @@ -0,0 +1,4 @@ +// SPDX-License-Identifier: Apache-2.0 +import JSON +import JSONSchema +_ = try JSON.withComposition { scope in let duplicate = copy scope; try duplicate.checkpoint() } diff --git a/Tools/CompositionQualification/ProofNegative/Escape.swift.txt b/Tools/CompositionQualification/ProofNegative/Escape.swift.txt new file mode 100644 index 0000000..9e5c604 --- /dev/null +++ b/Tools/CompositionQualification/ProofNegative/Escape.swift.txt @@ -0,0 +1,6 @@ +// SPDX-License-Identifier: Apache-2.0 +import JSON +import JSONSchema +func rejected(_ scope: borrowing JSONCompositionScope) -> () throws -> Void { + { try scope.checkpoint() } +} diff --git a/Tools/CompositionQualification/ProofNegative/ExplicitEscape.swift.txt b/Tools/CompositionQualification/ProofNegative/ExplicitEscape.swift.txt new file mode 100644 index 0000000..08a5b76 --- /dev/null +++ b/Tools/CompositionQualification/ProofNegative/ExplicitEscape.swift.txt @@ -0,0 +1,9 @@ +// SPDX-License-Identifier: Apache-2.0 +import JSON +func rejected() throws { + var escaped: (() throws -> Void)? + _ = try JSON.withComposition { (scope: borrowing JSONCompositionScope) in + escaped = { try scope.checkpoint() } + } + try escaped?() +} diff --git a/Tools/CompositionQualification/ProofNegative/Fork.swift.txt b/Tools/CompositionQualification/ProofNegative/Fork.swift.txt new file mode 100644 index 0000000..ecb8efe --- /dev/null +++ b/Tools/CompositionQualification/ProofNegative/Fork.swift.txt @@ -0,0 +1,4 @@ +// SPDX-License-Identifier: Apache-2.0 +import JSON +import JSONSchema +_ = try JSON.withComposition { scope in try scope.fork() } diff --git a/Tools/CompositionQualification/ProofNegative/PackageMeter.swift.txt b/Tools/CompositionQualification/ProofNegative/PackageMeter.swift.txt new file mode 100644 index 0000000..fa89bd9 --- /dev/null +++ b/Tools/CompositionQualification/ProofNegative/PackageMeter.swift.txt @@ -0,0 +1,4 @@ +// SPDX-License-Identifier: Apache-2.0 +import JSON +import JSONSchema +_ = try JSON.withComposition { scope in try scope.withMeter { _ in 0 } } diff --git a/Tools/CompositionQualification/ProofNegative/RawMeter.swift.txt b/Tools/CompositionQualification/ProofNegative/RawMeter.swift.txt new file mode 100644 index 0000000..75b2279 --- /dev/null +++ b/Tools/CompositionQualification/ProofNegative/RawMeter.swift.txt @@ -0,0 +1,4 @@ +// SPDX-License-Identifier: Apache-2.0 +import JSON +import JSONSchema +let meter = try JSONWorkMeter(limits: .balanced, domain: .core) diff --git a/Tools/CompositionQualification/ProofNegative/Refund.swift.txt b/Tools/CompositionQualification/ProofNegative/Refund.swift.txt new file mode 100644 index 0000000..b940bf7 --- /dev/null +++ b/Tools/CompositionQualification/ProofNegative/Refund.swift.txt @@ -0,0 +1,4 @@ +// SPDX-License-Identifier: Apache-2.0 +import JSON +import JSONSchema +_ = try JSON.withComposition { scope in try scope.release(1) } diff --git a/Tools/CompositionQualification/ProofNegative/Reset.swift.txt b/Tools/CompositionQualification/ProofNegative/Reset.swift.txt new file mode 100644 index 0000000..c50d1c7 --- /dev/null +++ b/Tools/CompositionQualification/ProofNegative/Reset.swift.txt @@ -0,0 +1,4 @@ +// SPDX-License-Identifier: Apache-2.0 +import JSON +import JSONSchema +_ = try JSON.withComposition { scope in try scope.reset() } diff --git a/Tools/CompositionQualification/ProofNegative/Return.swift.txt b/Tools/CompositionQualification/ProofNegative/Return.swift.txt new file mode 100644 index 0000000..2ebe84e --- /dev/null +++ b/Tools/CompositionQualification/ProofNegative/Return.swift.txt @@ -0,0 +1,4 @@ +// SPDX-License-Identifier: Apache-2.0 +import JSON +import JSONSchema +let scope = try JSON.withComposition { scope in return scope } diff --git a/Tools/CompositionQualification/ProofNegative/Sendable.swift.txt b/Tools/CompositionQualification/ProofNegative/Sendable.swift.txt new file mode 100644 index 0000000..be9a833 --- /dev/null +++ b/Tools/CompositionQualification/ProofNegative/Sendable.swift.txt @@ -0,0 +1,5 @@ +// SPDX-License-Identifier: Apache-2.0 +import JSON +import JSONSchema +func needsSendable(_ value: borrowing T) {} +_ = try JSON.withComposition { scope in needsSendable(scope) } diff --git a/Tools/CompositionQualification/ProofNegative/Store.swift.txt b/Tools/CompositionQualification/ProofNegative/Store.swift.txt new file mode 100644 index 0000000..37fd253 --- /dev/null +++ b/Tools/CompositionQualification/ProofNegative/Store.swift.txt @@ -0,0 +1,5 @@ +// SPDX-License-Identifier: Apache-2.0 +import JSON +import JSONSchema +var saved: JSONCompositionScope? +_ = try JSON.withComposition { scope in saved = consume scope } diff --git a/Tools/CompositionQualification/ProofNegative/Substitute.swift.txt b/Tools/CompositionQualification/ProofNegative/Substitute.swift.txt new file mode 100644 index 0000000..a679266 --- /dev/null +++ b/Tools/CompositionQualification/ProofNegative/Substitute.swift.txt @@ -0,0 +1,6 @@ +// SPDX-License-Identifier: Apache-2.0 +import JSON +import JSONSchema +_ = try JSON.withComposition { outer in + _ = try JSON.withComposition { inner in swap(&outer, &inner) } +} diff --git a/Tools/CompositionQualification/README.md b/Tools/CompositionQualification/README.md index f61927e..041ff77 100644 --- a/Tools/CompositionQualification/README.md +++ b/Tools/CompositionQualification/README.md @@ -46,3 +46,13 @@ and all 11 compiler rejection cases. Five cells complete and three deliberately those refusals remain operational errors. The initial direct 128/128 balanced capacity assumption failed and is preserved. Staging the reference array on the same scope qualifies that bounded target without changing core APIs or limits. See ../../Documentation/Architecture/CompositionQualification.md. + +## Reconciled independent accounting proof + +Run `python3 Scripts/ci.py --lane composition-proof` in addition to the reference-import +lane above. It runs 26 public functions in Debug/Release, two actual omitted-charge +mutants and 14 unrelated SwiftPM diagnostic builds. `Negative/` preserves the 11 +original compiler cases; `ProofNegative/` contains the independent 14-case suite. +Compiler crashes are retained separately under `CompilerBugs/` and never satisfy +an intended rejection. The package uses ordinary public JSON/JSONSchema products. +See [the reconciliation record](../../Documentation/Qualification/Composition/Reconciliation.md). diff --git a/Tools/CompositionQualification/Sources/CompositionBenchmark/main.swift b/Tools/CompositionQualification/Sources/CompositionBenchmark/main.swift new file mode 100644 index 0000000..46292bb --- /dev/null +++ b/Tools/CompositionQualification/Sources/CompositionBenchmark/main.swift @@ -0,0 +1,61 @@ +// SPDX-License-Identifier: Apache-2.0 +import Foundation +import JSON +import JSONSchema + +struct Sample: Codable { + let name: String + let inputBytes: Int + let iterations: Int + let seconds: [Double] + let work: UInt64? + let peak: UInt64? +} +// Deterministic 512-field exact/scalar workload, not an external conformance corpus. +let text = "{" + (0..<512).map { "\"field\($0)\":" + #"{"n":9007199254740993,"s":"é","a":[true,null,0.01]}"# }.joined(separator: ",") + "}" +let realPath = ProcessInfo.processInfo.environment["JCOMP_BENCHMARK_INPUT"] +let data = try realPath.map { try Data(contentsOf: URL(fileURLWithPath: $0)) } ?? Data(text.utf8) +let bytes = Array(data) +let expectedMembers = realPath == nil ? 512 : 2 +let root = try JSONParser().parse(bytes) +let schema = try JSONSchemaDocument(JSONParser().parse(#"{"type":"object","properties":{"n":{"type":"integer"},"s":{"type":"string"}},"required":["n"]}"#)) +let parserProbe = try JSONParser().parseWithReport(bytes) +let schemaProbe = try JSONSchemaCompiler().compileWithReport(schema) +let compositionProbe = try JSON.withComposition(limits: .offline) { try $0.parseDocument(data) } +let clock = ContinuousClock() +func seconds(_ duration: Duration) -> Double { Double(duration.components.seconds) + Double(duration.components.attoseconds) / 1e18 } +func measure(_ name: String, iterations: Int, work: UInt64? = nil, peak: UInt64? = nil, _ operation: () throws -> Void) throws -> Sample { + for _ in 0..<2 { for _ in 0.. JSONOperationReport { + try JSON.withComposition { (scope: borrowing JSONCompositionScope) in + let document = try scope.parseDocument(data) + try adapterStep(scope) + let path = try scope.pointer(tokens: ["count"]) + let uri = try scope.uri("https://example.invalid/schema.json#/$defs/count") + _ = try scope.removingFragment(uri) + return try scope.resolve(path, in: document.root) + } +} + +private func adapterStep(_ scope: borrowing JSONCompositionScope) throws { + try scope.reserveExternalLogicalBytes(1, count: 32, overhead: 32) + try scope.chargeExternalWork(1) +} diff --git a/Tools/CompositionQualification/Tests/CompositionPublicTests/CompositionTests.swift b/Tools/CompositionQualification/Tests/CompositionPublicTests/CompositionTests.swift new file mode 100644 index 0000000..72ad6b1 --- /dev/null +++ b/Tools/CompositionQualification/Tests/CompositionPublicTests/CompositionTests.swift @@ -0,0 +1,370 @@ +// SPDX-License-Identifier: Apache-2.0 +import Foundation +import Testing +import JSON +import JSONSchema + +private enum AuditError: Error { case workTrace, retentionTrace, caller } +private let source = Data(##"{"n":9007199254740993,"adjacent":9007199254740992,"long":123456789012345678901234567890.01234567890123456789,"fraction":0.01,"negative":-0,"é":true,"e\u0301":false,"a/b":{"~":null},"items":[false],"type":["string","null"],"$dynamicRef":"#node","$schema":"https://unknown.invalid/dialect"}"##.utf8) + +// Independent audited-adapter ledger, not derived from operation statistics. +// Three bounded buffer fills: one frame visit and one unit per written byte; +// 32 logical header bytes per buffer. Final native receipt copies 49 slots and +// settles its aggregate slot (50 work), retaining 32 + 49*8 logical bytes. +private let goldenWork: UInt64 = 197 +private let goldenBytes: UInt64 = 664 +private func fill(_ scope: borrowing JSONCompositionScope, count: Int, first: Bool, mutation: String?) throws -> [UInt8] { + let reservation = count - (first && mutation == "reservation" ? 1 : 0) + try scope.reserveExternalLogicalBytes(1, count: UInt64(reservation), overhead: 32) + try scope.chargeExternalWork(1) + var bytes: [UInt8] = [] + bytes.reserveCapacity(count) + for i in 0.. JSONOperationReport<([UInt8], [UInt8], [UInt8])> { + let mutation = ProcessInfo.processInfo.environment["JCOMP_TRACE_MUTATION"] + return try JSON.withComposition(limits: limits) { scope in + let queue = try fill(scope, count: 64, first: true, mutation: mutation) + let scratch = try fill(scope, count: 32, first: false, mutation: mutation) + let output = try fill(scope, count: 48, first: false, mutation: mutation) + try scope.chargeExternalWork(extra) + return (queue, scratch, output) + } +} +private func audit(_ report: JSONOperationReport<([UInt8], [UInt8], [UInt8])>) throws { + guard report.statistics[.workUnits] == goldenWork else { throw AuditError.workTrace } + guard report.statistics[.retainedBytes] == goldenBytes, + report.statistics.peakLogicalRetainedBytes == goldenBytes else { throw AuditError.retentionTrace } +} +private func capture(_ body: () throws -> Void) -> (any Error)? { + do { try body(); Issue.record("expected an error"); return nil } catch { return error } +} + +@Test func externalTraceMatchesIndependentLedger() throws { + let report = try traceAdapter() + #expect(report.result.0.count == 64 && report.result.1.count == 32 && report.result.2.count == 48) + try audit(report) + if let directory = ProcessInfo.processInfo.environment["JCOMP_TRACE_DIRECTORY"] { + let trace = "owner\twork\tlogical_bytes\nqueue\t65\t96\nscratch\t33\t64\nresult\t49\t80\nreceipt\t50\t424\n" + try trace.write(toFile: directory + "/owned-events.tsv", atomically: true, encoding: .utf8) + } +} + +@Test func exactWorkAndRetentionLimitsAreActive() throws { + var exact = JSONLimits.balanced + exact[.workUnits] = goldenWork; exact[.retainedBytes] = goldenBytes + try audit(traceAdapter(exact)) + for metric in [JSONResourceMetric.workUnits, .retainedBytes] { + var below = exact; below[metric] -= 1 + let failure = capture { _ = try traceAdapter(below) } as? JSONFailure + #expect(failure?.category == .resource && failure?.resource?.metric == metric) + } + #expect(throws: JSONFailure.self) { _ = try traceAdapter(exact, extra: 1) } +} + +@Test func reportOverlapIsAdmittedByFinalizer() throws { + var limits = JSONLimits.balanced + limits[.retainedBytes] = 424; limits[.workUnits] = 50 + let report = try JSON.withComposition(limits: limits) { _ in 7 } + #expect(report.result == 7 && report.statistics[.retainedBytes] == 424) + #expect(report.statistics[.workUnits] == 50 && report.statistics.peakLogicalRetainedBytes == 424) + limits[.retainedBytes] = 423 + #expect(throws: JSONFailure.self) { try JSON.withComposition(limits: limits) { _ in 7 } } + limits[.retainedBytes] = 424; limits[.workUnits] = 49 + #expect(throws: JSONFailure.self) { try JSON.withComposition(limits: limits) { _ in 7 } } +} + +@Test func finiteMaximumWorkDoesNotOverflowCompletion() throws { + var limits = JSONLimits.balanced; limits[.workUnits] = UInt64.max - 1 + let report = try JSON.withComposition(limits: limits) { scope in try scope.chargeExternalWork(UInt64.max - 51) } + #expect(report.statistics[.workUnits] == UInt64.max - 1) + #expect(throws: JSONFailure.self) { + try JSON.withComposition(limits: limits) { scope in try scope.chargeExternalWork(UInt64.max - 50) } + } +} + +@Test func checkedProductAndOverheadFailBeforeGrowth() throws { + for mode in 0..<2 { + var growth = false + let error = capture { + _ = try JSON.withComposition { scope in + do { + if mode == 0 { try scope.reserveExternalLogicalBytes(UInt64.max, count: 2) } + else { try scope.reserveExternalLogicalBytes(1, overhead: UInt64.max) } + growth = true + } catch let failure as JSONFailure { #expect(failure.code == .counterOverflow) } + return 0 + } + } as? JSONFailure + #expect(!growth && error?.code == .counterOverflow) + } +} + +@Test func zeroReservationAndZeroQuotaHaveDistinctMeaning() throws { + let report = try JSON.withComposition { try $0.reserveExternalLogicalBytes(0, count: 0) } + #expect(report.statistics[.retainedBytes] == 424) + var none = JSONLimits.balanced; none[.workUnits] = 0 + #expect(throws: JSONFailure.self) { + try JSON.withComposition(limits: none) { scope in do { try scope.chargeExternalWork(1) } catch { } } + } +} + +@Test func firstQuotaFailureSurvivesCatchCleanupAndCallerError() throws { + var limits = JSONLimits.balanced; limits[.workUnits] = 1 + let error = capture { + _ = try JSON.withComposition(limits: limits) { scope in + do { try scope.chargeExternalWork(2) } catch { } + do { try scope.reserveExternalLogicalBytes(UInt64.max, count: 2) } catch { } + throw AuditError.caller + } + } as? JSONFailure + #expect(error?.code == .limitExceeded && error?.resource?.metric == .workUnits) + #expect(error?.resource?.attemptedUsage == 2) +} + +@Test func firstOverflowSurvivesLaterCalls() throws { + let error = capture { + _ = try JSON.withComposition { scope in + try scope.chargeExternalWork(1) + do { try scope.chargeExternalWork(UInt64.max) } catch { } + do { _ = try scope.parseDocument(Data("{".utf8)) } catch { } + return 7 + } + } as? JSONFailure + #expect(error?.code == .counterOverflow && error?.resource?.metric == .workUnits) +} + +@Test func freshInvocationHasFreshLedgerAfterFailure() throws { + #expect(throws: JSONFailure.self) { + try JSON.withComposition { try $0.reserveExternalLogicalBytes(UInt64.max, count: 2) } + } + try audit(traceAdapter()) +} + +@Test func preCopyCapacityFailurePrecedesAnyCopyWork() throws { + var limits = JSONLimits.balanced; limits[.workUnits] = 0 + limits[.inputBytes] = UInt64(source.count - 1) + let input = capture { _ = try JSON.withComposition(limits: limits) { try $0.parseDocument(source) } } as? JSONFailure + #expect(input?.resource?.metric == .inputBytes && input?.resource?.attemptedUsage == UInt64(source.count)) + limits[.inputBytes] = UInt64(source.count); limits[.retainedBytes] = UInt64(source.count - 1) + let bytes = capture { _ = try JSON.withComposition(limits: limits) { try $0.parseDocument(source) } } as? JSONFailure + #expect(bytes?.resource?.metric == .retainedBytes && bytes?.resource?.attemptedUsage == UInt64(source.count)) +} + +@Test func sourceLexemesScalarKeysAndSpansArePreserved() throws { + let report = try JSON.withComposition { try $0.parseDocument(source) } + let doc = report.result; let ordinary = try JSONParser().parseDocument(source) + #expect(doc.root == ordinary.root && doc.sourceBytes == source) + #expect(doc.root.objectValue?["n"]?.numberValue?.originalLexeme == "9007199254740993") + #expect(doc.root.objectValue?["adjacent"]?.numberValue?.originalLexeme == "9007199254740992") + #expect(doc.root.objectValue?["long"]?.numberValue?.originalLexeme == "123456789012345678901234567890.01234567890123456789") + #expect(doc.root.objectValue?["fraction"]?.numberValue?.originalLexeme == "0.01") + #expect(doc.root.objectValue?["negative"]?.numberValue?.originalLexeme == "-0") + #expect(doc.root.objectValue?["é"] == true && doc.root.objectValue?["e\u{301}"] == false) + #expect(doc.sourceMap?.entries.count == ordinary.sourceMap?.entries.count) + #expect(report.statistics[.numericWorkUnits] > 0 && report.statistics[.sourceMapEntries] > 0) +} + +@Test func perDocumentInputBoundAndCumulativeResources() throws { + let bytes = Data("null".utf8) + var limits = JSONLimits.balanced; limits[.inputBytes] = 4 + let report = try JSON.withComposition(limits: limits) { scope in + let entry = try scope.parseDocument(bytes) + let a = try scope.parseDocument(bytes) + let b = try scope.parseDocument(bytes) + return (entry, a, b) + } + #expect(report.result.0.sourceBytes == bytes && report.result.1.sourceBytes == bytes && report.result.2.sourceBytes == bytes) + #expect(report.statistics[.inputBytes] == 12) + // Independent ownership lower bound: 3*(source+copy+node+map) + receipt. + #expect(report.statistics[.retainedBytes] >= 3 * (8 + 32 + 32) + 424) + limits[.inputBytes] = 3 + #expect(throws: JSONFailure.self) { try JSON.withComposition(limits: limits) { try $0.parseDocument(bytes) } } +} + +@Test func emptyAndLongSourceRemainStrictAndBounded() throws { + for bytes in [Data(), Data(" ".utf8), Data("null true".utf8)] { + #expect(throws: JSONFailure.self) { try JSON.withComposition { try $0.parseDocument(bytes) } } + } + let bytes = Data(("\"" + String(repeating: "x", count: 8192) + "\"").utf8) + let doc = try JSON.withComposition { try $0.parseDocument(bytes) }.result + #expect(doc.sourceBytes == bytes && doc.root.stringValue?.utf8.count == 8192) +} + +@Test func decodedDuplicateKeysStillFail() throws { + let bytes = Data(#"{"é":0,"\u00e9":1}"#.utf8) + #expect(throws: JSONFailure.self) { try JSON.withComposition { try $0.parseDocument(bytes) } } +} + +@Test func nativePointerEscapesAndScalarIdentity() throws { + try JSON.withComposition { scope in + let doc = try scope.parseDocument(source) + let root = try scope.pointer(parsing: "") + #expect(try scope.resolve(root, in: doc.root) == doc.root) + let path = try scope.pointer(tokens: ["a/b", "~"]) + #expect(try scope.resolve(path, in: doc.root) == .null) + let scalar = try scope.pointer(uriFragment: "#/%C3%A9") + #expect(try scope.resolve(scalar, in: doc.root) == true) + let decomposed = try scope.appending(root, key: "e\u{301}") + #expect(try scope.resolve(decomposed, in: doc.root) == false) + } +} + +@Test func nativePointerIndexAndPercentGrammarAreNotPermissive() throws { + for text in ["/01", "/-", "/-1", "/999999999999999999999999"] { + #expect(throws: JSONFailure.self) { + try JSON.withComposition { scope in try scope.resolve(scope.pointer(parsing: text), in: .array([.null])) } + } + } + for text in ["#/%", "#/%FF", "#/%7E2"] { + #expect(throws: JSONFailure.self) { try JSON.withComposition { try $0.pointer(uriFragment: text) } } + } + let pointer = try JSON.withComposition { try $0.pointer(uriFragment: "#/%252F") }.result + #expect(pointer.tokens.first?.string == "%2F") +} + +@Test func nativeURICompositionUsesCoreRatherThanCompiledPlanQuota() throws { + var limits = JSONLimits.balanced; limits[.compiledPlanBytes] = 0; limits[.compileWorkUnits] = 0 + let report = try JSON.withComposition(limits: limits) { scope in + let base = try scope.uri("https://EXAMPLE.invalid/a/../api.json#part") + let relative = try scope.uri("schemas.json#/n", relativeTo: base) + let resource = try scope.removingFragment(relative) + #expect(resource.string == "https://example.invalid/schemas.json") + return relative + } + let ordinary = try JSONSchemaURI("schemas.json#/n", relativeTo: JSONSchemaURI("https://EXAMPLE.invalid/a/../api.json#part")) + #expect(report.result == ordinary) + #expect(report.statistics[.compiledPlanBytes] == 0 && report.statistics[.compileWorkUnits] == 0) + #expect(report.statistics.domain == .core) +} + +@Test func URIDataFailureIsStickyAndNeverFetches() throws { + #expect(throws: (any Error).self) { + try JSON.withComposition { scope in do { _ = try scope.uri("https://example.invalid/%ZZ") } catch { }; return 0 } + } +} + +@Test func acceptedInputHistorySurvivesRestriction() throws { + let error = capture { + _ = try JSON.withComposition { scope in + _ = try scope.parseDocument(source) + // Retire the larger input count. The current input is now only four + // bytes; restriction must still consult accepted-capacity history. + _ = try scope.parseDocument(Data("null".utf8)) + var lower = JSONLimits.balanced; lower[.inputBytes] = UInt64(source.count - 1) + do { try scope.restrict(to: lower) } catch { } + return 0 + } + } as? JSONFailure + #expect(error?.code == .limitExceeded && error?.resource?.metric == .inputBytes) + #expect(error?.resource?.attemptedUsage == UInt64(source.count)) +} + +@Test func independentNestedInvocationCannotRestoreOuterHealth() throws { + var limits = JSONLimits.balanced; limits[.workUnits] = 0 + #expect(throws: JSONFailure.self) { + try JSON.withComposition(limits: limits) { outer in + do { try outer.chargeExternalWork(1) } catch { } + _ = try JSON.withComposition { _ in 0 } + return 0 + } + } +} + +@Test func publicScopeCancellationCannotBeCaughtIntoSuccess() async throws { + let task = Task.detached { + try JSON.withComposition { scope in + withUnsafeCurrentTask { $0?.cancel() } + do { try scope.checkpoint() } catch is CancellationError { } + return 9 + } + } + do { _ = try await task.value; Issue.record("cancelled scope finalized") } catch is CancellationError { } +} + +@Test func firstQuotaCauseWinsOverLaterCancellation() async throws { + let task = Task.detached { + var limits = JSONLimits.balanced; limits[.workUnits] = 0 + return try JSON.withComposition(limits: limits) { scope in + do { try scope.chargeExternalWork(1) } catch { } + withUnsafeCurrentTask { $0?.cancel() } + do { try scope.checkpoint() } catch { } + return 9 + } + } + do { _ = try await task.value; Issue.record("terminal scope finalized") } + catch let failure as JSONFailure { #expect(failure.resource?.metric == .workUnits && failure.resource?.attemptedUsage == 1) } +} + +@Test func independentConcurrentSessionsHaveDeterministicReports() async throws { + try await withThrowingTaskGroup(of: JSONOperationStatistics.self) { group in + for _ in 0..<16 { group.addTask { try traceAdapter().statistics } } + for try await report in group { #expect(report[.workUnits] == goldenWork && report.peakLogicalRetainedBytes == goldenBytes) } + } +} + +@Test func callerErrorsAndForgedReportsAreNotCertificates() throws { + #expect(throws: AuditError.self) { try JSON.withComposition { _ in throw AuditError.caller } } + let genuine = try traceAdapter() + let forged = JSONOperationReport(result: "not the audited result", statistics: genuine.statistics) + #expect(forged.result == "not the audited result") + #expect(forged.statistics[.workUnits] == goldenWork) +} + + +@Test func multiResourceSourceAndFinalOutputOverlap() throws { + // Independent native weight ledger for each scalar document: + // 4 copied bytes + 4 preserved source bytes + 32 value + 32 source-map entry. + // All three documents remain in the returned tuple, alongside two buffers. + // 3*72 + (64+32) + (48+32) + 424 report = 816 logical bytes. + let expected: UInt64 = 816 + for ceiling in [expected, expected - 1] { + var limits = JSONLimits.balanced; limits[.inputBytes] = 4; limits[.retainedBytes] = ceiling + do { + let report = try JSON.withComposition(limits: limits) { scope in + let entry = try scope.parseDocument(Data("null".utf8)) + let resourceA = try scope.parseDocument(Data("null".utf8)) + let resourceB = try scope.parseDocument(Data("null".utf8)) + let queue = try fill(scope, count: 64, first: false, mutation: nil) + let output = try fill(scope, count: 48, first: false, mutation: nil) + return (entry, resourceA, resourceB, queue, output) + } + #expect(ceiling == expected) + #expect(report.statistics[.retainedBytes] == expected) + #expect(report.statistics.peakLogicalRetainedBytes == expected) + #expect(report.result.0.sourceBytes == Data("null".utf8) && report.result.4.count == 48) + } catch let failure as JSONFailure { + #expect(ceiling == expected - 1 && failure.resource?.metric == .retainedBytes) + #expect(failure.resource?.attemptedUsage == expected) + } + } +} + +@Test func explicitBorrowAllowsNestedSynchronousHelperOnTheSameLedger() throws { + // Independently counted: 2 entry units, 3 helper units, then 50 finalizer + // units; the helper's 8-byte frame overlaps the 424-byte report. + var limits = JSONLimits.balanced + limits[.workUnits] = 55; limits[.retainedBytes] = 432 + let report = try JSON.withComposition(limits: limits) { (scope: borrowing JSONCompositionScope) in + func local() throws { + try scope.reserveExternalLogicalBytes(8) + try scope.chargeExternalWork(3) + } + try scope.chargeExternalWork(2) + try local() + return 7 + } + #expect(report.result == 7 && report.statistics[.workUnits] == 55) + #expect(report.statistics[.retainedBytes] == 432) + limits[.workUnits] = 54 + #expect(throws: JSONFailure.self) { + try JSON.withComposition(limits: limits) { (scope: borrowing JSONCompositionScope) in + func local() throws { try scope.chargeExternalWork(5) } + try local() + } + } +} From 7a754be8604492924489a9c9c5e26f759a310185 Mon Sep 17 00:00:00 2001 From: Thorsten Fuchs <76904008+ThorFuchs@users.noreply.github.com> Date: Thu, 15 Oct 2026 23:28:42 +0200 Subject: [PATCH 08/11] Preserve cumulative reservation overflow precedence --- Sources/JSON/Limits/JSONCompositionScope.swift | 6 +++--- Sources/JSONCLIEngine/CLIBuildIdentity.swift | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/Sources/JSON/Limits/JSONCompositionScope.swift b/Sources/JSON/Limits/JSONCompositionScope.swift index 5c4a3e8..0e0eeee 100644 --- a/Sources/JSON/Limits/JSONCompositionScope.swift +++ b/Sources/JSON/Limits/JSONCompositionScope.swift @@ -146,13 +146,13 @@ public struct JSONCompositionScope: ~Copyable { /// - Throws: Quota/overflow failure or cancellation, permanently terminal. public func reserveExternalLogicalBytes(_ bytes: UInt64, count: UInt64 = 1, overhead: UInt64) throws { try withMeter { meter in - let (total, overflow) = bytes.multipliedReportingOverflow(by: count) - guard !overflow else { + let (total, productOverflow) = bytes.multipliedReportingOverflow(by: count) + let (complete, sumOverflow) = total.addingReportingOverflow(overhead) + guard !productOverflow && !sumOverflow else { throw JSONFailure(.resource, code: .counterOverflow, resource: .init( domain: .core, metric: .retainedBytes, limit: meter.limits[.retainedBytes], attemptedUsage: nil, overflow: true)) } - let complete = try meter.adding(.retainedBytes, total, overhead) try meter.retain(complete) } } diff --git a/Sources/JSONCLIEngine/CLIBuildIdentity.swift b/Sources/JSONCLIEngine/CLIBuildIdentity.swift index d7546c2..9b6686e 100644 --- a/Sources/JSONCLIEngine/CLIBuildIdentity.swift +++ b/Sources/JSONCLIEngine/CLIBuildIdentity.swift @@ -2,6 +2,6 @@ // Generated by Scripts/generate_cli_identity.py; excludes this file. // SHA-256 of the ordered Package.swift + Sources content manifest, not a Git commit. package enum CLIBuildIdentity { - package static let revision = "sha256:a0890c08b95b4eb2620018832f2145737e2fa5ac39e687f1e8aba2af41cae752" + package static let revision = "sha256:349b55989a018bf722cce1d14e6cdede15a01e1d0d4d8bb37b81847faff8a876" package static let kind = "source-sha256" } From 7ac75818f7974ab17d1bc44a39b08214eab15f16 Mon Sep 17 00:00:00 2001 From: Thorsten Fuchs <76904008+ThorFuchs@users.noreply.github.com> Date: Thu, 15 Oct 2026 23:30:04 +0200 Subject: [PATCH 09/11] Compare completion ownership without summing logical stage peaks --- Documentation/Qualification/Composition/Reconciliation.md | 3 ++- Scripts/qualify_composition_performance.py | 4 ++-- .../Sources/CompositionBenchmark/main.swift | 7 ++++--- 3 files changed, 8 insertions(+), 6 deletions(-) diff --git a/Documentation/Qualification/Composition/Reconciliation.md b/Documentation/Qualification/Composition/Reconciliation.md index ddcfc1f..5b15c45 100644 --- a/Documentation/Qualification/Composition/Reconciliation.md +++ b/Documentation/Qualification/Composition/Reconciliation.md @@ -58,7 +58,8 @@ The controlled Release comparison uses main 30631e8 as its full pinned baseline. Both existing PRs' comparisons against 037e735 remain historical. Identical public harnesses, runtime/resource paths and fixtures compare unchanged APIs and the composition document route. Existing API counters must match; composition's -expected completion delta is exactly 50 work / 424 logical bytes. Five operations +expected completion delta is exactly 50 work / 424 final logical bytes. Peak is +max(previous peak, final retained plus report), rather than a sum of stage peaks. Five operations on two inputs, nine samples per process and ABBA ordering yield 18 samples per revision/row. Each pre/post admission requires three interval samples >=90% idle. The review trigger stays 10%; operational refusals and blocked/noisy timing are diff --git a/Scripts/qualify_composition_performance.py b/Scripts/qualify_composition_performance.py index aa1cc69..b64cb4b 100644 --- a/Scripts/qualify_composition_performance.py +++ b/Scripts/qualify_composition_performance.py @@ -121,12 +121,12 @@ def host(name): if sample['workload'] != workload: continue row = next(r for r in sample['samples'] if r['name']==operation) rows[sample['name']].extend(row['seconds']) - counters[sample['name']].add((row.get('work'),row.get('peak'),row['inputBytes'])) + counters[sample['name']].add((row.get('work'),row.get('peak'),row.get('retained'),row['inputBytes'])) if len(counters['baseline'])!=1 or len(counters['candidate'])!=1: raise ValueError('Nondeterministic accounting: '+operation) base_count = next(iter(counters['baseline'])); candidate_count = next(iter(counters['candidate'])) if operation == 'composition-document': - if candidate_count != (base_count[0]+50, base_count[1]+424, base_count[2]): + if candidate_count != (base_count[0]+50, max(base_count[1], base_count[2]+424), base_count[2]+424, base_count[3]): raise ValueError('Unexpected composition receipt delta') elif counters['baseline'] != counters['candidate']: raise ValueError('Changed old-API work/retention/input identity: '+operation) diff --git a/Tools/CompositionQualification/Sources/CompositionBenchmark/main.swift b/Tools/CompositionQualification/Sources/CompositionBenchmark/main.swift index 46292bb..99c5abe 100644 --- a/Tools/CompositionQualification/Sources/CompositionBenchmark/main.swift +++ b/Tools/CompositionQualification/Sources/CompositionBenchmark/main.swift @@ -10,6 +10,7 @@ struct Sample: Codable { let seconds: [Double] let work: UInt64? let peak: UInt64? + let retained: UInt64? } // Deterministic 512-field exact/scalar workload, not an external conformance corpus. let text = "{" + (0..<512).map { "\"field\($0)\":" + #"{"n":9007199254740993,"s":"é","a":[true,null,0.01]}"# }.joined(separator: ",") + "}" @@ -24,7 +25,7 @@ let schemaProbe = try JSONSchemaCompiler().compileWithReport(schema) let compositionProbe = try JSON.withComposition(limits: .offline) { try $0.parseDocument(data) } let clock = ContinuousClock() func seconds(_ duration: Duration) -> Double { Double(duration.components.seconds) + Double(duration.components.attoseconds) / 1e18 } -func measure(_ name: String, iterations: Int, work: UInt64? = nil, peak: UInt64? = nil, _ operation: () throws -> Void) throws -> Sample { +func measure(_ name: String, iterations: Int, work: UInt64? = nil, peak: UInt64? = nil, retained: UInt64? = nil, _ operation: () throws -> Void) throws -> Sample { for _ in 0..<2 { for _ in 0.. Date: Fri, 16 Oct 2026 02:22:45 +0200 Subject: [PATCH 10/11] Remove process-enumeration overhead from quiet-host admission --- .github/workflows/composition-performance.yml | 2 + .../Qualification/Composition/Performance.md | 65 +++++++++++ .../Qualification/Composition/README.md | 2 + Scripts/composition_host.py | 68 +++++++++++ Scripts/qualify_composition_performance.py | 108 ++++++++++++++---- Scripts/test_composition_performance.py | 63 ++++++++++ 6 files changed, 288 insertions(+), 20 deletions(-) create mode 100644 Documentation/Qualification/Composition/Performance.md create mode 100644 Scripts/composition_host.py create mode 100644 Scripts/test_composition_performance.py diff --git a/.github/workflows/composition-performance.yml b/.github/workflows/composition-performance.yml index 8d06730..4d31447 100644 --- a/.github/workflows/composition-performance.yml +++ b/.github/workflows/composition-performance.yml @@ -4,6 +4,8 @@ on: paths: - '.github/workflows/composition-performance.yml' - 'Scripts/qualify_composition_performance.py' + - 'Scripts/composition_host.py' + - 'Scripts/test_composition_performance.py' - 'Sources/JSON/Limits/JSONCompositionScope.swift' - 'Sources/JSON/Limits/JSONWorkMeter.swift' - 'Tools/CompositionQualification/Sources/CompositionBenchmark/**' diff --git a/Documentation/Qualification/Composition/Performance.md b/Documentation/Qualification/Composition/Performance.md new file mode 100644 index 0000000..94629f7 --- /dev/null +++ b/Documentation/Qualification/Composition/Performance.md @@ -0,0 +1,65 @@ +# Composition quiet-host qualification + +The Release comparison retains the full baseline +`30631e8fe60201518c00544c172340ec6576e0d5`, identical public benchmark source, +runtime/resource paths, two pinned inputs, five operations, ABBA ordering and +18 observations per revision/row. The review trigger remains a positive elapsed +increase greater than 10%. Every process must have three consecutive one-second +intervals at least 90% idle both before and immediately after execution. There +is no retry or settling period after a failed post window. Failed host admission +produces a blocked receipt, not a performance result or exception. + +## Observer investigation + +The three attempts of workflow `38810354356`, on composition proof head +`7ac75818f7974ab17d1bc44a39b08214eab15f16`, stopped at immediate post windows: +87.79%, 71.38% and 54.29% minimum idle. Those receipts remain immutable. They +show bursts of system CPU but contain no process attribution, so they cannot +establish which process caused every burst or quantify a regression. + +The former observer starts `top -l 4 -n 0 -s 1` for each window. Limiting its +display does not avoid system-wide process inspection. A local ABBA diagnostic +found roughly 1.2 CPU seconds per four-snapshot invocation even with framework +and memory-map display disabled. This is diagnostic evidence on a busy Mac, +not a controlled performance result. It motivates checking observer overhead +on the actual qualification host rather than rerunning the old gate blindly. + +`composition_host.py` samples the supported Darwin +`host_statistics(HOST_CPU_LOAD_INFO)` interface directly. SDK +`mach/host_info.h` and `mach/machine.h` define four unsigned 32-bit CPU tick +counters in user/system/idle/nice order. Four snapshots yield three one-second +interval deltas; boot totals are never treated as interval load. Checked shapes, +normal counter wrap, missing samples and nonfinite values have rejection tests. +All global activity, including the sampler's own CPU, remains in the denominator. +This removes process enumeration from admission without changing its threshold, +interval count or immediate-post policy. The owned Mach send right is released. + +Each new native receipt includes a pre-measurement ABBA observer experiment: +the exact former `top` command twice and sampler-only windows twice. It records +independent CPU ticks, observer CPU time, elapsed time and raw top logs. Process +and thermal snapshots are outside measurement windows; a failed post window +additionally captures diagnostics without changing the failure. Paging between +the pre/post windows now also blocks a comparison. No workload is stopped. + +Executable replacement, binary digest checks and preparation precede pre-run +admission. The first post-run native snapshot precedes JSON decoding, hashing +or receipt serialization. Eight harness tests reject omitted/changed workloads, +nonfinite/incomplete timings, malformed counters and sub-threshold intervals, +and preserve completion's non-additive peak-retention oracle. This Python +harness is distinct from the unchanged Swift Testing behavior suites. + +## Interpretation + +Unchanged native API counters must match. Composition completion intentionally +adds 50 work units and 424 final logical retained bytes; peak is +`max(previous peak, previous final retained + 424)`. Logical accounting does not +measure RSS. The pinned frozen schema resource is JSON/Pointer input here, not +protocol conformance or schema evaluation. Compiler ownership limitations and +bounded adapter-capacity refusals remain documented separately. + +Run `python3 -B Scripts/qualify_composition_performance.py --output ` +on the supported native host. Actual final-source outcomes, command exits, +source/fixture/binary hashes and materialized logs belong in the return evidence. +This contract does not claim an unexecuted pass. Marking the PR ready additionally +requires its current required native/safety/docs checks and relevant Linux and +archive/proof jobs; merge, tagging and publication remain separate actions. diff --git a/Documentation/Qualification/Composition/README.md b/Documentation/Qualification/Composition/README.md index aafc9e4..792bc7c 100644 --- a/Documentation/Qualification/Composition/README.md +++ b/Documentation/Qualification/Composition/README.md @@ -90,6 +90,8 @@ Every process requires at least 90% idle before and immediately after timing. A bounded settling period precedes runs; a failed post-run probe invalidates the controlled result. The 10% review trigger is unchanged. Exit 2 means blocked and exit 3 requires review; neither is a performance pass or automatic exception. +The [observer investigation and replay contract](Performance.md) specifies the +lightweight native CPU sampler, raw-counter evidence and paging checks. Logical retention and owned work remain distinct from RSS and caller instructions. The Composition performance workflow executes this exact harness when its benchmark files change in a PR, or through optional manual dispatch after default-branch diff --git a/Scripts/composition_host.py b/Scripts/composition_host.py new file mode 100644 index 0000000..d6cf29c --- /dev/null +++ b/Scripts/composition_host.py @@ -0,0 +1,68 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: Apache-2.0 +"""Low-observer-cost Darwin host admission; no process enumeration during timing. + +The SDK's mach/host_info.h defines HOST_CPU_LOAD_INFO=3 and four natural_t +cpu_ticks in USER/SYSTEM/IDLE/NICE order. Preserve raw snapshots in receipts. +All CPU activity, including this observer, remains in the denominator. +""" +import ctypes +import math +import platform +import resource +import time + + +def interval(before, after): + if len(before) != 4 or len(after) != 4 or any(type(v) is not int or not 0 <= v < 2**32 for v in [*before, *after]): + raise ValueError('Invalid HOST_CPU_LOAD_INFO counters') + delta = [(b-a) % 2**32 for a, b in zip(before, after)] + if not sum(delta): + raise ValueError('CPU counters did not advance') + return {'delta_ticks': delta, 'idle_percent': 100 * delta[2] / sum(delta)} + + +def admitted(window): + values = window['interval_idle_percent'] + return len(values) == 3 and all(math.isfinite(v) and 90 <= v <= 100 for v in values) + + +class DarwinCPU: + def __init__(self): + if platform.system() != 'Darwin': + raise OSError('Darwin host CPU counters required') + self.lib = ctypes.CDLL('/usr/lib/libSystem.B.dylib') + self.lib.mach_host_self.restype = ctypes.c_uint32 + self.lib.host_statistics.argtypes = [ctypes.c_uint32, ctypes.c_int, ctypes.POINTER(ctypes.c_uint32), ctypes.POINTER(ctypes.c_uint32)] + self.lib.host_statistics.restype = ctypes.c_int + self.lib.mach_port_deallocate.argtypes = [ctypes.c_uint32, ctypes.c_uint32] + self.lib.mach_port_deallocate.restype = ctypes.c_int + self.port = self.lib.mach_host_self() + + def close(self): + if self.port: + code = self.lib.mach_port_deallocate(ctypes.c_uint32.in_dll(self.lib, 'mach_task_self_').value, self.port) + self.port = 0 + if code: + raise OSError('mach_port_deallocate failed: '+str(code)) + + def snapshot(self): + ticks = (ctypes.c_uint32 * 4)() + count = ctypes.c_uint32(4) + code = self.lib.host_statistics(self.port, 3, ticks, ctypes.byref(count)) + if code or count.value != 4: + raise OSError('HOST_CPU_LOAD_INFO failed: '+str((code, count.value))) + return {'monotonic_seconds': time.monotonic(), 'cpu_ticks': list(ticks)} + + def window(self): + usage = resource.getrusage(resource.RUSAGE_SELF) + snapshots = [self.snapshot()] + for _ in range(3): + time.sleep(1) + snapshots.append(self.snapshot()) + rows = [interval(a['cpu_ticks'], b['cpu_ticks']) for a, b in zip(snapshots, snapshots[1:])] + after = resource.getrusage(resource.RUSAGE_SELF) + return {'method': 'host_statistics(HOST_CPU_LOAD_INFO)', 'snapshots': snapshots, + 'intervals': rows, 'interval_idle_percent': [r['idle_percent'] for r in rows], + 'first_cumulative_snapshot_excluded': True, + 'observer_cpu_seconds': after.ru_utime+after.ru_stime-usage.ru_utime-usage.ru_stime} diff --git a/Scripts/qualify_composition_performance.py b/Scripts/qualify_composition_performance.py index b64cb4b..4a4dda6 100644 --- a/Scripts/qualify_composition_performance.py +++ b/Scripts/qualify_composition_performance.py @@ -1,9 +1,10 @@ #!/usr/bin/env python3 # SPDX-License-Identifier: Apache-2.0 """Matched native Release ABBA comparison with mandatory quiet-host admission.""" -import argparse, datetime, hashlib, json, os, pathlib, re, shutil, statistics, subprocess, tarfile, time +import argparse, datetime, hashlib, json, math, os, pathlib, re, resource, shutil, statistics, subprocess, tarfile, time from preflight import collect from archive_candidate import files +from composition_host import DarwinCPU, admitted ROOT = pathlib.Path(__file__).resolve().parents[1] BASE = '30631e8fe60201518c00544c172340ec6576e0d5' @@ -17,6 +18,16 @@ def inventory(directory): def source_identity(): return [{'path':p.relative_to(ROOT).as_posix(), 'sha256':digest(p)} for p in files()] +def check_samples(rows): + if len(rows)!=5 or {row['name']:row['iterations'] for row in rows} != NAMES: + raise ValueError('Changed benchmark discovery/iterations') + if any(len(row['seconds'])!=9 or any(not math.isfinite(v) or v<=0 for v in row['seconds']) for row in rows): + raise ValueError('Missing/nonpositive/nonfinite timing samples') + +def completion_counters(baseline): + work, peak, retained, size = baseline + return (work+50, max(peak, retained+424), retained+424, size) + class Blocked(Exception): pass def main(): @@ -27,7 +38,8 @@ def main(): receipt = {'status':'running', 'baseline':BASE, 'candidate':None, 'environment':collect(), 'source_inputs':inputs, 'source_manifest_sha256':hashlib.sha256(json.dumps(inputs, sort_keys=True).encode()).hexdigest(), 'configuration':'release', 'build_system':'native', 'review_trigger_percent':10, - 'minimum_pre_and_post_idle_percent':90, 'commands':[], 'host_checks':[], 'runs':[], + 'minimum_pre_and_post_idle_percent':90, 'host_sampler':'Darwin HOST_CPU_LOAD_INFO; three one-second intervals; no process enumeration', + 'commands':[], 'host_checks':[], 'observer_experiment':[], 'runs':[], 'started_utc':datetime.datetime.now(datetime.timezone.utc).isoformat()} def save(): (output/'receipt.json').write_text(json.dumps(receipt, indent=2)+'\n') def run(name, command, env=None): @@ -39,19 +51,29 @@ def run(name, command, env=None): save() if result.returncode: raise ValueError('Command failed: '+name) return log.read_text() + cpu = None def host(name): - result = subprocess.run(['top','-l','4','-n','0','-s','1'], capture_output=True, text=True, timeout=15) - log = output/(name+'.log'); log.write_text(result.stdout+result.stderr) - matches = re.findall(r'([0-9.]+)% idle', result.stdout) - intervals = [float(v) for v in matches[-3:]] if result.returncode == 0 and len(matches) == 4 else [] - idle = min(intervals) if intervals else None - receipt['host_checks'].append({'name':name, 'command':['top','-l','4','-n','0','-s','1'], 'interval_idle_percent':intervals, - 'exit_code':result.returncode, 'idle_percent':idle, 'log':log.name, 'log_sha256':digest(log)}) + window = cpu.window() + log = output/(name+'.log'); log.write_text(json.dumps(window, indent=2)+'\n') + idle = min(window['interval_idle_percent']) + receipt['host_checks'].append(dict(window, name=name, idle_percent=idle, + admitted=admitted(window), log=log.name, log_sha256=digest(log))) save(); print(name, 'idle='+str(idle), flush=True) - return idle is not None and idle >= 90 + return admitted(window) + def diagnostics(name): + # Outside measurement/admission; never masks an immediate failed post window. + run(name+'-processes', ['ps','-Ao','pid,pcpu,state,comm','-r']) + run(name+'-thermal', ['pmset','-g','therm']) + def swap(name): + text = run(name+'-vm', ['vm_stat']) + values = [re.search(r'^'+key+r':\s*(\d+)\.', text, re.M) for key in ['Swapins','Swapouts']] + if not all(values): raise ValueError('Missing native swap counters') + return [int(v[1]) for v in values] try: save() if receipt['environment']['status'] != 'passed': raise Blocked('Supported native Mac/toolchain/SDK required') + run('harness-tests', ['python3','-B','Scripts/test_composition_performance.py']) + cpu = DarwinCPU() run('cpu', ['sysctl','-n','machdep.cpu.brand_string']) run('memory', ['sysctl','-n','hw.memsize']) run('os-build', ['sw_vers','-buildVersion']) @@ -91,28 +113,70 @@ def host(name): shutil.copytree(binaries['baseline'].parent/bundle, runtime/bundle) receipt['resource_files'] = resources receipt['binaries'] = {name:{'sha256':digest(path), 'bytes':path.stat().st_size} for name,path in binaries.items()} + # ABBA observer experiment, before qualification. The legacy top monitor + # performs system-wide per-process inspection even with -n 0. Record its + # own CPU and independent global ticks, alongside sampler-only windows. + for index, mode in enumerate(['legacy-top','native-only','native-only','legacy-top']): + label = 'observer-'+str(index)+'-'+mode + before = resource.getrusage(resource.RUSAGE_CHILDREN); start = time.monotonic() + if mode == 'legacy-top': + command = ['top','-l','4','-n','0','-s','1'] + with (output/(label+'.log')).open('w') as stream: + monitor = subprocess.Popen(command, stdout=stream, stderr=subprocess.STDOUT) + try: + window = cpu.window() + code = monitor.wait(timeout=20) + finally: + if monitor.poll() is None: + monitor.terminate() + monitor.wait(timeout=5) + if code: raise ValueError('Observer diagnostic command failed') + else: + window = cpu.window(); command = None + (output/(label+'.log')).write_text(json.dumps(window, indent=2)+'\n') + after = resource.getrusage(resource.RUSAGE_CHILDREN) + receipt['observer_experiment'].append(dict(name=label, command=command, + window=window, elapsed_seconds=time.monotonic()-start, + child_cpu_seconds=after.ru_utime+after.ru_stime-before.ru_utime-before.ru_stime, + log=label+'.log', log_sha256=digest(output/(label+'.log')))) + diagnostics('measurement-start') save() for workload in ['wide512','frozen-schema']: env = os.environ.copy(); env.pop('JCOMP_BENCHMARK_INPUT',None) if workload == 'frozen-schema': env['JCOMP_BENCHMARK_INPUT'] = str(fixture) for index, name in enumerate(['baseline','candidate','candidate','baseline']): label = workload+'-'+str(index)+'-'+name + # Finish executable replacement and receipt I/O before pre-admission. + shutil.copy2(binaries[name],runtime/'CompositionBenchmark') + if digest(runtime/'CompositionBenchmark') != receipt['binaries'][name]['sha256']: + raise ValueError('Changed runtime binary') + before_swap = swap(label+'-before') # A bounded settling period is allowed before a measurement. # Post-run eligibility is checked immediately, with no waiting to hide noise. for attempt in range(12): if host(label+'-before-'+str(attempt)): break time.sleep(3) else: raise Blocked('No 90% pre-run idle window: '+label) - shutil.copy2(binaries[name],runtime/'CompositionBenchmark') - text = run(label,[str(runtime/'CompositionBenchmark')],env) - rows = json.loads(text) - if len(rows)!=5 or {row['name']:row['iterations'] for row in rows} != NAMES: - raise ValueError('Changed benchmark discovery/iterations') - if any(len(row['seconds'])!=9 or any(v<=0 for v in row['seconds']) for row in rows): - raise ValueError('Missing/nonpositive timing samples') - receipt['runs'].append({'workload':workload, 'name':name, 'log':label+'.log', 'samples':rows}) + log = output/(label+'.log'); start = time.monotonic() + with log.open('w') as stream: + result = subprocess.run([str(runtime/'CompositionBenchmark')], cwd=ROOT, env=env, + stdout=stream, stderr=subprocess.STDOUT, timeout=1200) + elapsed = time.monotonic()-start + # No hashing, JSON decoding, receipt serialization or settling delay + # between process exit and the first post-run counter snapshot. + post_ok = host(label+'-after') + receipt['commands'].append(dict(name=label, command=[str(runtime/'CompositionBenchmark')], + exit_code=result.returncode, seconds=elapsed, log=log.name, log_sha256=digest(log))) + after_swap = swap(label+'-after') + if result.returncode: raise ValueError('Command failed: '+label) + rows = json.loads(log.read_text()); check_samples(rows) + receipt['runs'].append({'workload':workload, 'name':name, 'log':log.name, 'samples':rows, + 'swap_before':before_swap, 'swap_after':after_swap, 'post_admitted':post_ok}) save() - if not host(label+'-after'): raise Blocked('Post-run idle below 90%: '+label) + if not post_ok: + diagnostics(label+'-failed-post') + raise Blocked('Post-run idle below 90%: '+label) + if before_swap != after_swap: raise Blocked('Paging during measurement: '+label) comparison = [] for workload in ['wide512','frozen-schema']: for operation in NAMES: @@ -126,7 +190,7 @@ def host(name): raise ValueError('Nondeterministic accounting: '+operation) base_count = next(iter(counters['baseline'])); candidate_count = next(iter(counters['candidate'])) if operation == 'composition-document': - if candidate_count != (base_count[0]+50, max(base_count[1], base_count[2]+424), base_count[2]+424, base_count[3]): + if candidate_count != completion_counters(base_count): raise ValueError('Unexpected composition receipt delta') elif counters['baseline'] != counters['candidate']: raise ValueError('Changed old-API work/retention/input identity: '+operation) @@ -143,6 +207,10 @@ def host(name): receipt.update(status='blocked', error=str(error)) except (OSError,ValueError,subprocess.SubprocessError) as error: receipt.update(status='failed', error=str(error)) + finally: + if cpu is not None: + try: cpu.close() + except OSError as error: receipt.update(status='failed', error=str(error)) receipt['source_inputs_unchanged'] = source_identity() == inputs if not receipt['source_inputs_unchanged']: receipt.update(status='failed',error='Source changed during measurement') receipt['finished_utc'] = datetime.datetime.now(datetime.timezone.utc).isoformat(); save() diff --git a/Scripts/test_composition_performance.py b/Scripts/test_composition_performance.py new file mode 100644 index 0000000..597e594 --- /dev/null +++ b/Scripts/test_composition_performance.py @@ -0,0 +1,63 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: Apache-2.0 +"""Behavior tests for native admission and the matched comparison's rejection paths.""" +import copy +import unittest +from unittest.mock import patch +from composition_host import DarwinCPU, admitted, interval +from qualify_composition_performance import NAMES, check_samples, completion_counters + + +class PerformanceHarnessTests(unittest.TestCase): + def test_interval_uses_deltas_not_boot_totals(self): + row = interval([900, 900, 9000, 0], [903, 907, 9090, 0]) + self.assertEqual(row, {'delta_ticks': [3, 7, 90, 0], 'idle_percent': 90}) + + def test_natural_counter_wrap_is_preserved(self): + self.assertEqual(interval([2**32-2, 0, 0, 0], [1, 0, 27, 0])['idle_percent'], 90) + + def test_malformed_or_frozen_counters_are_errors(self): + for before, after in [([0]*4, [0]*4), ([0]*3, [1]*4), ([0]*4, [True, 0, 1, 0]), + ([0]*4, [0, 0, 2**32, 0]), ([0]*4, [0, -1, 2, 0])]: + with self.assertRaises(ValueError): interval(before, after) + + def test_every_interval_must_meet_unchanged_threshold(self): + self.assertTrue(admitted({'interval_idle_percent': [90, 90, 90]})) + for values in [[100, 89.999, 100], [99, 99], [], [99]*4, [100, float('nan'), 100], + [100, float('inf'), 100], [100, 101, 100]]: + self.assertFalse(admitted({'interval_idle_percent': values})) + + def test_window_excludes_initial_snapshot_and_waits_three_intervals(self): + cpu = object.__new__(DarwinCPU) + snapshots = [{'cpu_ticks': [900+i, 500+i, 100+i*18, 0]} for i in range(4)] + with patch.object(cpu, 'snapshot', side_effect=snapshots) as read, patch('composition_host.time.sleep') as sleep: + window = cpu.window() + self.assertEqual(read.call_count, 4) + self.assertEqual(sleep.call_args_list, [unittest.mock.call(1)]*3) + self.assertEqual(window['interval_idle_percent'], [90]*3) + self.assertTrue(window['first_cumulative_snapshot_excluded']) + + def test_missing_duplicate_or_changed_workload_is_rejected(self): + rows = [dict(name=n, iterations=i, seconds=[0.01]*9) for n, i in NAMES.items()] + check_samples(rows) + for changed in [rows[:-1], rows+[rows[0]], [dict(r, iterations=1) for r in rows]]: + with self.assertRaises(ValueError): check_samples(changed) + + def test_nonfinite_or_incomplete_samples_cannot_pass(self): + rows = [dict(name=n, iterations=i, seconds=[0.01]*9) for n, i in NAMES.items()] + for value in [0, -1, float('nan'), float('inf')]: + changed = copy.deepcopy(rows); changed[0]['seconds'][0] = value + with self.assertRaises(ValueError): check_samples(changed) + rows[0]['seconds'].pop() + with self.assertRaises(ValueError): check_samples(rows) + + def test_completion_overlap_does_not_add_disjoint_stage_peaks(self): + self.assertEqual(completion_counters((10, 1000, 500, 40)), (60, 1000, 924, 40)) + self.assertEqual(completion_counters((10, 1000, 900, 40)), (60, 1324, 1324, 40)) + + +if __name__ == '__main__': + result = unittest.TextTestRunner(verbosity=2).run(unittest.defaultTestLoader.loadTestsFromTestCase(PerformanceHarnessTests)) + complete = result.wasSuccessful() and result.testsRun == 8 + print('PERFORMANCE_HARNESS '+('passed' if complete else 'failed')+' cases='+str(result.testsRun)) + raise SystemExit(0 if complete else 1) From 47ea4ef71dd7286ef0ec81342a8bb527438ce4b5 Mon Sep 17 00:00:00 2001 From: Thorsten Fuchs <76904008+ThorFuchs@users.noreply.github.com> Date: Fri, 16 Oct 2026 02:36:02 +0200 Subject: [PATCH 11/11] Prepare native processes before admission and refresh CI inventory --- Documentation/CI/README.md | 15 +++++++++++++++ .../Qualification/Composition/Performance.md | 9 +++++++-- Scripts/qualify_composition_performance.py | 9 ++++++++- 3 files changed, 30 insertions(+), 3 deletions(-) diff --git a/Documentation/CI/README.md b/Documentation/CI/README.md index 24aa203..311f5fd 100644 --- a/Documentation/CI/README.md +++ b/Documentation/CI/README.md @@ -1,5 +1,20 @@ # Native CI +## Current composition proof + +The additive composition proof retains current main's Navigation product and +adds two native boundary functions: exact discovery is 599 native / 583 Linux / +495 WASI functions, with the same 325 native/Linux and 288 WASI parameter cases. +Shared-state expectations remain 43 native / 42 Linux. Native/Linux still require +33/29 commands, all 126 restored identities and 49 cap probes. The separate +composition-proof lane additionally checks 26 public functions in both modes, +two rejected omitted-charge mutants and 14 normal compiler rejections. Known +compiler crashes remain separate limitations. Native CI now has five matrix +lanes including composition-proof; protected names/settings are unchanged. +The [controlled performance contract](../Qualification/Composition/Performance.md) +requires genuine pre/post idle admission on its isolated native runner. The +following counts retain their original stage context; receipts establish passes. + ## Reconciled Navigation and composition increment Current source retains composition from main and adds the opt-in Navigation diff --git a/Documentation/Qualification/Composition/Performance.md b/Documentation/Qualification/Composition/Performance.md index 94629f7..1f17b1b 100644 --- a/Documentation/Qualification/Composition/Performance.md +++ b/Documentation/Qualification/Composition/Performance.md @@ -41,8 +41,13 @@ and thermal snapshots are outside measurement windows; a failed post window additionally captures diagnostics without changing the failure. Paging between the pre/post windows now also blocks a comparison. No workload is stopped. -Executable replacement, binary digest checks and preparation precede pre-run -admission. The first post-run native snapshot precedes JSON decoding, hashing +Executable replacement, binary digest checks and a separately labelled preparation +launch precede pre-run admission for every process, equally for both revisions. +Preparation outputs are retained and checked for discovery/nonfinite values but +are never qualification samples. This lets first-launch cache/resource setup +settle before admission; it does not establish which service caused an old burst. +The measured processes still each perform their original two warmup sets and +nine samples. The first post-run native snapshot precedes JSON decoding, hashing or receipt serialization. Eight harness tests reject omitted/changed workloads, nonfinite/incomplete timings, malformed counters and sub-threshold intervals, and preserve completion's non-additive peak-retention oracle. This Python diff --git a/Scripts/qualify_composition_performance.py b/Scripts/qualify_composition_performance.py index 4a4dda6..7ca3726 100644 --- a/Scripts/qualify_composition_performance.py +++ b/Scripts/qualify_composition_performance.py @@ -39,7 +39,7 @@ def main(): 'source_inputs':inputs, 'source_manifest_sha256':hashlib.sha256(json.dumps(inputs, sort_keys=True).encode()).hexdigest(), 'configuration':'release', 'build_system':'native', 'review_trigger_percent':10, 'minimum_pre_and_post_idle_percent':90, 'host_sampler':'Darwin HOST_CPU_LOAD_INFO; three one-second intervals; no process enumeration', - 'commands':[], 'host_checks':[], 'observer_experiment':[], 'runs':[], + 'commands':[], 'host_checks':[], 'observer_experiment':[], 'preparations':[], 'runs':[], 'started_utc':datetime.datetime.now(datetime.timezone.utc).isoformat()} def save(): (output/'receipt.json').write_text(json.dumps(receipt, indent=2)+'\n') def run(name, command, env=None): @@ -150,6 +150,13 @@ def swap(name): shutil.copy2(binaries[name],runtime/'CompositionBenchmark') if digest(runtime/'CompositionBenchmark') != receipt['binaries'][name]['sha256']: raise ValueError('Changed runtime binary') + # A separately labelled first launch precedes admission. This + # prepares Foundation/resources and OS executable caches equally + # for both revisions. Its samples are never qualification data. + prepared = json.loads(run(label+'-prepare', [str(runtime/'CompositionBenchmark')], env)) + check_samples(prepared) + receipt['preparations'].append(dict(workload=workload, name=name, + log=label+'-prepare.log', qualification_samples=False)) before_swap = swap(label+'-before') # A bounded settling period is allowed before a measurement. # Post-run eligibility is checked immediately, with no waiting to hide noise.