diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 4331df1c177..4deb436758e 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -25,6 +25,7 @@ jobs: solidity: ${{ steps.filter.outputs.solidity }} typescript: ${{ steps.filter.outputs.typescript }} content: ${{ steps.filter.outputs.content }} + llms: ${{ steps.filter.outputs.llms }} any_non_solidity: ${{ steps.filter.outputs.any_non_solidity }} steps: - uses: actions/checkout@v6 @@ -53,6 +54,21 @@ jobs: - 'lychee.toml' - 'vercel.json' - 'src/scripts/link-check/**' + llms: + - 'src/features/**' + - 'src/config/sidebar*' + - 'src/config/sidebar/**' + - 'src/config/llms.ts' + - 'src/config/sunset.ts' + - 'src/config/solutions/**' + - 'src/utils/solutionPages.ts' + - 'src/scripts/generate-llms.ts' + - 'src/scripts/validate-llms.ts' + - 'src/scripts/component-coverage-baseline.json' + - '.husky/pre-commit' + - 'package.json' + - 'package-lock.json' + - '.github/workflows/test.yml' any_non_solidity: - '**' - '!public/samples/**/*.sol' @@ -318,7 +334,7 @@ jobs: # Job 6: Validate LLM files are present and well-formed validate-llms: needs: [setup, changes] - if: needs.changes.outputs.content == 'true' + if: needs.changes.outputs.content == 'true' || needs.changes.outputs.llms == 'true' runs-on: ubuntu-latest steps: - name: Checkout Repo @@ -348,7 +364,17 @@ jobs: - name: Regenerate LLM files run: npm run generate:llms + - name: Upload component coverage report + if: always() + uses: actions/upload-artifact@v7 + with: + name: component-coverage-report + path: reports/component-coverage.json + retention-days: 30 + if-no-files-found: warn + - name: Check LLM files are up-to-date + if: ${{ !cancelled() }} run: | if ! git diff --exit-code 'src/content/**/llms-full*.txt'; then echo "LLM files are out of date. Run 'npm run generate:llms' and commit the result." @@ -356,6 +382,7 @@ jobs: fi - name: Validate LLM files + if: ${{ !cancelled() }} run: npm run check:llms - name: Check Markdown fidelity diff --git a/.gitignore b/.gitignore index d96e095bc4f..8042bdf1c18 100644 --- a/.gitignore +++ b/.gitignore @@ -7,7 +7,7 @@ dist/ node_modules/ temp/ -reports/ +/reports/ # logs npm-debug.log* @@ -49,4 +49,4 @@ selectors*.yml.backup selectors.yml.backup # Test coverage -coverage/ +coverage/ \ No newline at end of file diff --git a/.prettierignore b/.prettierignore index 03708937f23..379a5449fea 100644 --- a/.prettierignore +++ b/.prettierignore @@ -13,6 +13,7 @@ data-source typechain-types public/snippets public/search-index.json +public/reports/component-coverage.json vercel.json diff --git a/package.json b/package.json index 0f215743e2c..85c9696ac0c 100644 --- a/package.json +++ b/package.json @@ -68,7 +68,8 @@ "quality-check:full": "npm run typecheck && npm run lint && npm run audit:dead-code", "generate:llms": "tsx --require tsconfig-paths/register src/scripts/generate-llms.ts", "check:llms": "tsx --require tsconfig-paths/register src/scripts/validate-llms.ts", - "check:markdown-fidelity": "tsx --require tsconfig-paths/register src/scripts/check-markdown-fidelity.ts" + "check:markdown-fidelity": "tsx --require tsconfig-paths/register src/scripts/check-markdown-fidelity.ts", + "report:component-coverage": "tsx --require tsconfig-paths/register src/scripts/generate-llms.ts --coverage-only" }, "dependencies": { "@11ty/eleventy-fetch": "^5.1.3", diff --git a/public/reports/component-coverage.json b/public/reports/component-coverage.json new file mode 100644 index 00000000000..264480753e9 --- /dev/null +++ b/public/reports/component-coverage.json @@ -0,0 +1,656 @@ +{ + "schemaVersion": 1, + "generatedAt": "2026-10-14T21:47:54.251Z", + "pagesProcessed": 1093, + "transformModes": { + "normal": 1092, + "sanitized": 0, + "fallback": 0, + "replacement": 1 + }, + "components": [ + { + "name": "Common", + "pageCount": 51, + "instances": 51, + "approxChars": 0, + "pages": [ + "chainlink-local/api-reference/v0.2.1/aggregator-interface.mdx", + "chainlink-local/api-reference/v0.2.1/aggregator-v2-v3-interface.mdx", + "chainlink-local/api-reference/v0.2.1/aggregator-v3-interface.mdx", + "chainlink-local/api-reference/v0.2.1/burn-mint-erc677-helper.mdx", + "chainlink-local/api-reference/v0.2.1/ccip-local-simulator-fork-js.mdx", + "chainlink-local/api-reference/v0.2.1/ccip-local-simulator-fork.mdx", + "chainlink-local/api-reference/v0.2.1/ccip-local-simulator.mdx", + "chainlink-local/api-reference/v0.2.1/index.mdx", + "chainlink-local/api-reference/v0.2.1/link-token.mdx", + "chainlink-local/api-reference/v0.2.1/mock-evm2evm-offramp.mdx", + "chainlink-local/api-reference/v0.2.1/mock-offchain-aggregator.mdx", + "chainlink-local/api-reference/v0.2.1/mock-v3-aggregator.mdx", + "chainlink-local/api-reference/v0.2.1/register.mdx", + "chainlink-local/api-reference/v0.2.1/weth9.mdx", + "chainlink-local/api-reference/v0.2.2/aggregator-interface.mdx", + "chainlink-local/api-reference/v0.2.2/aggregator-v2-v3-interface.mdx", + "chainlink-local/api-reference/v0.2.2/aggregator-v3-interface.mdx", + "chainlink-local/api-reference/v0.2.2/burn-mint-erc677-helper.mdx", + "chainlink-local/api-reference/v0.2.2/ccip-local-simulator-fork-js.mdx", + "chainlink-local/api-reference/v0.2.2/ccip-local-simulator-fork.mdx", + "chainlink-local/api-reference/v0.2.2/ccip-local-simulator.mdx", + "chainlink-local/api-reference/v0.2.2/index.mdx", + "chainlink-local/api-reference/v0.2.2/link-token.mdx", + "chainlink-local/api-reference/v0.2.2/mock-offchain-aggregator.mdx", + "chainlink-local/api-reference/v0.2.2/mock-v3-aggregator.mdx", + "chainlink-local/api-reference/v0.2.2/register.mdx", + "chainlink-local/api-reference/v0.2.2/weth9.mdx", + "chainlink-local/api-reference/v0.2.3/aggregator-interface.mdx", + "chainlink-local/api-reference/v0.2.3/aggregator-v2-v3-interface.mdx", + "chainlink-local/api-reference/v0.2.3/aggregator-v3-interface.mdx", + "chainlink-local/api-reference/v0.2.3/burn-mint-erc677-helper.mdx", + "chainlink-local/api-reference/v0.2.3/ccip-local-simulator-fork-js.mdx", + "chainlink-local/api-reference/v0.2.3/ccip-local-simulator-fork.mdx", + "chainlink-local/api-reference/v0.2.3/ccip-local-simulator.mdx", + "chainlink-local/api-reference/v0.2.3/index.mdx", + "chainlink-local/api-reference/v0.2.3/link-token.mdx", + "chainlink-local/api-reference/v0.2.3/mock-offchain-aggregator.mdx", + "chainlink-local/api-reference/v0.2.3/mock-v3-aggregator.mdx", + "chainlink-local/api-reference/v0.2.3/register.mdx", + "chainlink-local/api-reference/v0.2.3/weth9.mdx", + "chainlink-local/build/ccip/foundry/cct-burn-and-mint-fork.mdx", + "chainlink-local/build/ccip/foundry/cct-lock-and-release-fork.mdx", + "chainlink-local/build/ccip/foundry/forking-mainnets.mdx", + "chainlink-local/build/ccip/foundry/index.mdx", + "chainlink-local/build/ccip/foundry/local-simulator-fork.mdx", + "chainlink-local/build/ccip/foundry/local-simulator.mdx", + "chainlink-local/build/ccip/hardhat/index.mdx", + "chainlink-local/build/ccip/hardhat/local-simulator-fork.mdx", + "chainlink-local/build/ccip/hardhat/local-simulator.mdx", + "chainlink-local/build/ccip/remix/index.mdx", + "chainlink-local/build/ccip/remix/local-simulator.mdx" + ] + }, + { + "name": "AceCallout", + "pageCount": 33, + "instances": 34, + "approxChars": 0, + "pages": [ + "ace/active-monitoring/concepts/decisions-and-audit.mdx", + "ace/active-monitoring/concepts/enforcement-and-security.mdx", + "ace/active-monitoring/concepts/how-it-works.mdx", + "ace/active-monitoring/concepts/monitored-tokens.mdx", + "ace/active-monitoring/concepts/monitoring-rules.mdx", + "ace/active-monitoring/concepts/screening.mdx", + "ace/active-monitoring/guides/configure-monitoring-rules.mdx", + "ace/active-monitoring/guides/configure-screening.mdx", + "ace/active-monitoring/guides/index.mdx", + "ace/active-monitoring/guides/manage-monitored-tokens.mdx", + "ace/active-monitoring/guides/manage-watchlist.mdx", + "ace/active-monitoring/guides/prepare-your-token.mdx", + "ace/active-monitoring/guides/review-decisions.mdx", + "ace/active-monitoring/guides/rule-examples.mdx", + "ace/active-monitoring/guides/troubleshooting.mdx", + "ace/active-monitoring/overview.mdx", + "ace/active-monitoring/quick-start.mdx", + "ace/active-monitoring/reference/api.mdx", + "ace/active-monitoring/reference/limits-and-values.mdx", + "ace/beta-scope.mdx", + "ace/concepts/architecture.mdx", + "ace/concepts/key-terms.mdx", + "ace/concepts/off-chain-policies.mdx", + "ace/concepts/policy-management.mdx", + "ace/concepts/preventive-vs-continuous.mdx", + "ace/getting-started/account-setup.mdx", + "ace/getting-started/identity-manager.mdx", + "ace/getting-started/index.mdx", + "ace/getting-started/policy-manager.mdx", + "ace/overview.mdx", + "ace/reference/apis.mdx", + "ace/release-notes.mdx", + "ace/supported-networks.mdx" + ] + }, + { + "name": "MethodTabs", + "pageCount": 21, + "instances": 66, + "approxChars": 91217, + "pages": [ + "ace/active-monitoring/guides/configure-monitoring-rules.mdx", + "ace/active-monitoring/guides/configure-screening.mdx", + "ace/active-monitoring/guides/manage-monitored-tokens.mdx", + "ace/active-monitoring/guides/manage-watchlist.mdx", + "ace/active-monitoring/guides/review-decisions.mdx", + "ace/active-monitoring/quick-start.mdx", + "ace/getting-started/account-setup.mdx", + "ace/getting-started/identity-manager.mdx", + "ace/getting-started/policy-manager.mdx", + "ace/guides/identity-manager/manage-identities.mdx", + "ace/guides/identity-manager/manage-registries.mdx", + "ace/guides/policy-manager/manage-engines.mdx", + "ace/guides/policy-manager/manage-policies.mdx", + "ace/guides/policy-manager/manage-protections.mdx", + "ace/guides/policy-manager/manage-targets.mdx", + "crec/guides/channels/manage-channels.mdx", + "crec/guides/events/poll-and-search.mdx", + "crec/guides/wallets/create-and-manage.mdx", + "crec/guides/watchers/create-with-abi.mdx", + "crec/guides/watchers/create-with-service.mdx", + "crec/guides/watchers/manage-lifecycle.mdx" + ] + }, + { + "name": "FeedList", + "pageCount": 13, + "instances": 14, + "approxChars": 0, + "pages": [ + "data-feeds/24-7-extended-hours-data-feeds.mdx", + "data-feeds/stellar/index.mdx", + "data-feeds/stellar/using-data-feeds-off-chain.mdx", + "data-feeds/stellar/using-data-feeds-on-chain.mdx", + "data-feeds/svr-feeds/searcher-onboarding-atlas.mdx", + "data-feeds/svr-feeds/searcher-onboarding-ethereum.mdx", + "data-feeds/tokenized-equity-feeds/coinbase.mdx", + "data-feeds/tokenized-equity-feeds/index.mdx", + "data-feeds/tokenized-equity-feeds/ondo.mdx", + "data-feeds/tokenized-equity-feeds/robinhood.mdx", + "data-streams/rwa-streams/24-5-us-equities-user-guide.mdx", + "data-streams/rwa-streams/apac-equities.mdx", + "datalink/provider-catalog/streams.mdx" + ] + }, + { + "name": "CrecCommon", + "pageCount": 10, + "instances": 10, + "approxChars": 0, + "pages": [ + "crec/getting-started/authentication.mdx", + "crec/getting-started/prerequisites.mdx", + "crec/getting-started/quickstart-send-operation.mdx", + "crec/getting-started/quickstart-watch-events.mdx", + "crec/getting-started/sdk-installation.mdx", + "crec/overview.mdx", + "crec/private-beta.mdx", + "crec/reference/rest-api.mdx", + "crec/release-notes.mdx", + "crec/supported-networks.mdx" + ] + }, + { + "name": "ReportSchemaTabs", + "pageCount": 10, + "instances": 10, + "approxChars": 0, + "pages": [ + "data-streams/reference/report-schema-overview.mdx", + "data-streams/reference/report-schema-v10.mdx", + "data-streams/reference/report-schema-v11.mdx", + "data-streams/reference/report-schema-v2.mdx", + "data-streams/reference/report-schema-v3-dex.mdx", + "data-streams/reference/report-schema-v3.mdx", + "data-streams/reference/report-schema-v4.mdx", + "data-streams/reference/report-schema-v7.mdx", + "data-streams/reference/report-schema-v8.mdx", + "data-streams/reference/report-schema-v9.mdx" + ] + }, + { + "name": "ToolsUtilitiesGrid", + "pageCount": 8, + "instances": 12, + "approxChars": 0, + "pages": [ + "ace/index.mdx", + "ccip/v1/index.mdx", + "ccip/v2/ccv-starter-kit/prerequisites.mdx", + "ccip/v2/index.mdx", + "cre/index.mdx", + "crec/index.mdx", + "data-feeds/index.mdx", + "data-streams/index.mdx" + ] + }, + { + "name": "Vrf2_5Common", + "pageCount": 8, + "instances": 8, + "approxChars": 0, + "pages": [ + "vrf/index.mdx", + "vrf/v2-5/best-practices.mdx", + "vrf/v2-5/migration-from-v1.mdx", + "vrf/v2-5/migration-from-v2.mdx", + "vrf/v2-5/overview/subscription.mdx", + "vrf/v2-5/subscription/create-manage.mdx", + "vrf/v2-5/subscription/get-a-random-number.mdx", + "vrf/v2-5/supported-networks.mdx" + ] + }, + { + "name": "Dta", + "pageCount": 7, + "instances": 7, + "approxChars": 0, + "pages": [ + "dta-technical-standard/actors.mdx", + "dta-technical-standard/concepts/architecture.mdx", + "dta-technical-standard/concepts/payment-modes.mdx", + "dta-technical-standard/concepts/request-lifecycle.mdx", + "dta-technical-standard/how-it-works.mdx", + "dta-technical-standard/index.mdx", + "dta-technical-standard/reference/glossary.mdx" + ] + }, + { + "name": "LayoutHero", + "pageCount": 7, + "instances": 7, + "approxChars": 0, + "pages": [ + "ace/index.mdx", + "ccip/v1/index.mdx", + "ccip/v2/index.mdx", + "cre/index.mdx", + "crec/index.mdx", + "data-feeds/index.mdx", + "data-streams/index.mdx" + ] + }, + { + "name": "img", + "pageCount": 7, + "instances": 153, + "approxChars": 0, + "pages": [ + "ace/supported-networks.mdx", + "ccip/v2/ccv-starter-kit/index.mdx", + "crec/supported-networks.mdx", + "data-feeds/contract-registry.mdx", + "data-feeds/l2-sequencer-feeds.mdx", + "resources/link-token-contracts.mdx", + "vrf/v2-5/supported-networks.mdx" + ] + }, + { + "name": "CCIPEvent", + "pageCount": 6, + "instances": 30, + "approxChars": 0, + "pages": [ + "ccip/v1/evm/api-reference/v1.5.0/events.mdx", + "ccip/v1/evm/api-reference/v1.5.1/events.mdx", + "ccip/v1/evm/api-reference/v1.6.0/events.mdx", + "ccip/v1/evm/api-reference/v1.6.1/events.mdx", + "ccip/v1/evm/api-reference/v1.6.2/events.mdx", + "ccip/v1/evm/api-reference/v1.6.3/events.mdx" + ] + }, + { + "name": "CCIPSendError", + "pageCount": 6, + "instances": 39, + "approxChars": 0, + "pages": [ + "ccip/v1/evm/api-reference/v1.5.0/errors.mdx", + "ccip/v1/evm/api-reference/v1.5.1/errors.mdx", + "ccip/v1/evm/api-reference/v1.6.0/errors.mdx", + "ccip/v1/evm/api-reference/v1.6.1/errors.mdx", + "ccip/v1/evm/api-reference/v1.6.2/errors.mdx", + "ccip/v1/evm/api-reference/v1.6.3/errors.mdx" + ] + }, + { + "name": "ChangelogSnippet", + "pageCount": 5, + "instances": 5, + "approxChars": 0, + "pages": [ + "ccip/v1/index.mdx", + "ccip/v2/index.mdx", + "cre/index.mdx", + "data-feeds/index.mdx", + "data-streams/index.mdx" + ] + }, + { + "name": "TabGrid", + "pageCount": 5, + "instances": 11, + "approxChars": 0, + "pages": [ + "ace/index.mdx", + "cre/index.mdx", + "crec/index.mdx", + "data-feeds/index.mdx", + "data-streams/index.mdx" + ] + }, + { + "name": "TokenizedEquityFeeds", + "pageCount": 5, + "instances": 5, + "approxChars": 0, + "pages": [ + "data-feeds/tokenized-equity-feeds/coinbase.mdx", + "data-feeds/tokenized-equity-feeds/index.mdx", + "data-feeds/tokenized-equity-feeds/ondo.mdx", + "data-feeds/tokenized-equity-feeds/providers.mdx", + "data-feeds/tokenized-equity-feeds/robinhood.mdx" + ] + }, + { + "name": "CopyText", + "pageCount": 3, + "instances": 10, + "approxChars": 0, + "pages": [ + "cre/getting-started/cli-installation/macos-linux.mdx", + "data-feeds/svr-feeds/searcher-onboarding-atlas.mdx", + "data-streams/tutorials/stellar-onchain-report-verification.mdx" + ] + }, + { + "name": "Icon", + "pageCount": 3, + "instances": 9, + "approxChars": 0, + "pages": [ + "data-feeds/api-reference.mdx", + "data-feeds/historical-data.mdx", + "datalink/push-delivery/api-reference.mdx" + ] + }, + { + "name": "YouTube", + "pageCount": 3, + "instances": 4, + "approxChars": 0, + "pages": [ + "chainlink-local/index.mdx", + "getting-started/conceptual-overview.mdx", + "resources/contributing-to-chainlink.mdx" + ] + }, + { + "name": "ChainAwareTabGrid", + "pageCount": 2, + "instances": 4, + "approxChars": 0, + "pages": [ + "ccip/v1/index.mdx", + "ccip/v2/index.mdx" + ] + }, + { + "name": "LandingVersionNote", + "pageCount": 2, + "instances": 2, + "approxChars": 0, + "pages": [ + "ccip/v1/index.mdx", + "ccip/v2/index.mdx" + ] + }, + { + "name": "LatestPrice", + "pageCount": 2, + "instances": 2, + "approxChars": 0, + "pages": [ + "data-feeds/using-data-feeds.mdx", + "vrf/v2-5/subscription/test-locally.mdx" + ] + }, + { + "name": "MarketEventsTabs", + "pageCount": 2, + "instances": 2, + "approxChars": 0, + "pages": [ + "data-streams/rwa-streams/handling-market-events-v11.mdx", + "data-streams/rwa-streams/handling-market-events.mdx" + ] + }, + { + "name": "MintTokenButton", + "pageCount": 2, + "instances": 2, + "approxChars": 0, + "pages": [ + "ccip/v1/test-tokens.mdx", + "ccip/v2/getting-started/test-tokens.mdx" + ] + }, + { + "name": "RateLimitBucket", + "pageCount": 2, + "instances": 2, + "approxChars": 0, + "pages": [ + "ccip/v2/concepts/cross-chain-token/overview.mdx", + "ccip/v2/evm/concepts/cross-chain-token/rate-limits/overview.mdx" + ] + }, + { + "name": "SVMTestTokensClient", + "pageCount": 2, + "instances": 2, + "approxChars": 0, + "pages": [ + "ccip/v1/test-tokens.mdx", + "ccip/v2/getting-started/test-tokens.mdx" + ] + }, + { + "name": "center", + "pageCount": 2, + "instances": 2, + "approxChars": 0, + "pages": [ + "cre/getting-started/cli-installation/macos-linux.mdx", + "cre/getting-started/cli-installation/windows.mdx" + ] + }, + { + "name": "em", + "pageCount": 2, + "instances": 2, + "approxChars": 24, + "pages": [ + "ccip/v2/evm/tutorials/cross-chain-tokens/update-rate-limiters-foundry.mdx", + "ccip/v2/evm/tutorials/cross-chain-tokens/update-rate-limiters-hardhat.mdx" + ] + }, + { + "name": "AdminSetupStep", + "pageCount": 1, + "instances": 2, + "approxChars": 0, + "pages": [ + "ccip/v1/evm/tutorials/cross-chain-tokens/register-from-eoa-remix.mdx" + ] + }, + { + "name": "AptosCCTCallout", + "pageCount": 1, + "instances": 1, + "approxChars": 0, + "pages": [ + "ccip/v1/concepts/cross-chain-token/overview.mdx" + ] + }, + { + "name": "ChainUpdateBuilderWrapper", + "pageCount": 1, + "instances": 2, + "approxChars": 0, + "pages": [ + "ccip/v1/evm/tutorials/cross-chain-tokens/register-from-eoa-remix.mdx" + ] + }, + { + "name": "ContentCommon", + "pageCount": 1, + "instances": 1, + "approxChars": 0, + "pages": [ + "vrf/v2-5/subscription/test-locally.mdx" + ] + }, + { + "name": "ContractsImportCard", + "pageCount": 1, + "instances": 1, + "approxChars": 0, + "pages": [ + "ccip/v1/evm/tutorials/cross-chain-tokens/register-from-eoa-remix.mdx" + ] + }, + { + "name": "DeployPoolStep", + "pageCount": 1, + "instances": 2, + "approxChars": 0, + "pages": [ + "ccip/v1/evm/tutorials/cross-chain-tokens/register-from-eoa-remix.mdx" + ] + }, + { + "name": "DeployTokenStep", + "pageCount": 1, + "instances": 2, + "approxChars": 0, + "pages": [ + "ccip/v1/evm/tutorials/cross-chain-tokens/register-from-eoa-remix.mdx" + ] + }, + { + "name": "DeveloperCommunicationsCallout", + "pageCount": 1, + "instances": 1, + "approxChars": 0, + "pages": [ + "resources/developer-communications.mdx" + ] + }, + { + "name": "EnsLookupForm", + "pageCount": 1, + "instances": 1, + "approxChars": 0, + "pages": [ + "data-feeds/ens.mdx" + ] + }, + { + "name": "EnsManualLookupForm", + "pageCount": 1, + "instances": 1, + "approxChars": 0, + "pages": [ + "data-feeds/ens.mdx" + ] + }, + { + "name": "GrantPrivilegesStep", + "pageCount": 1, + "instances": 2, + "approxChars": 0, + "pages": [ + "ccip/v1/evm/tutorials/cross-chain-tokens/register-from-eoa-remix.mdx" + ] + }, + { + "name": "HistoricalPrice", + "pageCount": 1, + "instances": 1, + "approxChars": 0, + "pages": [ + "data-feeds/historical-data.mdx" + ] + }, + { + "name": "MarketPricingRiskTiers", + "pageCount": 1, + "instances": 1, + "approxChars": 0, + "pages": [ + "data-streams/selecting-data-streams.mdx" + ] + }, + { + "name": "PoolConfigVerification", + "pageCount": 1, + "instances": 2, + "approxChars": 0, + "pages": [ + "ccip/v1/evm/tutorials/cross-chain-tokens/register-from-eoa-remix.mdx" + ] + }, + { + "name": "PrerequisitesCard", + "pageCount": 1, + "instances": 1, + "approxChars": 0, + "pages": [ + "ccip/v1/evm/tutorials/cross-chain-tokens/register-from-eoa-remix.mdx" + ] + }, + { + "name": "SectionWrapper", + "pageCount": 1, + "instances": 1, + "approxChars": 0, + "pages": [ + "data-streams/supported-networks.mdx" + ] + }, + { + "name": "SetPoolStep", + "pageCount": 1, + "instances": 2, + "approxChars": 0, + "pages": [ + "ccip/v1/evm/tutorials/cross-chain-tokens/register-from-eoa-remix.mdx" + ] + }, + { + "name": "StreamsNetworkAddressesTable", + "pageCount": 1, + "instances": 1, + "approxChars": 0, + "pages": [ + "data-streams/tutorials/stellar-onchain-report-verification.mdx" + ] + }, + { + "name": "StreamsPage", + "pageCount": 1, + "instances": 1, + "approxChars": 0, + "pages": [ + "data-streams/deprecating-streams.mdx" + ] + }, + { + "name": "TokenCalculator", + "pageCount": 1, + "instances": 1, + "approxChars": 0, + "pages": [ + "ccip/v1/billing.mdx" + ] + }, + { + "name": "TutorialBlockchainSelector", + "pageCount": 1, + "instances": 1, + "approxChars": 0, + "pages": [ + "ccip/v1/evm/tutorials/cross-chain-tokens/register-from-eoa-remix.mdx" + ] + } + ], + "newSinceBaseline": [] +} diff --git a/src/lib/markdown/__tests__/componentCoverage.test.ts b/src/lib/markdown/__tests__/componentCoverage.test.ts new file mode 100644 index 00000000000..a76ddac2e57 --- /dev/null +++ b/src/lib/markdown/__tests__/componentCoverage.test.ts @@ -0,0 +1,202 @@ +import fs from "node:fs/promises" +import os from "node:os" +import path from "node:path" +import { afterEach, describe, expect, jest, test } from "@jest/globals" +import { unified } from "unified" +import { transformMarkdown } from "../transformMarkdown.js" +import { renderLlmsPageMarkdown } from "../llms.js" +import { + createComponentCoverageCollector, + parseComponentCoverageBaseline, + writeComponentCoverageReport, +} from "../componentCoverage.js" +import type { DroppedComponent } from "../types.js" + +const PAGE = path.resolve("src/content/cre/fixture.mdx") +const SECOND_PAGE = path.resolve("src/content/cre/second.mdx") +const options = { baseline: {}, generatedAt: "2026-10-14T06:00:00.000Z" } + +afterEach(() => { + jest.restoreAllMocks() +}) + +describe("drop observation", () => { + test("measures text, inline code, and fenced code without changing Markdown", async () => { + const source = "Before.\n\n\n\nLost `inline`.\n\n```ts\nlostCode()\n```\n\n\n\nAfter." + const drops: DroppedComponent[] = [] + const plain = await transformMarkdown(source, PAGE) + const observed = await transformMarkdown(source, PAGE, { onDropComponent: (drop) => drops.push(drop) }) + expect(observed).toBe(plain) + expect(observed).not.toContain("lostCode") + expect(drops).toEqual([{ name: "Unknown", approxChars: 22 }]) + expect(JSON.stringify(drops)).not.toContain("lostCode") + }) + + test("reports empty flow and inline drops, and only the executed parent drop", async () => { + const drops: DroppedComponent[] = [] + await transformMarkdown( + "Before after.\n\n\n\n\n\nLost text\n\n", + PAGE, + { + onDropComponent: (drop) => drops.push(drop), + } + ) + expect(drops).toEqual([ + { name: "Inline", approxChars: 0 }, + { name: "Flow", approxChars: 0 }, + { name: "Outer", approxChars: 9 }, + ]) + }) + + test("handled components and code literals are not reported", async () => { + const sink = jest.fn() + await transformMarkdown("\n\n```mdx\n\n```", PAGE, { onDropComponent: sink }) + expect(sink).not.toHaveBeenCalled() + }) + + test("a throwing observer neither changes output nor triggers a retry", async () => { + const source = "Lost\n\nKept" + const plain = await renderLlmsPageMarkdown(source, PAGE, "cre") + const sink = jest.fn(() => { + throw new Error("observer failed") + }) + const observed = await renderLlmsPageMarkdown(source, PAGE, "cre", undefined, { onPageRendered: sink }) + expect(observed).toEqual(plain) + expect(observed.transformMode).toBe("normal") + expect(sink).toHaveBeenCalledTimes(1) + const dropSink = jest.fn(() => { + throw new Error("drop observer failed") + }) + expect(await transformMarkdown(source, PAGE, { onDropComponent: dropSink })).toBe( + await transformMarkdown(source, PAGE) + ) + expect(dropSink).toHaveBeenCalledTimes(1) + }) + + test("discards failed-attempt drops and reports the successful retry once", async () => { + jest.spyOn(Object.getPrototypeOf(unified()), "stringify").mockImplementationOnce(() => { + throw new Error("serialization failed after projection") + }) + const sink = jest.fn() + const rendered = await renderLlmsPageMarkdown("Lost\n\nKept", PAGE, "cre", undefined, { + onPageRendered: sink, + }) + expect(rendered.transformMode).toBe("sanitized") + expect(sink).toHaveBeenCalledTimes(1) + expect(sink).toHaveBeenCalledWith(PAGE, "sanitized", [{ name: "Unknown", approxChars: 4 }]) + }) + + test("emits no phantom drops when both AST attempts fail", async () => { + jest.spyOn(Object.getPrototypeOf(unified()), "stringify").mockImplementation(() => { + throw new Error("serialization failed after projection") + }) + const sink = jest.fn() + const rendered = await renderLlmsPageMarkdown("Lost\n\nKept", PAGE, "cre", undefined, { + onPageRendered: sink, + }) + expect(rendered.transformMode).toBe("fallback") + expect(sink).toHaveBeenCalledTimes(1) + expect(sink).toHaveBeenCalledWith(PAGE, "fallback", []) + }) +}) + +describe("component coverage aggregation", () => { + test("keeps separate maxima of occurrences and characters between Go/TS renders", () => { + const collector = createComponentCoverageCollector() + collector.observers.onPageRendered?.(PAGE, "normal", [ + { name: "Widget", approxChars: 2 }, + { name: "Widget", approxChars: 3 }, + ]) + collector.observers.onPageRendered?.(PAGE, "sanitized", [{ name: "Widget", approxChars: 20 }]) + collector.observers.onPageRendered?.(SECOND_PAGE, "normal", [{ name: "Widget", approxChars: 4 }]) + const report = collector.report(options) + expect(report.components).toEqual([ + { name: "Widget", pageCount: 2, instances: 3, approxChars: 24, pages: ["cre/fixture.mdx", "cre/second.mdx"] }, + ]) + expect(report.pagesProcessed).toBe(2) + expect(report.transformModes).toMatchObject({ normal: 2, sanitized: 1 }) + }) + + test("repeated source components are counted once across actual Go/TS renders", async () => { + const collector = createComponentCoverageCollector() + for (const language of ["go", "ts"]) { + await renderLlmsPageMarkdown( + "First\n\nSecond", + PAGE, + "cre", + language, + collector.observers + ) + } + const report = collector.report(options) + expect(report.components[0]).toMatchObject({ pageCount: 1, instances: 2, approxChars: 11 }) + expect(report.pagesProcessed).toBe(1) + expect(report.transformModes.normal).toBe(1) + }) + + test("reports names outside a justified baseline", () => { + const baseline = parseComponentCoverageBaseline({ Existing: "Reviewed interactive-only content" }) + const collector = createComponentCoverageCollector() + collector.observers.onPageRendered?.(PAGE, "normal", [ + { name: "Existing", approxChars: 0 }, + { name: "New", approxChars: 0 }, + ]) + expect(collector.report({ ...options, baseline }).newSinceBaseline).toEqual(["New"]) + expect(parseComponentCoverageBaseline({})).toEqual({}) + for (const invalid of [[], null, { "": "reason" }, { Existing: "" }, { Existing: 1 }]) { + expect(() => parseComponentCoverageBaseline(invalid)).toThrow() + } + }) + + test("a failed atomic replacement preserves the previous public report and cleans up the temporary file", async () => { + const directory = await fs.mkdtemp(path.join(os.tmpdir(), "component-coverage-write-")) + try { + const output = path.join(directory, "component-coverage.json") + await fs.writeFile(output, "previous public report") + jest.spyOn(fs, "rename").mockRejectedValueOnce(new Error("rename failed")) + await expect( + writeComponentCoverageReport(createComponentCoverageCollector().report(options), output) + ).rejects.toThrow("rename failed") + expect(await fs.readFile(output, "utf8")).toBe("previous public report") + expect(await fs.readdir(directory)).toEqual(["component-coverage.json"]) + } finally { + await fs.rm(directory, { recursive: true, force: true }) + } + }) + + test("sorts by affected pages then name and writes deterministic data", async () => { + const entries = [ + { + page: PAGE, + drops: [ + { name: "Zeta", approxChars: 1 }, + { name: "Alpha", approxChars: 0 }, + { name: "Big", approxChars: 999 }, + ], + }, + { + page: SECOND_PAGE, + drops: [ + { name: "Zeta", approxChars: 0 }, + { name: "Alpha", approxChars: 0 }, + ], + }, + ] + const reports = [entries, [...entries].reverse()].map((ordered) => { + const collector = createComponentCoverageCollector() + for (const { page, drops } of ordered) collector.observers.onPageRendered?.(page, "normal", drops) + return collector.report(options) + }) + expect(reports[0]).toEqual(reports[1]) + expect(reports[0].components.map(({ name }) => name)).toEqual(["Alpha", "Zeta", "Big"]) + expect(reports[0].generatedAt).toBe(options.generatedAt) + const directory = await fs.mkdtemp(path.join(os.tmpdir(), "component-coverage-")) + try { + const output = path.join(directory, "report.json") + await writeComponentCoverageReport(reports[0], output) + expect(await fs.readFile(output, "utf8")).toBe(JSON.stringify(reports[1], null, 2) + "\n") + } finally { + await fs.rm(directory, { recursive: true, force: true }) + } + }) +}) diff --git a/src/lib/markdown/buildMarkdownArtifact.ts b/src/lib/markdown/buildMarkdownArtifact.ts index 1a63fde172c..6e963ca7987 100644 --- a/src/lib/markdown/buildMarkdownArtifact.ts +++ b/src/lib/markdown/buildMarkdownArtifact.ts @@ -1,7 +1,7 @@ import fs from "node:fs/promises" import path from "node:path" import { transformPageToMarkdown } from "./transformMarkdown.js" -import type { MarkdownArtifact } from "./types.js" +import type { MarkdownArtifact, TransformConfig } from "./types.js" import { extractFrontmatter, getIsoStringOrUndefined, toCanonicalUrl, toContentRelative } from "./utils.js" import { isSunsetDocsPath } from "../../config/sunset.js" import { withSolutionLayoutContent } from "./solutionContent.js" @@ -110,7 +110,7 @@ export async function buildMarkdownArtifact( export async function transformPageBodyToMarkdown( body: string, mdxAbsPath: string, - options: { siteBase?: string; targetLanguage?: string } = {} + options: Pick, "siteBase" | "targetLanguage" | "onDropComponent"> = {} ): Promise { if (mdxAbsPath.includes("data-feeds/deprecating-feeds")) { return { @@ -129,6 +129,7 @@ https://docs.chain.link/data-feeds/deprecating-feeds const transformOptions = { siteBase: options.siteBase ?? SITE_BASE, targetLanguage: options.targetLanguage, + onDropComponent: options.onDropComponent, } const completeBody = withSolutionLayoutContent(body, mdxAbsPath) diff --git a/src/lib/markdown/componentCoverage.ts b/src/lib/markdown/componentCoverage.ts new file mode 100644 index 00000000000..3ab06b975b6 --- /dev/null +++ b/src/lib/markdown/componentCoverage.ts @@ -0,0 +1,137 @@ +/** + * Counts unhandled JSX drops during TXT generation; Go/TS use per-page/component maxima. + * approxChars = 0 can hide lost dynamic content; the baseline maps names to reasons for accepting losses. + * Local/CI output: reports/component-coverage.json, also uploaded as a CI artifact. + * Weekly, from up-to-date main, run npm run report:component-coverage (--coverage-only). + * It writes public/reports/component-coverage.json for the public /reports/component-coverage.json URL. + * Open a dedicated PR containing only that file; content PRs never update it. + */ +import fs from "node:fs/promises" +import path from "node:path" +import type { LlmsRenderObservers, MarkdownArtifact } from "./types.js" + +export const COMPONENT_COVERAGE_PATH = "reports/component-coverage.json" +export const COMPONENT_COVERAGE_PUBLIC_PATH = "public/reports/component-coverage.json" +export const COMPONENT_COVERAGE_BASELINE_PATH = "src/scripts/component-coverage-baseline.json" +export type ComponentCoverageBaseline = Record + +type Counters = { instances: number; approxChars: number } +type TransformMode = MarkdownArtifact["transformMode"] + +export type ComponentCoverageReport = { + /** Public artifact contract version for docs-agent-score. */ + schemaVersion: 1 + /** UTC ISO timestamp of this generation. */ + generatedAt: string + /** Unique source pages rendered, counting language variants once. */ + pagesProcessed: number + /** Unique source pages per mode; language variants can use different modes. */ + transformModes: Record + /** Removed components, sorted by descending pageCount, then name. */ + components: { + /** Unhandled JSX tag name. */ + name: string + /** Number of distinct source pages with this component removed. */ + pageCount: number + /** Sum of maximum removed occurrences per source page across variants. */ + instances: number + /** Removed AST text and code length in UTF-16 units, using per-page maxima. */ + approxChars: number + /** All affected paths relative to src/content, sorted and using / separators. */ + pages: string[] + }[] + /** Sorted observed names absent from the baseline; accepted names remain in components. */ + newSinceBaseline: string[] +} + +export function parseComponentCoverageBaseline(value: unknown): ComponentCoverageBaseline { + if ( + !value || + typeof value !== "object" || + Array.isArray(value) || + Object.entries(value).some(([name, reason]) => !name.trim() || typeof reason !== "string" || !reason.trim()) + ) { + throw new Error("Expected component names mapped to nonempty reasons") + } + return value as ComponentCoverageBaseline +} + +/** Per-page maxima avoid counting the Go/TS renders of a page twice. */ +export function createComponentCoverageCollector(contentRoot = path.resolve("src/content")) { + const pages = new Map; components: Map }>() + const observers: LlmsRenderObservers = { + onPageRendered(sourcePath, mode, drops) { + const relative = path.relative(contentRoot, path.resolve(sourcePath)) + if (!relative || relative.startsWith(`..${path.sep}`) || relative === ".." || path.isAbsolute(relative)) { + throw new Error("Component coverage only accepts pages inside the content root") + } + const page = relative.split(path.sep).join("/") + const previous = pages.get(page) ?? { modes: new Set(), components: new Map() } + previous.modes.add(mode) + const current = new Map() + for (const drop of drops) { + const counters = current.get(drop.name) ?? { instances: 0, approxChars: 0 } + counters.instances++ + counters.approxChars += drop.approxChars + current.set(drop.name, counters) + } + for (const [name, counters] of current) { + const old = previous.components.get(name) + previous.components.set(name, { + instances: Math.max(old?.instances ?? 0, counters.instances), + approxChars: Math.max(old?.approxChars ?? 0, counters.approxChars), + }) + } + pages.set(page, previous) + }, + } + + function report(options: { baseline: ComponentCoverageBaseline; generatedAt?: string }): ComponentCoverageReport { + const grouped = new Map() + const transformModes = { normal: 0, sanitized: 0, fallback: 0, replacement: 0 } + for (const [page, stats] of pages) { + for (const mode of stats.modes) transformModes[mode]++ + for (const [name, counters] of stats.components) { + const component = grouped.get(name) ?? { pages: [], instances: 0, approxChars: 0 } + component.pages.push(page) + component.instances += counters.instances + component.approxChars += counters.approxChars + grouped.set(name, component) + } + } + const components = [...grouped].map(([name, component]) => ({ + name, + pageCount: component.pages.length, + instances: component.instances, + approxChars: component.approxChars, + pages: component.pages.sort(), + })) + components.sort((a, b) => b.pageCount - a.pageCount || (a.name < b.name ? -1 : a.name > b.name ? 1 : 0)) + return { + schemaVersion: 1, + generatedAt: options.generatedAt ?? new Date().toISOString(), + pagesProcessed: pages.size, + transformModes, + components, + newSinceBaseline: components + .map((component) => component.name) + .filter((name) => !Object.hasOwn(options.baseline, name)) + .sort(), + } + } + + return { observers, report } +} + +export async function writeComponentCoverageReport(report: ComponentCoverageReport, outputPath: string): Promise { + await fs.mkdir(path.dirname(outputPath), { recursive: true }) + const temporaryPath = path.join(path.dirname(outputPath), `.${path.basename(outputPath)}.${process.pid}.tmp`) + try { + await fs.writeFile(temporaryPath, JSON.stringify(report, null, 2) + "\n", "utf8") + await fs.rename(temporaryPath, outputPath) + } finally { + await fs.rm(temporaryPath, { force: true }).catch(() => { + // Cleanup cannot replace the original write/rename error. + }) + } +} diff --git a/src/lib/markdown/llms.ts b/src/lib/markdown/llms.ts index e6be92aa3f6..976409096ef 100644 --- a/src/lib/markdown/llms.ts +++ b/src/lib/markdown/llms.ts @@ -1,6 +1,6 @@ import { transformPageBodyToMarkdown } from "./buildMarkdownArtifact.js" import { unescapeMarkdown } from "./formatters.js" -import type { MarkdownArtifact } from "./types.js" +import type { DroppedComponent, LlmsRenderObservers, MarkdownArtifact } from "./types.js" import { extractFrontmatter, getIsoStringOrUndefined, @@ -17,16 +17,19 @@ export async function renderLlmsPageMarkdown( raw: string, absFile: string, section: string, - targetLanguage?: string + targetLanguage?: string, + observers: LlmsRenderObservers = {} ): Promise<{ markdown: string; transformMode: MarkdownArtifact["transformMode"] }> { const { body, fmTitle, fmLastModified } = extractFrontmatter(raw) const relFromContent = toContentRelative(absFile) const sourceUrl = toCanonicalUrl(section, relFromContent, SITE_BASE) const title = fmTitle || inferTitleFromPath(relFromContent) const lastModified = getIsoStringOrUndefined(fmLastModified) + const drops: DroppedComponent[] = [] const transformed = await transformPageBodyToMarkdown(body, absFile, { siteBase: SITE_BASE, targetLanguage, + onDropComponent: observers.onPageRendered ? (drop) => drops.push(drop) : undefined, }) const cleanedPlain = unescapeMarkdown(transformed.markdown) const headerLines = [ @@ -36,10 +39,17 @@ export async function renderLlmsPageMarkdown( "", ] - return { + const result = { markdown: [...headerLines, cleanedPlain.trim()].join("\n"), transformMode: transformed.transformMode, } + try { + observers.onPageRendered?.(absFile, transformed.transformMode, drops) + } catch { + // Diagnostics must not change the generated document. + } + + return result } const CODE_EXAMPLE_DISCLAIMER_MARKERS = [ diff --git a/src/lib/markdown/solutionLlms.ts b/src/lib/markdown/solutionLlms.ts index 59f55b28b53..9442e003846 100644 --- a/src/lib/markdown/solutionLlms.ts +++ b/src/lib/markdown/solutionLlms.ts @@ -4,6 +4,7 @@ import type { Solution } from "../../config/solutions/index.ts" import { collectSolutionPages, getOrderedSolutionPages, type SolutionPage } from "../../utils/solutionPages.js" import { assembleLlmsDocument, renderLlmsPageMarkdown } from "./llms.js" import { extractFrontmatter, toContentRelative } from "./utils.js" +import type { LlmsRenderObservers } from "./types.js" const SOLUTIONS_ROOT = path.resolve("src/content/solutions") @@ -40,7 +41,11 @@ export async function readSolutionLlmsPages(solution: Solution, contentRoot = SO } /** Both the CLI and the Astro endpoint use this exact conversion, order, and disclaimer policy. */ -export async function renderSolutionLlmsDocument(solution: Solution, pages: readonly SolutionPage[]): Promise { +export async function renderSolutionLlmsDocument( + solution: Solution, + pages: readonly SolutionPage[], + observers: LlmsRenderObservers = {} +): Promise { const orderedPages = getOrderedSolutionPages(solution, collectSolutionPages(solution.slug, pages)) const outputs: string[] = [] for (const entry of orderedPages) { @@ -49,7 +54,7 @@ export async function renderSolutionLlmsDocument(solution: Solution, pages: read } const absFile = path.resolve(entry.filePath) const raw = await fs.readFile(absFile, "utf8") - const rendered = await renderLlmsPageMarkdown(raw, absFile, "solutions") + const rendered = await renderLlmsPageMarkdown(raw, absFile, "solutions", undefined, observers) if (rendered.transformMode !== "normal") { console.warn(`[llms] ${toContentRelative(absFile)}: transform mode ${rendered.transformMode}`) } diff --git a/src/lib/markdown/transformMarkdown.ts b/src/lib/markdown/transformMarkdown.ts index c92f505a73e..b3aa77de3e2 100644 --- a/src/lib/markdown/transformMarkdown.ts +++ b/src/lib/markdown/transformMarkdown.ts @@ -9,7 +9,7 @@ import remarkGfm from "remark-gfm" import remarkStringify from "remark-stringify" import { SKIP, visit } from "unist-util-visit" import type { Node, Parent, Literal } from "unist" -import type { TransformConfig, MdxJsxNode, ComponentContext } from "./types.js" +import type { TransformConfig, MdxJsxNode, ComponentContext, DroppedComponent } from "./types.js" import { handleCcipCommon, handleCodeHighlightBlock, @@ -137,6 +137,7 @@ export async function transformMarkdown( config: Partial = {} ): Promise { const { targetLanguage } = config + const drops: DroppedComponent[] = [] // Inline CcipCommon content before AST parsing so embedded components reach the normal AST handlers. const preprocessedMarkdown = preprocessCcipCommon(markdown) @@ -408,6 +409,23 @@ export async function transformMarkdown( node.type === "export" || node.type === "html" ) { + if (config.onDropComponent && (node.type === "mdxJsxFlowElement" || node.type === "mdxJsxTextElement")) { + const drop: DroppedComponent = { + name: (node as MdxJsxNode).name ?? "anonymous", + approxChars: 0, + } + try { + visit(node, (descendant) => { + if (descendant.type === "text" || descendant.type === "inlineCode" || descendant.type === "code") { + drop.approxChars += String((descendant as Literal).value ?? "").length + } + }) + } catch { + // Diagnostic measurement cannot change transformation behavior. + drop.approxChars = 0 + } + drops.push(drop) + } parent.children.splice(index, 1) return index } @@ -450,6 +468,15 @@ export async function transformMarkdown( .filter((line) => !line.trim().match(/^{\/\*.*?\*\/}$/)) .join("\n") + // A failed normal/sanitized attempt must not emit events for output that was abandoned. + for (const drop of drops) { + try { + config.onDropComponent?.(drop) + } catch { + // Observers cannot change Markdown output or trigger a fallback/retry. + } + } + return result } diff --git a/src/lib/markdown/types.ts b/src/lib/markdown/types.ts index 9548b2d5f59..8d0e2e743aa 100644 --- a/src/lib/markdown/types.ts +++ b/src/lib/markdown/types.ts @@ -75,6 +75,13 @@ export type ComponentHandler = ( context: ComponentContext ) => number | void +/** Counts captured before an unhandled JSX subtree is removed; no source content is retained. */ +export interface DroppedComponent { + name: string + /** UTF-16 length of descendant text, inline code, and fenced code values. */ + approxChars: number +} + /** * Configuration for markdown transformation */ @@ -87,6 +94,8 @@ export interface TransformConfig { includeFrontmatter?: boolean /** Custom component handlers */ componentHandlers?: Record + /** Synchronous, best-effort observer. Events are delivered only after a successful transform. */ + onDropComponent?: (drop: DroppedComponent) => void } /** @@ -118,3 +127,7 @@ export type MarkdownArtifact = { sourcePath?: string markdown: string } + +export interface LlmsRenderObservers { + onPageRendered?: (sourcePath: string, mode: MarkdownArtifact["transformMode"], drops: DroppedComponent[]) => void +} diff --git a/src/scripts/component-coverage-baseline.json b/src/scripts/component-coverage-baseline.json new file mode 100644 index 00000000000..23a426aa43f --- /dev/null +++ b/src/scripts/component-coverage-baseline.json @@ -0,0 +1,51 @@ +{ + "AceCallout": "Present when the diagnostic was introduced; pending review.", + "AdminSetupStep": "Present when the diagnostic was introduced; pending review.", + "AptosCCTCallout": "Present when the diagnostic was introduced; pending review.", + "CCIPEvent": "Present when the diagnostic was introduced; pending review.", + "CCIPSendError": "Present when the diagnostic was introduced; pending review.", + "ChainAwareTabGrid": "Present when the diagnostic was introduced; pending review.", + "ChainUpdateBuilderWrapper": "Present when the diagnostic was introduced; pending review.", + "ChangelogSnippet": "Present when the diagnostic was introduced; pending review.", + "Common": "Present when the diagnostic was introduced; pending review.", + "ContentCommon": "Present when the diagnostic was introduced; pending review.", + "ContractsImportCard": "Present when the diagnostic was introduced; pending review.", + "CopyText": "Present when the diagnostic was introduced; pending review.", + "CrecCommon": "Present when the diagnostic was introduced; pending review.", + "DeployPoolStep": "Present when the diagnostic was introduced; pending review.", + "DeployTokenStep": "Present when the diagnostic was introduced; pending review.", + "DeveloperCommunicationsCallout": "Present when the diagnostic was introduced; pending review.", + "Dta": "Present when the diagnostic was introduced; pending review.", + "EnsLookupForm": "Present when the diagnostic was introduced; pending review.", + "EnsManualLookupForm": "Present when the diagnostic was introduced; pending review.", + "FeedList": "Present when the diagnostic was introduced; pending review.", + "GrantPrivilegesStep": "Present when the diagnostic was introduced; pending review.", + "HistoricalPrice": "Present when the diagnostic was introduced; pending review.", + "Icon": "Present when the diagnostic was introduced; pending review.", + "LandingVersionNote": "Present when the diagnostic was introduced; pending review.", + "LatestPrice": "Present when the diagnostic was introduced; pending review.", + "LayoutHero": "Present when the diagnostic was introduced; pending review.", + "MarketEventsTabs": "Present when the diagnostic was introduced; pending review.", + "MarketPricingRiskTiers": "Present when the diagnostic was introduced; pending review.", + "MethodTabs": "Present when the diagnostic was introduced; pending review.", + "MintTokenButton": "Present when the diagnostic was introduced; pending review.", + "PoolConfigVerification": "Present when the diagnostic was introduced; pending review.", + "PrerequisitesCard": "Present when the diagnostic was introduced; pending review.", + "RateLimitBucket": "Present when the diagnostic was introduced; pending review.", + "ReportSchemaTabs": "Present when the diagnostic was introduced; pending review.", + "SVMTestTokensClient": "Present when the diagnostic was introduced; pending review.", + "SectionWrapper": "Present when the diagnostic was introduced; pending review.", + "SetPoolStep": "Present when the diagnostic was introduced; pending review.", + "StreamsNetworkAddressesTable": "Present when the diagnostic was introduced; pending review.", + "StreamsPage": "Present when the diagnostic was introduced; pending review.", + "TabGrid": "Present when the diagnostic was introduced; pending review.", + "TokenCalculator": "Present when the diagnostic was introduced; pending review.", + "TokenizedEquityFeeds": "Present when the diagnostic was introduced; pending review.", + "ToolsUtilitiesGrid": "Present when the diagnostic was introduced; pending review.", + "TutorialBlockchainSelector": "Present when the diagnostic was introduced; pending review.", + "Vrf2_5Common": "Present when the diagnostic was introduced; pending review.", + "YouTube": "Present when the diagnostic was introduced; pending review.", + "center": "Present when the diagnostic was introduced; pending review.", + "em": "Present when the diagnostic was introduced; pending review.", + "img": "Present when the diagnostic was introduced; pending review." +} diff --git a/src/scripts/component-coverage-cli.test.ts b/src/scripts/component-coverage-cli.test.ts new file mode 100644 index 00000000000..b8ff476bdb1 --- /dev/null +++ b/src/scripts/component-coverage-cli.test.ts @@ -0,0 +1,127 @@ +import fs from "node:fs/promises" +import os from "node:os" +import path from "node:path" +import { spawnSync } from "node:child_process" +import { afterEach, beforeEach, describe, expect, test } from "@jest/globals" +import { LLM_SECTIONS_CONFIG } from "../config/llms.js" +import { SOLUTIONS } from "../config/solutions/index.ts" + +const REPO_ROOT = process.cwd() +const GENERATOR = path.join(REPO_ROOT, "src/scripts/generate-llms.ts") +const LOADER = path.join(REPO_ROOT, "node_modules/tsx/dist/loader.mjs") +let directory: string + +beforeEach(async () => { + directory = await fs.mkdtemp(path.join(os.tmpdir(), "component-coverage-cli-")) + for (const dir of ["src/content/ace", "src/scripts", "reports", "public/reports"]) { + await fs.mkdir(path.join(directory, dir), { recursive: true }) + } + await fs.writeFile(path.join(directory, "src/content/ace/fixture.mdx"), "Lost") + await fs.writeFile(path.join(directory, "src/scripts/component-coverage-baseline.json"), "{}") + await fs.writeFile(path.join(directory, "public/reports/component-coverage.json"), "stale") +}) + +afterEach(async () => { + await fs.rm(directory, { recursive: true, force: true }) +}) + +function run(args: string[], env: Record = {}) { + return spawnSync(process.execPath, ["--import", LOADER, GENERATOR, ...args], { + cwd: directory, + encoding: "utf8", + env: { + ...process.env, + GITHUB_ACTIONS: "", + GITHUB_STEP_SUMMARY: "", + TSX_TSCONFIG_PATH: path.join(REPO_ROOT, "tsconfig.json"), + ...env, + }, + timeout: 20_000, + }) +} + +async function prepareFullCorpus() { + for (const config of Object.values(LLM_SECTIONS_CONFIG)) { + await fs.mkdir(path.join(directory, config.root), { recursive: true }) + } + for (const solution of SOLUTIONS.filter((entry) => !entry.hidden)) { + await fs.cp( + path.join(REPO_ROOT, "src/content/solutions", solution.slug), + path.join(directory, "src/content/solutions", solution.slug), + { + recursive: true, + filter: (file) => !file.endsWith("llms-full.txt"), + } + ) + } +} + +describe("component coverage CLI", () => { + test("coverage-only publishes directly from the full corpus without writing bundles or the LLM report", async () => { + await prepareFullCorpus() + const bundle = path.join(directory, "src/content/ace/llms-full.txt") + const llmsReport = path.join(directory, "reports/llms-report.json") + await fs.writeFile(bundle, "existing bundle") + await fs.writeFile(llmsReport, "existing report") + const result = run(["--coverage-only"]) + expect(result.status).toBe(0) + const report = JSON.parse(await fs.readFile(path.join(directory, "public/reports/component-coverage.json"), "utf8")) + expect(report.pagesProcessed).toBeGreaterThan(1) + expect(report.components.find((entry: { name: string }) => entry.name === "Unknown")).toMatchObject({ + instances: 1, + approxChars: 4, + }) + expect(await fs.readFile(bundle, "utf8")).toBe("existing bundle") + expect(await fs.readFile(llmsReport, "utf8")).toBe("existing report") + const bundles = (await fs.readdir(path.join(directory, "src/content"), { recursive: true })).filter((file) => + /llms-full.*\.txt$/.test(file) + ) + expect(bundles).toEqual(["ace/llms-full.txt"]) + await expect(fs.access(path.join(directory, "reports/component-coverage.json"))).rejects.toThrow() + }, 30_000) + + test("normal generation writes an ignored report and non-blocking CI warnings with a job summary", async () => { + const summary = path.join(directory, "summary.md") + const result = run(["--section", "ace"], { + GITHUB_ACTIONS: "true", + GITHUB_STEP_SUMMARY: summary, + }) + expect(result.status).toBe(0) + expect(result.stdout + result.stderr).toContain("::warning::") + expect(await fs.readFile(summary, "utf8")).toContain("Unknown") + const report = JSON.parse(await fs.readFile(path.join(directory, "reports/component-coverage.json"), "utf8")) + expect(report.newSinceBaseline).toEqual(["Unknown"]) + expect(report.components[0]).toMatchObject({ name: "Unknown", instances: 1, approxChars: 4 }) + expect(await fs.readFile(path.join(directory, "public/reports/component-coverage.json"), "utf8")).toBe("stale") + }) + + test("coverage generation failure exits unsuccessfully and preserves the previous public report", async () => { + await fs.rm(path.join(directory, "src/content/ace"), { recursive: true }) + await fs.writeFile(path.join(directory, "src/content/ace"), "not a directory") + const result = run(["--coverage-only"]) + expect(result.status).toBe(1) + expect(result.stderr).toContain("ENOTDIR") + expect(await fs.readFile(path.join(directory, "public/reports/component-coverage.json"), "utf8")).toBe("stale") + }) + + test.each(["--section", "--pages", "--report"])( + "rejects a missing value for %s before generating files", + async (flag) => { + const result = run([flag]) + expect(result.status).toBe(1) + expect(result.stderr).toContain("requires a value") + await expect(fs.access(path.join(directory, "src/content/ace/llms-full.txt"))).rejects.toThrow() + } + ) + + test.each([ + ["--section", "unknown-product"], + ["--unknown-option"], + ["--coverage-only", "--section", "ace"], + ["--coverage-only", "--pages", "pages.json"], + ["--coverage-only", "--report", "report.json"], + ])("rejects invalid CLI arguments %j", async (...args) => { + expect(run(args).status).toBe(1) + await expect(fs.access(path.join(directory, "src/content/ace/llms-full.txt"))).rejects.toThrow() + }) +}) diff --git a/src/scripts/generate-llms.test.ts b/src/scripts/generate-llms.test.ts index c23fa7afd05..22dcb06df65 100644 --- a/src/scripts/generate-llms.test.ts +++ b/src/scripts/generate-llms.test.ts @@ -20,6 +20,7 @@ import { collectSolutionPages } from "../utils/solutionPages.js" import { readSolutionLlmsPages, renderSolutionLlmsDocument } from "../lib/markdown/solutionLlms.js" import { llmsFilesToValidate, validateLlmsFile } from "./validate-llms.js" import { SOLUTION_DISCLAIMER } from "../config/solutions/disclaimer.ts" +import { createComponentCoverageCollector } from "../lib/markdown/componentCoverage.js" test("sidebar links order discovered pages without adding a solution to a product export", () => { const first = path.resolve("src/content/ccip/first.mdx") @@ -57,9 +58,11 @@ describe("solution CLI exports", () => { expect(await generateSolutionLlmsFiles([fixture], directory)).toEqual([]) expect(await fs.readdir(directory)).toEqual([]) const published = { ...fixture, hidden: false } - const reports = await generateSolutionLlmsFiles([published], directory) + const coverage = createComponentCoverageCollector() + const reports = await generateSolutionLlmsFiles([published], directory, coverage.observers) const document = await fs.readFile(path.join(directory, fixture.slug, "llms-full.txt"), "utf8") const pageCount = fixture.nav?.flatMap((group) => group.pages).length ?? 1 + expect(coverage.report({ baseline: {} }).pagesProcessed).toBe(pageCount) expect(reports).toHaveLength(1) expect(reports[0].pagesProcessed).toBe(pageCount) expect(reports[0].bytes).toBe(Buffer.byteLength(document)) diff --git a/src/scripts/generate-llms.ts b/src/scripts/generate-llms.ts index 36de5cc7f4f..c9031b19f9a 100644 --- a/src/scripts/generate-llms.ts +++ b/src/scripts/generate-llms.ts @@ -9,6 +9,16 @@ import { isSunsetContentFile } from "../config/sunset.js" import type { SectionEntry, SectionContent } from "../config/sidebar.js" import { assembleLlmsDocument, renderLlmsPageMarkdown } from "../lib/markdown/llms.js" import { SOLUTIONS, type Solution } from "../config/solutions/index.ts" +import type { LlmsRenderObservers } from "../lib/markdown/types.js" +import { + COMPONENT_COVERAGE_PATH, + COMPONENT_COVERAGE_PUBLIC_PATH, + COMPONENT_COVERAGE_BASELINE_PATH, + createComponentCoverageCollector, + parseComponentCoverageBaseline, + writeComponentCoverageReport, + type ComponentCoverageBaseline, +} from "../lib/markdown/componentCoverage.js" import { readSolutionLlmsPages, renderSolutionLlmsDocument, @@ -40,6 +50,7 @@ type CliArgs = { section?: string pagesFile?: string reportPath?: string + coverageOnly?: boolean } export function selectLlmsTargets(section?: string): { sections: string[]; solutions: Solution[] } { @@ -51,18 +62,22 @@ export function selectLlmsTargets(section?: string): { sections: string[]; solut if (!selected.length) throw new Error(`[llms] Unknown or hidden solution: ${section}`) return { sections: [], solutions: selected } } + if (!(SUPPORTED_LLM_SECTIONS as readonly string[]).includes(section)) { + throw new Error(`[llms] Unknown section: ${section}`) + } return { sections: [section], solutions: [] } } export async function generateSolutionLlmsFiles( solutions: readonly Solution[], - outputRoot?: string + outputRoot?: string, + observers: LlmsRenderObservers = {} ): Promise { const reports: SectionReport[] = [] for (const solution of solutions) { if (solution.hidden) continue const pages = await readSolutionLlmsPages(solution) - const document = await renderSolutionLlmsDocument(solution, pages) + const document = await renderSolutionLlmsDocument(solution, pages, observers) const outPath = solutionLlmsOutputPath(solution.slug, outputRoot) let prevBytes: number | undefined try { @@ -86,10 +101,28 @@ export async function generateSolutionLlmsFiles( async function main() { const args = parseArgs(process.argv.slice(2)) + if (args.coverageOnly && (args.section || args.pagesFile || args.reportPath)) { + throw new Error( + "[llms] --coverage-only uses the full corpus and cannot be combined with --section, --pages, or --report" + ) + } const { sections, solutions } = selectLlmsTargets(args.section) if (args.pagesFile && args.section?.startsWith("solutions")) { throw new Error("[llms] --pages is only supported for product sections; solutions use their complete navigation.") } + await generateLlms(args, sections, solutions) +} + +async function generateLlms(args: CliArgs, sections: string[], solutions: Solution[]) { + const started = performance.now() + const coverage = createComponentCoverageCollector() + let baseline: ComponentCoverageBaseline = {} + try { + baseline = parseComponentCoverageBaseline(JSON.parse(await fs.readFile(COMPONENT_COVERAGE_BASELINE_PATH, "utf8"))) + } catch (error) { + if (args.coverageOnly) throw error + warnCoverage(`Baseline unavailable; all observed types will be reported: ${errorMessage(error)}`) + } const report: Report = { startedAt: new Date().toISOString(), @@ -127,15 +160,16 @@ async function main() { // Skip if this page doesn't belong to this language if (!shouldIncludeInLanguageFile(pageLanguage, lang)) continue - const rendered = await renderLlmsPageMarkdown(raw, absFile, section, lang) + const rendered = await renderLlmsPageMarkdown(raw, absFile, section, lang, coverage.observers) if (rendered.transformMode !== "normal") { console.warn(`[llms] ${toContentRelative(absFile)}: transform mode ${rendered.transformMode}`) } - outputs.push(rendered.markdown) + if (!args.coverageOnly) outputs.push(rendered.markdown) processed += 1 } + if (args.coverageOnly) continue const finalOutput = assembleLlmsDocument(outputs) const outPath = path.resolve(`src/content/${section}/llms-full-${lang}.txt`) @@ -169,15 +203,16 @@ async function main() { if (!absFile.endsWith(".mdx") && !absFile.endsWith(".md")) continue if (isSunsetContentFile(absFile)) continue const raw = await fs.readFile(absFile, "utf-8") - const rendered = await renderLlmsPageMarkdown(raw, absFile, section) + const rendered = await renderLlmsPageMarkdown(raw, absFile, section, undefined, coverage.observers) if (rendered.transformMode !== "normal") { console.warn(`[llms] ${toContentRelative(absFile)}: transform mode ${rendered.transformMode}`) } - outputs.push(rendered.markdown) + if (!args.coverageOnly) outputs.push(rendered.markdown) processed += 1 } + if (args.coverageOnly) continue const finalOutput = assembleLlmsDocument(outputs) const outPath = path.resolve(`src/content/${section}/llms-full.txt`) @@ -204,19 +239,90 @@ async function main() { } } - report.sections.push(...(await generateSolutionLlmsFiles(solutions))) - report.finishedAt = new Date().toISOString() - await writeReports(report, args.reportPath) - console.log("LLM generation complete.") + if (args.coverageOnly) { + for (const solution of solutions) { + await renderSolutionLlmsDocument(solution, await readSolutionLlmsPages(solution), coverage.observers) + } + } else { + report.sections.push(...(await generateSolutionLlmsFiles(solutions, undefined, coverage.observers))) + report.finishedAt = new Date().toISOString() + await writeReports(report, args.reportPath) + } + const coverageReport = coverage.report({ baseline }) + const coveragePath = args.coverageOnly ? COMPONENT_COVERAGE_PUBLIC_PATH : COMPONENT_COVERAGE_PATH + try { + await writeComponentCoverageReport(coverageReport, coveragePath) + console.log(`[component-coverage] ${coveragePath}`) + } catch (error) { + if (args.coverageOnly) throw error + await fs.rm(coveragePath, { force: true }).catch(() => { + // Cleanup failure must also remain non-blocking. + }) + warnCoverage(`Report unavailable: ${errorMessage(error)}`) + } + const nextSteps = `Check the pages listed in ${coveragePath}, then add a Markdown handler or a "ComponentName": "Reason for accepting this loss" entry in ${COMPONENT_COVERAGE_BASELINE_PATH}.` + if (coverageReport.newSinceBaseline.length) { + warnCoverage(`Unhandled components outside baseline: ${coverageReport.newSinceBaseline.join(", ")}. ${nextSteps}`) + } + if (!args.coverageOnly && process.env.GITHUB_STEP_SUMMARY) { + try { + const findings = coverageReport.newSinceBaseline.length + ? `Outside baseline:\n\n${coverageReport.components + .filter((component) => coverageReport.newSinceBaseline.includes(component.name)) + .map( + (component) => + `- \`${component.name}\`: ${component.pages.map((page) => `\`src/content/${page}\``).join(", ")}` + ) + .join("\n")}\n\n${nextSteps}` + : "No component types outside baseline." + await fs.appendFile( + process.env.GITHUB_STEP_SUMMARY, + `\n### Markdown component coverage\n\n${coverageReport.pagesProcessed} source pages; ${coverageReport.components.length} dropped component types.\n\n${findings}\n\nTransform modes: ${Object.entries( + coverageReport.transformModes + ) + .map(([mode, count]) => `${mode}: ${count}`) + .join(", ")}.\n` + ) + } catch (error) { + warnCoverage(`Job summary unavailable: ${errorMessage(error)}`) + } + } + console.log( + args.coverageOnly + ? `[component-coverage] Coverage-only completed in ${((performance.now() - started) / 1000).toFixed(2)}s` + : "LLM generation complete." + ) +} + +function errorMessage(error: unknown): string { + return error instanceof Error ? error.message : String(error) +} + +function warnCoverage(message: string) { + const warning = `[component-coverage] ${message}` + console.warn( + process.env.GITHUB_ACTIONS === "true" + ? `::warning::${warning.replace(/%/g, "%25").replace(/\r/g, "%0D").replace(/\n/g, "%0A")}` + : warning + ) } function parseArgs(argv: string[]): CliArgs { const args: CliArgs = {} for (let i = 0; i < argv.length; i++) { const a = argv[i] - if (a === "--section") args.section = argv[++i] - if (a === "--pages") args.pagesFile = argv[++i] - if (a === "--report") args.reportPath = argv[++i] + if (a === "--coverage-only") { + args.coverageOnly = true + continue + } + if (!["--section", "--pages", "--report"].includes(a)) { + throw new Error(`[llms] Unknown option: ${a}`) + } + const value = argv[++i] + if (!value || value.startsWith("--")) throw new Error(`[llms] ${a} requires a value`) + if (a === "--section") args.section = value + if (a === "--pages") args.pagesFile = value + if (a === "--report") args.reportPath = value } return args } @@ -299,6 +405,6 @@ async function writeReports(report: Report, reportPath?: string) { if (process.argv[1] && path.resolve(process.argv[1]) === fileURLToPath(import.meta.url)) { main().catch((err) => { console.error(err) - process.exit(1) + process.exitCode = 1 }) }