From 24310b47faa8e68aa721e2781f069bc36738ae66 Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 1 Nov 2026 14:50:21 +0000 Subject: [PATCH 1/2] Put the whole family into the search index The index the home page and the playground read carried the manual and the three sample catalogues. It now carries the rest of what a reader looks for from the home page (scripts/lib/search-family.mjs): - where to go: the playground, the layout editor, the AI builder, the sister sites, releases, issues - with the words people use for them - every repository of abap2UI5 and abap2UI5-addons (GitHub API), with its README and issues as actions - the essays, the page for decision makers and the addons, from the search.json each of those sites now publishes - the client API, each method leading to its section and to its line in z2ui5_if_client, and every class of the framework outside src/99 - a sample's source on GitHub as an action beside it Every source may fail and costs its rows, never the build. The matcher answers three more things about a query: a scope (`repo:`, `api:`, ... narrow to one area), jumps (`#1234`, a class, a repository or a tool named whole) and elsewhere( ): the query handed to GitHub's code, issue and discussion search - links, so issues and discussions are never a day old and nothing is fetched from GitHub in the browser. A box that does not use them loses nothing. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_014YZFmFLTfBmZJXKPuRQWci --- .github/workflows/check.yml | 4 + .github/workflows/deploy.yml | 4 + AGENTS.md | 3 +- scripts/generate-search.mjs | 11 +- scripts/lib/search-engine.js | 127 ++++++++++++++- scripts/lib/search-family.mjs | 296 ++++++++++++++++++++++++++++++++++ scripts/lib/search-index.mjs | 22 ++- test/search-family.test.mjs | 148 +++++++++++++++++ 8 files changed, 604 insertions(+), 11 deletions(-) create mode 100644 scripts/lib/search-family.mjs create mode 100644 test/search-family.test.mjs diff --git a/.github/workflows/check.yml b/.github/workflows/check.yml index 76b163b1a..6e2a9f8be 100644 --- a/.github/workflows/check.yml +++ b/.github/workflows/check.yml @@ -106,6 +106,10 @@ jobs: # alone. A page that does not build is a page nobody can read. - name: build the site if: ${{ !cancelled() }} + # the search index lists the organisations' repositories and the + # framework's classes from the GitHub API (scripts/lib/search-family.mjs) + env: + GITHUB_TOKEN: ${{ github.token }} run: npm run docs:build # every link out of this deployment and into a neighbouring one on the diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index c18e46b2e..183c7bb34 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -176,6 +176,10 @@ jobs: # THE SITE THAT IS PUBLISHED: VitePress's build, from this repository # alone. - name: Build the site + # the search index lists the organisations' repositories and the + # framework's classes from the GitHub API (scripts/lib/search-family.mjs) + env: + GITHUB_TOKEN: ${{ github.token }} run: npm run docs:build # The one gate that cannot run before the build, because what it judges diff --git a/AGENTS.md b/AGENTS.md index 84838fab3..71f6a91d3 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -21,7 +21,8 @@ person reads the page. Do not put "as an AI, …" prose back into `docs/`. | `scripts/generate-api-reference.mjs` | Generates the client API reference — the block in `docs/resources/api.md` and `docs/public/api/client-api.json` — from `z2ui5_if_client` on the framework branch this site tracks (`main`); `--check` is the freshness gate | | `scripts/lib/client-interface.mjs` | Where `z2ui5_if_client` is fetched from (the ref comes from `lib/release.mjs`, shared with `check-api-names.mjs`; `A2UI5_HOME` reads the same ref out of a local clone of the framework instead, with `git show`, never its working tree) and the full parser `generate-api-reference.mjs` renders from; pinned by `test/client-interface.test.mjs` | | `scripts/check-version.mjs` | The release number in the bar's version dropdown, the deprecations page and the changelog, against the newest release tag of the framework | -| `scripts/generate-search.mjs` | Builds `docs/public/search-index.json` — the pages of this site plus every entry in the three sample catalogues. This site's own bar searches with VitePress's local search; the index is published for the playground's search box, which fetches it from here. `scripts/lib/search-engine.js` is the matcher the playground carries a copy of, pinned with the index by `test/search.test.mjs`. Runs inside `docs:build`, so the deploy publishes it; `scripts/lib/search-index.mjs` is what goes in | +| `scripts/generate-search.mjs` | Builds `docs/public/search-index.json` — the pages of this site, every entry in the three sample catalogues, and the rest of the family (below). This site's own bar searches with VitePress's local search; the index is published for the search box on the home page (https://abap2ui5.github.io/, abap2UI5/abap2UI5.github.io) and the playground's, which fetch it from here. `scripts/lib/search-engine.js` is the matcher both carry a copy of, pinned with the index by `test/search.test.mjs` and `test/search-family.test.mjs`. Runs inside `docs:build`, so the deploy publishes it; `scripts/lib/search-index.mjs` is what goes in | +| `scripts/lib/search-family.mjs` | The rest of the family in that index: where to go (the tools, as data here), every repository of `abap2UI5` and `abap2UI5-addons` (GitHub API, `GITHUB_TOKEN` in CI), the essays, the page for decision makers and the addons (each from the `search.json` its site publishes - one shape, `{ site, url, entries: [{ title, text, headings, terms, url, links }] }`), the client API (from the committed `client-api.json`, each method linked to its line in `z2ui5_if_client`) and every class of the framework outside `src/99`. Every source may fail and costs its rows, never the build. Issues and discussions are not indexed: the box hands the query to GitHub's search for them (`elsewhere( )` in the matcher), so they are never a day old | | `scripts/lib/pages.mjs` | What a page of this site IS: the sidebar walk, its title, its opening sentences, its headings, its words. Shared by `generate-llms.mjs` and `generate-search.mjs` so a page added to the sidebar reaches both; `summarise( )` — the description of 150 pages and the note in `llms.txt`, which nobody writes and nobody proofreads — is pinned by `test/summarise.test.mjs` | | `docs/.vitepress/playground.mjs` | Decides which fenced ABAP example gets a **Run** button, and wraps the fence; `theme/playground.js` is the browser half | | `scripts/list-runnable.mjs` | The measurement's worklist: every fenced example that carries a Run button, out of the same `playground.mjs` that decides the button. `--json` adds each example's ABAP verbatim - what the button sends - so the measurement below can be driven rather than clicked | diff --git a/scripts/generate-search.mjs b/scripts/generate-search.mjs index 1ec0b51e1..9427e774c 100644 --- a/scripts/generate-search.mjs +++ b/scripts/generate-search.mjs @@ -1,6 +1,8 @@ #!/usr/bin/env node -// The search index the bar reads: every page of this site, every sample in the -// three catalogues, one JSON document at /docs/search-index.json. +// The search index the boxes of the family read: every page of this site, every +// sample in the three catalogues and the rest of the family (repositories, +// tools, the sister sites, the API - scripts/lib/search-family.mjs), one JSON +// document at /docs/search-index.json. // // Runs inside `docs:build`, before vitepress, so the file is in docs/public/ // when the build copies it - the same arrangement as generate-llms.mjs, and @@ -24,7 +26,8 @@ const OUT = path.join(DOCS, 'public', 'search-index.json'); const index = await buildIndex(ROOT, { log: (line) => console.log(line) }); const docs = index.entries.filter((e) => e.area === 'docs').length; -const samples = index.entries.length - docs; +const samples = index.entries.filter((e) => e.area === 'samples').length; +const rest = index.entries.length - docs - samples; /* The floor. An index with no page in it is a broken walk - the sidebar moved, * or config.mjs stopped exporting one - and an empty search box that reports @@ -40,4 +43,4 @@ fs.mkdirSync(path.dirname(OUT), { recursive: true }); fs.writeFileSync(OUT, JSON.stringify(index)); const kb = Math.round(fs.statSync(OUT).size / 1024); -console.log(`search-index.json: ${docs} page(s) + ${samples} sample(s) (${kb} kB)`); +console.log(`search-index.json: ${docs} page(s) + ${samples} sample(s) + ${rest} from the rest of the family (${kb} kB)`); diff --git a/scripts/lib/search-engine.js b/scripts/lib/search-engine.js index 9c102451a..36770f876 100644 --- a/scripts/lib/search-engine.js +++ b/scripts/lib/search-engine.js @@ -1,5 +1,5 @@ /* - * The matcher behind the search box in the bar — framework-free on purpose. + * The matcher behind the search boxes of the family — framework-free on purpose. * * Plain JavaScript with no imports, and this site's own: the index it reads * is /docs/search-index.json, generated by scripts/generate-search.mjs from @@ -292,6 +292,21 @@ function nearest(entries, term) { * (above) and the list says so: `hits.relaxedTo` is the query that answered. */ export function search(entries, query, { limit = 30 } = {}) { + /* A scope - `repo: popup` - narrows the entries before anything is scored, + * and a scope alone - `repo:` - lists its area, which is how a reader asks + * "what repositories are there". */ + const scoped = parseScope(query); + if (scoped.area) { + entries = entries.filter((e) => e.area === scoped.area); + query = scoped.query; + if (!queryTerms(query).length) { + return entries + .slice() + .sort((a, b) => (a.title || '').localeCompare(b.title || '')) + .slice(0, limit) + .map((entry) => ({ entry, score: 0, heading: null, strong: true })); + } + } /* Eight words at most. Nobody narrows a list with a ninth, and a pasted * error message or a line of ABAP is thirty: every relaxing pass below * scored the whole index once per word, and the box froze for seconds on @@ -476,3 +491,113 @@ export function recallQuery() { const q = record.q.trim(); return q && q.length <= QUERY_MAX ? q : ''; } + +/* ── SCOPES, JUMPS AND THE WAY OUT ────────────────────────────────────────── + * + * The index carries more than the manual and the samples since the home page + * got its box: where to go (`tools`), the repositories (`repos`), the sister + * sites (`insights`, `why`, `addons`) and the code (`api`) - see + * scripts/lib/search-family.mjs. Three things go with that, all of them data + * about the QUERY rather than about the ranking, so a box that does not use + * them (the playground's, today) loses nothing: + * + * parseScope( ) `repo: popup`, `api: bind` - a word and a colon in front + * narrow the search to one area; + * jumps( ) the query that NAMES something - `#1234`, a class name, + * a repository, a tool - answered with that thing, before + * any list; + * elsewhere( ) the query handed to GitHub's own search, for what no index + * of ours carries: the code of every repository, the issues, + * the discussions. Links, not requests - the box fetches + * nothing from GitHub, and the answer is never a day old. + */ + +/** The words a scope can be written as, and the area each means. */ +export const SCOPES = { + doc: 'docs', docs: 'docs', + sample: 'samples', samples: 'samples', + repo: 'repos', repos: 'repos', + api: 'api', code: 'api', class: 'api', + tool: 'tools', tools: 'tools', go: 'tools', + insight: 'insights', insights: 'insights', essay: 'insights', + why: 'why', + addon: 'addons', addons: 'addons', +}; + +/** `{ area, query }`: the area a leading `word:` names, and the rest. A word + * that is no scope leaves the query as it is - `z2ui5:` is a search. */ +export function parseScope(query) { + const m = /^\s*([a-z]+):\s*(.*)$/is.exec(query || ''); + const area = m && SCOPES[m[1].toLowerCase()]; + return area ? { area, query: m[2] } : { area: null, query: query || '' }; +} + +const GITHUB = 'https://github.com'; +const FRAMEWORK = `${GITHUB}/abap2UI5/abap2UI5`; +const enc = encodeURIComponent; + +/** GitHub's code search over both organisations. */ +export const codeSearchUrl = (q) => `${GITHUB}/search?q=${enc(`(org:abap2UI5 OR org:abap2UI5-addons) ${q}`)}&type=code`; + +/** + * What the query NAMES, if it names something: at most three targets, each + * `{ entry }` (a hit of the index) or `{ title, text, url }` (a place + * outside it). Empty when the query is a question rather than a name - the + * list answers those. + * + * #1234, issue 1234, pr 1234 that issue or pull request of the framework + * a class, a method, a repo the entry whose `code` it is, whole + * a tool or a repository name the entry whose title it is, whole + * z2ui5_… in no entry the code search for it + */ +export function jumps(entries, query) { + const q = parseScope(query).query.trim(); + const out = []; + const m = /^(?:#|(?:issue|pr|pull)\s*#?)(\d{1,6})$/i.exec(q); + if (m) out.push({ title: `#${m[1]}`, text: 'Issue or pull request of abap2UI5/abap2UI5', url: `${FRAMEWORK}/issues/${m[1]}` }); + + const low = normalise(q).replace(/\s+/g, ' '); + if (low.length >= 3) { + for (const e of entries) { + const code = e.code || ''; + const named = code === low || code.split('/').pop() === low + || ((e.area === 'tools' || e.area === 'repos') && normalise(e.title) === low); + if (named) out.push({ entry: e }); + } + } + if (!out.length && /^z2ui5_\w{3,}$/i.test(q)) { + out.push({ title: q, text: 'Not in the index - every repository’s code, on GitHub', url: codeSearchUrl(q) }); + } + const seen = new Set(); + return out.filter((j) => { + const url = j.entry ? j.entry.url : j.url; + if (seen.has(url)) return false; + seen.add(url); + return true; + }).slice(0, 3); +} + +/** The query handed on to GitHub: `[{ label, url }]`, empty for an empty + * query. Issues and discussions are never indexed (a nightly copy of them + * is a day old exactly when it matters), so this is how they are searched. */ +export function elsewhere(query) { + const q = parseScope(query).query.trim(); + if (!q) return []; + return [ + { label: 'Code', url: codeSearchUrl(q) }, + { label: 'Issues', url: `${GITHUB}/search?q=${enc(`org:abap2UI5 org:abap2UI5-addons ${q}`)}&type=issues` }, + { label: 'Discussions', url: `${GITHUB}/search?q=${enc(`org:abap2UI5 ${q}`)}&type=discussions` }, + ]; +} + +/** The actions beside a hit: what its entry carries (`links` - the source of + * a sample, the README of a repository) and, for a repository, its code + * searched for what was typed. */ +export function actions(entry, query = '') { + const out = Array.isArray(entry?.links) ? entry.links.filter((l) => l && l.label && l.url) : []; + const q = parseScope(query).query.trim(); + if (entry?.area === 'repos' && entry.code && q && normalise(q) !== entry.code.split('/').pop()) { + out.push({ label: 'Search code', url: `${GITHUB}/search?q=${enc(`repo:${entry.code} ${q}`)}&type=code` }); + } + return out; +} diff --git a/scripts/lib/search-family.mjs b/scripts/lib/search-family.mjs new file mode 100644 index 000000000..528bb907f --- /dev/null +++ b/scripts/lib/search-family.mjs @@ -0,0 +1,296 @@ +/* + * search-family — the rest of the project in the one search index. + * + * search-index.mjs builds the two areas this repository owns or has always + * carried: the pages of the manual and the three sample catalogues. That + * answered "where is it explained" and "where is an example of it". It did + * not answer the other questions a reader brings to a box on the home page: + * WHERE do I go (a repository, a tool, a site of the family), WHAT does the + * rest of the family say about it (the essays, the page for decision makers, + * the addons), and WHERE in the code is it (a client method, a class of the + * framework, down to the file and the line). + * + * So, beside the two: + * + * tools the places of the family a reader goes to rather than reads - + * the playground, the layout editor, the AI builder - with the + * words people use for them. Data, here, below. + * repos every repository of github.com/abap2UI5 and abap2UI5-addons, + * from the GitHub API: name, description, topics. A hit opens the + * repository; its README and issues are actions beside it. + * insights the essays, why the page for decision makers, addons the + * why addons and the developer tools - each from the search.json its + * addons site publishes (one shape on every site, see SITES below). + * api the client API (docs/public/api/client-api.json, committed and + * held fresh by check:api-reference), each method leading to its + * section of resources/api.md and to its line in + * z2ui5_if_client; and every class and interface of the + * framework outside the frozen package, leading to its file. + * + * EVERY SOURCE MAY FAIL. The GitHub API, a sister site, raw.githubusercontent: + * an unreachable one costs its rows and is reported, never the build - the + * rule the sample catalogues already follow. An index with fewer areas is a + * worse search; a failed deploy of the manual over a sister site's outage is + * a worse documentation. + * + * Issues and discussions are deliberately NOT indexed: a nightly copy of them + * would be a day old when it mattered most. The box hands the query to + * GitHub's own search for those instead (`elsewhere( )` in search-engine.js). + */ +import fs from 'node:fs'; +import path from 'node:path'; +import { DOCS, headings } from './pages.mjs'; +import { fetchInterface } from './client-interface.mjs'; + +const GITHUB = 'https://github.com'; +const API = 'https://api.github.com'; +export const ORGS = ['abap2UI5', 'abap2UI5-addons']; + +/** The places a reader GOES to. `terms` are the words people use for them, + * which is the whole reason these are entries and not tiles on a page. */ +export const DESTINATIONS = [ + { title: 'Playground', text: 'Write ABAP in the browser and run the app beside it - no SAP system needed.', url: 'https://abap2ui5.github.io/playground/', terms: 'try online browser editor run abap without system sandbox' }, + { title: 'Layout Editor', text: 'Design a UI5 view by drag and drop, then export it as an ABAP class.', url: 'https://abap2ui5.github.io/layout-editor/', terms: 'drag drop designer wysiwyg view editor layout builder' }, + { title: 'AI App Builder', text: 'Describe the app you need; the AI writes it and you download it for abapGit.', url: 'https://abap2ui5.github.io/playground/ai/', terms: 'ai llm generate app chatgpt claude prompt builder' }, + { title: 'AI Agent', text: 'Let the AI operate a running app: describe the task in the chat and watch it work.', url: 'https://abap2ui5.github.io/playground/pilot/', terms: 'ai agent pilot automate operate chat' }, + { title: 'Sample catalogue', text: 'Every sample app, by control and UI5 release, each runnable in the playground.', url: 'https://abap2ui5.github.io/samples-site/', terms: 'samples examples demos catalogue catalog controls' }, + { title: 'Documentation', text: 'The manual: how it works, how to install it, how to build apps.', url: 'https://abap2ui5.github.io/docs/', terms: 'docs manual guide handbook' }, + { title: 'Insights', text: 'Short essays on the idea: how it works, where it belongs, running it.', url: 'https://abap2ui5.github.io/insights/', terms: 'essays articles blog background architecture' }, + { title: 'Why abap2UI5', text: 'For decision makers: what it builds, who runs it, what it costs.', url: 'https://abap2ui5.github.io/why/', terms: 'managers business case cost decision evaluate' }, + { title: 'Addons & Tools', text: 'Addons for your system, tools for your editor and pipeline, each with a page.', url: 'https://abap2ui5-addons.github.io/', terms: 'addons extensions plugins tools' }, + { title: 'Home', text: 'The entry page of abap2UI5.', url: 'https://abap2ui5.github.io/', terms: 'start home entry' }, + { title: 'Slack', text: 'The community channel #abap2ui5 on the abapGit Slack.', url: 'https://abapgit.slack.com/join/shared_invite/zt-46tqufaht-QlrxTzlDqlx85CWbeUnOqg#/shared-invite/email', terms: 'chat community help ask question' }, + { title: 'Releases', text: 'The releases of the framework, with their notes.', url: `${GITHUB}/abap2UI5/abap2UI5/releases`, terms: 'release changelog version download latest' }, + { title: 'Report an issue', text: 'Open an issue for the framework on GitHub.', url: `${GITHUB}/abap2UI5/abap2UI5/issues/new/choose`, terms: 'bug report issue problem error new' }, + { title: 'Discussions', text: 'Questions, ideas and show-and-tell, on GitHub.', url: `${GITHUB}/abap2UI5/abap2UI5/discussions`, terms: 'question idea forum ask discuss' }, +]; + +export function destinationEntries() { + return DESTINATIONS.map((d) => ({ area: 'tools', group: 'Go to', ...d })); +} + +/** The sites that publish a search.json, and the area each becomes. */ +export const SITES = [ + { area: 'insights', label: 'Insights', url: 'https://abap2ui5.github.io/insights/search.json' }, + { area: 'why', label: 'Why abap2UI5', url: 'https://abap2ui5.github.io/why/search.json' }, + { area: 'addons', label: 'Addons & Tools', url: 'https://abap2ui5-addons.github.io/search.json' }, +]; + +const timeout = () => AbortSignal.timeout(20000); +const ghHeaders = (token = process.env.GITHUB_TOKEN) => ({ + Accept: 'application/vnd.github+json', + 'User-Agent': 'abap2UI5-docs-search-index', + ...(token ? { Authorization: `Bearer ${token}` } : {}), +}); + +/** Is this an http(s) URL of the family or GitHub - the only places a feed + * may send a reader. A feed is another repository's file; a hit that leads + * off the project is not something it gets to publish into this box. */ +const OWN = /^https:\/\/(abap2ui5\.github\.io|abap2ui5-addons\.github\.io|github\.com\/abap2UI5(-addons)?)\//i; + +/** One site's search.json as entries of `area`. Only the fields the matcher + * reads, each checked for its type: a feed that changed shape costs its + * odd entries, not the index. */ +export function siteEntries(feed, { area, label }) { + const out = []; + for (const e of Array.isArray(feed?.entries) ? feed.entries : []) { + if (typeof e?.title !== 'string' || !e.title.trim() || typeof e.url !== 'string' || !OWN.test(e.url)) continue; + const entry = { area, group: label, title: e.title.trim(), text: typeof e.text === 'string' ? e.text : '', url: e.url }; + if (Array.isArray(e.headings)) { + const hs = e.headings.filter((h) => Array.isArray(h) && typeof h[0] === 'string' && typeof h[1] === 'string'); + if (hs.length) entry.headings = hs; + } + if (typeof e.terms === 'string' && e.terms) entry.terms = e.terms; + if (typeof e.code === 'string' && e.code) entry.code = e.code; + const links = linksOf(e.links); + if (links.length) entry.links = links; + out.push(entry); + } + return out; +} + +function linksOf(links) { + return (Array.isArray(links) ? links : []) + .filter((l) => typeof l?.label === 'string' && typeof l.url === 'string' && OWN.test(l.url)) + .map((l) => ({ label: l.label, url: l.url })); +} + +/** Every repository of the organisations, as entries. Archived ones are left + * out: a search that leads to a repository nobody maintains is a wrong + * answer that looks right. Forks of the organisation's own are kept - they + * are how an addon carries a library it adapted. */ +export function repoEntries(repos) { + return repos + .filter((r) => r && !r.archived && !r.private && typeof r.full_name === 'string') + .map((r) => { + const [org, name] = r.full_name.split('/'); + return { + area: 'repos', + group: 'Repositories', + title: name, + text: r.description || '', + /* the full name is what somebody pastes - abap2UI5-addons/popups */ + code: r.full_name.toLowerCase(), + terms: [org, ...(r.topics || []), r.language || '', r.homepage ? 'site' : ''].join(' ').toLowerCase().trim(), + url: r.html_url || `${GITHUB}/${r.full_name}`, + links: [ + { label: 'README', url: `${GITHUB}/${r.full_name}#readme` }, + { label: 'Issues', url: `${GITHUB}/${r.full_name}/issues` }, + ...(r.homepage && OWN.test(r.homepage) ? [{ label: 'Site', url: r.homepage }] : []), + ], + }; + }); +} + +async function fetchRepos(org, fetchFn) { + const all = []; + for (let page = 1; page <= 5; page++) { + const res = await fetchFn(`${API}/orgs/${org}/repos?type=public&per_page=100&page=${page}`, { headers: ghHeaders(), signal: timeout() }); + if (!res.ok) throw new Error(`HTTP ${res.status}`); + const rows = await res.json(); + all.push(...rows); + if (rows.length < 100) break; + } + return all; +} + +/* ── THE CODE ─────────────────────────────────────────────────────────────── */ + +const FRAMEWORK = 'abap2UI5/abap2UI5'; +const CLIENT_API = path.join(DOCS, 'public', 'api', 'client-api.json'); +const API_PAGE = 'https://abap2ui5.github.io/docs/resources/api.html'; + +/** The first sentence of a doc block - what a result row has room for. */ +const first = (lines) => { + const s = (Array.isArray(lines) ? lines.join(' ') : String(lines || '')).replace(/\s+/g, ' ').trim(); + const m = s.match(/^.{20,}?[.!?](\s|$)/); + return (m ? m[0] : s).trim(); +}; + +/** The line a member is declared on in the interface source, 1-based, or 0. */ +export function lineOf(source, name, kind = 'METHODS') { + if (!source) return 0; + const re = new RegExp(`^\\s*(${kind}|CLASS-${kind})\\s+${name.replace(/[.*+?^${}()|[\]\\]/g, '\\$&')}\\b`, 'im'); + const lines = source.split('\n'); + const i = lines.findIndex((l) => re.test(l)); + return i < 0 ? 0 : i + 1; +} + +/** The client API: one entry per method and per constant group, from the + * committed client-api.json, with the anchors resources/api.md really has. */ +export function clientApiEntries(api, apiPageBody, source) { + const anchors = new Map(headings(apiPageBody).map((h) => [h.text, h.anchor])); + const file = api.source || `${GITHUB}/${FRAMEWORK}/blob/main/src/02/z2ui5_if_client.intf.abap`; + const at = (name, kind) => { + const n = lineOf(source, name, kind); + return n ? `${file}#L${n}` : file; + }; + const out = []; + for (const m of api.methods || []) { + const anchor = anchors.get(m.name); + out.push({ + area: 'api', + group: 'API', + title: `client->${m.name}( )`, + text: first(m.doc), + code: m.name.toLowerCase(), + terms: [m.group, ...(m.parameters || []).map((p) => p.name)].join(' ').toLowerCase(), + url: anchor ? `${API_PAGE}#${anchor}` : API_PAGE, + links: [{ label: 'Source', url: at(m.name, 'METHODS') }], + }); + } + for (const c of api.constants || []) { + const anchor = anchors.get(c.name); + out.push({ + area: 'api', + group: 'API', + title: `client->${c.name}`, + text: first(c.doc) || `The ${c.name} constants of z2ui5_if_client.`, + code: c.name.toLowerCase(), + terms: (c.members || []).map((x) => x.name).join(' ').toLowerCase(), + url: anchor ? `${API_PAGE}#${anchor}` : API_PAGE, + links: [{ label: 'Source', url: at(c.name, 'CONSTANTS') }], + }); + } + return out; +} + +/** Every class and interface of the framework outside src/99 - the frozen + * package, whose names this site does not teach (check:api-names) and a + * search box should not offer. From the tree of main: one request. */ +export function sourceEntries(tree) { + const out = []; + for (const t of tree || []) { + const m = /^src\/(?!99\/)(?:.*\/)?(z2ui5_\w+)\.(clas|intf)\.abap$/i.exec(t?.path || ''); + if (!m || t.type !== 'blob') continue; + const [, name, kind] = m; + out.push({ + area: 'api', + group: 'Framework source', + title: name.toLowerCase(), + text: `${kind === 'intf' ? 'Interface' : 'Class'} of the framework - ${t.path.split('/').slice(1, -1).join('/') || 'src'}`, + code: name.toLowerCase(), + url: `${GITHUB}/${FRAMEWORK}/blob/main/${t.path}`, + }); + } + return out; +} + +/** + * The family's areas, each fetched on its own, each allowed to fail. + * `{ entries, areas }` - `areas` in the same shape search-index.mjs reports. + */ +export async function familyEntries({ fetchFn = globalThis.fetch, log = () => {}, readFile = fs.readFileSync, interfaceFn = fetchInterface } = {}) { + const entries = []; + const areas = []; + const add = (area, label, rows, from) => { + entries.push(...rows); + areas.push({ area, label, count: rows.length }); + log(` ${label}: ${rows.length} entr${rows.length === 1 ? 'y' : 'ies'} (${from})`); + }; + const tryArea = async (label, fn) => { + try { + await fn(); + } catch (err) { + log(` ${label}: not reachable (${err.message}) — not in this index`); + } + }; + + add('tools', 'Go to', destinationEntries(), 'scripts/lib/search-family.mjs'); + + await tryArea('Repositories', async () => { + const repos = []; + for (const org of ORGS) repos.push(...await fetchRepos(org, fetchFn)); + add('repos', 'Repositories', repoEntries(repos), `api.github.com, ${ORGS.join(' + ')}`); + }); + + for (const site of SITES) { + await tryArea(site.label, async () => { + const res = await fetchFn(site.url, { signal: timeout() }); + if (!res.ok) throw new Error(`HTTP ${res.status}`); + const rows = siteEntries(await res.json(), site); + if (!rows.length) throw new Error('no entry in the feed'); + add(site.area, site.label, rows, site.url); + }); + } + + await tryArea('API', async () => { + const api = JSON.parse(readFile(CLIENT_API, 'utf8')); + const page = readFile(path.join(DOCS, 'resources', 'api.md'), 'utf8'); + /* the line numbers are a nicety: without the source the links go to the + * file, which is still the right file */ + let source = ''; + try { source = await interfaceFn(api.ref || 'main'); } catch (err) { log(` API: interface source not reachable (${err.message}) — links without a line`); } + add('api', 'API', clientApiEntries(api, page, source), 'docs/public/api/client-api.json'); + }); + + await tryArea('Framework source', async () => { + const res = await fetchFn(`${API}/repos/${FRAMEWORK}/git/trees/main?recursive=1`, { headers: ghHeaders(), signal: timeout() }); + if (!res.ok) throw new Error(`HTTP ${res.status}`); + const rows = sourceEntries((await res.json()).tree); + if (!rows.length) throw new Error('no class in the tree'); + add('api', 'Framework source', rows, `api.github.com, ${FRAMEWORK}@main`); + }); + + return { entries, areas }; +} diff --git a/scripts/lib/search-index.mjs b/scripts/lib/search-index.mjs index 4dd1aa0b3..a865bb538 100644 --- a/scripts/lib/search-index.mjs +++ b/scripts/lib/search-index.mjs @@ -17,10 +17,12 @@ * summary and keywords the sample repositories maintain for it, and * a link to that sample's own page in the catalogue * - * The playground is not an area. It has no content to find: its URL carries + * The playground is not a corpus. It has no content to find: its URL carries * the code in the editor, and a result that opened an empty editor would - * answer no question anybody typed. It is a destination in the bar, not in the - * index — the same reasoning that keeps it out of the position memory. + * answer no question anybody typed. It is a DESTINATION - one row of the + * `tools` area, with the layout editor and the AI builder, in + * search-family.mjs, which also brings in the repositories, the sister sites + * and the API. * * WHERE IT IS PUBLISHED, AND WHY THAT IS ENOUGH. `/docs/search-index.json`, on * the origin all four documents share, fetched by whichever of them the reader @@ -34,6 +36,7 @@ import fs from 'node:fs'; import { entriesOf, loadCatalogue } from './catalogue.mjs'; import { sidebarPages, fileOf, describe, title, headings, terms, SITE } from './pages.mjs'; +import { familyEntries } from './search-family.mjs'; /** The three sample repositories, and where a reader is sent for a hit. */ export const CORPORA = [ @@ -75,7 +78,7 @@ export function docEntries(pages = sidebarPages(), read = (link) => fs.readFileS * three catalogues could be reached. A repository that could not is left out * and reported - it costs its rows, never the build, which is the rule every * other reader of these catalogues here already follows. */ -export function sampleEntries(catalogue, label) { +export function sampleEntries(catalogue, label, repo) { return entriesOf(catalogue).map((e) => { const cls = String(e.class); /* The three repositories describe an entry with slightly different fields @@ -102,6 +105,9 @@ export function sampleEntries(catalogue, label) { .toLowerCase().split(/[^\p{L}\p{N}._]+/u).filter(Boolean), )].join(' '), url: samplePage(cls), + /* the class itself, where its repository keeps it - the action beside + * the hit, for the reader who wants the code rather than the app */ + ...(repo && (e.file || e.path) ? { links: [{ label: 'Source', url: `https://github.com/abap2UI5/${repo}/blob/main/${e.file || e.path}` }] } : {}), }; }); } @@ -198,11 +204,17 @@ export async function buildIndex(root, { fetchFn = globalThis.fetch, log = () => log(` ${repo}: no catalogue reachable — its samples are not in this index`); continue; } - const rows = sampleEntries(found.catalogue, label); + const rows = sampleEntries(found.catalogue, label, repo); entries.push(...rows); areas.push({ area: 'samples', repo, label, count: rows.length }); log(` ${repo}: ${rows.length} sample(s) (${found.source})`); } + /* the rest of the family: where to go, the repositories, the sister + * sites, the API and the framework's source - search-family.mjs */ + const family = await familyEntries({ fetchFn, log }); + entries.push(...family.entries); + areas.push(...family.areas); + return { built: new Date().toISOString(), areas, entries: enrich(entries) }; } diff --git a/test/search-family.test.mjs b/test/search-family.test.mjs new file mode 100644 index 000000000..034c01c78 --- /dev/null +++ b/test/search-family.test.mjs @@ -0,0 +1,148 @@ +/* + * The rest of the family in the search index (scripts/lib/search-family.mjs) + * and the three things the matcher answers about a query beside its ranking + * (parseScope, jumps, elsewhere and actions in scripts/lib/search-engine.js). + * + * The sources are other repositories and the GitHub API, so what is pinned + * here is what this side does with what they send: a feed that leads off the + * project is not published, an archived repository is not offered, the frozen + * package is not taught, and a source that is down costs its rows and never + * the build. + * + * npm test + */ +import { test } from 'node:test'; +import assert from 'node:assert/strict'; +import { + siteEntries, repoEntries, sourceEntries, clientApiEntries, lineOf, destinationEntries, familyEntries, SITES, +} from '../scripts/lib/search-family.mjs'; +import { search, grouped, jumps, elsewhere, actions, parseScope } from '../scripts/lib/search-engine.js'; + +const INSIGHTS = SITES.find((s) => s.area === 'insights'); + +test('a feed of a sister site becomes entries of its area, and only what leads to the project', () => { + const rows = siteEntries({ + site: 'Insights', + entries: [ + { title: '#21 The Lock Is Gone by the Next Click', text: 'ENQUEUE inside one roundtrip.', headings: [['Why', 'why'], ['bad']], terms: 'enqueue lock', url: 'https://abap2ui5.github.io/insights/21-the-lock/' }, + { title: 'Elsewhere', text: 'a page of somebody else', url: 'https://example.com/' }, + { title: '', url: 'https://abap2ui5.github.io/insights/x/' }, + { title: 'No url' }, + { title: 'Addon', url: 'https://abap2ui5-addons.github.io/popups/', links: [{ label: 'Repository', url: 'https://github.com/abap2UI5-addons/popups' }, { label: 'Off', url: 'https://example.com/' }] }, + ], + }, INSIGHTS); + assert.deepEqual(rows.map((r) => r.title), ['#21 The Lock Is Gone by the Next Click', 'Addon']); + assert.equal(rows[0].area, 'insights'); + assert.equal(rows[0].group, 'Insights'); + assert.deepEqual(rows[0].headings, [['Why', 'why']], 'a malformed heading is dropped'); + assert.deepEqual(rows[1].links, [{ label: 'Repository', url: 'https://github.com/abap2UI5-addons/popups' }], 'a link off the project is dropped'); + assert.deepEqual(siteEntries(null, INSIGHTS), [], 'a feed that is not one is nothing'); +}); + +test('a repository is offered with its README and issues, an archived one is not', () => { + const rows = repoEntries([ + { full_name: 'abap2UI5/linter', description: 'A static code analysis tool', topics: ['abap'], html_url: 'https://github.com/abap2UI5/linter', homepage: 'https://abap2ui5.github.io/linter/' }, + { full_name: 'abap2UI5/old', archived: true }, + { full_name: 'abap2UI5-addons/popups', description: 'Value-Help, Search-Help', homepage: 'https://example.com/' }, + ]); + assert.deepEqual(rows.map((r) => r.code), ['abap2ui5/linter', 'abap2ui5-addons/popups']); + assert.deepEqual(rows[0].links.map((l) => l.label), ['README', 'Issues', 'Site']); + assert.deepEqual(rows[1].links.map((l) => l.label), ['README', 'Issues'], 'a homepage off the project is no action'); + assert.equal(rows[1].url, 'https://github.com/abap2UI5-addons/popups'); +}); + +test('the framework source leaves out the frozen package', () => { + const rows = sourceEntries([ + { path: 'src/02/z2ui5_if_client.intf.abap', type: 'blob' }, + { path: 'src/01/04/z2ui5_cl_ui5_view_builder.clas.abap', type: 'blob' }, + { path: 'src/01/04/z2ui5_cl_ui5_view_builder.clas.xml', type: 'blob' }, + { path: 'src/99/z2ui5_cl_xml_view.clas.abap', type: 'blob' }, + { path: 'src/01', type: 'tree' }, + ]); + assert.deepEqual(rows.map((r) => r.code), ['z2ui5_if_client', 'z2ui5_cl_ui5_view_builder']); + assert.equal(rows[1].url, 'https://github.com/abap2UI5/abap2UI5/blob/main/src/01/04/z2ui5_cl_ui5_view_builder.clas.abap'); +}); + +test('a client method leads to its section and to its line', () => { + const api = { + source: 'https://github.com/abap2UI5/abap2UI5/blob/main/src/02/z2ui5_if_client.intf.abap', + methods: [{ name: 'view_display', group: 'Main view', doc: ['Display the MAIN view. A new main view is a new screen.'], parameters: [{ name: 'val' }] }], + constants: [{ name: 'cs_event', doc: [], members: [{ name: 'popup_close' }] }], + }; + const page = '# Client API\n\n## Main view\n\n### `view_display`\n\ntext\n\n## Constants\n\n### `cs_event`\n'; + const source = 'INTERFACE z2ui5_if_client.\n CONSTANTS:\n BEGIN OF cs_event,\n METHODS view_display\n IMPORTING val TYPE clike.\n'; + const [method, constant] = clientApiEntries(api, page, source); + assert.equal(method.title, 'client->view_display( )'); + assert.equal(method.text, 'Display the MAIN view.'); + assert.equal(method.url, 'https://abap2ui5.github.io/docs/resources/api.html#view-display'); + assert.equal(method.links[0].url, `${api.source}#L4`); + assert.equal(constant.code, 'cs_event'); + assert.equal(lineOf(source, 'nothing_like_it'), 0); + assert.equal(clientApiEntries(api, page, '')[0].links[0].url, api.source, 'no source, no line - still the file'); +}); + +test('a source that is down costs its rows, never the build', async () => { + const lines = []; + const down = async () => ({ ok: false, status: 503, json: async () => ({}) }); + const { entries, areas } = await familyEntries({ fetchFn: down, interfaceFn: async () => { throw new Error('offline'); }, log: (l) => lines.push(l) }); + assert.ok(entries.some((e) => e.area === 'tools'), 'where to go needs no network'); + assert.ok(entries.some((e) => e.area === 'api' && e.group === 'API'), 'the client API is committed here'); + assert.ok(!areas.some((a) => a.area === 'repos'), 'no repositories without the API'); + assert.ok(lines.some((l) => /Repositories: not reachable/.test(l)), 'and the log says so'); +}); + +/* ── the matcher ─────────────────────────────────────────────────────────── */ + +const ENTRIES = [ + ...destinationEntries(), + ...repoEntries([{ full_name: 'abap2UI5/linter', description: 'A static code analysis tool', topics: [] }, { full_name: 'abap2UI5-addons/popups', description: 'Value-Help, Search-Help, Transport', topics: [] }]), + { area: 'samples', group: 'Controls', title: 'Carousel', text: 'A carousel.', code: 'z2ui5_cl_smpc_app_398', url: 'https://abap2ui5.github.io/playground/samples/z2ui5_cl_smpc_app_398/', links: [{ label: 'Source', url: 'https://github.com/abap2UI5/samples-controls/blob/main/src/x.clas.abap' }] }, + { area: 'api', group: 'API', title: 'client->view_display( )', text: 'Display the MAIN view.', code: 'view_display', url: 'https://abap2ui5.github.io/docs/resources/api.html#view-display' }, + { area: 'docs', group: 'Cookbook', title: 'Popup', text: 'One popup and what closes it.', url: 'https://abap2ui5.github.io/docs/cookbook/popup.html' }, +]; + +test('a scope narrows the search to its area, and alone it lists the area', () => { + assert.deepEqual(parseScope('repo: popup'), { area: 'repos', query: 'popup' }); + assert.deepEqual(parseScope('z2ui5: x'), { area: null, query: 'z2ui5: x' }, 'a word that is no scope is a search'); + assert.deepEqual(search(ENTRIES, 'repo: popup').map((h) => h.entry.area), ['repos']); + assert.ok(search(ENTRIES, 'popup').some((h) => h.entry.area === 'docs'), 'unscoped, the page is there too'); + assert.deepEqual(search(ENTRIES, 'repos:').map((h) => h.entry.title), ['linter', 'popups']); + assert.deepEqual(grouped(search(ENTRIES, 'repo:')).map((g) => g.label), ['Repositories']); +}); + +test('a query that names something jumps to it', () => { + const at = (q) => jumps(ENTRIES, q).map((j) => (j.entry ? j.entry.url : j.url)); + assert.deepEqual(at('#1234'), ['https://github.com/abap2UI5/abap2UI5/issues/1234']); + assert.deepEqual(at('issue 42'), ['https://github.com/abap2UI5/abap2UI5/issues/42']); + assert.deepEqual(at('Z2UI5_CL_SMPC_APP_398'), ['https://abap2ui5.github.io/playground/samples/z2ui5_cl_smpc_app_398/']); + assert.deepEqual(at('view_display'), ['https://abap2ui5.github.io/docs/resources/api.html#view-display']); + assert.deepEqual(at('linter'), ['https://github.com/abap2UI5/linter']); + assert.deepEqual(at('abap2UI5-addons/popups'), ['https://github.com/abap2UI5-addons/popups']); + assert.deepEqual(at('layout editor'), ['https://abap2ui5.github.io/layout-editor/']); + assert.match(at('z2ui5_cl_nowhere')[0], /^https:\/\/github\.com\/search\?q=.*z2ui5_cl_nowhere&type=code$/, 'a class in no entry: the code search'); + assert.deepEqual(at('how do i close a popup'), [], 'a question is answered by the list'); + assert.deepEqual(at('#12a'), []); +}); + +test('the way out: the query handed to GitHub, for what no index carries', () => { + const out = elsewhere('repo: popup close'); + assert.deepEqual(out.map((o) => o.label), ['Code', 'Issues', 'Discussions']); + for (const o of out) { + assert.match(o.url, /^https:\/\/github\.com\/search\?q=/); + assert.match(decodeURIComponent(o.url), /org:abap2UI5 .*popup close|org:abap2UI5-addons\) popup close/, 'the query without its scope'); + } + assert.match(out[0].url, /&type=code$/); + assert.match(out[1].url, /&type=issues$/); + assert.match(out[2].url, /&type=discussions$/); + assert.deepEqual(elsewhere(' '), []); +}); + +test('the actions of a hit: what it carries, and a repository searched for the query', () => { + const carousel = ENTRIES.find((e) => e.title === 'Carousel'); + assert.deepEqual(actions(carousel, 'carousel').map((a) => a.label), ['Source']); + const linter = ENTRIES.find((e) => e.code === 'abap2ui5/linter'); + assert.deepEqual(actions(linter, 'rule').map((a) => a.label), ['README', 'Issues', 'Search code']); + assert.match(actions(linter, 'rule').at(-1).url, /repo%3Aabap2ui5%2Flinter%20rule&type=code$/); + assert.deepEqual(actions(linter, 'linter').map((a) => a.label), ['README', 'Issues'], 'its own name is no code search'); + assert.deepEqual(actions(undefined), []); +}); From 9b63c36e1a0ddf3477db076233e62ebae7891160 Mon Sep 17 00:00:00 2001 From: Claude Date: Sun, 1 Nov 2026 14:50:56 +0000 Subject: [PATCH 2/2] Drop the headings of a feed entry whose url has an anchor A box appends a heading's anchor to the url; on a url that already names a section that made #a#b. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_014YZFmFLTfBmZJXKPuRQWci --- scripts/lib/search-family.mjs | 5 ++++- test/search-family.test.mjs | 2 ++ 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/scripts/lib/search-family.mjs b/scripts/lib/search-family.mjs index 528bb907f..9bf087da3 100644 --- a/scripts/lib/search-family.mjs +++ b/scripts/lib/search-family.mjs @@ -96,7 +96,10 @@ export function siteEntries(feed, { area, label }) { for (const e of Array.isArray(feed?.entries) ? feed.entries : []) { if (typeof e?.title !== 'string' || !e.title.trim() || typeof e.url !== 'string' || !OWN.test(e.url)) continue; const entry = { area, group: label, title: e.title.trim(), text: typeof e.text === 'string' ? e.text : '', url: e.url }; - if (Array.isArray(e.headings)) { + /* a url with a fragment already names its place on the page: a heading's + * anchor appended to it - which is what a box does with a heading hit - + * would make it `#a#b` */ + if (Array.isArray(e.headings) && !e.url.includes('#')) { const hs = e.headings.filter((h) => Array.isArray(h) && typeof h[0] === 'string' && typeof h[1] === 'string'); if (hs.length) entry.headings = hs; } diff --git a/test/search-family.test.mjs b/test/search-family.test.mjs index 034c01c78..d224ab893 100644 --- a/test/search-family.test.mjs +++ b/test/search-family.test.mjs @@ -37,6 +37,8 @@ test('a feed of a sister site becomes entries of its area, and only what leads t assert.deepEqual(rows[0].headings, [['Why', 'why']], 'a malformed heading is dropped'); assert.deepEqual(rows[1].links, [{ label: 'Repository', url: 'https://github.com/abap2UI5-addons/popups' }], 'a link off the project is dropped'); assert.deepEqual(siteEntries(null, INSIGHTS), [], 'a feed that is not one is nothing'); + const [section] = siteEntries({ entries: [{ title: 'Cost', url: 'https://abap2ui5.github.io/why/#what-it-costs', headings: [['Licence', 'what-it-costs']] }] }, INSIGHTS); + assert.equal(section.headings, undefined, 'a url with an anchor carries no headings of its own'); }); test('a repository is offered with its README and issues, an archived one is not', () => {