diff --git a/.github/workflows/check.yml b/.github/workflows/check.yml index 76b163b1..6e2a9f8b 100644 --- a/.github/workflows/check.yml +++ b/.github/workflows/check.yml @@ -106,6 +106,10 @@ jobs: # alone. A page that does not build is a page nobody can read. - name: build the site if: ${{ !cancelled() }} + # the search index lists the organisations' repositories and the + # framework's classes from the GitHub API (scripts/lib/search-family.mjs) + env: + GITHUB_TOKEN: ${{ github.token }} run: npm run docs:build # every link out of this deployment and into a neighbouring one on the diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml index c18e46b2..183c7bb3 100644 --- a/.github/workflows/deploy.yml +++ b/.github/workflows/deploy.yml @@ -176,6 +176,10 @@ jobs: # THE SITE THAT IS PUBLISHED: VitePress's build, from this repository # alone. - name: Build the site + # the search index lists the organisations' repositories and the + # framework's classes from the GitHub API (scripts/lib/search-family.mjs) + env: + GITHUB_TOKEN: ${{ github.token }} run: npm run docs:build # The one gate that cannot run before the build, because what it judges diff --git a/AGENTS.md b/AGENTS.md index 84838fab..71f6a91d 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -21,7 +21,8 @@ person reads the page. Do not put "as an AI, …" prose back into `docs/`. | `scripts/generate-api-reference.mjs` | Generates the client API reference — the block in `docs/resources/api.md` and `docs/public/api/client-api.json` — from `z2ui5_if_client` on the framework branch this site tracks (`main`); `--check` is the freshness gate | | `scripts/lib/client-interface.mjs` | Where `z2ui5_if_client` is fetched from (the ref comes from `lib/release.mjs`, shared with `check-api-names.mjs`; `A2UI5_HOME` reads the same ref out of a local clone of the framework instead, with `git show`, never its working tree) and the full parser `generate-api-reference.mjs` renders from; pinned by `test/client-interface.test.mjs` | | `scripts/check-version.mjs` | The release number in the bar's version dropdown, the deprecations page and the changelog, against the newest release tag of the framework | -| `scripts/generate-search.mjs` | Builds `docs/public/search-index.json` — the pages of this site plus every entry in the three sample catalogues. This site's own bar searches with VitePress's local search; the index is published for the playground's search box, which fetches it from here. `scripts/lib/search-engine.js` is the matcher the playground carries a copy of, pinned with the index by `test/search.test.mjs`. Runs inside `docs:build`, so the deploy publishes it; `scripts/lib/search-index.mjs` is what goes in | +| `scripts/generate-search.mjs` | Builds `docs/public/search-index.json` — the pages of this site, every entry in the three sample catalogues, and the rest of the family (below). This site's own bar searches with VitePress's local search; the index is published for the search box on the home page (https://abap2ui5.github.io/, abap2UI5/abap2UI5.github.io) and the playground's, which fetch it from here. `scripts/lib/search-engine.js` is the matcher both carry a copy of, pinned with the index by `test/search.test.mjs` and `test/search-family.test.mjs`. Runs inside `docs:build`, so the deploy publishes it; `scripts/lib/search-index.mjs` is what goes in | +| `scripts/lib/search-family.mjs` | The rest of the family in that index: where to go (the tools, as data here), every repository of `abap2UI5` and `abap2UI5-addons` (GitHub API, `GITHUB_TOKEN` in CI), the essays, the page for decision makers and the addons (each from the `search.json` its site publishes - one shape, `{ site, url, entries: [{ title, text, headings, terms, url, links }] }`), the client API (from the committed `client-api.json`, each method linked to its line in `z2ui5_if_client`) and every class of the framework outside `src/99`. Every source may fail and costs its rows, never the build. Issues and discussions are not indexed: the box hands the query to GitHub's search for them (`elsewhere( )` in the matcher), so they are never a day old | | `scripts/lib/pages.mjs` | What a page of this site IS: the sidebar walk, its title, its opening sentences, its headings, its words. Shared by `generate-llms.mjs` and `generate-search.mjs` so a page added to the sidebar reaches both; `summarise( )` — the description of 150 pages and the note in `llms.txt`, which nobody writes and nobody proofreads — is pinned by `test/summarise.test.mjs` | | `docs/.vitepress/playground.mjs` | Decides which fenced ABAP example gets a **Run** button, and wraps the fence; `theme/playground.js` is the browser half | | `scripts/list-runnable.mjs` | The measurement's worklist: every fenced example that carries a Run button, out of the same `playground.mjs` that decides the button. `--json` adds each example's ABAP verbatim - what the button sends - so the measurement below can be driven rather than clicked | diff --git a/scripts/generate-search.mjs b/scripts/generate-search.mjs index 1ec0b51e..9427e774 100644 --- a/scripts/generate-search.mjs +++ b/scripts/generate-search.mjs @@ -1,6 +1,8 @@ #!/usr/bin/env node -// The search index the bar reads: every page of this site, every sample in the -// three catalogues, one JSON document at /docs/search-index.json. +// The search index the boxes of the family read: every page of this site, every +// sample in the three catalogues and the rest of the family (repositories, +// tools, the sister sites, the API - scripts/lib/search-family.mjs), one JSON +// document at /docs/search-index.json. // // Runs inside `docs:build`, before vitepress, so the file is in docs/public/ // when the build copies it - the same arrangement as generate-llms.mjs, and @@ -24,7 +26,8 @@ const OUT = path.join(DOCS, 'public', 'search-index.json'); const index = await buildIndex(ROOT, { log: (line) => console.log(line) }); const docs = index.entries.filter((e) => e.area === 'docs').length; -const samples = index.entries.length - docs; +const samples = index.entries.filter((e) => e.area === 'samples').length; +const rest = index.entries.length - docs - samples; /* The floor. An index with no page in it is a broken walk - the sidebar moved, * or config.mjs stopped exporting one - and an empty search box that reports @@ -40,4 +43,4 @@ fs.mkdirSync(path.dirname(OUT), { recursive: true }); fs.writeFileSync(OUT, JSON.stringify(index)); const kb = Math.round(fs.statSync(OUT).size / 1024); -console.log(`search-index.json: ${docs} page(s) + ${samples} sample(s) (${kb} kB)`); +console.log(`search-index.json: ${docs} page(s) + ${samples} sample(s) + ${rest} from the rest of the family (${kb} kB)`); diff --git a/scripts/lib/search-engine.js b/scripts/lib/search-engine.js index 9c102451..36770f87 100644 --- a/scripts/lib/search-engine.js +++ b/scripts/lib/search-engine.js @@ -1,5 +1,5 @@ /* - * The matcher behind the search box in the bar — framework-free on purpose. + * The matcher behind the search boxes of the family — framework-free on purpose. * * Plain JavaScript with no imports, and this site's own: the index it reads * is /docs/search-index.json, generated by scripts/generate-search.mjs from @@ -292,6 +292,21 @@ function nearest(entries, term) { * (above) and the list says so: `hits.relaxedTo` is the query that answered. */ export function search(entries, query, { limit = 30 } = {}) { + /* A scope - `repo: popup` - narrows the entries before anything is scored, + * and a scope alone - `repo:` - lists its area, which is how a reader asks + * "what repositories are there". */ + const scoped = parseScope(query); + if (scoped.area) { + entries = entries.filter((e) => e.area === scoped.area); + query = scoped.query; + if (!queryTerms(query).length) { + return entries + .slice() + .sort((a, b) => (a.title || '').localeCompare(b.title || '')) + .slice(0, limit) + .map((entry) => ({ entry, score: 0, heading: null, strong: true })); + } + } /* Eight words at most. Nobody narrows a list with a ninth, and a pasted * error message or a line of ABAP is thirty: every relaxing pass below * scored the whole index once per word, and the box froze for seconds on @@ -476,3 +491,113 @@ export function recallQuery() { const q = record.q.trim(); return q && q.length <= QUERY_MAX ? q : ''; } + +/* ── SCOPES, JUMPS AND THE WAY OUT ────────────────────────────────────────── + * + * The index carries more than the manual and the samples since the home page + * got its box: where to go (`tools`), the repositories (`repos`), the sister + * sites (`insights`, `why`, `addons`) and the code (`api`) - see + * scripts/lib/search-family.mjs. Three things go with that, all of them data + * about the QUERY rather than about the ranking, so a box that does not use + * them (the playground's, today) loses nothing: + * + * parseScope( ) `repo: popup`, `api: bind` - a word and a colon in front + * narrow the search to one area; + * jumps( ) the query that NAMES something - `#1234`, a class name, + * a repository, a tool - answered with that thing, before + * any list; + * elsewhere( ) the query handed to GitHub's own search, for what no index + * of ours carries: the code of every repository, the issues, + * the discussions. Links, not requests - the box fetches + * nothing from GitHub, and the answer is never a day old. + */ + +/** The words a scope can be written as, and the area each means. */ +export const SCOPES = { + doc: 'docs', docs: 'docs', + sample: 'samples', samples: 'samples', + repo: 'repos', repos: 'repos', + api: 'api', code: 'api', class: 'api', + tool: 'tools', tools: 'tools', go: 'tools', + insight: 'insights', insights: 'insights', essay: 'insights', + why: 'why', + addon: 'addons', addons: 'addons', +}; + +/** `{ area, query }`: the area a leading `word:` names, and the rest. A word + * that is no scope leaves the query as it is - `z2ui5:` is a search. */ +export function parseScope(query) { + const m = /^\s*([a-z]+):\s*(.*)$/is.exec(query || ''); + const area = m && SCOPES[m[1].toLowerCase()]; + return area ? { area, query: m[2] } : { area: null, query: query || '' }; +} + +const GITHUB = 'https://github.com'; +const FRAMEWORK = `${GITHUB}/abap2UI5/abap2UI5`; +const enc = encodeURIComponent; + +/** GitHub's code search over both organisations. */ +export const codeSearchUrl = (q) => `${GITHUB}/search?q=${enc(`(org:abap2UI5 OR org:abap2UI5-addons) ${q}`)}&type=code`; + +/** + * What the query NAMES, if it names something: at most three targets, each + * `{ entry }` (a hit of the index) or `{ title, text, url }` (a place + * outside it). Empty when the query is a question rather than a name - the + * list answers those. + * + * #1234, issue 1234, pr 1234 that issue or pull request of the framework + * a class, a method, a repo the entry whose `code` it is, whole + * a tool or a repository name the entry whose title it is, whole + * z2ui5_… in no entry the code search for it + */ +export function jumps(entries, query) { + const q = parseScope(query).query.trim(); + const out = []; + const m = /^(?:#|(?:issue|pr|pull)\s*#?)(\d{1,6})$/i.exec(q); + if (m) out.push({ title: `#${m[1]}`, text: 'Issue or pull request of abap2UI5/abap2UI5', url: `${FRAMEWORK}/issues/${m[1]}` }); + + const low = normalise(q).replace(/\s+/g, ' '); + if (low.length >= 3) { + for (const e of entries) { + const code = e.code || ''; + const named = code === low || code.split('/').pop() === low + || ((e.area === 'tools' || e.area === 'repos') && normalise(e.title) === low); + if (named) out.push({ entry: e }); + } + } + if (!out.length && /^z2ui5_\w{3,}$/i.test(q)) { + out.push({ title: q, text: 'Not in the index - every repository’s code, on GitHub', url: codeSearchUrl(q) }); + } + const seen = new Set(); + return out.filter((j) => { + const url = j.entry ? j.entry.url : j.url; + if (seen.has(url)) return false; + seen.add(url); + return true; + }).slice(0, 3); +} + +/** The query handed on to GitHub: `[{ label, url }]`, empty for an empty + * query. Issues and discussions are never indexed (a nightly copy of them + * is a day old exactly when it matters), so this is how they are searched. */ +export function elsewhere(query) { + const q = parseScope(query).query.trim(); + if (!q) return []; + return [ + { label: 'Code', url: codeSearchUrl(q) }, + { label: 'Issues', url: `${GITHUB}/search?q=${enc(`org:abap2UI5 org:abap2UI5-addons ${q}`)}&type=issues` }, + { label: 'Discussions', url: `${GITHUB}/search?q=${enc(`org:abap2UI5 ${q}`)}&type=discussions` }, + ]; +} + +/** The actions beside a hit: what its entry carries (`links` - the source of + * a sample, the README of a repository) and, for a repository, its code + * searched for what was typed. */ +export function actions(entry, query = '') { + const out = Array.isArray(entry?.links) ? entry.links.filter((l) => l && l.label && l.url) : []; + const q = parseScope(query).query.trim(); + if (entry?.area === 'repos' && entry.code && q && normalise(q) !== entry.code.split('/').pop()) { + out.push({ label: 'Search code', url: `${GITHUB}/search?q=${enc(`repo:${entry.code} ${q}`)}&type=code` }); + } + return out; +} diff --git a/scripts/lib/search-family.mjs b/scripts/lib/search-family.mjs new file mode 100644 index 00000000..9bf087da --- /dev/null +++ b/scripts/lib/search-family.mjs @@ -0,0 +1,299 @@ +/* + * search-family — the rest of the project in the one search index. + * + * search-index.mjs builds the two areas this repository owns or has always + * carried: the pages of the manual and the three sample catalogues. That + * answered "where is it explained" and "where is an example of it". It did + * not answer the other questions a reader brings to a box on the home page: + * WHERE do I go (a repository, a tool, a site of the family), WHAT does the + * rest of the family say about it (the essays, the page for decision makers, + * the addons), and WHERE in the code is it (a client method, a class of the + * framework, down to the file and the line). + * + * So, beside the two: + * + * tools the places of the family a reader goes to rather than reads - + * the playground, the layout editor, the AI builder - with the + * words people use for them. Data, here, below. + * repos every repository of github.com/abap2UI5 and abap2UI5-addons, + * from the GitHub API: name, description, topics. A hit opens the + * repository; its README and issues are actions beside it. + * insights the essays, why the page for decision makers, addons the + * why addons and the developer tools - each from the search.json its + * addons site publishes (one shape on every site, see SITES below). + * api the client API (docs/public/api/client-api.json, committed and + * held fresh by check:api-reference), each method leading to its + * section of resources/api.md and to its line in + * z2ui5_if_client; and every class and interface of the + * framework outside the frozen package, leading to its file. + * + * EVERY SOURCE MAY FAIL. The GitHub API, a sister site, raw.githubusercontent: + * an unreachable one costs its rows and is reported, never the build - the + * rule the sample catalogues already follow. An index with fewer areas is a + * worse search; a failed deploy of the manual over a sister site's outage is + * a worse documentation. + * + * Issues and discussions are deliberately NOT indexed: a nightly copy of them + * would be a day old when it mattered most. The box hands the query to + * GitHub's own search for those instead (`elsewhere( )` in search-engine.js). + */ +import fs from 'node:fs'; +import path from 'node:path'; +import { DOCS, headings } from './pages.mjs'; +import { fetchInterface } from './client-interface.mjs'; + +const GITHUB = 'https://github.com'; +const API = 'https://api.github.com'; +export const ORGS = ['abap2UI5', 'abap2UI5-addons']; + +/** The places a reader GOES to. `terms` are the words people use for them, + * which is the whole reason these are entries and not tiles on a page. */ +export const DESTINATIONS = [ + { title: 'Playground', text: 'Write ABAP in the browser and run the app beside it - no SAP system needed.', url: 'https://abap2ui5.github.io/playground/', terms: 'try online browser editor run abap without system sandbox' }, + { title: 'Layout Editor', text: 'Design a UI5 view by drag and drop, then export it as an ABAP class.', url: 'https://abap2ui5.github.io/layout-editor/', terms: 'drag drop designer wysiwyg view editor layout builder' }, + { title: 'AI App Builder', text: 'Describe the app you need; the AI writes it and you download it for abapGit.', url: 'https://abap2ui5.github.io/playground/ai/', terms: 'ai llm generate app chatgpt claude prompt builder' }, + { title: 'AI Agent', text: 'Let the AI operate a running app: describe the task in the chat and watch it work.', url: 'https://abap2ui5.github.io/playground/pilot/', terms: 'ai agent pilot automate operate chat' }, + { title: 'Sample catalogue', text: 'Every sample app, by control and UI5 release, each runnable in the playground.', url: 'https://abap2ui5.github.io/samples-site/', terms: 'samples examples demos catalogue catalog controls' }, + { title: 'Documentation', text: 'The manual: how it works, how to install it, how to build apps.', url: 'https://abap2ui5.github.io/docs/', terms: 'docs manual guide handbook' }, + { title: 'Insights', text: 'Short essays on the idea: how it works, where it belongs, running it.', url: 'https://abap2ui5.github.io/insights/', terms: 'essays articles blog background architecture' }, + { title: 'Why abap2UI5', text: 'For decision makers: what it builds, who runs it, what it costs.', url: 'https://abap2ui5.github.io/why/', terms: 'managers business case cost decision evaluate' }, + { title: 'Addons & Tools', text: 'Addons for your system, tools for your editor and pipeline, each with a page.', url: 'https://abap2ui5-addons.github.io/', terms: 'addons extensions plugins tools' }, + { title: 'Home', text: 'The entry page of abap2UI5.', url: 'https://abap2ui5.github.io/', terms: 'start home entry' }, + { title: 'Slack', text: 'The community channel #abap2ui5 on the abapGit Slack.', url: 'https://abapgit.slack.com/join/shared_invite/zt-46tqufaht-QlrxTzlDqlx85CWbeUnOqg#/shared-invite/email', terms: 'chat community help ask question' }, + { title: 'Releases', text: 'The releases of the framework, with their notes.', url: `${GITHUB}/abap2UI5/abap2UI5/releases`, terms: 'release changelog version download latest' }, + { title: 'Report an issue', text: 'Open an issue for the framework on GitHub.', url: `${GITHUB}/abap2UI5/abap2UI5/issues/new/choose`, terms: 'bug report issue problem error new' }, + { title: 'Discussions', text: 'Questions, ideas and show-and-tell, on GitHub.', url: `${GITHUB}/abap2UI5/abap2UI5/discussions`, terms: 'question idea forum ask discuss' }, +]; + +export function destinationEntries() { + return DESTINATIONS.map((d) => ({ area: 'tools', group: 'Go to', ...d })); +} + +/** The sites that publish a search.json, and the area each becomes. */ +export const SITES = [ + { area: 'insights', label: 'Insights', url: 'https://abap2ui5.github.io/insights/search.json' }, + { area: 'why', label: 'Why abap2UI5', url: 'https://abap2ui5.github.io/why/search.json' }, + { area: 'addons', label: 'Addons & Tools', url: 'https://abap2ui5-addons.github.io/search.json' }, +]; + +const timeout = () => AbortSignal.timeout(20000); +const ghHeaders = (token = process.env.GITHUB_TOKEN) => ({ + Accept: 'application/vnd.github+json', + 'User-Agent': 'abap2UI5-docs-search-index', + ...(token ? { Authorization: `Bearer ${token}` } : {}), +}); + +/** Is this an http(s) URL of the family or GitHub - the only places a feed + * may send a reader. A feed is another repository's file; a hit that leads + * off the project is not something it gets to publish into this box. */ +const OWN = /^https:\/\/(abap2ui5\.github\.io|abap2ui5-addons\.github\.io|github\.com\/abap2UI5(-addons)?)\//i; + +/** One site's search.json as entries of `area`. Only the fields the matcher + * reads, each checked for its type: a feed that changed shape costs its + * odd entries, not the index. */ +export function siteEntries(feed, { area, label }) { + const out = []; + for (const e of Array.isArray(feed?.entries) ? feed.entries : []) { + if (typeof e?.title !== 'string' || !e.title.trim() || typeof e.url !== 'string' || !OWN.test(e.url)) continue; + const entry = { area, group: label, title: e.title.trim(), text: typeof e.text === 'string' ? e.text : '', url: e.url }; + /* a url with a fragment already names its place on the page: a heading's + * anchor appended to it - which is what a box does with a heading hit - + * would make it `#a#b` */ + if (Array.isArray(e.headings) && !e.url.includes('#')) { + const hs = e.headings.filter((h) => Array.isArray(h) && typeof h[0] === 'string' && typeof h[1] === 'string'); + if (hs.length) entry.headings = hs; + } + if (typeof e.terms === 'string' && e.terms) entry.terms = e.terms; + if (typeof e.code === 'string' && e.code) entry.code = e.code; + const links = linksOf(e.links); + if (links.length) entry.links = links; + out.push(entry); + } + return out; +} + +function linksOf(links) { + return (Array.isArray(links) ? links : []) + .filter((l) => typeof l?.label === 'string' && typeof l.url === 'string' && OWN.test(l.url)) + .map((l) => ({ label: l.label, url: l.url })); +} + +/** Every repository of the organisations, as entries. Archived ones are left + * out: a search that leads to a repository nobody maintains is a wrong + * answer that looks right. Forks of the organisation's own are kept - they + * are how an addon carries a library it adapted. */ +export function repoEntries(repos) { + return repos + .filter((r) => r && !r.archived && !r.private && typeof r.full_name === 'string') + .map((r) => { + const [org, name] = r.full_name.split('/'); + return { + area: 'repos', + group: 'Repositories', + title: name, + text: r.description || '', + /* the full name is what somebody pastes - abap2UI5-addons/popups */ + code: r.full_name.toLowerCase(), + terms: [org, ...(r.topics || []), r.language || '', r.homepage ? 'site' : ''].join(' ').toLowerCase().trim(), + url: r.html_url || `${GITHUB}/${r.full_name}`, + links: [ + { label: 'README', url: `${GITHUB}/${r.full_name}#readme` }, + { label: 'Issues', url: `${GITHUB}/${r.full_name}/issues` }, + ...(r.homepage && OWN.test(r.homepage) ? [{ label: 'Site', url: r.homepage }] : []), + ], + }; + }); +} + +async function fetchRepos(org, fetchFn) { + const all = []; + for (let page = 1; page <= 5; page++) { + const res = await fetchFn(`${API}/orgs/${org}/repos?type=public&per_page=100&page=${page}`, { headers: ghHeaders(), signal: timeout() }); + if (!res.ok) throw new Error(`HTTP ${res.status}`); + const rows = await res.json(); + all.push(...rows); + if (rows.length < 100) break; + } + return all; +} + +/* ── THE CODE ─────────────────────────────────────────────────────────────── */ + +const FRAMEWORK = 'abap2UI5/abap2UI5'; +const CLIENT_API = path.join(DOCS, 'public', 'api', 'client-api.json'); +const API_PAGE = 'https://abap2ui5.github.io/docs/resources/api.html'; + +/** The first sentence of a doc block - what a result row has room for. */ +const first = (lines) => { + const s = (Array.isArray(lines) ? lines.join(' ') : String(lines || '')).replace(/\s+/g, ' ').trim(); + const m = s.match(/^.{20,}?[.!?](\s|$)/); + return (m ? m[0] : s).trim(); +}; + +/** The line a member is declared on in the interface source, 1-based, or 0. */ +export function lineOf(source, name, kind = 'METHODS') { + if (!source) return 0; + const re = new RegExp(`^\\s*(${kind}|CLASS-${kind})\\s+${name.replace(/[.*+?^${}()|[\]\\]/g, '\\$&')}\\b`, 'im'); + const lines = source.split('\n'); + const i = lines.findIndex((l) => re.test(l)); + return i < 0 ? 0 : i + 1; +} + +/** The client API: one entry per method and per constant group, from the + * committed client-api.json, with the anchors resources/api.md really has. */ +export function clientApiEntries(api, apiPageBody, source) { + const anchors = new Map(headings(apiPageBody).map((h) => [h.text, h.anchor])); + const file = api.source || `${GITHUB}/${FRAMEWORK}/blob/main/src/02/z2ui5_if_client.intf.abap`; + const at = (name, kind) => { + const n = lineOf(source, name, kind); + return n ? `${file}#L${n}` : file; + }; + const out = []; + for (const m of api.methods || []) { + const anchor = anchors.get(m.name); + out.push({ + area: 'api', + group: 'API', + title: `client->${m.name}( )`, + text: first(m.doc), + code: m.name.toLowerCase(), + terms: [m.group, ...(m.parameters || []).map((p) => p.name)].join(' ').toLowerCase(), + url: anchor ? `${API_PAGE}#${anchor}` : API_PAGE, + links: [{ label: 'Source', url: at(m.name, 'METHODS') }], + }); + } + for (const c of api.constants || []) { + const anchor = anchors.get(c.name); + out.push({ + area: 'api', + group: 'API', + title: `client->${c.name}`, + text: first(c.doc) || `The ${c.name} constants of z2ui5_if_client.`, + code: c.name.toLowerCase(), + terms: (c.members || []).map((x) => x.name).join(' ').toLowerCase(), + url: anchor ? `${API_PAGE}#${anchor}` : API_PAGE, + links: [{ label: 'Source', url: at(c.name, 'CONSTANTS') }], + }); + } + return out; +} + +/** Every class and interface of the framework outside src/99 - the frozen + * package, whose names this site does not teach (check:api-names) and a + * search box should not offer. From the tree of main: one request. */ +export function sourceEntries(tree) { + const out = []; + for (const t of tree || []) { + const m = /^src\/(?!99\/)(?:.*\/)?(z2ui5_\w+)\.(clas|intf)\.abap$/i.exec(t?.path || ''); + if (!m || t.type !== 'blob') continue; + const [, name, kind] = m; + out.push({ + area: 'api', + group: 'Framework source', + title: name.toLowerCase(), + text: `${kind === 'intf' ? 'Interface' : 'Class'} of the framework - ${t.path.split('/').slice(1, -1).join('/') || 'src'}`, + code: name.toLowerCase(), + url: `${GITHUB}/${FRAMEWORK}/blob/main/${t.path}`, + }); + } + return out; +} + +/** + * The family's areas, each fetched on its own, each allowed to fail. + * `{ entries, areas }` - `areas` in the same shape search-index.mjs reports. + */ +export async function familyEntries({ fetchFn = globalThis.fetch, log = () => {}, readFile = fs.readFileSync, interfaceFn = fetchInterface } = {}) { + const entries = []; + const areas = []; + const add = (area, label, rows, from) => { + entries.push(...rows); + areas.push({ area, label, count: rows.length }); + log(` ${label}: ${rows.length} entr${rows.length === 1 ? 'y' : 'ies'} (${from})`); + }; + const tryArea = async (label, fn) => { + try { + await fn(); + } catch (err) { + log(` ${label}: not reachable (${err.message}) — not in this index`); + } + }; + + add('tools', 'Go to', destinationEntries(), 'scripts/lib/search-family.mjs'); + + await tryArea('Repositories', async () => { + const repos = []; + for (const org of ORGS) repos.push(...await fetchRepos(org, fetchFn)); + add('repos', 'Repositories', repoEntries(repos), `api.github.com, ${ORGS.join(' + ')}`); + }); + + for (const site of SITES) { + await tryArea(site.label, async () => { + const res = await fetchFn(site.url, { signal: timeout() }); + if (!res.ok) throw new Error(`HTTP ${res.status}`); + const rows = siteEntries(await res.json(), site); + if (!rows.length) throw new Error('no entry in the feed'); + add(site.area, site.label, rows, site.url); + }); + } + + await tryArea('API', async () => { + const api = JSON.parse(readFile(CLIENT_API, 'utf8')); + const page = readFile(path.join(DOCS, 'resources', 'api.md'), 'utf8'); + /* the line numbers are a nicety: without the source the links go to the + * file, which is still the right file */ + let source = ''; + try { source = await interfaceFn(api.ref || 'main'); } catch (err) { log(` API: interface source not reachable (${err.message}) — links without a line`); } + add('api', 'API', clientApiEntries(api, page, source), 'docs/public/api/client-api.json'); + }); + + await tryArea('Framework source', async () => { + const res = await fetchFn(`${API}/repos/${FRAMEWORK}/git/trees/main?recursive=1`, { headers: ghHeaders(), signal: timeout() }); + if (!res.ok) throw new Error(`HTTP ${res.status}`); + const rows = sourceEntries((await res.json()).tree); + if (!rows.length) throw new Error('no class in the tree'); + add('api', 'Framework source', rows, `api.github.com, ${FRAMEWORK}@main`); + }); + + return { entries, areas }; +} diff --git a/scripts/lib/search-index.mjs b/scripts/lib/search-index.mjs index 4dd1aa0b..a865bb53 100644 --- a/scripts/lib/search-index.mjs +++ b/scripts/lib/search-index.mjs @@ -17,10 +17,12 @@ * summary and keywords the sample repositories maintain for it, and * a link to that sample's own page in the catalogue * - * The playground is not an area. It has no content to find: its URL carries + * The playground is not a corpus. It has no content to find: its URL carries * the code in the editor, and a result that opened an empty editor would - * answer no question anybody typed. It is a destination in the bar, not in the - * index — the same reasoning that keeps it out of the position memory. + * answer no question anybody typed. It is a DESTINATION - one row of the + * `tools` area, with the layout editor and the AI builder, in + * search-family.mjs, which also brings in the repositories, the sister sites + * and the API. * * WHERE IT IS PUBLISHED, AND WHY THAT IS ENOUGH. `/docs/search-index.json`, on * the origin all four documents share, fetched by whichever of them the reader @@ -34,6 +36,7 @@ import fs from 'node:fs'; import { entriesOf, loadCatalogue } from './catalogue.mjs'; import { sidebarPages, fileOf, describe, title, headings, terms, SITE } from './pages.mjs'; +import { familyEntries } from './search-family.mjs'; /** The three sample repositories, and where a reader is sent for a hit. */ export const CORPORA = [ @@ -75,7 +78,7 @@ export function docEntries(pages = sidebarPages(), read = (link) => fs.readFileS * three catalogues could be reached. A repository that could not is left out * and reported - it costs its rows, never the build, which is the rule every * other reader of these catalogues here already follows. */ -export function sampleEntries(catalogue, label) { +export function sampleEntries(catalogue, label, repo) { return entriesOf(catalogue).map((e) => { const cls = String(e.class); /* The three repositories describe an entry with slightly different fields @@ -102,6 +105,9 @@ export function sampleEntries(catalogue, label) { .toLowerCase().split(/[^\p{L}\p{N}._]+/u).filter(Boolean), )].join(' '), url: samplePage(cls), + /* the class itself, where its repository keeps it - the action beside + * the hit, for the reader who wants the code rather than the app */ + ...(repo && (e.file || e.path) ? { links: [{ label: 'Source', url: `https://github.com/abap2UI5/${repo}/blob/main/${e.file || e.path}` }] } : {}), }; }); } @@ -198,11 +204,17 @@ export async function buildIndex(root, { fetchFn = globalThis.fetch, log = () => log(` ${repo}: no catalogue reachable — its samples are not in this index`); continue; } - const rows = sampleEntries(found.catalogue, label); + const rows = sampleEntries(found.catalogue, label, repo); entries.push(...rows); areas.push({ area: 'samples', repo, label, count: rows.length }); log(` ${repo}: ${rows.length} sample(s) (${found.source})`); } + /* the rest of the family: where to go, the repositories, the sister + * sites, the API and the framework's source - search-family.mjs */ + const family = await familyEntries({ fetchFn, log }); + entries.push(...family.entries); + areas.push(...family.areas); + return { built: new Date().toISOString(), areas, entries: enrich(entries) }; } diff --git a/test/search-family.test.mjs b/test/search-family.test.mjs new file mode 100644 index 00000000..d224ab89 --- /dev/null +++ b/test/search-family.test.mjs @@ -0,0 +1,150 @@ +/* + * The rest of the family in the search index (scripts/lib/search-family.mjs) + * and the three things the matcher answers about a query beside its ranking + * (parseScope, jumps, elsewhere and actions in scripts/lib/search-engine.js). + * + * The sources are other repositories and the GitHub API, so what is pinned + * here is what this side does with what they send: a feed that leads off the + * project is not published, an archived repository is not offered, the frozen + * package is not taught, and a source that is down costs its rows and never + * the build. + * + * npm test + */ +import { test } from 'node:test'; +import assert from 'node:assert/strict'; +import { + siteEntries, repoEntries, sourceEntries, clientApiEntries, lineOf, destinationEntries, familyEntries, SITES, +} from '../scripts/lib/search-family.mjs'; +import { search, grouped, jumps, elsewhere, actions, parseScope } from '../scripts/lib/search-engine.js'; + +const INSIGHTS = SITES.find((s) => s.area === 'insights'); + +test('a feed of a sister site becomes entries of its area, and only what leads to the project', () => { + const rows = siteEntries({ + site: 'Insights', + entries: [ + { title: '#21 The Lock Is Gone by the Next Click', text: 'ENQUEUE inside one roundtrip.', headings: [['Why', 'why'], ['bad']], terms: 'enqueue lock', url: 'https://abap2ui5.github.io/insights/21-the-lock/' }, + { title: 'Elsewhere', text: 'a page of somebody else', url: 'https://example.com/' }, + { title: '', url: 'https://abap2ui5.github.io/insights/x/' }, + { title: 'No url' }, + { title: 'Addon', url: 'https://abap2ui5-addons.github.io/popups/', links: [{ label: 'Repository', url: 'https://github.com/abap2UI5-addons/popups' }, { label: 'Off', url: 'https://example.com/' }] }, + ], + }, INSIGHTS); + assert.deepEqual(rows.map((r) => r.title), ['#21 The Lock Is Gone by the Next Click', 'Addon']); + assert.equal(rows[0].area, 'insights'); + assert.equal(rows[0].group, 'Insights'); + assert.deepEqual(rows[0].headings, [['Why', 'why']], 'a malformed heading is dropped'); + assert.deepEqual(rows[1].links, [{ label: 'Repository', url: 'https://github.com/abap2UI5-addons/popups' }], 'a link off the project is dropped'); + assert.deepEqual(siteEntries(null, INSIGHTS), [], 'a feed that is not one is nothing'); + const [section] = siteEntries({ entries: [{ title: 'Cost', url: 'https://abap2ui5.github.io/why/#what-it-costs', headings: [['Licence', 'what-it-costs']] }] }, INSIGHTS); + assert.equal(section.headings, undefined, 'a url with an anchor carries no headings of its own'); +}); + +test('a repository is offered with its README and issues, an archived one is not', () => { + const rows = repoEntries([ + { full_name: 'abap2UI5/linter', description: 'A static code analysis tool', topics: ['abap'], html_url: 'https://github.com/abap2UI5/linter', homepage: 'https://abap2ui5.github.io/linter/' }, + { full_name: 'abap2UI5/old', archived: true }, + { full_name: 'abap2UI5-addons/popups', description: 'Value-Help, Search-Help', homepage: 'https://example.com/' }, + ]); + assert.deepEqual(rows.map((r) => r.code), ['abap2ui5/linter', 'abap2ui5-addons/popups']); + assert.deepEqual(rows[0].links.map((l) => l.label), ['README', 'Issues', 'Site']); + assert.deepEqual(rows[1].links.map((l) => l.label), ['README', 'Issues'], 'a homepage off the project is no action'); + assert.equal(rows[1].url, 'https://github.com/abap2UI5-addons/popups'); +}); + +test('the framework source leaves out the frozen package', () => { + const rows = sourceEntries([ + { path: 'src/02/z2ui5_if_client.intf.abap', type: 'blob' }, + { path: 'src/01/04/z2ui5_cl_ui5_view_builder.clas.abap', type: 'blob' }, + { path: 'src/01/04/z2ui5_cl_ui5_view_builder.clas.xml', type: 'blob' }, + { path: 'src/99/z2ui5_cl_xml_view.clas.abap', type: 'blob' }, + { path: 'src/01', type: 'tree' }, + ]); + assert.deepEqual(rows.map((r) => r.code), ['z2ui5_if_client', 'z2ui5_cl_ui5_view_builder']); + assert.equal(rows[1].url, 'https://github.com/abap2UI5/abap2UI5/blob/main/src/01/04/z2ui5_cl_ui5_view_builder.clas.abap'); +}); + +test('a client method leads to its section and to its line', () => { + const api = { + source: 'https://github.com/abap2UI5/abap2UI5/blob/main/src/02/z2ui5_if_client.intf.abap', + methods: [{ name: 'view_display', group: 'Main view', doc: ['Display the MAIN view. A new main view is a new screen.'], parameters: [{ name: 'val' }] }], + constants: [{ name: 'cs_event', doc: [], members: [{ name: 'popup_close' }] }], + }; + const page = '# Client API\n\n## Main view\n\n### `view_display`\n\ntext\n\n## Constants\n\n### `cs_event`\n'; + const source = 'INTERFACE z2ui5_if_client.\n CONSTANTS:\n BEGIN OF cs_event,\n METHODS view_display\n IMPORTING val TYPE clike.\n'; + const [method, constant] = clientApiEntries(api, page, source); + assert.equal(method.title, 'client->view_display( )'); + assert.equal(method.text, 'Display the MAIN view.'); + assert.equal(method.url, 'https://abap2ui5.github.io/docs/resources/api.html#view-display'); + assert.equal(method.links[0].url, `${api.source}#L4`); + assert.equal(constant.code, 'cs_event'); + assert.equal(lineOf(source, 'nothing_like_it'), 0); + assert.equal(clientApiEntries(api, page, '')[0].links[0].url, api.source, 'no source, no line - still the file'); +}); + +test('a source that is down costs its rows, never the build', async () => { + const lines = []; + const down = async () => ({ ok: false, status: 503, json: async () => ({}) }); + const { entries, areas } = await familyEntries({ fetchFn: down, interfaceFn: async () => { throw new Error('offline'); }, log: (l) => lines.push(l) }); + assert.ok(entries.some((e) => e.area === 'tools'), 'where to go needs no network'); + assert.ok(entries.some((e) => e.area === 'api' && e.group === 'API'), 'the client API is committed here'); + assert.ok(!areas.some((a) => a.area === 'repos'), 'no repositories without the API'); + assert.ok(lines.some((l) => /Repositories: not reachable/.test(l)), 'and the log says so'); +}); + +/* ── the matcher ─────────────────────────────────────────────────────────── */ + +const ENTRIES = [ + ...destinationEntries(), + ...repoEntries([{ full_name: 'abap2UI5/linter', description: 'A static code analysis tool', topics: [] }, { full_name: 'abap2UI5-addons/popups', description: 'Value-Help, Search-Help, Transport', topics: [] }]), + { area: 'samples', group: 'Controls', title: 'Carousel', text: 'A carousel.', code: 'z2ui5_cl_smpc_app_398', url: 'https://abap2ui5.github.io/playground/samples/z2ui5_cl_smpc_app_398/', links: [{ label: 'Source', url: 'https://github.com/abap2UI5/samples-controls/blob/main/src/x.clas.abap' }] }, + { area: 'api', group: 'API', title: 'client->view_display( )', text: 'Display the MAIN view.', code: 'view_display', url: 'https://abap2ui5.github.io/docs/resources/api.html#view-display' }, + { area: 'docs', group: 'Cookbook', title: 'Popup', text: 'One popup and what closes it.', url: 'https://abap2ui5.github.io/docs/cookbook/popup.html' }, +]; + +test('a scope narrows the search to its area, and alone it lists the area', () => { + assert.deepEqual(parseScope('repo: popup'), { area: 'repos', query: 'popup' }); + assert.deepEqual(parseScope('z2ui5: x'), { area: null, query: 'z2ui5: x' }, 'a word that is no scope is a search'); + assert.deepEqual(search(ENTRIES, 'repo: popup').map((h) => h.entry.area), ['repos']); + assert.ok(search(ENTRIES, 'popup').some((h) => h.entry.area === 'docs'), 'unscoped, the page is there too'); + assert.deepEqual(search(ENTRIES, 'repos:').map((h) => h.entry.title), ['linter', 'popups']); + assert.deepEqual(grouped(search(ENTRIES, 'repo:')).map((g) => g.label), ['Repositories']); +}); + +test('a query that names something jumps to it', () => { + const at = (q) => jumps(ENTRIES, q).map((j) => (j.entry ? j.entry.url : j.url)); + assert.deepEqual(at('#1234'), ['https://github.com/abap2UI5/abap2UI5/issues/1234']); + assert.deepEqual(at('issue 42'), ['https://github.com/abap2UI5/abap2UI5/issues/42']); + assert.deepEqual(at('Z2UI5_CL_SMPC_APP_398'), ['https://abap2ui5.github.io/playground/samples/z2ui5_cl_smpc_app_398/']); + assert.deepEqual(at('view_display'), ['https://abap2ui5.github.io/docs/resources/api.html#view-display']); + assert.deepEqual(at('linter'), ['https://github.com/abap2UI5/linter']); + assert.deepEqual(at('abap2UI5-addons/popups'), ['https://github.com/abap2UI5-addons/popups']); + assert.deepEqual(at('layout editor'), ['https://abap2ui5.github.io/layout-editor/']); + assert.match(at('z2ui5_cl_nowhere')[0], /^https:\/\/github\.com\/search\?q=.*z2ui5_cl_nowhere&type=code$/, 'a class in no entry: the code search'); + assert.deepEqual(at('how do i close a popup'), [], 'a question is answered by the list'); + assert.deepEqual(at('#12a'), []); +}); + +test('the way out: the query handed to GitHub, for what no index carries', () => { + const out = elsewhere('repo: popup close'); + assert.deepEqual(out.map((o) => o.label), ['Code', 'Issues', 'Discussions']); + for (const o of out) { + assert.match(o.url, /^https:\/\/github\.com\/search\?q=/); + assert.match(decodeURIComponent(o.url), /org:abap2UI5 .*popup close|org:abap2UI5-addons\) popup close/, 'the query without its scope'); + } + assert.match(out[0].url, /&type=code$/); + assert.match(out[1].url, /&type=issues$/); + assert.match(out[2].url, /&type=discussions$/); + assert.deepEqual(elsewhere(' '), []); +}); + +test('the actions of a hit: what it carries, and a repository searched for the query', () => { + const carousel = ENTRIES.find((e) => e.title === 'Carousel'); + assert.deepEqual(actions(carousel, 'carousel').map((a) => a.label), ['Source']); + const linter = ENTRIES.find((e) => e.code === 'abap2ui5/linter'); + assert.deepEqual(actions(linter, 'rule').map((a) => a.label), ['README', 'Issues', 'Search code']); + assert.match(actions(linter, 'rule').at(-1).url, /repo%3Aabap2ui5%2Flinter%20rule&type=code$/); + assert.deepEqual(actions(linter, 'linter').map((a) => a.label), ['README', 'Issues'], 'its own name is no code search'); + assert.deepEqual(actions(undefined), []); +});