feat: FROM-aware autocompletion — alias/scope resolution + debounced column loading (#84)

CHANGELOG.md

@@ -9,6 +9,23 @@ auto-generated per-PR notes; this file is the curated, human-readable history.
 
 ## [Unreleased]
 
+### Added
+- **Schema-aware, FROM-driven autocompletion** (#84) — column completion now
+  fires *while you type*, driven by the statement's `FROM`/`JOIN` clause, so you
+  no longer have to expand a table in the sidebar first. A new pure module
+  `src/core/from-scope.js` resolves the caret's statement into its base tables
+  (`{db, table, alias}[]`, reusing the SQL tokenizer so strings/comments/`;`
+  never fool it), and completion uses it three ways: **aliases resolve**
+  (`e.` after `FROM events e` offers `events`' columns), **unqualified columns
+  are scoped** to the statement's tables (an unrelated loaded table's columns
+  are no longer suggested), and **columns load lazily** on a **debounced idle
+  tick** (300 ms, never on the keystroke path) — deduped via the existing
+  `'loading'` sentinel, cached per connection, and the open dropdown refreshes
+  when they arrive. `db.table.`/`table.` qualification still works; with no
+  FROM in view completion degrades gracefully to the global pool. Non-goals
+  (v1): CTE/subquery-derived scopes, `USING`, `SELECT *` expansion, table
+  functions. Builds directly on the CM6 editor (#21).
+
 ### Fixed
 - **Editor scrollbars are back, and the whole UI's scrollbars behave
   consistently again.** The console no longer renders at 1.2× via `html{zoom}`

src/core/completions.js

@@ -113,16 +113,18 @@ export function buildCompletions(ref, schema) {
  * The word being typed at the caret, whether it is qualified (after a dot —
  * `table.` → that table's columns), and whether it sits inside a FORMAT clause
  * (`afterFormat` — the preceding token is FORMAT → complete output-format names).
- * Returns {word, from, to, qualified, parent, afterFormat}.
+ * Returns {word, from, to, qualified, parent, afterFormat}. `toks` optionally
+ * supplies a pre-computed `tokenize(value)` so the completion path (which also
+ * runs from-scope) lexes the caret prefix once, not twice.
  */
-export function completionContext(value, pos) {
+export function completionContext(value, pos, toks) {
   // The word being typed is either inside an OPEN backtick-quoted identifier (a
   // `non-bare-name… still being typed) or a bare [A-Za-z0-9_] run. We use the SQL
   // tokenizer to find an open backtick so a backtick inside a string/comment (or
   // an escaped `\`` in a closed name) can't fool us. `from` is where an accepted
   // candidate replaces to — the opening backtick in the quoted case, so accepting
   // never doubles the backtick.
-  const open = openBacktickStart(value, pos);
+  const open = openBacktickStart(value, pos, toks);
   let from;
   let word;
   if (open >= 0) {
@@ -157,9 +159,9 @@ export function completionContext(value, pos) {
 // contains the caret, or -1. Uses the tokenizer so a backtick inside a string or
 // comment isn't mistaken for an identifier quote, and an escaped `\`` inside a
 // closed name doesn't desync a naive parity count.
-function openBacktickStart(value, pos) {
+function openBacktickStart(value, pos, toks) {
   let off = 0;
-  for (const [type, text] of tokenize(value)) {
+  for (const [type, text] of toks || tokenize(value)) {
     const end = off + text.length;
     if (off < pos && pos <= end && type === 'ident' && text[0] === '`') {
       // open = the caret is inside the run before any closing backtick (an
@@ -185,15 +187,41 @@ function identBefore(value, end) {
 }
 
 /**
- * Rank candidates for `ctx`. Qualified → only that table's columns. Otherwise
- * prefix matches before substring; columns/tables boosted over keywords once
- * ≥1 char is typed. Empty word (and not qualified) → keywords + tables only.
- * Capped for a tight dropdown.
+ * Resolve a qualifier (`ctx.parent`, the identifier before the dot) through the
+ * statement's FROM scope: an alias (`e` from `FROM events e`) resolves to its
+ * table name; a bare table name (or anything not in scope) is returned as-is so
+ * `events.` keeps working with no scope. Pure — `scope` is the
+ * `{db, table, alias}[]` from core/from-scope.js (#84), or absent.
+ */
+export function resolveScopeAlias(parent, scope) {
+  if (!scope || parent == null) return parent;
+  for (const r of scope) if (r.alias === parent) return r.table;
+  return parent;
+}
+
+// The set of in-scope table names (for unqualified column scoping), or null when
+// there is no FROM scope — null means "keep today's global behavior".
+function scopeTableSet(scope) {
+  if (!scope || !scope.length) return null;
+  const s = new Set();
+  for (const r of scope) if (r.table) s.add(r.table);
+  return s.size ? s : null;
+}
+
+/**
+ * Rank candidates for `ctx`. Qualified → only that table's columns (resolving an
+ * alias through `ctx.scope` first). Otherwise prefix matches before substring;
+ * columns/tables boosted over keywords once ≥1 char is typed. When a FROM scope
+ * is present, unqualified column suggestions are scoped to the statement's
+ * tables (out-of-scope columns dropped, in-scope ones boosted); with no scope
+ * the global pool is used unchanged. Empty word (and not qualified) → keywords +
+ * tables only. Capped for a tight dropdown.
  */
 export function rankCompletions(items, ctx) {
   const w = ctx.word.toLowerCase();
   if (ctx.qualified) {
-    const cols = items.filter((it) => it.kind === 'column' && it.parent === ctx.parent);
+    const target = resolveScopeAlias(ctx.parent, ctx.scope);
+    const cols = items.filter((it) => it.kind === 'column' && it.parent === target);
     return (w ? cols.filter((c) => c.label.toLowerCase().includes(w)) : cols).slice(0, 50);
   }
   if (ctx.afterFormat) {
@@ -205,14 +233,19 @@ export function rankCompletions(items, ctx) {
   if (!w) {
     return items.filter((it) => it.kind === 'keyword' || it.kind === 'table').slice(0, 40);
   }
+  const scopeTables = scopeTableSet(ctx.scope);
   const scored = [];
   for (const it of items) {
     if (it.kind === 'format') continue; // formats only inside a FORMAT clause
+    // FROM-scoped: only columns of the statement's tables compete unqualified;
+    // an unrelated loaded table's columns aren't suggested (#84). No scope → all.
+    if (it.kind === 'column' && scopeTables && !scopeTables.has(it.parent)) continue;
     const l = it.label.toLowerCase();
     const idx = l.indexOf(w);
     if (idx === -1) continue;
     let score = idx === 0 ? 0 : 100 + idx;              // prefix beats substring
     if (it.kind === 'column' || it.kind === 'table') score -= 10; // boost schema
+    if (it.kind === 'column' && scopeTables) score -= 15; // in-scope columns lead
     if (it.kind === 'keyword') {
       // A clause keyword sharing a name with an obscure function wins the tie
       // once enough of it is typed (≥3 chars, prefix) — e.g. `for` → FORMAT, not

src/core/from-scope.js

@@ -0,0 +1,166 @@
+// FROM/JOIN scope resolution for FROM-aware autocompletion (#84). Pure: no DOM,
+// no globals. Given editor text + a caret offset, `fromScopeAt` returns the base
+// tables in scope for the statement the caret sits in — each `{db, table, alias}`
+// — so completion can (1) resolve an alias (`e.` → `events`), (2) scope
+// unqualified column suggestions to the statement's FROM/JOIN tables, and (3)
+// drive the debounced lazy-load of those tables' columns.
+//
+// It reuses the shared SQL tokenizer (sql-highlight.js `tokenize`) so a `FROM`
+// inside a string/comment, or a `;` inside a literal, never fools the parse:
+// the tokenizer already classifies strings/comments/backtick-idents, and a
+// top-level `;` is always a bare `op` token.
+//
+// Non-goals (v1, per the issue): CTE / subquery-derived column scopes,
+// `USING`/correlated-subquery resolution, `SELECT *` expansion, table functions
+// (`FROM numbers(…)`). Those are skipped, not resolved — only real base tables
+// named in FROM/JOIN are returned.
+
+import { tokenize } from './sql-highlight.js';
+import { unquoteIdent } from './format.js';
+
+// Bare words that must never be read as a table alias but aren't in
+// SQL_KEYWORDS (so the tokenizer types them as `ident`, not `keyword`).
+// SQL_KEYWORDS members are already `keyword`-typed and rejected for free.
+const NON_ALIAS = new Set(['USING', 'WINDOW', 'QUALIFY']);
+
+// Tokenize `text` (or reuse a caller's `tokenize` output) and annotate each
+// token with its end offset — `tokenize` covers every character exactly once,
+// so a running length is enough to map a token to the caret. Only `end` is
+// needed (statement selection); the start is never read.
+function withOffsets(text, toks) {
+  const out = [];
+  let off = 0;
+  for (const [type, t] of toks || tokenize(text)) {
+    off += t.length;
+    out.push({ type, text: t, end: off });
+  }
+  return out;
+}
+
+// The tokens of the statement containing `pos`, split on top-level `;` (a bare
+// `op` `;` token — literals/comments are their own token types, so their `;`
+// never reaches here). `pos` picks the first statement whose text extends to or
+// past it; a `pos` sitting in the gap left by a `;` falls to the next statement.
+function statementTokensAt(toks, pos) {
+  const groups = [];
+  let cur = [];
+  for (const t of toks) {
+    if (t.type === 'op' && t.text === ';') { groups.push(cur); cur = []; continue; }
+    cur.push(t);
+  }
+  groups.push(cur);
+  for (const g of groups) {
+    if (g.length && g[g.length - 1].end >= pos) return g;
+  }
+  return groups[groups.length - 1];
+}
+
+const isDot = (t) => t && t.type === 'op' && t.text === '.';
+const isComma = (t) => t && t.type === 'op' && t.text === ',';
+const isOpenParen = (t) => t && t.type === 'op' && t.text === '(';
+const isIdent = (t) => t && t.type === 'ident';
+const isKeyword = (t, kw) => t && t.type === 'keyword' && t.text.toUpperCase() === kw;
+
+// Parse a single table reference starting at `i` in the significant-token list,
+// pushing `{db, table, alias}` to `refs` when it names a real base table.
+// Returns the index just past what it consumed. Bails (adds nothing) on a `(`
+// (subquery / table function) — those are non-goals.
+function parseTableRef(sig, i, refs) {
+  const t = sig[i];
+  if (!isIdent(t)) return i;
+  let db = null;
+  let table = unquoteIdent(t.text);
+  let j = i + 1;
+  if (isDot(sig[j]) && isIdent(sig[j + 1])) {
+    db = table;
+    table = unquoteIdent(sig[j + 1].text);
+    j += 2;
+  }
+  if (isOpenParen(sig[j])) return j; // table function / subquery alias form — skip
+  let alias = null;
+  if (isKeyword(sig[j], 'AS') && isIdent(sig[j + 1])) {
+    alias = unquoteIdent(sig[j + 1].text);
+    j += 2;
+  } else if (isIdent(sig[j]) && !NON_ALIAS.has(sig[j].text.toUpperCase())) {
+    alias = unquoteIdent(sig[j].text);
+    j += 1;
+  }
+  refs.push({ db, table, alias });
+  return j;
+}
+
+// Parse a comma-separated list of table refs (the FROM list) starting at `i`.
+function parseFromList(sig, i, refs) {
+  let j = parseTableRef(sig, i, refs);
+  while (isComma(sig[j])) j = parseTableRef(sig, j + 1, refs);
+  return j;
+}
+
+/**
+ * The base tables in scope for the statement containing `pos`: an array of
+ * `{db, table, alias}` (db/alias null when absent), in source order, deduped.
+ * Handles `db.table`, `table alias`, `table AS alias`, comma joins and `JOIN`s;
+ * a table function or a subquery in FROM position (`FROM (…) x`) contributes no
+ * ref. Not paren-aware: a subquery elsewhere (`WHERE id IN (SELECT … FROM b)`)
+ * still adds its base table `b` — a v1 over-approximation (it over-includes,
+ * never wrong-suppresses), since subquery-derived scoping is a non-goal.
+ * Returns `[]` when the statement has no FROM. `toks` optionally supplies a
+ * pre-computed `tokenize(text)` so the completion path lexes once. Pure.
+ */
+export function fromScopeAt(text, pos, toks) {
+  const s = String(text || '');
+  const p = Math.max(0, Math.min(pos | 0, s.length));
+  const stmt = statementTokensAt(withOffsets(s, toks), p);
+  const sig = stmt.filter((t) => t.type !== 'ws' && t.type !== 'comment');
+  const refs = [];
+  for (let i = 0; i < sig.length; i++) {
+    const t = sig[i];
+    if (t.type !== 'keyword') continue;
+    const kw = t.text.toUpperCase();
+    if (kw === 'FROM') {
+      i = parseFromList(sig, i + 1, refs) - 1;
+    } else if (kw === 'JOIN' && !isKeyword(sig[i - 1], 'ARRAY')) {
+      // `ARRAY JOIN arr` unnests an array column, not a table — don't scope it.
+      i = parseTableRef(sig, i + 1, refs) - 1;
+    }
+  }
+  return dedupe(refs);
+}
+
+// Drop duplicate refs (self-joins, repeated names) by db+table+alias identity.
+// JSON.stringify keys the tuple unambiguously — identifiers may contain spaces
+// (backtick-quoted), so a plain-delimiter key could collide.
+function dedupe(refs) {
+  const seen = new Set();
+  const out = [];
+  for (const r of refs) {
+    const key = JSON.stringify([r.db, r.table, r.alias]);
+    if (!seen.has(key)) { seen.add(key); out.push(r); }
+  }
+  return out;
+}
+
+/**
+ * Which of the scope's tables still need their columns fetched: the `{db, table}`
+ * entries present in `schema` whose `columns` are neither loaded (an array) nor
+ * in-flight (`'loading'`). Matched by db when the ref is db-qualified, else
+ * across every db that has a table of that name. Deduped by db+table. Feeds the
+ * editor's debounced idle-tick column loader (never the keystroke path). Pure.
+ */
+export function pendingColumnLoads(scope, schema) {
+  const out = [];
+  const seen = new Set();
+  for (const ref of scope || []) {
+    if (!ref.table) continue;
+    for (const d of schema || []) {
+      if (ref.db != null && d.db !== ref.db) continue;
+      for (const tb of d.tables || []) {
+        if (tb.name !== ref.table) continue;
+        if (Array.isArray(tb.columns) || tb.columns === 'loading') continue;
+        const key = JSON.stringify([d.db, tb.name]);
+        if (!seen.has(key)) { seen.add(key); out.push({ db: d.db, table: tb.name }); }
+      }
+    }
+  }
+  return out;
+}

src/editor/codemirror-adapter.js

@@ -19,11 +19,13 @@ import { EditorView, keymap, lineNumbers, drawSelection, dropCursor, hoverToolti
 import { history, historyKeymap, defaultKeymap } from '@codemirror/commands';
 import { bracketMatching, syntaxHighlighting, syntaxTree, HighlightStyle } from '@codemirror/language';
 import { sql, SQLDialect } from '@codemirror/lang-sql';
-import { autocompletion, closeBrackets, closeBracketsKeymap, acceptCompletion } from '@codemirror/autocomplete';
+import { autocompletion, closeBrackets, closeBracketsKeymap, acceptCompletion, startCompletion, completionStatus } from '@codemirror/autocomplete';
 import { search, searchKeymap } from '@codemirror/search';
 import { tags } from '@lezer/highlight';
 import { h } from '../ui/dom.js';
 import { completionContext, rankCompletions, wordAt } from '../core/completions.js';
+import { fromScopeAt, pendingColumnLoads } from '../core/from-scope.js';
+import { tokenize } from '../core/sql-highlight.js';
 import { toSubquery, clamp } from '../core/format.js';
 import { activeTab } from '../state.js';
 import { IDENT_MIME, SUBQUERY_MIME } from '../ui/dnd-mime.js';
@@ -138,8 +140,16 @@ export function completionSourceFor(app) {
     // (cutting AT the caret would misread an open backtick-identifier whose
     // escaped-backtick ends up last-before-the-cut as already closed).
     const doc = ctx.state.sliceDoc(0, ctx.state.doc.lineAt(ctx.pos).to);
-    const c = completionContext(doc, ctx.pos);
+    // Lex the caret prefix once and share it: both completionContext (open-
+    // backtick detection) and fromScopeAt need the same token stream.
+    const toks = tokenize(doc);
+    const c = completionContext(doc, ctx.pos, toks);
     if (!c.qualified && c.word.length < 1 && !ctx.explicit) return null;
+    // FROM-aware ranking (#84): resolve `e.` → events, and scope unqualified
+    // columns to the statement's FROM/JOIN tables. The slice already covers the
+    // lines before the caret, so a FROM above the caret is in view; a FROM below
+    // it degrades gracefully to the global pool (no scope).
+    c.scope = fromScopeAt(doc, ctx.pos, toks);
     const items = rankCompletions(app.completions || [], c);
     if (!items.length) return null;
     return {
@@ -280,6 +290,28 @@ export function insertTwoSpaces(view) {
   return true;
 }
 
+// Idle delay before the FROM-scope column prefetch runs (#84). Column metadata
+// is fetched on this debounced tick, NEVER on the keystroke path (the standing
+// editor rule) — long enough that a burst of typing collapses to one tick.
+const COLUMN_LOAD_DELAY_MS = 300;
+
+/**
+ * FROM-driven lazy column loading (#84): parse the statement around the caret,
+ * find its FROM/JOIN tables whose columns aren't loaded yet, and fetch them via
+ * the app's existing `loadColumns` (which writes the `'loading'` sentinel to
+ * dedupe, caches per connection, and rebuilds `app.completions`). Uses the whole
+ * document (not the keystroke-path line slice) so a FROM below the caret still
+ * prefetches. Resolves to whether it fetched anything — the caller refreshes an
+ * open dropdown only after re-checking the view is still live (destroy race).
+ * Exported for direct tests (timer-free).
+ */
+export function loadScopeColumns(app, view) {
+  const scope = fromScopeAt(view.state.doc.toString(), view.state.selection.main.head);
+  const pending = pendingColumnLoads(scope, app.state.schema.value);
+  if (!pending.length) return Promise.resolve(false);
+  return Promise.all(pending.map((p) => app.actions.loadColumns(p.db, p.table))).then(() => true);
+}
+
 /**
  * The CM6 editor behind the EditorPort seam. Port methods tolerate pre-mount
  * calls (no view yet → no-op / empty results). mount() is re-runnable: a
@@ -299,6 +331,25 @@ export function createCodeMirrorEditor(app) {
   let langExt = null;
   let view = null;
   let shownTabId = null;
+  let colTimer = null; // debounce handle for the FROM-scope column prefetch (#84)
+
+  // Schedule the debounced idle-tick column load (#84). Coalesces a typing
+  // burst into one tick; cleared on destroy so a torn-down view never fires.
+  // After the async fetch, re-check `view === v` (the file's replaceDocument
+  // idiom) before touching the view: a destroy() between tick and resolve nulls
+  // `view`, and re-running the completion source on a torn-down view would
+  // throw. Refresh a live, open completion so freshly-loaded columns appear.
+  const scheduleColumnLoad = () => {
+    if (colTimer) clearTimeout(colTimer);
+    colTimer = setTimeout(() => {
+      colTimer = null;
+      const v = view;
+      if (!v) return;
+      loadScopeColumns(app, v).then((loaded) => {
+        if (loaded && view === v && completionStatus(v.state)) startCompletion(v);
+      });
+    }, COLUMN_LOAD_DELAY_MS);
+  };
 
   const extensions = () => [
     lineNumbers(),
@@ -331,6 +382,7 @@ export function createCodeMirrorEditor(app) {
       // coalesces a user edit with a reconcile must still reach tab.sql.
       if (u.docChanged && !u.transactions.every((tr) => tr.annotation(syncTx))) {
         emit(u.state.doc.toString());
+        scheduleColumnLoad(); // user edit → prefetch the statement's FROM columns (#84)
       }
     }),
     EditorView.domEventHandlers({
@@ -359,6 +411,7 @@ export function createCodeMirrorEditor(app) {
     destroy: () => {
       subs.clear();
       tabStates.clear();
+      if (colTimer) { clearTimeout(colTimer); colTimer = null; }
       if (view) view.destroy();
       view = null;
     },