From d97b68e637d11402fe041e19020c1963fd2e41c7 Mon Sep 17 00:00:00 2001 From: janicelichtman Date: Tue, 13 Oct 2026 20:45:47 -0400 Subject: [PATCH 01/15] moving ocpf member matching to be scoped per court --- functions/src/ocpf/matchOcpfMembers.ts | 9 ++++++--- functions/src/ocpf/scrapeOcpfFinance.ts | 7 +++---- functions/src/ocpf/types.ts | 4 ++-- 3 files changed, 11 insertions(+), 9 deletions(-) diff --git a/functions/src/ocpf/matchOcpfMembers.ts b/functions/src/ocpf/matchOcpfMembers.ts index 169f2309d..980b12471 100644 --- a/functions/src/ocpf/matchOcpfMembers.ts +++ b/functions/src/ocpf/matchOcpfMembers.ts @@ -40,7 +40,10 @@ export const matchOcpfMembersV2 = onRequestV2(async (req, res) => { const filers = await downloadAndParseFilers() const members = await loadMembers() - const existingMappingDoc = await db.doc("/config/ocpfMemberMapping").get() + const mappingPath = `/generalCourts/${currentGeneralCourt}/ocpfMemberMappingConfig/ocpfMemberMapping` + const flagsPath = `/generalCourts/${currentGeneralCourt}/ocpfMemberMappingConfig/ocpfMemberMappingFlags` + + const existingMappingDoc = await db.doc(mappingPath).get() const existingMapping = (existingMappingDoc.data() ?? {}) as OcpfMemberMapping const mapping: OcpfMemberMapping = {} @@ -125,8 +128,8 @@ export const matchOcpfMembersV2 = onRequestV2(async (req, res) => { const flags: OcpfMemberMappingFlags = { unmatched, ambiguous } - await db.doc("/config/ocpfMemberMapping").set(mapping, { merge: true }) - await db.doc("/config/ocpfMemberMappingFlags").set(flags) + await db.doc(mappingPath).set(mapping, { merge: true }) + await db.doc(flagsPath).set(flags) logger.info("matchOcpfMembers complete", { matched: Object.keys(mapping).length, diff --git a/functions/src/ocpf/scrapeOcpfFinance.ts b/functions/src/ocpf/scrapeOcpfFinance.ts index 10c9ae8ad..c366e085d 100644 --- a/functions/src/ocpf/scrapeOcpfFinance.ts +++ b/functions/src/ocpf/scrapeOcpfFinance.ts @@ -307,11 +307,10 @@ async function runOcpfFinanceScrape( options: { cpfId?: number } = {} ): Promise<{ processed: number; years: string[] }> { // ── A. Load member mapping ───────────────────────────────────────────── - const mappingDoc = await db.doc("/config/ocpfMemberMapping").get() + const mappingPath = `/generalCourts/${currentGeneralCourt}/ocpfMemberMappingConfig/ocpfMemberMapping` + const mappingDoc = await db.doc(mappingPath).get() if (!mappingDoc.exists) { - logger.warn( - "config/ocpfMemberMapping not found; no members will be processed" - ) + logger.warn(`${mappingPath} not found; no members will be processed`) } const mapping = (mappingDoc.data() ?? {}) as OcpfMemberMapping diff --git a/functions/src/ocpf/types.ts b/functions/src/ocpf/types.ts index 37f639328..a935cd8ef 100644 --- a/functions/src/ocpf/types.ts +++ b/functions/src/ocpf/types.ts @@ -8,7 +8,7 @@ export interface OcpfFilerRow { closedDate: string // empty string = active } -// Firestore: /config/ocpfMemberMapping +// Firestore: /generalCourts/{court}/ocpfMemberMappingConfig/ocpfMemberMapping // memberCode → { cpfId, name }, e.g. { "SND1": { cpfId: 15031, name: "Sal N. DiDomenico" } } export interface OcpfMemberMappingEntry { cpfId: number @@ -22,7 +22,7 @@ export interface OcpfMemberMappingFlagsEntry { name: string } -// Firestore: /config/ocpfMemberMappingFlags +// Firestore: /generalCourts/{court}/ocpfMemberMappingConfig/ocpfMemberMappingFlags export interface OcpfMemberMappingFlags { unmatched: OcpfMemberMappingFlagsEntry[] ambiguous: OcpfMemberMappingFlagsEntry[] From bd784567e46ec2bd24307b1ffaea388d2cf5bb0f Mon Sep 17 00:00:00 2001 From: janicelichtman Date: Mon, 19 Oct 2026 16:18:35 -0400 Subject: [PATCH 02/15] renamed matchOcpfMembersV2 to matchOcpfMembers --- functions/src/index.ts | 2 +- functions/src/ocpf/matchOcpfMembers.ts | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/functions/src/index.ts b/functions/src/index.ts index 5a217d6f5..788b1857e 100644 --- a/functions/src/index.ts +++ b/functions/src/index.ts @@ -64,7 +64,7 @@ export { scrapeElections } from "./legislators" export { transcriptionV2 } from "./webhooks" -export { matchOcpfMembersV2 } from "./ocpf/matchOcpfMembers" +export { matchOcpfMembers } from "./ocpf/matchOcpfMembers" export { scrapeOcpfFinanceNightly, scrapeOcpfFinanceManual diff --git a/functions/src/ocpf/matchOcpfMembers.ts b/functions/src/ocpf/matchOcpfMembers.ts index 980b12471..b899a2461 100644 --- a/functions/src/ocpf/matchOcpfMembers.ts +++ b/functions/src/ocpf/matchOcpfMembers.ts @@ -14,7 +14,7 @@ import { OcpfMemberMappingFlagsEntry } from "./types" -export const matchOcpfMembersV2 = onRequestV2(async (req, res) => { +export const matchOcpfMembers = onRequestV2(async (req, res) => { if (req.method !== "POST") { res.status(405).send("Method Not Allowed. Use POST.") return From 8d7e8e49571461900eea47758980a20c46340d56 Mon Sep 17 00:00:00 2001 From: janicelichtman Date: Mon, 19 Oct 2026 16:33:42 -0400 Subject: [PATCH 03/15] warn on unclassified OCPF record types in finance scrape MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Report types already logged a warning when unrecognized, defaulting to a safe itemsOnly treatment. Record types fell through to a silent default with no distinction between deliberately-ignored codes (205, 220) and a genuinely new one — this makes that failure mode loud too. Co-Authored-By: Claude Sonnet 5 --- functions/src/ocpf/scrapeOcpfFinance.ts | 31 ++++++++++++++++++++++--- 1 file changed, 28 insertions(+), 3 deletions(-) diff --git a/functions/src/ocpf/scrapeOcpfFinance.ts b/functions/src/ocpf/scrapeOcpfFinance.ts index c366e085d..821b4e6b5 100644 --- a/functions/src/ocpf/scrapeOcpfFinance.ts +++ b/functions/src/ocpf/scrapeOcpfFinance.ts @@ -700,6 +700,7 @@ async function streamReportItems( let idx: Record = {} let processed = 0 let skipped = 0 + const warnedRecordTypeIds = new Set() for await (const line of rl) { if (!line.trim()) continue @@ -743,7 +744,15 @@ async function streamReportItems( col(cols, idx.zip) ) - accumulateItem(acc, recordTypeId, amount, dateMs, contributor, year) + accumulateItem( + acc, + recordTypeId, + amount, + dateMs, + contributor, + year, + warnedRecordTypeIds + ) processed++ } @@ -758,6 +767,12 @@ async function streamReportItems( // these are the itemized contributions behind totalRaised. const CONTRIBUTION_RECORD_TYPE_IDS = new Set([201, 202, 203]) +// Record types whose amounts are already counted elsewhere and are correctly +// excluded here — not unclassified, just sourced differently: 205 (Bank +// Interest) and 220 (Aggregated un-itemized) totals come from reports.txt, +// not report-items.txt. +const KNOWN_IGNORED_RECORD_TYPE_IDS = new Set([205, 220]) + // OCPF has no contributor ID, so a contributor is identified by // record type + name + first name + 5-digit ZIP. Committees and unions leave // First_Name blank and put the full name in Name. Record type is part of the key so an @@ -785,7 +800,8 @@ function accumulateItem( amount: number, dateMs: number, contributor: string | null, - year: string + year: string, + warnedRecordTypeIds: Set ): void { const addTo = (entry: MutableBreakdownEntry) => { entry.count++ @@ -852,8 +868,17 @@ function accumulateItem( case 319: // Payment-processor fee (see breakdown.processingFees doc comment) addTo(yb.processingFees) break - // 205 (Bank Interest) and 220 (Aggregated un-itemized) totals sourced from reports.txt, not items default: + if ( + !KNOWN_IGNORED_RECORD_TYPE_IDS.has(recordTypeId) && + !warnedRecordTypeIds.has(recordTypeId) + ) { + warnedRecordTypeIds.add(recordTypeId) + logger.warn( + "Unclassified OCPF record type — no amount counted; classify it in accumulateItem", + { recordTypeId, year } + ) + } break } } From 648362e11ee22a98be2b56cfb4bd852acf224b81 Mon Sep 17 00:00:00 2001 From: janicelichtman Date: Mon, 19 Oct 2026 16:57:17 -0400 Subject: [PATCH 04/15] extract matchMembersForCourt for per-court OCPF matching Pulls the matching logic out of the matchOcpfMembers HTTP handler into a reusable, court-parameterized function so the upcoming backfill script can match historical courts too. Active-filers-only filtering now applies only to the current court; historical courts drop it since a legislator who served years ago has likely closed their OCPF committee by now. Co-Authored-By: Claude Sonnet 5 --- functions/src/ocpf/matchOcpfMembers.ts | 68 ++++++++++++++++++-------- 1 file changed, 47 insertions(+), 21 deletions(-) diff --git a/functions/src/ocpf/matchOcpfMembers.ts b/functions/src/ocpf/matchOcpfMembers.ts index b899a2461..53c119059 100644 --- a/functions/src/ocpf/matchOcpfMembers.ts +++ b/functions/src/ocpf/matchOcpfMembers.ts @@ -37,11 +37,37 @@ export const matchOcpfMembers = onRequestV2(async (req, res) => { } } - const filers = await downloadAndParseFilers() - const members = await loadMembers() + const { matched, unmatched, ambiguous } = await matchMembersForCourt( + currentGeneralCourt + ) + + res.status(200).json({ + results: { + matched, + unmatched: unmatched.length, + ambiguous: ambiguous.length + }, + unmatched_members: unmatched, + ambiguous_members: ambiguous + }) +}) + +// Matches a general court's MAPLE members to their OCPF filer record by last +// name + branch, writing the result to that court's ocpfMemberMapping. The +// current court is matched against active filers only (OCPF Closed_Date +// empty); historical courts drop that filter. +export async function matchMembersForCourt(court: number): Promise<{ + matched: number + unmatched: OcpfMemberMappingFlagsEntry[] + ambiguous: OcpfMemberMappingFlagsEntry[] +}> { + const activeFilersOnly = court === currentGeneralCourt - const mappingPath = `/generalCourts/${currentGeneralCourt}/ocpfMemberMappingConfig/ocpfMemberMapping` - const flagsPath = `/generalCourts/${currentGeneralCourt}/ocpfMemberMappingConfig/ocpfMemberMappingFlags` + const filers = await downloadAndParseFilers(activeFilersOnly) + const members = await loadMembers(court) + + const mappingPath = `/generalCourts/${court}/ocpfMemberMappingConfig/ocpfMemberMapping` + const flagsPath = `/generalCourts/${court}/ocpfMemberMappingConfig/ocpfMemberMappingFlags` const existingMappingDoc = await db.doc(mappingPath).get() const existingMapping = (existingMappingDoc.data() ?? {}) as OcpfMemberMapping @@ -56,6 +82,9 @@ export const matchOcpfMembers = onRequestV2(async (req, res) => { if (!branch || (branch !== "Senate" && branch !== "House")) continue + // Branch is a hard filter: a legislator who switched chambers (House <-> + // Senate) between courts will come up unmatched for the earlier court. + // Expected for older courts, not a bug. const lastNameAndBranchMatches = filers.filter( f => f.lastName.toLowerCase() === lastName.toLowerCase() && @@ -90,6 +119,7 @@ export const matchOcpfMembers = onRequestV2(async (req, res) => { logger.warn( "Ambiguous OCPF match. Single last name match but first name did not align.", { + court, memberCode: member.MemberCode, name: member.Name, district: member.District, @@ -103,6 +133,7 @@ export const matchOcpfMembers = onRequestV2(async (req, res) => { } else if (candidates.length === 0) { unmatched.push({ memberCode: member.MemberCode, name: member.Name }) logger.warn("No OCPF match.", { + court, memberCode: member.MemberCode, name: member.Name, district: member.District, @@ -111,6 +142,7 @@ export const matchOcpfMembers = onRequestV2(async (req, res) => { } else { ambiguous.push({ memberCode: member.MemberCode, name: member.Name }) logger.warn("Ambiguous OCPF match.", { + court, memberCode: member.MemberCode, name: member.Name, district: member.District, @@ -131,24 +163,20 @@ export const matchOcpfMembers = onRequestV2(async (req, res) => { await db.doc(mappingPath).set(mapping, { merge: true }) await db.doc(flagsPath).set(flags) + const matched = Object.keys(mapping).length logger.info("matchOcpfMembers complete", { - matched: Object.keys(mapping).length, + court, + matched, unmatched: unmatched.length, ambiguous: ambiguous.length }) - res.status(200).json({ - results: { - matched: Object.keys(mapping).length, - unmatched: unmatched.length, - ambiguous: ambiguous.length - }, - unmatched_members: unmatched, - ambiguous_members: ambiguous - }) -}) + return { matched, unmatched, ambiguous } +} -async function downloadAndParseFilers(): Promise { +async function downloadAndParseFilers( + activeFilersOnly: boolean +): Promise { const response = await axios.get( "https://ocpf2.blob.core.windows.net/downloads/data2/ocpf-filers.zip", { responseType: "arraybuffer" } @@ -200,7 +228,7 @@ async function downloadAndParseFilers(): Promise { const closedDate = col(cols, colIndex.closedDate) const officeSought = col(cols, colIndex.officeSought) - if (closedDate !== "") continue + if (activeFilersOnly && closedDate !== "") continue if (officeSought !== "Senate" && officeSought !== "House") continue filers.push({ @@ -257,10 +285,8 @@ function buildColumnIndex( return index } -async function loadMembers(): Promise { - const snapshot = await db - .collection(`/generalCourts/${currentGeneralCourt}/members`) - .get() +async function loadMembers(court: number): Promise { + const snapshot = await db.collection(`/generalCourts/${court}/members`).get() return snapshot.docs .map(doc => { From c7d08daafb25d23c9be92c95ece73494ee30eb8e Mon Sep 17 00:00:00 2001 From: janicelichtman Date: Mon, 19 Oct 2026 17:26:49 -0400 Subject: [PATCH 05/15] add backfillOcpfMemberMapping script for historical OCPF member matching Co-Authored-By: Claude Sonnet 5 --- .../backfillOcpfMemberMapping.ts | 52 +++++++++++++++++++ 1 file changed, 52 insertions(+) create mode 100644 scripts/firebase-admin/backfillOcpfMemberMapping.ts diff --git a/scripts/firebase-admin/backfillOcpfMemberMapping.ts b/scripts/firebase-admin/backfillOcpfMemberMapping.ts new file mode 100644 index 000000000..59961cfce --- /dev/null +++ b/scripts/firebase-admin/backfillOcpfMemberMapping.ts @@ -0,0 +1,52 @@ +import { Record, String } from "runtypes" +import { Script } from "./types" +import { matchMembersForCourt } from "functions/src/ocpf/matchOcpfMembers" + +const Args = Record({ courts: String }) + +export const script: Script = async ({ args }) => { + const { courts } = Args.check(args) + const courtNumbers = courts.split(",").map(Number) + + const succeeded: number[] = [] + const failed: number[] = [] + + for (const court of courtNumbers) { + if (!Number.isInteger(court) || court <= 0) { + console.error(`Skipping invalid court number: ${court}`) + failed.push(court) + continue + } + + console.log(`\n── Court ${court} ──`) + try { + const { matched, unmatched, ambiguous } = await matchMembersForCourt( + court + ) + + console.log( + `Court ${court}: matched ${matched}, unmatched ${unmatched.length}, ambiguous ${ambiguous.length}` + ) + if (unmatched.length > 0) { + console.log( + " unmatched:", + unmatched.map(m => `${m.memberCode} (${m.name})`).join(", ") + ) + } + if (ambiguous.length > 0) { + console.log( + " ambiguous:", + ambiguous.map(m => `${m.memberCode} (${m.name})`).join(", ") + ) + } + succeeded.push(court) + } catch (err) { + console.error(`Court ${court} failed:`, err) + failed.push(court) + } + } + + console.log("\n── Summary ──") + console.log(`Succeeded: ${succeeded.join(", ") || "none"}`) + console.log(`Failed: ${failed.join(", ") || "none"}`) +} From 69a42fbdb894b71c1aa6dc6cc2ef38c42be400bf Mon Sep 17 00:00:00 2001 From: janicelichtman Date: Mon, 19 Oct 2026 17:45:02 -0400 Subject: [PATCH 06/15] add backfillOcpfFinance script and court-parameterize finance scraping MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Extracts runOcpfFinanceScrape into scrapeFinanceForCourt(court, options), deriving YEARS per-call from generalCourts instead of a module-level constant tied to the current court. Adds generalCourts entries for 189-191 so historical courts can be targeted. The two existing triggers are unaffected — both still just call scrapeFinanceForCourt(currentGeneralCourt, ...). Co-Authored-By: Claude Sonnet 5 --- functions/src/ocpf/scrapeOcpfFinance.ts | 60 +++++++++++-------- functions/src/shared/constants.ts | 18 ++++++ scripts/firebase-admin/backfillOcpfFinance.ts | 39 ++++++++++++ 3 files changed, 91 insertions(+), 26 deletions(-) create mode 100644 scripts/firebase-admin/backfillOcpfFinance.ts diff --git a/functions/src/ocpf/scrapeOcpfFinance.ts b/functions/src/ocpf/scrapeOcpfFinance.ts index 821b4e6b5..75b875947 100644 --- a/functions/src/ocpf/scrapeOcpfFinance.ts +++ b/functions/src/ocpf/scrapeOcpfFinance.ts @@ -19,12 +19,6 @@ import { const OCPF_BASE_URL = "https://ocpf2.blob.core.windows.net/downloads/data2" -const currentCourt = generalCourts[currentGeneralCourt] -if (!currentCourt) { - throw new Error(`No general court entry for court ${currentGeneralCourt}`) -} -const YEARS = [currentCourt.FirstYear, currentCourt.SecondYear].map(String) - // How each OCPF report type (reports.txt Report_Type_ID) is used: // totals — Receipts_Total/Expenditures_Total feed totalRaised/totalSpent; // items and Receipts_Unitemized_Total are also used @@ -182,7 +176,7 @@ function sumBreakdowns(breakdowns: MutableBreakdown[]): MutableBreakdown { } } -function newAccumulator(cpfId: number): MemberAccumulator { +function newAccumulator(cpfId: number, years: string[]): MemberAccumulator { const yearInit = (): YearAccumulator => ({ totalRaised: 0, totalSpent: 0, @@ -202,10 +196,10 @@ function newAccumulator(cpfId: number): MemberAccumulator { union: emptyEntry(), unitemized: { amount: 0 } }, - years: Object.fromEntries(YEARS.map(y => [y, yearInit()])), - yearEndCheck: Object.fromEntries(YEARS.map(y => [y, null])), + years: Object.fromEntries(years.map(y => [y, yearInit()])), + yearEndCheck: Object.fromEntries(years.map(y => [y, null])), reportTotals: Object.fromEntries( - YEARS.map(y => [y, { receipts: 0, expenditures: 0 }]) + years.map(y => [y, { receipts: 0, expenditures: 0 }]) ), pendingNonContributionReceipts: [] } @@ -248,7 +242,7 @@ export const scrapeOcpfFinanceNightly = onSchedule( retryCount: 1 }, async () => { - await runOcpfFinanceScrape() + await scrapeFinanceForCourt(currentGeneralCourt) } ) @@ -295,22 +289,33 @@ export const scrapeOcpfFinanceManual = onRequestV2( return } - const results = await runOcpfFinanceScrape({ cpfId: query.data.cpfId }) + const results = await scrapeFinanceForCourt(currentGeneralCourt, { + cpfId: query.data.cpfId + }) res.status(200).json({ results }) } ) -// ── Scrape run (shared by nightly and manual triggers) ──────────────────────── +// ── Scrape run (shared by nightly, manual and backfill triggers) ────────────── // Set cpfId to run a single member; omit to run all members. -async function runOcpfFinanceScrape( +export async function scrapeFinanceForCourt( + court: number, options: { cpfId?: number } = {} ): Promise<{ processed: number; years: string[] }> { + const courtInfo = generalCourts[court] + if (!courtInfo) { + throw new Error(`No general court entry for court ${court}`) + } + const years = [courtInfo.FirstYear, courtInfo.SecondYear].map(String) + // ── A. Load member mapping ───────────────────────────────────────────── - const mappingPath = `/generalCourts/${currentGeneralCourt}/ocpfMemberMappingConfig/ocpfMemberMapping` + const mappingPath = `/generalCourts/${court}/ocpfMemberMappingConfig/ocpfMemberMapping` const mappingDoc = await db.doc(mappingPath).get() if (!mappingDoc.exists) { - logger.warn(`${mappingPath} not found; no members will be processed`) + logger.warn(`${mappingPath} not found; no members will be processed`, { + court + }) } const mapping = (mappingDoc.data() ?? {}) as OcpfMemberMapping @@ -334,6 +339,7 @@ async function runOcpfFinanceScrape( } logger.info("Loaded member mapping", { + court, totalMembers: Object.keys(mapping).length, activeInRun: cpfIdToMemberCode.size }) @@ -347,20 +353,21 @@ async function runOcpfFinanceScrape( // items are read for in-kind records only const inKindOnlyReportIds = new Set() - for (const year of YEARS) { + for (const year of years) { const url = `${OCPF_BASE_URL}/ocpf-${year}-reports.zip` - logger.info(`Downloading ${url}`) + logger.info(`Downloading ${url}`, { court }) const buf = await downloadBuffer(url) await parseReports( buf, year, + years, cpfIdToMemberCode, accumulators, reportIdToMemberCode, inKindOnlyReportIds ) - logger.info(`Streaming report-items for ${year}`) + logger.info(`Streaming report-items for ${year}`, { court }) await streamReportItems( buf, reportIdToMemberCode, @@ -417,7 +424,7 @@ async function runOcpfFinanceScrape( // (Deposit Report items, before deduction of fees). That gap is tracked // via record type 319 (breakdown.processingFees). for (const [memberCode, acc] of accumulators) { - for (const year of YEARS) { + for (const year of years) { const check = acc.yearEndCheck[year] if (!check) continue const summedRaised = acc.reportTotals[year]?.receipts ?? 0 @@ -428,6 +435,7 @@ async function runOcpfFinanceScrape( logger.warn( "Year-end totals mismatch — investigate periodic report accumulation", { + court, memberCode, year, yearEnd: { @@ -452,9 +460,7 @@ async function runOcpfFinanceScrape( const batch = db.batch() for (const [memberCode, acc] of accumulators) { - const doc = db.doc( - `/generalCourts/${currentGeneralCourt}/membersFinance/${memberCode}` - ) + const doc = db.doc(`/generalCourts/${court}/membersFinance/${memberCode}`) const yearAccs = Object.values(acc.years) const data: MembersFinance = { ocpfCpfId: acc.cpfId, @@ -495,11 +501,12 @@ async function runOcpfFinanceScrape( await batch.commit() logger.info("scrapeOcpfFinance complete", { + court, processed: accumulators.size, - years: YEARS + years }) - return { processed: accumulators.size, years: YEARS } + return { processed: accumulators.size, years } } // ── Helpers ─────────────────────────────────────────────────────────────────── @@ -563,6 +570,7 @@ function col(cols: string[], idx: number): string { async function parseReports( buf: Buffer, year: string, + years: string[], cpfIdToMemberCode: Map, accumulators: Map, reportIdToMemberCode: Map, @@ -602,7 +610,7 @@ async function parseReports( let acc = accumulators.get(memberCode) if (!acc) { - acc = newAccumulator(cpfId) + acc = newAccumulator(cpfId, years) accumulators.set(memberCode, acc) } diff --git a/functions/src/shared/constants.ts b/functions/src/shared/constants.ts index ff375ec22..1857947cf 100644 --- a/functions/src/shared/constants.ts +++ b/functions/src/shared/constants.ts @@ -23,6 +23,24 @@ export const generalCourts: Record = { Number: 192, FirstYear: 2021, SecondYear: 2022 + }, + 191: { + Name: "191st (2019 - 2020)", + Number: 191, + FirstYear: 2019, + SecondYear: 2020 + }, + 190: { + Name: "190th (2017 - 2018)", + Number: 190, + FirstYear: 2017, + SecondYear: 2018 + }, + 189: { + Name: "189th (2015 - 2016)", + Number: 189, + FirstYear: 2015, + SecondYear: 2016 } } diff --git a/scripts/firebase-admin/backfillOcpfFinance.ts b/scripts/firebase-admin/backfillOcpfFinance.ts new file mode 100644 index 000000000..195ad25c8 --- /dev/null +++ b/scripts/firebase-admin/backfillOcpfFinance.ts @@ -0,0 +1,39 @@ +import { Record, String } from "runtypes" +import { Script } from "./types" +import { scrapeFinanceForCourt } from "functions/src/ocpf/scrapeOcpfFinance" + +const Args = Record({ courts: String }) + +export const script: Script = async ({ args }) => { + const { courts } = Args.check(args) + const courtNumbers = courts.split(",").map(Number) + + const succeeded: number[] = [] + const failed: number[] = [] + + for (const court of courtNumbers) { + if (!Number.isInteger(court) || court <= 0) { + console.error(`Skipping invalid court number: ${court}`) + failed.push(court) + continue + } + + console.log(`\n── Court ${court} ──`) + try { + const { processed, years } = await scrapeFinanceForCourt(court) + console.log( + `Court ${court}: processed ${processed} members for years ${years.join( + ", " + )}` + ) + succeeded.push(court) + } catch (err) { + console.error(`Court ${court} failed:`, err) + failed.push(court) + } + } + + console.log("\n── Summary ──") + console.log(`Succeeded: ${succeeded.join(", ") || "none"}`) + console.log(`Failed: ${failed.join(", ") || "none"}`) +} From 7042c3bb07baf03604e0b8e39ba63565b5c7b420 Mon Sep 17 00:00:00 2001 From: janicelichtman Date: Tue, 20 Oct 2026 17:49:53 -0400 Subject: [PATCH 07/15] enable scripts to take a single court in addition to a list of courts --- scripts/firebase-admin/backfillOcpfFinance.ts | 7 ++++--- scripts/firebase-admin/backfillOcpfMemberMapping.ts | 7 ++++--- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/scripts/firebase-admin/backfillOcpfFinance.ts b/scripts/firebase-admin/backfillOcpfFinance.ts index 195ad25c8..d14d56071 100644 --- a/scripts/firebase-admin/backfillOcpfFinance.ts +++ b/scripts/firebase-admin/backfillOcpfFinance.ts @@ -1,12 +1,13 @@ -import { Record, String } from "runtypes" +import { Number as RNumber, Record, String as RString, Union } from "runtypes" import { Script } from "./types" import { scrapeFinanceForCourt } from "functions/src/ocpf/scrapeOcpfFinance" -const Args = Record({ courts: String }) +// yargs parses a single value like "193" as a number +const Args = Record({ courts: Union(RString, RNumber) }) export const script: Script = async ({ args }) => { const { courts } = Args.check(args) - const courtNumbers = courts.split(",").map(Number) + const courtNumbers = String(courts).split(",").map(Number) const succeeded: number[] = [] const failed: number[] = [] diff --git a/scripts/firebase-admin/backfillOcpfMemberMapping.ts b/scripts/firebase-admin/backfillOcpfMemberMapping.ts index 59961cfce..8a466305d 100644 --- a/scripts/firebase-admin/backfillOcpfMemberMapping.ts +++ b/scripts/firebase-admin/backfillOcpfMemberMapping.ts @@ -1,12 +1,13 @@ -import { Record, String } from "runtypes" +import { Number as RNumber, Record, String as RString, Union } from "runtypes" import { Script } from "./types" import { matchMembersForCourt } from "functions/src/ocpf/matchOcpfMembers" -const Args = Record({ courts: String }) +// yargs parses a single value like "193" as a number +const Args = Record({ courts: Union(RString, RNumber) }) export const script: Script = async ({ args }) => { const { courts } = Args.check(args) - const courtNumbers = courts.split(",").map(Number) + const courtNumbers = String(courts).split(",").map(Number) const succeeded: number[] = [] const failed: number[] = [] From 6ce42604aac3ba899004840976ad1959c6e040c3 Mon Sep 17 00:00:00 2001 From: janicelichtman Date: Tue, 20 Oct 2026 18:04:05 -0400 Subject: [PATCH 08/15] using zod to parse input --- scripts/firebase-admin/backfillOcpfFinance.ts | 22 +++++++++---------- .../backfillOcpfMemberMapping.ts | 22 +++++++++---------- 2 files changed, 20 insertions(+), 24 deletions(-) diff --git a/scripts/firebase-admin/backfillOcpfFinance.ts b/scripts/firebase-admin/backfillOcpfFinance.ts index d14d56071..b43797e8d 100644 --- a/scripts/firebase-admin/backfillOcpfFinance.ts +++ b/scripts/firebase-admin/backfillOcpfFinance.ts @@ -1,24 +1,22 @@ -import { Number as RNumber, Record, String as RString, Union } from "runtypes" +import { z } from "zod" import { Script } from "./types" import { scrapeFinanceForCourt } from "functions/src/ocpf/scrapeOcpfFinance" -// yargs parses a single value like "193" as a number -const Args = Record({ courts: Union(RString, RNumber) }) +const Args = z.object({ + // yargs parses a single value like "193" as a number + courts: z + .union([z.string(), z.number()]) + .transform(v => String(v).split(",").map(Number)) + .pipe(z.array(z.number().int().positive())) +}) export const script: Script = async ({ args }) => { - const { courts } = Args.check(args) - const courtNumbers = String(courts).split(",").map(Number) + const { courts } = Args.parse(args) const succeeded: number[] = [] const failed: number[] = [] - for (const court of courtNumbers) { - if (!Number.isInteger(court) || court <= 0) { - console.error(`Skipping invalid court number: ${court}`) - failed.push(court) - continue - } - + for (const court of courts) { console.log(`\n── Court ${court} ──`) try { const { processed, years } = await scrapeFinanceForCourt(court) diff --git a/scripts/firebase-admin/backfillOcpfMemberMapping.ts b/scripts/firebase-admin/backfillOcpfMemberMapping.ts index 8a466305d..755bdda11 100644 --- a/scripts/firebase-admin/backfillOcpfMemberMapping.ts +++ b/scripts/firebase-admin/backfillOcpfMemberMapping.ts @@ -1,24 +1,22 @@ -import { Number as RNumber, Record, String as RString, Union } from "runtypes" +import { z } from "zod" import { Script } from "./types" import { matchMembersForCourt } from "functions/src/ocpf/matchOcpfMembers" -// yargs parses a single value like "193" as a number -const Args = Record({ courts: Union(RString, RNumber) }) +const Args = z.object({ + // yargs parses a single value like "193" as a number + courts: z + .union([z.string(), z.number()]) + .transform(v => String(v).split(",").map(Number)) + .pipe(z.array(z.number().int().positive())) +}) export const script: Script = async ({ args }) => { - const { courts } = Args.check(args) - const courtNumbers = String(courts).split(",").map(Number) + const { courts } = Args.parse(args) const succeeded: number[] = [] const failed: number[] = [] - for (const court of courtNumbers) { - if (!Number.isInteger(court) || court <= 0) { - console.error(`Skipping invalid court number: ${court}`) - failed.push(court) - continue - } - + for (const court of courts) { console.log(`\n── Court ${court} ──`) try { const { matched, unmatched, ambiguous } = await matchMembersForCourt( From 748a667d2e19843d70c349e66b6c358ef9097bdb Mon Sep 17 00:00:00 2001 From: janicelichtman Date: Tue, 20 Oct 2026 19:41:46 -0400 Subject: [PATCH 09/15] giving user nicer error message for bad script input --- scripts/firebase-admin/backfillOcpfFinance.ts | 3 ++- scripts/firebase-admin/backfillOcpfMemberMapping.ts | 3 ++- scripts/firebase-admin/index.ts | 9 ++++++++- 3 files changed, 12 insertions(+), 3 deletions(-) diff --git a/scripts/firebase-admin/backfillOcpfFinance.ts b/scripts/firebase-admin/backfillOcpfFinance.ts index b43797e8d..05e7fd7cd 100644 --- a/scripts/firebase-admin/backfillOcpfFinance.ts +++ b/scripts/firebase-admin/backfillOcpfFinance.ts @@ -1,5 +1,6 @@ import { z } from "zod" import { Script } from "./types" +import { parseArgs } from "./args" import { scrapeFinanceForCourt } from "functions/src/ocpf/scrapeOcpfFinance" const Args = z.object({ @@ -11,7 +12,7 @@ const Args = z.object({ }) export const script: Script = async ({ args }) => { - const { courts } = Args.parse(args) + const { courts } = parseArgs(Args, args) const succeeded: number[] = [] const failed: number[] = [] diff --git a/scripts/firebase-admin/backfillOcpfMemberMapping.ts b/scripts/firebase-admin/backfillOcpfMemberMapping.ts index 755bdda11..274db753a 100644 --- a/scripts/firebase-admin/backfillOcpfMemberMapping.ts +++ b/scripts/firebase-admin/backfillOcpfMemberMapping.ts @@ -1,5 +1,6 @@ import { z } from "zod" import { Script } from "./types" +import { parseArgs } from "./args" import { matchMembersForCourt } from "functions/src/ocpf/matchOcpfMembers" const Args = z.object({ @@ -11,7 +12,7 @@ const Args = z.object({ }) export const script: Script = async ({ args }) => { - const { courts } = Args.parse(args) + const { courts } = parseArgs(Args, args) const succeeded: number[] = [] const failed: number[] = [] diff --git a/scripts/firebase-admin/index.ts b/scripts/firebase-admin/index.ts index 81e18ef76..37d3ca5f3 100644 --- a/scripts/firebase-admin/index.ts +++ b/scripts/firebase-admin/index.ts @@ -5,6 +5,7 @@ import { hideBin } from "yargs/helpers" import { runAgainstEmulators, runAgainstProject } from "../configure" import { Context, Script, ScriptContext } from "./types" import { listAllUsers } from "./list-all-users" +import { ArgsError } from "./args" const Args = Record({ env: Union(L("local"), L("dev"), L("prod")), @@ -69,7 +70,13 @@ yargs(hideBin(process.argv)) const context: ScriptContext = { ...init(args), args: args as any } const script: Script = require(`./${scriptName}`).script - await script(context) + try { + await script(context) + } catch (err) { + if (!(err instanceof ArgsError)) throw err + console.error(err.message) + process.exitCode = 1 + } } ) .strictCommands() From 23dbecff7de1ae43fdeed2597f71a83b6a24f0f2 Mon Sep 17 00:00:00 2001 From: janicelichtman Date: Tue, 20 Oct 2026 21:11:56 -0400 Subject: [PATCH 10/15] further improvements to error message for bad script input --- scripts/firebase-admin/args.ts | 43 +++++++++++++++++++ scripts/firebase-admin/backfillOcpfFinance.ts | 8 +--- .../backfillOcpfMemberMapping.ts | 8 +--- scripts/firebase-admin/index.ts | 15 +++++++ 4 files changed, 62 insertions(+), 12 deletions(-) create mode 100644 scripts/firebase-admin/args.ts diff --git a/scripts/firebase-admin/args.ts b/scripts/firebase-admin/args.ts new file mode 100644 index 000000000..d9ebdddb6 --- /dev/null +++ b/scripts/firebase-admin/args.ts @@ -0,0 +1,43 @@ +import { z } from "zod" + +/** Thrown for invalid script arguments; the runner prints it without a stack trace. */ +export class ArgsError extends Error { + constructor(readonly issues: z.ZodIssue[]) { + super( + "Invalid arguments:\n" + + issues + .map(({ path: [arg, ...rest], message }) => { + // show 1-based positions, e.g. the 2nd value in --courts 193,abc + const item = rest.map(p => (typeof p === "number" ? p + 1 : p)) + const where = item.length ? ` (item ${item.join(".")})` : "" + return ` --${arg ?? "?"}${where}: ${message}` + }) + .join("\n") + ) + this.name = "ArgsError" + } +} + +/** A comma-separated list of general court numbers, e.g. --courts 192,193 */ +export const courtList = z + // yargs parses a single value like "193" as a number + .union([z.string(), z.number()]) + .transform(v => String(v).split(",")) + .pipe( + z.array( + z + .string() + .trim() + .min(1, "empty value, check for a stray comma") + .pipe(z.coerce.number().int().positive()) + ) + ) + +export function parseArgs( + schema: T, + args: unknown +): z.output { + const result = schema.safeParse(args) + if (!result.success) throw new ArgsError(result.error.issues) + return result.data +} diff --git a/scripts/firebase-admin/backfillOcpfFinance.ts b/scripts/firebase-admin/backfillOcpfFinance.ts index 05e7fd7cd..3e16cd4f8 100644 --- a/scripts/firebase-admin/backfillOcpfFinance.ts +++ b/scripts/firebase-admin/backfillOcpfFinance.ts @@ -1,14 +1,10 @@ import { z } from "zod" import { Script } from "./types" -import { parseArgs } from "./args" +import { courtList, parseArgs } from "./args" import { scrapeFinanceForCourt } from "functions/src/ocpf/scrapeOcpfFinance" const Args = z.object({ - // yargs parses a single value like "193" as a number - courts: z - .union([z.string(), z.number()]) - .transform(v => String(v).split(",").map(Number)) - .pipe(z.array(z.number().int().positive())) + courts: courtList }) export const script: Script = async ({ args }) => { diff --git a/scripts/firebase-admin/backfillOcpfMemberMapping.ts b/scripts/firebase-admin/backfillOcpfMemberMapping.ts index 274db753a..beedbc4f2 100644 --- a/scripts/firebase-admin/backfillOcpfMemberMapping.ts +++ b/scripts/firebase-admin/backfillOcpfMemberMapping.ts @@ -1,14 +1,10 @@ import { z } from "zod" import { Script } from "./types" -import { parseArgs } from "./args" +import { courtList, parseArgs } from "./args" import { matchMembersForCourt } from "functions/src/ocpf/matchOcpfMembers" const Args = z.object({ - // yargs parses a single value like "193" as a number - courts: z - .union([z.string(), z.number()]) - .transform(v => String(v).split(",").map(Number)) - .pipe(z.array(z.number().int().positive())) + courts: courtList }) export const script: Script = async ({ args }) => { diff --git a/scripts/firebase-admin/index.ts b/scripts/firebase-admin/index.ts index 37d3ca5f3..3c75bca84 100644 --- a/scripts/firebase-admin/index.ts +++ b/scripts/firebase-admin/index.ts @@ -66,6 +66,21 @@ yargs(hideBin(process.argv)) } }), async (args: unknown) => { + // A space in a list (--courts 192, 193) leaves 193 as a stray positional + // that yargs parses as a number + const stray = ((args as { argv?: unknown[] }).argv ?? []).filter( + a => typeof a !== "string" + ) + if (stray.length) { + console.error( + `Unexpected argument(s): ${stray.join(", ")}\n` + + `Comma-separated lists can't contain spaces, e.g. --courts 192,193 ` + + `(or quote them: --courts "192, 193")` + ) + process.exitCode = 1 + return + } + const { script: scriptName } = ScriptArgs.check(args) const context: ScriptContext = { ...init(args), args: args as any } const script: Script = require(`./${scriptName}`).script From ae28ed9a9940ef57157ab7cf49beb84777f68599 Mon Sep 17 00:00:00 2001 From: janicelichtman Date: Tue, 20 Oct 2026 21:15:35 -0400 Subject: [PATCH 11/15] quiet member-mapping backfill logs by default, add --verbose, print JSON summary Co-Authored-By: Claude Opus 5.5 --- functions/src/ocpf/matchOcpfMembers.ts | 32 ++++++++++------ .../backfillOcpfMemberMapping.ts | 37 +++++++++++-------- 2 files changed, 42 insertions(+), 27 deletions(-) diff --git a/functions/src/ocpf/matchOcpfMembers.ts b/functions/src/ocpf/matchOcpfMembers.ts index 53c119059..aa9d79f85 100644 --- a/functions/src/ocpf/matchOcpfMembers.ts +++ b/functions/src/ocpf/matchOcpfMembers.ts @@ -55,15 +55,20 @@ export const matchOcpfMembers = onRequestV2(async (req, res) => { // Matches a general court's MAPLE members to their OCPF filer record by last // name + branch, writing the result to that court's ocpfMemberMapping. The // current court is matched against active filers only (OCPF Closed_Date -// empty); historical courts drop that filter. -export async function matchMembersForCourt(court: number): Promise<{ +// empty); historical courts drop that filter. Pass quiet to suppress logging +// (e.g. when run from a CLI script that prints its own summary). +export async function matchMembersForCourt( + court: number, + { quiet = false }: { quiet?: boolean } = {} +): Promise<{ matched: number unmatched: OcpfMemberMappingFlagsEntry[] ambiguous: OcpfMemberMappingFlagsEntry[] }> { + const log = quiet ? silentLogger : logger const activeFilersOnly = court === currentGeneralCourt - const filers = await downloadAndParseFilers(activeFilersOnly) + const filers = await downloadAndParseFilers(activeFilersOnly, log) const members = await loadMembers(court) const mappingPath = `/generalCourts/${court}/ocpfMemberMappingConfig/ocpfMemberMapping` @@ -116,7 +121,7 @@ export async function matchMembersForCourt(court: number): Promise<{ // and another person with same last name is running for original office). } else if (candidates.length === 1 && firstNameMatches.length === 0) { ambiguous.push({ memberCode: member.MemberCode, name: member.Name }) - logger.warn( + log.warn( "Ambiguous OCPF match. Single last name match but first name did not align.", { court, @@ -132,7 +137,7 @@ export async function matchMembersForCourt(court: number): Promise<{ ) } else if (candidates.length === 0) { unmatched.push({ memberCode: member.MemberCode, name: member.Name }) - logger.warn("No OCPF match.", { + log.warn("No OCPF match.", { court, memberCode: member.MemberCode, name: member.Name, @@ -141,7 +146,7 @@ export async function matchMembersForCourt(court: number): Promise<{ }) } else { ambiguous.push({ memberCode: member.MemberCode, name: member.Name }) - logger.warn("Ambiguous OCPF match.", { + log.warn("Ambiguous OCPF match.", { court, memberCode: member.MemberCode, name: member.Name, @@ -164,7 +169,7 @@ export async function matchMembersForCourt(court: number): Promise<{ await db.doc(flagsPath).set(flags) const matched = Object.keys(mapping).length - logger.info("matchOcpfMembers complete", { + log.info("matchOcpfMembers complete", { court, matched, unmatched: unmatched.length, @@ -174,8 +179,13 @@ export async function matchMembersForCourt(court: number): Promise<{ return { matched, unmatched, ambiguous } } +type Log = Pick + +const silentLogger: Log = { info: () => {}, warn: () => {} } + async function downloadAndParseFilers( - activeFilersOnly: boolean + activeFilersOnly: boolean, + log: Log ): Promise { const response = await axios.get( "https://ocpf2.blob.core.windows.net/downloads/data2/ocpf-filers.zip", @@ -183,7 +193,7 @@ async function downloadAndParseFilers( ) const buffer = Buffer.from(response.data as ArrayBuffer) - logger.info("Downloaded ocpf-filers.zip", { + log.info("Downloaded ocpf-filers.zip", { status: response.status, contentType: response.headers["content-type"], bytes: buffer.length, @@ -204,7 +214,7 @@ async function downloadAndParseFilers( const lines = text.split(/\r?\n/) const rawHeaders = lines[0].split("\t").map(h => h.trim()) - logger.info("OCPF filers headers", { headers: rawHeaders }) + log.info("OCPF filers headers", { headers: rawHeaders }) const colIndex = buildColumnIndex(rawHeaders, [ "cpfId", @@ -241,7 +251,7 @@ async function downloadAndParseFilers( }) } - logger.info("Parsed active state legislators from OCPF", { + log.info("Parsed active state legislators from OCPF", { count: filers.length }) return filers diff --git a/scripts/firebase-admin/backfillOcpfMemberMapping.ts b/scripts/firebase-admin/backfillOcpfMemberMapping.ts index beedbc4f2..99c14b8f9 100644 --- a/scripts/firebase-admin/backfillOcpfMemberMapping.ts +++ b/scripts/firebase-admin/backfillOcpfMemberMapping.ts @@ -4,11 +4,14 @@ import { courtList, parseArgs } from "./args" import { matchMembersForCourt } from "functions/src/ocpf/matchOcpfMembers" const Args = z.object({ - courts: courtList + courts: courtList, + // Print matchMembersForCourt's detailed logs (download info, OCPF headers, + // per-member match warnings) + verbose: z.boolean().default(false) }) export const script: Script = async ({ args }) => { - const { courts } = parseArgs(Args, args) + const { courts, verbose } = parseArgs(Args, args) const succeeded: number[] = [] const failed: number[] = [] @@ -17,24 +20,26 @@ export const script: Script = async ({ args }) => { console.log(`\n── Court ${court} ──`) try { const { matched, unmatched, ambiguous } = await matchMembersForCourt( - court + court, + { quiet: !verbose } ) + // Same shape as the matchOcpfMembers HTTP response console.log( - `Court ${court}: matched ${matched}, unmatched ${unmatched.length}, ambiguous ${ambiguous.length}` - ) - if (unmatched.length > 0) { - console.log( - " unmatched:", - unmatched.map(m => `${m.memberCode} (${m.name})`).join(", ") - ) - } - if (ambiguous.length > 0) { - console.log( - " ambiguous:", - ambiguous.map(m => `${m.memberCode} (${m.name})`).join(", ") + JSON.stringify( + { + results: { + matched, + unmatched: unmatched.length, + ambiguous: ambiguous.length + }, + unmatched_members: unmatched, + ambiguous_members: ambiguous + }, + null, + 2 ) - } + ) succeeded.push(court) } catch (err) { console.error(`Court ${court} failed:`, err) From 6a010a731ecdd76a0865c97d2390926e8706662f Mon Sep 17 00:00:00 2001 From: janicelichtman Date: Wed, 21 Oct 2026 14:24:59 -0400 Subject: [PATCH 12/15] added comment for an edge case about name matching --- functions/src/ocpf/matchOcpfMembers.ts | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/functions/src/ocpf/matchOcpfMembers.ts b/functions/src/ocpf/matchOcpfMembers.ts index aa9d79f85..14888d05a 100644 --- a/functions/src/ocpf/matchOcpfMembers.ts +++ b/functions/src/ocpf/matchOcpfMembers.ts @@ -55,8 +55,20 @@ export const matchOcpfMembers = onRequestV2(async (req, res) => { // Matches a general court's MAPLE members to their OCPF filer record by last // name + branch, writing the result to that court's ocpfMemberMapping. The // current court is matched against active filers only (OCPF Closed_Date -// empty); historical courts drop that filter. Pass quiet to suppress logging +// empty); historical courts drop that filter. +// Pass quiet to suppress logging // (e.g. when run from a CLI script that prints its own summary). + +// NOTE: OCPF filer records reflect the filer's *current* name and office, so +// matching older courts can miss legislators whose record has changed since. +// Examples: +// - Chamber switch (House <-> Senate): branch is a hard filter, so they come +// up unmatched for the earlier court. +// - Last name change (e.g. Alyson M. Sullivan in court 192 is now filed as +// Alyson M. Sullivan-Almeida). Alyson M. Sullivan isn't found, so for court 192, +// they come up unmatched, or ambiguous if others share the old last name. +// Expected for older courts, not a bug; fix these by hand in ocpfMemberMapping. + export async function matchMembersForCourt( court: number, { quiet = false }: { quiet?: boolean } = {} @@ -87,9 +99,6 @@ export async function matchMembersForCourt( if (!branch || (branch !== "Senate" && branch !== "House")) continue - // Branch is a hard filter: a legislator who switched chambers (House <-> - // Senate) between courts will come up unmatched for the earlier court. - // Expected for older courts, not a bug. const lastNameAndBranchMatches = filers.filter( f => f.lastName.toLowerCase() === lastName.toLowerCase() && From 4d685405781acf07335f6446c3c4b857836d5b29 Mon Sep 17 00:00:00 2001 From: janicelichtman Date: Wed, 21 Oct 2026 17:13:31 -0400 Subject: [PATCH 13/15] add setOcpfMemberMapping script and manual flag so manual matches always win Co-Authored-By: Claude Opus 5.5 --- functions/src/ocpf/matchOcpfMembers.ts | 11 +- functions/src/ocpf/types.ts | 2 + .../firebase-admin/setOcpfMemberMapping.ts | 109 ++++++++++++++++++ 3 files changed, 119 insertions(+), 3 deletions(-) create mode 100644 scripts/firebase-admin/setOcpfMemberMapping.ts diff --git a/functions/src/ocpf/matchOcpfMembers.ts b/functions/src/ocpf/matchOcpfMembers.ts index 14888d05a..86c6a2dc8 100644 --- a/functions/src/ocpf/matchOcpfMembers.ts +++ b/functions/src/ocpf/matchOcpfMembers.ts @@ -67,7 +67,8 @@ export const matchOcpfMembers = onRequestV2(async (req, res) => { // - Last name change (e.g. Alyson M. Sullivan in court 192 is now filed as // Alyson M. Sullivan-Almeida). Alyson M. Sullivan isn't found, so for court 192, // they come up unmatched, or ambiguous if others share the old last name. -// Expected for older courts, not a bug; fix these by hand in ocpfMemberMapping. +// Expected for older courts, not a bug; fix these by hand in ocpfMemberMapping and +// set manual: true on the entry so later runs don't overwrite it. export async function matchMembersForCourt( court: number, @@ -114,14 +115,18 @@ export async function matchMembersForCourt( const candidates = firstNameMatches.length > 0 ? firstNameMatches : lastNameAndBranchMatches - if (firstNameMatches.length === 1) { + // Fixed by hand (manual: true). Always wins over automatic matching. + if (existingMapping[member.MemberCode]?.manual) { + continue + } else if (firstNameMatches.length === 1) { const entry: OcpfMemberMappingEntry = { cpfId: candidates[0].cpfId, name: member.Name } mapping[member.MemberCode] = entry - // Matching was likley fixed manually + // Matched on a previous run but not this one. Keep the existing entry + // rather than flagging. } else if (member.MemberCode in existingMapping) { continue diff --git a/functions/src/ocpf/types.ts b/functions/src/ocpf/types.ts index a935cd8ef..17ded9014 100644 --- a/functions/src/ocpf/types.ts +++ b/functions/src/ocpf/types.ts @@ -10,9 +10,11 @@ export interface OcpfFilerRow { // Firestore: /generalCourts/{court}/ocpfMemberMappingConfig/ocpfMemberMapping // memberCode → { cpfId, name }, e.g. { "SND1": { cpfId: 15031, name: "Sal N. DiDomenico" } } +// Set manual: true on entries fixed by hand so the matcher never overwrites them. export interface OcpfMemberMappingEntry { cpfId: number name: string + manual?: boolean } export type OcpfMemberMapping = Record diff --git a/scripts/firebase-admin/setOcpfMemberMapping.ts b/scripts/firebase-admin/setOcpfMemberMapping.ts new file mode 100644 index 000000000..528959e42 --- /dev/null +++ b/scripts/firebase-admin/setOcpfMemberMapping.ts @@ -0,0 +1,109 @@ +import { z } from "zod" +import { Script } from "./types" +import { parseArgs } from "./args" +import { MemberContent } from "functions/src/members/types" +import { + OcpfMemberMapping, + OcpfMemberMappingEntry, + OcpfMemberMappingFlags, + OcpfMemberMappingFlagsEntry +} from "functions/src/ocpf/types" + +// Manually maps one MAPLE member to their OCPF filer, for members that +// matchMembersForCourt flagged as unmatched or ambiguous. The entry is written +// with manual: true so later matcher runs leave it alone, and the member is +// removed from the court's flags doc. +// +// yarn firebase-admin run-script setOcpfMemberMapping --env dev \ +// --court 194 --memberCode SND1 --cpfId 15031 + +// yargs parses numeric values like "194" as numbers, and a flag given with no +// value as true +const stringOrNumber = z.union([z.string(), z.number()], { + errorMap: (_, ctx) => ({ + message: ctx.data === undefined ? "required" : "requires a value" + }) +}) + +const positiveInt = stringOrNumber.pipe( + z.coerce + .number({ invalid_type_error: "must be a number" }) + .int("must be a whole number") + .positive("must be greater than 0") +) + +const Args = z.object({ + court: positiveInt, + memberCode: stringOrNumber.pipe( + z.coerce.string().trim().min(1, "requires a value") + ), + cpfId: positiveInt +}) + +export const script: Script = async ({ db, args }) => { + const { court, memberCode, cpfId } = parseArgs(Args, args) + + const memberPath = `/generalCourts/${court}/members/${memberCode}` + const mappingPath = `/generalCourts/${court}/ocpfMemberMappingConfig/ocpfMemberMapping` + const flagsPath = `/generalCourts/${court}/ocpfMemberMappingConfig/ocpfMemberMappingFlags` + + const result = await db.runTransaction(async tx => { + const [memberDoc, mappingDoc, flagsDoc] = await Promise.all([ + tx.get(db.doc(memberPath)), + tx.get(db.doc(mappingPath)), + tx.get(db.doc(flagsPath)) + ]) + + const member = memberDoc.data()?.content as MemberContent | undefined + if (!member) return { found: false as const } + + const previous = (mappingDoc.data() as OcpfMemberMapping | undefined)?.[ + memberCode + ] + const entry: OcpfMemberMappingEntry = { + cpfId, + name: member.Name, + manual: true + } + tx.set(db.doc(mappingPath), { [memberCode]: entry }, { merge: true }) + + const flags = flagsDoc.data() as OcpfMemberMappingFlags | undefined + const removedFrom: string[] = [] + if (flags) { + const without = (list: OcpfMemberMappingFlagsEntry[], name: string) => { + const kept = list.filter(f => f.memberCode !== memberCode) + if (kept.length !== list.length) removedFrom.push(name) + return kept + } + const updated: OcpfMemberMappingFlags = { + unmatched: without(flags.unmatched ?? [], "unmatched"), + ambiguous: without(flags.ambiguous ?? [], "ambiguous") + } + if (removedFrom.length) tx.set(db.doc(flagsPath), updated) + } + + return { found: true as const, entry, previous, removedFrom } + }) + + if (!result.found) { + console.error( + `No member ${memberCode} in court ${court} (${memberPath}). Nothing written.` + ) + process.exitCode = 1 + return + } + + console.log( + JSON.stringify( + { + court, + memberCode, + previous: result.previous ?? null, + written: result.entry, + removedFromFlags: result.removedFrom + }, + null, + 2 + ) + ) +} From a18aeded79d421323cea76ee4b6118d90827c820 Mon Sep 17 00:00:00 2001 From: janicelichtman Date: Tue, 27 Oct 2026 15:37:54 -0400 Subject: [PATCH 14/15] fail member matching for a court with no members instead of writing empty mapping docs Co-Authored-By: Claude Opus 5.5 --- functions/src/ocpf/matchOcpfMembers.ts | 13 ++++++++++++- scripts/firebase-admin/backfillOcpfMemberMapping.ts | 10 ++++++++-- 2 files changed, 20 insertions(+), 3 deletions(-) diff --git a/functions/src/ocpf/matchOcpfMembers.ts b/functions/src/ocpf/matchOcpfMembers.ts index 86c6a2dc8..24891ac6c 100644 --- a/functions/src/ocpf/matchOcpfMembers.ts +++ b/functions/src/ocpf/matchOcpfMembers.ts @@ -70,6 +70,13 @@ export const matchOcpfMembers = onRequestV2(async (req, res) => { // Expected for older courts, not a bug; fix these by hand in ocpfMemberMapping and // set manual: true on the entry so later runs don't overwrite it. +/** Thrown for a court with no members (e.g. a mistyped court number). */ +export class NoMembersError extends Error { + constructor(court: number) { + super(`No members found for court ${court}. Nothing written.`) + } +} + export async function matchMembersForCourt( court: number, { quiet = false }: { quiet?: boolean } = {} @@ -81,8 +88,12 @@ export async function matchMembersForCourt( const log = quiet ? silentLogger : logger const activeFilersOnly = court === currentGeneralCourt - const filers = await downloadAndParseFilers(activeFilersOnly, log) + // Check members first so an unknown court fails before anything is written + // (writing the mapping docs would make the court appear in Firestore) const members = await loadMembers(court) + if (members.length === 0) throw new NoMembersError(court) + + const filers = await downloadAndParseFilers(activeFilersOnly, log) const mappingPath = `/generalCourts/${court}/ocpfMemberMappingConfig/ocpfMemberMapping` const flagsPath = `/generalCourts/${court}/ocpfMemberMappingConfig/ocpfMemberMappingFlags` diff --git a/scripts/firebase-admin/backfillOcpfMemberMapping.ts b/scripts/firebase-admin/backfillOcpfMemberMapping.ts index 99c14b8f9..559b60d78 100644 --- a/scripts/firebase-admin/backfillOcpfMemberMapping.ts +++ b/scripts/firebase-admin/backfillOcpfMemberMapping.ts @@ -1,7 +1,10 @@ import { z } from "zod" import { Script } from "./types" import { courtList, parseArgs } from "./args" -import { matchMembersForCourt } from "functions/src/ocpf/matchOcpfMembers" +import { + matchMembersForCourt, + NoMembersError +} from "functions/src/ocpf/matchOcpfMembers" const Args = z.object({ courts: courtList, @@ -42,7 +45,10 @@ export const script: Script = async ({ args }) => { ) succeeded.push(court) } catch (err) { - console.error(`Court ${court} failed:`, err) + // Expected for a bad court number, so skip the stack trace + if (err instanceof NoMembersError) + console.error(`Court ${court} failed: ${err}`) + else console.error(`Court ${court} failed:`, err) failed.push(court) } } From 6777f1bc1685bd41ff978ca01744819415d695f1 Mon Sep 17 00:00:00 2001 From: janicelichtman Date: Tue, 27 Oct 2026 15:48:34 -0400 Subject: [PATCH 15/15] print a one-line error instead of a stack trace for an unknown court in the finance backfill Co-Authored-By: Claude Opus 5.5 --- functions/src/ocpf/scrapeOcpfFinance.ts | 11 ++++++++--- scripts/firebase-admin/backfillOcpfFinance.ts | 10 ++++++++-- 2 files changed, 16 insertions(+), 5 deletions(-) diff --git a/functions/src/ocpf/scrapeOcpfFinance.ts b/functions/src/ocpf/scrapeOcpfFinance.ts index 75b875947..b2b80fd11 100644 --- a/functions/src/ocpf/scrapeOcpfFinance.ts +++ b/functions/src/ocpf/scrapeOcpfFinance.ts @@ -299,14 +299,19 @@ export const scrapeOcpfFinanceManual = onRequestV2( // ── Scrape run (shared by nightly, manual and backfill triggers) ────────────── // Set cpfId to run a single member; omit to run all members. +/** Thrown for a court not in generalCourts (e.g. a mistyped court number). */ +export class UnknownCourtError extends Error { + constructor(court: number) { + super(`No general court entry for court ${court}. Nothing written.`) + } +} + export async function scrapeFinanceForCourt( court: number, options: { cpfId?: number } = {} ): Promise<{ processed: number; years: string[] }> { const courtInfo = generalCourts[court] - if (!courtInfo) { - throw new Error(`No general court entry for court ${court}`) - } + if (!courtInfo) throw new UnknownCourtError(court) const years = [courtInfo.FirstYear, courtInfo.SecondYear].map(String) // ── A. Load member mapping ───────────────────────────────────────────── diff --git a/scripts/firebase-admin/backfillOcpfFinance.ts b/scripts/firebase-admin/backfillOcpfFinance.ts index 3e16cd4f8..16675731f 100644 --- a/scripts/firebase-admin/backfillOcpfFinance.ts +++ b/scripts/firebase-admin/backfillOcpfFinance.ts @@ -1,7 +1,10 @@ import { z } from "zod" import { Script } from "./types" import { courtList, parseArgs } from "./args" -import { scrapeFinanceForCourt } from "functions/src/ocpf/scrapeOcpfFinance" +import { + scrapeFinanceForCourt, + UnknownCourtError +} from "functions/src/ocpf/scrapeOcpfFinance" const Args = z.object({ courts: courtList @@ -24,7 +27,10 @@ export const script: Script = async ({ args }) => { ) succeeded.push(court) } catch (err) { - console.error(`Court ${court} failed:`, err) + // Expected for a bad court number, so skip the stack trace + if (err instanceof UnknownCourtError) + console.error(`Court ${court} failed: ${err}`) + else console.error(`Court ${court} failed:`, err) failed.push(court) } }