diff --git a/functions/src/index.ts b/functions/src/index.ts index 5a217d6f5..788b1857e 100644 --- a/functions/src/index.ts +++ b/functions/src/index.ts @@ -64,7 +64,7 @@ export { scrapeElections } from "./legislators" export { transcriptionV2 } from "./webhooks" -export { matchOcpfMembersV2 } from "./ocpf/matchOcpfMembers" +export { matchOcpfMembers } from "./ocpf/matchOcpfMembers" export { scrapeOcpfFinanceNightly, scrapeOcpfFinanceManual diff --git a/functions/src/ocpf/matchOcpfMembers.ts b/functions/src/ocpf/matchOcpfMembers.ts index 169f2309d..24891ac6c 100644 --- a/functions/src/ocpf/matchOcpfMembers.ts +++ b/functions/src/ocpf/matchOcpfMembers.ts @@ -14,7 +14,7 @@ import { OcpfMemberMappingFlagsEntry } from "./types" -export const matchOcpfMembersV2 = onRequestV2(async (req, res) => { +export const matchOcpfMembers = onRequestV2(async (req, res) => { if (req.method !== "POST") { res.status(405).send("Method Not Allowed. Use POST.") return @@ -37,10 +37,68 @@ export const matchOcpfMembersV2 = onRequestV2(async (req, res) => { } } - const filers = await downloadAndParseFilers() - const members = await loadMembers() + const { matched, unmatched, ambiguous } = await matchMembersForCourt( + currentGeneralCourt + ) + + res.status(200).json({ + results: { + matched, + unmatched: unmatched.length, + ambiguous: ambiguous.length + }, + unmatched_members: unmatched, + ambiguous_members: ambiguous + }) +}) + +// Matches a general court's MAPLE members to their OCPF filer record by last +// name + branch, writing the result to that court's ocpfMemberMapping. The +// current court is matched against active filers only (OCPF Closed_Date +// empty); historical courts drop that filter. +// Pass quiet to suppress logging +// (e.g. when run from a CLI script that prints its own summary). + +// NOTE: OCPF filer records reflect the filer's *current* name and office, so +// matching older courts can miss legislators whose record has changed since. +// Examples: +// - Chamber switch (House <-> Senate): branch is a hard filter, so they come +// up unmatched for the earlier court. +// - Last name change (e.g. Alyson M. Sullivan in court 192 is now filed as +// Alyson M. Sullivan-Almeida). Alyson M. Sullivan isn't found, so for court 192, +// they come up unmatched, or ambiguous if others share the old last name. +// Expected for older courts, not a bug; fix these by hand in ocpfMemberMapping and +// set manual: true on the entry so later runs don't overwrite it. + +/** Thrown for a court with no members (e.g. a mistyped court number). */ +export class NoMembersError extends Error { + constructor(court: number) { + super(`No members found for court ${court}. Nothing written.`) + } +} + +export async function matchMembersForCourt( + court: number, + { quiet = false }: { quiet?: boolean } = {} +): Promise<{ + matched: number + unmatched: OcpfMemberMappingFlagsEntry[] + ambiguous: OcpfMemberMappingFlagsEntry[] +}> { + const log = quiet ? silentLogger : logger + const activeFilersOnly = court === currentGeneralCourt + + // Check members first so an unknown court fails before anything is written + // (writing the mapping docs would make the court appear in Firestore) + const members = await loadMembers(court) + if (members.length === 0) throw new NoMembersError(court) - const existingMappingDoc = await db.doc("/config/ocpfMemberMapping").get() + const filers = await downloadAndParseFilers(activeFilersOnly, log) + + const mappingPath = `/generalCourts/${court}/ocpfMemberMappingConfig/ocpfMemberMapping` + const flagsPath = `/generalCourts/${court}/ocpfMemberMappingConfig/ocpfMemberMappingFlags` + + const existingMappingDoc = await db.doc(mappingPath).get() const existingMapping = (existingMappingDoc.data() ?? {}) as OcpfMemberMapping const mapping: OcpfMemberMapping = {} @@ -68,14 +126,18 @@ export const matchOcpfMembersV2 = onRequestV2(async (req, res) => { const candidates = firstNameMatches.length > 0 ? firstNameMatches : lastNameAndBranchMatches - if (firstNameMatches.length === 1) { + // Fixed by hand (manual: true). Always wins over automatic matching. + if (existingMapping[member.MemberCode]?.manual) { + continue + } else if (firstNameMatches.length === 1) { const entry: OcpfMemberMappingEntry = { cpfId: candidates[0].cpfId, name: member.Name } mapping[member.MemberCode] = entry - // Matching was likley fixed manually + // Matched on a previous run but not this one. Keep the existing entry + // rather than flagging. } else if (member.MemberCode in existingMapping) { continue @@ -84,9 +146,10 @@ export const matchOcpfMembersV2 = onRequestV2(async (req, res) => { // and another person with same last name is running for original office). } else if (candidates.length === 1 && firstNameMatches.length === 0) { ambiguous.push({ memberCode: member.MemberCode, name: member.Name }) - logger.warn( + log.warn( "Ambiguous OCPF match. Single last name match but first name did not align.", { + court, memberCode: member.MemberCode, name: member.Name, district: member.District, @@ -99,7 +162,8 @@ export const matchOcpfMembersV2 = onRequestV2(async (req, res) => { ) } else if (candidates.length === 0) { unmatched.push({ memberCode: member.MemberCode, name: member.Name }) - logger.warn("No OCPF match.", { + log.warn("No OCPF match.", { + court, memberCode: member.MemberCode, name: member.Name, district: member.District, @@ -107,7 +171,8 @@ export const matchOcpfMembersV2 = onRequestV2(async (req, res) => { }) } else { ambiguous.push({ memberCode: member.MemberCode, name: member.Name }) - logger.warn("Ambiguous OCPF match.", { + log.warn("Ambiguous OCPF match.", { + court, memberCode: member.MemberCode, name: member.Name, district: member.District, @@ -125,34 +190,35 @@ export const matchOcpfMembersV2 = onRequestV2(async (req, res) => { const flags: OcpfMemberMappingFlags = { unmatched, ambiguous } - await db.doc("/config/ocpfMemberMapping").set(mapping, { merge: true }) - await db.doc("/config/ocpfMemberMappingFlags").set(flags) + await db.doc(mappingPath).set(mapping, { merge: true }) + await db.doc(flagsPath).set(flags) - logger.info("matchOcpfMembers complete", { - matched: Object.keys(mapping).length, + const matched = Object.keys(mapping).length + log.info("matchOcpfMembers complete", { + court, + matched, unmatched: unmatched.length, ambiguous: ambiguous.length }) - res.status(200).json({ - results: { - matched: Object.keys(mapping).length, - unmatched: unmatched.length, - ambiguous: ambiguous.length - }, - unmatched_members: unmatched, - ambiguous_members: ambiguous - }) -}) + return { matched, unmatched, ambiguous } +} + +type Log = Pick + +const silentLogger: Log = { info: () => {}, warn: () => {} } -async function downloadAndParseFilers(): Promise { +async function downloadAndParseFilers( + activeFilersOnly: boolean, + log: Log +): Promise { const response = await axios.get( "https://ocpf2.blob.core.windows.net/downloads/data2/ocpf-filers.zip", { responseType: "arraybuffer" } ) const buffer = Buffer.from(response.data as ArrayBuffer) - logger.info("Downloaded ocpf-filers.zip", { + log.info("Downloaded ocpf-filers.zip", { status: response.status, contentType: response.headers["content-type"], bytes: buffer.length, @@ -173,7 +239,7 @@ async function downloadAndParseFilers(): Promise { const lines = text.split(/\r?\n/) const rawHeaders = lines[0].split("\t").map(h => h.trim()) - logger.info("OCPF filers headers", { headers: rawHeaders }) + log.info("OCPF filers headers", { headers: rawHeaders }) const colIndex = buildColumnIndex(rawHeaders, [ "cpfId", @@ -197,7 +263,7 @@ async function downloadAndParseFilers(): Promise { const closedDate = col(cols, colIndex.closedDate) const officeSought = col(cols, colIndex.officeSought) - if (closedDate !== "") continue + if (activeFilersOnly && closedDate !== "") continue if (officeSought !== "Senate" && officeSought !== "House") continue filers.push({ @@ -210,7 +276,7 @@ async function downloadAndParseFilers(): Promise { }) } - logger.info("Parsed active state legislators from OCPF", { + log.info("Parsed active state legislators from OCPF", { count: filers.length }) return filers @@ -254,10 +320,8 @@ function buildColumnIndex( return index } -async function loadMembers(): Promise { - const snapshot = await db - .collection(`/generalCourts/${currentGeneralCourt}/members`) - .get() +async function loadMembers(court: number): Promise { + const snapshot = await db.collection(`/generalCourts/${court}/members`).get() return snapshot.docs .map(doc => { diff --git a/functions/src/ocpf/scrapeOcpfFinance.ts b/functions/src/ocpf/scrapeOcpfFinance.ts index 10c9ae8ad..b2b80fd11 100644 --- a/functions/src/ocpf/scrapeOcpfFinance.ts +++ b/functions/src/ocpf/scrapeOcpfFinance.ts @@ -19,12 +19,6 @@ import { const OCPF_BASE_URL = "https://ocpf2.blob.core.windows.net/downloads/data2" -const currentCourt = generalCourts[currentGeneralCourt] -if (!currentCourt) { - throw new Error(`No general court entry for court ${currentGeneralCourt}`) -} -const YEARS = [currentCourt.FirstYear, currentCourt.SecondYear].map(String) - // How each OCPF report type (reports.txt Report_Type_ID) is used: // totals — Receipts_Total/Expenditures_Total feed totalRaised/totalSpent; // items and Receipts_Unitemized_Total are also used @@ -182,7 +176,7 @@ function sumBreakdowns(breakdowns: MutableBreakdown[]): MutableBreakdown { } } -function newAccumulator(cpfId: number): MemberAccumulator { +function newAccumulator(cpfId: number, years: string[]): MemberAccumulator { const yearInit = (): YearAccumulator => ({ totalRaised: 0, totalSpent: 0, @@ -202,10 +196,10 @@ function newAccumulator(cpfId: number): MemberAccumulator { union: emptyEntry(), unitemized: { amount: 0 } }, - years: Object.fromEntries(YEARS.map(y => [y, yearInit()])), - yearEndCheck: Object.fromEntries(YEARS.map(y => [y, null])), + years: Object.fromEntries(years.map(y => [y, yearInit()])), + yearEndCheck: Object.fromEntries(years.map(y => [y, null])), reportTotals: Object.fromEntries( - YEARS.map(y => [y, { receipts: 0, expenditures: 0 }]) + years.map(y => [y, { receipts: 0, expenditures: 0 }]) ), pendingNonContributionReceipts: [] } @@ -248,7 +242,7 @@ export const scrapeOcpfFinanceNightly = onSchedule( retryCount: 1 }, async () => { - await runOcpfFinanceScrape() + await scrapeFinanceForCourt(currentGeneralCourt) } ) @@ -295,23 +289,38 @@ export const scrapeOcpfFinanceManual = onRequestV2( return } - const results = await runOcpfFinanceScrape({ cpfId: query.data.cpfId }) + const results = await scrapeFinanceForCourt(currentGeneralCourt, { + cpfId: query.data.cpfId + }) res.status(200).json({ results }) } ) -// ── Scrape run (shared by nightly and manual triggers) ──────────────────────── +// ── Scrape run (shared by nightly, manual and backfill triggers) ────────────── // Set cpfId to run a single member; omit to run all members. -async function runOcpfFinanceScrape( +/** Thrown for a court not in generalCourts (e.g. a mistyped court number). */ +export class UnknownCourtError extends Error { + constructor(court: number) { + super(`No general court entry for court ${court}. Nothing written.`) + } +} + +export async function scrapeFinanceForCourt( + court: number, options: { cpfId?: number } = {} ): Promise<{ processed: number; years: string[] }> { + const courtInfo = generalCourts[court] + if (!courtInfo) throw new UnknownCourtError(court) + const years = [courtInfo.FirstYear, courtInfo.SecondYear].map(String) + // ── A. Load member mapping ───────────────────────────────────────────── - const mappingDoc = await db.doc("/config/ocpfMemberMapping").get() + const mappingPath = `/generalCourts/${court}/ocpfMemberMappingConfig/ocpfMemberMapping` + const mappingDoc = await db.doc(mappingPath).get() if (!mappingDoc.exists) { - logger.warn( - "config/ocpfMemberMapping not found; no members will be processed" - ) + logger.warn(`${mappingPath} not found; no members will be processed`, { + court + }) } const mapping = (mappingDoc.data() ?? {}) as OcpfMemberMapping @@ -335,6 +344,7 @@ async function runOcpfFinanceScrape( } logger.info("Loaded member mapping", { + court, totalMembers: Object.keys(mapping).length, activeInRun: cpfIdToMemberCode.size }) @@ -348,20 +358,21 @@ async function runOcpfFinanceScrape( // items are read for in-kind records only const inKindOnlyReportIds = new Set() - for (const year of YEARS) { + for (const year of years) { const url = `${OCPF_BASE_URL}/ocpf-${year}-reports.zip` - logger.info(`Downloading ${url}`) + logger.info(`Downloading ${url}`, { court }) const buf = await downloadBuffer(url) await parseReports( buf, year, + years, cpfIdToMemberCode, accumulators, reportIdToMemberCode, inKindOnlyReportIds ) - logger.info(`Streaming report-items for ${year}`) + logger.info(`Streaming report-items for ${year}`, { court }) await streamReportItems( buf, reportIdToMemberCode, @@ -418,7 +429,7 @@ async function runOcpfFinanceScrape( // (Deposit Report items, before deduction of fees). That gap is tracked // via record type 319 (breakdown.processingFees). for (const [memberCode, acc] of accumulators) { - for (const year of YEARS) { + for (const year of years) { const check = acc.yearEndCheck[year] if (!check) continue const summedRaised = acc.reportTotals[year]?.receipts ?? 0 @@ -429,6 +440,7 @@ async function runOcpfFinanceScrape( logger.warn( "Year-end totals mismatch — investigate periodic report accumulation", { + court, memberCode, year, yearEnd: { @@ -453,9 +465,7 @@ async function runOcpfFinanceScrape( const batch = db.batch() for (const [memberCode, acc] of accumulators) { - const doc = db.doc( - `/generalCourts/${currentGeneralCourt}/membersFinance/${memberCode}` - ) + const doc = db.doc(`/generalCourts/${court}/membersFinance/${memberCode}`) const yearAccs = Object.values(acc.years) const data: MembersFinance = { ocpfCpfId: acc.cpfId, @@ -496,11 +506,12 @@ async function runOcpfFinanceScrape( await batch.commit() logger.info("scrapeOcpfFinance complete", { + court, processed: accumulators.size, - years: YEARS + years }) - return { processed: accumulators.size, years: YEARS } + return { processed: accumulators.size, years } } // ── Helpers ─────────────────────────────────────────────────────────────────── @@ -564,6 +575,7 @@ function col(cols: string[], idx: number): string { async function parseReports( buf: Buffer, year: string, + years: string[], cpfIdToMemberCode: Map, accumulators: Map, reportIdToMemberCode: Map, @@ -603,7 +615,7 @@ async function parseReports( let acc = accumulators.get(memberCode) if (!acc) { - acc = newAccumulator(cpfId) + acc = newAccumulator(cpfId, years) accumulators.set(memberCode, acc) } @@ -701,6 +713,7 @@ async function streamReportItems( let idx: Record = {} let processed = 0 let skipped = 0 + const warnedRecordTypeIds = new Set() for await (const line of rl) { if (!line.trim()) continue @@ -744,7 +757,15 @@ async function streamReportItems( col(cols, idx.zip) ) - accumulateItem(acc, recordTypeId, amount, dateMs, contributor, year) + accumulateItem( + acc, + recordTypeId, + amount, + dateMs, + contributor, + year, + warnedRecordTypeIds + ) processed++ } @@ -759,6 +780,12 @@ async function streamReportItems( // these are the itemized contributions behind totalRaised. const CONTRIBUTION_RECORD_TYPE_IDS = new Set([201, 202, 203]) +// Record types whose amounts are already counted elsewhere and are correctly +// excluded here — not unclassified, just sourced differently: 205 (Bank +// Interest) and 220 (Aggregated un-itemized) totals come from reports.txt, +// not report-items.txt. +const KNOWN_IGNORED_RECORD_TYPE_IDS = new Set([205, 220]) + // OCPF has no contributor ID, so a contributor is identified by // record type + name + first name + 5-digit ZIP. Committees and unions leave // First_Name blank and put the full name in Name. Record type is part of the key so an @@ -786,7 +813,8 @@ function accumulateItem( amount: number, dateMs: number, contributor: string | null, - year: string + year: string, + warnedRecordTypeIds: Set ): void { const addTo = (entry: MutableBreakdownEntry) => { entry.count++ @@ -853,8 +881,17 @@ function accumulateItem( case 319: // Payment-processor fee (see breakdown.processingFees doc comment) addTo(yb.processingFees) break - // 205 (Bank Interest) and 220 (Aggregated un-itemized) totals sourced from reports.txt, not items default: + if ( + !KNOWN_IGNORED_RECORD_TYPE_IDS.has(recordTypeId) && + !warnedRecordTypeIds.has(recordTypeId) + ) { + warnedRecordTypeIds.add(recordTypeId) + logger.warn( + "Unclassified OCPF record type — no amount counted; classify it in accumulateItem", + { recordTypeId, year } + ) + } break } } diff --git a/functions/src/ocpf/types.ts b/functions/src/ocpf/types.ts index 37f639328..17ded9014 100644 --- a/functions/src/ocpf/types.ts +++ b/functions/src/ocpf/types.ts @@ -8,11 +8,13 @@ export interface OcpfFilerRow { closedDate: string // empty string = active } -// Firestore: /config/ocpfMemberMapping +// Firestore: /generalCourts/{court}/ocpfMemberMappingConfig/ocpfMemberMapping // memberCode → { cpfId, name }, e.g. { "SND1": { cpfId: 15031, name: "Sal N. DiDomenico" } } +// Set manual: true on entries fixed by hand so the matcher never overwrites them. export interface OcpfMemberMappingEntry { cpfId: number name: string + manual?: boolean } export type OcpfMemberMapping = Record @@ -22,7 +24,7 @@ export interface OcpfMemberMappingFlagsEntry { name: string } -// Firestore: /config/ocpfMemberMappingFlags +// Firestore: /generalCourts/{court}/ocpfMemberMappingConfig/ocpfMemberMappingFlags export interface OcpfMemberMappingFlags { unmatched: OcpfMemberMappingFlagsEntry[] ambiguous: OcpfMemberMappingFlagsEntry[] diff --git a/functions/src/shared/constants.ts b/functions/src/shared/constants.ts index ff375ec22..1857947cf 100644 --- a/functions/src/shared/constants.ts +++ b/functions/src/shared/constants.ts @@ -23,6 +23,24 @@ export const generalCourts: Record = { Number: 192, FirstYear: 2021, SecondYear: 2022 + }, + 191: { + Name: "191st (2019 - 2020)", + Number: 191, + FirstYear: 2019, + SecondYear: 2020 + }, + 190: { + Name: "190th (2017 - 2018)", + Number: 190, + FirstYear: 2017, + SecondYear: 2018 + }, + 189: { + Name: "189th (2015 - 2016)", + Number: 189, + FirstYear: 2015, + SecondYear: 2016 } } diff --git a/scripts/firebase-admin/args.ts b/scripts/firebase-admin/args.ts new file mode 100644 index 000000000..d9ebdddb6 --- /dev/null +++ b/scripts/firebase-admin/args.ts @@ -0,0 +1,43 @@ +import { z } from "zod" + +/** Thrown for invalid script arguments; the runner prints it without a stack trace. */ +export class ArgsError extends Error { + constructor(readonly issues: z.ZodIssue[]) { + super( + "Invalid arguments:\n" + + issues + .map(({ path: [arg, ...rest], message }) => { + // show 1-based positions, e.g. the 2nd value in --courts 193,abc + const item = rest.map(p => (typeof p === "number" ? p + 1 : p)) + const where = item.length ? ` (item ${item.join(".")})` : "" + return ` --${arg ?? "?"}${where}: ${message}` + }) + .join("\n") + ) + this.name = "ArgsError" + } +} + +/** A comma-separated list of general court numbers, e.g. --courts 192,193 */ +export const courtList = z + // yargs parses a single value like "193" as a number + .union([z.string(), z.number()]) + .transform(v => String(v).split(",")) + .pipe( + z.array( + z + .string() + .trim() + .min(1, "empty value, check for a stray comma") + .pipe(z.coerce.number().int().positive()) + ) + ) + +export function parseArgs( + schema: T, + args: unknown +): z.output { + const result = schema.safeParse(args) + if (!result.success) throw new ArgsError(result.error.issues) + return result.data +} diff --git a/scripts/firebase-admin/backfillOcpfFinance.ts b/scripts/firebase-admin/backfillOcpfFinance.ts new file mode 100644 index 000000000..16675731f --- /dev/null +++ b/scripts/firebase-admin/backfillOcpfFinance.ts @@ -0,0 +1,41 @@ +import { z } from "zod" +import { Script } from "./types" +import { courtList, parseArgs } from "./args" +import { + scrapeFinanceForCourt, + UnknownCourtError +} from "functions/src/ocpf/scrapeOcpfFinance" + +const Args = z.object({ + courts: courtList +}) + +export const script: Script = async ({ args }) => { + const { courts } = parseArgs(Args, args) + + const succeeded: number[] = [] + const failed: number[] = [] + + for (const court of courts) { + console.log(`\n── Court ${court} ──`) + try { + const { processed, years } = await scrapeFinanceForCourt(court) + console.log( + `Court ${court}: processed ${processed} members for years ${years.join( + ", " + )}` + ) + succeeded.push(court) + } catch (err) { + // Expected for a bad court number, so skip the stack trace + if (err instanceof UnknownCourtError) + console.error(`Court ${court} failed: ${err}`) + else console.error(`Court ${court} failed:`, err) + failed.push(court) + } + } + + console.log("\n── Summary ──") + console.log(`Succeeded: ${succeeded.join(", ") || "none"}`) + console.log(`Failed: ${failed.join(", ") || "none"}`) +} diff --git a/scripts/firebase-admin/backfillOcpfMemberMapping.ts b/scripts/firebase-admin/backfillOcpfMemberMapping.ts new file mode 100644 index 000000000..559b60d78 --- /dev/null +++ b/scripts/firebase-admin/backfillOcpfMemberMapping.ts @@ -0,0 +1,59 @@ +import { z } from "zod" +import { Script } from "./types" +import { courtList, parseArgs } from "./args" +import { + matchMembersForCourt, + NoMembersError +} from "functions/src/ocpf/matchOcpfMembers" + +const Args = z.object({ + courts: courtList, + // Print matchMembersForCourt's detailed logs (download info, OCPF headers, + // per-member match warnings) + verbose: z.boolean().default(false) +}) + +export const script: Script = async ({ args }) => { + const { courts, verbose } = parseArgs(Args, args) + + const succeeded: number[] = [] + const failed: number[] = [] + + for (const court of courts) { + console.log(`\n── Court ${court} ──`) + try { + const { matched, unmatched, ambiguous } = await matchMembersForCourt( + court, + { quiet: !verbose } + ) + + // Same shape as the matchOcpfMembers HTTP response + console.log( + JSON.stringify( + { + results: { + matched, + unmatched: unmatched.length, + ambiguous: ambiguous.length + }, + unmatched_members: unmatched, + ambiguous_members: ambiguous + }, + null, + 2 + ) + ) + succeeded.push(court) + } catch (err) { + // Expected for a bad court number, so skip the stack trace + if (err instanceof NoMembersError) + console.error(`Court ${court} failed: ${err}`) + else console.error(`Court ${court} failed:`, err) + failed.push(court) + } + } + + console.log("\n── Summary ──") + console.log(`Succeeded: ${succeeded.join(", ") || "none"}`) + console.log(`Failed: ${failed.join(", ") || "none"}`) +} diff --git a/scripts/firebase-admin/index.ts b/scripts/firebase-admin/index.ts index 81e18ef76..3c75bca84 100644 --- a/scripts/firebase-admin/index.ts +++ b/scripts/firebase-admin/index.ts @@ -5,6 +5,7 @@ import { hideBin } from "yargs/helpers" import { runAgainstEmulators, runAgainstProject } from "../configure" import { Context, Script, ScriptContext } from "./types" import { listAllUsers } from "./list-all-users" +import { ArgsError } from "./args" const Args = Record({ env: Union(L("local"), L("dev"), L("prod")), @@ -65,11 +66,32 @@ yargs(hideBin(process.argv)) } }), async (args: unknown) => { + // A space in a list (--courts 192, 193) leaves 193 as a stray positional + // that yargs parses as a number + const stray = ((args as { argv?: unknown[] }).argv ?? []).filter( + a => typeof a !== "string" + ) + if (stray.length) { + console.error( + `Unexpected argument(s): ${stray.join(", ")}\n` + + `Comma-separated lists can't contain spaces, e.g. --courts 192,193 ` + + `(or quote them: --courts "192, 193")` + ) + process.exitCode = 1 + return + } + const { script: scriptName } = ScriptArgs.check(args) const context: ScriptContext = { ...init(args), args: args as any } const script: Script = require(`./${scriptName}`).script - await script(context) + try { + await script(context) + } catch (err) { + if (!(err instanceof ArgsError)) throw err + console.error(err.message) + process.exitCode = 1 + } } ) .strictCommands() diff --git a/scripts/firebase-admin/setOcpfMemberMapping.ts b/scripts/firebase-admin/setOcpfMemberMapping.ts new file mode 100644 index 000000000..528959e42 --- /dev/null +++ b/scripts/firebase-admin/setOcpfMemberMapping.ts @@ -0,0 +1,109 @@ +import { z } from "zod" +import { Script } from "./types" +import { parseArgs } from "./args" +import { MemberContent } from "functions/src/members/types" +import { + OcpfMemberMapping, + OcpfMemberMappingEntry, + OcpfMemberMappingFlags, + OcpfMemberMappingFlagsEntry +} from "functions/src/ocpf/types" + +// Manually maps one MAPLE member to their OCPF filer, for members that +// matchMembersForCourt flagged as unmatched or ambiguous. The entry is written +// with manual: true so later matcher runs leave it alone, and the member is +// removed from the court's flags doc. +// +// yarn firebase-admin run-script setOcpfMemberMapping --env dev \ +// --court 194 --memberCode SND1 --cpfId 15031 + +// yargs parses numeric values like "194" as numbers, and a flag given with no +// value as true +const stringOrNumber = z.union([z.string(), z.number()], { + errorMap: (_, ctx) => ({ + message: ctx.data === undefined ? "required" : "requires a value" + }) +}) + +const positiveInt = stringOrNumber.pipe( + z.coerce + .number({ invalid_type_error: "must be a number" }) + .int("must be a whole number") + .positive("must be greater than 0") +) + +const Args = z.object({ + court: positiveInt, + memberCode: stringOrNumber.pipe( + z.coerce.string().trim().min(1, "requires a value") + ), + cpfId: positiveInt +}) + +export const script: Script = async ({ db, args }) => { + const { court, memberCode, cpfId } = parseArgs(Args, args) + + const memberPath = `/generalCourts/${court}/members/${memberCode}` + const mappingPath = `/generalCourts/${court}/ocpfMemberMappingConfig/ocpfMemberMapping` + const flagsPath = `/generalCourts/${court}/ocpfMemberMappingConfig/ocpfMemberMappingFlags` + + const result = await db.runTransaction(async tx => { + const [memberDoc, mappingDoc, flagsDoc] = await Promise.all([ + tx.get(db.doc(memberPath)), + tx.get(db.doc(mappingPath)), + tx.get(db.doc(flagsPath)) + ]) + + const member = memberDoc.data()?.content as MemberContent | undefined + if (!member) return { found: false as const } + + const previous = (mappingDoc.data() as OcpfMemberMapping | undefined)?.[ + memberCode + ] + const entry: OcpfMemberMappingEntry = { + cpfId, + name: member.Name, + manual: true + } + tx.set(db.doc(mappingPath), { [memberCode]: entry }, { merge: true }) + + const flags = flagsDoc.data() as OcpfMemberMappingFlags | undefined + const removedFrom: string[] = [] + if (flags) { + const without = (list: OcpfMemberMappingFlagsEntry[], name: string) => { + const kept = list.filter(f => f.memberCode !== memberCode) + if (kept.length !== list.length) removedFrom.push(name) + return kept + } + const updated: OcpfMemberMappingFlags = { + unmatched: without(flags.unmatched ?? [], "unmatched"), + ambiguous: without(flags.ambiguous ?? [], "ambiguous") + } + if (removedFrom.length) tx.set(db.doc(flagsPath), updated) + } + + return { found: true as const, entry, previous, removedFrom } + }) + + if (!result.found) { + console.error( + `No member ${memberCode} in court ${court} (${memberPath}). Nothing written.` + ) + process.exitCode = 1 + return + } + + console.log( + JSON.stringify( + { + court, + memberCode, + previous: result.previous ?? null, + written: result.entry, + removedFromFlags: result.removedFrom + }, + null, + 2 + ) + ) +}