Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 2 additions & 1 deletion apps/api/src/app.ts
Original file line number Diff line number Diff line change
Expand Up @@ -158,12 +158,13 @@ export async function buildApp(opts: BuildAppOptions = {}): Promise<FastifyInsta
await fastify.register(slugRedirectPlugin);
await fastify.register(legacyRedirectPlugin);
await fastify.register(eventRemindersPlugin);
await fastify.register(spamPurgePlugin);

// ----- 7. Session middleware (JWT auth) -----
// Before rate-limit and idempotency: all three use onRequest hooks, which run
// in registration order, and the other two key on request.session.person.
await fastify.register(sessionMiddlewarePlugin);
// After the session middleware: the sweep clears purged people's session metadata.
await fastify.register(spamPurgePlugin);

// ----- 8. Rate limiting -----
await fastify.register(rateLimitPlugin);
Expand Down
13 changes: 13 additions & 0 deletions apps/api/src/auth/session-metadata.ts
Original file line number Diff line number Diff line change
Expand Up @@ -54,6 +54,19 @@ export class SessionMetadataStore {
await this.flush(privateStore);
}

/** Forget every session of a purged person. */
async removeAllFor(personId: string, privateStore: PrivateStore): Promise<number> {
let removed = 0;
for (const [jti, meta] of this.#map) {
if (meta.personId === personId) {
this.#map.delete(jti);
removed++;
}
}
if (removed > 0) await this.flush(privateStore);
return removed;
}

getAll(personId: string): SessionMeta[] {
const result: SessionMeta[] = [];
for (const meta of this.#map.values()) {
Expand Down
8 changes: 8 additions & 0 deletions apps/api/src/env.ts
Original file line number Diff line number Diff line change
Expand Up @@ -74,6 +74,13 @@ export const EnvSchema = z.object({
* image; unset in dev (Vite owns 5173).
*/
CFP_WEB_DIST_PATH: z.string().optional(),
/**
* Whether this pod runs the daily scheduled spam purge
* (specs/behaviors/person-lifecycle.md#scheduled-spam-purge). Off by default:
* sandbox and production share `published`, so exactly one deployment —
* production — may run a destructive sweep against it.
*/
CFP_SCHEDULED_PURGE: z.enum(['true', 'false']).default('false'),
/**
* Host of the public-facing site (e.g. `codeforphilly.org` in prod,
* `next-v2.codeforphilly.org` in sandbox). Used by the server-side
Expand Down Expand Up @@ -147,6 +154,7 @@ export const envJsonSchema = {
SAML_ENTITY_ID: { type: 'string', default: SAML_ENTITY_ID_DEFAULT },
SLACK_TEAM_HOST: { type: 'string', default: 'codeforphilly.slack.com' },
CFP_WEB_DIST_PATH: { type: 'string' },
CFP_SCHEDULED_PURGE: { type: 'string', enum: ['true', 'false'], default: 'false' },
CFP_SITE_HOST: { type: 'string', default: 'codeforphilly.org' },
POSTMARK_SERVER_TOKEN: { type: 'string' },
POSTMARK_MESSAGE_STREAM: { type: 'string', default: 'outbound' },
Expand Down
17 changes: 14 additions & 3 deletions apps/api/src/plugins/spam-purge.ts
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,14 @@ import fp from 'fastify-plugin';

import { SPAM_PURGE_INTERVAL_MS, SpamPurgeService } from '../services/spam-purge.js';

/**
* Whether this pod runs the sweep: only when CFP_SCHEDULED_PURGE is "true",
* and never under tests (they call the sweep directly).
*/
export function scheduledPurgeEnabled(config: { NODE_ENV: string; CFP_SCHEDULED_PURGE: string }): boolean {
return config.NODE_ENV !== 'test' && config.CFP_SCHEDULED_PURGE === 'true';
}

declare module 'fastify' {
interface FastifyInstance {
spamPurge: SpamPurgeService;
Expand All @@ -20,6 +28,7 @@ async function spamPurgePlugin(fastify: FastifyInstance): Promise<void> {
fts: fastify.fts,
store: fastify.store,
log: fastify.log,
sessionMetadata: fastify.sessionMetadata,
});
fastify.decorate('spamPurge', service);

Expand All @@ -31,8 +40,10 @@ async function spamPurgePlugin(fastify: FastifyInstance): Promise<void> {

let interval: ReturnType<typeof setInterval> | undefined;
fastify.addHook('onReady', async () => {
// Tests boot many apps; the sweep is exercised directly there.
if (fastify.config.NODE_ENV === 'test') return;
if (!scheduledPurgeEnabled(fastify.config)) {
fastify.log.info('scheduled spam purge disabled (CFP_SCHEDULED_PURGE is not "true")');
return;
}
run();
interval = setInterval(run, SPAM_PURGE_INTERVAL_MS);
});
Expand All @@ -44,5 +55,5 @@ async function spamPurgePlugin(fastify: FastifyInstance): Promise<void> {
export default fp(spamPurgePlugin, {
name: 'spam-purge',
fastify: '5.x',
dependencies: ['services'],
dependencies: ['services', 'session-middleware'],
});
36 changes: 36 additions & 0 deletions apps/api/src/routes/moderation.ts
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@ import { ApiNotFoundError, ApiValidationError } from '../lib/errors.js';
import { getCallerSession } from '../services/permissions.js';
import { buildTransactionOptions } from '../store/commit-meta.js';
import type { MemberOrigin, VoteFilter } from '../services/moderation.js';
import { findOrphanPrivateData, purgeOrphanPrivateData } from '../services/private-orphans.js';

function requireStaffOr404(request: FastifyRequest): void {
const level = request.session.accountLevel;
Expand Down Expand Up @@ -139,4 +140,39 @@ export async function moderationRoutes(fastify: FastifyInstance): Promise<void>
});
},
);

// POST /api/admin/private-data/purge-orphans (administrator only)
// Spec: specs/api/moderation.md → POST /api/admin/private-data/purge-orphans
fastify.post(
'/api/admin/private-data/purge-orphans',
{
schema: {
tags: ['admin'],
summary: 'Delete private data of people with no public record (admin only)',
body: {
type: 'object',
properties: { apply: { type: 'boolean' } },
required: ['apply'],
additionalProperties: false,
},
},
},
async (request) => {
if (request.session.accountLevel !== 'administrator') throw new ApiNotFoundError('Not found');
const { apply } = request.body as { apply: boolean };
// Under the write lock: account creation writes the private profile
// before the public record, and must never look orphaned mid-write.
const found = await fastify.publicRepo.withLock(async () => {
const publicIds = new Set(fastify.inMemoryState.people.keys());
const result = await findOrphanPrivateData(publicIds, fastify.store.private, fastify.sessionMetadata);
if (apply && result.personIds.length > 0) {
await purgeOrphanPrivateData(result, fastify.store.private, fastify.sessionMetadata);
}
return result;
});
const { personIds, ...counts } = found;
request.log.info({ apply, orphanPeople: personIds.length, ...counts }, 'private orphan purge');
return ok({ applied: apply, orphanPeople: personIds.length, ...counts });
},
);
}
2 changes: 2 additions & 0 deletions apps/api/src/routes/people.ts
Original file line number Diff line number Diff line change
Expand Up @@ -239,6 +239,7 @@ export async function peopleRoutes(fastify: FastifyInstance): Promise<void> {
},
}, async (request, reply) => {
const { slug } = request.params as { slug: string };
const personId = fastify.inMemoryState.personIdBySlug.get(slug);
const result = await fastify.store.transact(
buildTransactionOptions({
request,
Expand All @@ -250,6 +251,7 @@ export async function peopleRoutes(fastify: FastifyInstance): Promise<void> {
async (tx) => fastify.services.peopleWrite.purge(tx, slug, request.session),
);
result.value.stateApply.apply(fastify.inMemoryState, fastify.fts);
if (personId) await fastify.sessionMetadata.removeAllFor(personId, fastify.store.private);
return reply.code(204).send();
});

Expand Down
3 changes: 3 additions & 0 deletions apps/api/src/services/person-purge.ts
Original file line number Diff line number Diff line change
Expand Up @@ -43,6 +43,9 @@ async function purgePersonRecords(
// 1. Delete the person record.
await tx.public.people.delete(existing);
stateApply.removePerson(personId, existing.slug);
// Their private data goes in the same transaction (profile, legacy password,
// reset tokens, claim requests).
tx.private.deletePersonData(personId);

// 2. Cascade-delete project-memberships.
const membershipIds = state.membershipsByPerson.get(existing.slug) ?? new Set<string>();
Expand Down
78 changes: 78 additions & 0 deletions apps/api/src/services/private-orphans.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,78 @@
/**
* Private data left by people who no longer exist publicly — the cleanup
* behind `POST /api/admin/private-data/purge-orphans`
* (specs/api/moderation.md). Purge deletes private data in the same
* transaction now; everyone purged or pruned before that left a private
* profile, legacy password, claim requests and session metadata behind.
*
* Runs inside the API on purpose: the API holds the private store in memory
* and rewrites whole files on each private write, so an out-of-process
* cleanup would be undone by the next write.
*/
import type { SessionMetadataStore } from '../auth/session-metadata.js';
import type { PrivateStore } from '../store/private/index.js';

export interface OrphanPrivateData {
/** Person ids with private data and no public record. */
readonly personIds: string[];
readonly profiles: number;
readonly legacyPasswords: number;
readonly claimRequests: number;
readonly sessions: number;
}

/** Find private data whose person is not in `publicIds`. */
export async function findOrphanPrivateData(
publicIds: ReadonlySet<string>,
privateStore: PrivateStore,
sessions: SessionMetadataStore,
): Promise<OrphanPrivateData> {
const orphans = new Set<string>();
let profiles = 0;
let legacyPasswords = 0;
let claimRequests = 0;
let sessionCount = 0;

for await (const profile of privateStore.listAllProfiles()) {
if (!publicIds.has(profile.personId)) {
orphans.add(profile.personId);
profiles++;
}
}
for (const personId of privateStore.indices.legacyPasswordByPersonId.keys()) {
if (!publicIds.has(personId)) {
orphans.add(personId);
legacyPasswords++;
}
}
for (const req of await privateStore.listAllClaimRequests()) {
const ids = [req.claimedPersonId, req.requesterPersonId].filter((id): id is string => !!id);
const gone = ids.filter((id) => !publicIds.has(id));
if (gone.length > 0) {
gone.forEach((id) => orphans.add(id));
claimRequests++;
}
}
for (const personId of orphans) sessionCount += sessions.getAll(personId).length;

return {
personIds: [...orphans].sort(),
profiles,
legacyPasswords,
claimRequests,
sessions: sessionCount,
};
}

/** Delete the private data of `found.personIds`. */
export async function purgeOrphanPrivateData(
found: OrphanPrivateData,
privateStore: PrivateStore,
sessions: SessionMetadataStore,
): Promise<void> {
await privateStore.transact(async (tx) => {
for (const personId of found.personIds) tx.deletePersonData(personId);
});
for (const personId of found.personIds) await sessions.removeAllFor(personId, privateStore);
}

5 changes: 5 additions & 0 deletions apps/api/src/services/spam-purge.ts
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@ import type { FtsEngine } from '../store/fts.js';
import type { InMemoryState } from '../store/memory/state.js';
import { StateApply } from '../store/state-apply.js';
import type { Store } from '../store/store.js';
import type { SessionMetadataStore } from '../auth/session-metadata.js';
import { purgePeople } from './person-purge.js';

/** How long a spam vote stands before the sweep acts on it. */
Expand Down Expand Up @@ -48,6 +49,7 @@ export interface SpamPurgeDeps {
readonly fts: FtsEngine;
readonly store: Store;
readonly log: FastifyBaseLogger;
readonly sessionMetadata: SessionMetadataStore;
}

export class SpamPurgeService {
Expand Down Expand Up @@ -86,6 +88,9 @@ export class SpamPurgeService {
throw err;
}
stateApply.apply(this.#d.state, this.#d.fts);
for (const person of queued) {
await this.#d.sessionMetadata.removeAllFor(person.id, this.#d.store.private);
}
this.#d.log.info({ purged: slugs }, 'scheduled spam purge');
return slugs;
}
Expand Down
29 changes: 26 additions & 3 deletions apps/api/src/store/private/base.ts
Original file line number Diff line number Diff line change
Expand Up @@ -174,13 +174,15 @@ export abstract class BasePrivateStore implements PrivateStore {
const profilesSnapshot = new Map(this.profiles);
const legacySnapshot = new Map(this.legacyPasswords);
const claimRequestsSnapshot = new Map(this.claimRequests);
const passwordTokensSnapshot = new Map(this.passwordTokens);

// Staged mutations applied only in-memory during the handler
const stagedProfilePuts: Map<string, PrivateProfile> = new Map();
const stagedProfileDeletes: Set<string> = new Set();
const stagedLegacyPuts: Map<string, LegacyPasswordCredential> = new Map();
const stagedLegacyDeletes: Set<string> = new Set();
const stagedClaimRequestPuts: Map<string, AccountClaimRequest> = new Map();
const stagedPersonPurges: Set<string> = new Set();

const tx: PrivateStoreTx = {
putProfile: (profile) => {
Expand All @@ -205,6 +207,11 @@ export abstract class BasePrivateStore implements PrivateStore {
const parsed = AccountClaimRequestSchema.parse(req);
stagedClaimRequestPuts.set(parsed.id, parsed);
},
deletePersonData: (personId) => {
stagedPersonPurges.add(personId);
stagedProfilePuts.delete(personId);
stagedLegacyPuts.delete(personId);
},
};

let result: T;
Expand All @@ -215,6 +222,7 @@ export abstract class BasePrivateStore implements PrivateStore {
this.profiles = profilesSnapshot;
this.legacyPasswords = legacySnapshot;
this.claimRequests = claimRequestsSnapshot;
this.passwordTokens = passwordTokensSnapshot;
this.rebuildIndices();
throw err;
}
Expand All @@ -235,19 +243,33 @@ export abstract class BasePrivateStore implements PrivateStore {
for (const [id, req] of stagedClaimRequestPuts) {
this.claimRequests.set(id, req);
}
for (const personId of stagedPersonPurges) {
this.profiles.delete(personId);
this.legacyPasswords.delete(personId);
for (const [id, req] of this.claimRequests) {
if (req.claimedPersonId === personId || req.requesterPersonId === personId) this.claimRequests.delete(id);
}
for (const [hash, token] of this.passwordTokens) {
if (token.personId === personId) this.passwordTokens.delete(hash);
}
}
this.rebuildIndices();

// Flush to backend
const flushOps: Promise<void>[] = [];
if (stagedProfilePuts.size > 0 || stagedProfileDeletes.size > 0) {
const purging = stagedPersonPurges.size > 0;
if (purging || stagedProfilePuts.size > 0 || stagedProfileDeletes.size > 0) {
flushOps.push(this.flushProfiles());
}
if (stagedLegacyPuts.size > 0 || stagedLegacyDeletes.size > 0) {
if (purging || stagedLegacyPuts.size > 0 || stagedLegacyDeletes.size > 0) {
flushOps.push(this.flushLegacyPasswords());
}
if (stagedClaimRequestPuts.size > 0) {
if (purging || stagedClaimRequestPuts.size > 0) {
flushOps.push(this.flushClaimRequests());
}
if (purging) {
flushOps.push(this.flushPasswordTokens());
}

try {
await Promise.all(flushOps);
Expand All @@ -257,6 +279,7 @@ export abstract class BasePrivateStore implements PrivateStore {
this.profiles = profilesSnapshot;
this.legacyPasswords = legacySnapshot;
this.claimRequests = claimRequestsSnapshot;
this.passwordTokens = passwordTokensSnapshot;
this.rebuildIndices();
throw new PrivateStoreError(
'private_store_unavailable',
Expand Down
6 changes: 6 additions & 0 deletions apps/api/src/store/private/interface.ts
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,12 @@ export interface PrivateStoreTx {
putLegacyPassword(cred: LegacyPasswordCredential): void;
deleteLegacyPassword(personId: string): void;
putClaimRequest(req: AccountClaimRequest): void;
/**
* Delete everything the private store holds for a person: profile, legacy
* password, password-reset tokens, and account-claim requests naming them
* (specs/behaviors/person-lifecycle.md → Purge).
*/
deletePersonData(personId: string): void;
}

/**
Expand Down
Loading
Loading