From b35a04f3b05117ab6ea628810fa99fe1b3df9285 Mon Sep 17 00:00:00 2001 From: Makisuo Date: Wed, 28 Oct 2026 15:58:26 +0100 Subject: [PATCH 01/15] feat(status-pages): status page service Org-scoped CRUD for status pages, components and incidents, break-glass token rotation, and typed errors plus record shapes in @maple/domain/http. Every write a visitor could see bumps the page revision in its own transaction, so the publisher picks it up. --- apps/api/src/runtime/service-graph.ts | 2 + .../status-pages/StatusPagesService.test.ts | 351 +++++ .../status-pages/StatusPagesService.ts | 1172 +++++++++++++++++ packages/domain/src/http/index.ts | 1 + packages/domain/src/http/status-pages.ts | 190 +++ 5 files changed, 1716 insertions(+) create mode 100644 packages/backend/src/services/status-pages/StatusPagesService.test.ts create mode 100644 packages/backend/src/services/status-pages/StatusPagesService.ts create mode 100644 packages/domain/src/http/status-pages.ts diff --git a/apps/api/src/runtime/service-graph.ts b/apps/api/src/runtime/service-graph.ts index 05a759e970..446eef9dfc 100644 --- a/apps/api/src/runtime/service-graph.ts +++ b/apps/api/src/runtime/service-graph.ts @@ -56,6 +56,7 @@ import { SupabaseOAuthService } from "@maple/backend/services/auth/SupabaseOAuth import { OrgOpenRouterKeyService } from "@maple/backend/services/org/OrgOpenRouterKeyService" import { ScrapeTargetsService } from "@maple/backend/services/integrations/ScrapeTargetsService" import { UptimeMonitorsService } from "@maple/backend/services/uptime/UptimeMonitorsService" +import { StatusPagesService } from "@maple/backend/services/status-pages/StatusPagesService" import { ChatWorkspaceService } from "@maple/backend/services/integrations/ChatWorkspaceService" import { TinybirdOrgTokenService } from "@maple/backend/services/integrations/TinybirdOrgTokenService" import { PlanetScaleWebhookQueue } from "@maple/backend/services/integrations/planetscale/PlanetScaleWebhookQueue" @@ -129,6 +130,7 @@ export const HttpServicesLive = Layer.mergeAll( CancellationReviewService.layer, ScrapeTargetsService.layer, UptimeMonitorsService.layer, + StatusPagesService.layer, PlanetScaleConnectionService.layer, PlanetScaleService.layer, PlanetScaleWebhookService.layer, diff --git a/packages/backend/src/services/status-pages/StatusPagesService.test.ts b/packages/backend/src/services/status-pages/StatusPagesService.test.ts new file mode 100644 index 0000000000..02cd7d06e6 --- /dev/null +++ b/packages/backend/src/services/status-pages/StatusPagesService.test.ts @@ -0,0 +1,351 @@ +import { afterEach, assert, describe, it } from "@effect/vitest" +import { Effect, Layer, Schema } from "effect" +import { TestClock } from "effect/testing" +import { OrgId, UserId } from "@maple/domain/http" +import { StatusPageComponentId, StatusPageId, UptimeMonitorId } from "@maple/domain/primitives" +import { STATUS_PAGE_BREAK_GLASS_PREFIX, sha256Base64Url } from "@maple/domain/status-page" +import { cleanupTestDbs, createTestDb, executeSql, type TestDb } from "@maple/backend/platform/test-pglite" +import { StatusPagesService, type CreateStatusPageIncidentInput } from "./StatusPagesService" + +const trackedDbs: TestDb[] = [] +afterEach(() => cleanupTestDbs(trackedDbs)) + +const ORG = Schema.decodeUnknownSync(OrgId)("org_status") +const OTHER_ORG = Schema.decodeUnknownSync(OrgId)("org_status_other") +const USER = Schema.decodeUnknownSync(UserId)("user_status") +const MONITOR = UptimeMonitorId.make("33333333-3333-4333-8333-333333333333") +const OTHER_MONITOR = UptimeMonitorId.make("44444444-4444-4444-8444-444444444444") +const DAY = 86_400_000 + +const makeLayer = (testDb: TestDb) => + Layer.effect(StatusPagesService, StatusPagesService.make).pipe(Layer.provide(testDb.layer)) + +const seedMonitor = (testDb: TestDb, orgId: OrgId, id: UptimeMonitorId) => + Effect.promise(() => + executeSql( + testDb, + `INSERT INTO uptime_monitors (id, org_id, name, kind, config_json, created_by, created_at, updated_at) + VALUES ($1, $2, 'API', 'heartbeat', '{"type":"heartbeat","graceSeconds":60}', $3, now(), now())`, + [id, orgId, USER], + ), + ) + +const pageRow = (testDb: TestDb, pageId: StatusPageId) => + Effect.promise(() => + testDb.pglite.query<{ revision: string; break_glass_token_sha256: string | null }>( + `SELECT revision::text, break_glass_token_sha256 FROM status_pages WHERE id = $1`, + [pageId], + ), + ).pipe(Effect.map((result) => result.rows[0])) + +const revision = (testDb: TestDb, pageId: StatusPageId) => + pageRow(testDb, pageId).pipe(Effect.map((row) => Number(row?.revision))) + +const createPage = (slug: string, orgId: OrgId = ORG) => + Effect.gen(function* () { + const service = yield* StatusPagesService + return yield* service.create(orgId, { name: `Page ${slug}`, slug, createdBy: USER }) + }) + +describe("StatusPagesService", () => { + it.effect("creates, lists, updates and deletes a page, scoped to its org", () => { + const testDb = createTestDb(trackedDbs) + return Effect.gen(function* () { + const service = yield* StatusPagesService + const page = yield* service.create(ORG, { + name: " Acme ", + slug: "acme", + websiteUrl: "https://acme.test", + brandColor: "#AABBCC", + createdBy: USER, + }) + assert.strictEqual(page.name, "Acme") + assert.strictEqual(page.brandColor, "#aabbcc") + assert.strictEqual(page.revision, 1) + assert.strictEqual(page.publishedRevision, 0) + yield* createPage("elsewhere", OTHER_ORG) + + const listed = yield* service.list(ORG) + assert.deepStrictEqual( + listed.map((row) => row.slug), + ["acme"], + ) + const crossOrg = yield* Effect.flip(service.get(OTHER_ORG, page.id)) + assert.strictEqual(crossOrg._tag, "@maple/http/errors/StatusPageNotFoundError") + + const updated = yield* service.update(ORG, page.id, { name: "Acme Inc", description: " " }) + assert.strictEqual(updated.name, "Acme Inc") + assert.isNull(updated.description) + assert.strictEqual(updated.revision, 2) + + const badUrl = yield* Effect.flip( + service.update(ORG, page.id, { logoUrl: "http://acme.test/logo.png" }), + ) + assert.strictEqual(badUrl._tag, "@maple/http/errors/StatusPageValidationError") + const badColor = yield* Effect.flip(service.update(ORG, page.id, { brandColor: "red" })) + assert.strictEqual(badColor._tag, "@maple/http/errors/StatusPageValidationError") + + assert.strictEqual(yield* service.remove(ORG, page.id), page.id) + const gone = yield* Effect.flip(service.get(ORG, page.id)) + assert.strictEqual(gone._tag, "@maple/http/errors/StatusPageNotFoundError") + }).pipe(Effect.provide(makeLayer(testDb))) + }) + + it.effect("refuses a slug another org already uses, on create and on rename", () => { + const testDb = createTestDb(trackedDbs) + return Effect.gen(function* () { + const service = yield* StatusPagesService + yield* createPage("taken", OTHER_ORG) + const onCreate = yield* Effect.flip(createPage("taken")) + assert.strictEqual(onCreate._tag, "@maple/http/errors/StatusPageSlugTakenError") + const mine = yield* createPage("mine") + const onRename = yield* Effect.flip(service.update(ORG, mine.id, { slug: "taken" })) + assert.strictEqual(onRename._tag, "@maple/http/errors/StatusPageSlugTakenError") + // Keeping your own slug is not a conflict. + const same = yield* service.update(ORG, mine.id, { slug: "mine" }) + assert.strictEqual(same.slug, "mine") + }).pipe(Effect.provide(makeLayer(testDb))) + }) + + it.effect("links components only to monitors of the same org and reorders them", () => { + const testDb = createTestDb(trackedDbs) + return Effect.gen(function* () { + const service = yield* StatusPagesService + yield* seedMonitor(testDb, ORG, MONITOR) + yield* seedMonitor(testDb, OTHER_ORG, OTHER_MONITOR) + const page = yield* createPage("acme") + + const foreign = yield* Effect.flip( + service.createComponent(ORG, page.id, { name: "API", monitorId: OTHER_MONITOR }), + ) + assert.strictEqual(foreign._tag, "@maple/http/errors/StatusPageValidationError") + + const api = yield* service.createComponent(ORG, page.id, { name: "API", monitorId: MONITOR }) + const web = yield* service.createComponent(ORG, page.id, { name: "Web" }) + assert.deepStrictEqual([api.position, web.position], [0, 1]) + + const reordered = yield* service.reorderComponents(ORG, page.id, [web.id, api.id]) + assert.deepStrictEqual( + reordered.map((row) => [row.name, row.position]), + [ + ["Web", 0], + ["API", 1], + ], + ) + const partial = yield* Effect.flip(service.reorderComponents(ORG, page.id, [web.id])) + assert.strictEqual(partial._tag, "@maple/http/errors/StatusPageValidationError") + }).pipe(Effect.provide(makeLayer(testDb))) + }) + + it.effect("opens an incident with its first update and resolves it", () => { + const testDb = createTestDb(trackedDbs) + return Effect.gen(function* () { + const service = yield* StatusPagesService + const page = yield* createPage("acme") + const other = yield* createPage("other") + const api = yield* service.createComponent(ORG, page.id, { name: "API" }) + const foreignComponent = yield* service.createComponent(ORG, other.id, { name: "Elsewhere" }) + const before = yield* revision(testDb, page.id) + + const wrongPage = yield* Effect.flip( + service.createIncident(ORG, page.id, { + kind: "incident", + title: "Errors", + impact: "major", + status: "investigating", + body: "Looking", + componentIds: [foreignComponent.id], + source: "dashboard", + }), + ) + assert.strictEqual(wrongPage._tag, "@maple/http/errors/StatusPageValidationError") + + const incident = yield* service.createIncident(ORG, page.id, { + kind: "incident", + title: "Elevated errors", + impact: "major", + status: "investigating", + body: "We are looking into it", + componentIds: [api.id], + source: "dashboard", + createdBy: USER, + }) + assert.deepStrictEqual(incident.componentIds, [api.id]) + assert.deepStrictEqual( + incident.updates.map((update) => [update.status, update.body]), + [["investigating", "We are looking into it"]], + ) + assert.isNull(incident.resolvedAt) + assert.strictEqual(yield* revision(testDb, page.id), before + 1) + + yield* TestClock.adjust("1 minute") + const resolved = yield* service.postUpdate(ORG, incident.id, { + status: "resolved", + body: "Fixed", + source: "dashboard", + }) + assert.strictEqual(resolved.status, "resolved") + assert.isNotNull(resolved.resolvedAt) + assert.deepStrictEqual( + resolved.updates.map((update) => update.status), + ["resolved", "investigating"], + ) + assert.strictEqual(yield* revision(testDb, page.id), before + 2) + // Resolved incidents leave the default list. + assert.lengthOf(yield* service.listIncidents(ORG, page.id), 0) + assert.lengthOf(yield* service.listIncidents(ORG, page.id, { includeResolved: true }), 1) + + const reopened = yield* service.postUpdate(ORG, incident.id, { + status: "monitoring", + body: "Watching again", + source: "api", + }) + assert.isNull(reopened.resolvedAt) + + const maintenanceStatus = yield* Effect.flip( + service.postUpdate(ORG, incident.id, { status: "completed", body: "x", source: "api" }), + ) + assert.strictEqual(maintenanceStatus._tag, "@maple/http/errors/StatusPageValidationError") + const crossOrg = yield* Effect.flip(service.getIncident(OTHER_ORG, incident.id)) + assert.strictEqual(crossOrg._tag, "@maple/http/errors/StatusPageIncidentNotFoundError") + }).pipe(Effect.provide(makeLayer(testDb))) + }) + + it.effect("requires a valid window and maintenance statuses for maintenance", () => { + const testDb = createTestDb(trackedDbs) + return Effect.gen(function* () { + const service = yield* StatusPagesService + const page = yield* createPage("acme") + const start = Date.UTC(2026, 10, 1, 2) + const base: CreateStatusPageIncidentInput = { + kind: "maintenance", + title: "Database upgrade", + impact: "minor", + status: "scheduled", + body: "Planned", + componentIds: [], + source: "dashboard", + } + const cases: ReadonlyArray> = [ + { scheduledStart: start, scheduledEnd: null }, + { scheduledStart: start, scheduledEnd: start }, + { status: "investigating", scheduledStart: start, scheduledEnd: start + DAY }, + ] + yield* Effect.forEach(cases, (input) => + Effect.flip(service.createIncident(ORG, page.id, { ...base, ...input })).pipe( + Effect.map((error) => + assert.strictEqual(error._tag, "@maple/http/errors/StatusPageValidationError"), + ), + ), + ) + const ok = yield* service.createIncident(ORG, page.id, { + ...base, + status: "scheduled", + scheduledStart: start, + scheduledEnd: start + DAY, + }) + assert.strictEqual(ok.startedAt, start) + const shrunk = yield* Effect.flip(service.updateIncident(ORG, ok.id, { scheduledEnd: start - 1 })) + assert.strictEqual(shrunk._tag, "@maple/http/errors/StatusPageValidationError") + }).pipe(Effect.provide(makeLayer(testDb))) + }) + + it.effect("rotates a break-glass token, storing only its hash and suffix", () => { + const testDb = createTestDb(trackedDbs) + return Effect.gen(function* () { + const service = yield* StatusPagesService + const page = yield* createPage("acme") + const token = yield* service.rotateBreakGlassToken(ORG, page.id) + assert.isTrue(token.startsWith(STATUS_PAGE_BREAK_GLASS_PREFIX)) + const row = yield* pageRow(testDb, page.id) + assert.strictEqual( + row?.break_glass_token_sha256, + yield* Effect.promise(() => sha256Base64Url(token)), + ) + // The raw token appears in no column of the page. + const dump = yield* Effect.promise(() => + testDb.pglite.query<{ dump: string }>( + `SELECT row_to_json(p)::text AS dump FROM status_pages p`, + ), + ) + assert.isFalse(dump.rows.some((entry) => entry.dump.includes(token))) + const record = yield* service.get(ORG, page.id) + assert.strictEqual(record.breakGlassTokenSuffix, token.slice(-6)) + + yield* service.revokeBreakGlassToken(ORG, page.id) + assert.isNull((yield* pageRow(testDb, page.id))?.break_glass_token_sha256) + assert.isNull((yield* service.get(ORG, page.id)).breakGlassTokenSuffix) + }).pipe(Effect.provide(makeLayer(testDb))) + }) + + it.effect("bumps the page revision on every visible mutation", () => { + const testDb = createTestDb(trackedDbs) + return Effect.gen(function* () { + const service = yield* StatusPagesService + const page = yield* createPage("acme") + const first = yield* service.createComponent(ORG, page.id, { name: "First" }) + const incident = yield* service.createIncident(ORG, page.id, { + kind: "incident", + title: "Outage", + impact: "critical", + status: "investigating", + body: "Down", + componentIds: [first.id], + source: "dashboard", + }) + const extra = StatusPageComponentId.make("55555555-5555-4555-8555-555555555555") + const mutations: ReadonlyArray]> = [ + ["update", service.update(ORG, page.id, { showPoweredBy: false })], + ["rotateBreakGlassToken", service.rotateBreakGlassToken(ORG, page.id)], + ["revokeBreakGlassToken", service.revokeBreakGlassToken(ORG, page.id)], + ["createComponent", service.createComponent(ORG, page.id, { name: "Second" })], + ["updateComponent", service.updateComponent(ORG, first.id, { manualStatus: "degraded" })], + [ + "reorderComponents", + service + .listComponents(ORG, page.id) + .pipe( + Effect.flatMap((rows) => + service.reorderComponents(ORG, page.id, rows.map((row) => row.id).reverse()), + ), + ), + ], + [ + "createIncident", + service.createIncident(ORG, page.id, { + kind: "incident", + title: "More", + impact: "minor", + status: "identified", + body: "b", + componentIds: [], + source: "api", + }), + ], + [ + "postUpdate", + service.postUpdate(ORG, incident.id, { + status: "identified", + body: "Found it", + source: "api", + }), + ], + ["updateIncident", service.updateIncident(ORG, incident.id, { title: "Outage in us-east" })], + ["removeIncident", service.removeIncident(ORG, incident.id)], + ["removeComponent", service.removeComponent(ORG, first.id)], + ] + yield* Effect.forEach(mutations, ([name, mutation]) => + Effect.gen(function* () { + const before = yield* revision(testDb, page.id) + yield* mutation + assert.strictEqual(yield* revision(testDb, page.id), before + 1, name) + }), + ) + // A failed mutation rolls back its bump with the write. + const before = yield* revision(testDb, page.id) + yield* Effect.flip(service.updateComponent(ORG, extra, { name: "Nope" })) + yield* Effect.flip(service.update(ORG, page.id, { brandColor: "nope" })) + assert.strictEqual(yield* revision(testDb, page.id), before) + }).pipe(Effect.provide(makeLayer(testDb))) + }) +}) diff --git a/packages/backend/src/services/status-pages/StatusPagesService.ts b/packages/backend/src/services/status-pages/StatusPagesService.ts new file mode 100644 index 0000000000..1cdabcec13 --- /dev/null +++ b/packages/backend/src/services/status-pages/StatusPagesService.ts @@ -0,0 +1,1172 @@ +import { randomBytes, randomUUID } from "node:crypto" +import * as PG from "@maple-dev/effect-orm/postgres" +import type { MapleDb } from "@maple/db/client" +import { + StatusPageComponents, + StatusPageIncidentComponents, + StatusPageIncidentUpdates, + StatusPageIncidents, + StatusPages, + UptimeMonitors, + type StatusPageComponentRow, + type StatusPageIncidentRow, + type StatusPageIncidentUpdateRow, + type StatusPageRow, +} from "@maple/db/tables" +import { + StatusPageComponentNotFoundError, + StatusPageIncidentNotFoundError, + StatusPageNotFoundError, + StatusPagePersistenceError, + StatusPageSlugTakenError, + StatusPageValidationError, + type StatusPageComponentRecord, + type StatusPageIncidentRecord, + type StatusPageIncidentUpdateRecord, + type StatusPageRecord, +} from "@maple/domain/http" +import { + StatusPageComponentId, + StatusPageId, + StatusPageIncidentId, + StatusPageIncidentUpdateId, + type OrgId, + type StatusPageAutoIncidents, + type StatusPageComponentStatus, + type StatusPageIncidentImpact, + type StatusPageIncidentKind, + type StatusPageIncidentStatus, + type StatusPageUpdateSource, + type UptimeMonitorId, + type UserId, +} from "@maple/domain/primitives" +import { STATUS_PAGE_BREAK_GLASS_PREFIX, StatusPageSlug, sha256Base64Url } from "@maple/domain/status-page" +import { Clock, Context, Effect, Layer, Schema } from "effect" +import { Database, type DatabaseError } from "@maple/backend/platform/DatabaseLive" +import { makeDbExecute } from "@maple/backend/platform/db-execute" +import { postgresSqlState } from "@maple/backend/platform/postgres-errors" +import { bumpPage } from "./outbox" + +export interface CreateStatusPageInput { + readonly name: string + readonly slug: string + readonly description?: string | null + readonly websiteUrl?: string | null + readonly logoUrl?: string | null + readonly brandColor?: string | null + readonly showPoweredBy?: boolean + readonly autoIncidents?: StatusPageAutoIncidents + readonly createdBy: UserId +} + +export interface UpdateStatusPageInput { + readonly name?: string + readonly slug?: string + readonly description?: string | null + readonly websiteUrl?: string | null + readonly logoUrl?: string | null + readonly brandColor?: string | null + readonly showPoweredBy?: boolean + readonly autoIncidents?: StatusPageAutoIncidents +} + +export interface CreateStatusPageComponentInput { + readonly name: string + readonly description?: string | null + readonly groupName?: string | null + readonly monitorId?: UptimeMonitorId | null + readonly manualStatus?: StatusPageComponentStatus | null + readonly showUptime?: boolean +} + +export type UpdateStatusPageComponentInput = Partial + +export interface ListStatusPageIncidentsOptions { + /** Resolved incidents and completed maintenance too; open ones only by default. */ + readonly includeResolved?: boolean +} + +export interface CreateStatusPageIncidentInput { + readonly kind: StatusPageIncidentKind + readonly title: string + readonly impact: StatusPageIncidentImpact + readonly status: StatusPageIncidentStatus + readonly body: string + readonly componentIds: ReadonlyArray + readonly published?: boolean + readonly scheduledStart?: number | null + readonly scheduledEnd?: number | null + readonly source: StatusPageUpdateSource + readonly createdBy?: UserId | null +} + +export interface PostStatusPageIncidentUpdateInput { + readonly status: StatusPageIncidentStatus + readonly body: string + readonly impact?: StatusPageIncidentImpact + readonly componentIds?: ReadonlyArray + readonly source: StatusPageUpdateSource + readonly createdBy?: UserId | null +} + +export interface UpdateStatusPageIncidentInput { + readonly title?: string + readonly impact?: StatusPageIncidentImpact + readonly published?: boolean + readonly scheduledStart?: number | null + readonly scheduledEnd?: number | null + readonly componentIds?: ReadonlyArray +} + +type PageError = StatusPageNotFoundError | StatusPagePersistenceError +type PageWriteError = PageError | StatusPageValidationError +type ComponentError = PageWriteError | StatusPageComponentNotFoundError +type IncidentError = PageWriteError | StatusPageIncidentNotFoundError + +export interface StatusPagesServiceApi { + readonly list: ( + orgId: OrgId, + ) => Effect.Effect, StatusPagePersistenceError> + readonly get: (orgId: OrgId, pageId: StatusPageId) => Effect.Effect + readonly create: ( + orgId: OrgId, + input: CreateStatusPageInput, + ) => Effect.Effect< + StatusPageRecord, + StatusPageValidationError | StatusPageSlugTakenError | StatusPagePersistenceError + > + readonly update: ( + orgId: OrgId, + pageId: StatusPageId, + patch: UpdateStatusPageInput, + ) => Effect.Effect + readonly remove: (orgId: OrgId, pageId: StatusPageId) => Effect.Effect + /** Returns the new raw token; only its hash and last 6 characters are stored. */ + readonly rotateBreakGlassToken: (orgId: OrgId, pageId: StatusPageId) => Effect.Effect + readonly revokeBreakGlassToken: (orgId: OrgId, pageId: StatusPageId) => Effect.Effect + readonly listComponents: ( + orgId: OrgId, + pageId: StatusPageId, + ) => Effect.Effect, PageError> + readonly createComponent: ( + orgId: OrgId, + pageId: StatusPageId, + input: CreateStatusPageComponentInput, + ) => Effect.Effect + readonly updateComponent: ( + orgId: OrgId, + componentId: StatusPageComponentId, + patch: UpdateStatusPageComponentInput, + ) => Effect.Effect + readonly removeComponent: ( + orgId: OrgId, + componentId: StatusPageComponentId, + ) => Effect.Effect + /** `componentIds` must list every component of the page once; position = index. */ + readonly reorderComponents: ( + orgId: OrgId, + pageId: StatusPageId, + componentIds: ReadonlyArray, + ) => Effect.Effect, PageWriteError> + readonly listIncidents: ( + orgId: OrgId, + pageId: StatusPageId, + options?: ListStatusPageIncidentsOptions, + ) => Effect.Effect, PageError> + readonly getIncident: ( + orgId: OrgId, + incidentId: StatusPageIncidentId, + ) => Effect.Effect + readonly createIncident: ( + orgId: OrgId, + pageId: StatusPageId, + input: CreateStatusPageIncidentInput, + ) => Effect.Effect + readonly postUpdate: ( + orgId: OrgId, + incidentId: StatusPageIncidentId, + input: PostStatusPageIncidentUpdateInput, + ) => Effect.Effect + readonly updateIncident: ( + orgId: OrgId, + incidentId: StatusPageIncidentId, + patch: UpdateStatusPageIncidentInput, + ) => Effect.Effect + readonly removeIncident: ( + orgId: OrgId, + incidentId: StatusPageIncidentId, + ) => Effect.Effect +} + +const MAX_NAME_LENGTH = 120 +const MAX_TEXT_LENGTH = 2000 +const MAX_URL_LENGTH = 2048 +const BRAND_COLOR_PATTERN = /^#[0-9a-fA-F]{6}$/u +const INCIDENT_STATUSES: ReadonlySet = new Set([ + "investigating", + "identified", + "monitoring", + "resolved", +]) +const MAINTENANCE_STATUSES: ReadonlySet = new Set([ + "scheduled", + "in_progress", + "completed", +]) +const isClosedStatus = (status: StatusPageIncidentStatus) => status === "resolved" || status === "completed" + +const toPersistenceError = (error: unknown) => + new StatusPagePersistenceError({ + message: error instanceof Error ? error.message : "Status page persistence failed", + cause: error, + }) + +const invalid = (message: string) => new StatusPageValidationError({ message }) +const pageNotFound = (pageId: StatusPageId) => + new StatusPageNotFoundError({ pageId, message: "Status page not found" }) +const componentNotFound = (componentId: StatusPageComponentId) => + new StatusPageComponentNotFoundError({ componentId, message: "Status page component not found" }) +const incidentNotFound = (incidentId: StatusPageIncidentId) => + new StatusPageIncidentNotFoundError({ incidentId, message: "Status page incident not found" }) +const slugTaken = (slug: string) => + new StatusPageSlugTakenError({ slug, message: `The address '${slug}' is already taken` }) + +const isValidSlug = Schema.is(StatusPageSlug) + +const validateName = (raw: string, what: string) => { + const name = raw.trim() + if (name.length === 0) return Effect.fail(invalid(`${what} is required`)) + if (name.length > MAX_NAME_LENGTH) + return Effect.fail(invalid(`${what} must be at most ${MAX_NAME_LENGTH} characters`)) + return Effect.succeed(name) +} + +/** Optional free text: blank becomes null. */ +const validateText = (raw: string | null, what: string) => { + const text = raw?.trim() ?? "" + if (text.length > MAX_TEXT_LENGTH) + return Effect.fail(invalid(`${what} must be at most ${MAX_TEXT_LENGTH} characters`)) + return Effect.succeed(text.length === 0 ? null : text) +} + +const validateSlug = (slug: string) => + isValidSlug(slug) + ? Effect.succeed(slug) + : Effect.fail( + invalid("Slug must be 3-48 lowercase letters, digits or inner hyphens (e.g. 'acme-status')"), + ) + +const validateHttpsUrl = (raw: string | null, what: string) => { + const value = raw?.trim() ?? "" + if (value.length === 0) return Effect.succeed(null) + if (value.length > MAX_URL_LENGTH || !URL.canParse(value)) + return Effect.fail(invalid(`${what} must be a valid URL`)) + if (new URL(value).protocol !== "https:") return Effect.fail(invalid(`${what} must use https`)) + return Effect.succeed(value) +} + +const validateBrandColor = (raw: string | null) => { + if (raw === null) return Effect.succeed(null) + return BRAND_COLOR_PATTERN.test(raw) + ? Effect.succeed(raw.toLowerCase()) + : Effect.fail(invalid("Brand color must be a hex color like #1a2b3c")) +} + +const validateStatusForKind = (kind: StatusPageIncidentKind, status: StatusPageIncidentStatus) => { + const allowed = kind === "maintenance" ? MAINTENANCE_STATUSES : INCIDENT_STATUSES + return allowed.has(status) + ? Effect.void + : Effect.fail( + invalid( + `Status '${status}' does not apply to ${kind === "maintenance" ? "maintenance" : "an incident"}`, + ), + ) +} + +const validateWindow = ( + kind: StatusPageIncidentKind, + scheduledStart: number | null, + scheduledEnd: number | null, +) => { + if (kind === "incident") { + return scheduledStart === null && scheduledEnd === null + ? Effect.void + : Effect.fail(invalid("Only maintenance has a scheduled window")) + } + if (scheduledStart === null || scheduledEnd === null) + return Effect.fail(invalid("Maintenance needs a scheduled start and end")) + return scheduledEnd > scheduledStart + ? Effect.void + : Effect.fail(invalid("Maintenance must end after it starts")) +} + +const validateUpdateBody = (raw: string) => { + const body = raw.trim() + if (body.length === 0) return Effect.fail(invalid("An update needs a message")) + if (body.length > MAX_TEXT_LENGTH) + return Effect.fail(invalid(`An update must be at most ${MAX_TEXT_LENGTH} characters`)) + return Effect.succeed(body) +} + +const toPageRecord = (row: StatusPageRow): StatusPageRecord => ({ + id: row.id, + orgId: row.orgId, + name: row.name, + slug: row.slug, + description: row.description, + websiteUrl: row.websiteUrl, + logoUrl: row.logoUrl, + brandColor: row.brandColor, + showPoweredBy: row.showPoweredBy, + autoIncidents: row.autoIncidents, + breakGlassTokenSuffix: row.breakGlassTokenSuffix, + revision: row.revision, + publishedRevision: row.publishedRevision, + publishedAt: row.publishedAt, + lastPublishError: row.lastPublishError, + createdBy: row.createdBy, + createdAt: row.createdAt, + updatedAt: row.updatedAt, +}) + +const toComponentRecord = (row: StatusPageComponentRow): StatusPageComponentRecord => ({ + id: row.id, + pageId: row.pageId, + name: row.name, + description: row.description, + groupName: row.groupName, + position: row.position, + monitorId: row.monitorId, + manualStatus: row.manualStatus, + showUptime: row.showUptime, + createdAt: row.createdAt, + updatedAt: row.updatedAt, +}) + +const toUpdateRecord = (row: StatusPageIncidentUpdateRow): StatusPageIncidentUpdateRecord => ({ + id: row.id, + status: row.status, + body: row.body, + source: row.source, + createdBy: row.createdBy, + createdAt: row.createdAt, +}) + +/** Incidents with their component links and updates (newest first), in two org-scoped reads. */ +const loadIncidents = Effect.fnUntraced(function* ( + db: MapleDb, + orgId: OrgId, + rows: ReadonlyArray, +) { + const [first, ...rest] = rows.map((row) => row.id) + if (first === undefined) return [] + const links = yield* db.run( + PG.from(StatusPageIncidentComponents) + .select() + .where(($) => [$.orgId.eq(orgId), $.incidentId.in_(first, ...rest)]), + ) + const updates = yield* db.run( + PG.from(StatusPageIncidentUpdates) + .select() + .where(($) => [$.orgId.eq(orgId), $.incidentId.in_(first, ...rest)]) + .orderBy(["createdAt", "desc"], ["id", "desc"]), + ) + return rows.map((row): StatusPageIncidentRecord => ({ + id: row.id, + pageId: row.pageId, + kind: row.kind, + title: row.title, + impact: row.impact, + status: row.status, + published: row.published, + scheduledStart: row.scheduledStart, + scheduledEnd: row.scheduledEnd, + startedAt: row.startedAt, + resolvedAt: row.resolvedAt, + componentIds: links.filter((link) => link.incidentId === row.id).map((link) => link.componentId), + updates: updates.filter((update) => update.incidentId === row.id).map(toUpdateRecord), + createdBy: row.createdBy, + createdAt: row.createdAt, + updatedAt: row.updatedAt, + })) +}) + +const findPage = (db: MapleDb, orgId: OrgId, pageId: StatusPageId) => + db + .run( + PG.from(StatusPages) + .select() + .where(($) => [$.orgId.eq(orgId), $.id.eq(pageId)]) + .limit(1), + ) + .pipe( + Effect.flatMap((rows) => + rows[0] === undefined ? Effect.fail(pageNotFound(pageId)) : Effect.succeed(rows[0]), + ), + ) + +const findComponent = (db: MapleDb, orgId: OrgId, componentId: StatusPageComponentId) => + db + .run( + PG.from(StatusPageComponents) + .select() + .where(($) => [$.orgId.eq(orgId), $.id.eq(componentId)]) + .limit(1), + ) + .pipe( + Effect.flatMap((rows) => + rows[0] === undefined ? Effect.fail(componentNotFound(componentId)) : Effect.succeed(rows[0]), + ), + ) + +const findIncident = (db: MapleDb, orgId: OrgId, incidentId: StatusPageIncidentId) => + db + .run( + PG.from(StatusPageIncidents) + .select() + .where(($) => [$.orgId.eq(orgId), $.id.eq(incidentId)]) + .limit(1), + ) + .pipe( + Effect.flatMap((rows) => + rows[0] === undefined ? Effect.fail(incidentNotFound(incidentId)) : Effect.succeed(rows[0]), + ), + ) + +/** The slug check is deliberately global: one hosted address per slug across all orgs. */ +const ensureSlugFree = Effect.fnUntraced(function* ( + db: MapleDb, + slug: string, + exceptPageId: StatusPageId | null, +) { + const rows = yield* db.run( + PG.from(StatusPages) + .select("id") + .where(($) => [$.slug.eq(slug), exceptPageId === null ? undefined : $.id.neq(exceptPageId)]) + .limit(1) + .crossTenant(), + ) + if (rows.length > 0) return yield* slugTaken(slug) +}) + +const ensureMonitorInOrg = Effect.fnUntraced(function* ( + db: MapleDb, + orgId: OrgId, + monitorId: UptimeMonitorId, +) { + const rows = yield* db.run( + PG.from(UptimeMonitors) + .select("id") + .where(($) => [$.orgId.eq(orgId), $.id.eq(monitorId)]) + .limit(1), + ) + if (rows.length === 0) return yield* invalid(`Unknown uptime monitor ${monitorId}`) +}) + +/** Every id must name a component of this page. */ +const ensureComponentsOnPage = Effect.fnUntraced(function* ( + db: MapleDb, + orgId: OrgId, + pageId: StatusPageId, + componentIds: ReadonlyArray, +) { + const [first, ...rest] = componentIds + if (first === undefined) return + const found = yield* db.run( + PG.from(StatusPageComponents) + .select("id") + .where(($) => [$.orgId.eq(orgId), $.pageId.eq(pageId), $.id.in_(first, ...rest)]), + ) + const known = new Set(found.map((row) => row.id)) + const missing = componentIds.find((id) => !known.has(id)) + if (missing !== undefined) return yield* invalid(`Component ${missing} is not on this page`) +}) + +const replaceIncidentComponents = Effect.fnUntraced(function* ( + db: MapleDb, + orgId: OrgId, + incidentId: StatusPageIncidentId, + componentIds: ReadonlyArray, +) { + yield* db.run( + PG.deleteFrom(StatusPageIncidentComponents).where(($) => [ + $.orgId.eq(orgId), + $.incidentId.eq(incidentId), + ]), + ) + if (componentIds.length === 0) return + yield* db.run( + PG.insertInto(StatusPageIncidentComponents).values( + componentIds.map((componentId) => ({ incidentId, componentId, orgId })), + ), + ) +}) + +const newBreakGlassToken = () => `${STATUS_PAGE_BREAK_GLASS_PREFIX}${randomBytes(32).toString("base64url")}` + +export class StatusPagesService extends Context.Service()( + "@maple/backend/services/status-pages/StatusPagesService", + { + make: Effect.gen(function* () { + const database = yield* Database + const dbExecute = makeDbExecute(database, "StatusPagesService", toPersistenceError) + // A slug race that slips past the in-transaction check lands on the unique index. + const dbExecuteSlug = (slug: string) => + makeDbExecute(database, "StatusPagesService", (error: DatabaseError) => + postgresSqlState(error) === "23505" ? slugTaken(slug) : toPersistenceError(error), + ) + + const requirePage = (orgId: OrgId, pageId: StatusPageId) => + dbExecute((db) => findPage(db, orgId, pageId)).pipe(Effect.map(toPageRecord)) + + const requireIncident = (orgId: OrgId, incidentId: StatusPageIncidentId) => + dbExecute((db) => + findIncident(db, orgId, incidentId).pipe( + Effect.flatMap((row) => loadIncidents(db, orgId, [row])), + ), + ).pipe( + Effect.flatMap((records) => + records[0] === undefined + ? Effect.fail(incidentNotFound(incidentId)) + : Effect.succeed(records[0]), + ), + ) + + const list = Effect.fn("StatusPagesService.list")(function* (orgId: OrgId) { + yield* Effect.annotateCurrentSpan({ orgId }) + const rows = yield* dbExecute((db) => + db.run( + PG.from(StatusPages) + .select() + .where(($) => [$.orgId.eq(orgId)]) + .orderBy(["createdAt", "asc"], ["id", "asc"]), + ), + ) + return rows.map(toPageRecord) + }) + + const get = Effect.fn("StatusPagesService.get")(function* (orgId: OrgId, pageId: StatusPageId) { + yield* Effect.annotateCurrentSpan({ orgId, "maple.status_page.id": pageId }) + return yield* requirePage(orgId, pageId) + }) + + const create = Effect.fn("StatusPagesService.create")(function* ( + orgId: OrgId, + input: CreateStatusPageInput, + ) { + yield* Effect.annotateCurrentSpan({ orgId }) + const name = yield* validateName(input.name, "Name") + const slug = yield* validateSlug(input.slug) + const description = yield* validateText(input.description ?? null, "Description") + const websiteUrl = yield* validateHttpsUrl(input.websiteUrl ?? null, "Website URL") + const logoUrl = yield* validateHttpsUrl(input.logoUrl ?? null, "Logo URL") + const brandColor = yield* validateBrandColor(input.brandColor ?? null) + const now = yield* Clock.currentTimeMillis + const id = StatusPageId.make(randomUUID()) + yield* dbExecuteSlug(slug)((db) => + db.transaction( + Effect.gen(function* () { + yield* ensureSlugFree(db, slug, null) + // A new page starts at revision 1 > published 0, so the publisher picks it up. + yield* db.run( + PG.insertInto(StatusPages).values({ + id, + orgId, + name, + slug, + description, + websiteUrl, + logoUrl, + brandColor, + showPoweredBy: input.showPoweredBy ?? true, + autoIncidents: input.autoIncidents ?? "off", + breakGlassTokenSha256: null, + breakGlassTokenSuffix: null, + publishedAt: null, + lastPublishError: null, + createdBy: input.createdBy, + createdAt: now, + updatedAt: now, + }), + ) + }), + ), + ) + return yield* requirePage(orgId, id).pipe( + Effect.catchTag("@maple/http/errors/StatusPageNotFoundError", () => + Effect.fail( + new StatusPagePersistenceError({ message: "Created status page not found" }), + ), + ), + ) + }) + + const update = Effect.fn("StatusPagesService.update")(function* ( + orgId: OrgId, + pageId: StatusPageId, + patch: UpdateStatusPageInput, + ) { + yield* Effect.annotateCurrentSpan({ orgId, "maple.status_page.id": pageId }) + const name = patch.name === undefined ? undefined : yield* validateName(patch.name, "Name") + const slug = patch.slug === undefined ? undefined : yield* validateSlug(patch.slug) + const description = + patch.description === undefined + ? undefined + : yield* validateText(patch.description, "Description") + const websiteUrl = + patch.websiteUrl === undefined + ? undefined + : yield* validateHttpsUrl(patch.websiteUrl, "Website URL") + const logoUrl = + patch.logoUrl === undefined + ? undefined + : yield* validateHttpsUrl(patch.logoUrl, "Logo URL") + const brandColor = + patch.brandColor === undefined ? undefined : yield* validateBrandColor(patch.brandColor) + const now = yield* Clock.currentTimeMillis + yield* dbExecuteSlug(slug ?? "")((db) => + db.transaction( + Effect.gen(function* () { + yield* findPage(db, orgId, pageId) + if (slug !== undefined) yield* ensureSlugFree(db, slug, pageId) + yield* db.run( + PG.update(StatusPages) + .set({ + ...(name === undefined ? undefined : { name }), + ...(slug === undefined ? undefined : { slug }), + ...(description === undefined ? undefined : { description }), + ...(websiteUrl === undefined ? undefined : { websiteUrl }), + ...(logoUrl === undefined ? undefined : { logoUrl }), + ...(brandColor === undefined ? undefined : { brandColor }), + ...(patch.showPoweredBy === undefined + ? undefined + : { showPoweredBy: patch.showPoweredBy }), + ...(patch.autoIncidents === undefined + ? undefined + : { autoIncidents: patch.autoIncidents }), + updatedAt: now, + }) + .where(($) => [$.orgId.eq(orgId), $.id.eq(pageId)]), + ) + yield* bumpPage(db, orgId, pageId) + }), + ), + ) + return yield* requirePage(orgId, pageId) + }) + + /** Components, incidents and updates go with it through the foreign-key cascade. */ + const remove = Effect.fn("StatusPagesService.remove")(function* ( + orgId: OrgId, + pageId: StatusPageId, + ) { + yield* Effect.annotateCurrentSpan({ orgId, "maple.status_page.id": pageId }) + const removed = yield* dbExecute((db) => + db.run( + PG.deleteFrom(StatusPages) + .where(($) => [$.orgId.eq(orgId), $.id.eq(pageId)]) + .returning("id"), + ), + ) + if (removed.length === 0) return yield* pageNotFound(pageId) + return pageId + }) + + const rotateBreakGlassToken = Effect.fn("StatusPagesService.rotateBreakGlassToken")(function* ( + orgId: OrgId, + pageId: StatusPageId, + ) { + yield* Effect.annotateCurrentSpan({ orgId, "maple.status_page.id": pageId }) + const token = newBreakGlassToken() + const hash = yield* Effect.promise(() => sha256Base64Url(token)) + const now = yield* Clock.currentTimeMillis + yield* dbExecute((db) => + db.transaction( + Effect.gen(function* () { + yield* findPage(db, orgId, pageId) + yield* db.run( + PG.update(StatusPages) + .set({ + breakGlassTokenSha256: hash, + breakGlassTokenSuffix: token.slice(-6), + updatedAt: now, + }) + .where(($) => [$.orgId.eq(orgId), $.id.eq(pageId)]), + ) + // The edge checks the published hash, so a rotation must republish. + yield* bumpPage(db, orgId, pageId) + }), + ), + ) + return token + }) + + const revokeBreakGlassToken = Effect.fn("StatusPagesService.revokeBreakGlassToken")(function* ( + orgId: OrgId, + pageId: StatusPageId, + ) { + yield* Effect.annotateCurrentSpan({ orgId, "maple.status_page.id": pageId }) + const now = yield* Clock.currentTimeMillis + yield* dbExecute((db) => + db.transaction( + Effect.gen(function* () { + yield* findPage(db, orgId, pageId) + yield* db.run( + PG.update(StatusPages) + .set({ + breakGlassTokenSha256: null, + breakGlassTokenSuffix: null, + updatedAt: now, + }) + .where(($) => [$.orgId.eq(orgId), $.id.eq(pageId)]), + ) + yield* bumpPage(db, orgId, pageId) + }), + ), + ) + }) + + const selectComponents = (db: MapleDb, orgId: OrgId, pageId: StatusPageId) => + db.run( + PG.from(StatusPageComponents) + .select() + .where(($) => [$.orgId.eq(orgId), $.pageId.eq(pageId)]) + .orderBy(["position", "asc"], ["createdAt", "asc"], ["id", "asc"]), + ) + + const listComponents = Effect.fn("StatusPagesService.listComponents")(function* ( + orgId: OrgId, + pageId: StatusPageId, + ) { + yield* Effect.annotateCurrentSpan({ orgId, "maple.status_page.id": pageId }) + const rows = yield* dbExecute((db) => + findPage(db, orgId, pageId).pipe(Effect.andThen(selectComponents(db, orgId, pageId))), + ) + return rows.map(toComponentRecord) + }) + + const createComponent = Effect.fn("StatusPagesService.createComponent")(function* ( + orgId: OrgId, + pageId: StatusPageId, + input: CreateStatusPageComponentInput, + ) { + yield* Effect.annotateCurrentSpan({ orgId, "maple.status_page.id": pageId }) + const name = yield* validateName(input.name, "Name") + const description = yield* validateText(input.description ?? null, "Description") + const groupName = yield* validateText(input.groupName ?? null, "Group name") + const monitorId = input.monitorId ?? null + const now = yield* Clock.currentTimeMillis + const id = StatusPageComponentId.make(randomUUID()) + const row = yield* dbExecute((db) => + db.transaction( + Effect.gen(function* () { + yield* findPage(db, orgId, pageId) + if (monitorId !== null) yield* ensureMonitorInOrg(db, orgId, monitorId) + const existing = yield* selectComponents(db, orgId, pageId) + const inserted = yield* db.run( + PG.insertInto(StatusPageComponents) + .values({ + id, + orgId, + pageId, + name, + description, + groupName, + position: existing.length, + monitorId, + manualStatus: input.manualStatus ?? null, + showUptime: input.showUptime ?? true, + createdAt: now, + updatedAt: now, + }) + .returning(), + ) + yield* bumpPage(db, orgId, pageId) + return inserted + }), + ), + ) + const created = row[0] + if (created === undefined) + return yield* new StatusPagePersistenceError({ + message: "Created component not returned", + }) + return toComponentRecord(created) + }) + + const updateComponent = Effect.fn("StatusPagesService.updateComponent")(function* ( + orgId: OrgId, + componentId: StatusPageComponentId, + patch: UpdateStatusPageComponentInput, + ) { + yield* Effect.annotateCurrentSpan({ orgId, "maple.status_page.component_id": componentId }) + const name = patch.name === undefined ? undefined : yield* validateName(patch.name, "Name") + const description = + patch.description === undefined + ? undefined + : yield* validateText(patch.description, "Description") + const groupName = + patch.groupName === undefined + ? undefined + : yield* validateText(patch.groupName, "Group name") + const monitorId = patch.monitorId + const now = yield* Clock.currentTimeMillis + const rows = yield* dbExecute((db) => + db.transaction( + Effect.gen(function* () { + const current = yield* findComponent(db, orgId, componentId) + if (monitorId !== undefined && monitorId !== null) + yield* ensureMonitorInOrg(db, orgId, monitorId) + const updated = yield* db.run( + PG.update(StatusPageComponents) + .set({ + ...(name === undefined ? undefined : { name }), + ...(description === undefined ? undefined : { description }), + ...(groupName === undefined ? undefined : { groupName }), + ...(monitorId === undefined ? undefined : { monitorId }), + ...(patch.manualStatus === undefined + ? undefined + : { manualStatus: patch.manualStatus }), + ...(patch.showUptime === undefined + ? undefined + : { showUptime: patch.showUptime }), + updatedAt: now, + }) + .where(($) => [$.orgId.eq(orgId), $.id.eq(componentId)]) + .returning(), + ) + yield* bumpPage(db, orgId, current.pageId) + return updated + }), + ), + ) + const row = rows[0] + if (row === undefined) return yield* componentNotFound(componentId) + return toComponentRecord(row) + }) + + const removeComponent = Effect.fn("StatusPagesService.removeComponent")(function* ( + orgId: OrgId, + componentId: StatusPageComponentId, + ) { + yield* Effect.annotateCurrentSpan({ orgId, "maple.status_page.component_id": componentId }) + yield* dbExecute((db) => + db.transaction( + Effect.gen(function* () { + const current = yield* findComponent(db, orgId, componentId) + yield* db.run( + PG.deleteFrom(StatusPageComponents).where(($) => [ + $.orgId.eq(orgId), + $.id.eq(componentId), + ]), + ) + yield* bumpPage(db, orgId, current.pageId) + }), + ), + ) + return componentId + }) + + const reorderComponents = Effect.fn("StatusPagesService.reorderComponents")(function* ( + orgId: OrgId, + pageId: StatusPageId, + componentIds: ReadonlyArray, + ) { + yield* Effect.annotateCurrentSpan({ orgId, "maple.status_page.id": pageId }) + const now = yield* Clock.currentTimeMillis + const rows = yield* dbExecute((db) => + db.transaction( + Effect.gen(function* () { + yield* findPage(db, orgId, pageId) + const existing = yield* selectComponents(db, orgId, pageId) + const requested = new Set(componentIds) + const sameSet = + requested.size === componentIds.length && + requested.size === existing.length && + existing.every((row) => requested.has(row.id)) + if (!sameSet) + return yield* invalid( + "componentIds must list every component of the page exactly once", + ) + yield* Effect.forEach( + componentIds, + (componentId, position) => + db.run( + PG.update(StatusPageComponents) + .set({ position, updatedAt: now }) + .where(($) => [ + $.orgId.eq(orgId), + $.pageId.eq(pageId), + $.id.eq(componentId), + ]), + ), + { discard: true }, + ) + yield* bumpPage(db, orgId, pageId) + return yield* selectComponents(db, orgId, pageId) + }), + ), + ) + return rows.map(toComponentRecord) + }) + + const listIncidents = Effect.fn("StatusPagesService.listIncidents")(function* ( + orgId: OrgId, + pageId: StatusPageId, + options?: ListStatusPageIncidentsOptions, + ) { + yield* Effect.annotateCurrentSpan({ orgId, "maple.status_page.id": pageId }) + const includeResolved = options?.includeResolved === true + return yield* dbExecute((db) => + Effect.gen(function* () { + yield* findPage(db, orgId, pageId) + const rows = yield* db.run( + PG.from(StatusPageIncidents) + .select() + .where(($) => [ + $.orgId.eq(orgId), + $.pageId.eq(pageId), + includeResolved ? undefined : $.resolvedAt.isNull(), + ]) + .orderBy(["startedAt", "desc"], ["id", "desc"]), + ) + return yield* loadIncidents(db, orgId, rows) + }), + ) + }) + + const getIncident = Effect.fn("StatusPagesService.getIncident")(function* ( + orgId: OrgId, + incidentId: StatusPageIncidentId, + ) { + yield* Effect.annotateCurrentSpan({ orgId, "maple.status_page.incident_id": incidentId }) + return yield* requireIncident(orgId, incidentId) + }) + + const createIncident = Effect.fn("StatusPagesService.createIncident")(function* ( + orgId: OrgId, + pageId: StatusPageId, + input: CreateStatusPageIncidentInput, + ) { + yield* Effect.annotateCurrentSpan({ + orgId, + "maple.status_page.id": pageId, + "maple.status_page.incident_kind": input.kind, + }) + const title = yield* validateName(input.title, "Title") + const body = yield* validateUpdateBody(input.body) + const scheduledStart = input.scheduledStart ?? null + const scheduledEnd = input.scheduledEnd ?? null + yield* validateStatusForKind(input.kind, input.status) + yield* validateWindow(input.kind, scheduledStart, scheduledEnd) + const componentIds = [...new Set(input.componentIds)] + const now = yield* Clock.currentTimeMillis + const id = StatusPageIncidentId.make(randomUUID()) + const createdBy = input.createdBy ?? null + yield* dbExecute((db) => + db.transaction( + Effect.gen(function* () { + yield* findPage(db, orgId, pageId) + yield* ensureComponentsOnPage(db, orgId, pageId, componentIds) + yield* db.run( + PG.insertInto(StatusPageIncidents).values({ + id, + orgId, + pageId, + kind: input.kind, + title, + impact: input.impact, + status: input.status, + published: input.published ?? true, + alertIncidentId: null, + scheduledStart, + scheduledEnd, + startedAt: scheduledStart ?? now, + resolvedAt: isClosedStatus(input.status) ? now : null, + createdBy, + createdAt: now, + updatedAt: now, + }), + ) + yield* db.run( + PG.insertInto(StatusPageIncidentUpdates).values({ + id: StatusPageIncidentUpdateId.make(randomUUID()), + orgId, + incidentId: id, + status: input.status, + body, + source: input.source, + createdBy, + overlayEntryId: null, + createdAt: now, + }), + ) + yield* replaceIncidentComponents(db, orgId, id, componentIds) + yield* bumpPage(db, orgId, pageId) + }), + ), + ) + return yield* requireIncident(orgId, id).pipe( + Effect.catchTag("@maple/http/errors/StatusPageIncidentNotFoundError", () => + Effect.fail( + new StatusPagePersistenceError({ message: "Created incident not found" }), + ), + ), + ) + }) + + const postUpdate = Effect.fn("StatusPagesService.postUpdate")(function* ( + orgId: OrgId, + incidentId: StatusPageIncidentId, + input: PostStatusPageIncidentUpdateInput, + ) { + yield* Effect.annotateCurrentSpan({ orgId, "maple.status_page.incident_id": incidentId }) + const body = yield* validateUpdateBody(input.body) + const componentIds = + input.componentIds === undefined ? undefined : [...new Set(input.componentIds)] + const now = yield* Clock.currentTimeMillis + yield* dbExecute((db) => + db.transaction( + Effect.gen(function* () { + const current = yield* findIncident(db, orgId, incidentId) + yield* validateStatusForKind(current.kind, input.status) + if (componentIds !== undefined) + yield* ensureComponentsOnPage(db, orgId, current.pageId, componentIds) + const closed = isClosedStatus(input.status) + yield* db.run( + PG.update(StatusPageIncidents) + .set({ + status: input.status, + ...(input.impact === undefined + ? undefined + : { impact: input.impact }), + resolvedAt: closed ? (current.resolvedAt ?? now) : null, + updatedAt: now, + }) + .where(($) => [$.orgId.eq(orgId), $.id.eq(incidentId)]), + ) + yield* db.run( + PG.insertInto(StatusPageIncidentUpdates).values({ + id: StatusPageIncidentUpdateId.make(randomUUID()), + orgId, + incidentId, + status: input.status, + body, + source: input.source, + createdBy: input.createdBy ?? null, + overlayEntryId: null, + createdAt: now, + }), + ) + if (componentIds !== undefined) + yield* replaceIncidentComponents(db, orgId, incidentId, componentIds) + yield* bumpPage(db, orgId, current.pageId) + }), + ), + ) + return yield* requireIncident(orgId, incidentId) + }) + + const updateIncident = Effect.fn("StatusPagesService.updateIncident")(function* ( + orgId: OrgId, + incidentId: StatusPageIncidentId, + patch: UpdateStatusPageIncidentInput, + ) { + yield* Effect.annotateCurrentSpan({ orgId, "maple.status_page.incident_id": incidentId }) + const title = + patch.title === undefined ? undefined : yield* validateName(patch.title, "Title") + const componentIds = + patch.componentIds === undefined ? undefined : [...new Set(patch.componentIds)] + const now = yield* Clock.currentTimeMillis + yield* dbExecute((db) => + db.transaction( + Effect.gen(function* () { + const current = yield* findIncident(db, orgId, incidentId) + const scheduledStart = + patch.scheduledStart === undefined + ? current.scheduledStart + : patch.scheduledStart + const scheduledEnd = + patch.scheduledEnd === undefined ? current.scheduledEnd : patch.scheduledEnd + const windowChanged = + patch.scheduledStart !== undefined || patch.scheduledEnd !== undefined + if (windowChanged) + yield* validateWindow(current.kind, scheduledStart, scheduledEnd) + if (componentIds !== undefined) + yield* ensureComponentsOnPage(db, orgId, current.pageId, componentIds) + yield* db.run( + PG.update(StatusPageIncidents) + .set({ + ...(title === undefined ? undefined : { title }), + ...(patch.impact === undefined + ? undefined + : { impact: patch.impact }), + ...(patch.published === undefined + ? undefined + : { published: patch.published }), + ...(windowChanged ? { scheduledStart, scheduledEnd } : undefined), + ...(windowChanged && + current.kind === "maintenance" && + scheduledStart !== null + ? { startedAt: scheduledStart } + : undefined), + updatedAt: now, + }) + .where(($) => [$.orgId.eq(orgId), $.id.eq(incidentId)]), + ) + if (componentIds !== undefined) + yield* replaceIncidentComponents(db, orgId, incidentId, componentIds) + yield* bumpPage(db, orgId, current.pageId) + }), + ), + ) + return yield* requireIncident(orgId, incidentId) + }) + + const removeIncident = Effect.fn("StatusPagesService.removeIncident")(function* ( + orgId: OrgId, + incidentId: StatusPageIncidentId, + ) { + yield* Effect.annotateCurrentSpan({ orgId, "maple.status_page.incident_id": incidentId }) + yield* dbExecute((db) => + db.transaction( + Effect.gen(function* () { + const current = yield* findIncident(db, orgId, incidentId) + yield* db.run( + PG.deleteFrom(StatusPageIncidents).where(($) => [ + $.orgId.eq(orgId), + $.id.eq(incidentId), + ]), + ) + yield* bumpPage(db, orgId, current.pageId) + }), + ), + ) + return incidentId + }) + + return { + list, + get, + create, + update, + remove, + rotateBreakGlassToken, + revokeBreakGlassToken, + listComponents, + createComponent, + updateComponent, + removeComponent, + reorderComponents, + listIncidents, + getIncident, + createIncident, + postUpdate, + updateIncident, + removeIncident, + } satisfies StatusPagesServiceApi + }), + }, +) { + static readonly layer = Layer.effect(this, this.make) +} diff --git a/packages/domain/src/http/index.ts b/packages/domain/src/http/index.ts index b02ff89486..4d279c9c6a 100644 --- a/packages/domain/src/http/index.ts +++ b/packages/domain/src/http/index.ts @@ -57,6 +57,7 @@ export * from "./scrape-targets" export * from "./scraper-internal" export * from "./security-audit" export * from "./uptime" +export * from "./status-pages" export * from "./session-replay" export * from "./share" export * from "./share-chart" diff --git a/packages/domain/src/http/status-pages.ts b/packages/domain/src/http/status-pages.ts new file mode 100644 index 0000000000..2e3b9e178c --- /dev/null +++ b/packages/domain/src/http/status-pages.ts @@ -0,0 +1,190 @@ +import { Schema } from "effect" +import { + OrgId, + StatusPageAutoIncidents, + StatusPageComponentId, + StatusPageComponentStatus, + StatusPageId, + StatusPageIncidentId, + StatusPageIncidentImpact, + StatusPageIncidentKind, + StatusPageIncidentStatus, + StatusPageIncidentUpdateId, + StatusPageUpdateSource, + UptimeMonitorId, + UserId, +} from "../primitives" +import { HttpTaggedError } from "./error-policy" + +export class StatusPageNotFoundError extends HttpTaggedError()( + "@maple/http/errors/StatusPageNotFoundError", + { + pageId: StatusPageId, + message: Schema.String, + }, + { + status: 404, + code: "status_page_not_found", + title: "Status page not found", + message: "No such status page.", + param: "id", + retry: "never", + recovery: "none", + exposure: "redacted", + }, +) {} + +export class StatusPageComponentNotFoundError extends HttpTaggedError()( + "@maple/http/errors/StatusPageComponentNotFoundError", + { + componentId: StatusPageComponentId, + message: Schema.String, + }, + { + status: 404, + code: "status_page_component_not_found", + title: "Status page component not found", + message: "No such status page component.", + param: "componentId", + retry: "never", + recovery: "none", + exposure: "redacted", + }, +) {} + +export class StatusPageIncidentNotFoundError extends HttpTaggedError()( + "@maple/http/errors/StatusPageIncidentNotFoundError", + { + incidentId: StatusPageIncidentId, + message: Schema.String, + }, + { + status: 404, + code: "status_page_incident_not_found", + title: "Status page incident not found", + message: "No such status page incident.", + param: "incidentId", + retry: "never", + recovery: "none", + exposure: "redacted", + }, +) {} + +export class StatusPageValidationError extends HttpTaggedError()( + "@maple/http/errors/StatusPageValidationError", + { + message: Schema.String, + }, + { + status: 400, + code: "status_page_invalid", + title: "Invalid status page", + retry: "never", + recovery: "fix_request", + exposure: "public_message", + }, +) {} + +/** Slugs are global: the hosted address is `.`. */ +export class StatusPageSlugTakenError extends HttpTaggedError()( + "@maple/http/errors/StatusPageSlugTakenError", + { + slug: Schema.String, + message: Schema.String, + }, + { + status: 409, + code: "status_page_slug_taken", + title: "Status page address is taken", + param: "slug", + retry: "never", + recovery: "fix_request", + exposure: "public_message", + }, +) {} + +export class StatusPagePersistenceError extends HttpTaggedError()( + "@maple/http/errors/StatusPagePersistenceError", + { + message: Schema.String, + cause: Schema.optionalKey(Schema.Defect()), + }, + { + status: 503, + code: "status_pages_unavailable", + title: "Status pages are temporarily unavailable", + message: "Status pages are temporarily unavailable. Retry in a few seconds.", + retry: "backoff", + recovery: "retry", + exposure: "redacted", + }, +) {} + +/** A page as its owners see it; the break-glass hash never leaves the service. */ +export const StatusPageRecord = Schema.Struct({ + id: StatusPageId, + orgId: OrgId, + name: Schema.String, + slug: Schema.String, + description: Schema.NullOr(Schema.String), + websiteUrl: Schema.NullOr(Schema.String), + logoUrl: Schema.NullOr(Schema.String), + brandColor: Schema.NullOr(Schema.String), + showPoweredBy: Schema.Boolean, + autoIncidents: StatusPageAutoIncidents, + breakGlassTokenSuffix: Schema.NullOr(Schema.String), + revision: Schema.Number, + publishedRevision: Schema.Number, + publishedAt: Schema.NullOr(Schema.Number), + lastPublishError: Schema.NullOr(Schema.String), + createdBy: UserId, + createdAt: Schema.Number, + updatedAt: Schema.Number, +}) +export type StatusPageRecord = Schema.Schema.Type + +export const StatusPageComponentRecord = Schema.Struct({ + id: StatusPageComponentId, + pageId: StatusPageId, + name: Schema.String, + description: Schema.NullOr(Schema.String), + groupName: Schema.NullOr(Schema.String), + position: Schema.Number, + monitorId: Schema.NullOr(UptimeMonitorId), + manualStatus: Schema.NullOr(StatusPageComponentStatus), + showUptime: Schema.Boolean, + createdAt: Schema.Number, + updatedAt: Schema.Number, +}) +export type StatusPageComponentRecord = Schema.Schema.Type + +export const StatusPageIncidentUpdateRecord = Schema.Struct({ + id: StatusPageIncidentUpdateId, + status: StatusPageIncidentStatus, + body: Schema.String, + source: StatusPageUpdateSource, + createdBy: Schema.NullOr(UserId), + createdAt: Schema.Number, +}) +export type StatusPageIncidentUpdateRecord = Schema.Schema.Type + +export const StatusPageIncidentRecord = Schema.Struct({ + id: StatusPageIncidentId, + pageId: StatusPageId, + kind: StatusPageIncidentKind, + title: Schema.String, + impact: StatusPageIncidentImpact, + status: StatusPageIncidentStatus, + published: Schema.Boolean, + scheduledStart: Schema.NullOr(Schema.Number), + scheduledEnd: Schema.NullOr(Schema.Number), + startedAt: Schema.Number, + resolvedAt: Schema.NullOr(Schema.Number), + componentIds: Schema.Array(StatusPageComponentId), + /** Newest first. */ + updates: Schema.Array(StatusPageIncidentUpdateRecord), + createdBy: Schema.NullOr(UserId), + createdAt: Schema.Number, + updatedAt: Schema.Number, +}) +export type StatusPageIncidentRecord = Schema.Schema.Type From 2936fba59e5210e595c8180a95d10998b4c86621 Mon Sep 17 00:00:00 2001 From: Makisuo Date: Wed, 28 Oct 2026 16:04:05 +0100 Subject: [PATCH 02/15] feat(status-pages): publisher and edge store The publisher turns a status page into its snapshot and writes it to edge storage (KV, mirrored to R2) whenever the page's revision lags, skipping the write when nothing a visitor sees changed (status_pages.published_hash, new migration). A sweep in the alerting minute cron catches up lagging and stale pages, writes the heartbeat and imports break-glass overlay entries into Postgres idempotently. Uptime alert incidents open and resolve status page incidents on pages that opted in to auto incidents. The alerting Worker runs this only when STATUS_PAGES_KV and STATUS_PAGES_BUCKET are bound, and otherwise logs and skips. --- apps/alerting/src/scheduled.test.ts | 6 +- apps/alerting/src/scheduled.ts | 47 ++- .../src/services/alerts/AlertsService.ts | 54 +++ .../alerts/AlertsService.uptime.test.ts | 86 ++++ .../status-pages/StatusPagePublisher.test.ts | 180 +++++++++ .../status-pages/StatusPagePublisher.ts | 195 +++++++++ .../status-pages/StatusPageReconciler.test.ts | 169 ++++++++ .../status-pages/StatusPageReconciler.ts | 278 +++++++++++++ .../StatusPageSnapshotBuilder.test.ts | 239 ++++++++++++ .../status-pages/StatusPageSnapshotBuilder.ts | 369 ++++++++++++++++++ .../services/status-pages/StatusPageStore.ts | 249 ++++++++++++ .../services/status-pages/auto-incidents.ts | 184 +++++++++ .../services/status-pages/publish-errors.ts | 14 + .../status-pages/status-page-test-support.ts | 221 +++++++++++ 14 files changed, 2288 insertions(+), 3 deletions(-) create mode 100644 packages/backend/src/services/status-pages/StatusPagePublisher.test.ts create mode 100644 packages/backend/src/services/status-pages/StatusPagePublisher.ts create mode 100644 packages/backend/src/services/status-pages/StatusPageReconciler.test.ts create mode 100644 packages/backend/src/services/status-pages/StatusPageReconciler.ts create mode 100644 packages/backend/src/services/status-pages/StatusPageSnapshotBuilder.test.ts create mode 100644 packages/backend/src/services/status-pages/StatusPageSnapshotBuilder.ts create mode 100644 packages/backend/src/services/status-pages/StatusPageStore.ts create mode 100644 packages/backend/src/services/status-pages/auto-incidents.ts create mode 100644 packages/backend/src/services/status-pages/publish-errors.ts create mode 100644 packages/backend/src/services/status-pages/status-page-test-support.ts diff --git a/apps/alerting/src/scheduled.test.ts b/apps/alerting/src/scheduled.test.ts index f5bbc45f6f..13f86ff909 100644 --- a/apps/alerting/src/scheduled.test.ts +++ b/apps/alerting/src/scheduled.test.ts @@ -20,7 +20,10 @@ const cronCases = [ ], ["*/15 * * * *", ["digest", "supabaseInsights"]], ["0 * * * *", ["serviceMapRollup", "supabaseDiscovery", "awsConnections"]], - ["* * * * *", ["alert", "alertLongQuery", "error", "escalation", "fixVerification", "supabaseLogs"]], + [ + "* * * * *", + ["alert", "alertLongQuery", "error", "escalation", "fixVerification", "statusPages", "supabaseLogs"], + ], ] as const /** Every tick recording its name when it runs, with `overrides` swapped in. */ @@ -45,6 +48,7 @@ const recordingTicks = ( prReviewPostMerge: tick("prReviewPostMerge"), railwayMetrics: tick("railwayMetrics"), serviceMapRollup: tick("serviceMapRollup"), + statusPages: tick("statusPages"), supabaseDiscovery: tick("supabaseDiscovery"), supabaseLogs: tick("supabaseLogs"), supabaseInsights: tick("supabaseInsights"), diff --git a/apps/alerting/src/scheduled.ts b/apps/alerting/src/scheduled.ts index 5c7f7ab311..e2f2a10617 100644 --- a/apps/alerting/src/scheduled.ts +++ b/apps/alerting/src/scheduled.ts @@ -28,17 +28,30 @@ import { SupabaseConnectionService } from "@maple/backend/services/integrations/ import { SupabaseInsightsService } from "@maple/backend/services/integrations/SupabaseInsightsService" import { SupabaseLogsService } from "@maple/backend/services/integrations/SupabaseLogsService" import { ServiceMapRollupService } from "@maple/backend/services/dashboards/ServiceMapRollupService" +import { StatusPagePublisher } from "@maple/backend/services/status-pages/StatusPagePublisher" +import { StatusPageReconciler } from "@maple/backend/services/status-pages/StatusPageReconciler" +import { StatusPageStore, statusPageBindingsFrom } from "@maple/backend/services/status-pages/StatusPageStore" import { layerMapleDbConnection } from "@maple/backend/platform/pg-connection-source" import { summarizeCause } from "@maple/backend/platform/describe-cause" import { withPgConnectionScope } from "@maple/backend/platform/pg-connection-scope" import { EmailSender, type EmailSenderClient } from "@maple/backend/platform/bindings" import { type ChatSessionNamespace, chatSessionsLayerIfBound } from "@maple/backend/platform/chat-sessions" import { envPorts } from "@maple/backend/platform/env-ports" -import { Cause, Effect, Layer, Match, Option } from "effect" +import { Cause, Clock, Effect, Layer, Match, Option } from "effect" import { FetchHttpClient } from "effect/http" import { isLaneOf, scheduleFor, type TickLane } from "./schedule.ts" import type { AlertingWorkerEnv } from "./worker.ts" +/** The status page publisher, only when its KV and R2 bindings exist on this Worker. */ +const statusPagesLayerIfBound = (env: AlertingWorkerEnv) => + Option.match(statusPageBindingsFrom(env), { + onNone: () => Layer.empty, + onSome: ({ kv, bucket }) => + Layer.mergeAll(StatusPagePublisher.layer, StatusPageReconciler.layer).pipe( + Layer.provide(StatusPageStore.fromBindings(kv, bucket)), + ), + }) + /** * The tick graph for one fire. Deliberately without a tracer or logger of its * own: those come from the SDK the Worker bridge builds into each fire's @@ -73,6 +86,7 @@ export const layerTick = ( // Read by `maybeEnqueueTriage` when present; its absence means every // incident opened here is investigated unclassified. IncidentClassifier.layer, + statusPagesLayerIfBound(env), ).pipe( Layer.provide(PullRequestLookupLive), Layer.provide(Layer.mergeAll(Env.layer, DatabasePgLive, EdgeCacheServiceLive, FetchHttpClient.layer)), @@ -400,6 +414,32 @@ const prReviewPostMergeTick = makeTick( : undefined, ) +/** Imports break-glass updates, publishes lagging pages and writes the heartbeat; each step isolated. */ +const statusPagesTick = Effect.gen(function* () { + const publisher = yield* Effect.serviceOption(StatusPagePublisher) + const reconciler = yield* Effect.serviceOption(StatusPageReconciler) + if (Option.isNone(publisher) || Option.isNone(reconciler)) { + return yield* Effect.logInfo("Status page storage is not bound; skipping status page publishing") + } + const nowMs = yield* Clock.currentTimeMillis + yield* makeTick(publisher.value.heartbeat(nowMs), "status_page_heartbeat", () => undefined) + yield* makeTick(reconciler.value.reconcileAll(), "status_page_reconcile", (result) => + result.imported + result.failed > 0 + ? { statusPageImported: result.imported, statusPageFailures: result.failed } + : undefined, + ) + yield* makeTick(publisher.value.sweep(nowMs), "status_page_publish", (result) => + result.due > 0 + ? { + statusPagesDue: result.due, + statusPagesWritten: result.written, + statusPagesUnchanged: result.unchanged, + statusPageFailures: result.failed, + } + : undefined, + ) +}) + export interface ScheduledTickPrograms { readonly alert: Effect.Effect readonly alertLongQuery: Effect.Effect @@ -416,6 +456,7 @@ export interface ScheduledTickPrograms { readonly prReviewPostMerge: Effect.Effect readonly railwayMetrics: Effect.Effect readonly serviceMapRollup: Effect.Effect + readonly statusPages: Effect.Effect readonly supabaseDiscovery: Effect.Effect readonly supabaseLogs: Effect.Effect readonly supabaseInsights: Effect.Effect @@ -430,7 +471,8 @@ const laneProgram = (lane: TickLane, ticks: ScheduledTickPrograms): Effect // a window this minute's error tick just refuted must already be settled // when the verification tick looks at it. Match.when("error", () => Effect.andThen(ticks.error, ticks.fixVerification)), - Match.when("escalation", () => ticks.escalation), + // Status pages ride the escalation lane: the minute cron is one connection under its cap. + Match.when("escalation", () => Effect.andThen(ticks.escalation, ticks.statusPages)), Match.when("anomaly", () => ticks.anomaly), Match.when("cloudflare_analytics", () => ticks.cloudflareAnalytics), Match.when("gcp_metrics", () => ticks.gcpMetrics), @@ -516,6 +558,7 @@ export const scheduledTicks: ScheduledTickPrograms = { prReviewPostMerge: prReviewPostMergeTick, railwayMetrics: railwayMetricsTick, serviceMapRollup: serviceMapRollupTick, + statusPages: statusPagesTick, supabaseDiscovery: supabaseDiscoveryTick, supabaseLogs: supabaseLogsTick, supabaseInsights: supabaseInsightsTick, diff --git a/packages/backend/src/services/alerts/AlertsService.ts b/packages/backend/src/services/alerts/AlertsService.ts index 89a084ccf1..efa0d6b499 100644 --- a/packages/backend/src/services/alerts/AlertsService.ts +++ b/packages/backend/src/services/alerts/AlertsService.ts @@ -161,6 +161,10 @@ import { makeAlertDestinationDelivery, parseAlertDestinationEncryptionKey } from import { AlertReadModelsService, type AlertReadModelsServiceApi } from "./AlertReadModelsService" import { AlertRulesService, makeAlertRulePersistence, type AlertRulesServiceApi } from "./AlertRulesService" import { UptimeAlertSource } from "@maple/backend/services/uptime/UptimeAlertSource" +import { + openForAlertIncident, + resolveForAlertIncident, +} from "@maple/backend/services/status-pages/auto-incidents" import { decodeStoredAlertRuleMetadata, storedRuleCondition, @@ -653,6 +657,27 @@ export class AlertsService extends Context.Service( + signalType: string, + write: (db: MapleDb) => Effect.Effect, + ) => + signalType !== "uptime" + ? Effect.void + : database + .execute((db) => db.transaction(write(db))) + .pipe( + Effect.asVoid, + Effect.catch((error) => + Effect.logWarning("Status page auto incident sync failed").pipe( + Effect.annotateLogs({ + "error.message": summarizeCause(Cause.fail(error)), + }), + ), + ), + ) + /** * Run `write` only while `claim` still owns the rule. The claim row is * locked FOR UPDATE and re-read in the same transaction as the write, so a @@ -3126,6 +3151,14 @@ export class AlertsService extends Context.Service + openForAlertIncident(db, { + orgId: row.orgId, + ruleId: row.id, + alertIncidentId: incidentId, + at: timestamp, + }), + ) if (lifecycle.notificationSuppression === "flapping") { yield* Effect.logInfo("Skipping trigger notification for flapping incident").pipe( Effect.annotateLogs({ @@ -3291,6 +3324,13 @@ export class AlertsService extends Context.Service + resolveForAlertIncident(db, { + orgId: row.orgId, + alertIncidentId: openIncident.id, + at: timestamp, + }), + ) if (openIncident.holdReason !== null) { yield* Metric.update(AlertingMetrics.incidentsResolvedAfterHoldTotal, 1) } @@ -3577,6 +3617,13 @@ export class AlertsService extends Context.Service + resolveForAlertIncident(db, { + orgId, + alertIncidentId: incident.id, + at: timestamp, + }), + ) if (incident.holdReason !== null) { yield* Metric.update(AlertingMetrics.incidentsResolvedAfterHoldTotal, 1) } @@ -3750,6 +3797,13 @@ export class AlertsService extends Context.Service + resolveForAlertIncident(db, { + orgId, + alertIncidentId: incident.id, + at: timestamp, + }), + ) if (incident.holdReason !== null) { yield* Metric.update(AlertingMetrics.incidentsResolvedAfterHoldTotal, 1) } diff --git a/packages/backend/src/services/alerts/AlertsService.uptime.test.ts b/packages/backend/src/services/alerts/AlertsService.uptime.test.ts index d0c3c1f9ea..e7d68f66e7 100644 --- a/packages/backend/src/services/alerts/AlertsService.uptime.test.ts +++ b/packages/backend/src/services/alerts/AlertsService.uptime.test.ts @@ -205,6 +205,92 @@ describe("AlertsService uptime rules", () => { }).pipe(Effect.provide(makeLayer(testDb, makeWarehouseStub({}), { fetch: fetchImpl }))) }) + it.effect("an uptime incident opens and resolves status page incidents on opted-in pages", () => { + const testDb = createTestDb(trackedDbs) + return Effect.gen(function* () { + yield* TestClock.setTime(DEFAULT_CLOCK_EPOCH_MS) + const alerts = yield* AlertsService + const orgId = asOrgId("org_uptime_status_page") + const { monitorId } = yield* seedUptimeMonitor(testDb, { + orgId, + name: "Checkout health", + serviceName: "checkout-api", + destinationIds: [], + config: httpMonitorConfig, + status: "down", + }) + const pages = [ + { id: crypto.randomUUID(), slug: "published-page", autoIncidents: "publish" }, + { id: crypto.randomUUID(), slug: "draft-page", autoIncidents: "draft" }, + { id: crypto.randomUUID(), slug: "off-page", autoIncidents: "off" }, + ] + yield* Effect.forEach(pages, (page) => + Effect.promise(() => + executeSql( + testDb, + `INSERT INTO status_pages (id, org_id, name, slug, auto_incidents, created_by, created_at, updated_at) + VALUES ($1, $2, 'Acme', $3, $4, 'user_uptime', now(), now())`, + [page.id, orgId, page.slug, page.autoIncidents], + ).then(() => + executeSql( + testDb, + `INSERT INTO status_page_components (id, org_id, page_id, name, monitor_id, created_at, updated_at) + VALUES ($1, $2, $3, 'Checkout', $4, now(), now())`, + [crypto.randomUUID(), orgId, page.id, monitorId], + ), + ), + ), + ) + const pageIncidents = () => + Effect.promise(() => + testDb.pglite.query<{ + slug: string + title: string + status: string + published: boolean + alert_incident_id: string | null + updates: number + }>( + `SELECT p.slug, i.title, i.status, i.published, i.alert_incident_id, + (SELECT count(*)::int FROM status_page_incident_updates u WHERE u.incident_id = i.id) AS updates + FROM status_page_incidents i JOIN status_pages p ON p.id = i.page_id ORDER BY p.slug`, + ), + ).pipe(Effect.map((result) => result.rows)) + + yield* alerts.runSchedulerTick() + const alertIncidentId = (yield* alerts.listIncidents(orgId)).incidents[0]?.id + assert.deepStrictEqual(yield* pageIncidents(), [ + { + slug: "draft-page", + title: "Checkout is down", + status: "investigating", + published: false, + alert_incident_id: alertIncidentId ?? null, + updates: 1, + }, + { + slug: "published-page", + title: "Checkout is down", + status: "investigating", + published: true, + alert_incident_id: alertIncidentId ?? null, + updates: 1, + }, + ]) + + yield* setMonitorStatus(testDb, monitorId, "up") + yield* TestClock.adjust(Duration.minutes(1)) + yield* alerts.runSchedulerTick() + assert.deepStrictEqual( + (yield* pageIncidents()).map((row) => [row.slug, row.status, row.updates]), + [ + ["draft-page", "resolved", 2], + ["published-page", "resolved", 2], + ], + ) + }).pipe(Effect.provide(makeLayer(testDb, makeWarehouseStub({})))) + }) + it.effect("a monitor created before backing rules gets one, once, and then fires", () => { const testDb = createTestDb(trackedDbs) return Effect.gen(function* () { diff --git a/packages/backend/src/services/status-pages/StatusPagePublisher.test.ts b/packages/backend/src/services/status-pages/StatusPagePublisher.test.ts new file mode 100644 index 0000000000..34973116d0 --- /dev/null +++ b/packages/backend/src/services/status-pages/StatusPagePublisher.test.ts @@ -0,0 +1,180 @@ +import { afterEach, assert, describe, it } from "@effect/vitest" +import { Effect, Exit, Layer, Option } from "effect" +import { TestClock } from "effect/testing" +import { decodeStatusPageSnapshotJson, StatusPageKeys } from "@maple/domain/status-page" +import { cleanupTestDbs, createTestDb, type TestDb } from "@maple/backend/platform/test-pglite" +import { StatusPagePublisher } from "./StatusPagePublisher" +import { StatusPageStore, type InMemoryStatusPageStore } from "./StatusPageStore" +import { bumpRevision, NOW, newPageId, readPage, seedComponent, seedPage } from "./status-page-test-support" + +const trackedDbs: TestDb[] = [] +afterEach(() => cleanupTestDbs(trackedDbs)) + +/** Runs `body` with a publisher over a fresh database and an in-memory store. */ +const withPublisher = ( + body: (ctx: { + readonly testDb: TestDb + readonly store: InMemoryStatusPageStore + readonly publisher: StatusPagePublisher["Service"] + }) => Effect.Effect, +) => + Effect.gen(function* () { + yield* TestClock.setTime(NOW) + const testDb = createTestDb(trackedDbs) + const store = yield* StatusPageStore.makeInMemory + const layer = StatusPagePublisher.layer.pipe( + Layer.provide(Layer.succeed(StatusPageStore, store.api)), + Layer.provide(testDb.layer), + ) + return yield* StatusPagePublisher.use((publisher) => body({ testDb, store, publisher })).pipe( + Effect.provide(layer), + ) + }) + +describe("StatusPagePublisher.publish", () => { + it.effect("writes the snapshot and routes, then advances published_revision", () => + withPublisher(({ testDb, store, publisher }) => + Effect.gen(function* () { + const pageId = newPageId() + yield* seedPage(testDb, { id: pageId, slug: "acme", revision: 3 }) + yield* seedComponent(testDb, { id: crypto.randomUUID(), pageId, name: "API", position: 0 }) + + const result = yield* publisher.publish(pageId) + assert.isTrue(result.written) + const { kv, objects } = yield* store.contents + const snapshot = decodeStatusPageSnapshotJson(kv.get(StatusPageKeys.snapshot(pageId))) + assert.isTrue(Option.isSome(snapshot)) + if (Option.isSome(snapshot)) { + assert.strictEqual(snapshot.value.revision, 3) + assert.deepStrictEqual( + snapshot.value.components.map((c) => c.name), + ["API"], + ) + } + assert.strictEqual( + objects.get(StatusPageKeys.snapshotObject(pageId)), + kv.get(StatusPageKeys.snapshot(pageId)), + ) + assert.strictEqual(kv.get(StatusPageKeys.slug("acme")), pageId) + const row = yield* readPage(testDb, pageId) + assert.strictEqual(row?.published_revision, "3") + assert.isNotNull(row?.published_hash) + }), + ), + ) + + it.effect("skips the edge write when nothing visible changed", () => + withPublisher(({ testDb, store, publisher }) => + Effect.gen(function* () { + const pageId = newPageId() + yield* seedPage(testDb, { id: pageId, slug: "acme-same" }) + yield* publisher.publish(pageId) + const first = (yield* store.contents).kv.get(StatusPageKeys.snapshot(pageId)) + + yield* bumpRevision(testDb, pageId) + yield* TestClock.adjust("1 minute") + const second = yield* publisher.publish(pageId) + assert.isFalse(second.written) + assert.strictEqual((yield* store.contents).kv.get(StatusPageKeys.snapshot(pageId)), first) + assert.strictEqual((yield* readPage(testDb, pageId))?.published_revision, "2") + }), + ), + ) + + it.effect("records the store failure and leaves published_revision behind", () => + withPublisher(({ testDb, store, publisher }) => + Effect.gen(function* () { + const pageId = newPageId() + yield* seedPage(testDb, { id: pageId, slug: "acme-down", revision: 2 }) + yield* store.setFailing(true) + const exit = yield* Effect.exit(publisher.publish(pageId)) + assert.isTrue(Exit.isFailure(exit)) + const row = yield* readPage(testDb, pageId) + assert.strictEqual(row?.published_revision, "0") + assert.include(row?.last_publish_error ?? "", "kv.put") + + yield* store.setFailing(false) + yield* publisher.publish(pageId) + assert.isNull((yield* readPage(testDb, pageId))?.last_publish_error) + }), + ), + ) +}) + +describe("StatusPagePublisher.sweep", () => { + it.effect("publishes only lagging or stale pages", () => + withPublisher(({ testDb, store, publisher }) => + Effect.gen(function* () { + const lagging = newPageId() + const current = newPageId() + const stale = newPageId() + yield* seedPage(testDb, { + id: lagging, + slug: "lagging", + revision: 2, + publishedRevision: 1, + publishedAt: NOW, + }) + yield* seedPage(testDb, { + id: current, + slug: "current", + revision: 1, + publishedRevision: 1, + publishedAt: NOW, + }) + yield* seedPage(testDb, { + id: stale, + slug: "stale", + revision: 1, + publishedRevision: 1, + publishedAt: NOW - 20 * 60_000, + }) + + const result = yield* publisher.sweep(NOW) + assert.strictEqual(result.due, 2) + const { kv } = yield* store.contents + assert.isTrue(kv.has(StatusPageKeys.snapshot(lagging))) + assert.isTrue(kv.has(StatusPageKeys.snapshot(stale))) + assert.isFalse(kv.has(StatusPageKeys.snapshot(current))) + }), + ), + ) + + it.effect("keeps going past a page that fails", () => + withPublisher(({ testDb, store, publisher }) => + Effect.gen(function* () { + const first = newPageId() + const second = newPageId() + yield* seedPage(testDb, { id: first, slug: "first" }) + yield* seedPage(testDb, { id: second, slug: "second" }) + yield* store.setFailing((key) => key.includes(first)) + + const partial = yield* publisher.sweep(NOW) + assert.deepStrictEqual(partial, { due: 2, written: 1, unchanged: 0, failed: 1 }) + assert.isTrue((yield* store.contents).kv.has(StatusPageKeys.snapshot(second))) + + yield* store.setFailing(false) + const recovered = yield* publisher.sweep(NOW) + assert.deepStrictEqual(recovered, { due: 1, written: 1, unchanged: 0, failed: 0 }) + assert.deepStrictEqual(yield* publisher.sweep(NOW), { + due: 0, + written: 0, + unchanged: 0, + failed: 0, + }) + }), + ), + ) + + it.effect("writes the heartbeat", () => + withPublisher(({ store, publisher }) => + Effect.gen(function* () { + yield* publisher.heartbeat(NOW) + assert.strictEqual( + (yield* store.contents).kv.get(StatusPageKeys.heartbeat), + JSON.stringify({ at: NOW }), + ) + }), + ), + ) +}) diff --git a/packages/backend/src/services/status-pages/StatusPagePublisher.ts b/packages/backend/src/services/status-pages/StatusPagePublisher.ts new file mode 100644 index 0000000000..35742355d6 --- /dev/null +++ b/packages/backend/src/services/status-pages/StatusPagePublisher.ts @@ -0,0 +1,195 @@ +import * as PG from "@maple-dev/effect-orm/postgres" +import { StatusPages } from "@maple/db/tables" +import type { StatusPageId } from "@maple/domain/primitives" +import { Clock, Context, Effect, Layer } from "effect" +import { Database } from "@maple/backend/platform/DatabaseLive" +import { makeDbExecute, makePersistenceErrorMapper } from "@maple/backend/platform/db-execute" +import { bumpPage } from "./outbox" +import { StatusPageGoneError, StatusPagePublishDbError } from "./publish-errors" +import { StatusPageSnapshotBuilder } from "./StatusPageSnapshotBuilder" +import { StatusPageStore, type StatusPageStoreError } from "./StatusPageStore" + +/** Pages published longer ago than this are republished so bars and freshness roll. */ +export const STATUS_PAGE_REPUBLISH_AFTER_MS = 15 * 60_000 +/** Pages one sweep publishes at most; the rest wait for the next minute. */ +export const STATUS_PAGE_SWEEP_BATCH = 100 +const SWEEP_CONCURRENCY = 4 + +export interface StatusPagePublishResult { + readonly pageId: StatusPageId + readonly revision: number + /** False when the content matched the last publish and the edge write was skipped. */ + readonly written: boolean +} + +export interface StatusPageSweepResult { + readonly due: number + readonly written: number + readonly unchanged: number + readonly failed: number +} + +export type StatusPagePublishError = StatusPageGoneError | StatusPagePublishDbError | StatusPageStoreError + +export interface StatusPagePublisherApi { + readonly publish: (pageId: StatusPageId) => Effect.Effect + /** Publishes pages whose revision lags or whose last publish is stale; a failing page is logged and skipped. */ + readonly sweep: (nowMs: number) => Effect.Effect + readonly heartbeat: (nowMs: number) => Effect.Effect +} + +const toHex = (buffer: ArrayBuffer) => + Array.from(new Uint8Array(buffer), (byte) => byte.toString(16).padStart(2, "0")).join("") + +/** SHA-256 hex of the text. */ +export const sha256Hex = (text: string) => + Effect.promise(() => crypto.subtle.digest("SHA-256", new TextEncoder().encode(text))).pipe( + Effect.map(toHex), + ) + +const toDbError = makePersistenceErrorMapper(StatusPagePublishDbError, "Status page publish write failed") + +export class StatusPagePublisher extends Context.Service()( + "@maple/backend/services/status-pages/StatusPagePublisher", + { + make: Effect.gen(function* () { + const database = yield* Database + const store = yield* StatusPageStore + const builder = yield* StatusPageSnapshotBuilder + const dbExecute = makeDbExecute(database, "StatusPagePublisher", toDbError) + + const recordFailure = (pageId: StatusPageId, error: StatusPageStoreError) => + dbExecute((db) => + db.run( + PG.update(StatusPages) + .set({ lastPublishError: error.message }) + .where(($) => [$.id.eq(pageId)]), + ), + ).pipe(Effect.ignore, Effect.andThen(Effect.fail(error))) + + const publish = Effect.fn("StatusPagePublisher.publish")(function* (pageId: StatusPageId) { + yield* Effect.annotateCurrentSpan({ "maple.status_page.id": pageId }) + const [current] = yield* dbExecute((db) => + db.run( + PG.from(StatusPages) + .select(($) => ({ + orgId: $.orgId, + slug: $.slug, + revision: $.revision, + publishedHash: $.publishedHash, + })) + .where(($) => [$.id.eq(pageId)]) + .limit(1), + ), + ) + if (current === undefined) { + return yield* new StatusPageGoneError({ + message: `Status page ${pageId} not found`, + pageId, + }) + } + const nowMs = yield* Clock.currentTimeMillis + const snapshot = yield* builder.build(pageId, nowMs) + // The read revision, not the build's: a bump during the build republishes next sweep. + const revision = current.revision + // What a visitor sees: a bump or a stale republish with no visible change writes nothing. + const hash = yield* sha256Hex(JSON.stringify({ ...snapshot, generatedAt: 0, revision: 0 })) + const written = hash !== current.publishedHash + if (written) { + yield* store.putSnapshot(pageId, JSON.stringify(snapshot)).pipe( + Effect.andThen(store.putRoutes(pageId, { slug: snapshot.page.slug, hostnames: [] })), + Effect.catch((error) => recordFailure(pageId, error)), + ) + } + const advanced = yield* dbExecute((db) => + db.run( + PG.update(StatusPages) + .set({ + publishedRevision: revision, + publishedAt: nowMs, + publishedHash: hash, + lastPublishError: null, + }) + .where(($) => [$.id.eq(pageId), $.publishedRevision.lt(revision)]) + .returning("id"), + ), + ) + if (advanced.length === 0) { + // A concurrent publish got further; ours may have overwritten its snapshot, so go again. + yield* dbExecute((db) => + db.run( + PG.update(StatusPages) + .set({ publishedAt: nowMs, lastPublishError: null }) + .where(($) => [$.id.eq(pageId)]), + ), + ) + if (written) yield* dbExecute((db) => bumpPage(db, current.orgId, pageId)) + } + yield* Effect.annotateCurrentSpan({ + "maple.status_page.revision": revision, + "maple.status_page.written": written, + }) + return { pageId, revision, written } + }) + + const sweep = Effect.fn("StatusPagePublisher.sweep")(function* (nowMs: number) { + const staleBefore = nowMs - STATUS_PAGE_REPUBLISH_AFTER_MS + const due = yield* dbExecute((db) => + db.run( + PG.from(StatusPages) + .select(($) => ({ id: $.id })) + .where(($) => [ + PG.or( + $.revision.gt($.publishedRevision), + $.publishedAt.isNull(), + $.publishedAt.lt(staleBefore), + ), + ]) + .orderBy(["id", "asc"]) + .limit(STATUS_PAGE_SWEEP_BATCH), + ), + ) + const outcomes = yield* Effect.forEach( + due, + ({ id }) => + publish(id).pipe( + Effect.map((result) => + result.written ? ("written" as const) : ("unchanged" as const), + ), + Effect.catch((error) => + Effect.logWarning("Status page publish failed").pipe( + Effect.annotateLogs({ + "maple.status_page.id": id, + "error.type": error._tag, + "error.message": error.message, + }), + Effect.as("failed" as const), + ), + ), + ), + { concurrency: SWEEP_CONCURRENCY }, + ) + const result: StatusPageSweepResult = { + due: due.length, + written: outcomes.filter((o) => o === "written").length, + unchanged: outcomes.filter((o) => o === "unchanged").length, + failed: outcomes.filter((o) => o === "failed").length, + } + yield* Effect.annotateCurrentSpan({ + "maple.status_page.due_count": result.due, + "maple.status_page.written_count": result.written, + "maple.status_page.failed_count": result.failed, + }) + return result + }) + + const heartbeat = Effect.fn("StatusPagePublisher.heartbeat")(function* (nowMs: number) { + yield* store.putHeartbeat(nowMs) + }) + + return { publish, sweep, heartbeat } + }), + }, +) { + static readonly layer = Layer.effect(this, this.make).pipe(Layer.provide(StatusPageSnapshotBuilder.layer)) +} diff --git a/packages/backend/src/services/status-pages/StatusPageReconciler.test.ts b/packages/backend/src/services/status-pages/StatusPageReconciler.test.ts new file mode 100644 index 0000000000..7cba2f18bc --- /dev/null +++ b/packages/backend/src/services/status-pages/StatusPageReconciler.test.ts @@ -0,0 +1,169 @@ +import { afterEach, assert, describe, it } from "@effect/vitest" +import { Effect, Exit, Layer } from "effect" +import { overlayIncidentId, StatusPageKeys, type StatusPageOverlayEntry } from "@maple/domain/status-page" +import { cleanupTestDbs, createTestDb, type TestDb } from "@maple/backend/platform/test-pglite" +import { StatusPageReconciler } from "./StatusPageReconciler" +import { StatusPageStore } from "./StatusPageStore" +import { newPageId, NOW, readPage, seedComponent, seedPage } from "./status-page-test-support" + +const trackedDbs: TestDb[] = [] +afterEach(() => cleanupTestDbs(trackedDbs)) + +const entry = (overrides: Partial & Pick) => ({ + incidentId: null, + title: null, + status: "investigating" as const, + impact: "major" as const, + body: "Update", + componentIds: [], + ...overrides, +}) + +const incidentsOf = (testDb: TestDb, pageId: string) => + Effect.promise(() => + testDb.pglite.query<{ + id: string + title: string + status: string + resolved: boolean + components: number + }>( + `SELECT i.id, i.title, i.status, i.resolved_at IS NOT NULL AS resolved, + (SELECT count(*)::int FROM status_page_incident_components c WHERE c.incident_id = i.id) AS components + FROM status_page_incidents i WHERE i.page_id = $1`, + [pageId], + ), + ).pipe(Effect.map((result) => result.rows)) + +const updatesOf = (testDb: TestDb, incidentId: string) => + Effect.promise(() => + testDb.pglite.query<{ overlay_entry_id: string; source: string }>( + `SELECT overlay_entry_id, source FROM status_page_incident_updates WHERE incident_id = $1 ORDER BY created_at`, + [incidentId], + ), + ).pipe(Effect.map((result) => result.rows)) + +describe("StatusPageReconciler", () => { + it.effect("imports a new incident and its follow-up once, then clears only those entries", () => + Effect.gen(function* () { + const testDb = createTestDb(trackedDbs) + const store = yield* StatusPageStore.makeInMemory + const pageId = newPageId() + const componentId = crypto.randomUUID() + yield* seedPage(testDb, { id: pageId, slug: "acme-bg", breakGlass: true }) + yield* seedComponent(testDb, { id: componentId, pageId, name: "API", position: 0 }) + + const opening = crypto.randomUUID() + const followUp = crypto.randomUUID() + const entries = [ + // Out of order on purpose: import goes by `at`. + entry({ + id: followUp, + at: NOW + 60_000, + incidentId: overlayIncidentId(opening), + status: "resolved", + body: "Fixed", + }), + entry({ + id: opening, + at: NOW, + title: "Login broken", + componentIds: [componentId, "not-a-component"], + }), + ] + yield* store.putKv(StatusPageKeys.overlay(pageId), JSON.stringify({ entries })) + + const layer = StatusPageReconciler.layer.pipe( + Layer.provide(Layer.succeed(StatusPageStore, store.api)), + Layer.provide(testDb.layer), + ) + const reconciler = yield* StatusPageReconciler.pipe(Effect.provide(layer)) + + const first = yield* reconciler.reconcile(pageId) + assert.strictEqual(first.imported, 2) + const incidents = yield* incidentsOf(testDb, pageId) + assert.deepStrictEqual( + incidents.map(({ title, status, resolved, components }) => ({ + title, + status, + resolved, + components, + })), + [{ title: "Login broken", status: "resolved", resolved: true, components: 1 }], + ) + const incidentId = incidents[0]?.id ?? "" + assert.deepStrictEqual(yield* updatesOf(testDb, incidentId), [ + { overlay_entry_id: opening, source: "break_glass" }, + { overlay_entry_id: followUp, source: "break_glass" }, + ]) + assert.isFalse((yield* store.contents).kv.has(StatusPageKeys.overlay(pageId))) + assert.strictEqual((yield* readPage(testDb, pageId))?.revision, "3") + + // The edge re-posts the same entries plus a new one (a re-import must not duplicate). + const later = crypto.randomUUID() + yield* store.putKv( + StatusPageKeys.overlay(pageId), + JSON.stringify({ + entries: [ + ...entries, + entry({ + id: later, + at: NOW + 120_000, + incidentId, + status: "monitoring", + body: "Watching", + }), + ], + }), + ) + const second = yield* reconciler.reconcileAll() + assert.deepStrictEqual(second, { pages: 1, imported: 3, failed: 0 }) + assert.strictEqual((yield* incidentsOf(testDb, pageId)).length, 1) + assert.strictEqual((yield* updatesOf(testDb, incidentId)).length, 3) + assert.strictEqual((yield* incidentsOf(testDb, pageId))[0]?.status, "monitoring") + }), + ) + + it.effect("leaves the overlay when clearing it fails, and a rerun does not import twice", () => + Effect.gen(function* () { + const testDb = createTestDb(trackedDbs) + const store = yield* StatusPageStore.makeInMemory + const pageId = newPageId() + yield* seedPage(testDb, { id: pageId, slug: "acme-keep", breakGlass: true }) + const only = entry({ id: crypto.randomUUID(), at: NOW, title: "Outage" }) + yield* store.putKv(StatusPageKeys.overlay(pageId), JSON.stringify({ entries: [only] })) + yield* store.setFailing((key) => key === StatusPageKeys.overlay(pageId)) + + const layer = StatusPageReconciler.layer.pipe( + Layer.provide(Layer.succeed(StatusPageStore, store.api)), + Layer.provide(testDb.layer), + ) + const reconciler = yield* StatusPageReconciler.pipe(Effect.provide(layer)) + const failed = yield* Effect.exit(reconciler.reconcile(pageId)) + assert.isTrue(Exit.isFailure(failed)) + assert.isTrue((yield* store.contents).kv.has(StatusPageKeys.overlay(pageId))) + + yield* store.setFailing(false) + assert.strictEqual((yield* reconciler.reconcile(pageId)).imported, 1) + assert.strictEqual((yield* incidentsOf(testDb, pageId)).length, 1) + assert.isFalse((yield* store.contents).kv.has(StatusPageKeys.overlay(pageId))) + }), + ) +}) + +describe("StatusPageStore.removeOverlayEntries", () => { + it.effect("removes only the given entries", () => + Effect.gen(function* () { + const store = yield* StatusPageStore.makeInMemory + const pageId = newPageId() + const a = entry({ id: "a", at: NOW }) + const b = entry({ id: "b", at: NOW + 1 }) + yield* store.putKv(StatusPageKeys.overlay(pageId), JSON.stringify({ entries: [a, b] })) + yield* store.api.removeOverlayEntries(pageId, ["a"]) + assert.deepStrictEqual( + (yield* store.api.readOverlay(pageId)).entries.map((e) => e.id), + ["b"], + ) + }), + ) +}) diff --git a/packages/backend/src/services/status-pages/StatusPageReconciler.ts b/packages/backend/src/services/status-pages/StatusPageReconciler.ts new file mode 100644 index 0000000000..54b8ff04ae --- /dev/null +++ b/packages/backend/src/services/status-pages/StatusPageReconciler.ts @@ -0,0 +1,278 @@ +import { randomUUID } from "node:crypto" +import * as PG from "@maple-dev/effect-orm/postgres" +import type { MapleDb } from "@maple/db/client" +import { + StatusPageComponents, + StatusPageIncidentComponents, + StatusPageIncidents, + StatusPageIncidentUpdates, + StatusPages, +} from "@maple/db/tables" +import { + type OrgId, + type StatusPageComponentId, + type StatusPageId, + StatusPageIncidentId, + StatusPageIncidentUpdateId, +} from "@maple/domain/primitives" +import type { StatusPageOverlayEntry } from "@maple/domain/status-page" +import { Context, Effect, Layer, Option, Ref, Result, Schema } from "effect" +import { Database } from "@maple/backend/platform/DatabaseLive" +import { makeDbExecute, makePersistenceErrorMapper } from "@maple/backend/platform/db-execute" +import { bumpPage } from "./outbox" +import { StatusPagePublishDbError } from "./publish-errors" +import { StatusPageStore, type StatusPageStoreError } from "./StatusPageStore" + +// Imports break-glass overlay entries (posted to the status Worker while Maple was down) into +// Postgres. Each update keeps `overlay_entry_id`, unique, so importing an entry twice is a no-op. + +const RECONCILE_BATCH = 100 +const RECONCILE_CONCURRENCY = 4 +const OVERLAY_INCIDENT_PREFIX = "bg-" + +export interface StatusPageReconcileResult { + /** Entries now in Postgres (including ones a previous run already imported). */ + readonly imported: number +} + +export interface StatusPageReconcileAllResult { + readonly pages: number + readonly imported: number + readonly failed: number +} + +export type StatusPageReconcileError = StatusPagePublishDbError | StatusPageStoreError + +export interface StatusPageReconcilerApi { + readonly reconcile: ( + pageId: StatusPageId, + ) => Effect.Effect + /** Reconciles every page with a break-glass token; a failing page is logged and skipped. */ + readonly reconcileAll: () => Effect.Effect +} + +const isResolved = (status: StatusPageOverlayEntry["status"]) => + status === "resolved" || status === "completed" +const decodeIncidentId = Schema.decodeUnknownOption(StatusPageIncidentId) + +interface PageScope { + readonly orgId: OrgId + readonly pageId: StatusPageId + readonly componentIds: ReadonlyArray +} + +/** The real incident an entry targets, if it still exists on this page. */ +const findTarget = (db: MapleDb, scope: PageScope, entry: StatusPageOverlayEntry) => + Effect.gen(function* () { + if (entry.incidentId === null) return Option.none() + if (entry.incidentId.startsWith(OVERLAY_INCIDENT_PREFIX)) { + // An incident the overlay opened: its first update carries the opening entry's id. + const openingEntryId = entry.incidentId.slice(OVERLAY_INCIDENT_PREFIX.length) + const rows = yield* db.run( + PG.from(StatusPageIncidentUpdates) + .select(($) => ({ incidentId: $.incidentId })) + .where(($) => [$.orgId.eq(scope.orgId), $.overlayEntryId.eq(openingEntryId)]) + .limit(1), + ) + return Option.fromNullishOr(rows[0]?.incidentId) + } + const id = decodeIncidentId(entry.incidentId) + if (Option.isNone(id)) return Option.none() + const rows = yield* db.run( + PG.from(StatusPageIncidents) + .select(($) => ({ id: $.id })) + .where(($) => [$.orgId.eq(scope.orgId), $.pageId.eq(scope.pageId), $.id.eq(id.value)]) + .limit(1), + ) + return Option.fromNullishOr(rows[0]?.id) + }) + +const linkComponents = ( + db: MapleDb, + scope: PageScope, + incidentId: StatusPageIncidentId, + requested: ReadonlyArray, +) => { + const componentIds = scope.componentIds.filter((id) => requested.includes(id)) + if (componentIds.length === 0) return Effect.void + return db.run( + PG.insertInto(StatusPageIncidentComponents) + .values(componentIds.map((componentId) => ({ incidentId, componentId, orgId: scope.orgId }))) + .onConflictDoNothing(), + ) +} + +/** Imports one entry in its own transaction. */ +const importEntry = (db: MapleDb, scope: PageScope, entry: StatusPageOverlayEntry) => + db.transaction( + Effect.gen(function* () { + const already = yield* db.run( + PG.from(StatusPageIncidentUpdates) + .select(($) => ({ id: $.id })) + .where(($) => [$.orgId.eq(scope.orgId), $.overlayEntryId.eq(entry.id)]) + .limit(1), + ) + if (already.length > 0) return + const resolvedAt = isResolved(entry.status) ? entry.at : null + const target = yield* findTarget(db, scope, entry) + // An entry whose target is gone opens its own incident rather than losing the customer's words. + const incidentId = yield* Option.match(target, { + onSome: (id) => + db + .run( + PG.update(StatusPageIncidents) + .set({ + status: entry.status, + impact: entry.impact, + resolvedAt, + updatedAt: entry.at, + ...(entry.title === null ? undefined : { title: entry.title }), + }) + .where(($) => [$.orgId.eq(scope.orgId), $.id.eq(id)]), + ) + .pipe(Effect.as(id)), + onNone: () => { + const id = StatusPageIncidentId.make(randomUUID()) + return db + .run( + PG.insertInto(StatusPageIncidents).values({ + id, + orgId: scope.orgId, + pageId: scope.pageId, + kind: "incident", + title: entry.title ?? "Service disruption", + impact: entry.impact, + status: entry.status, + published: true, + alertIncidentId: null, + scheduledStart: null, + scheduledEnd: null, + startedAt: entry.at, + resolvedAt, + createdBy: null, + createdAt: entry.at, + updatedAt: entry.at, + }), + ) + .pipe(Effect.as(id)) + }, + }) + yield* linkComponents(db, scope, incidentId, entry.componentIds) + yield* db.run( + PG.insertInto(StatusPageIncidentUpdates) + .values({ + id: StatusPageIncidentUpdateId.make(randomUUID()), + orgId: scope.orgId, + incidentId, + status: entry.status, + body: entry.body, + source: "break_glass", + createdBy: null, + overlayEntryId: entry.id, + createdAt: entry.at, + }) + .onConflictDoNothing(), + ) + yield* bumpPage(db, scope.orgId, scope.pageId) + }), + ) + +const toDbError = makePersistenceErrorMapper(StatusPagePublishDbError, "Status page overlay import failed") + +export class StatusPageReconciler extends Context.Service()( + "@maple/backend/services/status-pages/StatusPageReconciler", + { + make: Effect.gen(function* () { + const database = yield* Database + const store = yield* StatusPageStore + const dbExecute = makeDbExecute(database, "StatusPageReconciler", toDbError) + + const reconcile = Effect.fn("StatusPageReconciler.reconcile")(function* (pageId: StatusPageId) { + yield* Effect.annotateCurrentSpan({ "maple.status_page.id": pageId }) + const overlay = yield* store.readOverlay(pageId) + if (overlay.entries.length === 0) return { imported: 0 } + const scope = yield* dbExecute((db) => + Effect.gen(function* () { + const [page] = yield* db.run( + PG.from(StatusPages) + .select(($) => ({ orgId: $.orgId })) + .where(($) => [$.id.eq(pageId)]) + .limit(1), + ) + if (page === undefined) return null + const components = yield* db.run( + PG.from(StatusPageComponents) + .select(($) => ({ id: $.id })) + .where(($) => [$.orgId.eq(page.orgId), $.pageId.eq(pageId)]), + ) + return { orgId: page.orgId, pageId, componentIds: components.map((c) => c.id) } + }), + ) + if (scope === null) return { imported: 0 } + const entries = overlay.entries.toSorted((a, b) => a.at - b.at) + const imported = yield* Ref.make>([]) + // In order: a later entry can target the incident an earlier one opened. + const outcome = yield* Effect.result( + Effect.forEach( + entries, + (entry) => + dbExecute((db) => importEntry(db, scope, entry)).pipe( + Effect.andThen(Ref.update(imported, (ids) => [...ids, entry.id])), + ), + { discard: true }, + ), + ) + const importedIds = yield* Ref.get(imported) + yield* store.removeOverlayEntries(pageId, importedIds) + yield* Effect.annotateCurrentSpan({ "maple.status_page.imported_count": importedIds.length }) + if (Result.isFailure(outcome)) return yield* Effect.fail(outcome.failure) + return { imported: importedIds.length } + }) + + const reconcileAll = Effect.fn("StatusPageReconciler.reconcileAll")(function* () { + const pages = yield* dbExecute((db) => + db.run( + PG.from(StatusPages) + .select(($) => ({ id: $.id })) + .where(($) => [$.breakGlassTokenSha256.isNotNull()]) + .orderBy(["id", "asc"]) + .limit(RECONCILE_BATCH), + ), + ) + const outcomes = yield* Effect.forEach( + pages, + ({ id }) => + reconcile(id).pipe( + Effect.map((result) => result.imported), + Effect.catch((error) => + Effect.logWarning("Status page overlay import failed").pipe( + Effect.annotateLogs({ + "maple.status_page.id": id, + "error.type": error._tag, + "error.message": error.message, + }), + Effect.as(null), + ), + ), + ), + { concurrency: RECONCILE_CONCURRENCY }, + ) + const result: StatusPageReconcileAllResult = { + pages: pages.length, + imported: outcomes.reduce((sum, n) => sum + (n ?? 0), 0), + failed: outcomes.filter((n) => n === null).length, + } + yield* Effect.annotateCurrentSpan({ + "maple.status_page.page_count": result.pages, + "maple.status_page.imported_count": result.imported, + "maple.status_page.failed_count": result.failed, + }) + return result + }) + + return { reconcile, reconcileAll } + }), + }, +) { + static readonly layer = Layer.effect(this, this.make) +} diff --git a/packages/backend/src/services/status-pages/StatusPageSnapshotBuilder.test.ts b/packages/backend/src/services/status-pages/StatusPageSnapshotBuilder.test.ts new file mode 100644 index 0000000000..c1576c6dcf --- /dev/null +++ b/packages/backend/src/services/status-pages/StatusPageSnapshotBuilder.test.ts @@ -0,0 +1,239 @@ +import { afterEach, assert, describe, it } from "@effect/vitest" +import { Effect, Exit } from "effect" +import { StatusPageComponentId, StatusPageIncidentId, UptimeMonitorId } from "@maple/domain/primitives" +import { utcDay } from "@maple/domain/status-page" +import { cleanupTestDbs, createTestDb, type TestDb } from "@maple/backend/platform/test-pglite" +import { StatusPageSnapshotBuilder, toSnapshot, type SnapshotInput } from "./StatusPageSnapshotBuilder" +import { + DAY_MS, + newPageId, + NOW, + seedComponent, + seedDaily, + seedIncident, + seedMonitor, + seedPage, +} from "./status-page-test-support" + +const trackedDbs: TestDb[] = [] +afterEach(() => cleanupTestDbs(trackedDbs)) + +const COMPONENT = StatusPageComponentId.make("22222222-2222-4222-8222-222222222222") +const MONITOR = UptimeMonitorId.make("33333333-3333-4333-8333-333333333333") +const INCIDENT = StatusPageIncidentId.make("44444444-4444-4444-8444-444444444444") + +const API_COMPONENT: SnapshotInput["components"][number] = { + id: COMPONENT, + name: "API", + description: null, + groupName: null, + monitorId: MONITOR, + manualStatus: null, + showUptime: true, + monitorStatus: "up", +} + +const baseInput = (overrides: Partial = {}): SnapshotInput => ({ + page: { + id: newPageId(), + name: "Acme", + slug: "acme", + description: null, + websiteUrl: null, + logoUrl: null, + brandColor: null, + showPoweredBy: true, + breakGlassTokenSha256: null, + revision: 3, + }, + components: [API_COMPONENT], + daily: [], + incidents: [], + updates: [], + incidentComponents: [], + nowMs: NOW, + ...overrides, +}) + +const incident = (overrides: Partial = {}) => ({ + id: INCIDENT, + kind: "incident" as const, + title: "API errors", + impact: "minor" as const, + status: "investigating" as const, + published: true, + startedAt: NOW - 60_000, + resolvedAt: null, + scheduledStart: null, + scheduledEnd: null, + ...overrides, +}) + +const componentWith = (overrides: Partial) => [ + { ...API_COMPONENT, ...overrides }, +] + +describe("toSnapshot", () => { + it("maps a down monitor to a major outage", () => { + const snapshot = toSnapshot(baseInput({ components: componentWith({ monitorStatus: "down" }) })) + assert.strictEqual(snapshot.components[0]?.status, "major_outage") + }) + + it("applies a manual override when it is worse than the monitor", () => { + const snapshot = toSnapshot(baseInput({ components: componentWith({ manualStatus: "degraded" }) })) + assert.strictEqual(snapshot.components[0]?.status, "degraded") + }) + + it("applies an active incident's impact to the components it names", () => { + const snapshot = toSnapshot( + baseInput({ + incidents: [incident({ impact: "major" })], + incidentComponents: [{ incidentId: INCIDENT, componentId: COMPONENT }], + }), + ) + assert.strictEqual(snapshot.components[0]?.status, "partial_outage") + assert.deepStrictEqual(snapshot.incidents[0]?.componentIds, [COMPONENT]) + }) + + it("leaves drafts and old resolved incidents off the page", () => { + const snapshot = toSnapshot( + baseInput({ + incidents: [ + incident({ published: false, impact: "critical" }), + incident({ + id: StatusPageIncidentId.make("55555555-5555-4555-8555-555555555555"), + status: "resolved", + resolvedAt: NOW - 20 * DAY_MS, + }), + ], + incidentComponents: [{ incidentId: INCIDENT, componentId: COMPONENT }], + }), + ) + assert.deepStrictEqual(snapshot.incidents, []) + assert.strictEqual(snapshot.components[0]?.status, "operational") + }) + + it("lists upcoming maintenance without affecting status", () => { + const snapshot = toSnapshot( + baseInput({ + incidents: [ + incident({ kind: "maintenance", status: "scheduled", scheduledEnd: NOW + DAY_MS }), + ], + incidentComponents: [{ incidentId: INCIDENT, componentId: COMPONENT }], + }), + ) + assert.strictEqual(snapshot.incidents.length, 1) + assert.strictEqual(snapshot.components[0]?.status, "operational") + }) + + it("fills 90 days of bars and takes probesSeenAt from the newest check", () => { + const today = utcDay(NOW) + const snapshot = toSnapshot( + baseInput({ + daily: [ + { + monitorId: MONITOR, + day: today, + checks: 10, + failures: 1, + downChecks: 0, + lastCheckAt: NOW - 5_000, + }, + { + monitorId: MONITOR, + day: utcDay(NOW - DAY_MS), + checks: 20, + failures: 0, + downChecks: 0, + lastCheckAt: NOW - DAY_MS, + }, + ], + }), + ) + const days = snapshot.components[0]?.days ?? [] + assert.strictEqual(days.length, 90) + assert.deepStrictEqual(days.at(-1), { day: today, checks: 10, failures: 1, downChecks: 0 }) + assert.deepStrictEqual(days[0], { + day: utcDay(NOW - 89 * DAY_MS), + checks: 0, + failures: 0, + downChecks: 0, + }) + assert.strictEqual(snapshot.probesSeenAt, NOW - 5_000) + }) + + it("gives components without a monitor or with hidden uptime no bars", () => { + const hidden = toSnapshot(baseInput({ components: componentWith({ showUptime: false }) })) + const unmonitored = toSnapshot(baseInput({ components: componentWith({ monitorId: null }) })) + assert.deepStrictEqual(hidden.components[0]?.days, []) + assert.deepStrictEqual(unmonitored.components[0]?.days, []) + assert.strictEqual(unmonitored.probesSeenAt, null) + }) +}) + +describe("StatusPageSnapshotBuilder.build", () => { + it.effect("reads a page from Postgres into its snapshot", () => { + const testDb = createTestDb(trackedDbs) + const pageId = newPageId() + const monitorId = crypto.randomUUID() + const downComponent = crypto.randomUUID() + const manualComponent = crypto.randomUUID() + return Effect.gen(function* () { + yield* seedPage(testDb, { id: pageId, slug: "acme-build", revision: 4 }) + yield* seedMonitor(testDb, { id: monitorId, status: "down" }) + yield* seedDaily(testDb, { + monitorId, + day: utcDay(NOW), + checks: 5, + failures: 2, + downChecks: 1, + lastCheckAt: NOW - 30_000, + }) + yield* seedComponent(testDb, { + id: manualComponent, + pageId, + name: "Docs", + position: 1, + manualStatus: "degraded", + }) + yield* seedComponent(testDb, { id: downComponent, pageId, name: "API", position: 0, monitorId }) + yield* seedIncident(testDb, { + id: crypto.randomUUID(), + pageId, + title: "Draft", + status: "investigating", + published: false, + startedAt: NOW - 1000, + componentIds: [manualComponent], + }) + yield* seedIncident(testDb, { + id: crypto.randomUUID(), + pageId, + title: "Docs slow", + impact: "critical", + status: "identified", + startedAt: NOW - 2000, + componentIds: [manualComponent], + }) + + const builder = yield* StatusPageSnapshotBuilder + const snapshot = yield* builder.build(pageId, NOW) + assert.strictEqual(snapshot.revision, 4) + assert.deepStrictEqual( + snapshot.components.map((c) => [c.name, c.status, c.days.length]), + [ + ["API", "major_outage", 90], + ["Docs", "major_outage", 0], + ], + ) + assert.deepStrictEqual( + snapshot.incidents.map((i) => [i.title, i.updates.length]), + [["Docs slow", 1]], + ) + assert.strictEqual(snapshot.probesSeenAt, NOW - 30_000) + + const missing = yield* Effect.exit(builder.build(newPageId(), NOW)) + assert.isTrue(Exit.isFailure(missing)) + }).pipe(Effect.provide(StatusPageSnapshotBuilder.layer), Effect.provide(testDb.layer)) + }) +}) diff --git a/packages/backend/src/services/status-pages/StatusPageSnapshotBuilder.ts b/packages/backend/src/services/status-pages/StatusPageSnapshotBuilder.ts new file mode 100644 index 0000000000..4d6ed4679a --- /dev/null +++ b/packages/backend/src/services/status-pages/StatusPageSnapshotBuilder.ts @@ -0,0 +1,369 @@ +import * as PG from "@maple-dev/effect-orm/postgres" +import { + StatusPageComponents, + StatusPageIncidentComponents, + StatusPageIncidents, + StatusPageIncidentUpdates, + StatusPages, + UptimeMonitorDaily, + UptimeMonitorStates, +} from "@maple/db/tables" +import type { + StatusPageComponentId, + StatusPageComponentStatus, + StatusPageId, + StatusPageIncidentId, + StatusPageIncidentImpact, + StatusPageIncidentKind, + StatusPageIncidentStatus, + StatusPageUpdateSource, + UptimeMonitorId, + UptimeMonitorStatus, +} from "@maple/domain/primitives" +import { + deriveComponentStatus, + fillDays, + incidentComponentStatus, + isActiveIncident, + lastDays, + STATUS_PAGE_HISTORY_DAYS, + STATUS_PAGE_INCIDENT_HISTORY_DAYS, + STATUS_PAGE_SNAPSHOT_VERSION, + type StatusPageSnapshot, + type StatusPageSnapshotIncident, +} from "@maple/domain/status-page" +import { Context, Effect, Layer } from "effect" +import { Database } from "@maple/backend/platform/DatabaseLive" +import { makeDbExecute, makePersistenceErrorMapper } from "@maple/backend/platform/db-execute" +import { StatusPageGoneError, StatusPagePublishDbError } from "./publish-errors" + +const DAY_MS = 86_400_000 + +export interface SnapshotPageInput { + readonly id: StatusPageId + readonly name: string + readonly slug: string + readonly description: string | null + readonly websiteUrl: string | null + readonly logoUrl: string | null + readonly brandColor: string | null + readonly showPoweredBy: boolean + readonly breakGlassTokenSha256: string | null + readonly revision: number +} + +export interface SnapshotComponentInput { + readonly id: StatusPageComponentId + readonly name: string + readonly description: string | null + readonly groupName: string | null + readonly monitorId: UptimeMonitorId | null + readonly manualStatus: StatusPageComponentStatus | null + readonly showUptime: boolean + /** Null when there is no monitor or it has no state row yet. */ + readonly monitorStatus: UptimeMonitorStatus | null +} + +export interface SnapshotDailyInput { + readonly monitorId: UptimeMonitorId + readonly day: string + readonly checks: number + readonly failures: number + readonly downChecks: number + readonly lastCheckAt: number +} + +export interface SnapshotIncidentInput { + readonly id: StatusPageIncidentId + readonly kind: StatusPageIncidentKind + readonly title: string + readonly impact: StatusPageIncidentImpact + readonly status: StatusPageIncidentStatus + readonly published: boolean + readonly startedAt: number + readonly resolvedAt: number | null + readonly scheduledStart: number | null + readonly scheduledEnd: number | null +} + +export interface SnapshotUpdateInput { + readonly id: string + readonly incidentId: StatusPageIncidentId + readonly status: StatusPageIncidentStatus + readonly body: string + readonly source: StatusPageUpdateSource + readonly createdAt: number +} + +export interface SnapshotInput { + readonly page: SnapshotPageInput + readonly components: ReadonlyArray + readonly daily: ReadonlyArray + readonly incidents: ReadonlyArray + readonly updates: ReadonlyArray + readonly incidentComponents: ReadonlyArray<{ + readonly incidentId: StatusPageIncidentId + readonly componentId: StatusPageComponentId + }> + readonly nowMs: number +} + +/** Whether a visitor should see the incident at `nowMs`: active, upcoming maintenance, or recently resolved. */ +export const isListedIncident = (incident: SnapshotIncidentInput, nowMs: number): boolean => { + if (!incident.published) return false + if (isActiveIncident(incident)) return true + if (incident.kind === "maintenance" && incident.status === "scheduled") { + return incident.scheduledEnd !== null && incident.scheduledEnd > nowMs + } + return ( + incident.resolvedAt !== null && + incident.resolvedAt >= nowMs - STATUS_PAGE_INCIDENT_HISTORY_DAYS * DAY_MS + ) +} + +/** The snapshot for already-read rows; no I/O, so the rules are testable without a database. */ +export const toSnapshot = (input: SnapshotInput): StatusPageSnapshot => { + const days = lastDays(input.nowMs, STATUS_PAGE_HISTORY_DAYS) + const firstDay = days[0] ?? "" + const daily = input.daily.filter((row) => row.day >= firstDay) + + const incidents: ReadonlyArray = input.incidents + .filter((incident) => isListedIncident(incident, input.nowMs)) + .toSorted((a, b) => b.startedAt - a.startedAt) + .map((incident) => ({ + id: incident.id, + kind: incident.kind, + title: incident.title, + impact: incident.impact, + status: incident.status, + startedAt: incident.startedAt, + resolvedAt: incident.resolvedAt, + scheduledStart: incident.scheduledStart, + scheduledEnd: incident.scheduledEnd, + componentIds: input.incidentComponents + .filter((link) => link.incidentId === incident.id) + .map((link) => link.componentId), + updates: input.updates + .filter((update) => update.incidentId === incident.id) + .toSorted((a, b) => b.createdAt - a.createdAt) + .map((update) => ({ + id: update.id, + status: update.status, + body: update.body, + at: update.createdAt, + source: update.source, + })), + })) + const active = incidents.filter(isActiveIncident) + + const components = input.components.map((component) => ({ + id: component.id, + name: component.name, + description: component.description, + group: component.groupName, + status: deriveComponentStatus({ + monitorStatus: component.monitorStatus, + manualStatus: component.manualStatus, + incidentStatuses: active + .filter((incident) => incident.componentIds.includes(component.id)) + .map(incidentComponentStatus), + }), + showUptime: component.showUptime, + days: + component.monitorId === null || !component.showUptime + ? [] + : fillDays( + days, + daily + .filter((row) => row.monitorId === component.monitorId) + .map((row) => ({ + day: row.day, + checks: row.checks, + failures: row.failures, + downChecks: row.downChecks, + })), + ), + })) + + const monitorIds = new Set(input.components.flatMap((c) => (c.monitorId === null ? [] : [c.monitorId]))) + const seen = daily.filter((row) => monitorIds.has(row.monitorId)).map((row) => row.lastCheckAt) + + return { + schemaVersion: STATUS_PAGE_SNAPSHOT_VERSION, + pageId: input.page.id, + revision: input.page.revision, + generatedAt: input.nowMs, + probesSeenAt: seen.length === 0 ? null : Math.max(...seen), + page: { + name: input.page.name, + slug: input.page.slug, + description: input.page.description, + websiteUrl: input.page.websiteUrl, + logoUrl: input.page.logoUrl, + brandColor: input.page.brandColor, + showPoweredBy: input.page.showPoweredBy, + }, + breakGlassTokenSha256: input.page.breakGlassTokenSha256, + components, + incidents, + } +} + +export interface StatusPageSnapshotBuilderApi { + /** Reads the page by id (a system path: every later query is scoped to its org). */ + readonly build: ( + pageId: StatusPageId, + nowMs: number, + ) => Effect.Effect +} + +const toDbError = makePersistenceErrorMapper(StatusPagePublishDbError, "Status page snapshot read failed") + +export class StatusPageSnapshotBuilder extends Context.Service< + StatusPageSnapshotBuilder, + StatusPageSnapshotBuilderApi +>()("@maple/backend/services/status-pages/StatusPageSnapshotBuilder", { + make: Effect.gen(function* () { + const database = yield* Database + const dbExecute = makeDbExecute(database, "StatusPageSnapshotBuilder", toDbError) + + const build = Effect.fn("StatusPageSnapshotBuilder.build")(function* ( + pageId: StatusPageId, + nowMs: number, + ) { + yield* Effect.annotateCurrentSpan({ "maple.status_page.id": pageId }) + const firstDay = lastDays(nowMs, STATUS_PAGE_HISTORY_DAYS)[0] ?? "" + const resolvedCutoff = nowMs - STATUS_PAGE_INCIDENT_HISTORY_DAYS * DAY_MS + const input = yield* dbExecute((db) => + Effect.gen(function* () { + const [page] = yield* db.run( + PG.from(StatusPages) + .select(($) => ({ + id: $.id, + orgId: $.orgId, + name: $.name, + slug: $.slug, + description: $.description, + websiteUrl: $.websiteUrl, + logoUrl: $.logoUrl, + brandColor: $.brandColor, + showPoweredBy: $.showPoweredBy, + breakGlassTokenSha256: $.breakGlassTokenSha256, + revision: $.revision, + })) + .where(($) => [$.id.eq(pageId)]) + .limit(1), + ) + if (page === undefined) return null + const orgId = page.orgId + const components = yield* db.run( + PG.from(StatusPageComponents, "c") + .leftJoin(UptimeMonitorStates, "s", (c, s) => + c.monitorId.eq(s.monitorId).and(c.orgId.eq(s.orgId)), + ) + .select(($) => ({ + id: $.id, + name: $.name, + description: $.description, + groupName: $.groupName, + monitorId: $.monitorId, + manualStatus: $.manualStatus, + showUptime: $.showUptime, + monitorStatus: $.s.status, + position: $.position, + })) + .where(($) => [$.orgId.eq(orgId), $.pageId.eq(pageId)]) + .orderBy(["position", "asc"], ["id", "asc"]), + ) + const monitorIds = components.flatMap((c) => (c.monitorId === null ? [] : [c.monitorId])) + const daily = + monitorIds.length === 0 + ? [] + : yield* db.run( + PG.from(UptimeMonitorDaily) + .select(($) => ({ + monitorId: $.monitorId, + day: $.day, + checks: $.checks, + failures: $.failures, + downChecks: $.downChecks, + lastCheckAt: $.lastCheckAt, + })) + .where(($) => [ + $.orgId.eq(orgId), + PG.inList($.monitorId, monitorIds), + $.day.gte(firstDay), + ]), + ) + const incidents = yield* db.run( + PG.from(StatusPageIncidents) + .select(($) => ({ + id: $.id, + kind: $.kind, + title: $.title, + impact: $.impact, + status: $.status, + published: $.published, + startedAt: $.startedAt, + resolvedAt: $.resolvedAt, + scheduledStart: $.scheduledStart, + scheduledEnd: $.scheduledEnd, + })) + .where(($) => [ + $.orgId.eq(orgId), + $.pageId.eq(pageId), + $.published.eq(true), + PG.or($.resolvedAt.isNull(), $.resolvedAt.gte(resolvedCutoff)), + ]), + ) + const incidentIds = incidents.map((incident) => incident.id) + const updates = + incidentIds.length === 0 + ? [] + : yield* db.run( + PG.from(StatusPageIncidentUpdates) + .select(($) => ({ + id: $.id, + incidentId: $.incidentId, + status: $.status, + body: $.body, + source: $.source, + createdAt: $.createdAt, + })) + .where(($) => [ + $.orgId.eq(orgId), + PG.inList($.incidentId, incidentIds), + ]), + ) + const incidentComponents = + incidentIds.length === 0 + ? [] + : yield* db.run( + PG.from(StatusPageIncidentComponents) + .select(($) => ({ + incidentId: $.incidentId, + componentId: $.componentId, + })) + .where(($) => [ + $.orgId.eq(orgId), + PG.inList($.incidentId, incidentIds), + ]), + ) + return { page, components, daily, incidents, updates, incidentComponents } + }), + ) + if (input === null) { + return yield* new StatusPageGoneError({ message: `Status page ${pageId} not found`, pageId }) + } + yield* Effect.annotateCurrentSpan({ + "maple.status_page.components": input.components.length, + "maple.status_page.incidents": input.incidents.length, + }) + return toSnapshot({ ...input, nowMs }) + }) + + return { build } + }), +}) { + static readonly layer = Layer.effect(this, this.make) +} diff --git a/packages/backend/src/services/status-pages/StatusPageStore.ts b/packages/backend/src/services/status-pages/StatusPageStore.ts new file mode 100644 index 0000000000..9538fe5704 --- /dev/null +++ b/packages/backend/src/services/status-pages/StatusPageStore.ts @@ -0,0 +1,249 @@ +import type { StatusPageId } from "@maple/domain/primitives" +import { + decodeStatusPageOverlayJson, + StatusPageKeys, + type StatusPageOverlay, +} from "@maple/domain/status-page" +import { Context, Effect, Layer, Option, Ref, Schema } from "effect" + +// Edge storage for status pages: the snapshot (KV, mirrored to R2), routes, the publisher +// heartbeat and the break-glass overlay the status Worker writes while Maple is down. + +export class StatusPageStoreError extends Schema.TaggedError()( + "@maple/backend/status-pages/StatusPageStoreError", + { + message: Schema.String, + operation: Schema.String, + key: Schema.String, + cause: Schema.Defect(), + }, +) {} + +/** The subset of a Workers KV namespace the store uses. */ +export interface StatusPageKvNamespace { + get(key: string, type: "text"): Promise + put(key: string, value: string): Promise + delete(key: string): Promise +} + +/** The subset of an R2 bucket the store uses. */ +export interface StatusPageBucket { + put( + key: string, + value: string, + options?: { httpMetadata?: { contentType?: string } }, + ): Promise + delete(key: string): Promise +} + +export interface StatusPageRoutes { + readonly slug: string + readonly hostnames: ReadonlyArray +} + +export interface StatusPageStoreApi { + /** Writes the snapshot to KV and its R2 copy. */ + readonly putSnapshot: (pageId: StatusPageId, json: string) => Effect.Effect + readonly putRoutes: ( + pageId: StatusPageId, + routes: StatusPageRoutes, + ) => Effect.Effect + readonly deleteRoutes: (routes: StatusPageRoutes) => Effect.Effect + /** Removes the snapshot (KV and R2) and the overlay. */ + readonly deletePage: (pageId: StatusPageId) => Effect.Effect + readonly putHeartbeat: (atMs: number) => Effect.Effect + /** The page's break-glass overlay; missing or undecodable reads as empty. */ + readonly readOverlay: (pageId: StatusPageId) => Effect.Effect + /** Drops only these entries, keeping any the edge added since the read. */ + readonly removeOverlayEntries: ( + pageId: StatusPageId, + entryIds: ReadonlyArray, + ) => Effect.Effect +} + +const EMPTY_OVERLAY: StatusPageOverlay = { entries: [] } + +const routeKeys = (routes: StatusPageRoutes): ReadonlyArray => [ + StatusPageKeys.slug(routes.slug), + ...routes.hostnames.map(StatusPageKeys.host), +] + +/** The overlay as stored, or empty when absent; an undecodable value is logged and ignored. */ +const parseOverlay = (pageId: StatusPageId, raw: string | null) => + raw === null + ? Effect.succeed(EMPTY_OVERLAY) + : Option.match(decodeStatusPageOverlayJson(raw), { + onNone: () => + Effect.logWarning("Undecodable status page overlay ignored").pipe( + Effect.annotateLogs({ "maple.status_page.id": pageId }), + Effect.as(EMPTY_OVERLAY), + ), + onSome: Effect.succeed, + }) + +const withoutEntries = (overlay: StatusPageOverlay, entryIds: ReadonlyArray): StatusPageOverlay => ({ + entries: overlay.entries.filter((entry) => !entryIds.includes(entry.id)), +}) + +/** Low-level key/value operations both layers implement. */ +interface StoreBackend { + readonly kvGet: (key: string) => Effect.Effect + readonly kvPut: (key: string, value: string) => Effect.Effect + readonly kvDelete: (key: string) => Effect.Effect + readonly objectPut: (key: string, value: string) => Effect.Effect + readonly objectDelete: (key: string) => Effect.Effect +} + +const makeApi = (backend: StoreBackend): StatusPageStoreApi => { + const readOverlay = (pageId: StatusPageId) => + backend.kvGet(StatusPageKeys.overlay(pageId)).pipe(Effect.flatMap((raw) => parseOverlay(pageId, raw))) + return { + putSnapshot: (pageId, json) => + Effect.all( + [ + backend.kvPut(StatusPageKeys.snapshot(pageId), json), + backend.objectPut(StatusPageKeys.snapshotObject(pageId), json), + ], + { concurrency: 2, discard: true }, + ), + putRoutes: (pageId, routes) => + Effect.forEach(routeKeys(routes), (key) => backend.kvPut(key, pageId), { + concurrency: 4, + discard: true, + }), + deleteRoutes: (routes) => + Effect.forEach(routeKeys(routes), backend.kvDelete, { concurrency: 4, discard: true }), + deletePage: (pageId) => + Effect.all( + [ + backend.kvDelete(StatusPageKeys.snapshot(pageId)), + backend.kvDelete(StatusPageKeys.overlay(pageId)), + backend.objectDelete(StatusPageKeys.snapshotObject(pageId)), + ], + { concurrency: 3, discard: true }, + ), + putHeartbeat: (atMs) => backend.kvPut(StatusPageKeys.heartbeat, JSON.stringify({ at: atMs })), + readOverlay, + removeOverlayEntries: (pageId, entryIds) => + entryIds.length === 0 + ? Effect.void + : readOverlay(pageId).pipe( + Effect.map((overlay) => withoutEntries(overlay, entryIds)), + Effect.flatMap((remaining) => + remaining.entries.length === 0 + ? backend.kvDelete(StatusPageKeys.overlay(pageId)) + : backend.kvPut(StatusPageKeys.overlay(pageId), JSON.stringify(remaining)), + ), + ), + } +} + +const call = (operation: string, key: string, run: () => Promise) => + Effect.tryPromise({ + try: run, + catch: (cause) => + new StatusPageStoreError({ + message: `Status page store ${operation} failed for ${key}`, + operation, + key, + cause, + }), + }) + +const isBucket = (value: unknown): value is StatusPageBucket => + typeof value === "object" && + value !== null && + "put" in value && + typeof value.put === "function" && + "delete" in value && + typeof value.delete === "function" + +const isKvNamespace = (value: unknown): value is StatusPageKvNamespace => + isBucket(value) && "get" in value && typeof value.get === "function" + +/** The KV namespace and R2 bucket from a Worker env, when both are bound. */ +export const statusPageBindingsFrom = ( + env: Record, +): Option.Option<{ readonly kv: StatusPageKvNamespace; readonly bucket: StatusPageBucket }> => { + const kv = env.STATUS_PAGES_KV + const bucket = env.STATUS_PAGES_BUCKET + return isKvNamespace(kv) && isBucket(bucket) ? Option.some({ kv, bucket }) : Option.none() +} + +/** What the in-memory store holds, for test assertions. */ +export interface StatusPageStoreContents { + readonly kv: ReadonlyMap + readonly objects: ReadonlyMap +} + +export interface InMemoryStatusPageStore { + readonly api: StatusPageStoreApi + readonly contents: Effect.Effect + /** While set, writes fail (all, or those whose key matches): a stand-in for an unavailable KV. */ + readonly setFailing: (failing: boolean | ((key: string) => boolean)) => Effect.Effect + /** Writes a raw KV value, as the status Worker would. */ + readonly putKv: (key: string, value: string) => Effect.Effect +} + +export class StatusPageStore extends Context.Service()( + "@maple/backend/services/status-pages/StatusPageStore", +) { + /** The store over Worker bindings: pass `env.STATUS_PAGES_KV` and `env.STATUS_PAGES_BUCKET`. */ + static readonly fromBindings = (kv: StatusPageKvNamespace, bucket: StatusPageBucket) => + Layer.succeed( + this, + makeApi({ + kvGet: (key) => call("kv.get", key, () => kv.get(key, "text")), + kvPut: (key, value) => call("kv.put", key, () => kv.put(key, value)).pipe(Effect.asVoid), + kvDelete: (key) => call("kv.delete", key, () => kv.delete(key)).pipe(Effect.asVoid), + objectPut: (key, value) => + call("r2.put", key, () => + bucket.put(key, value, { httpMetadata: { contentType: "application/json" } }), + ).pipe(Effect.asVoid), + objectDelete: (key) => call("r2.delete", key, () => bucket.delete(key)).pipe(Effect.asVoid), + }), + ) + + /** A `Ref`-backed store plus handles on its contents. */ + static readonly makeInMemory: Effect.Effect = Effect.gen(function* () { + const kv = yield* Ref.make>(new Map()) + const objects = yield* Ref.make>(new Map()) + const failing = yield* Ref.make<(key: string) => boolean>(() => false) + const guardWrite = (operation: string, key: string) => + Ref.get(failing).pipe( + Effect.flatMap((fails) => + fails(key) + ? Effect.fail( + new StatusPageStoreError({ + message: `Status page store ${operation} failed for ${key}`, + operation, + key, + cause: new Error("store unavailable"), + }), + ) + : Effect.void, + ), + ) + const set = (ref: Ref.Ref>, key: string, value: string) => + Ref.update(ref, (map) => new Map([...map, [key, value]])) + const remove = (ref: Ref.Ref>, key: string) => + Ref.update(ref, (map) => new Map([...map].filter(([k]) => k !== key))) + const api = makeApi({ + kvGet: (key) => Ref.get(kv).pipe(Effect.map((map) => map.get(key) ?? null)), + kvPut: (key, value) => guardWrite("kv.put", key).pipe(Effect.andThen(set(kv, key, value))), + kvDelete: (key) => guardWrite("kv.delete", key).pipe(Effect.andThen(remove(kv, key))), + objectPut: (key, value) => + guardWrite("r2.put", key).pipe(Effect.andThen(set(objects, key, value))), + objectDelete: (key) => guardWrite("r2.delete", key).pipe(Effect.andThen(remove(objects, key))), + }) + return { + api, + contents: Effect.all({ kv: Ref.get(kv), objects: Ref.get(objects) }), + setFailing: (fail) => Ref.set(failing, typeof fail === "function" ? fail : () => fail), + putKv: (key, value) => set(kv, key, value), + } + }) + + /** An empty in-memory store, for tests that only need the service. */ + static readonly inMemory = Layer.effect(this, this.makeInMemory.pipe(Effect.map((store) => store.api))) +} diff --git a/packages/backend/src/services/status-pages/auto-incidents.ts b/packages/backend/src/services/status-pages/auto-incidents.ts new file mode 100644 index 0000000000..7066ff85e4 --- /dev/null +++ b/packages/backend/src/services/status-pages/auto-incidents.ts @@ -0,0 +1,184 @@ +import { randomUUID } from "node:crypto" +import * as PG from "@maple-dev/effect-orm/postgres" +import type { MapleDb } from "@maple/db/client" +import { + StatusPageComponents, + StatusPageIncidentComponents, + StatusPageIncidents, + StatusPageIncidentUpdates, + StatusPages, + UptimeMonitors, +} from "@maple/db/tables" +import { + type AlertIncidentId, + type AlertRuleId, + type OrgId, + type StatusPageAutoIncidents, + type StatusPageComponentId, + type StatusPageId, + StatusPageIncidentId, + StatusPageIncidentUpdateId, +} from "@maple/domain/primitives" +import { Effect } from "effect" +import { bumpPage } from "./outbox" + +// Status page incidents opened and resolved by an uptime monitor's alert incident, on every +// page that opted in (`auto_incidents`) and shows a component on that monitor. + +export interface OpenForAlertIncidentInput { + readonly orgId: OrgId + readonly ruleId: AlertRuleId + readonly alertIncidentId: AlertIncidentId + readonly at: number +} + +export interface ResolveForAlertIncidentInput { + readonly orgId: OrgId + readonly alertIncidentId: AlertIncidentId + readonly at: number +} + +interface PageComponents { + readonly pageId: StatusPageId + readonly publish: boolean + readonly components: ReadonlyArray<{ readonly id: StatusPageComponentId; readonly name: string }> +} + +const groupByPage = ( + rows: ReadonlyArray<{ + readonly pageId: StatusPageId + readonly autoIncidents: StatusPageAutoIncidents + readonly componentId: StatusPageComponentId + readonly componentName: string + }>, +): ReadonlyArray => + [...new Set(rows.map((row) => row.pageId))].map((pageId) => { + const forPage = rows.filter((row) => row.pageId === pageId) + return { + pageId, + publish: forPage[0]?.autoIncidents === "publish", + components: forPage.map((row) => ({ id: row.componentId, name: row.componentName })), + } + }) + +/** Opens a status page incident per opted-in page showing the rule's monitor; run inside a transaction. */ +export const openForAlertIncident = (db: MapleDb, input: OpenForAlertIncidentInput) => + Effect.gen(function* () { + const rows = yield* db.run( + PG.from(StatusPageComponents, "c") + .innerJoin(UptimeMonitors, "m", (c, m) => c.monitorId.eq(m.id).and(c.orgId.eq(m.orgId))) + .innerJoin(StatusPages, "p", (c, p) => c.pageId.eq(p.id).and(c.orgId.eq(p.orgId))) + .select(($) => ({ + pageId: $.pageId, + autoIncidents: $.p.autoIncidents, + componentId: $.id, + componentName: $.name, + position: $.position, + })) + .where(($) => [ + $.orgId.eq(input.orgId), + $.m.alertRuleId.eq(input.ruleId), + $.p.autoIncidents.neq("off"), + ]) + .orderBy(["position", "asc"], ["componentId", "asc"]), + ) + if (rows.length === 0) return 0 + const pages = groupByPage(rows) + const existing = yield* db.run( + PG.from(StatusPageIncidents) + .select(($) => ({ pageId: $.pageId })) + .where(($) => [$.orgId.eq(input.orgId), $.alertIncidentId.eq(input.alertIncidentId)]), + ) + const toOpen = pages.filter((page) => !existing.some((row) => row.pageId === page.pageId)) + yield* Effect.forEach( + toOpen, + (page) => + Effect.gen(function* () { + const id = StatusPageIncidentId.make(randomUUID()) + const name = page.components[0]?.name ?? "A service" + yield* db.run( + PG.insertInto(StatusPageIncidents).values({ + id, + orgId: input.orgId, + pageId: page.pageId, + kind: "incident", + title: `${name} is down`, + impact: "major", + status: "investigating", + published: page.publish, + alertIncidentId: input.alertIncidentId, + scheduledStart: null, + scheduledEnd: null, + startedAt: input.at, + resolvedAt: null, + createdBy: null, + createdAt: input.at, + updatedAt: input.at, + }), + ) + yield* db.run( + PG.insertInto(StatusPageIncidentComponents) + .values( + page.components.map((component) => ({ + incidentId: id, + componentId: component.id, + orgId: input.orgId, + })), + ) + .onConflictDoNothing(), + ) + yield* db.run( + PG.insertInto(StatusPageIncidentUpdates).values({ + id: StatusPageIncidentUpdateId.make(randomUUID()), + orgId: input.orgId, + incidentId: id, + status: "investigating", + body: `Automated checks for ${name} are failing. We are investigating.`, + source: "auto", + createdBy: null, + overlayEntryId: null, + createdAt: input.at, + }), + ) + yield* bumpPage(db, input.orgId, page.pageId) + }), + { discard: true }, + ) + return toOpen.length + }) + +/** Resolves the open status page incidents this alert incident opened; run inside a transaction. */ +export const resolveForAlertIncident = (db: MapleDb, input: ResolveForAlertIncidentInput) => + Effect.gen(function* () { + const resolved = yield* db.run( + PG.update(StatusPageIncidents) + .set({ status: "resolved", resolvedAt: input.at, updatedAt: input.at }) + .where(($) => [ + $.orgId.eq(input.orgId), + $.alertIncidentId.eq(input.alertIncidentId), + $.status.neq("resolved"), + ]) + .returning(($) => ({ id: $.id, pageId: $.pageId })), + ) + yield* Effect.forEach( + resolved, + (incident) => + db + .run( + PG.insertInto(StatusPageIncidentUpdates).values({ + id: StatusPageIncidentUpdateId.make(randomUUID()), + orgId: input.orgId, + incidentId: incident.id, + status: "resolved", + body: "Automated checks are passing again. This incident has been resolved.", + source: "auto", + createdBy: null, + overlayEntryId: null, + createdAt: input.at, + }), + ) + .pipe(Effect.andThen(bumpPage(db, input.orgId, incident.pageId))), + { discard: true }, + ) + return resolved.length + }) diff --git a/packages/backend/src/services/status-pages/publish-errors.ts b/packages/backend/src/services/status-pages/publish-errors.ts new file mode 100644 index 0000000000..e36992cdd5 --- /dev/null +++ b/packages/backend/src/services/status-pages/publish-errors.ts @@ -0,0 +1,14 @@ +import { StatusPageId } from "@maple/domain/primitives" +import { Schema } from "effect" + +/** The page was deleted (or never existed) by the time the publisher read it. */ +export class StatusPageGoneError extends Schema.TaggedError()( + "@maple/backend/status-pages/StatusPageGoneError", + { message: Schema.String, pageId: StatusPageId }, +) {} + +/** A Postgres read or write on the publish path failed. */ +export class StatusPagePublishDbError extends Schema.TaggedError()( + "@maple/backend/status-pages/StatusPagePublishDbError", + { message: Schema.String, cause: Schema.optionalKey(Schema.String) }, +) {} diff --git a/packages/backend/src/services/status-pages/status-page-test-support.ts b/packages/backend/src/services/status-pages/status-page-test-support.ts new file mode 100644 index 0000000000..5d23387369 --- /dev/null +++ b/packages/backend/src/services/status-pages/status-page-test-support.ts @@ -0,0 +1,221 @@ +// Seeds status pages, monitors and incidents for the status-pages tests. +import { Effect, Schema } from "effect" +import { OrgId, StatusPageId } from "@maple/domain/primitives" +import { executeSql, queryFirstRow, type TestDb } from "@maple/backend/platform/test-pglite" + +export const ORG = Schema.decodeUnknownSync(OrgId)("org_status_pages") +export const USER = "user_status_pages" +/** 2026-10-28T12:00:00Z */ +export const NOW = Date.UTC(2026, 9, 28, 12) +export const DAY_MS = 86_400_000 + +const iso = (ms: number) => new Date(ms).toISOString() +const sql = (testDb: TestDb, text: string, params: unknown[]) => + Effect.promise(() => executeSql(testDb, text, params)) + +export const newPageId = () => StatusPageId.make(crypto.randomUUID()) + +export const seedPage = ( + testDb: TestDb, + input: { + readonly id: StatusPageId + readonly slug: string + readonly autoIncidents?: "off" | "draft" | "publish" + readonly breakGlass?: boolean + readonly revision?: number + readonly publishedRevision?: number + readonly publishedAt?: number | null + }, +) => + sql( + testDb, + `INSERT INTO status_pages (id, org_id, name, slug, auto_incidents, break_glass_token_sha256, + revision, published_revision, published_at, created_by, created_at, updated_at) + VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $11)`, + [ + input.id, + ORG, + `Page ${input.slug}`, + input.slug, + input.autoIncidents ?? "off", + input.breakGlass === true ? "token-hash" : null, + input.revision ?? 1, + input.publishedRevision ?? 0, + input.publishedAt === undefined || input.publishedAt === null ? null : iso(input.publishedAt), + USER, + iso(NOW - DAY_MS), + ], + ) + +export const seedMonitor = ( + testDb: TestDb, + input: { readonly id: string; readonly status: string; readonly alertRuleId?: string | null }, +) => + sql( + testDb, + `INSERT INTO uptime_monitors (id, org_id, name, kind, config_json, alert_rule_id, created_by, created_at, updated_at) + VALUES ($1, $2, 'Monitor', 'http', $3, $4, $5, $6, $6)`, + [ + input.id, + ORG, + JSON.stringify({ + _tag: "http", + url: "https://example.com/health", + method: "GET", + headers: {}, + assertions: [], + timeoutMs: 5000, + }), + input.alertRuleId ?? null, + USER, + iso(NOW - DAY_MS), + ], + ).pipe( + Effect.andThen( + sql( + testDb, + `INSERT INTO uptime_monitor_states (monitor_id, org_id, status, recent_json, updated_at) + VALUES ($1, $2, $3, '[]', $4)`, + [input.id, ORG, input.status, iso(NOW)], + ), + ), + ) + +export const seedDaily = ( + testDb: TestDb, + input: { + readonly monitorId: string + readonly day: string + readonly checks: number + readonly failures: number + readonly downChecks: number + readonly lastCheckAt: number + }, +) => + sql( + testDb, + `INSERT INTO uptime_monitor_daily (monitor_id, day, org_id, checks, failures, down_checks, last_check_at) + VALUES ($1, $2, $3, $4, $5, $6, $7)`, + [ + input.monitorId, + input.day, + ORG, + input.checks, + input.failures, + input.downChecks, + iso(input.lastCheckAt), + ], + ) + +export const seedComponent = ( + testDb: TestDb, + input: { + readonly id: string + readonly pageId: string + readonly name: string + readonly position: number + readonly monitorId?: string | null + readonly manualStatus?: string | null + readonly showUptime?: boolean + }, +) => + sql( + testDb, + `INSERT INTO status_page_components (id, org_id, page_id, name, position, monitor_id, manual_status, + show_uptime, created_at, updated_at) + VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $9)`, + [ + input.id, + ORG, + input.pageId, + input.name, + input.position, + input.monitorId ?? null, + input.manualStatus ?? null, + input.showUptime ?? true, + iso(NOW - DAY_MS), + ], + ) + +export const seedIncident = ( + testDb: TestDb, + input: { + readonly id: string + readonly pageId: string + readonly title: string + readonly kind?: "incident" | "maintenance" + readonly impact?: string + readonly status: string + readonly published?: boolean + readonly startedAt: number + readonly resolvedAt?: number | null + readonly scheduledEnd?: number | null + readonly componentIds?: ReadonlyArray + }, +) => + sql( + testDb, + `INSERT INTO status_page_incidents (id, org_id, page_id, kind, title, impact, status, published, + scheduled_end, started_at, resolved_at, created_at, updated_at) + VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $10, $10)`, + [ + input.id, + ORG, + input.pageId, + input.kind ?? "incident", + input.title, + input.impact ?? "major", + input.status, + input.published ?? true, + input.scheduledEnd === undefined || input.scheduledEnd === null ? null : iso(input.scheduledEnd), + iso(input.startedAt), + input.resolvedAt === undefined || input.resolvedAt === null ? null : iso(input.resolvedAt), + ], + ).pipe( + Effect.andThen( + Effect.forEach( + input.componentIds ?? [], + (componentId) => + sql( + testDb, + `INSERT INTO status_page_incident_components (incident_id, component_id, org_id) VALUES ($1, $2, $3)`, + [input.id, componentId, ORG], + ), + { discard: true }, + ), + ), + Effect.andThen( + sql( + testDb, + `INSERT INTO status_page_incident_updates (id, org_id, incident_id, status, body, source, created_at) + VALUES ($1, $2, $3, $4, $5, 'dashboard', $6)`, + [ + crypto.randomUUID(), + ORG, + input.id, + input.status, + `${input.title} update`, + iso(input.startedAt), + ], + ), + ), + ) + +export const readPage = (testDb: TestDb, pageId: string) => + Effect.promise(() => + queryFirstRow<{ + revision: string + published_revision: string + published_at: string | null + published_hash: string | null + last_publish_error: string | null + }>( + testDb, + `SELECT revision::text, published_revision::text, published_at::text, published_hash, last_publish_error + FROM status_pages WHERE id = $1`, + [pageId], + ), + ) + +export const bumpRevision = (testDb: TestDb, pageId: string) => + sql(testDb, `UPDATE status_pages SET revision = revision + 1 WHERE id = $1`, [pageId]) From f3ce64d67c32c0756f08a61fcc65c5e25c776f6f Mon Sep 17 00:00:00 2001 From: Makisuo Date: Wed, 28 Oct 2026 16:08:27 +0100 Subject: [PATCH 03/15] feat(status-pages): bind snapshot KV and R2 to the alerting Worker --- apps/alerting/src/scheduled.ts | 15 ++++++++----- apps/alerting/src/worker.ts | 16 +++++++++++++- apps/status/src/worker.ts | 8 +++++-- .../services/status-pages/StatusPageStore.ts | 22 ++++--------------- packages/infra/src/cloudflare/index.ts | 1 + .../infra/src/cloudflare/status-pages.ts | 2 +- 6 files changed, 37 insertions(+), 27 deletions(-) rename apps/status/src/resources.ts => packages/infra/src/cloudflare/status-pages.ts (95%) diff --git a/apps/alerting/src/scheduled.ts b/apps/alerting/src/scheduled.ts index e2f2a10617..a2162743c3 100644 --- a/apps/alerting/src/scheduled.ts +++ b/apps/alerting/src/scheduled.ts @@ -30,7 +30,10 @@ import { SupabaseLogsService } from "@maple/backend/services/integrations/Supaba import { ServiceMapRollupService } from "@maple/backend/services/dashboards/ServiceMapRollupService" import { StatusPagePublisher } from "@maple/backend/services/status-pages/StatusPagePublisher" import { StatusPageReconciler } from "@maple/backend/services/status-pages/StatusPageReconciler" -import { StatusPageStore, statusPageBindingsFrom } from "@maple/backend/services/status-pages/StatusPageStore" +import { + type StatusPageBindings, + StatusPageStore, +} from "@maple/backend/services/status-pages/StatusPageStore" import { layerMapleDbConnection } from "@maple/backend/platform/pg-connection-source" import { summarizeCause } from "@maple/backend/platform/describe-cause" import { withPgConnectionScope } from "@maple/backend/platform/pg-connection-scope" @@ -43,8 +46,8 @@ import { isLaneOf, scheduleFor, type TickLane } from "./schedule.ts" import type { AlertingWorkerEnv } from "./worker.ts" /** The status page publisher, only when its KV and R2 bindings exist on this Worker. */ -const statusPagesLayerIfBound = (env: AlertingWorkerEnv) => - Option.match(statusPageBindingsFrom(env), { +const statusPagesLayerIfBound = (bindings: Option.Option) => + Option.match(bindings, { onNone: () => Layer.empty, onSome: ({ kv, bucket }) => Layer.mergeAll(StatusPagePublisher.layer, StatusPageReconciler.layer).pipe( @@ -63,6 +66,7 @@ export const layerTick = ( env: AlertingWorkerEnv, email: Option.Option = Option.none(), chatSessions?: ChatSessionNamespace, + statusPages: Option.Option = Option.none(), ) => Layer.mergeAll( AlertsService.layer, @@ -86,7 +90,7 @@ export const layerTick = ( // Read by `maybeEnqueueTriage` when present; its absence means every // incident opened here is investigated unclassified. IncidentClassifier.layer, - statusPagesLayerIfBound(env), + statusPagesLayerIfBound(statusPages), ).pipe( Layer.provide(PullRequestLookupLive), Layer.provide(Layer.mergeAll(Env.layer, DatabasePgLive, EdgeCacheServiceLive, FetchHttpClient.layer)), @@ -577,10 +581,11 @@ export const runScheduled = ( env: AlertingWorkerEnv, email: Option.Option, chatSessions: ChatSessionNamespace, + statusPages: Option.Option, ): Effect.Effect => withPgConnectionScope(selectLaneProgram(cron, lane, scheduledTicks)).pipe( // One lane is one application run: the layer is built here and released // with it, as the async entry's ManagedRuntime was. // oxlint-disable-next-line effecttsgo/strict-effect-provide - Effect.provide(layerTick(env, email, chatSessions)), + Effect.provide(layerTick(env, email, chatSessions, statusPages)), ) diff --git a/apps/alerting/src/worker.ts b/apps/alerting/src/worker.ts index 9e272c71b2..9c3afd2322 100644 --- a/apps/alerting/src/worker.ts +++ b/apps/alerting/src/worker.ts @@ -13,6 +13,8 @@ import { MapleStack, type MapleStage, mapleWorkerProps, + StatusSnapshotsBucket, + StatusSnapshotsKv, } from "@maple/infra/cloudflare" import { apnsEnv, @@ -36,6 +38,7 @@ import { bindChatSessions } from "@maple/backend/platform/chat-sessions" import { bindEmailSender } from "@maple/backend/platform/email-sender" import { chatConnectorOutboundConfigKeys } from "@maple/chat-platform" import * as Cloudflare from "@alchemy.run/cloudflare" +import { RuntimeContext } from "alchemy/RuntimeContext" import { Cause, Config, Effect, Layer, Option, Ref } from "effect" import { FetchHttpClient, HttpClient, HttpServerResponse } from "effect/http" import { ALERTING_CRONS, scheduleFor } from "./schedule.ts" @@ -49,6 +52,11 @@ import { CHAT_ANTICIPATED_ERROR_IDENTIFIERS } from "@maple/chat-platform/anticip */ export type AlertingWorkerEnv = Record +/** Discharge alchemy's phantom color, the way alchemy's own runtime helpers do. */ +const runtime = (effect: Effect.Effect): Effect.Effect => + // oxlint-disable-next-line effecttsgo/strict-effect-provide + Effect.provide(effect, RuntimeContext.phantom) + /** Config-sourced env; largely shared with api via `@maple/infra/env`. */ const configuredEnv = (stage: MapleStage, region: MapleRegion, domains: MapleDomains) => merge( @@ -151,6 +159,9 @@ export default class Alerting extends Cloudflare.Worker()( const email = yield* bindEmailSender // maple-ai's chat Durable Object, where ticks start investigations; handed to each fire's graph. const chatSessions = yield* bindChatSessions + // Where published status pages live; ticks publish snapshots and import break-glass posts. + const statusKv = yield* Cloudflare.KV.ReadWriteNamespace(StatusSnapshotsKv) + const statusBucket = yield* Cloudflare.R2.ReadWriteBucket(StatusSnapshotsBucket) // Once per isolate, not once per fire. const loggedNonProdSkip = yield* Ref.make(false) // Once per isolate too: an isolate never changes data center. A failed lookup stays @@ -167,7 +178,8 @@ export default class Alerting extends Cloudflare.Worker()( const runTick = (cron: string, lane: string, env: Record) => Effect.gen(function* () { const { runScheduled } = yield* scheduled - yield* runScheduled(cron, lane, env, email, chatSessions) + const [kv, bucket] = yield* runtime(Effect.all([statusKv.raw, statusBucket.raw])) + yield* runScheduled(cron, lane, env, email, chatSessions, Option.some({ kv, bucket })) }).pipe( // Interrupts are isolate teardown, not a failed run. Effect.catchCause((cause) => @@ -284,6 +296,8 @@ export default class Alerting extends Cloudflare.Worker()( Layer.mergeAll( Cloudflare.Hyperdrive.ConnectBinding, Cloudflare.Email.SendBinding, + Cloudflare.KV.ReadWriteNamespaceBinding, + Cloudflare.R2.ReadWriteBucketBinding, Cloudflare.Workers.CronEventSourceLive, FetchHttpClient.layer, WorkerTelemetry({ diff --git a/apps/status/src/worker.ts b/apps/status/src/worker.ts index c5f3624885..cc62d82f2e 100644 --- a/apps/status/src/worker.ts +++ b/apps/status/src/worker.ts @@ -4,7 +4,12 @@ * No service bindings or Hyperdrive: serving never calls Maple. Traces export to Maple's ingest after * the response, best effort, so an outage loses them without touching the page. */ -import { MapleStack, mapleWorkerProps } from "@maple/infra/cloudflare" +import { + MapleStack, + mapleWorkerProps, + StatusSnapshotsBucket, + StatusSnapshotsKv, +} from "@maple/infra/cloudflare" import { selfObservabilityEnv } from "@maple/infra/env" import { WorkerTelemetry } from "@maple/infra/worker-telemetry" import * as Cloudflare from "@alchemy.run/cloudflare" @@ -12,7 +17,6 @@ import { RuntimeContext } from "alchemy/RuntimeContext" import { Clock, Effect, Layer } from "effect" import { HttpServerRequest, HttpServerResponse } from "effect/http" import { handleRequest } from "./handler" -import { StatusSnapshotsBucket, StatusSnapshotsKv } from "./resources" import { nativeStorage } from "./storage" /** `__ALCHEMY_RUNTIME__` folds to `true` in the bundle, so the stack-side branch is tree-shaken. */ diff --git a/packages/backend/src/services/status-pages/StatusPageStore.ts b/packages/backend/src/services/status-pages/StatusPageStore.ts index 9538fe5704..9ff31b02bd 100644 --- a/packages/backend/src/services/status-pages/StatusPageStore.ts +++ b/packages/backend/src/services/status-pages/StatusPageStore.ts @@ -150,24 +150,10 @@ const call = (operation: string, key: string, run: () => Promise) => }), }) -const isBucket = (value: unknown): value is StatusPageBucket => - typeof value === "object" && - value !== null && - "put" in value && - typeof value.put === "function" && - "delete" in value && - typeof value.delete === "function" - -const isKvNamespace = (value: unknown): value is StatusPageKvNamespace => - isBucket(value) && "get" in value && typeof value.get === "function" - -/** The KV namespace and R2 bucket from a Worker env, when both are bound. */ -export const statusPageBindingsFrom = ( - env: Record, -): Option.Option<{ readonly kv: StatusPageKvNamespace; readonly bucket: StatusPageBucket }> => { - const kv = env.STATUS_PAGES_KV - const bucket = env.STATUS_PAGES_BUCKET - return isKvNamespace(kv) && isBucket(bucket) ? Option.some({ kv, bucket }) : Option.none() +/** The KV namespace and R2 bucket a Worker binds for publishing. */ +export interface StatusPageBindings { + readonly kv: StatusPageKvNamespace + readonly bucket: StatusPageBucket } /** What the in-memory store holds, for test assertions. */ diff --git a/packages/infra/src/cloudflare/index.ts b/packages/infra/src/cloudflare/index.ts index bb3eb60e6f..807ba6d569 100644 --- a/packages/infra/src/cloudflare/index.ts +++ b/packages/infra/src/cloudflare/index.ts @@ -1,6 +1,7 @@ export * from "./cached-recoverable.ts" export * from "./device-token-key.ts" export * from "./email.ts" +export * from "./status-pages.ts" export * from "./maple-db.ts" export * from "./observability.ts" export * from "./stack.ts" diff --git a/apps/status/src/resources.ts b/packages/infra/src/cloudflare/status-pages.ts similarity index 95% rename from apps/status/src/resources.ts rename to packages/infra/src/cloudflare/status-pages.ts index aa2f768fe2..bed181e855 100644 --- a/apps/status/src/resources.ts +++ b/packages/infra/src/cloudflare/status-pages.ts @@ -3,7 +3,7 @@ * the status Worker reads them and writes only break-glass entry keys. Stage-isolated, and * retained: a replaced namespace or bucket would blank every customer's page. */ -import { stageProps } from "@maple/infra/cloudflare" +import { stageProps } from "./stack.ts" import * as Cloudflare from "@alchemy.run/cloudflare" import * as RemovalPolicy from "alchemy/RemovalPolicy" From 27bf74fcfb59dfc7b71f38bbd63cc04762f0e9fb Mon Sep 17 00:00:00 2001 From: Makisuo Date: Wed, 28 Oct 2026 16:29:26 +0100 Subject: [PATCH 04/15] fix(status-pages): edge cleanup drain, per-entry break-glass reconcile, heartbeat after a good sweep, auto-incident repair - Adopt the snapshot contract: monitored components, overlayEntryId on updates, openedByOverlayEntryId on incidents; heartbeat carries probesSeenAt. - Page delete and slug rename queue status_page_cleanups; the sweep drains them oldest first, guarding slugs another page owns, and writes R2 slug objects with routes. - Reconciler lists overlay: keys with a cursor, imports each entry in one transaction claimed by the unique overlay_entry_id, and deletes a key once a publish carries it. - Sweep orders by publish_attempted_at nulls first and stamps every attempt; a stale republish advances and stores its hash. - Minute tick resolves auto incidents whose alert incident is resolved or gone. --- apps/alerting/src/scheduled.ts | 23 +- .../status-pages/StatusPagePublisher.test.ts | 204 +++++++- .../status-pages/StatusPagePublisher.ts | 152 +++++- .../status-pages/StatusPageReconciler.test.ts | 455 +++++++++++++----- .../status-pages/StatusPageReconciler.ts | 397 +++++++++------ .../StatusPageSnapshotBuilder.test.ts | 63 ++- .../status-pages/StatusPageSnapshotBuilder.ts | 50 +- .../services/status-pages/StatusPageStore.ts | 334 ++++++++----- .../status-pages/StatusPagesService.test.ts | 26 + .../status-pages/StatusPagesService.ts | 21 +- .../services/status-pages/auto-incidents.ts | 91 +++- .../status-pages/status-page-test-support.ts | 73 ++- 12 files changed, 1422 insertions(+), 467 deletions(-) diff --git a/apps/alerting/src/scheduled.ts b/apps/alerting/src/scheduled.ts index a2162743c3..402c011e87 100644 --- a/apps/alerting/src/scheduled.ts +++ b/apps/alerting/src/scheduled.ts @@ -418,27 +418,38 @@ const prReviewPostMergeTick = makeTick( : undefined, ) -/** Imports break-glass updates, publishes lagging pages and writes the heartbeat; each step isolated. */ +/** + * Imports break-glass updates, repairs auto incidents, then sweeps: edge cleanups, lagging pages + * and the heartbeat (written by the sweep only once it reached the database). Each step isolated. + */ const statusPagesTick = Effect.gen(function* () { const publisher = yield* Effect.serviceOption(StatusPagePublisher) const reconciler = yield* Effect.serviceOption(StatusPageReconciler) if (Option.isNone(publisher) || Option.isNone(reconciler)) { return yield* Effect.logInfo("Status page storage is not bound; skipping status page publishing") } - const nowMs = yield* Clock.currentTimeMillis - yield* makeTick(publisher.value.heartbeat(nowMs), "status_page_heartbeat", () => undefined) yield* makeTick(reconciler.value.reconcileAll(), "status_page_reconcile", (result) => - result.imported + result.failed > 0 - ? { statusPageImported: result.imported, statusPageFailures: result.failed } + result.imported + result.cleared + result.failed > 0 + ? { + statusPageImported: result.imported, + statusPageCleared: result.cleared, + statusPageFailures: result.failed, + } : undefined, ) + yield* makeTick(reconciler.value.repairAutoIncidents(), "status_page_repair", (repaired) => + repaired > 0 ? { statusPageAutoIncidentsRepaired: repaired } : undefined, + ) + const nowMs = yield* Clock.currentTimeMillis yield* makeTick(publisher.value.sweep(nowMs), "status_page_publish", (result) => - result.due > 0 + result.due + result.cleaned + result.cleanupsFailed > 0 ? { statusPagesDue: result.due, statusPagesWritten: result.written, statusPagesUnchanged: result.unchanged, statusPageFailures: result.failed, + statusPagesCleaned: result.cleaned, + statusPageCleanupFailures: result.cleanupsFailed, } : undefined, ) diff --git a/packages/backend/src/services/status-pages/StatusPagePublisher.test.ts b/packages/backend/src/services/status-pages/StatusPagePublisher.test.ts index 34973116d0..242abe2030 100644 --- a/packages/backend/src/services/status-pages/StatusPagePublisher.test.ts +++ b/packages/backend/src/services/status-pages/StatusPagePublisher.test.ts @@ -1,11 +1,24 @@ import { afterEach, assert, describe, it } from "@effect/vitest" import { Effect, Exit, Layer, Option } from "effect" import { TestClock } from "effect/testing" -import { decodeStatusPageSnapshotJson, StatusPageKeys } from "@maple/domain/status-page" -import { cleanupTestDbs, createTestDb, type TestDb } from "@maple/backend/platform/test-pglite" -import { StatusPagePublisher } from "./StatusPagePublisher" +import { decodeStatusPageSnapshotJson, StatusPageKeys, utcDay } from "@maple/domain/status-page" +import { StatusPageId } from "@maple/domain/primitives" +import { cleanupTestDbs, createTestDb, executeSql, type TestDb } from "@maple/backend/platform/test-pglite" +import { STATUS_PAGE_SWEEP_BATCH, StatusPagePublisher } from "./StatusPagePublisher" import { StatusPageStore, type InMemoryStatusPageStore } from "./StatusPageStore" -import { bumpRevision, NOW, newPageId, readPage, seedComponent, seedPage } from "./status-page-test-support" +import { + bumpRevision, + cleanupRows, + DAY_MS, + NOW, + newPageId, + readPage, + seedCleanup, + seedComponent, + seedDaily, + seedMonitor, + seedPage, +} from "./status-page-test-support" const trackedDbs: TestDb[] = [] afterEach(() => cleanupTestDbs(trackedDbs)) @@ -56,6 +69,8 @@ describe("StatusPagePublisher.publish", () => { kv.get(StatusPageKeys.snapshot(pageId)), ) assert.strictEqual(kv.get(StatusPageKeys.slug("acme")), pageId) + // The edge's fallback when KV fails. + assert.strictEqual(objects.get(StatusPageKeys.slugObject("acme")), pageId) const row = yield* readPage(testDb, pageId) assert.strictEqual(row?.published_revision, "3") assert.isNotNull(row?.published_hash) @@ -81,6 +96,30 @@ describe("StatusPagePublisher.publish", () => { ), ) + it.effect("stores the hash of a stale republish without bumping the page again", () => + withPublisher(({ testDb, store, publisher }) => + Effect.gen(function* () { + const pageId = newPageId() + yield* seedPage(testDb, { + id: pageId, + slug: "acme-stale", + revision: 4, + publishedRevision: 4, + publishedAt: NOW - 20 * 60_000, + }) + const result = yield* publisher.publish(pageId) + assert.isTrue(result.written) + assert.isTrue((yield* store.contents).kv.has(StatusPageKeys.snapshot(pageId))) + const row = yield* readPage(testDb, pageId) + assert.strictEqual(row?.revision, "4") + assert.strictEqual(row?.published_revision, "4") + assert.isNotNull(row?.published_hash) + // The stored hash makes the next stale republish a no-op write. + assert.isFalse((yield* publisher.publish(pageId)).written) + }), + ), + ) + it.effect("records the store failure and leaves published_revision behind", () => withPublisher(({ testDb, store, publisher }) => Effect.gen(function* () { @@ -149,32 +188,177 @@ describe("StatusPagePublisher.sweep", () => { yield* seedPage(testDb, { id: second, slug: "second" }) yield* store.setFailing((key) => key.includes(first)) + const counts = { cleaned: 0, cleanupsFailed: 0 } const partial = yield* publisher.sweep(NOW) - assert.deepStrictEqual(partial, { due: 2, written: 1, unchanged: 0, failed: 1 }) + assert.deepStrictEqual(partial, { due: 2, written: 1, unchanged: 0, failed: 1, ...counts }) assert.isTrue((yield* store.contents).kv.has(StatusPageKeys.snapshot(second))) + // A failed attempt still counts, so it waits behind pages not yet tried. + assert.isNotNull((yield* readPage(testDb, first))?.publish_attempted_at) yield* store.setFailing(false) const recovered = yield* publisher.sweep(NOW) - assert.deepStrictEqual(recovered, { due: 1, written: 1, unchanged: 0, failed: 0 }) + assert.deepStrictEqual(recovered, { due: 1, written: 1, unchanged: 0, failed: 0, ...counts }) assert.deepStrictEqual(yield* publisher.sweep(NOW), { due: 0, written: 0, unchanged: 0, failed: 0, + ...counts, }) }), ), ) - it.effect("writes the heartbeat", () => - withPublisher(({ store, publisher }) => + it.effect("takes the longest-waiting attempts first, so a page never tried before is not starved", () => + withPublisher(({ testDb, store, publisher }) => + Effect.gen(function* () { + // Sorts last by id, so an id-ordered sweep would never reach it. + const starved = StatusPageId.make("ffffffff-ffff-4fff-8fff-ffffffffffff") + yield* seedPage(testDb, { id: starved, slug: "starved", publishAttemptedAt: null }) + yield* Effect.forEach( + Array.from({ length: STATUS_PAGE_SWEEP_BATCH }, (_, index) => index), + (index) => + seedPage(testDb, { + id: newPageId(), + slug: `busy-${index}`, + publishAttemptedAt: NOW - 60_000, + }), + { discard: true }, + ) + const result = yield* publisher.sweep(NOW) + assert.strictEqual(result.due, STATUS_PAGE_SWEEP_BATCH) + assert.isTrue((yield* store.contents).kv.has(StatusPageKeys.snapshot(starved))) + }), + ), + ) + + it.effect("writes the heartbeat with the newest scheduled check after a sweep", () => + withPublisher(({ testDb, store, publisher }) => Effect.gen(function* () { - yield* publisher.heartbeat(NOW) + const probe = crypto.randomUUID() + const heartbeatMonitor = crypto.randomUUID() + const paused = crypto.randomUUID() + yield* seedMonitor(testDb, { id: probe, status: "up" }) + yield* seedMonitor(testDb, { id: heartbeatMonitor, status: "up", kind: "heartbeat" }) + yield* seedMonitor(testDb, { id: paused, status: "up", enabled: false }) + const daily = (monitorId: string, day: string, lastCheckAt: number) => + seedDaily(testDb, { monitorId, day, checks: 1, failures: 0, downChecks: 0, lastCheckAt }) + yield* daily(probe, utcDay(NOW - DAY_MS), NOW - DAY_MS) + yield* daily(probe, utcDay(NOW), NOW - 90_000) + // Neither a heartbeat's synthetic miss nor a paused monitor proves the probes run. + yield* daily(heartbeatMonitor, utcDay(NOW), NOW - 1_000) + yield* daily(paused, utcDay(NOW), NOW - 2_000) + + yield* publisher.sweep(NOW) assert.strictEqual( (yield* store.contents).kv.get(StatusPageKeys.heartbeat), - JSON.stringify({ at: NOW }), + JSON.stringify({ at: NOW, probesSeenAt: NOW - 90_000 }), ) }), ), ) + + it.effect("writes no heartbeat when the sweep cannot read the database", () => + withPublisher(({ testDb, store, publisher }) => + Effect.gen(function* () { + yield* Effect.promise(() => + executeSql(testDb, `ALTER TABLE status_pages RENAME TO status_pages_gone`, []), + ) + const exit = yield* Effect.exit(publisher.sweep(NOW)) + assert.isTrue(Exit.isFailure(exit)) + assert.isFalse((yield* store.contents).kv.has(StatusPageKeys.heartbeat)) + }), + ), + ) +}) + +describe("StatusPagePublisher.sweep edge cleanups", () => { + it.effect("drops a renamed page's old slug route, in KV and R2, and the queue row", () => + withPublisher(({ testDb, store, publisher }) => + Effect.gen(function* () { + const pageId = newPageId() + yield* seedPage(testDb, { id: pageId, slug: "old-name" }) + yield* publisher.sweep(NOW) + yield* Effect.promise(() => + executeSql( + testDb, + `UPDATE status_pages SET slug = 'new-name', revision = revision + 1 WHERE id = $1`, + [pageId], + ), + ) + yield* seedCleanup(testDb, { pageId, slug: "old-name", deleteSnapshot: false, at: NOW }) + + const result = yield* publisher.sweep(NOW + 60_000) + assert.strictEqual(result.cleaned, 1) + const { kv, objects } = yield* store.contents + assert.isFalse(kv.has(StatusPageKeys.slug("old-name"))) + assert.isFalse(objects.has(StatusPageKeys.slugObject("old-name"))) + assert.strictEqual(kv.get(StatusPageKeys.slug("new-name")), pageId) + assert.isTrue(kv.has(StatusPageKeys.snapshot(pageId))) + assert.deepStrictEqual(yield* cleanupRows(testDb), []) + }), + ), + ) + + it.effect("leaves a slug route alone once another page owns that slug", () => + withPublisher(({ testDb, store, publisher }) => + Effect.gen(function* () { + const renamed = newPageId() + const taker = newPageId() + yield* seedPage(testDb, { id: taker, slug: "shared" }) + yield* publisher.sweep(NOW) + // The rename's cleanup drains after the new owner already took the slug. + yield* seedCleanup(testDb, { + pageId: renamed, + slug: "shared", + deleteSnapshot: false, + at: NOW, + }) + yield* publisher.sweep(NOW + 60_000) + const { kv, objects } = yield* store.contents + assert.strictEqual(kv.get(StatusPageKeys.slug("shared")), taker) + assert.strictEqual(objects.get(StatusPageKeys.slugObject("shared")), taker) + assert.deepStrictEqual(yield* cleanupRows(testDb), []) + }), + ), + ) + + it.effect("removes a deleted page's snapshot, routes and break-glass entries", () => + withPublisher(({ testDb, store, publisher }) => + Effect.gen(function* () { + const pageId = newPageId() + const other = newPageId() + yield* seedPage(testDb, { id: pageId, slug: "doomed" }) + yield* seedPage(testDb, { id: other, slug: "kept" }) + yield* publisher.sweep(NOW) + yield* store.putKv(StatusPageKeys.overlayEntry(pageId, "e1"), "{}") + yield* store.putKv(StatusPageKeys.overlayEntry(pageId, "e2"), "{}") + yield* store.putKv(StatusPageKeys.overlayEntry(other, "e3"), "{}") + yield* Effect.promise(() => + executeSql(testDb, `DELETE FROM status_pages WHERE id = $1`, [pageId]), + ) + yield* seedCleanup(testDb, { pageId, slug: "doomed", deleteSnapshot: true, at: NOW }) + + // A failing edge delete keeps the row for the next sweep. + yield* store.setFailing((key) => key === StatusPageKeys.snapshotObject(pageId)) + const failed = yield* publisher.sweep(NOW + 60_000) + assert.deepStrictEqual([failed.cleaned, failed.cleanupsFailed], [0, 1]) + assert.strictEqual((yield* cleanupRows(testDb)).length, 1) + + yield* store.setFailing(false) + const done = yield* publisher.sweep(NOW + 120_000) + assert.strictEqual(done.cleaned, 1) + const { kv, objects } = yield* store.contents + assert.isFalse(kv.has(StatusPageKeys.snapshot(pageId))) + assert.isFalse(objects.has(StatusPageKeys.snapshotObject(pageId))) + assert.isFalse(kv.has(StatusPageKeys.slug("doomed"))) + assert.isFalse(objects.has(StatusPageKeys.slugObject("doomed"))) + assert.isFalse(kv.has(StatusPageKeys.overlayEntry(pageId, "e1"))) + assert.isFalse(kv.has(StatusPageKeys.overlayEntry(pageId, "e2"))) + assert.isTrue(kv.has(StatusPageKeys.overlayEntry(other, "e3"))) + assert.isTrue(kv.has(StatusPageKeys.snapshot(other))) + assert.deepStrictEqual(yield* cleanupRows(testDb), []) + }), + ), + ) }) diff --git a/packages/backend/src/services/status-pages/StatusPagePublisher.ts b/packages/backend/src/services/status-pages/StatusPagePublisher.ts index 35742355d6..539c670b6c 100644 --- a/packages/backend/src/services/status-pages/StatusPagePublisher.ts +++ b/packages/backend/src/services/status-pages/StatusPagePublisher.ts @@ -1,6 +1,8 @@ import * as PG from "@maple-dev/effect-orm/postgres" -import { StatusPages } from "@maple/db/tables" +import { StatusPageCleanups, StatusPages, UptimeMonitorDaily, UptimeMonitors } from "@maple/db/tables" +import type { StatusPageCleanupRow } from "@maple/db/tables" import type { StatusPageId } from "@maple/domain/primitives" +import { utcDay } from "@maple/domain/status-page" import { Clock, Context, Effect, Layer } from "effect" import { Database } from "@maple/backend/platform/DatabaseLive" import { makeDbExecute, makePersistenceErrorMapper } from "@maple/backend/platform/db-execute" @@ -13,7 +15,10 @@ import { StatusPageStore, type StatusPageStoreError } from "./StatusPageStore" export const STATUS_PAGE_REPUBLISH_AFTER_MS = 15 * 60_000 /** Pages one sweep publishes at most; the rest wait for the next minute. */ export const STATUS_PAGE_SWEEP_BATCH = 100 +/** Edge cleanups one sweep drains at most, oldest first. */ +export const STATUS_PAGE_CLEANUP_BATCH = 50 const SWEEP_CONCURRENCY = 4 +const DAY_MS = 86_400_000 export interface StatusPagePublishResult { readonly pageId: StatusPageId @@ -27,15 +32,25 @@ export interface StatusPageSweepResult { readonly written: number readonly unchanged: number readonly failed: number + /** Edge cleanups done (their rows deleted) and ones left for the next sweep. */ + readonly cleaned: number + readonly cleanupsFailed: number } export type StatusPagePublishError = StatusPageGoneError | StatusPagePublishDbError | StatusPageStoreError export interface StatusPagePublisherApi { readonly publish: (pageId: StatusPageId) => Effect.Effect - /** Publishes pages whose revision lags or whose last publish is stale; a failing page is logged and skipped. */ + /** + * Drains edge cleanups, publishes pages whose revision lags or whose last publish is stale + * (a failing page is logged and skipped), then writes the heartbeat. A database failure + * fails the whole sweep, so no heartbeat claims a live publisher. + */ readonly sweep: (nowMs: number) => Effect.Effect - readonly heartbeat: (nowMs: number) => Effect.Effect + /** Writes the heartbeat with the newest scheduled check across all monitors. */ + readonly heartbeat: ( + nowMs: number, + ) => Effect.Effect } const toHex = (buffer: ArrayBuffer) => @@ -58,11 +73,11 @@ export class StatusPagePublisher extends Context.Service + const recordFailure = (pageId: StatusPageId, nowMs: number, error: StatusPageStoreError) => dbExecute((db) => db.run( PG.update(StatusPages) - .set({ lastPublishError: error.message }) + .set({ lastPublishError: error.message, publishAttemptedAt: nowMs }) .where(($) => [$.id.eq(pageId)]), ), ).pipe(Effect.ignore, Effect.andThen(Effect.fail(error))) @@ -74,8 +89,8 @@ export class StatusPagePublisher extends Context.Service ({ orgId: $.orgId, - slug: $.slug, revision: $.revision, + publishedRevision: $.publishedRevision, publishedHash: $.publishedHash, })) .where(($) => [$.id.eq(pageId)]) @@ -98,9 +113,10 @@ export class StatusPagePublisher extends Context.Service recordFailure(pageId, error)), + Effect.catch((error) => recordFailure(pageId, nowMs, error)), ) } + // Advances only while no other publish moved it since our read, stale republishes included. const advanced = yield* dbExecute((db) => db.run( PG.update(StatusPages) @@ -108,9 +124,13 @@ export class StatusPagePublisher extends Context.Service [$.id.eq(pageId), $.publishedRevision.lt(revision)]) + .where(($) => [ + $.id.eq(pageId), + $.publishedRevision.eq(current.publishedRevision), + ]) .returning("id"), ), ) @@ -119,7 +139,11 @@ export class StatusPagePublisher extends Context.Service db.run( PG.update(StatusPages) - .set({ publishedAt: nowMs, lastPublishError: null }) + .set({ + publishedAt: nowMs, + publishAttemptedAt: nowMs, + lastPublishError: null, + }) .where(($) => [$.id.eq(pageId)]), ), ) @@ -132,8 +156,83 @@ export class StatusPagePublisher extends Context.Service + Effect.gen(function* () { + const owners = yield* dbExecute((db) => + db.run( + PG.from(StatusPages) + .select(($) => ({ id: $.id })) + .where(($) => [$.slug.eq(row.slug)]) + .limit(1), + ), + ) + const routedTo = yield* store.readSlugRoute(row.slug) + const slugIsOurs = owners.length === 0 && (routedTo === null || routedTo === row.pageId) + if (slugIsOurs) yield* store.deleteRoutes({ slug: row.slug, hostnames: [] }) + if (row.deleteSnapshot) yield* store.deletePage(row.pageId) + }) + + /** Oldest first; a row goes only once its edge deletes succeeded. */ + const drainCleanups = Effect.gen(function* () { + const rows = yield* dbExecute((db) => + db.run( + PG.from(StatusPageCleanups) + .select() + .orderBy(["createdAt", "asc"], ["id", "asc"]) + .limit(STATUS_PAGE_CLEANUP_BATCH), + ), + ) + // In order: a rename's slug cleanup lands before a later delete of the same page. + const outcomes = yield* Effect.forEach(rows, (row) => + cleanEdge(row).pipe( + Effect.andThen( + dbExecute((db) => + db.run(PG.deleteFrom(StatusPageCleanups).where(($) => [$.id.eq(row.id)])), + ), + ), + Effect.as(true), + Effect.catchTag("@maple/backend/status-pages/StatusPageStoreError", (error) => + Effect.logWarning("Status page edge cleanup failed").pipe( + Effect.annotateLogs({ + "maple.status_page.id": row.pageId, + "error.message": error.message, + }), + Effect.as(false), + ), + ), + ), + ) + return { + cleaned: outcomes.filter((ok) => ok).length, + cleanupsFailed: outcomes.filter((ok) => !ok).length, + } + }) + + const heartbeat = Effect.fn("StatusPagePublisher.heartbeat")(function* (nowMs: number) { + // The daily counters fold in every scheduled check; their PK (monitor, day) serves this + // per enabled monitor. Heartbeat monitors are excluded: their misses are not probes. + const [seen] = yield* dbExecute((db) => + db.run( + PG.from(UptimeMonitorDaily, "d") + .innerJoin(UptimeMonitors, "m", (d, m) => + d.monitorId.eq(m.id).and(d.orgId.eq(m.orgId)), + ) + .select(($) => ({ at: PG.max($.lastCheckAt) })) + .where(($) => [ + $.m.enabled.eq(true), + $.m.kind.neq("heartbeat"), + PG.inList($.day, [utcDay(nowMs), utcDay(nowMs - DAY_MS)]), + ]), + ), + ) + yield* store.putHeartbeat({ at: nowMs, probesSeenAt: seen?.at ?? null }) + }) + const sweep = Effect.fn("StatusPagePublisher.sweep")(function* (nowMs: number) { + const cleanups = yield* drainCleanups const staleBefore = nowMs - STATUS_PAGE_REPUBLISH_AFTER_MS + // Longest-waiting attempt first (never-attempted before all), so failing pages can't starve the rest. const due = yield* dbExecute((db) => db.run( PG.from(StatusPages) @@ -145,10 +244,29 @@ export class StatusPagePublisher extends Context.Service [ + [PG.asBoolean($.publishAttemptedAt.isNotNull()), "asc"], + [$.publishAttemptedAt, "asc"], + [$.id, "asc"], + ]) .limit(STATUS_PAGE_SWEEP_BATCH), ), ) + // Marked up front, so a page whose build fails still moves to the back of the queue. + if (due.length > 0) { + yield* dbExecute((db) => + db.run( + PG.update(StatusPages) + .set({ publishAttemptedAt: nowMs }) + .where(($) => [ + PG.inList( + $.id, + due.map((row) => row.id), + ), + ]), + ), + ) + } const outcomes = yield* Effect.forEach( due, ({ id }) => @@ -174,19 +292,25 @@ export class StatusPagePublisher extends Context.Service o === "written").length, unchanged: outcomes.filter((o) => o === "unchanged").length, failed: outcomes.filter((o) => o === "failed").length, + ...cleanups, } + // Only after a sweep that reached the database; a store failure here is logged. + yield* heartbeat(nowMs).pipe( + Effect.catchTag("@maple/backend/status-pages/StatusPageStoreError", (error) => + Effect.logWarning("Status page heartbeat write failed").pipe( + Effect.annotateLogs({ "error.message": error.message }), + ), + ), + ) yield* Effect.annotateCurrentSpan({ "maple.status_page.due_count": result.due, "maple.status_page.written_count": result.written, "maple.status_page.failed_count": result.failed, + "maple.status_page.cleaned_count": result.cleaned, }) return result }) - const heartbeat = Effect.fn("StatusPagePublisher.heartbeat")(function* (nowMs: number) { - yield* store.putHeartbeat(nowMs) - }) - return { publish, sweep, heartbeat } }), }, diff --git a/packages/backend/src/services/status-pages/StatusPageReconciler.test.ts b/packages/backend/src/services/status-pages/StatusPageReconciler.test.ts index 7cba2f18bc..d8de88b9d4 100644 --- a/packages/backend/src/services/status-pages/StatusPageReconciler.test.ts +++ b/packages/backend/src/services/status-pages/StatusPageReconciler.test.ts @@ -1,19 +1,33 @@ import { afterEach, assert, describe, it } from "@effect/vitest" -import { Effect, Exit, Layer } from "effect" +import { Effect, Layer } from "effect" +import { TestClock } from "effect/testing" import { overlayIncidentId, StatusPageKeys, type StatusPageOverlayEntry } from "@maple/domain/status-page" -import { cleanupTestDbs, createTestDb, type TestDb } from "@maple/backend/platform/test-pglite" -import { StatusPageReconciler } from "./StatusPageReconciler" -import { StatusPageStore } from "./StatusPageStore" -import { newPageId, NOW, readPage, seedComponent, seedPage } from "./status-page-test-support" +import type { StatusPageId } from "@maple/domain/primitives" +import { cleanupTestDbs, createTestDb, executeSql, type TestDb } from "@maple/backend/platform/test-pglite" +import { Database } from "@maple/backend/platform/DatabaseLive" +import { importEntry, StatusPageReconciler } from "./StatusPageReconciler" +import { type InMemoryStatusPageStore, StatusPageStore } from "./StatusPageStore" +import { + NOW, + newPageId, + ORG, + readPage, + seedAlertIncident, + seedComponent, + seedIncident, + seedPage, +} from "./status-page-test-support" const trackedDbs: TestDb[] = [] afterEach(() => cleanupTestDbs(trackedDbs)) -const entry = (overrides: Partial & Pick) => ({ +const entry = ( + overrides: Partial & Pick, +): StatusPageOverlayEntry => ({ incidentId: null, title: null, - status: "investigating" as const, - impact: "major" as const, + status: "investigating", + impact: "major", body: "Update", componentIds: [], ...overrides, @@ -25,145 +39,346 @@ const incidentsOf = (testDb: TestDb, pageId: string) => id: string title: string status: string + impact: string resolved: boolean components: number }>( - `SELECT i.id, i.title, i.status, i.resolved_at IS NOT NULL AS resolved, + `SELECT i.id, i.title, i.status, i.impact, i.resolved_at IS NOT NULL AS resolved, (SELECT count(*)::int FROM status_page_incident_components c WHERE c.incident_id = i.id) AS components - FROM status_page_incidents i WHERE i.page_id = $1`, + FROM status_page_incidents i WHERE i.page_id = $1 ORDER BY i.started_at`, [pageId], ), ).pipe(Effect.map((result) => result.rows)) const updatesOf = (testDb: TestDb, incidentId: string) => Effect.promise(() => - testDb.pglite.query<{ overlay_entry_id: string; source: string }>( - `SELECT overlay_entry_id, source FROM status_page_incident_updates WHERE incident_id = $1 ORDER BY created_at`, + testDb.pglite.query<{ overlay_entry_id: string | null; source: string; status: string }>( + `SELECT overlay_entry_id, source, status FROM status_page_incident_updates WHERE incident_id = $1 ORDER BY created_at`, [incidentId], ), ).pipe(Effect.map((result) => result.rows)) +const putEntry = (store: InMemoryStatusPageStore, pageId: StatusPageId, value: StatusPageOverlayEntry) => + store.putKv(StatusPageKeys.overlayEntry(pageId, value.id), JSON.stringify(value)) + +/** Marks the page published at `at`, as the publisher would after a sweep. */ +const markPublished = (testDb: TestDb, pageId: string, at: number) => + Effect.promise(() => + executeSql( + testDb, + `UPDATE status_pages SET published_revision = revision, published_at = $2 WHERE id = $1`, + [pageId, new Date(at).toISOString()], + ), + ) + +/** Runs `body` with a reconciler over a fresh database and an in-memory store. */ +const withReconciler = ( + body: (ctx: { + readonly testDb: TestDb + readonly store: InMemoryStatusPageStore + readonly reconciler: StatusPageReconciler["Service"] + }) => Effect.Effect, + options: { readonly listPageSize?: number } = {}, +) => + Effect.gen(function* () { + yield* TestClock.setTime(NOW) + const testDb = createTestDb(trackedDbs) + const store = yield* StatusPageStore.makeInMemoryWith(options) + const layer = StatusPageReconciler.layer.pipe( + Layer.provide(Layer.succeed(StatusPageStore, store.api)), + Layer.provide(testDb.layer), + ) + return yield* StatusPageReconciler.use((reconciler) => body({ testDb, store, reconciler })).pipe( + Effect.provide(layer), + ) + }) + describe("StatusPageReconciler", () => { - it.effect("imports a new incident and its follow-up once, then clears only those entries", () => - Effect.gen(function* () { - const testDb = createTestDb(trackedDbs) - const store = yield* StatusPageStore.makeInMemory - const pageId = newPageId() - const componentId = crypto.randomUUID() - yield* seedPage(testDb, { id: pageId, slug: "acme-bg", breakGlass: true }) - yield* seedComponent(testDb, { id: componentId, pageId, name: "API", position: 0 }) - - const opening = crypto.randomUUID() - const followUp = crypto.randomUUID() - const entries = [ - // Out of order on purpose: import goes by `at`. - entry({ - id: followUp, - at: NOW + 60_000, - incidentId: overlayIncidentId(opening), - status: "resolved", - body: "Fixed", - }), - entry({ - id: opening, + it.effect("imports a new incident and its `bg-` follow-up once, keeping keys until published", () => + withReconciler(({ testDb, store, reconciler }) => + Effect.gen(function* () { + const pageId = newPageId() + const componentId = crypto.randomUUID() + // No token: entries posted before a revoke still import. + yield* seedPage(testDb, { id: pageId, slug: "acme-bg" }) + yield* seedComponent(testDb, { id: componentId, pageId, name: "API", position: 0 }) + + const opening = entry({ + id: crypto.randomUUID(), at: NOW, title: "Login broken", componentIds: [componentId, "not-a-component"], + }) + const followUp = entry({ + id: crypto.randomUUID(), + at: NOW + 60_000, + incidentId: overlayIncidentId(opening.id), + status: "resolved", + impact: null, + body: "Fixed", + }) + yield* putEntry(store, pageId, opening) + yield* putEntry(store, pageId, followUp) + + const first = yield* reconciler.reconcile(pageId) + assert.deepStrictEqual(first, { imported: 2, cleared: 0 }) + const incidents = yield* incidentsOf(testDb, pageId) + assert.deepStrictEqual( + incidents.map(({ title, status, impact, resolved, components }) => ({ + title, + status, + impact, + resolved, + components, + })), + [ + { + title: "Login broken", + status: "resolved", + impact: "major", + resolved: true, + components: 1, + }, + ], + ) + const incidentId = incidents[0]?.id ?? "" + assert.deepStrictEqual( + (yield* updatesOf(testDb, incidentId)).map((u) => [u.overlay_entry_id, u.source]), + [ + [opening.id, "break_glass"], + [followUp.id, "break_glass"], + ], + ) + assert.strictEqual((yield* readPage(testDb, pageId))?.revision, "3") + + // Not yet in a published snapshot: the edge must keep overlaying them. + const unpublished = yield* reconciler.reconcileAll() + assert.deepStrictEqual(unpublished, { pages: 1, imported: 0, cleared: 0, failed: 0 }) + assert.isTrue((yield* store.contents).kv.has(StatusPageKeys.overlayEntry(pageId, opening.id))) + + yield* markPublished(testDb, pageId, NOW + 5 * 60_000) + const published = yield* reconciler.reconcileAll() + assert.deepStrictEqual(published, { pages: 1, imported: 0, cleared: 2, failed: 0 }) + const { kv } = yield* store.contents + assert.isFalse(kv.has(StatusPageKeys.overlayEntry(pageId, opening.id))) + assert.isFalse(kv.has(StatusPageKeys.overlayEntry(pageId, followUp.id))) + assert.strictEqual((yield* incidentsOf(testDb, pageId)).length, 1) + assert.strictEqual((yield* updatesOf(testDb, incidentId)).length, 2) + }), + ), + ) + + it.effect("keeps a key whose page published before the import's revision", () => + withReconciler(({ testDb, store, reconciler }) => + Effect.gen(function* () { + const pageId = newPageId() + yield* seedPage(testDb, { id: pageId, slug: "acme-lag" }) + const only = entry({ id: crypto.randomUUID(), at: NOW }) + yield* putEntry(store, pageId, only) + yield* reconciler.reconcileAll() + // Published, but a later bump (revision past published_revision) is still pending. + yield* markPublished(testDb, pageId, NOW + 60_000) + yield* Effect.promise(() => + executeSql(testDb, `UPDATE status_pages SET revision = revision + 1 WHERE id = $1`, [ + pageId, + ]), + ) + assert.strictEqual((yield* reconciler.reconcileAll()).cleared, 0) + assert.isTrue((yield* store.contents).kv.has(StatusPageKeys.overlayEntry(pageId, only.id))) + }), + ), + ) + + it.effect("reconciles every page across KV list pages", () => + withReconciler( + ({ testDb, store, reconciler }) => + Effect.gen(function* () { + const pageIds = Array.from({ length: 4 }, () => newPageId()) + yield* Effect.forEach( + pageIds, + (pageId, index) => + seedPage(testDb, { id: pageId, slug: `many-${index}` }).pipe( + Effect.andThen( + putEntry(store, pageId, entry({ id: crypto.randomUUID(), at: NOW })), + ), + Effect.andThen( + putEntry(store, pageId, entry({ id: crypto.randomUUID(), at: NOW + 1 })), + ), + ), + { discard: true }, + ) + // Eight keys, three per list call. + const result = yield* reconciler.reconcileAll() + assert.deepStrictEqual(result, { pages: 4, imported: 8, cleared: 0, failed: 0 }) + const counts = yield* Effect.forEach(pageIds, (pageId) => + incidentsOf(testDb, pageId).pipe(Effect.map((rows) => rows.length)), + ) + assert.deepStrictEqual(counts, [2, 2, 2, 2]) }), - ] - yield* store.putKv(StatusPageKeys.overlay(pageId), JSON.stringify({ entries })) - - const layer = StatusPageReconciler.layer.pipe( - Layer.provide(Layer.succeed(StatusPageStore, store.api)), - Layer.provide(testDb.layer), - ) - const reconciler = yield* StatusPageReconciler.pipe(Effect.provide(layer)) - - const first = yield* reconciler.reconcile(pageId) - assert.strictEqual(first.imported, 2) - const incidents = yield* incidentsOf(testDb, pageId) - assert.deepStrictEqual( - incidents.map(({ title, status, resolved, components }) => ({ - title, - status, - resolved, - components, - })), - [{ title: "Login broken", status: "resolved", resolved: true, components: 1 }], - ) - const incidentId = incidents[0]?.id ?? "" - assert.deepStrictEqual(yield* updatesOf(testDb, incidentId), [ - { overlay_entry_id: opening, source: "break_glass" }, - { overlay_entry_id: followUp, source: "break_glass" }, - ]) - assert.isFalse((yield* store.contents).kv.has(StatusPageKeys.overlay(pageId))) - assert.strictEqual((yield* readPage(testDb, pageId))?.revision, "3") - - // The edge re-posts the same entries plus a new one (a re-import must not duplicate). - const later = crypto.randomUUID() - yield* store.putKv( - StatusPageKeys.overlay(pageId), - JSON.stringify({ - entries: [ - ...entries, + { listPageSize: 3 }, + ), + ) + + it.effect("imports an entry once when two reconciles race", () => + withReconciler(({ testDb, store, reconciler }) => + Effect.gen(function* () { + const pageId = newPageId() + yield* seedPage(testDb, { id: pageId, slug: "acme-race" }) + const only = entry({ id: crypto.randomUUID(), at: NOW, title: "Outage" }) + yield* putEntry(store, pageId, only) + const results = yield* Effect.all( + [reconciler.reconcile(pageId), reconciler.reconcile(pageId)], + { + concurrency: 2, + }, + ) + assert.strictEqual( + results.reduce((sum, r) => sum + r.imported, 0), + 1, + ) + const incidents = yield* incidentsOf(testDb, pageId) + assert.strictEqual(incidents.length, 1) + assert.strictEqual((yield* updatesOf(testDb, incidents[0]?.id ?? "")).length, 1) + }), + ), + ) + + it.effect("rolls back a losing import, leaving no orphan incident", () => + withReconciler(({ testDb }) => + Effect.gen(function* () { + const pageId = newPageId() + yield* seedPage(testDb, { id: pageId, slug: "acme-lost" }) + const only = entry({ id: crypto.randomUUID(), at: NOW, title: "Outage" }) + const scope = { orgId: ORG, pageId, componentIds: [] } + const database = yield* Database + // The second import read no update row, as a concurrent run would have. + const won = yield* database.execute((db) => importEntry(db, scope, only)) + const lost = yield* database.execute((db) => importEntry(db, scope, only)) + assert.deepStrictEqual([won, lost], [true, false]) + assert.strictEqual((yield* incidentsOf(testDb, pageId)).length, 1) + assert.strictEqual((yield* readPage(testDb, pageId))?.revision, "2") + }).pipe(Effect.provide(testDb.layer)), + ), + ) + + it.effect( + "keeps impact and status on a no-change entry, and ignores a status of the other lifecycle", + () => + withReconciler(({ testDb, store, reconciler }) => + Effect.gen(function* () { + const pageId = newPageId() + const incidentId = crypto.randomUUID() + yield* seedPage(testDb, { id: pageId, slug: "acme-keep" }) + yield* seedIncident(testDb, { + id: incidentId, + pageId, + title: "Slow API", + impact: "critical", + status: "identified", + startedAt: NOW - 60_000, + }) + yield* putEntry( + store, + pageId, entry({ - id: later, - at: NOW + 120_000, + id: crypto.randomUUID(), + at: NOW, incidentId, - status: "monitoring", - body: "Watching", + status: null, + impact: null, + body: "Still on it", }), - ], + ) + yield* putEntry( + store, + pageId, + entry({ + id: crypto.randomUUID(), + at: NOW + 1, + incidentId, + status: "completed", + impact: null, + }), + ) + assert.strictEqual((yield* reconciler.reconcile(pageId)).imported, 2) + const [incident] = yield* incidentsOf(testDb, pageId) + assert.deepStrictEqual( + [incident?.status, incident?.impact, incident?.resolved], + ["identified", "critical", false], + ) + assert.deepStrictEqual( + (yield* updatesOf(testDb, incidentId)).map((u) => u.status), + ["identified", "identified", "identified"], + ) }), - ) - const second = yield* reconciler.reconcileAll() - assert.deepStrictEqual(second, { pages: 1, imported: 3, failed: 0 }) - assert.strictEqual((yield* incidentsOf(testDb, pageId)).length, 1) - assert.strictEqual((yield* updatesOf(testDb, incidentId)).length, 3) - assert.strictEqual((yield* incidentsOf(testDb, pageId))[0]?.status, "monitoring") - }), + ), ) - it.effect("leaves the overlay when clearing it fails, and a rerun does not import twice", () => - Effect.gen(function* () { - const testDb = createTestDb(trackedDbs) - const store = yield* StatusPageStore.makeInMemory - const pageId = newPageId() - yield* seedPage(testDb, { id: pageId, slug: "acme-keep", breakGlass: true }) - const only = entry({ id: crypto.randomUUID(), at: NOW, title: "Outage" }) - yield* store.putKv(StatusPageKeys.overlay(pageId), JSON.stringify({ entries: [only] })) - yield* store.setFailing((key) => key === StatusPageKeys.overlay(pageId)) - - const layer = StatusPageReconciler.layer.pipe( - Layer.provide(Layer.succeed(StatusPageStore, store.api)), - Layer.provide(testDb.layer), - ) - const reconciler = yield* StatusPageReconciler.pipe(Effect.provide(layer)) - const failed = yield* Effect.exit(reconciler.reconcile(pageId)) - assert.isTrue(Exit.isFailure(failed)) - assert.isTrue((yield* store.contents).kv.has(StatusPageKeys.overlay(pageId))) - - yield* store.setFailing(false) - assert.strictEqual((yield* reconciler.reconcile(pageId)).imported, 1) - assert.strictEqual((yield* incidentsOf(testDb, pageId)).length, 1) - assert.isFalse((yield* store.contents).kv.has(StatusPageKeys.overlay(pageId))) - }), + it.effect("opens a new incident as investigating and major when the entry leaves them null", () => + withReconciler(({ testDb, store, reconciler }) => + Effect.gen(function* () { + const pageId = newPageId() + yield* seedPage(testDb, { id: pageId, slug: "acme-defaults" }) + yield* putEntry( + store, + pageId, + entry({ id: crypto.randomUUID(), at: NOW, status: null, impact: null }), + ) + yield* reconciler.reconcile(pageId) + const [incident] = yield* incidentsOf(testDb, pageId) + assert.deepStrictEqual([incident?.status, incident?.impact], ["investigating", "major"]) + }), + ), ) }) -describe("StatusPageStore.removeOverlayEntries", () => { - it.effect("removes only the given entries", () => - Effect.gen(function* () { - const store = yield* StatusPageStore.makeInMemory - const pageId = newPageId() - const a = entry({ id: "a", at: NOW }) - const b = entry({ id: "b", at: NOW + 1 }) - yield* store.putKv(StatusPageKeys.overlay(pageId), JSON.stringify({ entries: [a, b] })) - yield* store.api.removeOverlayEntries(pageId, ["a"]) - assert.deepStrictEqual( - (yield* store.api.readOverlay(pageId)).entries.map((e) => e.id), - ["b"], - ) - }), +describe("StatusPageReconciler.repairAutoIncidents", () => { + it.effect("resolves auto incidents whose alert incident is resolved or gone, and only those", () => + withReconciler(({ testDb, reconciler }) => + Effect.gen(function* () { + const pageId = newPageId() + yield* seedPage(testDb, { id: pageId, slug: "acme-auto" }) + const resolvedAlert = crypto.randomUUID() + const openAlert = crypto.randomUUID() + yield* seedAlertIncident(testDb, { id: resolvedAlert, status: "resolved" }) + yield* seedAlertIncident(testDb, { id: openAlert, status: "open" }) + const orphanOfResolved = crypto.randomUUID() + const orphanOfMissing = crypto.randomUUID() + const stillOpen = crypto.randomUUID() + const seed = (id: string, alertIncidentId: string, startedAt: number) => + seedIncident(testDb, { + id, + pageId, + title: id, + status: "investigating", + startedAt, + alertIncidentId, + }) + yield* seed(orphanOfResolved, resolvedAlert, NOW - 3_000) + yield* seed(orphanOfMissing, crypto.randomUUID(), NOW - 2_000) + yield* seed(stillOpen, openAlert, NOW - 1_000) + const before = Number((yield* readPage(testDb, pageId))?.revision) + + assert.strictEqual(yield* reconciler.repairAutoIncidents(), 2) + const incidents = yield* incidentsOf(testDb, pageId) + assert.deepStrictEqual( + incidents.map((i) => [i.id, i.status]), + [ + [orphanOfResolved, "resolved"], + [orphanOfMissing, "resolved"], + [stillOpen, "investigating"], + ], + ) + assert.deepStrictEqual( + (yield* updatesOf(testDb, orphanOfMissing)).map((u) => [u.source, u.status]), + [ + ["dashboard", "investigating"], + ["auto", "resolved"], + ], + ) + assert.strictEqual(Number((yield* readPage(testDb, pageId))?.revision), before + 2) + assert.strictEqual(yield* reconciler.repairAutoIncidents(), 0) + }), + ), ) }) diff --git a/packages/backend/src/services/status-pages/StatusPageReconciler.ts b/packages/backend/src/services/status-pages/StatusPageReconciler.ts index 54b8ff04ae..7f34642e3e 100644 --- a/packages/backend/src/services/status-pages/StatusPageReconciler.ts +++ b/packages/backend/src/services/status-pages/StatusPageReconciler.ts @@ -13,31 +13,41 @@ import { type StatusPageComponentId, type StatusPageId, StatusPageIncidentId, + type StatusPageIncidentImpact, + type StatusPageIncidentKind, + type StatusPageIncidentStatus, StatusPageIncidentUpdateId, } from "@maple/domain/primitives" -import type { StatusPageOverlayEntry } from "@maple/domain/status-page" -import { Context, Effect, Layer, Option, Ref, Result, Schema } from "effect" +import { StatusPageKeys, statusFitsKind, type StatusPageOverlayEntry } from "@maple/domain/status-page" +import { Clock, Context, Effect, Layer, Option, Schema } from "effect" import { Database } from "@maple/backend/platform/DatabaseLive" import { makeDbExecute, makePersistenceErrorMapper } from "@maple/backend/platform/db-execute" +import { repairOrphanedAutoIncidents } from "./auto-incidents" import { bumpPage } from "./outbox" import { StatusPagePublishDbError } from "./publish-errors" -import { StatusPageStore, type StatusPageStoreError } from "./StatusPageStore" +import { parseOverlayKey, StatusPageStore, type StatusPageStoreError } from "./StatusPageStore" -// Imports break-glass overlay entries (posted to the status Worker while Maple was down) into -// Postgres. Each update keeps `overlay_entry_id`, unique, so importing an entry twice is a no-op. +// Imports break-glass entries (one KV key each, posted to the status Worker while Maple was +// down) into Postgres. Each update keeps `overlay_entry_id`, unique, so an import is idempotent; +// a key is deleted only once a published snapshot carries its update. -const RECONCILE_BATCH = 100 +/** KV list calls one run makes at most (1000 keys each). */ +const RECONCILE_LIST_PAGES = 10 const RECONCILE_CONCURRENCY = 4 +const ENTRY_READ_CONCURRENCY = 8 const OVERLAY_INCIDENT_PREFIX = "bg-" export interface StatusPageReconcileResult { - /** Entries now in Postgres (including ones a previous run already imported). */ + /** Entries imported by this run. */ readonly imported: number + /** Entry keys deleted because a published snapshot carries them. */ + readonly cleared: number } export interface StatusPageReconcileAllResult { readonly pages: number readonly imported: number + readonly cleared: number readonly failed: number } @@ -47,44 +57,68 @@ export interface StatusPageReconcilerApi { readonly reconcile: ( pageId: StatusPageId, ) => Effect.Effect - /** Reconciles every page with a break-glass token; a failing page is logged and skipped. */ - readonly reconcileAll: () => Effect.Effect + /** Reconciles every page with break-glass entries in KV; a failing page is logged and skipped. */ + readonly reconcileAll: () => Effect.Effect + /** Resolves auto incidents whose alert incident is resolved or gone; returns how many. */ + readonly repairAutoIncidents: () => Effect.Effect } -const isResolved = (status: StatusPageOverlayEntry["status"]) => - status === "resolved" || status === "completed" +/** A concurrent import inserted the entry's update first; rolls this attempt back. */ +class StatusPageOverlayImportRaced extends Schema.TaggedError()( + "@maple/backend/status-pages/StatusPageOverlayImportRaced", + { message: Schema.String, entryId: Schema.String }, +) {} + +const isResolved = (status: StatusPageIncidentStatus) => status === "resolved" || status === "completed" const decodeIncidentId = Schema.decodeUnknownOption(StatusPageIncidentId) -interface PageScope { +export interface PageScope { readonly orgId: OrgId readonly pageId: StatusPageId readonly componentIds: ReadonlyArray } +interface TargetIncident { + readonly id: StatusPageIncidentId + readonly kind: StatusPageIncidentKind + readonly status: StatusPageIncidentStatus + readonly impact: StatusPageIncidentImpact + readonly resolvedAt: number | null +} + +const targetColumns = PG.from(StatusPageIncidents).select(($) => ({ + id: $.id, + kind: $.kind, + status: $.status, + impact: $.impact, + resolvedAt: $.resolvedAt, +})) + /** The real incident an entry targets, if it still exists on this page. */ const findTarget = (db: MapleDb, scope: PageScope, entry: StatusPageOverlayEntry) => Effect.gen(function* () { - if (entry.incidentId === null) return Option.none() + if (entry.incidentId === null) return Option.none() + const byId = (id: StatusPageIncidentId) => + db + .run( + targetColumns + .where(($) => [$.orgId.eq(scope.orgId), $.pageId.eq(scope.pageId), $.id.eq(id)]) + .limit(1), + ) + .pipe(Effect.map((rows) => Option.fromNullishOr(rows[0]))) if (entry.incidentId.startsWith(OVERLAY_INCIDENT_PREFIX)) { - // An incident the overlay opened: its first update carries the opening entry's id. + // `bg-`: the incident that entry opened (or updated) holds its update. const openingEntryId = entry.incidentId.slice(OVERLAY_INCIDENT_PREFIX.length) - const rows = yield* db.run( + const [opening] = yield* db.run( PG.from(StatusPageIncidentUpdates) .select(($) => ({ incidentId: $.incidentId })) .where(($) => [$.orgId.eq(scope.orgId), $.overlayEntryId.eq(openingEntryId)]) .limit(1), ) - return Option.fromNullishOr(rows[0]?.incidentId) + return opening === undefined ? Option.none() : yield* byId(opening.incidentId) } const id = decodeIncidentId(entry.incidentId) - if (Option.isNone(id)) return Option.none() - const rows = yield* db.run( - PG.from(StatusPageIncidents) - .select(($) => ({ id: $.id })) - .where(($) => [$.orgId.eq(scope.orgId), $.pageId.eq(scope.pageId), $.id.eq(id.value)]) - .limit(1), - ) - return Option.fromNullishOr(rows[0]?.id) + return Option.isNone(id) ? Option.none() : yield* byId(id.value) }) const linkComponents = ( @@ -102,80 +136,98 @@ const linkComponents = ( ) } -/** Imports one entry in its own transaction. */ -const importEntry = (db: MapleDb, scope: PageScope, entry: StatusPageOverlayEntry) => - db.transaction( - Effect.gen(function* () { - const already = yield* db.run( - PG.from(StatusPageIncidentUpdates) - .select(($) => ({ id: $.id })) - .where(($) => [$.orgId.eq(scope.orgId), $.overlayEntryId.eq(entry.id)]) - .limit(1), - ) - if (already.length > 0) return - const resolvedAt = isResolved(entry.status) ? entry.at : null - const target = yield* findTarget(db, scope, entry) - // An entry whose target is gone opens its own incident rather than losing the customer's words. - const incidentId = yield* Option.match(target, { - onSome: (id) => - db - .run( - PG.update(StatusPageIncidents) - .set({ - status: entry.status, - impact: entry.impact, - resolvedAt, - updatedAt: entry.at, - ...(entry.title === null ? undefined : { title: entry.title }), - }) - .where(($) => [$.orgId.eq(scope.orgId), $.id.eq(id)]), - ) - .pipe(Effect.as(id)), - onNone: () => { - const id = StatusPageIncidentId.make(randomUUID()) - return db - .run( - PG.insertInto(StatusPageIncidents).values({ - id, - orgId: scope.orgId, - pageId: scope.pageId, - kind: "incident", - title: entry.title ?? "Service disruption", - impact: entry.impact, - status: entry.status, - published: true, - alertIncidentId: null, - scheduledStart: null, - scheduledEnd: null, - startedAt: entry.at, - resolvedAt, - createdBy: null, - createdAt: entry.at, - updatedAt: entry.at, - }), - ) - .pipe(Effect.as(id)) - }, - }) - yield* linkComponents(db, scope, incidentId, entry.componentIds) +/** Writes the incident change for an entry; returns the incident and the status its update records. */ +const applyToIncident = (db: MapleDb, scope: PageScope, entry: StatusPageOverlayEntry) => + Effect.gen(function* () { + const target = yield* findTarget(db, scope, entry) + if (Option.isSome(target)) { + const existing = target.value + // Null keeps the current value; a status from the other lifecycle is ignored. + const status = + entry.status !== null && statusFitsKind(existing.kind, entry.status) + ? entry.status + : existing.status yield* db.run( - PG.insertInto(StatusPageIncidentUpdates) - .values({ - id: StatusPageIncidentUpdateId.make(randomUUID()), - orgId: scope.orgId, - incidentId, - status: entry.status, - body: entry.body, - source: "break_glass", - createdBy: null, - overlayEntryId: entry.id, - createdAt: entry.at, + PG.update(StatusPageIncidents) + .set({ + status, + impact: entry.impact ?? existing.impact, + resolvedAt: isResolved(status) ? (existing.resolvedAt ?? entry.at) : null, + updatedAt: entry.at, + ...(entry.title === null ? undefined : { title: entry.title }), }) - .onConflictDoNothing(), + .where(($) => [$.orgId.eq(scope.orgId), $.id.eq(existing.id)]), ) - yield* bumpPage(db, scope.orgId, scope.pageId) - }), - ) + return { incidentId: existing.id, status } + } + // No target (or it is gone): open an incident rather than lose the customer's words. + const id = StatusPageIncidentId.make(randomUUID()) + const status = + entry.status !== null && statusFitsKind("incident", entry.status) ? entry.status : "investigating" + yield* db.run( + PG.insertInto(StatusPageIncidents).values({ + id, + orgId: scope.orgId, + pageId: scope.pageId, + kind: "incident", + title: entry.title ?? "Service disruption", + impact: entry.impact ?? "major", + status, + published: true, + alertIncidentId: null, + scheduledStart: null, + scheduledEnd: null, + startedAt: entry.at, + resolvedAt: isResolved(status) ? entry.at : null, + createdBy: null, + createdAt: entry.at, + updatedAt: entry.at, + }), + ) + return { incidentId: id, status } + }) + +/** + * Imports one entry in one transaction. The update insert is the claim: when a concurrent + * import already holds `overlay_entry_id`, the transaction rolls back, incident writes included. + */ +export const importEntry = (db: MapleDb, scope: PageScope, entry: StatusPageOverlayEntry) => + db + .transaction( + Effect.gen(function* () { + const { incidentId, status } = yield* applyToIncident(db, scope, entry) + yield* linkComponents(db, scope, incidentId, entry.componentIds) + const claimed = yield* db.run( + PG.insertInto(StatusPageIncidentUpdates) + .values({ + id: StatusPageIncidentUpdateId.make(randomUUID()), + orgId: scope.orgId, + incidentId, + status, + body: entry.body, + source: "break_glass", + createdBy: null, + overlayEntryId: entry.id, + createdAt: entry.at, + }) + .onConflictDoNothing() + .returning("id"), + ) + if (claimed.length === 0) { + return yield* new StatusPageOverlayImportRaced({ + message: `Break-glass entry ${entry.id} was imported concurrently`, + entryId: entry.id, + }) + } + yield* bumpPage(db, scope.orgId, scope.pageId) + }), + ) + .pipe( + Effect.as(true), + Effect.catchTag("@maple/backend/status-pages/StatusPageOverlayImportRaced", () => + Effect.succeed(false), + ), + ) const toDbError = makePersistenceErrorMapper(StatusPagePublishDbError, "Status page overlay import failed") @@ -187,90 +239,157 @@ export class StatusPageReconciler extends Context.Service + /** Imports the page's entries and clears the keys a published snapshot already carries. */ + const reconcileEntries = Effect.fn("StatusPageReconciler.reconcilePage")(function* ( + pageId: StatusPageId, + entryIds: ReadonlyArray, + ) { + yield* Effect.annotateCurrentSpan({ + "maple.status_page.id": pageId, + "maple.status_page.entry_count": entryIds.length, + }) + const read = yield* Effect.forEach( + entryIds, + (entryId) => store.readOverlayEntry(StatusPageKeys.overlayEntry(pageId, entryId)), + { concurrency: ENTRY_READ_CONCURRENCY }, + ) + // A key names its entry; one that disagrees is not trusted. + const entries = read + .flatMap((entry) => + Option.isSome(entry) && entryIds.includes(entry.value.id) ? [entry.value] : [], + ) + .toSorted((a, b) => a.at - b.at) + if (entries.length === 0) return { imported: 0, cleared: 0 } + const page = yield* dbExecute((db) => Effect.gen(function* () { - const [page] = yield* db.run( + const [row] = yield* db.run( PG.from(StatusPages) - .select(($) => ({ orgId: $.orgId })) + .select(($) => ({ + orgId: $.orgId, + revision: $.revision, + publishedRevision: $.publishedRevision, + publishedAt: $.publishedAt, + })) .where(($) => [$.id.eq(pageId)]) .limit(1), ) - if (page === undefined) return null + if (row === undefined) return null const components = yield* db.run( PG.from(StatusPageComponents) .select(($) => ({ id: $.id })) - .where(($) => [$.orgId.eq(page.orgId), $.pageId.eq(pageId)]), + .where(($) => [$.orgId.eq(row.orgId), $.pageId.eq(pageId)]), + ) + const existing = yield* db.run( + PG.from(StatusPageIncidentUpdates) + .select(($) => ({ overlayEntryId: $.overlayEntryId, createdAt: $.createdAt })) + .where(($) => [ + $.orgId.eq(row.orgId), + // OR of equalities: the unique index serves it on a nullable column. + PG.or(...entries.map((entry) => $.overlayEntryId.eq(entry.id))), + ]), ) - return { orgId: page.orgId, pageId, componentIds: components.map((c) => c.id) } + return { ...row, componentIds: components.map((c) => c.id), existing } }), ) - if (scope === null) return { imported: 0 } - const entries = overlay.entries.toSorted((a, b) => a.at - b.at) - const imported = yield* Ref.make>([]) + // A deleted page's keys go with its edge cleanup. + if (page === null) return { imported: 0, cleared: 0 } + const scope: PageScope = { orgId: page.orgId, pageId, componentIds: page.componentIds } + const imported = new Set( + page.existing.flatMap((row) => (row.overlayEntryId === null ? [] : [row.overlayEntryId])), + ) // In order: a later entry can target the incident an earlier one opened. - const outcome = yield* Effect.result( - Effect.forEach( - entries, - (entry) => - dbExecute((db) => importEntry(db, scope, entry)).pipe( - Effect.andThen(Ref.update(imported, (ids) => [...ids, entry.id])), - ), - { discard: true }, - ), + const outcomes = yield* Effect.forEach( + entries.filter((entry) => !imported.has(entry.id)), + (entry) => dbExecute((db) => importEntry(db, scope, entry)), + ) + // Published at or past the revision read before this run's imports, and after the update. + const { publishedAt } = page + const carried = + page.publishedRevision >= page.revision && publishedAt !== null + ? page.existing.flatMap((row) => + row.overlayEntryId !== null && publishedAt > row.createdAt + ? [row.overlayEntryId] + : [], + ) + : [] + yield* Effect.forEach(carried, (entryId) => store.deleteOverlayEntry(pageId, entryId), { + concurrency: ENTRY_READ_CONCURRENCY, + discard: true, + }) + const result = { imported: outcomes.filter((ok) => ok).length, cleared: carried.length } + yield* Effect.annotateCurrentSpan({ + "maple.status_page.imported_count": result.imported, + "maple.status_page.cleared_count": result.cleared, + }) + return result + }) + + /** Entry ids per page among the listed keys. */ + const groupByPage = (keys: ReadonlyArray) => + keys.reduce>>((groups, key) => { + const parsed = parseOverlayKey(key) + if (Option.isNone(parsed)) return groups + const { pageId, entryId } = parsed.value + return new Map([...groups, [pageId, [...(groups.get(pageId) ?? []), entryId]]]) + }, new Map()) + + const reconcile = Effect.fn("StatusPageReconciler.reconcile")(function* (pageId: StatusPageId) { + const listing = yield* store.listKeys( + StatusPageKeys.overlayPage(pageId), + RECONCILE_LIST_PAGES, ) - const importedIds = yield* Ref.get(imported) - yield* store.removeOverlayEntries(pageId, importedIds) - yield* Effect.annotateCurrentSpan({ "maple.status_page.imported_count": importedIds.length }) - if (Result.isFailure(outcome)) return yield* Effect.fail(outcome.failure) - return { imported: importedIds.length } + const entryIds = groupByPage(listing.keys).get(pageId) ?? [] + if (entryIds.length === 0) return { imported: 0, cleared: 0 } + return yield* reconcileEntries(pageId, entryIds) }) const reconcileAll = Effect.fn("StatusPageReconciler.reconcileAll")(function* () { - const pages = yield* dbExecute((db) => - db.run( - PG.from(StatusPages) - .select(($) => ({ id: $.id })) - .where(($) => [$.breakGlassTokenSha256.isNotNull()]) - .orderBy(["id", "asc"]) - .limit(RECONCILE_BATCH), - ), - ) + // Whether or not the page still has a token: entries posted before a revoke still count. + const listing = yield* store.listKeys(StatusPageKeys.overlayAll, RECONCILE_LIST_PAGES) + const pages = [...groupByPage(listing.keys)] const outcomes = yield* Effect.forEach( pages, - ({ id }) => - reconcile(id).pipe( - Effect.map((result) => result.imported), + ([pageId, entryIds]) => + reconcileEntries(pageId, entryIds).pipe( + Effect.map(Option.some), Effect.catch((error) => Effect.logWarning("Status page overlay import failed").pipe( Effect.annotateLogs({ - "maple.status_page.id": id, + "maple.status_page.id": pageId, "error.type": error._tag, "error.message": error.message, }), - Effect.as(null), + Effect.as(Option.none()), ), ), ), { concurrency: RECONCILE_CONCURRENCY }, ) + const done = outcomes.flatMap((outcome) => (Option.isSome(outcome) ? [outcome.value] : [])) const result: StatusPageReconcileAllResult = { pages: pages.length, - imported: outcomes.reduce((sum, n) => sum + (n ?? 0), 0), - failed: outcomes.filter((n) => n === null).length, + imported: done.reduce((sum, r) => sum + r.imported, 0), + cleared: done.reduce((sum, r) => sum + r.cleared, 0), + failed: outcomes.length - done.length, } yield* Effect.annotateCurrentSpan({ "maple.status_page.page_count": result.pages, "maple.status_page.imported_count": result.imported, + "maple.status_page.cleared_count": result.cleared, "maple.status_page.failed_count": result.failed, + "maple.status_page.listing_complete": listing.complete, }) return result }) - return { reconcile, reconcileAll } + const repairAutoIncidents = Effect.fn("StatusPageReconciler.repairAutoIncidents")(function* () { + const nowMs = yield* Clock.currentTimeMillis + const repaired = yield* dbExecute((db) => repairOrphanedAutoIncidents(db, nowMs)) + yield* Effect.annotateCurrentSpan({ "maple.status_page.repaired_count": repaired }) + return repaired + }) + + return { reconcile, reconcileAll, repairAutoIncidents } }), }, ) { diff --git a/packages/backend/src/services/status-pages/StatusPageSnapshotBuilder.test.ts b/packages/backend/src/services/status-pages/StatusPageSnapshotBuilder.test.ts index c1576c6dcf..ac5155b47f 100644 --- a/packages/backend/src/services/status-pages/StatusPageSnapshotBuilder.test.ts +++ b/packages/backend/src/services/status-pages/StatusPageSnapshotBuilder.test.ts @@ -1,5 +1,5 @@ import { afterEach, assert, describe, it } from "@effect/vitest" -import { Effect, Exit } from "effect" +import { Effect, Exit, Layer } from "effect" import { StatusPageComponentId, StatusPageIncidentId, UptimeMonitorId } from "@maple/domain/primitives" import { utcDay } from "@maple/domain/status-page" import { cleanupTestDbs, createTestDb, type TestDb } from "@maple/backend/platform/test-pglite" @@ -126,7 +126,7 @@ describe("toSnapshot", () => { assert.strictEqual(snapshot.components[0]?.status, "operational") }) - it("fills 90 days of bars and takes probesSeenAt from the newest check", () => { + it("fills 90 days of bars", () => { const today = utcDay(NOW) const snapshot = toSnapshot( baseInput({ @@ -137,7 +137,6 @@ describe("toSnapshot", () => { checks: 10, failures: 1, downChecks: 0, - lastCheckAt: NOW - 5_000, }, { monitorId: MONITOR, @@ -145,7 +144,6 @@ describe("toSnapshot", () => { checks: 20, failures: 0, downChecks: 0, - lastCheckAt: NOW - DAY_MS, }, ], }), @@ -159,15 +157,59 @@ describe("toSnapshot", () => { failures: 0, downChecks: 0, }) - assert.strictEqual(snapshot.probesSeenAt, NOW - 5_000) }) - it("gives components without a monitor or with hidden uptime no bars", () => { + it("gives components without a monitor or with hidden uptime no bars, but still marks monitored ones", () => { const hidden = toSnapshot(baseInput({ components: componentWith({ showUptime: false }) })) const unmonitored = toSnapshot(baseInput({ components: componentWith({ monitorId: null }) })) assert.deepStrictEqual(hidden.components[0]?.days, []) assert.deepStrictEqual(unmonitored.components[0]?.days, []) - assert.strictEqual(unmonitored.probesSeenAt, null) + assert.isTrue(hidden.components[0]?.monitored) + assert.isFalse(unmonitored.components[0]?.monitored) + }) + + it("links break-glass updates to their entries and names the entry that opened an incident", () => { + const update = ( + id: string, + at: number, + source: "break_glass" | "dashboard", + entry: string | null, + ) => ({ + id, + incidentId: INCIDENT, + status: "investigating" as const, + body: id, + source, + overlayEntryId: entry, + createdAt: at, + }) + const opened = toSnapshot( + baseInput({ + incidents: [incident()], + updates: [ + update("later", NOW - 1_000, "dashboard", null), + update("first", NOW - 60_000, "break_glass", "entry-1"), + ], + }), + ) + assert.strictEqual(opened.incidents[0]?.openedByOverlayEntryId, "entry-1") + assert.deepStrictEqual( + opened.incidents[0]?.updates.map((u) => [u.id, u.overlayEntryId]), + [ + ["later", null], + ["first", "entry-1"], + ], + ) + const followedUp = toSnapshot( + baseInput({ + incidents: [incident()], + updates: [ + update("first", NOW - 60_000, "dashboard", null), + update("later", NOW - 1_000, "break_glass", "entry-2"), + ], + }), + ) + assert.isNull(followedUp.incidents[0]?.openedByOverlayEntryId) }) }) @@ -230,10 +272,13 @@ describe("StatusPageSnapshotBuilder.build", () => { snapshot.incidents.map((i) => [i.title, i.updates.length]), [["Docs slow", 1]], ) - assert.strictEqual(snapshot.probesSeenAt, NOW - 30_000) + assert.deepStrictEqual( + snapshot.components.map((c) => c.monitored), + [true, false], + ) const missing = yield* Effect.exit(builder.build(newPageId(), NOW)) assert.isTrue(Exit.isFailure(missing)) - }).pipe(Effect.provide(StatusPageSnapshotBuilder.layer), Effect.provide(testDb.layer)) + }).pipe(Effect.provide(StatusPageSnapshotBuilder.layer.pipe(Layer.provide(testDb.layer)))) }) }) diff --git a/packages/backend/src/services/status-pages/StatusPageSnapshotBuilder.ts b/packages/backend/src/services/status-pages/StatusPageSnapshotBuilder.ts index 4d6ed4679a..5607ab8794 100644 --- a/packages/backend/src/services/status-pages/StatusPageSnapshotBuilder.ts +++ b/packages/backend/src/services/status-pages/StatusPageSnapshotBuilder.ts @@ -70,7 +70,6 @@ export interface SnapshotDailyInput { readonly checks: number readonly failures: number readonly downChecks: number - readonly lastCheckAt: number } export interface SnapshotIncidentInput { @@ -92,6 +91,8 @@ export interface SnapshotUpdateInput { readonly status: StatusPageIncidentStatus readonly body: string readonly source: StatusPageUpdateSource + /** The break-glass entry it was imported from, if any. */ + readonly overlayEntryId: string | null readonly createdAt: number } @@ -130,30 +131,36 @@ export const toSnapshot = (input: SnapshotInput): StatusPageSnapshot => { const incidents: ReadonlyArray = input.incidents .filter((incident) => isListedIncident(incident, input.nowMs)) .toSorted((a, b) => b.startedAt - a.startedAt) - .map((incident) => ({ - id: incident.id, - kind: incident.kind, - title: incident.title, - impact: incident.impact, - status: incident.status, - startedAt: incident.startedAt, - resolvedAt: incident.resolvedAt, - scheduledStart: incident.scheduledStart, - scheduledEnd: incident.scheduledEnd, - componentIds: input.incidentComponents - .filter((link) => link.incidentId === incident.id) - .map((link) => link.componentId), - updates: input.updates + .map((incident) => { + const updates = input.updates .filter((update) => update.incidentId === incident.id) - .toSorted((a, b) => b.createdAt - a.createdAt) - .map((update) => ({ + .toSorted((a, b) => b.createdAt - a.createdAt || b.id.localeCompare(a.id)) + const first = updates.at(-1) + return { + id: incident.id, + kind: incident.kind, + title: incident.title, + impact: incident.impact, + status: incident.status, + startedAt: incident.startedAt, + resolvedAt: incident.resolvedAt, + scheduledStart: incident.scheduledStart, + scheduledEnd: incident.scheduledEnd, + componentIds: input.incidentComponents + .filter((link) => link.incidentId === incident.id) + .map((link) => link.componentId), + // The first update came from break-glass: `bg-` follow-ups still target it. + openedByOverlayEntryId: first?.source === "break_glass" ? first.overlayEntryId : null, + updates: updates.map((update) => ({ id: update.id, status: update.status, body: update.body, at: update.createdAt, source: update.source, + overlayEntryId: update.overlayEntryId, })), - })) + } + }) const active = incidents.filter(isActiveIncident) const components = input.components.map((component) => ({ @@ -169,6 +176,7 @@ export const toSnapshot = (input: SnapshotInput): StatusPageSnapshot => { .map(incidentComponentStatus), }), showUptime: component.showUptime, + monitored: component.monitorId !== null, days: component.monitorId === null || !component.showUptime ? [] @@ -185,15 +193,11 @@ export const toSnapshot = (input: SnapshotInput): StatusPageSnapshot => { ), })) - const monitorIds = new Set(input.components.flatMap((c) => (c.monitorId === null ? [] : [c.monitorId]))) - const seen = daily.filter((row) => monitorIds.has(row.monitorId)).map((row) => row.lastCheckAt) - return { schemaVersion: STATUS_PAGE_SNAPSHOT_VERSION, pageId: input.page.id, revision: input.page.revision, generatedAt: input.nowMs, - probesSeenAt: seen.length === 0 ? null : Math.max(...seen), page: { name: input.page.name, slug: input.page.slug, @@ -287,7 +291,6 @@ export class StatusPageSnapshotBuilder extends Context.Service< checks: $.checks, failures: $.failures, downChecks: $.downChecks, - lastCheckAt: $.lastCheckAt, })) .where(($) => [ $.orgId.eq(orgId), @@ -328,6 +331,7 @@ export class StatusPageSnapshotBuilder extends Context.Service< status: $.status, body: $.body, source: $.source, + overlayEntryId: $.overlayEntryId, createdAt: $.createdAt, })) .where(($) => [ diff --git a/packages/backend/src/services/status-pages/StatusPageStore.ts b/packages/backend/src/services/status-pages/StatusPageStore.ts index 9ff31b02bd..ea1b311b0f 100644 --- a/packages/backend/src/services/status-pages/StatusPageStore.ts +++ b/packages/backend/src/services/status-pages/StatusPageStore.ts @@ -1,13 +1,14 @@ -import type { StatusPageId } from "@maple/domain/primitives" +import { StatusPageId } from "@maple/domain/primitives" import { - decodeStatusPageOverlayJson, + decodeStatusPageOverlayEntryJson, StatusPageKeys, - type StatusPageOverlay, + type StatusPageOverlayEntry, + type StatusPagePublisherHeartbeat, } from "@maple/domain/status-page" import { Context, Effect, Layer, Option, Ref, Schema } from "effect" // Edge storage for status pages: the snapshot (KV, mirrored to R2), routes, the publisher -// heartbeat and the break-glass overlay the status Worker writes while Maple is down. +// heartbeat and the break-glass entries the status Worker writes while Maple is down. export class StatusPageStoreError extends Schema.TaggedError()( "@maple/backend/status-pages/StatusPageStoreError", @@ -19,11 +20,19 @@ export class StatusPageStoreError extends Schema.TaggedError + readonly list_complete: boolean + readonly cursor?: string +} + /** The subset of a Workers KV namespace the store uses. */ export interface StatusPageKvNamespace { get(key: string, type: "text"): Promise put(key: string, value: string): Promise delete(key: string): Promise + list(options: { prefix: string; limit: number; cursor?: string }): Promise } /** The subset of an R2 bucket the store uses. */ @@ -41,102 +50,166 @@ export interface StatusPageRoutes { readonly hostnames: ReadonlyArray } +/** One page of KV keys under a prefix; `cursor` is null once the listing is complete. */ +export interface StatusPageKeyPage { + readonly keys: ReadonlyArray + readonly cursor: string | null +} + +/** Keys listed across pages; `complete` is false when the page bound stopped the listing. */ +export interface StatusPageKeyListing { + readonly keys: ReadonlyArray + readonly complete: boolean +} + export interface StatusPageStoreApi { /** Writes the snapshot to KV and its R2 copy. */ readonly putSnapshot: (pageId: StatusPageId, json: string) => Effect.Effect + /** Points the slug and hostnames at the page, in KV and in R2 for the edge's fallback. */ readonly putRoutes: ( pageId: StatusPageId, routes: StatusPageRoutes, ) => Effect.Effect + /** The page id the slug's KV route points at, or null. */ + readonly readSlugRoute: (slug: string) => Effect.Effect + /** Removes the routes from KV and R2. */ readonly deleteRoutes: (routes: StatusPageRoutes) => Effect.Effect - /** Removes the snapshot (KV and R2) and the overlay. */ + /** Removes the snapshot (KV and R2) and every break-glass entry of the page. */ readonly deletePage: (pageId: StatusPageId) => Effect.Effect - readonly putHeartbeat: (atMs: number) => Effect.Effect - /** The page's break-glass overlay; missing or undecodable reads as empty. */ - readonly readOverlay: (pageId: StatusPageId) => Effect.Effect - /** Drops only these entries, keeping any the edge added since the read. */ - readonly removeOverlayEntries: ( + readonly putHeartbeat: ( + heartbeat: StatusPagePublisherHeartbeat, + ) => Effect.Effect + /** KV keys under the prefix, following the list cursor for at most `maxPages` list calls. */ + readonly listKeys: ( + prefix: string, + maxPages: number, + ) => Effect.Effect + /** A break-glass entry by key; missing or undecodable reads as none. */ + readonly readOverlayEntry: ( + key: string, + ) => Effect.Effect, StatusPageStoreError> + readonly deleteOverlayEntry: ( pageId: StatusPageId, - entryIds: ReadonlyArray, + entryId: string, ) => Effect.Effect } -const EMPTY_OVERLAY: StatusPageOverlay = { entries: [] } - -const routeKeys = (routes: StatusPageRoutes): ReadonlyArray => [ - StatusPageKeys.slug(routes.slug), - ...routes.hostnames.map(StatusPageKeys.host), -] - -/** The overlay as stored, or empty when absent; an undecodable value is logged and ignored. */ -const parseOverlay = (pageId: StatusPageId, raw: string | null) => - raw === null - ? Effect.succeed(EMPTY_OVERLAY) - : Option.match(decodeStatusPageOverlayJson(raw), { - onNone: () => - Effect.logWarning("Undecodable status page overlay ignored").pipe( - Effect.annotateLogs({ "maple.status_page.id": pageId }), - Effect.as(EMPTY_OVERLAY), - ), - onSome: Effect.succeed, - }) +/** KV list page size; Workers KV caps it at 1000. */ +export const STATUS_PAGE_KV_LIST_LIMIT = 1000 -const withoutEntries = (overlay: StatusPageOverlay, entryIds: ReadonlyArray): StatusPageOverlay => ({ - entries: overlay.entries.filter((entry) => !entryIds.includes(entry.id)), -}) +const decodePageId = Schema.decodeUnknownOption(StatusPageId) + +/** The page and entry ids in an `overlay::` key. */ +export const parseOverlayKey = ( + key: string, +): Option.Option<{ readonly pageId: StatusPageId; readonly entryId: string }> => { + if (!key.startsWith(StatusPageKeys.overlayAll)) return Option.none() + const rest = key.slice(StatusPageKeys.overlayAll.length) + const split = rest.indexOf(":") + if (split <= 0 || split === rest.length - 1) return Option.none() + return Option.map(decodePageId(rest.slice(0, split)), (pageId) => ({ + pageId, + entryId: rest.slice(split + 1), + })) +} /** Low-level key/value operations both layers implement. */ interface StoreBackend { readonly kvGet: (key: string) => Effect.Effect readonly kvPut: (key: string, value: string) => Effect.Effect readonly kvDelete: (key: string) => Effect.Effect + readonly kvList: ( + prefix: string, + cursor: string | null, + ) => Effect.Effect readonly objectPut: (key: string, value: string) => Effect.Effect readonly objectDelete: (key: string) => Effect.Effect } -const makeApi = (backend: StoreBackend): StatusPageStoreApi => { - const readOverlay = (pageId: StatusPageId) => - backend.kvGet(StatusPageKeys.overlay(pageId)).pipe(Effect.flatMap((raw) => parseOverlay(pageId, raw))) - return { - putSnapshot: (pageId, json) => - Effect.all( - [ - backend.kvPut(StatusPageKeys.snapshot(pageId), json), - backend.objectPut(StatusPageKeys.snapshotObject(pageId), json), - ], - { concurrency: 2, discard: true }, - ), - putRoutes: (pageId, routes) => - Effect.forEach(routeKeys(routes), (key) => backend.kvPut(key, pageId), { - concurrency: 4, - discard: true, - }), - deleteRoutes: (routes) => - Effect.forEach(routeKeys(routes), backend.kvDelete, { concurrency: 4, discard: true }), - deletePage: (pageId) => - Effect.all( - [ - backend.kvDelete(StatusPageKeys.snapshot(pageId)), - backend.kvDelete(StatusPageKeys.overlay(pageId)), - backend.objectDelete(StatusPageKeys.snapshotObject(pageId)), - ], - { concurrency: 3, discard: true }, - ), - putHeartbeat: (atMs) => backend.kvPut(StatusPageKeys.heartbeat, JSON.stringify({ at: atMs })), - readOverlay, - removeOverlayEntries: (pageId, entryIds) => - entryIds.length === 0 - ? Effect.void - : readOverlay(pageId).pipe( - Effect.map((overlay) => withoutEntries(overlay, entryIds)), - Effect.flatMap((remaining) => - remaining.entries.length === 0 - ? backend.kvDelete(StatusPageKeys.overlay(pageId)) - : backend.kvPut(StatusPageKeys.overlay(pageId), JSON.stringify(remaining)), - ), +/** KV keys under the prefix, following the cursor until the listing ends or `pagesLeft` runs out. */ +const listFrom = ( + backend: StoreBackend, + prefix: string, + cursor: string | null, + seen: ReadonlyArray, + pagesLeft: number, +): Effect.Effect => + backend.kvList(prefix, cursor).pipe( + Effect.flatMap((page) => { + const keys = [...seen, ...page.keys] + if (page.cursor === null) return Effect.succeed({ keys, complete: true }) + if (pagesLeft <= 1) return Effect.succeed({ keys, complete: false }) + return listFrom(backend, prefix, page.cursor, keys, pagesLeft - 1) + }), + ) + +/** List calls one page's break-glass cleanup may make; the edge caps a page's entries well below. */ +const PAGE_OVERLAY_LIST_PAGES = 10 + +const makeApi = (backend: StoreBackend): StatusPageStoreApi => ({ + putSnapshot: (pageId, json) => + Effect.all( + [ + backend.kvPut(StatusPageKeys.snapshot(pageId), json), + backend.objectPut(StatusPageKeys.snapshotObject(pageId), json), + ], + { concurrency: 2, discard: true }, + ), + putRoutes: (pageId, routes) => + Effect.all( + [ + backend.kvPut(StatusPageKeys.slug(routes.slug), pageId), + backend.objectPut(StatusPageKeys.slugObject(routes.slug), pageId), + ...routes.hostnames.flatMap((hostname) => [ + backend.kvPut(StatusPageKeys.host(hostname), pageId), + backend.objectPut(StatusPageKeys.hostObject(hostname), pageId), + ]), + ], + { concurrency: 4, discard: true }, + ), + readSlugRoute: (slug) => backend.kvGet(StatusPageKeys.slug(slug)), + deleteRoutes: (routes) => + Effect.all( + [ + backend.kvDelete(StatusPageKeys.slug(routes.slug)), + backend.objectDelete(StatusPageKeys.slugObject(routes.slug)), + ...routes.hostnames.flatMap((hostname) => [ + backend.kvDelete(StatusPageKeys.host(hostname)), + backend.objectDelete(StatusPageKeys.hostObject(hostname)), + ]), + ], + { concurrency: 4, discard: true }, + ), + deletePage: (pageId) => + Effect.all( + [ + backend.kvDelete(StatusPageKeys.snapshot(pageId)), + backend.objectDelete(StatusPageKeys.snapshotObject(pageId)), + listFrom(backend, StatusPageKeys.overlayPage(pageId), null, [], PAGE_OVERLAY_LIST_PAGES).pipe( + Effect.flatMap((listing) => + Effect.forEach(listing.keys, backend.kvDelete, { concurrency: 4, discard: true }), ), - } -} + ), + ], + { concurrency: 3, discard: true }, + ), + putHeartbeat: (heartbeat) => backend.kvPut(StatusPageKeys.heartbeat, JSON.stringify(heartbeat)), + listKeys: (prefix, maxPages) => listFrom(backend, prefix, null, [], maxPages), + readOverlayEntry: (key) => + backend.kvGet(key).pipe( + Effect.flatMap((raw) => { + if (raw === null) return Effect.succeed(Option.none()) + const entry = decodeStatusPageOverlayEntryJson(raw) + return Option.isSome(entry) + ? Effect.succeed(entry) + : Effect.logWarning("Undecodable break-glass entry ignored").pipe( + Effect.annotateLogs({ "maple.status_page.overlay_key": key }), + Effect.as(Option.none()), + ) + }), + ), + deleteOverlayEntry: (pageId, entryId) => backend.kvDelete(StatusPageKeys.overlayEntry(pageId, entryId)), +}) const call = (operation: string, key: string, run: () => Promise) => Effect.tryPromise({ @@ -182,6 +255,19 @@ export class StatusPageStore extends Context.Service call("kv.get", key, () => kv.get(key, "text")), kvPut: (key, value) => call("kv.put", key, () => kv.put(key, value)).pipe(Effect.asVoid), kvDelete: (key) => call("kv.delete", key, () => kv.delete(key)).pipe(Effect.asVoid), + kvList: (prefix, cursor) => + call("kv.list", prefix, () => + kv.list({ + prefix, + limit: STATUS_PAGE_KV_LIST_LIMIT, + ...(cursor === null ? undefined : { cursor }), + }), + ).pipe( + Effect.map((page) => ({ + keys: page.keys.map((key) => key.name), + cursor: page.list_complete ? null : (page.cursor ?? null), + })), + ), objectPut: (key, value) => call("r2.put", key, () => bucket.put(key, value, { httpMetadata: { contentType: "application/json" } }), @@ -190,45 +276,67 @@ export class StatusPageStore extends Context.Service = Effect.gen(function* () { - const kv = yield* Ref.make>(new Map()) - const objects = yield* Ref.make>(new Map()) - const failing = yield* Ref.make<(key: string) => boolean>(() => false) - const guardWrite = (operation: string, key: string) => - Ref.get(failing).pipe( - Effect.flatMap((fails) => - fails(key) - ? Effect.fail( - new StatusPageStoreError({ - message: `Status page store ${operation} failed for ${key}`, - operation, - key, - cause: new Error("store unavailable"), - }), - ) - : Effect.void, - ), - ) - const set = (ref: Ref.Ref>, key: string, value: string) => - Ref.update(ref, (map) => new Map([...map, [key, value]])) - const remove = (ref: Ref.Ref>, key: string) => - Ref.update(ref, (map) => new Map([...map].filter(([k]) => k !== key))) - const api = makeApi({ - kvGet: (key) => Ref.get(kv).pipe(Effect.map((map) => map.get(key) ?? null)), - kvPut: (key, value) => guardWrite("kv.put", key).pipe(Effect.andThen(set(kv, key, value))), - kvDelete: (key) => guardWrite("kv.delete", key).pipe(Effect.andThen(remove(kv, key))), - objectPut: (key, value) => - guardWrite("r2.put", key).pipe(Effect.andThen(set(objects, key, value))), - objectDelete: (key) => guardWrite("r2.delete", key).pipe(Effect.andThen(remove(objects, key))), + /** A `Ref`-backed store plus handles on its contents; `listPageSize` shrinks KV list pages. */ + static readonly makeInMemoryWith = (options: { + readonly listPageSize?: number + }): Effect.Effect => + Effect.gen(function* () { + const kv = yield* Ref.make>(new Map()) + const objects = yield* Ref.make>(new Map()) + const failing = yield* Ref.make<(key: string) => boolean>(() => false) + const guardWrite = (operation: string, key: string) => + Ref.get(failing).pipe( + Effect.flatMap((fails) => + fails(key) + ? Effect.fail( + new StatusPageStoreError({ + message: `Status page store ${operation} failed for ${key}`, + operation, + key, + cause: new Error("store unavailable"), + }), + ) + : Effect.void, + ), + ) + const set = (ref: Ref.Ref>, key: string, value: string) => + Ref.update(ref, (map) => new Map([...map, [key, value]])) + const remove = (ref: Ref.Ref>, key: string) => + Ref.update(ref, (map) => new Map([...map].filter(([k]) => k !== key))) + const api = makeApi({ + kvGet: (key) => Ref.get(kv).pipe(Effect.map((map) => map.get(key) ?? null)), + kvPut: (key, value) => guardWrite("kv.put", key).pipe(Effect.andThen(set(kv, key, value))), + kvDelete: (key) => guardWrite("kv.delete", key).pipe(Effect.andThen(remove(kv, key))), + objectPut: (key, value) => + guardWrite("r2.put", key).pipe(Effect.andThen(set(objects, key, value))), + objectDelete: (key) => + guardWrite("r2.delete", key).pipe(Effect.andThen(remove(objects, key))), + kvList: (prefix, cursor) => + Ref.get(kv).pipe( + Effect.map((map) => { + const size = options.listPageSize ?? STATUS_PAGE_KV_LIST_LIMIT + const offset = cursor === null ? 0 : Number(cursor) + const matching = [...map.keys()] + .filter((key) => key.startsWith(prefix)) + .toSorted() + const end = offset + size + return { + keys: matching.slice(offset, end), + cursor: end < matching.length ? String(end) : null, + } + }), + ), + }) + return { + api, + contents: Effect.all({ kv: Ref.get(kv), objects: Ref.get(objects) }), + setFailing: (fail) => Ref.set(failing, typeof fail === "function" ? fail : () => fail), + putKv: (key, value) => set(kv, key, value), + } }) - return { - api, - contents: Effect.all({ kv: Ref.get(kv), objects: Ref.get(objects) }), - setFailing: (fail) => Ref.set(failing, typeof fail === "function" ? fail : () => fail), - putKv: (key, value) => set(kv, key, value), - } - }) + + /** A `Ref`-backed store plus handles on its contents. */ + static readonly makeInMemory: Effect.Effect = this.makeInMemoryWith({}) /** An empty in-memory store, for tests that only need the service. */ static readonly inMemory = Layer.effect(this, this.makeInMemory.pipe(Effect.map((store) => store.api))) diff --git a/packages/backend/src/services/status-pages/StatusPagesService.test.ts b/packages/backend/src/services/status-pages/StatusPagesService.test.ts index 02cd7d06e6..758988a23a 100644 --- a/packages/backend/src/services/status-pages/StatusPagesService.test.ts +++ b/packages/backend/src/services/status-pages/StatusPagesService.test.ts @@ -107,6 +107,32 @@ describe("StatusPagesService", () => { }).pipe(Effect.provide(makeLayer(testDb))) }) + it.effect("queues edge cleanup for a renamed slug and a deleted page", () => { + const testDb = createTestDb(trackedDbs) + const cleanups = Effect.promise(() => + testDb.pglite.query<{ page_id: string; slug: string; delete_snapshot: boolean }>( + `SELECT page_id, slug, delete_snapshot FROM status_page_cleanups ORDER BY created_at, delete_snapshot`, + ), + ).pipe(Effect.map((result) => result.rows)) + return Effect.gen(function* () { + const service = yield* StatusPagesService + const page = yield* createPage("before") + yield* service.update(ORG, page.id, { slug: "before" }) + assert.deepStrictEqual(yield* cleanups, []) + + yield* service.update(ORG, page.id, { slug: "after" }) + assert.deepStrictEqual(yield* cleanups, [ + { page_id: page.id, slug: "before", delete_snapshot: false }, + ]) + + yield* service.remove(ORG, page.id) + assert.deepStrictEqual(yield* cleanups, [ + { page_id: page.id, slug: "before", delete_snapshot: false }, + { page_id: page.id, slug: "after", delete_snapshot: true }, + ]) + }).pipe(Effect.provide(makeLayer(testDb))) + }) + it.effect("links components only to monitors of the same org and reorders them", () => { const testDb = createTestDb(trackedDbs) return Effect.gen(function* () { diff --git a/packages/backend/src/services/status-pages/StatusPagesService.ts b/packages/backend/src/services/status-pages/StatusPagesService.ts index 1cdabcec13..91ceca914a 100644 --- a/packages/backend/src/services/status-pages/StatusPagesService.ts +++ b/packages/backend/src/services/status-pages/StatusPagesService.ts @@ -45,7 +45,7 @@ import { Clock, Context, Effect, Layer, Schema } from "effect" import { Database, type DatabaseError } from "@maple/backend/platform/DatabaseLive" import { makeDbExecute } from "@maple/backend/platform/db-execute" import { postgresSqlState } from "@maple/backend/platform/postgres-errors" -import { bumpPage } from "./outbox" +import { bumpPage, queuePageRemoval, queueSlugRemoval } from "./outbox" export interface CreateStatusPageInput { readonly name: string @@ -627,8 +627,11 @@ export class StatusPagesService extends Context.Service db.transaction( Effect.gen(function* () { - yield* findPage(db, orgId, pageId) + const current = yield* findPage(db, orgId, pageId) if (slug !== undefined) yield* ensureSlugFree(db, slug, pageId) + // The old slug route stays at the edge until the publisher drains this. + if (slug !== undefined && slug !== current.slug) + yield* queueSlugRemoval(db, orgId, pageId, current.slug) yield* db.run( PG.update(StatusPages) .set({ @@ -662,10 +665,16 @@ export class StatusPagesService extends Context.Service - db.run( - PG.deleteFrom(StatusPages) - .where(($) => [$.orgId.eq(orgId), $.id.eq(pageId)]) - .returning("id"), + db.transaction( + Effect.gen(function* () { + // Queued before the delete reads the row; the publisher removes the edge copy. + yield* queuePageRemoval(db, orgId, pageId) + return yield* db.run( + PG.deleteFrom(StatusPages) + .where(($) => [$.orgId.eq(orgId), $.id.eq(pageId)]) + .returning("id"), + ) + }), ), ) if (removed.length === 0) return yield* pageNotFound(pageId) diff --git a/packages/backend/src/services/status-pages/auto-incidents.ts b/packages/backend/src/services/status-pages/auto-incidents.ts index 7066ff85e4..70a4bff840 100644 --- a/packages/backend/src/services/status-pages/auto-incidents.ts +++ b/packages/backend/src/services/status-pages/auto-incidents.ts @@ -2,6 +2,7 @@ import { randomUUID } from "node:crypto" import * as PG from "@maple-dev/effect-orm/postgres" import type { MapleDb } from "@maple/db/client" import { + AlertIncidents, StatusPageComponents, StatusPageIncidentComponents, StatusPageIncidents, @@ -147,6 +148,34 @@ export const openForAlertIncident = (db: MapleDb, input: OpenForAlertIncidentInp return toOpen.length }) +/** Posts the auto resolve update on each just-resolved incident and republishes its page. */ +const announceResolved = ( + db: MapleDb, + orgId: OrgId, + resolved: ReadonlyArray<{ readonly id: StatusPageIncidentId; readonly pageId: StatusPageId }>, + at: number, +) => + Effect.forEach( + resolved, + (incident) => + db + .run( + PG.insertInto(StatusPageIncidentUpdates).values({ + id: StatusPageIncidentUpdateId.make(randomUUID()), + orgId, + incidentId: incident.id, + status: "resolved", + body: "Automated checks are passing again. This incident has been resolved.", + source: "auto", + createdBy: null, + overlayEntryId: null, + createdAt: at, + }), + ) + .pipe(Effect.andThen(bumpPage(db, orgId, incident.pageId))), + { discard: true }, + ) + /** Resolves the open status page incidents this alert incident opened; run inside a transaction. */ export const resolveForAlertIncident = (db: MapleDb, input: ResolveForAlertIncidentInput) => Effect.gen(function* () { @@ -160,25 +189,49 @@ export const resolveForAlertIncident = (db: MapleDb, input: ResolveForAlertIncid ]) .returning(($) => ({ id: $.id, pageId: $.pageId })), ) - yield* Effect.forEach( - resolved, - (incident) => - db - .run( - PG.insertInto(StatusPageIncidentUpdates).values({ - id: StatusPageIncidentUpdateId.make(randomUUID()), - orgId: input.orgId, - incidentId: incident.id, - status: "resolved", - body: "Automated checks are passing again. This incident has been resolved.", - source: "auto", - createdBy: null, - overlayEntryId: null, - createdAt: input.at, - }), + yield* announceResolved(db, input.orgId, resolved, input.at) + return resolved.length + }) + +/** Auto incidents one repair pass resolves at most. */ +export const AUTO_INCIDENT_REPAIR_BATCH = 50 + +/** + * Open auto incidents whose alert incident is resolved or gone: the resolve hook runs after + * the alert commit and may have failed. Resolves each in its own transaction; returns how many. + */ +export const repairOrphanedAutoIncidents = (db: MapleDb, at: number) => + Effect.gen(function* () { + const orphans = yield* db.run( + PG.from(StatusPageIncidents, "i") + .leftJoin(AlertIncidents, "a", (i, a) => i.alertIncidentId.eq(a.id).and(i.orgId.eq(a.orgId))) + .select(($) => ({ id: $.id, orgId: $.orgId })) + .where(($) => [ + $.alertIncidentId.isNotNull(), + $.status.neq("resolved"), + PG.or($.a.id.isNull(), $.a.status.eq("resolved")), + ]) + .orderBy(["id", "asc"]) + .limit(AUTO_INCIDENT_REPAIR_BATCH), + ) + const counts = yield* Effect.forEach(orphans, (orphan) => + db.transaction( + Effect.gen(function* () { + // Guarded on status: a resolve landing meanwhile wins and posts its own update. + const resolved = yield* db.run( + PG.update(StatusPageIncidents) + .set({ status: "resolved", resolvedAt: at, updatedAt: at }) + .where(($) => [ + $.orgId.eq(orphan.orgId), + $.id.eq(orphan.id), + $.status.neq("resolved"), + ]) + .returning(($) => ({ id: $.id, pageId: $.pageId })), ) - .pipe(Effect.andThen(bumpPage(db, input.orgId, incident.pageId))), - { discard: true }, + yield* announceResolved(db, orphan.orgId, resolved, at) + return resolved.length + }), + ), ) - return resolved.length + return counts.reduce((sum, n) => sum + n, 0) }) diff --git a/packages/backend/src/services/status-pages/status-page-test-support.ts b/packages/backend/src/services/status-pages/status-page-test-support.ts index 5d23387369..53e1518722 100644 --- a/packages/backend/src/services/status-pages/status-page-test-support.ts +++ b/packages/backend/src/services/status-pages/status-page-test-support.ts @@ -25,13 +25,14 @@ export const seedPage = ( readonly revision?: number readonly publishedRevision?: number readonly publishedAt?: number | null + readonly publishAttemptedAt?: number | null }, ) => sql( testDb, `INSERT INTO status_pages (id, org_id, name, slug, auto_incidents, break_glass_token_sha256, - revision, published_revision, published_at, created_by, created_at, updated_at) - VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $11)`, + revision, published_revision, published_at, created_by, created_at, updated_at, publish_attempted_at) + VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $11, $12)`, [ input.id, ORG, @@ -44,17 +45,26 @@ export const seedPage = ( input.publishedAt === undefined || input.publishedAt === null ? null : iso(input.publishedAt), USER, iso(NOW - DAY_MS), + input.publishAttemptedAt === undefined || input.publishAttemptedAt === null + ? null + : iso(input.publishAttemptedAt), ], ) export const seedMonitor = ( testDb: TestDb, - input: { readonly id: string; readonly status: string; readonly alertRuleId?: string | null }, + input: { + readonly id: string + readonly status: string + readonly alertRuleId?: string | null + readonly kind?: "http" | "heartbeat" + readonly enabled?: boolean + }, ) => sql( testDb, - `INSERT INTO uptime_monitors (id, org_id, name, kind, config_json, alert_rule_id, created_by, created_at, updated_at) - VALUES ($1, $2, 'Monitor', 'http', $3, $4, $5, $6, $6)`, + `INSERT INTO uptime_monitors (id, org_id, name, kind, config_json, alert_rule_id, created_by, created_at, updated_at, enabled) + VALUES ($1, $2, 'Monitor', $7, $3, $4, $5, $6, $6, $8)`, [ input.id, ORG, @@ -69,6 +79,8 @@ export const seedMonitor = ( input.alertRuleId ?? null, USER, iso(NOW - DAY_MS), + input.kind ?? "http", + input.enabled ?? true, ], ).pipe( Effect.andThen( @@ -151,13 +163,14 @@ export const seedIncident = ( readonly resolvedAt?: number | null readonly scheduledEnd?: number | null readonly componentIds?: ReadonlyArray + readonly alertIncidentId?: string | null }, ) => sql( testDb, `INSERT INTO status_page_incidents (id, org_id, page_id, kind, title, impact, status, published, - scheduled_end, started_at, resolved_at, created_at, updated_at) - VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $10, $10)`, + scheduled_end, started_at, resolved_at, created_at, updated_at, alert_incident_id) + VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $10, $10, $12)`, [ input.id, ORG, @@ -170,6 +183,7 @@ export const seedIncident = ( input.scheduledEnd === undefined || input.scheduledEnd === null ? null : iso(input.scheduledEnd), iso(input.startedAt), input.resolvedAt === undefined || input.resolvedAt === null ? null : iso(input.resolvedAt), + input.alertIncidentId ?? null, ], ).pipe( Effect.andThen( @@ -209,9 +223,11 @@ export const readPage = (testDb: TestDb, pageId: string) => published_at: string | null published_hash: string | null last_publish_error: string | null + publish_attempted_at: string | null }>( testDb, - `SELECT revision::text, published_revision::text, published_at::text, published_hash, last_publish_error + `SELECT revision::text, published_revision::text, published_at::text, published_hash, last_publish_error, + publish_attempted_at::text FROM status_pages WHERE id = $1`, [pageId], ), @@ -219,3 +235,44 @@ export const readPage = (testDb: TestDb, pageId: string) => export const bumpRevision = (testDb: TestDb, pageId: string) => sql(testDb, `UPDATE status_pages SET revision = revision + 1 WHERE id = $1`, [pageId]) + +/** Queues an edge cleanup row, as `outbox.ts` would. */ +export const seedCleanup = ( + testDb: TestDb, + input: { + readonly pageId: string + readonly slug: string + readonly deleteSnapshot: boolean + readonly at: number + }, +) => + sql( + testDb, + `INSERT INTO status_page_cleanups (id, org_id, page_id, slug, delete_snapshot, created_at) + VALUES ($1, $2, $3, $4, $5, $6)`, + [crypto.randomUUID(), ORG, input.pageId, input.slug, input.deleteSnapshot, iso(input.at)], + ) + +export const cleanupRows = (testDb: TestDb) => + Effect.promise(() => + testDb.pglite.query<{ page_id: string; slug: string }>( + `SELECT page_id, slug FROM status_page_cleanups ORDER BY created_at`, + ), + ).pipe(Effect.map((result) => result.rows)) + +/** An alert incident row, for auto-incident repair. */ +export const seedAlertIncident = (testDb: TestDb, input: { readonly id: string; readonly status: string }) => + sql( + testDb, + `INSERT INTO alert_incidents (id, org_id, rule_id, incident_key, rule_name, signal_type, severity, status, + comparator, threshold, first_triggered_at, last_triggered_at, resolved_at, dedupe_key, created_at, updated_at) + VALUES ($1, $2, $3, $1, 'Uptime', 'uptime', 'critical', $4, 'gt', 0, $5, $5, $6, $1, $5, $5)`, + [ + input.id, + ORG, + crypto.randomUUID(), + input.status, + iso(NOW - DAY_MS), + input.status === "resolved" ? iso(NOW - 60_000) : null, + ], + ) From 684f60c35adad665d2b0166eee0c5b2ab967cefd Mon Sep 17 00:00:00 2001 From: Makisuo Date: Wed, 28 Oct 2026 16:39:27 +0100 Subject: [PATCH 05/15] fix(status-pages): snapshot components carry their incident-free base status --- .../services/status-pages/StatusPageSnapshotBuilder.test.ts | 2 ++ .../src/services/status-pages/StatusPageSnapshotBuilder.ts | 5 +++++ 2 files changed, 7 insertions(+) diff --git a/packages/backend/src/services/status-pages/StatusPageSnapshotBuilder.test.ts b/packages/backend/src/services/status-pages/StatusPageSnapshotBuilder.test.ts index ac5155b47f..edf9c16fa0 100644 --- a/packages/backend/src/services/status-pages/StatusPageSnapshotBuilder.test.ts +++ b/packages/backend/src/services/status-pages/StatusPageSnapshotBuilder.test.ts @@ -92,6 +92,8 @@ describe("toSnapshot", () => { }), ) assert.strictEqual(snapshot.components[0]?.status, "partial_outage") + // The base leaves incidents out, so the edge can drop the impact when one resolves there. + assert.strictEqual(snapshot.components[0]?.baseStatus, "operational") assert.deepStrictEqual(snapshot.incidents[0]?.componentIds, [COMPONENT]) }) diff --git a/packages/backend/src/services/status-pages/StatusPageSnapshotBuilder.ts b/packages/backend/src/services/status-pages/StatusPageSnapshotBuilder.ts index 5607ab8794..844c7239d6 100644 --- a/packages/backend/src/services/status-pages/StatusPageSnapshotBuilder.ts +++ b/packages/backend/src/services/status-pages/StatusPageSnapshotBuilder.ts @@ -175,6 +175,11 @@ export const toSnapshot = (input: SnapshotInput): StatusPageSnapshot => { .filter((incident) => incident.componentIds.includes(component.id)) .map(incidentComponentStatus), }), + baseStatus: deriveComponentStatus({ + monitorStatus: component.monitorStatus, + manualStatus: component.manualStatus, + incidentStatuses: [], + }), showUptime: component.showUptime, monitored: component.monitorId !== null, days: From b91a72829d635db3eefac6a74e76727f23f9bb8f Mon Sep 17 00:00:00 2001 From: Makisuo Date: Wed, 28 Oct 2026 16:47:11 +0100 Subject: [PATCH 06/15] fix(status-pages): client spans on KV and R2 calls, ids on the auto-incident warning, bounded incident lists, in-progress maintenance test --- .../src/services/alerts/AlertsService.ts | 65 ++++++++++++------- .../StatusPageSnapshotBuilder.test.ts | 14 ++++ .../services/status-pages/StatusPageStore.ts | 11 +++- .../status-pages/StatusPagesService.ts | 6 +- 4 files changed, 69 insertions(+), 27 deletions(-) diff --git a/packages/backend/src/services/alerts/AlertsService.ts b/packages/backend/src/services/alerts/AlertsService.ts index efa0d6b499..c615dadc35 100644 --- a/packages/backend/src/services/alerts/AlertsService.ts +++ b/packages/backend/src/services/alerts/AlertsService.ts @@ -661,6 +661,7 @@ export class AlertsService extends Context.Service( signalType: string, + context: { readonly orgId: OrgId; readonly alertIncidentId: string }, write: (db: MapleDb) => Effect.Effect, ) => signalType !== "uptime" @@ -672,6 +673,8 @@ export class AlertsService extends Context.Service Effect.logWarning("Status page auto incident sync failed").pipe( Effect.annotateLogs({ + orgId: context.orgId, + "maple.alert.incident_id": context.alertIncidentId, "error.message": summarizeCause(Cause.fail(error)), }), ), @@ -3151,13 +3154,16 @@ export class AlertsService extends Context.Service - openForAlertIncident(db, { - orgId: row.orgId, - ruleId: row.id, - alertIncidentId: incidentId, - at: timestamp, - }), + yield* syncStatusPages( + normalized.signalType, + { orgId: row.orgId, alertIncidentId: incidentId }, + (db) => + openForAlertIncident(db, { + orgId: row.orgId, + ruleId: row.id, + alertIncidentId: incidentId, + at: timestamp, + }), ) if (lifecycle.notificationSuppression === "flapping") { yield* Effect.logInfo("Skipping trigger notification for flapping incident").pipe( @@ -3324,12 +3330,15 @@ export class AlertsService extends Context.Service - resolveForAlertIncident(db, { - orgId: row.orgId, - alertIncidentId: openIncident.id, - at: timestamp, - }), + yield* syncStatusPages( + normalized.signalType, + { orgId: row.orgId, alertIncidentId: openIncident.id }, + (db) => + resolveForAlertIncident(db, { + orgId: row.orgId, + alertIncidentId: openIncident.id, + at: timestamp, + }), ) if (openIncident.holdReason !== null) { yield* Metric.update(AlertingMetrics.incidentsResolvedAfterHoldTotal, 1) @@ -3617,12 +3626,15 @@ export class AlertsService extends Context.Service - resolveForAlertIncident(db, { - orgId, - alertIncidentId: incident.id, - at: timestamp, - }), + yield* syncStatusPages( + normalized.signalType, + { orgId, alertIncidentId: incident.id }, + (db) => + resolveForAlertIncident(db, { + orgId, + alertIncidentId: incident.id, + at: timestamp, + }), ) if (incident.holdReason !== null) { yield* Metric.update(AlertingMetrics.incidentsResolvedAfterHoldTotal, 1) @@ -3797,12 +3809,15 @@ export class AlertsService extends Context.Service - resolveForAlertIncident(db, { - orgId, - alertIncidentId: incident.id, - at: timestamp, - }), + yield* syncStatusPages( + normalized.signalType, + { orgId, alertIncidentId: incident.id }, + (db) => + resolveForAlertIncident(db, { + orgId, + alertIncidentId: incident.id, + at: timestamp, + }), ) if (incident.holdReason !== null) { yield* Metric.update(AlertingMetrics.incidentsResolvedAfterHoldTotal, 1) diff --git a/packages/backend/src/services/status-pages/StatusPageSnapshotBuilder.test.ts b/packages/backend/src/services/status-pages/StatusPageSnapshotBuilder.test.ts index edf9c16fa0..8243c6cc46 100644 --- a/packages/backend/src/services/status-pages/StatusPageSnapshotBuilder.test.ts +++ b/packages/backend/src/services/status-pages/StatusPageSnapshotBuilder.test.ts @@ -128,6 +128,20 @@ describe("toSnapshot", () => { assert.strictEqual(snapshot.components[0]?.status, "operational") }) + it("marks the components of in-progress maintenance as under maintenance", () => { + const snapshot = toSnapshot( + baseInput({ + incidents: [ + incident({ kind: "maintenance", status: "in_progress", scheduledEnd: NOW + DAY_MS }), + ], + incidentComponents: [{ incidentId: INCIDENT, componentId: COMPONENT }], + }), + ) + assert.strictEqual(snapshot.incidents.length, 1) + assert.strictEqual(snapshot.components[0]?.status, "maintenance") + assert.strictEqual(snapshot.components[0]?.baseStatus, "operational") + }) + it("fills 90 days of bars", () => { const today = utcDay(NOW) const snapshot = toSnapshot( diff --git a/packages/backend/src/services/status-pages/StatusPageStore.ts b/packages/backend/src/services/status-pages/StatusPageStore.ts index ea1b311b0f..b6d4ee94b1 100644 --- a/packages/backend/src/services/status-pages/StatusPageStore.ts +++ b/packages/backend/src/services/status-pages/StatusPageStore.ts @@ -221,7 +221,16 @@ const call = (operation: string, key: string, run: () => Promise) => key, cause, }), - }) + }).pipe( + // A Client span per KV/R2 call, so edge storage shows on the service map. + Effect.withSpan(`StatusPageStore.${operation}`, { + kind: "client", + attributes: { + "peer.service": operation.startsWith("r2.") ? "cloudflare-r2" : "cloudflare-kv", + "maple.status_page.key": key, + }, + }), + ) /** The KV namespace and R2 bucket a Worker binds for publishing. */ export interface StatusPageBindings { diff --git a/packages/backend/src/services/status-pages/StatusPagesService.ts b/packages/backend/src/services/status-pages/StatusPagesService.ts index 91ceca914a..facbc91cc2 100644 --- a/packages/backend/src/services/status-pages/StatusPagesService.ts +++ b/packages/backend/src/services/status-pages/StatusPagesService.ts @@ -504,6 +504,9 @@ const replaceIncidentComponents = Effect.fnUntraced(function* ( const newBreakGlassToken = () => `${STATUS_PAGE_BREAK_GLASS_PREFIX}${randomBytes(32).toString("base64url")}` +/** Newest incidents a list returns: with history included, a long-lived page stays bounded. */ +const INCIDENT_LIST_LIMIT = 200 + export class StatusPagesService extends Context.Service()( "@maple/backend/services/status-pages/StatusPagesService", { @@ -937,7 +940,8 @@ export class StatusPagesService extends Context.Service Date: Wed, 28 Oct 2026 16:56:29 +0100 Subject: [PATCH 07/15] fix(status-pages): cleanup never deletes a page that still exists --- .../status-pages/StatusPagePublisher.test.ts | 17 +++++++++++++++++ .../status-pages/StatusPagePublisher.ts | 15 +++++++++++++-- 2 files changed, 30 insertions(+), 2 deletions(-) diff --git a/packages/backend/src/services/status-pages/StatusPagePublisher.test.ts b/packages/backend/src/services/status-pages/StatusPagePublisher.test.ts index 242abe2030..e570113744 100644 --- a/packages/backend/src/services/status-pages/StatusPagePublisher.test.ts +++ b/packages/backend/src/services/status-pages/StatusPagePublisher.test.ts @@ -300,6 +300,23 @@ describe("StatusPagePublisher.sweep edge cleanups", () => { ), ) + it.effect("keeps a live page's edge data when its delete never committed", () => + withPublisher(({ testDb, store, publisher }) => + Effect.gen(function* () { + const pageId = newPageId() + yield* seedPage(testDb, { id: pageId, slug: "still-here" }) + yield* publisher.sweep(NOW) + // Queued by an org purge that failed after the queue insert committed. + yield* seedCleanup(testDb, { pageId, slug: "still-here", deleteSnapshot: true, at: NOW }) + yield* publisher.sweep(NOW + 60_000) + const { kv } = yield* store.contents + assert.isTrue(kv.has(StatusPageKeys.snapshot(pageId))) + assert.strictEqual(kv.get(StatusPageKeys.slug("still-here")), pageId) + assert.deepStrictEqual(yield* cleanupRows(testDb), []) + }), + ), + ) + it.effect("leaves a slug route alone once another page owns that slug", () => withPublisher(({ testDb, store, publisher }) => Effect.gen(function* () { diff --git a/packages/backend/src/services/status-pages/StatusPagePublisher.ts b/packages/backend/src/services/status-pages/StatusPagePublisher.ts index 539c670b6c..3dc4cf024f 100644 --- a/packages/backend/src/services/status-pages/StatusPagePublisher.ts +++ b/packages/backend/src/services/status-pages/StatusPagePublisher.ts @@ -156,9 +156,20 @@ export class StatusPagePublisher extends Context.Service Effect.gen(function* () { + const alive = yield* dbExecute((db) => + db.run( + PG.from(StatusPages) + .select(($) => ({ id: $.id })) + .where(($) => [$.id.eq(row.pageId)]) + .limit(1), + ), + ) const owners = yield* dbExecute((db) => db.run( PG.from(StatusPages) @@ -170,7 +181,7 @@ export class StatusPagePublisher extends Context.Service Date: Wed, 28 Oct 2026 16:59:27 +0100 Subject: [PATCH 08/15] fix(status-pages): new components land after the highest position, fixed span name for store calls --- .../src/services/status-pages/StatusPageStore.ts | 3 ++- .../services/status-pages/StatusPagesService.test.ts | 12 ++++++++++++ .../src/services/status-pages/StatusPagesService.ts | 6 +++++- 3 files changed, 19 insertions(+), 2 deletions(-) diff --git a/packages/backend/src/services/status-pages/StatusPageStore.ts b/packages/backend/src/services/status-pages/StatusPageStore.ts index b6d4ee94b1..ab258a7a3a 100644 --- a/packages/backend/src/services/status-pages/StatusPageStore.ts +++ b/packages/backend/src/services/status-pages/StatusPageStore.ts @@ -223,9 +223,10 @@ const call = (operation: string, key: string, run: () => Promise) => }), }).pipe( // A Client span per KV/R2 call, so edge storage shows on the service map. - Effect.withSpan(`StatusPageStore.${operation}`, { + Effect.withSpan("StatusPageStore.call", { kind: "client", attributes: { + "maple.status_page.operation": operation, "peer.service": operation.startsWith("r2.") ? "cloudflare-r2" : "cloudflare-kv", "maple.status_page.key": key, }, diff --git a/packages/backend/src/services/status-pages/StatusPagesService.test.ts b/packages/backend/src/services/status-pages/StatusPagesService.test.ts index 758988a23a..1ad2454d6f 100644 --- a/packages/backend/src/services/status-pages/StatusPagesService.test.ts +++ b/packages/backend/src/services/status-pages/StatusPagesService.test.ts @@ -160,6 +160,18 @@ describe("StatusPagesService", () => { ) const partial = yield* Effect.flip(service.reorderComponents(ORG, page.id, [web.id])) assert.strictEqual(partial._tag, "@maple/http/errors/StatusPageValidationError") + + // After a delete leaves a gap, a new component still lands last, not on top of one. + yield* service.removeComponent(ORG, web.id) + const docs = yield* service.createComponent(ORG, page.id, { name: "Docs" }) + assert.deepStrictEqual( + (yield* service.listComponents(ORG, page.id)).map((row) => [row.name, row.position]), + [ + ["API", 1], + ["Docs", 2], + ], + ) + assert.strictEqual(docs.position, 2) }).pipe(Effect.provide(makeLayer(testDb))) }) diff --git a/packages/backend/src/services/status-pages/StatusPagesService.ts b/packages/backend/src/services/status-pages/StatusPagesService.ts index facbc91cc2..ab045d0310 100644 --- a/packages/backend/src/services/status-pages/StatusPagesService.ts +++ b/packages/backend/src/services/status-pages/StatusPagesService.ts @@ -784,7 +784,11 @@ export class StatusPagesService extends Context.Service Math.max(top, row.position + 1), + 0, + ), monitorId, manualStatus: input.manualStatus ?? null, showUptime: input.showUptime ?? true, From a725719ad45e36a6319d969f891cd0c95c484c22 Mon Sep 17 00:00:00 2001 From: Makisuo Date: Wed, 28 Oct 2026 17:04:56 +0100 Subject: [PATCH 09/15] feat(status-pages): postUpdate can publish a draft in the same write --- .../status-pages/StatusPagesService.test.ts | 43 +++++++++++++++++++ .../status-pages/StatusPagesService.ts | 3 ++ 2 files changed, 46 insertions(+) diff --git a/packages/backend/src/services/status-pages/StatusPagesService.test.ts b/packages/backend/src/services/status-pages/StatusPagesService.test.ts index 1ad2454d6f..56d23ebecb 100644 --- a/packages/backend/src/services/status-pages/StatusPagesService.test.ts +++ b/packages/backend/src/services/status-pages/StatusPagesService.test.ts @@ -175,6 +175,49 @@ describe("StatusPagesService", () => { }).pipe(Effect.provide(makeLayer(testDb))) }) + it.effect("publishes a draft in the same write as its update, or not at all", () => { + const testDb = createTestDb(trackedDbs) + return Effect.gen(function* () { + const service = yield* StatusPagesService + const page = yield* createPage("drafts") + const draft = yield* service.createIncident(ORG, page.id, { + kind: "incident", + title: "Elevated errors", + impact: "major", + status: "investigating", + body: "Looking into it", + componentIds: [], + published: false, + source: "mcp", + createdBy: USER, + }) + assert.isFalse(draft.published) + + // A rejected update (maintenance status on an incident) leaves the draft and its updates as they were. + yield* Effect.flip( + service.postUpdate(ORG, draft.id, { + status: "in_progress", + body: "x", + source: "mcp", + publish: true, + }), + ) + const unchanged = yield* service.getIncident(ORG, draft.id) + assert.isFalse(unchanged.published) + assert.lengthOf(unchanged.updates, 1) + + yield* TestClock.adjust("1 minute") + const published = yield* service.postUpdate(ORG, draft.id, { + status: "identified", + body: "Rolling back", + source: "mcp", + publish: true, + }) + assert.isTrue(published.published) + assert.lengthOf(published.updates, 2) + }).pipe(Effect.provide(makeLayer(testDb))) + }) + it.effect("opens an incident with its first update and resolves it", () => { const testDb = createTestDb(trackedDbs) return Effect.gen(function* () { diff --git a/packages/backend/src/services/status-pages/StatusPagesService.ts b/packages/backend/src/services/status-pages/StatusPagesService.ts index ab045d0310..ba80d3bdb5 100644 --- a/packages/backend/src/services/status-pages/StatusPagesService.ts +++ b/packages/backend/src/services/status-pages/StatusPagesService.ts @@ -107,6 +107,8 @@ export interface PostStatusPageIncidentUpdateInput { readonly componentIds?: ReadonlyArray readonly source: StatusPageUpdateSource readonly createdBy?: UserId | null + /** Publishes a draft in the same write, so the update and its visibility never split. */ + readonly publish?: boolean } export interface UpdateStatusPageIncidentInput { @@ -1057,6 +1059,7 @@ export class StatusPagesService extends Context.Service Date: Wed, 28 Oct 2026 17:06:22 +0100 Subject: [PATCH 10/15] chore(status-pages): regenerate anticipated error identifiers for the status page errors --- .../domain/src/generated/anticipated-error-identifiers.ts | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/packages/domain/src/generated/anticipated-error-identifiers.ts b/packages/domain/src/generated/anticipated-error-identifiers.ts index ba306b02d2..a571a548b2 100644 --- a/packages/domain/src/generated/anticipated-error-identifiers.ts +++ b/packages/domain/src/generated/anticipated-error-identifiers.ts @@ -106,6 +106,11 @@ export const ANTICIPATED_ERROR_IDENTIFIER_LIST: ReadonlyArray = [ "@maple/http/errors/ShareWrongOrgError", "@maple/http/errors/SourceNotFoundError", "@maple/http/errors/SourceValidationError", + "@maple/http/errors/StatusPageComponentNotFoundError", + "@maple/http/errors/StatusPageIncidentNotFoundError", + "@maple/http/errors/StatusPageNotFoundError", + "@maple/http/errors/StatusPageSlugTakenError", + "@maple/http/errors/StatusPageValidationError", "@maple/http/errors/UnauthorizedError", "@maple/http/errors/UnknownVcsProviderError", "@maple/http/errors/UptimeMonitorLimitReachedError", From 765825eaad374640204ac37d15ed290250e598c3 Mon Sep 17 00:00:00 2001 From: Makisuo Date: Wed, 28 Oct 2026 17:12:39 +0100 Subject: [PATCH 11/15] fix(status-pages): bound the auto-incident page write so it cannot hold the alert notification --- packages/backend/src/services/alerts/AlertsService.ts | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/packages/backend/src/services/alerts/AlertsService.ts b/packages/backend/src/services/alerts/AlertsService.ts index c615dadc35..02dc4697b7 100644 --- a/packages/backend/src/services/alerts/AlertsService.ts +++ b/packages/backend/src/services/alerts/AlertsService.ts @@ -601,6 +601,9 @@ export interface AlertsServiceApi > } +/** Longest an uptime incident's status-page write may delay the alert tick. */ +const STATUS_PAGE_SYNC_TIMEOUT = "3 seconds" + export class AlertsService extends Context.Service()( "@maple/backend/services/AlertsService", { @@ -658,7 +661,8 @@ export class AlertsService extends Context.Service( signalType: string, context: { readonly orgId: OrgId; readonly alertIncidentId: string }, @@ -669,6 +673,7 @@ export class AlertsService extends Context.Service db.transaction(write(db))) .pipe( + Effect.timeout(STATUS_PAGE_SYNC_TIMEOUT), Effect.asVoid, Effect.catch((error) => Effect.logWarning("Status page auto incident sync failed").pipe( From 73428db0a7323297fb71184e8486fbd64b5c7ff9 Mon Sep 17 00:00:00 2001 From: Makisuo Date: Wed, 28 Oct 2026 17:18:18 +0100 Subject: [PATCH 12/15] fix(status-pages): the repair step opens page incidents a recent alert missed --- .../status-pages/StatusPageReconciler.test.ts | 43 ++++++++++++++ .../status-pages/StatusPageReconciler.ts | 7 ++- .../services/status-pages/auto-incidents.ts | 58 +++++++++++++++++-- .../status-pages/status-page-test-support.ts | 14 ++++- 4 files changed, 113 insertions(+), 9 deletions(-) diff --git a/packages/backend/src/services/status-pages/StatusPageReconciler.test.ts b/packages/backend/src/services/status-pages/StatusPageReconciler.test.ts index d8de88b9d4..63570e9b19 100644 --- a/packages/backend/src/services/status-pages/StatusPageReconciler.test.ts +++ b/packages/backend/src/services/status-pages/StatusPageReconciler.test.ts @@ -8,12 +8,14 @@ import { Database } from "@maple/backend/platform/DatabaseLive" import { importEntry, StatusPageReconciler } from "./StatusPageReconciler" import { type InMemoryStatusPageStore, StatusPageStore } from "./StatusPageStore" import { + DAY_MS, NOW, newPageId, ORG, readPage, seedAlertIncident, seedComponent, + seedMonitor, seedIncident, seedPage, } from "./status-page-test-support" @@ -333,6 +335,47 @@ describe("StatusPageReconciler", () => { }) describe("StatusPageReconciler.repairAutoIncidents", () => { + it.effect("opens the page incident a recent alert missed, once, and leaves older alerts alone", () => + withReconciler(({ testDb, reconciler }) => + Effect.gen(function* () { + const pageId = newPageId() + const ruleId = crypto.randomUUID() + const monitorId = crypto.randomUUID() + yield* seedPage(testDb, { id: pageId, slug: "acme-missed", autoIncidents: "publish" }) + yield* seedMonitor(testDb, { id: monitorId, status: "down", alertRuleId: ruleId }) + yield* seedComponent(testDb, { + id: crypto.randomUUID(), + pageId, + name: "Checkout", + position: 0, + monitorId, + }) + const missed = crypto.randomUUID() + yield* seedAlertIncident(testDb, { + id: missed, + status: "open", + ruleId, + firstTriggeredAt: NOW - 60_000, + }) + // Opened long ago: its page incident may have been deleted on purpose. + yield* seedAlertIncident(testDb, { + id: crypto.randomUUID(), + status: "open", + ruleId, + firstTriggeredAt: NOW - DAY_MS, + }) + + assert.strictEqual(yield* reconciler.repairAutoIncidents(), 1) + const incidents = yield* incidentsOf(testDb, pageId) + assert.deepStrictEqual( + incidents.map((i) => [i.title, i.status]), + [["Checkout is down", "investigating"]], + ) + assert.strictEqual(yield* reconciler.repairAutoIncidents(), 0) + }), + ), + ) + it.effect("resolves auto incidents whose alert incident is resolved or gone, and only those", () => withReconciler(({ testDb, reconciler }) => Effect.gen(function* () { diff --git a/packages/backend/src/services/status-pages/StatusPageReconciler.ts b/packages/backend/src/services/status-pages/StatusPageReconciler.ts index 7f34642e3e..bda83c4a11 100644 --- a/packages/backend/src/services/status-pages/StatusPageReconciler.ts +++ b/packages/backend/src/services/status-pages/StatusPageReconciler.ts @@ -22,7 +22,7 @@ import { StatusPageKeys, statusFitsKind, type StatusPageOverlayEntry } from "@ma import { Clock, Context, Effect, Layer, Option, Schema } from "effect" import { Database } from "@maple/backend/platform/DatabaseLive" import { makeDbExecute, makePersistenceErrorMapper } from "@maple/backend/platform/db-execute" -import { repairOrphanedAutoIncidents } from "./auto-incidents" +import { repairMissingAutoIncidents, repairOrphanedAutoIncidents } from "./auto-incidents" import { bumpPage } from "./outbox" import { StatusPagePublishDbError } from "./publish-errors" import { parseOverlayKey, StatusPageStore, type StatusPageStoreError } from "./StatusPageStore" @@ -384,7 +384,10 @@ export class StatusPageReconciler extends Context.Service repairOrphanedAutoIncidents(db, nowMs)) + // Opens what the open hook missed, then resolves what the resolve hook missed. + const opened = yield* dbExecute((db) => repairMissingAutoIncidents(db, nowMs)) + const resolved = yield* dbExecute((db) => repairOrphanedAutoIncidents(db, nowMs)) + const repaired = opened + resolved yield* Effect.annotateCurrentSpan({ "maple.status_page.repaired_count": repaired }) return repaired }) diff --git a/packages/backend/src/services/status-pages/auto-incidents.ts b/packages/backend/src/services/status-pages/auto-incidents.ts index 70a4bff840..24772d0f1e 100644 --- a/packages/backend/src/services/status-pages/auto-incidents.ts +++ b/packages/backend/src/services/status-pages/auto-incidents.ts @@ -11,16 +11,18 @@ import { UptimeMonitors, } from "@maple/db/tables" import { - type AlertIncidentId, - type AlertRuleId, - type OrgId, + AlertIncidentId, + AlertRuleId, + OrgId, type StatusPageAutoIncidents, type StatusPageComponentId, type StatusPageId, StatusPageIncidentId, StatusPageIncidentUpdateId, } from "@maple/domain/primitives" -import { Effect } from "effect" +import * as Orm from "@maple-dev/effect-orm/database" +import { msToSqlTimestamp } from "@maple/backend/platform/time" +import { Effect, Schema } from "effect" import { bumpPage } from "./outbox" // Status page incidents opened and resolved by an uptime monitor's alert incident, on every @@ -193,6 +195,54 @@ export const resolveForAlertIncident = (db: MapleDb, input: ResolveForAlertIncid return resolved.length }) +const MissingAutoIncidentRow = Schema.Struct({ + id: AlertIncidentId, + org_id: OrgId, + rule_id: AlertRuleId, + first_triggered_ms: Schema.Number, +}) + +/** How long after an alert opens the repair still opens its missing page incidents. */ +export const AUTO_INCIDENT_OPEN_RETRY_MS = 15 * 60_000 + +/** + * Opens page incidents the open hook missed (it runs after the alert commit, bounded, and may + * fail). Only for alerts opened in the last 15 minutes, so a page incident someone deleted while + * its alert stays open is not brought back. Each alert in its own transaction; returns how many. + */ +export const repairMissingAutoIncidents = (db: MapleDb, at: number) => + Effect.gen(function* () { + // Four tables across two join keys (rule, then monitor): a typed query, not the builder. + const rows = yield* db.query( + Orm.sql` + SELECT DISTINCT a.id, a.org_id, a.rule_id, + (extract(epoch from a.first_triggered_at) * 1000)::float8 AS first_triggered_ms + FROM alert_incidents a + JOIN uptime_monitors m ON m.alert_rule_id = a.rule_id AND m.org_id = a.org_id + JOIN status_page_components c ON c.monitor_id = m.id AND c.org_id = a.org_id + JOIN status_pages p ON p.id = c.page_id AND p.org_id = a.org_id + LEFT JOIN status_page_incidents i ON i.alert_incident_id = a.id AND i.page_id = p.id + WHERE a.status <> 'resolved' + AND a.first_triggered_at >= ${msToSqlTimestamp(at - AUTO_INCIDENT_OPEN_RETRY_MS)}::timestamptz + AND p.auto_incidents <> 'off' + AND i.id IS NULL + ORDER BY first_triggered_ms + LIMIT ${AUTO_INCIDENT_REPAIR_BATCH}`, + MissingAutoIncidentRow, + ) + const counts = yield* Effect.forEach(rows, (row) => + db.transaction( + openForAlertIncident(db, { + orgId: row.org_id, + ruleId: row.rule_id, + alertIncidentId: row.id, + at: row.first_triggered_ms, + }), + ), + ) + return counts.reduce((sum, n) => sum + n, 0) + }) + /** Auto incidents one repair pass resolves at most. */ export const AUTO_INCIDENT_REPAIR_BATCH = 50 diff --git a/packages/backend/src/services/status-pages/status-page-test-support.ts b/packages/backend/src/services/status-pages/status-page-test-support.ts index 53e1518722..58f023fe29 100644 --- a/packages/backend/src/services/status-pages/status-page-test-support.ts +++ b/packages/backend/src/services/status-pages/status-page-test-support.ts @@ -261,7 +261,15 @@ export const cleanupRows = (testDb: TestDb) => ).pipe(Effect.map((result) => result.rows)) /** An alert incident row, for auto-incident repair. */ -export const seedAlertIncident = (testDb: TestDb, input: { readonly id: string; readonly status: string }) => +export const seedAlertIncident = ( + testDb: TestDb, + input: { + readonly id: string + readonly status: string + readonly ruleId?: string + readonly firstTriggeredAt?: number + }, +) => sql( testDb, `INSERT INTO alert_incidents (id, org_id, rule_id, incident_key, rule_name, signal_type, severity, status, @@ -270,9 +278,9 @@ export const seedAlertIncident = (testDb: TestDb, input: { readonly id: string; [ input.id, ORG, - crypto.randomUUID(), + input.ruleId ?? crypto.randomUUID(), input.status, - iso(NOW - DAY_MS), + iso(input.firstTriggeredAt ?? NOW - DAY_MS), input.status === "resolved" ? iso(NOW - 60_000) : null, ], ) From 44745685270b5c644f436462a595c8af76968c99 Mon Sep 17 00:00:00 2001 From: Makisuo Date: Wed, 28 Oct 2026 17:18:18 +0100 Subject: [PATCH 13/15] fix(status-pages): an imported break-glass update keeps its entry id in the snapshot --- .../services/status-pages/StatusPageSnapshotBuilder.test.ts | 3 ++- .../src/services/status-pages/StatusPageSnapshotBuilder.ts | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/packages/backend/src/services/status-pages/StatusPageSnapshotBuilder.test.ts b/packages/backend/src/services/status-pages/StatusPageSnapshotBuilder.test.ts index 8243c6cc46..ebe5190203 100644 --- a/packages/backend/src/services/status-pages/StatusPageSnapshotBuilder.test.ts +++ b/packages/backend/src/services/status-pages/StatusPageSnapshotBuilder.test.ts @@ -209,11 +209,12 @@ describe("toSnapshot", () => { }), ) assert.strictEqual(opened.incidents[0]?.openedByOverlayEntryId, "entry-1") + // An imported entry keeps its overlay id, so the feed does not show it twice after import. assert.deepStrictEqual( opened.incidents[0]?.updates.map((u) => [u.id, u.overlayEntryId]), [ ["later", null], - ["first", "entry-1"], + ["entry-1", "entry-1"], ], ) const followedUp = toSnapshot( diff --git a/packages/backend/src/services/status-pages/StatusPageSnapshotBuilder.ts b/packages/backend/src/services/status-pages/StatusPageSnapshotBuilder.ts index 844c7239d6..81b977a5d8 100644 --- a/packages/backend/src/services/status-pages/StatusPageSnapshotBuilder.ts +++ b/packages/backend/src/services/status-pages/StatusPageSnapshotBuilder.ts @@ -152,7 +152,8 @@ export const toSnapshot = (input: SnapshotInput): StatusPageSnapshot => { // The first update came from break-glass: `bg-` follow-ups still target it. openedByOverlayEntryId: first?.source === "break_glass" ? first.overlayEntryId : null, updates: updates.map((update) => ({ - id: update.id, + // An imported break-glass update keeps the id visitors and feed readers already saw. + id: update.overlayEntryId ?? update.id, status: update.status, body: update.body, at: update.createdAt, From 1d3e768bbdcaf6d220df67488bc184815b257836 Mon Sep 17 00:00:00 2001 From: Makisuo Date: Wed, 28 Oct 2026 17:22:54 +0100 Subject: [PATCH 14/15] perf(status-pages): anchor the auto-incident repair at opted-in pages so every join is indexed --- .../src/services/status-pages/auto-incidents.ts | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/packages/backend/src/services/status-pages/auto-incidents.ts b/packages/backend/src/services/status-pages/auto-incidents.ts index 24772d0f1e..ab6ec9aceb 100644 --- a/packages/backend/src/services/status-pages/auto-incidents.ts +++ b/packages/backend/src/services/status-pages/auto-incidents.ts @@ -212,19 +212,22 @@ export const AUTO_INCIDENT_OPEN_RETRY_MS = 15 * 60_000 */ export const repairMissingAutoIncidents = (db: MapleDb, at: number) => Effect.gen(function* () { - // Four tables across two join keys (rule, then monitor): a typed query, not the builder. + // Anchored at the few opted-in pages (MATERIALIZED keeps the planner there), then down indexed + // keys: components by page, monitors by id, alerts by (org, rule), page incidents by alert. const rows = yield* db.query( Orm.sql` + WITH pages AS MATERIALIZED ( + SELECT id, org_id FROM status_pages WHERE auto_incidents <> 'off' + ) SELECT DISTINCT a.id, a.org_id, a.rule_id, (extract(epoch from a.first_triggered_at) * 1000)::float8 AS first_triggered_ms - FROM alert_incidents a - JOIN uptime_monitors m ON m.alert_rule_id = a.rule_id AND m.org_id = a.org_id - JOIN status_page_components c ON c.monitor_id = m.id AND c.org_id = a.org_id - JOIN status_pages p ON p.id = c.page_id AND p.org_id = a.org_id + FROM pages p + JOIN status_page_components c ON c.page_id = p.id AND c.org_id = p.org_id + JOIN uptime_monitors m ON m.id = c.monitor_id AND m.org_id = p.org_id + JOIN alert_incidents a ON a.org_id = m.org_id AND a.rule_id = m.alert_rule_id LEFT JOIN status_page_incidents i ON i.alert_incident_id = a.id AND i.page_id = p.id WHERE a.status <> 'resolved' AND a.first_triggered_at >= ${msToSqlTimestamp(at - AUTO_INCIDENT_OPEN_RETRY_MS)}::timestamptz - AND p.auto_incidents <> 'off' AND i.id IS NULL ORDER BY first_triggered_ms LIMIT ${AUTO_INCIDENT_REPAIR_BATCH}`, From 4b26f136eb3f97e43bb771f5f32e7e4e80246d77 Mon Sep 17 00:00:00 2001 From: Makisuo Date: Sat, 31 Oct 2026 01:46:14 +0100 Subject: [PATCH 15/15] fix(status-pages): publish restores lost routes, recovers from lost and deleted races, auto incidents serialize, sync defects never abort the alert tick --- .../src/services/alerts/AlertsService.ts | 5 +- .../status-pages/StatusPagePublisher.test.ts | 84 ++++++++++++++++++- .../status-pages/StatusPagePublisher.ts | 28 ++++++- .../services/status-pages/auto-incidents.ts | 4 + 4 files changed, 113 insertions(+), 8 deletions(-) diff --git a/packages/backend/src/services/alerts/AlertsService.ts b/packages/backend/src/services/alerts/AlertsService.ts index 02dc4697b7..5f36c9dd6e 100644 --- a/packages/backend/src/services/alerts/AlertsService.ts +++ b/packages/backend/src/services/alerts/AlertsService.ts @@ -675,12 +675,13 @@ export class AlertsService extends Context.Service + // A defect too: a status page sync must never abort the alert tick. + Effect.catchCause((cause) => Effect.logWarning("Status page auto incident sync failed").pipe( Effect.annotateLogs({ orgId: context.orgId, "maple.alert.incident_id": context.alertIncidentId, - "error.message": summarizeCause(Cause.fail(error)), + "error.message": summarizeCause(cause), }), ), ), diff --git a/packages/backend/src/services/status-pages/StatusPagePublisher.test.ts b/packages/backend/src/services/status-pages/StatusPagePublisher.test.ts index e570113744..e40eba7bfb 100644 --- a/packages/backend/src/services/status-pages/StatusPagePublisher.test.ts +++ b/packages/backend/src/services/status-pages/StatusPagePublisher.test.ts @@ -5,7 +5,7 @@ import { decodeStatusPageSnapshotJson, StatusPageKeys, utcDay } from "@maple/dom import { StatusPageId } from "@maple/domain/primitives" import { cleanupTestDbs, createTestDb, executeSql, type TestDb } from "@maple/backend/platform/test-pglite" import { STATUS_PAGE_SWEEP_BATCH, StatusPagePublisher } from "./StatusPagePublisher" -import { StatusPageStore, type InMemoryStatusPageStore } from "./StatusPageStore" +import { StatusPageStore, type InMemoryStatusPageStore, type StatusPageStoreApi } from "./StatusPageStore" import { bumpRevision, cleanupRows, @@ -30,13 +30,15 @@ const withPublisher = ( readonly store: InMemoryStatusPageStore readonly publisher: StatusPagePublisher["Service"] }) => Effect.Effect, + /** Wraps the store the publisher sees, to land a concurrent change mid-publish. */ + wrap: (api: StatusPageStoreApi, testDb: TestDb) => StatusPageStoreApi = (api) => api, ) => Effect.gen(function* () { yield* TestClock.setTime(NOW) const testDb = createTestDb(trackedDbs) const store = yield* StatusPageStore.makeInMemory const layer = StatusPagePublisher.layer.pipe( - Layer.provide(Layer.succeed(StatusPageStore, store.api)), + Layer.provide(Layer.succeed(StatusPageStore, wrap(store.api, testDb))), Layer.provide(testDb.layer), ) return yield* StatusPagePublisher.use((publisher) => body({ testDb, store, publisher })).pipe( @@ -120,6 +122,84 @@ describe("StatusPagePublisher.publish", () => { ), ) + it.effect("rewrites a slug route lost at the edge even when the content is unchanged", () => + withPublisher(({ testDb, store, publisher }) => + Effect.gen(function* () { + const pageId = newPageId() + yield* seedPage(testDb, { id: pageId, slug: "acme-route" }) + yield* publisher.publish(pageId) + yield* store.api.deleteRoutes({ slug: "acme-route", hostnames: [] }) + + yield* bumpRevision(testDb, pageId) + yield* TestClock.adjust("1 minute") + assert.isFalse((yield* publisher.publish(pageId)).written) + assert.strictEqual((yield* store.contents).kv.get(StatusPageKeys.slug("acme-route")), pageId) + }), + ), + ) + + it.effect("clears the hash when a concurrent publish advanced first, so the next one rewrites", () => { + const pageId = newPageId() + const race = { armed: false } + return withPublisher( + ({ testDb, publisher }) => + Effect.gen(function* () { + yield* seedPage(testDb, { id: pageId, slug: "acme-race", revision: 3 }) + yield* publisher.publish(pageId) + yield* seedComponent(testDb, { + id: crypto.randomUUID(), + pageId, + name: "API", + position: 0, + }) + yield* bumpRevision(testDb, pageId) + race.armed = true + yield* publisher.publish(pageId) + const row = yield* readPage(testDb, pageId) + assert.isNull(row?.published_hash) + // Bumped, so the next sweep publishes it again. + assert.strictEqual(row?.revision, "5") + }), + (api, testDb) => ({ + ...api, + // The other publish lands its revision between our edge write and our compare-and-set. + putSnapshot: (id, json) => + (race.armed + ? Effect.promise(() => + executeSql( + testDb, + "UPDATE status_pages SET published_revision = 9 WHERE id = $1", + [id], + ), + ) + : Effect.void + ).pipe(Effect.andThen(api.putSnapshot(id, json))), + }), + ) + }) + + it.effect("takes back its edge writes when the page is deleted mid-publish", () => { + const pageId = newPageId() + return withPublisher( + ({ testDb, store, publisher }) => + Effect.gen(function* () { + yield* seedPage(testDb, { id: pageId, slug: "acme-gone" }) + const exit = yield* Effect.exit(publisher.publish(pageId)) + assert.isTrue(Exit.isFailure(exit)) + const { kv } = yield* store.contents + assert.isFalse(kv.has(StatusPageKeys.snapshot(pageId))) + assert.isFalse(kv.has(StatusPageKeys.slug("acme-gone"))) + }), + (api, testDb) => ({ + ...api, + putSnapshot: (id, json) => + Effect.promise(() => + executeSql(testDb, "DELETE FROM status_pages WHERE id = $1", [id]), + ).pipe(Effect.andThen(api.putSnapshot(id, json))), + }), + ) + }) + it.effect("records the store failure and leaves published_revision behind", () => withPublisher(({ testDb, store, publisher }) => Effect.gen(function* () { diff --git a/packages/backend/src/services/status-pages/StatusPagePublisher.ts b/packages/backend/src/services/status-pages/StatusPagePublisher.ts index 3dc4cf024f..d26768645c 100644 --- a/packages/backend/src/services/status-pages/StatusPagePublisher.ts +++ b/packages/backend/src/services/status-pages/StatusPagePublisher.ts @@ -110,9 +110,18 @@ export class StatusPagePublisher extends Context.Service recordFailure(pageId, nowMs, error)), + ) + } else { + // The hash covers content only, so a route lost at the edge is checked for on its own. + yield* store.readSlugRoute(routes.slug).pipe( + Effect.flatMap((routedTo) => + routedTo === pageId ? Effect.void : store.putRoutes(pageId, routes), + ), Effect.catch((error) => recordFailure(pageId, nowMs, error)), ) } @@ -136,17 +145,28 @@ export class StatusPagePublisher extends Context.Service + // Clearing the hash makes that next publish rewrite the body even if it builds the same. + const [still] = yield* dbExecute((db) => db.run( PG.update(StatusPages) .set({ publishedAt: nowMs, publishAttemptedAt: nowMs, lastPublishError: null, + ...(written ? { publishedHash: null } : undefined), }) - .where(($) => [$.id.eq(pageId)]), + .where(($) => [$.id.eq(pageId)]) + .returning("id"), ), ) + if (still === undefined) { + // Deleted mid-publish, maybe after its cleanup drained: take back what we wrote. + if (written) yield* cleanEdge({ pageId, slug: routes.slug, deleteSnapshot: true }) + return yield* new StatusPageGoneError({ + message: `Status page ${pageId} not found`, + pageId, + }) + } if (written) yield* dbExecute((db) => bumpPage(db, current.orgId, pageId)) } yield* Effect.annotateCurrentSpan({ @@ -160,7 +180,7 @@ export class StatusPagePublisher extends Context.Service + const cleanEdge = (row: Pick) => Effect.gen(function* () { const alive = yield* dbExecute((db) => db.run( diff --git a/packages/backend/src/services/status-pages/auto-incidents.ts b/packages/backend/src/services/status-pages/auto-incidents.ts index ab6ec9aceb..64c3755ee2 100644 --- a/packages/backend/src/services/status-pages/auto-incidents.ts +++ b/packages/backend/src/services/status-pages/auto-incidents.ts @@ -87,6 +87,10 @@ export const openForAlertIncident = (db: MapleDb, input: OpenForAlertIncidentInp ) if (rows.length === 0) return 0 const pages = groupByPage(rows) + // Two ticks opening the same alert incident would both miss `existing`; this serializes them. + yield* db.execute( + Orm.sql`select pg_advisory_xact_lock(hashtext(${input.alertIncidentId}), hashtext('status_page_auto_incident'))`, + ) const existing = yield* db.run( PG.from(StatusPageIncidents) .select(($) => ({ pageId: $.pageId }))