diff --git a/apps/web/src/test/drizzle-journal-monotonic.test.ts b/apps/web/src/test/drizzle-journal-monotonic.test.ts new file mode 100644 index 0000000..bad19ca --- /dev/null +++ b/apps/web/src/test/drizzle-journal-monotonic.test.ts @@ -0,0 +1,59 @@ +import { describe, it, expect } from "vitest"; +import { readFileSync } from "node:fs"; +import { join } from "node:path"; +import { + assertJournalMonotonic, + formatJournalViolations, + type JournalEntry, +} from "@cmbot/db/journal-check"; + +/** + * CI guard against the recurring drizzle journal-skip bug. + * + * Drizzle's migrator orders entries by `when` (not `idx`) and only + * applies entries whose `when` is greater than the latest applied + * row's recorded `created_at`. We've shipped two breaking deploys + * (0010/0011 and 0012/0013) where freshly-generated migrations had + * `when` values older than a prior manually-bumped entry — `pnpm + * migrate` printed "Migrations applied." while silently skipping + * the new SQL, and production 500'd until we hand-fixed the journal. + * + * This test reads the committed _journal.json and fails if the + * entries aren't strictly monotonically increasing by `when` in the + * same order as `idx`. Catches a bad commit at PR time instead of + * at the next deploy. + */ +describe("drizzle journal monotonicity (regression guard)", () => { + const journalPath = join( + __dirname, + "..", + "..", + "..", + "..", + "packages", + "db", + "migrations", + "meta", + "_journal.json", + ); + + const raw = JSON.parse(readFileSync(journalPath, "utf8")) as { + entries: JournalEntry[]; + }; + + it("loads at least one journal entry (sanity)", () => { + expect(raw.entries.length).toBeGreaterThan(0); + }); + + it("`when` timestamps are strictly increasing in `idx` order", () => { + const result = assertJournalMonotonic(raw.entries); + if (!result.ok) { + // Print the same actionable message migrate.ts prints, so a + // failed CI run reads exactly like a failed local migrate. + // eslint-disable-next-line no-console + console.error(formatJournalViolations(result)); + } + expect(result.violations).toEqual([]); + expect(result.ok).toBe(true); + }); +}); diff --git a/packages/db/package.json b/packages/db/package.json index db4ff15..b652635 100644 --- a/packages/db/package.json +++ b/packages/db/package.json @@ -13,6 +13,10 @@ "./schema": { "types": "./dist/schema.d.ts", "default": "./dist/schema.js" + }, + "./journal-check": { + "types": "./dist/journal-check.d.ts", + "default": "./dist/journal-check.js" } }, "scripts": { diff --git a/packages/db/src/journal-check.ts b/packages/db/src/journal-check.ts new file mode 100644 index 0000000..e0aa3de --- /dev/null +++ b/packages/db/src/journal-check.ts @@ -0,0 +1,90 @@ +/** + * Drizzle journal monotonicity guard. + * + * Background — twice already we hit this regression: a `pnpm migrate` + * silently skipped a freshly-generated migration because its `when` + * timestamp was older than the previous migration's `when`. Drizzle's + * migrator orders the entries by `when` (not by `idx`) and only + * applies entries whose `when` is strictly greater than the latest + * row's `created_at` in `pgboss... drizzle.__drizzle_migrations`. + * + * Symptom: migrate prints "Migrations applied." while the schema in + * the live DB is missing whatever 0012 / 0013 were supposed to add. + * Web 500's on every authenticated request because the code expects + * the new columns. + * + * This module is the first line of defence: + * - `assertJournalMonotonic(entries)` is a pure check the test + * suite runs against the committed journal file. CI fails on a + * bad commit before it can ship. + * - migrate.ts calls it on boot. If the live journal in source + * control has slipped out of monotonic order, migrate refuses + * to run and prints the offending entries with the smallest + * bump that would unbreak each one. + */ + +export interface JournalEntry { + idx: number; + tag: string; + when: number; +} + +export interface JournalCheckResult { + ok: boolean; + /** Entries whose `when` is <= the previous entry's `when`. */ + violations: Array<{ + idx: number; + tag: string; + when: number; + /** The previous entry's when — the new bound that this one must beat. */ + previousWhen: number; + previousTag: string; + /** A `when` value that would make THIS entry monotonic again. */ + suggestedWhen: number; + }>; +} + +/** + * Walk the journal entries in idx order and report any whose `when` + * is not strictly greater than the previous entry's `when`. The + * journal can have any starting timestamp; we only care about the + * relative ordering matching idx. Equal timestamps are also a + * violation — drizzle requires strictly greater. + */ +export function assertJournalMonotonic(entries: JournalEntry[]): JournalCheckResult { + const sorted = [...entries].sort((a, b) => a.idx - b.idx); + const violations: JournalCheckResult["violations"] = []; + for (let i = 1; i < sorted.length; i++) { + const prev = sorted[i - 1]!; + const cur = sorted[i]!; + if (cur.when <= prev.when) { + violations.push({ + idx: cur.idx, + tag: cur.tag, + when: cur.when, + previousWhen: prev.when, + previousTag: prev.tag, + suggestedWhen: prev.when + 1000, + }); + } + } + return { ok: violations.length === 0, violations }; +} + +/** Format the check result into a multi-line human message. */ +export function formatJournalViolations(result: JournalCheckResult): string { + if (result.ok) return ""; + const lines: string[] = [ + "Drizzle journal is not monotonic — migrate would silently skip these entries:", + ]; + for (const v of result.violations) { + lines.push( + ` ${v.tag} (idx ${v.idx}) when=${v.when} <= ${v.previousTag}.when=${v.previousWhen}`, + ); + lines.push( + ` fix: set ${v.tag}.when to >= ${v.suggestedWhen} in ` + + `packages/db/migrations/meta/_journal.json`, + ); + } + return lines.join("\n"); +} diff --git a/packages/db/src/migrate.ts b/packages/db/src/migrate.ts index dc14764..0b899ac 100644 --- a/packages/db/src/migrate.ts +++ b/packages/db/src/migrate.ts @@ -1,5 +1,13 @@ import { migrate } from "drizzle-orm/node-postgres/migrator"; +import { readFileSync } from "node:fs"; +import { join, dirname } from "node:path"; +import { fileURLToPath } from "node:url"; import { createClient } from "./index.js"; +import { + assertJournalMonotonic, + formatJournalViolations, + type JournalEntry, +} from "./journal-check.js"; const databaseUrl = process.env.DATABASE_URL; if (!databaseUrl) { @@ -7,6 +15,27 @@ if (!databaseUrl) { process.exit(1); } +// --- Pre-flight: refuse to run if the journal is non-monotonic. ----------- +// Drizzle silently skips entries whose `when` is older than the previous +// entry's `when`. We've hit this twice now (0010/0011, then 0012/0013), +// each time the symptom was "Migrations applied." with no schema change +// and a 500 in production for the missing column. Catch it before we +// hand the journal to drizzle. +const __dirname = dirname(fileURLToPath(import.meta.url)); +const journalPath = join(__dirname, "..", "migrations", "meta", "_journal.json"); +const journal = JSON.parse(readFileSync(journalPath, "utf8")) as { + entries: JournalEntry[]; +}; +const check = assertJournalMonotonic(journal.entries); +if (!check.ok) { + console.error(formatJournalViolations(check)); + console.error( + "\nRefusing to run drizzle migrate. Bump the offending `when` values in\n" + + "_journal.json so they're strictly increasing in the same order as `idx`.", + ); + process.exit(2); +} + const { db, pool } = createClient(databaseUrl); console.log("Applying migrations..."); await migrate(db, { migrationsFolder: "./migrations" });