feat(db): auto-guard against drizzle journal-skip regression

Twice now we've shipped a deploy that 500'd in production because
drizzle silently skipped freshly-generated migrations whose `when`
timestamps were older than a prior manually-bumped entry (0010/0011
in 1b7f553, then 0012/0013 in 2731888). Both times pnpm migrate
printed "Migrations applied." while the live DB schema lagged the
code's expectations.

Three layers of defence:

1. packages/db/src/journal-check.ts — pure helpers
   - assertJournalMonotonic(entries): walks idx-sorted entries and
     returns each one whose `when` <= the previous entry's `when`,
     plus a suggested `when` value to bump it to.
   - formatJournalViolations(result): renders an actionable
     multi-line message that points at the offending file path.

2. packages/db/src/migrate.ts — pre-flight
   Reads _journal.json BEFORE handing it to drizzle.migrate(). If
   the journal is non-monotonic, it prints the violations + bump
   instructions and exits with code 2. No more "Migrations applied."
   while silently skipping.

3. apps/web/src/test/drizzle-journal-monotonic.test.ts — CI guard
   Reads the committed _journal.json at test time. CI fails on the
   PR before the bad commit can ship. Imports the helper through a
   new "./journal-check" subpath export on @cmbot/db so the test
   doesn't rely on a deep path into the package.

Together: a bad commit fails CI; if it somehow got through, migrate
itself refuses to run; if migrate is bypassed, the previous deploy's
schema stays intact (drizzle wouldn't have skipped anything in any
case where the journal is monotonic).

Web suite 480 → 482 tests, all green.

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>

apps/web/src/test/drizzle-journal-monotonic.test.ts

@@ -0,0 +1,59 @@
+import { describe, it, expect } from "vitest";
+import { readFileSync } from "node:fs";
+import { join } from "node:path";
+import {
+  assertJournalMonotonic,
+  formatJournalViolations,
+  type JournalEntry,
+} from "@cmbot/db/journal-check";
+
+/**
+ * CI guard against the recurring drizzle journal-skip bug.
+ *
+ * Drizzle's migrator orders entries by `when` (not `idx`) and only
+ * applies entries whose `when` is greater than the latest applied
+ * row's recorded `created_at`. We've shipped two breaking deploys
+ * (0010/0011 and 0012/0013) where freshly-generated migrations had
+ * `when` values older than a prior manually-bumped entry — `pnpm
+ * migrate` printed "Migrations applied." while silently skipping
+ * the new SQL, and production 500'd until we hand-fixed the journal.
+ *
+ * This test reads the committed _journal.json and fails if the
+ * entries aren't strictly monotonically increasing by `when` in the
+ * same order as `idx`. Catches a bad commit at PR time instead of
+ * at the next deploy.
+ */
+describe("drizzle journal monotonicity (regression guard)", () => {
+  const journalPath = join(
+    __dirname,
+    "..",
+    "..",
+    "..",
+    "..",
+    "packages",
+    "db",
+    "migrations",
+    "meta",
+    "_journal.json",
+  );
+
+  const raw = JSON.parse(readFileSync(journalPath, "utf8")) as {
+    entries: JournalEntry[];
+  };
+
+  it("loads at least one journal entry (sanity)", () => {
+    expect(raw.entries.length).toBeGreaterThan(0);
+  });
+
+  it("`when` timestamps are strictly increasing in `idx` order", () => {
+    const result = assertJournalMonotonic(raw.entries);
+    if (!result.ok) {
+      // Print the same actionable message migrate.ts prints, so a
+      // failed CI run reads exactly like a failed local migrate.
+      // eslint-disable-next-line no-console
+      console.error(formatJournalViolations(result));
+    }
+    expect(result.violations).toEqual([]);
+    expect(result.ok).toBe(true);
+  });
+});

packages/db/package.json

@@ -13,6 +13,10 @@
     "./schema": {
       "types": "./dist/schema.d.ts",
       "default": "./dist/schema.js"
+    },
+    "./journal-check": {
+      "types": "./dist/journal-check.d.ts",
+      "default": "./dist/journal-check.js"
     }
   },
   "scripts": {

packages/db/src/journal-check.ts

@@ -0,0 +1,90 @@
+/**
+ * Drizzle journal monotonicity guard.
+ *
+ * Background — twice already we hit this regression: a `pnpm migrate`
+ * silently skipped a freshly-generated migration because its `when`
+ * timestamp was older than the previous migration's `when`. Drizzle's
+ * migrator orders the entries by `when` (not by `idx`) and only
+ * applies entries whose `when` is strictly greater than the latest
+ * row's `created_at` in `pgboss... drizzle.__drizzle_migrations`.
+ *
+ * Symptom: migrate prints "Migrations applied." while the schema in
+ * the live DB is missing whatever 0012 / 0013 were supposed to add.
+ * Web 500's on every authenticated request because the code expects
+ * the new columns.
+ *
+ * This module is the first line of defence:
+ *   - `assertJournalMonotonic(entries)` is a pure check the test
+ *     suite runs against the committed journal file. CI fails on a
+ *     bad commit before it can ship.
+ *   - migrate.ts calls it on boot. If the live journal in source
+ *     control has slipped out of monotonic order, migrate refuses
+ *     to run and prints the offending entries with the smallest
+ *     bump that would unbreak each one.
+ */
+
+export interface JournalEntry {
+  idx: number;
+  tag: string;
+  when: number;
+}
+
+export interface JournalCheckResult {
+  ok: boolean;
+  /** Entries whose `when` is <= the previous entry's `when`. */
+  violations: Array<{
+    idx: number;
+    tag: string;
+    when: number;
+    /** The previous entry's when — the new bound that this one must beat. */
+    previousWhen: number;
+    previousTag: string;
+    /** A `when` value that would make THIS entry monotonic again. */
+    suggestedWhen: number;
+  }>;
+}
+
+/**
+ * Walk the journal entries in idx order and report any whose `when`
+ * is not strictly greater than the previous entry's `when`. The
+ * journal can have any starting timestamp; we only care about the
+ * relative ordering matching idx. Equal timestamps are also a
+ * violation — drizzle requires strictly greater.
+ */
+export function assertJournalMonotonic(entries: JournalEntry[]): JournalCheckResult {
+  const sorted = [...entries].sort((a, b) => a.idx - b.idx);
+  const violations: JournalCheckResult["violations"] = [];
+  for (let i = 1; i < sorted.length; i++) {
+    const prev = sorted[i - 1]!;
+    const cur = sorted[i]!;
+    if (cur.when <= prev.when) {
+      violations.push({
+        idx: cur.idx,
+        tag: cur.tag,
+        when: cur.when,
+        previousWhen: prev.when,
+        previousTag: prev.tag,
+        suggestedWhen: prev.when + 1000,
+      });
+    }
+  }
+  return { ok: violations.length === 0, violations };
+}
+
+/** Format the check result into a multi-line human message. */
+export function formatJournalViolations(result: JournalCheckResult): string {
+  if (result.ok) return "";
+  const lines: string[] = [
+    "Drizzle journal is not monotonic — migrate would silently skip these entries:",
+  ];
+  for (const v of result.violations) {
+    lines.push(
+      `  ${v.tag} (idx ${v.idx}) when=${v.when} <= ${v.previousTag}.when=${v.previousWhen}`,
+    );
+    lines.push(
+      `    fix: set ${v.tag}.when to >= ${v.suggestedWhen} in ` +
+        `packages/db/migrations/meta/_journal.json`,
+    );
+  }
+  return lines.join("\n");
+}

packages/db/src/migrate.ts

@@ -1,5 +1,13 @@
 import { migrate } from "drizzle-orm/node-postgres/migrator";
+import { readFileSync } from "node:fs";
+import { join, dirname } from "node:path";
+import { fileURLToPath } from "node:url";
 import { createClient } from "./index.js";
+import {
+  assertJournalMonotonic,
+  formatJournalViolations,
+  type JournalEntry,
+} from "./journal-check.js";
 
 const databaseUrl = process.env.DATABASE_URL;
 if (!databaseUrl) {
@@ -7,6 +15,27 @@ if (!databaseUrl) {
   process.exit(1);
 }
 
+// --- Pre-flight: refuse to run if the journal is non-monotonic. -----------
+// Drizzle silently skips entries whose `when` is older than the previous
+// entry's `when`. We've hit this twice now (0010/0011, then 0012/0013),
+// each time the symptom was "Migrations applied." with no schema change
+// and a 500 in production for the missing column. Catch it before we
+// hand the journal to drizzle.
+const __dirname = dirname(fileURLToPath(import.meta.url));
+const journalPath = join(__dirname, "..", "migrations", "meta", "_journal.json");
+const journal = JSON.parse(readFileSync(journalPath, "utf8")) as {
+  entries: JournalEntry[];
+};
+const check = assertJournalMonotonic(journal.entries);
+if (!check.ok) {
+  console.error(formatJournalViolations(check));
+  console.error(
+    "\nRefusing to run drizzle migrate. Bump the offending `when` values in\n" +
+      "_journal.json so they're strictly increasing in the same order as `idx`.",
+  );
+  process.exit(2);
+}
+
 const { db, pool } = createClient(databaseUrl);
 console.log("Applying migrations...");
 await migrate(db, { migrationsFolder: "./migrations" });