feat(db,web): pg_trgm + indexes + Postgres-backed cache and rate-limit

- Add cacheEntries and rateLimitBuckets tables to schema
- Generate migration 0002_left_jimmy_woo.sql with pg_trgm extension and all indexes
- Implement cache.ts (get/set/delete/getOrSet/sweep) backed by Postgres
- Implement rate-limit.ts (sliding-window UPSERT) backed by Postgres
- Implement search.ts (trigramMatch / trigramRank helpers)
- Add vitest 2.1.9 + vitest.config.ts; 7 unit tests pass (4 cache + 3 rate-limit)

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
yiekheng 2026-05-09 23:03:10 +08:00
parent 04e3a8d6ed
commit 17f9ee179f
14 changed files with 1356 additions and 5 deletions

6
apps/web/next-env.d.ts vendored Normal file
View File

@ -0,0 +1,6 @@
/// <reference types="next" />
/// <reference types="next/image-types/global" />
import "./.next/dev/types/routes.d.ts";
// NOTE: This file should not be edited
// see https://nextjs.org/docs/app/api-reference/config/typescript for more information.

View File

@ -8,7 +8,8 @@
"build": "next build",
"start": "next start --hostname 0.0.0.0",
"lint": "next lint",
"typecheck": "tsc --noEmit"
"typecheck": "tsc --noEmit",
"test": "vitest run"
},
"dependencies": {
"@cmbot/db": "workspace:*",
@ -43,7 +44,9 @@
"@types/qrcode": "^1.5.5",
"@types/react": "^19.0.0",
"@types/react-dom": "^19.0.0",
"@vitest/ui": "^2.1.9",
"tailwindcss": "^4.0.0",
"typescript": "^5.5.0"
"typescript": "^5.5.0",
"vitest": "^2.1.9"
}
}

View File

@ -0,0 +1,46 @@
import { describe, it, expect, beforeEach, afterAll } from "vitest";
import { cacheGet, cacheSet, cacheGetOrSet, cacheDelete } from "./cache";
import { db, pool } from "./db";
import { cacheEntries } from "@cmbot/db";
import { sql } from "drizzle-orm";
describe("cache helpers", () => {
beforeEach(async () => {
await db.delete(cacheEntries).where(sql`true`);
});
afterAll(async () => {
await pool.end();
});
it("set + get round-trip", async () => {
await cacheSet("k1", { hello: "world" }, 60);
const v = await cacheGet<{ hello: string }>("k1");
expect(v).toEqual({ hello: "world" });
});
it("getOrSet computes once, then returns cached", async () => {
let calls = 0;
const compute = async () => {
calls++;
return { stamp: 42 };
};
const a = await cacheGetOrSet("k2", 60, compute);
const b = await cacheGetOrSet("k2", 60, compute);
expect(a).toEqual({ stamp: 42 });
expect(b).toEqual({ stamp: 42 });
expect(calls).toBe(1);
});
it("expired entries are skipped on get", async () => {
await cacheSet("k3", { stale: true }, -1); // already expired
const v = await cacheGet("k3");
expect(v).toBeNull();
});
it("delete removes the entry", async () => {
await cacheSet("k4", { x: 1 }, 60);
await cacheDelete("k4");
expect(await cacheGet("k4")).toBeNull();
});
});

43
apps/web/src/lib/cache.ts Normal file
View File

@ -0,0 +1,43 @@
import "server-only";
import { sql, eq } from "drizzle-orm";
import { cacheEntries } from "@cmbot/db";
import { db } from "./db";
export async function cacheGet<T>(key: string): Promise<T | null> {
const row = await db.query.cacheEntries.findFirst({
where: (c, { eq, and, gt }) => and(eq(c.key, key), gt(c.expiresAt, new Date())),
});
return (row?.value ?? null) as T | null;
}
export async function cacheSet(key: string, value: unknown, ttlSeconds: number): Promise<void> {
const expiresAt = new Date(Date.now() + ttlSeconds * 1000);
await db
.insert(cacheEntries)
.values({ key, value: value as never, expiresAt })
.onConflictDoUpdate({
target: cacheEntries.key,
set: { value: value as never, expiresAt },
});
}
export async function cacheDelete(key: string): Promise<void> {
await db.delete(cacheEntries).where(eq(cacheEntries.key, key));
}
export async function cacheGetOrSet<T>(
key: string,
ttlSeconds: number,
compute: () => Promise<T>,
): Promise<T> {
const hit = await cacheGet<T>(key);
if (hit !== null) return hit;
const fresh = await compute();
await cacheSet(key, fresh, ttlSeconds);
return fresh;
}
export async function cacheSweep(): Promise<{ removed: number }> {
const r = await db.execute(sql`DELETE FROM cache_entries WHERE expires_at < now() RETURNING key`);
return { removed: r.rowCount ?? 0 };
}

View File

@ -0,0 +1,40 @@
import { describe, it, expect, beforeEach, afterAll } from "vitest";
import { checkRateLimit } from "./rate-limit";
import { db, pool } from "./db";
import { rateLimitBuckets } from "@cmbot/db";
import { sql } from "drizzle-orm";
describe("checkRateLimit", () => {
beforeEach(async () => {
await db.delete(rateLimitBuckets).where(sql`true`);
});
afterAll(async () => {
await pool.end();
});
it("allows requests under the limit", async () => {
for (let i = 0; i < 5; i++) {
const r = await checkRateLimit("test:1", { max: 5, windowSec: 10 });
expect(r.limited).toBe(false);
}
});
it("blocks the (max+1)th request within the window", async () => {
for (let i = 0; i < 3; i++) {
await checkRateLimit("test:2", { max: 3, windowSec: 10 });
}
const r = await checkRateLimit("test:2", { max: 3, windowSec: 10 });
expect(r.limited).toBe(true);
expect(r.count).toBeGreaterThan(3);
});
it("isolates buckets by key", async () => {
await checkRateLimit("a", { max: 1, windowSec: 10 });
await checkRateLimit("a", { max: 1, windowSec: 10 });
const aLimited = await checkRateLimit("a", { max: 1, windowSec: 10 });
const bFresh = await checkRateLimit("b", { max: 1, windowSec: 10 });
expect(aLimited.limited).toBe(true);
expect(bFresh.limited).toBe(false);
});
});

View File

@ -0,0 +1,43 @@
import "server-only";
import { sql } from "drizzle-orm";
import { db } from "./db";
export type RateLimitOptions = { max: number; windowSec: number };
export type RateLimitResult = { limited: boolean; count: number };
/**
* Sliding-window rate limit using a single atomic Postgres UPSERT.
* Returns { limited: count > max, count }. The UPSERT resets the window
* when the existing row is older than windowSec, otherwise increments
* the count in place.
*/
export async function checkRateLimit(
key: string,
opts: RateLimitOptions,
): Promise<RateLimitResult> {
const { windowSec } = opts;
const result = await db.execute(sql`
INSERT INTO rate_limit_buckets (key, window_start, count, expires_at)
VALUES (${key}, now(), 1, now() + (${windowSec} * interval '1 second'))
ON CONFLICT (key) DO UPDATE
SET count = CASE
WHEN rate_limit_buckets.window_start < now() - (${windowSec} * interval '1 second')
THEN 1
ELSE rate_limit_buckets.count + 1
END,
window_start = CASE
WHEN rate_limit_buckets.window_start < now() - (${windowSec} * interval '1 second')
THEN now()
ELSE rate_limit_buckets.window_start
END,
expires_at = now() + (${windowSec} * interval '1 second')
RETURNING count;
`);
const count = Number((result.rows[0] as { count: number }).count);
return { limited: count > opts.max, count };
}
export async function rateLimitSweep(): Promise<{ removed: number }> {
const r = await db.execute(sql`DELETE FROM rate_limit_buckets WHERE expires_at < now()`);
return { removed: r.rowCount ?? 0 };
}

View File

@ -0,0 +1,22 @@
import "server-only";
import { sql, type SQL } from "drizzle-orm";
/**
* Build a Drizzle WHERE fragment that fuzzy-matches `column` against `query`
* using pg_trgm's similarity operator. Returns `true` (no filter) when query is
* empty so callers can compose unconditionally.
*
* Caller must ensure a `gin_trgm_ops` index exists on the column.
*/
export function trigramMatch(column: SQL, query: string | null | undefined): SQL {
const q = (query ?? "").trim();
if (!q) return sql`true`;
return sql`${column} % ${q}`;
}
/** Order-by fragment that ranks rows by similarity descending. */
export function trigramRank(column: SQL, query: string | null | undefined): SQL {
const q = (query ?? "").trim();
if (!q) return sql`1`;
return sql`similarity(${column}, ${q}) DESC`;
}

View File

@ -0,0 +1,2 @@
// Empty stub — vitest replaces "server-only" imports with this so they don't crash.
export {};

16
apps/web/vitest.config.ts Normal file
View File

@ -0,0 +1,16 @@
import { defineConfig } from "vitest/config";
import path from "node:path";
export default defineConfig({
test: {
environment: "node",
include: ["src/**/*.test.ts"],
},
resolve: {
alias: {
"@": path.resolve(__dirname, "src"),
// Stub out next's "server-only" import — vitest can't load it
"server-only": path.resolve(__dirname, "src/test/server-only-stub.ts"),
},
},
});

View File

@ -0,0 +1,40 @@
CREATE EXTENSION IF NOT EXISTS pg_trgm;
CREATE TABLE IF NOT EXISTS "cache_entries" (
"key" text PRIMARY KEY NOT NULL,
"value" jsonb NOT NULL,
"expires_at" timestamp with time zone NOT NULL
);
--> statement-breakpoint
CREATE TABLE IF NOT EXISTS "rate_limit_buckets" (
"key" text PRIMARY KEY NOT NULL,
"window_start" timestamp with time zone NOT NULL,
"count" integer NOT NULL,
"expires_at" timestamp with time zone NOT NULL
);
-- Trigram fuzzy search indexes (pg_trgm)
CREATE INDEX IF NOT EXISTS whatsapp_groups_name_trgm
ON whatsapp_groups USING gin (name gin_trgm_ops);
CREATE INDEX IF NOT EXISTS reminders_name_trgm
ON reminders USING gin (name gin_trgm_ops);
-- Common-filter B-tree indexes
CREATE INDEX IF NOT EXISTS reminders_status_idx
ON reminders (status);
CREATE INDEX IF NOT EXISTS reminders_account_scheduled_idx
ON reminders (account_id, scheduled_at DESC NULLS LAST);
CREATE INDEX IF NOT EXISTS reminder_runs_reminder_fired_idx
ON reminder_runs (reminder_id, fired_at DESC);
-- BRIN indexes for append-mostly time-series columns
CREATE INDEX IF NOT EXISTS reminder_runs_fired_at_brin
ON reminder_runs USING brin (fired_at);
CREATE INDEX IF NOT EXISTS audit_log_created_at_brin
ON audit_log USING brin (created_at);
-- Expiry indexes for the new utility tables
CREATE INDEX IF NOT EXISTS cache_entries_expires_idx
ON cache_entries (expires_at);
CREATE INDEX IF NOT EXISTS rate_limit_buckets_expires_idx
ON rate_limit_buckets (expires_at);

File diff suppressed because it is too large Load Diff

View File

@ -15,6 +15,13 @@
"when": 1778320434707,
"tag": "0001_smart_vertigo",
"breakpoints": true
},
{
"idx": 2,
"version": "7",
"when": 1778338808600,
"tag": "0002_left_jimmy_woo",
"breakpoints": true
}
]
}

View File

@ -153,6 +153,19 @@ export const authSessions = pgTable("auth_sessions", {
userAgent: text("user_agent"),
});
export const cacheEntries = pgTable("cache_entries", {
key: text("key").primaryKey(),
value: jsonb("value").notNull(),
expiresAt: timestamp("expires_at", { withTimezone: true }).notNull(),
});
export const rateLimitBuckets = pgTable("rate_limit_buckets", {
key: text("key").primaryKey(),
windowStart: timestamp("window_start", { withTimezone: true }).notNull(),
count: integer("count").notNull(),
expiresAt: timestamp("expires_at", { withTimezone: true }).notNull(),
});
export type Operator = typeof operators.$inferSelect;
export type NewOperator = typeof operators.$inferInsert;
export type WhatsappAccount = typeof whatsappAccounts.$inferSelect;

75
pnpm-lock.yaml generated
View File

@ -71,7 +71,7 @@ importers:
version: 5.9.3
vitest:
specifier: ^2.1.0
version: 2.1.9(@types/node@22.19.18)(lightningcss@1.32.0)(msw@2.14.5(@types/node@22.19.18)(typescript@5.9.3))
version: 2.1.9(@types/node@22.19.18)(@vitest/ui@2.1.9)(lightningcss@1.32.0)(msw@2.14.5(@types/node@22.19.18)(typescript@5.9.3))
apps/web:
dependencies:
@ -166,12 +166,18 @@ importers:
'@types/react-dom':
specifier: ^19.0.0
version: 19.2.3(@types/react@19.2.14)
'@vitest/ui':
specifier: ^2.1.9
version: 2.1.9(vitest@2.1.9)
tailwindcss:
specifier: ^4.0.0
version: 4.3.0
typescript:
specifier: ^5.5.0
version: 5.9.3
vitest:
specifier: ^2.1.9
version: 2.1.9(@types/node@22.19.18)(@vitest/ui@2.1.9)(lightningcss@1.32.0)(msw@2.14.5(@types/node@22.19.18)(typescript@5.9.3))
packages/db:
dependencies:
@ -218,7 +224,7 @@ importers:
version: 5.9.3
vitest:
specifier: ^2.1.0
version: 2.1.9(@types/node@22.19.18)(lightningcss@1.32.0)(msw@2.14.5(@types/node@22.19.18)(typescript@5.9.3))
version: 2.1.9(@types/node@22.19.18)(@vitest/ui@2.1.9)(lightningcss@1.32.0)(msw@2.14.5(@types/node@22.19.18)(typescript@5.9.3))
packages:
@ -1292,6 +1298,9 @@ packages:
'@pinojs/redact@0.4.0':
resolution: {integrity: sha512-k2ENnmBugE/rzQfEcdWHcCY+/FM3VLzH9cYEsbdsoqrvzAKRhUZeRNhAZvB8OitQJ1TBed3yqWtdjzS6wJKBwg==}
'@polka/url@1.0.0-next.29':
resolution: {integrity: sha512-wwQAWhWSuHaag8c4q/KN/vCoeOJYshAIvMQwD4GpSb3OiZklFfvAgmj0VCBBImRpuF/aFgIRzllXlVX93Jevww==}
'@protobufjs/aspromise@1.1.2':
resolution: {integrity: sha512-j+gKExEuLmKwvz3OgROXtrJ2UG2x8Ch2YZUxahh+s1F2HZ+wAceUNLkvy6zKCPVRkU++ZWQrdxsUeQXmcg4uoQ==}
@ -2353,6 +2362,11 @@ packages:
'@vitest/spy@2.1.9':
resolution: {integrity: sha512-E1B35FwzXXTs9FHNK6bDszs7mtydNi5MIfUWpceJ8Xbfb1gBMscAnwLbEu+B44ed6W3XjL9/ehLPHR1fkf1KLQ==}
'@vitest/ui@2.1.9':
resolution: {integrity: sha512-izzd2zmnk8Nl5ECYkW27328RbQ1nKvkm6Bb5DAaz1Gk59EbLkiCMa6OLT0NoaAYTjOFS6N+SMYW1nh4/9ljPiw==}
peerDependencies:
vitest: 2.1.9
'@vitest/utils@2.1.9':
resolution: {integrity: sha512-v0psaMSkNJ3A2NMrUEHFRzJtDPFn+/VWZ5WxImB21T9fjucJRmS7xCS3ppEnARb9y11OAzaD+P2Ps+b+BGX5iQ==}
@ -2989,6 +3003,9 @@ packages:
resolution: {integrity: sha512-7yAQpD2UMJzLi1Dqv7qFYnPbaPx7ZfFK6PiIxQ4PfkGPyNyl2Ugx+a/umUonmKqjhM4DnfbMvdX6otXq83soQQ==}
engines: {node: ^12.20 || >= 14.13}
fflate@0.8.2:
resolution: {integrity: sha512-cPJU47OaAoCbg0pBvzsgpTPhmhqI5eJjh/JIu8tPj5q+T7iLvW/JAYUqmE7KOB4R1ZyEhzBaIQpQpardBF5z8A==}
figures@6.1.0:
resolution: {integrity: sha512-d+l3qxjSesT4V7v2fh+QnmFnUWv9lSpjarhShNTgBOfA0ttejbQUAlHLitbjkoRiDulW0OPoQPYIGhIC8ohejg==}
engines: {node: '>=18'}
@ -3009,6 +3026,9 @@ packages:
resolution: {integrity: sha512-PpOwAdQ/YlXQ2vj8a3h8IipDuYRi3wceVQQGYWxNINccq40Anw7BlsEXCMbt1Zt+OLA6Fq9suIpIWD0OsnISlw==}
engines: {node: '>=8'}
flatted@3.4.2:
resolution: {integrity: sha512-PjDse7RzhcPkIJwy5t7KPWQSZ9cAbzQXcafsetQoD7sOJRQlGikNbx7yZp2OotDnJyrDcbyRq3Ttb18iYOqkxA==}
formdata-polyfill@4.0.10:
resolution: {integrity: sha512-buewHzMvYL29jdeQTVILecSaZKnt/RJWjoZCF5OW60Z67/GmSLBkOFM7qh1PI3zFNtJbaZL5eQu1vLfazOwj4g==}
engines: {node: '>=12.20.0'}
@ -3456,6 +3476,10 @@ packages:
mpg123-decoder@1.0.3:
resolution: {integrity: sha512-+fjxnWigodWJm3+4pndi+KUg9TBojgn31DPk85zEsim7C6s0X5Ztc/hQYdytXkwuGXH+aB0/aEkG40Emukv6oQ==}
mrmime@2.0.1:
resolution: {integrity: sha512-Y3wQdFg2Va6etvQ5I82yUhGdsKrcYox6p7FfL1LbK2J4V01F9TGlepTIhnK24t7koZibmg82KGglhA1XK5IsLQ==}
engines: {node: '>=10'}
ms@2.1.3:
resolution: {integrity: sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==}
@ -4029,6 +4053,10 @@ packages:
simple-yenc@1.0.4:
resolution: {integrity: sha512-5gvxpSd79e9a3V4QDYUqnqxeD4HGlhCakVpb6gMnDD7lexJggSBJRBO5h52y/iJrdXRilX9UCuDaIJhSWm5OWw==}
sirv@3.0.2:
resolution: {integrity: sha512-2wcC/oGxHis/BoHkkPwldgiPSYcpZK3JU28WoMVv55yHJgcZ8rlXvuG9iZggz+sU1d4bRgIGASwyWqjxu3FM0g==}
engines: {node: '>=18'}
sisteransi@1.0.5:
resolution: {integrity: sha512-bLGGlR1QxBcynn2d5YmDX4MGjlZvy2MRBDRNHLJ8VI6l6+9FUiyTFNJ0IveOSP0bcXgVDPRcfGqA0pjaqUpfVg==}
@ -4155,6 +4183,10 @@ packages:
tinyexec@0.3.2:
resolution: {integrity: sha512-KQQR9yN7R5+OSwaK0XQoj22pwHoTlgYqmUscPYoknOoWCWfj/5/ABTMRi69FrKU5ffPVh5QcFikpWJI/P1ocHA==}
tinyglobby@0.2.16:
resolution: {integrity: sha512-pn99VhoACYR8nFHhxqix+uvsbXineAasWm5ojXoN8xEwK5Kd3/TrhNn1wByuD52UxWRLy8pu+kRMniEi6Eq9Zg==}
engines: {node: '>=12.0.0'}
tinypool@1.1.1:
resolution: {integrity: sha512-Zba82s87IFq9A9XmjiX5uZA/ARWDrB03OHlq+Vw1fSdt0I+4/Kutwy8BP4Y/y/aORMo61FQ0vIb5j44vSo5Pkg==}
engines: {node: ^18.0.0 || >=20.0.0}
@ -4186,6 +4218,10 @@ packages:
resolution: {integrity: sha512-dRXchy+C0IgK8WPC6xvCHFRIWYUbqqdEIKPaKo/AcTUNzwLTK6AH7RjdLWsEZcAN/TBdtfUw3PYEgPr5VPr6ww==}
engines: {node: '>=14.16'}
totalist@3.0.1:
resolution: {integrity: sha512-sf4i37nQ2LBx4m3wB74y+ubopq6W/dIzXg0FDGjsYnZHVa1Da8FH853wlL2gtUhg+xJXjfk3kUZS3BRoQeoQBQ==}
engines: {node: '>=6'}
tough-cookie@6.0.1:
resolution: {integrity: sha512-LktZQb3IeoUWB9lqR5EWTHgW/VTITCXg4D21M+lvybRVdylLrRMnqaIONLVb5mav8vM19m44HIcGq4qASeu2Qw==}
engines: {node: '>=16'}
@ -5252,6 +5288,8 @@ snapshots:
'@pinojs/redact@0.4.0': {}
'@polka/url@1.0.0-next.29': {}
'@protobufjs/aspromise@1.1.2': {}
'@protobufjs/base64@1.1.2': {}
@ -6292,6 +6330,17 @@ snapshots:
dependencies:
tinyspy: 3.0.2
'@vitest/ui@2.1.9(vitest@2.1.9)':
dependencies:
'@vitest/utils': 2.1.9
fflate: 0.8.2
flatted: 3.4.2
pathe: 1.1.2
sirv: 3.0.2
tinyglobby: 0.2.16
tinyrainbow: 1.2.0
vitest: 2.1.9(@types/node@22.19.18)(@vitest/ui@2.1.9)(lightningcss@1.32.0)(msw@2.14.5(@types/node@22.19.18)(typescript@5.9.3))
'@vitest/utils@2.1.9':
dependencies:
'@vitest/pretty-format': 2.1.9
@ -6931,6 +6980,8 @@ snapshots:
node-domexception: 1.0.0
web-streams-polyfill: 3.3.3
fflate@0.8.2: {}
figures@6.1.0:
dependencies:
is-unicode-supported: 2.1.0
@ -6964,6 +7015,8 @@ snapshots:
locate-path: 5.0.0
path-exists: 4.0.0
flatted@3.4.2: {}
formdata-polyfill@4.0.10:
dependencies:
fetch-blob: 3.2.0
@ -7301,6 +7354,8 @@ snapshots:
'@wasm-audio-decoders/common': 9.0.7
optional: true
mrmime@2.0.1: {}
ms@2.1.3: {}
msw@2.14.5(@types/node@22.19.18)(typescript@5.9.3):
@ -8073,6 +8128,12 @@ snapshots:
simple-yenc@1.0.4:
optional: true
sirv@3.0.2:
dependencies:
'@polka/url': 1.0.0-next.29
mrmime: 2.0.1
totalist: 3.0.1
sisteransi@1.0.5: {}
sonic-boom@4.2.1:
@ -8172,6 +8233,11 @@ snapshots:
tinyexec@0.3.2: {}
tinyglobby@0.2.16:
dependencies:
fdir: 6.5.0(picomatch@4.0.4)
picomatch: 4.0.4
tinypool@1.1.1: {}
tinyrainbow@1.2.0: {}
@ -8196,6 +8262,8 @@ snapshots:
'@tokenizer/token': 0.3.0
ieee754: 1.2.1
totalist@3.0.1: {}
tough-cookie@6.0.1:
dependencies:
tldts: 7.0.30
@ -8314,7 +8382,7 @@ snapshots:
fsevents: 2.3.3
lightningcss: 1.32.0
vitest@2.1.9(@types/node@22.19.18)(lightningcss@1.32.0)(msw@2.14.5(@types/node@22.19.18)(typescript@5.9.3)):
vitest@2.1.9(@types/node@22.19.18)(@vitest/ui@2.1.9)(lightningcss@1.32.0)(msw@2.14.5(@types/node@22.19.18)(typescript@5.9.3)):
dependencies:
'@vitest/expect': 2.1.9
'@vitest/mocker': 2.1.9(msw@2.14.5(@types/node@22.19.18)(typescript@5.9.3))(vite@5.4.21(@types/node@22.19.18)(lightningcss@1.32.0))
@ -8338,6 +8406,7 @@ snapshots:
why-is-node-running: 2.3.0
optionalDependencies:
'@types/node': 22.19.18
'@vitest/ui': 2.1.9(vitest@2.1.9)
transitivePeerDependencies:
- less
- lightningcss