From 6bb85222d15e50e1bda2ebb475b1b785f831b793 Mon Sep 17 00:00:00 2001 From: yiekheng Date: Sun, 3 May 2026 11:29:34 +0800 Subject: [PATCH] perf(web): server-side pagination + infinite-scroll for accounts/users MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit For 3k+ row deployments, returning the full table in one shot is the bottleneck — the JSON payload alone is hundreds of KB and the client mounts thousands of EditableCell instances on every visit. Pagination with auto-fetch on scroll shrinks both the wire payload and the initial render to a single page (200 rows). Server (app/cm_api.py): - /acc/ and /user/ accept ?limit, ?offset, ?prefix, ?dir, plus ?sort on /user/ (f_username | last_update_time). Defaults: limit=200 (capped at 1000), offset=0, dir=desc. - ORDER BY done in SQL with prefix-priority: rows whose username starts with the configured CM_PREFIX_PATTERN come first, then asc/desc by the sort column. The 'dir' value is whitelisted to ASC|DESC before string interpolation; everything else goes through parameterised binding. - Schema verification (verify_tables_once) deferred to first request via a Flask before_request hook — keeps create_app() free of MySQL touches so unit tests + gunicorn preload still work without a live DB. Web client: - web/lib/api.ts: getAccountsPage / getUsersPage return { rows, hasMore }. hasMore = (rows.length === PAGE_SIZE), so the client knows when to stop fetching. Each page is its own Next.js cache entry (the URL is the cache key) — caching from the previous commit still applies. - web/app/actions.ts: loadMoreAccounts / loadMoreUsers Server Actions for next-page requests; refreshAccounts / refreshUsers force-evict the cache via revalidateTag before refetching page 1. - web/app/page.tsx + users/page.tsx: only fetch the first page now. - web/components/{accounts,users}-table.tsx: rewrote state model. Rows accumulate as the user scrolls. An IntersectionObserver on a sentinel div near the bottom triggers loadMore when it enters the viewport (300px rootMargin so the next page starts loading before the user reaches the end). useOptimistic wraps the accumulated rows for in- flight edits; on success the row is committed locally so the change survives even though we no longer router.refresh. - Sort toggle now refetches from page 1 with the new dir/sort param. Local sort over a partial set would be inconsistent. - Mutations: delete filters from local state; create + refresh both reset to page 1 so the row appears in its sorted position. - Header count shows '+' when more pages exist so the operator knows what they're seeing isn't the full table. Removed AutoRefresh: - web/app/layout.tsx no longer mounts AutoRefresh. - web/components/auto-refresh.tsx deleted. - Reason: router.refresh every 30s would yank the user back to page 1 every time, losing scroll position and accumulated rows. Manual Refresh button replaces it (now wired to refreshAccounts/refreshUsers which evict cache + refetch). Tests: deferred verify_tables_once() means tests.test_bot_cli's CreateAppFactoryTests pass without DB env vars again. All 38 existing tests pass. --- app/cm_api.py | 109 ++++++++++++---- web/app/actions.ts | 40 +++++- web/app/layout.tsx | 2 - web/app/page.tsx | 12 +- web/app/users/page.tsx | 16 ++- web/components/accounts-table.tsx | 166 +++++++++++++++++++------ web/components/auto-refresh.tsx | 24 ---- web/components/users-table.tsx | 199 ++++++++++++++++++++---------- web/lib/api.ts | 67 ++++++++-- 9 files changed, 460 insertions(+), 175 deletions(-) delete mode 100644 web/components/auto-refresh.tsx diff --git a/app/cm_api.py b/app/cm_api.py index f4a28c3..bea4d1b 100644 --- a/app/cm_api.py +++ b/app/cm_api.py @@ -15,7 +15,7 @@ def _debug_enabled() -> bool: class CM_API: - + def __init__(self): self.app = Flask(__name__) # No CORS middleware: api-server is internal-only (no host port @@ -25,6 +25,19 @@ class CM_API: # default that becomes an attack surface if a host port is ever # accidentally re-exposed. self._register_routes() + # Schema verification is deferred to the first request so that + # constructing the WSGI app (e.g., in tests, or via gunicorn's + # preload phase before MySQL is reachable) doesn't require the + # DB to be up. The first request hits this hook, validates the + # schema, and flips the latch — subsequent requests skip it. + self._schema_verified = False + self.app.before_request(self._verify_schema_once) + + def _verify_schema_once(self): + if self._schema_verified: + return + verify_tables_once() + self._schema_verified = True def _get_database_connection(self): """Return a DB handle backed by the shared connection pool. @@ -74,36 +87,86 @@ class CM_API: is_available, db, error_response = self._check_database_available() if not is_available: return error_response - + try: if username: query = "SELECT username, password, status, link FROM acc WHERE username = %s" - query_params = [username] + results = db.query(query, [username]) + return jsonify(results) + + # Listing path — pagination + prefix-priority sort. + try: + limit = max(1, min(int(request.args.get('limit', 200)), 1000)) + offset = max(0, int(request.args.get('offset', 0))) + except (TypeError, ValueError): + return jsonify({"error": "limit and offset must be integers"}), 400 + prefix = (request.args.get('prefix') or '').strip() + # Whitelist direction so it's safe to interpolate into the + # ORDER BY clause (parameterised binding doesn't apply to + # column names or sort directions). + direction = 'ASC' if request.args.get('dir', 'desc').lower() == 'asc' else 'DESC' + + if prefix: + query = ( + "SELECT username, password, status, link FROM acc " + "ORDER BY (CASE WHEN username LIKE %s THEN 0 ELSE 1 END), " + f"username {direction} " + "LIMIT %s OFFSET %s" + ) + params = [f"{prefix}%", limit, offset] else: - query = "SELECT username, password, status, link FROM acc" - query_params = [] - - results = db.query(query, query_params) - return jsonify(results) - + query = ( + "SELECT username, password, status, link FROM acc " + f"ORDER BY username {direction} " + "LIMIT %s OFFSET %s" + ) + params = [limit, offset] + + return jsonify(db.query(query, params)) + except Exception as error: return self._handle_error(error, "Not Found"), 404 - + def get_user(self, username=None): is_available, db, error_response = self._check_database_available() if not is_available: return error_response - + try: if username: query = "SELECT f_username, f_password, t_username, t_password FROM user WHERE f_username = %s" - query_params = [username] + results = db.query(query, [username]) + return jsonify(results) + + try: + limit = max(1, min(int(request.args.get('limit', 200)), 1000)) + offset = max(0, int(request.args.get('offset', 0))) + except (TypeError, ValueError): + return jsonify({"error": "limit and offset must be integers"}), 400 + prefix = (request.args.get('prefix') or '').strip() + sort_arg = request.args.get('sort', 'last_update_time') + sort_col = 'f_username' if sort_arg == 'f_username' else 'last_update_time' + direction = 'ASC' if request.args.get('dir', 'desc').lower() == 'asc' else 'DESC' + + if prefix: + query = ( + "SELECT f_username, f_password, t_username, t_password, last_update_time " + "FROM user " + "ORDER BY (CASE WHEN f_username LIKE %s THEN 0 ELSE 1 END), " + f"{sort_col} {direction} " + "LIMIT %s OFFSET %s" + ) + params = [f"{prefix}%", limit, offset] else: - query = "SELECT f_username, f_password, t_username, t_password, last_update_time FROM user" - query_params = [] - - results = db.query(query, query_params) - return jsonify(results) + query = ( + "SELECT f_username, f_password, t_username, t_password, last_update_time " + "FROM user " + f"ORDER BY {sort_col} {direction} " + "LIMIT %s OFFSET %s" + ) + params = [limit, offset] + + return jsonify(db.query(query, params)) except Exception as error: return self._handle_error(error, "Not Found"), 404 @@ -298,13 +361,13 @@ class CM_API: def create_app(): """WSGI factory used by gunicorn (`app.cm_api:create_app()`). - Returns the Flask app object so gunicorn can serve it. Validates the - schema once at boot (so a misconfigured DB fails fast) — request-time - handlers don't repeat the check. + Returns the Flask app object. Schema verification runs lazily on the + first request (see CM_API._verify_schema_once) so the factory itself + never touches MySQL — keeps gunicorn's preload phase unaffected by a + momentarily-unavailable DB and lets unit tests construct the app + without DB env wiring. """ - app = CM_API().app - verify_tables_once() - return app + return CM_API().app if __name__ == '__main__': diff --git a/web/app/actions.ts b/web/app/actions.ts index d3a79bb..37294f7 100644 --- a/web/app/actions.ts +++ b/web/app/actions.ts @@ -1,14 +1,22 @@ "use server"; import { revalidatePath, revalidateTag } from "next/cache"; -import { ACCOUNTS_TAG, USERS_TAG, fetchApi } from "@/lib/api"; -import type { AccUpdate, UserUpdate } from "@/lib/types"; +import { + ACCOUNTS_TAG, + USERS_TAG, + fetchApi, + getAccountsPage, + getUsersPage, + type AccountsPageOpts, + type Page, + type UsersPageOpts, +} from "@/lib/api"; +import type { Acc, AccUpdate, User, UserUpdate } from "@/lib/types"; export type ActionResult = { ok: true } | { ok: false; error: string }; // Each mutation evicts the matching tag so the next GET bypasses the -// 30s data cache and re-reads from MySQL. revalidatePath then tells -// Next.js to re-render the dependent route on the next request. +// 30s data cache and re-reads from MySQL. export async function updateAccount(data: AccUpdate): Promise { try { @@ -75,3 +83,27 @@ export async function deleteUser(f_username: string): Promise { return { ok: false, error: err instanceof Error ? err.message : String(err) }; } } + +// ---- Pagination + force-refresh ---- + +export async function loadMoreAccounts(opts: AccountsPageOpts): Promise> { + return getAccountsPage(opts); +} + +export async function loadMoreUsers(opts: UsersPageOpts): Promise> { + return getUsersPage(opts); +} + +// Force-refresh evicts the cached tag before refetching the first page, +// so manual Refresh always returns DB-fresh data even if the cache is +// still warm. + +export async function refreshAccounts(opts: AccountsPageOpts = {}): Promise> { + revalidateTag(ACCOUNTS_TAG); + return getAccountsPage({ ...opts, offset: 0 }); +} + +export async function refreshUsers(opts: UsersPageOpts = {}): Promise> { + revalidateTag(USERS_TAG); + return getUsersPage({ ...opts, offset: 0 }); +} diff --git a/web/app/layout.tsx b/web/app/layout.tsx index f9fbed0..df66366 100644 --- a/web/app/layout.tsx +++ b/web/app/layout.tsx @@ -1,7 +1,6 @@ import "./globals.css"; import type { Metadata, Viewport } from "next"; import Nav from "@/components/nav"; -import AutoRefresh from "@/components/auto-refresh"; import { getSession } from "@/lib/auth"; export const metadata: Metadata = { @@ -27,7 +26,6 @@ export default async function RootLayout({
{children}
- ); diff --git a/web/app/page.tsx b/web/app/page.tsx index 88a6007..dc7a7fd 100644 --- a/web/app/page.tsx +++ b/web/app/page.tsx @@ -1,9 +1,15 @@ -import { getAccounts } from "@/lib/api"; +import { getAccountsPage } from "@/lib/api"; import AccountsTable from "@/components/accounts-table"; const PREFIX_PATTERN = process.env.NEXT_PUBLIC_CM_PREFIX_PATTERN ?? "13c"; export default async function AccountsPage() { - const accounts = await getAccounts(); - return ; + const page = await getAccountsPage({ prefix: PREFIX_PATTERN, dir: "desc" }); + return ( + + ); } diff --git a/web/app/users/page.tsx b/web/app/users/page.tsx index da8c03e..d5c9a3d 100644 --- a/web/app/users/page.tsx +++ b/web/app/users/page.tsx @@ -1,9 +1,19 @@ -import { getUsers } from "@/lib/api"; +import { getUsersPage } from "@/lib/api"; import UsersTable from "@/components/users-table"; const PREFIX_PATTERN = process.env.NEXT_PUBLIC_CM_PREFIX_PATTERN ?? "13c"; export default async function UsersPage() { - const users = await getUsers(); - return ; + const page = await getUsersPage({ + prefix: PREFIX_PATTERN, + sort: "last_update_time", + dir: "desc", + }); + return ( + + ); } diff --git a/web/components/accounts-table.tsx b/web/components/accounts-table.tsx index 57bb8c2..66f8025 100644 --- a/web/components/accounts-table.tsx +++ b/web/components/accounts-table.tsx @@ -1,30 +1,26 @@ "use client"; -import { useMemo, useOptimistic, useState, useTransition } from "react"; -import { useRouter } from "next/navigation"; +import { useEffect, useOptimistic, useRef, useState, useTransition } from "react"; import type { Acc } from "@/lib/types"; -import { deleteAccount, updateAccount } from "@/app/actions"; +import { + deleteAccount, + loadMoreAccounts, + refreshAccounts, + updateAccount, +} from "@/app/actions"; import EditableCell from "./editable-cell"; import ConfirmDialog from "./confirm-dialog"; import CreateAccountDialog from "./create-account-dialog"; import Toast, { type ToastMessage } from "./toast"; -type Props = { initial: Acc[]; prefixPattern: string }; +type Props = { + initial: Acc[]; + initialHasMore: boolean; + prefixPattern: string; +}; type SortDir = "asc" | "desc"; type OptimisticPatch = { username: string; field: keyof Acc; value: string }; -function sortAccounts(rows: Acc[], dir: SortDir, prefix: string): Acc[] { - return [...rows].sort((a, b) => { - const ap = a.username.startsWith(prefix); - const bp = b.username.startsWith(prefix); - if (ap && !bp) return -1; - if (!ap && bp) return 1; - return dir === "asc" - ? a.username.localeCompare(b.username) - : b.username.localeCompare(a.username); - }); -} - function StatusBadge({ status }: { status: string }) { const map: Record = { "": { bg: "bg-zinc-100", fg: "text-zinc-600", label: "available" }, @@ -62,52 +58,111 @@ function DeleteButton({ ); } -export default function AccountsTable({ initial, prefixPattern }: Props) { - const router = useRouter(); +export default function AccountsTable({ + initial, + initialHasMore, + prefixPattern, +}: Props) { const [sortDir, setSortDir] = useState("desc"); const [editingKey, setEditingKey] = useState(null); const [, startTransition] = useTransition(); const [refreshing, setRefreshing] = useState(false); + const [loadingMore, setLoadingMore] = useState(false); const [deleteTarget, setDeleteTarget] = useState(null); const [deleting, setDeleting] = useState(false); const [deleteError, setDeleteError] = useState(null); const [createOpen, setCreateOpen] = useState(false); const [toast, setToast] = useState(null); + // Accumulated rows from initial server-side fetch + every loadMore. + const [rows, setRows] = useState(initial); + const [hasMore, setHasMore] = useState(initialHasMore); + const sentinelRef = useRef(null); + const [optimistic, applyOptimistic] = useOptimistic( - initial, + rows, (state, patch) => state.map((row) => row.username === patch.username ? { ...row, [patch.field]: patch.value } : row, ), ); - const sorted = useMemo( - () => sortAccounts(optimistic, sortDir, prefixPattern), - [optimistic, sortDir, prefixPattern], - ); - function saveCell(username: string, field: keyof Acc, value: string) { return new Promise<{ ok: boolean; error?: string }>((resolve) => { startTransition(async () => { applyOptimistic({ username, field, value }); - const row = initial.find((r) => r.username === username); + const row = rows.find((r) => r.username === username); if (!row) return resolve({ ok: false, error: "row not found" }); const next: Acc = { ...row, [field]: value }; const result = await updateAccount(next); - resolve(result.ok ? { ok: true } : { ok: false, error: result.error }); + if (result.ok) { + setRows((prev) => prev.map((r) => (r.username === username ? next : r))); + resolve({ ok: true }); + } else { + resolve({ ok: false, error: result.error }); + } }); }); } - function refresh() { + async function refresh() { setRefreshing(true); - startTransition(() => { - router.refresh(); - setTimeout(() => setRefreshing(false), 400); - }); + try { + const page = await refreshAccounts({ prefix: prefixPattern, dir: sortDir }); + setRows(page.rows); + setHasMore(page.hasMore); + } finally { + setRefreshing(false); + } } + async function changeSort(next: SortDir) { + if (next === sortDir) return; + setSortDir(next); + setLoadingMore(true); + try { + const page = await loadMoreAccounts({ + offset: 0, + prefix: prefixPattern, + dir: next, + }); + setRows(page.rows); + setHasMore(page.hasMore); + } finally { + setLoadingMore(false); + } + } + + // Infinite scroll: when sentinel enters viewport, fetch the next page. + // 300px rootMargin so the next page starts loading before the user + // hits the bottom — feels seamless when scrolling fast. + useEffect(() => { + if (!hasMore || loadingMore || refreshing) return; + const sentinel = sentinelRef.current; + if (!sentinel) return; + + const observer = new IntersectionObserver( + (entries) => { + if (!entries.some((e) => e.isIntersecting)) return; + setLoadingMore(true); + loadMoreAccounts({ + offset: rows.length, + prefix: prefixPattern, + dir: sortDir, + }) + .then((page) => { + setRows((prev) => [...prev, ...page.rows]); + setHasMore(page.hasMore); + }) + .catch((err) => console.error("loadMoreAccounts failed:", err)) + .finally(() => setLoadingMore(false)); + }, + { rootMargin: "300px 0px" }, + ); + observer.observe(sentinel); + return () => observer.disconnect(); + }, [hasMore, loadingMore, refreshing, rows.length, prefixPattern, sortDir]); + async function confirmDelete() { if (!deleteTarget) return; setDeleting(true); @@ -116,6 +171,7 @@ export default function AccountsTable({ initial, prefixPattern }: Props) { setDeleting(false); if (result.ok) { const deleted = deleteTarget; + setRows((prev) => prev.filter((r) => r.username !== deleted)); setDeleteTarget(null); setToast({ type: "success", message: `Account ${deleted} deleted` }); } else { @@ -123,11 +179,13 @@ export default function AccountsTable({ initial, prefixPattern }: Props) { } } - if (initial.length === 0) { + if (rows.length === 0) { return (
setCreateOpen(true)} @@ -140,6 +198,12 @@ export default function AccountsTable({ initial, prefixPattern }: Props) { setCreateOpen(false)} + onSuccess={async (name) => { + setToast({ type: "success", message: `Account ${name} created` }); + // Force-refresh from page 1 so the new row appears in its + // sorted position. (We don't know where it ranks otherwise.) + await refresh(); + }} prefixPattern={prefixPattern} />
@@ -150,6 +214,8 @@ export default function AccountsTable({ initial, prefixPattern }: Props) {
setCreateOpen(true)} @@ -163,7 +229,7 @@ export default function AccountsTable({ initial, prefixPattern }: Props) {
); } @@ -201,6 +250,7 @@ export default function UsersTable({ initial, prefixPattern }: Props) {
setCreateOpen(true)} @@ -229,7 +279,7 @@ export default function UsersTable({ initial, prefixPattern }: Props) { - {sorted.map((row) => { + {optimistic.map((row) => { const k = (f: string) => `${row.f_username}::${f}`; return ( @@ -286,7 +336,7 @@ export default function UsersTable({ initial, prefixPattern }: Props) {
- {sorted.map((row) => { + {optimistic.map((row) => { const k = (f: string) => `${row.f_username}::${f}`; return (
@@ -344,12 +394,25 @@ export default function UsersTable({ initial, prefixPattern }: Props) { })}
+