Add chapter-level DB tools, dim sync, retries, and all-genres
- Check missing pages: compare site page count vs R2, detect NULL/0 width/height in DB and fix by reading WebP bytes from R2 (no re-upload needed when dims are just missing). - Delete specific chapter(s): multi-select or all, removes from R2+DB. - Chapter pickers now offer "All chapters" as first option. - Save all genres comma-separated in Manga.genre (was only the first). - Sync refreshes title/description/genre for existing manga records. - Page inserts now save width and height from PIL (schema update). - Retry failed page fetches up to 3 attempts with 2s backoff instead of skipping after one failure. - Cover detection polls DOM up to 8s and tries broader selectors. Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
This commit is contained in:
parent
e037996c5c
commit
723b82c9fc
475
manga.py
475
manga.py
@ -205,6 +205,23 @@ def with_browser(func):
|
|||||||
# ── Cloudflare ─────────────────────────────────────────────
|
# ── Cloudflare ─────────────────────────────────────────────
|
||||||
|
|
||||||
|
|
||||||
|
def _wait_for_cf_on_page(page, timeout=120):
|
||||||
|
"""Wait for CF to resolve on a specific page."""
|
||||||
|
for i in range(timeout):
|
||||||
|
try:
|
||||||
|
title = page.title()
|
||||||
|
except Exception:
|
||||||
|
time.sleep(1)
|
||||||
|
continue
|
||||||
|
if "Just a moment" in title or "challenge" in page.url:
|
||||||
|
time.sleep(1)
|
||||||
|
continue
|
||||||
|
if title and ("嗨皮漫画" in title or "happymh" in page.url):
|
||||||
|
return True
|
||||||
|
time.sleep(1)
|
||||||
|
return False
|
||||||
|
|
||||||
|
|
||||||
def wait_for_cloudflare(session, timeout=120):
|
def wait_for_cloudflare(session, timeout=120):
|
||||||
"""Wait for CF to resolve. User solves in the visible browser window."""
|
"""Wait for CF to resolve. User solves in the visible browser window."""
|
||||||
page = session.page
|
page = session.page
|
||||||
@ -406,7 +423,7 @@ def fetch_metadata(page):
|
|||||||
# ── Happymh: image download ───────────────────────────────
|
# ── Happymh: image download ───────────────────────────────
|
||||||
|
|
||||||
|
|
||||||
def _try_get_chapter_images(session, slug, chapter_id):
|
def _try_get_chapter_images(page, slug, chapter_id):
|
||||||
"""Single attempt to get chapter images. Returns (images, api_status)."""
|
"""Single attempt to get chapter images. Returns (images, api_status)."""
|
||||||
captured_images = []
|
captured_images = []
|
||||||
api_info = {"found": False, "status": None, "error": None}
|
api_info = {"found": False, "status": None, "error": None}
|
||||||
@ -443,10 +460,8 @@ def _try_get_chapter_images(session, slug, chapter_id):
|
|||||||
except Exception as e:
|
except Exception as e:
|
||||||
api_info["error"] = str(e)
|
api_info["error"] = str(e)
|
||||||
|
|
||||||
page = session.page
|
|
||||||
page.on("response", on_response)
|
page.on("response", on_response)
|
||||||
reader_url = f"{BASE_URL}/mangaread/{slug}/{chapter_id}"
|
reader_url = f"{BASE_URL}/mangaread/{slug}/{chapter_id}"
|
||||||
print(" Loading reader...")
|
|
||||||
try:
|
try:
|
||||||
page.evaluate(f"window.location.href = '{reader_url}'")
|
page.evaluate(f"window.location.href = '{reader_url}'")
|
||||||
except Exception:
|
except Exception:
|
||||||
@ -459,17 +474,13 @@ def _try_get_chapter_images(session, slug, chapter_id):
|
|||||||
except Exception:
|
except Exception:
|
||||||
pass
|
pass
|
||||||
|
|
||||||
print(" Waiting for page...")
|
if not _wait_for_cf_on_page(page, timeout=90):
|
||||||
if not wait_for_cloudflare(session, timeout=90):
|
|
||||||
page = session.page
|
|
||||||
try:
|
try:
|
||||||
page.remove_listener("response", on_response)
|
page.remove_listener("response", on_response)
|
||||||
except Exception:
|
except Exception:
|
||||||
pass
|
pass
|
||||||
return [], api_info
|
return [], api_info
|
||||||
|
|
||||||
page = session.page
|
|
||||||
print(" Waiting for API...")
|
|
||||||
deadline = time.time() + 20
|
deadline = time.time() + 20
|
||||||
while time.time() < deadline:
|
while time.time() < deadline:
|
||||||
if captured_images:
|
if captured_images:
|
||||||
@ -539,28 +550,39 @@ def _try_get_chapter_images(session, slug, chapter_id):
|
|||||||
return captured_images, api_info
|
return captured_images, api_info
|
||||||
|
|
||||||
|
|
||||||
def get_chapter_images(session, slug, chapter_id):
|
def get_chapter_images(page, slug, chapter_id):
|
||||||
"""Get chapter images. On API 403 (CF expired), navigate to solve and retry."""
|
"""Get chapter images using given page. On API 403, returns empty (caller should handle CF)."""
|
||||||
images, api_info = _try_get_chapter_images(session, slug, chapter_id)
|
images, api_info = _try_get_chapter_images(page, slug, chapter_id)
|
||||||
if images:
|
return images, api_info
|
||||||
return images
|
|
||||||
|
|
||||||
if api_info.get("status") == 403:
|
|
||||||
print(" CF expired — solve in browser...")
|
|
||||||
page = session.page
|
|
||||||
try:
|
|
||||||
page.goto(f"{BASE_URL}/mangaread/{slug}/{chapter_id}", wait_until="commit", timeout=60000)
|
|
||||||
except Exception:
|
|
||||||
pass
|
|
||||||
if wait_for_cloudflare(session, timeout=120):
|
|
||||||
images, _ = _try_get_chapter_images(session, slug, chapter_id)
|
|
||||||
|
|
||||||
return images
|
|
||||||
|
|
||||||
|
|
||||||
def fetch_image_bytes(session, img):
|
def fetch_all_pages(page, images, max_attempts=3):
|
||||||
"""Fetch image via browser network stack, return raw bytes or None."""
|
"""Fetch all pages with retry using given page. Returns {page_num: bytes}."""
|
||||||
page = session.page
|
total = len(images)
|
||||||
|
page_bytes = {}
|
||||||
|
pending = list(enumerate(images, 1))
|
||||||
|
|
||||||
|
for attempt in range(1, max_attempts + 1):
|
||||||
|
if not pending:
|
||||||
|
break
|
||||||
|
if attempt > 1:
|
||||||
|
time.sleep(2)
|
||||||
|
|
||||||
|
next_pending = []
|
||||||
|
for pn, img in pending:
|
||||||
|
body = fetch_image_bytes(page, img)
|
||||||
|
if body:
|
||||||
|
page_bytes[pn] = body
|
||||||
|
else:
|
||||||
|
next_pending.append((pn, img))
|
||||||
|
time.sleep(0.1)
|
||||||
|
pending = next_pending
|
||||||
|
|
||||||
|
return page_bytes
|
||||||
|
|
||||||
|
|
||||||
|
def fetch_image_bytes(page, img):
|
||||||
|
"""Fetch image via browser network stack using given page."""
|
||||||
url = img["url"]
|
url = img["url"]
|
||||||
ref_policy = "no-referrer" if img.get("no_referrer") else "origin"
|
ref_policy = "no-referrer" if img.get("no_referrer") else "origin"
|
||||||
try:
|
try:
|
||||||
@ -571,18 +593,16 @@ def fetch_image_bytes(session, img):
|
|||||||
body = response.body()
|
body = response.body()
|
||||||
if body and len(body) > 100:
|
if body and len(body) > 100:
|
||||||
return body
|
return body
|
||||||
except Exception as e:
|
except Exception:
|
||||||
if not hasattr(fetch_image_bytes, "_err_logged"):
|
pass
|
||||||
fetch_image_bytes._err_logged = True
|
|
||||||
print(f"\n First error: {e}")
|
|
||||||
return None
|
return None
|
||||||
|
|
||||||
|
|
||||||
def download_image(session, img, save_path):
|
def download_image(page, img, save_path):
|
||||||
"""Fetch image and save to disk."""
|
"""Fetch image and save to disk."""
|
||||||
if save_path.exists():
|
if save_path.exists():
|
||||||
return True
|
return True
|
||||||
body = fetch_image_bytes(session, img)
|
body = fetch_image_bytes(page, img)
|
||||||
if body:
|
if body:
|
||||||
save_path.parent.mkdir(parents=True, exist_ok=True)
|
save_path.parent.mkdir(parents=True, exist_ok=True)
|
||||||
save_path.write_bytes(body)
|
save_path.write_bytes(body)
|
||||||
@ -606,6 +626,22 @@ def convert_to_webp(source, quality=WEBP_QUALITY):
|
|||||||
return _to_webp_bytes(Image.open(source), quality)
|
return _to_webp_bytes(Image.open(source), quality)
|
||||||
|
|
||||||
|
|
||||||
|
def probe_and_webp(source, quality=WEBP_QUALITY):
|
||||||
|
"""Open once; return (width, height, webp_bytes)."""
|
||||||
|
with Image.open(source) as img:
|
||||||
|
return img.width, img.height, _to_webp_bytes(img, quality)
|
||||||
|
|
||||||
|
|
||||||
|
def insert_pages(cur, chapter_id, page_urls):
|
||||||
|
"""page_urls: {page_num: (url, width, height)}. Inserts in page_num order."""
|
||||||
|
for pn in sorted(page_urls):
|
||||||
|
url, w, h = page_urls[pn]
|
||||||
|
cur.execute(
|
||||||
|
'INSERT INTO "Page" ("chapterId", number, "imageUrl", width, height) VALUES (%s, %s, %s, %s, %s)',
|
||||||
|
(chapter_id, pn, url, w, h),
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
def make_cover(source, width=400, height=560):
|
def make_cover(source, width=400, height=560):
|
||||||
img = Image.open(source)
|
img = Image.open(source)
|
||||||
target_ratio = width / height
|
target_ratio = width / height
|
||||||
@ -789,7 +825,7 @@ def download_chapter(session, slug, chapter_index, chapter, manga_dir):
|
|||||||
folder_name = f"{chapter_index} {ch_name}"
|
folder_name = f"{chapter_index} {ch_name}"
|
||||||
chapter_dir = manga_dir / folder_name
|
chapter_dir = manga_dir / folder_name
|
||||||
|
|
||||||
images = get_chapter_images(session, slug, ch_id)
|
images, _ = get_chapter_images(session.page, slug, ch_id)
|
||||||
if not images:
|
if not images:
|
||||||
print(f" No images")
|
print(f" No images")
|
||||||
return False
|
return False
|
||||||
@ -797,30 +833,16 @@ def download_chapter(session, slug, chapter_index, chapter, manga_dir):
|
|||||||
print(f" {len(images)} pages")
|
print(f" {len(images)} pages")
|
||||||
chapter_dir.mkdir(parents=True, exist_ok=True)
|
chapter_dir.mkdir(parents=True, exist_ok=True)
|
||||||
|
|
||||||
|
page_bytes = fetch_all_pages(session.page, images)
|
||||||
ok = 0
|
ok = 0
|
||||||
failed = []
|
for pn, body in page_bytes.items():
|
||||||
for pn, img in enumerate(images, 1):
|
|
||||||
save_path = chapter_dir / f"{pn}.jpg"
|
save_path = chapter_dir / f"{pn}.jpg"
|
||||||
if download_image(session, img, save_path):
|
save_path.write_bytes(body)
|
||||||
ok += 1
|
ok += 1
|
||||||
print(f" {pn}/{len(images)}", end="\r")
|
|
||||||
else:
|
|
||||||
failed.append((pn, img))
|
|
||||||
time.sleep(0.1)
|
|
||||||
|
|
||||||
if failed:
|
|
||||||
time.sleep(1)
|
|
||||||
for pn, img in failed:
|
|
||||||
save_path = chapter_dir / f"{pn}.jpg"
|
|
||||||
if download_image(session, img, save_path):
|
|
||||||
ok += 1
|
|
||||||
else:
|
|
||||||
print(f" {pn}/{len(images)} FAIL")
|
|
||||||
time.sleep(0.3)
|
|
||||||
|
|
||||||
print(f" {ok}/{len(images)} downloaded" + " " * 20)
|
print(f" {ok}/{len(images)} downloaded" + " " * 20)
|
||||||
|
|
||||||
if ok == 0:
|
if ok < len(images):
|
||||||
try:
|
try:
|
||||||
chapter_dir.rmdir()
|
chapter_dir.rmdir()
|
||||||
except Exception:
|
except Exception:
|
||||||
@ -910,17 +932,19 @@ def upload_manga_to_r2(manga_name, conn):
|
|||||||
def process_page(args, _slug=slug, _order=order_num):
|
def process_page(args, _slug=slug, _order=order_num):
|
||||||
j, pf = args
|
j, pf = args
|
||||||
r2_key = f"manga/{_slug}/chapters/{_order}/{j}.webp"
|
r2_key = f"manga/{_slug}/chapters/{_order}/{j}.webp"
|
||||||
if not r2_key_exists(r2_key):
|
if r2_key_exists(r2_key):
|
||||||
return j, upload_to_r2(r2_key, convert_to_webp(pf))
|
with Image.open(pf) as img:
|
||||||
return j, f"{PUBLIC_URL}/{r2_key}"
|
return j, f"{PUBLIC_URL}/{r2_key}", img.width, img.height
|
||||||
|
w, h, webp = probe_and_webp(pf)
|
||||||
|
return j, upload_to_r2(r2_key, webp), w, h
|
||||||
|
|
||||||
page_urls = {}
|
page_urls = {}
|
||||||
done = 0
|
done = 0
|
||||||
with ThreadPoolExecutor(max_workers=UPLOAD_WORKERS) as pool:
|
with ThreadPoolExecutor(max_workers=UPLOAD_WORKERS) as pool:
|
||||||
futures = {pool.submit(process_page, (j, f)): j for j, f in enumerate(page_files, 1)}
|
futures = {pool.submit(process_page, (j, f)): j for j, f in enumerate(page_files, 1)}
|
||||||
for future in as_completed(futures):
|
for future in as_completed(futures):
|
||||||
j, url = future.result()
|
j, url, w, h = future.result()
|
||||||
page_urls[j] = url
|
page_urls[j] = (url, w, h)
|
||||||
done += 1
|
done += 1
|
||||||
print(f" {done}/{len(page_files)}", end="\r")
|
print(f" {done}/{len(page_files)}", end="\r")
|
||||||
|
|
||||||
@ -934,8 +958,7 @@ def upload_manga_to_r2(manga_name, conn):
|
|||||||
(manga_id, order_num, chapter_title),
|
(manga_id, order_num, chapter_title),
|
||||||
)
|
)
|
||||||
chapter_id = cur.fetchone()[0]
|
chapter_id = cur.fetchone()[0]
|
||||||
for j in sorted(page_urls):
|
insert_pages(cur, chapter_id, page_urls)
|
||||||
cur.execute('INSERT INTO "Page" ("chapterId", number, "imageUrl") VALUES (%s, %s, %s)', (chapter_id, j, page_urls[j]))
|
|
||||||
conn.commit()
|
conn.commit()
|
||||||
print(f" {len(page_files)} pages uploaded" + " " * 10)
|
print(f" {len(page_files)} pages uploaded" + " " * 10)
|
||||||
|
|
||||||
@ -1127,80 +1150,71 @@ def cmd_sync(manga_url=None):
|
|||||||
cur.execute('SELECT number FROM "Chapter" WHERE "mangaId" = %s', (manga_id,))
|
cur.execute('SELECT number FROM "Chapter" WHERE "mangaId" = %s', (manga_id,))
|
||||||
existing_numbers = {row[0] for row in cur.fetchall()}
|
existing_numbers = {row[0] for row in cur.fetchall()}
|
||||||
|
|
||||||
new_count = 0
|
# 3. Collect chapters to sync
|
||||||
for i, ch in enumerate(chapters, 1):
|
todo = [(i, ch) for i, ch in enumerate(chapters, 1) if i not in existing_numbers]
|
||||||
|
|
||||||
|
if not todo:
|
||||||
|
print(" Already up to date!")
|
||||||
|
continue
|
||||||
|
|
||||||
|
print(f" {len(todo)} new chapters to sync")
|
||||||
|
|
||||||
|
completed = 0
|
||||||
|
skipped = 0
|
||||||
|
for i, ch in todo:
|
||||||
if esc.stop.is_set():
|
if esc.stop.is_set():
|
||||||
break
|
break
|
||||||
ch_name = ch["chapterName"]
|
ch_name = ch["chapterName"]
|
||||||
if i in existing_numbers:
|
|
||||||
continue
|
|
||||||
|
|
||||||
new_count += 1
|
|
||||||
print(f" [{i}/{len(chapters)}] {ch_name} (id={ch['id']})")
|
print(f" [{i}/{len(chapters)}] {ch_name} (id={ch['id']})")
|
||||||
|
|
||||||
# Get image URLs from reader page
|
images, api_info = get_chapter_images(session.page, slug, ch["id"])
|
||||||
images = get_chapter_images(session, slug, ch["id"])
|
if not images and api_info.get("status") == 403:
|
||||||
|
print(f" CF blocked — run Setup and try again")
|
||||||
|
esc.stop.set()
|
||||||
|
break
|
||||||
if not images:
|
if not images:
|
||||||
print(f" No images")
|
print(f" No images")
|
||||||
|
skipped += 1
|
||||||
continue
|
continue
|
||||||
|
|
||||||
print(f" {len(images)} pages")
|
print(f" {len(images)} pages")
|
||||||
|
page_bytes = fetch_all_pages(session.page, images)
|
||||||
# Fetch each image into RAM, convert to WebP, upload to R2
|
if len(page_bytes) < len(images):
|
||||||
page_bytes = {} # page_num -> raw bytes
|
missing = [pn for pn in range(1, len(images) + 1) if pn not in page_bytes]
|
||||||
ok = 0
|
print(f" Could not fetch pages: {missing}, skipping chapter")
|
||||||
for pn, img in enumerate(images, 1):
|
skipped += 1
|
||||||
body = fetch_image_bytes(session, img)
|
|
||||||
if body:
|
|
||||||
page_bytes[pn] = body
|
|
||||||
ok += 1
|
|
||||||
print(f" Fetched {pn}/{len(images)}", end="\r")
|
|
||||||
else:
|
|
||||||
print(f" {pn}/{len(images)} FAIL")
|
|
||||||
time.sleep(0.1)
|
|
||||||
|
|
||||||
if not page_bytes:
|
|
||||||
print(f" No images fetched, skip")
|
|
||||||
continue
|
continue
|
||||||
|
|
||||||
# Upload to R2 first
|
def upload_one(args, _slug=slug, _i=i):
|
||||||
def upload_page(args, _slug=slug, _i=i):
|
|
||||||
pn, raw = args
|
pn, raw = args
|
||||||
r2_key = f"manga/{_slug}/chapters/{_i}/{pn}.webp"
|
r2_key = f"manga/{_slug}/chapters/{_i}/{pn}.webp"
|
||||||
webp = convert_to_webp(io.BytesIO(raw))
|
w, h, webp = probe_and_webp(io.BytesIO(raw))
|
||||||
return pn, upload_to_r2(r2_key, webp)
|
return pn, upload_to_r2(r2_key, webp), w, h
|
||||||
|
|
||||||
page_urls = {}
|
page_urls = {}
|
||||||
done = 0
|
done = 0
|
||||||
with ThreadPoolExecutor(max_workers=UPLOAD_WORKERS) as pool:
|
with ThreadPoolExecutor(max_workers=UPLOAD_WORKERS) as pool:
|
||||||
futures = {pool.submit(upload_page, (pn, raw)): pn for pn, raw in page_bytes.items()}
|
for pn, r2_url, w, h in pool.map(upload_one, page_bytes.items()):
|
||||||
for future in as_completed(futures):
|
page_urls[pn] = (r2_url, w, h)
|
||||||
pn, r2_url = future.result()
|
|
||||||
page_urls[pn] = r2_url
|
|
||||||
done += 1
|
done += 1
|
||||||
print(f" R2: {done}/{len(page_bytes)}", end="\r")
|
print(f" R2: {done}/{len(page_bytes)}", end="\r")
|
||||||
|
|
||||||
if not page_urls:
|
if not page_urls:
|
||||||
print(f" R2 upload failed, skip")
|
skipped += 1
|
||||||
continue
|
continue
|
||||||
|
|
||||||
# Only create DB records after R2 upload succeeds
|
|
||||||
cur.execute(
|
cur.execute(
|
||||||
'INSERT INTO "Chapter" ("mangaId", number, title) VALUES (%s, %s, %s) RETURNING id',
|
'INSERT INTO "Chapter" ("mangaId", number, title) VALUES (%s, %s, %s) RETURNING id',
|
||||||
(manga_id, i, ch_name),
|
(manga_id, i, ch_name),
|
||||||
)
|
)
|
||||||
chapter_id = cur.fetchone()[0]
|
chapter_id = cur.fetchone()[0]
|
||||||
for pn in sorted(page_urls):
|
insert_pages(cur, chapter_id, page_urls)
|
||||||
cur.execute('INSERT INTO "Page" ("chapterId", number, "imageUrl") VALUES (%s, %s, %s)', (chapter_id, pn, page_urls[pn]))
|
|
||||||
conn.commit()
|
conn.commit()
|
||||||
|
completed += 1
|
||||||
print(f" {len(page_urls)} pages synced" + " " * 20)
|
print(f" {len(page_urls)} pages synced" + " " * 20)
|
||||||
|
|
||||||
time.sleep(REQUEST_DELAY)
|
time.sleep(REQUEST_DELAY)
|
||||||
|
|
||||||
if new_count == 0:
|
print(f" Synced {completed}/{len(todo)} chapters ({skipped} skipped)")
|
||||||
print(" Already up to date!")
|
|
||||||
else:
|
|
||||||
print(f" Synced {new_count} new chapters")
|
|
||||||
|
|
||||||
try:
|
try:
|
||||||
with_browser(run)
|
with_browser(run)
|
||||||
@ -1584,12 +1598,261 @@ def tui_edit_manga():
|
|||||||
conn.close()
|
conn.close()
|
||||||
|
|
||||||
|
|
||||||
|
def _pick_manga_and_chapters(conn, prompt="Select chapters", multi=True):
|
||||||
|
"""Helper: pick manga from DB, then pick chapter(s). Returns (slug, [(ch_id, ch_num, ch_title), ...]) or None."""
|
||||||
|
cur = conn.cursor()
|
||||||
|
cur.execute('SELECT id, slug, title FROM "Manga" ORDER BY title')
|
||||||
|
mangas = cur.fetchall()
|
||||||
|
if not mangas:
|
||||||
|
print(" No manga in DB")
|
||||||
|
return None
|
||||||
|
|
||||||
|
items = [f"{i+1}. {title} ({slug})" for i, (_, slug, title) in enumerate(mangas)]
|
||||||
|
sel = tui_select("Select manga (/ to search):", items, search=True)
|
||||||
|
if sel < 0:
|
||||||
|
return None
|
||||||
|
manga_id, slug, _ = mangas[sel]
|
||||||
|
|
||||||
|
cur.execute('SELECT id, number, title FROM "Chapter" WHERE "mangaId" = %s ORDER BY number', (manga_id,))
|
||||||
|
chapters = cur.fetchall()
|
||||||
|
if not chapters:
|
||||||
|
print(" No chapters in DB for this manga")
|
||||||
|
return None
|
||||||
|
|
||||||
|
if multi:
|
||||||
|
scope = tui_select(f"{prompt}: {len(chapters)} chapters", [
|
||||||
|
"All chapters",
|
||||||
|
"Select specific chapters",
|
||||||
|
])
|
||||||
|
if scope == -1:
|
||||||
|
return None
|
||||||
|
if scope == 0:
|
||||||
|
return slug, list(chapters)
|
||||||
|
|
||||||
|
items = [f"{num}. {title}" for _, num, title in chapters]
|
||||||
|
menu = TerminalMenu(
|
||||||
|
items,
|
||||||
|
title="Space=toggle, Enter=confirm, /=search:",
|
||||||
|
multi_select=True,
|
||||||
|
show_multi_select_hint=True,
|
||||||
|
search_key="/",
|
||||||
|
show_search_hint=True,
|
||||||
|
)
|
||||||
|
selected = menu.show()
|
||||||
|
if not selected:
|
||||||
|
return None
|
||||||
|
if isinstance(selected, int):
|
||||||
|
selected = (selected,)
|
||||||
|
picked = [chapters[i] for i in selected]
|
||||||
|
else:
|
||||||
|
items = [f"{num}. {title}" for _, num, title in chapters]
|
||||||
|
sel = tui_select(f"{prompt} (/ to search):", items, search=True)
|
||||||
|
if sel < 0:
|
||||||
|
return None
|
||||||
|
picked = [chapters[sel]]
|
||||||
|
|
||||||
|
return slug, picked
|
||||||
|
|
||||||
|
|
||||||
|
def tui_delete_chapter():
|
||||||
|
"""Delete specific chapter(s) from R2 + DB."""
|
||||||
|
try:
|
||||||
|
conn = get_db()
|
||||||
|
except Exception as e:
|
||||||
|
print(f" DB error: {e}")
|
||||||
|
return
|
||||||
|
try:
|
||||||
|
result = _pick_manga_and_chapters(conn, "Select chapters to delete")
|
||||||
|
if not result:
|
||||||
|
return
|
||||||
|
slug, to_delete = result
|
||||||
|
confirm = input(f" Delete {len(to_delete)} chapter(s) from R2 + DB? [y/N] ").strip().lower()
|
||||||
|
if confirm != "y":
|
||||||
|
print(" Cancelled.")
|
||||||
|
return
|
||||||
|
|
||||||
|
cur = conn.cursor()
|
||||||
|
for ch_id, ch_num, ch_title in to_delete:
|
||||||
|
print(f" Deleting [{ch_num}] {ch_title}...")
|
||||||
|
r2_delete_prefix(f"manga/{slug}/chapters/{ch_num}/")
|
||||||
|
cur.execute('DELETE FROM "Page" WHERE "chapterId" = %s', (ch_id,))
|
||||||
|
cur.execute('DELETE FROM "Chapter" WHERE id = %s', (ch_id,))
|
||||||
|
conn.commit()
|
||||||
|
print(f" Done.")
|
||||||
|
finally:
|
||||||
|
conn.close()
|
||||||
|
|
||||||
|
|
||||||
|
def tui_check_missing_pages():
|
||||||
|
"""Check selected chapters against the site's actual page count and re-upload if mismatched."""
|
||||||
|
try:
|
||||||
|
conn = get_db()
|
||||||
|
except Exception as e:
|
||||||
|
print(f" DB error: {e}")
|
||||||
|
return
|
||||||
|
|
||||||
|
try:
|
||||||
|
result = _pick_manga_and_chapters(conn, "Select chapters to check")
|
||||||
|
if not result:
|
||||||
|
return
|
||||||
|
slug, selected_chapters = result
|
||||||
|
|
||||||
|
if slug not in [slug_from_url(u) for u in load_manga_urls()]:
|
||||||
|
print(f" {slug} not in manga.json — cannot re-fetch pages")
|
||||||
|
return
|
||||||
|
except Exception:
|
||||||
|
conn.close()
|
||||||
|
raise
|
||||||
|
|
||||||
|
# Load reader pages and compare site's actual page count vs R2
|
||||||
|
def run(session):
|
||||||
|
with EscListener() as esc:
|
||||||
|
result = load_manga_page(session, slug)
|
||||||
|
if not result:
|
||||||
|
return
|
||||||
|
chapters, _, _ = result
|
||||||
|
if not chapters:
|
||||||
|
return
|
||||||
|
|
||||||
|
cur2 = conn.cursor()
|
||||||
|
to_reupload = []
|
||||||
|
to_fix_dims = []
|
||||||
|
|
||||||
|
print(f"\n Checking {len(selected_chapters)} chapters...")
|
||||||
|
for ch_id, ch_num, ch_title in selected_chapters:
|
||||||
|
if esc.stop.is_set():
|
||||||
|
break
|
||||||
|
if ch_num > len(chapters):
|
||||||
|
print(f" [{ch_num}] {ch_title}: out of range on site")
|
||||||
|
continue
|
||||||
|
|
||||||
|
ch = chapters[ch_num - 1]
|
||||||
|
images, api_info = get_chapter_images(session.page, slug, ch["id"])
|
||||||
|
if not images:
|
||||||
|
if api_info.get("status") == 403:
|
||||||
|
print(f" [{ch_num}] CF blocked — run Setup")
|
||||||
|
esc.stop.set()
|
||||||
|
break
|
||||||
|
print(f" [{ch_num}] {ch_title}: no images from site")
|
||||||
|
continue
|
||||||
|
|
||||||
|
site_count = len(images)
|
||||||
|
r2_count = r2_count_by_prefix(f"manga/{slug}/chapters/{ch_num}/")
|
||||||
|
|
||||||
|
if site_count != r2_count:
|
||||||
|
print(f" [{ch_num}] {ch_title}: site={site_count}, R2={r2_count} — re-upload")
|
||||||
|
to_reupload.append((ch_id, ch_num, ch_title, ch, images))
|
||||||
|
continue
|
||||||
|
|
||||||
|
# Count matches — check if DB has valid width/height for all pages
|
||||||
|
cur2.execute(
|
||||||
|
'SELECT COUNT(*), '
|
||||||
|
'COUNT(*) FILTER (WHERE width IS NULL OR width <= 0), '
|
||||||
|
'COUNT(*) FILTER (WHERE height IS NULL OR height <= 0), '
|
||||||
|
'MIN(width), MAX(width), MIN(height), MAX(height) '
|
||||||
|
'FROM "Page" WHERE "chapterId" = %s',
|
||||||
|
(ch_id,),
|
||||||
|
)
|
||||||
|
db_count, bad_w, bad_h, min_w, max_w, min_h, max_h = cur2.fetchone()
|
||||||
|
bad_count = max(bad_w, bad_h)
|
||||||
|
if bad_count > 0:
|
||||||
|
print(f" [{ch_num}] {ch_title}: {bad_count} pages need dims (w {min_w}-{max_w}, h {min_h}-{max_h}) — fix from R2")
|
||||||
|
to_fix_dims.append((ch_id, ch_num, ch_title))
|
||||||
|
else:
|
||||||
|
print(f" [{ch_num}] {ch_title}: {site_count} pages OK (w {min_w}-{max_w}, h {min_h}-{max_h})")
|
||||||
|
|
||||||
|
# Fix dimensions by reading existing R2 objects (no re-upload)
|
||||||
|
if to_fix_dims:
|
||||||
|
print(f"\n Fixing dimensions for {len(to_fix_dims)} chapter(s)...")
|
||||||
|
for ch_id, ch_num, ch_title in to_fix_dims:
|
||||||
|
if esc.stop.is_set():
|
||||||
|
break
|
||||||
|
cur2.execute(
|
||||||
|
'SELECT id, number, "imageUrl" FROM "Page" WHERE "chapterId" = %s '
|
||||||
|
'AND (width IS NULL OR width = 0 OR height IS NULL OR height = 0) '
|
||||||
|
'ORDER BY number',
|
||||||
|
(ch_id,),
|
||||||
|
)
|
||||||
|
pages = cur2.fetchall()
|
||||||
|
|
||||||
|
def read_dims(args, _slug=slug, _n=ch_num):
|
||||||
|
page_id, pn, _url = args
|
||||||
|
r2_key = f"manga/{_slug}/chapters/{_n}/{pn}.webp"
|
||||||
|
try:
|
||||||
|
data = s3.get_object(Bucket=BUCKET, Key=r2_key)["Body"].read()
|
||||||
|
with Image.open(io.BytesIO(data)) as img:
|
||||||
|
return page_id, img.width, img.height
|
||||||
|
except Exception:
|
||||||
|
return page_id, None, None
|
||||||
|
|
||||||
|
updated = 0
|
||||||
|
with ThreadPoolExecutor(max_workers=UPLOAD_WORKERS) as pool:
|
||||||
|
for page_id, w, h in pool.map(read_dims, pages):
|
||||||
|
if w and h:
|
||||||
|
cur2.execute(
|
||||||
|
'UPDATE "Page" SET width = %s, height = %s WHERE id = %s',
|
||||||
|
(w, h, page_id),
|
||||||
|
)
|
||||||
|
updated += 1
|
||||||
|
conn.commit()
|
||||||
|
print(f" [{ch_num}] {ch_title}: {updated}/{len(pages)} dims updated")
|
||||||
|
|
||||||
|
if not to_reupload:
|
||||||
|
if not to_fix_dims:
|
||||||
|
print("\n All selected chapters are complete.")
|
||||||
|
return
|
||||||
|
|
||||||
|
print(f"\n {len(to_reupload)} chapter(s) need re-upload")
|
||||||
|
|
||||||
|
for ch_id, ch_num, ch_title, ch, images in to_reupload:
|
||||||
|
if esc.stop.is_set():
|
||||||
|
break
|
||||||
|
print(f"\n Re-uploading [{ch_num}] {ch_title}")
|
||||||
|
page_bytes = fetch_all_pages(session.page, images)
|
||||||
|
if len(page_bytes) < len(images):
|
||||||
|
missing = [pn for pn in range(1, len(images) + 1) if pn not in page_bytes]
|
||||||
|
print(f" Could not fetch pages: {missing}, skipping")
|
||||||
|
continue
|
||||||
|
|
||||||
|
# Upload to R2 (overwrites existing)
|
||||||
|
def upload_page(args, _slug=slug, _n=ch_num):
|
||||||
|
pn, raw = args
|
||||||
|
r2_key = f"manga/{_slug}/chapters/{_n}/{pn}.webp"
|
||||||
|
w, h, webp = probe_and_webp(io.BytesIO(raw))
|
||||||
|
return pn, upload_to_r2(r2_key, webp), w, h
|
||||||
|
|
||||||
|
page_urls = {}
|
||||||
|
done = 0
|
||||||
|
with ThreadPoolExecutor(max_workers=UPLOAD_WORKERS) as pool:
|
||||||
|
futures = {pool.submit(upload_page, (pn, raw)): pn for pn, raw in page_bytes.items()}
|
||||||
|
for future in as_completed(futures):
|
||||||
|
pn, r2_url, w, h = future.result()
|
||||||
|
page_urls[pn] = (r2_url, w, h)
|
||||||
|
done += 1
|
||||||
|
print(f" R2: {done}/{len(page_bytes)}", end="\r")
|
||||||
|
|
||||||
|
# Replace Page records
|
||||||
|
cur2.execute('DELETE FROM "Page" WHERE "chapterId" = %s', (ch_id,))
|
||||||
|
insert_pages(cur2, ch_id, page_urls)
|
||||||
|
conn.commit()
|
||||||
|
print(f" {len(page_urls)} pages restored" + " " * 20)
|
||||||
|
|
||||||
|
try:
|
||||||
|
with_browser(run)
|
||||||
|
finally:
|
||||||
|
conn.close()
|
||||||
|
|
||||||
|
print("\nCheck complete!")
|
||||||
|
|
||||||
|
|
||||||
def tui_r2_manage():
|
def tui_r2_manage():
|
||||||
while True:
|
while True:
|
||||||
idx = tui_select("R2 / DB Management", [
|
idx = tui_select("R2 / DB Management", [
|
||||||
"Status",
|
"Status",
|
||||||
"Edit manga info",
|
"Edit manga info",
|
||||||
"Delete specific manga",
|
"Delete specific manga",
|
||||||
|
"Delete specific chapter",
|
||||||
|
"Check missing pages",
|
||||||
"Clear ALL (R2 + DB)",
|
"Clear ALL (R2 + DB)",
|
||||||
"Recompress manga (quality 65)",
|
"Recompress manga (quality 65)",
|
||||||
])
|
])
|
||||||
@ -1651,6 +1914,12 @@ def tui_r2_manage():
|
|||||||
print(f" DB error: {e}")
|
print(f" DB error: {e}")
|
||||||
|
|
||||||
elif idx == 3:
|
elif idx == 3:
|
||||||
|
tui_delete_chapter()
|
||||||
|
|
||||||
|
elif idx == 4:
|
||||||
|
tui_check_missing_pages()
|
||||||
|
|
||||||
|
elif idx == 5:
|
||||||
confirm = input(" Delete ALL R2 + DB? [y/N] ").strip().lower()
|
confirm = input(" Delete ALL R2 + DB? [y/N] ").strip().lower()
|
||||||
if confirm == "y":
|
if confirm == "y":
|
||||||
r2_delete_prefix("")
|
r2_delete_prefix("")
|
||||||
@ -1665,7 +1934,7 @@ def tui_r2_manage():
|
|||||||
except Exception as e:
|
except Exception as e:
|
||||||
print(f" DB error: {e}")
|
print(f" DB error: {e}")
|
||||||
|
|
||||||
elif idx == 4:
|
elif idx == 6:
|
||||||
slugs = r2_list_prefixes()
|
slugs = r2_list_prefixes()
|
||||||
if not slugs:
|
if not slugs:
|
||||||
print(" R2 is empty")
|
print(" R2 is empty")
|
||||||
|
|||||||
Loading…
x
Reference in New Issue
Block a user