From d73439698a5057d544ca4e97397b0e356b9d9e0d Mon Sep 17 00:00:00 2001 From: yiekheng Date: Sun, 19 Oct 2025 22:22:55 +0800 Subject: [PATCH] Refactor Docker layout for Gitea publishing - move Python sources into app package and switch services to module entrypoints - relocate Dockerfiles under docker/, add buildx publish script, override compose for local builds - configure images to pull from gitea.04080616.xyz/yiekheng with env-driven tags and limits - harden installs and transfer worker logging/concurrency for cleaner container output --- .dockerignore | 57 +------------ .env | 2 + Dockerfile.transfer | 17 ---- app/__init__.py | 2 + cm_api.py => app/cm_api.py | 4 +- cm_bot.py => app/cm_bot.py | 0 cm_bot_hal.py => app/cm_bot_hal.py | 4 +- cm_telegram.py => app/cm_telegram.py | 4 +- app/cm_transfer_credit.py | 65 +++++++++++++++ cm_web_view.py => app/cm_web_view.py | 0 db.py => app/db.py | 0 cm_transfer_credit.py | 61 -------------- docker-compose.override.yml | 29 +++++++ docker-compose.yml | 20 ++--- Dockerfile.api => docker/api/Dockerfile | 10 +-- .../telegram/Dockerfile | 10 +-- docker/transfer/Dockerfile | 17 ++++ Dockerfile.web => docker/web/Dockerfile | 10 +-- scripts/local_build.sh | 1 + scripts/publish.sh | 83 +++++++++++++++++++ 20 files changed, 232 insertions(+), 164 deletions(-) create mode 100644 .env delete mode 100644 Dockerfile.transfer create mode 100644 app/__init__.py rename cm_api.py => app/cm_api.py (99%) rename cm_bot.py => app/cm_bot.py (100%) rename cm_bot_hal.py => app/cm_bot_hal.py (99%) rename cm_telegram.py => app/cm_telegram.py (98%) create mode 100644 app/cm_transfer_credit.py rename cm_web_view.py => app/cm_web_view.py (100%) rename db.py => app/db.py (100%) delete mode 100644 cm_transfer_credit.py create mode 100644 docker-compose.override.yml rename Dockerfile.api => docker/api/Dockerfile (57%) rename Dockerfile.telegram => docker/telegram/Dockerfile (52%) create mode 100644 docker/transfer/Dockerfile rename Dockerfile.web => docker/web/Dockerfile (56%) create mode 100755 scripts/local_build.sh create mode 100755 scripts/publish.sh diff --git a/.dockerignore b/.dockerignore index cdcc516..a25256f 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1,55 +1,6 @@ -# Git -.git -.gitignore - -# Python -__pycache__ -*.pyc -*.pyo -*.pyd -.Python -env -pip-log.txt -pip-delete-this-directory.txt -.tox -.coverage -.coverage.* -.cache -nosetests.xml -coverage.xml -*.cover +__pycache__/ +*.py[cod] *.log .git -.mypy_cache -.pytest_cache -.hypothesis - -# OS -.DS_Store -.DS_Store? -._* -.Spotlight-V100 -.Trashes -ehthumbs.db -Thumbs.db - -# IDE -.vscode -.idea -*.swp -*.swo -*~ - -# Docker -Dockerfile* -docker-compose*.yml -.dockerignore - -# Documentation -README.md -*.md - -# Other -.env -.env.local -.env.*.local +logs/ +node_modules/ diff --git a/.env b/.env new file mode 100644 index 0000000..c4c8c05 --- /dev/null +++ b/.env @@ -0,0 +1,2 @@ +CM_IMAGE_PREFIX=gitea.04080616.xyz/yiekheng +DOCKER_IMAGE_TAG=latest diff --git a/Dockerfile.transfer b/Dockerfile.transfer deleted file mode 100644 index 66b0aec..0000000 --- a/Dockerfile.transfer +++ /dev/null @@ -1,17 +0,0 @@ -FROM python:3.9-slim - -WORKDIR /app - -RUN apt-get update - -COPY requirements.txt . -RUN pip install --no-cache-dir -r requirements.txt - -# Copy application files -COPY . . - -# Set environment variables -ENV PYTHONUNBUFFERED=1 - -# Run the API server -CMD ["python", "cm_transfer_credit.py"] \ No newline at end of file diff --git a/app/__init__.py b/app/__init__.py new file mode 100644 index 0000000..e91af5d --- /dev/null +++ b/app/__init__.py @@ -0,0 +1,2 @@ +"""Shared CM bot application package.""" + diff --git a/cm_api.py b/app/cm_api.py similarity index 99% rename from cm_api.py rename to app/cm_api.py index 92d68a3..0bb59d9 100644 --- a/cm_api.py +++ b/app/cm_api.py @@ -1,7 +1,7 @@ import threading from flask import Flask, jsonify, request from flask_cors import CORS -from db import DB +from .db import DB class CM_API: @@ -188,4 +188,4 @@ class CM_API: if __name__ == '__main__': api = CM_API() - api.run(port = 3000) \ No newline at end of file + api.run(port = 3000) diff --git a/cm_bot.py b/app/cm_bot.py similarity index 100% rename from cm_bot.py rename to app/cm_bot.py diff --git a/cm_bot_hal.py b/app/cm_bot_hal.py similarity index 99% rename from cm_bot_hal.py rename to app/cm_bot_hal.py index 9022fb2..e5833c7 100644 --- a/cm_bot_hal.py +++ b/app/cm_bot_hal.py @@ -1,7 +1,7 @@ import requests, re from bs4 import BeautifulSoup -from cm_bot import CM_BOT -from db import DB +from .cm_bot import CM_BOT +from .db import DB import secrets, string diff --git a/cm_telegram.py b/app/cm_telegram.py similarity index 98% rename from cm_telegram.py rename to app/cm_telegram.py index 13df0cf..071583e 100644 --- a/cm_telegram.py +++ b/app/cm_telegram.py @@ -3,7 +3,7 @@ import threading, logging, time, asyncio from telegram import ForceReply, Update from telegram.ext import Application, CommandHandler, ContextTypes, MessageHandler, filters -from cm_bot_hal import CM_BOT_HAL +from .cm_bot_hal import CM_BOT_HAL logging.basicConfig( format="%(asctime)s - %(name)s - %(levelname)s - %(message)s", level=logging.INFO @@ -107,4 +107,4 @@ def main() -> None: application.run_polling(allowed_updates=Update.ALL_TYPES) if __name__ == "__main__": - main() \ No newline at end of file + main() diff --git a/app/cm_transfer_credit.py b/app/cm_transfer_credit.py new file mode 100644 index 0000000..ae72fd6 --- /dev/null +++ b/app/cm_transfer_credit.py @@ -0,0 +1,65 @@ +import logging, time, requests, json, os, threading +from concurrent.futures import ThreadPoolExecutor +from datetime import datetime + +from .cm_bot_hal import CM_BOT_HAL + +# Suppress httpx logs +logging.getLogger("httpx").setLevel(logging.WARNING) + +# api_url = 'https://api.luckytown888.net' +api_url = os.getenv('API_BASE_URL', 'http://api-server:3000') +max_threading = int(os.getenv('CM_TRANSFER_MAX_THREADS', '20')) + +def transfer(data: dict, local_logger): + bot = CM_BOT_HAL() + thread_name = threading.current_thread().name + local_logger.info(f"[Thread-{thread_name}] [Start] Transfer Credit from {data['f_username']} to {data['t_username']}") + result = bot.transfer_credit_api(data['f_username'], data['f_password'], data['t_username'], data['t_password']) + local_logger.info(f"[Thread-{thread_name}] {result}") + local_logger.info(f"[Thread-{thread_name}] [Done] {data['f_username']} transfer done!") + del bot + time.sleep(5) + +def main(): + while True: + weekday = int(datetime.now().strftime("%w")) + hour = int(datetime.now().strftime("%H")) + if weekday == 1 and (hour >= 6 and hour < 12): + local_logger = logging.getLogger(__name__) + if not local_logger.handlers: + handler = logging.StreamHandler() + handler.setLevel(logging.INFO) + formatter = logging.Formatter('%(asctime)s - %(levelname)s - %(message)s') + handler.setFormatter(formatter) + local_logger.addHandler(handler) + local_logger.setLevel(logging.INFO) + + local_logger.info("=" * 80) + local_logger.info( + "Transfer window triggered | weekday=%s | hour=%s | max_threads=%s | api_url=%s", + weekday, + hour, + max_threading, + api_url, + ) + local_logger.info("=" * 80) + + response = requests.get(f'{api_url}/user') + items = json.loads(response.text) + total_items = len(items) if isinstance(items, list) else 0 + + if total_items == 0: + local_logger.info("No items to process.") + local_logger.info("=" * 80) + else: + local_logger.info(f"Processing {total_items} transfer items...") + with ThreadPoolExecutor(max_workers=max_threading) as executor: + list(executor.map(lambda item: transfer(item, local_logger), items)) + local_logger.info(f"Completed processing {total_items} transfer items.") + local_logger.info("=" * 80) + time.sleep(10 * 60) + + +if __name__ == "__main__": + main() diff --git a/cm_web_view.py b/app/cm_web_view.py similarity index 100% rename from cm_web_view.py rename to app/cm_web_view.py diff --git a/db.py b/app/db.py similarity index 100% rename from db.py rename to app/db.py diff --git a/cm_transfer_credit.py b/cm_transfer_credit.py deleted file mode 100644 index 684bafa..0000000 --- a/cm_transfer_credit.py +++ /dev/null @@ -1,61 +0,0 @@ -import logging, time, requests, json, os, threading -from concurrent.futures import ThreadPoolExecutor -from datetime import datetime - -from cm_bot_hal import CM_BOT_HAL - -# Suppress httpx logs -logging.getLogger("httpx").setLevel(logging.WARNING) - -# Create logs directory if it doesn't exist -logs_dir = "logs" -if not os.path.exists(logs_dir): - os.makedirs(logs_dir) - -# api_url = 'https://api.luckytown888.net' -api_url = 'http://api-server:3000' -max_threading = 20 - -def transfer(data: dict, local_logger): - bot = CM_BOT_HAL() - thread_name = threading.current_thread().name - local_logger.info(f"[Thread-{thread_name}] [Start] Transfer Credit from {data['f_username']} to {data['t_username']}") - result = bot.transfer_credit_api(data['f_username'], data['f_password'], data['t_username'], data['t_password']) - local_logger.info(f"[Thread-{thread_name}] {result}") - local_logger.info(f"[Thread-{thread_name}] [Done] {data['f_username']} transfer done!") - del bot - time.sleep(5) - -while True: - weekday = int(datetime.now().strftime("%w")) - hour = int(datetime.now().strftime("%H")) - minutes = int(datetime.now().strftime("%M")) - if weekday == 1 and (hour >= 6 and hour < 12): - local_logger = logging.getLogger(f"{__name__}") - - # Configure file handler for logging to logs folder - log_date_dir = f"{logs_dir}/{datetime.now().strftime('%Y%m%d')}" - if not os.path.exists(log_date_dir): - os.makedirs(log_date_dir) - log_filename = f"{log_date_dir}/transfer_credit_{datetime.now().strftime('%Y%m%d_%H%M')}.log" - file_handler = logging.FileHandler(log_filename) - file_handler.setLevel(logging.INFO) - formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s') - file_handler.setFormatter(formatter) - - # Add file handler to logger if not already added - if not any(isinstance(handler, logging.FileHandler) for handler in local_logger.handlers): - local_logger.addHandler(file_handler) - local_logger.setLevel(logging.INFO) - - response = requests.get(f'{api_url}/user') - items = json.loads(response.text) - total_items = len(items) if isinstance(items, list) else 0 - if total_items == 0: - local_logger.info("No items to process.") - else: - local_logger.info(f"Processing {total_items} transfer items...") - with ThreadPoolExecutor(max_workers=max_threading) as executor: - results = list(executor.map(lambda item: transfer(item, local_logger), items)) - local_logger.info(f"Completed processing {total_items} transfer items.") - time.sleep(10 * 60) diff --git a/docker-compose.override.yml b/docker-compose.override.yml new file mode 100644 index 0000000..7cbe8e6 --- /dev/null +++ b/docker-compose.override.yml @@ -0,0 +1,29 @@ +services: + telegram-bot: + build: + context: . + dockerfile: docker/telegram/Dockerfile + image: "${CM_IMAGE_PREFIX:-local}/cm-telegram:${DOCKER_IMAGE_TAG:-dev}" + + api-server: + build: + context: . + dockerfile: docker/api/Dockerfile + image: "${CM_IMAGE_PREFIX:-local}/cm-api:${DOCKER_IMAGE_TAG:-dev}" + + web-view: + build: + context: . + dockerfile: docker/web/Dockerfile + image: "${CM_IMAGE_PREFIX:-local}/cm-web:${DOCKER_IMAGE_TAG:-dev}" + + transfer-bot: + build: + context: . + dockerfile: docker/transfer/Dockerfile + image: "${CM_IMAGE_PREFIX:-local}/cm-transfer:${DOCKER_IMAGE_TAG:-dev}" + environment: + - API_BASE_URL=http://api-server:3000 + - CM_TRANSFER_MAX_THREADS=1 + mem_limit: 2g + cpus: 2 diff --git a/docker-compose.yml b/docker-compose.yml index 349e258..815dc8c 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,9 +1,7 @@ services: # Telegram Bot Service telegram-bot: - build: - context: . - dockerfile: Dockerfile.telegram + image: "${CM_IMAGE_PREFIX:-your-registry/namespace}/cm-telegram:${DOCKER_IMAGE_TAG:-latest}" container_name: cm-telegram-bot restart: unless-stopped environment: @@ -18,9 +16,7 @@ services: # API Server Service api-server: - build: - context: . - dockerfile: Dockerfile.api + image: "${CM_IMAGE_PREFIX:-your-registry/namespace}/cm-api:${DOCKER_IMAGE_TAG:-latest}" container_name: cm-api-server restart: unless-stopped ports: @@ -35,9 +31,7 @@ services: # Web View Service web-view: - build: - context: . - dockerfile: Dockerfile.web + image: "${CM_IMAGE_PREFIX:-your-registry/namespace}/cm-web:${DOCKER_IMAGE_TAG:-latest}" container_name: cm-web-view restart: unless-stopped ports: @@ -54,16 +48,18 @@ services: - api-server transfer-bot: - build: - context: . - dockerfile: Dockerfile.transfer + image: "${CM_IMAGE_PREFIX:-your-registry/namespace}/cm-transfer:${DOCKER_IMAGE_TAG:-latest}" container_name: cm-transfer-bot restart: unless-stopped environment: - PYTHONUNBUFFERED=1 + - API_BASE_URL=http://api-server:3000 + - CM_TRANSFER_MAX_THREADS=20 volumes: - /etc/timezone:/etc/timezone:ro - /etc/localtime:/etc/localtime:ro + mem_limit: 6g + cpus: 2 networks: - cm-network depends_on: diff --git a/Dockerfile.api b/docker/api/Dockerfile similarity index 57% rename from Dockerfile.api rename to docker/api/Dockerfile index 08d17be..9b4b209 100644 --- a/Dockerfile.api +++ b/docker/api/Dockerfile @@ -1,14 +1,14 @@ FROM python:3.9-slim +ENV PIP_DEFAULT_TIMEOUT=120 + WORKDIR /app -RUN apt-get update - COPY requirements.txt . -RUN pip install --no-cache-dir -r requirements.txt +RUN pip install --no-cache-dir --retries 5 -r requirements.txt # Copy application files -COPY . . +COPY app ./app # Set environment variables ENV PYTHONUNBUFFERED=1 @@ -17,4 +17,4 @@ ENV PYTHONUNBUFFERED=1 EXPOSE 3000 # Run the API server -CMD ["python", "cm_api.py"] \ No newline at end of file +CMD ["python", "-m", "app.cm_api"] diff --git a/Dockerfile.telegram b/docker/telegram/Dockerfile similarity index 52% rename from Dockerfile.telegram rename to docker/telegram/Dockerfile index 98958d8..73313fa 100644 --- a/Dockerfile.telegram +++ b/docker/telegram/Dockerfile @@ -1,17 +1,17 @@ FROM python:3.9-slim +ENV PIP_DEFAULT_TIMEOUT=120 + WORKDIR /app -RUN apt-get update - COPY requirements.txt . -RUN pip install --no-cache-dir -r requirements.txt +RUN pip install --no-cache-dir --retries 5 -r requirements.txt # Copy application files -COPY . . +COPY app ./app # Set environment variables ENV PYTHONUNBUFFERED=1 # Run the telegram bot -CMD ["python", "cm_telegram.py"] +CMD ["python", "-m", "app.cm_telegram"] diff --git a/docker/transfer/Dockerfile b/docker/transfer/Dockerfile new file mode 100644 index 0000000..43e8f0f --- /dev/null +++ b/docker/transfer/Dockerfile @@ -0,0 +1,17 @@ +FROM python:3.9-slim + +ENV PIP_DEFAULT_TIMEOUT=120 + +WORKDIR /app + +COPY requirements.txt . +RUN pip install --no-cache-dir --retries 5 -r requirements.txt + +# Copy application files +COPY app ./app + +# Set environment variables +ENV PYTHONUNBUFFERED=1 + +# Run the transfer credit worker +CMD ["python", "-m", "app.cm_transfer_credit"] diff --git a/Dockerfile.web b/docker/web/Dockerfile similarity index 56% rename from Dockerfile.web rename to docker/web/Dockerfile index 2ed3850..9234f9b 100644 --- a/Dockerfile.web +++ b/docker/web/Dockerfile @@ -1,14 +1,14 @@ FROM python:3.9-slim +ENV PIP_DEFAULT_TIMEOUT=120 + WORKDIR /app -RUN apt-get update - COPY requirements.txt . -RUN pip install --no-cache-dir -r requirements.txt +RUN pip install --no-cache-dir --retries 5 -r requirements.txt # Copy application files -COPY . . +COPY app ./app # Set environment variables ENV PYTHONUNBUFFERED=1 @@ -17,4 +17,4 @@ ENV PYTHONUNBUFFERED=1 EXPOSE 8000 # Run the web view -CMD ["python", "cm_web_view.py"] +CMD ["python", "-m", "app.cm_web_view"] diff --git a/scripts/local_build.sh b/scripts/local_build.sh new file mode 100755 index 0000000..e1a95dd --- /dev/null +++ b/scripts/local_build.sh @@ -0,0 +1 @@ +sudo docker compose -f docker-compose.yml -f docker-compose.override.yml up --build \ No newline at end of file diff --git a/scripts/publish.sh b/scripts/publish.sh new file mode 100755 index 0000000..266cbdd --- /dev/null +++ b/scripts/publish.sh @@ -0,0 +1,83 @@ +#!/usr/bin/env bash +set -euo pipefail + +REGISTRY_PREFIX="gitea.04080616.xyz/yiekheng" + +usage() { + cat <<'EOF' +Build and push CM Bot service images to gitea.04080616.xyz/yiekheng. + +Usage: + scripts/publish.sh [tag] + +Arguments: + tag Optional tag to publish (default: latest). Override with DOCKER_IMAGE_TAG. + +Environment: + DOCKER_IMAGE_TAG Alternative way to set the tag (overrides CLI argument). + BUILD_ARGS Extra arguments passed to each docker build command. + +Make sure you are authenticated first: + docker login gitea.04080616.xyz +EOF +} + +if [[ "${1:-}" == "-h" || "${1:-}" == "--help" ]]; then + usage + exit 0 +fi + +if ! docker info >/dev/null 2>&1; then + echo "Docker daemon is not reachable. Please start Docker and retry." >&2 + exit 1 +fi + +if ! docker system info --format '{{json .IndexServerAddress}}' | grep -q "gitea.04080616.xyz" 2>/dev/null; then + cat <<'EOF' >&2 +Reminder: run 'docker login gitea.04080616.xyz' before publishing so pushes succeed. +EOF +fi + +IMAGE_TAG="${1:-${DOCKER_IMAGE_TAG:-latest}}" +ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +PLATFORMS="${CM_IMAGE_PLATFORMS:-linux/amd64}" + +if ! docker buildx version >/dev/null 2>&1; then + cat <<'EOF' >&2 +Docker Buildx is required for producing registry-compatible images. +Install/enable buildx and rerun, for example: + docker buildx create --use --name cm-bot-builder + docker buildx inspect --bootstrap +EOF + exit 1 +fi + +echo "Using buildx with platforms: ${PLATFORMS}" +echo + +SERVICES=( + "api docker/api/Dockerfile" + "telegram docker/telegram/Dockerfile" + "web docker/web/Dockerfile" + "transfer docker/transfer/Dockerfile" +) + +echo "Publishing CM Bot images to ${REGISTRY_PREFIX}/cm-:${IMAGE_TAG}" +echo + +for ENTRY in "${SERVICES[@]}"; do + SERVICE="${ENTRY%% *}" + DOCKERFILE="${ENTRY#* }" + IMAGE_NAME="${REGISTRY_PREFIX}/cm-${SERVICE}:${IMAGE_TAG}" + + echo "==> Building and pushing ${IMAGE_NAME} (${DOCKERFILE})" + docker buildx build ${BUILD_ARGS:-} \ + --platform "${PLATFORMS}" \ + -f "${ROOT_DIR}/${DOCKERFILE}" \ + -t "${IMAGE_NAME}" \ + --push \ + "${ROOT_DIR}" + echo +done + +echo "All images pushed to ${REGISTRY_PREFIX} with tag '${IMAGE_TAG}'."