From d9c0824cb49a4a5de1552e6f8f5608b771b9ecbe Mon Sep 17 00:00:00 2001 From: Lee Chapman Date: Thu, 5 Mar 2026 00:43:04 -0800 Subject: [PATCH] feat: local Postgres, schema on first use, initial data pull - Add Postgres service to docker-compose; API uses DATABASE_URL - Makefile: postgres-up, postgres-down, local-up, local-down - .env.example: PostgreSQL URL for local runs - Ensure DB schema created on first use (_get_pool calls _init_db) - Run initial data collection on startup so MCP/UI have data immediately Made-with: Cursor --- .env.example | 3 ++- Makefile | 24 ++++++++++++++++++++++-- docker-compose.yml | 31 +++++++++++++++++++++++++++---- src/collector.py | 8 +++++++- src/database.py | 29 +++++++++++++++++++---------- 5 files changed, 77 insertions(+), 18 deletions(-) diff --git a/.env.example b/.env.example index a67ba6f..61fa5af 100644 --- a/.env.example +++ b/.env.example @@ -1,7 +1,8 @@ FLASK_ENV=development FLASK_DEBUG=True API_PORT=5000 -DATABASE_URL=sqlite:///helldivers2.db +# PostgreSQL (required). For local runs use postgres from docker: make postgres-up +DATABASE_URL=postgresql://helldivers:helldivers@localhost:5432/helldivers2 LOG_LEVEL=INFO SCRAPE_INTERVAL=300 diff --git a/Makefile b/Makefile index b58bede..7e165c3 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,4 @@ -.PHONY: help install dev test lint format clean build docker-build docker-login docker-push docker-pull docker-run run check check-all commit-changes release venv +.PHONY: help install dev test lint format clean build docker-build docker-login docker-push docker-pull docker-run run check check-all commit-changes release venv postgres-up postgres-down local-up local-down # Force use of bash shell (required for make to work properly with line continuations) SHELL := /bin/bash @@ -22,7 +22,11 @@ help: @echo " venv Create virtual environment" @echo " install Install dependencies" @echo " dev Install development dependencies and run with reload" - @echo " run Run the API server" + @echo " run Run the API server (requires DATABASE_URL; use postgres-up first for local)" + @echo " postgres-up Start local PostgreSQL in Docker (port 5432)" + @echo " postgres-down Stop local PostgreSQL" + @echo " local-up Start Postgres then run API (local development)" + @echo " local-down Stop Postgres" @echo " test Run tests with coverage" @echo " test-fast Run tests without coverage" @echo " lint Run linters (ruff, mypy)" @@ -53,6 +57,22 @@ dev: venv run: venv $(PYTHON) -m uvicorn src.app:app --host 0.0.0.0 --port $(PORT) +# Local PostgreSQL for development (docker compose) +postgres-up: + docker compose up -d postgres + @echo "Waiting for Postgres to be ready..." + @until docker compose exec -T postgres pg_isready -U helldivers -d helldivers2 2>/dev/null; do sleep 1; done + @echo "Postgres is ready. Set DATABASE_URL=postgresql://helldivers:helldivers@localhost:5432/helldivers2 (see .env.example) and run: make run" + +postgres-down: + docker compose stop postgres + +local-up: postgres-up venv + @echo "Starting API with local Postgres..." + @DATABASE_URL=$${DATABASE_URL:-postgresql://helldivers:helldivers@localhost:5432/helldivers2} $(PYTHON) -m uvicorn src.app:app --host 0.0.0.0 --port $(PORT) + +local-down: postgres-down + test: $(PYTEST) diff --git a/docker-compose.yml b/docker-compose.yml index 113cd8e..1b2d7c5 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,21 +1,41 @@ +# Local development and run: use Postgres. Start with: docker compose up -d postgres && make run +# Or run API in Docker too: docker compose up --build version: '3.8' services: + postgres: + image: postgres:16-alpine + container_name: high-command-postgres + environment: + POSTGRES_USER: helldivers + POSTGRES_PASSWORD: helldivers + POSTGRES_DB: helldivers2 + ports: + - "5432:5432" + volumes: + - high-command-pgdata:/var/lib/postgresql/data + healthcheck: + test: ["CMD-SHELL", "pg_isready -U helldivers -d helldivers2"] + interval: 5s + timeout: 5s + retries: 5 + restart: unless-stopped + api: build: context: . dockerfile: Dockerfile image: high-command-api:latest - # Alternative: Use Harbor image for production deployments - # image: harbor.dataknife.net/library/high-command-api:latest ports: - "5000:5000" environment: + - DATABASE_URL=postgresql://helldivers:helldivers@postgres:5432/helldivers2 - FLASK_ENV=production - API_PORT=5000 - SCRAPE_INTERVAL=300 - volumes: - - ./helldivers2.db:/app/helldivers2.db + depends_on: + postgres: + condition: service_healthy restart: unless-stopped healthcheck: test: ["CMD", "curl", "-f", "http://localhost:5000/api/health"] @@ -23,3 +43,6 @@ services: timeout: 10s retries: 3 start_period: 40s + +volumes: + high-command-pgdata: diff --git a/src/collector.py b/src/collector.py index 39d581e..84e970b 100644 --- a/src/collector.py +++ b/src/collector.py @@ -1,4 +1,5 @@ import logging +import threading from apscheduler.schedulers.background import BackgroundScheduler from src.scraper import HellDivers2Scraper from src.database import Database @@ -17,7 +18,7 @@ def __init__(self, db: Database, interval: int = 300): self.is_running = False def start(self): - """Start the data collection scheduler""" + """Start the data collection scheduler and run an initial pull so MCP/UI have data immediately.""" if self.is_running: logger.warning("Data collector is already running") return @@ -29,6 +30,11 @@ def start(self): self.is_running = True logger.info(f"Data collector started with {self.interval}s interval") + # Initial pull so MCP and UI have data while the interval runs + thread = threading.Thread(target=self.collect_all_data, daemon=True) + thread.start() + logger.info("Initial data collection started in background") + def stop(self): """Stop the data collection scheduler""" if not self.is_running: diff --git a/src/database.py b/src/database.py index e088f92..1181b13 100644 --- a/src/database.py +++ b/src/database.py @@ -50,10 +50,11 @@ def __init__(self, database_url: Optional[str] = None): self.database_url = database_url self._initialized = False + self._schema_initialized = False self._pool: Optional[pool.ThreadedConnectionPool] = None def _get_pool(self) -> pool.ThreadedConnectionPool: - """Get or create the connection pool (lazy init)""" + """Get or create the connection pool (lazy init). Ensures schema exists on first use.""" if self._pool is None: import os maxconn = DEFAULT_POOL_MAX_CONN @@ -68,6 +69,8 @@ def _get_pool(self) -> pool.ThreadedConnectionPool: dsn=self.database_url, ) logger.info("Database connection pool initialized") + if not self._schema_initialized and self._init_db(): + self._schema_initialized = True return self._pool def _get_connection(self): @@ -102,8 +105,8 @@ def _parse_expiration_time(expiration_time: str) -> Optional[datetime]: except (ValueError, AttributeError): return None - def _init_db(self): - """Initialize database schema (lazy - called on first use)""" + def _init_db(self) -> bool: + """Initialize database schema (lazy - called on first use). Returns True on success.""" conn = None try: conn = self._get_connection() @@ -232,13 +235,19 @@ def _init_db(self): conn.commit() conn.close() - except psycopg2.OperationalError: - # Database connection failed - this is OK during tests/imports - # Schema will be created when first actual operation happens - pass - except Exception: - # Any other error during init is also OK - will be handled on first use - pass + return True + except psycopg2.OperationalError as e: + logger.debug("Schema init skipped (connection): %s", e) + return False + except Exception as e: + logger.warning("Schema init failed: %s", e) + return False + finally: + if conn is not None: + try: + conn.close() + except Exception: + pass def save_war_status(self, data: Dict) -> bool: """Save war status to database"""