feat: local Postgres, schema on first use, initial data pull

.env.example

@@ -1,7 +1,8 @@
 FLASK_ENV=development
 FLASK_DEBUG=True
 API_PORT=5000
-DATABASE_URL=sqlite:///helldivers2.db
+# PostgreSQL (required). For local runs use postgres from docker: make postgres-up
+DATABASE_URL=postgresql://helldivers:helldivers@localhost:5432/helldivers2
 LOG_LEVEL=INFO
 SCRAPE_INTERVAL=300
 

Makefile

@@ -1,4 +1,4 @@
-.PHONY: help install dev test lint format clean build docker-build docker-login docker-push docker-pull docker-run run check check-all commit-changes release venv
+.PHONY: help install dev test lint format clean build docker-build docker-login docker-push docker-pull docker-run run check check-all commit-changes release venv postgres-up postgres-down local-up local-down
 
 # Force use of bash shell (required for make to work properly with line continuations)
 SHELL := /bin/bash
@@ -22,7 +22,11 @@ help:
 	@echo "  venv           Create virtual environment"
 	@echo "  install        Install dependencies"
 	@echo "  dev            Install development dependencies and run with reload"
-	@echo "  run            Run the API server"
+	@echo "  run            Run the API server (requires DATABASE_URL; use postgres-up first for local)"
+	@echo "  postgres-up    Start local PostgreSQL in Docker (port 5432)"
+	@echo "  postgres-down  Stop local PostgreSQL"
+	@echo "  local-up       Start Postgres then run API (local development)"
+	@echo "  local-down    Stop Postgres"
 	@echo "  test           Run tests with coverage"
 	@echo "  test-fast      Run tests without coverage"
 	@echo "  lint           Run linters (ruff, mypy)"
@@ -53,6 +57,22 @@ dev: venv
 run: venv
 	$(PYTHON) -m uvicorn src.app:app --host 0.0.0.0 --port $(PORT)
 
+# Local PostgreSQL for development (docker compose)
+postgres-up:
+	docker compose up -d postgres
+	@echo "Waiting for Postgres to be ready..."
+	@until docker compose exec -T postgres pg_isready -U helldivers -d helldivers2 2>/dev/null; do sleep 1; done
+	@echo "Postgres is ready. Set DATABASE_URL=postgresql://helldivers:helldivers@localhost:5432/helldivers2 (see .env.example) and run: make run"
+
+postgres-down:
+	docker compose stop postgres
+
+local-up: postgres-up venv
+	@echo "Starting API with local Postgres..."
+	@DATABASE_URL=$${DATABASE_URL:-postgresql://helldivers:helldivers@localhost:5432/helldivers2} $(PYTHON) -m uvicorn src.app:app --host 0.0.0.0 --port $(PORT)
+
+local-down: postgres-down
+
 test:
 	$(PYTEST)
 

docker-compose.yml

@@ -1,25 +1,48 @@
+# Local development and run: use Postgres. Start with: docker compose up -d postgres && make run
+# Or run API in Docker too: docker compose up --build
 version: '3.8'
 
 services:
+  postgres:
+    image: postgres:16-alpine
+    container_name: high-command-postgres
+    environment:
+      POSTGRES_USER: helldivers
+      POSTGRES_PASSWORD: helldivers
+      POSTGRES_DB: helldivers2
+    ports:
+      - "5432:5432"
+    volumes:
+      - high-command-pgdata:/var/lib/postgresql/data
+    healthcheck:
+      test: ["CMD-SHELL", "pg_isready -U helldivers -d helldivers2"]
+      interval: 5s
+      timeout: 5s
+      retries: 5
+    restart: unless-stopped
+
   api:
     build:
       context: .
       dockerfile: Dockerfile
     image: high-command-api:latest
-    # Alternative: Use Harbor image for production deployments
-    # image: harbor.dataknife.net/library/high-command-api:latest
     ports:
       - "5000:5000"
     environment:
+      - DATABASE_URL=postgresql://helldivers:helldivers@postgres:5432/helldivers2
       - FLASK_ENV=production
       - API_PORT=5000
       - SCRAPE_INTERVAL=300
-    volumes:
-      - ./helldivers2.db:/app/helldivers2.db
+    depends_on:
+      postgres:
+        condition: service_healthy
     restart: unless-stopped
     healthcheck:
       test: ["CMD", "curl", "-f", "http://localhost:5000/api/health"]
       interval: 30s
       timeout: 10s
       retries: 3
       start_period: 40s
+
+volumes:
+  high-command-pgdata:

src/collector.py

@@ -1,4 +1,5 @@
 import logging
+import threading
 from apscheduler.schedulers.background import BackgroundScheduler
 from src.scraper import HellDivers2Scraper
 from src.database import Database
@@ -17,7 +18,7 @@ def __init__(self, db: Database, interval: int = 300):
         self.is_running = False
 
     def start(self):
-        """Start the data collection scheduler"""
+        """Start the data collection scheduler and run an initial pull so MCP/UI have data immediately."""
         if self.is_running:
             logger.warning("Data collector is already running")
             return
@@ -29,6 +30,11 @@ def start(self):
         self.is_running = True
         logger.info(f"Data collector started with {self.interval}s interval")
 
+        # Initial pull so MCP and UI have data while the interval runs
+        thread = threading.Thread(target=self.collect_all_data, daemon=True)
+        thread.start()
+        logger.info("Initial data collection started in background")
+
     def stop(self):
         """Stop the data collection scheduler"""
         if not self.is_running:

src/database.py

@@ -50,10 +50,11 @@ def __init__(self, database_url: Optional[str] = None):
 
         self.database_url = database_url
         self._initialized = False
+        self._schema_initialized = False
         self._pool: Optional[pool.ThreadedConnectionPool] = None
 
     def _get_pool(self) -> pool.ThreadedConnectionPool:
-        """Get or create the connection pool (lazy init)"""
+        """Get or create the connection pool (lazy init). Ensures schema exists on first use."""
         if self._pool is None:
             import os
             maxconn = DEFAULT_POOL_MAX_CONN
@@ -68,6 +69,8 @@ def _get_pool(self) -> pool.ThreadedConnectionPool:
                 dsn=self.database_url,
             )
             logger.info("Database connection pool initialized")
+        if not self._schema_initialized and self._init_db():
+            self._schema_initialized = True
         return self._pool
 
     def _get_connection(self):
@@ -102,8 +105,8 @@ def _parse_expiration_time(expiration_time: str) -> Optional[datetime]:
         except (ValueError, AttributeError):
             return None
 
-    def _init_db(self):
-        """Initialize database schema (lazy - called on first use)"""
+    def _init_db(self) -> bool:
+        """Initialize database schema (lazy - called on first use). Returns True on success."""
         conn = None
         try:
             conn = self._get_connection()
@@ -232,13 +235,19 @@ def _init_db(self):
 
             conn.commit()
             conn.close()
-        except psycopg2.OperationalError:
-            # Database connection failed - this is OK during tests/imports
-            # Schema will be created when first actual operation happens
-            pass
-        except Exception:
-            # Any other error during init is also OK - will be handled on first use
-            pass
+            return True
+        except psycopg2.OperationalError as e:
+            logger.debug("Schema init skipped (connection): %s", e)
+            return False
+        except Exception as e:
+            logger.warning("Schema init failed: %s", e)
+            return False
+        finally:
+            if conn is not None:
+                try:
+                    conn.close()
+                except Exception:
+                    pass
 
     def save_war_status(self, data: Dict) -> bool:
         """Save war status to database"""