diff --git a/AGENT.md b/AGENT.md index 8cc715a3..4a256855 100644 --- a/AGENT.md +++ b/AGENT.md @@ -21,7 +21,7 @@ Developer reference for agents and contributors. User-facing overview: [README.m - `web/src/` -- React SPA (`AppRoutes.tsx`, `views/`, `components/`); pipeline UI: `PipelineRunnerFab`, `PipelineContext` - `alembic/` -- schema migrations -**Local dev:** `./local-run` (Postgres in Docker `wp-pg`, FileService `:8080`, Data `:8091`, AiService `:8092`, ReportService `:8094`, FastAPI `:8001`, BFF `:8090`, Vite `:3000`; default `DATABASE_URL`: `postgres://postgres:dev@127.0.0.1:5432/website_profiling`). See `scripts/local-run.sh`. **Local tests:** `./local-test` runs **three** Python coverage gates (core 100%, reporting 100%, tools 100%) plus web and .NET checks — mirrors CI; Docker CI is separate (see `.github/workflows/ci.yml`). `./local-test browser` for `@pytest.mark.browser` integration tests — see `scripts/local-test.sh`. Mocked browser unit tests: `tests/test_browser_fetcher_unit.py`. +**Local dev:** `./local-run` (Postgres in Docker `wp-pg`, FileService `:8097`, Data `:8091`, AiService `:8092`, ReportService `:8094`, FastAPI `:8096`, BFF `:8090`, Vite `:3000`; default `DATABASE_URL`: `postgres://postgres:dev@127.0.0.1:5432/website_profiling`). See `scripts/local-run.sh`. **Local tests:** `./local-test` runs **three** Python coverage gates (core 100%, reporting 100%, tools 100%) plus web and .NET checks — mirrors CI; Docker CI is separate (see `.github/workflows/ci.yml`). `./local-test browser` for `@pytest.mark.browser` integration tests — see `scripts/local-test.sh`. Mocked browser unit tests: `tests/test_browser_fetcher_unit.py`. **JavaScript crawl (optional):** Config keys `crawl_render_mode` (`static` | `javascript` | `auto`) and `crawl_js_*` in pipeline config / `pipelineConfigSchema.ts`. JS/auto crawls can capture browser console errors and uncaught exceptions (`crawl_js_capture_console`, stored under `page_analysis.browser`). **Auto mode** uses static-first fetch, pre-parse SPA heuristics (`needs_js_render`), then post-parse low-outlink fallback (`needs_js_render_after_parse`) in `crawler.py`. **Preflight:** `GET /api/crawl/browser-status` (localhost) spawns Python `browser_status()`; Run audit settings/run validation calls it when render mode is `javascript` or `auto`. Browser deps: Playwright from `requirements.txt` (installed by `./local-run setup` and `./local-test`). Runtime needs Chromium on `PATH` or `CHROME_PATH` (Docker sets `CHROME_PATH=/usr/bin/chromium`). Integration tests: `@pytest.mark.browser` — excluded by default in `pytest.ini`; Docker CI runs `tests/test_crawl_fetchers.py` and `tests/test_crawler_browser_e2e.py -m browser`; locally `./local-test browser`. diff --git a/AGENTS.md b/AGENTS.md index fb1f98c5..2385fa1b 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -18,7 +18,7 @@ This file is the canonical entry point for agents. For full detail see [AGENT.md - `services/ReportService/` — .NET report build + pipeline orchestration (port 8094). See [services/ReportService/README.md](services/ReportService/README.md) - `services/AiService/` — .NET AI service (Microsoft.Extensions.AI, chat, enrichment, MCP, **secrets/llm-settings writes**; port 8092). See [services/AiService/README.md](services/AiService/README.md) - `services/IntegrationsService/` — .NET Google/Bing integrations (GSC/GA4 fetch, OAuth, page-live, keyword reads; port 8093). See [services/IntegrationsService/README.md](services/IntegrationsService/README.md) -- `services/FileService/` — .NET PDF + Excel workbook export (port 8080). HTTP-only via `REPORT_API_URL`; no Postgres. Profiles: `executive|standard|full|premium`. Details: [services/FileService/README.md](services/FileService/README.md). Env: `FILE_SERVICE_URL` (MCP), `REPORT_API_URL` (FileService). +- `services/FileService/` — .NET PDF + Excel workbook export (port 8097). HTTP-only via `REPORT_API_URL`; no Postgres. Profiles: `executive|standard|full|premium`. Details: [services/FileService/README.md](services/FileService/README.md). Env: `FILE_SERVICE_URL` (MCP), `REPORT_API_URL` (FileService). - `alembic/` — DB migrations - `services/WebsiteProfiling.slnx` — unified .NET solution (all six services + shared libs) - `docs/` — documentation index diff --git a/Dockerfile b/Dockerfile index d2dc985d..5fcd75b8 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,5 +1,5 @@ # syntax=docker/dockerfile:1 -# WebsiteProfiling: FastAPI (port 8001) + Python worker + pipeline. +# WebsiteProfiling: FastAPI (port 8096) + Python worker + pipeline. # Web UI is a separate image: web/Dockerfile (Vite SPA + nginx). # Build from repository root: docker build -t website-profiling . # BuildKit cache mounts (default in Docker Desktop) reuse pip/npm downloads across rebuilds. @@ -72,6 +72,6 @@ ENV NODE_ENV=production # Persisted data directory (secrets + shadow config) RUN mkdir -p /data && chmod +x /app/docker-entrypoint.sh -EXPOSE 8001 +EXPOSE 8096 CMD ["/app/docker-entrypoint.sh"] diff --git a/README.md b/README.md index e5c61239..7948734f 100644 --- a/README.md +++ b/README.md @@ -149,12 +149,12 @@ flowchart TB subgraph BffRoutes["BFF upstream routing"] direction LR - BFF --> FastAPI["fastapi :8001
Crawl · jobs · pipeline settings"] + BFF --> FastAPI["fastapi :8096
Crawl · jobs · pipeline settings"] BFF --> Report["report :8094
Report build · orchestration"] BFF --> Integrations["integrations :8093
Google/Bing · GSC/GA4 · keywords"] BFF --> Ai["ai :8092
Chat · LLM · secrets · MCP"] BFF --> Data["data :8091
Report reads · portfolio · issues"] - BFF --> Files["files :8080
PDF · Excel export"] + BFF --> Files["files :8097
PDF · Excel export"] end subgraph Pipeline["Background pipeline"] @@ -208,7 +208,7 @@ flowchart TB ``` WebsiteProfiling/ ├── src/website_profiling/ # Python audit engine (CLI: python -m src) -│ ├── api/ # FastAPI app (uvicorn :8001) +│ ├── api/ # FastAPI app (uvicorn :8096) │ ├── worker/ # Background pipeline job runner │ ├── crawl/ # Crawler, fetchers, JS rendering │ ├── reporting/ # Report builder (Python bridge; native build in ReportService) @@ -233,7 +233,7 @@ WebsiteProfiling/ ├── services/IntegrationsService/ # .NET Google/Bing integrations — OAuth, GSC/GA4, keywords (port 8093) ├── services/AiService/ # .NET AI — chat, secrets, LLM config, MCP, enrichment (port 8092) ├── services/Data/ # .NET read service — report/portfolio/issue reads (port 8091) -├── services/FileService/ # .NET PDF + Excel workbook export (port 8080) +├── services/FileService/ # .NET PDF + Excel workbook export (port 8097) ├── alembic/versions/ # PostgreSQL schema migrations ├── tests/ # pytest suite + fixtures ├── docs/ # Glossary, MCP, ops, brand assets @@ -291,7 +291,7 @@ Build and run the full dev stack from source: docker compose up --build ``` -Services: **postgres**, **fastapi** (`:8001`, internal), **worker**, **report** (`:8094`, internal), **integrations** (`:8093`, internal), **ai** (`:8092`, internal), **data** (`:8091`, internal), **bff** (`:8090`), **web** (`:3000`), **files** (`:8080`, internal). +Services: **postgres**, **fastapi** (`:8096`, internal), **worker**, **report** (`:8094`, internal), **integrations** (`:8093`, internal), **ai** (`:8092`, internal), **data** (`:8091`, internal), **bff** (`:8090`), **web** (`:3000`), **files** (`:8097`, internal). Open [http://localhost:3000/home](http://localhost:3000/home). The browser talks only to the **BFF** (`:8090`); the BFF proxies to FastAPI (crawl/pipeline), IntegrationsService (Google/Bing), AiService (AI/secrets), Data (report reads), and FileService (PDF/workbook export). @@ -308,7 +308,7 @@ Production deployment: `docker-compose.prod.yml` — set `POSTGRES_USER`, `POSTG ./local-prod # Same DB, Vite production build + preview (no hot reload) ``` -`./local-run` starts (in order): **FileService** `:8080`, **Data** `:8091`, **AiService** `:8092` (MCP HTTP enabled), **ReportService** `:8094`, **IntegrationsService** `:8093`, **ConfigService** `:8095`, **FastAPI** `:8001` (Python bridge), **BFF** `:8090`, and **Vite** `:3000`. Use `localhost` (not `127.0.0.1`) for pipeline APIs so CORS and cookies match the BFF origin. +`./local-run` starts (in order): **FileService** `:8097`, **Data** `:8091`, **AiService** `:8092` (MCP HTTP enabled), **ReportService** `:8094`, **IntegrationsService** `:8093`, **ConfigService** `:8095`, **FastAPI** `:8096` (Python bridge), **BFF** `:8090`, and **Vite** `:3000`. Use `localhost` (not `127.0.0.1`) for pipeline APIs so CORS and cookies match the BFF origin. Default local `DATABASE_URL`: `postgres://postgres:dev@127.0.0.1:5432/website_profiling` (Docker Compose dev stack uses `profiling:profiling`). @@ -324,8 +324,8 @@ With `./local-run`, each service runs in **Development** and exposes interactive | **IntegrationsService** | 8093 | [http://localhost:8093/docs](http://localhost:8093/docs) | `/swagger/v1/swagger.json` | | **ReportService** | 8094 | [http://localhost:8094/docs](http://localhost:8094/docs) | `/swagger/v1/swagger.json` | | **ConfigService** | 8095 | [http://localhost:8095/docs](http://localhost:8095/docs) | `/swagger/v1/swagger.json` | -| **FileService** | 8080 | [http://localhost:8080/docs](http://localhost:8080/docs) | `/swagger/v1/swagger.json` | -| **Python bridge** (FastAPI) | 8001 | [http://localhost:8001/docs](http://localhost:8001/docs) | `/openapi.json` | +| **FileService** | 8097 | [http://localhost:8097/docs](http://localhost:8097/docs) | `/swagger/v1/swagger.json` | +| **Python bridge** (FastAPI) | 8096 | [http://localhost:8096/docs](http://localhost:8096/docs) | `/openapi.json` | Swagger UI is disabled when `ASPNETCORE_ENVIRONMENT=Production`. The legacy `web/openapi.json` reflects the old monolithic FastAPI app and is not a complete map of the current C# services. diff --git a/diagram.html b/diagram.html new file mode 100644 index 00000000..6c7bb005 --- /dev/null +++ b/diagram.html @@ -0,0 +1,619 @@ + + + + + + Interactive Architecture Canvas + + + + + + + +
+ + +
+
+ +
+ + + + + + + + + + + +
+
+
+ + +
+ + +
+ +
+ + + +
+ + + + \ No newline at end of file diff --git a/docker-compose.prod.yml b/docker-compose.prod.yml index b8c76630..dcb08397 100644 --- a/docker-compose.prod.yml +++ b/docker-compose.prod.yml @@ -40,7 +40,7 @@ services: volumes: - profiling-data:/data healthcheck: - test: ['CMD', 'node', '-e', "require('http').get('http://127.0.0.1:8001/api/health', (r) => process.exit(r.statusCode === 200 ? 0 : 1)).on('error', () => process.exit(1))"] + test: ['CMD', 'node', '-e', "require('http').get('http://127.0.0.1:8096/api/health', (r) => process.exit(r.statusCode === 200 ? 0 : 1)).on('error', () => process.exit(1))"] interval: 30s timeout: 5s retries: 3 @@ -71,7 +71,7 @@ services: dockerfile: AiService/Dockerfile environment: DATABASE_URL: postgres://${POSTGRES_USER}:${POSTGRES_PASSWORD}@postgres:5432/${POSTGRES_DB:-website_profiling} - FASTAPI_URL: http://fastapi:8001 + FASTAPI_URL: http://fastapi:8096 ASPNETCORE_URLS: http://+:8092 WP_MCP_HTTP: "1" depends_on: @@ -94,7 +94,7 @@ services: environment: DATABASE_URL: postgres://${POSTGRES_USER}:${POSTGRES_PASSWORD}@postgres:5432/${POSTGRES_DB:-website_profiling} ASPNETCORE_URLS: http://+:8093 - FASTAPI_URL: http://fastapi:8001 + FASTAPI_URL: http://fastapi:8096 USE_FASTAPI_PYTHON_BRIDGE: "1" ASPNETCORE_ENVIRONMENT: Production AUTH_SECRET: ${AUTH_SECRET:?set AUTH_SECRET} @@ -120,7 +120,7 @@ services: environment: DATABASE_URL: postgres://${POSTGRES_USER}:${POSTGRES_PASSWORD}@postgres:5432/${POSTGRES_DB:-website_profiling} ASPNETCORE_URLS: http://+:8094 - FASTAPI_URL: http://fastapi:8001 + FASTAPI_URL: http://fastapi:8096 INTEGRATIONS_SERVICE_URL: http://integrations:8093 AISERVICE_URL: http://ai:8092 REPORT_SERVICE_USE_PYTHON_BRIDGE: "0" @@ -147,7 +147,7 @@ services: dockerfile: FileService/Dockerfile environment: DATABASE_URL: postgres://${POSTGRES_USER}:${POSTGRES_PASSWORD}@postgres:5432/${POSTGRES_DB:-website_profiling} - REPORT_API_URL: http://fastapi:8001 + REPORT_API_URL: http://fastapi:8096 depends_on: postgres: condition: service_healthy @@ -174,8 +174,8 @@ services: ports: - '${BFF_PORT:-8090}:8090' environment: - FASTAPI_URL: http://fastapi:8001 - FILE_SERVICE_URL: http://files:8080 + FASTAPI_URL: http://fastapi:8096 + FILE_SERVICE_URL: http://files:8097 DATA_SERVICE_URL: http://data:8091 AI_SERVICE_URL: http://ai:8092 INTEGRATIONS_SERVICE_URL: http://integrations:8093 @@ -220,7 +220,7 @@ services: condition: service_healthy environment: DATABASE_URL: postgres://${POSTGRES_USER}:${POSTGRES_PASSWORD}@postgres:5432/${POSTGRES_DB:-website_profiling} - FASTAPI_URL: http://fastapi:8001 + FASTAPI_URL: http://fastapi:8096 ASPNETCORE_URLS: http://+:8092 WP_MCP_HTTP: "1" WP_MCP_DOMAIN: ${WP_MCP_DOMAIN:-core} diff --git a/docker-compose.pull.yml b/docker-compose.pull.yml index c258d7fe..b46b6057 100644 --- a/docker-compose.pull.yml +++ b/docker-compose.pull.yml @@ -42,7 +42,7 @@ services: volumes: - profiling-data:/data healthcheck: - test: ["CMD", "node", "-e", "require('http').get('http://127.0.0.1:8001/api/health', (r) => process.exit(r.statusCode === 200 ? 0 : 1)).on('error', () => process.exit(1))"] + test: ["CMD", "node", "-e", "require('http').get('http://127.0.0.1:8096/api/health', (r) => process.exit(r.statusCode === 200 ? 0 : 1)).on('error', () => process.exit(1))"] interval: 30s timeout: 5s retries: 3 @@ -73,7 +73,7 @@ services: dockerfile: AiService/Dockerfile environment: DATABASE_URL: postgres://profiling:profiling@postgres:5432/website_profiling - FASTAPI_URL: http://fastapi:8001 + FASTAPI_URL: http://fastapi:8096 ASPNETCORE_URLS: http://+:8092 depends_on: postgres: @@ -95,7 +95,7 @@ services: environment: DATABASE_URL: postgres://profiling:profiling@postgres:5432/website_profiling ASPNETCORE_URLS: http://+:8093 - FASTAPI_URL: http://fastapi:8001 + FASTAPI_URL: http://fastapi:8096 USE_FASTAPI_PYTHON_BRIDGE: "1" AUTH_SECRET: ${AUTH_SECRET:-} GOOGLE_REDIRECT_URI: ${GOOGLE_REDIRECT_URI:-http://localhost:8090/api/integrations/google/callback} @@ -120,7 +120,7 @@ services: environment: DATABASE_URL: postgres://profiling:profiling@postgres:5432/website_profiling ASPNETCORE_URLS: http://+:8094 - FASTAPI_URL: http://fastapi:8001 + FASTAPI_URL: http://fastapi:8096 INTEGRATIONS_SERVICE_URL: http://integrations:8093 REPORT_SERVICE_USE_PYTHON_BRIDGE: "0" USE_FASTAPI_PYTHON_BRIDGE: "1" @@ -146,7 +146,7 @@ services: dockerfile: FileService/Dockerfile environment: DATABASE_URL: postgres://profiling:profiling@postgres:5432/website_profiling - REPORT_API_URL: http://fastapi:8001 + REPORT_API_URL: http://fastapi:8096 depends_on: postgres: condition: service_healthy @@ -160,8 +160,8 @@ services: ports: - "8090:8090" environment: - FASTAPI_URL: http://fastapi:8001 - FILE_SERVICE_URL: http://files:8080 + FASTAPI_URL: http://fastapi:8096 + FILE_SERVICE_URL: http://files:8097 DATA_SERVICE_URL: http://data:8091 AI_SERVICE_URL: http://ai:8092 INTEGRATIONS_SERVICE_URL: http://integrations:8093 diff --git a/docker-compose.yml b/docker-compose.yml index 4cba43e6..0eb48aaf 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -41,7 +41,7 @@ services: volumes: - profiling-data:/data healthcheck: - test: ["CMD", "node", "-e", "require('http').get('http://127.0.0.1:8001/api/health', (r) => process.exit(r.statusCode === 200 ? 0 : 1)).on('error', () => process.exit(1))"] + test: ["CMD", "node", "-e", "require('http').get('http://127.0.0.1:8096/api/health', (r) => process.exit(r.statusCode === 200 ? 0 : 1)).on('error', () => process.exit(1))"] interval: 30s timeout: 5s retries: 3 @@ -74,7 +74,7 @@ services: dockerfile: AiService/Dockerfile environment: DATABASE_URL: postgres://profiling:profiling@postgres:5432/website_profiling - FASTAPI_URL: http://fastapi:8001 + FASTAPI_URL: http://fastapi:8096 ASPNETCORE_URLS: http://+:8092 WP_MCP_HTTP: "1" depends_on: @@ -97,7 +97,7 @@ services: environment: DATABASE_URL: postgres://profiling:profiling@postgres:5432/website_profiling ASPNETCORE_URLS: http://+:8093 - FASTAPI_URL: http://fastapi:8001 + FASTAPI_URL: http://fastapi:8096 USE_FASTAPI_PYTHON_BRIDGE: "1" AUTH_SECRET: ${AUTH_SECRET:-} GOOGLE_REDIRECT_URI: ${GOOGLE_REDIRECT_URI:-http://localhost:8090/api/integrations/google/callback} @@ -122,7 +122,7 @@ services: environment: DATABASE_URL: postgres://profiling:profiling@postgres:5432/website_profiling ASPNETCORE_URLS: http://+:8094 - FASTAPI_URL: http://fastapi:8001 + FASTAPI_URL: http://fastapi:8096 INTEGRATIONS_SERVICE_URL: http://integrations:8093 AISERVICE_URL: http://ai:8092 REPORT_SERVICE_USE_PYTHON_BRIDGE: "0" @@ -169,8 +169,8 @@ services: ports: - "8090:8090" environment: - FASTAPI_URL: http://fastapi:8001 - FILE_SERVICE_URL: http://files:8080 + FASTAPI_URL: http://fastapi:8096 + FILE_SERVICE_URL: http://files:8097 BFF_ALLOWED_ORIGINS: "http://localhost:3000" DATA_SERVICE_URL: http://data:8091 AI_SERVICE_URL: http://ai:8092 @@ -226,14 +226,14 @@ services: dockerfile: FileService/Dockerfile environment: DATABASE_URL: postgres://profiling:profiling@postgres:5432/website_profiling - REPORT_API_URL: http://fastapi:8001 + REPORT_API_URL: http://fastapi:8096 depends_on: postgres: condition: service_healthy fastapi: condition: service_started healthcheck: - test: ["CMD-SHELL", "curl -fsS http://127.0.0.1:8080/health || exit 1"] + test: ["CMD-SHELL", "curl -fsS http://127.0.0.1:8097/health || exit 1"] interval: 30s timeout: 5s retries: 3 diff --git a/docker-entrypoint.sh b/docker-entrypoint.sh index 6b93de80..c26b722a 100644 --- a/docker-entrypoint.sh +++ b/docker-entrypoint.sh @@ -76,7 +76,7 @@ migrate() { start_uvicorn_foreground() { exec /opt/venv/bin/uvicorn website_profiling.api.main:app \ - --host 0.0.0.0 --port 8001 --workers 1 + --host 0.0.0.0 --port 8096 --workers 1 } case "$ROLE" in @@ -109,7 +109,7 @@ case "$ROLE" in WORKER_PID=$! /opt/venv/bin/uvicorn website_profiling.api.main:app \ - --host 0.0.0.0 --port 8001 --workers 1 & + --host 0.0.0.0 --port 8096 --workers 1 & UVICORN_PID=$! while kill -0 "$UVICORN_PID" 2>/dev/null; do diff --git a/docs/MCP.md b/docs/MCP.md index 5e79c553..4f259755 100644 --- a/docs/MCP.md +++ b/docs/MCP.md @@ -28,12 +28,12 @@ The same tool catalog powers in-app **AI Chat** at `/chat` (also served by AiSer ## Prerequisites -[.NET SDK 10+](https://dotnet.microsoft.com/download), Postgres, and `./local-run` (or AiService + FastAPI manually). AiService needs FastAPI on `:8001` for the audit-tool bridge until all tools are native C#. +[.NET SDK 10+](https://dotnet.microsoft.com/download), Postgres, and `./local-run` (or AiService + FastAPI manually). AiService needs FastAPI on `:8096` for the audit-tool bridge until all tools are native C#. ```bash export DATABASE_URL=postgres://profiling:profiling@localhost:5432/website_profiling # Docker default # ./local-run default: postgres://postgres:dev@127.0.0.1:5432/website_profiling -export FASTAPI_URL=http://127.0.0.1:8001 +export FASTAPI_URL=http://127.0.0.1:8096 ``` ### Local stdio (IDE subprocess) @@ -43,7 +43,7 @@ From the repo root, with Postgres running: ```bash cd services/AiService export DATABASE_URL=postgres://postgres:dev@127.0.0.1:5432/website_profiling -export FASTAPI_URL=http://127.0.0.1:8001 +export FASTAPI_URL=http://127.0.0.1:8096 dotnet run --project src/AiService.Api ``` @@ -85,7 +85,7 @@ Add to `.cursor/mcp.json` or your MCP client settings (stdio — spawn AiService "args": ["run", "--project", "services/AiService/src/AiService.Api", "--no-launch-profile"], "env": { "DATABASE_URL": "postgres://postgres:dev@127.0.0.1:5432/website_profiling", - "FASTAPI_URL": "http://127.0.0.1:8001", + "FASTAPI_URL": "http://127.0.0.1:8096", "WP_MCP_DOMAIN": "core", "WP_PROPERTY_ID": "1" } @@ -95,7 +95,7 @@ Add to `.cursor/mcp.json` or your MCP client settings (stdio — spawn AiService "args": ["run", "--project", "services/AiService/src/AiService.Api", "--no-launch-profile"], "env": { "DATABASE_URL": "postgres://postgres:dev@127.0.0.1:5432/website_profiling", - "FASTAPI_URL": "http://127.0.0.1:8001", + "FASTAPI_URL": "http://127.0.0.1:8096", "WP_MCP_DOMAIN": "google", "WP_PROPERTY_ID": "1" } @@ -114,7 +114,7 @@ Add to `.cursor/mcp.json` or your MCP client settings (stdio — spawn AiService "args": ["run", "--project", "services/AiService/src/AiService.Api", "--no-launch-profile"], "env": { "DATABASE_URL": "postgres://postgres:dev@127.0.0.1:5432/website_profiling", - "FASTAPI_URL": "http://127.0.0.1:8001", + "FASTAPI_URL": "http://127.0.0.1:8096", "WP_MCP_DOMAIN": "full", "WP_PROPERTY_ID": "1" } @@ -135,7 +135,7 @@ Configure access on **MCP settings** (`/mcp`) in the web UI (recommended), or se ```bash export DATABASE_URL=postgres://profiling:profiling@localhost:5432/website_profiling -export FASTAPI_URL=http://127.0.0.1:8001 +export FASTAPI_URL=http://127.0.0.1:8096 export ASPNETCORE_URLS=http://0.0.0.0:8092 export WP_MCP_HTTP=1 export WP_MCP_DOMAIN=core @@ -227,7 +227,7 @@ Export tools write artifact files with a 24-hour TTL; in-app chat renders downlo | Deliverable | Generator | Env / notes | |-------------|-----------|-------------| -| PDF | **FileService** (`GET /v1/reports/{id}/pdf`) | `FILE_SERVICE_URL` on MCP/web (default `http://127.0.0.1:8080`); FileService must be running | +| PDF | **FileService** (`GET /v1/reports/{id}/pdf`) | `FILE_SERVICE_URL` on MCP/web (default `http://127.0.0.1:8097`); FileService must be running | | Excel workbook | **FileService** (`GET /v1/reports/{id}/workbook`) | Same as PDF | | CSV / JSON audit export | Python (`export_audit_report`, `GET /api/report/export`) | Reads Postgres via report payload — no FileService required | | Compare / list CSV | Python | `export_compare_csv`, `export_list_as_csv` | diff --git a/docs/OPS.md b/docs/OPS.md index 2411e258..a35089e9 100644 --- a/docs/OPS.md +++ b/docs/OPS.md @@ -261,12 +261,12 @@ Do not run the application container in isolation with `docker run` unless you p ### FileService (PDF and workbook export) -The `files` service (port **8080**) renders audit PDFs and Excel workbooks. It reads report data over HTTP from the `web` service — no Postgres connection. +The `files` service (port **8097**) renders audit PDFs and Excel workbooks. It reads report data over HTTP from the `web` service — no Postgres connection. | Variable | Service | Purpose | |----------|---------|---------| -| `FILE_SERVICE_URL` | `web`, MCP | Where clients call FileService (default `http://files:8080` in Compose) | -| `REPORT_API_URL` | `files` | Report API base URL (Compose: `http://web:8001`) | +| `FILE_SERVICE_URL` | `web`, MCP | Where clients call FileService (default `http://files:8097` in Compose) | +| `REPORT_API_URL` | `files` | Report API base URL (Compose: `http://web:8096`) | PDF or workbook downloads fail if `files` is not running. See [services/FileService/README.md](../services/FileService/README.md). diff --git a/docs/README.md b/docs/README.md index 5cc4b272..b5b64d08 100644 --- a/docs/README.md +++ b/docs/README.md @@ -29,12 +29,12 @@ All `/api/*` calls from the SPA go to the **BFF** (`:8090`). The BFF forwards su | Upstream | Examples | |----------|----------| -| **FastAPI** (`:8001`) | `/api/run`, `/api/jobs/*`, `/api/pipeline-config`, `/api/pipeline-settings`, `/api/ui-preferences`, crawl, properties | +| **FastAPI** (`:8096`) | `/api/run`, `/api/jobs/*`, `/api/pipeline-config`, `/api/pipeline-settings`, `/api/ui-preferences`, crawl, properties | | **ReportService** (`:8094`, internal) | Report build and full-audit orchestration (worker; not browser-facing) | | **IntegrationsService** (`:8093`) | `/api/integrations/google/*`, `/api/integrations/bing/*`, property Google config | | **AiService** (`:8092`) | `/api/chat`, `/api/secrets`, `/api/llm-settings`, MCP-related APIs | | **Data** (`:8091`) | Report payload reads, portfolio, issue status, saved filters | -| **FileService** (`:8080`) | PDF and Excel export | +| **FileService** (`:8097`) | PDF and Excel export | **Internal service-to-service:** ReportService reads Google/keyword snapshots from IntegrationsService (`GET /internal/integrations/report/enrichment`) during native report build — not via the BFF. @@ -52,8 +52,8 @@ During local development (`./local-run`, `ASPNETCORE_ENVIRONMENT=Development`), | IntegrationsService | 8093 | [http://localhost:8093/docs](http://localhost:8093/docs) | | ReportService | 8094 | [http://localhost:8094/docs](http://localhost:8094/docs) | | ConfigService | 8095 | [http://localhost:8095/docs](http://localhost:8095/docs) | -| FileService | 8080 | [http://localhost:8080/docs](http://localhost:8080/docs) | -| Python bridge | 8001 | [http://localhost:8001/docs](http://localhost:8001/docs) | +| FileService | 8097 | [http://localhost:8097/docs](http://localhost:8097/docs) | +| Python bridge | 8096 | [http://localhost:8096/docs](http://localhost:8096/docs) | OpenAPI JSON for .NET services: `/swagger/v1/swagger.json`. See the main [README.md](../README.md#api-documentation-swagger) for details. diff --git a/scripts/local-prod.sh b/scripts/local-prod.sh index a408442e..2abbe8dd 100755 --- a/scripts/local-prod.sh +++ b/scripts/local-prod.sh @@ -165,7 +165,7 @@ cmd_start() { stop_service "Vite preview" "$NPM_PID" NPM_PID="" stop_host_dotnet_stack stop_service - stop_service "FastAPI" "$UVICORN_PID" 8001 + stop_service "FastAPI" "$UVICORN_PID" 8096 UVICORN_PID="" stop_service "pipeline worker" "$WORKER_PID" WORKER_PID="" @@ -184,21 +184,21 @@ cmd_start() { export AI_SERVICE_URL="${AI_SERVICE_URL:-http://127.0.0.1:8092}" export INTEGRATIONS_SERVICE_URL="${INTEGRATIONS_SERVICE_URL:-http://127.0.0.1:8093}" - export FILE_SERVICE_URL="${FILE_SERVICE_URL:-http://127.0.0.1:8080}" + export FILE_SERVICE_URL="${FILE_SERVICE_URL:-http://127.0.0.1:8097}" export REPORT_SERVICE_URL="${REPORT_SERVICE_URL:-http://127.0.0.1:8094}" export PIPELINE_ORCHESTRATE_VIA_REPORT_SERVICE="${PIPELINE_ORCHESTRATE_VIA_REPORT_SERVICE:-1}" export PYTHON="${PYTHON:-$ROOT/.venv/bin/python}" export WEBSITE_PROFILING_ROOT="$ROOT" log "Pipeline jobs run in ReportService C# worker" - log "Starting Python bridge (audit-tool + keyword enrich CLI) on port 8001" - export FASTAPI_URL="http://127.0.0.1:8001" + log "Starting Python bridge (audit-tool + keyword enrich CLI) on port 8096" + export FASTAPI_URL="http://127.0.0.1:8096" export FASTAPI_ALLOWED_ORIGINS="http://localhost:8090" "$ROOT/.venv/bin/uvicorn" website_profiling.api.main:app \ - --host 0.0.0.0 --port 8001 --workers 1 & + --host 0.0.0.0 --port 8096 --workers 1 & UVICORN_PID=$! disown_bg "$UVICORN_PID" - wait_for_http "http://127.0.0.1:8001/api/health" "FastAPI" 90 || die "FastAPI failed to start" + wait_for_http "http://127.0.0.1:8096/api/health" "FastAPI" 90 || die "FastAPI failed to start" start_host_integrations_bff "$ROOT" Production disown_bg "$INTEGRATIONS_PID" diff --git a/scripts/local-run-common.sh b/scripts/local-run-common.sh index c2d8abbf..397e9c39 100644 --- a/scripts/local-run-common.sh +++ b/scripts/local-run-common.sh @@ -32,16 +32,16 @@ start_host_dotnet_base() { local root="$1" local mode="${2:-Development}" - export REPORT_API_URL="${REPORT_API_URL:-http://127.0.0.1:8001}" + export REPORT_API_URL="${REPORT_API_URL:-http://127.0.0.1:8096}" export AI_SERVICE_URL="${AI_SERVICE_URL:-http://127.0.0.1:8092}" export INTEGRATIONS_SERVICE_URL="${INTEGRATIONS_SERVICE_URL:-http://127.0.0.1:8093}" export REPORT_SERVICE_URL="${REPORT_SERVICE_URL:-http://127.0.0.1:8094}" - export FILE_SERVICE_URL="${FILE_SERVICE_URL:-http://127.0.0.1:8080}" + export FILE_SERVICE_URL="${FILE_SERVICE_URL:-http://127.0.0.1:8097}" - free_port 8080 - printf '\033[1;36m→\033[0m Starting FileService on port 8080\n' + free_port 8097 + printf '\033[1;36m→\033[0m Starting FileService on port 8097\n' (cd "$root/services/FileService" && \ - ASPNETCORE_URLS="http://127.0.0.1:8080" \ + ASPNETCORE_URLS="http://127.0.0.1:8097" \ ASPNETCORE_ENVIRONMENT="$mode" \ dotnet run --project src/FileService.Api --no-launch-profile) & FILE_SERVICE_PID=$! @@ -69,14 +69,14 @@ start_host_dotnet_base() { printf '\033[1;36m→\033[0m Starting AiService on port 8092\n' (cd "$root/services/AiService" && \ DATABASE_URL="$DATABASE_URL" \ - FASTAPI_URL="http://127.0.0.1:8001" \ + FASTAPI_URL="http://127.0.0.1:8096" \ ASPNETCORE_URLS="http://127.0.0.1:8092" \ ASPNETCORE_ENVIRONMENT="$mode" \ WP_MCP_HTTP=1 \ dotnet run --project src/AiService.Api --no-launch-profile) & AI_PID=$! - wait_for_http "http://127.0.0.1:8080/health" "FileService" + wait_for_http "http://127.0.0.1:8097/health" "FileService" wait_for_http "http://127.0.0.1:8091/health" "Data service" wait_for_http "http://127.0.0.1:8095/health" "ConfigService" wait_for_http "http://127.0.0.1:8092/health" "AiService" @@ -90,7 +90,7 @@ start_host_report_service() { printf '\033[1;36m→\033[0m Starting ReportService on port 8094\n' (cd "$root/services/ReportService" && \ DATABASE_URL="$DATABASE_URL" \ - FASTAPI_URL="http://127.0.0.1:8001" \ + FASTAPI_URL="http://127.0.0.1:8096" \ INTEGRATIONS_SERVICE_URL="$INTEGRATIONS_SERVICE_URL" \ AISERVICE_URL="${AISERVICE_URL:-${AI_SERVICE_URL:-http://127.0.0.1:8092}}" \ WEBSITE_PROFILING_ROOT="$root" \ @@ -114,7 +114,7 @@ start_host_integrations_bff() { printf '\033[1;36m→\033[0m Starting IntegrationsService on port 8093\n' (cd "$root/services/IntegrationsService" && \ DATABASE_URL="$DATABASE_URL" \ - FASTAPI_URL="http://127.0.0.1:8001" \ + FASTAPI_URL="http://127.0.0.1:8096" \ USE_FASTAPI_PYTHON_BRIDGE="${USE_FASTAPI_PYTHON_BRIDGE:-1}" \ ASPNETCORE_URLS="http://127.0.0.1:8093" \ ASPNETCORE_ENVIRONMENT="$mode" \ @@ -129,7 +129,7 @@ start_host_integrations_bff() { free_port 8090 printf '\033[1;36m→\033[0m Starting BFF on port 8090\n' (cd "$root/services/Bff" && \ - FASTAPI_URL="http://127.0.0.1:8001" \ + FASTAPI_URL="http://127.0.0.1:8096" \ FILE_SERVICE_URL="$FILE_SERVICE_URL" \ DATA_SERVICE_URL="http://127.0.0.1:8091" \ AI_SERVICE_URL="$AI_SERVICE_URL" \ @@ -175,6 +175,6 @@ stop_host_dotnet_stack() { CONFIG_PID="" "$stop_service_fn" "Data" "${DATA_PID:-}" 8091 DATA_PID="" - "$stop_service_fn" "FileService" "${FILE_SERVICE_PID:-}" 8080 + "$stop_service_fn" "FileService" "${FILE_SERVICE_PID:-}" 8097 FILE_SERVICE_PID="" } diff --git a/scripts/local-run.ps1 b/scripts/local-run.ps1 index cc6538bc..4ba40ff8 100644 --- a/scripts/local-run.ps1 +++ b/scripts/local-run.ps1 @@ -256,11 +256,11 @@ function Invoke-Start { Invoke-WebDeps $bffBase = if ($env:VITE_BFF_BASE_URL) { $env:VITE_BFF_BASE_URL } else { "http://localhost:8090" } - $fileServiceUrl = if ($env:FILE_SERVICE_URL) { $env:FILE_SERVICE_URL } else { "http://127.0.0.1:8080" } + $fileServiceUrl = if ($env:FILE_SERVICE_URL) { $env:FILE_SERVICE_URL } else { "http://127.0.0.1:8097" } if (Get-Command dotnet -ErrorAction SilentlyContinue) { - Write-Log "Starting FileService on port 8080" - $env:REPORT_API_URL = "http://127.0.0.1:8001" + Write-Log "Starting FileService on port 8097" + $env:REPORT_API_URL = "http://127.0.0.1:8096" Start-Process -FilePath "dotnet" ` -ArgumentList @("run", "--project", "src/FileService.Api", "--no-launch-profile") ` -WorkingDirectory (Join-Path $ROOT "services/FileService") ` @@ -270,18 +270,18 @@ function Invoke-Start { } Write-Log "Pipeline jobs run in ReportService C# worker" - Write-Log "Starting Python bridge on port 8001" - $env:FASTAPI_URL = "http://127.0.0.1:8001" + Write-Log "Starting Python bridge on port 8096" + $env:FASTAPI_URL = "http://127.0.0.1:8096" $env:FASTAPI_ALLOWED_ORIGINS = "http://localhost:8090" Start-Process -FilePath $VENV_PYTHON ` - -ArgumentList @("-m", "uvicorn", "website_profiling.api.main:app", "--host", "0.0.0.0", "--port", "8001", "--workers", "1") ` + -ArgumentList @("-m", "uvicorn", "website_profiling.api.main:app", "--host", "0.0.0.0", "--port", "8096", "--workers", "1") ` -WorkingDirectory $ROOT ` -WindowStyle Minimized | Out-Null if (Get-Command dotnet -ErrorAction SilentlyContinue) { Write-Log "Starting BFF on port 8090" $bffDir = Join-Path $ROOT "services/Bff" - $env:FASTAPI_URL = "http://127.0.0.1:8001" + $env:FASTAPI_URL = "http://127.0.0.1:8096" $env:FILE_SERVICE_URL = $fileServiceUrl $env:BFF_ALLOWED_ORIGINS = "http://localhost:3000" $env:ASPNETCORE_URLS = "http://127.0.0.1:8090" diff --git a/scripts/local-run.sh b/scripts/local-run.sh index 5591e75e..587e9414 100755 --- a/scripts/local-run.sh +++ b/scripts/local-run.sh @@ -234,7 +234,7 @@ cmd_start() { if command -v dotnet >/dev/null 2>&1; then stop_host_dotnet_stack stop_service fi - stop_service "FastAPI" "$UVICORN_PID" 8001 + stop_service "FastAPI" "$UVICORN_PID" 8096 UVICORN_PID="" stop_service "pipeline worker" "$WORKER_PID" WORKER_PID="" @@ -261,23 +261,23 @@ cmd_start() { export AI_SERVICE_URL="${AI_SERVICE_URL:-http://127.0.0.1:8092}" export INTEGRATIONS_SERVICE_URL="${INTEGRATIONS_SERVICE_URL:-http://127.0.0.1:8093}" - export FILE_SERVICE_URL="${FILE_SERVICE_URL:-http://127.0.0.1:8080}" + export FILE_SERVICE_URL="${FILE_SERVICE_URL:-http://127.0.0.1:8097}" export REPORT_SERVICE_URL="${REPORT_SERVICE_URL:-http://127.0.0.1:8094}" export PIPELINE_ORCHESTRATE_VIA_REPORT_SERVICE="${PIPELINE_ORCHESTRATE_VIA_REPORT_SERVICE:-1}" export PYTHON="${PYTHON:-$VENV/bin/python}" export WEBSITE_PROFILING_ROOT="$ROOT" log "Pipeline jobs run in ReportService C# worker (Python subprocess for crawl/lighthouse only)" - log "Starting Python bridge (audit-tool + keyword enrich CLI only) on port 8001" - export FASTAPI_URL="http://127.0.0.1:8001" + log "Starting Python bridge (audit-tool + keyword enrich CLI only) on port 8096" + export FASTAPI_URL="http://127.0.0.1:8096" export FASTAPI_ALLOWED_ORIGINS="http://localhost:8090" export DEPRECATE_PYTHON_INTEGRATIONS="${DEPRECATE_PYTHON_INTEGRATIONS:-1}" export DEPRECATE_PYTHON_REPORT_ROUTES="${DEPRECATE_PYTHON_REPORT_ROUTES:-1}" "$VENV/bin/uvicorn" website_profiling.api.main:app \ - --host 0.0.0.0 --port 8001 --workers 1 & + --host 0.0.0.0 --port 8096 --workers 1 & UVICORN_PID=$! disown_bg "$UVICORN_PID" - wait_for_http_logged "http://127.0.0.1:8001/api/health" "FastAPI" + wait_for_http_logged "http://127.0.0.1:8096/api/health" "FastAPI" if command -v dotnet >/dev/null 2>&1; then start_host_integrations_bff "$ROOT" Development @@ -292,10 +292,10 @@ cmd_start() { log "DATA_DIR=$DATA_DIR" log "PYTHON=$PYTHON" log "VITE_BFF_BASE_URL=${VITE_BFF_BASE_URL:-http://localhost:8090}" - log "FILE_SERVICE_URL=${FILE_SERVICE_URL:-http://127.0.0.1:8080}" + log "FILE_SERVICE_URL=${FILE_SERVICE_URL:-http://127.0.0.1:8097}" log "REPORT_SERVICE_URL=${REPORT_SERVICE_URL:-http://127.0.0.1:8094}" log "DATA_ROUTES=${DATA_ROUTES:-/api/report/meta,...}" - export FILE_SERVICE_URL="${FILE_SERVICE_URL:-http://127.0.0.1:8080}" + export FILE_SERVICE_URL="${FILE_SERVICE_URL:-http://127.0.0.1:8097}" export VITE_BFF_BASE_URL="${VITE_BFF_BASE_URL:-http://localhost:8090}" cd "$WEB" set +e diff --git a/services/AiService/README.md b/services/AiService/README.md index 03b4bfa9..67e26f7f 100644 --- a/services/AiService/README.md +++ b/services/AiService/README.md @@ -6,12 +6,12 @@ Python FastAPI retains crawl, pipeline jobs, and integrations OAuth reads. AiSer ## Run locally -Prerequisites: [.NET SDK 10+](https://dotnet.microsoft.com/download), Postgres, FastAPI on port 8001 (audit-tool bridge for tools not yet native in C#). +Prerequisites: [.NET SDK 10+](https://dotnet.microsoft.com/download), Postgres, FastAPI on port 8096 (audit-tool bridge for tools not yet native in C#). ```bash cd services/AiService export DATABASE_URL=postgres://postgres:dev@127.0.0.1:5432/website_profiling -export FASTAPI_URL=http://127.0.0.1:8001 +export FASTAPI_URL=http://127.0.0.1:8096 dotnet run --project src/AiService.Api ``` @@ -60,7 +60,7 @@ Pipeline domain settings (`crawl_settings`, `report_settings`, …) are written | Variable | Default | Description | |----------|---------|-------------| | `DATABASE_URL` | — | Postgres (`llm_settings`, `llm_cache`, `chat_sessions`, typed pipeline tables, `google_app_settings`, `report_payload`) | -| `FASTAPI_URL` | `http://127.0.0.1:8001` | Python audit-tool bridge for unported tools | +| `FASTAPI_URL` | `http://127.0.0.1:8096` | Python audit-tool bridge for unported tools | | `AI_SERVICE_URL` | — | Used by Python worker (`ai_service_client.py`) | | `ASPNETCORE_URLS` | `http://+:8092` | Bind address | | `WP_MCP_HTTP` | — | Set `1` to expose MCP at `/mcp` | diff --git a/services/AiService/src/AiService.Api/appsettings.json b/services/AiService/src/AiService.Api/appsettings.json index ed5a72cd..8d852d66 100644 --- a/services/AiService/src/AiService.Api/appsettings.json +++ b/services/AiService/src/AiService.Api/appsettings.json @@ -7,7 +7,7 @@ "CommandTimeoutSeconds": 30 }, "FastApi": { - "BaseUrl": "http://127.0.0.1:8001" + "BaseUrl": "http://127.0.0.1:8096" }, "Logging": { "LogLevel": { diff --git a/services/AiService/src/AiService.Tools/Options/FastApiOptions.cs b/services/AiService/src/AiService.Tools/Options/FastApiOptions.cs index 265e509d..9872ba84 100644 --- a/services/AiService/src/AiService.Tools/Options/FastApiOptions.cs +++ b/services/AiService/src/AiService.Tools/Options/FastApiOptions.cs @@ -9,5 +9,5 @@ public sealed class FastApiOptions public const string SectionName = "FastApi"; /// FastAPI base URL. Default matches local compose / BFF upstream. - public string BaseUrl { get; set; } = "http://127.0.0.1:8001"; + public string BaseUrl { get; set; } = "http://127.0.0.1:8096"; } diff --git a/services/Bff/src/Bff.Api/appsettings.json b/services/Bff/src/Bff.Api/appsettings.json index 5f86f66b..b8f40616 100644 --- a/services/Bff/src/Bff.Api/appsettings.json +++ b/services/Bff/src/Bff.Api/appsettings.json @@ -1,8 +1,8 @@ { "Urls": "http://+:8090", "Upstream": { - "FastApiBaseUrl": "http://127.0.0.1:8001", - "FileServiceBaseUrl": "http://127.0.0.1:8080", + "FastApiBaseUrl": "http://127.0.0.1:8096", + "FileServiceBaseUrl": "http://127.0.0.1:8097", "TimeoutSeconds": 120 }, "Auth": { diff --git a/services/Bff/src/Bff.Application/Options/UpstreamOptions.cs b/services/Bff/src/Bff.Application/Options/UpstreamOptions.cs index d1bc527c..54c91f31 100644 --- a/services/Bff/src/Bff.Application/Options/UpstreamOptions.cs +++ b/services/Bff/src/Bff.Application/Options/UpstreamOptions.cs @@ -9,10 +9,10 @@ public sealed class UpstreamOptions public const string SectionName = "Upstream"; /// FastAPI base URL (env override: FASTAPI_URL). Default matches the new internal compose service. - public string FastApiBaseUrl { get; set; } = "http://127.0.0.1:8001"; + public string FastApiBaseUrl { get; set; } = "http://127.0.0.1:8096"; /// FileService base URL (env override: FILE_SERVICE_URL). - public string FileServiceBaseUrl { get; set; } = "http://127.0.0.1:8080"; + public string FileServiceBaseUrl { get; set; } = "http://127.0.0.1:8097"; /// Timeout for normal (non-streaming) upstream calls. Parity with the TS proxy (120s). public int TimeoutSeconds { get; set; } = 120; diff --git a/services/Bff/tests/Bff.Tests/ServiceRegistrationValidationTests.cs b/services/Bff/tests/Bff.Tests/ServiceRegistrationValidationTests.cs index 38e27553..12182b2a 100644 --- a/services/Bff/tests/Bff.Tests/ServiceRegistrationValidationTests.cs +++ b/services/Bff/tests/Bff.Tests/ServiceRegistrationValidationTests.cs @@ -13,7 +13,7 @@ public void Web_host_resolves_core_services() { using var env = ServiceRegistrationTestEnvironment.Push(); env.SetDefaultsForPostgresServices(); - env.Set("FILE_SERVICE_URL", "http://127.0.0.1:8080"); + env.Set("FILE_SERVICE_URL", "http://127.0.0.1:8097"); env.Set("DATA_SERVICE_URL", "http://127.0.0.1:8091"); env.Set("AI_SERVICE_URL", "http://127.0.0.1:8092"); env.Set("INTEGRATIONS_SERVICE_URL", "http://127.0.0.1:8093"); diff --git a/services/FileService/Dockerfile b/services/FileService/Dockerfile index c09f2268..b23c0143 100644 --- a/services/FileService/Dockerfile +++ b/services/FileService/Dockerfile @@ -12,7 +12,7 @@ RUN dotnet publish FileService/src/FileService.Api/FileService.Api.csproj -c Rel FROM mcr.microsoft.com/dotnet/aspnet:10.0 AS runtime WORKDIR /app -ENV ASPNETCORE_URLS=http://+:8080 -EXPOSE 8080 +ENV ASPNETCORE_URLS=http://+:8097 +EXPOSE 8097 COPY --from=build /app/publish . ENTRYPOINT ["dotnet", "FileService.Api.dll"] diff --git a/services/FileService/README.md b/services/FileService/README.md index d2d84dc2..e25a200c 100644 --- a/services/FileService/README.md +++ b/services/FileService/README.md @@ -6,24 +6,24 @@ Python owns crawl data and CSV/JSON exports. FileService renders PDF and Excel w ## Run locally -Prerequisites: [.NET SDK 10+](https://dotnet.microsoft.com/download), Site Audit report API on port 8001 (started by `./local-run` or Docker `web` service). +Prerequisites: [.NET SDK 10+](https://dotnet.microsoft.com/download), Site Audit report API on port 8096 (started by `./local-run` or Docker `web` service). ```bash cd services/FileService -export REPORT_API_URL=http://127.0.0.1:8001 +export REPORT_API_URL=http://127.0.0.1:8096 dotnet run --project src/FileService.Api ``` -Service listens on **http://localhost:8080** (`ASPNETCORE_URLS` / `appsettings.json`). +Service listens on **http://localhost:8097** (`ASPNETCORE_URLS` / `appsettings.json`). -In **Development**, Swagger UI is at **http://localhost:8080/docs** and the OpenAPI JSON at **http://localhost:8080/swagger/v1/swagger.json**. +In **Development**, Swagger UI is at **http://localhost:8097/docs** and the OpenAPI JSON at **http://localhost:8097/swagger/v1/swagger.json**. ## Environment variables | Variable | Default | Description | |----------|---------|-------------| -| `REPORT_API_URL` | `http://127.0.0.1:8001` | Base URL for report payload, meta, and ui-preferences HTTP API | -| `ASPNETCORE_URLS` | `http://127.0.0.1:8080` | Bind address (Docker sets `http://+:8080`) | +| `REPORT_API_URL` | `http://127.0.0.1:8096` | Base URL for report payload, meta, and ui-preferences HTTP API | +| `ASPNETCORE_URLS` | `http://127.0.0.1:8097` | Bind address (Docker sets `http://+:8097`) | ## Upstream HTTP contract diff --git a/services/FileService/src/FileService.Api/Properties/launchSettings.json b/services/FileService/src/FileService.Api/Properties/launchSettings.json index f90e2975..4bf9048a 100644 --- a/services/FileService/src/FileService.Api/Properties/launchSettings.json +++ b/services/FileService/src/FileService.Api/Properties/launchSettings.json @@ -6,7 +6,7 @@ "dotnetRunMessages": true, "launchBrowser": true, "launchUrl": "docs", - "applicationUrl": "http://localhost:8080", + "applicationUrl": "http://localhost:8097", "environmentVariables": { "ASPNETCORE_ENVIRONMENT": "Development" } diff --git a/services/FileService/src/FileService.Api/appsettings.json b/services/FileService/src/FileService.Api/appsettings.json index 85bfa513..33fa3663 100644 --- a/services/FileService/src/FileService.Api/appsettings.json +++ b/services/FileService/src/FileService.Api/appsettings.json @@ -1,7 +1,7 @@ { - "Urls": "http://+:8080", + "Urls": "http://+:8097", "ReportApi": { - "BaseUrl": "http://127.0.0.1:8001", + "BaseUrl": "http://127.0.0.1:8096", "TimeoutSeconds": 120 }, "Logging": { diff --git a/services/FileService/src/FileService.Application/Options/ReportApiOptions.cs b/services/FileService/src/FileService.Application/Options/ReportApiOptions.cs index 9102692b..32a095f5 100644 --- a/services/FileService/src/FileService.Application/Options/ReportApiOptions.cs +++ b/services/FileService/src/FileService.Application/Options/ReportApiOptions.cs @@ -9,6 +9,6 @@ public sealed class ReportApiOptions { public const string SectionName = "ReportApi"; - public string BaseUrl { get; set; } = "http://127.0.0.1:8001"; + public string BaseUrl { get; set; } = "http://127.0.0.1:8096"; public int TimeoutSeconds { get; set; } = 120; } diff --git a/services/FileService/tests/FileService.Tests/ServiceRegistrationValidationTests.cs b/services/FileService/tests/FileService.Tests/ServiceRegistrationValidationTests.cs index 8cd77d1f..63ec1f10 100644 --- a/services/FileService/tests/FileService.Tests/ServiceRegistrationValidationTests.cs +++ b/services/FileService/tests/FileService.Tests/ServiceRegistrationValidationTests.cs @@ -14,7 +14,7 @@ public void Web_host_resolves_core_services() { using var env = ServiceRegistrationTestEnvironment.Push(); env.SetDefaultsForPostgresServices(); - env.Set("REPORT_API_URL", "http://127.0.0.1:8001"); + env.Set("REPORT_API_URL", "http://127.0.0.1:8096"); using var factory = new WebApplicationFactory().WithWebHostBuilder(builder => { diff --git a/services/IntegrationsService/README.md b/services/IntegrationsService/README.md index 7893093f..59fa6485 100644 --- a/services/IntegrationsService/README.md +++ b/services/IntegrationsService/README.md @@ -11,7 +11,7 @@ Internal .NET microservice for Google Search Console, GA4, Bing sync, and keywor ```bash cd services/IntegrationsService DATABASE_URL=postgres://profiling:profiling@127.0.0.1:5432/website_profiling \ - FASTAPI_URL=http://127.0.0.1:8001 \ + FASTAPI_URL=http://127.0.0.1:8096 \ USE_FASTAPI_PYTHON_BRIDGE=1 \ ASPNETCORE_URLS=http://127.0.0.1:8093 \ dotnet run --project src/IntegrationsService.Api --no-launch-profile diff --git a/services/IntegrationsService/src/IntegrationsService.Api/Controllers/PropertyGoogleController.cs b/services/IntegrationsService/src/IntegrationsService.Api/Controllers/PropertyGoogleController.cs index c046a00c..1670c1d1 100644 --- a/services/IntegrationsService/src/IntegrationsService.Api/Controllers/PropertyGoogleController.cs +++ b/services/IntegrationsService/src/IntegrationsService.Api/Controllers/PropertyGoogleController.cs @@ -31,7 +31,6 @@ public async Task Status(long propertyId, CancellationToken cance return NotFound(new { error = "Property not found" }); } - var appCfg = await appSettings.ReadAsync(cancellationToken); var publicStatus = PropertyGoogleStatusMapper.ToPublicStatus(prop); return Ok(new { @@ -42,7 +41,7 @@ public async Task Status(long propertyId, CancellationToken cance dateRangeDays = publicStatus.DateRangeDays, connectedEmail = publicStatus.ConnectedEmail, connectedAt = publicStatus.ConnectedAt, - hasClientId = !string.IsNullOrWhiteSpace(appCfg.ClientId), + hasClientId = await appSettings.HasClientCredentialsAsync(cancellationToken), lastFetchedAt = await googleData.GetLastFetchedAtAsync(propertyId, cancellationToken), propertyId, }); diff --git a/services/IntegrationsService/src/IntegrationsService.Application/Google/FastApiPythonBridge.cs b/services/IntegrationsService/src/IntegrationsService.Application/Google/FastApiPythonBridge.cs index a3cca697..6dfb82db 100644 --- a/services/IntegrationsService/src/IntegrationsService.Application/Google/FastApiPythonBridge.cs +++ b/services/IntegrationsService/src/IntegrationsService.Application/Google/FastApiPythonBridge.cs @@ -133,7 +133,7 @@ public async Task RunKeywordEnrichAsync( private HttpClient CreateClient() { var client = httpClientFactory.CreateClient(nameof(FastApiPythonBridge)); - var baseUrl = (Environment.GetEnvironmentVariable("FASTAPI_URL") ?? "http://127.0.0.1:8001").Trim().TrimEnd('/'); + var baseUrl = (Environment.GetEnvironmentVariable("FASTAPI_URL") ?? "http://127.0.0.1:8096").Trim().TrimEnd('/'); client.BaseAddress = new Uri(baseUrl + "/"); client.Timeout = TimeSpan.FromSeconds(120); return client; diff --git a/services/IntegrationsService/src/IntegrationsService.Application/Repositories/GoogleAppSettingsRepository.cs b/services/IntegrationsService/src/IntegrationsService.Application/Repositories/GoogleAppSettingsRepository.cs index 113842be..080532e1 100644 --- a/services/IntegrationsService/src/IntegrationsService.Application/Repositories/GoogleAppSettingsRepository.cs +++ b/services/IntegrationsService/src/IntegrationsService.Application/Repositories/GoogleAppSettingsRepository.cs @@ -33,6 +33,13 @@ public async Task HasServiceAccountAsync(CancellationToken cancellationTok && row.ServiceAccountJson != "null"; } + public async Task HasClientCredentialsAsync(CancellationToken cancellationToken = default) + { + var row = await ReadAsync(cancellationToken); + var clientId = (row.ClientId ?? Environment.GetEnvironmentVariable("GOOGLE_CLIENT_ID") ?? "").Trim(); + return !string.IsNullOrWhiteSpace(clientId); + } + public async Task<(string ClientId, string ClientSecret)> AppClientCredentialsAsync( CancellationToken cancellationToken = default) { diff --git a/services/IntegrationsService/src/IntegrationsService.Providers/Google/GoogleCredentialFactory.cs b/services/IntegrationsService/src/IntegrationsService.Providers/Google/GoogleCredentialFactory.cs index 55358b87..00ba4a77 100644 --- a/services/IntegrationsService/src/IntegrationsService.Providers/Google/GoogleCredentialFactory.cs +++ b/services/IntegrationsService/src/IntegrationsService.Providers/Google/GoogleCredentialFactory.cs @@ -51,7 +51,24 @@ public async Task BuildCredentialsAsync( propertyId.ToString(), new TokenResponse { RefreshToken = token }); - if (!await credential.RefreshTokenAsync(cancellationToken)) + bool refreshed; + try + { + refreshed = await credential.RefreshTokenAsync(cancellationToken); + } + catch (TokenResponseException ex) when (ex.Error?.Error == "invalid_grant") + { + await properties.DisconnectGoogleAsync(propertyId, cancellationToken); + throw new InvalidOperationException( + "Google connection expired — reconnect Google for this site."); + } + catch (TokenResponseException) + { + throw new InvalidOperationException( + "Google connection expired — reconnect Google for this site."); + } + + if (!refreshed) { throw new InvalidOperationException( "Google connection expired — reconnect Google for this site."); diff --git a/services/ReportService/src/ReportService.Api/appsettings.json b/services/ReportService/src/ReportService.Api/appsettings.json index ab59f4e2..98069c97 100644 --- a/services/ReportService/src/ReportService.Api/appsettings.json +++ b/services/ReportService/src/ReportService.Api/appsettings.json @@ -6,7 +6,7 @@ } }, "FastApi": { - "BaseUrl": "http://127.0.0.1:8001", + "BaseUrl": "http://127.0.0.1:8096", "TimeoutSeconds": 1800 }, "ReportService": { diff --git a/services/ReportService/src/ReportService.Application/Options/FastApiOptions.cs b/services/ReportService/src/ReportService.Application/Options/FastApiOptions.cs index ad69f508..71cac715 100644 --- a/services/ReportService/src/ReportService.Application/Options/FastApiOptions.cs +++ b/services/ReportService/src/ReportService.Application/Options/FastApiOptions.cs @@ -4,7 +4,7 @@ public sealed class FastApiOptions { public const string SectionName = "FastApi"; - public string BaseUrl { get; set; } = "http://127.0.0.1:8001"; + public string BaseUrl { get; set; } = "http://127.0.0.1:8096"; public int TimeoutSeconds { get; set; } = 1800; } diff --git a/services/Shared/WebsiteProfiling.Testing/ServiceRegistrationTestEnvironment.cs b/services/Shared/WebsiteProfiling.Testing/ServiceRegistrationTestEnvironment.cs index b50affed..090d3ebe 100644 --- a/services/Shared/WebsiteProfiling.Testing/ServiceRegistrationTestEnvironment.cs +++ b/services/Shared/WebsiteProfiling.Testing/ServiceRegistrationTestEnvironment.cs @@ -26,7 +26,7 @@ public void Set(string key, string? value) public void SetDefaultsForPostgresServices() { Set("DATABASE_URL", "Host=127.0.0.1;Port=5432;Database=wp_di_test;Username=test;Password=test"); - Set("FASTAPI_URL", "http://127.0.0.1:8001"); + Set("FASTAPI_URL", "http://127.0.0.1:8096"); } public void Dispose() diff --git a/src/website_profiling/clients/file_service.py b/src/website_profiling/clients/file_service.py index 3285c3b1..0a7b704a 100644 --- a/src/website_profiling/clients/file_service.py +++ b/src/website_profiling/clients/file_service.py @@ -7,7 +7,7 @@ import requests -_DEFAULT_BASE = "http://127.0.0.1:8080" +_DEFAULT_BASE = "http://127.0.0.1:8097" _TIMEOUT_SECONDS = 120 diff --git a/src/website_profiling/commands/report_build.py b/src/website_profiling/commands/report_build.py index 9fb72569..1e330c4e 100644 --- a/src/website_profiling/commands/report_build.py +++ b/src/website_profiling/commands/report_build.py @@ -149,7 +149,7 @@ def call_fastapi_report_bridge(cfg: dict, property_id: int, crawl_run_id: int | import urllib.error import urllib.request - base = (os.environ.get("FASTAPI_URL") or "http://127.0.0.1:8001").strip().rstrip("/") + base = (os.environ.get("FASTAPI_URL") or "http://127.0.0.1:8096").strip().rstrip("/") body = { "propertyId": int(property_id), "crawlRunId": crawl_run_id, diff --git a/tests/test_ai_migration_coverage.py b/tests/test_ai_migration_coverage.py index 8dc15634..c9cb60c0 100644 --- a/tests/test_ai_migration_coverage.py +++ b/tests/test_ai_migration_coverage.py @@ -418,7 +418,7 @@ def __exit__(self, *_a): with pytest.raises(ConnectionError): report_build.call_report_service({}, 1) - monkeypatch.setenv("FASTAPI_URL", "http://fastapi:8001") + monkeypatch.setenv("FASTAPI_URL", "http://fastapi:8096") with patch("urllib.request.urlopen", return_value=_Resp()): out2 = report_build.call_fastapi_report_bridge({"a": "1"}, 3) assert out2["ok"] is True diff --git a/web/src/lib/mcpClientConfig.test.ts b/web/src/lib/mcpClientConfig.test.ts index 4d66f15f..5d274dd1 100644 --- a/web/src/lib/mcpClientConfig.test.ts +++ b/web/src/lib/mcpClientConfig.test.ts @@ -40,7 +40,7 @@ describe('mcpClientConfig', () => { }); expect(json).toContain('"command": "dotnet"'); expect(json).toContain('services/AiService/src/AiService.Api'); - expect(json).toContain('"FASTAPI_URL": "http://127.0.0.1:8001"'); + expect(json).toContain('"FASTAPI_URL": "http://127.0.0.1:8096"'); expect(json).toContain('"WP_PROPERTY_ID": "2"'); }); diff --git a/web/src/lib/mcpClientConfig.ts b/web/src/lib/mcpClientConfig.ts index fb55ed52..ee19c0b0 100644 --- a/web/src/lib/mcpClientConfig.ts +++ b/web/src/lib/mcpClientConfig.ts @@ -78,7 +78,7 @@ export function buildLocalStdioConfig(input: McpClientConfigInput): string { args: ['run', '--project', 'services/AiService/src/AiService.Api', '--no-launch-profile'], env: { DATABASE_URL: databaseUrl, - FASTAPI_URL: 'http://127.0.0.1:8001', + FASTAPI_URL: 'http://127.0.0.1:8096', WP_MCP_DOMAIN: domain, WP_PROPERTY_ID: propertyId, }, @@ -96,7 +96,7 @@ export function buildHttpStartCommand(): string { return [ 'cd services/AiService', 'export DATABASE_URL=postgres://USER:PASS@localhost:5432/website_profiling', - 'export FASTAPI_URL=http://127.0.0.1:8001', + 'export FASTAPI_URL=http://127.0.0.1:8096', 'export ASPNETCORE_URLS=http://0.0.0.0:8092', 'export WP_MCP_HTTP=1', 'dotnet run --project src/AiService.Api',