diff --git a/.env.template b/.env.template index e96d4a48..e8084ca7 100644 --- a/.env.template +++ b/.env.template @@ -45,12 +45,6 @@ WEB_ARCHIVE_HYPERBROWSER_CAPTCHA=true # (`pip install cloakbrowser`). Exposes cloakbrowser.launch() -> Browser. WEB_ARCHIVE_CLOAKBROWSER_IMPORT=cloakbrowser WEB_ARCHIVE_PDF_MAX_PAGES=50 -# Operator-only: database DSN for `harvest-db` (reads a bot's cited URLs straight -# from Postgres). libpq DSN or postgresql:// URL — e.g. a Neon connection string. -# This DSN is a real secret. PREFER the macOS Keychain (item `metaculus-db-dsn`) -# over this file — see the source_archive README "DSN resolution". Leave blank to -# use the Keychain / local default. -METACULUS_DB_DSN= # Disable if in Streamlit Cloud FILE_WRITING_ALLOWED=TRUE diff --git a/code_tests/unit_tests/test_agents_and_tools/test_source_archive/test_metaculus_db.py b/code_tests/unit_tests/test_agents_and_tools/test_source_archive/test_metaculus_db.py deleted file mode 100644 index 54cab175..00000000 --- a/code_tests/unit_tests/test_agents_and_tools/test_source_archive/test_metaculus_db.py +++ /dev/null @@ -1,124 +0,0 @@ -from __future__ import annotations - -from forecasting_tools.agents_and_tools.source_archive.ingest.metaculus_db import ( - LOCAL_DEFAULT_DSN, - MetaculusDbHarvester, - resolve_dsn, -) - - -def test_harvest_post_builds_records_with_provenance(): - rows = [ - { - "comment_id": 1, - "on_post_id": 42, - "text": "see https://a.test/x and https://b.test/y", - "username": "alpha", - "author_id": 7, - }, - { - "comment_id": 2, - "on_post_id": 42, - "text": "https://a.test/x again", - "username": "beta", - "author_id": 8, - }, - ] - seen = {} - - def query(sql, params): - seen["sql"], seen["params"] = sql, params - return rows - - records = MetaculusDbHarvester(query).harvest_post(42) - - assert seen["params"] == (42,) - assert {r.url for r in records} == {"https://a.test/x", "https://b.test/y"} - r0 = next(r for r in records if r.url == "https://a.test/x") - assert r0.origin == "metaculus_comment" - assert r0.question_id == "42" - assert r0.question_url == "https://www.metaculus.com/questions/42/" - assert r0.bot in ("alpha", "beta") - # one record per (URL, comment): a.test/x is cited in both comments - assert sum(r.url == "https://a.test/x" for r in records) == 2 - - -def test_harvest_recent_passes_days_and_limit(): - seen = {} - - def query(sql, params): - seen["sql"], seen["params"] = sql, params - return [] - - MetaculusDbHarvester(query).harvest_recent(days=3, limit=50) - assert seen["params"] == (3, 50) - assert "limit %s" in seen["sql"] - - -def test_harvest_recent_uncapped_by_default(): - seen = {} - - def query(sql, params): - seen["sql"], seen["params"] = sql, params - return [] - - # A daily sweep wants every row from the latest day, not a 1000-row cap. - MetaculusDbHarvester(query).harvest_recent(days=1) - assert seen["params"] == (1,) - assert "limit" not in seen["sql"].lower() - - -def test_includes_private_bot_comments_by_default(): - seen = {} - - def query(sql, params): - seen["sql"] = sql - return [] - - # The day-behind replica's value is the now-private bot reasoning, so the - # default read must NOT filter private rows out. - MetaculusDbHarvester(query).harvest_recent(days=1) - assert "is_private" not in seen["sql"] - assert "u.is_bot" in seen["sql"] - - -def test_public_only_filters_private_comments(): - seen = {} - - def query(sql, params): - seen["sql"] = sql - return [] - - MetaculusDbHarvester(query).harvest_post(42, include_private=False) - assert "not c.is_private" in seen["sql"] - - -def test_resolve_dsn_prefers_explicit_then_env_then_keychain(): - # explicit flag wins over everything - assert ( - resolve_dsn( - "postgresql://flag", - env={"METACULUS_DB_DSN": "postgresql://env"}, - keychain_reader=lambda: "postgresql://kc", - ) - == "postgresql://flag" - ) - # then the env var - assert ( - resolve_dsn( - None, - env={"METACULUS_DB_DSN": "postgresql://env"}, - keychain_reader=lambda: "postgresql://kc", - ) - == "postgresql://env" - ) - # then the keychain (the private path) - assert ( - resolve_dsn(None, env={}, keychain_reader=lambda: "postgresql://kc") - == "postgresql://kc" - ) - - -def test_resolve_dsn_falls_back_to_local_default(): - # nothing configured and no keychain item -> local dev DB, not a crash - assert resolve_dsn(None, env={}, keychain_reader=lambda: None) == LOCAL_DEFAULT_DSN diff --git a/forecasting_tools/agents_and_tools/source_archive/README.md b/forecasting_tools/agents_and_tools/source_archive/README.md index b86ffe91..9c815175 100644 --- a/forecasting_tools/agents_and_tools/source_archive/README.md +++ b/forecasting_tools/agents_and_tools/source_archive/README.md @@ -229,49 +229,17 @@ The pipeline dedupes URLs within the manifest before fetching. ## Where the manifest comes from -You can write a manifest yourself, or generate one from a forecasting bot's -reasoning — the source links a bot used are recorded in the comment it posts and, -more completely, in its run traces. +You write a manifest yourself, or generate one from a bot's recorded reasoning. -**From the database (operator path).** `harvest-db` reads the URLs a bot cited -straight from the platform's Postgres database and emits a manifest. Point it at -a database (a `postgresql://…` URL works — e.g. a Neon connection string): +**From text.** `extract_urls(text)` / `extract_citation_records(...)` in +`ingest.url_extraction` pull URLs out of any markdown/text (markdown links, +autolinks, and bare URLs) — point them at whatever record of a bot's reasoning +you have. -```bash -# one post, or the latest day of activity -source-archive harvest-db --post 29495 --dedupe --out run.jsonl -source-archive harvest-db --days 1 --dedupe --upload --run-id "$(date -u +%F)" -``` - -It reads `comments_comment ⋈ users_user (is_bot)` and emits the same manifest. -`--days` is uncapped by default; `--limit N` caps the row count for spot checks. -`--public-only` restricts to public comments (all comments are read by default). - -**DSN resolution (keep the credential off disk).** The DSN is resolved in this -order: `--dsn` flag → `$METACULUS_DB_DSN` → macOS Keychain item -`metaculus-db-dsn` → local default `dbname=metaculus`. The DSN is a real secret -(it grants database read access), so prefer the **Keychain** over `.env` / a -shell export — those land in files and shell history that any editor or coding -agent can read. Store it once (you'll be prompted to paste it, so it never -appears in your shell history): - -```bash -security add-generic-password -U -a "$USER" -s metaculus-db-dsn -w -# paste the full postgresql://USER:PASS@HOST/dbname?sslmode=require string, return -``` - -For the strongest guard, open **Keychain Access.app → login → `metaculus-db-dsn` -→ Access Control → "Confirm before allowing access"** and clear the always-allow -list. Every read then raises a GUI confirm: a human running the harvest clicks -*Allow* (not *Always Allow*), but an automated agent driving a shell can't. With -that set, the harvester works with no DSN in any file — `source-archive -harvest-db --days 1` just prompts you once per run. - -**From text or traces.** The lower-level `extract_urls(text)` / -`extract_citation_records(...)` helpers in `ingest.url_extraction` pull URLs out -of any markdown/text (markdown links, autolinks, and bare URLs). For bots you -control, an instrumented trace (`ingest-traces`) gives the fullest URL list; a -comment gives a shallower one, since it is length-truncated when posted. +**From instrumented traces.** For bots you control, a trace is the fullest +source. `source-archive ingest-traces ` walks a traced run and emits a +manifest of every URL the bot touched, with provenance (trace, tool, search +query). ## How it's organized @@ -279,7 +247,7 @@ comment gives a shallower one, since it is length-truncated when posted. | --- | --- | | `config.py` | Environment-driven `ArchiveConfig` | | `models.py` | `CaptureResult`, `StoredCapture`, `CitationRecord` | -| `ingest/` | Build a manifest: URL extraction + Metaculus comment harvester | +| `ingest/` | Build a manifest: URL extraction from text + traced bot runs | | `fetchers/` | Playwright (primary) + CloakBrowser / Hyperbrowser / Firecrawl / PDF backups, tiered orchestrator | | `benchmark.py` | Backend bake-off: reliability + cost per backend over a manifest | | `quality.py` | Reject 404s, block pages, and thin content before archiving | diff --git a/forecasting_tools/agents_and_tools/source_archive/cli.py b/forecasting_tools/agents_and_tools/source_archive/cli.py index f180ffb7..9a1e021c 100644 --- a/forecasting_tools/agents_and_tools/source_archive/cli.py +++ b/forecasting_tools/agents_and_tools/source_archive/cli.py @@ -190,44 +190,6 @@ def _cmd_catalog(args, config: ArchiveConfig) -> int: return 0 -def _cmd_harvest_db(args, config: ArchiveConfig) -> int: - from forecasting_tools.agents_and_tools.source_archive.ingest import ( - MetaculusDbHarvester, - dedupe_records, - resolve_dsn, - ) - - dsn = resolve_dsn(args.dsn) - include_private = not args.public_only - harvester = MetaculusDbHarvester.from_dsn(dsn) - if args.post: - records = harvester.harvest_post( - args.post, run_id=args.run_id, include_private=include_private - ) - run_id = args.run_id or f"metaculus-db-post-{args.post}" - else: - records = harvester.harvest_recent( - days=args.days, - limit=args.limit, - run_id=args.run_id, - include_private=include_private, - ) - run_id = args.run_id or f"metaculus-db-recent-{args.days}d" - if args.dedupe: - records = dedupe_records(records) - print(f"Harvested {len(records)} citation record(s) from the Metaculus DB") - - out_path = args.out or f"{run_id}.jsonl" - if not args.upload or args.out: - manifest_io.write_file(out_path, records) - print(f"Wrote manifest -> {out_path}") - if args.upload: - store = _make_blob_store(config, None, args.bucket) - manifest_io.write_blob(store, run_id, records, config) - print(f"Uploaded manifest -> {config.s3_prefix}/manifests/{run_id}.jsonl") - return 0 - - def _cmd_coverage(args, config: ArchiveConfig) -> int: from pathlib import Path @@ -336,41 +298,6 @@ def main(argv: list[str] | None = None) -> int: ) cat.add_argument("--bucket", help="override the S3 bucket") - hdb = sub.add_parser( - "harvest-db", - help="read a bot's cited URLs from the platform Postgres database (operator)", - ) - grp = hdb.add_mutually_exclusive_group(required=True) - grp.add_argument("--post", help="harvest one post id") - grp.add_argument("--days", type=int, help="harvest the most recent N days") - hdb.add_argument( - "--limit", - type=int, - default=None, - help="cap rows when using --days (default: uncapped — a daily sweep wants all)", - ) - hdb.add_argument( - "--public-only", - action="store_true", - help="read only public comments (default: read all of a bot's comments)", - ) - hdb.add_argument( - "--dsn", - help="libpq DSN or postgresql:// URL. Default resolution: --dsn > " - "$METACULUS_DB_DSN > macOS Keychain item 'metaculus-db-dsn' > " - "dbname=metaculus. Prefer the Keychain for the real secret " - "(a --dsn value lands in shell history).", - ) - hdb.add_argument("--out", metavar="FILE", help="write the manifest to this .jsonl") - hdb.add_argument("--run-id", help="run id (default derived from the slice)") - hdb.add_argument( - "--dedupe", action="store_true", help="keep one record per URL (first seen)" - ) - hdb.add_argument( - "--upload", action="store_true", help="upload the manifest to S3 manifests/" - ) - hdb.add_argument("--bucket", help="override the S3 bucket") - cov = sub.add_parser( "coverage", help="report what %% of cited sources were archived (trace vs comments)", @@ -396,8 +323,6 @@ def main(argv: list[str] | None = None) -> int: return _cmd_capture(args, config) if args.command == "ingest-traces": return _cmd_ingest_traces(args, config) - if args.command == "harvest-db": - return _cmd_harvest_db(args, config) if args.command == "catalog": return _cmd_catalog(args, config) if args.command == "coverage": diff --git a/forecasting_tools/agents_and_tools/source_archive/ingest/__init__.py b/forecasting_tools/agents_and_tools/source_archive/ingest/__init__.py index 8b689781..5b9729db 100644 --- a/forecasting_tools/agents_and_tools/source_archive/ingest/__init__.py +++ b/forecasting_tools/agents_and_tools/source_archive/ingest/__init__.py @@ -4,14 +4,9 @@ from a bot's published reasoning: - :mod:`url_extraction` — pull URLs out of free text / markdown. - - :mod:`metaculus_db` — read a bot's cited URLs from the platform database. - :mod:`trace_extraction` — build a manifest from a traced bot run (fullest path). """ -from forecasting_tools.agents_and_tools.source_archive.ingest.metaculus_db import ( - MetaculusDbHarvester, - resolve_dsn, -) from forecasting_tools.agents_and_tools.source_archive.ingest.trace_extraction import ( extract_records_from_events, extract_records_from_question_dir, @@ -25,7 +20,6 @@ ) __all__ = [ - "MetaculusDbHarvester", "dedupe_records", "extract_citation_records", "extract_records_from_events", @@ -33,5 +27,4 @@ "extract_records_from_trace_file", "extract_urls", "harvest_run", - "resolve_dsn", ] diff --git a/forecasting_tools/agents_and_tools/source_archive/ingest/metaculus_db.py b/forecasting_tools/agents_and_tools/source_archive/ingest/metaculus_db.py deleted file mode 100644 index c0221bbf..00000000 --- a/forecasting_tools/agents_and_tools/source_archive/ingest/metaculus_db.py +++ /dev/null @@ -1,215 +0,0 @@ -"""Read a bot's cited URLs from the platform Postgres database (operator tooling). - -For operators with database access, this reads the URLs a forecasting bot cited -straight from Postgres (``comments_comment`` joined to ``users_user.is_bot``) and -emits the same :class:`CitationRecord`s as every other ingestion path, so the -catalog / coverage / capture stages downstream are unchanged. By default it reads -all of a bot's comments (``include_private=True``); pass ``include_private=False`` -for the public ones only. Only ``u.is_bot`` accounts are ever read. - -The DB call is **injected** (``query``) so the core is driver-agnostic and unit -testable; :meth:`from_dsn` wires a psycopg2 connection for real use (a libpq DSN -or a ``postgresql://…`` URL — e.g. a Neon connection string). Reads only; no -secrets are stored — the DSN comes from the caller / environment. -""" - -from __future__ import annotations - -import os -from typing import Any, Callable, Mapping, Sequence - -from forecasting_tools.agents_and_tools.source_archive.ingest.url_extraction import ( - extract_citation_records, -) -from forecasting_tools.agents_and_tools.source_archive.models import CitationRecord - -QueryFn = Callable[[str, Sequence[Any]], list[dict]] - -# Keychain service name the DSN is stored under (see resolve_dsn / README). -KEYCHAIN_SERVICE = "metaculus-db-dsn" -LOCAL_DEFAULT_DSN = "dbname=metaculus" - -_WEB = "https://www.metaculus.com" - -# The windowed/post-scoped comment set is computed in a MATERIALIZED CTE so -# Postgres evaluates it FIRST, then joins users_user by primary key. Without the -# CTE the planner's stale stats misjudge the date window at ~300k rows (it is -# really ~2k/day) and pick a join order that times out on the remote pooler. -_OUTER = ( - "select r.id as comment_id, r.on_post_id, r.text, " - "u.username, r.author_id " - "from recent r join users_user u on u.id = r.author_id where u.is_bot" -) - - -def _recent_cte(scope: str, include_private: bool) -> str: - """A MATERIALIZED ``recent`` CTE of link-bearing, non-deleted comments. - - ``scope`` is the row-narrowing predicate (a post id or a created_at window). - Private comments are included unless ``include_private`` is False. - - ``strpos(text,'http') > 0`` is a cheap substring pre-filter (a regex `~` scan - times out on the pooler; ``like`` would need ``%%`` escaping under psycopg2). - The real URL parsing happens in extract_citation_records, so over-matching - here just costs a few empty rows. - """ - clauses = ["not c.is_soft_deleted", "strpos(c.text, 'http') > 0", scope] - if not include_private: - clauses.append("not c.is_private") - where = " and ".join(clauses) - return ( - "with recent as materialized (" - "select c.id, c.on_post_id, c.text, c.author_id, c.created_at " - f"from comments_comment c where {where}) " - ) - - -def _dsn_from_keychain(service: str = KEYCHAIN_SERVICE) -> str | None: - """Read the DSN from the macOS login Keychain, or ``None`` if unavailable. - - Uses ``security find-generic-password -w`` so the credential lives only in - the Keychain — never in ``.env``, a shell rc, or shell history. If the - Keychain item's ACL is set to confirm on access, this call raises a GUI - prompt: a human can approve it, an automated agent driving the shell cannot. - Returns ``None`` off macOS or when the item is absent / access is denied, so - callers fall through to the next source. - """ - import shutil - import subprocess - - if not shutil.which("security"): # not macOS - return None - try: - proc = subprocess.run( - ["security", "find-generic-password", "-s", service, "-w"], - capture_output=True, - text=True, - timeout=30, - ) - except (OSError, subprocess.SubprocessError): - return None - if proc.returncode != 0: - return None - return proc.stdout.strip() or None - - -def resolve_dsn( - explicit: str | None = None, - *, - env: Mapping[str, str] | None = None, - keychain_reader: Callable[[], str | None] | None = None, -) -> str: - """Resolve the DB DSN without ever persisting it to disk. - - Resolution order, first hit wins: - 1. ``explicit`` (e.g. a ``--dsn`` flag — convenient, but lands in shell - history, so prefer the Keychain for the real secret), - 2. ``$METACULUS_DB_DSN``, - 3. the macOS Keychain item ``metaculus-db-dsn`` (the private path), - 4. the local default ``dbname=metaculus``. - ``env`` / ``keychain_reader`` are injectable for tests. - """ - if explicit: - return explicit - environ = env if env is not None else os.environ - from_env = environ.get("METACULUS_DB_DSN") - if from_env: - return from_env - reader = keychain_reader or _dsn_from_keychain - from_keychain = reader() - if from_keychain: - return from_keychain - return LOCAL_DEFAULT_DSN - - -class MetaculusDbHarvester: - """Reads bot comments from Postgres. ``query(sql, params) -> list[dict]``.""" - - def __init__(self, query: QueryFn): - self._query = query - - @classmethod - def from_dsn(cls, dsn: str = "dbname=metaculus") -> "MetaculusDbHarvester": - try: - import psycopg2 - import psycopg2.extras - except ImportError as e: # pragma: no cover - optional operator dep - raise ImportError( - "psycopg2 is required for DB harvesting " - "(`pip install psycopg2-binary`)." - ) from e - conn = psycopg2.connect(dsn) - - def query(sql: str, params: Sequence[Any]) -> list[dict]: - with conn.cursor(cursor_factory=psycopg2.extras.RealDictCursor) as cur: - cur.execute(sql, params) - return [dict(r) for r in cur.fetchall()] - - return cls(query) - - def _records(self, rows: list[dict], run_id: str | None) -> list[CitationRecord]: - out: list[CitationRecord] = [] - for r in rows: - post_id = r.get("on_post_id") - pid = str(post_id) if post_id is not None else None - comment_id = r.get("comment_id") - out.extend( - extract_citation_records( - r.get("text"), - run_id=run_id, - bot=r.get("username") or str(r.get("author_id")), - question_id=pid, - metaculus_id=pid, - question_url=( - f"{_WEB}/questions/{post_id}/" if post_id is not None else None - ), - comment_id=str(comment_id) if comment_id is not None else None, - origin="metaculus_comment", - ) - ) - return out - - def harvest_post( - self, - post_id: int | str, - *, - run_id: str | None = None, - include_private: bool = True, - ) -> list[CitationRecord]: - """Every bot-cited URL in the comments on one post.""" - run_id = run_id or f"metaculus-db-post-{post_id}" - sql = ( - _recent_cte("c.on_post_id = %s", include_private) - + _OUTER - + " order by r.created_at" - ) - return self._records(self._query(sql, (post_id,)), run_id) - - def harvest_recent( - self, - *, - days: int = 1, - limit: int | None = None, - run_id: str | None = None, - include_private: bool = True, - ) -> list[CitationRecord]: - """Bot-cited URLs from the most recent ``days`` of comments. - - "Recent" is measured against ``max(created_at)`` in the table, not wall - clock, so a replica that lags real time by a day still returns its latest - day with ``days=1``. ``limit`` caps the row count; ``None`` (the default) - is uncapped, which is what a daily sweep wants. - """ - run_id = run_id or f"metaculus-db-recent-{days}d" - scope = ( - "c.created_at >= " - "(select max(created_at) from comments_comment) - (%s * interval '1 day')" - ) - sql = ( - _recent_cte(scope, include_private) + _OUTER + " order by r.created_at desc" - ) - params: list[Any] = [days] - if limit: - sql += " limit %s" - params.append(limit) - return self._records(self._query(sql, tuple(params)), run_id) diff --git a/forecasting_tools/agents_and_tools/source_archive/ingest/trace_extraction.py b/forecasting_tools/agents_and_tools/source_archive/ingest/trace_extraction.py index c330eccb..1c63fe08 100644 --- a/forecasting_tools/agents_and_tools/source_archive/ingest/trace_extraction.py +++ b/forecasting_tools/agents_and_tools/source_archive/ingest/trace_extraction.py @@ -3,8 +3,7 @@ When the template bot is run with tracing enabled it writes one JSONL trace per forecast attempt, recording the agent loop step by step. Those traces are the *fullest* record of what the bot actually looked at — richer than the reasoning -comment it posts, which is length-truncated (see :mod:`metaculus_db` for the -shallower comment path). +comment it posts, which is length-truncated. This module walks those traces and pulls out every external URL the bot touched, turning each into a :class:`CitationRecord` with provenance (which trace, which