source-archive trimming

.env.template

@@ -45,12 +45,6 @@ WEB_ARCHIVE_HYPERBROWSER_CAPTCHA=true
 # (`pip install cloakbrowser`). Exposes cloakbrowser.launch() -> Browser.
 WEB_ARCHIVE_CLOAKBROWSER_IMPORT=cloakbrowser
 WEB_ARCHIVE_PDF_MAX_PAGES=50
-# Operator-only: database DSN for `harvest-db` (reads a bot's cited URLs straight
-# from Postgres). libpq DSN or postgresql:// URL — e.g. a Neon connection string.
-# This DSN is a real secret. PREFER the macOS Keychain (item `metaculus-db-dsn`)
-# over this file — see the source_archive README "DSN resolution". Leave blank to
-# use the Keychain / local default.
-METACULUS_DB_DSN=
 
 # Disable if in Streamlit Cloud
 FILE_WRITING_ALLOWED=TRUE

forecasting_tools/agents_and_tools/source_archive/README.md

@@ -229,57 +229,25 @@ The pipeline dedupes URLs within the manifest before fetching.
 
 ## Where the manifest comes from
 
-You can write a manifest yourself, or generate one from a forecasting bot's
-reasoning — the source links a bot used are recorded in the comment it posts and,
-more completely, in its run traces.
+You write a manifest yourself, or generate one from a bot's recorded reasoning.
 
-**From the database (operator path).** `harvest-db` reads the URLs a bot cited
-straight from the platform's Postgres database and emits a manifest. Point it at
-a database (a `postgresql://…` URL works — e.g. a Neon connection string):
+**From text.** `extract_urls(text)` / `extract_citation_records(...)` in
+`ingest.url_extraction` pull URLs out of any markdown/text (markdown links,
+autolinks, and bare URLs) — point them at whatever record of a bot's reasoning
+you have.
 
-```bash
-# one post, or the latest day of activity
-source-archive harvest-db --post 29495 --dedupe --out run.jsonl
-source-archive harvest-db --days 1 --dedupe --upload --run-id "$(date -u +%F)"
-```
-
-It reads `comments_comment ⋈ users_user (is_bot)` and emits the same manifest.
-`--days` is uncapped by default; `--limit N` caps the row count for spot checks.
-`--public-only` restricts to public comments (all comments are read by default).
-
-**DSN resolution (keep the credential off disk).** The DSN is resolved in this
-order: `--dsn` flag → `$METACULUS_DB_DSN` → macOS Keychain item
-`metaculus-db-dsn` → local default `dbname=metaculus`. The DSN is a real secret
-(it grants database read access), so prefer the **Keychain** over `.env` / a
-shell export — those land in files and shell history that any editor or coding
-agent can read. Store it once (you'll be prompted to paste it, so it never
-appears in your shell history):
-
-```bash
-security add-generic-password -U -a "$USER" -s metaculus-db-dsn -w
-# paste the full postgresql://USER:PASS@HOST/dbname?sslmode=require string, return
-```
-
-For the strongest guard, open **Keychain Access.app → login → `metaculus-db-dsn`
-→ Access Control → "Confirm before allowing access"** and clear the always-allow
-list. Every read then raises a GUI confirm: a human running the harvest clicks
-*Allow* (not *Always Allow*), but an automated agent driving a shell can't. With
-that set, the harvester works with no DSN in any file — `source-archive
-harvest-db --days 1` just prompts you once per run.
-
-**From text or traces.** The lower-level `extract_urls(text)` /
-`extract_citation_records(...)` helpers in `ingest.url_extraction` pull URLs out
-of any markdown/text (markdown links, autolinks, and bare URLs). For bots you
-control, an instrumented trace (`ingest-traces`) gives the fullest URL list; a
-comment gives a shallower one, since it is length-truncated when posted.
+**From instrumented traces.** For bots you control, a trace is the fullest
+source. `source-archive ingest-traces <run-dir>` walks a traced run and emits a
+manifest of every URL the bot touched, with provenance (trace, tool, search
+query).
 
 ## How it's organized
 
 | Module | Responsibility |
 | --- | --- |
 | `config.py` | Environment-driven `ArchiveConfig` |
 | `models.py` | `CaptureResult`, `StoredCapture`, `CitationRecord` |
-| `ingest/` | Build a manifest: URL extraction + Metaculus comment harvester |
+| `ingest/` | Build a manifest: URL extraction from text + traced bot runs |
 | `fetchers/` | Playwright (primary) + CloakBrowser / Hyperbrowser / Firecrawl / PDF backups, tiered orchestrator |
 | `benchmark.py` | Backend bake-off: reliability + cost per backend over a manifest |
 | `quality.py` | Reject 404s, block pages, and thin content before archiving |

forecasting_tools/agents_and_tools/source_archive/cli.py

@@ -190,44 +190,6 @@ def _cmd_catalog(args, config: ArchiveConfig) -> int:
     return 0
 
 
-def _cmd_harvest_db(args, config: ArchiveConfig) -> int:
-    from forecasting_tools.agents_and_tools.source_archive.ingest import (
-        MetaculusDbHarvester,
-        dedupe_records,
-        resolve_dsn,
-    )
-
-    dsn = resolve_dsn(args.dsn)
-    include_private = not args.public_only
-    harvester = MetaculusDbHarvester.from_dsn(dsn)
-    if args.post:
-        records = harvester.harvest_post(
-            args.post, run_id=args.run_id, include_private=include_private
-        )
-        run_id = args.run_id or f"metaculus-db-post-{args.post}"
-    else:
-        records = harvester.harvest_recent(
-            days=args.days,
-            limit=args.limit,
-            run_id=args.run_id,
-            include_private=include_private,
-        )
-        run_id = args.run_id or f"metaculus-db-recent-{args.days}d"
-    if args.dedupe:
-        records = dedupe_records(records)
-    print(f"Harvested {len(records)} citation record(s) from the Metaculus DB")
-
-    out_path = args.out or f"{run_id}.jsonl"
-    if not args.upload or args.out:
-        manifest_io.write_file(out_path, records)
-        print(f"Wrote manifest -> {out_path}")
-    if args.upload:
-        store = _make_blob_store(config, None, args.bucket)
-        manifest_io.write_blob(store, run_id, records, config)
-        print(f"Uploaded manifest -> {config.s3_prefix}/manifests/{run_id}.jsonl")
-    return 0
-
-
 def _cmd_coverage(args, config: ArchiveConfig) -> int:
     from pathlib import Path
 
@@ -336,41 +298,6 @@ def main(argv: list[str] | None = None) -> int:
     )
     cat.add_argument("--bucket", help="override the S3 bucket")
 
-    hdb = sub.add_parser(
-        "harvest-db",
-        help="read a bot's cited URLs from the platform Postgres database (operator)",
-    )
-    grp = hdb.add_mutually_exclusive_group(required=True)
-    grp.add_argument("--post", help="harvest one post id")
-    grp.add_argument("--days", type=int, help="harvest the most recent N days")
-    hdb.add_argument(
-        "--limit",
-        type=int,
-        default=None,
-        help="cap rows when using --days (default: uncapped — a daily sweep wants all)",
-    )
-    hdb.add_argument(
-        "--public-only",
-        action="store_true",
-        help="read only public comments (default: read all of a bot's comments)",
-    )
-    hdb.add_argument(
-        "--dsn",
-        help="libpq DSN or postgresql:// URL. Default resolution: --dsn > "
-        "$METACULUS_DB_DSN > macOS Keychain item 'metaculus-db-dsn' > "
-        "dbname=metaculus. Prefer the Keychain for the real secret "
-        "(a --dsn value lands in shell history).",
-    )
-    hdb.add_argument("--out", metavar="FILE", help="write the manifest to this .jsonl")
-    hdb.add_argument("--run-id", help="run id (default derived from the slice)")
-    hdb.add_argument(
-        "--dedupe", action="store_true", help="keep one record per URL (first seen)"
-    )
-    hdb.add_argument(
-        "--upload", action="store_true", help="upload the manifest to S3 manifests/"
-    )
-    hdb.add_argument("--bucket", help="override the S3 bucket")
-
     cov = sub.add_parser(
         "coverage",
         help="report what %% of cited sources were archived (trace vs comments)",
@@ -396,8 +323,6 @@ def main(argv: list[str] | None = None) -> int:
         return _cmd_capture(args, config)
     if args.command == "ingest-traces":
         return _cmd_ingest_traces(args, config)
-    if args.command == "harvest-db":
-        return _cmd_harvest_db(args, config)
     if args.command == "catalog":
         return _cmd_catalog(args, config)
     if args.command == "coverage":

forecasting_tools/agents_and_tools/source_archive/ingest/__init__.py

@@ -4,14 +4,9 @@
 from a bot's published reasoning:
 
   - :mod:`url_extraction` — pull URLs out of free text / markdown.
-  - :mod:`metaculus_db` — read a bot's cited URLs from the platform database.
   - :mod:`trace_extraction` — build a manifest from a traced bot run (fullest path).
 """
 
-from forecasting_tools.agents_and_tools.source_archive.ingest.metaculus_db import (
-    MetaculusDbHarvester,
-    resolve_dsn,
-)
 from forecasting_tools.agents_and_tools.source_archive.ingest.trace_extraction import (
     extract_records_from_events,
     extract_records_from_question_dir,
@@ -25,13 +20,11 @@
 )
 
 __all__ = [
-    "MetaculusDbHarvester",
     "dedupe_records",
     "extract_citation_records",
     "extract_records_from_events",
     "extract_records_from_question_dir",
     "extract_records_from_trace_file",
     "extract_urls",
     "harvest_run",
-    "resolve_dsn",
 ]