Add configurable discovery

CHANGELOG.md

@@ -1,5 +1,25 @@
 # Changelog
 
+## v1.5.0
+
+### Added
+- Support for datamasque-python 1.1.8.
+  - `dm discover schema-results` handles matches with no label.
+  - Validation errors are now printed.
+  - `dm rulesets validate` and `dm libraries validate` now fail (return 4)
+    on invalid rulesets/libraries.
+  - `dm discover db-report` writes a zip archive returned for large reports to
+    `--output`, aborting with a hint rather than dumping binary data to stdout.
+- Support for Configurable Discovery:
+  - `dm discover configs` — list, get, defaults, create, delete, and validate
+    discovery configs (`database` or `file`).
+  - `dm discover libraries` — list, get, create, delete, and validate discovery
+    config libraries.
+  - `dm discover schema --config <name>` and `dm discover file
+    [--config <name>]` start discovery runs with or without a specific config.
+  - `dm discover config-snapshot <run-id>` downloads the discovery config a run
+    actually used.
+
 ## v1.4.0
 
 ### Added

README.md

@@ -216,11 +216,36 @@ dm users delete <username>                      # Delete a user
 ### Discovery
 
 ```console
-dm discover schema <connection>                 # Start a schema-discovery run
-dm discover schema-results <run-id>             # List schema-discovery results once the run finishes
-dm discover sdd-report <run-id>                 # Sensitive data discovery report
-dm discover db-report <run-id>                  # Database discovery CSV
-dm discover file-report <run-id>                # File discovery report
+dm discover schema <connection>                      # Schema discovery (built-in keyword-driven)
+dm discover schema <connection> --config <name>      # Schema discovery from a saved database config
+dm discover schema-results <run-id>                  # List schema-discovery results once the run finishes
+dm discover file <connection>                        # File data discovery (built-in keyword-driven)
+dm discover file <connection> --config <name>        # File data discovery from a saved file config
+dm discover sdd-report <run-id>                      # Sensitive data discovery report
+dm discover db-report <run-id>                       # Database discovery CSV
+dm discover file-report <run-id>                     # File discovery report
+dm discover config-snapshot <run-id> -o used.yaml    # Download the discovery config a run actually used
+```
+
+#### Discovery configs
+
+```console
+dm discover configs list [--type database|file]                    # List configs
+dm discover configs get <name> [--type database] [--yaml]          # Show details or raw YAML
+dm discover configs defaults [--type database|file] -o cfg.yaml    # Built-in default as a starting point
+dm discover configs create --name <n> --type database -f cfg.yaml  # Create/update from YAML
+dm discover configs delete <name> [--type database]                # Delete a config
+dm discover configs validate -f cfg.yaml --type database           # Validate a YAML file against the server
+```
+
+#### Discovery config libraries
+
+```console
+dm discover libraries list [--type database|file]
+dm discover libraries get <name> [--type database] [--namespace org] [--yaml]
+dm discover libraries create --name <n> --type database --namespace org -f lib.yaml
+dm discover libraries delete <name> [--type database] [--namespace org] [--force]  # --force if imported by configs
+dm discover libraries validate -f lib.yaml --type database
 ```
 
 ### Seeds

pyproject.toml

@@ -12,7 +12,7 @@ requires-python = ">=3.11"
 dependencies = [
     "typer>=0.15.0",
     "tomli-w>=1.0.0",
-    "datamasque-python>=1.0.0,<2",
+    "datamasque-python>=1.1.8,<2",
 ]
 classifiers = [
     "Development Status :: 4 - Beta",

src/datamasque_cli/commands/discovery.py

@@ -8,12 +8,21 @@
 import typer
 from datamasque.client import DataMasqueClient, RunId
 from datamasque.client.models.connection import ConnectionId
-from datamasque.client.models.discovery import SchemaDiscoveryRequest
+from datamasque.client.models.discovery import (
+    FileDataDiscoveryFromConfigRequest,
+    FileDataDiscoveryRequest,
+    SchemaDiscoveryFromConfigRequest,
+    SchemaDiscoveryRequest,
+)
+from datamasque.client.models.discovery_config import DiscoveryConfigId, DiscoveryConfigType
 
 from datamasque_cli.client import get_client
+from datamasque_cli.commands import discovery_config_libraries, discovery_configs
 from datamasque_cli.output import ErrorCode, abort, print_json, print_success, render_output, should_emit_json
 
 app = typer.Typer(help="Data discovery operations.", no_args_is_help=True)
+app.add_typer(discovery_configs.app, name="configs")
+app.add_typer(discovery_config_libraries.app, name="libraries")
 
 
 def _write_or_echo(content: str, output: Path | None, success_label: str) -> None:
@@ -33,9 +42,40 @@ def _resolve_connection_id(client: DataMasqueClient, name_or_id: str) -> str:
     return str(match.id)
 
 
+def _resolve_discovery_config_id(
+    client: DataMasqueClient, name: str, expected_type: DiscoveryConfigType
+) -> DiscoveryConfigId:
+    """Resolve a discovery config name to its UUID, requiring it to be of `expected_type`."""
+    named = [c for c in client.list_discovery_configs() if c.name == name]
+    matches = [c for c in named if c.config_type is expected_type]
+
+    if not matches:
+        if named:
+            existing = ", ".join(c.config_type.value for c in named)
+            abort(
+                f"Discovery config '{name}' exists as {existing}, "
+                f"but {expected_type.value} discovery needs a {expected_type.value} config.",
+                code=ErrorCode.INVALID_INPUT,
+            )
+        abort(f"Discovery config '{name}' not found.", code=ErrorCode.NOT_FOUND)
+    if len(matches) > 1:
+        options = "\n  ".join(f"id={c.id}" for c in matches)
+        abort(
+            f"Multiple {expected_type.value} discovery configs named '{name}':\n  {options}",
+            code=ErrorCode.AMBIGUOUS,
+        )
+
+    config_id = matches[0].id
+    assert config_id is not None
+    return config_id
+
+
 @app.command("schema")
 def schema_discovery(
     connection: str = typer.Argument(help="Connection name or ID"),
+    config: str | None = typer.Option(
+        None, "--config", "-c", help="Run with a saved database discovery config (configurable discovery)"
+    ),
     profile: str | None = typer.Option(None, "--profile", "-p", help="Profile to use"),
 ) -> None:
     """Start a schema-discovery run on a connection.
@@ -47,14 +87,54 @@ def schema_discovery(
     client = get_client(profile)
     conn_id = _resolve_connection_id(client, connection)
 
-    request = SchemaDiscoveryRequest(connection=ConnectionId(conn_id))
-    run_id = client.start_schema_discovery_run(request)
+    if config is not None:
+        config_id = _resolve_discovery_config_id(client, config, DiscoveryConfigType.database)
+        from_config = SchemaDiscoveryFromConfigRequest(connection=ConnectionId(conn_id), discovery_config=config_id)
+        run_id = client.start_schema_discovery_run_from_config(from_config)
+        source = f"config '{config}'"
+    else:
+        request = SchemaDiscoveryRequest(connection=ConnectionId(conn_id))
+        run_id = client.start_schema_discovery_run(request)
+        source = "default discovery"
+
     print_success(
-        f"Schema discovery run {run_id} started for connection '{connection}'. "
+        f"Schema discovery run {run_id} started for connection '{connection}' ({source}). "
         f"Once finished, list results with: dm discover schema-results {run_id}"
     )
 
 
+@app.command("file")
+def file_discovery(
+    connection: str = typer.Argument(help="Connection name or ID"),
+    config: str | None = typer.Option(
+        None, "--config", "-c", help="Run with a saved file discovery config (configurable discovery)"
+    ),
+    profile: str | None = typer.Option(None, "--profile", "-p", help="Profile to use"),
+) -> None:
+    """Start a file-data-discovery run on a file connection.
+
+    Once finished, download the report with `dm discover file-report <run-id>`
+    (poll with `dm run status <run-id>`).
+    """
+    client = get_client(profile)
+    conn_id = _resolve_connection_id(client, connection)
+
+    if config is not None:
+        config_id = _resolve_discovery_config_id(client, config, DiscoveryConfigType.file)
+        from_config = FileDataDiscoveryFromConfigRequest(connection=ConnectionId(conn_id), discovery_config=config_id)
+        run_id = client.start_file_data_discovery_run_from_config(from_config)
+        source = f"config '{config}'"
+    else:
+        request = FileDataDiscoveryRequest(connection=ConnectionId(conn_id))
+        run_id = client.start_file_data_discovery_run(request)
+        source = "default discovery"
+
+    print_success(
+        f"File data discovery run {run_id} started for connection '{connection}' ({source}). "
+        f"Once finished, download the report with: dm discover file-report {run_id}"
+    )
+
+
 @app.command("schema-results")
 def schema_results(
     run_id: int = typer.Argument(help="Schema discovery run ID"),
@@ -77,7 +157,7 @@ def schema_results(
             "table": r.table,
             "column": r.column,
             "data_type": r.data.data_type or "",
-            "matches": ", ".join(m.label for m in r.data.discovery_matches) or "-",
+            "matches": ", ".join(m.label for m in r.data.discovery_matches if m.label) or "-",
             "constraint": r.data.constraint or "",
         }
         for r in results
@@ -111,6 +191,19 @@ def db_discovery_report(
     """Download database discovery report (CSV) for a run."""
     client = get_client(profile)
     report = client.get_db_discovery_result_report(RunId(run_id))
+
+    if isinstance(report, bytes):
+        if output is None:
+            abort(
+                f"Database discovery report for run {run_id} is a zip archive of CSV parts; "
+                "refusing to write binary data to stdout.",
+                code=ErrorCode.INVALID_INPUT,
+                hint="Pass --output <file>.zip to save it.",
+            )
+        output.write_bytes(report)
+        print_success(f"Database discovery report written to {output}")
+        return
+
     _write_or_echo(report, output, "Database discovery report")
 
 
@@ -134,3 +227,15 @@ def file_discovery_report(
         print_json(report)
     else:
         render_output(report, is_json=False, title=f"File Discovery: Run {run_id}")
+
+
+@app.command("config-snapshot")
+def config_snapshot(
+    run_id: int = typer.Argument(help="Discovery run ID"),
+    output: Path | None = typer.Option(None, "--output", "-o", help="Write YAML to this path"),
+    profile: str | None = typer.Option(None, "--profile", "-p", help="Profile to use"),
+) -> None:
+    """Download the discovery config a run used (the run's snapshot)."""
+    client = get_client(profile)
+    snapshot = client.get_discovery_run_config_snapshot_yaml(RunId(run_id))
+    _write_or_echo(snapshot, output, "Discovery config snapshot")