diff --git a/CHANGELOG.md b/CHANGELOG.md index 3205275..c44b3f9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,27 @@ # Changelog +## v1.5.0 + +### Added +- Support for datamasque-python 1.1.8. + - `dm discover schema-results` handles matches with no label. + - Validation errors are now printed. + - `dm rulesets validate` and `dm libraries validate` now fail (return 4) + on invalid rulesets/libraries. + - `dm discover db-report` writes a zip archive returned for large reports to + `--output`, aborting with a hint rather than dumping binary data to stdout. +- Support for Configurable Discovery: + - `dm discover configs` — list, get, defaults, create, delete, and validate + discovery configs (`database` or `file`). + - `dm discover libraries` — list, get, create, delete, and validate discovery + config libraries. + - `dm discover schema --config ` and `dm discover file + [--config ]` start discovery runs with or without a specific config. + - `dm discover config-snapshot ` downloads the discovery config a run + actually used. +- Safe Data Preview: `dm discover schema-results` and `dm discover file-report` + include `safe_data_preview` in their `--json` output. + ## v1.4.0 ### Added diff --git a/README.md b/README.md index 07fe69f..bc9c940 100644 --- a/README.md +++ b/README.md @@ -216,11 +216,36 @@ dm users delete # Delete a user ### Discovery ```console -dm discover schema # Start a schema-discovery run -dm discover schema-results # List schema-discovery results once the run finishes -dm discover sdd-report # Sensitive data discovery report -dm discover db-report # Database discovery CSV -dm discover file-report # File discovery report +dm discover schema # Schema discovery (built-in keyword-driven) +dm discover schema --config # Schema discovery from a saved database config +dm discover schema-results # List schema-discovery results once the run finishes +dm discover file # File data discovery (built-in keyword-driven) +dm discover file --config # File data discovery from a saved file config +dm discover sdd-report # Sensitive data discovery report +dm discover db-report # Database discovery CSV +dm discover file-report # File discovery report +dm discover config-snapshot -o used.yaml # Download the discovery config a run actually used +``` + +#### Discovery configs + +```console +dm discover configs list [--type database|file] # List configs +dm discover configs get [--type database] [--yaml] # Show details or raw YAML +dm discover configs defaults [--type database|file] -o cfg.yaml # Built-in default as a starting point +dm discover configs create --name --type database -f cfg.yaml # Create/update from YAML +dm discover configs delete [--type database] # Delete a config +dm discover configs validate -f cfg.yaml --type database # Validate a YAML file against the server +``` + +#### Discovery config libraries + +```console +dm discover libraries list [--type database|file] +dm discover libraries get [--type database] [--namespace org] [--yaml] +dm discover libraries create --name --type database --namespace org -f lib.yaml +dm discover libraries delete [--type database] [--namespace org] [--force] # --force if imported by configs +dm discover libraries validate -f lib.yaml --type database ``` ### Seeds diff --git a/claude-skills/datamasque-cli/skills/datamasque-cli/SKILL.md b/claude-skills/datamasque-cli/skills/datamasque-cli/SKILL.md index ad70cda..afd74e6 100644 --- a/claude-skills/datamasque-cli/skills/datamasque-cli/SKILL.md +++ b/claude-skills/datamasque-cli/skills/datamasque-cli/SKILL.md @@ -89,6 +89,14 @@ Pass repeated `--options key=value` for server-side knobs then fetch results with `dm discover schema-results ` / `sdd-report` / `db-report` / `file-report`. +- **Configurable discovery and Safe Data Preview.** Save a discovery config + with `dm discover configs create` (start from `dm discover configs defaults`), + then run `dm discover schema --config `. When the config + enables in-data discovery with safe data preview, `dm discover schema-results + --json` carries a `safe_data_preview` per column — value distributions, + patterns, and cardinality worth reading before choosing masks. It is JSON-only; + `file-report --json` exposes the same per locator. + - **`dm rulesets validate --file --type `** runs server-side validation without committing the ruleset. Use this before `create` when you want a clean failure mode for bad YAML. diff --git a/pyproject.toml b/pyproject.toml index b4dfa47..f729be0 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -12,7 +12,7 @@ requires-python = ">=3.11" dependencies = [ "typer>=0.15.0", "tomli-w>=1.0.0", - "datamasque-python>=1.0.0,<2", + "datamasque-python>=1.1.8,<2", ] classifiers = [ "Development Status :: 4 - Beta", diff --git a/src/datamasque_cli/commands/discovery.py b/src/datamasque_cli/commands/discovery.py index 9db83af..0698671 100644 --- a/src/datamasque_cli/commands/discovery.py +++ b/src/datamasque_cli/commands/discovery.py @@ -8,12 +8,21 @@ import typer from datamasque.client import DataMasqueClient, RunId from datamasque.client.models.connection import ConnectionId -from datamasque.client.models.discovery import SchemaDiscoveryRequest +from datamasque.client.models.discovery import ( + FileDataDiscoveryFromConfigRequest, + FileDataDiscoveryRequest, + SchemaDiscoveryFromConfigRequest, + SchemaDiscoveryRequest, +) +from datamasque.client.models.discovery_config import DiscoveryConfigId, DiscoveryConfigType from datamasque_cli.client import get_client +from datamasque_cli.commands import discovery_config_libraries, discovery_configs from datamasque_cli.output import ErrorCode, abort, print_json, print_success, render_output, should_emit_json app = typer.Typer(help="Data discovery operations.", no_args_is_help=True) +app.add_typer(discovery_configs.app, name="configs") +app.add_typer(discovery_config_libraries.app, name="libraries") def _write_or_echo(content: str, output: Path | None, success_label: str) -> None: @@ -33,9 +42,40 @@ def _resolve_connection_id(client: DataMasqueClient, name_or_id: str) -> str: return str(match.id) +def _resolve_discovery_config_id( + client: DataMasqueClient, name: str, expected_type: DiscoveryConfigType +) -> DiscoveryConfigId: + """Resolve a discovery config name to its UUID, requiring it to be of `expected_type`.""" + named = [c for c in client.list_discovery_configs() if c.name == name] + matches = [c for c in named if c.config_type is expected_type] + + if not matches: + if named: + existing = ", ".join(c.config_type.value for c in named) + abort( + f"Discovery config '{name}' exists as {existing}, " + f"but {expected_type.value} discovery needs a {expected_type.value} config.", + code=ErrorCode.INVALID_INPUT, + ) + abort(f"Discovery config '{name}' not found.", code=ErrorCode.NOT_FOUND) + if len(matches) > 1: + options = "\n ".join(f"id={c.id}" for c in matches) + abort( + f"Multiple {expected_type.value} discovery configs named '{name}':\n {options}", + code=ErrorCode.AMBIGUOUS, + ) + + config_id = matches[0].id + assert config_id is not None + return config_id + + @app.command("schema") def schema_discovery( connection: str = typer.Argument(help="Connection name or ID"), + config: str | None = typer.Option( + None, "--config", "-c", help="Run with a saved database discovery config (configurable discovery)" + ), profile: str | None = typer.Option(None, "--profile", "-p", help="Profile to use"), ) -> None: """Start a schema-discovery run on a connection. @@ -47,14 +87,54 @@ def schema_discovery( client = get_client(profile) conn_id = _resolve_connection_id(client, connection) - request = SchemaDiscoveryRequest(connection=ConnectionId(conn_id)) - run_id = client.start_schema_discovery_run(request) + if config is not None: + config_id = _resolve_discovery_config_id(client, config, DiscoveryConfigType.database) + from_config = SchemaDiscoveryFromConfigRequest(connection=ConnectionId(conn_id), discovery_config=config_id) + run_id = client.start_schema_discovery_run_from_config(from_config) + source = f"config '{config}'" + else: + request = SchemaDiscoveryRequest(connection=ConnectionId(conn_id)) + run_id = client.start_schema_discovery_run(request) + source = "default discovery" + print_success( - f"Schema discovery run {run_id} started for connection '{connection}'. " + f"Schema discovery run {run_id} started for connection '{connection}' ({source}). " f"Once finished, list results with: dm discover schema-results {run_id}" ) +@app.command("file") +def file_discovery( + connection: str = typer.Argument(help="Connection name or ID"), + config: str | None = typer.Option( + None, "--config", "-c", help="Run with a saved file discovery config (configurable discovery)" + ), + profile: str | None = typer.Option(None, "--profile", "-p", help="Profile to use"), +) -> None: + """Start a file-data-discovery run on a file connection. + + Once finished, download the report with `dm discover file-report ` + (poll with `dm run status `). + """ + client = get_client(profile) + conn_id = _resolve_connection_id(client, connection) + + if config is not None: + config_id = _resolve_discovery_config_id(client, config, DiscoveryConfigType.file) + from_config = FileDataDiscoveryFromConfigRequest(connection=ConnectionId(conn_id), discovery_config=config_id) + run_id = client.start_file_data_discovery_run_from_config(from_config) + source = f"config '{config}'" + else: + request = FileDataDiscoveryRequest(connection=ConnectionId(conn_id)) + run_id = client.start_file_data_discovery_run(request) + source = "default discovery" + + print_success( + f"File data discovery run {run_id} started for connection '{connection}' ({source}). " + f"Once finished, download the report with: dm discover file-report {run_id}" + ) + + @app.command("schema-results") def schema_results( run_id: int = typer.Argument(help="Schema discovery run ID"), @@ -77,8 +157,11 @@ def schema_results( "table": r.table, "column": r.column, "data_type": r.data.data_type or "", - "matches": ", ".join(m.label for m in r.data.discovery_matches) or "-", + "matches": ", ".join(m.label for m in r.data.discovery_matches if m.label) or "-", "constraint": r.data.constraint or "", + "safe_data_preview": ( + r.data.safe_data_preview.model_dump(mode="json") if r.data.safe_data_preview else None + ), } for r in results ] @@ -111,6 +194,19 @@ def db_discovery_report( """Download database discovery report (CSV) for a run.""" client = get_client(profile) report = client.get_db_discovery_result_report(RunId(run_id)) + + if isinstance(report, bytes): + if output is None: + abort( + f"Database discovery report for run {run_id} is a zip archive of CSV parts; " + "refusing to write binary data to stdout.", + code=ErrorCode.INVALID_INPUT, + hint="Pass --output .zip to save it.", + ) + output.write_bytes(report) + print_success(f"Database discovery report written to {output}") + return + _write_or_echo(report, output, "Database discovery report") @@ -124,13 +220,43 @@ def file_discovery_report( """Download file discovery report for a run.""" client = get_client(profile) report = client.get_file_data_discovery_report(RunId(run_id)) + full = [result.model_dump(mode="json") for result in report] if output is not None: - output.write_text(json.dumps(report, indent=2, default=str)) + output.write_text(json.dumps(full, indent=2, default=str)) print_success(f"File discovery report written to {output}") return if should_emit_json(is_json): - print_json(report) - else: - render_output(report, is_json=False, title=f"File Discovery: Run {run_id}") + print_json(full) + return + + rows = [ + { + "id": result.id, + "files": ", ".join(f.path for f in result.files), + "locator": locator.locator, + "matches": ", ".join(m.label for m in locator.matches if m.label) or "-", + "data_types": ", ".join(locator.data_types) or "-", + } + for result in report + for locator in result.results + ] + render_output( + rows, + is_json=False, + columns=["id", "files", "locator", "matches", "data_types"], + title=f"File Discovery: Run {run_id}", + ) + + +@app.command("config-snapshot") +def config_snapshot( + run_id: int = typer.Argument(help="Discovery run ID"), + output: Path | None = typer.Option(None, "--output", "-o", help="Write YAML to this path"), + profile: str | None = typer.Option(None, "--profile", "-p", help="Profile to use"), +) -> None: + """Download the discovery config a run used (the run's snapshot).""" + client = get_client(profile) + snapshot = client.get_discovery_run_config_snapshot_yaml(RunId(run_id)) + _write_or_echo(snapshot, output, "Discovery config snapshot") diff --git a/src/datamasque_cli/commands/discovery_config_libraries.py b/src/datamasque_cli/commands/discovery_config_libraries.py new file mode 100644 index 0000000..d375f65 --- /dev/null +++ b/src/datamasque_cli/commands/discovery_config_libraries.py @@ -0,0 +1,217 @@ +"""Discovery config library management commands (configurable discovery).""" + +from __future__ import annotations + +from pathlib import Path + +import typer +from datamasque.client import DataMasqueClient +from datamasque.client.models.discovery_config import DiscoveryConfigType +from datamasque.client.models.discovery_config_library import DiscoveryConfigLibrary +from datamasque.client.models.status import ValidationStatus + +from datamasque_cli.client import get_client +from datamasque_cli.output import ErrorCode, abort, print_info, print_success, render_output + +app = typer.Typer(help="Manage discovery config libraries (configurable discovery).", no_args_is_help=True) + + +def _label(name: str, namespace: str) -> str: + """Render a library's display label as `namespace/name`, or bare `name` in the default namespace.""" + return f"{namespace}/{name}" if namespace else name + + +def _find_by_name( + client: DataMasqueClient, + name: str, + config_type: DiscoveryConfigType | None = None, + namespace: str | None = None, +) -> list[DiscoveryConfigLibrary]: + """Return all libraries matching `name`, optionally narrowed by `namespace` and `config_type`.""" + matches = [lib for lib in client.list_discovery_config_libraries() if lib.name == name] + if namespace is not None: + matches = [lib for lib in matches if lib.namespace == namespace] + if config_type is not None: + matches = [lib for lib in matches if lib.config_type is config_type] + return matches + + +def _pick_single(matches: list[DiscoveryConfigLibrary], name: str) -> DiscoveryConfigLibrary: + """Return the sole match or abort with a disambiguation message.""" + if not matches: + abort(f"Discovery config library '{name}' not found.", code=ErrorCode.NOT_FOUND) + if len(matches) > 1: + options = "\n ".join( + f"id={lib.id} namespace={lib.namespace or '(default)'} type={lib.config_type.value}" for lib in matches + ) + abort( + f"Multiple discovery config libraries named '{name}':\n {options}", + code=ErrorCode.AMBIGUOUS, + hint="Pass --type file|database and/or --namespace to disambiguate.", + ) + return matches[0] + + +@app.command("list") +def list_libraries( + config_type: str | None = typer.Option(None, "--type", "-t", help="Filter by type: database or file"), + profile: str | None = typer.Option(None, "--profile", "-p", help="Profile to use"), + is_json: bool = typer.Option(False, "--json", help="Output as JSON"), +) -> None: + """List all discovery config libraries.""" + client = get_client(profile) + libraries = client.list_discovery_config_libraries() + + if config_type is not None: + wanted = DiscoveryConfigType(config_type) + libraries = [lib for lib in libraries if lib.config_type is wanted] + + data = [ + { + "id": lib.id, + "namespace": lib.namespace or "", + "name": lib.name, + "type": lib.config_type.value, + "valid": lib.is_valid.value if lib.is_valid else "unknown", + } + for lib in libraries + ] + + render_output( + data, + is_json=is_json, + columns=["id", "namespace", "name", "type", "valid"], + title="Discovery Config Libraries", + ) + + +@app.command("get") +def get_library( + name: str = typer.Argument(help="Library name"), + config_type: str | None = typer.Option(None, "--type", "-t", help="Required when two libraries share a name"), + namespace: str = typer.Option("", "--namespace", "-n", help="Library namespace"), + profile: str | None = typer.Option(None, "--profile", "-p", help="Profile to use"), + is_yaml: bool = typer.Option(False, "--yaml", help="Output raw YAML content only"), + is_json: bool = typer.Option(False, "--json", help="Output as JSON"), +) -> None: + """Show a discovery config library's details or YAML content.""" + client = get_client(profile) + wanted = DiscoveryConfigType(config_type) if config_type is not None else None + match = _pick_single(_find_by_name(client, name, wanted, namespace), name) + + # `list_discovery_config_libraries` omits the YAML body; fetch the single library for it. + assert match.id is not None + full = client.get_discovery_config_library(match.id) + + if is_yaml: + typer.echo(full.yaml) + return + + data: dict[str, object] = { + "id": full.id, + "namespace": full.namespace, + "name": full.name, + "type": full.config_type.value, + "valid": full.is_valid.value if full.is_valid else "unknown", + "created": full.created, + "modified": full.modified, + } + render_output(data, is_json=is_json, title=f"Discovery Config Library: {full.name}") + + +@app.command("create") +def create_library( + name: str = typer.Option(..., help="Library name"), + file: Path = typer.Option(..., "--file", "-f", help="Path to YAML library file", exists=True, readable=True), + config_type: str | None = typer.Option( + None, + "--type", + "-t", + help=( + "Config type: database or file. " + "Required when the library does not yet exist; defaults to the existing type on updates." + ), + ), + namespace: str = typer.Option("", "--namespace", "-n", help="Library namespace"), + profile: str | None = typer.Option(None, "--profile", "-p", help="Profile to use"), +) -> None: + """Create or update a discovery config library from a YAML file.""" + client = get_client(profile) + existing = _find_by_name(client, name, namespace=namespace) + explicit = DiscoveryConfigType(config_type) if config_type is not None else None + + if explicit is not None: + lib_type = explicit + elif len(existing) == 1: + lib_type = existing[0].config_type + print_info(f"Updating existing {lib_type.value}-type library '{_label(name, namespace)}'.") + elif not existing: + abort( + f"No discovery config library named '{_label(name, namespace)}' exists.", + code=ErrorCode.NOT_FOUND, + hint="Pass --type file|database to create a new one.", + ) + else: + options = ", ".join(lib.config_type.value for lib in existing) + abort( + f"Multiple libraries named '{_label(name, namespace)}' ({options}).", + code=ErrorCode.AMBIGUOUS, + hint="Pass --type file|database to pick which one to update.", + ) + + yaml_content = file.read_text() + library = DiscoveryConfigLibrary(name=name, namespace=namespace, yaml=yaml_content, config_type=lib_type) + client.create_or_update_discovery_config_library(library) + print_success(f"Discovery config library '{_label(name, namespace)}' ({lib_type.value}) created/updated.") + + +@app.command("delete") +def delete_library( + name: str = typer.Argument(help="Library name to delete"), + config_type: str | None = typer.Option(None, "--type", "-t", help="Required when two libraries share a name"), + namespace: str = typer.Option("", "--namespace", "-n", help="Library namespace"), + force: bool = typer.Option(False, "--force", help="Force delete even if imported by discovery configs"), + profile: str | None = typer.Option(None, "--profile", "-p", help="Profile to use"), + is_confirmed: bool = typer.Option(False, "--yes", "-y", help="Skip confirmation"), +) -> None: + """Delete a discovery config library by name. + + If the library is imported by any discovery configs, + the server rejects the delete unless --force is passed. + """ + client = get_client(profile) + wanted = DiscoveryConfigType(config_type) if config_type is not None else None + match = _pick_single(_find_by_name(client, name, wanted, namespace), name) + label = _label(name, namespace) + + if not is_confirmed: + typer.confirm(f"Delete discovery config library '{label}' ({match.config_type.value})?", abort=True) + + assert match.id is not None + client.delete_discovery_config_library_by_id_if_exists(match.id, force=force) + print_success(f"Discovery config library '{label}' deleted.") + + +@app.command("validate") +def validate_library( + file: Path = typer.Option(..., "--file", "-f", help="Path to YAML library file", exists=True, readable=True), + config_type: str = typer.Option(..., "--type", "-t", help="Config type: database or file"), + namespace: str = typer.Option("", "--namespace", "-n", help="Library namespace"), + profile: str | None = typer.Option(None, "--profile", "-p", help="Profile to use"), +) -> None: + """Validate a discovery config library YAML file against the DataMasque server.""" + yaml_content = file.read_text() + lib_type = DiscoveryConfigType(config_type) + + client = get_client(profile) + library = DiscoveryConfigLibrary(name=file.stem, namespace=namespace, yaml=yaml_content, config_type=lib_type) + validated = client.validate_discovery_config_library(library) + + if validated.is_valid is ValidationStatus.invalid: + abort( + f'Discovery config library "{file.name}" is invalid: {validated.validation_error}', + code=ErrorCode.INVALID_INPUT, + ) + + status = validated.is_valid.value if validated.is_valid else "unknown" + print_success(f'Discovery config library "{file.name}" validation status: {status}') diff --git a/src/datamasque_cli/commands/discovery_configs.py b/src/datamasque_cli/commands/discovery_configs.py new file mode 100644 index 0000000..1f26939 --- /dev/null +++ b/src/datamasque_cli/commands/discovery_configs.py @@ -0,0 +1,213 @@ +"""Discovery config management commands (configurable discovery).""" + +from __future__ import annotations + +from pathlib import Path + +import typer +from datamasque.client import DataMasqueClient +from datamasque.client.models.discovery_config import DiscoveryConfig, DiscoveryConfigType +from datamasque.client.models.status import ValidationStatus + +from datamasque_cli.client import get_client +from datamasque_cli.output import ErrorCode, abort, print_info, print_success, render_output + +app = typer.Typer(help="Manage discovery configs (configurable discovery).", no_args_is_help=True) + + +def _find_by_name( + client: DataMasqueClient, + name: str, + config_type: DiscoveryConfigType | None = None, +) -> list[DiscoveryConfig]: + """Return all discovery configs matching `name`, optionally narrowed by `config_type`.""" + matches = [c for c in client.list_discovery_configs() if c.name == name] + if config_type is not None: + matches = [c for c in matches if c.config_type is config_type] + return matches + + +def _pick_single(matches: list[DiscoveryConfig], name: str) -> DiscoveryConfig: + """Return the sole match or abort with a disambiguation message.""" + if not matches: + abort(f"Discovery config '{name}' not found.", code=ErrorCode.NOT_FOUND) + if len(matches) > 1: + options = "\n ".join(f"id={c.id} type={c.config_type.value}" for c in matches) + abort( + f"Multiple discovery configs named '{name}':\n {options}", + code=ErrorCode.AMBIGUOUS, + hint="Pass --type file|database to disambiguate.", + ) + return matches[0] + + +@app.command("list") +def list_configs( + config_type: str | None = typer.Option(None, "--type", "-t", help="Filter by type: database or file"), + profile: str | None = typer.Option(None, "--profile", "-p", help="Profile to use"), + is_json: bool = typer.Option(False, "--json", help="Output as JSON"), +) -> None: + """List all discovery configs.""" + client = get_client(profile) + configs = client.list_discovery_configs() + + if config_type is not None: + wanted = DiscoveryConfigType(config_type) + configs = [c for c in configs if c.config_type is wanted] + + data = [ + { + "id": c.id, + "name": c.name, + "type": c.config_type.value, + "valid": c.is_valid.value if c.is_valid else "unknown", + } + for c in configs + ] + + render_output(data, is_json=is_json, columns=["id", "name", "type", "valid"], title="Discovery Configs") + + +@app.command("get") +def get_config( + name: str = typer.Argument(help="Discovery config name"), + config_type: str | None = typer.Option(None, "--type", "-t", help="Required when two configs share a name"), + profile: str | None = typer.Option(None, "--profile", "-p", help="Profile to use"), + is_yaml: bool = typer.Option(False, "--yaml", help="Output raw YAML content only"), + is_json: bool = typer.Option(False, "--json", help="Output as JSON"), +) -> None: + """Show a discovery config's details or YAML content.""" + client = get_client(profile) + wanted = DiscoveryConfigType(config_type) if config_type is not None else None + match = _pick_single(_find_by_name(client, name, wanted), name) + + assert match.id is not None + full = client.get_discovery_config(match.id) + + if is_yaml: + typer.echo(full.yaml) + return + + data: dict[str, object] = { + "id": full.id, + "name": full.name, + "type": full.config_type.value, + "valid": full.is_valid.value if full.is_valid else "unknown", + "created": full.created, + "modified": full.modified, + } + render_output(data, is_json=is_json, title=f"Discovery Config: {full.name}") + + +@app.command("defaults") +def config_defaults( + config_type: str = typer.Option("database", "--type", "-t", help="Config type: database or file"), + output: Path | None = typer.Option(None, "--output", "-o", help="Write YAML to this path"), + profile: str | None = typer.Option(None, "--profile", "-p", help="Profile to use"), +) -> None: + """Print the server's built-in default discovery config as YAML.""" + client = get_client(profile) + wanted = DiscoveryConfigType(config_type) + # `get_default_discovery_config_yaml` takes no config type, so call `make_request` to pass one. + response = client.make_request("GET", "/api/discovery/configs/defaults/", params={"config_type": wanted.value}) + yaml_content = response.content.decode("utf-8") + + if output is not None: + output.write_text(yaml_content) + print_success(f"Default {wanted.value} discovery config written to {output}") + return + + typer.echo(yaml_content) + + +@app.command("create") +def create_config( + name: str = typer.Option(..., help="Discovery config name"), + file: Path = typer.Option(..., "--file", "-f", help="Path to YAML config file", exists=True, readable=True), + config_type: str | None = typer.Option( + None, + "--type", + "-t", + help=( + "Config type: database or file. " + "Required when the config does not yet exist; defaults to the existing type on updates." + ), + ), + profile: str | None = typer.Option(None, "--profile", "-p", help="Profile to use"), +) -> None: + """Create or update a discovery config from a YAML file. + + A brand-new config needs --type because there is no stored row to copy the + type from; an update defaults to whatever the existing row is stored as. + """ + client = get_client(profile) + existing = _find_by_name(client, name) + explicit = DiscoveryConfigType(config_type) if config_type is not None else None + + if explicit is not None: + cfg_type = explicit + elif len(existing) == 1: + cfg_type = existing[0].config_type + print_info(f"Updating existing {cfg_type.value}-type discovery config '{name}'.") + elif not existing: + abort( + f"No discovery config named '{name}' exists.", + code=ErrorCode.NOT_FOUND, + hint="Pass --type file|database to create a new one.", + ) + else: + options = ", ".join(c.config_type.value for c in existing) + abort( + f"Multiple discovery configs named '{name}' ({options}).", + code=ErrorCode.AMBIGUOUS, + hint="Pass --type file|database to pick which one to update.", + ) + + yaml_content = file.read_text() + config = DiscoveryConfig(name=name, yaml=yaml_content, config_type=cfg_type) + client.create_or_update_discovery_config(config) + print_success(f"Discovery config '{name}' ({cfg_type.value}) created/updated.") + + +@app.command("delete") +def delete_config( + name: str = typer.Argument(help="Discovery config name to delete"), + config_type: str | None = typer.Option(None, "--type", "-t", help="Required when two configs share a name"), + profile: str | None = typer.Option(None, "--profile", "-p", help="Profile to use"), + is_confirmed: bool = typer.Option(False, "--yes", "-y", help="Skip confirmation"), +) -> None: + """Delete a discovery config by name.""" + client = get_client(profile) + wanted = DiscoveryConfigType(config_type) if config_type is not None else None + match = _pick_single(_find_by_name(client, name, wanted), name) + + if not is_confirmed: + typer.confirm(f"Delete discovery config '{name}' ({match.config_type.value})?", abort=True) + + assert match.id is not None + client.delete_discovery_config_by_id_if_exists(match.id) + print_success(f"Discovery config '{name}' ({match.config_type.value}) deleted.") + + +@app.command("validate") +def validate_config( + file: Path = typer.Option(..., "--file", "-f", help="Path to YAML config file", exists=True, readable=True), + config_type: str = typer.Option(..., "--type", "-t", help="Config type: database or file"), + profile: str | None = typer.Option(None, "--profile", "-p", help="Profile to use"), +) -> None: + """Validate a discovery config YAML file against the DataMasque server.""" + yaml_content = file.read_text() + cfg_type = DiscoveryConfigType(config_type) + + client = get_client(profile) + config = DiscoveryConfig(name=file.stem, yaml=yaml_content, config_type=cfg_type) + validated = client.validate_discovery_config(config) + + if validated.is_valid is ValidationStatus.invalid: + abort( + f'Discovery config "{file.name}" is invalid: {validated.validation_error}', + code=ErrorCode.INVALID_INPUT, + ) + + status = validated.is_valid.value if validated.is_valid else "unknown" + print_success(f'Discovery config "{file.name}" validation status: {status}') diff --git a/src/datamasque_cli/commands/ruleset_libraries.py b/src/datamasque_cli/commands/ruleset_libraries.py index 675b73b..b34e960 100644 --- a/src/datamasque_cli/commands/ruleset_libraries.py +++ b/src/datamasque_cli/commands/ruleset_libraries.py @@ -8,7 +8,7 @@ from datamasque.client.models.ruleset_library import RulesetLibrary from datamasque_cli.client import get_client -from datamasque_cli.output import ErrorCode, abort, print_success, render_output +from datamasque_cli.output import ErrorCode, abort, abort_if_invalid, print_success, render_output app = typer.Typer(help="Manage ruleset libraries.", no_args_is_help=True) @@ -114,16 +114,18 @@ def validate_library( Triggers a server-side validation pass on an existing library and reports the result. """ + label = f"{namespace}/{name}" if namespace else name + client = get_client(profile) lib = client.get_ruleset_library_by_name(name, namespace) if lib is None: - label = f"{namespace}/{name}" if namespace else name abort(f"Library '{label}' not found.", code=ErrorCode.NOT_FOUND) validated = client.validate_ruleset_library(lib.id) + abort_if_invalid(f"Library '{label}'", validated.is_valid, validated.validation_errors) + status = validated.is_valid.value if validated.is_valid else "unknown" - label = f"{namespace}/{name}" if namespace else name print_success(f"Library '{label}' validation status: {status}") diff --git a/src/datamasque_cli/commands/rulesets.py b/src/datamasque_cli/commands/rulesets.py index 80100e1..df38f7f 100644 --- a/src/datamasque_cli/commands/rulesets.py +++ b/src/datamasque_cli/commands/rulesets.py @@ -13,7 +13,16 @@ from datamasque.client.models.ruleset import Ruleset, RulesetType from datamasque_cli.client import get_client -from datamasque_cli.output import ErrorCode, abort, print_error, print_info, print_success, print_warning, render_output +from datamasque_cli.output import ( + ErrorCode, + abort, + abort_if_invalid, + print_error, + print_info, + print_success, + print_warning, + render_output, +) app = typer.Typer(help="Manage masking rulesets.", no_args_is_help=True) @@ -204,6 +213,7 @@ def validate_ruleset( # `try/finally` so a Ctrl-C or unexpected exception between create and # delete still cleans up the temp ruleset on the server. try: + abort_if_invalid(f"Ruleset '{file.name}' ({rs_type.value})", created.is_valid, created.validation_errors) print_success(f"Ruleset '{file.name}' ({rs_type.value}) is valid.") finally: if created.id is not None: diff --git a/src/datamasque_cli/output.py b/src/datamasque_cli/output.py index 58dafaa..5a45819 100644 --- a/src/datamasque_cli/output.py +++ b/src/datamasque_cli/output.py @@ -18,6 +18,7 @@ from typing import Any, NoReturn import typer +from datamasque.client.models.status import ValidationErrorDetails, ValidationStatus from rich.console import Console from rich.table import Table from rich.text import Text @@ -248,3 +249,13 @@ def abort(message: str, *, code: ErrorCode = ErrorCode.ERROR, hint: str | None = if hint: console.print(f"[dim]Hint: {hint}[/dim]") raise SystemExit(EXIT_CODES[code]) + + +def abort_if_invalid(subject: str, is_valid: ValidationStatus | None, errors: list[ValidationErrorDetails]) -> None: + """Print each server-side validation error for `subject` and exit, if it failed validation.""" + if is_valid is not ValidationStatus.invalid and not errors: + return + for error in errors: + location = f" (line {error.line_number})" if error.line_number is not None else "" + print_error(f"{error.message}{location}") + abort(f"{subject} is invalid.", code=ErrorCode.INVALID_INPUT) diff --git a/tests/commands/test_catalog.py b/tests/commands/test_catalog.py index 3083057..3ae1e37 100644 --- a/tests/commands/test_catalog.py +++ b/tests/commands/test_catalog.py @@ -22,6 +22,12 @@ def test_catalog_compact_json_lists_every_subcommand(monkeypatch: pytest.MonkeyP assert "run start" in paths assert "auth login" in paths + # Nested discovery-config groups surface as `discover ` paths. + assert "discover schema" in paths + assert "discover configs list" in paths + assert "discover libraries create" in paths + assert "discover config-snapshot" in paths + def test_catalog_full_includes_options(monkeypatch: pytest.MonkeyPatch, runner: CliRunner) -> None: monkeypatch.setenv("DM_OUTPUT", "json") diff --git a/tests/commands/test_discovery.py b/tests/commands/test_discovery.py index c35218b..56d6a33 100644 --- a/tests/commands/test_discovery.py +++ b/tests/commands/test_discovery.py @@ -1,9 +1,26 @@ from __future__ import annotations +import json from pathlib import Path from types import SimpleNamespace from unittest.mock import MagicMock, patch +from datamasque.client.models.discovery import ( + FileDiscoveryFile, + FileDiscoveryLocatorResult, + FileDiscoveryResult, +) +from datamasque.client.models.discovery_config import DiscoveryConfigType +from datamasque.client.models.runs import RunConnectionRef +from datamasque.client.models.safe_data_preview import ( + CommonStatistics, + LengthsStatistics, + NumericPreview, + NumericStatistics, + NumericSummaries, + StringPreview, + StringStatistics, +) from typer.testing import CliRunner from datamasque_cli.main import app @@ -11,6 +28,24 @@ MODULE = "datamasque_cli.commands.discovery" +def _string_preview() -> StringPreview: + return StringPreview( + statistics_common=CommonStatistics(count_row=100, count_null=0, count_distinct=76), + statistics_kind=StringStatistics( + lengths=LengthsStatistics(min=8, max=30, mean=13.4, median=13.0, most_common=[]), + ), + ) + + +def _numeric_preview() -> NumericPreview: + return NumericPreview( + statistics_common=CommonStatistics(count_row=500, count_null=0, count_distinct=500), + statistics_kind=NumericStatistics( + summaries=NumericSummaries(mean=1.9e8, q1=9e7, q2=2.15e8, q3=2.7e8, p5=4.6e7, p95=2.78e8), + ), + ) + + @patch(f"{MODULE}.get_client") def test_sdd_report_writes_to_output_file(mock_get_client: MagicMock, runner: CliRunner, tmp_path: Path) -> None: client = MagicMock() @@ -49,17 +84,77 @@ def test_db_report_writes_to_output_file(mock_get_client: MagicMock, runner: Cli assert out.read_text() == "header\nrow1\n" +@patch(f"{MODULE}.get_client") +def test_db_report_writes_zip_bytes_to_output_file( + mock_get_client: MagicMock, runner: CliRunner, tmp_path: Path +) -> None: + client = MagicMock() + mock_get_client.return_value = client + client.get_db_discovery_result_report.return_value = b"PK\x03\x04fake-zip-bytes" + + out = tmp_path / "db.zip" + result = runner.invoke(app, ["discover", "db-report", "42", "--output", str(out)]) + + assert result.exit_code == 0 + assert out.read_bytes() == b"PK\x03\x04fake-zip-bytes" + assert "zip" in result.stderr + + +@patch(f"{MODULE}.get_client") +def test_db_report_zip_without_output_aborts_with_hint(mock_get_client: MagicMock, runner: CliRunner) -> None: + client = MagicMock() + mock_get_client.return_value = client + client.get_db_discovery_result_report.return_value = b"PK\x03\x04fake-zip-bytes" + + result = runner.invoke(app, ["discover", "db-report", "42"]) + + assert result.exit_code == 4 # invalid_input + assert "--output" in result.stderr + assert "PK" not in result.stdout + + +def _file_report() -> list[FileDiscoveryResult]: + return [ + FileDiscoveryResult( + id=7, + connection=RunConnectionRef(id="c1", name="myinput"), + file_type="csv", + files=[FileDiscoveryFile(path="data.csv", file_type="csv")], + results=[ + FileDiscoveryLocatorResult( + locator="phone", matches=[], data_types=["int"], safe_data_preview=_numeric_preview() + ), + ], + ), + ] + + @patch(f"{MODULE}.get_client") def test_file_report_writes_json_to_output(mock_get_client: MagicMock, runner: CliRunner, tmp_path: Path) -> None: client = MagicMock() mock_get_client.return_value = client - client.get_file_data_discovery_report.return_value = [{"file": "a"}] + client.get_file_data_discovery_report.return_value = _file_report() out = tmp_path / "file.json" - result = runner.invoke(app, ["discover", "file-report", "42", "--output", str(out)]) + result = runner.invoke(app, ["discover", "file-report", "7", "--output", str(out)]) + + assert result.exit_code == 0 + payload = json.loads(out.read_text()) + assert payload[0]["results"][0]["safe_data_preview"]["kind"] == "numeric" + + +@patch(f"{MODULE}.get_client") +def test_file_report_table_lists_locators(mock_get_client: MagicMock, runner: CliRunner) -> None: + client = MagicMock() + mock_get_client.return_value = client + client.get_file_data_discovery_report.return_value = _file_report() + + result = runner.invoke(app, ["discover", "file-report", "7"]) assert result.exit_code == 0 - assert '"file": "a"' in out.read_text() + assert "phone" in result.stdout + assert "data.csv" in result.stdout + assert "safe_data_preview" not in result.stdout # -- schema discovery trigger --------------------------------------------- @@ -97,6 +192,7 @@ def test_schema_results_lists_with_flattened_rows(mock_get_client: MagicMock, ru data_type="varchar", discovery_matches=[SimpleNamespace(label="EMAIL_ADDRESS")], constraint="", + safe_data_preview=None, ), ), SimpleNamespace( @@ -111,6 +207,7 @@ def test_schema_results_lists_with_flattened_rows(mock_get_client: MagicMock, ru SimpleNamespace(label="PII"), ], constraint="Primary", + safe_data_preview=None, ), ), ] @@ -123,3 +220,165 @@ def test_schema_results_lists_with_flattened_rows(mock_get_client: MagicMock, ru assert '"EMAIL_ADDRESS"' in result.stdout assert '"US_SSN, PII"' in result.stdout assert '"Primary"' in result.stdout + + +@patch(f"{MODULE}.get_client") +def test_schema_results_skips_unlabelled_matches(mock_get_client: MagicMock, runner: CliRunner) -> None: + client = MagicMock() + mock_get_client.return_value = client + client.list_schema_discovery_results.return_value = [ + SimpleNamespace( + id=1, + column="email", + table="users", + schema_name="public", + data=SimpleNamespace( + data_type="varchar", + discovery_matches=[ + SimpleNamespace(label="EMAIL_ADDRESS"), + SimpleNamespace(label=None), + ], + constraint="", + safe_data_preview=None, + ), + ), + SimpleNamespace( + id=2, + column="notes", + table="users", + schema_name="public", + data=SimpleNamespace( + data_type="text", + discovery_matches=[SimpleNamespace(label=None)], + constraint="", + safe_data_preview=None, + ), + ), + ] + + result = runner.invoke(app, ["discover", "schema-results", "42", "--json"]) + + assert result.exit_code == 0 + rows = json.loads(result.stdout) + assert rows[0]["matches"] == "EMAIL_ADDRESS" + assert rows[1]["matches"] == "-" + + +@patch(f"{MODULE}.get_client") +def test_schema_results_includes_safe_data_preview_in_json(mock_get_client: MagicMock, runner: CliRunner) -> None: + client = MagicMock() + mock_get_client.return_value = client + client.list_schema_discovery_results.return_value = [ + SimpleNamespace( + id=1, + column="author", + table="books", + schema_name="public", + data=SimpleNamespace( + data_type="varchar", + discovery_matches=[SimpleNamespace(label="name")], + constraint="", + safe_data_preview=_string_preview(), + ), + ), + ] + + result = runner.invoke(app, ["discover", "schema-results", "42", "--json"]) + + assert result.exit_code == 0 + rows = json.loads(result.stdout) + assert rows[0]["safe_data_preview"]["kind"] == "string" + assert rows[0]["safe_data_preview"]["statistics_kind"]["lengths"]["max"] == 30 + + table = runner.invoke(app, ["discover", "schema-results", "42"]) + assert table.exit_code == 0 + assert "safe_data_preview" not in table.stdout + + +# -- configurable-discovery run triggers ---------------------------------- + + +@patch(f"{MODULE}.get_client") +def test_schema_with_config_runs_from_saved_config(mock_get_client: MagicMock, runner: CliRunner) -> None: + client = MagicMock() + mock_get_client.return_value = client + client.list_connections.return_value = [SimpleNamespace(id="abc-123", name="my_db", mask_type="database")] + client.list_discovery_configs.return_value = [ + SimpleNamespace(id="cfg-1", name="emp", config_type=DiscoveryConfigType.database), + ] + client.start_schema_discovery_run_from_config.return_value = 77 + + result = runner.invoke(app, ["discover", "schema", "my_db", "--config", "emp"]) + + assert result.exit_code == 0 + client.start_schema_discovery_run.assert_not_called() + (call,) = client.start_schema_discovery_run_from_config.call_args_list + (request,) = call.args + assert request.connection == "abc-123" + assert request.discovery_config == "cfg-1" + assert "dm discover schema-results 77" in result.stderr + + +@patch(f"{MODULE}.get_client") +def test_schema_config_wrong_type_aborts(mock_get_client: MagicMock, runner: CliRunner) -> None: + client = MagicMock() + mock_get_client.return_value = client + client.list_connections.return_value = [SimpleNamespace(id="abc-123", name="my_db", mask_type="database")] + client.list_discovery_configs.return_value = [ + SimpleNamespace(id="cfg-2", name="docs", config_type=DiscoveryConfigType.file), + ] + + result = runner.invoke(app, ["discover", "schema", "my_db", "--config", "docs"]) + + assert result.exit_code == 4 # invalid_input + client.start_schema_discovery_run_from_config.assert_not_called() + + +@patch(f"{MODULE}.get_client") +def test_file_without_config_runs_keyword_discovery(mock_get_client: MagicMock, runner: CliRunner) -> None: + client = MagicMock() + mock_get_client.return_value = client + client.list_connections.return_value = [SimpleNamespace(id="fs-1", name="my_files", mask_type="file")] + client.start_file_data_discovery_run.return_value = 88 + + result = runner.invoke(app, ["discover", "file", "my_files"]) + + assert result.exit_code == 0 + client.start_file_data_discovery_run_from_config.assert_not_called() + (call,) = client.start_file_data_discovery_run.call_args_list + (request,) = call.args + assert request.connection == "fs-1" + assert "dm discover file-report 88" in result.stderr + + +@patch(f"{MODULE}.get_client") +def test_file_with_config_runs_from_saved_config(mock_get_client: MagicMock, runner: CliRunner) -> None: + client = MagicMock() + mock_get_client.return_value = client + client.list_connections.return_value = [SimpleNamespace(id="fs-1", name="my_files", mask_type="file")] + client.list_discovery_configs.return_value = [ + SimpleNamespace(id="cfg-3", name="docs", config_type=DiscoveryConfigType.file), + ] + client.start_file_data_discovery_run_from_config.return_value = 89 + + result = runner.invoke(app, ["discover", "file", "my_files", "--config", "docs"]) + + assert result.exit_code == 0 + client.start_file_data_discovery_run.assert_not_called() + (call,) = client.start_file_data_discovery_run_from_config.call_args_list + (request,) = call.args + assert request.discovery_config == "cfg-3" + + +@patch(f"{MODULE}.get_client") +def test_config_snapshot_writes_to_output(mock_get_client: MagicMock, runner: CliRunner, tmp_path: Path) -> None: + client = MagicMock() + mock_get_client.return_value = client + client.get_discovery_run_config_snapshot_yaml.return_value = "# provenance\nlabels: []\n" + + out = tmp_path / "used.yaml" + result = runner.invoke(app, ["discover", "config-snapshot", "42", "--output", str(out)]) + + assert result.exit_code == 0 + assert out.read_text() == "# provenance\nlabels: []\n" + client.get_discovery_run_config_snapshot_yaml.assert_called_once_with(42) diff --git a/tests/commands/test_discovery_config_libraries.py b/tests/commands/test_discovery_config_libraries.py new file mode 100644 index 0000000..ceb2dc7 --- /dev/null +++ b/tests/commands/test_discovery_config_libraries.py @@ -0,0 +1,119 @@ +from __future__ import annotations + +from types import SimpleNamespace +from unittest.mock import MagicMock, patch + +from datamasque.client.models.discovery_config import DiscoveryConfigType +from datamasque.client.models.status import ValidationStatus +from typer.testing import CliRunner + +from datamasque_cli.main import app + +MODULE = "datamasque_cli.commands.discovery_config_libraries" + + +def _library( + name: str, + config_type: DiscoveryConfigType = DiscoveryConfigType.database, + namespace: str = "", + library_id: str = "lib-uuid", + is_valid: ValidationStatus | None = ValidationStatus.valid, + yaml: str | None = None, +) -> SimpleNamespace: + return SimpleNamespace( + id=library_id, + name=name, + namespace=namespace, + config_type=config_type, + is_valid=is_valid, + validation_error=None, + created=None, + modified=None, + yaml=yaml, + ) + + +@patch(f"{MODULE}.get_client") +def test_list_shows_namespace_and_type(mock_get_client: MagicMock, runner: CliRunner) -> None: + client = MagicMock() + mock_get_client.return_value = client + client.list_discovery_config_libraries.return_value = [ + _library("finance", namespace="org"), + ] + + result = runner.invoke(app, ["discover", "libraries", "list", "--json"]) + + assert result.exit_code == 0 + assert '"finance"' in result.stdout + assert '"org"' in result.stdout + + +@patch(f"{MODULE}.get_client") +def test_get_yaml_fetches_full_library(mock_get_client: MagicMock, runner: CliRunner) -> None: + client = MagicMock() + mock_get_client.return_value = client + client.list_discovery_config_libraries.return_value = [_library("finance", namespace="org")] + client.get_discovery_config_library.return_value = _library("finance", namespace="org", yaml="labels: []\n") + + result = runner.invoke(app, ["discover", "libraries", "get", "finance", "--namespace", "org", "--yaml"]) + + assert result.exit_code == 0 + assert "labels: []" in result.stdout + client.get_discovery_config_library.assert_called_once_with("lib-uuid") + + +@patch(f"{MODULE}.get_client") +def test_get_namespace_scopes_lookup(mock_get_client: MagicMock, runner: CliRunner) -> None: + client = MagicMock() + mock_get_client.return_value = client + client.list_discovery_config_libraries.return_value = [_library("finance", namespace="org")] + + result = runner.invoke(app, ["discover", "libraries", "get", "finance"]) + + assert result.exit_code == 3 + client.get_discovery_config_library.assert_not_called() + + +@patch(f"{MODULE}.get_client") +def test_create_new_requires_type(mock_get_client: MagicMock, runner: CliRunner, tmp_path) -> None: + client = MagicMock() + mock_get_client.return_value = client + client.list_discovery_config_libraries.return_value = [] + lib = tmp_path / "lib.yaml" + lib.write_text("labels: []\n") + + result = runner.invoke( + app, + ["discover", "libraries", "create", "--name", "finance", "-n", "org", "-f", str(lib), "--type", "database"], + ) + + assert result.exit_code == 0 + client.create_or_update_discovery_config_library.assert_called_once() + + +@patch(f"{MODULE}.get_client") +def test_delete_force_passes_through(mock_get_client: MagicMock, runner: CliRunner) -> None: + client = MagicMock() + mock_get_client.return_value = client + client.list_discovery_config_libraries.return_value = [_library("finance", namespace="org")] + + result = runner.invoke(app, ["discover", "libraries", "delete", "finance", "-n", "org", "--force", "--yes"]) + + assert result.exit_code == 0 + client.delete_discovery_config_library_by_id_if_exists.assert_called_once_with("lib-uuid", force=True) + + +@patch(f"{MODULE}.get_client") +def test_validate_invalid_exits_4(mock_get_client: MagicMock, runner: CliRunner, tmp_path) -> None: + client = MagicMock() + mock_get_client.return_value = client + client.validate_discovery_config_library.return_value = SimpleNamespace( + is_valid=ValidationStatus.invalid, validation_error="duplicate label 'email'" + ) + lib = tmp_path / "lib.yaml" + lib.write_text("labels: []\n") + + result = runner.invoke(app, ["discover", "libraries", "validate", "-f", str(lib), "--type", "database"]) + + assert result.exit_code == 4 + assert "duplicate label 'email'" in result.stderr diff --git a/tests/commands/test_discovery_configs.py b/tests/commands/test_discovery_configs.py new file mode 100644 index 0000000..509c98e --- /dev/null +++ b/tests/commands/test_discovery_configs.py @@ -0,0 +1,197 @@ +from __future__ import annotations + +from types import SimpleNamespace +from unittest.mock import MagicMock, patch + +from datamasque.client.models.discovery_config import DiscoveryConfigType +from datamasque.client.models.status import ValidationStatus +from typer.testing import CliRunner + +from datamasque_cli.main import app + +MODULE = "datamasque_cli.commands.discovery_configs" + + +def _config( + name: str, + config_type: DiscoveryConfigType = DiscoveryConfigType.database, + config_id: str = "cfg-uuid", + is_valid: ValidationStatus | None = ValidationStatus.valid, + yaml: str | None = None, +) -> SimpleNamespace: + return SimpleNamespace( + id=config_id, + name=name, + config_type=config_type, + is_valid=is_valid, + validation_error=None, + created=None, + modified=None, + yaml=yaml, + ) + + +@patch(f"{MODULE}.get_client") +def test_list_filters_by_type(mock_get_client: MagicMock, runner: CliRunner) -> None: + client = MagicMock() + mock_get_client.return_value = client + client.list_discovery_configs.return_value = [ + _config("emp", DiscoveryConfigType.database), + _config("docs", DiscoveryConfigType.file), + ] + + result = runner.invoke(app, ["discover", "configs", "list", "--type", "file"]) + + assert result.exit_code == 0 + assert "docs" in result.stdout + assert "emp" not in result.stdout + + +@patch(f"{MODULE}.get_client") +def test_get_yaml_fetches_full_config(mock_get_client: MagicMock, runner: CliRunner) -> None: + client = MagicMock() + mock_get_client.return_value = client + client.list_discovery_configs.return_value = [_config("emp")] + client.get_discovery_config.return_value = _config("emp", yaml="labels: []\n") + + result = runner.invoke(app, ["discover", "configs", "get", "emp", "--yaml"]) + + assert result.exit_code == 0 + assert "labels: []" in result.stdout + client.get_discovery_config.assert_called_once_with("cfg-uuid") + + +@patch(f"{MODULE}.get_client") +def test_get_ambiguous_name_aborts(mock_get_client: MagicMock, runner: CliRunner) -> None: + client = MagicMock() + mock_get_client.return_value = client + client.list_discovery_configs.return_value = [ + _config("shared", DiscoveryConfigType.database, config_id="a"), + _config("shared", DiscoveryConfigType.file, config_id="b"), + ] + + result = runner.invoke(app, ["discover", "configs", "get", "shared"]) + + assert result.exit_code == 5 + client.get_discovery_config.assert_not_called() + + +@patch(f"{MODULE}.get_client") +def test_get_ambiguous_resolved_by_type(mock_get_client: MagicMock, runner: CliRunner) -> None: + client = MagicMock() + mock_get_client.return_value = client + client.list_discovery_configs.return_value = [ + _config("shared", DiscoveryConfigType.database, config_id="a"), + _config("shared", DiscoveryConfigType.file, config_id="b"), + ] + client.get_discovery_config.return_value = _config("shared", DiscoveryConfigType.file, config_id="b") + + result = runner.invoke(app, ["discover", "configs", "get", "shared", "--type", "file"]) + + assert result.exit_code == 0 + client.get_discovery_config.assert_called_once_with("b") + + +@patch(f"{MODULE}.get_client") +def test_defaults_requests_typed_default(mock_get_client: MagicMock, runner: CliRunner) -> None: + client = MagicMock() + mock_get_client.return_value = client + client.make_request.return_value = SimpleNamespace(content=b"labels: []\n") + + result = runner.invoke(app, ["discover", "configs", "defaults", "--type", "file"]) + + assert result.exit_code == 0 + assert "labels: []" in result.stdout + client.make_request.assert_called_once_with( + "GET", "/api/discovery/configs/defaults/", params={"config_type": "file"} + ) + + +@patch(f"{MODULE}.get_client") +def test_create_new_requires_type(mock_get_client: MagicMock, runner: CliRunner, tmp_path) -> None: + client = MagicMock() + mock_get_client.return_value = client + client.list_discovery_configs.return_value = [] + cfg = tmp_path / "cfg.yaml" + cfg.write_text("labels: []\n") + + missing_type = runner.invoke(app, ["discover", "configs", "create", "--name", "emp", "-f", str(cfg)]) + assert missing_type.exit_code == 3 + client.create_or_update_discovery_config.assert_not_called() + + with_type = runner.invoke( + app, ["discover", "configs", "create", "--name", "emp", "-f", str(cfg), "--type", "database"] + ) + assert with_type.exit_code == 0 + client.create_or_update_discovery_config.assert_called_once() + + +@patch(f"{MODULE}.get_client") +def test_create_update_defaults_to_existing_type(mock_get_client: MagicMock, runner: CliRunner, tmp_path) -> None: + client = MagicMock() + mock_get_client.return_value = client + client.list_discovery_configs.return_value = [_config("emp", DiscoveryConfigType.database)] + cfg = tmp_path / "cfg.yaml" + cfg.write_text("labels: []\n") + + result = runner.invoke(app, ["discover", "configs", "create", "--name", "emp", "-f", str(cfg)]) + + assert result.exit_code == 0 + client.create_or_update_discovery_config.assert_called_once() + + +@patch(f"{MODULE}.get_client") +def test_delete_proceeds_when_present(mock_get_client: MagicMock, runner: CliRunner) -> None: + client = MagicMock() + mock_get_client.return_value = client + client.list_discovery_configs.return_value = [_config("emp")] + + result = runner.invoke(app, ["discover", "configs", "delete", "emp", "--yes"]) + + assert result.exit_code == 0 + client.delete_discovery_config_by_id_if_exists.assert_called_once_with("cfg-uuid") + + +@patch(f"{MODULE}.get_client") +def test_delete_aborts_when_missing(mock_get_client: MagicMock, runner: CliRunner) -> None: + client = MagicMock() + mock_get_client.return_value = client + client.list_discovery_configs.return_value = [] + + result = runner.invoke(app, ["discover", "configs", "delete", "nope", "--yes"]) + + assert result.exit_code == 3 + client.delete_discovery_config_by_id_if_exists.assert_not_called() + + +@patch(f"{MODULE}.get_client") +def test_validate_reports_valid(mock_get_client: MagicMock, runner: CliRunner, tmp_path) -> None: + client = MagicMock() + mock_get_client.return_value = client + client.validate_discovery_config.return_value = SimpleNamespace( + is_valid=ValidationStatus.valid, validation_error=None + ) + cfg = tmp_path / "cfg.yaml" + cfg.write_text("labels: []\n") + + result = runner.invoke(app, ["discover", "configs", "validate", "-f", str(cfg), "--type", "database"]) + + assert result.exit_code == 0 + assert "valid" in result.stderr + client.validate_discovery_config.assert_called_once() + + +@patch(f"{MODULE}.get_client") +def test_validate_invalid_exits_4(mock_get_client: MagicMock, runner: CliRunner, tmp_path) -> None: + client = MagicMock() + mock_get_client.return_value = client + client.validate_discovery_config.return_value = SimpleNamespace( + is_valid=ValidationStatus.invalid, validation_error="unknown label 'foo'" + ) + cfg = tmp_path / "cfg.yaml" + cfg.write_text("labels: []\n") + + result = runner.invoke(app, ["discover", "configs", "validate", "-f", str(cfg), "--type", "database"]) + + assert result.exit_code == 4 + assert "unknown label 'foo'" in result.stderr diff --git a/tests/commands/test_ruleset_libraries.py b/tests/commands/test_ruleset_libraries.py index 522de9f..ecacdd1 100644 --- a/tests/commands/test_ruleset_libraries.py +++ b/tests/commands/test_ruleset_libraries.py @@ -3,6 +3,7 @@ from types import SimpleNamespace from unittest.mock import MagicMock, patch +from datamasque.client.models.status import ValidationStatus from typer.testing import CliRunner from datamasque_cli.main import app @@ -10,6 +11,13 @@ MODULE = "datamasque_cli.commands.ruleset_libraries" +def _validated_library( + is_valid: ValidationStatus | None, + validation_errors: list[SimpleNamespace] | None = None, +) -> SimpleNamespace: + return SimpleNamespace(id="lib-uuid", is_valid=is_valid, validation_errors=validation_errors or []) + + @patch(f"{MODULE}.get_client") def test_delete_library_aborts_when_missing(mock_get_client: MagicMock, runner: CliRunner) -> None: client = MagicMock() @@ -38,13 +46,8 @@ def test_delete_library_proceeds_when_present(mock_get_client: MagicMock, runner def test_validate_library_reports_status(mock_get_client: MagicMock, runner: CliRunner) -> None: client = MagicMock() mock_get_client.return_value = client - original = MagicMock() - original.id = "lib-uuid" - client.get_ruleset_library_by_name.return_value = original - - validated = MagicMock() - validated.is_valid = MagicMock(value="valid") - client.validate_ruleset_library.return_value = validated + client.get_ruleset_library_by_name.return_value = SimpleNamespace(id="lib-uuid", name="my-lib", namespace="") + client.validate_ruleset_library.return_value = _validated_library(ValidationStatus.valid) result = runner.invoke(app, ["libraries", "validate", "my-lib"]) @@ -63,3 +66,37 @@ def test_validate_library_aborts_when_missing(mock_get_client: MagicMock, runner assert result.exit_code != 0 client.validate_ruleset_library.assert_not_called() + + +@patch(f"{MODULE}.get_client") +def test_validate_library_invalid_prints_errors_and_exits_4(mock_get_client: MagicMock, runner: CliRunner) -> None: + client = MagicMock() + mock_get_client.return_value = client + client.get_ruleset_library_by_name.return_value = SimpleNamespace(id="lib-uuid", name="my-lib", namespace="") + client.validate_ruleset_library.return_value = _validated_library( + ValidationStatus.invalid, + [ + SimpleNamespace(message="unknown mask type 'from_nowhere'", line_number=3), + SimpleNamespace(message="duplicate anchor 'email'", line_number=None), + ], + ) + + result = runner.invoke(app, ["libraries", "validate", "my-lib"]) + + assert result.exit_code == 4 # invalid_input + assert "unknown mask type 'from_nowhere'" in result.stderr + assert "line 3" in result.stderr + assert "duplicate anchor 'email'" in result.stderr + + +@patch(f"{MODULE}.get_client") +def test_validate_library_nonterminal_status_passes_through(mock_get_client: MagicMock, runner: CliRunner) -> None: + client = MagicMock() + mock_get_client.return_value = client + client.get_ruleset_library_by_name.return_value = SimpleNamespace(id="lib-uuid", name="my-lib", namespace="") + client.validate_ruleset_library.return_value = _validated_library(ValidationStatus.in_progress) + + result = runner.invoke(app, ["libraries", "validate", "my-lib"]) + + assert result.exit_code == 0 + assert "in_progress" in result.stderr diff --git a/tests/commands/test_rulesets.py b/tests/commands/test_rulesets.py index 677f99f..fc418ee 100644 --- a/tests/commands/test_rulesets.py +++ b/tests/commands/test_rulesets.py @@ -1,11 +1,14 @@ from __future__ import annotations +from collections.abc import Callable from pathlib import Path from types import SimpleNamespace from unittest.mock import MagicMock, patch +import pytest from datamasque.client.exceptions import DataMasqueApiError from datamasque.client.models.ruleset import RulesetType +from datamasque.client.models.status import ValidationStatus from typer.testing import CliRunner from datamasque_cli.main import app @@ -17,6 +20,20 @@ def _ruleset(id_: int, name: str, rs_type: RulesetType) -> SimpleNamespace: return SimpleNamespace(id=id_, name=name, ruleset_type=rs_type, yaml="") +def _create_returning( + is_valid: ValidationStatus | None, + validation_errors: list[SimpleNamespace] | None = None, +) -> Callable[[object], object]: + + def fake_create(rs: object) -> object: + rs.id = 99 # type: ignore[attr-defined] + rs.is_valid = is_valid # type: ignore[attr-defined] + rs.validation_errors = validation_errors or [] # type: ignore[attr-defined] + return rs + + return fake_create + + # -- create (type resolution via server lookup) ---------------------------- @@ -239,12 +256,7 @@ def test_validate_uses_unique_temp_name_and_cleans_by_id( ) -> None: client = MagicMock() mock_get_client.return_value = client - - def fake_create(rs: object) -> object: - rs.id = 99 # type: ignore[attr-defined] - return rs - - client.create_or_update_ruleset.side_effect = fake_create + client.create_or_update_ruleset.side_effect = _create_returning(ValidationStatus.valid) yaml_file = tmp_path / "rs.yaml" yaml_file.write_text("tasks:\n - type: mask_table\n") @@ -268,12 +280,7 @@ def test_validate_cleans_up_when_print_success_interrupted( """`try/finally` guarantees the temp ruleset is deleted even if a later step raises.""" client = MagicMock() mock_get_client.return_value = client - - def fake_create(rs: object) -> object: - rs.id = 99 # type: ignore[attr-defined] - return rs - - client.create_or_update_ruleset.side_effect = fake_create + client.create_or_update_ruleset.side_effect = _create_returning(ValidationStatus.valid) yaml_file = tmp_path / "rs.yaml" yaml_file.write_text("tasks:\n - type: mask_table\n") @@ -287,12 +294,7 @@ def fake_create(rs: object) -> object: def test_validate_warns_when_cleanup_fails(mock_get_client: MagicMock, runner: CliRunner, tmp_path: Path) -> None: client = MagicMock() mock_get_client.return_value = client - - def fake_create(rs: object) -> object: - rs.id = 99 # type: ignore[attr-defined] - return rs - - client.create_or_update_ruleset.side_effect = fake_create + client.create_or_update_ruleset.side_effect = _create_returning(ValidationStatus.valid) client.delete_ruleset_by_id_if_exists.side_effect = DataMasqueApiError("boom", response=MagicMock()) yaml_file = tmp_path / "rs.yaml" @@ -304,6 +306,50 @@ def fake_create(rs: object) -> object: assert "left on server" in result.stderr +@patch(f"{MODULE}.get_client") +def test_validate_sync_invalid_prints_errors_and_cleans_up( + mock_get_client: MagicMock, runner: CliRunner, tmp_path: Path +) -> None: + client = MagicMock() + mock_get_client.return_value = client + client.create_or_update_ruleset.side_effect = _create_returning( + ValidationStatus.invalid, + [ + SimpleNamespace(message="unknown mask type 'from_nowhere'", line_number=7), + SimpleNamespace(message="tasks must not be empty", line_number=None), + ], + ) + + yaml_file = tmp_path / "rs.yaml" + yaml_file.write_text("tasks: []\n") + + result = runner.invoke(app, ["rulesets", "validate", "--file", str(yaml_file), "--type", "database"]) + + assert result.exit_code == 4 # invalid_input + assert "unknown mask type 'from_nowhere'" in result.stderr + assert "line 7" in result.stderr + assert "tasks must not be empty" in result.stderr + client.delete_ruleset_by_id_if_exists.assert_called_once_with(99) + + +@pytest.mark.parametrize("initial_status", [None, ValidationStatus.in_progress]) +@patch(f"{MODULE}.get_client") +def test_validate_nonterminal_status_reports_valid( + mock_get_client: MagicMock, runner: CliRunner, tmp_path: Path, initial_status: ValidationStatus | None +) -> None: + client = MagicMock() + mock_get_client.return_value = client + client.create_or_update_ruleset.side_effect = _create_returning(initial_status) + + yaml_file = tmp_path / "rs.yaml" + yaml_file.write_text("tasks:\n - type: mask_table\n") + + result = runner.invoke(app, ["rulesets", "validate", "--file", str(yaml_file), "--type", "database"]) + + assert result.exit_code == 0 + client.delete_ruleset_by_id_if_exists.assert_called_once_with(99) + + # -- export-bundle / import-bundle ---------------------------------------- diff --git a/tests/integration/conftest.py b/tests/integration/conftest.py index 024f1bd..7ed9c71 100644 --- a/tests/integration/conftest.py +++ b/tests/integration/conftest.py @@ -165,3 +165,93 @@ def db_yaml(tmp_path: Path) -> Path: " value: redacted@example.com\n" ) return path + + +DISCOVERY_TEST_NAMESPACE = "dm_int_ns" + + +@pytest.fixture() +def discovery_config_name(runner: CliRunner) -> Iterator[str]: + name = f"dm_int_{uuid.uuid4().hex[:8]}" + yield name + for config_type in ("file", "database"): + runner.invoke(app, ["discover", "configs", "delete", name, "--type", config_type, "--yes"]) + + +@pytest.fixture() +def discovery_library_name(runner: CliRunner) -> Iterator[str]: + name = f"dm_int_{uuid.uuid4().hex[:8]}" + yield name + for namespace in ("", DISCOVERY_TEST_NAMESPACE): + for config_type in ("file", "database"): + args = ["discover", "libraries", "delete", name, "--type", config_type, "--yes", "--force"] + if namespace: + args += ["--namespace", namespace] + runner.invoke(app, args) + + +@pytest.fixture() +def db_discovery_config(runner: CliRunner, tmp_path: Path) -> Path: + """The server's built-in database discovery config.""" + path = tmp_path / "db_config.yaml" + result = runner.invoke(app, ["discover", "configs", "defaults", "--type", "database", "-o", str(path)]) + if result.exit_code != 0 or not path.exists(): + pytest.skip("Could not fetch the default database discovery config from the instance") + return path + + +@pytest.fixture() +def file_discovery_config(runner: CliRunner, tmp_path: Path) -> Path: + """The server's built-in file discovery config.""" + path = tmp_path / "file_config.yaml" + result = runner.invoke(app, ["discover", "configs", "defaults", "--type", "file", "-o", str(path)]) + if result.exit_code != 0 or not path.exists(): + pytest.skip("Could not fetch the default file discovery config from the instance") + return path + + +@pytest.fixture() +def discovery_library_yaml(tmp_path: Path) -> Path: + """Minimal valid discovery config library.""" + path = tmp_path / "library.yaml" + path.write_text("labels: []\nmetadata_rules: []\nidd_rules: []\n") + return path + + +@pytest.fixture() +def invalid_discovery_yaml(tmp_path: Path) -> Path: + """YAML the discovery parser rejects.""" + path = tmp_path / "invalid.yaml" + path.write_text("this: is\nnot: a valid discovery config\ngarbage: true\n") + return path + + +@pytest.fixture() +def any_connection(runner: CliRunner) -> str: + """Name of any connection on the instance.""" + result = runner.invoke(app, ["connections", "list", "--json"]) + if result.exit_code != 0: + pytest.skip("Could not list connections") + conns = json.loads(result.stdout) + if not conns: + pytest.skip("No connections on this instance") + return str(conns[0]["name"]) + + +@pytest.fixture() +def database_connection(runner: CliRunner) -> str: + """Name of a database-type source connection.""" + override = os.environ.get("DM_TEST_DB_CONN") + if override: + return override + result = runner.invoke(app, ["connections", "list", "--json"]) + if result.exit_code != 0: + pytest.skip("Could not list connections to find a database source") + conns = json.loads(result.stdout) + match = next( + (c["name"] for c in conns if c["type"] == "Database" and c["role"] in {"source", "source+destination"}), + None, + ) + if not match: + pytest.skip("No database-type source connection on this instance; set DM_TEST_DB_CONN to override") + return str(match) diff --git a/tests/integration/test_discovery.py b/tests/integration/test_discovery.py new file mode 100644 index 0000000..0b13cc3 --- /dev/null +++ b/tests/integration/test_discovery.py @@ -0,0 +1,333 @@ +"""Live-instance tests for configurable discovery.""" + +from __future__ import annotations + +import re +from pathlib import Path + +import pytest +from typer.testing import CliRunner + +from datamasque_cli.main import app +from tests.integration.conftest import DISCOVERY_TEST_NAMESPACE + +pytestmark = pytest.mark.integration + + +# --- discovery configs ------------------------------------------------------- + + +def test_config_create_get_delete_lifecycle( + runner: CliRunner, + discovery_config_name: str, + db_discovery_config: Path, +) -> None: + create = runner.invoke( + app, + [ + "discover", + "configs", + "create", + "--name", + discovery_config_name, + "--type", + "database", + "-f", + str(db_discovery_config), + ], + ) + assert create.exit_code == 0, create.stdout + + get_yaml = runner.invoke(app, ["discover", "configs", "get", discovery_config_name, "--yaml"]) + assert get_yaml.exit_code == 0 + assert "labels:" in get_yaml.stdout + + listing = runner.invoke(app, ["discover", "configs", "list"]) + assert discovery_config_name in listing.stdout + + delete = runner.invoke(app, ["discover", "configs", "delete", discovery_config_name, "--yes"]) + assert delete.exit_code == 0 + + gone = runner.invoke(app, ["discover", "configs", "get", discovery_config_name]) + assert gone.exit_code == 3 + + +def test_config_validate_accepts_default_config(runner: CliRunner, db_discovery_config: Path) -> None: + result = runner.invoke( + app, ["discover", "configs", "validate", "-f", str(db_discovery_config), "--type", "database"] + ) + assert result.exit_code == 0, result.stdout + + +def test_config_validate_rejects_invalid_yaml(runner: CliRunner, invalid_discovery_yaml: Path) -> None: + result = runner.invoke( + app, ["discover", "configs", "validate", "-f", str(invalid_discovery_yaml), "--type", "database"] + ) + assert result.exit_code == 4 + assert "invalid" in result.stderr.lower() + + +def test_config_same_name_coexists_across_types( + runner: CliRunner, + discovery_config_name: str, + db_discovery_config: Path, + file_discovery_config: Path, +) -> None: + db = runner.invoke( + app, + [ + "discover", + "configs", + "create", + "--name", + discovery_config_name, + "--type", + "database", + "-f", + str(db_discovery_config), + ], + ) + file = runner.invoke( + app, + [ + "discover", + "configs", + "create", + "--name", + discovery_config_name, + "--type", + "file", + "-f", + str(file_discovery_config), + ], + ) + assert db.exit_code == 0, db.stdout + assert file.exit_code == 0, file.stdout + + listing = runner.invoke(app, ["discover", "configs", "list"]) + matches = [line for line in listing.stdout.splitlines() if discovery_config_name in line] + assert len(matches) == 2 + + +def test_config_create_without_type_aborts_when_ambiguous( + runner: CliRunner, + discovery_config_name: str, + db_discovery_config: Path, + file_discovery_config: Path, +) -> None: + runner.invoke( + app, + [ + "discover", + "configs", + "create", + "--name", + discovery_config_name, + "--type", + "database", + "-f", + str(db_discovery_config), + ], + ) + runner.invoke( + app, + [ + "discover", + "configs", + "create", + "--name", + discovery_config_name, + "--type", + "file", + "-f", + str(file_discovery_config), + ], + ) + + result = runner.invoke( + app, ["discover", "configs", "create", "--name", discovery_config_name, "-f", str(db_discovery_config)] + ) + + assert result.exit_code != 0 + assert "Multiple discovery configs" in result.stderr + + +def test_config_get_missing_is_not_found(runner: CliRunner) -> None: + result = runner.invoke(app, ["discover", "configs", "get", "dm_int_does_not_exist"]) + assert result.exit_code == 3 + + +# --- discovery config libraries ---------------------------------------------- + + +def test_library_create_get_delete_lifecycle( + runner: CliRunner, + discovery_library_name: str, + discovery_library_yaml: Path, +) -> None: + create = runner.invoke( + app, + [ + "discover", + "libraries", + "create", + "--name", + discovery_library_name, + "--type", + "database", + "-f", + str(discovery_library_yaml), + ], + ) + assert create.exit_code == 0, create.stdout + + get_yaml = runner.invoke(app, ["discover", "libraries", "get", discovery_library_name, "--yaml"]) + assert get_yaml.exit_code == 0 + + listing = runner.invoke(app, ["discover", "libraries", "list"]) + assert discovery_library_name in listing.stdout + + delete = runner.invoke(app, ["discover", "libraries", "delete", discovery_library_name, "--yes"]) + assert delete.exit_code == 0 + + gone = runner.invoke(app, ["discover", "libraries", "get", discovery_library_name]) + assert gone.exit_code == 3 + + +def test_library_namespace_is_isolated( + runner: CliRunner, + discovery_library_name: str, + discovery_library_yaml: Path, +) -> None: + created = runner.invoke( + app, + [ + "discover", + "libraries", + "create", + "--name", + discovery_library_name, + "--type", + "database", + "--namespace", + DISCOVERY_TEST_NAMESPACE, + "-f", + str(discovery_library_yaml), + ], + ) + assert created.exit_code == 0, created.stdout + + in_namespace = runner.invoke( + app, ["discover", "libraries", "get", discovery_library_name, "--namespace", DISCOVERY_TEST_NAMESPACE] + ) + assert in_namespace.exit_code == 0 + + default_namespace = runner.invoke(app, ["discover", "libraries", "get", discovery_library_name]) + assert default_namespace.exit_code == 3 + + +def test_library_validate_rejects_invalid_yaml(runner: CliRunner, invalid_discovery_yaml: Path) -> None: + result = runner.invoke( + app, ["discover", "libraries", "validate", "-f", str(invalid_discovery_yaml), "--type", "database"] + ) + assert result.exit_code == 4 + + +# --- `--config` resolution guards (abort before any run starts) -------------- + + +def test_schema_config_type_mismatch_aborts( + runner: CliRunner, + any_connection: str, + discovery_config_name: str, + file_discovery_config: Path, +) -> None: + runner.invoke( + app, + [ + "discover", + "configs", + "create", + "--name", + discovery_config_name, + "--type", + "file", + "-f", + str(file_discovery_config), + ], + ) + result = runner.invoke(app, ["discover", "schema", any_connection, "--config", discovery_config_name]) + assert result.exit_code == 4 + assert "database config" in result.stderr + + +def test_file_config_type_mismatch_aborts( + runner: CliRunner, + any_connection: str, + discovery_config_name: str, + db_discovery_config: Path, +) -> None: + runner.invoke( + app, + [ + "discover", + "configs", + "create", + "--name", + discovery_config_name, + "--type", + "database", + "-f", + str(db_discovery_config), + ], + ) + result = runner.invoke(app, ["discover", "file", any_connection, "--config", discovery_config_name]) + assert result.exit_code == 4 + assert "file config" in result.stderr + + +def test_schema_config_not_found_aborts(runner: CliRunner, any_connection: str) -> None: + result = runner.invoke(app, ["discover", "schema", any_connection, "--config", "dm_int_no_such_config"]) + assert result.exit_code == 3 + + +# --- run from config + config snapshot (env-gated) --------------------------- + + +def test_schema_run_from_config_and_snapshot( + runner: CliRunner, + database_connection: str, + discovery_config_name: str, + db_discovery_config: Path, + tmp_path: Path, +) -> None: + create = runner.invoke( + app, + [ + "discover", + "configs", + "create", + "--name", + discovery_config_name, + "--type", + "database", + "-f", + str(db_discovery_config), + ], + ) + assert create.exit_code == 0, create.stdout + + start = runner.invoke(app, ["discover", "schema", database_connection, "--config", discovery_config_name]) + if start.exit_code != 0: + pytest.skip(f"Could not start schema discovery on '{database_connection}': {start.stdout}{start.stderr}") + + output = " ".join(start.stderr.split()) + assert f"config '{discovery_config_name}'" in output + match = re.search(r"run (\d+)", output) + assert match, f"no run id in output: {output}" + run_id = match.group(1) + + snapshot = tmp_path / "snapshot.yaml" + snap_result = runner.invoke(app, ["discover", "config-snapshot", run_id, "-o", str(snapshot)]) + assert snap_result.exit_code == 0, snap_result.stdout + assert snapshot.exists() and snapshot.read_text().strip() diff --git a/uv.lock b/uv.lock index 13b056f..f1a8b16 100644 --- a/uv.lock +++ b/uv.lock @@ -159,7 +159,7 @@ dev = [ [package.metadata] requires-dist = [ - { name = "datamasque-python", specifier = ">=1.0.0,<2" }, + { name = "datamasque-python", specifier = ">=1.1.8,<2" }, { name = "tomli-w", specifier = ">=1.0.0" }, { name = "typer", specifier = ">=0.15.0" }, ] @@ -174,15 +174,14 @@ dev = [ [[package]] name = "datamasque-python" -version = "1.0.4" -source = { registry = "https://pypi.org/simple" } +version = "1.1.8" +source = { registry = "../../.claude/jobs/0f1e3a41/tmp/dist" } dependencies = [ { name = "pydantic" }, { name = "requests" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/e0/52/1acd8c73b15e07c417a7a90060facc15b8823d5cf3207c6a51a8f9510be6/datamasque_python-1.0.4.tar.gz", hash = "sha256:45d1020364e16cd8200b972960f2bf72a683a2633cacde0c0d3eb8b00a80191a", size = 164063, upload-time = "2026-06-09T06:35:57.719Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/50/8e/7323a24cd06116cd2b1fcb3a664df86660fc11dbc49c470afc91d2744819/datamasque_python-1.0.4-py3-none-any.whl", hash = "sha256:893eb5e63814d2862d3f2d5d0b26850cb7367f54cd315bb7e2716ed4cdd69cd3", size = 50839, upload-time = "2026-06-09T06:35:56.328Z" }, + { path = "datamasque_python-1.1.8-py3-none-any.whl" }, ] [[package]]