diff --git a/docs/commands.md b/docs/commands.md index bd576b9..6448da2 100644 --- a/docs/commands.md +++ b/docs/commands.md @@ -18,6 +18,9 @@ The commands available to you are: thinks is the current number of files for a given dataset at each storage element, compared to what is observed. If a discrepancy is found at Minoc, the user can choose to create the file replicas missing for Minoc. +- `unregistered`: This provides insight into datasets which failed to register + with Datatrail, either summarised across the whole unregistered bucket + (`summary`) or for a single event (`search`). - `version`: List the CLI and server version. Detailed information on all of the CLI commands can be found on the diff --git a/docs/unregistered.md b/docs/unregistered.md new file mode 100644 index 0000000..e3824b6 --- /dev/null +++ b/docs/unregistered.md @@ -0,0 +1,237 @@ +# ๐Ÿšง Inspecting failures with `unregistered` + + + +```bash +โฏ datatrail unregistered --help +Usage: datatrail unregistered [OPTIONS] COMMAND [ARGS]... + + Commands related to unregistered datasets. + +Options: + --help Show this message and exit. + +Commands: + search Check whether an event is an unregistered dataset. + summary Summarise the reasons for unregistered datasets. +``` + +## Overview + +When Datatrail cannot register a dataset, the attempt is recorded in the +unregistered datasets bucket along with the reason it failed. The +`unregistered` commands let you inspect that bucket, either as a whole with +`summary`, or for a single event with `search`. + +This is usually the first place to look when an event you expect to find with +[`scout`](scout.md) or [`ps`](ps.md) is missing entirely from Datatrail. + +## `search` + +```bash +โฏ datatrail unregistered search --help +Usage: datatrail unregistered search [OPTIONS] EVENT + + Check whether an event is an unregistered dataset. + +Options: + -s, --scope TEXT Only search within this scope. + -p, --partial Match events containing EVENT. + -v, --verbose Verbosity: v=INFO, vv=DEBUG. + -q, --quiet Only errors shown in logs. + --json Output as JSON. + --help Show this message and exit. +``` + +`search` answers the question "why is this event not in Datatrail?". Each +record found is shown as a separate block, most recently recorded first, +detailing the scope and site the registration was attempted for, the parent +dataset it was to be attached to, and the reason the attempt failed. + +An event can have more than one record, e.g. one per site, or one per failed +attempt. + +=== "All scopes" + + ```bash + โฏ datatrail unregistered search 1172713191 + โš  1172713191 is an unregistered dataset โš  + 4 records found. + Event 1172713191 + Scope hco.event.baseband.raw + Site hco + Parent dataset CHAMPS_Localization.J0408 + Recorded 2026-08-08 19:15 UTC + Reason Could not attach datasets: 1172713191 and + CHAMPS_Localization.J0408. ERROR: dataset + CHAMPS_Localization.J0408, hco.event.baseband.raw not found.. + + Event 1172713191 + Scope gbo.event.baseband.raw + Site gbo + Parent dataset CHAMPS_Localization.J0408 + Recorded 2026-08-08 19:09 UTC + Reason Could not attach datasets: 1172713191 and + CHAMPS_Localization.J0408. ERROR: dataset + CHAMPS_Localization.J0408, gbo.event.baseband.raw not found.. + + Event 1172713191 + Scope kko.event.baseband.raw + Site kko + Parent dataset CHAMPS_Localization.J0408 + Recorded 2026-08-08 18:54 UTC + Reason Could not attach datasets: 1172713191 and + CHAMPS_Localization.J0408. ERROR: dataset + CHAMPS_Localization.J0408, kko.event.baseband.raw not found.. + + Event 1172713191 + Scope chime.event.intensity.raw + Site chime + Parent dataset CHAMPS_Localization.J0408 + Recorded 2026-08-08 14:28 UTC + Reason Could not attach datasets: 1172713191 and + CHAMPS_Localization.J0408. ERROR: dataset + CHAMPS_Localization.J0408, chime.event.intensity.raw not + found.. + ``` + +=== "Filtering by scope" + + ```bash + โฏ datatrail unregistered search 1172713191 --scope kko.event.baseband.raw + โš  1172713191 is an unregistered dataset โš  + 1 record found. + Event 1172713191 + Scope kko.event.baseband.raw + Site kko + Parent dataset CHAMPS_Localization.J0408 + Recorded 2026-08-08 18:54 UTC + Reason Could not attach datasets: 1172713191 and + CHAMPS_Localization.J0408. ERROR: dataset + CHAMPS_Localization.J0408, kko.event.baseband.raw not found.. + ``` + +=== "No records found" + + ```bash + โฏ datatrail unregistered search 382085503 + 382085503 is not an unregistered dataset. + Use --partial to search for events containing this name. + ``` + +### Partial matches + +By default the event name has to match exactly. With `--partial`, any event +name _containing_ the given string is returned, which is useful when you only +know part of the name, or when you want to check a whole family of events such +as a set of commissioning datasets. + +```bash +โฏ datatrail unregistered search 11728169 --partial + โš  11728169 is an unregistered dataset โš  + 2 records found. +Event 1172816953 +Scope chime.event.intensity.raw +Site chime +Parent dataset CHAMPS_Localization.J0408 +Recorded 2026-08-09 11:14 UTC +Reason Could not attach datasets: 1172816953 and + CHAMPS_Localization.J0408. ERROR: dataset + CHAMPS_Localization.J0408, chime.event.intensity.raw not + found.. + +Event 1172816944 +Scope chime.event.intensity.raw +Site chime +Parent dataset CHAMPS_Localization.J0408 +Recorded 2026-08-09 10:16 UTC +Reason Could not attach datasets: 1172816944 and + CHAMPS_Localization.J0408. ERROR: dataset + CHAMPS_Localization.J0408, chime.event.intensity.raw not + found.. +``` + +!!! note "Result limit" + + A maximum of 100 records are returned. If you hit that limit with + `--partial`, narrow the search with a longer event name or a `--scope`. + +### Machine readable output + +`--json` prints the full records, including fields not shown in the table, +which is handy for scripting or for piping into `jq`. + +```bash +โฏ datatrail unregistered search 1172713191 --scope kko.event.baseband.raw --json +{ + "event": "1172713191", + "scope": "kko.event.baseband.raw", + "partial": false, + "unregistered": [ + { + "dataset_name": "1172713191", + "dataset_scope": "kko.event.baseband.raw", + "storage_name": "kko", + "storage_element_captured_at": "kko", + "attach_to_dataset": "CHAMPS_Localization.J0408", + "reason": "Could not attach datasets: 1172713191 and CHAMPS_Localization.J0408. ERROR: dataset CHAMPS_Localization.J0408, kko.event.baseband.raw not found..", + "associated_event": {}, + "locked": false + } + ] +} +``` + +!!! tip "Errors in JSON mode" + + With `--json`, a failed query is reported as `{"error": "..."}` on stdout + and the command exits with status `1`, so failures can be detected in a + pipeline. + +## `summary` + +```bash +โฏ datatrail unregistered summary --help +Usage: datatrail unregistered summary [OPTIONS] + + Summarise the reasons for unregistered datasets. + +Options: + -v, --verbose Verbosity: v=INFO, vv=DEBUG. + -q, --quiet Only errors shown in logs. + --help Show this message and exit. +``` + +`summary` groups every record in the bucket by the reason it failed, giving an +overview of what is currently going wrong. Reasons are grouped into categories, +e.g. `ATTACH_MISSING` for datasets whose parent dataset does not exist, or +`POSTGRES` for database errors. + +```bash +โฏ datatrail unregistered summary + Summary of reasons โ€” 886 unregistered datasets +โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”ณโ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”ณโ”โ”โ”โ”โ”โ”โ”โ”ณโ”โ”โ”โ”โ”โ”โ”โ”“ +โ”ƒ Category โ”ƒ Detail โ”ƒ Count โ”ƒ % โ”ƒ +โ”กโ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ•‡โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ”โ•‡โ”โ”โ”โ”โ”โ”โ”โ•‡โ”โ”โ”โ”โ”โ”โ”โ”ฉ +โ”‚ POSTGRES โ”‚ (psycopg2.errors.UniqueViolation) โ”‚ 627 โ”‚ 70.8% โ”‚ +โ”‚ โ”‚ duplicate key value violates unique โ”‚ โ”‚ โ”‚ +โ”‚ โ”‚ constraint "name_se_captured_at_pair" โ”‚ โ”‚ โ”‚ +โ”‚ โ”‚ DETAIL: Key โ”‚ โ”‚ โ”‚ +โ”œโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”ผโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”ผโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”ผโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”ค +โ”‚ STATUS โ”‚ (no reason recorded) โ”‚ 110 โ”‚ 12.4% โ”‚ +โ”‚ โ”‚ event-missing-verification โ”‚ 85 โ”‚ 9.6% โ”‚ +โ”‚ โ”‚ pending-tsar-classification โ”‚ 23 โ”‚ 2.6% โ”‚ +โ”œโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”ผโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”ผโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”ผโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”ค +โ”‚ ATTACH_MISSING โ”‚ CHAMPS_Localization.J0408 โ†’ โ”‚ 18 โ”‚ 2.0% โ”‚ +โ”‚ โ”‚ chime.event.intensity.raw โ”‚ โ”‚ โ”‚ +โ”‚ โ”‚ outrigger.commissioning.B0834 โ†’ โ”‚ 12 โ”‚ 1.4% โ”‚ +โ”‚ โ”‚ chime.event.baseband.raw โ”‚ โ”‚ โ”‚ +โ”œโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”ผโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”ผโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”ผโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”ค +โ”‚ CREATE_DUPLICATE โ”‚ โ†’ chime.event.baseband.raw โ”‚ 3 โ”‚ 0.3% โ”‚ +โ””โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”ดโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”ดโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”ดโ”€โ”€โ”€โ”€โ”€โ”€โ”€โ”˜ +``` + +For `ATTACH_MISSING` and `CREATE_DUPLICATE`, the detail column reads as +`dataset โ†’ scope`. In the `CREATE_DUPLICATE` case the digits of the dataset +name are replaced with ``, so that the same failure for many events is +counted as a single row. diff --git a/dtcli/src/functions.py b/dtcli/src/functions.py index a9c64f0..49283a4 100644 --- a/dtcli/src/functions.py +++ b/dtcli/src/functions.py @@ -486,6 +486,35 @@ def get_unregistered_dataset(dataset: str, scope: str) -> Optional[Dict[str, Any return response[0] +def find_unregistered_datasets( + event: str, + scope: Optional[str] = None, + partial: bool = False, + limit: int = 100, +) -> List[Dict[str, Any]]: + """Find unregistered datasets recorded for an event. + + Args: + event (str): Name of the event, i.e. the dataset name. + scope (Optional[str]): Only return records for this scope. + partial (bool): Match events containing `event` rather than exactly. + limit (int): Maximum number of records to return. + + Returns: + List[Dict[str, Any]]: Unregistered dataset records for the event. + """ + name: Any = {"$regex": re.escape(event)} if partial else event + query: Dict[str, Any] = {"results.dataset_name": name} + if scope: + query["results.dataset_scope"] = scope + return view_results( + pipeline="datatrail-unregistered-datasets", + query=query, + projection={"results.files": 0}, + limit=limit, + ) + + ATTACH_RE = re.compile( r"Could not attach datasets: .+? ERROR: \"?dataset (.+?), (.+?) not found" # noqa: E501 ) diff --git a/dtcli/unregistered.py b/dtcli/unregistered.py index ad393be..19cc12c 100644 --- a/dtcli/unregistered.py +++ b/dtcli/unregistered.py @@ -1,8 +1,11 @@ """Datatrail Unregistered datasets commands.""" +import json import logging +import re from collections import defaultdict -from typing import DefaultDict, List, Tuple +from datetime import datetime, timezone +from typing import Any, DefaultDict, Dict, List, Optional, Tuple import click from rich.console import Console @@ -100,3 +103,126 @@ def summary( ) console.print(table) + + +@unregistered.command(help="Check whether an event is an unregistered dataset.") +@click.argument("event", required=True, type=click.STRING, nargs=1) +@click.option("-s", "--scope", type=click.STRING, help="Only search within this scope.") +@click.option("-p", "--partial", is_flag=True, help="Match events containing EVENT.") +@click.option("-v", "--verbose", count=True, help="Verbosity: v=INFO, vv=DEBUG.") +@click.option("-q", "--quiet", is_flag=True, help="Only errors shown in logs.") +@click.option("--json", "output_json", is_flag=True, help="Output as JSON.") +@click.pass_context +def search( + ctx: click.Context, + event: str, + scope: Optional[str] = None, + partial: bool = False, + verbose: int = 0, + quiet: bool = False, + output_json: bool = False, +): + """Search the unregistered datasets for an event. + + Args: + ctx (click.Context): Click context. + event (str): Name of the event to search for. + scope (str): Only search within this scope. + partial (bool): Match events containing EVENT. + verbose (int): Verbosity: v=INFO, vv=DEBUG. + quiet (bool): Only errors shown in logs. + output_json (bool): Output as JSON. + """ + # Set logging level. + set_log_level(logger, verbose, quiet) + logger.debug("`search` called with:") + logger.debug(f"event: {event} [{type(event)}]") + logger.debug(f"scope: {scope} [{type(scope)}]") + logger.debug(f"partial: {partial} [{type(partial)}]") + logger.debug(f"verbose: {verbose} [{type(verbose)}]") + logger.debug(f"quiet: {quiet} [{type(quiet)}]") + + try: + results = functions.find_unregistered_datasets(event, scope, partial) + except Exception as error: + logger.debug(error) + if output_json: + print(json.dumps({"error": str(error)}, indent=2)) + ctx.exit(1) + error_console.print(error) + return None + + if output_json: + print( + json.dumps( + { + "event": event, + "scope": scope, + "partial": partial, + "unregistered": [result["results"] for result in results], + }, + indent=2, + ) + ) + return None + + if not results: + within = f" in {scope}" if scope else "" + console.print(f"{event} is not an unregistered dataset{within}.", style="green") + if not partial: + console.print("Use --partial to search for events containing this name.") + return None + + console.print( + f":warning: {event} is an unregistered dataset :warning:", + style="bold yellow", + justify="center", + ) + console.print( + f"{len(results):,} record{'s' if len(results) > 1 else ''} found.", + style="bold magenta", + justify="center", + ) + results = sorted( + results, key=lambda result: result.get("creation") or 0, reverse=True + ) + for index, result in enumerate(results): + if index: + console.print() + console.print(create_record_table(result)) + + +def create_record_table(result: Dict[str, Any]) -> Table: + """Create a table detailing a single unregistered record. + + Args: + result (Dict[str, Any]): Unregistered dataset record. + + Returns: + Table: Table of the record details. + """ + record = result["results"] + creation = result.get("creation") + recorded = ( + datetime.fromtimestamp(creation, tz=timezone.utc).strftime("%Y-%m-%d %H:%M UTC") + if creation + else "unknown" + ) + + table = Table(box=None, show_header=False, padding=(0, 2, 0, 0)) + table.add_column(style="bold") + table.add_column(overflow="fold") + table.add_row("Event", str(record.get("dataset_name"))) + table.add_row("Scope", str(record.get("dataset_scope"))) + table.add_row("Site", str(result.get("site"))) + table.add_row("Parent dataset", str(record.get("attach_to_dataset") or "-")) + table.add_row("Recorded", recorded) + table.add_row( + "Reason", + Text( + re.sub(r"\n+", "\n", str(record.get("reason") or "")).strip() + or "(no reason recorded)", + style="red", + ), + ) + return table diff --git a/mkdocs.yml b/mkdocs.yml index 2b1839e..53fe8aa 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -14,6 +14,7 @@ nav: - ps: ps.md - pull: pull.md - scout: scout.md + - unregistered: unregistered.md - Command Line Interface: - Commands: commands.md - Reference: cli.md diff --git a/tests/test_cli.py b/tests/test_cli.py index 1074ec6..88a7e95 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -194,6 +194,39 @@ def test_cli_clear_help(runner: CliRunner) -> None: assert result.output == expected_response +def test_cli_unregistered_search_help(runner: CliRunner) -> None: + """Test CLI unregistered search help page. + + Args: + runner (CliRunner): Click runner. + """ + result = runner.invoke(datatrail, ["unregistered", "search", "--help"]) + assert result.exit_code == 0 + assert "Usage: cli unregistered search [OPTIONS] EVENT" in result.output + assert "Check whether an event is an unregistered dataset" in result.output + assert "--scope" in result.output + assert "--partial" in result.output + assert "--json" in result.output + + +def test_cli_unregistered_search_not_found(runner: CliRunner) -> None: + """Test CLI unregistered search for an event that is registered. + + Args: + runner (CliRunner): Click runner. + """ + import json + + result = runner.invoke( + datatrail, ["unregistered", "search", "not-an-event", "--json"] + ) + assert result.exit_code == 0 + json_start = result.output.find("{") + output_data = json.loads(result.output[json_start:]) + assert output_data["event"] == "not-an-event" + assert output_data["unregistered"] == [] + + def test_cli_config_init(runner: CliRunner) -> None: """Test CLI configuration initialisation. diff --git a/tests/test_functions.py b/tests/test_functions.py index 88d66d2..4955062 100644 --- a/tests/test_functions.py +++ b/tests/test_functions.py @@ -5,7 +5,11 @@ import pytest -from dtcli.src.functions import get_unregistered_dataset, view_results +from dtcli.src.functions import ( + find_unregistered_datasets, + get_unregistered_dataset, + view_results, +) def test_view_results() -> None: @@ -52,3 +56,35 @@ def test_get_unregistered_dataset() -> None: assert "reason" in unregistered_dataset["results"].keys() else: pytest.skip("No unregistered datasets found.") + + +def test_find_unregistered_datasets() -> None: + """Test find_unregistered_datasets.""" + pipeline: str = "datatrail-unregistered-datasets" + projection: Dict[str, Any] = {"results.dataset_name": 1, "results.dataset_scope": 1} + try: + known: Dict[str, Any] = view_results(pipeline, {}, projection, 1)[0] + except IndexError: + pytest.skip("No unregistered datasets found.") + dataset_name: str = known["results"]["dataset_name"] + dataset_scope: str = known["results"]["dataset_scope"] + + results: List[Dict[str, Any]] = find_unregistered_datasets(dataset_name) + assert len(results) > 0 + assert all(r["results"]["dataset_name"] == dataset_name for r in results) + assert "reason" in results[0]["results"].keys() + + # Scope of the dataset filters nothing out, an unrelated scope filters all. + assert find_unregistered_datasets(dataset_name, scope=dataset_scope) + assert find_unregistered_datasets(dataset_name, scope="not.a.scope") == [] + + # A partial search finds at least the datasets an exact search does. + partial: List[Dict[str, Any]] = find_unregistered_datasets( + dataset_name[:-1], partial=True + ) + assert len(partial) >= len(results) + + +def test_find_unregistered_datasets_no_match() -> None: + """Test find_unregistered_datasets with an event that is not unregistered.""" + assert find_unregistered_datasets("not-an-event") == []