From 8c007cf7b9de0f61c6ead77b8ab27241cee8e1a7 Mon Sep 17 00:00:00 2001 From: Christian Pinto Date: Thu, 4 Jun 2026 14:41:29 +0100 Subject: [PATCH 1/7] feat(benchmarks): Added dummy benchmark for testing Signed-off-by: Christian Pinto --- .../test-package/pyproject.toml | 11 +++++++ .../test-package/test_package/__init__.py | 8 +++++ .../test-package/test_package/experiments.py | 31 +++++++++++++++++++ .../test_benchmark/space.yaml | 13 ++++++++ packages/terratorch/nexus.yaml | 6 ++++ 5 files changed, 69 insertions(+) create mode 100644 packages/terratorch/benchmark_packages/test-package/pyproject.toml create mode 100644 packages/terratorch/benchmark_packages/test-package/test_package/__init__.py create mode 100644 packages/terratorch/benchmark_packages/test-package/test_package/experiments.py create mode 100644 packages/terratorch/models/prithvi/benchmark_instances/test_benchmark/space.yaml diff --git a/packages/terratorch/benchmark_packages/test-package/pyproject.toml b/packages/terratorch/benchmark_packages/test-package/pyproject.toml new file mode 100644 index 0000000..5fdc6d4 --- /dev/null +++ b/packages/terratorch/benchmark_packages/test-package/pyproject.toml @@ -0,0 +1,11 @@ +[project] +name = "test-package" +version = "0.1.0" +description = "Custom experiment package for terratorch benchmarks" +requires-python = ">=3.10" +dependencies = [ + "orchestrator", +] + +[project.entry-points."ado.custom_experiments"] +my_experiment = "test_package.experiments" diff --git a/packages/terratorch/benchmark_packages/test-package/test_package/__init__.py b/packages/terratorch/benchmark_packages/test-package/test_package/__init__.py new file mode 100644 index 0000000..1c62e13 --- /dev/null +++ b/packages/terratorch/benchmark_packages/test-package/test_package/__init__.py @@ -0,0 +1,8 @@ +# Copyright IBM Corp. 2026 +# SPDX-License-Identifier: Apache-2.0 + +"""Test package for custom experiments.""" + +__version__ = "0.1.0" + +# Made with Bob diff --git a/packages/terratorch/benchmark_packages/test-package/test_package/experiments.py b/packages/terratorch/benchmark_packages/test-package/test_package/experiments.py new file mode 100644 index 0000000..13e8149 --- /dev/null +++ b/packages/terratorch/benchmark_packages/test-package/test_package/experiments.py @@ -0,0 +1,31 @@ +# Copyright IBM Corporation 2025, 2026 +# SPDX-License-Identifier: MIT + +"""Custom experiments for terratorch benchmarks.""" + +from typing import Any + +from orchestrator.modules.actuators.custom_experiments import custom_experiment + + +@custom_experiment(output_property_identifiers=["result"]) +def echo_experiment(input_string: str) -> dict[str, Any]: + """ + Simple custom experiment that echoes the input string. + + This is a minimal example demonstrating the structure required + by ADO (Automated Design Optimization). + + Parameters + ---------- + input_string : str + The string to echo back + + Returns + ------- + dict[str, Any] + Dictionary containing 'result' key with the input string + """ + return { + "result": input_string, + } diff --git a/packages/terratorch/models/prithvi/benchmark_instances/test_benchmark/space.yaml b/packages/terratorch/models/prithvi/benchmark_instances/test_benchmark/space.yaml new file mode 100644 index 0000000..f30613c --- /dev/null +++ b/packages/terratorch/models/prithvi/benchmark_instances/test_benchmark/space.yaml @@ -0,0 +1,13 @@ +# ADO Discovery Space Configuration for Test Benchmark +# This benchmark tests the echo_experiment custom experiment + +# Entity space defining parameter values for the benchmark +entitySpace: + - identifier: input_string + propertyDomain: + values: ["hello world", "test string", "benchmark test"] + +# List of experiments to run +experiments: + - actuatorIdentifier: custom_experiments + experimentIdentifier: echo_experiment diff --git a/packages/terratorch/nexus.yaml b/packages/terratorch/nexus.yaml index adeaede..b0a9dc6 100644 --- a/packages/terratorch/nexus.yaml +++ b/packages/terratorch/nexus.yaml @@ -1,2 +1,8 @@ package: name: terratorch + + # Register benchmark packages with custom experiments + benchmark_packages: + - requirement_specifier: "./packages/terratorch/benchmark_packages/test-package" + experiments: + - "echo_experiment" From be1b2a4f2ec7ce6b0f46eeeb7e6a2adc837c01c6 Mon Sep 17 00:00:00 2001 From: Christian Pinto Date: Thu, 4 Jun 2026 14:48:00 +0100 Subject: [PATCH 2/7] feat(benchmarks): Added dummy benchmark for testing Signed-off-by: Christian Pinto --- .../benchmark_instances/test_benchmark/space.yaml | 15 +++++++++++++++ 1 file changed, 15 insertions(+) create mode 100644 packages/terratorch/models/terramind_base_fire/benchmark_instances/test_benchmark/space.yaml diff --git a/packages/terratorch/models/terramind_base_fire/benchmark_instances/test_benchmark/space.yaml b/packages/terratorch/models/terramind_base_fire/benchmark_instances/test_benchmark/space.yaml new file mode 100644 index 0000000..5ad9d91 --- /dev/null +++ b/packages/terratorch/models/terramind_base_fire/benchmark_instances/test_benchmark/space.yaml @@ -0,0 +1,15 @@ +# ADO Discovery Space Configuration for Test Benchmark +# This benchmark tests the echo_experiment custom experiment + +# Entity space defining parameter values for the benchmark +entitySpace: + - identifier: input_string + propertyDomain: + values: ["hello world", "test string", "benchmark test"] + +# List of experiments to run +experiments: + - actuatorIdentifier: custom_experiments + experimentIdentifier: echo_experiment + +# Made with Bob From bad622fdc19381631685ddcafbab4a1664ae795a Mon Sep 17 00:00:00 2001 From: Christian Pinto <55737893+christian-pinto@users.noreply.github.com> Date: Fri, 5 Jun 2026 14:40:41 +0100 Subject: [PATCH 3/7] Update packages/terratorch/benchmark_packages/test-package/pyproject.toml Signed-off-by: Christian Pinto <55737893+christian-pinto@users.noreply.github.com> --- .../terratorch/benchmark_packages/test-package/pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/terratorch/benchmark_packages/test-package/pyproject.toml b/packages/terratorch/benchmark_packages/test-package/pyproject.toml index 5fdc6d4..fe658c0 100644 --- a/packages/terratorch/benchmark_packages/test-package/pyproject.toml +++ b/packages/terratorch/benchmark_packages/test-package/pyproject.toml @@ -4,7 +4,7 @@ version = "0.1.0" description = "Custom experiment package for terratorch benchmarks" requires-python = ">=3.10" dependencies = [ - "orchestrator", + "ado-core", ] [project.entry-points."ado.custom_experiments"] From 4a915a6135cbf6f170e58b86d80d129db6e9a473 Mon Sep 17 00:00:00 2001 From: Christian Pinto <55737893+christian-pinto@users.noreply.github.com> Date: Tue, 9 Jun 2026 15:25:15 +0100 Subject: [PATCH 4/7] feat(benchmarks): Added benchmark instance in package base --- .../test_benchmark_base/space.yaml | 13 +++++++++++++ 1 file changed, 13 insertions(+) create mode 100644 packages/terratorch/benchmark_instances/test_benchmark_base/space.yaml diff --git a/packages/terratorch/benchmark_instances/test_benchmark_base/space.yaml b/packages/terratorch/benchmark_instances/test_benchmark_base/space.yaml new file mode 100644 index 0000000..f30613c --- /dev/null +++ b/packages/terratorch/benchmark_instances/test_benchmark_base/space.yaml @@ -0,0 +1,13 @@ +# ADO Discovery Space Configuration for Test Benchmark +# This benchmark tests the echo_experiment custom experiment + +# Entity space defining parameter values for the benchmark +entitySpace: + - identifier: input_string + propertyDomain: + values: ["hello world", "test string", "benchmark test"] + +# List of experiments to run +experiments: + - actuatorIdentifier: custom_experiments + experimentIdentifier: echo_experiment From 152dc1358f3c06d3d19a4bf2e8b8af93f5bd15dd Mon Sep 17 00:00:00 2001 From: Christian Pinto <55737893+christian-pinto@users.noreply.github.com> Date: Tue, 9 Jun 2026 15:40:30 +0100 Subject: [PATCH 5/7] Added benchmarks from remote repo --- .../benchmark_instances/test_benchmark_base/space.yaml | 2 +- packages/terratorch/nexus.yaml | 3 +++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/packages/terratorch/benchmark_instances/test_benchmark_base/space.yaml b/packages/terratorch/benchmark_instances/test_benchmark_base/space.yaml index f30613c..705093a 100644 --- a/packages/terratorch/benchmark_instances/test_benchmark_base/space.yaml +++ b/packages/terratorch/benchmark_instances/test_benchmark_base/space.yaml @@ -10,4 +10,4 @@ entitySpace: # List of experiments to run experiments: - actuatorIdentifier: custom_experiments - experimentIdentifier: echo_experiment + experimentIdentifier: echo_experiment_2 diff --git a/packages/terratorch/nexus.yaml b/packages/terratorch/nexus.yaml index b0a9dc6..c5be6a2 100644 --- a/packages/terratorch/nexus.yaml +++ b/packages/terratorch/nexus.yaml @@ -6,3 +6,6 @@ package: - requirement_specifier: "./packages/terratorch/benchmark_packages/test-package" experiments: - "echo_experiment" + - requirement_specifier: "https://github.com/christian-pinto/test-ado-custom-experiment" + experiments: + - "echo_experiment_2" From 6a62aaf6d09eca325bc804a259256da98c143133 Mon Sep 17 00:00:00 2001 From: Christian Pinto Date: Mon, 22 Jun 2026 15:26:40 +0100 Subject: [PATCH 6/7] feat(cli): Allow running all benchmark instances for a specific package Signed-off-by: Christian Pinto --- .../commands/benchmark_manager.py | 47 ++++++----- src/algorithm_nexus/commands/run.py | 83 +++++++++++++++---- tests/test_cli_run.py | 62 ++++++++++++++ 3 files changed, 153 insertions(+), 39 deletions(-) diff --git a/src/algorithm_nexus/commands/benchmark_manager.py b/src/algorithm_nexus/commands/benchmark_manager.py index 26046bb..c595840 100644 --- a/src/algorithm_nexus/commands/benchmark_manager.py +++ b/src/algorithm_nexus/commands/benchmark_manager.py @@ -967,38 +967,37 @@ def _create_operation( "ray_job_id": ray_job_id, } - def run(self) -> dict[str, Any]: + def run( + self, + packages_root: Path | None = None, + package_filter: str | None = None, + ) -> dict[str, Any]: """Main execution method. + Args: + packages_root: Path to packages directory (for all/package mode) + package_filter: Optional package name to filter by + Returns: Dictionary with execution results """ try: - console.print("Analyzing PR for new or changed benchmark instances...") - console.print(f"PR URL: {self.pr_url}") - - # Always check if we need to checkout PR code at the beginning - if not self.is_local_repo_on_pr_commit(): - console.print("[yellow]Local repository is not on PR commit[/yellow]") - console.print("Checking out PR code to temporary directory...") - self.checkout_pr_to_temp() - else: - console.print( - "[green]✓[/green] Using local repository (already on PR commit)" - ) + # Print mode header + self._print_mode_header(packages_root, package_filter, "Executing") - changed_files = self.get_changed_files() + # Discover benchmark instances + benchmark_instances = self._discover_instances( + packages_root, package_filter + ) - benchmark_instances = self.find_benchmark_instances(changed_files) + # Print found instances (show list for dry-run/non-execution mode only) + self._print_instances_found( + benchmark_instances, package_filter, show_list=not self.execute + ) if not benchmark_instances: - console.print( - "[yellow]No new or changed benchmark instances found in this PR.[/yellow]" - ) return {"instances": []} - console.print(f"Found {len(benchmark_instances)} benchmark instance(s):") - results: dict[str, Any] = { "instances": [], } @@ -1013,7 +1012,7 @@ def run(self) -> dict[str, Any]: exec_result = self.execute_benchmark(instance_path) results["instances"].append(exec_result.model_dump()) - if exec_result.status == "success": + if exec_result.status in ("success", "started"): successful += 1 else: failed += 1 @@ -1024,9 +1023,13 @@ def run(self) -> dict[str, Any]: console.print(f" Successful: {successful}") console.print(f" Failed: {failed}") console.print("=" * 60) + + results["summary"] = { + "successful": successful, + "failed": failed, + } else: for instance_path in benchmark_instances: - console.print(f" {instance_path}") results["instances"].append({"instance_path": str(instance_path)}) return results diff --git a/src/algorithm_nexus/commands/run.py b/src/algorithm_nexus/commands/run.py index 4fbf987..1668f18 100644 --- a/src/algorithm_nexus/commands/run.py +++ b/src/algorithm_nexus/commands/run.py @@ -34,12 +34,27 @@ def run_benchmarks( pr: Annotated[ - str, + str | None, typer.Option( "--pr", - help="GitHub Pull Request URL (e.g., https://github.com/IBM/algorithm-nexus/pull/123)", + help="GitHub Pull Request URL (e.g., https://github.com/IBM/algorithm-nexus/pull/123). " + "If not provided, runs benchmark instances from packages.", + ), + ] = None, + packages_root: Annotated[ + Path, + typer.Option( + "--packages-root", + help="Path to packages directory", + ), + ] = Path("./packages"), + package: Annotated[ + str | None, + typer.Option( + "--package", + help="Run only benchmark instances from a specific package", ), - ], + ] = None, remote: Annotated[ Path | None, typer.Option( @@ -87,28 +102,60 @@ def run_benchmarks( ), ] = None, ) -> None: - """Execute benchmarks from a GitHub Pull Request. + """Execute benchmarks. - Identifies new or changed benchmark instances in a PR and optionally - executes them using the ADO CLI. When executing with --remote flag, - automatically installs required benchmark packages in the Ray environment. + This command supports three modes: + 1. PR mode: Execute instances modified in a PR (provide --pr) + 2. All mode: Execute all benchmark instances (no --pr) + 3. Package mode: Execute instances from a specific package (use --package) - The command automatically checks if the local repository is on the same - commit as the PR. If not, it will checkout the PR code in a temporary - directory. + Identifies benchmark instances and optionally executes them using the ADO CLI. + When executing with --remote flag, automatically installs required benchmark + packages in the Ray environment. """ # Validate output format if specified if output_format: validate_output_format(output_format, allow_yaml=True, allow_csv=False) - try: - manager = BenchmarkManager( - pr_url=pr, - execute=not dry_run, - remote_context_file=remote, - context_file=context, + # Warn if both package filter and PR URL are provided (package is ignored in PR mode) + if package and pr: + console.print( + "[yellow]Warning:[/yellow] --package is ignored when --pr is specified. " + "In PR mode, only instances changed in the PR are executed." ) - results = manager.run() + + # Validate package exists if package filter is specified (non-PR mode only) + if package and not pr: + package_path = packages_root / package + if not package_path.is_dir(): + console.print( + f"[red]Error:[/red] Package '{package}' not found in {packages_root.resolve()}" + ) + console.print( + "\nTo see available packages, run: [cyan]nexus list packages[/cyan]" + ) + raise typer.Exit(code=1) + + try: + if pr: + manager = BenchmarkManager( + pr_url=pr, + execute=not dry_run, + remote_context_file=remote, + context_file=context, + ) + results = manager.run() + else: + manager = BenchmarkManager( + pr_url=None, + execute=not dry_run, + remote_context_file=remote, + context_file=context, + ) + results = manager.run( + packages_root=packages_root, + package_filter=package, + ) # Output results # Determine output format @@ -138,6 +185,8 @@ def run_benchmarks( if not dry_run and results.get("summary") and results["summary"]["failed"] > 0: raise typer.Exit(code=1) + except typer.Exit: + raise except KeyboardInterrupt: console_err.print("\n[yellow]Interrupted by user[/yellow]") raise typer.Exit(code=130) diff --git a/tests/test_cli_run.py b/tests/test_cli_run.py index b544829..c877c1a 100644 --- a/tests/test_cli_run.py +++ b/tests/test_cli_run.py @@ -305,4 +305,66 @@ def test_model_dump(self) -> None: assert data["operation_id"] is None +class TestRunBenchmarksCommand: + """Tests for run benchmarks CLI command.""" + + def test_run_benchmarks_nonexistent_package(self, tmp_path, capsys) -> None: + """Test run benchmarks with nonexistent package.""" + import typer + + from algorithm_nexus.commands.run import run_benchmarks + + packages_root = tmp_path / "packages" + packages_root.mkdir() + (packages_root / "existing-package").mkdir() + + with pytest.raises(typer.Exit) as exc_info: + run_benchmarks( + pr=None, + packages_root=packages_root, + package="nonexistent-package", + dry_run=True, + ) + assert exc_info.value.exit_code == 1 + + captured = capsys.readouterr() + assert "nexus list packages" in captured.out + + def test_run_benchmarks_both_package_and_pr_warns( + self, tmp_path, capsys, monkeypatch + ) -> None: + """Test that specifying both --package and --pr prints a warning.""" + from algorithm_nexus.commands.run import run_benchmarks + + packages_root = tmp_path / "packages" + packages_root.mkdir() + (packages_root / "test-package").mkdir() + + # Mock BenchmarkManager to avoid real operations + class DummyManager: + def __init__(self, *args, **kwargs): + pass + + def run(self, *args, **kwargs): + return {"instances": []} + + monkeypatch.setattr( + "algorithm_nexus.commands.run.BenchmarkManager", DummyManager + ) + + import contextlib + + with contextlib.suppress(Exception): + run_benchmarks( + pr="https://github.com/IBM/algorithm-nexus/pull/123", + packages_root=packages_root, + package="test-package", + dry_run=True, + ) + + captured = capsys.readouterr() + assert "Warning" in captured.out + assert "--package is ignored when --pr is specified" in captured.out + + # Made with Bob From 0be9c72f9b61fe2a367b855d1d56bc834b4a31b5 Mon Sep 17 00:00:00 2001 From: Christian Pinto Date: Mon, 22 Jun 2026 15:26:40 +0100 Subject: [PATCH 7/7] feat(cli): Allow running all benchmark instances for a specific package Signed-off-by: Christian Pinto --- .../commands/benchmark_manager.py | 101 +++++++++--------- src/algorithm_nexus/commands/run.py | 83 +++++++++++--- tests/test_cli_run.py | 62 +++++++++++ 3 files changed, 178 insertions(+), 68 deletions(-) diff --git a/src/algorithm_nexus/commands/benchmark_manager.py b/src/algorithm_nexus/commands/benchmark_manager.py index 26046bb..dd48e34 100644 --- a/src/algorithm_nexus/commands/benchmark_manager.py +++ b/src/algorithm_nexus/commands/benchmark_manager.py @@ -711,21 +711,24 @@ def _create_discoveryspace( space_config = DiscoverySpaceConfiguration.model_validate(space_config_dict) # Generate descriptive name and description from instance path - # Extract PR number from URL (pr_url may be None in non-PR mode) - pr_number = self.pr_url.rstrip("/").split("/")[-1] if self.pr_url else "unknown" - # Parse instance path to get package, model, and instance names package_name, model_name, instance_name = self._parse_instance_path( instance_path ) - # Create descriptive name: space-pr123-package-model-instance - space_name = f"space-pr{pr_number}-{package_name}-{model_name}-{instance_name}" - space_description = f"Discovery space for benchmark instance from PR #{pr_number}: {package_name}/{model_name}/{instance_name}" - - # Build custom labels with algorithm-nexus fields + # Build name, description, and labels depending on whether we are in PR mode labels = space_config.metadata.labels or {} - labels["algorithm-nexus.pr_url"] = self.pr_url + if self.pr_url: + pr_number = self.pr_url.rstrip("/").split("/")[-1] + space_name = ( + f"space-pr{pr_number}-{package_name}-{model_name}-{instance_name}" + ) + space_description = f"Discovery space for benchmark instance from PR #{pr_number}: {package_name}/{model_name}/{instance_name}" + labels["algorithm-nexus.pr_url"] = self.pr_url + else: + space_name = f"space-{package_name}-{model_name}-{instance_name}" + space_description = f"Discovery space for benchmark instance: {package_name}/{model_name}/{instance_name}" + labels["algorithm-nexus.instance_path"] = str(instance_path) # Update metadata with descriptive name, description, and labels @@ -863,35 +866,28 @@ def _create_operation( Returns: Dictionary with operation_id and ray_job_id (if remote execution) """ - # Generate descriptive name and description from instance path + # Generate descriptive name, description, and labels depending on mode + custom_metadata = {} if instance_path: - # Extract PR number from URL (pr_url may be None in non-PR mode) - pr_number = ( - self.pr_url.rstrip("/").split("/")[-1] if self.pr_url else "unknown" - ) - - # Parse instance path to get package, model, and instance names package_name, model_name, instance_name = self._parse_instance_path( instance_path ) + custom_metadata["algorithm-nexus.instance_path"] = str(instance_path) - # Create descriptive name: randomwalk-pr123-package-model-instance - operation_name = ( - f"randomwalk-pr{pr_number}-{package_name}-{model_name}-{instance_name}" - ) - operation_description = f"Random walk for benchmark instance from PR #{pr_number}: {package_name}/{model_name}/{instance_name}" + if self.pr_url: + pr_number = self.pr_url.rstrip("/").split("/")[-1] + operation_name = f"randomwalk-pr{pr_number}-{package_name}-{model_name}-{instance_name}" + operation_description = f"Random walk for benchmark instance from PR #{pr_number}: {package_name}/{model_name}/{instance_name}" + custom_metadata["algorithm-nexus.pr_url"] = self.pr_url + else: + operation_name = ( + f"randomwalk-{package_name}-{model_name}-{instance_name}" + ) + operation_description = f"Random walk for benchmark instance: {package_name}/{model_name}/{instance_name}" else: operation_name = "randomwalk-all" operation_description = "Perform a random walk on all points in a space" - # Create custom metadata with algorithm-nexus fields - custom_metadata = { - "algorithm-nexus.pr_url": self.pr_url or "", - "algorithm-nexus.instance_path": str(instance_path) - if instance_path - else "", - } - # Create operation config using the factory function operation_config = create_random_walk_operation_config( space_id=space_id, @@ -967,38 +963,37 @@ def _create_operation( "ray_job_id": ray_job_id, } - def run(self) -> dict[str, Any]: + def run( + self, + packages_root: Path | None = None, + package_filter: str | None = None, + ) -> dict[str, Any]: """Main execution method. + Args: + packages_root: Path to packages directory (for all/package mode) + package_filter: Optional package name to filter by + Returns: Dictionary with execution results """ try: - console.print("Analyzing PR for new or changed benchmark instances...") - console.print(f"PR URL: {self.pr_url}") - - # Always check if we need to checkout PR code at the beginning - if not self.is_local_repo_on_pr_commit(): - console.print("[yellow]Local repository is not on PR commit[/yellow]") - console.print("Checking out PR code to temporary directory...") - self.checkout_pr_to_temp() - else: - console.print( - "[green]✓[/green] Using local repository (already on PR commit)" - ) + # Print mode header + self._print_mode_header(packages_root, package_filter, "Executing") - changed_files = self.get_changed_files() + # Discover benchmark instances + benchmark_instances = self._discover_instances( + packages_root, package_filter + ) - benchmark_instances = self.find_benchmark_instances(changed_files) + # Print found instances (show list for dry-run/non-execution mode only) + self._print_instances_found( + benchmark_instances, package_filter, show_list=not self.execute + ) if not benchmark_instances: - console.print( - "[yellow]No new or changed benchmark instances found in this PR.[/yellow]" - ) return {"instances": []} - console.print(f"Found {len(benchmark_instances)} benchmark instance(s):") - results: dict[str, Any] = { "instances": [], } @@ -1013,7 +1008,7 @@ def run(self) -> dict[str, Any]: exec_result = self.execute_benchmark(instance_path) results["instances"].append(exec_result.model_dump()) - if exec_result.status == "success": + if exec_result.status in ("success", "started"): successful += 1 else: failed += 1 @@ -1024,9 +1019,13 @@ def run(self) -> dict[str, Any]: console.print(f" Successful: {successful}") console.print(f" Failed: {failed}") console.print("=" * 60) + + results["summary"] = { + "successful": successful, + "failed": failed, + } else: for instance_path in benchmark_instances: - console.print(f" {instance_path}") results["instances"].append({"instance_path": str(instance_path)}) return results diff --git a/src/algorithm_nexus/commands/run.py b/src/algorithm_nexus/commands/run.py index 4fbf987..1668f18 100644 --- a/src/algorithm_nexus/commands/run.py +++ b/src/algorithm_nexus/commands/run.py @@ -34,12 +34,27 @@ def run_benchmarks( pr: Annotated[ - str, + str | None, typer.Option( "--pr", - help="GitHub Pull Request URL (e.g., https://github.com/IBM/algorithm-nexus/pull/123)", + help="GitHub Pull Request URL (e.g., https://github.com/IBM/algorithm-nexus/pull/123). " + "If not provided, runs benchmark instances from packages.", + ), + ] = None, + packages_root: Annotated[ + Path, + typer.Option( + "--packages-root", + help="Path to packages directory", + ), + ] = Path("./packages"), + package: Annotated[ + str | None, + typer.Option( + "--package", + help="Run only benchmark instances from a specific package", ), - ], + ] = None, remote: Annotated[ Path | None, typer.Option( @@ -87,28 +102,60 @@ def run_benchmarks( ), ] = None, ) -> None: - """Execute benchmarks from a GitHub Pull Request. + """Execute benchmarks. - Identifies new or changed benchmark instances in a PR and optionally - executes them using the ADO CLI. When executing with --remote flag, - automatically installs required benchmark packages in the Ray environment. + This command supports three modes: + 1. PR mode: Execute instances modified in a PR (provide --pr) + 2. All mode: Execute all benchmark instances (no --pr) + 3. Package mode: Execute instances from a specific package (use --package) - The command automatically checks if the local repository is on the same - commit as the PR. If not, it will checkout the PR code in a temporary - directory. + Identifies benchmark instances and optionally executes them using the ADO CLI. + When executing with --remote flag, automatically installs required benchmark + packages in the Ray environment. """ # Validate output format if specified if output_format: validate_output_format(output_format, allow_yaml=True, allow_csv=False) - try: - manager = BenchmarkManager( - pr_url=pr, - execute=not dry_run, - remote_context_file=remote, - context_file=context, + # Warn if both package filter and PR URL are provided (package is ignored in PR mode) + if package and pr: + console.print( + "[yellow]Warning:[/yellow] --package is ignored when --pr is specified. " + "In PR mode, only instances changed in the PR are executed." ) - results = manager.run() + + # Validate package exists if package filter is specified (non-PR mode only) + if package and not pr: + package_path = packages_root / package + if not package_path.is_dir(): + console.print( + f"[red]Error:[/red] Package '{package}' not found in {packages_root.resolve()}" + ) + console.print( + "\nTo see available packages, run: [cyan]nexus list packages[/cyan]" + ) + raise typer.Exit(code=1) + + try: + if pr: + manager = BenchmarkManager( + pr_url=pr, + execute=not dry_run, + remote_context_file=remote, + context_file=context, + ) + results = manager.run() + else: + manager = BenchmarkManager( + pr_url=None, + execute=not dry_run, + remote_context_file=remote, + context_file=context, + ) + results = manager.run( + packages_root=packages_root, + package_filter=package, + ) # Output results # Determine output format @@ -138,6 +185,8 @@ def run_benchmarks( if not dry_run and results.get("summary") and results["summary"]["failed"] > 0: raise typer.Exit(code=1) + except typer.Exit: + raise except KeyboardInterrupt: console_err.print("\n[yellow]Interrupted by user[/yellow]") raise typer.Exit(code=130) diff --git a/tests/test_cli_run.py b/tests/test_cli_run.py index b544829..c877c1a 100644 --- a/tests/test_cli_run.py +++ b/tests/test_cli_run.py @@ -305,4 +305,66 @@ def test_model_dump(self) -> None: assert data["operation_id"] is None +class TestRunBenchmarksCommand: + """Tests for run benchmarks CLI command.""" + + def test_run_benchmarks_nonexistent_package(self, tmp_path, capsys) -> None: + """Test run benchmarks with nonexistent package.""" + import typer + + from algorithm_nexus.commands.run import run_benchmarks + + packages_root = tmp_path / "packages" + packages_root.mkdir() + (packages_root / "existing-package").mkdir() + + with pytest.raises(typer.Exit) as exc_info: + run_benchmarks( + pr=None, + packages_root=packages_root, + package="nonexistent-package", + dry_run=True, + ) + assert exc_info.value.exit_code == 1 + + captured = capsys.readouterr() + assert "nexus list packages" in captured.out + + def test_run_benchmarks_both_package_and_pr_warns( + self, tmp_path, capsys, monkeypatch + ) -> None: + """Test that specifying both --package and --pr prints a warning.""" + from algorithm_nexus.commands.run import run_benchmarks + + packages_root = tmp_path / "packages" + packages_root.mkdir() + (packages_root / "test-package").mkdir() + + # Mock BenchmarkManager to avoid real operations + class DummyManager: + def __init__(self, *args, **kwargs): + pass + + def run(self, *args, **kwargs): + return {"instances": []} + + monkeypatch.setattr( + "algorithm_nexus.commands.run.BenchmarkManager", DummyManager + ) + + import contextlib + + with contextlib.suppress(Exception): + run_benchmarks( + pr="https://github.com/IBM/algorithm-nexus/pull/123", + packages_root=packages_root, + package="test-package", + dry_run=True, + ) + + captured = capsys.readouterr() + assert "Warning" in captured.out + assert "--package is ignored when --pr is specified" in captured.out + + # Made with Bob