From fe626e6646ec3090bfc061d6da72874d1018a4b6 Mon Sep 17 00:00:00 2001 From: Michele Gazzetti Date: Mon, 29 Jun 2026 13:12:58 +0100 Subject: [PATCH 1/5] feat(benchmark): Add vllm_performance benchmark for prithvi Signed-off-by: Michele Gazzetti --- .../vllm_performance/space.yaml | 96 +++++++++++++++++++ packages/terratorch/nexus.yaml | 6 ++ 2 files changed, 102 insertions(+) create mode 100644 packages/terratorch/models/prithvi/benchmark_instances/vllm_performance/space.yaml diff --git a/packages/terratorch/models/prithvi/benchmark_instances/vllm_performance/space.yaml b/packages/terratorch/models/prithvi/benchmark_instances/vllm_performance/space.yaml new file mode 100644 index 0000000..1bb12e2 --- /dev/null +++ b/packages/terratorch/models/prithvi/benchmark_instances/vllm_performance/space.yaml @@ -0,0 +1,96 @@ + +entitySpace: + - identifier: "model" + propertyDomain: + variableType: "CATEGORICAL_VARIABLE_TYPE" + values: ["ibm-nasa-geospatial/Prithvi-EO-2.0-300M-TL-Sen1Floods11"] + + - identifier: "request_rate" + propertyDomain: + variableType: "DISCRETE_VARIABLE_TYPE" + values: [1, 16, 23, 32, 64] + + # Docker images: one with threadpool support (v0.20.1), one without (v0.18.0) + - identifier: "image" + metadata: + description: "Docker image with vLLM + terratorch" + propertyDomain: + variableType: "CATEGORICAL_VARIABLE_TYPE" + values: + - "quay.io/mgazzetti/vllm-performance/vllm:vllm.v0.20.1-tt.v1.2.7" + + # Optional: Number of GPUs + - identifier: "n_gpus" + propertyDomain: + variableType: "DISCRETE_VARIABLE_TYPE" + values: [1] + + # Optional: GPU type + - identifier: "gpu_type" + propertyDomain: + variableType: "CATEGORICAL_VARIABLE_TYPE" + values: ["NVIDIA-A100-80GB-PCIe"] + + - identifier: "max_num_seq" + propertyDomain: + variableType: "DISCRETE_VARIABLE_TYPE" + values: [256] + + - identifier: "max_batch_tokens" + propertyDomain: + variableType: "DISCRETE_VARIABLE_TYPE" + values: [16384] + + # Optional: Number of CPUs + - identifier: "n_cpus" + propertyDomain: + variableType: "DISCRETE_VARIABLE_TYPE" + values: [16] + + - identifier: "memory" + propertyDomain: + variableType: "CATEGORICAL_VARIABLE_TYPE" + values: ['64Gi'] + + - identifier: "gpu_memory_utilization" + propertyDomain: + variableType: "DISCRETE_VARIABLE_TYPE" + values: [0.9] + + - identifier: "dtype" + propertyDomain: + variableType: "CATEGORICAL_VARIABLE_TYPE" + values: ["float16"] + + - identifier: "num_prompts" + propertyDomain: + variableType: "DISCRETE_VARIABLE_TYPE" + values: [500] + + - identifier: "dataset" + propertyDomain: + variableType: "CATEGORICAL_VARIABLE_TYPE" + values: ["india_url_in_b64_out"] + + # Geospatial-specific: Skip tokenizer initialization + - identifier: "skip_tokenizer_init" + propertyDomain: + variableType: "DISCRETE_VARIABLE_TYPE" + values: [1] + + # Geospatial-specific: Enforce eager mode + - identifier: "enforce_eager" + propertyDomain: + variableType: "DISCRETE_VARIABLE_TYPE" + values: [1] + + # Geospatial-specific: IO processor plugin + - identifier: "io_processor_plugin" + propertyDomain: + variableType: "CATEGORICAL_VARIABLE_TYPE" + values: ["terratorch_segmentation"] + + +experiments: + - actuatorIdentifier: "vllm_performance" + experimentIdentifier: "test-geospatial-deployment-custom-dataset-v1" diff --git a/packages/terratorch/nexus.yaml b/packages/terratorch/nexus.yaml index adeaede..e68258b 100644 --- a/packages/terratorch/nexus.yaml +++ b/packages/terratorch/nexus.yaml @@ -1,2 +1,8 @@ package: name: terratorch + + benchmark_packages: + # Local package stored in this repository + - requirement_specifier: "ado-vllm-performance==1.8.0" + experiments: + - "test-geospatial-deployment-custom-dataset-v1" From 90759c3959eec8ebb31725f13245ec8453638717 Mon Sep 17 00:00:00 2001 From: Michele Gazzetti Date: Wed, 15 Jul 2026 11:04:04 +0100 Subject: [PATCH 2/5] build(benchmark): bump terratorch benchmark package version to >=1.9.6 Signed-off-by: Michele Gazzetti --- packages/terratorch/nexus.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/terratorch/nexus.yaml b/packages/terratorch/nexus.yaml index e68258b..b64f468 100644 --- a/packages/terratorch/nexus.yaml +++ b/packages/terratorch/nexus.yaml @@ -3,6 +3,6 @@ package: benchmark_packages: # Local package stored in this repository - - requirement_specifier: "ado-vllm-performance==1.8.0" + - requirement_specifier: "ado-vllm-performance>=1.9.6" experiments: - "test-geospatial-deployment-custom-dataset-v1" From d7c66e727baeab946734178fe1c67464c1bf490e Mon Sep 17 00:00:00 2001 From: Michele Gazzetti Date: Thu, 6 Aug 2026 13:05:01 +0100 Subject: [PATCH 3/5] build(benchmark): update Prithvi benchmark package - bump ado-vllm-performance version to 1.13.2 - update experiment name to geospatial-vllm-bench-deployment-custom-dataset Signed-off-by: Michele Gazzetti --- packages/terratorch/nexus.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/terratorch/nexus.yaml b/packages/terratorch/nexus.yaml index b64f468..d09077c 100644 --- a/packages/terratorch/nexus.yaml +++ b/packages/terratorch/nexus.yaml @@ -3,6 +3,6 @@ package: benchmark_packages: # Local package stored in this repository - - requirement_specifier: "ado-vllm-performance>=1.9.6" + - requirement_specifier: "ado-vllm-performance>=1.13.2" experiments: - - "test-geospatial-deployment-custom-dataset-v1" + - "geospatial-vllm-bench-deployment-custom-dataset" From 0b1dbb5b5ea520da9c18039699a3efee198b7d38 Mon Sep 17 00:00:00 2001 From: Michele Gazzetti Date: Thu, 6 Aug 2026 13:38:42 +0100 Subject: [PATCH 4/5] fix(benchmark): update Prithvi experiment indentifier Signed-off-by: Michele Gazzetti --- .../prithvi/benchmark_instances/vllm_performance/space.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/terratorch/models/prithvi/benchmark_instances/vllm_performance/space.yaml b/packages/terratorch/models/prithvi/benchmark_instances/vllm_performance/space.yaml index 1bb12e2..463a490 100644 --- a/packages/terratorch/models/prithvi/benchmark_instances/vllm_performance/space.yaml +++ b/packages/terratorch/models/prithvi/benchmark_instances/vllm_performance/space.yaml @@ -93,4 +93,4 @@ entitySpace: experiments: - actuatorIdentifier: "vllm_performance" - experimentIdentifier: "test-geospatial-deployment-custom-dataset-v1" + experimentIdentifier: "geospatial-vllm-bench-deployment-custom-dataset" From c64418ed992139bcbc86020b9ae5b1d40df632f1 Mon Sep 17 00:00:00 2001 From: Michele Gazzetti Date: Thu, 6 Aug 2026 16:02:34 +0100 Subject: [PATCH 5/5] fix(benchmark): add experimentVersion to the Prithvi experiment Signed-off-by: Michele Gazzetti --- .../prithvi/benchmark_instances/vllm_performance/space.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/packages/terratorch/models/prithvi/benchmark_instances/vllm_performance/space.yaml b/packages/terratorch/models/prithvi/benchmark_instances/vllm_performance/space.yaml index 463a490..6a0fb4f 100644 --- a/packages/terratorch/models/prithvi/benchmark_instances/vllm_performance/space.yaml +++ b/packages/terratorch/models/prithvi/benchmark_instances/vllm_performance/space.yaml @@ -94,3 +94,4 @@ entitySpace: experiments: - actuatorIdentifier: "vllm_performance" experimentIdentifier: "geospatial-vllm-bench-deployment-custom-dataset" + experimentVersion: "1.0.0"