Skip to content

Feature/8754 gridded products variables across zarr stores #317

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
utas-raymondng merged 3 commits into from
Aug 11, 2026
Merged

Feature/8754 gridded products variables across zarr stores #317

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
46b8e47
remove request-time elasticsearch dataset membership validation
weited Aug 5, 2026
579f419
publish tile_types from DAS capability rather than variable arity
weited Aug 5, 2026
4b7ef78
drop the now-unused Objects import from DasTilerService
weited Aug 5, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
20 changes: 1 addition & 19 deletions server/src/main/java/au/org/aodn/ogcapi/server/core/service/das/DasTilerService.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,8 +2,6 @@

import au.org.aodn.ogcapi.server.core.configuration.Config;
import au.org.aodn.ogcapi.server.core.exception.DasUpstreamException;
import au.org.aodn.ogcapi.server.core.service.Search;
import au.org.aodn.stac.model.StacCollectionModel;
import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.ObjectMapper;
import lombok.extern.slf4j.Slf4j;
Expand All @@ -24,7 +22,6 @@
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.Set;

/**
Expand All @@ -45,7 +42,6 @@ public class DasTilerService {
private static final String DATA_TILES_BASE = "/api/v1/das/tiler/data_tiles";

protected final DasProperties dasProperties;
protected final Search search;

/**
* The shared DAS client — short timeouts, and it attaches the DAS API key to every request
Expand All @@ -55,11 +51,10 @@ public class DasTilerService {
private final ObjectMapper mapper;

public DasTilerService(
DasProperties dasProperties, Search search,
DasProperties dasProperties,
@Qualifier(Config.DAS_REST_TEMPLATE) RestTemplate httpClient,
ObjectMapper mapper) {
this.dasProperties = dasProperties;
this.search = search;
this.httpClient = httpClient;
this.mapper = mapper;
}
Expand Down Expand Up @@ -217,19 +212,6 @@ public List<JsonNode> productsForCollection(String collectionId) {
return result;
}

public boolean isDatasetInCollection(String collectionId, String dataset) {
var result = search.searchCollections(collectionId);
if (result == null || result.getCollections() == null) {
return false;
}
return result.getCollections().stream()
.map(StacCollectionModel::getAssets)
.filter(Objects::nonNull)
.flatMap(assets -> assets.keySet().stream())
.map(key -> key.contains(".") ? key.substring(0, key.indexOf('.')) : key)
.anyMatch(dataset::equals);
}

private DasTileResult exchangeForImage(UriComponentsBuilder builder, Map<String, Object> params, String fallbackContentType) {
String url = builder.encode().toUriString();
try {
Expand Down
7 changes: 2 additions & 5 deletions server/src/main/java/au/org/aodn/ogcapi/server/tile/RestApi.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -105,7 +105,8 @@ public ResponseEntity<String> collectionMapGetTile(String tileMatrix, Integer ti
content = @Content(mediaType = "application/json",
schema = @Schema(implementation = ErrorResponse.class))),
@ApiResponse(responseCode = "404", description = "`tileMatrixSetId` is not `WebMercatorQuad`, " +
"`dataset` is not in the collection, or there is no data for that date.",
"or DAS reported an unknown product (`{dataset}:{variable}`) or an unavailable date. " +
"The DAS cases are forwarded from DAS, which owns the product catalogue.",
content = @Content(mediaType = "application/json",
schema = @Schema(implementation = ErrorResponse.class))),
@ApiResponse(responseCode = "422", description = "The product cannot be rendered as a visual " +
Expand Down Expand Up @@ -211,10 +212,6 @@ public ResponseEntity<byte[]> getCollectionVisualMapTile(
if (!"png".equals(f) && !"webp".equals(f)) {
throw new InvalidParameterException("f must be 'png' or 'webp'");
}
if (!dasTilerService.isDatasetInCollection(collectionId, dataset)) {
throw new ResourceNotFoundException(
"dataset '" + dataset + "' not found in collection '" + collectionId + "'");
}

// DAS identifies a renderable product by the combined {dataset}:{variable} id.
String product = dataset + ":" + variable;
Expand Down
53 changes: 32 additions & 21 deletions server/src/main/java/au/org/aodn/ogcapi/server/tile/RestExtApi.java
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,7 +1,6 @@
package au.org.aodn.ogcapi.server.tile;

import au.org.aodn.ogcapi.server.core.exception.InvalidParameterException;
import au.org.aodn.ogcapi.server.core.exception.ResourceNotFoundException;
import au.org.aodn.ogcapi.server.core.model.ErrorResponse;
import au.org.aodn.ogcapi.server.core.service.das.DasTilerService;
import com.fasterxml.jackson.databind.JsonNode;
Expand Down Expand Up @@ -43,8 +42,12 @@ public class RestExtApi {
description = "The discovery call a map client makes before requesting tiles: it supplies every " +
"value the tile routes need, including ready-to-use URL templates.\n\n" +
"`tile_types` is a capability list — what this service can serve today, not a property of " +
"the data. Single-variable products give `[\"visual\", \"data\"]`; two-variable products " +
"(e.g. `ucur+vcur`) give `[\"data\"]` (they cannot be colourised as visual tiles). Each " +
"the data. It reflects the capability DAS reports per product: a product DAS marks as " +
"visual-capable gives `[\"visual\", \"data\"]`, one it does not gives `[\"data\"]`. " +
"Two-variable products (e.g. `ucur+vcur`) are always `[\"data\"]` — they cannot be " +
"colourised — and a single-variable product may be `[\"data\"]` too when its variable is " +
"not renderable. Variable arity is used only as a fallback against an older DAS that does " +
"not report capability. Each " +
"capability carries its own template(s): `visual_tile_url_template` + `legend_url` when " +
"`\"visual\"` is present, `data_tile_url_template` + `data_manifest_url_template` when " +
"`\"data\"` is present. For two-variable products the `variable` array order (e.g. " +
Expand Down Expand Up @@ -77,6 +80,15 @@ public class RestExtApi {
"full_date_range": {"start": "2020-01-01", "end": "2024-01-02"},
"data_tile_url_template": "/api/v1/ogc/ext/tiles/collections/0c9eb39c-9cbe-4c6a-8a10-5867087e703a/data_tiles/{lod}/{x}/{y}?dataset=model_sea_level_anomaly_gridded_realtime&variable=ucur%2Bvcur&datetime={datetime}",
"data_manifest_url_template": "/api/v1/ogc/ext/tiles/collections/0c9eb39c-9cbe-4c6a-8a10-5867087e703a/data_tiles/manifest?dataset=model_sea_level_anomaly_gridded_realtime&variable=ucur%2Bvcur&datetime={datetime}"
},
{
"id": "model_sea_level_anomaly_gridded_realtime:wdir",
"variable": "WDIR",
"tile_types": ["data"],
"available_dates": ["2024-01-01", "2024-01-02"],
"full_date_range": {"start": "2020-01-01", "end": "2024-01-02"},
"data_tile_url_template": "/api/v1/ogc/ext/tiles/collections/0c9eb39c-9cbe-4c6a-8a10-5867087e703a/data_tiles/{lod}/{x}/{y}?dataset=model_sea_level_anomaly_gridded_realtime&variable=wdir&datetime={datetime}",
"data_manifest_url_template": "/api/v1/ogc/ext/tiles/collections/0c9eb39c-9cbe-4c6a-8a10-5867087e703a/data_tiles/manifest?dataset=model_sea_level_anomaly_gridded_realtime&variable=wdir&datetime={datetime}"
}
]
}"""))),
Expand Down Expand Up @@ -113,10 +125,15 @@ public ResponseEntity<JsonNode> getCollectionProducts(
entry.set("variable", variable);

// tile_types is a capability list: what THIS service can serve today, not a property of
// the data. Visual tiles are colourised scalars only; DAS rejects multi-variable products
// there. Data tiles support one or exactly two variables (the shader packs one or two
// channels); three or more is unsupported by the protocol, so nothing is servable.
boolean canVisual = variableCount == 1;
// the data. Visual capability now comes from DAS, which knows whether a variable is
// actually renderable — arity cannot tell a colourisable scalar from one the renderer
// has no sensible colouring for. The arity rule survives only as a fallback for a DAS
// old enough to have no `visual` field, which the OGC-first deployment order requires.
// Data tiles still follow arity: the shader packs one or two channels, and DAS config
// validation guarantees nothing longer reaches here.
boolean canVisual = product.has("visual")
? product.path("visual").asBoolean()
: variableCount == 1;
boolean canData = variableCount == 1 || variableCount == 2;
ArrayNode tileTypes = mapper.createArrayNode();
if (canVisual) {
Expand All @@ -133,8 +150,8 @@ public ResponseEntity<JsonNode> getCollectionProducts(
entry.set("full_date_range", availability != null && !availability.isMissingNode()
? availability.path("full_date_range") : mapper.createObjectNode());

// The tile routes take dataset and variable separately (they check dataset membership
// against the collection's assets), so split the product id on its first ':'. The
// The tile routes take dataset and variable separately, so split the product id on its
// first ':'. That split is the only place the id is treated as anything but opaque. The
// variable half of a two-variable product contains '+' (e.g. ucur+vcur), which URLEncoder
// renders as %2B — without that a query string would decode it back to a space.
int sep = id.indexOf(':');
Expand Down Expand Up @@ -204,8 +221,9 @@ public ResponseEntity<JsonNode> getCollectionProducts(
"negative `x`/`y`.",
content = @Content(mediaType = "application/json",
schema = @Schema(implementation = ErrorResponse.class))),
@ApiResponse(responseCode = "404", description = "`dataset` is not in the collection, or DAS has no " +
"such product, LOD or chunk, or no data for that date.",
@ApiResponse(responseCode = "404", description = "DAS reported an unknown product " +
"(`{dataset}:{variable}`), an unavailable date, or an out-of-range LOD or chunk. " +
"Forwarded from DAS, which owns the product catalogue.",
content = @Content(mediaType = "application/json",
schema = @Schema(implementation = ErrorResponse.class))),
@ApiResponse(responseCode = "422", description = "DAS could not process the request (e.g. a " +
Expand Down Expand Up @@ -278,10 +296,6 @@ public ResponseEntity<byte[]> getCollectionDataTile(
throw new InvalidParameterException("x and y (chunk column/row) must be >= 0");
}
validateProductParams(dataset, variable, datetime);
if (!dasTilerService.isDatasetInCollection(collectionId, dataset)) {
throw new ResourceNotFoundException(
"dataset '" + dataset + "' not found in collection '" + collectionId + "'");
}

// DAS identifies a product by the combined {dataset}:{variable} id.
String product = dataset + ":" + variable;
Expand Down Expand Up @@ -314,8 +328,9 @@ public ResponseEntity<byte[]> getCollectionDataTile(
"containing a space (an unencoded `+`), or `datetime` not `YYYY-MM-DD`.",
content = @Content(mediaType = "application/json",
schema = @Schema(implementation = ErrorResponse.class))),
@ApiResponse(responseCode = "404", description = "`dataset` is not in the collection, or DAS has " +
"no such product, or no data for that date.",
@ApiResponse(responseCode = "404", description = "DAS reported an unknown product " +
"(`{dataset}:{variable}`) or an unavailable date. Forwarded from DAS, which owns " +
"the product catalogue.",
content = @Content(mediaType = "application/json",
schema = @Schema(implementation = ErrorResponse.class))),
@ApiResponse(responseCode = "422", description = "DAS could not process the request (e.g. a " +
Expand Down Expand Up @@ -361,10 +376,6 @@ public ResponseEntity<JsonNode> getCollectionDataManifest(
@RequestParam(required = false) String datetime) {

validateProductParams(dataset, variable, datetime);
if (!dasTilerService.isDatasetInCollection(collectionId, dataset)) {
throw new ResourceNotFoundException(
"dataset '" + dataset + "' not found in collection '" + collectionId + "'");
}

String product = dataset + ":" + variable;
DasTilerService.DasJsonResult manifest = dasTilerService.getDataManifest(product, datetime);
Expand Down
47 changes: 1 addition & 46 deletions server/src/test/java/au/org/aodn/ogcapi/server/core/service/das/DasTilerServiceTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,10 +1,6 @@
package au.org.aodn.ogcapi.server.core.service.das;

import au.org.aodn.ogcapi.server.core.exception.DasUpstreamException;
import au.org.aodn.ogcapi.server.core.service.ElasticSearchBase;
import au.org.aodn.ogcapi.server.core.service.Search;
import au.org.aodn.stac.model.AssetModel;
import au.org.aodn.stac.model.StacCollectionModel;
import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.databind.node.ObjectNode;
Expand Down Expand Up @@ -45,20 +41,18 @@ public class DasTilerServiceTest {
private static final String PRODUCT_ID = "model_sea_level_anomaly_gridded_realtime:gsla";

private RestTemplate httpClient;
private Search search;
private DasTilerService service;

@BeforeEach
public void setUp() {
httpClient = mock(RestTemplate.class);
search = mock(Search.class);

DasProperties config = new DasProperties(
HOST, null,"test-secret", "internal-secret",
Duration.ofSeconds(5), Duration.ofSeconds(30)
);

service = new DasTilerService(config, search, httpClient, new ObjectMapper());
service = new DasTilerService(config, httpClient, new ObjectMapper());
}

private HttpHeaders imageHeaders() {
Expand Down Expand Up @@ -374,45 +368,6 @@ public void testProductsForCollectionFiltersByMetadataUuid() {
assertTrue(service.productsForCollection("unknown-uuid").isEmpty());
}

@Test
public void testIsDatasetInCollectionChecksAssetKeys() {
// es-indexer keys assets by the cloud-optimised file name, which carries a format extension,
// while DAS product ids use the bare stem — so everything from the first dot on is dropped
// before matching. Membership is a stem lookup against the cached searchCollections result.
// The extension is not enumerated, so an unknown/future format works the same way.
StacCollectionModel model = StacCollectionModel.builder()
.uuid("uuid-a")
.assets(Map.of(
"satellite_austemp_heatwave_8day.zarr",
AssetModel.builder().role(AssetModel.Role.SUMMARY).build(),
"mooring_temperature_logger_delayed.parquet",
AssetModel.builder().role(AssetModel.Role.SUMMARY).build(),
"some_future_format_dataset.nc4",
AssetModel.builder().role(AssetModel.Role.SUMMARY).build()))
.build();
ElasticSearchBase.SearchResult<StacCollectionModel> found = new ElasticSearchBase.SearchResult<>();
found.setCollections(List.of(model));
when(search.searchCollections("uuid-a")).thenReturn(found);

assertTrue(service.isDatasetInCollection("uuid-a", "satellite_austemp_heatwave_8day"),
"the .zarr extension on the asset key must be ignored when matching the dataset");
assertTrue(service.isDatasetInCollection("uuid-a", "mooring_temperature_logger_delayed"),
"the .parquet extension on the asset key must be ignored when matching the dataset");
assertTrue(service.isDatasetInCollection("uuid-a", "some_future_format_dataset"),
"the extension is not hard-coded, so an unknown format is stripped the same way");
assertFalse(service.isDatasetInCollection("uuid-a", "some_other_dataset"),
"a dataset that is not an asset key is not in the collection");
}

@Test
public void testIsDatasetInCollectionFalseWhenCollectionMissing() {
ElasticSearchBase.SearchResult<StacCollectionModel> empty = new ElasticSearchBase.SearchResult<>();
empty.setCollections(List.of());
when(search.searchCollections("uuid-missing")).thenReturn(empty);

assertFalse(service.isDatasetInCollection("uuid-missing", "model_sea_level_anomaly_gridded_realtime"));
}

private record CapturedRequest(String url, Map<String, Object> params) {
}
}
Loading
Loading