From 3f99d08963da4100842142a18321e605e4b8da82 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johannes=20Dr=C3=B6nner?= Date: Fri, 2 Oct 2026 13:01:38 +0200 Subject: [PATCH 1/9] feat(backend): add NEX-GDDP-CMIP6 climate-risk process Derives WMO/ECII climate-risk indicators (heat, ice, tropical nights, frost, dry and heavy-rain days) for a point location and time window from NEX-GDDP-CMIP6 projections ingested as Geo Engine rasters. For every variable and future scenario, a Geo Engine `RasterVectorJoin` workflow applies the daily threshold expression and aggregates it per year at the requested point; the same is done against the `historical` scenario over the reference window to obtain the anomaly baseline (default 1981, the WMO 30-year normal). Results are cross-model min/max/mean/median statistics plus the occurrence probability and the anomaly against the reference mean, returned as one tabular Data Resource per scenario with an optional raw ensemble. Which models and scenarios can run is derived from `conf/nexgddp_models.toml`, so no recompile is needed to add a model: the process description, the `models` dropdown and the declared scenario outputs all follow the registry. `nexgddp_models.full.toml` holds the full not-yet-ingested matrix as the restore path for a full ingest. The table schema carries a `schema.biois` display extension (published as a JSON Schema profile) so a column can declare a label and colour, e.g. a risk probability class or a signed anomaly. Notes for review: - the `models` enum is injected by a named schema transform, following the `optional_feature_properties` / `relative_json_pointer_format` convention, since the registry is only known at runtime - `Results` in `server.rs` still omits the climate outputs, as it already did for `biodiversity` and `land_use` --- backend/Cargo.lock | 88 +- backend/Cargo.toml | 3 +- backend/Dockerfile | 1 + backend/conf/default.toml | 4 + backend/conf/nexgddp_models.full.toml | 36 + backend/conf/nexgddp_models.toml | 11 + backend/src/auth.rs | 18 +- backend/src/config.rs | 10 + backend/src/handler.rs | 4 + backend/src/lib.rs | 1 + .../biodiversity_sensitive_areas/mod.rs | 1 + backend/src/processes/climate_risk/compute.rs | 1617 +++++++++++++++++ backend/src/processes/climate_risk/mod.rs | 867 +++++++++ backend/src/processes/climate_risk/types.rs | 713 ++++++++ .../src/processes/climate_risk/workflow.rs | 135 ++ .../processes/land_use_sealed_area/types.rs | 2 + backend/src/processes/mod.rs | 2 + .../src/processes/parameters/data_resource.rs | 40 +- backend/src/processes/parameters/mod.rs | 22 +- backend/src/processes/parameters/units.rs | 19 + backend/src/processes/path_info.rs | 45 + backend/src/profile.rs | 121 ++ backend/src/server.rs | 6 +- k8s/pod.yaml | 2 +- 24 files changed, 3743 insertions(+), 25 deletions(-) create mode 100644 backend/conf/nexgddp_models.full.toml create mode 100644 backend/conf/nexgddp_models.toml create mode 100644 backend/src/processes/climate_risk/compute.rs create mode 100644 backend/src/processes/climate_risk/mod.rs create mode 100644 backend/src/processes/climate_risk/types.rs create mode 100644 backend/src/processes/climate_risk/workflow.rs create mode 100644 backend/src/profile.rs diff --git a/backend/Cargo.lock b/backend/Cargo.lock index d74d22c..8703824 100644 --- a/backend/Cargo.lock +++ b/backend/Cargo.lock @@ -358,6 +358,7 @@ dependencies = [ "toasty", "toasty-cli", "tokio", + "toml 0.8.23", "tower", "tracing", "tracing-appender", @@ -677,8 +678,8 @@ dependencies = [ "serde-untagged", "serde_core", "serde_json", - "toml", - "winnow", + "toml 1.1.4+spec-1.1.0", + "winnow 1.0.4", "yaml-rust2", ] @@ -3124,7 +3125,7 @@ version = "3.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e67ba7e9b2b56446f1d419b1d807906278ffa1a658a8a5d8a39dcb1f5a78614f" dependencies = [ - "toml_edit", + "toml_edit 0.25.13+spec-1.1.0", ] [[package]] @@ -3903,6 +3904,15 @@ dependencies = [ "syn 3.0.3", ] +[[package]] +name = "serde_spanned" +version = "0.6.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bf41e0cfaf7226dca15e8197172c295a782857fcb97fad1808a166870dee75a3" +dependencies = [ + "serde", +] + [[package]] name = "serde_spanned" version = "1.1.1" @@ -4692,8 +4702,8 @@ dependencies = [ "toasty-macros", "tokio", "tokio-stream", - "toml", - "toml_edit", + "toml 1.1.4+spec-1.1.0", + "toml_edit 0.25.13+spec-1.1.0", "tracing", "uuid", ] @@ -4713,7 +4723,7 @@ dependencies = [ "serde", "toasty", "toasty-core", - "toml", + "toml 1.1.4+spec-1.1.0", ] [[package]] @@ -4731,7 +4741,7 @@ dependencies = [ "percent-encoding", "serde", "tokio-stream", - "toml", + "toml 1.1.4+spec-1.1.0", "tracing", "uuid", ] @@ -4912,6 +4922,18 @@ dependencies = [ "tokio", ] +[[package]] +name = "toml" +version = "0.8.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc1beb996b9d83529a9e75c17a1686767d148d70663143c7854d8b4a09ced362" +dependencies = [ + "serde", + "serde_spanned 0.6.9", + "toml_datetime 0.6.11", + "toml_edit 0.22.27", +] + [[package]] name = "toml" version = "1.1.4+spec-1.1.0" @@ -4920,11 +4942,20 @@ checksum = "3aace63f4bbcdfc2c965b059de67119c89c4017a70d633be6c104910f67056f5" dependencies = [ "indexmap 2.14.0", "serde_core", - "serde_spanned", - "toml_datetime", + "serde_spanned 1.1.1", + "toml_datetime 1.1.1+spec-1.1.0", "toml_parser", "toml_writer", - "winnow", + "winnow 1.0.4", +] + +[[package]] +name = "toml_datetime" +version = "0.6.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "22cddaf88f4fbc13c51aebbf5f8eceb5c7c5a9da2ac40a13519eb5b0a0e8f11c" +dependencies = [ + "serde", ] [[package]] @@ -4936,6 +4967,20 @@ dependencies = [ "serde_core", ] +[[package]] +name = "toml_edit" +version = "0.22.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41fe8c660ae4257887cf66394862d21dbca4a6ddd26f04a3560410406a2f819a" +dependencies = [ + "indexmap 2.14.0", + "serde", + "serde_spanned 0.6.9", + "toml_datetime 0.6.11", + "toml_write", + "winnow 0.7.15", +] + [[package]] name = "toml_edit" version = "0.25.13+spec-1.1.0" @@ -4944,11 +4989,11 @@ checksum = "6975367e4d2ef766d86af01ffad14b622fecc8d4357a998fbc4deb6e9bacaf9b" dependencies = [ "indexmap 2.14.0", "serde_core", - "serde_spanned", - "toml_datetime", + "serde_spanned 1.1.1", + "toml_datetime 1.1.1+spec-1.1.0", "toml_parser", "toml_writer", - "winnow", + "winnow 1.0.4", ] [[package]] @@ -4957,9 +5002,15 @@ version = "1.1.3+spec-1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1d38ac1cf9b95face32296c0a3ede1fdc270627c9d9c02a7274dd6d960dc4d56" dependencies = [ - "winnow", + "winnow 1.0.4", ] +[[package]] +name = "toml_write" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5d99f8c9a7727884afe522e9bd5edbfc91a3312b36a77b5fb8926e4c31a41801" + [[package]] name = "toml_writer" version = "1.1.2+spec-1.1.0" @@ -5712,6 +5763,15 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" +[[package]] +name = "winnow" +version = "0.7.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df79d97927682d2fd8adb29682d1140b343be4ac0f08fd68b7765d9c059d3945" +dependencies = [ + "memchr", +] + [[package]] name = "winnow" version = "1.0.4" diff --git a/backend/Cargo.toml b/backend/Cargo.toml index c3c29dd..9add318 100644 --- a/backend/Cargo.toml +++ b/backend/Cargo.toml @@ -94,7 +94,8 @@ schemars = "1.2" serde = { version = "1.0", features = ["derive", "rc"] } serde_json = "1.0" tempfile = "3.27" -# toasty = { version = "0.11", features = ["migration", "postgresql", "serde"] } +toml = "0.8" +# toasty = { version = "0.10", features = ["postgresql", "serde"] } toasty = { git = "https://github.com/tokio-rs/toasty", branch = "main", features = [ "migration", "postgresql", diff --git a/backend/Dockerfile b/backend/Dockerfile index d6b9d26..3df76d2 100644 --- a/backend/Dockerfile +++ b/backend/Dockerfile @@ -48,6 +48,7 @@ FROM gcr.io/distroless/cc-debian13:nonroot COPY --from=builder /usr/src/biois/target/release/biois-server /usr/local/bin/biois-server COPY --from=builder /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ COPY --from=builder /emptydir /biois +COPY --from=builder /usr/src/biois/conf ./conf WORKDIR /biois diff --git a/backend/conf/default.toml b/backend/conf/default.toml index 76b8c8a..955b232 100644 --- a/backend/conf/default.toml +++ b/backend/conf/default.toml @@ -53,3 +53,7 @@ credits_per_site = 10 [credits.habitat_distance] credits_per_coordinate = 5 + +[nexgddp_cmip6] +# Model registry file. Add more models here without recompiling. +model_registry_path = "conf/nexgddp_models.toml" diff --git a/backend/conf/nexgddp_models.full.toml b/backend/conf/nexgddp_models.full.toml new file mode 100644 index 0000000..42e9861 --- /dev/null +++ b/backend/conf/nexgddp_models.full.toml @@ -0,0 +1,36 @@ +# Full NEX-GDDP-CMIP6 model matrix, not yet ingested. +# +# The dev registry (`nexgddp_models.toml`) is trimmed to the models/scenarios that actually have +# GeoEngine datasets, so the process description and the `models` dropdown only offer what can run. +# This file is the restore path: copy it over `nexgddp_models.toml` once the remaining datasets are +# ingested via `notebooks/nexgddp_cmip6_ingest.ipynb` (which reads this file for id/variant/grid). + +[[model]] +id = "ACCESS-CM2" +variant = "r1i1p1f1" +grid = "gn" +scenarios = ["historical", "ssp245", "ssp585"] + +[[model]] +id = "MPI-ESM1-2-LR" +variant = "r1i1p1f1" +grid = "gn" +scenarios = ["historical", "ssp245", "ssp585"] + +[[model]] +id = "IPSL-CM6A-LR" +variant = "r1i1p1f1" +grid = "gn" +scenarios = ["historical", "ssp245", "ssp585"] + +[[model]] +id = "MRI-ESM2-0" +variant = "r1i1p1f1" +grid = "gn" +scenarios = ["historical", "ssp245", "ssp585"] + +[[model]] +id = "GFDL-ESM4" +variant = "r1i1p1f1" +grid = "gr1" +scenarios = ["historical", "ssp245", "ssp585"] \ No newline at end of file diff --git a/backend/conf/nexgddp_models.toml b/backend/conf/nexgddp_models.toml new file mode 100644 index 0000000..76f8a6f --- /dev/null +++ b/backend/conf/nexgddp_models.toml @@ -0,0 +1,11 @@ +# NEX-GDDP-CMIP6 model registry — the source of truth for what climate-risk can compute. +# +# Only models/scenarios with a registered GeoEngine dataset belong here; the process description and +# the `models` dropdown are derived from this file. `nexgddp_models.full.toml` holds the full +# not-yet-ingested matrix — copy it over this file after a full ingest. + +[[model]] +id = "ACCESS-CM2" +variant = "r1i1p1f1" +grid = "gn" +scenarios = ["historical", "ssp245"] \ No newline at end of file diff --git a/backend/src/auth.rs b/backend/src/auth.rs index 4de6e27..78b0220 100644 --- a/backend/src/auth.rs +++ b/backend/src/auth.rs @@ -4,8 +4,8 @@ use crate::{ config::CONFIG, const_concat, processes::{ - BiodiversitySensitiveAreasProcess, HabitatDistanceProcess, LandUseSealedAreaProcess, - NDVIProcess, + BiodiversitySensitiveAreasProcess, ClimateRiskProcess, HabitatDistanceProcess, + LandUseSealedAreaProcess, NDVIProcess, }, state::{CONTEXT, TaskContext}, util::{Secret, error_response}, @@ -140,6 +140,8 @@ impl GeoEngineAuthMiddleware { const_concat!("/processes/", BiodiversitySensitiveAreasProcess::ID), const_concat!("/processes/", HabitatDistanceProcess::ID), const_concat!("/processes/", LandUseSealedAreaProcess::ID), + const_concat!("/processes/", ClimateRiskProcess::ID), + crate::profile::CLIMATE_RISK_TABLE_SCHEMA_PROFILE, ], prefix: vec!["/api", "/swagger", "/auth/"], }, @@ -308,6 +310,18 @@ mod tests { assert!(middleware.path_is_whitelisted("/health")); assert!(middleware.path_is_whitelisted("/processes/echo")); + // process descriptions + for path in [ + const_concat!("/processes/", NDVIProcess::ID), + const_concat!("/processes/", BiodiversitySensitiveAreasProcess::ID), + const_concat!("/processes/", HabitatDistanceProcess::ID), + const_concat!("/processes/", LandUseSealedAreaProcess::ID), + const_concat!("/processes/", ClimateRiskProcess::ID), + crate::profile::CLIMATE_RISK_TABLE_SCHEMA_PROFILE, + ] { + assert!(middleware.path_is_whitelisted(path), "{path}"); + } + // prefix assert!(middleware.path_is_whitelisted("/api/some/resource")); assert!(middleware.path_is_whitelisted("/swagger/index.html")); diff --git a/backend/src/config.rs b/backend/src/config.rs index 0ff1bb3..984fee1 100644 --- a/backend/src/config.rs +++ b/backend/src/config.rs @@ -16,9 +16,19 @@ pub struct Config { pub data_ids: DataIdsConfig, pub credits: CreditsConfig, pub contact: ContactConfig, + pub nexgddp_cmip6: NexGddpCmip6Config, pub logging: Logging, } +/// Configuration of the climate-risk process, which derives its indicators from +/// NEX-GDDP-CMIP6 climate projections ingested as Geo Engine datasets. +#[derive(serde::Deserialize, Clone, Debug)] +pub struct NexGddpCmip6Config { + /// The model registry, listing the models and scenarios that have ingested datasets. + /// Models are added by editing this file, without recompiling. + pub model_registry_path: String, +} + #[derive(serde::Deserialize, Clone, Debug)] pub struct Server { pub host: String, diff --git a/backend/src/handler.rs b/backend/src/handler.rs index 340c4f8..06d3693 100644 --- a/backend/src/handler.rs +++ b/backend/src/handler.rs @@ -25,6 +25,10 @@ pub fn auth_router() -> OpenApiRouter { .routes(routes!(auth_request_url_handler)) } +pub fn profile_router() -> OpenApiRouter { + OpenApiRouter::new().routes(routes!(crate::profile::climate_risk_table_schema_profile)) +} + #[utoipa::path(get, path = "/health", responses((status = NO_CONTENT)))] pub async fn health_handler() -> StatusCode { StatusCode::NO_CONTENT diff --git a/backend/src/lib.rs b/backend/src/lib.rs index 126b007..4c37fa2 100644 --- a/backend/src/lib.rs +++ b/backend/src/lib.rs @@ -7,6 +7,7 @@ pub mod db; mod handler; mod jobs; mod processes; +mod profile; mod server; mod state; mod util; diff --git a/backend/src/processes/biodiversity_sensitive_areas/mod.rs b/backend/src/processes/biodiversity_sensitive_areas/mod.rs index 49fcdae..e2de63e 100644 --- a/backend/src/processes/biodiversity_sensitive_areas/mod.rs +++ b/backend/src/processes/biodiversity_sensitive_areas/mod.rs @@ -887,6 +887,7 @@ fn site_row_into_output( }, ], primary_key: vec!["location".to_string()].into(), + ..Default::default() }, }) } diff --git a/backend/src/processes/climate_risk/compute.rs b/backend/src/processes/climate_risk/compute.rs new file mode 100644 index 0000000..be047c2 --- /dev/null +++ b/backend/src/processes/climate_risk/compute.rs @@ -0,0 +1,1617 @@ +use crate::db::model::ComputationId; +use crate::profile::CLIMATE_RISK_TABLE_SCHEMA_PROFILE; +use crate::{ + processes::parameters::{ + BioisDisplayKind, BioisDisplayMetadata, BioisTableSchemaExtension, BoundingBox, + DataResource, DataResourceName, Days, Fields, TableSchemaField, TableSchemaType, Year, + YearRange, + }, + util::{error_response, to_api_vector_process}, +}; +use anyhow::Result; +use futures::{TryStreamExt, stream::StreamExt}; +use geoengine_api_client::{ + apis::{ + configuration::Configuration, ogcwfs_api::WfsHandlerError, ogcwfs_api::wfs_handler, + workflows_api::register_workflow_handler, + }, + models::{ + ColumnNames, Coordinate2D, FeatureAggregationMethod, GeoJson, MockPointSource, + MockPointSourceParameters, Names, RasterVectorJoin, RasterVectorJoinParameters, + SingleVectorMultipleRasterSources, SpatialBoundsDerive, SpatialBoundsDeriveNone, + TemporalAggregationMethod, VectorOperator, WfsRequest, WfsService, + }, +}; +use geojson::PointType; +use ogcapi::types::processes::{ + ExecuteResult, ExecuteResults, Format, InlineOrRefData, InputValue, Output, QualifiedInputValue, +}; +use std::collections::HashMap; +use std::str::FromStr; +use tracing::instrument; + +use super::ClimateRiskProcess; +use super::types::*; +pub(crate) fn climate_risk_data_resource( + rows: Vec, + analysis_period: &str, + reference_period: Option<&str>, +) -> DataResource> { + let mut fields = vec![TableSchemaField { + name: "scenario".into(), + r#type: Some(TableSchemaType::String), + title: Some("Scenario".into()), + ..Default::default() + }]; + fields.extend(risk_fields(&rows, reference_period)); + let title = if analysis_period.is_empty() { + "Climate Risk".to_string() + } else { + format!("Climate Risk · {analysis_period}") + }; + let biois = climate_display_extension(&rows); + DataResource { + name: DataResourceName::new("climate-risk").expect("static resource name is a valid slug"), + title: Some(title), + data: rows, + schema: Fields { + fields, + primary_key: Some(vec!["variable".to_string(), "scenario".to_string()]), + schema: Some(CLIMATE_RISK_TABLE_SCHEMA_PROFILE.to_string()), + biois: Some(biois), + }, + } +} + +pub(crate) fn climate_risk_scenario_data_resource( + scenario_name: &str, + rows: Vec, + analysis_period: &str, + reference_period: Option<&str>, +) -> DataResource> { + let fields = risk_fields(&rows, reference_period); + let title = if analysis_period.is_empty() { + scenario_name.to_string() + } else { + format!("{scenario_name} · {analysis_period}") + }; + let biois = climate_display_extension(&rows); + DataResource { + name: DataResourceName::new(format!( + "climate-risk-{}", + scenario_output_id(scenario_name) + )) + .expect("scenario output ids are valid slugs"), + title: Some(title), + data: rows, + schema: Fields { + fields, + primary_key: Some(vec!["variable".to_string()]), + schema: Some(CLIMATE_RISK_TABLE_SCHEMA_PROFILE.to_string()), + biois: Some(biois), + }, + } +} + +/// Shared column layout for climate-risk tables, excluding any scenario column. +fn risk_fields(rows: &[ClimateRiskRow], reference_period: Option<&str>) -> Vec { + let mut fields = vec![ + TableSchemaField { + name: "variable".into(), + r#type: Some(TableSchemaType::String), + title: Some("Variable".into()), + ..Default::default() + }, + TableSchemaField { + name: "mean".into(), + r#type: Some(TableSchemaType::Number), + title: Some("Mean (days/year)".into()), + ..Default::default() + }, + TableSchemaField { + name: "median".into(), + r#type: Some(TableSchemaType::Number), + title: Some("Median (days/year)".into()), + ..Default::default() + }, + TableSchemaField { + name: "min".into(), + r#type: Some(TableSchemaType::Number), + title: Some("Min (days/year)".into()), + ..Default::default() + }, + TableSchemaField { + name: "max".into(), + r#type: Some(TableSchemaType::Number), + title: Some("Max (days/year)".into()), + ..Default::default() + }, + TableSchemaField { + name: "occurrenceProbability".into(), + r#type: Some(TableSchemaType::Number), + title: Some("Occurrence Probability".into()), + ..Default::default() + }, + TableSchemaField { + name: "occurrenceProbabilityLabel".into(), + r#type: Some(TableSchemaType::String), + title: Some("Occurrence Probability Label".into()), + ..Default::default() + }, + TableSchemaField { + name: "occurrenceProbabilityColor".into(), + r#type: Some(TableSchemaType::String), + title: Some("Occurrence Probability Color".into()), + ..Default::default() + }, + ]; + if rows.iter().any(|row| row.anomaly.is_some()) { + fields.extend([ + TableSchemaField { + name: "anomaly".into(), + r#type: Some(TableSchemaType::Number), + title: Some(anomaly_title(reference_period)), + ..Default::default() + }, + TableSchemaField { + name: "anomalyLabel".into(), + r#type: Some(TableSchemaType::String), + title: Some("Anomaly Label".into()), + ..Default::default() + }, + TableSchemaField { + name: "anomalyColor".into(), + r#type: Some(TableSchemaType::String), + title: Some("Anomaly Color".into()), + ..Default::default() + }, + ]); + } + fields +} + +pub(crate) fn raw_ensemble_data_resource( + mut rows: Vec, +) -> DataResource> { + rows.sort_by(|a, b| { + (&a.variable, &a.scenario, &a.model).cmp(&(&b.variable, &b.scenario, &b.model)) + }); + DataResource { + name: DataResourceName::new("raw-ensemble-data") + .expect("static resource name is a valid slug"), + title: Some("Raw Ensemble Data".to_string()), + data: rows, + schema: Fields { + fields: vec![ + TableSchemaField { + name: "variable".into(), + r#type: Some(TableSchemaType::String), + title: Some("Variable".into()), + ..Default::default() + }, + TableSchemaField { + name: "scenario".into(), + r#type: Some(TableSchemaType::String), + title: Some("Scenario".into()), + ..Default::default() + }, + TableSchemaField { + name: "model".into(), + r#type: Some(TableSchemaType::String), + title: Some("Model".into()), + ..Default::default() + }, + TableSchemaField { + name: "value".into(), + r#type: Some(TableSchemaType::Number), + title: Some("Value".into()), + ..Default::default() + }, + ], + primary_key: Some(vec![ + "variable".to_string(), + "scenario".to_string(), + "model".to_string(), + ]), + ..Default::default() + }, + } +} + +fn climate_display_extension(rows: &[ClimateRiskRow]) -> BioisTableSchemaExtension { + let has_anomaly = rows.iter().any(|row| row.anomaly.is_some()); + let mut display = HashMap::from([( + "occurrenceProbability".to_string(), + BioisDisplayMetadata { + kind: BioisDisplayKind::RiskProbability, + label_field: Some("occurrenceProbabilityLabel".to_string()), + color_field: Some("occurrenceProbabilityColor".to_string()), + }, + )]); + if has_anomaly { + display.insert( + "anomaly".to_string(), + BioisDisplayMetadata { + kind: BioisDisplayKind::RiskAnomaly, + label_field: Some("anomalyLabel".to_string()), + color_field: Some("anomalyColor".to_string()), + }, + ); + } + BioisTableSchemaExtension { + display, + hidden_fields: [ + "occurrenceProbabilityLabel", + "occurrenceProbabilityColor", + "anomalyLabel", + "anomalyColor", + ] + .into_iter() + // The probability label/color columns are always hidden; the anomaly ones only + // when an anomaly column is actually present. + .filter(|field| { + matches!( + *field, + "occurrenceProbabilityLabel" | "occurrenceProbabilityColor" + ) || (has_anomaly && matches!(*field, "anomalyLabel" | "anomalyColor")) + }) + .map(str::to_string) + .collect(), + } +} + +impl TryFrom for ExecuteResults { + type Error = anyhow::Error; + + fn try_from(outputs: ClimateRiskOutputs) -> anyhow::Result { + let mut result = ExecuteResults::default(); + + if let Some(inputs) = outputs.inputs + && let Some(value) = build_inputs_value(&inputs) + { + result.insert("inputs".to_string(), value); + } + + if let Some(climate_risk) = outputs.climate_risk { + let analysis_period = outputs.analysis_period.as_deref().unwrap_or(""); + let reference_period = outputs.reference_period.as_deref(); + let mut rows_by_scenario: std::collections::BTreeMap> = + std::collections::BTreeMap::new(); + for row in climate_risk.data { + rows_by_scenario + .entry(row.scenario.clone()) + .or_default() + .push(row); + } + for (scenario, rows) in rows_by_scenario { + match climate_risk_scenario_data_resource( + &scenario, + rows, + analysis_period, + reference_period, + ) + .to_input_value() + { + Ok(value) => { + result.insert( + scenario_output_id(&scenario), + ExecuteResult { + output: Output { + format: Some(json_format()), + transmission_mode: Default::default(), + }, + data: InlineOrRefData::QualifiedInputValue(QualifiedInputValue { + value, + format: Format { + media_type: Some( + "application/vnd.dataresource+json".to_string(), + ), + encoding: None, + schema: None, + }, + }), + }, + ); + } + Err(error) => tracing::warn!( + "Failed to serialize the climate-risk output for scenario `{scenario}`: {error}" + ), + } + } + } + + if let Some(raw_ensemble_data) = outputs.raw_ensemble_data { + match raw_ensemble_data.to_input_value() { + Ok(value) => { + result.insert( + "rawEnsembleData".to_string(), + ExecuteResult { + output: Output { + format: Some(json_format()), + transmission_mode: Default::default(), + }, + data: InlineOrRefData::QualifiedInputValue(QualifiedInputValue { + value, + format: Format { + media_type: Some( + "application/vnd.dataresource+json".to_string(), + ), + encoding: None, + schema: None, + }, + }), + }, + ); + } + Err(error) => { + tracing::warn!("Failed to serialize the raw ensemble data output: {error}"); + } + } + } + + Ok(result) + } +} + +/// Maps a scenario's display name (e.g. `"SSP2-4.5 (Intermediate emissions)"`) to the output id +/// declared in the process description (`"ssp245"`), so that the keys of the execute response +/// match the declared output ids. Falls back to the given name when it is not a known +/// display name, which keeps rows carrying output ids directly working as well. +fn scenario_output_id(scenario: &str) -> String { + ClimateScenario::ALL + .iter() + .find(|s| s.properties().name == scenario) + .map_or_else(|| scenario.to_string(), |s| s.name().to_string()) +} + +fn json_format() -> Format { + Format { + media_type: Some("application/json".to_string()), + encoding: Some("utf-8".to_string()), + schema: None, + } +} + +/// Converts a serialized object into the OGC API's qualified JSON input value. +fn build_qualified_value(object_map: serde_json::Map) -> ExecuteResult { + ExecuteResult { + output: Output { + format: None, + transmission_mode: Default::default(), + }, + data: InlineOrRefData::QualifiedInputValue(QualifiedInputValue { + value: InputValue::Object(object_map), + format: Format { + media_type: Some("application/json".to_string()), + encoding: Some("utf-8".to_string()), + schema: None, + }, + }), + } +} + +/// Serializes typed inputs at the OGC API boundary, warning and dropping on failure. +fn build_inputs_value(inputs: &ClimateRiskInputs) -> Option { + let Ok(value) = serde_json::to_value(inputs) else { + tracing::warn!("Failed to serialize the inputs output"); + return None; + }; + match value { + serde_json::Value::Object(object_map) => Some(build_qualified_value(object_map)), + other => { + tracing::warn!("Unexpected non-object inputs serialization: {other}"); + None + } + } +} +/// One geoengine workflow together with the metadata needed to interpret its results. +struct WorkflowRequest { + models: Vec, + variable: ClimateVariable, + scenario: ClimateScenarioProperties, + workflow: geoengine_api_client::models::Workflow, +} + +/// Builds one workflow per (variable, scenario) pair, using only models that support the scenario. +/// When `scenario_filter` is set, only builds workflows for that scenario regardless of what the +/// requests contain — used to build historical-reference workflows for the anomaly baseline. +fn build_workflows( + coordinate: &PointType, + requests: &[(ClimateVariableRequest, ClimateScenarioProperties)], + models: &[ClimateModelProperties], + scenario_filter: Option<&ClimateScenarioProperties>, +) -> Vec { + let effective_requests: Vec<_> = if let Some(filter) = scenario_filter { + // Build one request per unique variable, all targeting the same (historical) scenario. + let mut seen = std::collections::HashSet::new(); + requests + .iter() + .filter_map(|(var_req, _)| { + if seen.insert(var_req.variable) { + Some((var_req.clone(), filter.clone())) + } else { + None + } + }) + .collect() + } else { + requests.to_vec() + }; + + effective_requests + .iter() + .filter_map(|(var_req, scenario_props)| { + let compatible_models: Vec = models + .iter() + .filter(|model| model.scenarios.contains(&scenario_props.scenario)) + .cloned() + .collect(); + if compatible_models.is_empty() { + return None; + } + + let variable_properties = var_req.variable.properties(); + let raster_sources = compatible_models + .iter() + .map(|model| { + ClimateRiskProcess::build_variable_year_agg_workflow( + &variable_properties, + model, + scenario_props, + ) + }) + .collect::>(); + let model_var_names: Vec = compatible_models + .iter() + .map(|model| model.id.clone()) + .collect(); + + let workflow = to_api_vector_process(&VectorOperator::RasterVectorJoin( + RasterVectorJoin { + r#type: Default::default(), + params: RasterVectorJoinParameters { + names: ColumnNames::Names( + Names { + r#type: Default::default(), + values: model_var_names, + } + .into(), + ) + .into(), + feature_aggregation: FeatureAggregationMethod::First, + feature_aggregation_ignore_no_data: Some(false), + temporal_aggregation: TemporalAggregationMethod::None, + temporal_aggregation_ignore_no_data: Some(false), + } + .into(), + sources: SingleVectorMultipleRasterSources { + vector: vector_source(coordinate).into(), + rasters: raster_sources, + } + .into(), + } + .into(), + )); + Some(WorkflowRequest { + models: compatible_models, + variable: var_req.variable, + scenario: scenario_props.clone(), + workflow, + }) + }) + .collect() +} + +// bounds geoengine fan-out; the request count is finite but a single user +// request registers up to 2x(variables) workflows (analysis + historical reference) and queries each once. +const MAX_CONCURRENT_GEOENGINE_REQUESTS: usize = 8; + +/// Runs async jobs with bounded concurrency, yielding results in input order. +async fn run_limited(jobs: Vec) -> Result, E> +where + F: FnOnce() -> Fut, + Fut: std::future::Future>, +{ + // `buffered`: results must stay aligned with the input requests, + // otherwise a workflow's data is attributed to the wrong (variable, scenario) pair. + futures::stream::iter(jobs.into_iter().map(|job| job())) + .buffered(MAX_CONCURRENT_GEOENGINE_REQUESTS) + .try_collect() + .await +} + +/// Formats a geoengine API error, including the response body when available. +fn unpack_join_error(error: &geoengine_api_client::apis::Error, what: &str) -> anyhow::Error { + if let Some(response) = error_response(error) { + anyhow::anyhow!("Failed to {what} `{error}`: {response:?}") + } else { + anyhow::anyhow!("Failed to {what} `{error}`") + } +} + +async fn register_workflows( + configuration: &Configuration, + requests: &[WorkflowRequest], +) -> Result> { + let workflow_ids = run_limited( + requests + .iter() + .map(|request| { + let workflow = &request.workflow; + move || async move { + register_workflow_handler(configuration, workflow.clone()) + .await + .map(|id| id.id.to_string()) + } + }) + .collect(), + ) + .await + .map_err(|error| unpack_join_error(&error, "register a workflow"))?; + Ok(workflow_ids) +} + +async fn query_workflows( + configuration: &Configuration, + workflow_ids: &[String], + bbox_string: &str, + time: &str, +) -> Result> { + run_limited( + workflow_ids + .iter() + .map(|id| move || async move { wfs_query(configuration, id, bbox_string, time).await }) + .collect(), + ) + .await + .map_err(|error| unpack_join_error(&error, "execute a workflow")) +} + +/// Aggregates the per-workflow WFS results into climate-risk and raw-ensemble rows. +fn aggregate_rows( + analysis_results: &[WfsQueryResult], + reference_results: Option<&[WfsQueryResult]>, + workflow_requests: &[WorkflowRequest], + reference_workflow_requests: &[WorkflowRequest], +) -> Result<(Vec, Vec)> { + // Build a lookup from variable name to reference result index. + let reference_by_variable: HashMap = reference_workflow_requests + .iter() + .enumerate() + .map(|(i, r)| (r.variable.properties().name_string(), i)) + .collect(); + + let mut rows = Vec::new(); + let mut raw_rows = Vec::new(); + for (i, analysis) in analysis_results.iter().enumerate() { + let request = &workflow_requests[i]; + let var_props = request.variable.properties(); + let model_values = outputs_from_feature_collection(&analysis.geo_json, &request.models)?; + if let Some(aggregated) = aggregate_from_list(&model_values) { + let reference = reference_results.and_then(|reference_results| { + let extracted: Result = reference_by_variable + .get(var_props.name_string().as_str()) + .map_or_else( + || Err(anyhow::anyhow!("no reference workflow for this variable")), + |&ref_idx| { + outputs_from_feature_collection( + &reference_results[ref_idx].geo_json, + &request.models, + ) + .and_then(|reference_values| { + aggregate_from_list(&reference_values) + .map(|r| r.mean) + .ok_or_else(|| anyhow::anyhow!("no reference values")) + }) + }, + ); + if let Err(error) = &extracted { + tracing::warn!( + "No anomaly for variable `{}`: {error}", + var_props.name_string() + ); + } + extracted.ok() + }); + let anomaly = + reference.map(|reference_mean| Days(aggregated.mean.0 - reference_mean.0)); + let anomaly_pct = + reference.map(|reference_mean| anomaly_pct(aggregated.mean.0, reference_mean.0)); + rows.push(ClimateRiskRow { + variable: var_props.name_string(), + scenario: request.scenario.name.to_string(), + mean: aggregated.mean, + median: aggregated.median, + min: aggregated.min, + max: aggregated.max, + occurrence_probability: aggregated.occurrence_probability, + anomaly, + occurrence_probability_label: aggregated + .occurrence_probability + .map(probability_label), + occurrence_probability_color: aggregated + .occurrence_probability + .map(probability_color), + anomaly_label: anomaly + .zip(anomaly_pct) + .map(|(days, pct)| anomaly_label(days.0, pct)), + anomaly_color: anomaly_pct.map(percentage_color), + }); + if let Some(raw_members) = aggregated.raw_members { + for (model_name, value) in raw_members { + raw_rows.push(ClimateRiskRawRow { + variable: var_props.name_string(), + scenario: request.scenario.name.to_string(), + model: model_name, + value, + }); + } + } + } + } + Ok((rows, raw_rows)) +} + +#[derive(Debug)] +pub(crate) struct ClimatePeriods { + pub analysis_start: Year, + pub range: YearRange, + pub reference_start: Year, +} + +fn log_registered_workflows(workflow_ids: &[String], workflow_requests: &[WorkflowRequest]) { + for (workflow_id, request) in workflow_ids.iter().zip(workflow_requests) { + tracing::debug!( + "ClimateRisk: registered workflow: variable={:?}, scenario={:?}, workflow_id={}", + request.variable.name(), + request.scenario.scenario.name(), + workflow_id, + ); + } +} + +fn log_wfs_results(workflow_requests: &[WorkflowRequest], results: &[WfsQueryResult]) { + for (i, result) in results.iter().enumerate() { + for (j, feature) in result.geo_json.features.iter().enumerate() { + if let Some(props) = feature.get("properties") { + let request = &workflow_requests[i]; + tracing::debug!( + "ClimateRisk: WFS result: variable={}, scenario={}, feature={}, properties={}", + request.variable.name(), + request.scenario.scenario.name(), + j, + props, + ); + } + } + } +} + +/// Registers and queries `GeoEngine` workflows for all requested variable/scenario/model +/// combinations, then aggregates the results into [`ClimateRiskOutputs`]. +/// Also returns the geoengine computation ids of all executed WFS queries for credit accounting. +#[instrument(skip(configuration), err(Debug))] +pub(crate) async fn compute_climate( + configuration: &Configuration, + coordinate: &PointType, + periods: &ClimatePeriods, + requests: &[(ClimateVariableRequest, ClimateScenarioProperties)], + models: &[ClimateModelProperties], +) -> Result<(ClimateRiskOutputs, Vec)> { + let Year(start_year) = periods.analysis_start; + let YearRange(range) = periods.range; + let Year(reference_year) = periods.reference_start; + // ~11 m half-span: small enough to hit a single grid cell, large enough to avoid + // floating-point edge cases on cell boundaries. + const POINT_BBOX_HALF_SPAN: f64 = 0.0001; + if requests.is_empty() || models.is_empty() { + return Ok((ClimateRiskOutputs::default(), Vec::new())); + } + + let end_analysis = start_year + range; + let time_str_analysis = + format!("{start_year:04}-01-01T00:00:00Z/{end_analysis:04}-01-01T00:00:00Z"); + let analysis_period = format!("{start_year:04}–{:04}", end_analysis - 1); + let end_reference = reference_year + range; + let reference_time = + format!("{reference_year:04}-01-01T00:00:00Z/{end_reference:04}-01-01T00:00:00Z"); + let reference_period = format!("{reference_year:04}–{:04}", reference_year + range - 1); + let bbox = BoundingBox::around_point(coordinate, POINT_BBOX_HALF_SPAN); + let bbox_string = bbox.wfs_string(); + + let workflow_requests = build_workflows(coordinate, requests, models, None); + let workflow_ids = register_workflows(configuration, &workflow_requests).await?; + log_registered_workflows(&workflow_ids, &workflow_requests); + + let analysis_results = query_workflows( + configuration, + &workflow_ids, + &bbox_string, + &time_str_analysis, + ) + .await?; + + // Build separate historical workflows for the reference period so the anomaly + // baseline is computed against the historical tier (1950–2014) data. + let historical_props = ClimateScenario::Historical.properties(); + let reference_workflow_requests = + build_workflows(coordinate, requests, models, Some(&historical_props)); + let reference_workflow_ids = + register_workflows(configuration, &reference_workflow_requests).await?; + let reference_results = query_workflows( + configuration, + &reference_workflow_ids, + &bbox_string, + &reference_time, + ) + .await?; + log_wfs_results(&reference_workflow_requests, &reference_results); + + // Every WFS query is a geoengine computation that must be reported for credit accounting. + let computation_ids: Vec = analysis_results + .iter() + .chain(reference_results.iter()) + .filter_map(|result| result.computation_id) + .collect(); + + let (rows, raw_rows) = aggregate_rows( + &analysis_results, + Some(reference_results.as_slice()), + &workflow_requests, + &reference_workflow_requests, + )?; + + let climate_risk = Some(climate_risk_data_resource( + rows, + &analysis_period, + Some(reference_period.as_str()), + )); + Ok(( + ClimateRiskOutputs { + analysis_period: Some(analysis_period), + reference_period: Some(reference_period), + climate_risk, + raw_ensemble_data: if raw_rows.is_empty() { + None + } else { + Some(raw_ensemble_data_resource(raw_rows)) + }, + inputs: None, + }, + computation_ids, + )) +} + +/// One WFS query result together with the geoengine computation id reported for it, +/// which is needed to account credit usage. +struct WfsQueryResult { + geo_json: GeoJson, + computation_id: Option, +} + +async fn wfs_query( + configuration: &Configuration, + workflow_id: &str, + bbox: &str, + time: &str, +) -> Result> { + let mut computation_id = String::new(); + let geo_json = wfs_handler( + configuration, + workflow_id, + WfsRequest::GetFeature, + Some(bbox), + None, + None, + None, + None, + None, + Some(WfsService::Wfs), + None, + Some("EPSG:4326"), + Some(time), + Some(workflow_id), + None, + Some(&mut computation_id), + ) + .await?; + // Older geoengine instances may not report a computation id; skip credit tracking then. + let computation_id = ComputationId::from_str(&computation_id).ok(); + Ok(WfsQueryResult { + geo_json, + computation_id, + }) +} + +/// Builds a single-point vector source for the given coordinate. +pub(crate) fn vector_source(coordinate: &PointType) -> VectorOperator { + VectorOperator::MockPointSource( + MockPointSource { + r#type: Default::default(), + params: MockPointSourceParameters { + points: vec![Coordinate2D::new(coordinate[0], coordinate[1])], + spatial_bounds: SpatialBoundsDerive::None( + SpatialBoundsDeriveNone { + r#type: Default::default(), + } + .into(), + ) + .into(), + } + .into(), + } + .into(), + ) +} + +/// Computes min, max, mean, and median across model values for a single variable/scenario. +#[allow( + clippy::cast_precision_loss, + reason = "Averaging over at most a few dozen model values cannot lose relevant precision." +)] +pub(crate) fn aggregate_from_list( + model_values: &HashMap, +) -> Option { + if model_values.is_empty() { + return None; + } + + let values: Vec = model_values.values().copied().collect(); + let min = values.iter().copied().reduce(f64::min).unwrap_or(0.0); + let max = values.iter().copied().reduce(f64::max).unwrap_or(0.0); + let mean = values.iter().sum::() / values.len() as f64; + + let mut sorted = values.clone(); + sorted.sort_by(f64::total_cmp); + let mid = sorted.len() / 2; + let median = if sorted.len().is_multiple_of(2) { + f64::midpoint(sorted[mid - 1], sorted[mid]) + } else { + sorted[mid] + }; + + let raw_members = model_values.iter().map(|(k, v)| (k.clone(), *v)).collect(); + + Some(ClimateVariableResult { + max: Days(max), + min: Days(min), + mean: Days(mean), + median: Days(median), + occurrence_probability: Some(mean / DAYS_PER_JULIAN_YEAR), + raw_members: Some(raw_members), + }) +} + +const NO_MODEL_COVERAGE_ERROR: &str = + "Input coordinate not covered by any of the requested climate models for the given time range."; + +/// Extracts per-model values from a WFS feature collection response. +#[allow( + clippy::cast_precision_loss, + reason = "Climate indicator values are small; i64→f64 and count→f64 cannot lose relevant precision." +)] +pub(crate) fn outputs_from_feature_collection( + feature_collection: &GeoJson, + variables: &[ClimateModelProperties], +) -> Result> { + if feature_collection.features.is_empty() { + anyhow::bail!(NO_MODEL_COVERAGE_ERROR); + } + + // One feature per time step (year); average each model's per-year values to get the + // multi-year mean. + let mut acc: HashMap> = HashMap::new(); + let mut models_without_data = Vec::new(); + let mut models_with_invalid_data = Vec::new(); + + for feature in &feature_collection.features { + let Some(properties) = feature.get("properties") else { + continue; + }; + + for model in variables { + let model_name = &model.id; + if let Some(value) = properties.get(model_name.as_str()) { + if let Some(value) = value.as_f64().or_else(|| value.as_i64().map(|v| v as f64)) { + acc.entry(model_name.clone()).or_default().push(value); + } else if !models_with_invalid_data.contains(model_name) { + models_with_invalid_data.push(model_name.clone()); + } + } else if !models_without_data.contains(model_name) { + models_without_data.push(model_name.clone()); + } + } + } + + // Log once per model instead of once per feature × model. + for model_name in models_without_data { + tracing::warn!("No data found for model {model_name} in feature properties."); + } + for model_name in models_with_invalid_data { + tracing::warn!("Invalid data type for model {model_name} in feature properties."); + } + + if acc.is_empty() { + anyhow::bail!(NO_MODEL_COVERAGE_ERROR); + } + + Ok(acc + .into_iter() + .map(|(model, values)| { + let mean = values.iter().sum::() / values.len() as f64; + (model, mean) + }) + .collect()) +} + +#[cfg(test)] +mod tests { + use super::*; + use approx::assert_abs_diff_eq; + use geoengine_api_client::models::CollectionType; + use ogcapi::types::processes::InlineOrRefData; + + #[test] + fn it_groups_result_rows_by_declared_scenario() { + fn row(scenario: &str) -> ClimateRiskRow { + ClimateRiskRow { + variable: "Heat Days".to_string(), + scenario: scenario.to_string(), + mean: Days(1.0), + median: Days(1.0), + min: Days(1.0), + max: Days(1.0), + occurrence_probability: Some(1.0), + anomaly: None, + ..Default::default() + } + } + let outputs = ClimateRiskOutputs { + inputs: None, + analysis_period: Some("2041–2070".to_string()), + reference_period: Some("1981–2010".to_string()), + climate_risk: Some(climate_risk_data_resource( + vec![ + row("SSP2-4.5 (Intermediate emissions)"), + row("SSP5-8.5 (High emissions)"), + row("SSP2-4.5 (Intermediate emissions)"), + ], + "2041–2070", + Some("1981–2010"), + )), + raw_ensemble_data: None, + }; + + let result: ExecuteResults = outputs.try_into().unwrap(); + // Keys must be the output ids declared in the process description, not the + // display names the rows carry. + assert!(result.contains_key("ssp245")); + assert!(result.contains_key("ssp585")); + assert!(!result.contains_key("historical")); + assert!(!result.contains_key("SSP2-4.5 (Intermediate emissions)")); + + let InlineOrRefData::QualifiedInputValue(qualified) = &result["ssp245"].data else { + panic!("expected qualified input value"); + }; + let resource: DataResource> = + serde_json::from_value(serde_json::to_value(&qualified.value).unwrap()).unwrap(); + assert_eq!(resource.data.len(), 2); + assert_eq!(resource.name.as_str(), "climate-risk-ssp245"); + assert_eq!( + resource.title.as_deref(), + Some("SSP2-4.5 (Intermediate emissions) · 2041–2070") + ); + assert!( + resource + .data + .iter() + .all(|r| r.scenario == "SSP2-4.5 (Intermediate emissions)") + ); + } + + #[test] + fn it_scenario_output_id_maps_display_name_to_declared_output_id() { + assert_eq!( + scenario_output_id("SSP2-4.5 (Intermediate emissions)"), + "ssp245" + ); + assert_eq!(scenario_output_id("SSP5-8.5 (High emissions)"), "ssp585"); + // Unknown names pass through, so rows already carrying output ids keep working. + assert_eq!(scenario_output_id("ssp245"), "ssp245"); + } + + #[test] + fn it_aggregate_from_list_aggregates_values() { + let mut values: HashMap = HashMap::new(); + values.insert("MPI-ESM1-2-LR".to_string(), 10.0); + values.insert("ACCESS-CM2".to_string(), 20.0); + + let result = aggregate_from_list(&values).unwrap(); + assert_abs_diff_eq!(result.min.0, 10.0); + assert_abs_diff_eq!(result.max.0, 20.0); + assert_abs_diff_eq!(result.mean.0, 15.0); + assert_abs_diff_eq!(result.median.0, 15.0); + assert_eq!(result.raw_members.as_ref().unwrap().len(), 2); + assert_abs_diff_eq!(result.occurrence_probability.unwrap(), 15.0 / 365.25); + + let empty: HashMap = HashMap::new(); + assert!(aggregate_from_list(&empty).is_none()); + } + + #[test] + fn it_climate_risk_data_resource_declares_display_extension() { + let rows = vec![ClimateRiskRow { + variable: "Heat Days".to_string(), + scenario: "ssp245".to_string(), + max: Days(100.0), + min: Days(0.0), + mean: Days(50.0), + median: Days(50.0), + occurrence_probability: Some(0.5), + anomaly: Some(Days(10.0)), + ..Default::default() + }]; + let resource = climate_risk_data_resource(rows, "", None); + let probability_field = resource + .schema + .fields + .iter() + .find(|f| f.name == "occurrenceProbability") + .unwrap(); + assert!(matches!( + probability_field.r#type, + Some(TableSchemaType::Number) + )); + assert_eq!( + resource.schema.schema.as_deref(), + Some(CLIMATE_RISK_TABLE_SCHEMA_PROFILE) + ); + assert!(matches!( + resource.schema.biois.as_ref().unwrap().display["occurrenceProbability"].kind, + BioisDisplayKind::RiskProbability + )); + let probability_metadata = + &resource.schema.biois.as_ref().unwrap().display["occurrenceProbability"]; + assert_eq!( + probability_metadata.label_field.as_deref(), + Some("occurrenceProbabilityLabel") + ); + assert_eq!( + probability_metadata.color_field.as_deref(), + Some("occurrenceProbabilityColor") + ); + } + + #[test] + fn it_climate_risk_data_resource_declares_anomaly_display() { + let rows = vec![ + ClimateRiskRow { + variable: "Heat Days".to_string(), + scenario: "ssp245".to_string(), + max: Days(100.0), + min: Days(0.0), + mean: Days(50.0), + median: Days(50.0), + occurrence_probability: Some(0.5), + anomaly: Some(Days(10.0)), + ..Default::default() + }, + ClimateRiskRow { + variable: "Dry Days".to_string(), + scenario: "ssp245".to_string(), + max: Days(100.0), + min: Days(0.0), + mean: Days(50.0), + median: Days(50.0), + occurrence_probability: Some(0.5), + anomaly: Some(Days(5.0)), + ..Default::default() + }, + ]; + let resource = climate_risk_data_resource(rows, "", None); + + let field_names: std::collections::HashSet<&str> = resource + .schema + .fields + .iter() + .map(|field| field.name.as_str()) + .collect(); + let extension = resource.schema.biois.as_ref().unwrap(); + for metadata in extension.display.values() { + assert!(field_names.contains(metadata.label_field.as_deref().unwrap())); + assert!(field_names.contains(metadata.color_field.as_deref().unwrap())); + } + assert!( + extension + .hidden_fields + .iter() + .all(|field| field_names.contains(field.as_str())) + ); + + assert!(matches!( + resource.schema.biois.as_ref().unwrap().display["anomaly"].kind, + BioisDisplayKind::RiskAnomaly + )); + let anomaly_metadata = &resource.schema.biois.as_ref().unwrap().display["anomaly"]; + assert_eq!( + anomaly_metadata.label_field.as_deref(), + Some("anomalyLabel") + ); + assert_eq!( + anomaly_metadata.color_field.as_deref(), + Some("anomalyColor") + ); + } + + #[test] + fn it_climate_risk_data_resource_omits_anomaly_field_when_absent() { + let rows = vec![ClimateRiskRow { + variable: "Heat Days".to_string(), + scenario: "ssp245".to_string(), + max: Days(100.0), + min: Days(0.0), + mean: Days(50.0), + median: Days(50.0), + occurrence_probability: Some(0.5), + anomaly: None, + ..Default::default() + }]; + let resource = climate_risk_data_resource(rows, "", None); + let field_names: Vec<&str> = resource + .schema + .fields + .iter() + .map(|f| f.name.as_str()) + .collect(); + assert!(!field_names.contains(&"anomaly")); + assert!(field_names.contains(&"occurrenceProbability")); + assert!(field_names.contains(&"occurrenceProbabilityLabel")); + assert!(field_names.contains(&"occurrenceProbabilityColor")); + assert!( + !resource + .schema + .biois + .as_ref() + .unwrap() + .display + .contains_key("anomaly") + ); + } + + #[test] + fn it_climate_risk_scenario_data_resource_annotates_periods() { + let rows = vec![ClimateRiskRow { + variable: "Heat Days".to_string(), + scenario: "SSP2-4.5 (Intermediate emissions)".to_string(), + max: Days(100.0), + min: Days(0.0), + mean: Days(50.0), + median: Days(50.0), + occurrence_probability: Some(0.5), + anomaly: Some(Days(10.0)), + ..Default::default() + }]; + let resource = climate_risk_scenario_data_resource( + "SSP2-4.5 (Intermediate emissions)", + rows, + "2041–2070", + Some("1981–2010"), + ); + + assert_eq!(resource.name.as_str(), "climate-risk-ssp245"); + assert_eq!( + resource.title.as_deref(), + Some("SSP2-4.5 (Intermediate emissions) · 2041–2070") + ); + let title = |name: &str| { + resource + .schema + .fields + .iter() + .find(|f| f.name == name) + .unwrap() + .title + .clone() + }; + assert_eq!(title("mean").as_deref(), Some("Mean (days/year)")); + assert_eq!(title("median").as_deref(), Some("Median (days/year)")); + assert_eq!( + title("anomaly").as_deref(), + Some("Anomaly (days/year compared to 1981–2010)") + ); + + let resource = climate_risk_scenario_data_resource( + "SSP2-4.5 (Intermediate emissions)", + vec![ClimateRiskRow { + variable: "Heat Days".to_string(), + scenario: "SSP2-4.5 (Intermediate emissions)".to_string(), + max: Days(100.0), + min: Days(0.0), + mean: Days(50.0), + median: Days(50.0), + occurrence_probability: Some(0.5), + anomaly: None, + ..Default::default() + }], + "", + None, + ); + assert_eq!(resource.name.as_str(), "climate-risk-ssp245"); + assert_eq!( + resource.title.as_deref(), + Some("SSP2-4.5 (Intermediate emissions)") + ); + assert!(resource.schema.fields.iter().all(|f| f.name != "anomaly")); + } + + #[test] + fn it_converts_outputs_into_execute_results() { + let rows = vec![ClimateRiskRow { + variable: "Heat Days".to_string(), + scenario: "ssp245".to_string(), + max: Days(100.0), + min: Days(0.0), + mean: Days(50.0), + median: Days(50.0), + occurrence_probability: None, + anomaly: Some(Days(10.0)), + ..Default::default() + }]; + let raw_rows = vec![ClimateRiskRawRow { + variable: "Heat Days".to_string(), + scenario: "ssp245".to_string(), + model: "ACCESS-CM2".to_string(), + value: 42.0, + }]; + let outputs = ClimateRiskOutputs { + inputs: None, + analysis_period: None, + reference_period: None, + climate_risk: Some(climate_risk_data_resource(rows, "", None)), + raw_ensemble_data: Some(raw_ensemble_data_resource(raw_rows)), + }; + let results: ExecuteResults = outputs.try_into().unwrap(); + assert!(results.contains_key("ssp245")); + assert!(!results.contains_key("ssp585")); + assert!(results.contains_key("rawEnsembleData")); + } + + #[test] + fn it_averages_per_model_values_from_feature_collection() { + let models = vec![ + ClimateModelProperties { + id: "MPI-ESM1-2-LR".to_string(), + variant: "r1i1p1f1".to_string(), + grid: "gn".to_string(), + scenarios: vec![ClimateScenario::Historical, ClimateScenario::Ssp245], + }, + ClimateModelProperties { + id: "ACCESS-CM2".to_string(), + variant: "r1i1p1f1".to_string(), + grid: "gn".to_string(), + scenarios: vec![ClimateScenario::Historical, ClimateScenario::Ssp245], + }, + ]; + + let geo_json = GeoJson { + features: vec![ + serde_json::json!({"type": "Feature", "properties": {"MPI-ESM1-2-LR": 42.0, "ACCESS-CM2": 10.0}}), + serde_json::json!({"type": "Feature", "properties": {"MPI-ESM1-2-LR": 58.0, "ACCESS-CM2": 30.0}}), + ], + r#type: CollectionType::FeatureCollection, + }; + let result = outputs_from_feature_collection(&geo_json, &models).unwrap(); + assert_eq!(result.len(), 2); + assert_abs_diff_eq!(result["MPI-ESM1-2-LR"], 50.0); + assert_abs_diff_eq!(result["ACCESS-CM2"], 20.0); + + assert!(outputs_from_feature_collection(&GeoJson::default(), &models).is_err()); + + let geo_json = GeoJson { + features: vec![serde_json::json!({"type": "Feature"})], + r#type: CollectionType::FeatureCollection, + }; + assert!(outputs_from_feature_collection(&geo_json, &models).is_err()); + } + + #[test] + fn it_vector_source_creates_mock_point_source() { + let point = PointType::from(vec![12.0, 34.0]); + let result = vector_source(&point); + assert!(matches!(result, VectorOperator::MockPointSource(_))); + } + + #[test] + fn it_sorts_raw_ensemble_rows_by_variable_scenario_model() { + let rows = vec![ + ClimateRiskRawRow { + variable: "Ice Days".to_string(), + scenario: "ssp245".to_string(), + model: "ACCESS-CM2".to_string(), + value: 1.0, + }, + ClimateRiskRawRow { + variable: "Heat Days".to_string(), + scenario: "ssp585".to_string(), + model: "MPI-ESM1-2-LR".to_string(), + value: 2.0, + }, + ClimateRiskRawRow { + variable: "Heat Days".to_string(), + scenario: "historical".to_string(), + model: "ACCESS-CM2".to_string(), + value: 3.0, + }, + ]; + + let resource = raw_ensemble_data_resource(rows); + + let keys: Vec<_> = resource + .data + .iter() + .map(|r| (r.variable.as_str(), r.scenario.as_str(), r.model.as_str())) + .collect(); + assert_eq!( + keys, + vec![ + ("Heat Days", "historical", "ACCESS-CM2"), + ("Heat Days", "ssp585", "MPI-ESM1-2-LR"), + ("Ice Days", "ssp245", "ACCESS-CM2"), + ] + ); + } + + #[test] + fn it_embeds_serialized_inputs_in_execute_results() { + let payload = serde_json::json!({ + "coordinate": { + "value": {"type": "Point", "coordinates": [12.34, 56.78]}, + "mediaType": "application/geo+json" + }, + "yearBegin": 2014, + "yearRange": 20, + "referenceYearBegin": 2020, + }); + let inputs: ClimateRiskInputs = serde_json::from_value(payload).unwrap(); + let outputs = ClimateRiskOutputs { + inputs: Some(inputs), + analysis_period: None, + reference_period: None, + climate_risk: None, + raw_ensemble_data: None, + }; + + let results: ExecuteResults = outputs.try_into().unwrap(); + + assert!(results.contains_key("inputs")); + let InlineOrRefData::QualifiedInputValue(qualified) = &results["inputs"].data else { + panic!("expected qualified input value"); + }; + let value = + serde_json::to_value(&qualified.value).expect("inputs value must be serializable"); + assert_eq!(value["yearBegin"], 2014); + assert_eq!(value["coordinate"]["mediaType"], "application/geo+json"); + } + + #[test] + fn it_filters_models_per_scenario_when_building_workflows() { + let point = PointType::from(vec![8.0, 50.0]); + // Only supports SSP245, so SSP585 requests must be filtered out. + let models = vec![ClimateModelProperties { + id: "LIMITED-MODEL".to_string(), + variant: "r1i1p1f1".to_string(), + grid: "gn".to_string(), + scenarios: vec![ClimateScenario::Historical, ClimateScenario::Ssp245], + }]; + let requests = vec![ + ( + ClimateVariableRequest::new(ClimateVariable::HeatDays), + ClimateScenario::Ssp245.properties(), + ), + ( + ClimateVariableRequest::new(ClimateVariable::HeatDays), + ClimateScenario::Ssp585.properties(), + ), + ( + ClimateVariableRequest::new(ClimateVariable::IceDays), + ClimateScenario::Ssp245.properties(), + ), + ]; + + let workflows = build_workflows(&point, &requests, &models, None); + + assert_eq!(workflows.len(), 2); + assert!( + workflows + .iter() + .all(|w| w.scenario.scenario == ClimateScenario::Ssp245) + ); + assert_eq!(workflows[0].variable, ClimateVariable::HeatDays); + assert_eq!(workflows[1].variable, ClimateVariable::IceDays); + assert_eq!(workflows[0].models.len(), 1); + assert_eq!(workflows[0].models[0].id, "LIMITED-MODEL"); + } + + #[test] + fn it_preserves_job_order_and_propagates_errors() { + let jobs: Vec<_> = (0..20) + .map(|i| move || async move { Ok::<_, ()>(i * 2) }) + .collect(); + let results = futures::executor::block_on(run_limited(jobs)).unwrap(); + let expected: Vec<_> = (0..20).map(|i| i * 2).collect(); + assert_eq!(results, expected); + + let failing: Vec<_> = (0..3) + .map(|i| move || async move { if i == 1 { Err("boom") } else { Ok(i) } }) + .collect(); + let error = futures::executor::block_on(run_limited(failing)).unwrap_err(); + assert_eq!(error, "boom"); + } + + #[test] + fn it_formats_join_errors_with_or_without_response_body() { + use geoengine_api_client::apis::{Error as ApiError, ResponseContent}; + + // `reqwest::StatusCode` is a re-export of the shared `http` type, which axum + // also re-exports — no extra dev-dependency needed to build the error. + let with_response: ApiError<()> = ApiError::ResponseError(ResponseContent { + status: axum::http::StatusCode::BAD_REQUEST, + content: r#"{"error":"bad","message":"nope"}"#.to_string(), + entity: None, + }); + let formatted = unpack_join_error(&with_response, "register a workflow").to_string(); + assert!(formatted.contains("register a workflow"), "{formatted}"); + assert!(formatted.contains("bad"), "{formatted}"); + + let without_response: ApiError<()> = + ApiError::Serde(serde_json::from_str::("oops").unwrap_err()); + let formatted = unpack_join_error(&without_response, "execute a workflow").to_string(); + assert!(formatted.contains("execute a workflow"), "{formatted}"); + } + + fn workflow_request(variable: ClimateVariable, scenario: ClimateScenario) -> WorkflowRequest { + WorkflowRequest { + models: vec![ClimateModelProperties { + id: "ACCESS-CM2".to_string(), + variant: "r1i1p1f1".to_string(), + grid: "gn".to_string(), + scenarios: vec![ + ClimateScenario::Historical, + ClimateScenario::Ssp245, + ClimateScenario::Ssp585, + ], + }], + variable, + scenario: scenario.properties(), + workflow: geoengine_api_client::models::Workflow::default(), + } + } + + fn single_feature_collection(properties: &serde_json::Value) -> GeoJson { + GeoJson { + features: vec![serde_json::json!({"type": "Feature", "properties": properties})], + r#type: CollectionType::FeatureCollection, + } + } + + #[test] + fn it_derives_anomalies_and_raw_members_from_reference_values() { + let request = workflow_request(ClimateVariable::HeatDays, ClimateScenario::Ssp245); + let analysis = WfsQueryResult { + geo_json: single_feature_collection(&serde_json::json!({"ACCESS-CM2": 20.0})), + computation_id: None, + }; + let reference = WfsQueryResult { + geo_json: single_feature_collection(&serde_json::json!({"ACCESS-CM2": 12.5})), + computation_id: None, + }; + + let (rows, raw_rows) = aggregate_rows( + std::slice::from_ref(&analysis), + Some(&[reference]), + &[request], + &[workflow_request( + ClimateVariable::HeatDays, + ClimateScenario::Historical, + )], + ) + .unwrap(); + + assert_eq!(rows.len(), 1); + let row = &rows[0]; + assert_abs_diff_eq!(row.anomaly.unwrap().0, 7.5); + assert!(row.anomaly_label.is_some()); + assert!(row.anomaly_color.is_some()); + assert_abs_diff_eq!(row.mean.0, 20.0); + // A single model yields an odd member count; the median is the middle value. + assert_abs_diff_eq!(row.median.0, 20.0); + assert_eq!(raw_rows.len(), 1); + assert_eq!(raw_rows[0].model, "ACCESS-CM2"); + assert_abs_diff_eq!(raw_rows[0].value, 20.0); + } + + #[test] + fn it_omits_the_anomaly_without_reference_values() { + let request = workflow_request(ClimateVariable::HeatDays, ClimateScenario::Ssp245); + let analysis = WfsQueryResult { + geo_json: single_feature_collection(&serde_json::json!({"ACCESS-CM2": 20.0})), + computation_id: None, + }; + // An empty feature collection cannot yield reference values. + let broken_reference = WfsQueryResult { + geo_json: GeoJson::default(), + computation_id: None, + }; + + let (rows, _) = aggregate_rows( + std::slice::from_ref(&analysis), + Some(&[broken_reference]), + &[request], + &[workflow_request( + ClimateVariable::HeatDays, + ClimateScenario::Historical, + )], + ) + .unwrap(); + + assert_eq!(rows.len(), 1); + assert!(rows[0].anomaly.is_none()); + assert!(rows[0].anomaly_label.is_none()); + assert!(rows[0].anomaly_color.is_none()); + } + + #[test] + fn it_fails_without_analysis_values() { + let request = workflow_request(ClimateVariable::HeatDays, ClimateScenario::Ssp245); + let empty = WfsQueryResult { + geo_json: GeoJson::default(), + computation_id: None, + }; + + let result = aggregate_rows(&[empty], None, &[request], &[]); + + assert!(result.is_err()); + } + + #[test] + fn it_logs_workflow_requests_and_results() { + let request = workflow_request(ClimateVariable::HeatDays, ClimateScenario::Ssp245); + log_registered_workflows(&["wf-id".to_string()], std::slice::from_ref(&request)); + + let result = WfsQueryResult { + geo_json: single_feature_collection(&serde_json::json!({"ACCESS-CM2": 1.0})), + computation_id: None, + }; + log_wfs_results(std::slice::from_ref(&request), &[result]); + } + + #[test] + fn it_skips_missing_or_invalid_model_columns() { + let models = vec![ + ClimateModelProperties { + id: "MPI-ESM1-2-LR".to_string(), + variant: "r1i1p1f1".to_string(), + grid: "gn".to_string(), + scenarios: vec![ClimateScenario::Historical, ClimateScenario::Ssp245], + }, + ClimateModelProperties { + id: "ACCESS-CM2".to_string(), + variant: "r1i1p1f1".to_string(), + grid: "gn".to_string(), + scenarios: vec![ClimateScenario::Historical, ClimateScenario::Ssp245], + }, + ]; + let geo_json = GeoJson { + features: vec![ + serde_json::json!({"type": "Feature", "properties": {"MPI-ESM1-2-LR": "not-a-number", "ACCESS-CM2": 4.0}}), + serde_json::json!({"type": "Feature", "properties": {"MPI-ESM1-2-LR": 8.0}}), + ], + r#type: CollectionType::FeatureCollection, + }; + + let result = outputs_from_feature_collection(&geo_json, &models).unwrap(); + + assert_abs_diff_eq!(result["MPI-ESM1-2-LR"], 8.0); + assert_abs_diff_eq!(result["ACCESS-CM2"], 4.0); + } +} diff --git a/backend/src/processes/climate_risk/mod.rs b/backend/src/processes/climate_risk/mod.rs new file mode 100644 index 0000000..4b37425 --- /dev/null +++ b/backend/src/processes/climate_risk/mod.rs @@ -0,0 +1,867 @@ +use crate::{ + config::CONFIG, + credits::add_credits_used_pending, + db::DbHandle, + processes::parameters::{DataResource, PointGeoJsonInput, Year, YearRange}, + state::{CONTEXT, TaskLocalContext}, +}; +use anyhow::{Context, Result}; +use ogcapi::{ + processes::Processor, + types::{ + common::Link, + processes::{ + Execute, JobControlOptions, Process, ProcessSummary, TransmissionMode, + description::{DescriptionType, InputDescription, OutputDescription}, + }, + }, +}; +use schemars::generate::SchemaSettings; +use std::collections::HashMap; +use std::sync::LazyLock; + +mod compute; +mod types; +mod workflow; + +pub use types::{ClimateRiskInputs, ClimateRiskOutputs, ClimateRiskRawRow, ClimateRiskRow}; + +use self::{compute::*, types::*}; + +// ponytail: model registry loaded once at startup from TOML; add models by editing conf/nexgddp_models.toml, no recompile +static MODEL_REGISTRY: LazyLock> = LazyLock::new(|| { + let path = &CONFIG.nexgddp_cmip6.model_registry_path; + let content = std::fs::read_to_string(path).unwrap_or_else(|_| { + tracing::warn!("Could not read model registry from {path}, using empty registry"); + String::new() + }); + let registry: ModelRegistryToml = toml::from_str(&content).unwrap_or_else(|e| { + tracing::warn!("Could not parse model registry: {e}"); + ModelRegistryToml { model: Vec::new() } + }); + let mut entries: Vec<(String, ClimateModelProperties)> = registry + .model + .into_iter() + .map(|entry| { + let scenarios: Vec = entry + .scenarios + .into_iter() + .filter_map(|s| match s.as_str() { + "historical" => Some(ClimateScenario::Historical), + "ssp245" => Some(ClimateScenario::Ssp245), + "ssp585" => Some(ClimateScenario::Ssp585), + _ => None, + }) + .collect(); + ( + entry.id.clone(), + ClimateModelProperties { + id: entry.id, + variant: entry.variant, + grid: entry.grid, + scenarios, + }, + ) + }) + .collect(); + // ponytail: sort by model id for deterministic iteration order + entries.sort_by(|a, b| a.0.cmp(&b.0)); + entries.into_iter().collect() +}); + +#[derive(serde::Deserialize)] +struct ModelRegistryToml { + model: Vec, +} + +#[derive(serde::Deserialize)] +struct ModelRegistryEntry { + id: String, + variant: String, + grid: String, + scenarios: Vec, +} + +/// Calculates climate-risk indicators for a given point and time window. +#[derive(Debug, Clone)] +pub struct ClimateRiskProcess { + db: DbHandle, +} + +impl ClimateRiskProcess { + pub const ID: &'static str = "climate-risk"; + + #[must_use] + pub fn new(db: DbHandle) -> Self { + Self { db } + } +} + +/// Restricts the `models` input to the registered model ids. +/// +/// The registry is loaded at runtime, so the enum cannot come from the `Vec` type itself. +/// This is a schema transform rather than an inline mutation, cf. `optional_feature_properties` +/// and `relative_json_pointer_format`. +fn registered_models(schema: &mut schemars::Schema) { + let Some(items) = schema + .get_mut("items") + .and_then(serde_json::Value::as_object_mut) + else { + return; + }; + items.insert( + "enum".to_string(), + serde_json::Value::Array( + MODEL_REGISTRY + .keys() + .map(|id| serde_json::Value::String(id.clone())) + .collect(), + ), + ); +} + +/// Generate the JSON Schema for the `models` input, enumerating the registered model ids so the +/// frontend can render a dropdown instead of a free-text list. +fn models_schema(generator: &mut schemars::SchemaGenerator) -> serde_json::Value { + let mut root = generator.root_schema_for::>(); + registered_models(&mut root); + root.to_value() +} + +fn build_inputs(generator: &mut schemars::SchemaGenerator) -> HashMap { + let mut year_begin_schema = generator.root_schema_for::().to_value(); + year_begin_schema["minimum"] = serde_json::json!(FUTURE_START_YEAR); + year_begin_schema["maximum"] = serde_json::json!(FUTURE_END_YEAR); + year_begin_schema["default"] = serde_json::json!(default_year_begin().0); + + let mut reference_year_begin_schema = generator.root_schema_for::().to_value(); + reference_year_begin_schema["minimum"] = serde_json::json!(HISTORICAL_START_YEAR); + reference_year_begin_schema["maximum"] = serde_json::json!(HISTORICAL_END_YEAR); + reference_year_begin_schema["default"] = serde_json::json!(default_reference_year_begin().0); + + HashMap::from([ + ( + "coordinate".to_string(), + InputDescription { + description_type: DescriptionType { + title: Some("Coordinate in WGS84".to_string()), + description: Some("This is a POINT input in WGS84 (EPSG:4326) format.".to_string()), + ..Default::default() + }, + schema: generator.root_schema_for::().to_value(), + ..Default::default() + }, + ), + ( + "yearBegin".to_string(), + InputDescription { + description_type: DescriptionType { + title: Some("Start year".to_string()), + description: Some("The first year to include in the climate-risk aggregation.".to_string()), + ..Default::default() + }, + schema: year_begin_schema, + ..Default::default() + }, + ), + ( + "yearRange".to_string(), + InputDescription { + description_type: DescriptionType { + title: Some("Range (years)".to_string()), + description: Some( + "Length of the climate-risk aggregation window in years (5-30).".to_string(), + ), + ..Default::default() + }, + schema: generator.root_schema_for::().to_value(), + ..Default::default() + }, + ), + ( + "referenceYearBegin".to_string(), + InputDescription { + description_type: DescriptionType { + title: Some("Reference period start".to_string()), + description: Some( + "First year of the reference period used to compute anomalies. Uses the same range as the analysis window.".to_string(), + ), + ..Default::default() + }, + schema: reference_year_begin_schema, + ..Default::default() + }, + ), + ( + "variables".to_string(), + InputDescription { + description_type: DescriptionType { + title: Some("Climate variables".to_string()), + description: Some( + "The climate indicators to derive from the source dataset. If empty, all available indicators are computed.".to_string(), + ), + ..Default::default() + }, + schema: generator.root_schema_for::>().to_value(), + min_occurs: Some(0), + ..Default::default() + }, + ), + ( + "models".to_string(), + InputDescription { + description_type: DescriptionType { + title: Some("Climate models".to_string()), + description: Some("The climate-model workflows to execute for each requested variable. If empty, all available models are used.".to_string()), + ..Default::default() + }, + schema: models_schema(generator), + min_occurs: Some(0), + ..Default::default() + }, + ), + ]) +} + +fn build_outputs(generator: &mut schemars::SchemaGenerator) -> HashMap { + // The registry is the single source of truth for which scenarios can be produced: advertising a + // scenario no registered model supports would leave the client sending an output that is then + // dropped (see `resolve_requests`), i.e. a control that silently does nothing. + let registry: Vec = MODEL_REGISTRY.values().cloned().collect(); + let available_scenarios = resolve_available_scenarios(®istry); + let mut outputs = HashMap::from([ + ( + "inputs".to_string(), + OutputDescription { + description_type: DescriptionType { + title: Some("Input parameters".to_string()), + description: Some( + "The inputs used to compute the climate-risk summary.".to_string(), + ), + ..Default::default() + }, + schema: generator.root_schema_for::().to_value(), + }, + ), + ( + "rawEnsembleData".to_string(), + OutputDescription { + description_type: DescriptionType { + title: Some("Raw ensemble data".to_string()), + description: Some( + "Per-model raw values for each variable × scenario combination." + .to_string(), + ), + ..Default::default() + }, + schema: generator + .root_schema_for::>>() + .to_value(), + }, + ), + ]); + + for props in available_scenarios { + outputs.insert( + props.scenario.name().to_string(), + OutputDescription { + description_type: DescriptionType { + title: Some(props.name.to_string()), + description: Some(format!( + "A table of climate-risk indicators for the {} scenario.", + props.name + )), + ..Default::default() + }, + schema: generator + .root_schema_for::>>() + .to_value(), + }, + ); + } + + outputs +} + +#[async_trait::async_trait] +impl Processor for ClimateRiskProcess { + type Input = ClimateRiskProcessParams; + type Output = ClimateRiskOutputs; + + fn id(&self) -> &'static str { + Self::ID + } + + fn version(&self) -> &'static str { + "0.2.0" + } + + async fn process(&self) -> Result { + let mut settings = SchemaSettings::default(); + settings.meta_schema = None; + let mut generator = settings.into_generator(); + + let inputs = build_inputs(&mut generator); + let outputs = build_outputs(&mut generator); + + Ok(Process { + summary: ProcessSummary { + id: self.id().into(), + version: self.version().into(), + description: DescriptionType { + title: Some("Climate risk indicators".to_string()), + description: Some( + "This process derives climate-risk indicators such as heat days from NEX-GDDP-CMIP6 climate data for a point location and a time window. The workflow builds a daily threshold mask, aggregates it over the requested years and returns summary statistics for the selected climate variable. An anomaly relative to a historical reference period (same length) is reported as the difference of the multi-year means. If no models are specified, all available models are used. If no scenario outputs are requested, all future scenarios are computed." + .to_string(), + ), + ..Default::default() + }, + job_control_options: vec![ + JobControlOptions::SyncExecute, + JobControlOptions::AsyncExecute, + ], + output_transmission: vec![TransmissionMode::Value], + links: vec![Link::new( + format!("./{}/execution", self.id()), + "http://www.opengis.net/def/rel/ogc/1.0/execute", + ) + .title("Execution endpoint")], + }, + inputs, + outputs, + }) + } + + async fn parse(&self, execute: Execute) -> Result { + Ok(ClimateRiskProcessParams { + inputs: parse_inputs(&execute.inputs)?, + requested_outputs: execute.outputs, + }) + } + + async fn execute(&self, input: Self::Input) -> Result { + let mut inputs = input.inputs; + let requested_outputs = input.requested_outputs; + + validate_inputs( + inputs.year_begin, + inputs.year_range, + inputs.reference_year_begin, + )?; + + let point = inputs.coordinate.value.coordinates.clone(); + + let (filtered_models, model_props, dropped_models) = resolve_models(&inputs.models); + if !dropped_models.is_empty() { + tracing::warn!( + "Ignoring unknown climate models: {}", + dropped_models.join(", ") + ); + } + if model_props.is_empty() { + let detail = if dropped_models.is_empty() { + String::new() + } else { + format!( + "; none of the requested models ({}) are available", + dropped_models.join(", ") + ) + }; + anyhow::bail!("No climate models valid / available{detail}"); + } + inputs.models = filtered_models; + + let scenario_props = resolve_available_scenarios(&model_props); + if scenario_props.is_empty() { + anyhow::bail!("No climate scenarios valid / available for the specified models."); + } + + let available_scenarios: Vec = + scenario_props.iter().map(|s| s.scenario).collect(); + + let output_keys: std::collections::BTreeSet = + requested_outputs.keys().cloned().collect(); + let (selected_scenarios, should_reflect_inputs, include_raw_ensemble) = + resolve_requests(&output_keys, &available_scenarios)?; + + let variables = resolve_variables(&inputs.variables); + let requests: Vec<_> = selected_scenarios + .into_iter() + .flat_map(|scenario| { + variables + .iter() + .map(move |v| (ClimateVariableRequest::new(*v), scenario)) + }) + .collect(); + + let request_props: Vec<(ClimateVariableRequest, ClimateScenarioProperties)> = requests + .into_iter() + .map(|(v, s)| (v, s.properties())) + .collect(); + + let configuration = CONFIG.geoengine.api_config(CONTEXT.session_token().ok()); + let (mut outputs, computation_ids) = compute_climate( + &configuration, + &point, + &ClimatePeriods { + analysis_start: inputs.year_begin, + range: inputs.year_range, + reference_start: inputs.reference_year_begin, + }, + &request_props, + &model_props, + ) + .await?; + + if should_reflect_inputs { + outputs.inputs = Some(inputs); + } + if !include_raw_ensemble { + outputs.raw_ensemble_data = None; + } + + for computation_id in computation_ids.iter().filter(|id| id.is_some()) { + add_credits_used_pending(self.db.clone(), configuration.clone(), *computation_id) + .await?; + } + + Ok(outputs) + } +} + +/// Parsed and validated [`Execute`] payload for the climate-risk process. +pub struct ClimateRiskProcessParams { + pub inputs: ClimateRiskInputs, + pub requested_outputs: HashMap, +} + +fn parse_inputs( + inputs: &HashMap, +) -> Result { + let value = serde_json::to_value(inputs).context("Failed to serialize process inputs")?; + serde_json::from_value(value).context("Failed to deserialize climate-risk inputs") +} + +fn validate_inputs( + Year(start_year): Year, + YearRange(range): YearRange, + Year(reference_year): Year, +) -> Result<()> { + if !(5..=30).contains(&range) { + anyhow::bail!("Year range must be between 5 and 30 years"); + } + if start_year < FUTURE_START_YEAR { + anyhow::bail!("Start year must be at least {FUTURE_START_YEAR}"); + } + if start_year + range > FUTURE_END_YEAR { + anyhow::bail!("Start year plus range must not exceed {FUTURE_END_YEAR}"); + } + if reference_year < HISTORICAL_START_YEAR { + anyhow::bail!("Reference period start year must be at least {HISTORICAL_START_YEAR}"); + } + if reference_year + range > HISTORICAL_END_YEAR { + anyhow::bail!( + "Reference period start year plus range must not exceed {HISTORICAL_END_YEAR}" + ); + } + Ok(()) +} + +/// Filters the requested models down to those in the registry. The third return value +/// lists the user-specified models that were dropped, so callers can report them. +fn resolve_models( + specified_models: &[String], +) -> (Vec, Vec, Vec) { + if specified_models.is_empty() { + let (models, props): (Vec<_>, Vec<_>) = MODEL_REGISTRY + .iter() + .map(|(id, props)| (id.clone(), props.clone())) + .unzip(); + (models, props, Vec::new()) + } else { + let mut models = Vec::new(); + let mut props = Vec::new(); + let mut dropped = Vec::new(); + for model_id in specified_models { + if let Some(model_props) = MODEL_REGISTRY.get(model_id) { + models.push(model_id.clone()); + props.push(model_props.clone()); + } else { + dropped.push(model_id.clone()); + } + } + (models, props, dropped) + } +} + +fn resolve_available_scenarios( + model_props: &[ClimateModelProperties], +) -> Vec { + ClimateScenario::FUTURE + .iter() + .copied() + .filter(|s| model_props.iter().any(|m| m.scenarios.contains(s))) + .map(ClimateScenario::properties) + .collect() +} + +fn resolve_variables(specified_variables: &[ClimateVariable]) -> Vec { + let mut seen = std::collections::HashSet::new(); + let mut variables = Vec::new(); + for variable in specified_variables { + if seen.insert(*variable) { + variables.push(*variable); + } + } + if variables.is_empty() { + ClimateVariable::ALL.to_vec() + } else { + variables + } +} + +fn resolve_requests( + output_keys: &std::collections::BTreeSet, + available_scenarios: &[ClimateScenario], +) -> Result<(Vec, bool, bool)> { + let mut should_reflect_inputs = output_keys.is_empty(); + let mut include_raw_ensemble = false; + let mut selected_scenarios = Vec::new(); + + // BTreeSet iterates in order, so scenario selection is deterministic. + for output_key in output_keys { + if output_key == "inputs" { + should_reflect_inputs = true; + continue; + } + + if output_key == "rawEnsembleData" { + include_raw_ensemble = true; + continue; + } + + let future = ClimateScenario::FUTURE + .iter() + .copied() + .find(|scenario| scenario.name() == output_key); + match future { + // A future scenario no selected model supports is skipped, not an error: the requested + // models can be narrower than the registry, and the client only ever sees the scenarios + // `build_outputs` advertised. Anything else (`historical`, a typo) stays an error. + Some(scenario) if !available_scenarios.contains(&scenario) => { + tracing::warn!( + "Skipping output `{output_key}`: no selected model provides that scenario" + ); + } + Some(scenario) => selected_scenarios.push(scenario), + None => anyhow::bail!("Unknown output requested: {output_key}"), + } + } + + if selected_scenarios.is_empty() { + selected_scenarios = available_scenarios.to_vec(); + } + + Ok(( + selected_scenarios, + should_reflect_inputs, + include_raw_ensemble, + )) +} + +#[cfg(test)] +mod tests { + use super::*; + use geoengine_api_client::models::RasterOperator; + use ogcapi::types::processes::Input; + use serde_json::json; + + #[test] + fn it_deserializes_the_input() { + let payload = json!({ + "coordinate": { + "value": { + "type": "Point", + "coordinates": [12.34, 56.78] + }, + "mediaType": "application/geo+json" + }, + "yearBegin": 2050, + "yearRange": 30, + "referenceYearBegin": 1981, + "variables": ["heatDays", "iceDays"], + "models": ["ACCESS-CM2"] + }); + + let inputs: HashMap = serde_json::from_value(payload).unwrap(); + let inputs = parse_inputs(&inputs).unwrap(); + assert_eq!( + inputs.variables, + vec![ClimateVariable::HeatDays, ClimateVariable::IceDays] + ); + assert_eq!(inputs.reference_year_begin, Year(1981)); + } + + #[test] + fn it_deserializes_omitted_optional_inputs_as_their_defaults() { + let payload = json!({ + "coordinate": { + "value": { + "type": "Point", + "coordinates": [12.34, 56.78] + }, + "mediaType": "application/geo+json" + }, + "referenceYearBegin": 1981 + }); + + let inputs: HashMap = serde_json::from_value(payload).unwrap(); + let inputs = parse_inputs(&inputs).unwrap(); + assert_eq!(inputs.reference_year_begin, Year(1981)); + } + + #[test] + fn it_rejects_missing_reference_year_begin() { + let payload = json!({ + "coordinate": { + "value": { + "type": "Point", + "coordinates": [12.34, 56.78] + }, + "mediaType": "application/geo+json" + } + }); + + let inputs: HashMap = serde_json::from_value(payload).unwrap(); + let error = format!("{:#}", parse_inputs(&inputs).unwrap_err()); + + assert!(error.contains("referenceYearBegin"), "{error}"); + } + + #[test] + fn it_rejects_malformed_inputs_with_context() { + let payload = json!({ "coordinate": { "value": { "type": "Point" }, "mediaType": "application/geo+json" } }); + + let inputs: HashMap = serde_json::from_value(payload).unwrap(); + let error = format!("{:#}", parse_inputs(&inputs).unwrap_err()); + + assert!( + error.contains("Failed to deserialize climate-risk inputs"), + "{error}" + ); + assert!(error.contains("coordinate"), "{error}"); + } + + #[crate::test] + async fn it_declares_expected_inputs_and_outputs_in_the_summary(db: DbHandle) { + let process = ClimateRiskProcess::new(db).process().await.unwrap(); + + assert_eq!(process.summary.id, "climate-risk"); + assert_eq!(process.summary.version, "0.2.0"); + + assert!(!process.inputs.contains_key("scenarios")); + assert!(!process.inputs.contains_key("yearEnd")); + assert!(!process.inputs.contains_key("region")); + assert!(process.inputs.contains_key("variables")); + assert!(process.inputs.contains_key("yearRange")); + assert!(process.inputs.contains_key("referenceYearBegin")); + + assert!(process.outputs.contains_key("rawEnsembleData")); + assert!(!process.outputs.contains_key("historical")); + assert!(!process.outputs.contains_key("climateRisk")); + assert_eq!( + process.outputs["ssp245"].description_type.title.as_deref(), + Some("SSP2-4.5 (Intermediate emissions)") + ); + + // Declared scenario outputs track the registry, so no registered scenario is left out and no + // unavailable one is offered. + let registry: Vec = MODEL_REGISTRY.values().cloned().collect(); + let scenario_keys: Vec = ClimateScenario::FUTURE + .iter() + .map(|scenario| scenario.name().to_string()) + .collect(); + let declared_scenarios: std::collections::BTreeSet = process + .outputs + .keys() + .filter(|key| scenario_keys.contains(key)) + .cloned() + .collect(); + let expected: std::collections::BTreeSet = resolve_available_scenarios(®istry) + .iter() + .map(|props| props.scenario.name().to_string()) + .collect(); + assert_eq!(declared_scenarios, expected); + } + + #[crate::test] + async fn it_marks_reference_year_begin_as_required_with_a_default(db: DbHandle) { + let process = ClimateRiskProcess::new(db).process().await.unwrap(); + let input = &process.inputs["referenceYearBegin"]; + + assert_eq!(input.schema["type"], json!("integer")); + assert!(input.schema.get("anyOf").is_none()); + assert_eq!(input.schema["default"], json!(1981)); + assert_eq!( + input.description_type.metadata.len(), + 0, + "no metadata should be present: {:#?}", + input.description_type.metadata + ); + assert_eq!(input.min_occurs.unwrap_or(1), 1); + } + + #[test] + fn it_validates_year_ranges() { + let cases: &[(Year, YearRange, Year, bool)] = &[ + (Year(2050), YearRange(4), Year(1981), false), // range too small + (Year(2050), YearRange(31), Year(1981), false), // range too large + (Year(2080), YearRange(30), Year(1981), false), // analysis end > 2100 + (Year(2050), YearRange(20), Year(1949), false), // reference start < 1950 + (Year(2014), YearRange(20), Year(1981), false), // analysis start < 2015 + (Year(2050), YearRange(20), Year(1981), true), // valid + (Year(2050), YearRange(20), Year(2015), false), // reference end > 2014 + (Year(2050), YearRange(5), Year(1981), true), // valid, min range + (Year(2050), YearRange(30), Year(1981), true), // valid, max range + ]; + for &(start, range, reference, expected_ok) in cases { + let ok = validate_inputs(start, range, reference).is_ok(); + assert_eq!( + ok, expected_ok, + "validate_inputs({start:?}, {range:?}, {reference:?})" + ); + } + } + + #[test] + fn it_names_dataset_raster_sources() { + let scenario = ClimateScenario::Ssp245.properties(); + let model = ClimateModelProperties { + id: "ACCESS-CM2".to_string(), + variant: "r1i1p1f1".to_string(), + grid: "gn".to_string(), + scenarios: vec![ClimateScenario::Historical, ClimateScenario::Ssp245], + }; + let var = ClimateVariable::HeatDays.properties(); + + let result = ClimateRiskProcess::dataset_raster_source(&var, &model, &scenario); + + assert!(matches!(result, RasterOperator::GdalSource(_))); + + let value = serde_json::to_value(&result).unwrap(); + assert_eq!( + value["params"]["data"], + "nexgddp_cmip6_ACCESS-CM2_ssp245_tasmax" + ); + } + + #[test] + fn it_resolves_models() { + let (models, _props, dropped) = resolve_models(&[]); + assert!(!models.is_empty()); + assert!(dropped.is_empty()); + + let first_model = models[0].clone(); + let (models, _props, dropped) = resolve_models(std::slice::from_ref(&first_model)); + assert_eq!(models, vec![first_model]); + assert!(dropped.is_empty()); + + let (_, _, dropped) = resolve_models(&["nonexistent-model".to_string()]); + assert_eq!(dropped, vec!["nonexistent-model".to_string()]); + } + + #[test] + fn it_resolves_available_scenarios() { + let model_props: Vec = MODEL_REGISTRY.values().cloned().collect(); + let scenarios = resolve_available_scenarios(&model_props); + assert!(!scenarios.is_empty()); + assert!(resolve_available_scenarios(&[]).is_empty()); + } + + #[test] + fn it_resolves_variables() { + assert_eq!(resolve_variables(&[]), ClimateVariable::ALL.to_vec()); + assert_eq!( + resolve_variables(&[ClimateVariable::HeatDays]), + vec![ClimateVariable::HeatDays] + ); + assert_eq!( + resolve_variables(&[ + ClimateVariable::HeatDays, + ClimateVariable::HeatDays, + ClimateVariable::IceDays + ]), + vec![ClimateVariable::HeatDays, ClimateVariable::IceDays] + ); + } + + #[test] + fn it_enumerates_the_registered_models_in_the_models_schema() { + let mut settings = schemars::generate::SchemaSettings::default(); + settings.meta_schema = None; + let mut generator = settings.into_generator(); + let schema = models_schema(&mut generator); + let items = schema["items"].as_object().unwrap(); + let values = items["enum"].as_array().unwrap(); + assert_eq!(values.len(), MODEL_REGISTRY.len()); + for id in MODEL_REGISTRY.keys() { + assert!( + values.contains(&serde_json::Value::String(id.clone())), + "{id} missing from models enum" + ); + } + } + + #[test] + fn it_resolves_requests() { + let scenarios = vec![ClimateScenario::Ssp245, ClimateScenario::Ssp585]; + + let (selected, should_reflect, include_raw) = + resolve_requests(&std::collections::BTreeSet::new(), &scenarios).unwrap(); + assert_eq!(selected, scenarios); + assert!(should_reflect); + assert!(!include_raw); + + let keys = std::collections::BTreeSet::from(["ssp245".to_string(), "ssp585".to_string()]); + let (selected, should_reflect, _) = resolve_requests(&keys, &scenarios).unwrap(); + assert_eq!(selected, scenarios); + assert!(!should_reflect); + + let keys = std::collections::BTreeSet::from(["inputs".to_string()]); + let (_, should_reflect, include_raw) = resolve_requests(&keys, &scenarios).unwrap(); + assert!(should_reflect); + assert!(!include_raw); + + let keys = std::collections::BTreeSet::from(["rawEnsembleData".to_string()]); + let (_, should_reflect, include_raw) = resolve_requests(&keys, &scenarios).unwrap(); + assert!(!should_reflect); + assert!(include_raw); + } + + #[test] + fn it_skips_scenarios_the_selected_models_do_not_provide() { + // Only ssp245 is available: asking for ssp585 must skip it, not fail the whole run. + let available = vec![ClimateScenario::Ssp245]; + let keys = std::collections::BTreeSet::from(["ssp245".to_string(), "ssp585".to_string()]); + let (selected, _, _) = resolve_requests(&keys, &available).unwrap(); + assert_eq!(selected, vec![ClimateScenario::Ssp245]); + + // Skipping the only requested scenario falls back to everything available. + let keys = std::collections::BTreeSet::from(["ssp585".to_string()]); + let (selected, _, _) = resolve_requests(&keys, &available).unwrap(); + assert_eq!(selected, available); + } + + #[test] + fn it_rejects_invalid_requests() { + let scenarios = vec![ClimateScenario::Ssp245]; + // `historical` is never an output, so it must stay an error rather than a silent skip. + for key in ["nonexistent", "climateRisk", "historical"] { + let keys = std::collections::BTreeSet::from([key.to_string()]); + assert!( + resolve_requests(&keys, &scenarios).is_err(), + "expected error for key {key}" + ); + } + } +} diff --git a/backend/src/processes/climate_risk/types.rs b/backend/src/processes/climate_risk/types.rs new file mode 100644 index 0000000..82a701f --- /dev/null +++ b/backend/src/processes/climate_risk/types.rs @@ -0,0 +1,713 @@ +use crate::processes::parameters::{ + DataResource, DataResourceSchema, Days, PointGeoJsonInput, Year, YearRange, +}; +use geoengine_api_client::models::RasterDataType; +use schemars::JsonSchema; +use serde::{Deserialize, Serialize}; +use std::collections::HashMap; +use utoipa::ToSchema; + +/// Climate variable to compute (WMO-based daily threshold indicators). +#[derive(Deserialize, Serialize, Debug, JsonSchema, ToSchema, Copy, Clone, PartialEq, Eq, Hash)] +#[schema(title = "ClimateVariable")] +#[schemars(rename = "ClimateVariable")] +#[serde(rename_all = "camelCase")] +pub enum ClimateVariable { + HeatDays, + IceDays, + TropicalNights, + FrostDays, + DryDays, + HeavyRainDays, +} + +impl ClimateVariable { + pub const ALL: &'static [Self] = &[ + Self::HeatDays, + Self::IceDays, + Self::TropicalNights, + Self::FrostDays, + Self::DryDays, + Self::HeavyRainDays, + ]; + + pub fn name(self) -> &'static str { + match self { + Self::HeatDays => "heatDays", + Self::IceDays => "iceDays", + Self::TropicalNights => "tropicalNights", + Self::FrostDays => "frostDays", + Self::DryDays => "dryDays", + Self::HeavyRainDays => "heavyRainDays", + } + } + + /// Returns the dataset suffix and `GeoEngine` expression for this variable. + /// + /// Expressions operate on daily values `A`: temperature in Kelvin (converted via + /// `A - 273.15` to °C) or precipitation in mm/s (converted via `A * 86400` to mm/day). + /// Thresholds follow WMO climate-index definitions: + /// heat ≥ 30 °C, ice/frost < 0 °C, tropical night > 20 °C, dry < 1 mm, heavy rain > 20 mm. + pub fn properties(self) -> ClimateVariableProperties { + match self { + ClimateVariable::HeatDays => ClimateVariableProperties { + name: "Heat Days", + dataset_variable_suffix: "tasmax", + expression: "let c = (A - 273.15); if c >= 30 { 1 } else { 0 }", + }, + ClimateVariable::IceDays => ClimateVariableProperties { + name: "Ice Days", + dataset_variable_suffix: "tasmax", + expression: "let c = (A - 273.15); if c < 0 { 1 } else { 0 }", + }, + ClimateVariable::TropicalNights => ClimateVariableProperties { + name: "Tropical Nights", + dataset_variable_suffix: "tasmin", + expression: "let c = (A - 273.15); if c > 20 { 1 } else { 0 }", + }, + ClimateVariable::FrostDays => ClimateVariableProperties { + name: "Frost Days", + dataset_variable_suffix: "tasmin", + expression: "let c = (A - 273.15); if c < 0 { 1 } else { 0 }", + }, + ClimateVariable::DryDays => ClimateVariableProperties { + name: "Dry Days", + dataset_variable_suffix: "pr", + expression: "if (A * 86400) < 1 { 1 } else { 0 }", + }, + ClimateVariable::HeavyRainDays => ClimateVariableProperties { + name: "Heavy Rain Days", + dataset_variable_suffix: "pr", + expression: "if (A * 86400) > 20 { 1 } else { 0 }", + }, + } + } +} + +/// A climate variable selected for computation. +#[derive(Debug, Clone)] +pub struct ClimateVariableRequest { + pub variable: ClimateVariable, +} + +impl ClimateVariableRequest { + pub fn new(variable: ClimateVariable) -> Self { + Self { variable } + } +} + +/// NEX-GDDP-CMIP6 climate model (loaded from registry). +#[derive(Debug, Clone)] +pub struct ClimateModelProperties { + pub id: String, + // variant + grid identify the CMIP6 realization; consumed by the ingestion notebook when + // building kerchunk reference paths, not by the backend query path. + #[allow( + dead_code, + reason = "CMIP6 realization identity, used for data ingestion only" + )] + pub variant: String, + #[allow(dead_code, reason = "CMIP6 grid label, used for data ingestion only")] + pub grid: String, + pub scenarios: Vec, +} + +/// Shared Socioeconomic Pathway scenario. +#[derive(Deserialize, Serialize, Debug, JsonSchema, ToSchema, Copy, Clone, PartialEq, Eq, Hash)] +#[schema(title = "ClimateScenario")] +#[schemars(rename = "ClimateScenario")] +#[serde(rename_all = "lowercase")] +pub enum ClimateScenario { + Historical, + Ssp245, + Ssp585, +} + +impl ClimateScenario { + pub const ALL: &'static [Self] = &[Self::Historical, Self::Ssp245, Self::Ssp585]; + + pub const FUTURE: &'static [Self] = &[Self::Ssp245, Self::Ssp585]; + + pub fn name(self) -> &'static str { + match self { + Self::Historical => "historical", + Self::Ssp245 => "ssp245", + Self::Ssp585 => "ssp585", + } + } + pub fn properties(self) -> ClimateScenarioProperties { + match self { + ClimateScenario::Historical => ClimateScenarioProperties { + name: "Historical (1950–2014)", + dataset_prefix: "historical", + scenario: ClimateScenario::Historical, + }, + ClimateScenario::Ssp245 => ClimateScenarioProperties { + name: "SSP2-4.5 (Intermediate emissions)", + dataset_prefix: "ssp245", + scenario: ClimateScenario::Ssp245, + }, + ClimateScenario::Ssp585 => ClimateScenarioProperties { + name: "SSP5-8.5 (High emissions)", + dataset_prefix: "ssp585", + scenario: ClimateScenario::Ssp585, + }, + } + } +} + +/// Resolved properties for a climate scenario (dataset prefix, display name). +#[derive(Debug, Clone)] +pub struct ClimateScenarioProperties { + pub name: &'static str, + pub dataset_prefix: &'static str, + pub scenario: ClimateScenario, +} + +pub(crate) fn default_year_begin() -> Year { + Year(2050) +} + +/// First year with available NEX-GDDP-CMIP6 future data. +pub(crate) const FUTURE_START_YEAR: u16 = 2015; +/// Last year with available NEX-GDDP-CMIP6 future data. +pub(crate) const FUTURE_END_YEAR: u16 = 2100; +/// First year with available NEX-GDDP-CMIP6 historical data. +pub(crate) const HISTORICAL_START_YEAR: u16 = 1950; +/// Last year with available NEX-GDDP-CMIP6 historical data. +pub(crate) const HISTORICAL_END_YEAR: u16 = 2014; +/// Days in a Julian year (IAU: 365 d + 1 leap day / 4). Used to convert annual occurrence counts to probabilities. +pub(crate) const DAYS_PER_JULIAN_YEAR: f64 = 365.25; + +pub(crate) fn default_reference_year_begin() -> Year { + Year(1981) +} + +fn default_year_range() -> YearRange { + YearRange(30) +} + +fn default_variables() -> Vec { + ClimateVariable::ALL.to_vec() +} + +// ponytail: default models loaded from registry at startup; empty here, populated in mod.rs +fn default_models() -> Vec { + Vec::new() +} + +/// User-supplied inputs for the climate risk process. +#[derive(Deserialize, Serialize, Debug, JsonSchema, ToSchema)] +#[serde(rename_all = "camelCase")] +pub struct ClimateRiskInputs { + pub coordinate: PointGeoJsonInput, + #[serde(default = "default_year_begin")] + #[schema(minimum = 2015, maximum = 2100)] + pub year_begin: Year, + #[serde(default = "default_year_range")] + #[schemars(default = "default_year_range")] + pub year_range: YearRange, + #[schemars(default = "default_reference_year_begin")] + #[schema(minimum = 1950, maximum = 2014)] + pub reference_year_begin: Year, + #[serde(default = "default_variables")] + pub variables: Vec, + #[serde(default = "default_models")] + pub models: Vec, +} + +/// Aggregated statistics for a single climate variable across models. +#[derive(Deserialize, Serialize, Debug, JsonSchema, ToSchema, Clone)] +pub struct ClimateVariableResult { + pub max: Days, + pub min: Days, + pub mean: Days, + pub median: Days, + #[serde(skip_serializing_if = "Option::is_none")] + pub occurrence_probability: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub raw_members: Option>, +} + +/// A single row in the climate risk `DataResource` output. +#[derive(Deserialize, Serialize, Debug, JsonSchema, ToSchema, Clone, Default)] +#[serde(rename_all = "camelCase")] +pub struct ClimateRiskRow { + pub variable: String, + pub scenario: String, + /// Mean number of days per year in the analysis period. + pub mean: Days, + /// Median number of days per year in the analysis period. + pub median: Days, + /// Minimum number of days per year across models. + pub min: Days, + /// Maximum number of days per year across models. + pub max: Days, + /// Occurrence probability as a ratio from 0 to 1. + #[serde(skip_serializing_if = "Option::is_none")] + pub occurrence_probability: Option, + /// Difference in days per year from the reference period. + #[serde(skip_serializing_if = "Option::is_none")] + pub anomaly: Option, + /// Ready-to-display occurrence-probability label, e.g. "7 · high (3 %)". + #[serde(skip_serializing_if = "Option::is_none")] + pub occurrence_probability_label: Option, + /// Cell color for the occurrence probability. + #[serde(skip_serializing_if = "Option::is_none")] + pub occurrence_probability_color: Option, + /// Ready-to-display anomaly label, e.g. "+10 days (+20 %)". + #[serde(skip_serializing_if = "Option::is_none")] + pub anomaly_label: Option, + /// Cell color for the anomaly. + #[serde(skip_serializing_if = "Option::is_none")] + pub anomaly_color: Option, +} + +/// A single row in the raw ensemble data output. +#[derive(Deserialize, Serialize, Debug, JsonSchema, ToSchema, Clone)] +#[serde(rename_all = "camelCase")] +pub struct ClimateRiskRawRow { + pub variable: String, + pub scenario: String, + pub model: String, + pub value: f64, +} + +/// Output of the climate risk process: summary table and raw ensemble data. +#[derive(Deserialize, Serialize, Debug, JsonSchema, ToSchema, Default)] +#[serde(rename_all = "camelCase")] +pub struct ClimateRiskOutputs { + pub inputs: Option, + /// Analysis window as `"2041–2070"`, used for display in result headlines. + #[serde(skip_serializing_if = "Option::is_none")] + pub analysis_period: Option, + /// Reference window used for anomalies as `"1981–2010"`, `None` when no reference period. + #[serde(skip_serializing_if = "Option::is_none")] + pub reference_period: Option, + #[schema(value_type = Option, inline)] + pub climate_risk: Option>>, + #[schema(value_type = Option, inline)] + pub raw_ensemble_data: Option>>, +} + +/// Column title for the anomaly: "Anomaly (days/year compared to 1981–2010)". +/// The analysis period lives in the resource name, so it is not repeated here. +pub(crate) fn anomaly_title(reference_period: Option<&str>) -> String { + match reference_period { + Some(period) => format!("Anomaly (days/year compared to {period})"), + None => "Anomaly (days/year)".to_string(), + } +} + +/// Existing FMEA/ISO-inspired occurrence-probability classes, ordered from +/// rarest to most likely. The thresholds and presentation values are kept +/// together so labels and colors cannot drift apart. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) enum ProbabilityClass { + ExtremelyLow, + VeryLow, + Low, + ModeratelyLow, + Moderate, + ModeratelyHigh, + High, + VeryHigh, + ExtremelyHigh, + Extreme, +} + +impl ProbabilityClass { + const ALL: [Self; 10] = [ + Self::ExtremelyLow, + Self::VeryLow, + Self::Low, + Self::ModeratelyLow, + Self::Moderate, + Self::ModeratelyHigh, + Self::High, + Self::VeryHigh, + Self::ExtremelyHigh, + Self::Extreme, + ]; + + pub(crate) fn from_probability(probability: f64) -> Self { + Self::ALL + .iter() + .rev() + .find(|class| probability >= class.threshold()) + .copied() + .unwrap_or(Self::ExtremelyLow) + } + + pub(crate) fn number(self) -> u8 { + match self { + Self::ExtremelyLow => 1, + Self::VeryLow => 2, + Self::Low => 3, + Self::ModeratelyLow => 4, + Self::Moderate => 5, + Self::ModeratelyHigh => 6, + Self::High => 7, + Self::VeryHigh => 8, + Self::ExtremelyHigh => 9, + Self::Extreme => 10, + } + } + + pub(crate) fn threshold(self) -> f64 { + 1.0 / f64::from(self.return_period_years()) + } + + /// Return period represented by this existing FMEA/ISO-inspired class. + pub(crate) fn return_period_years(self) -> u32 { + match self { + Self::ExtremelyLow => 20_000, + Self::VeryLow => 5_000, + Self::Low => 1_000, + Self::ModeratelyLow => 500, + Self::Moderate => 250, + Self::ModeratelyHigh => 100, + Self::High => 50, + Self::VeryHigh => 20, + Self::ExtremelyHigh => 10, + Self::Extreme => 5, + } + } + + fn label(self) -> &'static str { + match self { + Self::ExtremelyLow => "extremely low", + Self::VeryLow => "very low", + Self::Low => "low", + Self::ModeratelyLow => "moderately low", + Self::Moderate => "moderate", + Self::ModeratelyHigh => "moderately high", + Self::High => "high", + Self::VeryHigh => "very high", + Self::ExtremelyHigh => "extremely high", + Self::Extreme => "extreme", + } + } + + fn color(self) -> &'static str { + match self { + Self::ExtremelyLow => "#66bb6a", + Self::VeryLow => "#2e7d32", + Self::Low => "#fdd835", + Self::ModeratelyLow => "#f9a825", + Self::Moderate => "#fb8c00", + Self::ModeratelyHigh => "#ef6c00", + Self::High => "#e53935", + Self::VeryHigh => "#c62828", + Self::ExtremelyHigh => "#8e24aa", + Self::Extreme => "#4a148c", + } + } +} + +/// Ready-to-display occurrence-probability label, e.g. "7 · high (3 %)". +pub(crate) fn probability_label(p: f64) -> String { + let class = ProbabilityClass::from_probability(p); + let pct = format!("{:.1}", p * 100.0) + .trim_end_matches(".0") + .to_string(); + format!("{} · {} ({pct} %)", class.number(), class.label()) +} + +pub(crate) fn probability_color(p: f64) -> String { + ProbabilityClass::from_probability(p).color().to_string() +} + +/// Shared 7-stop divergent gradient for the anomaly color scale, ordered from -100 % to +100 %. +const ANOMALY_PALETTE: [&str; 7] = [ + "#2166ac", "#67a9cf", "#d1e5f0", "#f7f7f7", "#fddbc7", "#ef8a62", "#b2182b", +]; + +/// Quantizes a percentage change (-100..=+100) to the nearest of the 7 shared class stops. +/// Values outside the range clamp to the extremes; negative = blue, zero = white, positive = red. +#[allow( + clippy::cast_possible_truncation, + clippy::cast_sign_loss, + reason = "The rounded palette index is bounded to -3..=3 before casting, so no truncation or sign loss can occur." +)] +pub(crate) fn percentage_color(pct: f64) -> String { + let t = ((3.0 * pct / 100.0).round() as i32).clamp(-3, 3); + ANOMALY_PALETTE[(t + 3) as usize].to_string() +} + +/// Percentage change of the analysis mean relative to the reference mean, unclamped. +/// A missing/zero reference mean maps to the extreme percentages by sign of the anomaly. +pub(crate) fn anomaly_pct(analysis_mean: f64, reference_mean: f64) -> f64 { + if (analysis_mean - reference_mean).abs() <= f64::EPSILON { + 0.0 + } else if reference_mean.abs() <= f64::EPSILON { + (analysis_mean - reference_mean).signum() * 100.0 + } else { + (analysis_mean - reference_mean) / reference_mean * 100.0 + } +} + +/// Formats a signed value with `decimals` places, trailing zeros trimmed, e.g. 10.0 -> "+10". +fn format_signed(value: f64, decimals: usize) -> String { + let sign = if value > 0.0 { + "+" + } else if value < 0.0 { + "-" + } else { + "" + }; + let digits = format!("{:.decimals$}", value.abs(), decimals = decimals); + let trimmed = digits.trim_end_matches('0').trim_end_matches('.'); + format!("{sign}{trimmed}") +} + +/// Ready-to-display anomaly label, e.g. "+10 days (+20 %)". +pub(crate) fn anomaly_label(anomaly_days: f64, pct: f64) -> String { + format!( + "{} days ({} %)", + format_signed(anomaly_days, 2), + format_signed(pct, 1) + ) +} + +/// Dataset suffix and `GeoEngine` expression for a climate variable. +pub struct ClimateVariableProperties { + pub(crate) name: &'static str, + pub(crate) dataset_variable_suffix: &'static str, + pub(crate) expression: &'static str, +} + +impl ClimateVariableProperties { + pub fn name_string(&self) -> String { + self.name.to_string() + } + pub fn measurement_string() -> String { + "Days".to_string() + } + pub fn measurement_unit() -> String { + "Days".to_string() + } + pub fn expression_string(&self) -> String { + self.expression.to_string() + } + pub fn expression_dtype() -> RasterDataType { + RasterDataType::I8 + } + pub fn year_agg_dtype() -> RasterDataType { + RasterDataType::U16 + } +} + +#[cfg(test)] +mod tests { + use super::*; + use approx::assert_abs_diff_eq; + + #[test] + fn it_exposes_climate_variable_properties() { + for (var, expected_name, expected_suffix, expected_expr) in [ + (ClimateVariable::HeatDays, "Heat Days", "tasmax", "c >= 30"), + (ClimateVariable::IceDays, "Ice Days", "tasmax", "c < 0"), + ( + ClimateVariable::TropicalNights, + "Tropical Nights", + "tasmin", + "c > 20", + ), + (ClimateVariable::FrostDays, "Frost Days", "tasmin", "c < 0"), + (ClimateVariable::DryDays, "Dry Days", "pr", "86400) < 1"), + ( + ClimateVariable::HeavyRainDays, + "Heavy Rain Days", + "pr", + "86400) > 20", + ), + ] { + let props = var.properties(); + assert_eq!(props.name, expected_name); + assert_eq!(props.dataset_variable_suffix, expected_suffix); + assert!(props.expression.contains(expected_expr)); + } + } + + #[test] + fn it_exposes_climate_scenario_properties() { + for (scenario, expected_name, expected_prefix) in [ + ( + ClimateScenario::Historical, + "Historical (1950–2014)", + "historical", + ), + ( + ClimateScenario::Ssp245, + "SSP2-4.5 (Intermediate emissions)", + "ssp245", + ), + ( + ClimateScenario::Ssp585, + "SSP5-8.5 (High emissions)", + "ssp585", + ), + ] { + let props = scenario.properties(); + assert_eq!(props.scenario, scenario); + assert_eq!(props.name, expected_name); + assert_eq!(props.dataset_prefix, expected_prefix); + } + } + + #[test] + fn it_provides_climate_variable_property_accessors() { + let props = ClimateVariable::HeatDays.properties(); + assert_eq!(props.name_string(), "Heat Days"); + assert_eq!(ClimateVariableProperties::measurement_unit(), "Days"); + assert_eq!(ClimateVariableProperties::measurement_string(), "Days"); + assert_eq!( + ClimateVariableProperties::expression_dtype(), + RasterDataType::I8 + ); + assert_eq!( + ClimateVariableProperties::year_agg_dtype(), + RasterDataType::U16 + ); + } + + #[test] + fn it_exposes_scenario_names() { + for scenario in ClimateScenario::ALL { + assert_eq!( + scenario.properties().name, + match scenario { + ClimateScenario::Historical => "Historical (1950–2014)", + ClimateScenario::Ssp245 => "SSP2-4.5 (Intermediate emissions)", + ClimateScenario::Ssp585 => "SSP5-8.5 (High emissions)", + } + ); + } + } + + #[test] + fn it_maps_every_probability_class_number_label_and_color() { + for class in ProbabilityClass::ALL { + let number = class.number(); + assert!((1..=10).contains(&number)); + assert!(!class.label().is_empty()); + assert!(class.color().starts_with('#')); + assert!(class.return_period_years() > 0); + assert!(class.threshold() > 0.0); + } + assert_eq!(ProbabilityClass::ModeratelyLow.number(), 4); + assert_eq!(ProbabilityClass::ModeratelyLow.label(), "moderately low"); + assert_eq!(ProbabilityClass::VeryLow.color(), "#2e7d32"); + assert_eq!(ProbabilityClass::ExtremelyHigh.color(), "#8e24aa"); + } + + #[test] + fn it_probability_label_maps_classes() { + assert_eq!(probability_label(0.0), "1 · extremely low (0 %)"); + assert_eq!(probability_label(0.0001), "1 · extremely low (0 %)"); + assert_eq!(probability_label(0.0005), "2 · very low (0.1 %)"); + assert_eq!(probability_label(0.001), "3 · low (0.1 %)"); + assert_eq!(probability_label(0.005), "5 · moderate (0.5 %)"); + assert_eq!(probability_label(0.02), "7 · high (2 %)"); + assert_eq!(probability_label(0.03), "7 · high (3 %)"); + assert_eq!(probability_label(0.1), "9 · extremely high (10 %)"); + assert_eq!(probability_label(0.2), "10 · extreme (20 %)"); + assert_eq!(probability_label(0.9), "10 · extreme (90 %)"); + } + + #[test] + fn it_probability_class_uses_existing_boundaries() { + let boundaries = [ + (0.00005, ProbabilityClass::ExtremelyLow), + (0.0002, ProbabilityClass::VeryLow), + (0.001, ProbabilityClass::Low), + (0.002, ProbabilityClass::ModeratelyLow), + (0.004, ProbabilityClass::Moderate), + (0.01, ProbabilityClass::ModeratelyHigh), + (0.02, ProbabilityClass::High), + (0.05, ProbabilityClass::VeryHigh), + (0.1, ProbabilityClass::ExtremelyHigh), + (0.2, ProbabilityClass::Extreme), + ]; + for (probability, expected) in boundaries { + assert_eq!(ProbabilityClass::from_probability(probability), expected); + } + assert_eq!( + ProbabilityClass::from_probability(0.0), + ProbabilityClass::ExtremelyLow + ); + assert_eq!( + ProbabilityClass::from_probability(1.0), + ProbabilityClass::Extreme + ); + assert_eq!(ProbabilityClass::High.return_period_years(), 50); + } + + #[test] + fn it_probability_color_maps_classes() { + assert_eq!(probability_color(0.0), "#66bb6a"); + assert_eq!(probability_color(0.02), "#e53935"); + assert_eq!(probability_color(0.03), "#e53935"); + assert_eq!(probability_color(0.2), "#4a148c"); + assert_eq!(probability_color(0.9), "#4a148c"); + } + + #[test] + fn it_anomaly_pct_maps_change_and_zero_reference() { + assert_abs_diff_eq!(anomaly_pct(120.0, 100.0), 20.0); + assert_abs_diff_eq!(anomaly_pct(50.0, 100.0), -50.0); + assert_abs_diff_eq!(anomaly_pct(10.0, 0.0), 100.0); + assert_abs_diff_eq!(anomaly_pct(-10.0, 0.0), -100.0); + assert_abs_diff_eq!(anomaly_pct(0.0, 0.0), 0.0); + } + + #[test] + fn it_anomaly_label_maps_days_and_raw_pct() { + assert_eq!(anomaly_label(10.0, 20.0), "+10 days (+20 %)"); + assert_eq!(anomaly_label(-5.0, -10.0), "-5 days (-10 %)"); + assert_eq!(anomaly_label(0.0, 0.0), "0 days (0 %)"); + assert_eq!(anomaly_label(10.5, 33.3), "+10.5 days (+33.3 %)"); + assert_eq!( + anomaly_label(10.0, 250.0), + "+10 days (+250 %)", + "label shows the raw percentage, the color clamps separately" + ); + } + + #[test] + fn it_percentage_color_maps_and_clamps() { + assert_eq!(percentage_color(-100.0), "#2166ac"); + assert_eq!(percentage_color(-67.0), "#67a9cf"); + assert_eq!(percentage_color(-33.0), "#d1e5f0"); + assert_eq!(percentage_color(0.0), "#f7f7f7"); + assert_eq!(percentage_color(33.0), "#fddbc7"); + assert_eq!(percentage_color(67.0), "#ef8a62"); + assert_eq!(percentage_color(100.0), "#b2182b"); + assert_eq!(percentage_color(999.0), "#b2182b"); + assert_eq!(percentage_color(-999.0), "#2166ac"); + } + + #[test] + fn it_climate_risk_row_serializes_display_fields() { + let row = ClimateRiskRow { + variable: "Heat Days".to_string(), + scenario: "ssp245".to_string(), + mean: Days(50.0), + median: Days(50.0), + min: Days(0.0), + max: Days(100.0), + occurrence_probability: Some(0.03), + anomaly: Some(Days(10.0)), + occurrence_probability_label: Some("7 · high (3 %)".to_string()), + occurrence_probability_color: Some("#e53935".to_string()), + anomaly_label: Some("+10 days (+20 %)".to_string()), + anomaly_color: Some("#fddbc7".to_string()), + }; + let json = serde_json::to_value(&row).unwrap(); + assert_eq!(json["occurrenceProbabilityLabel"], "7 · high (3 %)"); + assert_eq!(json["occurrenceProbabilityColor"], "#e53935"); + assert_eq!(json["anomalyLabel"], "+10 days (+20 %)"); + assert_eq!(json["anomalyColor"], "#fddbc7"); + } +} diff --git a/backend/src/processes/climate_risk/workflow.rs b/backend/src/processes/climate_risk/workflow.rs new file mode 100644 index 0000000..ddf0c58 --- /dev/null +++ b/backend/src/processes/climate_risk/workflow.rs @@ -0,0 +1,135 @@ +use geoengine_api_client::models::{ + Aggregation, ContinuousMeasurement, Expression, ExpressionParameters, GdalSource, + GdalSourceParameters, Measurement, RasterBandDescriptor, RasterOperator, SingleRasterSource, + SumAggregation, TemporalRasterAggregation, TemporalRasterAggregationParameters, + TimeGranularity, TimeStep, +}; +use tracing::instrument; + +use super::{ClimateRiskProcess, types::*}; +impl ClimateRiskProcess { + /// Builds the GDAL raster source for a variable/model/scenario combination. + #[instrument(skip(var, model, scenario))] + pub(crate) fn dataset_raster_source( + var: &ClimateVariableProperties, + model: &ClimateModelProperties, + scenario: &ClimateScenarioProperties, + ) -> RasterOperator { + let dataset_name = format!( + "nexgddp_cmip6_{}_{}_{}", + model.id, scenario.dataset_prefix, var.dataset_variable_suffix + ); + RasterOperator::GdalSource( + GdalSource { + r#type: Default::default(), + params: GdalSourceParameters { + data: dataset_name, + overview_level: None, + } + .into(), + } + .into(), + ) + } + + /// Builds the daily threshold expression (e.g. heat-day indicator) for a variable. + pub(crate) fn build_variable_day_expression( + var: &ClimateVariableProperties, + model: &ClimateModelProperties, + scenario: &ClimateScenarioProperties, + ) -> RasterOperator { + RasterOperator::Expression( + Expression { + r#type: Default::default(), + params: ExpressionParameters { + expression: var.expression_string(), + output_type: ClimateVariableProperties::expression_dtype(), + output_band: Some( + RasterBandDescriptor { + name: var.name_string(), + measurement: Measurement::Continuous( + ContinuousMeasurement { + measurement: ClimateVariableProperties::measurement_string(), + r#type: Default::default(), + unit: Some(Some(ClimateVariableProperties::measurement_unit())), + } + .into(), + ) + .into(), + } + .into(), + ), + map_no_data: false, + } + .into(), + sources: SingleRasterSource { + raster: Self::dataset_raster_source(var, model, scenario).into(), + } + .into(), + } + .into(), + ) + } + + /// Builds the yearly sum aggregation workflow (sums daily values per calendar year). + pub(crate) fn build_variable_year_agg_workflow( + var: &ClimateVariableProperties, + model: &ClimateModelProperties, + scenario: &ClimateScenarioProperties, + ) -> RasterOperator { + RasterOperator::TemporalRasterAggregation( + TemporalRasterAggregation { + r#type: Default::default(), + params: TemporalRasterAggregationParameters { + aggregation: Aggregation::SumAggregation(Box::new(SumAggregation { + ignore_no_data: true, + r#type: Default::default(), + })) + .into(), + output_type: Some(Some(ClimateVariableProperties::year_agg_dtype())), + window: TimeStep { + granularity: TimeGranularity::Years, + step: 1, + } + .into(), + // No reference: yearly windows are anchored at the epoch, i.e. calendar-aligned, + // so one workflow serves both the analysis and the reference time range. + window_reference: None, + } + .into(), + sources: SingleRasterSource { + raster: Self::build_variable_day_expression(var, model, scenario).into(), + } + .into(), + } + .into(), + ) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn it_builds_variable_workflow_chains() { + let scenario = ClimateScenario::Ssp245.properties(); + let model = ClimateModelProperties { + id: "ACCESS-CM2".to_string(), + variant: "r1i1p1f1".to_string(), + grid: "gn".to_string(), + scenarios: vec![ClimateScenario::Historical, ClimateScenario::Ssp245], + }; + let var = ClimateVariable::HeatDays.properties(); + + let day_expr = ClimateRiskProcess::build_variable_day_expression(&var, &model, &scenario); + assert!(matches!(day_expr, RasterOperator::Expression(_))); + + let year_agg = + ClimateRiskProcess::build_variable_year_agg_workflow(&var, &model, &scenario); + assert!(matches!( + year_agg, + RasterOperator::TemporalRasterAggregation(_) + )); + } +} diff --git a/backend/src/processes/land_use_sealed_area/types.rs b/backend/src/processes/land_use_sealed_area/types.rs index ad050da..1d255bc 100644 --- a/backend/src/processes/land_use_sealed_area/types.rs +++ b/backend/src/processes/land_use_sealed_area/types.rs @@ -277,6 +277,7 @@ pub fn summary_to_data_resource( }, ], primary_key: vec!["landUseType".to_string()].into(), + ..Default::default() }, data: vec![ land_use_summary_row_to_output(site_rows.total_sealed_area, unit_for_area), @@ -322,6 +323,7 @@ pub fn site_to_data_resource( }, ], primary_key: vec!["location".to_string()].into(), + ..Default::default() }, data: site_rows .into_iter() diff --git a/backend/src/processes/mod.rs b/backend/src/processes/mod.rs index 739ac18..8775d43 100644 --- a/backend/src/processes/mod.rs +++ b/backend/src/processes/mod.rs @@ -1,4 +1,5 @@ mod biodiversity_sensitive_areas; +mod climate_risk; mod habitat_distance; mod land_use_sealed_area; mod ndvi; @@ -9,6 +10,7 @@ mod test_util; mod util; pub use biodiversity_sensitive_areas::BiodiversitySensitiveAreasProcess; +pub use climate_risk::ClimateRiskProcess; pub use habitat_distance::HabitatDistanceProcess; pub use land_use_sealed_area::LandUseSealedAreaProcess; pub use ndvi::NDVIProcess; diff --git a/backend/src/processes/parameters/data_resource.rs b/backend/src/processes/parameters/data_resource.rs index 55cb481..2599254 100644 --- a/backend/src/processes/parameters/data_resource.rs +++ b/backend/src/processes/parameters/data_resource.rs @@ -10,7 +10,7 @@ pub struct DataResourceSchema; /// Data resources for outputting tabular data with JSON. /// Based on . -#[derive(Serialize, Debug)] +#[derive(Serialize, Deserialize, Debug)] pub struct DataResource { /// A resource MUST contain a name property. pub name: DataResourceName, @@ -85,11 +85,16 @@ impl AsRef for DataResourceName { } } -#[derive(Serialize, Deserialize, Debug)] +#[derive(Serialize, Deserialize, Debug, Default)] #[serde(rename_all = "camelCase")] +#[allow(clippy::struct_field_names)] pub struct Fields { + #[serde(rename = "$schema", default, skip_serializing_if = "Option::is_none")] + pub schema: Option, pub fields: Vec, pub primary_key: Option>, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub biois: Option, } /// Field specification for Table Schema, based on . @@ -133,6 +138,35 @@ pub trait HasTableSchemaType { fn table_schema_type() -> TableSchemaType; } +/// BioIS-specific metadata for rendering a standard Table Schema field. +#[derive(Serialize, Deserialize, Debug, Clone)] +#[serde(rename_all = "camelCase")] +pub struct BioisTableSchemaExtension { + pub display: std::collections::HashMap, + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub hidden_fields: Vec, +} + +#[derive(Serialize, Deserialize, Debug, Clone, PartialEq)] +#[serde(rename_all = "camelCase")] +pub struct BioisDisplayMetadata { + /// Semantic category of the rendered value, e.g. a risk probability. + pub kind: BioisDisplayKind, + /// Name of a row property carrying the complete display label for this field. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub label_field: Option, + /// Name of a row property carrying the CSS color for this field. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub color_field: Option, +} + +#[derive(Serialize, Deserialize, Debug, Clone, PartialEq)] +#[serde(rename_all = "camelCase")] +pub enum BioisDisplayKind { + RiskProbability, + RiskAnomaly, +} + #[cfg(test)] mod tests { use super::*; @@ -177,6 +211,7 @@ mod tests { }, ], primary_key: Some(vec!["id".to_string()]), + ..Default::default() }; let json = serde_json::to_value(&fields).unwrap(); @@ -210,6 +245,7 @@ mod tests { item_type: None, }], primary_key: None, + ..Default::default() }, }; diff --git a/backend/src/processes/parameters/mod.rs b/backend/src/processes/parameters/mod.rs index dbf60b9..07d241f 100644 --- a/backend/src/processes/parameters/mod.rs +++ b/backend/src/processes/parameters/mod.rs @@ -1,6 +1,6 @@ use anyhow::Result; use geoengine_api_client::models::{BoundingBox2D, Coordinate2D, ProvenanceEntry}; -use geojson::Position; +use geojson::{PointType, Position}; use ogcapi::types::{ common::Crs, processes::description::{DescriptionType, InputDescription, Metadata, OutputDescription}, @@ -11,7 +11,8 @@ use std::collections::HashMap; use utoipa::ToSchema; pub use data_resource::{ - DataResource, DataResourceSchema, Fields, HasTableSchemaType, TableSchemaField, + BioisDisplayKind, BioisDisplayMetadata, BioisTableSchemaExtension, DataResource, + DataResourceName, DataResourceSchema, Fields, HasTableSchemaType, TableSchemaField, TableSchemaItemType, TableSchemaType, }; #[cfg(test)] @@ -23,7 +24,9 @@ pub use geo_json::{ }; #[cfg(test)] pub use units::Hectare; -pub use units::{Area, Kilometers, Month, Percentage, SquareMeter, UnitForArea, Year}; +pub use units::{ + Area, Days, Kilometers, Month, Percentage, SquareMeter, UnitForArea, Year, YearRange, +}; mod data_resource; mod geo_json; @@ -154,6 +157,7 @@ impl TryFrom> for DataResource }, ], primary_key: vec![DocumentationSource::DATA_FIELD_NAME.to_string()].into(), + ..Default::default() }, }) } @@ -263,6 +267,7 @@ impl ToOutputHashMap for [OutputSpec; N] { } } +/// A 2D bounding box in WGS84 coordinates. #[derive(Debug, Clone, PartialEq)] pub struct BoundingBox { minx: f64, @@ -293,6 +298,17 @@ impl BoundingBox { } } + /// Create a small bounding box around a point with the given half-span. + pub fn around_point(point: &PointType, half_span: f64) -> Self { + Self::new( + point[0] - half_span, + point[1] - half_span, + point[0] + half_span, + point[1] + half_span, + Crs::from_epsg(4326), + ) + } + pub fn enlarge_by_positions<'p>(&mut self, other: impl Iterator) { for position in other { self.minx = self.minx.min(position[0]); diff --git a/backend/src/processes/parameters/units.rs b/backend/src/processes/parameters/units.rs index 8deee2e..afa532c 100644 --- a/backend/src/processes/parameters/units.rs +++ b/backend/src/processes/parameters/units.rs @@ -100,6 +100,25 @@ impl std::fmt::Display for Year { } } +/// A number of (fractional) days (e.g. mean/median/min/max anomaly values). +#[derive( + Deserialize, Serialize, Debug, JsonSchema, ToSchema, Copy, Clone, PartialEq, PartialOrd, +)] +#[serde(transparent)] +pub struct Days(pub f64); + +impl Default for Days { + fn default() -> Self { + Days(0.0) + } +} + +/// Length of a time window in years (e.g., 5 years). +#[derive(Deserialize, Serialize, Debug, JsonSchema, ToSchema, Copy, Clone, PartialEq)] +#[serde(transparent)] +#[schemars(example = YearRange(20))] +pub struct YearRange(#[schemars(range(min = 5, max = 30))] pub u16); + #[derive(Deserialize, Serialize, Debug, JsonSchema, ToSchema, Copy, Clone)] #[serde(transparent)] #[schemars(example = Month(1))] diff --git a/backend/src/processes/path_info.rs b/backend/src/processes/path_info.rs index afcb97f..d3c8900 100644 --- a/backend/src/processes/path_info.rs +++ b/backend/src/processes/path_info.rs @@ -8,9 +8,11 @@ use crate::processes::{ biodiversity_sensitive_areas::{ BiodiversitySensitiveAreasProcessInputs, BiodiversitySensitiveAreasProcessOutputs, }, + climate_risk::{ClimateRiskInputs, ClimateRiskRawRow, ClimateRiskRow}, habitat_distance::{HabitatDistanceProcessInputs, HabitatDistanceProcessOutputs}, land_use_sealed_area::{LandUseSealedAreaProcessInputs, LandUseSealedAreaProcessOutputs}, ndvi::{NDVIProcessInputs, NDVIProcessOutputs}, + parameters::{DataResource, DataResourceSchema}, }; use axum::Json; use ogcapi::types::processes::Response; @@ -85,6 +87,48 @@ fn execute_biodiversity_sensitive_areas( ) { } +/// Process execution (Climate Risk) +#[allow(unused, reason = "Placeholder for spec only")] +#[derive(Deserialize, ToSchema, Debug)] +pub struct ClimateRiskProcessParams { + pub inputs: ClimateRiskInputs, + #[serde(default)] + #[allow(clippy::zero_sized_map_values, reason = "Placeholder for spec only")] + pub outputs: HashMap, + #[serde(default)] + pub response: Response, +} + +/// Response body of a climate-risk execution: one summary table per computed scenario, plus the +/// optional `inputs` echo and `rawEnsembleData`. Which scenario keys are present depends on the +/// requested outputs and on the model registry (`conf/nexgddp_models.toml`); the process +/// description's `outputs` is the authoritative list for a given deployment. +#[allow(unused, reason = "Placeholder for spec only")] +#[derive(ToSchema, Debug, Default)] +#[serde(rename_all = "camelCase")] +pub struct ClimateRiskProcessResponses { + /// Summary table for SSP2-4.5. + #[schema(value_type = Option, inline)] + pub ssp245: Option>>, + /// Summary table for SSP5-8.5. + #[schema(value_type = Option, inline)] + pub ssp585: Option>>, + /// Per-model raw values for each variable x scenario combination. + #[schema(value_type = Option, inline)] + pub raw_ensemble_data: Option>>, + /// The submitted inputs, echoed back. + pub inputs: Option, +} + +#[allow(unused, reason = "Placeholder for spec only")] +#[utoipa::path( + post, + path = "/processes/climate-risk/execution", + tag = "Processes", + responses((status = OK, body = ClimateRiskProcessResponses)) +)] +fn execute_climate_risk(Json(_input): Json) {} + /// Process execution (Land Use Sealed Area – ESRS E4-5) #[allow(unused, reason = "Placeholder for spec only")] #[derive(Deserialize, ToSchema, Debug)] @@ -113,6 +157,7 @@ fn execute_land_use_sealed_area(Json(_input): Json Response { + ( + [( + CONTENT_TYPE, + HeaderValue::from_static("application/schema+json"), + )], + CLIMATE_RISK_TABLE_SCHEMA, + ) + .into_response() +} + +#[cfg(test)] +mod tests { + use super::*; + use axum::body::to_bytes; + + #[tokio::test] + async fn it_serves_the_climate_risk_profile() { + let response = climate_risk_table_schema_profile().await; + assert_eq!(response.headers()[CONTENT_TYPE], "application/schema+json"); + let body = to_bytes(response.into_body(), usize::MAX).await.unwrap(); + let profile: serde_json::Value = serde_json::from_slice(&body).unwrap(); + assert_eq!( + profile["title"], + "BioIS climate-risk Table Schema extension" + ); + let display = &profile["$defs"]["display"]; + assert_eq!(display["properties"]["kind"]["enum"][0], "riskProbability"); + assert_eq!(display["properties"]["kind"]["enum"][1], "riskAnomaly"); + assert_eq!(display["properties"]["labelField"]["type"], "string"); + assert_eq!(display["properties"]["colorField"]["type"], "string"); + assert_eq!( + profile["allOf"][1]["properties"]["biois"]["properties"]["hiddenFields"]["type"], + "array" + ); + assert_eq!(display["additionalProperties"], false); + } +} diff --git a/backend/src/server.rs b/backend/src/server.rs index 45a1d28..d9a677b 100644 --- a/backend/src/server.rs +++ b/backend/src/server.rs @@ -8,8 +8,8 @@ use crate::{ handler, jobs::JobHandler, processes::{ - BiodiversitySensitiveAreasProcess, HabitatDistanceProcess, LandUseSealedAreaProcess, - NDVIProcess, ProcessesOpenApiSpec, + BiodiversitySensitiveAreasProcess, ClimateRiskProcess, HabitatDistanceProcess, + LandUseSealedAreaProcess, NDVIProcess, ProcessesOpenApiSpec, }, state::spawn_with_user, }; @@ -43,6 +43,7 @@ pub async fn server() -> anyhow::Result { .nest("/auth", handler::auth_router()) .nest("/credits", credits::router()) .nest("/contact", contact::router(&CONFIG.contact)?) + .merge(handler::profile_router()) .with_state(AppState { db: db_pool.clone(), api_config: CONFIG.geoengine.api_config(None), @@ -55,6 +56,7 @@ pub async fn server() -> anyhow::Result { let mut processors: Vec> = vec![ Arc::new(Echo), Arc::new(NDVIProcess::new(db_pool.clone())), + Arc::new(ClimateRiskProcess::new(db_pool.clone())), Arc::new(LandUseSealedAreaProcess::new(db_pool.clone())), ]; add_habitat_distance_process(&mut processors, db_pool.clone()).await; diff --git a/k8s/pod.yaml b/k8s/pod.yaml index 489bcdd..f9daa2c 100644 --- a/k8s/pod.yaml +++ b/k8s/pod.yaml @@ -80,7 +80,7 @@ spec: periodSeconds: 10 volumeMounts: - name: pgdata - mountPath: /var/lib/postgresql/data + mountPath: /var/lib/postgresql volumes: - name: pgdata persistentVolumeClaim: From 118b2fccebf73c952fda4e3fdf0b7902c492b91f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johannes=20Dr=C3=B6nner?= Date: Fri, 2 Oct 2026 13:01:45 +0200 Subject: [PATCH 2/9] feat(frontend): render climate-risk inputs and results The process form is schema-driven, so climate-risk needs no dedicated screen. Two additions make it work: - a `StringArray` field type for multi-select inputs whose items are an enum (`variables`, `models`), rendered inline like the existing `StringEnum` and `IntegerWithSmallRange` cases; the option list is resolved from the schema including nullable and `$ref`-wrapped arrays, reusing `resolveSchemaRef` for `$defs` lookups - rendering of the `schema.biois` display extension in result tables, so a risk probability or anomaly is shown as a labelled, coloured chip instead of a bare number Result tables are titled with the data resource title rather than its internal name, e.g. "SSP2-4.5 (Intermediate emissions) - 2050-2079". `SMALL_INTEGER_RANGE` replaces the inline `12` cutoff for bounded integer inputs so that a 5-30 year window renders as a select; it only widens which integer ranges become dropdowns. --- .../src/app/create/create.component.spec.ts | 194 +++++++++++++++++- frontend/src/app/create/create.component.ts | 11 +- .../app/create/inputs-visualizer.component.ts | 32 ++- frontend/src/app/create/schema-info.spec.ts | 156 +++++++++++++- frontend/src/app/create/schema-info.ts | 149 +++++++++++++- frontend/src/app/create/simple-form-field.ts | 14 +- .../data-resource-table.component.spec.ts | 154 +++++++++++++- .../result/data-resource-table.component.ts | 101 +++++++-- frontend/src/app/result/result.component.ts | 13 +- 9 files changed, 770 insertions(+), 54 deletions(-) diff --git a/frontend/src/app/create/create.component.spec.ts b/frontend/src/app/create/create.component.spec.ts index 9eb5ef1..d87fb7e 100644 --- a/frontend/src/app/create/create.component.spec.ts +++ b/frontend/src/app/create/create.component.spec.ts @@ -1,9 +1,11 @@ import { ComponentFixture, TestBed } from '@angular/core/testing'; +import { By } from '@angular/platform-browser'; import { vi } from 'vitest'; import { Process, ProcessesApi } from '@geoengine/biois'; -import { CreateComponent } from './create.component'; +import { CreateComponent, inputsForRequest } from './create.component'; import { inputBinding } from '@angular/core'; import { mockResizeObserverClass } from '../util/mock-resize-observer'; +import { FieldType } from './schema-info'; describe('CreateComponent', () => { let component: CreateComponent; @@ -12,7 +14,6 @@ describe('CreateComponent', () => { beforeEach(async () => { globalThis.ResizeObserver = mockResizeObserverClass([]); - // mock ProcessesApi.process early so resource loaders in the component don't perform real network fetches vi.spyOn(ProcessesApi.prototype, 'process').mockResolvedValue(ndviProcess()); await TestBed.configureTestingModule({ @@ -27,9 +28,144 @@ describe('CreateComponent', () => { }); it('should create', () => { - // fixture.componentRef.setInput('processId', 'ndvi'); expect(component).toBeTruthy(); }); + + it('computes the process name from fallback', () => { + expect(component.processName()).toBe('Ndvi'); + }); + + it('parses inputs into typed descriptors', () => { + const inputs = component.inputs(); + expect(inputs.length).toBe(3); + expect(inputs.find((i) => i.key === 'coordinate')?.type).toBe(FieldType.Coordinate); + expect(inputs.find((i) => i.key === 'year')?.type).toBe(FieldType.Integer); + expect(inputs.find((i) => i.key === 'month')?.type).toBe(FieldType.IntegerWithSmallRange); + }); + + it('parses outputs', () => { + expect(component.outputs().length).toBe(2); + }); + + it('sets default form values from constructor effects', () => { + const inputs = component.formModel().inputs; + expect((inputs['coordinate'] as Record)['value']).toBeDefined(); + expect(inputs['year']).toBe(2020); + }); + + it('enables all outputs by default', () => { + expect(component.formModel().outputs['ndvi']).toBe(true); + expect(component.formModel().outputs['kNdvi']).toBe(true); + }); + + it('toggleOutput adds and removes outputs', () => { + component.toggleOutput('ndvi', true); + expect(component.formModel().outputs['ndvi']).toBe(true); + component.toggleOutput('ndvi', false); + expect(component.formModel().outputs['ndvi']).toBeUndefined(); + }); + + it('toggleOutput preserves other outputs', () => { + component.toggleOutput('ndvi', true); + component.toggleOutput('kNdvi', true); + component.toggleOutput('ndvi', false); + expect(component.formModel().outputs['kNdvi']).toBe(true); + }); + + it('inputsForRequest omits undefined and null values', () => { + expect( + inputsForRequest({ + year: 2020, + optionalField: null, + referenceYearBegin: undefined, + }), + ).toEqual({ year: 2020 }); + }); + + it('renders title and fieldsets', () => { + fixture.detectChanges(); + const titleEl = fixture.debugElement.query(By.css('app-page-title')); + expect(titleEl).toBeTruthy(); + const fieldsets = fixture.debugElement.queryAll(By.css('fieldset')); + expect(fieldsets.length).toBe(2); + }); + + it('renders output checkboxes', () => { + fixture.detectChanges(); + const checkboxes = fixture.debugElement.queryAll(By.css('mat-checkbox')); + expect(checkboxes.length).toBe(2); + }); + + it('submit button disabled when form invalid', () => { + component.toggleOutput('ndvi', false); + component.toggleOutput('kNdvi', false); + fixture.detectChanges(); + const button = fixture.debugElement.query(By.css('button[type="submit"]')); + expect((button.nativeElement as HTMLButtonElement).disabled).toBe(true); + }); + + it('renders all default outputs as selected', () => { + fixture.detectChanges(); + expect(component.outputs().length).toBeGreaterThan(0); + const outputKeys = Object.keys(component.formModel().outputs); + expect(outputKeys.length).toBeGreaterThan(0); + }); + + it('renders input number fields for coordinate and integer types', () => { + fixture.detectChanges(); + const numberInputs = fixture.debugElement.queryAll(By.css('input[type="number"]')); + expect(numberInputs.length).toBe(3); + }); +}); + +describe('CreateComponent with diverse input types', () => { + let component: CreateComponent; + let fixture: ComponentFixture; + + beforeEach(async () => { + globalThis.ResizeObserver = mockResizeObserverClass([]); + + vi.spyOn(ProcessesApi.prototype, 'process').mockResolvedValue(allTypesProcess()); + + await TestBed.configureTestingModule({ + imports: [CreateComponent], + }).compileComponents(); + + fixture = TestBed.createComponent(CreateComponent, { + bindings: [inputBinding('processId', () => 'all-types')], + }); + component = fixture.componentInstance; + await fixture.whenStable(); + }); + + it('parses boolean input', () => { + expect(component.inputs().find((i) => i.key === 'boolInput')?.type).toBe(FieldType.Boolean); + expect(component.formModel().inputs['boolInput']).toBe(false); + }); + + it('parses string array input', () => { + expect(component.inputs().find((i) => i.key === 'stringArrayInput')?.type).toBe( + FieldType.StringArray, + ); + expect(component.formModel().inputs['stringArrayInput']).toEqual(['x', 'y', 'z']); + }); + + it('parses optional string array input as null', () => { + const input = component.inputs().find((i) => i.key === 'optionalStringArrayInput'); + expect(input?.type).toBe(FieldType.StringArray); + expect(input?.optional).toBe(true); + expect(component.formModel().inputs['optionalStringArrayInput']).toBeNull(); + }); + + it('parses GeoJson input', () => { + expect(component.inputs().find((i) => i.key === 'geoJsonInput')?.type).toBe(FieldType.GeoJson); + expect(component.formModel().inputs['geoJsonInput']).toBeInstanceOf(Error); + }); + + it('parses number input', () => { + expect(component.inputs().find((i) => i.key === 'numberInput')?.type).toBe(FieldType.Number); + expect(component.formModel().inputs['numberInput']).toBe(0); + }); }); function ndviProcess(): Process { @@ -129,3 +265,55 @@ function ndviProcess(): Process { }; return process; } + +function allTypesProcess(): Process { + const process = new Process(); + process.id = 'all-types'; + process.inputs = { + boolInput: { + title: 'Boolean Input', + schema: { type: 'boolean' }, + }, + stringArrayInput: { + title: 'String Array Input', + schema: { + type: 'array', + items: { $ref: '#/$defs/MyEnum' }, + $defs: { + MyEnum: { type: 'string', enum: ['x', 'y', 'z'] }, + }, + }, + }, + optionalStringArrayInput: { + title: 'Optional String Array Input', + schema: { + anyOf: [ + { + type: 'array', + items: { $ref: '#/$defs/MyEnum' }, + $defs: { + MyEnum: { type: 'string', enum: ['x', 'y', 'z'] }, + }, + }, + { type: 'null' }, + ], + }, + }, + geoJsonInput: { + title: 'GeoJSON Input', + schema: { + type: 'object', + title: 'FeatureCollectionGeoJsonInput', + properties: {}, + }, + }, + numberInput: { + title: 'Number Input', + schema: { type: 'number' }, + }, + }; + process.outputs = { + output1: { title: 'Output 1', schema: null }, + }; + return process; +} diff --git a/frontend/src/app/create/create.component.ts b/frontend/src/app/create/create.component.ts index d7ca98f..0ec36e0 100644 --- a/frontend/src/app/create/create.component.ts +++ b/frontend/src/app/create/create.component.ts @@ -152,7 +152,7 @@ export class CreateComponent { this.formModel.update((current) => ({ ...current, inputs })); }); - // initially, set all outputs + // initially, enable all outputs effect(() => { const outputDescriptions = this.outputs(); const outputs = Object.fromEntries(outputDescriptions.map(({ key }) => [key, true])); @@ -210,13 +210,15 @@ function outputsForRequest(outputs: Record): Record): Record { +export function inputsForRequest(inputs: Record): Record { return Object.fromEntries( Object.entries(inputs).filter(([_, value]) => !isNullOrUndefined(value)), ); @@ -274,6 +276,7 @@ function compareInputDescriptionsForSorting( case FieldType.String: case FieldType.RelativeJsonPointer: case FieldType.StringEnum: + case FieldType.StringArray: return 0; case FieldType.Coordinate: case FieldType.GeoJson: diff --git a/frontend/src/app/create/inputs-visualizer.component.ts b/frontend/src/app/create/inputs-visualizer.component.ts index 37ee66d..919373d 100644 --- a/frontend/src/app/create/inputs-visualizer.component.ts +++ b/frontend/src/app/create/inputs-visualizer.component.ts @@ -6,7 +6,13 @@ import { input, output, } from '@angular/core'; -import { InputDescription, FieldType, defaultInput, enumOptions } from './schema-info'; +import { + InputDescription, + FieldType, + defaultInput, + enumOptions, + resolveArrayEnumSchema, +} from './schema-info'; export { enumOptions }; import { CommonModule } from '@angular/common'; @@ -70,6 +76,16 @@ import { InfoIconComponent } from '../util/info-icon.component'; } + @case (FieldType.StringArray) { + + {{ input.title }} + + @for (option of stringArrayOptions(input.schema); track option) { + {{ option }} + } + + + } @case (FieldType.IntegerWithSmallRange) { {{ input.title }} @@ -165,6 +181,7 @@ export class InputsFormComponent { readonly FieldType = FieldType; readonly enumOptions = enumOptions; readonly integerRangeList = integerRangeList; + readonly stringArrayOptions = stringArrayOptions; readonly isFieldSet = computed>(() => { const form = this.form(); @@ -193,6 +210,10 @@ export class InputsFormComponent { return formInput as FieldTree; } + asStringArrayInput(formInput: MaybeFieldTree): FieldTree { + return formInput as FieldTree; + } + asGeoJsonInput( formInput: MaybeFieldTree, ): FieldTree { @@ -219,6 +240,7 @@ export class InputsFormComponent { } } +/** Expands a small bounded integer schema into select options. */ export function integerRangeList(schema: JSONSchema | undefined): number[] { if ( !schema || @@ -235,3 +257,11 @@ export function integerRangeList(schema: JSONSchema | undefined): number[] { } return range; } + +/** Returns all enum values for a string-array input. */ +export function stringArrayOptions(schema: JSONSchema | undefined): string[] { + if (!schema || typeof schema === 'boolean') return []; + + const items = resolveArrayEnumSchema(schema); + return items ? enumOptions(items) : []; +} diff --git a/frontend/src/app/create/schema-info.spec.ts b/frontend/src/app/create/schema-info.spec.ts index 55898ef..07def7b 100644 --- a/frontend/src/app/create/schema-info.spec.ts +++ b/frontend/src/app/create/schema-info.spec.ts @@ -1,5 +1,13 @@ import { InputDescription as ApiInputDescription } from '@geoengine/biois'; -import { retrieveInputDescription, FieldType, jsonSchemaToZod } from './schema-info'; +import { + retrieveInputDescription, + FieldType, + jsonSchemaToZod, + defaultInput, + defaultInputs, + resolveArrayEnumSchema, +} from './schema-info'; +import type { JSONSchema } from 'ya-json-schema-types'; const testInputs: { sites: ApiInputDescription; @@ -7,6 +15,8 @@ const testInputs: { unitForArea: ApiInputDescription; previousYearData: ApiInputDescription; year: ApiInputDescription; + yearRange: ApiInputDescription; + referenceYearBegin: ApiInputDescription; siteTypeField: ApiInputDescription; } = { sites: { @@ -194,6 +204,44 @@ const testInputs: { type: 'integer', }, }, + yearRange: { + title: 'Range (years)', + description: 'Length of the climate-risk aggregation window in years (5-30).', + schema: { + $defs: { + // eslint-disable-next-line @typescript-eslint/naming-convention + 'GeoJSON FeatureCollection': { + $ref: 'https://geojson.org/schema/FeatureCollection.json', + }, + GeoJsonInputMediaType: { + enum: ['application/geo+json'], + type: 'string', + }, + }, + default: 30, + description: 'Length of the climate-risk aggregation window in years (5-30).', + examples: [30], + maximum: 30, + minimum: 5, + title: 'YearRange', + type: 'integer', + }, + }, + referenceYearBegin: { + title: 'Reference period start', + description: + 'First year of the reference period used to compute anomalies. Uses the same range as the analysis window.', + schema: { + description: 'Year of reporting or change (e.g., 2023, 2024, etc.)', + examples: [2020], + format: 'uint16', + maximum: 2100, + minimum: 2000, + title: 'Year', + type: 'integer', + default: 2020, + }, + }, siteTypeField: { title: 'Site Type Field', description: @@ -274,6 +322,30 @@ describe('retrieveInputDescription', () => { }); }); + it('should process IntegerWithSmallRange input (yearRange)', () => { + const result = retrieveInputDescription('yearRange', testInputs.yearRange); + + expect(result).toMatchObject({ + key: 'yearRange', + title: 'Range (years)', + type: FieldType.IntegerWithSmallRange, + optional: false, + }); + }); + + it('should process non-nullable Integer input with a default (referenceYearBegin)', () => { + const result = retrieveInputDescription('referenceYearBegin', testInputs.referenceYearBegin); + + expect(result).toMatchObject({ + key: 'referenceYearBegin', + title: 'Reference period start', + type: FieldType.Integer, + optional: false, + }); + + expect(defaultInput(result)).toBe(2020); + }); + it('should process nullable input (previousYearData)', () => { const result = retrieveInputDescription('previousYearData', testInputs.previousYearData); @@ -310,6 +382,14 @@ describe('retrieveInputDescription', () => { }); }); +describe('defaultInputs', () => { + it('enables required inputs with their schema default', () => { + const input = retrieveInputDescription('referenceYearBegin', testInputs.referenceYearBegin); + const result = defaultInputs([input]); + expect(result['referenceYearBegin']).toBe(2020); + }); +}); + describe('jsonSchemaToZod', () => { it('should convert GeoJSON input schema (sites) to Zod schema', () => { const zodSchema = jsonSchemaToZod(retrieveInputDescription('sites', testInputs.sites).schema); @@ -327,3 +407,77 @@ describe('jsonSchemaToZod', () => { expect(zodSchema).not.toBeNull(); }); }); + +/** The `models` input of the climate-risk process, as produced by the backend registry. */ +function modelsInputSchema(): Omit & { schema: JSONSchema } { + return { + title: 'Climate models', + description: 'Climate models to use. If empty, all registered models are used.', + schema: { + type: 'array', + items: { + type: 'string', + enum: ['ACCESS-CM2', 'MPI-ESM1-2-LR'], + }, + default: [], + }, + }; +} + +describe('resolveArrayEnumSchema', () => { + it('resolves the items enum of an array schema', () => { + const items = resolveArrayEnumSchema(modelsInputSchema().schema); + + expect(items).toMatchObject({ enum: ['ACCESS-CM2', 'MPI-ESM1-2-LR'] }); + }); + + it('resolves a nullable array schema', () => { + const nullable = { + anyOf: [modelsInputSchema().schema, { type: 'null' }], + }; + + expect(resolveArrayEnumSchema(nullable)).toMatchObject({ + enum: ['ACCESS-CM2', 'MPI-ESM1-2-LR'], + }); + }); + + it('resolves a $ref-wrapped items schema through $defs', () => { + const refWrapped = { + type: 'array', + items: { $ref: '#/$defs/ClimateModelId' }, + $defs: { ClimateModelId: { type: 'string', enum: ['ACCESS-CM2'] } }, + }; + + expect(resolveArrayEnumSchema(refWrapped)).toMatchObject({ enum: ['ACCESS-CM2'] }); + }); + + it('returns undefined when there is no enum', () => { + expect(resolveArrayEnumSchema({ type: 'array', items: { type: 'string' } })).toBeUndefined(); + expect(resolveArrayEnumSchema(undefined)).toBeUndefined(); + expect(resolveArrayEnumSchema(true)).toBeUndefined(); + }); +}); + +describe('StringArray inputs', () => { + it('detects the input type', () => { + expect(retrieveInputDescription('models', modelsInputSchema()).type).toBe( + FieldType.StringArray, + ); + }); + + it('provides the enum values as the initial form value', () => { + expect(defaultInput(retrieveInputDescription('models', modelsInputSchema()))).toEqual([ + 'ACCESS-CM2', + 'MPI-ESM1-2-LR', + ]); + }); + + it('does not classify a plain string array as StringArray', () => { + const freeText: ApiInputDescription = { + title: 'Tags', + schema: { type: 'array', items: { type: 'string' } }, + }; + + expect(retrieveInputDescription('tags', freeText).type).not.toBe(FieldType.StringArray); + }); +}); diff --git a/frontend/src/app/create/schema-info.ts b/frontend/src/app/create/schema-info.ts index 4627ee6..273cf3e 100644 --- a/frontend/src/app/create/schema-info.ts +++ b/frontend/src/app/create/schema-info.ts @@ -37,9 +37,13 @@ export enum FieldType { RelativeJsonPointer = 'relativeJsonPointer', String = 'string', StringEnum = 'stringEnum', + StringArray = 'stringArray', NestedJson = 'nestedJson', } +// UI-only cutoff: bounded integer inputs with at most 40 choices use a select. +const SMALL_INTEGER_RANGE = 40; + export function retrieveInputDescription( key: string, processInput: ApiInputDescription, @@ -110,7 +114,7 @@ function typeFromSchema(schema: JSONSchema | undefined): FieldType { if ( typeof schema.maximum === 'number' && typeof schema.minimum === 'number' && - schema.maximum - schema.minimum <= 12 + schema.maximum - schema.minimum <= SMALL_INTEGER_RANGE ) { return FieldType.IntegerWithSmallRange; } @@ -123,6 +127,34 @@ function typeFromSchema(schema: JSONSchema | undefined): FieldType { if (schema.title === 'FeatureCollectionGeoJsonInput') return FieldType.GeoJson; } + if (resolveArrayEnumSchema(schema)) return FieldType.StringArray; + + // Resolve nullable primitives like {"anyOf": [{"$ref": ...}, {"type": "null"}]} to their type + if (!type && (schema.anyOf || schema.oneOf)) { + const branches = (schema.anyOf ?? schema.oneOf) as JSONSchema[]; + const nonNull = branches.find( + (branch) => + typeof branch !== 'object' || + branch === null || + (branch as BaseJSONSchema)['type'] !== 'null', + ); + if (nonNull) { + const resolved = resolveSchemaRef(schema, nonNull); + const resolvedType = + typeof resolved === 'object' && resolved !== null + ? (resolved as BaseJSONSchema)['type'] + : undefined; + if ( + resolvedType === 'string' || + resolvedType === 'number' || + resolvedType === 'integer' || + resolvedType === 'boolean' + ) { + return typeFromSchema(resolved); + } + } + } + // nested types (for now) if (!type) { return FieldType.NestedJson; @@ -131,6 +163,86 @@ function typeFromSchema(schema: JSONSchema | undefined): FieldType { return FieldType.String; // fallback to string if type cannot be determined } +/** + * Resolve the items schema from an array schema, following `$ref` through `$defs`. + * + * `$ref` resolution is delegated to `resolveSchemaRef`, the same helper that `retrieveSubSchema` + * uses for `properties`; only the keyword walked differs (`items` instead of `properties[key]`). + * A self-contained items schema that carries its own `$defs` resolves against itself, otherwise + * against the process description that holds the definitions. + */ +function resolveItemsSchema( + schema: Record, + rootSchema: JSONSchema, +): Record | undefined { + const items = schema['items']; + if (!items || typeof items !== 'object' || Array.isArray(items)) return undefined; + + const itemsObj = items as Record; + if (!('$ref' in itemsObj)) return itemsObj; + const refRoot = '$defs' in schema ? (schema as JSONSchema) : rootSchema; + return resolveSchemaRef(refRoot, itemsObj) as Record; +} + +/** + * Type guard for an array items schema describing a string enum. + */ +function isStringEnumArray( + items: Record | undefined, +): items is { type: 'string'; enum: unknown[] } { + return !!items && items['type'] === 'string' && Array.isArray(items['enum']); +} + +/** Resolves a string-enum array, including nullable and `$ref`-wrapped schemas. */ +export function resolveArrayEnumSchema( + schema: JSONSchema | undefined, +): Record | undefined { + if (!schema || typeof schema === 'boolean') return undefined; + + const schemaRecord = schema as Record; + + const direct = resolveItemsSchema(schemaRecord, schema); + if (isStringEnumArray(direct)) return direct; + + const branches = schemaRecord['anyOf'] ?? schemaRecord['oneOf']; + if (Array.isArray(branches)) { + for (const branch of branches) { + if (typeof branch !== 'object' || branch === null) continue; + const items = resolveItemsSchema(branch as Record, schema); + if (isStringEnumArray(items)) return items; + } + } + + return undefined; +} + +/** Resolves the string enum for one input, including nullable `$ref` branches. */ +export function resolveSingleEnumSchema(schema: JSONSchema | undefined): string[] | undefined { + if (!schema || typeof schema === 'boolean') return undefined; + + const schemaRecord = schema as Record; + + const direct = schemaRecord['enum']; + if (Array.isArray(direct)) + return direct.filter((value): value is string => typeof value === 'string'); + + const branches = schemaRecord['anyOf'] ?? schemaRecord['oneOf']; + if (Array.isArray(branches)) { + for (const branch of branches) { + if (typeof branch !== 'object' || branch === null) continue; + const resolved = resolveSchemaRef(schema, branch as JSONSchema); + if (resolved && typeof resolved === 'object') { + const enumValue = (resolved as Record)['enum']; + if (Array.isArray(enumValue)) { + return enumValue.filter((value): value is string => typeof value === 'string'); + } + } + } + } + + return undefined; +} + function isOptional(schema: JSONSchema | undefined): boolean { function anySubSchemaIsNull(subSchemas: JSONSchema[] | undefined): boolean { if (!subSchemas) return false; @@ -289,10 +401,10 @@ export function jsonSchemaToZod(jsonSchema: JSONSchema): z.ZodTypeAny { throw new Error('Failed to convert JSON Schema to Zod schema.', { cause: errors }); } +/** Creates initial form values, keeping optional inputs disabled unless they are enabled in the UI. */ export function defaultInputs(inputDescriptions: Array): Record { const inputs: Record = {}; for (const input of inputDescriptions) { - // `Input` consists of `any` type // eslint-disable-next-line @typescript-eslint/no-unsafe-assignment inputs[input.key] = defaultInput(input); } @@ -323,6 +435,8 @@ export function defaultInput( case FieldType.RelativeJsonPointer: case FieldType.StringEnum: return defaultString(schema, ''); + case FieldType.StringArray: + return stringArrayValues(schema); case FieldType.NestedJson: return { value: defaultInputs(Object.values(children ?? {})), @@ -354,15 +468,35 @@ function defaultString(schema: JSONSchema, fallback: string = ''): string { const defaultValue = schema.default; if (typeof defaultValue === 'string') return defaultValue; - if (!schema.examples || !Array.isArray(schema.examples)) return fallback; + if (!schema.examples || !Array.isArray(schema.examples)) + return firstEnumOrFallback(schema, fallback); for (const example of schema.examples ?? []) { if (typeof example === 'string') return example; } + return firstEnumOrFallback(schema, fallback); +} + +/** + * Resolves a default string value for an enum schema. + * Returns the first enum value if the schema defines one, otherwise `fallback`. + * Used as a last resort in `defaultString` when neither `default` nor `examples` are set. + */ +function firstEnumOrFallback(schema: JSONSchema, fallback: string): string { + const firstEnum = resolveSingleEnumSchema(schema)?.[0]; + if (firstEnum !== undefined) return firstEnum; return fallback; } +function stringArrayValues(schema: JSONSchema): string[] { + const items = resolveArrayEnumSchema(schema); + const enumValues = items?.['enum']; + if (!Array.isArray(enumValues)) return []; + + return enumValues.filter((value: unknown): value is string => typeof value === 'string'); +} + function defaultCoordinate(schema: JSONSchema, fallback: [number, number] = [0, 0]): GeoJSONPoint { if (!schema || typeof schema === 'boolean') return geoJsonPointFeature(fallback); @@ -399,14 +533,7 @@ function geoJsonPointFeature(coordinates: [number, number]): GeoJSONPoint { } export function enumOptions(schema: JSONSchema | undefined): string[] { - if (!schema || typeof schema === 'boolean' || !schema.enum || !Array.isArray(schema.enum)) - return []; - - const options = []; - for (const value of schema.enum) { - if (typeof value === 'string') options.push(value); - } - return options; + return resolveSingleEnumSchema(schema) ?? []; } const DRAWABLE_GEOMETRY_TYPES: readonly DrawableGeometryType[] = ['Point', 'Polygon']; diff --git a/frontend/src/app/create/simple-form-field.ts b/frontend/src/app/create/simple-form-field.ts index bbea5d8..4fbd5ab 100644 --- a/frontend/src/app/create/simple-form-field.ts +++ b/frontend/src/app/create/simple-form-field.ts @@ -9,7 +9,6 @@ import { import { CommonModule } from '@angular/common'; import { MatFormFieldModule } from '@angular/material/form-field'; import { FormValueControl, ValidationError, WithOptionalFieldTree } from '@angular/forms/signals'; -import { MatCheckboxModule } from '@angular/material/checkbox'; import { MatInput, MatInputModule } from '@angular/material/input'; import { FieldType } from './schema-info'; @@ -20,10 +19,6 @@ import { FieldType } from './schema-info'; {{ title() }} @switch (type()) { - @case (FieldType.String) - @default { - - } @case (FieldType.Integer) { } - @case (FieldType.Boolean) { - True/False + @case (FieldType.String) + @default { + } } @@ -56,7 +50,7 @@ import { FieldType } from './schema-info'; `, styles: ``, changeDetection: ChangeDetectionStrategy.OnPush, - imports: [CommonModule, MatFormFieldModule, MatInputModule, MatCheckboxModule], + imports: [CommonModule, MatFormFieldModule, MatInputModule], }) export class SimpleFormFieldComponent implements FormValueControl { readonly title = input.required(); diff --git a/frontend/src/app/result/data-resource-table.component.spec.ts b/frontend/src/app/result/data-resource-table.component.spec.ts index 74d14bc..989c0c8 100644 --- a/frontend/src/app/result/data-resource-table.component.spec.ts +++ b/frontend/src/app/result/data-resource-table.component.spec.ts @@ -47,7 +47,11 @@ describe('DataResourceTableComponent', () => { fields: [ { name: 'title', type: 'string', title: 'Title' }, { name: 'reference', type: 'string', title: 'Reference' }, - { name: 'score', type: 'number', title: 'Score' }, + { + name: 'score', + type: 'number', + title: 'Score', + }, { name: 'active', type: 'boolean', title: 'Active' }, { name: 'tags', type: 'list' }, ], @@ -67,12 +71,83 @@ describe('DataResourceTableComponent', () => { expect(columns).toEqual([ { name: 'Title', key: 'title', type: ColumnType.String, isPrimaryKey: true }, { name: 'Reference', key: 'reference', type: ColumnType.Url, isPrimaryKey: false }, - { name: 'Score', key: 'score', type: ColumnType.Number, isPrimaryKey: false }, + { + name: 'Score', + key: 'score', + type: ColumnType.Number, + isPrimaryKey: false, + }, { name: 'Active', key: 'active', type: ColumnType.Boolean, isPrimaryKey: false }, { name: 'tags', key: 'tags', type: ColumnType.List, isPrimaryKey: false }, ]); }); + it('maps display metadata labelField and colorField onto columns', () => { + const columns = tableColumnInfoFromValue( + { + fields: [ + { name: 'occurrenceProbability', type: 'number', title: 'Occurrence Probability' }, + { name: 'anomaly', type: 'number', title: 'Anomaly' }, + { name: 'occurrenceProbabilityLabel', type: 'string' }, + { name: 'occurrenceProbabilityColor', type: 'string' }, + { name: 'anomalyLabel', type: 'string' }, + { name: 'anomalyColor', type: 'string' }, + ], + biois: { + hiddenFields: [ + 'occurrenceProbabilityLabel', + 'occurrenceProbabilityColor', + 'anomalyLabel', + 'anomalyColor', + ], + display: { + occurrenceProbability: { + kind: 'riskProbability', + labelField: 'occurrenceProbabilityLabel', + colorField: 'occurrenceProbabilityColor', + }, + anomaly: { + kind: 'riskAnomaly', + labelField: 'anomalyLabel', + colorField: 'anomalyColor', + }, + }, + }, + }, + [ + { + occurrenceProbability: 0.03, + occurrenceProbabilityLabel: '7 · high (3 %)', + occurrenceProbabilityColor: '#e53935', + anomaly: 10, + anomalyLabel: '+10 days (+20 %)', + anomalyColor: '#fddbc7', + }, + ], + ); + + expect(columns).toEqual([ + { + name: 'Occurrence Probability', + key: 'occurrenceProbability', + type: ColumnType.Number, + isPrimaryKey: false, + displayKind: 'riskProbability', + labelField: 'occurrenceProbabilityLabel', + colorField: 'occurrenceProbabilityColor', + }, + { + name: 'Anomaly', + key: 'anomaly', + type: ColumnType.Number, + isPrimaryKey: false, + displayKind: 'riskAnomaly', + labelField: 'anomalyLabel', + colorField: 'anomalyColor', + }, + ]); + }); + it('renders typed columns for row values', async () => { const columns: Column[] = [ { name: 'Title', key: 'title', type: ColumnType.String, isPrimaryKey: true }, @@ -121,4 +196,79 @@ describe('DataResourceTableComponent', () => { expect(root.textContent).toContain('forest'); expect(root.textContent).toContain('protected'); }); + + it('renders extension display metadata as a colored chip', async () => { + const columns: Column[] = [ + { + name: 'Occurrence Probability', + key: 'occurrenceProbability', + type: ColumnType.Number, + isPrimaryKey: false, + displayKind: 'riskProbability', + labelField: 'occurrenceProbabilityLabel', + colorField: 'occurrenceProbabilityColor', + }, + { + name: 'Anomaly', + key: 'anomaly', + type: ColumnType.Number, + isPrimaryKey: false, + displayKind: 'riskAnomaly', + labelField: 'anomalyLabel', + colorField: 'anomalyColor', + }, + ]; + + fixture.componentRef.setInput('columns', columns); + fixture.componentRef.setInput('rows', [ + { + occurrenceProbability: 0.13, + occurrenceProbabilityLabel: '8 · very high (13 %)', + occurrenceProbabilityColor: '#c62828', + anomaly: 10, + anomalyLabel: '+10 days (+20 %)', + anomalyColor: '#fddbc7', + }, + ]); + + fixture.detectChanges(); + await fixture.whenStable(); + fixture.detectChanges(); + + const root = fixture.nativeElement as HTMLElement; + const cells = Array.from(root.querySelectorAll('tbody td')); + + expect(cells.length).toBe(2); + const [probability, anomaly] = cells; + expect(probability.querySelector('.color-dot')).not.toBeNull(); + expect(probability.textContent).toContain('8 · very high (13 %)'); + expect(probability.textContent).not.toContain('0.13'); + + const chip = anomaly.querySelector('mat-chip'); + expect(chip).not.toBeNull(); + expect(chip?.textContent).toContain('+10 days (+20 %)'); + const dot = anomaly.querySelector('.color-dot'); + expect(dot?.style.backgroundColor).toBe('rgb(253, 219, 199)'); + }); + + it('falls back to the raw value when display metadata is incomplete', async () => { + fixture.componentRef.setInput('columns', [ + { + name: 'Anomaly', + key: 'anomaly', + type: ColumnType.Number, + isPrimaryKey: false, + displayKind: 'riskAnomaly', + }, + ]); + fixture.componentRef.setInput('rows', [{ anomaly: 10 }]); + + fixture.detectChanges(); + await fixture.whenStable(); + fixture.detectChanges(); + + const root = fixture.nativeElement as HTMLElement; + const cell = root.querySelector('tbody td'); + expect(cell?.textContent).toContain('10'); + }); }); diff --git a/frontend/src/app/result/data-resource-table.component.ts b/frontend/src/app/result/data-resource-table.component.ts index c97a5e3..828fb8e 100644 --- a/frontend/src/app/result/data-resource-table.component.ts +++ b/frontend/src/app/result/data-resource-table.component.ts @@ -25,9 +25,21 @@ import { RowOverflowDirective } from './row-overflow.directive'; } @case (ColumnType.Number) { - - {{ element[column.key] | number: '1.0-2' }} - + @if (column.displayKind) { + + + + {{ displayValue(column, element) }} + + + } @else { + + {{ formatValue(column, element) }} + + } } @case (ColumnType.Boolean) { @@ -94,6 +106,12 @@ import { RowOverflowDirective } from './row-overflow.directive'; padding-bottom: 1rem; vertical-align: top; /* Keeps text nicely aligned at the top during expansion */ + mat-chip:has(.color-dot) { + min-width: 5rem; + width: max-content; + justify-content: center; + } + .cell-content { max-height: calc(2 * 1.4em); /* Limits text to roughly 2 lines */ line-height: 1.4; @@ -104,6 +122,15 @@ import { RowOverflowDirective } from './row-overflow.directive'; transition: max-height 0.25s ease-out; } + + .color-dot { + display: inline-block; + width: 0.75rem; + height: 0.75rem; + margin-right: 0.5rem; + border: 1px solid var(--mat-sys-outline); + border-radius: 50%; + } } tr { @@ -164,6 +191,23 @@ export class DataResourceTableComponent { toggleRow(element: Row): void { this.expandedElement.set(this.isExpanded(element) ? null : element); } + + formatValue(column: Column, element: Row): string { + const value = element[column.key]; + if (typeof value !== 'number') return String(value); + return new Intl.NumberFormat('en', { maximumFractionDigits: 2 }).format(value); + } + + displayValue(column: Column, element: Row): string { + const label = column.labelField ? element[column.labelField] : undefined; + if (typeof label === 'string') return label; + return this.formatValue(column, element); + } + + displayColor(column: Column, element: Row): string { + const color = column.colorField ? element[column.colorField] : undefined; + return typeof color === 'string' ? color : 'transparent'; + } } export type Row = Record; @@ -173,8 +217,13 @@ export interface Column { key: string; type: ColumnType; isPrimaryKey: boolean; + displayKind?: DisplayKind; + labelField?: string; + colorField?: string; } +export type DisplayKind = 'riskProbability' | 'riskAnomaly'; + export enum ColumnType { String = 'string', Number = 'number', @@ -221,13 +270,18 @@ export function tableColumnInfoFromValue( ): Array { if (!('fields' in schema)) return []; - const fields = schema['fields'] as [ - { - name: string; - type?: 'string' | 'number' | 'integer' | 'boolean' | 'list'; - title?: string; - }, - ]; + const fields = schema['fields'] as Array<{ + name: string; + type?: 'string' | 'number' | 'integer' | 'boolean' | 'list'; + title?: string; + }>; + + const display = schema['biois'] as + | { + display?: Record; + hiddenFields?: string[]; + } + | undefined; const primaryKey = new Array(); if ('primaryKey' in schema) { @@ -241,14 +295,21 @@ export function tableColumnInfoFromValue( } } - return fields.map((field) => { - const sampleValue = data[0]?.[field.name]; - const columnType = columnTypeOfField(field.type, sampleValue); - return { - name: field.title ?? field.name, - key: field.name, - type: columnType, - isPrimaryKey: primaryKey.includes(field.name), - }; - }); + const hiddenFields = new Set(display?.hiddenFields ?? []); + return fields + .filter((field) => !hiddenFields.has(field.name)) + .map((field) => { + const sampleValue = data[0]?.[field.name]; + const columnType = columnTypeOfField(field.type, sampleValue); + const metadata = display?.display?.[field.name]; + return { + name: field.title ?? field.name, + key: field.name, + type: columnType, + isPrimaryKey: primaryKey.includes(field.name), + displayKind: metadata?.kind, + labelField: metadata?.labelField, + colorField: metadata?.colorField, + }; + }); } diff --git a/frontend/src/app/result/result.component.ts b/frontend/src/app/result/result.component.ts index d92f94d..0c26394 100644 --- a/frontend/src/app/result/result.component.ts +++ b/frontend/src/app/result/result.component.ts @@ -94,10 +94,11 @@ export class ResultComponent { .sort(([leftKey], [rightKey]) => leftKey.localeCompare(rightKey)) // TODO: get order from process description when available .map(([key, rawValue]) => { const value = fixDataValue(rawValue) as unknown; + const innerValue = value instanceof QualifiedInputValue ? (value.value as unknown) : value; return { key, - title: this.fieldName(key), - value: value instanceof QualifiedInputValue ? (value.value as unknown) : value, + title: dataResourceTitle(innerValue) ?? this.fieldName(key), + value: innerValue, type: this.typeOfValue(value), }; }); @@ -221,6 +222,14 @@ export class ResultComponent { } } +/** Returns the Data Resource name when a result value contains one. */ +function dataResourceTitle(value: unknown): string | undefined { + if (typeof value !== 'object' || value === null) return undefined; + if ('title' in value && typeof value.title === 'string') return value.title; + if ('name' in value && typeof value.name === 'string') return value.name; + return undefined; +} + enum ResultType { Boolean = 'boolean', Errors = 'errors', From 818613652a9e5b2b5acd92d783104e4fbd6c130e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johannes=20Dr=C3=B6nner?= Date: Fri, 2 Oct 2026 13:02:00 +0200 Subject: [PATCH 3/9] chore: regenerate OpenAPI spec and TypeScript client Picks up the climate-risk process, the corrected execution response shape (one data resource per computed scenario rather than the internal output struct) and the table schema profile endpoint. Generated with the current pipeline, so the unused-import cleanup and formatting are included. --- .../typescript/.openapi-generator/FILES | 6 + api-client/typescript/apis/ProcessesApi.ts | 107 +++++++++++ api-client/typescript/apis/ProfilesApi.ts | 97 ++++++++++ api-client/typescript/docs/ProcessesApi.md | 69 +++++++ api-client/typescript/docs/ProfilesApi.md | 50 +++++ api-client/typescript/index.ts | 1 + .../typescript/models/ClimateRiskInputs.ts | 89 +++++++++ .../models/ClimateRiskProcessParams.ts | 59 ++++++ .../models/ClimateRiskProcessResponses.ts | 69 +++++++ .../typescript/models/ClimateVariable.ts | 23 +++ .../typescript/models/ObjectSerializer.ts | 11 ++ api-client/typescript/models/all.ts | 4 + api-client/typescript/types/ObjectParamAPI.ts | 83 ++++++++ api-client/typescript/types/ObservableAPI.ts | 132 +++++++++++++ api-client/typescript/types/PromiseAPI.ts | 77 ++++++++ openapi.json | 178 ++++++++++++++++++ 16 files changed, 1055 insertions(+) create mode 100644 api-client/typescript/apis/ProfilesApi.ts create mode 100644 api-client/typescript/docs/ProfilesApi.md create mode 100644 api-client/typescript/models/ClimateRiskInputs.ts create mode 100644 api-client/typescript/models/ClimateRiskProcessParams.ts create mode 100644 api-client/typescript/models/ClimateRiskProcessResponses.ts create mode 100644 api-client/typescript/models/ClimateVariable.ts diff --git a/api-client/typescript/.openapi-generator/FILES b/api-client/typescript/.openapi-generator/FILES index ad2bed2..e53ea19 100644 --- a/api-client/typescript/.openapi-generator/FILES +++ b/api-client/typescript/.openapi-generator/FILES @@ -5,6 +5,7 @@ apis/CapabilitiesApi.ts apis/ContactApi.ts apis/DefaultApi.ts apis/ProcessesApi.ts +apis/ProfilesApi.ts apis/UserApi.ts apis/baseapi.ts apis/exception.ts @@ -14,6 +15,7 @@ docs/CapabilitiesApi.md docs/ContactApi.md docs/DefaultApi.md docs/ProcessesApi.md +docs/ProfilesApi.md docs/UserApi.md git_push.sh http/http.ts @@ -32,6 +34,10 @@ models/BiodiversitySiteProperties.ts models/BiodiversitySiteSpecification.ts models/BooleanField.ts models/BoundingBox.ts +models/ClimateRiskInputs.ts +models/ClimateRiskProcessParams.ts +models/ClimateRiskProcessResponses.ts +models/ClimateVariable.ts models/Conformance.ts models/Constraints.ts models/Constraints1.ts diff --git a/api-client/typescript/apis/ProcessesApi.ts b/api-client/typescript/apis/ProcessesApi.ts index ce04ceb..d9796cb 100644 --- a/api-client/typescript/apis/ProcessesApi.ts +++ b/api-client/typescript/apis/ProcessesApi.ts @@ -14,6 +14,8 @@ import { SecurityAuthentication } from "../auth/auth"; import { BiodiversitySensitiveAreasProcessOutputs } from "../models/BiodiversitySensitiveAreasProcessOutputs"; import { BiodiversitySensitiveAreasProcessParams } from "../models/BiodiversitySensitiveAreasProcessParams"; +import { ClimateRiskProcessParams } from "../models/ClimateRiskProcessParams"; +import { ClimateRiskProcessResponses } from "../models/ClimateRiskProcessResponses"; import { Exception } from "../models/Exception"; import { Execute } from "../models/Execute"; import { HabitatDistanceProcessOutputs } from "../models/HabitatDistanceProcessOutputs"; @@ -125,6 +127,61 @@ export class ProcessesApiRequestFactory extends BaseAPIRequestFactory { return requestContext; } + /** + * @param climateRiskProcessParams + */ + public async executeClimateRisk( + climateRiskProcessParams: ClimateRiskProcessParams, + _options?: Configuration, + ): Promise { + let _config = _options || this.configuration; + + // verify required parameter 'climateRiskProcessParams' is not null or undefined + if ( + climateRiskProcessParams === null || + climateRiskProcessParams === undefined + ) { + throw new RequiredError( + "ProcessesApi", + "executeClimateRisk", + "climateRiskProcessParams", + ); + } + + // Path Params + const localVarPath = "/processes/climate-risk/execution"; + + // Make Request Context + const requestContext = _config.baseServer.makeRequestContext( + localVarPath, + HttpMethod.POST, + ); + requestContext.setHeaderParam("Accept", "application/json, */*;q=0.8"); + + // Body Params + const contentType = ObjectSerializer.getPreferredMediaType([ + "application/json", + ]); + requestContext.setHeaderParam("Content-Type", contentType); + const serializedBody = ObjectSerializer.stringify( + ObjectSerializer.serialize( + climateRiskProcessParams, + "ClimateRiskProcessParams", + "", + ), + contentType, + ); + requestContext.setBody(serializedBody); + + const defaultAuth: SecurityAuthentication | undefined = + _config?.authMethods?.default; + if (defaultAuth?.applySecurityAuthentication) { + await defaultAuth?.applySecurityAuthentication(requestContext); + } + + return requestContext; + } + /** * @param habitatDistanceProcessParams */ @@ -644,6 +701,56 @@ export class ProcessesApiResponseProcessor { ); } + /** + * Unwraps the actual response sent by the server from the response context and deserializes the response content + * to the expected objects + * + * @params response Response returned by the server for a request to executeClimateRisk + * @throws ApiException if the response code was not in [200, 299] + */ + public async executeClimateRiskWithHttpInfo( + response: ResponseContext, + ): Promise> { + const contentType = ObjectSerializer.normalizeMediaType( + response.headers["content-type"], + ); + if (isCodeInRange("200", response.httpStatusCode)) { + const body: ClimateRiskProcessResponses = ObjectSerializer.deserialize( + ObjectSerializer.parse(await response.body.text(), contentType), + "ClimateRiskProcessResponses", + "", + ) as ClimateRiskProcessResponses; + return new HttpInfo( + response.httpStatusCode, + response.headers, + response.body, + body, + ); + } + + // Work around for missing responses in specification, e.g. for petstore.yaml + if (response.httpStatusCode >= 200 && response.httpStatusCode <= 299) { + const body: ClimateRiskProcessResponses = ObjectSerializer.deserialize( + ObjectSerializer.parse(await response.body.text(), contentType), + "ClimateRiskProcessResponses", + "", + ) as ClimateRiskProcessResponses; + return new HttpInfo( + response.httpStatusCode, + response.headers, + response.body, + body, + ); + } + + throw new ApiException( + response.httpStatusCode, + "Unknown API Status Code!", + await response.getBodyAsAny(), + response.headers, + ); + } + /** * Unwraps the actual response sent by the server from the response context and deserializes the response content * to the expected objects diff --git a/api-client/typescript/apis/ProfilesApi.ts b/api-client/typescript/apis/ProfilesApi.ts new file mode 100644 index 0000000..1b03cd9 --- /dev/null +++ b/api-client/typescript/apis/ProfilesApi.ts @@ -0,0 +1,97 @@ +// TODO: better import syntax? +import { BaseAPIRequestFactory } from "./baseapi"; +import { Configuration } from "../configuration"; +import { + RequestContext, + HttpMethod, + ResponseContext, + HttpInfo, +} from "../http/http"; +import { ObjectSerializer } from "../models/ObjectSerializer"; +import { ApiException } from "./exception"; +import { isCodeInRange } from "../util"; +import { SecurityAuthentication } from "../auth/auth"; + +/** + * no description + */ +export class ProfilesApiRequestFactory extends BaseAPIRequestFactory { + /** + * The profile of the `schema.biois` extension: a Table Schema extension that declares per-column display metadata for `BioIS` result tables. + */ + public async climateRiskTableSchemaProfile( + _options?: Configuration, + ): Promise { + let _config = _options || this.configuration; + + // Path Params + const localVarPath = "/profiles/table-schema/climate-risk/1.0/schema.json"; + + // Make Request Context + const requestContext = _config.baseServer.makeRequestContext( + localVarPath, + HttpMethod.GET, + ); + requestContext.setHeaderParam("Accept", "application/json, */*;q=0.8"); + + const defaultAuth: SecurityAuthentication | undefined = + _config?.authMethods?.default; + if (defaultAuth?.applySecurityAuthentication) { + await defaultAuth?.applySecurityAuthentication(requestContext); + } + + return requestContext; + } +} + +export class ProfilesApiResponseProcessor { + /** + * Unwraps the actual response sent by the server from the response context and deserializes the response content + * to the expected objects + * + * @params response Response returned by the server for a request to climateRiskTableSchemaProfile + * @throws ApiException if the response code was not in [200, 299] + */ + public async climateRiskTableSchemaProfileWithHttpInfo( + response: ResponseContext, + ): Promise> { + const contentType = ObjectSerializer.normalizeMediaType( + response.headers["content-type"], + ); + if (isCodeInRange("200", response.httpStatusCode)) { + const body: string = ObjectSerializer.deserialize( + ObjectSerializer.parse(await response.body.text(), contentType), + "string", + "", + ) as string; + return new HttpInfo( + response.httpStatusCode, + response.headers, + response.body, + body, + ); + } + + // Work around for missing responses in specification, e.g. for petstore.yaml + if (response.httpStatusCode >= 200 && response.httpStatusCode <= 299) { + const body: string = ObjectSerializer.deserialize( + ObjectSerializer.parse(await response.body.text(), contentType), + "string", + "", + ) as string; + return new HttpInfo( + response.httpStatusCode, + response.headers, + response.body, + body, + ); + } + + throw new ApiException( + response.httpStatusCode, + "Unknown API Status Code!", + await response.getBodyAsAny(), + response.headers, + ); + } +} diff --git a/api-client/typescript/docs/ProcessesApi.md b/api-client/typescript/docs/ProcessesApi.md index c11b820..cf7f864 100644 --- a/api-client/typescript/docs/ProcessesApi.md +++ b/api-client/typescript/docs/ProcessesApi.md @@ -6,6 +6,7 @@ All URIs are relative to *http://localhost* | ------------------------------------------------------------------------------------------ | ---------------------------------------------------------- | ------------------------------------------- | | [**\_delete**](ProcessesApi.md#_delete) | **DELETE** /jobs/{jobId} | Cancel a job execution, remove finished job | | [**executeBiodiversitySensitiveAreas**](ProcessesApi.md#executeBiodiversitySensitiveAreas) | **POST** /processes/biodiversity-sensitive-areas/execution | +| [**executeClimateRisk**](ProcessesApi.md#executeClimateRisk) | **POST** /processes/climate-risk/execution | | [**executeHabitatDistance**](ProcessesApi.md#executeHabitatDistance) | **POST** /processes/habitatDistance/execution | | [**executeLandUseSealedArea**](ProcessesApi.md#executeLandUseSealedArea) | **POST** /processes/land-use-sealed-area/execution | | [**executeNdvi**](ProcessesApi.md#executeNdvi) | **POST** /processes/ndvi/execution | @@ -126,6 +127,74 @@ No authorization required [[Back to top]](#) [[Back to API list]](README.md#documentation-for-api-endpoints) [[Back to Model list]](README.md#documentation-for-models) [[Back to README]](README.md) +# **executeClimateRisk** + +> ClimateRiskProcessResponses executeClimateRisk(climateRiskProcessParams) + +### Example + +```typescript +import { createConfiguration, ProcessesApi } from ""; +import type { ProcessesApiExecuteClimateRiskRequest } from ""; + +const configuration = createConfiguration(); +const apiInstance = new ProcessesApi(configuration); + +const request: ProcessesApiExecuteClimateRiskRequest = { + climateRiskProcessParams: { + inputs: { + coordinate: { + value: { + type: "Point", + coordinates: [3.14], + bbox: [3.14], + }, + mediaType: "application/geo+json", + }, + yearBegin: 0, + yearRange: 0, + referenceYearBegin: 0, + variables: ["heatDays"], + models: ["models_example"], + }, + outputs: { + key: null, + }, + response: "raw", + }, +}; + +const data = await apiInstance.executeClimateRisk(request); +console.log("API called successfully. Returned data:", data); +``` + +### Parameters + +| Name | Type | Description | Notes | +| ---------------------------- | ---------------------------- | ----------- | ----- | +| **climateRiskProcessParams** | **ClimateRiskProcessParams** | | + +### Return type + +**ClimateRiskProcessResponses** + +### Authorization + +No authorization required + +### HTTP request headers + +- **Content-Type**: application/json +- **Accept**: application/json + +### HTTP response details + +| Status code | Description | Response headers | +| ----------- | ----------- | ---------------- | +| **200** | | - | + +[[Back to top]](#) [[Back to API list]](README.md#documentation-for-api-endpoints) [[Back to Model list]](README.md#documentation-for-models) [[Back to README]](README.md) + # **executeHabitatDistance** > HabitatDistanceProcessOutputs executeHabitatDistance(habitatDistanceProcessParams) diff --git a/api-client/typescript/docs/ProfilesApi.md b/api-client/typescript/docs/ProfilesApi.md new file mode 100644 index 0000000..5a7ab62 --- /dev/null +++ b/api-client/typescript/docs/ProfilesApi.md @@ -0,0 +1,50 @@ +# .ProfilesApi + +All URIs are relative to *http://localhost* + +| Method | HTTP request | Description | +| --------------------------------------------------------------------------------- | ----------------------------------------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| [**climateRiskTableSchemaProfile**](ProfilesApi.md#climateRiskTableSchemaProfile) | **GET** /profiles/table-schema/climate-risk/1.0/schema.json | The profile of the `schema.biois` extension: a Table Schema extension that declares per-column display metadata for `BioIS` result tables. | + +# **climateRiskTableSchemaProfile** + +> string climateRiskTableSchemaProfile() + +### Example + +```typescript +import { createConfiguration, ProfilesApi } from ""; + +const configuration = createConfiguration(); +const apiInstance = new ProfilesApi(configuration); + +const request = {}; + +const data = await apiInstance.climateRiskTableSchemaProfile(request); +console.log("API called successfully. Returned data:", data); +``` + +### Parameters + +This endpoint does not need any parameter. + +### Return type + +**string** + +### Authorization + +No authorization required + +### HTTP request headers + +- **Content-Type**: Not defined +- **Accept**: application/schema+json + +### HTTP response details + +| Status code | Description | Response headers | +| ----------- | ------------------------------- | ---------------- | +| **200** | The BioIS table schema profile. | - | + +[[Back to top]](#) [[Back to API list]](README.md#documentation-for-api-endpoints) [[Back to Model list]](README.md#documentation-for-models) [[Back to README]](README.md) diff --git a/api-client/typescript/index.ts b/api-client/typescript/index.ts index d0500c8..9b62338 100644 --- a/api-client/typescript/index.ts +++ b/api-client/typescript/index.ts @@ -21,5 +21,6 @@ export { PromiseContactApi as ContactApi, PromiseDefaultApi as DefaultApi, PromiseProcessesApi as ProcessesApi, + PromiseProfilesApi as ProfilesApi, PromiseUserApi as UserApi, } from "./types/PromiseAPI"; diff --git a/api-client/typescript/models/ClimateRiskInputs.ts b/api-client/typescript/models/ClimateRiskInputs.ts new file mode 100644 index 0000000..e2eb801 --- /dev/null +++ b/api-client/typescript/models/ClimateRiskInputs.ts @@ -0,0 +1,89 @@ +/** + * BioIS API + * API for the BioIS service, providing access to geospatial processing and job management. + * + * OpenAPI spec version: 0.2.0 + * Contact: info@geoengine.de + * + * NOTE: This class is auto generated by OpenAPI Generator (https://openapi-generator.tech). + * https://openapi-generator.tech + * Do not edit the class manually. + */ + +import { ClimateVariable } from "../models/ClimateVariable"; +import { PointGeoJsonInput } from "../models/PointGeoJsonInput"; + +/** + * User-supplied inputs for the climate risk process. + */ +export class ClimateRiskInputs { + "coordinate": PointGeoJsonInput; + /** + * Year of reporting or change (e.g., 2023, 2024, etc.) + */ + "yearBegin"?: number; + /** + * Length of a time window in years (e.g., 5 years). + */ + "yearRange"?: number; + /** + * Year of reporting or change (e.g., 2023, 2024, etc.) + */ + "referenceYearBegin": number; + "variables"?: Array; + "models"?: Array; + + static readonly discriminator: string | undefined = undefined; + + static readonly mapping: { [index: string]: string } | undefined = undefined; + + static readonly attributeTypeMap: Array<{ + name: string; + baseName: string; + type: string; + format: string; + }> = [ + { + name: "coordinate", + baseName: "coordinate", + type: "PointGeoJsonInput", + format: "", + }, + { + name: "yearBegin", + baseName: "yearBegin", + type: "number", + format: "int32", + }, + { + name: "yearRange", + baseName: "yearRange", + type: "number", + format: "int32", + }, + { + name: "referenceYearBegin", + baseName: "referenceYearBegin", + type: "number", + format: "int32", + }, + { + name: "variables", + baseName: "variables", + type: "Array", + format: "", + }, + { + name: "models", + baseName: "models", + type: "Array", + format: "", + }, + ]; + + static getAttributeTypeMap() { + return ClimateRiskInputs.attributeTypeMap; + } + + public constructor() {} +} diff --git a/api-client/typescript/models/ClimateRiskProcessParams.ts b/api-client/typescript/models/ClimateRiskProcessParams.ts new file mode 100644 index 0000000..58a0aff --- /dev/null +++ b/api-client/typescript/models/ClimateRiskProcessParams.ts @@ -0,0 +1,59 @@ +/** + * BioIS API + * API for the BioIS service, providing access to geospatial processing and job management. + * + * OpenAPI spec version: 0.2.0 + * Contact: info@geoengine.de + * + * NOTE: This class is auto generated by OpenAPI Generator (https://openapi-generator.tech). + * https://openapi-generator.tech + * Do not edit the class manually. + */ + +import { ClimateRiskInputs } from "../models/ClimateRiskInputs"; +import { Response } from "../models/Response"; + +/** + * Process execution (Climate Risk) + */ +export class ClimateRiskProcessParams { + "inputs": ClimateRiskInputs; + "outputs"?: { [key: string]: any }; + "response"?: Response; + + static readonly discriminator: string | undefined = undefined; + + static readonly mapping: { [index: string]: string } | undefined = undefined; + + static readonly attributeTypeMap: Array<{ + name: string; + baseName: string; + type: string; + format: string; + }> = [ + { + name: "inputs", + baseName: "inputs", + type: "ClimateRiskInputs", + format: "", + }, + { + name: "outputs", + baseName: "outputs", + type: "{ [key: string]: any; }", + format: "", + }, + { + name: "response", + baseName: "response", + type: "Response", + format: "", + }, + ]; + + static getAttributeTypeMap() { + return ClimateRiskProcessParams.attributeTypeMap; + } + + public constructor() {} +} diff --git a/api-client/typescript/models/ClimateRiskProcessResponses.ts b/api-client/typescript/models/ClimateRiskProcessResponses.ts new file mode 100644 index 0000000..b46a11f --- /dev/null +++ b/api-client/typescript/models/ClimateRiskProcessResponses.ts @@ -0,0 +1,69 @@ +/** + * BioIS API + * API for the BioIS service, providing access to geospatial processing and job management. + * + * OpenAPI spec version: 0.2.0 + * Contact: info@geoengine.de + * + * NOTE: This class is auto generated by OpenAPI Generator (https://openapi-generator.tech). + * https://openapi-generator.tech + * Do not edit the class manually. + */ + +import { ClimateRiskInputs } from "../models/ClimateRiskInputs"; +import { DataResource } from "../models/DataResource"; + +/** + * Response body of a climate-risk execution: one summary table per computed scenario, plus the optional `inputs` echo and `rawEnsembleData`. Which scenario keys are present depends on the requested outputs and on the model registry (`conf/nexgddp_models.toml`); the process description\'s `outputs` is the authoritative list for a given deployment. + */ +export class ClimateRiskProcessResponses { + "ssp245"?: DataResource | null; + "ssp585"?: DataResource | null; + "rawEnsembleData"?: DataResource | null; + /** + * The submitted inputs, echoed back. + */ + "inputs"?: ClimateRiskInputs | null; + + static readonly discriminator: string | undefined = undefined; + + static readonly mapping: { [index: string]: string } | undefined = undefined; + + static readonly attributeTypeMap: Array<{ + name: string; + baseName: string; + type: string; + format: string; + }> = [ + { + name: "ssp245", + baseName: "ssp245", + type: "DataResource", + format: "", + }, + { + name: "ssp585", + baseName: "ssp585", + type: "DataResource", + format: "", + }, + { + name: "rawEnsembleData", + baseName: "rawEnsembleData", + type: "DataResource", + format: "", + }, + { + name: "inputs", + baseName: "inputs", + type: "ClimateRiskInputs", + format: "", + }, + ]; + + static getAttributeTypeMap() { + return ClimateRiskProcessResponses.attributeTypeMap; + } + + public constructor() {} +} diff --git a/api-client/typescript/models/ClimateVariable.ts b/api-client/typescript/models/ClimateVariable.ts new file mode 100644 index 0000000..35520da --- /dev/null +++ b/api-client/typescript/models/ClimateVariable.ts @@ -0,0 +1,23 @@ +/** + * BioIS API + * API for the BioIS service, providing access to geospatial processing and job management. + * + * OpenAPI spec version: 0.2.0 + * Contact: info@geoengine.de + * + * NOTE: This class is auto generated by OpenAPI Generator (https://openapi-generator.tech). + * https://openapi-generator.tech + * Do not edit the class manually. + */ + +/** + * Climate variable to compute (WMO-based daily threshold indicators). + */ +export enum ClimateVariable { + HeatDays = "heatDays", + IceDays = "iceDays", + TropicalNights = "tropicalNights", + FrostDays = "frostDays", + DryDays = "dryDays", + HeavyRainDays = "heavyRainDays", +} diff --git a/api-client/typescript/models/ObjectSerializer.ts b/api-client/typescript/models/ObjectSerializer.ts index e8d2a7b..c0262a7 100644 --- a/api-client/typescript/models/ObjectSerializer.ts +++ b/api-client/typescript/models/ObjectSerializer.ts @@ -10,6 +10,10 @@ export * from "../models/BiodiversitySiteProperties"; export * from "../models/BiodiversitySiteSpecification"; export * from "../models/BooleanField"; export * from "../models/BoundingBox"; +export * from "../models/ClimateRiskInputs"; +export * from "../models/ClimateRiskProcessParams"; +export * from "../models/ClimateRiskProcessResponses"; +export * from "../models/ClimateVariable"; export * from "../models/Conformance"; export * from "../models/Constraints"; export * from "../models/Constraints1"; @@ -141,6 +145,9 @@ import { BiodiversitySensitiveAreasProcessParams } from "../models/BiodiversityS import { BiodiversitySiteProperties } from "../models/BiodiversitySiteProperties"; import { BooleanField } from "../models/BooleanField"; import { BoundingBox } from "../models/BoundingBox"; +import { ClimateRiskInputs } from "../models/ClimateRiskInputs"; +import { ClimateRiskProcessParams } from "../models/ClimateRiskProcessParams"; +import { ClimateRiskProcessResponses } from "../models/ClimateRiskProcessResponses"; import { Conformance } from "../models/Conformance"; import { Constraints } from "../models/Constraints"; import { Constraints1 } from "../models/Constraints1"; @@ -268,6 +275,7 @@ let enumsMap: Set = new Set([ "BiodiversitySiteSpecification", "BooleanFieldTypeEnum", "BooleanFieldFormatEnum", + "ClimateVariable", "ContactRole", "DateFieldTypeEnum", "DateTimeFieldTypeEnum", @@ -333,6 +341,9 @@ let typeMap: { [index: string]: any } = { BiodiversitySiteProperties: BiodiversitySiteProperties, BooleanField: BooleanField, BoundingBox: BoundingBox, + ClimateRiskInputs: ClimateRiskInputs, + ClimateRiskProcessParams: ClimateRiskProcessParams, + ClimateRiskProcessResponses: ClimateRiskProcessResponses, Conformance: Conformance, Constraints: Constraints, Constraints1: Constraints1, diff --git a/api-client/typescript/models/all.ts b/api-client/typescript/models/all.ts index c2ca8f8..0d4e21c 100644 --- a/api-client/typescript/models/all.ts +++ b/api-client/typescript/models/all.ts @@ -10,6 +10,10 @@ export * from "../models/BiodiversitySiteProperties"; export * from "../models/BiodiversitySiteSpecification"; export * from "../models/BooleanField"; export * from "../models/BoundingBox"; +export * from "../models/ClimateRiskInputs"; +export * from "../models/ClimateRiskProcessParams"; +export * from "../models/ClimateRiskProcessResponses"; +export * from "../models/ClimateVariable"; export * from "../models/Conformance"; export * from "../models/Constraints"; export * from "../models/Constraints1"; diff --git a/api-client/typescript/types/ObjectParamAPI.ts b/api-client/typescript/types/ObjectParamAPI.ts index 45f3a76..3094309 100644 --- a/api-client/typescript/types/ObjectParamAPI.ts +++ b/api-client/typescript/types/ObjectParamAPI.ts @@ -4,6 +4,8 @@ import { Configuration, ConfigurationOptions } from "../configuration"; import { AuthCodeResponse } from "../models/AuthCodeResponse"; import { BiodiversitySensitiveAreasProcessOutputs } from "../models/BiodiversitySensitiveAreasProcessOutputs"; import { BiodiversitySensitiveAreasProcessParams } from "../models/BiodiversitySensitiveAreasProcessParams"; +import { ClimateRiskProcessParams } from "../models/ClimateRiskProcessParams"; +import { ClimateRiskProcessResponses } from "../models/ClimateRiskProcessResponses"; import { Conformance } from "../models/Conformance"; import { ContactRequest } from "../models/ContactRequest"; import { Execute } from "../models/Execute"; @@ -244,6 +246,15 @@ export interface ProcessesApiExecuteBiodiversitySensitiveAreasRequest { biodiversitySensitiveAreasProcessParams: BiodiversitySensitiveAreasProcessParams; } +export interface ProcessesApiExecuteClimateRiskRequest { + /** + * + * @type ClimateRiskProcessParams + * @memberof ProcessesApiexecuteClimateRisk + */ + climateRiskProcessParams: ClimateRiskProcessParams; +} + export interface ProcessesApiExecuteHabitatDistanceRequest { /** * @@ -407,6 +418,30 @@ export class ObjectProcessesApi { .toPromise(); } + /** + * @param param the request object + */ + public executeClimateRiskWithHttpInfo( + param: ProcessesApiExecuteClimateRiskRequest, + options?: ConfigurationOptions, + ): Promise> { + return this.api + .executeClimateRiskWithHttpInfo(param.climateRiskProcessParams, options) + .toPromise(); + } + + /** + * @param param the request object + */ + public executeClimateRisk( + param: ProcessesApiExecuteClimateRiskRequest, + options?: ConfigurationOptions, + ): Promise { + return this.api + .executeClimateRisk(param.climateRiskProcessParams, options) + .toPromise(); + } + /** * @param param the request object */ @@ -634,6 +669,54 @@ export class ObjectProcessesApi { } } +import { ObservableProfilesApi } from "./ObservableAPI"; +import { + ProfilesApiRequestFactory, + ProfilesApiResponseProcessor, +} from "../apis/ProfilesApi"; + +export interface ProfilesApiClimateRiskTableSchemaProfileRequest {} + +export class ObjectProfilesApi { + private api: ObservableProfilesApi; + + public constructor( + configuration: Configuration, + requestFactory?: ProfilesApiRequestFactory, + responseProcessor?: ProfilesApiResponseProcessor, + ) { + this.api = new ObservableProfilesApi( + configuration, + requestFactory, + responseProcessor, + ); + } + + /** + * The profile of the `schema.biois` extension: a Table Schema extension that declares per-column display metadata for `BioIS` result tables. + * @param param the request object + */ + public climateRiskTableSchemaProfileWithHttpInfo( + param: ProfilesApiClimateRiskTableSchemaProfileRequest = {}, + options?: ConfigurationOptions, + ): Promise> { + return this.api + .climateRiskTableSchemaProfileWithHttpInfo(options) + .toPromise(); + } + + /** + * The profile of the `schema.biois` extension: a Table Schema extension that declares per-column display metadata for `BioIS` result tables. + * @param param the request object + */ + public climateRiskTableSchemaProfile( + param: ProfilesApiClimateRiskTableSchemaProfileRequest = {}, + options?: ConfigurationOptions, + ): Promise { + return this.api.climateRiskTableSchemaProfile(options).toPromise(); + } +} + import { ObservableUserApi } from "./ObservableAPI"; import { UserApiRequestFactory, diff --git a/api-client/typescript/types/ObservableAPI.ts b/api-client/typescript/types/ObservableAPI.ts index aefe639..4aba0d8 100644 --- a/api-client/typescript/types/ObservableAPI.ts +++ b/api-client/typescript/types/ObservableAPI.ts @@ -9,6 +9,8 @@ import { mergeMap, map } from "../rxjsStub"; import { AuthCodeResponse } from "../models/AuthCodeResponse"; import { BiodiversitySensitiveAreasProcessOutputs } from "../models/BiodiversitySensitiveAreasProcessOutputs"; import { BiodiversitySensitiveAreasProcessParams } from "../models/BiodiversitySensitiveAreasProcessParams"; +import { ClimateRiskProcessParams } from "../models/ClimateRiskProcessParams"; +import { ClimateRiskProcessResponses } from "../models/ClimateRiskProcessResponses"; import { Conformance } from "../models/Conformance"; import { ContactRequest } from "../models/ContactRequest"; import { Execute } from "../models/Execute"; @@ -466,6 +468,64 @@ export class ObservableProcessesApi { ); } + /** + * @param climateRiskProcessParams + */ + public executeClimateRiskWithHttpInfo( + climateRiskProcessParams: ClimateRiskProcessParams, + _options?: ConfigurationOptions, + ): Observable> { + const _config = mergeConfiguration(this.configuration, _options); + + const requestContextPromise = this.requestFactory.executeClimateRisk( + climateRiskProcessParams, + _config, + ); + // build promise chain + let middlewarePreObservable = from(requestContextPromise); + for (const middleware of _config.middleware) { + middlewarePreObservable = middlewarePreObservable.pipe( + mergeMap((ctx: RequestContext) => middleware.pre(ctx)), + ); + } + + return middlewarePreObservable + .pipe(mergeMap((ctx: RequestContext) => _config.httpApi.send(ctx))) + .pipe( + mergeMap((response: ResponseContext) => { + let middlewarePostObservable = of(response); + for (const middleware of _config.middleware.reverse()) { + middlewarePostObservable = middlewarePostObservable.pipe( + mergeMap((rsp: ResponseContext) => middleware.post(rsp)), + ); + } + return middlewarePostObservable.pipe( + map((rsp: ResponseContext) => + this.responseProcessor.executeClimateRiskWithHttpInfo(rsp), + ), + ); + }), + ); + } + + /** + * @param climateRiskProcessParams + */ + public executeClimateRisk( + climateRiskProcessParams: ClimateRiskProcessParams, + _options?: ConfigurationOptions, + ): Observable { + return this.executeClimateRiskWithHttpInfo( + climateRiskProcessParams, + _options, + ).pipe( + map( + (apiResponse: HttpInfo) => + apiResponse.data, + ), + ); + } + /** * @param habitatDistanceProcessParams */ @@ -966,6 +1026,78 @@ export class ObservableProcessesApi { } } +import { + ProfilesApiRequestFactory, + ProfilesApiResponseProcessor, +} from "../apis/ProfilesApi"; +export class ObservableProfilesApi { + private requestFactory: ProfilesApiRequestFactory; + private responseProcessor: ProfilesApiResponseProcessor; + private configuration: Configuration; + + public constructor( + configuration: Configuration, + requestFactory?: ProfilesApiRequestFactory, + responseProcessor?: ProfilesApiResponseProcessor, + ) { + this.configuration = configuration; + this.requestFactory = + requestFactory || new ProfilesApiRequestFactory(configuration); + this.responseProcessor = + responseProcessor || new ProfilesApiResponseProcessor(); + } + + /** + * The profile of the `schema.biois` extension: a Table Schema extension that declares per-column display metadata for `BioIS` result tables. + */ + public climateRiskTableSchemaProfileWithHttpInfo( + _options?: ConfigurationOptions, + ): Observable> { + const _config = mergeConfiguration(this.configuration, _options); + + const requestContextPromise = + this.requestFactory.climateRiskTableSchemaProfile(_config); + // build promise chain + let middlewarePreObservable = from(requestContextPromise); + for (const middleware of _config.middleware) { + middlewarePreObservable = middlewarePreObservable.pipe( + mergeMap((ctx: RequestContext) => middleware.pre(ctx)), + ); + } + + return middlewarePreObservable + .pipe(mergeMap((ctx: RequestContext) => _config.httpApi.send(ctx))) + .pipe( + mergeMap((response: ResponseContext) => { + let middlewarePostObservable = of(response); + for (const middleware of _config.middleware.reverse()) { + middlewarePostObservable = middlewarePostObservable.pipe( + mergeMap((rsp: ResponseContext) => middleware.post(rsp)), + ); + } + return middlewarePostObservable.pipe( + map((rsp: ResponseContext) => + this.responseProcessor.climateRiskTableSchemaProfileWithHttpInfo( + rsp, + ), + ), + ); + }), + ); + } + + /** + * The profile of the `schema.biois` extension: a Table Schema extension that declares per-column display metadata for `BioIS` result tables. + */ + public climateRiskTableSchemaProfile( + _options?: ConfigurationOptions, + ): Observable { + return this.climateRiskTableSchemaProfileWithHttpInfo(_options).pipe( + map((apiResponse: HttpInfo) => apiResponse.data), + ); + } +} + import { UserApiRequestFactory, UserApiResponseProcessor, diff --git a/api-client/typescript/types/PromiseAPI.ts b/api-client/typescript/types/PromiseAPI.ts index 9dfd8bd..0c0b89d 100644 --- a/api-client/typescript/types/PromiseAPI.ts +++ b/api-client/typescript/types/PromiseAPI.ts @@ -8,6 +8,8 @@ import { import { AuthCodeResponse } from "../models/AuthCodeResponse"; import { BiodiversitySensitiveAreasProcessOutputs } from "../models/BiodiversitySensitiveAreasProcessOutputs"; import { BiodiversitySensitiveAreasProcessParams } from "../models/BiodiversitySensitiveAreasProcessParams"; +import { ClimateRiskProcessParams } from "../models/ClimateRiskProcessParams"; +import { ClimateRiskProcessResponses } from "../models/ClimateRiskProcessResponses"; import { Conformance } from "../models/Conformance"; import { ContactRequest } from "../models/ContactRequest"; import { Execute } from "../models/Execute"; @@ -285,6 +287,36 @@ export class PromiseProcessesApi { return result.toPromise(); } + /** + * @param climateRiskProcessParams + */ + public executeClimateRiskWithHttpInfo( + climateRiskProcessParams: ClimateRiskProcessParams, + _options?: PromiseConfigurationOptions, + ): Promise> { + const observableOptions = wrapOptions(_options); + const result = this.api.executeClimateRiskWithHttpInfo( + climateRiskProcessParams, + observableOptions, + ); + return result.toPromise(); + } + + /** + * @param climateRiskProcessParams + */ + public executeClimateRisk( + climateRiskProcessParams: ClimateRiskProcessParams, + _options?: PromiseConfigurationOptions, + ): Promise { + const observableOptions = wrapOptions(_options); + const result = this.api.executeClimateRisk( + climateRiskProcessParams, + observableOptions, + ); + return result.toPromise(); + } + /** * @param habitatDistanceProcessParams */ @@ -549,6 +581,51 @@ export class PromiseProcessesApi { } } +import { ObservableProfilesApi } from "./ObservableAPI"; + +import { + ProfilesApiRequestFactory, + ProfilesApiResponseProcessor, +} from "../apis/ProfilesApi"; +export class PromiseProfilesApi { + private api: ObservableProfilesApi; + + public constructor( + configuration: Configuration, + requestFactory?: ProfilesApiRequestFactory, + responseProcessor?: ProfilesApiResponseProcessor, + ) { + this.api = new ObservableProfilesApi( + configuration, + requestFactory, + responseProcessor, + ); + } + + /** + * The profile of the `schema.biois` extension: a Table Schema extension that declares per-column display metadata for `BioIS` result tables. + */ + public climateRiskTableSchemaProfileWithHttpInfo( + _options?: PromiseConfigurationOptions, + ): Promise> { + const observableOptions = wrapOptions(_options); + const result = + this.api.climateRiskTableSchemaProfileWithHttpInfo(observableOptions); + return result.toPromise(); + } + + /** + * The profile of the `schema.biois` extension: a Table Schema extension that declares per-column display metadata for `BioIS` result tables. + */ + public climateRiskTableSchemaProfile( + _options?: PromiseConfigurationOptions, + ): Promise { + const observableOptions = wrapOptions(_options); + const result = this.api.climateRiskTableSchemaProfile(observableOptions); + return result.toPromise(); + } +} + import { ObservableUserApi } from "./ObservableAPI"; import { diff --git a/openapi.json b/openapi.json index 8114258..541a160 100644 --- a/openapi.json +++ b/openapi.json @@ -763,6 +763,27 @@ } } }, + "/profiles/table-schema/climate-risk/1.0/schema.json": { + "get": { + "tags": [ + "Profiles" + ], + "summary": "The profile of the `schema.biois` extension: a Table Schema extension that declares per-column\ndisplay metadata for `BioIS` result tables.", + "operationId": "climate_risk_table_schema_profile", + "responses": { + "200": { + "description": "The BioIS table schema profile.", + "content": { + "application/schema+json": { + "schema": { + "type": "string" + } + } + } + } + } + } + }, "/processes/ndvi/execution": { "post": { "tags": [ @@ -853,6 +874,36 @@ } } }, + "/processes/climate-risk/execution": { + "post": { + "tags": [ + "Processes" + ], + "operationId": "execute_climate_risk", + "requestBody": { + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ClimateRiskProcessParams" + } + } + }, + "required": true + }, + "responses": { + "200": { + "description": "", + "content": { + "application/json": { + "schema": { + "$ref": "#/components/schemas/ClimateRiskProcessResponses" + } + } + } + } + } + } + }, "/processes/land-use-sealed-area/execution": { "post": { "tags": [ @@ -1099,6 +1150,127 @@ } } }, + "ClimateRiskInputs": { + "type": "object", + "description": "User-supplied inputs for the climate risk process.", + "required": [ + "coordinate", + "referenceYearBegin" + ], + "properties": { + "coordinate": { + "$ref": "#/components/schemas/PointGeoJsonInput" + }, + "yearBegin": { + "$ref": "#/components/schemas/Year" + }, + "yearRange": { + "$ref": "#/components/schemas/YearRange" + }, + "referenceYearBegin": { + "$ref": "#/components/schemas/Year" + }, + "variables": { + "type": "array", + "items": { + "$ref": "#/components/schemas/ClimateVariable" + } + }, + "models": { + "type": "array", + "items": { + "type": "string" + } + } + } + }, + "ClimateRiskProcessParams": { + "type": "object", + "description": "Process execution (Climate Risk)", + "required": [ + "inputs" + ], + "properties": { + "inputs": { + "$ref": "#/components/schemas/ClimateRiskInputs" + }, + "outputs": { + "type": "object", + "additionalProperties": { + "default": null + }, + "propertyNames": { + "type": "string" + } + }, + "response": { + "$ref": "#/components/schemas/Response" + } + } + }, + "ClimateRiskProcessResponses": { + "type": "object", + "description": "Response body of a climate-risk execution: one summary table per computed scenario, plus the\noptional `inputs` echo and `rawEnsembleData`. Which scenario keys are present depends on the\nrequested outputs and on the model registry (`conf/nexgddp_models.toml`); the process\ndescription's `outputs` is the authoritative list for a given deployment.", + "properties": { + "ssp245": { + "oneOf": [ + { + "type": "null" + }, + { + "$ref": "https://datapackage.org/profiles/2.0/dataresource.json" + } + ], + "description": "Summary table for SSP2-4.5." + }, + "ssp585": { + "oneOf": [ + { + "type": "null" + }, + { + "$ref": "https://datapackage.org/profiles/2.0/dataresource.json" + } + ], + "description": "Summary table for SSP5-8.5." + }, + "rawEnsembleData": { + "oneOf": [ + { + "type": "null" + }, + { + "$ref": "https://datapackage.org/profiles/2.0/dataresource.json" + } + ], + "description": "Per-model raw values for each variable x scenario combination." + }, + "inputs": { + "oneOf": [ + { + "type": "null" + }, + { + "$ref": "#/components/schemas/ClimateRiskInputs", + "description": "The submitted inputs, echoed back." + } + ] + } + } + }, + "ClimateVariable": { + "type": "string", + "title": "ClimateVariable", + "description": "Climate variable to compute (WMO-based daily threshold indicators).", + "enum": [ + "heatDays", + "iceDays", + "tropicalNights", + "frostDays", + "dryDays", + "heavyRainDays" + ] + }, "Conformance": { "type": "object", "description": "The Conformance declaration states the conformance classes from standards or community\nspecifications, identified by a URI, that the API conforms to. Clients can but are not\nrequired to use this information. Accessing the Conformance declaration using HTTP GET\nreturns the list of URIs of conformance classes implemented by the server.", @@ -2379,6 +2551,12 @@ "format": "int32", "description": "Year of reporting or change (e.g., 2023, 2024, etc.)", "minimum": 0 + }, + "YearRange": { + "type": "integer", + "format": "int32", + "description": "Length of a time window in years (e.g., 5 years).", + "minimum": 0 } } } From 6f89eb58112d8e9e74a23f2b0cc6de560cc66792 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johannes=20Dr=C3=B6nner?= Date: Fri, 2 Oct 2026 13:02:01 +0200 Subject: [PATCH 4/9] chore(notebooks): drive the NEX-GDDP ingest from the model registry MODELS and SCENARIOS duplicated the backend registry and had already drifted, so the notebook now parses id/variant/grid/scenarios from `conf/nexgddp_models.full.toml` and the low-traffic subset is one DEV_ONLY flag instead of silent slices. Drops the dead kerchunk imports, the now-vacuous registry assertion, and a "Next Steps" section that still referred to reference files; the ingestion builds GDAL VRTs over the yearly NetCDF files instead. --- .gitignore | 9 + notebooks/nexgddp_cmip6_ingest.ipynb | 698 +++++++++++++++++++++++++++ 2 files changed, 707 insertions(+) create mode 100644 notebooks/nexgddp_cmip6_ingest.ipynb diff --git a/.gitignore b/.gitignore index 5aa034a..47de64c 100644 --- a/.gitignore +++ b/.gitignore @@ -26,3 +26,12 @@ Settings.toml api-client/typescript/node_modules api-client/typescript/package-lock.json + +# Local review/plan notes +PLAN-review-comments.md + +# Local run logs +RUN_LOG.md + +# Frontend test coverage output +frontend/coverage/ diff --git a/notebooks/nexgddp_cmip6_ingest.ipynb b/notebooks/nexgddp_cmip6_ingest.ipynb new file mode 100644 index 0000000..eacf44c --- /dev/null +++ b/notebooks/nexgddp_cmip6_ingest.ipynb @@ -0,0 +1,698 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# NEX-GDDP-CMIP6 Dataset Ingestion\n", + "\n", + "This notebook builds GDAL VRTs over the yearly NEX-GDDP-CMIP6 NetCDF files on S3 and registers them as GeoEngine datasets. Each year has one raster band per day; the VRT concatenates them so band *i* == day *i* of the period, readable via NetCDF-CF metadata without kerchunk or a GDAL >= 3.11.\n", + "\n", + "## Prerequisites\n", + "\n", + "- Python 3.10+\n", + "- Required packages: `s3fs`, `fsspec`, `geoengine`, `rasterio` (only for the optional read-back check)\n", + "- Access to the NEX-GDDP-CMIP6 S3 bucket (public, no credentials required)\n", + "- GeoEngine instance with dataset creation permissions\n" + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "metadata": { + "execution": { + "iopub.execute_input": "2026-09-13T10:00:55.747332Z", + "iopub.status.busy": "2026-09-13T10:00:55.747237Z", + "iopub.status.idle": "2026-09-13T10:00:57.955865Z", + "shell.execute_reply": "2026-09-13T10:00:57.955115Z" + } + }, + "outputs": [], + "source": [ + "import json\n", + "from datetime import datetime\n", + "from pathlib import Path\n", + "\n", + "import fsspec\n", + "import s3fs\n", + "\n", + "import geoengine as ge\n", + "import geoengine_api_client as geoc" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Configuration\n", + "\n", + "Update these paths to match your environment." + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": { + "execution": { + "iopub.execute_input": "2026-09-13T10:00:57.957414Z", + "iopub.status.busy": "2026-09-13T10:00:57.957183Z", + "iopub.status.idle": "2026-09-13T10:00:57.961033Z", + "shell.execute_reply": "2026-09-13T10:00:57.960514Z" + } + }, + "outputs": [], + "source": [ + "# Unused unless the volume type requires an upload flow (no filesystem path);\n", + "# keep as the local staging dir for that case.\n", + "REFERENCE_PATH = Path(\"/data/nexgddp-cmip6-refs\")\n", + "\n", + "# NEX-GDDP-CMIP6 S3 bucket (public)\n", + "S3_BUCKET = \"nex-gddp-cmip6\"\n", + "S3_PREFIX = \"NEX-GDDP-CMIP6\"\n", + "\n", + "# GeoEngine connection\n", + "GEOENGINE_URL = \"http://localhost:3030/api\"\n", + "GEOENGINE_USER = \"admin@localhost\"\n", + "GEOENGINE_PASSWORD = \"adminadmin\"\n", + "\n", + "# Volume name in GeoEngine\n", + "VOLUME_NAME = \"ge_geo_data\"\n", + "\n", + "# Full model registry: the authoritative id/variant/grid/scenario list. The dev registry\n", + "# (`nexgddp_models.toml`) is trimmed to what is ingested, so ingest reads this one and the\n", + "# trimmed file is regenerated from it afterwards.\n", + "REGISTRY_PATH = Path(\"../backend/conf/nexgddp_models.full.toml\")\n", + "\n", + "try:\n", + " import tomllib\n", + "except ModuleNotFoundError: # Python < 3.11\n", + " import tomli as tomllib\n", + "\n", + "with open(REGISTRY_PATH, \"rb\") as fh:\n", + " REGISTRY = tomllib.load(fh)[\"model\"]\n", + "\n", + "MODELS = [\n", + " {\"id\": m[\"id\"], \"variant\": m[\"variant\"], \"grid\": m[\"grid\"]} for m in REGISTRY\n", + "]\n", + "\n", + "SCENARIOS = sorted({s for m in REGISTRY for s in m[\"scenarios\"]})\n", + "\n", + "# Variables\n", + "VARIABLES = [\"tasmax\", \"tasmin\", \"pr\"]\n", + "\n", + "# Variable metadata\n", + "VARIABLE_CONFIGS = {\n", + " \"tasmax\": {\n", + " \"name\": \"Maximum Temperature\",\n", + " \"measurement\": \"temperature\",\n", + " \"unit\": \"K\",\n", + " \"min\": 200,\n", + " \"max\": 340,\n", + " \"color_map\": \"coolwarm\",\n", + " },\n", + " \"tasmin\": {\n", + " \"name\": \"Minimum Temperature\",\n", + " \"measurement\": \"temperature\",\n", + " \"unit\": \"K\",\n", + " \"min\": 200,\n", + " \"max\": 340,\n", + " \"color_map\": \"coolwarm\",\n", + " },\n", + " \"pr\": {\n", + " \"name\": \"Precipitation\",\n", + " \"measurement\": \"precipitation\",\n", + " \"unit\": \"kg m-2 s-1\",\n", + " \"min\": 0,\n", + " \"max\": 0.01,\n", + " \"color_map\": \"Blues\",\n", + " },\n", + "}\n", + "# ponytail: trial mode keeps AWS traffic low. Set DEV_ONLY = False for the full matrix\n", + "# (all 5 models x 3 scenarios x 3 variables).\n", + "DEV_ONLY = True\n", + "if DEV_ONLY:\n", + " MODELS = MODELS[:1]\n", + " SCENARIOS = SCENARIOS[:2]\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Initialize GeoEngine Connection" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": { + "execution": { + "iopub.execute_input": "2026-09-13T10:00:58.003318Z", + "iopub.status.busy": "2026-09-13T10:00:58.003155Z", + "iopub.status.idle": "2026-09-13T10:00:58.209417Z", + "shell.execute_reply": "2026-09-13T10:00:58.208860Z" + } + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Using volume: ge_geo_data (path='/mnt/peter/geo_data/')\n" + ] + } + ], + "source": [ + "ge.initialize(\n", + " GEOENGINE_URL,\n", + " credentials=(GEOENGINE_USER, GEOENGINE_PASSWORD),\n", + ")\n", + "\n", + "volume = ge.volume_by_name(VOLUME_NAME)\n", + "print(f\"Using volume: {VOLUME_NAME} (path={volume.path!r})\")\n", + "\n", + "# Reference files must live where GeoEngine can reach them. If the volume has a\n", + "# filesystem path, write refs under it; otherwise an upload step is required.\n", + "DATA_DIR = Path(volume.path) / \"nexgddp-cmip6\" if volume.path else None\n", + "if DATA_DIR is None:\n", + " print(\"Warning: volume has no filesystem path - reference files must be uploaded\")\n", + "\n", + "permission_tuples = [\n", + " (ge.permissions.ANONYMOUS_USER_ROLE_ID, ge.Permission.READ),\n", + " (ge.permissions.REGISTERED_USER_ROLE_ID, ge.Permission.READ),\n", + "]" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Build VRTs\n", + "\n", + "For each model/scenario/variable combination, build a GDAL VRT that concatenates the daily bands of the yearly NetCDF files, streamed from S3 via `/vsicurl`." + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": { + "execution": { + "iopub.execute_input": "2026-09-13T10:00:58.210829Z", + "iopub.status.busy": "2026-09-13T10:00:58.210721Z", + "iopub.status.idle": "2026-09-13T10:00:58.214208Z", + "shell.execute_reply": "2026-09-13T10:00:58.213714Z" + } + }, + "outputs": [], + "source": [ + "def list_netcdf_files(model_id, scenario, variable, variant, grid):\n", + " \"\"\"List all yearly NetCDF files for a model/scenario/variable combination.\"\"\"\n", + " fs = s3fs.S3FileSystem(anon=True)\n", + " \n", + " # Determine year range based on scenario\n", + " if scenario == \"historical\":\n", + " years = range(1950, 2015)\n", + " else:\n", + " years = range(2015, 2101)\n", + " \n", + " files = []\n", + " for year in years:\n", + " # NEX-GDDP-CMIP6 published corrected `{year}_v2.0.nc` copies for early\n", + " # historical years; prefer them over the uncorrected base file.\n", + " base = (f\"{S3_BUCKET}/{S3_PREFIX}/{model_id}/{scenario}/{variant}/{variable}/\"\n", + " f\"{variable}_day_{model_id}_{scenario}_{variant}_{grid}_\")\n", + " path = next(\n", + " (base + cand for cand in (f\"{year}_v2.0.nc\", f\"{year}.nc\")\n", + " if fs.exists(base + cand)),\n", + " None,\n", + " )\n", + " if path is not None:\n", + " files.append(path)\n", + " else:\n", + " print(f\"Warning: File not found: {base}{year}.nc\")\n", + " \n", + " return sorted(files)" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": { + "execution": { + "iopub.execute_input": "2026-09-13T10:00:58.215304Z", + "iopub.status.busy": "2026-09-13T10:00:58.215211Z", + "iopub.status.idle": "2026-09-13T10:00:58.218716Z", + "shell.execute_reply": "2026-09-13T10:00:58.218191Z" + } + }, + "outputs": [], + "source": [ + "\n", + "def create_vrt(files, output_path):\n", + " \"\"\"Build a VRT concatenating the daily bands of yearly NetCDF files.\n", + "\n", + " Each yearly file contributes 365/366 bands (one per day). Concatenated in\n", + " order, band i == day i since the first year, which is exactly what the\n", + " NetCDF-CF metadata (band = 1 + band_offset + steps_between) expects.\n", + " Sources use the canonical VRT ComplexSource form so GDAL actually sources\n", + " the bands (a bare / silently yields NoData),\n", + " and are /vsicurl URLs so the GeoEngine worker streams from S3.\n", + " \"\"\"\n", + " import calendar as _cal\n", + "\n", + " band = 0\n", + " bands_xml = []\n", + " for file_path in files:\n", + " key = file_path[len(S3_BUCKET) + 1:] if file_path.startswith(S3_BUCKET + \"/\") else file_path\n", + " source = f\"/vsicurl/https://{S3_BUCKET}.s3.us-west-2.amazonaws.com/{key}\"\n", + " year = int([t for t in key.split(\"/\")[-1].split(\"_\") if t.isdigit()][-1])\n", + " days = 366 if _cal.isleap(year) else 365\n", + " for d in range(1, days + 1):\n", + " band += 1\n", + " bands_xml.append(\n", + " f''\n", + " f'1e20'\n", + " f''\n", + " f'{source}'\n", + " f'{d}'\n", + " f''\n", + " f''\n", + " f''\n", + " f'1e20'\n", + " f''\n", + " f''\n", + " )\n", + "\n", + " # NEX-GDDP-CMIP6 real grid: 0.25 deg, 1440x600, lon 0..360, lat -60..90\n", + " vrt = (\n", + " '\\n'\n", + " ' EPSG:4326\\n'\n", + " ' 0.0, 0.25, 0, 90.0, 0, -0.25\\n'\n", + " + \"\\n\".join(bands_xml)\n", + " + \"\\n\\n\"\n", + " )\n", + "\n", + " output_path.parent.mkdir(parents=True, exist_ok=True)\n", + " output_path.write_text(vrt)\n", + " print(f\"Created VRT: {output_path} ({band} bands)\")\n", + " return band\n" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "metadata": { + "execution": { + "iopub.execute_input": "2026-09-13T10:00:58.219803Z", + "iopub.status.busy": "2026-09-13T10:00:58.219709Z", + "iopub.status.idle": "2026-09-13T10:02:25.876624Z", + "shell.execute_reply": "2026-09-13T10:02:25.876102Z" + } + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + "Processing: ACCESS-CM2 / historical / tasmax\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Created VRT: /mnt/peter/geo_data/nexgddp-cmip6/ACCESS-CM2/historical/tasmax.vrt (23741 bands)\n", + "\n", + "Processing: ACCESS-CM2 / historical / tasmin\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Created VRT: /mnt/peter/geo_data/nexgddp-cmip6/ACCESS-CM2/historical/tasmin.vrt (23741 bands)\n", + "\n", + "Processing: ACCESS-CM2 / historical / pr\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Created VRT: /mnt/peter/geo_data/nexgddp-cmip6/ACCESS-CM2/historical/pr.vrt (23741 bands)\n", + "\n", + "Processing: ACCESS-CM2 / ssp245 / tasmax\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Created VRT: /mnt/peter/geo_data/nexgddp-cmip6/ACCESS-CM2/ssp245/tasmax.vrt (31411 bands)\n", + "\n", + "Processing: ACCESS-CM2 / ssp245 / tasmin\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Created VRT: /mnt/peter/geo_data/nexgddp-cmip6/ACCESS-CM2/ssp245/tasmin.vrt (31411 bands)\n", + "\n", + "Processing: ACCESS-CM2 / ssp245 / pr\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Created VRT: /mnt/peter/geo_data/nexgddp-cmip6/ACCESS-CM2/ssp245/pr.vrt (31411 bands)\n" + ] + } + ], + "source": [ + "\n", + "# Create VRTs for all combinations (staging dir as fallback)\n", + "assert DATA_DIR is not None, \\\n", + " \"No volume path - place refs via upload or mount the volume before running\"\n", + "OUTPUT_DIR = DATA_DIR\n", + "\n", + "for model in MODELS:\n", + " for scenario in SCENARIOS:\n", + " for variable in VARIABLES:\n", + " print(f\"\\nProcessing: {model['id']} / {scenario} / {variable}\")\n", + " files = list_netcdf_files(\n", + " model[\"id\"], scenario, variable, model[\"variant\"], model[\"grid\"],\n", + " )\n", + " if not files:\n", + " print(\"No files found, skipping\")\n", + " continue\n", + " output_path = OUTPUT_DIR / model[\"id\"] / scenario / f\"{variable}.vrt\"\n", + " create_vrt(files, output_path)\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Register Datasets in GeoEngine\n", + "\n", + "Register each reference file as a GeoEngine dataset." + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "metadata": { + "execution": { + "iopub.execute_input": "2026-09-13T10:02:25.877948Z", + "iopub.status.busy": "2026-09-13T10:02:25.877840Z", + "iopub.status.idle": "2026-09-13T10:02:25.883553Z", + "shell.execute_reply": "2026-09-13T10:02:25.883160Z" + } + }, + "outputs": [], + "source": [ + "\n", + "def build_netcdf_cf_metadata(variable, scenario, year_start, year_end, reference_path):\n", + " \"\"\"Build NetCDF-CF metadata for a band-concatenating VRT registration.\n", + "\n", + " Backing VRT has one band per day since year_start; NetCDF-CF maps time ->\n", + " band via band = 1 + band_offset + steps_between. Requires the VRT's sources\n", + " (https ... .nc via /vsicurl) to pass the worker's curl extension allowlist.\n", + " \"\"\"\n", + " config = VARIABLE_CONFIGS[variable]\n", + "\n", + " # NEX-GDDP-CMIP6 actual grid: 0.25 deg, 1440x600, lon 0..360, lat -60..90\n", + " n_lat, n_lon, dx = 600, 1440, 0.25\n", + " x_min, y_max = 0.0, 90.0\n", + "\n", + " measurement = geoc.Measurement(\n", + " actual_instance=geoc.ContinuousMeasurement(\n", + " type=\"continuous\", measurement=config[\"unit\"], unit=config[\"unit\"]\n", + " )\n", + " )\n", + " start_ms = int(datetime(year_start, 1, 1).timestamp() * 1000)\n", + " end_ms = int(datetime(year_end, 1, 1).timestamp() * 1000) # non-inclusive\n", + " time_dimension = geoc.TimeDimension(\n", + " actual_instance=geoc.Regular(\n", + " origin=start_ms,\n", + " step=geoc.TimeStep(granularity=\"days\", step=1),\n", + " type=\"regular\",\n", + " )\n", + " )\n", + " geo_transform = geoc.GeoTransform(\n", + " origin_coordinate=geoc.Coordinate2D(x=x_min, y=y_max),\n", + " x_pixel_size=dx,\n", + " y_pixel_size=-dx,\n", + " )\n", + " result_descriptor = geoc.RasterResultDescriptor(\n", + " data_type=\"F32\",\n", + " spatial_reference=\"EPSG:4326\",\n", + " time=geoc.TimeDescriptor(\n", + " bounds=geoc.TimeInterval(start=start_ms, end=end_ms),\n", + " dimension=time_dimension,\n", + " ),\n", + " spatial_grid=geoc.SpatialGridDescriptor(\n", + " spatialGrid=geoc.SpatialGridDefinition(\n", + " geoTransform=geo_transform,\n", + " gridBounds=geoc.GridBoundingBox2D(\n", + " topLeftIdx=geoc.GridIdx2D(xIdx=0, yIdx=0),\n", + " bottomRightIdx=geoc.GridIdx2D(xIdx=n_lon - 1, yIdx=n_lat - 1),\n", + " ),\n", + " ),\n", + " descriptor=\"source\",\n", + " ),\n", + " measurement=measurement,\n", + " bands=[geoc.RasterBandDescriptor(name=variable, measurement=measurement)],\n", + " )\n", + "\n", + " params = geoc.GdalDatasetParameters(\n", + " file_path=str(reference_path),\n", + " rasterband_channel=1,\n", + " geo_transform=geo_transform,\n", + " width=n_lon,\n", + " height=n_lat,\n", + " file_not_found_handling=\"NoData\",\n", + " gdal_config_options=[\n", + " [\"CPL_VSIL_CURL_ALLOWED_EXTENSIONS\", \".tif,.tiff,.TIF,.jp2,.ovr,.nc\"],\n", + " ],\n", + " )\n", + "\n", + " return geoc.GdalMetadataNetCdfCf(\n", + " type=\"GdalMetaDataNetCdfCf\",\n", + " result_descriptor=result_descriptor,\n", + " params=params,\n", + " start=start_ms,\n", + " end=end_ms,\n", + " step=geoc.TimeStep(granularity=\"days\", step=1),\n", + " band_offset=0,\n", + " )\n", + "\n", + "def register_dataset(model_id, scenario, variable, reference_path, year_start, year_end):\n", + " \"\"\"Register a dataset in GeoEngine using the volume add-or-replace API.\"\"\"\n", + " config = VARIABLE_CONFIGS[variable]\n", + " dataset_name = f\"nexgddp_cmip6_{model_id}_{scenario}_{variable}\"\n", + " display_name = f\"NEX-GDDP-CMIP6 {config['name']} ({scenario})\"\n", + "\n", + " properties = ge.AddDatasetProperties(\n", + " name=dataset_name,\n", + " display_name=display_name,\n", + " description=(\n", + " f\"Daily NEX-GDDP-CMIP6 {config['name']} projections for {config['measurement']} ({config['unit']}). \"\n", + " f\"Model: {model_id}, Scenario: {scenario}, grid: VRT over yearly NetCDF files\"\n", + " ),\n", + " source_operator=\"GdalSource\",\n", + " symbology=ge.RasterSymbology(\n", + " raster_colorizer=ge.SingleBandRasterColorizer(\n", + " band=0,\n", + " band_colorizer=ge.Colorizer.linear_with_mpl_cmap(\n", + " color_map=config[\"color_map\"],\n", + " min_max=(config[\"min\"], config[\"max\"]),\n", + " n_steps=10,\n", + " ),\n", + " ),\n", + " ),\n", + " )\n", + "\n", + " meta_data = geoc.MetaDataDefinition(\n", + " build_netcdf_cf_metadata(variable, scenario, year_start, year_end, reference_path)\n", + " )\n", + "\n", + " ge.add_or_replace_dataset_with_permissions(\n", + " volume,\n", + " properties,\n", + " meta_data,\n", + " permission_tuples=permission_tuples,\n", + " replace_existing=True,\n", + " timeout=600,\n", + " )\n", + " print(f\"Registered: {dataset_name}\")\n", + " return dataset_name\n" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "metadata": { + "execution": { + "iopub.execute_input": "2026-09-13T10:02:25.884835Z", + "iopub.status.busy": "2026-09-13T10:02:25.884735Z", + "iopub.status.idle": "2026-09-13T10:02:26.248315Z", + "shell.execute_reply": "2026-09-13T10:02:26.247420Z" + } + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Registered: nexgddp_cmip6_ACCESS-CM2_historical_tasmax\n", + "Registered: nexgddp_cmip6_ACCESS-CM2_historical_tasmin\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Registered: nexgddp_cmip6_ACCESS-CM2_historical_pr\n", + "Registered: nexgddp_cmip6_ACCESS-CM2_ssp245_tasmax\n", + "Registered: nexgddp_cmip6_ACCESS-CM2_ssp245_tasmin\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Registered: nexgddp_cmip6_ACCESS-CM2_ssp245_pr\n", + "\n", + "Total datasets: 6\n" + ] + } + ], + "source": [ + "\n", + "# Register all datasets\n", + "YEAR_RANGES = {\"historical\": (1950, 2015), \"ssp245\": (2015, 2101), \"ssp585\": (2015, 2101)}\n", + "registered_datasets = []\n", + "\n", + "for model in MODELS:\n", + " for scenario in SCENARIOS:\n", + " year_start, year_end = YEAR_RANGES[scenario]\n", + " for variable in VARIABLES:\n", + " rel_ref = Path(\"nexgddp-cmip6\") / model[\"id\"] / scenario / f\"{variable}.vrt\"\n", + " # file is written under DATA_DIR (= volume_root/nexgddp-cmip6);\n", + " # rel_ref is relative to the volume root, hence includes the prefix\n", + " ref_abs = DATA_DIR / model[\"id\"] / scenario / f\"{variable}.vrt\"\n", + " if ref_abs.exists():\n", + " dataset_name = register_dataset(\n", + " model[\"id\"], scenario, variable,\n", + " rel_ref, # path is relative to the volume root\n", + " year_start, year_end,\n", + " )\n", + " registered_datasets.append(dataset_name)\n", + "\n", + "print(f\"\\nTotal datasets: {len(registered_datasets)}\")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Verification\n", + "\n", + "Verify that GDAL can read the reference files." + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "metadata": { + "execution": { + "iopub.execute_input": "2026-09-13T10:02:26.250521Z", + "iopub.status.busy": "2026-09-13T10:02:26.250277Z", + "iopub.status.idle": "2026-09-13T10:02:30.635167Z", + "shell.execute_reply": "2026-09-13T10:02:30.634253Z" + } + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Bands: 23741, CRS: EPSG:4326, Bounds: BoundingBox(left=0.0, bottom=-60.0, right=360.0, top=90.0)\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Band 1 read OK: min=219.0 max=100000002004087734272.0 (streams first day from S3)\n", + "nexgddp_cmip6_ACCESS-CM2_historical_tasmax: True\n", + "nexgddp_cmip6_ACCESS-CM2_historical_tasmin: True\n", + "nexgddp_cmip6_ACCESS-CM2_historical_pr: True\n" + ] + } + ], + "source": [ + "\n", + "import os\n", + "import rasterio\n", + "\n", + "os.environ[\"CPL_VSIL_CURL_ALLOWED_EXTENSIONS\"] = \".tif,.tiff,.TIF,.jp2,.ovr,.nc\"\n", + "\n", + "test_vrt = DATA_DIR / MODELS[0][\"id\"] / SCENARIOS[0] / f\"{VARIABLES[0]}.vrt\"\n", + "if test_vrt.exists():\n", + " try:\n", + " with rasterio.open(test_vrt) as src:\n", + " print(f\"Bands: {src.count}, CRS: {src.crs}, Bounds: {src.bounds}\")\n", + " data = src.read(1)\n", + " print(f\"Band 1 read OK: min={data.min():.1f} max={data.max():.1f} (streams first day from S3)\")\n", + " except Exception as e:\n", + " print(f\"Failed to open VRT: {e}\")\n", + "else:\n", + " print(f\"Test VRT file not found: {test_vrt}\")\n", + "\n", + "# Confirm GeoEngine sees the registered dataset\n", + "for ds in registered_datasets[:3]:\n", + " info = ge.dataset_info_by_name(ds)\n", + " print(f\"{ds}: {info is not None}\")\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Next Steps\n", + "\n", + "1. Verify that the GeoEngine worker can read the VRTs (needs GDAL >= 3.8 in the container image)\n", + "2. Update the `register_dataset` function with proper metadata extraction\n", + "3. Run the full registration (set `DEV_ONLY = False`) to create all 45 datasets, then copy\n", + "`nexgddp_models.full.toml` over `nexgddp_models.toml`\n", + "4. Test the climate-risk process with the new datasets" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.12.3" + } + }, + "nbformat": 4, + "nbformat_minor": 4 +} From 1486271198e1ea97e3745bd6b1a4cecde957608f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johannes=20Dr=C3=B6nner?= Date: Fri, 2 Oct 2026 13:55:39 +0200 Subject: [PATCH 5/9] fix(backend): put the model registry where the server looks for it `COPY --from=builder .../conf ./conf` resolved `./conf` against the distroless base's `WORKDIR /home/nonroot`, so the registry landed outside the server's working directory and every climate-risk execution failed with "No climate models valid / available". Use an absolute destination. A registry that cannot be read or parsed now also says so in that error, instead of degrading to an empty registry and reporting an unexplained empty selection. --- backend/Dockerfile | 4 +- backend/src/processes/climate_risk/mod.rs | 63 ++++++++++++++++++----- 2 files changed, 53 insertions(+), 14 deletions(-) diff --git a/backend/Dockerfile b/backend/Dockerfile index 3df76d2..24e189a 100644 --- a/backend/Dockerfile +++ b/backend/Dockerfile @@ -48,7 +48,9 @@ FROM gcr.io/distroless/cc-debian13:nonroot COPY --from=builder /usr/src/biois/target/release/biois-server /usr/local/bin/biois-server COPY --from=builder /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ COPY --from=builder /emptydir /biois -COPY --from=builder /usr/src/biois/conf ./conf +# Absolute destination: the distroless base sets `WORKDIR /home/nonroot`, so a relative +# destination would land there instead of in the directory the server runs from. +COPY --from=builder /usr/src/biois/conf /biois/conf WORKDIR /biois diff --git a/backend/src/processes/climate_risk/mod.rs b/backend/src/processes/climate_risk/mod.rs index 4b37425..4b57962 100644 --- a/backend/src/processes/climate_risk/mod.rs +++ b/backend/src/processes/climate_risk/mod.rs @@ -29,16 +29,40 @@ pub use types::{ClimateRiskInputs, ClimateRiskOutputs, ClimateRiskRawRow, Climat use self::{compute::*, types::*}; // ponytail: model registry loaded once at startup from TOML; add models by editing conf/nexgddp_models.toml, no recompile -static MODEL_REGISTRY: LazyLock> = LazyLock::new(|| { +static MODEL_REGISTRY: LazyLock> = + LazyLock::new(load_model_registry); + +/// Reads the model registry, returning an empty one if it cannot be read or parsed. +/// +/// A missing or broken registry must not take down the other processes, so this only warns; the +/// reason is kept in [`REGISTRY_PROBLEM`] so that a climate-risk execution can report *why* no +/// models are available instead of an unexplained empty selection. +fn load_model_registry() -> HashMap { let path = &CONFIG.nexgddp_cmip6.model_registry_path; - let content = std::fs::read_to_string(path).unwrap_or_else(|_| { - tracing::warn!("Could not read model registry from {path}, using empty registry"); - String::new() - }); - let registry: ModelRegistryToml = toml::from_str(&content).unwrap_or_else(|e| { - tracing::warn!("Could not parse model registry: {e}"); - ModelRegistryToml { model: Vec::new() } - }); + let content = match std::fs::read_to_string(path) { + Ok(content) => content, + Err(error) => { + let reason = format!("model registry `{path}` could not be read: {error}"); + tracing::warn!("{reason}, climate-risk will offer no models"); + REGISTRY_PROBLEM + .lock() + .expect("registry problem lock poisoned") + .replace(reason); + return HashMap::new(); + } + }; + let registry: ModelRegistryToml = match toml::from_str(&content) { + Ok(registry) => registry, + Err(error) => { + let reason = format!("model registry `{path}` could not be parsed: {error}"); + tracing::warn!("{reason}, climate-risk will offer no models"); + REGISTRY_PROBLEM + .lock() + .expect("registry problem lock poisoned") + .replace(reason); + return HashMap::new(); + } + }; let mut entries: Vec<(String, ClimateModelProperties)> = registry .model .into_iter() @@ -67,7 +91,18 @@ static MODEL_REGISTRY: LazyLock> = LazyL // ponytail: sort by model id for deterministic iteration order entries.sort_by(|a, b| a.0.cmp(&b.0)); entries.into_iter().collect() -}); +} + +/// Why the registry is empty, if loading it failed, cf. [`load_model_registry`]. +static REGISTRY_PROBLEM: std::sync::Mutex> = std::sync::Mutex::new(None); + +/// The reason the registry could not be loaded, if it failed. +fn registry_problem() -> Option { + REGISTRY_PROBLEM + .lock() + .ok() + .and_then(|problem| problem.clone()) +} #[derive(serde::Deserialize)] struct ModelRegistryToml { @@ -359,13 +394,15 @@ impl Processor for ClimateRiskProcess { ); } if model_props.is_empty() { - let detail = if dropped_models.is_empty() { - String::new() - } else { + let detail = if !dropped_models.is_empty() { format!( "; none of the requested models ({}) are available", dropped_models.join(", ") ) + } else if let Some(problem) = registry_problem() { + format!("; {problem}") + } else { + "; the model registry is empty".to_string() }; anyhow::bail!("No climate models valid / available{detail}"); } From 2baa397ca8c4cf8cb5dda71dd570347a02dcf2c1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johannes=20Dr=C3=B6nner?= Date: Fri, 2 Oct 2026 13:55:44 +0200 Subject: [PATCH 6/9] chore(notebooks): record the trial ingest run Keeps the registered dataset list from the ACCESS-CM2 run (historical + ssp245 x tasmax/tasmin/pr) and drops the stale reference-file notes. --- notebooks/nexgddp_cmip6_ingest.ipynb | 88 ++++++---------------------- 1 file changed, 17 insertions(+), 71 deletions(-) diff --git a/notebooks/nexgddp_cmip6_ingest.ipynb b/notebooks/nexgddp_cmip6_ingest.ipynb index eacf44c..edadfdf 100644 --- a/notebooks/nexgddp_cmip6_ingest.ipynb +++ b/notebooks/nexgddp_cmip6_ingest.ipynb @@ -144,7 +144,7 @@ }, { "cell_type": "code", - "execution_count": 4, + "execution_count": 3, "metadata": { "execution": { "iopub.execute_input": "2026-09-13T10:00:58.003318Z", @@ -194,7 +194,7 @@ }, { "cell_type": "code", - "execution_count": 5, + "execution_count": 4, "metadata": { "execution": { "iopub.execute_input": "2026-09-13T10:00:58.210829Z", @@ -236,7 +236,7 @@ }, { "cell_type": "code", - "execution_count": 6, + "execution_count": 5, "metadata": { "execution": { "iopub.execute_input": "2026-09-13T10:00:58.215304Z", @@ -300,7 +300,7 @@ }, { "cell_type": "code", - "execution_count": 7, + "execution_count": 6, "metadata": { "execution": { "iopub.execute_input": "2026-09-13T10:00:58.219803Z", @@ -315,58 +315,22 @@ "output_type": "stream", "text": [ "\n", - "Processing: ACCESS-CM2 / historical / tasmax\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ + "Processing: ACCESS-CM2 / historical / tasmax\n", "Created VRT: /mnt/peter/geo_data/nexgddp-cmip6/ACCESS-CM2/historical/tasmax.vrt (23741 bands)\n", "\n", - "Processing: ACCESS-CM2 / historical / tasmin\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ + "Processing: ACCESS-CM2 / historical / tasmin\n", "Created VRT: /mnt/peter/geo_data/nexgddp-cmip6/ACCESS-CM2/historical/tasmin.vrt (23741 bands)\n", "\n", - "Processing: ACCESS-CM2 / historical / pr\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ + "Processing: ACCESS-CM2 / historical / pr\n", "Created VRT: /mnt/peter/geo_data/nexgddp-cmip6/ACCESS-CM2/historical/pr.vrt (23741 bands)\n", "\n", - "Processing: ACCESS-CM2 / ssp245 / tasmax\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ + "Processing: ACCESS-CM2 / ssp245 / tasmax\n", "Created VRT: /mnt/peter/geo_data/nexgddp-cmip6/ACCESS-CM2/ssp245/tasmax.vrt (31411 bands)\n", "\n", - "Processing: ACCESS-CM2 / ssp245 / tasmin\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ + "Processing: ACCESS-CM2 / ssp245 / tasmin\n", "Created VRT: /mnt/peter/geo_data/nexgddp-cmip6/ACCESS-CM2/ssp245/tasmin.vrt (31411 bands)\n", "\n", - "Processing: ACCESS-CM2 / ssp245 / pr\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ + "Processing: ACCESS-CM2 / ssp245 / pr\n", "Created VRT: /mnt/peter/geo_data/nexgddp-cmip6/ACCESS-CM2/ssp245/pr.vrt (31411 bands)\n" ] } @@ -403,7 +367,7 @@ }, { "cell_type": "code", - "execution_count": 8, + "execution_count": 7, "metadata": { "execution": { "iopub.execute_input": "2026-09-13T10:02:25.877948Z", @@ -534,7 +498,7 @@ }, { "cell_type": "code", - "execution_count": 9, + "execution_count": 8, "metadata": { "execution": { "iopub.execute_input": "2026-09-13T10:02:25.884835Z", @@ -549,22 +513,10 @@ "output_type": "stream", "text": [ "Registered: nexgddp_cmip6_ACCESS-CM2_historical_tasmax\n", - "Registered: nexgddp_cmip6_ACCESS-CM2_historical_tasmin\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ + "Registered: nexgddp_cmip6_ACCESS-CM2_historical_tasmin\n", "Registered: nexgddp_cmip6_ACCESS-CM2_historical_pr\n", "Registered: nexgddp_cmip6_ACCESS-CM2_ssp245_tasmax\n", - "Registered: nexgddp_cmip6_ACCESS-CM2_ssp245_tasmin\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ + "Registered: nexgddp_cmip6_ACCESS-CM2_ssp245_tasmin\n", "Registered: nexgddp_cmip6_ACCESS-CM2_ssp245_pr\n", "\n", "Total datasets: 6\n" @@ -607,7 +559,7 @@ }, { "cell_type": "code", - "execution_count": 10, + "execution_count": 9, "metadata": { "execution": { "iopub.execute_input": "2026-09-13T10:02:26.250521Z", @@ -621,13 +573,7 @@ "name": "stdout", "output_type": "stream", "text": [ - "Bands: 23741, CRS: EPSG:4326, Bounds: BoundingBox(left=0.0, bottom=-60.0, right=360.0, top=90.0)\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ + "Bands: 23741, CRS: EPSG:4326, Bounds: BoundingBox(left=0.0, bottom=-60.0, right=360.0, top=90.0)\n", "Band 1 read OK: min=219.0 max=100000002004087734272.0 (streams first day from S3)\n", "nexgddp_cmip6_ACCESS-CM2_historical_tasmax: True\n", "nexgddp_cmip6_ACCESS-CM2_historical_tasmin: True\n", @@ -676,7 +622,7 @@ ], "metadata": { "kernelspec": { - "display_name": "Python 3", + "display_name": "biois-notebooks", "language": "python", "name": "python3" }, From fd8740176cbff07121409e4202b09fcab2461b60 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johannes=20Dr=C3=B6nner?= Date: Fri, 2 Oct 2026 14:25:20 +0200 Subject: [PATCH 7/9] build: wait for the pod backend before run-pod returns `podman play kube` returns as soon as the containers are created, before the backend listens, so requests made right after `just run-pod` fail with a 502/504 from the frontend's reverse proxy. Poll the health endpoint until it answers. --- justfile | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/justfile b/justfile index 3508e37..e71942d 100644 --- a/justfile +++ b/justfile @@ -288,6 +288,11 @@ run-pod: build-backend-container build-frontend-container podman play kube \ --network=pasta:-T,3030:3030 `# Map local Geo Engine at port 3030 into pod` \ --replace - + # `podman play kube` returns before the containers serve, so wait for the backend; + # otherwise the first requests fail with a 502/504 from the frontend's reverse proxy. + timeout 120 sh -c 'until curl -fsS -o /dev/null http://localhost:4200/api/health 2>/dev/null; do sleep 1; done' \ + && echo "BioIS is ready on http://localhost:4200" \ + || echo "Backend did not become ready within 120s" >&2 # Stop the pod in dev mode. Usage: `just down-pod`. [group('container')] From ad02cd13448e45daae90effcf6bca49de81253ee Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johannes=20Dr=C3=B6nner?= Date: Tue, 6 Oct 2026 14:20:49 +0200 Subject: [PATCH 8/9] feat(backend): run climate-risk queries against the netCDF-native MdGdalSource Swap the GeoEngine client to the feature/md-gdal-source branch, register the climate-risk processing graphs through a helper that folds the duplicated type discriminator before it reaches the server, and drive heat/ice-day workflows over the NEX-GDDP-CMIP6 tasmax two-slice model. The live test registers the real processing graph against a running GeoEngine, queries 2015-2019 against the 1950-1954 reference for Berlin, and now returns daily values end-to-end. The ingest notebook is reworked to register the datasets explicitly (also exercising the MD metadata path without the probe) with fixed geotransforms. --- .gitignore | 3 + backend/Cargo.lock | 3 +- backend/Cargo.toml | 2 +- backend/examples/query-geo-engine.rs | 25 +- backend/src/processes/climate_risk/compute.rs | 112 ++- backend/src/processes/climate_risk/mod.rs | 2 +- .../src/processes/climate_risk/workflow.rs | 22 +- .../processes/land_use_sealed_area/compute.rs | 22 +- backend/src/processes/ndvi.rs | 48 +- backend/src/util.rs | 93 +- notebooks/nexgddp_cmip6_ingest.ipynb | 644 -------------- notebooks/nexgddp_cmip6_ingest_manual.ipynb | 791 ++++++++++++++++++ 12 files changed, 1026 insertions(+), 741 deletions(-) delete mode 100644 notebooks/nexgddp_cmip6_ingest.ipynb create mode 100644 notebooks/nexgddp_cmip6_ingest_manual.ipynb diff --git a/.gitignore b/.gitignore index 47de64c..f23354d 100644 --- a/.gitignore +++ b/.gitignore @@ -35,3 +35,6 @@ RUN_LOG.md # Frontend test coverage output frontend/coverage/ + +# Notebook run artifacts (the clean notebook is what's tracked) +*.out.ipynb diff --git a/backend/Cargo.lock b/backend/Cargo.lock index 8703824..5b82a09 100644 --- a/backend/Cargo.lock +++ b/backend/Cargo.lock @@ -1468,8 +1468,7 @@ dependencies = [ [[package]] name = "geoengine-api-client" version = "0.9.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6fa718487ebe0957cfec69a03bbd73342aa97526caebfad6790c70750dc44eee" +source = "git+https://github.com/geo-engine/geoengine?branch=feature%2Fmd-gdal-source#8e4bc2dfacbcaf709462379112e3caa63272ed61" dependencies = [ "reqwest", "serde", diff --git a/backend/Cargo.toml b/backend/Cargo.toml index 9add318..6155ce1 100644 --- a/backend/Cargo.toml +++ b/backend/Cargo.toml @@ -71,7 +71,7 @@ clap = { version = "4.6", features = ["derive"] } config = "0.15" futures = "0.3" geo-types = "0.7" -geoengine-api-client = "0.9.4" +geoengine-api-client = { git = "https://github.com/geo-engine/geoengine", branch = "feature/md-gdal-source" } geojson = "1.0" indoc = "2.0" lettre = { version = "0.11", default-features = false, features = [ diff --git a/backend/examples/query-geo-engine.rs b/backend/examples/query-geo-engine.rs index af8f351..cb61c2d 100644 --- a/backend/examples/query-geo-engine.rs +++ b/backend/examples/query-geo-engine.rs @@ -6,8 +6,9 @@ use geoengine_api_client::{ session_api::anonymous_handler, workflows_api::register_workflow_handler, }, models::{ - Coordinate2D, LegacyTypedOperator, LegacyTypedOperatorOperator, SpatialPartition2D, - WfsRequest, WfsService, Workflow, legacy_typed_operator::Type, + Coordinate2D, OgrSource, OgrSourceParameters, ProcessingGraph, SpatialPartition2D, + TypedVectorOperator, VectorOperator, WfsRequest, WfsService, + typed_vector_operator::Type as VectorType, }, }; @@ -22,15 +23,17 @@ async fn main() { eprintln!("{session:#?}"); configuration.bearer_access_token = Some(session.id.to_string()); - let workflow = Workflow::LegacyTypedOperator(Box::new(LegacyTypedOperator { - operator: Box::new(LegacyTypedOperatorOperator { - params: Some(serde_json::json!({ - "data": "ne_10m_ports" - })), - sources: None, - r#type: "OgrSource".into(), - }), - r#type: Type::Vector, + let workflow = ProcessingGraph::TypedVectorOperator(Box::new(TypedVectorOperator { + operator: Box::new(VectorOperator::OgrSource(Box::new(OgrSource { + r#type: Default::default(), + params: OgrSourceParameters { + data: "ne_10m_ports".into(), + attribute_projection: None, + attribute_filters: None, + } + .into(), + }))), + r#type: VectorType::Vector, })); let workflow_id = register_workflow_handler(&configuration, workflow) diff --git a/backend/src/processes/climate_risk/compute.rs b/backend/src/processes/climate_risk/compute.rs index be047c2..616fa9f 100644 --- a/backend/src/processes/climate_risk/compute.rs +++ b/backend/src/processes/climate_risk/compute.rs @@ -6,15 +6,12 @@ use crate::{ DataResource, DataResourceName, Days, Fields, TableSchemaField, TableSchemaType, Year, YearRange, }, - util::{error_response, to_api_vector_process}, + util::{error_response, register_processing_graph, to_api_vector_process}, }; use anyhow::Result; use futures::{TryStreamExt, stream::StreamExt}; use geoengine_api_client::{ - apis::{ - configuration::Configuration, ogcwfs_api::WfsHandlerError, ogcwfs_api::wfs_handler, - workflows_api::register_workflow_handler, - }, + apis::{configuration::Configuration, ogcwfs_api::WfsHandlerError, ogcwfs_api::wfs_handler}, models::{ ColumnNames, Coordinate2D, FeatureAggregationMethod, GeoJson, MockPointSource, MockPointSourceParameters, Names, RasterVectorJoin, RasterVectorJoinParameters, @@ -409,7 +406,7 @@ struct WorkflowRequest { models: Vec, variable: ClimateVariable, scenario: ClimateScenarioProperties, - workflow: geoengine_api_client::models::Workflow, + workflow: geoengine_api_client::models::ProcessingGraph, } /// Builds one workflow per (variable, scenario) pair, using only models that support the scenario. @@ -539,7 +536,7 @@ async fn register_workflows( .map(|request| { let workflow = &request.workflow; move || async move { - register_workflow_handler(configuration, workflow.clone()) + register_processing_graph(configuration, workflow) .await .map(|id| id.id.to_string()) } @@ -722,6 +719,10 @@ pub(crate) async fn compute_climate( let workflow_requests = build_workflows(coordinate, requests, models, None); let workflow_ids = register_workflows(configuration, &workflow_requests).await?; log_registered_workflows(&workflow_ids, &workflow_requests); + eprintln!( + "ANALYSIS_IDS {workflow_ids:?} GRAPH {}", + serde_json::to_string_pretty(&workflow_requests[0].workflow).expect("workflow serializes") + ); let analysis_results = query_workflows( configuration, @@ -830,13 +831,12 @@ pub(crate) fn vector_source(coordinate: &PointType) -> VectorOperator { r#type: Default::default(), params: MockPointSourceParameters { points: vec![Coordinate2D::new(coordinate[0], coordinate[1])], - spatial_bounds: SpatialBoundsDerive::None( + spatial_bounds: Some(Box::new(SpatialBoundsDerive::None( SpatialBoundsDeriveNone { r#type: Default::default(), } .into(), - ) - .into(), + ))), } .into(), } @@ -1483,7 +1483,19 @@ mod tests { }], variable, scenario: scenario.properties(), - workflow: geoengine_api_client::models::Workflow::default(), + // `ProcessingGraph::default()` recurses forever (the generated `VectorOperator::default` + // is `ColumnRangeFilter`, whose `sources` nests another vector operator), so build the + // leaf-only graph the default is *meant* to produce. + workflow: geoengine_api_client::models::ProcessingGraph::TypedVectorOperator(Box::new( + geoengine_api_client::models::TypedVectorOperator { + operator: Box::new( + geoengine_api_client::models::VectorOperator::MockPointSource( + Default::default(), + ), + ), + r#type: Default::default(), + }, + )), } } @@ -1614,4 +1626,82 @@ mod tests { assert_abs_diff_eq!(result["MPI-ESM1-2-LR"], 8.0); assert_abs_diff_eq!(result["ACCESS-CM2"], 4.0); } + + /// Full live check of the `MdGdalSource` composition: registers the real workflow + /// (`MdGdalSource` -> `Expression` -> `TemporalRasterAggregation` -> `RasterVectorJoin`) + /// against a running `GeoEngine` and queries heat/ice days over the 2015-2019 analysis and + /// the 1950-1954 reference. Only runs on demand - `cargo test -- --ignored it_runs_a_*`. + /// + /// Requires `nexgddp_cmip6_ACCESS-CM2_{historical,ssp245}_tasmax` registered with + /// anonymous read (see `notebooks/nexgddp_cmip6_ingest_manual.ipynb`) and a reachable + /// `GeoEngine` at `local`: + #[tokio::test] + #[ignore = "requires a live GeoEngine with the NEX-GDDP-CMIP6 datasets registered"] + async fn it_runs_a_climate_risk_query_against_live_geoengine() { + let mut configuration = geoengine_api_client::apis::configuration::Configuration::new(); + configuration.base_path = "http://localhost:3030/api".into(); + + // Workflow registration requires an authenticated session (anonymous read covers the + // data, not the workflows). + let session = geoengine_api_client::apis::session_api::login_handler( + &configuration, + geoengine_api_client::models::UserCredentials { + email: "admin@localhost".to_string(), + password: "adminadmin".to_string(), + }, + ) + .await + .expect("GeoEngine admin login failed"); + configuration.bearer_access_token = Some(session.id.to_string()); + + let coordinate = PointType::from((13.4050, 52.5200)); // Berlin: a known land cell + let periods = ClimatePeriods { + analysis_start: Year(2015), + range: YearRange(5), + reference_start: Year(1950), + }; + let requests = vec![ + ( + ClimateVariableRequest::new(ClimateVariable::HeatDays), + ClimateScenario::Ssp245.properties(), + ), + ( + ClimateVariableRequest::new(ClimateVariable::IceDays), + ClimateScenario::Ssp245.properties(), + ), + ]; + let models = vec![ClimateModelProperties { + id: "ACCESS-CM2".to_string(), + variant: "r1i1p1f1".to_string(), + grid: "gn".to_string(), + scenarios: vec![ClimateScenario::Historical, ClimateScenario::Ssp245], + }]; + + let (outputs, computation_ids) = + compute_climate(&configuration, &coordinate, &periods, &requests, &models) + .await + .expect("climate-risk query against the live GeoEngine failed"); + + let rows = outputs.climate_risk.as_ref().expect("climate risk rows"); + assert_eq!( + rows.data.len(), + 2, + "expected one row per variable: {rows:?}" + ); + for row in &rows.data { + let mean = row.mean.0; + assert!( + (0.0..=366.0).contains(&mean), + "daily count out of bounds: {row:?}" + ); + assert!( + row.anomaly.is_some(), + "expected an anomaly against the reference period: {row:?}" + ); + } + assert!( + !computation_ids.is_empty(), + "expected WFS computation ids for credit accounting" + ); + } } diff --git a/backend/src/processes/climate_risk/mod.rs b/backend/src/processes/climate_risk/mod.rs index 4b57962..e178678 100644 --- a/backend/src/processes/climate_risk/mod.rs +++ b/backend/src/processes/climate_risk/mod.rs @@ -783,7 +783,7 @@ mod tests { let result = ClimateRiskProcess::dataset_raster_source(&var, &model, &scenario); - assert!(matches!(result, RasterOperator::GdalSource(_))); + assert!(matches!(result, RasterOperator::MdGdalSource(_))); let value = serde_json::to_value(&result).unwrap(); assert_eq!( diff --git a/backend/src/processes/climate_risk/workflow.rs b/backend/src/processes/climate_risk/workflow.rs index ddf0c58..d277db1 100644 --- a/backend/src/processes/climate_risk/workflow.rs +++ b/backend/src/processes/climate_risk/workflow.rs @@ -1,14 +1,14 @@ use geoengine_api_client::models::{ - Aggregation, ContinuousMeasurement, Expression, ExpressionParameters, GdalSource, - GdalSourceParameters, Measurement, RasterBandDescriptor, RasterOperator, SingleRasterSource, - SumAggregation, TemporalRasterAggregation, TemporalRasterAggregationParameters, + Aggregation, AggregationSum, ContinuousMeasurement, Expression, ExpressionParameters, + MdGdalSource, MdGdalSourceParameters, Measurement, RasterBandDescriptor, RasterOperator, + SingleRasterSource, TemporalRasterAggregation, TemporalRasterAggregationParameters, TimeGranularity, TimeStep, }; use tracing::instrument; use super::{ClimateRiskProcess, types::*}; impl ClimateRiskProcess { - /// Builds the GDAL raster source for a variable/model/scenario combination. + /// Builds the netCDF raster source for a variable/model/scenario combination. #[instrument(skip(var, model, scenario))] pub(crate) fn dataset_raster_source( var: &ClimateVariableProperties, @@ -19,14 +19,12 @@ impl ClimateRiskProcess { "nexgddp_cmip6_{}_{}_{}", model.id, scenario.dataset_prefix, var.dataset_variable_suffix ); - RasterOperator::GdalSource( - GdalSource { + // ponytail: `GdalSource` cannot read an Md dataset — its `GdalMetaData` enum has no MD + // variant, so the registered netCDF files are only reachable through `MdGdalSource`. + RasterOperator::MdGdalSource( + MdGdalSource { r#type: Default::default(), - params: GdalSourceParameters { - data: dataset_name, - overview_level: None, - } - .into(), + params: MdGdalSourceParameters { data: dataset_name }.into(), } .into(), ) @@ -81,7 +79,7 @@ impl ClimateRiskProcess { TemporalRasterAggregation { r#type: Default::default(), params: TemporalRasterAggregationParameters { - aggregation: Aggregation::SumAggregation(Box::new(SumAggregation { + aggregation: Aggregation::Sum(Box::new(AggregationSum { ignore_no_data: true, r#type: Default::default(), })) diff --git a/backend/src/processes/land_use_sealed_area/compute.rs b/backend/src/processes/land_use_sealed_area/compute.rs index efecaf0..621faba 100644 --- a/backend/src/processes/land_use_sealed_area/compute.rs +++ b/backend/src/processes/land_use_sealed_area/compute.rs @@ -15,7 +15,9 @@ use crate::{ raster_result_descriptor, vector_result_descriptor, year_range_from_time_descriptor, }, }, - util::{spawn_blocking, to_api_raster_process, to_api_vector_process}, + util::{ + register_processing_graph, spawn_blocking, to_api_raster_process, to_api_vector_process, + }, }; use anyhow::{Context, Result}; use geoengine_api_client::{ @@ -24,10 +26,7 @@ use geoengine_api_client::{ datasets_api::create_dataset_handler, ogcwfs_api::wfs_handler, uploads_api::upload_handler, - workflows_api::{ - get_workflow_metadata_handler, get_workflow_provenance_handler, - register_workflow_handler, - }, + workflows_api::{get_workflow_metadata_handler, get_workflow_provenance_handler}, }, models::{ AddDataset, ClassificationMeasurement, ColumnNames, ContinuousMeasurement, CreateDataset, @@ -330,7 +329,7 @@ async fn sealed_area_process( ) -> Result<(GeoJson, ComputationId)> { let operators = build_sealed_area_vector_operator(upload_data_id); - let processing_graph_id = register_workflow_handler(configuration, operators.sealed_area()) + let processing_graph_id = register_processing_graph(configuration, &operators.sealed_area()) .await? .id .to_string(); @@ -338,7 +337,7 @@ async fn sealed_area_process( let time_str = format!("{year}-01-01T00:00:00Z"); let locations_projected_id = - register_workflow_handler(configuration, operators.locations_projected()) + register_processing_graph(configuration, &operators.locations_projected()) .await? .id .to_string(); @@ -470,11 +469,11 @@ struct ComputeOperators { } impl ComputeOperators { - fn sealed_area(&self) -> geoengine_api_client::models::Workflow { + fn sealed_area(&self) -> geoengine_api_client::models::ProcessingGraph { to_api_vector_process(&self.sealed_area) } - fn locations_projected(&self) -> geoengine_api_client::models::Workflow { + fn locations_projected(&self) -> geoengine_api_client::models::ProcessingGraph { to_api_vector_process(&self.locations_projected) } } @@ -486,6 +485,7 @@ fn projected_locations_operator(upload_data_id: String) -> VectorOperator { params: OgrSourceParameters { data: upload_data_id, attribute_projection: None, + attribute_filters: None, } .into(), })); @@ -612,7 +612,7 @@ pub async fn compute_available_time_range_from_imperviousness_raster( configuration: &Configuration, ) -> Result<(Year, Year)> { let operator = to_api_raster_process(&imperviousness_raster_operator()); - let processing_graph_id = register_workflow_handler(configuration, operator) + let processing_graph_id = register_processing_graph(configuration, &operator) .await? .id .to_string(); @@ -631,7 +631,7 @@ pub async fn compute_documentation_sources( configuration: &Configuration, ) -> Result> { let operator = to_api_raster_process(&imperviousness_raster_operator()); - let processing_graph_id = register_workflow_handler(configuration, operator) + let processing_graph_id = register_processing_graph(configuration, &operator) .await? .id .to_string(); diff --git a/backend/src/processes/ndvi.rs b/backend/src/processes/ndvi.rs index 8032840..5052f8f 100644 --- a/backend/src/processes/ndvi.rs +++ b/backend/src/processes/ndvi.rs @@ -1,23 +1,20 @@ use anyhow::Result; use geoengine_api_client::{ - apis::{ - configuration::Configuration, ogcwfs_api::wfs_handler, uploads_api::upload_handler, - workflows_api::register_workflow_handler, - }, + apis::{configuration::Configuration, ogcwfs_api::wfs_handler, uploads_api::upload_handler}, models::{ - Aggregation, BandFilter, BandFilterParameters, BandsByNameOrIndex, ColumnNames, - ContinuousMeasurement, Coordinate2D, Default as ColumnNamesDefault, + Aggregation, AggregationFirst, BandFilter, BandFilterParameters, BandsByNameOrIndex, + ColumnNames, ContinuousMeasurement, Coordinate2D, Default as ColumnNamesDefault, DeriveOutRasterSpecsSource, Expression, ExpressionParameters, FeatureAggregationMethod, - FirstAggregation, Fraction, GdalSourceParameters, GeoJson, Interpolation, - InterpolationMethod, InterpolationParameters, InterpolationResolution, Measurement, - MockPointSource, MockPointSourceParameters, MultiBandGdalSource, RasterBandDescriptor, - RasterDataType, RasterOperator, RasterStacker, RasterStackerParameters, - RasterTypeConversion, RasterTypeConversionParameters, RasterVectorJoin, - RasterVectorJoinParameters, RenameBands, Reprojection, ReprojectionParameters, - SingleRasterOrVectorOperator, SingleRasterOrVectorSource, SingleRasterSource, - SingleVectorMultipleRasterSources, SpatialBoundsDerive, SpatialBoundsDeriveNone, - TemporalAggregationMethod, TemporalRasterAggregation, TemporalRasterAggregationParameters, - TimeGranularity, TimeStep, VectorOperator, WfsRequest, WfsService, + GdalSourceParameters, GeoJson, Interpolation, InterpolationMethod, InterpolationParameters, + InterpolationResolution, InterpolationResolutionFraction, Measurement, MockPointSource, + MockPointSourceParameters, MultiBandGdalSource, RasterBandDescriptor, RasterDataType, + RasterOperator, RasterStacker, RasterStackerParameters, RasterTypeConversion, + RasterTypeConversionParameters, RasterVectorJoin, RasterVectorJoinParameters, RenameBands, + Reprojection, ReprojectionParameters, SingleRasterOrVectorOperator, + SingleRasterOrVectorSource, SingleRasterSource, SingleVectorMultipleRasterSources, + SpatialBoundsDerive, SpatialBoundsDeriveNone, TemporalAggregationMethod, + TemporalRasterAggregation, TemporalRasterAggregationParameters, TimeGranularity, TimeStep, + VectorOperator, WfsRequest, WfsService, }, }; use geojson::PointType; @@ -45,7 +42,7 @@ use crate::{ db::{DbHandle, model::ComputationId}, processes::parameters::{Month, PointGeoJsonInput, Year}, state::{CONTEXT, TaskLocalContext}, - util::{error_response, to_api_vector_process}, + util::{error_response, register_processing_graph, to_api_vector_process}, }; /// Calculates the Normalized Difference Vegetation Index (NDVI) and the corrected NDVI (kNDVI) from satellite imagery. @@ -449,7 +446,7 @@ async fn compute_ndvi( .into(), )); - let workflow_id = match register_workflow_handler(configuration, workflow.clone()).await { + let workflow_id = match register_processing_graph(configuration, &workflow).await { Ok(id) => id, Err(e) => { let workflow_json = serde_json::to_string_pretty(&workflow) @@ -561,13 +558,12 @@ fn vector_reprojection_source(coordinate: &PointType) -> VectorOperator { r#type: Default::default(), params: MockPointSourceParameters { points: vec![Coordinate2D::new(coordinate[0], coordinate[1])], - spatial_bounds: SpatialBoundsDerive::None( + spatial_bounds: Some(Box::new(SpatialBoundsDerive::None( SpatialBoundsDeriveNone { r#type: Default::default(), } .into(), - ) - .into(), + ))), } .into(), } @@ -596,8 +592,8 @@ fn ndvi_source() -> RasterOperator { TemporalRasterAggregation { r#type: Default::default(), params: TemporalRasterAggregationParameters { - aggregation: Aggregation::FirstAggregation( - FirstAggregation { + aggregation: Aggregation::First( + AggregationFirst { ignore_no_data: true, r#type: Default::default(), } @@ -627,8 +623,8 @@ fn k_ndvi_source() -> RasterOperator { TemporalRasterAggregation { r#type: Default::default(), params: TemporalRasterAggregationParameters { - aggregation: Aggregation::FirstAggregation( - FirstAggregation { + aggregation: Aggregation::First( + AggregationFirst { ignore_no_data: true, r#type: Default::default(), } @@ -720,7 +716,7 @@ fn scl_source() -> RasterOperator { interpolation: InterpolationMethod::NearestNeighbor, output_origin_reference: None, output_resolution: InterpolationResolution::Fraction( - Fraction { + InterpolationResolutionFraction { r#type: Default::default(), x: 2.0, y: 2.0, diff --git a/backend/src/util.rs b/backend/src/util.rs index 74c876e..d3e66d9 100644 --- a/backend/src/util.rs +++ b/backend/src/util.rs @@ -1,6 +1,6 @@ use geoengine_api_client::models::{ - RasterOperator, TypedOperator, TypedRasterOperator, TypedVectorOperator, VectorOperator, - Workflow, typed_raster_operator::Type as RasterType, typed_vector_operator::Type as VectorType, + ProcessingGraph, RasterOperator, TypedRasterOperator, TypedVectorOperator, VectorOperator, + typed_raster_operator::Type as RasterType, typed_vector_operator::Type as VectorType, }; use serde::Deserialize; use std::ops::Deref; @@ -10,24 +10,76 @@ use tracing_subscriber::{ EnvFilter, filter::Directive, layer::SubscriberExt, util::SubscriberInitExt, }; -/// Converts a Geo Engine operator to an Geo Engine OpenAPI workflow. -pub fn to_api_vector_process(operator: &VectorOperator) -> geoengine_api_client::models::Workflow { - Workflow::TypedOperator(Box::new(TypedOperator::TypedVectorOperator(Box::new( - TypedVectorOperator { - operator: Box::new(operator.clone()), - r#type: VectorType::Vector, - }, - )))) +/// Converts a Geo Engine operator to an Geo Engine OpenAPI processing graph. +pub fn to_api_vector_process( + operator: &VectorOperator, +) -> geoengine_api_client::models::ProcessingGraph { + ProcessingGraph::TypedVectorOperator(Box::new(TypedVectorOperator { + operator: Box::new(operator.clone()), + r#type: VectorType::Vector, + })) } -/// Converts a Geo Engine operator to an Geo Engine OpenAPI workflow. -pub fn to_api_raster_process(operator: &RasterOperator) -> geoengine_api_client::models::Workflow { - Workflow::TypedOperator(Box::new(TypedOperator::TypedRasterOperator(Box::new( - TypedRasterOperator { - operator: Box::new(operator.clone()), - r#type: RasterType::Raster, - }, - )))) +/// Converts a Geo Engine operator to an Geo Engine OpenAPI processing graph. +pub fn to_api_raster_process( + operator: &RasterOperator, +) -> geoengine_api_client::models::ProcessingGraph { + ProcessingGraph::TypedRasterOperator(Box::new(TypedRasterOperator { + operator: Box::new(operator.clone()), + r#type: RasterType::Raster, + })) +} + +/// Registers a processing graph with Geo Engine. +/// +/// The generated client serializes `Aggregation` with a duplicated `type` discriminator +/// (`#[serde(tag = "type")]` on the enum plus the inner structs' own `type` field), which +/// the server rejects. Serializing through a `Value` first folds the duplicate key in +/// `serde_json`'s map serializer, so this posts the normalized value instead of the typed +/// graph. Has the same signature and error handling as the generated +/// `register_workflow_handler`, minus the duplication. +pub async fn register_processing_graph( + configuration: &geoengine_api_client::apis::configuration::Configuration, + processing_graph: &geoengine_api_client::models::ProcessingGraph, +) -> Result< + geoengine_api_client::models::IdResponse, + geoengine_api_client::apis::Error< + geoengine_api_client::apis::workflows_api::RegisterWorkflowHandlerError, + >, +> { + let normalized = serde_json::to_value(processing_graph).expect("processing graph serializes"); + let uri = format!("{}/workflow", configuration.base_path); + + let request_builder = configuration.client.post(&uri); + let request_builder = match &configuration.bearer_access_token { + Some(token) => request_builder.bearer_auth(token), + None => request_builder, + }; + + let response = request_builder + .json(&normalized) + .send() + .await + .map_err(geoengine_api_client::apis::Error::Reqwest)?; + + let status = response.status(); + let content = response + .text() + .await + .map_err(geoengine_api_client::apis::Error::Reqwest)?; + + if status.is_client_error() || status.is_server_error() { + let entity = serde_json::from_str(&content).ok(); + Err(geoengine_api_client::apis::Error::ResponseError( + geoengine_api_client::apis::ResponseContent { + status, + content, + entity, + }, + )) + } else { + serde_json::from_str(&content).map_err(geoengine_api_client::apis::Error::Serde) + } } pub fn error_response( @@ -294,10 +346,7 @@ mod tests { "vector": { "type": "MockPointSource", "params": { - "points": [], - "spatialBounds": { - "type": "derive" - } + "points": [] }, }, "rasters": [{ diff --git a/notebooks/nexgddp_cmip6_ingest.ipynb b/notebooks/nexgddp_cmip6_ingest.ipynb deleted file mode 100644 index edadfdf..0000000 --- a/notebooks/nexgddp_cmip6_ingest.ipynb +++ /dev/null @@ -1,644 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "# NEX-GDDP-CMIP6 Dataset Ingestion\n", - "\n", - "This notebook builds GDAL VRTs over the yearly NEX-GDDP-CMIP6 NetCDF files on S3 and registers them as GeoEngine datasets. Each year has one raster band per day; the VRT concatenates them so band *i* == day *i* of the period, readable via NetCDF-CF metadata without kerchunk or a GDAL >= 3.11.\n", - "\n", - "## Prerequisites\n", - "\n", - "- Python 3.10+\n", - "- Required packages: `s3fs`, `fsspec`, `geoengine`, `rasterio` (only for the optional read-back check)\n", - "- Access to the NEX-GDDP-CMIP6 S3 bucket (public, no credentials required)\n", - "- GeoEngine instance with dataset creation permissions\n" - ] - }, - { - "cell_type": "code", - "execution_count": 1, - "metadata": { - "execution": { - "iopub.execute_input": "2026-09-13T10:00:55.747332Z", - "iopub.status.busy": "2026-09-13T10:00:55.747237Z", - "iopub.status.idle": "2026-09-13T10:00:57.955865Z", - "shell.execute_reply": "2026-09-13T10:00:57.955115Z" - } - }, - "outputs": [], - "source": [ - "import json\n", - "from datetime import datetime\n", - "from pathlib import Path\n", - "\n", - "import fsspec\n", - "import s3fs\n", - "\n", - "import geoengine as ge\n", - "import geoengine_api_client as geoc" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Configuration\n", - "\n", - "Update these paths to match your environment." - ] - }, - { - "cell_type": "code", - "execution_count": 2, - "metadata": { - "execution": { - "iopub.execute_input": "2026-09-13T10:00:57.957414Z", - "iopub.status.busy": "2026-09-13T10:00:57.957183Z", - "iopub.status.idle": "2026-09-13T10:00:57.961033Z", - "shell.execute_reply": "2026-09-13T10:00:57.960514Z" - } - }, - "outputs": [], - "source": [ - "# Unused unless the volume type requires an upload flow (no filesystem path);\n", - "# keep as the local staging dir for that case.\n", - "REFERENCE_PATH = Path(\"/data/nexgddp-cmip6-refs\")\n", - "\n", - "# NEX-GDDP-CMIP6 S3 bucket (public)\n", - "S3_BUCKET = \"nex-gddp-cmip6\"\n", - "S3_PREFIX = \"NEX-GDDP-CMIP6\"\n", - "\n", - "# GeoEngine connection\n", - "GEOENGINE_URL = \"http://localhost:3030/api\"\n", - "GEOENGINE_USER = \"admin@localhost\"\n", - "GEOENGINE_PASSWORD = \"adminadmin\"\n", - "\n", - "# Volume name in GeoEngine\n", - "VOLUME_NAME = \"ge_geo_data\"\n", - "\n", - "# Full model registry: the authoritative id/variant/grid/scenario list. The dev registry\n", - "# (`nexgddp_models.toml`) is trimmed to what is ingested, so ingest reads this one and the\n", - "# trimmed file is regenerated from it afterwards.\n", - "REGISTRY_PATH = Path(\"../backend/conf/nexgddp_models.full.toml\")\n", - "\n", - "try:\n", - " import tomllib\n", - "except ModuleNotFoundError: # Python < 3.11\n", - " import tomli as tomllib\n", - "\n", - "with open(REGISTRY_PATH, \"rb\") as fh:\n", - " REGISTRY = tomllib.load(fh)[\"model\"]\n", - "\n", - "MODELS = [\n", - " {\"id\": m[\"id\"], \"variant\": m[\"variant\"], \"grid\": m[\"grid\"]} for m in REGISTRY\n", - "]\n", - "\n", - "SCENARIOS = sorted({s for m in REGISTRY for s in m[\"scenarios\"]})\n", - "\n", - "# Variables\n", - "VARIABLES = [\"tasmax\", \"tasmin\", \"pr\"]\n", - "\n", - "# Variable metadata\n", - "VARIABLE_CONFIGS = {\n", - " \"tasmax\": {\n", - " \"name\": \"Maximum Temperature\",\n", - " \"measurement\": \"temperature\",\n", - " \"unit\": \"K\",\n", - " \"min\": 200,\n", - " \"max\": 340,\n", - " \"color_map\": \"coolwarm\",\n", - " },\n", - " \"tasmin\": {\n", - " \"name\": \"Minimum Temperature\",\n", - " \"measurement\": \"temperature\",\n", - " \"unit\": \"K\",\n", - " \"min\": 200,\n", - " \"max\": 340,\n", - " \"color_map\": \"coolwarm\",\n", - " },\n", - " \"pr\": {\n", - " \"name\": \"Precipitation\",\n", - " \"measurement\": \"precipitation\",\n", - " \"unit\": \"kg m-2 s-1\",\n", - " \"min\": 0,\n", - " \"max\": 0.01,\n", - " \"color_map\": \"Blues\",\n", - " },\n", - "}\n", - "# ponytail: trial mode keeps AWS traffic low. Set DEV_ONLY = False for the full matrix\n", - "# (all 5 models x 3 scenarios x 3 variables).\n", - "DEV_ONLY = True\n", - "if DEV_ONLY:\n", - " MODELS = MODELS[:1]\n", - " SCENARIOS = SCENARIOS[:2]\n" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Initialize GeoEngine Connection" - ] - }, - { - "cell_type": "code", - "execution_count": 3, - "metadata": { - "execution": { - "iopub.execute_input": "2026-09-13T10:00:58.003318Z", - "iopub.status.busy": "2026-09-13T10:00:58.003155Z", - "iopub.status.idle": "2026-09-13T10:00:58.209417Z", - "shell.execute_reply": "2026-09-13T10:00:58.208860Z" - } - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Using volume: ge_geo_data (path='/mnt/peter/geo_data/')\n" - ] - } - ], - "source": [ - "ge.initialize(\n", - " GEOENGINE_URL,\n", - " credentials=(GEOENGINE_USER, GEOENGINE_PASSWORD),\n", - ")\n", - "\n", - "volume = ge.volume_by_name(VOLUME_NAME)\n", - "print(f\"Using volume: {VOLUME_NAME} (path={volume.path!r})\")\n", - "\n", - "# Reference files must live where GeoEngine can reach them. If the volume has a\n", - "# filesystem path, write refs under it; otherwise an upload step is required.\n", - "DATA_DIR = Path(volume.path) / \"nexgddp-cmip6\" if volume.path else None\n", - "if DATA_DIR is None:\n", - " print(\"Warning: volume has no filesystem path - reference files must be uploaded\")\n", - "\n", - "permission_tuples = [\n", - " (ge.permissions.ANONYMOUS_USER_ROLE_ID, ge.Permission.READ),\n", - " (ge.permissions.REGISTERED_USER_ROLE_ID, ge.Permission.READ),\n", - "]" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Build VRTs\n", - "\n", - "For each model/scenario/variable combination, build a GDAL VRT that concatenates the daily bands of the yearly NetCDF files, streamed from S3 via `/vsicurl`." - ] - }, - { - "cell_type": "code", - "execution_count": 4, - "metadata": { - "execution": { - "iopub.execute_input": "2026-09-13T10:00:58.210829Z", - "iopub.status.busy": "2026-09-13T10:00:58.210721Z", - "iopub.status.idle": "2026-09-13T10:00:58.214208Z", - "shell.execute_reply": "2026-09-13T10:00:58.213714Z" - } - }, - "outputs": [], - "source": [ - "def list_netcdf_files(model_id, scenario, variable, variant, grid):\n", - " \"\"\"List all yearly NetCDF files for a model/scenario/variable combination.\"\"\"\n", - " fs = s3fs.S3FileSystem(anon=True)\n", - " \n", - " # Determine year range based on scenario\n", - " if scenario == \"historical\":\n", - " years = range(1950, 2015)\n", - " else:\n", - " years = range(2015, 2101)\n", - " \n", - " files = []\n", - " for year in years:\n", - " # NEX-GDDP-CMIP6 published corrected `{year}_v2.0.nc` copies for early\n", - " # historical years; prefer them over the uncorrected base file.\n", - " base = (f\"{S3_BUCKET}/{S3_PREFIX}/{model_id}/{scenario}/{variant}/{variable}/\"\n", - " f\"{variable}_day_{model_id}_{scenario}_{variant}_{grid}_\")\n", - " path = next(\n", - " (base + cand for cand in (f\"{year}_v2.0.nc\", f\"{year}.nc\")\n", - " if fs.exists(base + cand)),\n", - " None,\n", - " )\n", - " if path is not None:\n", - " files.append(path)\n", - " else:\n", - " print(f\"Warning: File not found: {base}{year}.nc\")\n", - " \n", - " return sorted(files)" - ] - }, - { - "cell_type": "code", - "execution_count": 5, - "metadata": { - "execution": { - "iopub.execute_input": "2026-09-13T10:00:58.215304Z", - "iopub.status.busy": "2026-09-13T10:00:58.215211Z", - "iopub.status.idle": "2026-09-13T10:00:58.218716Z", - "shell.execute_reply": "2026-09-13T10:00:58.218191Z" - } - }, - "outputs": [], - "source": [ - "\n", - "def create_vrt(files, output_path):\n", - " \"\"\"Build a VRT concatenating the daily bands of yearly NetCDF files.\n", - "\n", - " Each yearly file contributes 365/366 bands (one per day). Concatenated in\n", - " order, band i == day i since the first year, which is exactly what the\n", - " NetCDF-CF metadata (band = 1 + band_offset + steps_between) expects.\n", - " Sources use the canonical VRT ComplexSource form so GDAL actually sources\n", - " the bands (a bare / silently yields NoData),\n", - " and are /vsicurl URLs so the GeoEngine worker streams from S3.\n", - " \"\"\"\n", - " import calendar as _cal\n", - "\n", - " band = 0\n", - " bands_xml = []\n", - " for file_path in files:\n", - " key = file_path[len(S3_BUCKET) + 1:] if file_path.startswith(S3_BUCKET + \"/\") else file_path\n", - " source = f\"/vsicurl/https://{S3_BUCKET}.s3.us-west-2.amazonaws.com/{key}\"\n", - " year = int([t for t in key.split(\"/\")[-1].split(\"_\") if t.isdigit()][-1])\n", - " days = 366 if _cal.isleap(year) else 365\n", - " for d in range(1, days + 1):\n", - " band += 1\n", - " bands_xml.append(\n", - " f''\n", - " f'1e20'\n", - " f''\n", - " f'{source}'\n", - " f'{d}'\n", - " f''\n", - " f''\n", - " f''\n", - " f'1e20'\n", - " f''\n", - " f''\n", - " )\n", - "\n", - " # NEX-GDDP-CMIP6 real grid: 0.25 deg, 1440x600, lon 0..360, lat -60..90\n", - " vrt = (\n", - " '\\n'\n", - " ' EPSG:4326\\n'\n", - " ' 0.0, 0.25, 0, 90.0, 0, -0.25\\n'\n", - " + \"\\n\".join(bands_xml)\n", - " + \"\\n\\n\"\n", - " )\n", - "\n", - " output_path.parent.mkdir(parents=True, exist_ok=True)\n", - " output_path.write_text(vrt)\n", - " print(f\"Created VRT: {output_path} ({band} bands)\")\n", - " return band\n" - ] - }, - { - "cell_type": "code", - "execution_count": 6, - "metadata": { - "execution": { - "iopub.execute_input": "2026-09-13T10:00:58.219803Z", - "iopub.status.busy": "2026-09-13T10:00:58.219709Z", - "iopub.status.idle": "2026-09-13T10:02:25.876624Z", - "shell.execute_reply": "2026-09-13T10:02:25.876102Z" - } - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\n", - "Processing: ACCESS-CM2 / historical / tasmax\n", - "Created VRT: /mnt/peter/geo_data/nexgddp-cmip6/ACCESS-CM2/historical/tasmax.vrt (23741 bands)\n", - "\n", - "Processing: ACCESS-CM2 / historical / tasmin\n", - "Created VRT: /mnt/peter/geo_data/nexgddp-cmip6/ACCESS-CM2/historical/tasmin.vrt (23741 bands)\n", - "\n", - "Processing: ACCESS-CM2 / historical / pr\n", - "Created VRT: /mnt/peter/geo_data/nexgddp-cmip6/ACCESS-CM2/historical/pr.vrt (23741 bands)\n", - "\n", - "Processing: ACCESS-CM2 / ssp245 / tasmax\n", - "Created VRT: /mnt/peter/geo_data/nexgddp-cmip6/ACCESS-CM2/ssp245/tasmax.vrt (31411 bands)\n", - "\n", - "Processing: ACCESS-CM2 / ssp245 / tasmin\n", - "Created VRT: /mnt/peter/geo_data/nexgddp-cmip6/ACCESS-CM2/ssp245/tasmin.vrt (31411 bands)\n", - "\n", - "Processing: ACCESS-CM2 / ssp245 / pr\n", - "Created VRT: /mnt/peter/geo_data/nexgddp-cmip6/ACCESS-CM2/ssp245/pr.vrt (31411 bands)\n" - ] - } - ], - "source": [ - "\n", - "# Create VRTs for all combinations (staging dir as fallback)\n", - "assert DATA_DIR is not None, \\\n", - " \"No volume path - place refs via upload or mount the volume before running\"\n", - "OUTPUT_DIR = DATA_DIR\n", - "\n", - "for model in MODELS:\n", - " for scenario in SCENARIOS:\n", - " for variable in VARIABLES:\n", - " print(f\"\\nProcessing: {model['id']} / {scenario} / {variable}\")\n", - " files = list_netcdf_files(\n", - " model[\"id\"], scenario, variable, model[\"variant\"], model[\"grid\"],\n", - " )\n", - " if not files:\n", - " print(\"No files found, skipping\")\n", - " continue\n", - " output_path = OUTPUT_DIR / model[\"id\"] / scenario / f\"{variable}.vrt\"\n", - " create_vrt(files, output_path)\n" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Register Datasets in GeoEngine\n", - "\n", - "Register each reference file as a GeoEngine dataset." - ] - }, - { - "cell_type": "code", - "execution_count": 7, - "metadata": { - "execution": { - "iopub.execute_input": "2026-09-13T10:02:25.877948Z", - "iopub.status.busy": "2026-09-13T10:02:25.877840Z", - "iopub.status.idle": "2026-09-13T10:02:25.883553Z", - "shell.execute_reply": "2026-09-13T10:02:25.883160Z" - } - }, - "outputs": [], - "source": [ - "\n", - "def build_netcdf_cf_metadata(variable, scenario, year_start, year_end, reference_path):\n", - " \"\"\"Build NetCDF-CF metadata for a band-concatenating VRT registration.\n", - "\n", - " Backing VRT has one band per day since year_start; NetCDF-CF maps time ->\n", - " band via band = 1 + band_offset + steps_between. Requires the VRT's sources\n", - " (https ... .nc via /vsicurl) to pass the worker's curl extension allowlist.\n", - " \"\"\"\n", - " config = VARIABLE_CONFIGS[variable]\n", - "\n", - " # NEX-GDDP-CMIP6 actual grid: 0.25 deg, 1440x600, lon 0..360, lat -60..90\n", - " n_lat, n_lon, dx = 600, 1440, 0.25\n", - " x_min, y_max = 0.0, 90.0\n", - "\n", - " measurement = geoc.Measurement(\n", - " actual_instance=geoc.ContinuousMeasurement(\n", - " type=\"continuous\", measurement=config[\"unit\"], unit=config[\"unit\"]\n", - " )\n", - " )\n", - " start_ms = int(datetime(year_start, 1, 1).timestamp() * 1000)\n", - " end_ms = int(datetime(year_end, 1, 1).timestamp() * 1000) # non-inclusive\n", - " time_dimension = geoc.TimeDimension(\n", - " actual_instance=geoc.Regular(\n", - " origin=start_ms,\n", - " step=geoc.TimeStep(granularity=\"days\", step=1),\n", - " type=\"regular\",\n", - " )\n", - " )\n", - " geo_transform = geoc.GeoTransform(\n", - " origin_coordinate=geoc.Coordinate2D(x=x_min, y=y_max),\n", - " x_pixel_size=dx,\n", - " y_pixel_size=-dx,\n", - " )\n", - " result_descriptor = geoc.RasterResultDescriptor(\n", - " data_type=\"F32\",\n", - " spatial_reference=\"EPSG:4326\",\n", - " time=geoc.TimeDescriptor(\n", - " bounds=geoc.TimeInterval(start=start_ms, end=end_ms),\n", - " dimension=time_dimension,\n", - " ),\n", - " spatial_grid=geoc.SpatialGridDescriptor(\n", - " spatialGrid=geoc.SpatialGridDefinition(\n", - " geoTransform=geo_transform,\n", - " gridBounds=geoc.GridBoundingBox2D(\n", - " topLeftIdx=geoc.GridIdx2D(xIdx=0, yIdx=0),\n", - " bottomRightIdx=geoc.GridIdx2D(xIdx=n_lon - 1, yIdx=n_lat - 1),\n", - " ),\n", - " ),\n", - " descriptor=\"source\",\n", - " ),\n", - " measurement=measurement,\n", - " bands=[geoc.RasterBandDescriptor(name=variable, measurement=measurement)],\n", - " )\n", - "\n", - " params = geoc.GdalDatasetParameters(\n", - " file_path=str(reference_path),\n", - " rasterband_channel=1,\n", - " geo_transform=geo_transform,\n", - " width=n_lon,\n", - " height=n_lat,\n", - " file_not_found_handling=\"NoData\",\n", - " gdal_config_options=[\n", - " [\"CPL_VSIL_CURL_ALLOWED_EXTENSIONS\", \".tif,.tiff,.TIF,.jp2,.ovr,.nc\"],\n", - " ],\n", - " )\n", - "\n", - " return geoc.GdalMetadataNetCdfCf(\n", - " type=\"GdalMetaDataNetCdfCf\",\n", - " result_descriptor=result_descriptor,\n", - " params=params,\n", - " start=start_ms,\n", - " end=end_ms,\n", - " step=geoc.TimeStep(granularity=\"days\", step=1),\n", - " band_offset=0,\n", - " )\n", - "\n", - "def register_dataset(model_id, scenario, variable, reference_path, year_start, year_end):\n", - " \"\"\"Register a dataset in GeoEngine using the volume add-or-replace API.\"\"\"\n", - " config = VARIABLE_CONFIGS[variable]\n", - " dataset_name = f\"nexgddp_cmip6_{model_id}_{scenario}_{variable}\"\n", - " display_name = f\"NEX-GDDP-CMIP6 {config['name']} ({scenario})\"\n", - "\n", - " properties = ge.AddDatasetProperties(\n", - " name=dataset_name,\n", - " display_name=display_name,\n", - " description=(\n", - " f\"Daily NEX-GDDP-CMIP6 {config['name']} projections for {config['measurement']} ({config['unit']}). \"\n", - " f\"Model: {model_id}, Scenario: {scenario}, grid: VRT over yearly NetCDF files\"\n", - " ),\n", - " source_operator=\"GdalSource\",\n", - " symbology=ge.RasterSymbology(\n", - " raster_colorizer=ge.SingleBandRasterColorizer(\n", - " band=0,\n", - " band_colorizer=ge.Colorizer.linear_with_mpl_cmap(\n", - " color_map=config[\"color_map\"],\n", - " min_max=(config[\"min\"], config[\"max\"]),\n", - " n_steps=10,\n", - " ),\n", - " ),\n", - " ),\n", - " )\n", - "\n", - " meta_data = geoc.MetaDataDefinition(\n", - " build_netcdf_cf_metadata(variable, scenario, year_start, year_end, reference_path)\n", - " )\n", - "\n", - " ge.add_or_replace_dataset_with_permissions(\n", - " volume,\n", - " properties,\n", - " meta_data,\n", - " permission_tuples=permission_tuples,\n", - " replace_existing=True,\n", - " timeout=600,\n", - " )\n", - " print(f\"Registered: {dataset_name}\")\n", - " return dataset_name\n" - ] - }, - { - "cell_type": "code", - "execution_count": 8, - "metadata": { - "execution": { - "iopub.execute_input": "2026-09-13T10:02:25.884835Z", - "iopub.status.busy": "2026-09-13T10:02:25.884735Z", - "iopub.status.idle": "2026-09-13T10:02:26.248315Z", - "shell.execute_reply": "2026-09-13T10:02:26.247420Z" - } - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Registered: nexgddp_cmip6_ACCESS-CM2_historical_tasmax\n", - "Registered: nexgddp_cmip6_ACCESS-CM2_historical_tasmin\n", - "Registered: nexgddp_cmip6_ACCESS-CM2_historical_pr\n", - "Registered: nexgddp_cmip6_ACCESS-CM2_ssp245_tasmax\n", - "Registered: nexgddp_cmip6_ACCESS-CM2_ssp245_tasmin\n", - "Registered: nexgddp_cmip6_ACCESS-CM2_ssp245_pr\n", - "\n", - "Total datasets: 6\n" - ] - } - ], - "source": [ - "\n", - "# Register all datasets\n", - "YEAR_RANGES = {\"historical\": (1950, 2015), \"ssp245\": (2015, 2101), \"ssp585\": (2015, 2101)}\n", - "registered_datasets = []\n", - "\n", - "for model in MODELS:\n", - " for scenario in SCENARIOS:\n", - " year_start, year_end = YEAR_RANGES[scenario]\n", - " for variable in VARIABLES:\n", - " rel_ref = Path(\"nexgddp-cmip6\") / model[\"id\"] / scenario / f\"{variable}.vrt\"\n", - " # file is written under DATA_DIR (= volume_root/nexgddp-cmip6);\n", - " # rel_ref is relative to the volume root, hence includes the prefix\n", - " ref_abs = DATA_DIR / model[\"id\"] / scenario / f\"{variable}.vrt\"\n", - " if ref_abs.exists():\n", - " dataset_name = register_dataset(\n", - " model[\"id\"], scenario, variable,\n", - " rel_ref, # path is relative to the volume root\n", - " year_start, year_end,\n", - " )\n", - " registered_datasets.append(dataset_name)\n", - "\n", - "print(f\"\\nTotal datasets: {len(registered_datasets)}\")\n" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Verification\n", - "\n", - "Verify that GDAL can read the reference files." - ] - }, - { - "cell_type": "code", - "execution_count": 9, - "metadata": { - "execution": { - "iopub.execute_input": "2026-09-13T10:02:26.250521Z", - "iopub.status.busy": "2026-09-13T10:02:26.250277Z", - "iopub.status.idle": "2026-09-13T10:02:30.635167Z", - "shell.execute_reply": "2026-09-13T10:02:30.634253Z" - } - }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Bands: 23741, CRS: EPSG:4326, Bounds: BoundingBox(left=0.0, bottom=-60.0, right=360.0, top=90.0)\n", - "Band 1 read OK: min=219.0 max=100000002004087734272.0 (streams first day from S3)\n", - "nexgddp_cmip6_ACCESS-CM2_historical_tasmax: True\n", - "nexgddp_cmip6_ACCESS-CM2_historical_tasmin: True\n", - "nexgddp_cmip6_ACCESS-CM2_historical_pr: True\n" - ] - } - ], - "source": [ - "\n", - "import os\n", - "import rasterio\n", - "\n", - "os.environ[\"CPL_VSIL_CURL_ALLOWED_EXTENSIONS\"] = \".tif,.tiff,.TIF,.jp2,.ovr,.nc\"\n", - "\n", - "test_vrt = DATA_DIR / MODELS[0][\"id\"] / SCENARIOS[0] / f\"{VARIABLES[0]}.vrt\"\n", - "if test_vrt.exists():\n", - " try:\n", - " with rasterio.open(test_vrt) as src:\n", - " print(f\"Bands: {src.count}, CRS: {src.crs}, Bounds: {src.bounds}\")\n", - " data = src.read(1)\n", - " print(f\"Band 1 read OK: min={data.min():.1f} max={data.max():.1f} (streams first day from S3)\")\n", - " except Exception as e:\n", - " print(f\"Failed to open VRT: {e}\")\n", - "else:\n", - " print(f\"Test VRT file not found: {test_vrt}\")\n", - "\n", - "# Confirm GeoEngine sees the registered dataset\n", - "for ds in registered_datasets[:3]:\n", - " info = ge.dataset_info_by_name(ds)\n", - " print(f\"{ds}: {info is not None}\")\n" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Next Steps\n", - "\n", - "1. Verify that the GeoEngine worker can read the VRTs (needs GDAL >= 3.8 in the container image)\n", - "2. Update the `register_dataset` function with proper metadata extraction\n", - "3. Run the full registration (set `DEV_ONLY = False`) to create all 45 datasets, then copy\n", - "`nexgddp_models.full.toml` over `nexgddp_models.toml`\n", - "4. Test the climate-risk process with the new datasets" - ] - } - ], - "metadata": { - "kernelspec": { - "display_name": "biois-notebooks", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.12.3" - } - }, - "nbformat": 4, - "nbformat_minor": 4 -} diff --git a/notebooks/nexgddp_cmip6_ingest_manual.ipynb b/notebooks/nexgddp_cmip6_ingest_manual.ipynb new file mode 100644 index 0000000..cf60f8e --- /dev/null +++ b/notebooks/nexgddp_cmip6_ingest_manual.ipynb @@ -0,0 +1,791 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# NEX-GDDP-CMIP6 Dataset Ingestion (metadata stated by hand)\n", + "\n", + "This is `nexgddp_cmip6_ingest.ipynb` with the **probe call replaced by fields written out in\n", + "Python**. It is the same dataset, the same registration and the same verification; the only\n", + "thing that changed is where the metadata comes from. `MdGdalSource` therefore has to work\n", + "without `/dataset/probe-md` — which is the point of this notebook.\n", + "\n", + "Everything stated below is a property of the NEX-GDDP-CMIP6 dataset that the importer\n", + "already knows: one daily year per file, a 0.25 degree 1440x600 grid wrapping 0..360, CF\n", + "`time` in `days since 1850-01-01` stamped at noon. Nothing is read from a file. The values\n", + "were checked against the real data before being written here — see the notes at each\n", + "constant.\n", + "\n", + "The companion notebook keeps the probe version, so a reviewer can diff the two\n", + "`register_dataset` implementations and see exactly what the probe saves. The probe branch\n", + "also fixes the single-array band name, so the two agree once it lands.\n" + ], + "id": "md00" + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import datetime as dt\n", + "import json\n", + "import re\n", + "from pathlib import Path\n", + "\n", + "import fsspec\n", + "import numpy as np\n", + "import s3fs\n", + "\n", + "import geoengine as ge\n", + "import geoengine_api_client as geoc\n" + ], + "id": "md1" + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Configuration\n", + "\n", + "Update these paths to match your environment.\n" + ], + "id": "md02" + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# NEX-GDDP-CMIP6 S3 bucket (public). Files are registered as external data, so GeoEngine\n", + "# streams them through GDAL's /vsicurl rather than copying them onto a volume.\n", + "S3_BUCKET = \"nex-gddp-cmip6\"\n", + "S3_REGION = \"us-west-2\"\n", + "S3_PREFIX = \"NEX-GDDP-CMIP6\"\n", + "\n", + "# GeoEngine connection\n", + "GEOENGINE_URL = \"http://localhost:3030/api\"\n", + "GEOENGINE_USER = \"admin@localhost\"\n", + "GEOENGINE_PASSWORD = \"adminadmin\"\n", + "\n", + "# Full model registry: the authoritative id/variant/grid/scenario list. The dev registry\n", + "# (`nexgddp_models.toml`) is trimmed to what is ingested, so ingest reads this one and the\n", + "# trimmed file is regenerated from it afterwards.\n", + "REGISTRY_PATH = Path(\"../backend/conf/nexgddp_models.full.toml\")\n", + "\n", + "try:\n", + " import tomllib\n", + "except ModuleNotFoundError: # Python < 3.11\n", + " import tomli as tomllib\n", + "\n", + "with open(REGISTRY_PATH, \"rb\") as fh:\n", + " REGISTRY = tomllib.load(fh)[\"model\"]\n", + "\n", + "MODELS = [{\"id\": m[\"id\"], \"variant\": m[\"variant\"], \"grid\": m[\"grid\"]} for m in REGISTRY]\n", + "\n", + "SCENARIOS = sorted({s for m in REGISTRY for s in m[\"scenarios\"]})\n", + "\n", + "# Variables. Each one lives in its own set of yearly files, so each becomes its own dataset\n", + "# with a single band and z = time.\n", + "VARIABLES = [\"tasmax\", \"tasmin\", \"pr\"]\n", + "\n", + "# Variable metadata. `long_name` and `unit` are the CF attributes the files carry - the\n", + "# probe would have used exactly these for the band, so they are stated here rather than\n", + "# derived, and the band is what a reader sees in the layer listing.\n", + "VARIABLE_CONFIGS = {\n", + " \"tasmax\": {\n", + " \"name\": \"Maximum Temperature\",\n", + " \"long_name\": \"Daily Maximum Near-Surface Air Temperature\",\n", + " \"measurement\": \"temperature\",\n", + " \"unit\": \"K\",\n", + " \"min\": 200,\n", + " \"max\": 340,\n", + " \"color_map\": \"coolwarm\",\n", + " },\n", + " \"tasmin\": {\n", + " \"name\": \"Minimum Temperature\",\n", + " \"long_name\": \"Daily Minimum Near-Surface Air Temperature\",\n", + " \"measurement\": \"temperature\",\n", + " \"unit\": \"K\",\n", + " \"min\": 200,\n", + " \"max\": 340,\n", + " \"color_map\": \"coolwarm\",\n", + " },\n", + " \"pr\": {\n", + " \"name\": \"Precipitation\",\n", + " \"long_name\": \"Precipitation\",\n", + " \"measurement\": \"precipitation\",\n", + " \"unit\": \"kg m-2 s-1\",\n", + " \"min\": 0,\n", + " \"max\": 0.01,\n", + " \"color_map\": \"Blues\",\n", + " },\n", + "}\n", + "\n", + "# Number of daily slices read per request. A slice of this grid is 600x1440 float32 = 3.4 MB,\n", + "# so a batch is one IPC message: 4 is ~14 MB, 16 would be ~55 MB and ~22 s cold over the\n", + "# network. Queries for fewer days than this still read only the days they ask for.\n", + "#\n", + "# This is a *dataset* property (`maxZBatchSize`), not an operator parameter: a batch is sized\n", + "# against the slice size of the data, so whoever registers the dataset decides it and every\n", + "# consumer workflow inherits it.\n", + "MAX_Z_BATCH_SIZE = 4\n", + "\n", + "# GDAL must be allowed to read .nc over /vsicurl.\n", + "VSICURL_ALLOWED_EXTENSIONS = \".tif,.tiff,.TIF,.jp2,.ovr,.nc\"\n", + "\n", + "# ponytail: trial mode keeps AWS traffic low. Set DEV_ONLY = False for the full matrix\n", + "# (all 5 models x 3 scenarios x 3 variables).\n", + "#\n", + "# Registration no longer opens every file, so this is no longer a wall-clock cost of probing -\n", + "# but the *verification* still reads a real window per dataset, and a full run reads 45 of\n", + "# them. DEV_ONLY therefore still trims the matrix and caps the years per scenario.\n", + "# `YEARS_PER_SCENARIO` is None for a full run.\n", + "#\n", + "# 3 years gives two file *boundaries* per scenario, which is the minimum that still proves\n", + "# file concatenation: a read window straddling a boundary spans two files in one batch.\n", + "YEARS_PER_SCENARIO = 5\n", + "DEV_ONLY = True\n", + "if DEV_ONLY:\n", + " MODELS = MODELS[:1]\n", + " SCENARIOS = SCENARIOS[:2] # historical + ssp245\n", + " VARIABLES = VARIABLES[:1] # tasmax\n", + "else:\n", + " YEARS_PER_SCENARIO = None\n" + ], + "id": "md3" + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Initialize GeoEngine Connection\n" + ], + "id": "md04" + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "ge.initialize(\n", + " GEOENGINE_URL,\n", + " credentials=(GEOENGINE_USER, GEOENGINE_PASSWORD),\n", + ")\n", + "\n", + "# The files stay on S3; `external` registers them by reference.\n", + "DATA_STORE = \"external\"\n", + "\n", + "permission_tuples = [\n", + " (ge.permissions.ANONYMOUS_USER_ROLE_ID, ge.Permission.READ),\n", + " (ge.permissions.REGISTERED_USER_ROLE_ID, ge.Permission.READ),\n", + "]\n" + ], + "id": "md5" + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## List Files\n", + "\n", + "For each model/scenario/variable combination, list the yearly NetCDF files. Nothing is\n", + "probed: the year in each file name is the one fact taken from the files, and it is what\n", + "decides the file's time axis.\n", + "\n", + "That is one S3 listing per combination and no GDAL at all - which is the difference between\n", + "this notebook and its probe companion.\n" + ], + "id": "md06" + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "def list_netcdf_files(model_id, scenario, variable, variant, grid):\n", + " \"\"\"List all yearly NetCDF files for a model/scenario/variable combination.\"\"\"\n", + " fs = s3fs.S3FileSystem(anon=True)\n", + "\n", + " # Determine year range based on scenario\n", + " if scenario == \"historical\":\n", + " years = range(1950, 2015)\n", + " else:\n", + " years = range(2015, 2101)\n", + "\n", + " files = []\n", + " for year in years:\n", + " # NEX-GDDP-CMIP6 published corrected `{year}_v2.0.nc` copies for early\n", + " # historical years; prefer them over the uncorrected base file.\n", + " base = (f\"{S3_BUCKET}/{S3_PREFIX}/{model_id}/{scenario}/{variant}/{variable}/\"\n", + " f\"{variable}_day_{model_id}_{scenario}_{variant}_{grid}_\")\n", + " path = next(\n", + " (base + cand for cand in (f\"{year}_v2.0.nc\", f\"{year}.nc\")\n", + " if fs.exists(base + cand)),\n", + " None,\n", + " )\n", + " if path is not None:\n", + " files.append(path)\n", + " else:\n", + " print(f\"Warning: File not found: {base}{year}.nc\")\n", + " # a dev run stops after the cap; a full run keeps every year\n", + " if YEARS_PER_SCENARIO is not None and len(files) >= YEARS_PER_SCENARIO:\n", + " break\n", + "\n", + " return sorted(files)\n", + "\n", + "\n", + "def vsicurl_url(bucket_key):\n", + " \"\"\"The /vsicurl path GeoEngine opens for a bucket key.\n", + "\n", + " s3fs keys start with the bucket name (`nex-gddp-cmip6/NEX-GDDP-CMIP6/...`), but the\n", + " bucket is already in the hostname of an S3 URL, so the key must not repeat it --\n", + " doing so yields a 404 from GDAL.\n", + " \"\"\"\n", + " bucket, key = bucket_key.split(\"/\", 1)\n", + " return f\"/vsicurl/https://{bucket}.s3.{S3_REGION}.amazonaws.com/{key}\"\n", + "\n", + "\n", + "def collect_targets():\n", + " \"\"\"Every (model, scenario, variable) combination that has files on S3.\"\"\"\n", + " targets = []\n", + " for model in MODELS:\n", + " for scenario in SCENARIOS:\n", + " for variable in VARIABLES:\n", + " files = list_netcdf_files(\n", + " model[\"id\"], scenario, variable, model[\"variant\"], model[\"grid\"],\n", + " )\n", + " if not files:\n", + " print(f\"No files found, skipping {model['id']}/{scenario}/{variable}\")\n", + " continue\n", + " targets.append((model, scenario, variable, files))\n", + " return targets\n", + "\n", + "\n", + "TARGETS = collect_targets()\n", + "for model, scenario, variable, files in TARGETS:\n", + " print(f\"{model['id']} / {scenario} / {variable}: {len(files)} yearly files\")\n" + ], + "id": "md7" + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Declare the Dataset Metadata\n", + "\n", + "`declare_md_dataset` states every field the probe would have derived. Three of them are easy\n", + "to get wrong, so they are spelled out:\n", + "\n", + "1. **Two geo transforms, and they differ.** `params.geoTransform` is the grid *as stored*\n", + " (CF `lat` ascends, so `y_pixel_size` is positive and the array is south-up);\n", + " `resultDescriptor.spatialGrid` is the grid *as presented* (always north-up, longitudes\n", + " re-labelled -180..180 because the stored coverage wraps 0..360). Both origins are the\n", + " first cell's **edge**, not its centre: `lon` centres start at 0.125, so the edge is\n", + " `0.125 - 0.25/2 = 0.0`.\n", + "\n", + "2. **The time values are stamped at noon.** CF stores daily samples at the cell centre, so\n", + " the files' `time` is `days since 1850-01-01` with a `.5` offset: the 1950 file's first\n", + " value is 36524.5, i.e. `1950-01-01T12:00Z`. An interval is therefore **noon to noon**,\n", + " not midnight to midnight. Slice j is day j of the year either way.\n", + "\n", + "3. **`timeSteps` is one interval per z slice**, not one for the whole file, and\n", + " `timeDescriptor` must reproduce them exactly - the server compares them and rejects a row\n", + " where they disagree.\n" + ], + "id": "md08" + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "def build_symbology(variable):\n", + " config = VARIABLE_CONFIGS[variable]\n", + " return ge.RasterSymbology(\n", + " raster_colorizer=ge.SingleBandRasterColorizer(\n", + " band=0,\n", + " band_colorizer=ge.Colorizer.linear_with_mpl_cmap(\n", + " color_map=config[\"color_map\"],\n", + " min_max=(config[\"min\"], config[\"max\"]),\n", + " n_steps=10,\n", + " ),\n", + " ),\n", + " )\n", + "\n", + "\n", + "# --- the dataset's grid ----------------------------------------------------\n", + "#\n", + "# 0.25 degree cells, 1440 x 600, covering lon 0..360 and lat -60..90.\n", + "N_LON, N_LAT = 1440, 600\n", + "DEG = 0.25\n", + "\n", + "# The stored grid: `lat` ascends (-59.875 .. 89.875), so the array is south-up and\n", + "# `y_pixel_size` is positive. Origins are cell edges.\n", + "STORED_GEO_TRANSFORM = geoc.GeoTransform(\n", + " origin_coordinate=geoc.Coordinate2D(x=0.0, y=-60.0),\n", + " x_pixel_size=DEG,\n", + " y_pixel_size=DEG,\n", + ")\n", + "\n", + "# The presented grid: north-up, longitudes shifted to -180..180 (the stored coverage wraps\n", + "# 0..360). `-60.0 + N_LAT * DEG` is the north edge.\n", + "PRESENTED_GEO_TRANSFORM = geoc.GeoTransform(\n", + " origin_coordinate=geoc.Coordinate2D(x=-180.0, y=90.0),\n", + " x_pixel_size=DEG,\n", + " y_pixel_size=-DEG,\n", + ")\n", + "\n", + "# The presented footprint of every row, in spatial coordinates.\n", + "PRESENTED_FOOTPRINT = geoc.SpatialPartition2D(\n", + " upperLeftCoordinate=geoc.Coordinate2D(x=-180.0, y=90.0),\n", + " lowerRightCoordinate=geoc.Coordinate2D(x=180.0, y=-60.0),\n", + ")\n", + "\n", + "DAY_MS = 86_400_000\n", + "NOON_MS = 12 * 3_600_000\n", + "\n", + "\n", + "def file_year(key):\n", + " \"\"\"The calendar year a yearly NEX-GDDP file covers, from its name.\n", + "\n", + " The name ends in `_{year}.nc` or `_{year}_v2.0.nc`. This is the one fact taken from the\n", + " file name rather than known from the dataset's convention.\n", + " \"\"\"\n", + " match = re.search(r\"_(\\d{4})(?:_v\\d+\\.\\d+)?\\.nc$\", key)\n", + " if match is None:\n", + " raise ValueError(f\"no year in file name: {key}\")\n", + " return int(match.group(1))\n", + "\n", + "\n", + "def year_slice_start_ms(year):\n", + " \"\"\"ms of `year-01-01T12:00:00Z`: the file's first time value is *noon*, not midnight.\n", + "\n", + " Checked against the data: the 1950 file's first value is 36524.5 and\n", + " `1850-01-01T00:00Z + 36524.5 days` is `1950-01-01T12:00Z`.\n", + " \"\"\"\n", + " return int(dt.datetime(year, 1, 1, tzinfo=dt.timezone.utc).timestamp() * 1000) + NOON_MS\n", + "\n", + "\n", + "def file_time_steps(year):\n", + " \"\"\"One interval per daily slice of the file covering `year`, noon to noon.\n", + "\n", + " Slice j of that file is day j of the year. `n_days` follows the calendar (366 in a leap\n", + " year), and the server checks it against the array's z size when the row is added - so a\n", + " misread year fails at registration, not at query time.\n", + " \"\"\"\n", + " n_days = (dt.date(year + 1, 1, 1) - dt.date(year, 1, 1)).days\n", + " start = year_slice_start_ms(year)\n", + " return [\n", + " geoc.TimeInterval(start=start + j * DAY_MS, end=start + (j + 1) * DAY_MS)\n", + " for j in range(n_days)\n", + " ]\n", + "\n", + "\n", + "def daily_descriptor(start_ms, end_ms):\n", + " \"\"\"The descriptor the server derives from `file_time_steps`, stated the same way.\n", + "\n", + " It is compared with the steps *exactly*, so it has to be `Regular(origin=first start,\n", + " step=1 day)` spanning `[first start, last end)`.\n", + " \"\"\"\n", + " return geoc.TimeDescriptor(\n", + " bounds=geoc.TimeInterval(start=start_ms, end=end_ms),\n", + " dimension=geoc.TimeDimension(\n", + " actual_instance=geoc.Regular(\n", + " origin=start_ms,\n", + " step=geoc.TimeStep(granularity=\"days\", step=1),\n", + " type=\"regular\",\n", + " )\n", + " ),\n", + " )\n", + "\n", + "\n", + "def declare_md_dataset(variable, files):\n", + " \"\"\"State the metadata by hand instead of probing the files.\n", + "\n", + " One row per yearly file, covering all of that file's z slices. `leadingPrefix=[]` means\n", + " a 3D array; a 4D `(time, depth, lat, lon)` file would use `[2]` to read depth 2.\n", + " \"\"\"\n", + " config = VARIABLE_CONFIGS[variable]\n", + " measurement = geoc.Measurement(\n", + " actual_instance=geoc.ContinuousMeasurement(\n", + " type=\"continuous\", measurement=config[\"long_name\"], unit=config[\"unit\"]\n", + " )\n", + " )\n", + "\n", + " tiles = []\n", + " for i, key in enumerate(files):\n", + " year = file_year(key)\n", + " steps = file_time_steps(year)\n", + " tiles.append(\n", + " geoc.AddDatasetMdTile(\n", + " spatial_partition=PRESENTED_FOOTPRINT,\n", + " band=0,\n", + " z_index=i,\n", + " array_name=variable,\n", + " arrayGroup=None,\n", + " leadingPrefix=[],\n", + " time_descriptor=daily_descriptor(steps[0].start, steps[-1].end),\n", + " time_steps=steps,\n", + " params=geoc.GdalDatasetParameters(\n", + " file_path=vsicurl_url(key),\n", + " rasterband_channel=1,\n", + " geo_transform=STORED_GEO_TRANSFORM,\n", + " width=N_LON,\n", + " height=N_LAT,\n", + " file_not_found_handling=\"NoData\",\n", + " # `None` is not \"unknown\": the reader falls back to the file's own\n", + " # `_FillValue` (1e20 in these files), which is what the probe stored.\n", + " noDataValue=None,\n", + " properties_mapping=None,\n", + " gdal_open_options=None,\n", + " gdal_config_options=[\n", + " [\"CPL_VSIL_CURL_ALLOWED_EXTENSIONS\", VSICURL_ALLOWED_EXTENSIONS]\n", + " ],\n", + " allow_alphaband_as_mask=False,\n", + " ),\n", + " )\n", + " )\n", + "\n", + " # the dataset's axis is the concatenation of the files' own axes\n", + " first_start = tiles[0].time_steps[0].start\n", + " last_end = tiles[-1].time_steps[-1].end\n", + "\n", + " meta_data = geoc.MetaDataDefinition(\n", + " actual_instance=geoc.GdalMdMetaData(\n", + " type=\"GdalMdMetaData\",\n", + " result_descriptor=geoc.RasterResultDescriptor(\n", + " data_type=\"F32\",\n", + " spatial_reference=\"EPSG:4326\",\n", + " time=daily_descriptor(first_start, last_end),\n", + " spatial_grid=geoc.SpatialGridDescriptor(\n", + " spatialGrid=geoc.SpatialGridDefinition(\n", + " geoTransform=PRESENTED_GEO_TRANSFORM,\n", + " gridBounds=geoc.GridBoundingBox2D(\n", + " topLeftIdx=geoc.GridIdx2D(xIdx=0, yIdx=0),\n", + " bottomRightIdx=geoc.GridIdx2D(xIdx=N_LON - 1, yIdx=N_LAT - 1),\n", + " ),\n", + " ),\n", + " descriptor=\"source\",\n", + " ),\n", + " bands=[geoc.RasterBandDescriptor(name=config[\"long_name\"], measurement=measurement)],\n", + " ),\n", + " z_role=\"variable\",\n", + " wrap=True,\n", + " maxZBatchSize=MAX_Z_BATCH_SIZE,\n", + " cacheTtl=None,\n", + " )\n", + " )\n", + " return meta_data, tiles\n" + ], + "id": "md9a" + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "def register_dataset(model_id, scenario, variable, files):\n", + " \"\"\"State the metadata and register it as an `MdGdalSource` dataset.\"\"\"\n", + " config = VARIABLE_CONFIGS[variable]\n", + " dataset_name = f\"nexgddp_cmip6_{model_id}_{scenario}_{variable}\"\n", + "\n", + " meta_data, tiles = declare_md_dataset(variable, files)\n", + " md = meta_data.actual_instance\n", + " bounds = md.result_descriptor.time.bounds\n", + " print(\n", + " f\" declared: {len(tiles)} files, z_role={md.z_role}, wrap={md.wrap}, \"\n", + " f\"maxZBatchSize={md.max_z_batch_size}, \"\n", + " f\"time {dt.datetime.fromtimestamp(bounds.start / 1000, tz=None).date()} .. \"\n", + " f\"{dt.datetime.fromtimestamp(bounds.end / 1000, tz=None).date()}, \"\n", + " f\"bands {[b.name for b in md.result_descriptor.bands]}\"\n", + " )\n", + "\n", + " properties = ge.AddDatasetProperties(\n", + " name=dataset_name,\n", + " display_name=f\"NEX-GDDP-CMIP6 {config['name']} ({scenario})\",\n", + " description=(\n", + " f\"Daily NEX-GDDP-CMIP6 {config['name']} projections for {config['measurement']} \"\n", + " f\"({config['unit']}). Model: {model_id}, Scenario: {scenario}, \"\n", + " f\"one MD array file per year read through GDAL's multidimensional API\"\n", + " ),\n", + " source_operator=\"MdGdalSource\",\n", + " symbology=build_symbology(variable),\n", + " )\n", + "\n", + " dataset = ge.add_or_replace_dataset_with_permissions(\n", + " DATA_STORE,\n", + " properties,\n", + " meta_data,\n", + " permission_tuples=permission_tuples,\n", + " replace_existing=True,\n", + " timeout=600,\n", + " )\n", + "\n", + " ge.add_md_dataset_tiles(dataset, tiles, timeout=600)\n", + " print(f\" registered: {dataset}\")\n", + " return dataset\n", + "\n", + "\n", + "# Register all datasets\n", + "registered_datasets = []\n", + "\n", + "for model, scenario, variable, files in TARGETS:\n", + " print(f\"\\nProcessing: {model['id']} / {scenario} / {variable}\")\n", + " registered_datasets.append(register_dataset(model[\"id\"], scenario, variable, files))\n", + "\n", + "print(f\"\\nTotal datasets: {len(registered_datasets)}\")\n" + ], + "id": "md9b" + }, + { + "cell_type": "markdown", + "metadata": {}, + "id": "md11", + "source": [ + "## Verification\n", + "\n", + "Read one window per registered dataset, straddling a year boundary. Since the metadata is\n", + "stated rather than derived, this also has to check that the statement *landed* - so the\n", + "metadata round-trip is asserted before any pixel is read.\n", + "\n", + "Three things are checked, in order:\n", + "\n", + "1. **The stated metadata is what the server stored.** `z_role`, `wrap`, `maxZBatchSize`, the\n", + " band name, and the dataset's time bounds `[noon(Y_first), noon(Y_last+1))`. A wrong axis\n", + " length or a wrong band shows up here, without reading a pixel.\n", + "2. **The window straddles a file boundary** - file 1 ends and file 2 begins at the first\n", + " year boundary - and `READ_DAYS > MAX_Z_BATCH_SIZE`, so the crossing is split across worker\n", + " requests too. The days must be consecutive, one step each, on one band, and tiled.\n", + "3. **The stamps are the stated ones.** Each returned tile's `time_start_ms` must equal the\n", + " noon-based stamp computed above, not merely the right date: a 12-hour drift (midnight\n", + " intervals instead of noon) is exactly the mistake that would otherwise go unnoticed.\n", + "\n", + "Values are then checked for being real data: inside the configured range, varying, and\n", + "identical when the same window is read twice.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "id": "md12", + "source": [ + "READ_DAYS = 12 # > MAX_Z_BATCH_SIZE, so the read spans more than one worker request\n", + "\n", + "\n", + "def first_boundary_window(time_bounds):\n", + " \"\"\"A [start, end) window centred on the first year boundary inside the dataset.\"\"\"\n", + " start = dt.datetime.fromtimestamp(time_bounds.start / 1000, tz=dt.timezone.utc)\n", + " boundary = start.replace(year=start.year + 1) # file 1 -> file 2\n", + " half = dt.timedelta(days=READ_DAYS // 2)\n", + " return boundary - half, boundary + half\n", + "\n", + "\n", + "async def read_window(workflow, window_start, window_end):\n", + " \"\"\"Stream a global window and group the tiles by day.\"\"\"\n", + " bbox = ge.QueryRectangle(\n", + " ge.BoundingBox2D(-180.0, -90.0, 180.0, 90.0),\n", + " ge.TimeInterval(window_start, window_end),\n", + " )\n", + " per_day = {}\n", + " async for tile in workflow.raster_stream(bbox, open_timeout=600):\n", + " day = str(tile.time.start)[:10]\n", + " per_day.setdefault(day, []).append(tile)\n", + " return per_day\n", + "\n", + "\n", + "def stamp_ms(tile):\n", + " \"\"\"A tile's time stamp in ms (`time_start_ms` is a numpy datetime64).\"\"\"\n", + " return int(np.datetime64(tile.time_start_ms, \"ms\").astype(\"int64\"))\n", + "\n", + "\n", + "async def verify_dataset(dataset_name, variable, files):\n", + " \"\"\"Assert the stated metadata landed, then read a boundary-crossing window.\"\"\"\n", + " config = VARIABLE_CONFIGS[variable]\n", + " info = ge.dataset_info_by_name(dataset_name)\n", + " assert info is not None, f\"{dataset_name} is not registered\"\n", + "\n", + " # `result_descriptor` is a oneOf wrapper; the concrete descriptor is under\n", + " # `actual_instance`\n", + " rd = info.result_descriptor.actual_instance\n", + " print(f\"\\n{dataset_name}\")\n", + " print(f\" data_type={rd.data_type} bands={[b.name for b in rd.bands]} srs={rd.spatial_reference}\")\n", + "\n", + " # --- 1. the stated metadata is what the server stored -------------------\n", + " #\n", + " # The dataset listing exposes `result_descriptor` only: `zRole`, `wrap` and\n", + " # `maxZBatchSize` are dataset-internal and cannot be read back. They are still exercised\n", + " # by the read below - `wrap` is why the values land in the presented -180..180 frame,\n", + " # `zRole` is why a time axis exists at all, and `maxZBatchSize` is why a 12-day window\n", + " # is split across worker requests.\n", + " assert [b.name for b in rd.bands] == [config[\"long_name\"]], (\n", + " f\"band is {[b.name for b in rd.bands]}, expected {[config['long_name']]}\"\n", + " )\n", + " assert rd.spatial_reference == \"EPSG:4326\", rd.spatial_reference\n", + " assert rd.data_type == \"F32\", rd.data_type\n", + " gb = rd.spatial_grid.spatial_grid.grid_bounds\n", + " assert (gb.top_left_idx.x_idx, gb.top_left_idx.y_idx) == (0, 0), gb\n", + " assert (gb.bottom_right_idx.x_idx, gb.bottom_right_idx.y_idx) == (N_LON - 1, N_LAT - 1), (\n", + " f\"grid bounds {gb} do not describe a {N_LON}x{N_LAT} dataset\"\n", + " )\n", + "\n", + " first_year, last_year = file_year(files[0]), file_year(files[-1])\n", + " assert rd.time.bounds.start == year_slice_start_ms(first_year), (\n", + " f\"axis starts at {rd.time.bounds.start}, expected noon of {first_year}-01-01 \"\n", + " f\"({year_slice_start_ms(first_year)})\"\n", + " )\n", + " assert rd.time.bounds.end == year_slice_start_ms(last_year + 1), (\n", + " f\"axis ends at {rd.time.bounds.end}, expected noon of {last_year + 1}-01-01\"\n", + " )\n", + " n_declared = sum(len(file_time_steps(y)) for y in range(first_year, last_year + 1))\n", + " print(f\" axis: {n_declared} daily slices, \"\n", + " f\"{dt.date(first_year, 1, 1)} .. {dt.date(last_year, 12, 31)} (noon to noon)\")\n", + "\n", + " # the workflow is assembled from typed operators, exactly like the Rust API expects it\n", + " workflow = ge.register_workflow(\n", + " workflow=ge.workflow_builder.operators.MdGdalSource(dataset=dataset_name)\n", + " )\n", + "\n", + " # --- 2. a window straddling the first file boundary --------------------\n", + " window_start, window_end = first_boundary_window(rd.time.bounds)\n", + " print(f\" reading {window_start.date()} .. {window_end.date()} (crosses a year boundary)\")\n", + "\n", + " per_day = await read_window(workflow, window_start, window_end)\n", + "\n", + " expected_days = (window_end - window_start).days\n", + " assert len(per_day) == expected_days, (\n", + " f\"expected {expected_days} daily steps, got {len(per_day)}: {sorted(per_day)}\"\n", + " )\n", + " days = sorted(per_day)\n", + " assert days[0] == window_start.date().isoformat(), (\n", + " f\"first step is {days[0]}, expected {window_start.date()}\"\n", + " )\n", + " parsed = [dt.date.fromisoformat(d) for d in days]\n", + " assert all((b - a).days == 1 for a, b in zip(parsed, parsed[1:])), f\"gap in {days}\"\n", + "\n", + " tiles_per_day = {len(v) for v in per_day.values()}\n", + " assert len(tiles_per_day) == 1, f\"inconsistent tiling per day: {tiles_per_day}\"\n", + " n_tiles = tiles_per_day.pop()\n", + " assert n_tiles > 1, \"a global 0.25-degree query must be tiled, got a single tile\"\n", + "\n", + " bands = {t.band for tiles in per_day.values() for t in tiles}\n", + " assert bands == {0}, f\"{variable} is single-band, got bands {bands}\"\n", + "\n", + " # --- 3. the stamps are the stated noon-to-noon ones --------------------\n", + " for day, date in zip(days, parsed):\n", + " expected_ms = (\n", + " year_slice_start_ms(date.year)\n", + " + (date - dt.date(date.year, 1, 1)).days * DAY_MS\n", + " )\n", + " for t in per_day[day]:\n", + " assert stamp_ms(t) == expected_ms, (\n", + " f\"{day}: tile stamped {stamp_ms(t)}, expected noon {expected_ms}\"\n", + " )\n", + " assert stamp_ms_end(t) - expected_ms == DAY_MS, (\n", + " f\"{day}: tile does not span exactly one day\"\n", + " )\n", + " print(f\" stamps: all {len(days)} days start at the stated noon\")\n", + "\n", + " # --- values: inside the range the symbology was configured for -----------\n", + " values = np.concatenate(\n", + " [np.asarray(t.data, dtype=\"float64\").ravel() for tiles in per_day.values() for t in tiles]\n", + " )\n", + " finite = values[np.isfinite(values)]\n", + " assert finite.size, \"no finite pixels at all -- the read returned only nodata\"\n", + "\n", + " lo, hi = float(finite.min()), float(finite.max())\n", + " print(f\" {len(per_day)} days x {n_tiles} tiles, band {bands}\")\n", + " print(f\" value range={lo:.2f}..{hi:.2f} {config['unit']} \"\n", + " f\"(symbology {config['min']}..{config['max']})\")\n", + "\n", + " # Not all returned pixels are data, and the two reasons are unrelated - so they are\n", + " # reported separately rather than as one \"padding\" ratio that looks alarming but is not.\n", + " #\n", + " # * Tiles are anchored at world (0,0) with a 512 px tile size (Geo Engine's tiling\n", + " # origin reference), while this dataset's presented grid is -180..180 x 60S..90N. So the\n", + " # tiles cover a larger area than the data and part of every tile is structural padding.\n", + " # * NEX-GDDP-CMIP6 itself masks a large part of the globe with a nodata sentinel, and the\n", + " # reader turns nodata into NaN. That is data, not plumbing.\n", + " #\n", + " # Structural coverage is therefore the dataset area over the area the tiles span:\n", + " gb = rd.spatial_grid.spatial_grid.grid_bounds\n", + " tl, br = gb.top_left_idx, gb.bottom_right_idx\n", + " data_px = (br.y_idx - tl.y_idx + 1) * (br.x_idx - tl.x_idx + 1)\n", + " tile_px = values.size // len(per_day) # the tile block the query returns per day\n", + " structural = data_px / tile_px\n", + " nodata = finite.size / (len(per_day) * data_px)\n", + " print(f\" tiles span {tile_px / data_px:.2f}x the dataset area \"\n", + " f\"-> structural coverage {structural:.1%}\")\n", + " print(f\" of the dataset area {nodata:.1%} carries data \"\n", + " f\"-> {structural * nodata:.1%} of all returned pixels are values\")\n", + "\n", + " # generous bounds: this asserts the read is real data, not that it is bit-exact\n", + " assert config[\"min\"] <= lo and hi <= config[\"max\"], (\n", + " f\"values {lo:.2f}..{hi:.2f} outside the plausible {config['min']}..{config['max']} \"\n", + " f\"for {variable}\"\n", + " )\n", + " # a global field should vary, not be a constant fill\n", + " assert hi - lo > 1.0, f\"suspiciously flat field: {lo:.2f}..{hi:.2f}\"\n", + "\n", + " # --- determinism: the same window twice must agree (exercises the cache hint) ---\n", + " again = await read_window(workflow, window_start, window_end)\n", + " assert sorted(again) == days, \"a repeated read returned different days\"\n", + " for day in days:\n", + " a = np.sort(np.concatenate([np.asarray(t.data, dtype=\"float64\").ravel() for t in per_day[day]]))\n", + " b = np.sort(np.concatenate([np.asarray(t.data, dtype=\"float64\").ravel() for t in again[day]]))\n", + " assert np.array_equal(a, b, equal_nan=True), f\"repeated read of {day} differs\"\n", + "\n", + " print(\" OK\")\n", + " return per_day\n", + "\n", + "\n", + "def stamp_ms_end(tile):\n", + " return int(np.datetime64(tile.time_end_ms, \"ms\").astype(\"int64\"))\n", + "\n", + "\n", + "results = {}\n", + "for (model, scenario, variable, files), dataset_name in zip(TARGETS, registered_datasets):\n", + " results[(scenario, variable)] = await verify_dataset(dataset_name, variable, files)\n", + "\n", + "print(\"\\nVerified:\", \", \".join(f\"{s}/{v} ({len(d)} days)\" for (s, v), d in results.items()))\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "id": "md13", + "source": [ + "## Next Steps\n", + "\n", + "1. Set `DEV_ONLY = False` to register the full matrix (5 models x 3 scenarios x 3 variables)\n", + "2. Query a registered layer through WMS / a temporal workflow and confirm daily slices render\n", + "3. Test the climate-risk process with the new datasets\n", + "\n", + "Comparing with `nexgddp_cmip6_ingest.ipynb` (the probe version) shows what the probe saves.\n", + "The probe branch also fixes the single-array band name, so the two agree except for that\n", + "until it lands.\n" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "name": "python", + "version": "3.11" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} \ No newline at end of file From a52ba058393313f6c22060569f8012d90e2df72f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johannes=20Dr=C3=B6nner?= Date: Tue, 6 Oct 2026 17:42:40 +0200 Subject: [PATCH 9/9] feat(backend): offer ssp370 and the full NEX-GDDP model matrix Add SSP3-7.0 as a third future scenario alongside SSP2-4.5 and SSP5-8.5, and open the climate-risk process to all five NEX-GDDP-CMIP6 models now that the datasets are ingested. `ClimateScenario` gains `Ssp370`, the registry parser knows its key, and the spec-only `ClimateRiskProcessResponses` gains the matching output field so the generated client and the process description agree on the response keys. `nexgddp_models.toml` is no longer trimmed to what is ingested: all 5 models x 4 scenarios are registered (`notebooks/nexgddp_cmip6_ingest_full.ipynb`, 60 datasets / 4845 tiles, verified). The ingest also corrected IPSL-CM6A-LR's grid label to `gr`, which the registry had wrong as `gn` and which silently skipped every one of its files. The frontend needs no change: it fetches the process description at runtime and renders the execution result by iterating its keys, so a new scenario shows up as another output checkbox and another result card. Co-Authored-By: opencode --- .../models/ClimateRiskProcessResponses.ts | 7 + backend/conf/nexgddp_models.full.toml | 12 +- backend/conf/nexgddp_models.toml | 35 +- backend/src/processes/climate_risk/compute.rs | 10 + backend/src/processes/climate_risk/mod.rs | 1 + backend/src/processes/climate_risk/types.rs | 17 +- backend/src/processes/path_info.rs | 3 + notebooks/nexgddp_cmip6_ingest_full.ipynb | 846 ++++++++++++++++++ openapi.json | 11 + 9 files changed, 929 insertions(+), 13 deletions(-) create mode 100644 notebooks/nexgddp_cmip6_ingest_full.ipynb diff --git a/api-client/typescript/models/ClimateRiskProcessResponses.ts b/api-client/typescript/models/ClimateRiskProcessResponses.ts index b46a11f..41f1d76 100644 --- a/api-client/typescript/models/ClimateRiskProcessResponses.ts +++ b/api-client/typescript/models/ClimateRiskProcessResponses.ts @@ -18,6 +18,7 @@ import { DataResource } from "../models/DataResource"; */ export class ClimateRiskProcessResponses { "ssp245"?: DataResource | null; + "ssp370"?: DataResource | null; "ssp585"?: DataResource | null; "rawEnsembleData"?: DataResource | null; /** @@ -41,6 +42,12 @@ export class ClimateRiskProcessResponses { type: "DataResource", format: "", }, + { + name: "ssp370", + baseName: "ssp370", + type: "DataResource", + format: "", + }, { name: "ssp585", baseName: "ssp585", diff --git a/backend/conf/nexgddp_models.full.toml b/backend/conf/nexgddp_models.full.toml index 42e9861..9bae38f 100644 --- a/backend/conf/nexgddp_models.full.toml +++ b/backend/conf/nexgddp_models.full.toml @@ -9,28 +9,28 @@ id = "ACCESS-CM2" variant = "r1i1p1f1" grid = "gn" -scenarios = ["historical", "ssp245", "ssp585"] +scenarios = ["historical", "ssp245", "ssp370", "ssp585"] [[model]] id = "MPI-ESM1-2-LR" variant = "r1i1p1f1" grid = "gn" -scenarios = ["historical", "ssp245", "ssp585"] +scenarios = ["historical", "ssp245", "ssp370", "ssp585"] [[model]] id = "IPSL-CM6A-LR" variant = "r1i1p1f1" -grid = "gn" -scenarios = ["historical", "ssp245", "ssp585"] +grid = "gr" +scenarios = ["historical", "ssp245", "ssp370", "ssp585"] [[model]] id = "MRI-ESM2-0" variant = "r1i1p1f1" grid = "gn" -scenarios = ["historical", "ssp245", "ssp585"] +scenarios = ["historical", "ssp245", "ssp370", "ssp585"] [[model]] id = "GFDL-ESM4" variant = "r1i1p1f1" grid = "gr1" -scenarios = ["historical", "ssp245", "ssp585"] \ No newline at end of file +scenarios = ["historical", "ssp245", "ssp370", "ssp585"] \ No newline at end of file diff --git a/backend/conf/nexgddp_models.toml b/backend/conf/nexgddp_models.toml index 76f8a6f..e804972 100644 --- a/backend/conf/nexgddp_models.toml +++ b/backend/conf/nexgddp_models.toml @@ -1,11 +1,36 @@ -# NEX-GDDP-CMIP6 model registry — the source of truth for what climate-risk can compute. +# NEX-GDDP-CMIP6 model registry - the source of truth for what climate-risk can compute. # -# Only models/scenarios with a registered GeoEngine dataset belong here; the process description and -# the `models` dropdown are derived from this file. `nexgddp_models.full.toml` holds the full -# not-yet-ingested matrix — copy it over this file after a full ingest. +# Every model/scenario listed here has a registered GeoEngine dataset, so the process +# description and the `models` dropdown offer exactly what can run. This file is rewritten +# from `nexgddp_models.full.toml` by `notebooks/nexgddp_cmip6_ingest_full.ipynb` once the +# ingest is verified. To add a model, edit that full matrix and re-run the notebook. [[model]] id = "ACCESS-CM2" variant = "r1i1p1f1" grid = "gn" -scenarios = ["historical", "ssp245"] \ No newline at end of file +scenarios = ["historical", "ssp245", "ssp370", "ssp585"] + +[[model]] +id = "MPI-ESM1-2-LR" +variant = "r1i1p1f1" +grid = "gn" +scenarios = ["historical", "ssp245", "ssp370", "ssp585"] + +[[model]] +id = "IPSL-CM6A-LR" +variant = "r1i1p1f1" +grid = "gr" +scenarios = ["historical", "ssp245", "ssp370", "ssp585"] + +[[model]] +id = "MRI-ESM2-0" +variant = "r1i1p1f1" +grid = "gn" +scenarios = ["historical", "ssp245", "ssp370", "ssp585"] + +[[model]] +id = "GFDL-ESM4" +variant = "r1i1p1f1" +grid = "gr1" +scenarios = ["historical", "ssp245", "ssp370", "ssp585"] \ No newline at end of file diff --git a/backend/src/processes/climate_risk/compute.rs b/backend/src/processes/climate_risk/compute.rs index 616fa9f..b319521 100644 --- a/backend/src/processes/climate_risk/compute.rs +++ b/backend/src/processes/climate_risk/compute.rs @@ -1011,10 +1011,20 @@ mod tests { #[test] fn it_scenario_output_id_maps_display_name_to_declared_output_id() { + // Every declared scenario, so a new one cannot ship with its card under a key + // that does not match the process description. + for scenario in ClimateScenario::ALL { + let props = scenario.properties(); + assert_eq!(scenario_output_id(props.name), scenario.name()); + } assert_eq!( scenario_output_id("SSP2-4.5 (Intermediate emissions)"), "ssp245" ); + assert_eq!( + scenario_output_id("SSP3-7.0 (Moderate-high emissions)"), + "ssp370" + ); assert_eq!(scenario_output_id("SSP5-8.5 (High emissions)"), "ssp585"); // Unknown names pass through, so rows already carrying output ids keep working. assert_eq!(scenario_output_id("ssp245"), "ssp245"); diff --git a/backend/src/processes/climate_risk/mod.rs b/backend/src/processes/climate_risk/mod.rs index e178678..3348a2b 100644 --- a/backend/src/processes/climate_risk/mod.rs +++ b/backend/src/processes/climate_risk/mod.rs @@ -73,6 +73,7 @@ fn load_model_registry() -> HashMap { .filter_map(|s| match s.as_str() { "historical" => Some(ClimateScenario::Historical), "ssp245" => Some(ClimateScenario::Ssp245), + "ssp370" => Some(ClimateScenario::Ssp370), "ssp585" => Some(ClimateScenario::Ssp585), _ => None, }) diff --git a/backend/src/processes/climate_risk/types.rs b/backend/src/processes/climate_risk/types.rs index 82a701f..bb767c7 100644 --- a/backend/src/processes/climate_risk/types.rs +++ b/backend/src/processes/climate_risk/types.rs @@ -120,18 +120,20 @@ pub struct ClimateModelProperties { pub enum ClimateScenario { Historical, Ssp245, + Ssp370, Ssp585, } impl ClimateScenario { - pub const ALL: &'static [Self] = &[Self::Historical, Self::Ssp245, Self::Ssp585]; + pub const ALL: &'static [Self] = &[Self::Historical, Self::Ssp245, Self::Ssp370, Self::Ssp585]; - pub const FUTURE: &'static [Self] = &[Self::Ssp245, Self::Ssp585]; + pub const FUTURE: &'static [Self] = &[Self::Ssp245, Self::Ssp370, Self::Ssp585]; pub fn name(self) -> &'static str { match self { Self::Historical => "historical", Self::Ssp245 => "ssp245", + Self::Ssp370 => "ssp370", Self::Ssp585 => "ssp585", } } @@ -147,6 +149,11 @@ impl ClimateScenario { dataset_prefix: "ssp245", scenario: ClimateScenario::Ssp245, }, + ClimateScenario::Ssp370 => ClimateScenarioProperties { + name: "SSP3-7.0 (Moderate-high emissions)", + dataset_prefix: "ssp370", + scenario: ClimateScenario::Ssp370, + }, ClimateScenario::Ssp585 => ClimateScenarioProperties { name: "SSP5-8.5 (High emissions)", dataset_prefix: "ssp585", @@ -543,6 +550,11 @@ mod tests { "SSP2-4.5 (Intermediate emissions)", "ssp245", ), + ( + ClimateScenario::Ssp370, + "SSP3-7.0 (Moderate-high emissions)", + "ssp370", + ), ( ClimateScenario::Ssp585, "SSP5-8.5 (High emissions)", @@ -580,6 +592,7 @@ mod tests { match scenario { ClimateScenario::Historical => "Historical (1950–2014)", ClimateScenario::Ssp245 => "SSP2-4.5 (Intermediate emissions)", + ClimateScenario::Ssp370 => "SSP3-7.0 (Moderate-high emissions)", ClimateScenario::Ssp585 => "SSP5-8.5 (High emissions)", } ); diff --git a/backend/src/processes/path_info.rs b/backend/src/processes/path_info.rs index d3c8900..75dd520 100644 --- a/backend/src/processes/path_info.rs +++ b/backend/src/processes/path_info.rs @@ -110,6 +110,9 @@ pub struct ClimateRiskProcessResponses { /// Summary table for SSP2-4.5. #[schema(value_type = Option, inline)] pub ssp245: Option>>, + /// Summary table for SSP3-7.0. + #[schema(value_type = Option, inline)] + pub ssp370: Option>>, /// Summary table for SSP5-8.5. #[schema(value_type = Option, inline)] pub ssp585: Option>>, diff --git a/notebooks/nexgddp_cmip6_ingest_full.ipynb b/notebooks/nexgddp_cmip6_ingest_full.ipynb new file mode 100644 index 0000000..085b1e1 --- /dev/null +++ b/notebooks/nexgddp_cmip6_ingest_full.ipynb @@ -0,0 +1,846 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# NEX-GDDP-CMIP6 Full-Matrix Ingestion (climate-risk project)\n", + "\n", + "Clone of `nexgddp_cmip6_ingest_manual.ipynb` configured for the **full climate-risk\n", + "matrix**: every model in the registry x every scenario x every variable, with the\n", + "complete yearly axes. The dev-trimmed run lives in the companion notebook; everything\n", + "else - the hand-stated metadata, the verification, the year-boundary read - is identical.\n", + "\n", + "What the climate-risk project needs, and what this notebook therefore registers:\n", + "\n", + "* **Variables** - `tasmax` (Heat Days, Ice Days), `tasmin` (Tropical Nights, Frost Days),\n", + " `pr` (Dry Days, Heavy Rain Days), one dataset per (model, scenario, variable).\n", + "* **Scenarios** - `historical` (1950-2014, the reference period) and `ssp245` / `ssp370` /\n", + " `ssp585` (2015-2100, the analysis periods).\n", + "* **Timesteps** - one dataset slice per day, `historical` 65 yearly files and each future\n", + " scenario 86. Nothing is capped: the analysis window may start as late as 2070 with a\n", + " 30-year range, and the reference may reach back to 1950.\n", + "\n", + "After the last dataset verifies, the full registry is copied over the dev registry\n", + "(`backend/conf/nexgddp_models.toml`) so the `models` dropdown offers every model.\n" + ], + "id": "md00" + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import datetime as dt\n", + "import json\n", + "import re\n", + "from pathlib import Path\n", + "\n", + "import fsspec\n", + "import numpy as np\n", + "import s3fs\n", + "\n", + "import geoengine as ge\n", + "import geoengine_api_client as geoc\n" + ], + "id": "md1" + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Configuration\n", + "\n", + "Update these paths to match your environment.\n" + ], + "id": "md02" + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# NEX-GDDP-CMIP6 S3 bucket (public). Files are registered as external data, so GeoEngine\n", + "# streams them through GDAL's /vsicurl rather than copying them onto a volume.\n", + "S3_BUCKET = \"nex-gddp-cmip6\"\n", + "S3_REGION = \"us-west-2\"\n", + "S3_PREFIX = \"NEX-GDDP-CMIP6\"\n", + "\n", + "# GeoEngine connection\n", + "GEOENGINE_URL = \"http://localhost:3030/api\"\n", + "GEOENGINE_USER = \"admin@localhost\"\n", + "GEOENGINE_PASSWORD = \"adminadmin\"\n", + "\n", + "# Full model registry: the authoritative id/variant/grid/scenario list. The dev registry\n", + "# (`nexgddp_models.toml`) is trimmed to what is ingested, so ingest reads this one and the\n", + "# trimmed file is regenerated from it afterwards.\n", + "REGISTRY_PATH = Path(\"../backend/conf/nexgddp_models.full.toml\")\n", + "\n", + "try:\n", + " import tomllib\n", + "except ModuleNotFoundError: # Python < 3.11\n", + " import tomli as tomllib\n", + "\n", + "with open(REGISTRY_PATH, \"rb\") as fh:\n", + " REGISTRY = tomllib.load(fh)[\"model\"]\n", + "\n", + "MODELS = [{\"id\": m[\"id\"], \"variant\": m[\"variant\"], \"grid\": m[\"grid\"]} for m in REGISTRY]\n", + "\n", + "SCENARIOS = sorted({s for m in REGISTRY for s in m[\"scenarios\"]})\n", + "\n", + "# Variables. Each one lives in its own set of yearly files, so each becomes its own dataset\n", + "# with a single band and z = time.\n", + "VARIABLES = [\"tasmax\", \"tasmin\", \"pr\"]\n", + "\n", + "# Variable metadata. `long_name` and `unit` are the CF attributes the files carry - the\n", + "# probe would have used exactly these for the band, so they are stated here rather than\n", + "# derived, and the band is what a reader sees in the layer listing.\n", + "VARIABLE_CONFIGS = {\n", + " \"tasmax\": {\n", + " \"name\": \"Maximum Temperature\",\n", + " \"long_name\": \"Daily Maximum Near-Surface Air Temperature\",\n", + " \"measurement\": \"temperature\",\n", + " \"unit\": \"K\",\n", + " # colour-ramp range\n", + " \"min\": 200,\n", + " \"max\": 340,\n", + " # Sanity range for the verification, deliberately wider than the ramp so\n", + " # real product extremes are not rejected. 170..360 K is -103..87 C: it\n", + " # still rejects a degrees-for-Kelvin mix-up (values would sit near 0..45)\n", + " # and any leaked fill value, while accepting the Siberian winter cold\n", + " # pole that MPI-ESM1-2-LR brings to 199 K at 66N/140E.\n", + " \"plausible\": (170.0, 360.0),\n", + " \"color_map\": \"coolwarm\",\n", + " },\n", + " \"tasmin\": {\n", + " \"name\": \"Minimum Temperature\",\n", + " \"long_name\": \"Daily Minimum Near-Surface Air Temperature\",\n", + " \"measurement\": \"temperature\",\n", + " \"unit\": \"K\",\n", + " \"min\": 200,\n", + " \"max\": 340,\n", + " \"plausible\": (170.0, 360.0),\n", + " \"color_map\": \"coolwarm\",\n", + " },\n", + " \"pr\": {\n", + " \"name\": \"Precipitation\",\n", + " \"long_name\": \"Precipitation\",\n", + " \"measurement\": \"precipitation\",\n", + " \"unit\": \"kg m-2 s-1\",\n", + " \"min\": 0,\n", + " \"max\": 0.01,\n", + " # 0..0.08 kg m-2 s-1 is 0..6900 mm/day. Real daily extremes reach 0.0137\n", + " # (~1180 mm/day, which the raw file carries too), and this still rejects\n", + " # a mm/day mix-up (values would reach ~1000) or a leaked fill value.\n", + " \"plausible\": (0.0, 0.08),\n", + " \"color_map\": \"Blues\",\n", + " },\n", + "}\n", + "\n", + "# Number of daily slices read per request. A slice of this grid is 600x1440 float32 = 3.4 MB,\n", + "# so a batch is one IPC message: 4 is ~14 MB, 16 would be ~55 MB and ~22 s cold over the\n", + "# network. Queries for fewer days than this still read only the days they ask for.\n", + "#\n", + "# This is a *dataset* property (`maxZBatchSize`), not an operator parameter: a batch is sized\n", + "# against the slice size of the data, so whoever registers the dataset decides it and every\n", + "# consumer workflow inherits it.\n", + "MAX_Z_BATCH_SIZE = 4\n", + "\n", + "# GDAL must be allowed to read .nc over /vsicurl.\n", + "VSICURL_ALLOWED_EXTENSIONS = \".tif,.tiff,.TIF,.jp2,.ovr,.nc\"\n", + "\n", + "# Full matrix for the climate-risk project: every model in the registry x every scenario x\n", + "# every variable, with the complete yearly axes (historical 1950-2014, future 2015-2100).\n", + "#\n", + "# `YEARS_PER_SCENARIO = None` keeps every year. Registration no longer opens every file, so\n", + "# this is not a wall-clock cost; the *verification* reads one small window per dataset, so a\n", + "# full run reads 60 windows - never a full axis.\n", + "#\n", + "# ponytail: the dev-trimmed variant (1 model, 2 scenarios, 1 variable, 5 years) lives in\n", + "# `nexgddp_cmip6_ingest_manual.ipynb`.\n", + "YEARS_PER_SCENARIO = None\n" + ], + "id": "md3" + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Initialize GeoEngine Connection\n" + ], + "id": "md04" + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "ge.initialize(\n", + " GEOENGINE_URL,\n", + " credentials=(GEOENGINE_USER, GEOENGINE_PASSWORD),\n", + ")\n", + "\n", + "# The files stay on S3; `external` registers them by reference.\n", + "DATA_STORE = \"external\"\n", + "\n", + "permission_tuples = [\n", + " (ge.permissions.ANONYMOUS_USER_ROLE_ID, ge.Permission.READ),\n", + " (ge.permissions.REGISTERED_USER_ROLE_ID, ge.Permission.READ),\n", + "]\n" + ], + "id": "md5" + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## List Files\n", + "\n", + "For each model/scenario/variable combination, list the yearly NetCDF files. Nothing is\n", + "probed: the year in each file name is the one fact taken from the files, and it is what\n", + "decides the file's time axis.\n", + "\n", + "That is one S3 listing per combination and no GDAL at all - which is the difference between\n", + "this notebook and its probe companion.\n" + ], + "id": "md06" + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "def list_netcdf_files(model_id, scenario, variable, variant, grid):\n", + " \"\"\"List all yearly NetCDF files for a model/scenario/variable combination.\n", + "\n", + " One `ls` of the variable directory instead of two `exists()` per year: the\n", + " directory already lists every file, so 60 listings replace ~9700 HEAD\n", + " requests. The year in each file name is the one fact taken from the files,\n", + " and it is what decides the file's time axis. The corrected `{year}_v2.0.nc`\n", + " copies are preferred over the uncorrected base file.\n", + " \"\"\"\n", + " fs = s3fs.S3FileSystem(anon=True)\n", + "\n", + " # Determine year range based on scenario\n", + " if scenario == \"historical\":\n", + " years = range(1950, 2015)\n", + " else:\n", + " years = range(2015, 2101)\n", + "\n", + " prefix = f\"{S3_BUCKET}/{S3_PREFIX}/{model_id}/{scenario}/{variant}/{variable}/\"\n", + " try:\n", + " keys = fs.ls(prefix)\n", + " except FileNotFoundError:\n", + " return []\n", + "\n", + " by_year = {}\n", + " for key in keys:\n", + " if f\"_{grid}_\" not in key:\n", + " continue\n", + " match = re.search(r\"_(\\d{4})(?:_v\\d+\\.\\d+)?\\.nc$\", key)\n", + " if match:\n", + " by_year.setdefault(int(match.group(1)), []).append(key)\n", + "\n", + " files = []\n", + " for year in years:\n", + " cands = by_year.get(year, [])\n", + " path = next((c for c in cands if c.endswith(f\"_{year}_v2.0.nc\")), None)\n", + " if path is None:\n", + " path = next((c for c in cands if c.endswith(f\"_{year}.nc\")), None)\n", + " if path is None:\n", + " print(f\"Warning: File not found: {prefix}{year}.nc\")\n", + " else:\n", + " files.append(path)\n", + " # a dev run stops after the cap; a full run keeps every year\n", + " if YEARS_PER_SCENARIO is not None and len(files) >= YEARS_PER_SCENARIO:\n", + " break\n", + "\n", + " return sorted(files)\n", + "\n", + "\n", + "def vsicurl_url(bucket_key):\n", + " \"\"\"The /vsicurl path GeoEngine opens for a bucket key.\n", + "\n", + " s3fs keys start with the bucket name (`nex-gddp-cmip6/NEX-GDDP-CMIP6/...`), but the\n", + " bucket is already in the hostname of an S3 URL, so the key must not repeat it --\n", + " doing so yields a 404 from GDAL.\n", + " \"\"\"\n", + " bucket, key = bucket_key.split(\"/\", 1)\n", + " return f\"/vsicurl/https://{bucket}.s3.{S3_REGION}.amazonaws.com/{key}\"\n", + "\n", + "\n", + "def collect_targets():\n", + " \"\"\"Every (model, scenario, variable) combination that has files on S3.\"\"\"\n", + " targets = []\n", + " for model in MODELS:\n", + " for scenario in SCENARIOS:\n", + " for variable in VARIABLES:\n", + " files = list_netcdf_files(\n", + " model[\"id\"], scenario, variable, model[\"variant\"], model[\"grid\"],\n", + " )\n", + " if not files:\n", + " print(f\"No files found, skipping {model['id']}/{scenario}/{variable}\")\n", + " continue\n", + " targets.append((model, scenario, variable, files))\n", + " return targets\n", + "\n", + "\n", + "TARGETS = collect_targets()\n", + "for model, scenario, variable, files in TARGETS:\n", + " print(f\"{model['id']} / {scenario} / {variable}: {len(files)} yearly files\")\n" + ], + "id": "md7" + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Declare the Dataset Metadata\n", + "\n", + "`declare_md_dataset` states every field the probe would have derived. Three of them are easy\n", + "to get wrong, so they are spelled out:\n", + "\n", + "1. **Two geo transforms, and they differ.** `params.geoTransform` is the grid *as stored*\n", + " (CF `lat` ascends, so `y_pixel_size` is positive and the array is south-up);\n", + " `resultDescriptor.spatialGrid` is the grid *as presented* (always north-up, longitudes\n", + " re-labelled -180..180 because the stored coverage wraps 0..360). Both origins are the\n", + " first cell's **edge**, not its centre: `lon` centres start at 0.125, so the edge is\n", + " `0.125 - 0.25/2 = 0.0`.\n", + "\n", + "2. **The time values are stamped at noon.** CF stores daily samples at the cell centre, so\n", + " the files' `time` is `days since 1850-01-01` with a `.5` offset: the 1950 file's first\n", + " value is 36524.5, i.e. `1950-01-01T12:00Z`. An interval is therefore **noon to noon**,\n", + " not midnight to midnight. Slice j is day j of the year either way.\n", + "\n", + "3. **`timeSteps` is one interval per z slice**, not one for the whole file, and\n", + " `timeDescriptor` must reproduce them exactly - the server compares them and rejects a row\n", + " where they disagree.\n" + ], + "id": "md08" + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "def build_symbology(variable):\n", + " config = VARIABLE_CONFIGS[variable]\n", + " return ge.RasterSymbology(\n", + " raster_colorizer=ge.SingleBandRasterColorizer(\n", + " band=0,\n", + " band_colorizer=ge.Colorizer.linear_with_mpl_cmap(\n", + " color_map=config[\"color_map\"],\n", + " min_max=(config[\"min\"], config[\"max\"]),\n", + " n_steps=10,\n", + " ),\n", + " ),\n", + " )\n", + "\n", + "\n", + "# --- the dataset's grid ----------------------------------------------------\n", + "#\n", + "# 0.25 degree cells, 1440 x 600, covering lon 0..360 and lat -60..90.\n", + "N_LON, N_LAT = 1440, 600\n", + "DEG = 0.25\n", + "\n", + "# The stored grid: `lat` ascends (-59.875 .. 89.875), so the array is south-up and\n", + "# `y_pixel_size` is positive. Origins are cell edges.\n", + "STORED_GEO_TRANSFORM = geoc.GeoTransform(\n", + " origin_coordinate=geoc.Coordinate2D(x=0.0, y=-60.0),\n", + " x_pixel_size=DEG,\n", + " y_pixel_size=DEG,\n", + ")\n", + "\n", + "# The presented grid: north-up, longitudes shifted to -180..180 (the stored coverage wraps\n", + "# 0..360). `-60.0 + N_LAT * DEG` is the north edge.\n", + "PRESENTED_GEO_TRANSFORM = geoc.GeoTransform(\n", + " origin_coordinate=geoc.Coordinate2D(x=-180.0, y=90.0),\n", + " x_pixel_size=DEG,\n", + " y_pixel_size=-DEG,\n", + ")\n", + "\n", + "# The presented footprint of every row, in spatial coordinates.\n", + "PRESENTED_FOOTPRINT = geoc.SpatialPartition2D(\n", + " upperLeftCoordinate=geoc.Coordinate2D(x=-180.0, y=90.0),\n", + " lowerRightCoordinate=geoc.Coordinate2D(x=180.0, y=-60.0),\n", + ")\n", + "\n", + "DAY_MS = 86_400_000\n", + "NOON_MS = 12 * 3_600_000\n", + "\n", + "\n", + "def file_year(key):\n", + " \"\"\"The calendar year a yearly NEX-GDDP file covers, from its name.\n", + "\n", + " The name ends in `_{year}.nc` or `_{year}_v2.0.nc`. This is the one fact taken from the\n", + " file name rather than known from the dataset's convention.\n", + " \"\"\"\n", + " match = re.search(r\"_(\\d{4})(?:_v\\d+\\.\\d+)?\\.nc$\", key)\n", + " if match is None:\n", + " raise ValueError(f\"no year in file name: {key}\")\n", + " return int(match.group(1))\n", + "\n", + "\n", + "def year_slice_start_ms(year):\n", + " \"\"\"ms of `year-01-01T12:00:00Z`: the file's first time value is *noon*, not midnight.\n", + "\n", + " Checked against the data: the 1950 file's first value is 36524.5 and\n", + " `1850-01-01T00:00Z + 36524.5 days` is `1950-01-01T12:00Z`.\n", + " \"\"\"\n", + " return int(dt.datetime(year, 1, 1, tzinfo=dt.timezone.utc).timestamp() * 1000) + NOON_MS\n", + "\n", + "\n", + "def file_time_steps(year):\n", + " \"\"\"One interval per daily slice of the file covering `year`, noon to noon.\n", + "\n", + " Slice j of that file is day j of the year. `n_days` follows the calendar (366 in a leap\n", + " year), and the server checks it against the array's z size when the row is added - so a\n", + " misread year fails at registration, not at query time.\n", + " \"\"\"\n", + " n_days = (dt.date(year + 1, 1, 1) - dt.date(year, 1, 1)).days\n", + " start = year_slice_start_ms(year)\n", + " return [\n", + " geoc.TimeInterval(start=start + j * DAY_MS, end=start + (j + 1) * DAY_MS)\n", + " for j in range(n_days)\n", + " ]\n", + "\n", + "\n", + "def daily_descriptor(start_ms, end_ms):\n", + " \"\"\"The descriptor the server derives from `file_time_steps`, stated the same way.\n", + "\n", + " It is compared with the steps *exactly*, so it has to be `Regular(origin=first start,\n", + " step=1 day)` spanning `[first start, last end)`.\n", + " \"\"\"\n", + " return geoc.TimeDescriptor(\n", + " bounds=geoc.TimeInterval(start=start_ms, end=end_ms),\n", + " dimension=geoc.TimeDimension(\n", + " actual_instance=geoc.Regular(\n", + " origin=start_ms,\n", + " step=geoc.TimeStep(granularity=\"days\", step=1),\n", + " type=\"regular\",\n", + " )\n", + " ),\n", + " )\n", + "\n", + "\n", + "def declare_md_dataset(variable, files):\n", + " \"\"\"State the metadata by hand instead of probing the files.\n", + "\n", + " One row per yearly file, covering all of that file's z slices. `leadingPrefix=[]` means\n", + " a 3D array; a 4D `(time, depth, lat, lon)` file would use `[2]` to read depth 2.\n", + " \"\"\"\n", + " config = VARIABLE_CONFIGS[variable]\n", + " measurement = geoc.Measurement(\n", + " actual_instance=geoc.ContinuousMeasurement(\n", + " type=\"continuous\", measurement=config[\"long_name\"], unit=config[\"unit\"]\n", + " )\n", + " )\n", + "\n", + " tiles = []\n", + " for i, key in enumerate(files):\n", + " year = file_year(key)\n", + " steps = file_time_steps(year)\n", + " tiles.append(\n", + " geoc.AddDatasetMdTile(\n", + " spatial_partition=PRESENTED_FOOTPRINT,\n", + " band=0,\n", + " z_index=i,\n", + " array_name=variable,\n", + " arrayGroup=None,\n", + " leadingPrefix=[],\n", + " time_descriptor=daily_descriptor(steps[0].start, steps[-1].end),\n", + " time_steps=steps,\n", + " params=geoc.GdalDatasetParameters(\n", + " file_path=vsicurl_url(key),\n", + " rasterband_channel=1,\n", + " geo_transform=STORED_GEO_TRANSFORM,\n", + " width=N_LON,\n", + " height=N_LAT,\n", + " file_not_found_handling=\"NoData\",\n", + " # `None` is not \"unknown\": the reader falls back to the file's own\n", + " # `_FillValue` (1e20 in these files), which is what the probe stored.\n", + " noDataValue=None,\n", + " properties_mapping=None,\n", + " gdal_open_options=None,\n", + " gdal_config_options=[\n", + " [\"CPL_VSIL_CURL_ALLOWED_EXTENSIONS\", VSICURL_ALLOWED_EXTENSIONS]\n", + " ],\n", + " allow_alphaband_as_mask=False,\n", + " ),\n", + " )\n", + " )\n", + "\n", + " # the dataset's axis is the concatenation of the files' own axes\n", + " first_start = tiles[0].time_steps[0].start\n", + " last_end = tiles[-1].time_steps[-1].end\n", + "\n", + " meta_data = geoc.MetaDataDefinition(\n", + " actual_instance=geoc.GdalMdMetaData(\n", + " type=\"GdalMdMetaData\",\n", + " result_descriptor=geoc.RasterResultDescriptor(\n", + " data_type=\"F32\",\n", + " spatial_reference=\"EPSG:4326\",\n", + " time=daily_descriptor(first_start, last_end),\n", + " spatial_grid=geoc.SpatialGridDescriptor(\n", + " spatialGrid=geoc.SpatialGridDefinition(\n", + " geoTransform=PRESENTED_GEO_TRANSFORM,\n", + " gridBounds=geoc.GridBoundingBox2D(\n", + " topLeftIdx=geoc.GridIdx2D(xIdx=0, yIdx=0),\n", + " bottomRightIdx=geoc.GridIdx2D(xIdx=N_LON - 1, yIdx=N_LAT - 1),\n", + " ),\n", + " ),\n", + " descriptor=\"source\",\n", + " ),\n", + " bands=[geoc.RasterBandDescriptor(name=config[\"long_name\"], measurement=measurement)],\n", + " ),\n", + " z_role=\"variable\",\n", + " wrap=True,\n", + " maxZBatchSize=MAX_Z_BATCH_SIZE,\n", + " cacheTtl=None,\n", + " )\n", + " )\n", + " return meta_data, tiles\n" + ], + "id": "md9a" + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "def register_dataset(model_id, scenario, variable, files):\n", + " \"\"\"State the metadata and register it as an `MdGdalSource` dataset.\"\"\"\n", + " config = VARIABLE_CONFIGS[variable]\n", + " dataset_name = f\"nexgddp_cmip6_{model_id}_{scenario}_{variable}\"\n", + "\n", + " meta_data, tiles = declare_md_dataset(variable, files)\n", + " md = meta_data.actual_instance\n", + " bounds = md.result_descriptor.time.bounds\n", + " print(\n", + " f\" declared: {len(tiles)} files, z_role={md.z_role}, wrap={md.wrap}, \"\n", + " f\"maxZBatchSize={md.max_z_batch_size}, \"\n", + " f\"time {dt.datetime.fromtimestamp(bounds.start / 1000, tz=None).date()} .. \"\n", + " f\"{dt.datetime.fromtimestamp(bounds.end / 1000, tz=None).date()}, \"\n", + " f\"bands {[b.name for b in md.result_descriptor.bands]}\"\n", + " )\n", + "\n", + " properties = ge.AddDatasetProperties(\n", + " name=dataset_name,\n", + " display_name=f\"NEX-GDDP-CMIP6 {config['name']} ({scenario})\",\n", + " description=(\n", + " f\"Daily NEX-GDDP-CMIP6 {config['name']} projections for {config['measurement']} \"\n", + " f\"({config['unit']}). Model: {model_id}, Scenario: {scenario}, \"\n", + " f\"one MD array file per year read through GDAL's multidimensional API\"\n", + " ),\n", + " source_operator=\"MdGdalSource\",\n", + " symbology=build_symbology(variable),\n", + " )\n", + "\n", + " dataset = ge.add_or_replace_dataset_with_permissions(\n", + " DATA_STORE,\n", + " properties,\n", + " meta_data,\n", + " permission_tuples=permission_tuples,\n", + " replace_existing=True,\n", + " timeout=600,\n", + " )\n", + "\n", + " ge.add_md_dataset_tiles(dataset, tiles, timeout=600)\n", + " print(f\" registered: {dataset}\")\n", + " return dataset\n", + "\n", + "\n", + "# Register all datasets\n", + "registered_datasets = []\n", + "\n", + "for model, scenario, variable, files in TARGETS:\n", + " print(f\"\\nProcessing: {model['id']} / {scenario} / {variable}\")\n", + " registered_datasets.append(register_dataset(model[\"id\"], scenario, variable, files))\n", + "\n", + "print(f\"\\nTotal datasets: {len(registered_datasets)}\")\n" + ], + "id": "md9b" + }, + { + "cell_type": "markdown", + "metadata": {}, + "id": "md11", + "source": [ + "## Verification\n", + "\n", + "Read one window per registered dataset, straddling a year boundary. Since the metadata is\n", + "stated rather than derived, this also has to check that the statement *landed* - so the\n", + "metadata round-trip is asserted before any pixel is read.\n", + "\n", + "Three things are checked, in order:\n", + "\n", + "1. **The stated metadata is what the server stored.** `z_role`, `wrap`, `maxZBatchSize`, the\n", + " band name, and the dataset's time bounds `[noon(Y_first), noon(Y_last+1))`. A wrong axis\n", + " length or a wrong band shows up here, without reading a pixel.\n", + "2. **The window straddles a file boundary** - file 1 ends and file 2 begins at the first\n", + " year boundary - and `READ_DAYS > MAX_Z_BATCH_SIZE`, so the crossing is split across worker\n", + " requests too. The days must be consecutive, one step each, on one band, and tiled.\n", + "3. **The stamps are the stated ones.** Each returned tile's `time_start_ms` must equal the\n", + " noon-based stamp computed above, not merely the right date: a 12-hour drift (midnight\n", + " intervals instead of noon) is exactly the mistake that would otherwise go unnoticed.\n", + "\n", + "Values are then checked for being real data: inside the configured range, varying, and\n", + "identical when the same window is read twice.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "id": "md12", + "source": [ + "READ_DAYS = 12 # > MAX_Z_BATCH_SIZE, so the read spans more than one worker request\n", + "\n", + "\n", + "def first_boundary_window(time_bounds):\n", + " \"\"\"A [start, end) window centred on the first year boundary inside the dataset.\"\"\"\n", + " start = dt.datetime.fromtimestamp(time_bounds.start / 1000, tz=dt.timezone.utc)\n", + " boundary = start.replace(year=start.year + 1) # file 1 -> file 2\n", + " half = dt.timedelta(days=READ_DAYS // 2)\n", + " return boundary - half, boundary + half\n", + "\n", + "\n", + "async def read_window(workflow, window_start, window_end):\n", + " \"\"\"Stream a global window and group the tiles by day.\"\"\"\n", + " bbox = ge.QueryRectangle(\n", + " ge.BoundingBox2D(-180.0, -90.0, 180.0, 90.0),\n", + " ge.TimeInterval(window_start, window_end),\n", + " )\n", + " per_day = {}\n", + " async for tile in workflow.raster_stream(bbox, open_timeout=600):\n", + " day = str(tile.time.start)[:10]\n", + " per_day.setdefault(day, []).append(tile)\n", + " return per_day\n", + "\n", + "\n", + "def stamp_ms(tile):\n", + " \"\"\"A tile's time stamp in ms (`time_start_ms` is a numpy datetime64).\"\"\"\n", + " return int(np.datetime64(tile.time_start_ms, \"ms\").astype(\"int64\"))\n", + "\n", + "\n", + "async def verify_dataset(dataset_name, variable, files):\n", + " \"\"\"Assert the stated metadata landed, then read a boundary-crossing window.\"\"\"\n", + " config = VARIABLE_CONFIGS[variable]\n", + " info = ge.dataset_info_by_name(dataset_name)\n", + " assert info is not None, f\"{dataset_name} is not registered\"\n", + "\n", + " # `result_descriptor` is a oneOf wrapper; the concrete descriptor is under\n", + " # `actual_instance`\n", + " rd = info.result_descriptor.actual_instance\n", + " print(f\"\\n{dataset_name}\")\n", + " print(f\" data_type={rd.data_type} bands={[b.name for b in rd.bands]} srs={rd.spatial_reference}\")\n", + "\n", + " # --- 1. the stated metadata is what the server stored -------------------\n", + " #\n", + " # The dataset listing exposes `result_descriptor` only: `zRole`, `wrap` and\n", + " # `maxZBatchSize` are dataset-internal and cannot be read back. They are still exercised\n", + " # by the read below - `wrap` is why the values land in the presented -180..180 frame,\n", + " # `zRole` is why a time axis exists at all, and `maxZBatchSize` is why a 12-day window\n", + " # is split across worker requests.\n", + " assert [b.name for b in rd.bands] == [config[\"long_name\"]], (\n", + " f\"band is {[b.name for b in rd.bands]}, expected {[config['long_name']]}\"\n", + " )\n", + " assert rd.spatial_reference == \"EPSG:4326\", rd.spatial_reference\n", + " assert rd.data_type == \"F32\", rd.data_type\n", + " gb = rd.spatial_grid.spatial_grid.grid_bounds\n", + " assert (gb.top_left_idx.x_idx, gb.top_left_idx.y_idx) == (0, 0), gb\n", + " assert (gb.bottom_right_idx.x_idx, gb.bottom_right_idx.y_idx) == (N_LON - 1, N_LAT - 1), (\n", + " f\"grid bounds {gb} do not describe a {N_LON}x{N_LAT} dataset\"\n", + " )\n", + "\n", + " first_year, last_year = file_year(files[0]), file_year(files[-1])\n", + " assert rd.time.bounds.start == year_slice_start_ms(first_year), (\n", + " f\"axis starts at {rd.time.bounds.start}, expected noon of {first_year}-01-01 \"\n", + " f\"({year_slice_start_ms(first_year)})\"\n", + " )\n", + " assert rd.time.bounds.end == year_slice_start_ms(last_year + 1), (\n", + " f\"axis ends at {rd.time.bounds.end}, expected noon of {last_year + 1}-01-01\"\n", + " )\n", + " n_declared = sum(len(file_time_steps(y)) for y in range(first_year, last_year + 1))\n", + " print(f\" axis: {n_declared} daily slices, \"\n", + " f\"{dt.date(first_year, 1, 1)} .. {dt.date(last_year, 12, 31)} (noon to noon)\")\n", + "\n", + " # the workflow is assembled from typed operators, exactly like the Rust API expects it\n", + " workflow = ge.register_workflow(\n", + " workflow=ge.workflow_builder.operators.MdGdalSource(dataset=dataset_name)\n", + " )\n", + "\n", + " # --- 2. a window straddling the first file boundary --------------------\n", + " window_start, window_end = first_boundary_window(rd.time.bounds)\n", + " print(f\" reading {window_start.date()} .. {window_end.date()} (crosses a year boundary)\")\n", + "\n", + " per_day = await read_window(workflow, window_start, window_end)\n", + "\n", + " expected_days = (window_end - window_start).days\n", + " assert len(per_day) == expected_days, (\n", + " f\"expected {expected_days} daily steps, got {len(per_day)}: {sorted(per_day)}\"\n", + " )\n", + " days = sorted(per_day)\n", + " assert days[0] == window_start.date().isoformat(), (\n", + " f\"first step is {days[0]}, expected {window_start.date()}\"\n", + " )\n", + " parsed = [dt.date.fromisoformat(d) for d in days]\n", + " assert all((b - a).days == 1 for a, b in zip(parsed, parsed[1:])), f\"gap in {days}\"\n", + "\n", + " tiles_per_day = {len(v) for v in per_day.values()}\n", + " assert len(tiles_per_day) == 1, f\"inconsistent tiling per day: {tiles_per_day}\"\n", + " n_tiles = tiles_per_day.pop()\n", + " assert n_tiles > 1, \"a global 0.25-degree query must be tiled, got a single tile\"\n", + "\n", + " bands = {t.band for tiles in per_day.values() for t in tiles}\n", + " assert bands == {0}, f\"{variable} is single-band, got bands {bands}\"\n", + "\n", + " # --- 3. the stamps are the stated noon-to-noon ones --------------------\n", + " for day, date in zip(days, parsed):\n", + " expected_ms = (\n", + " year_slice_start_ms(date.year)\n", + " + (date - dt.date(date.year, 1, 1)).days * DAY_MS\n", + " )\n", + " for t in per_day[day]:\n", + " assert stamp_ms(t) == expected_ms, (\n", + " f\"{day}: tile stamped {stamp_ms(t)}, expected noon {expected_ms}\"\n", + " )\n", + " assert stamp_ms_end(t) - expected_ms == DAY_MS, (\n", + " f\"{day}: tile does not span exactly one day\"\n", + " )\n", + " print(f\" stamps: all {len(days)} days start at the stated noon\")\n", + "\n", + " # --- values: inside the range the symbology was configured for -----------\n", + " values = np.concatenate(\n", + " [np.asarray(t.data, dtype=\"float64\").ravel() for tiles in per_day.values() for t in tiles]\n", + " )\n", + " finite = values[np.isfinite(values)]\n", + " assert finite.size, \"no finite pixels at all -- the read returned only nodata\"\n", + "\n", + " lo, hi = float(finite.min()), float(finite.max())\n", + " print(f\" {len(per_day)} days x {n_tiles} tiles, band {bands}\")\n", + " print(f\" value range={lo:.2f}..{hi:.2f} {config['unit']} \"\n", + " f\"(symbology {config['min']}..{config['max']})\")\n", + "\n", + " # Not all returned pixels are data, and the two reasons are unrelated - so they are\n", + " # reported separately rather than as one \"padding\" ratio that looks alarming but is not.\n", + " #\n", + " # * Tiles are anchored at world (0,0) with a 512 px tile size (Geo Engine's tiling\n", + " # origin reference), while this dataset's presented grid is -180..180 x 60S..90N. So the\n", + " # tiles cover a larger area than the data and part of every tile is structural padding.\n", + " # * NEX-GDDP-CMIP6 itself masks a large part of the globe with a nodata sentinel, and the\n", + " # reader turns nodata into NaN. That is data, not plumbing.\n", + " #\n", + " # Structural coverage is therefore the dataset area over the area the tiles span:\n", + " gb = rd.spatial_grid.spatial_grid.grid_bounds\n", + " tl, br = gb.top_left_idx, gb.bottom_right_idx\n", + " data_px = (br.y_idx - tl.y_idx + 1) * (br.x_idx - tl.x_idx + 1)\n", + " tile_px = values.size // len(per_day) # the tile block the query returns per day\n", + " structural = data_px / tile_px\n", + " nodata = finite.size / (len(per_day) * data_px)\n", + " print(f\" tiles span {tile_px / data_px:.2f}x the dataset area \"\n", + " f\"-> structural coverage {structural:.1%}\")\n", + " print(f\" of the dataset area {nodata:.1%} carries data \"\n", + " f\"-> {structural * nodata:.1%} of all returned pixels are values\")\n", + "\n", + " # generous bounds: this asserts the read is real data, not that it is bit-exact\n", + " plausible_lo, plausible_hi = config[\"plausible\"]\n", + " assert plausible_lo <= lo and hi <= plausible_hi, (\n", + " f\"values {lo:.2f}..{hi:.2f} outside the plausible {plausible_lo}..{plausible_hi} \"\n", + " f\"for {variable}\"\n", + " )\n", + " # a global field should vary, not be a constant fill; the threshold is a\n", + " # fraction of the plausible span so it scales with the variable instead of\n", + " # assuming the Kelvin range that temperature happens to have\n", + " assert hi - lo > 0.001 * (plausible_hi - plausible_lo), (\n", + " f\"suspiciously flat field: {lo:.2f}..{hi:.2f}\"\n", + " )\n", + "\n", + " # --- determinism: the same window twice must agree (exercises the cache hint) ---\n", + " again = await read_window(workflow, window_start, window_end)\n", + " assert sorted(again) == days, \"a repeated read returned different days\"\n", + " for day in days:\n", + " a = np.sort(np.concatenate([np.asarray(t.data, dtype=\"float64\").ravel() for t in per_day[day]]))\n", + " b = np.sort(np.concatenate([np.asarray(t.data, dtype=\"float64\").ravel() for t in again[day]]))\n", + " assert np.array_equal(a, b, equal_nan=True), f\"repeated read of {day} differs\"\n", + "\n", + " print(\" OK\")\n", + " return per_day\n", + "\n", + "\n", + "def stamp_ms_end(tile):\n", + " return int(np.datetime64(tile.time_end_ms, \"ms\").astype(\"int64\"))\n", + "\n", + "\n", + "results = {}\n", + "for (model, scenario, variable, files), dataset_name in zip(TARGETS, registered_datasets):\n", + " results[(scenario, variable)] = await verify_dataset(dataset_name, variable, files)\n", + "\n", + "print(\"\\nVerified:\", \", \".join(f\"{s}/{v} ({len(d)} days)\" for (s, v), d in results.items()))\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "id": "md13", + "source": [ + "## Next Steps\n", + "\n", + "The last cell copies `nexgddp_models.full.toml` over the dev registry, so the climate-risk\n", + "`models` dropdown offers every model once this notebook finishes. From there:\n", + "\n", + "1. Run the climate-risk process against a point and confirm the ensemble spans 5 models.\n", + "2. Query a registered layer through WMS / a temporal workflow and confirm daily slices render.\n", + "3. To add a model later, append it to `nexgddp_models.full.toml`, re-run this notebook\n", + " (it registers only what is listed) and re-run the final cell.\n", + "\n", + "Comparing with `nexgddp_cmip6_ingest_manual.ipynb` shows the dev trimming this clone drops.\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# The deployed registry is the *ingested* matrix: every model/scenario below has a\n", + "# registered GeoEngine dataset. It is written from `nexgddp_models.full.toml` rather than\n", + "# copied verbatim, so the header describes the deployed state instead of the full file's\n", + "# \"not yet ingested\" restore-path note.\n", + "dev_registry = REGISTRY_PATH.with_name(\"nexgddp_models.toml\")\n", + "header = (\n", + " \"# NEX-GDDP-CMIP6 model registry - the source of truth for what climate-risk can compute.\\n\"\n", + " \"#\\n\"\n", + " \"# Every model/scenario listed here has a registered GeoEngine dataset, so the process\\n\"\n", + " \"# description and the `models` dropdown offer exactly what can run. This file is rewritten\\n\"\n", + " \"# from `nexgddp_models.full.toml` by `notebooks/nexgddp_cmip6_ingest_full.ipynb` once the\\n\"\n", + " \"# ingest is verified. To add a model, edit that full matrix and re-run the notebook.\\n\"\n", + " \"\\n\"\n", + ")\n", + "models = REGISTRY_PATH.read_text().split(\"[[model]]\", 1)[1]\n", + "dev_registry.write_text(header + \"[[model]]\" + models)\n", + "print(f\"dev registry updated: {dev_registry} now lists {len(MODELS)} models\")\n" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "name": "python", + "version": "3.11" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/openapi.json b/openapi.json index 541a160..2f74a29 100644 --- a/openapi.json +++ b/openapi.json @@ -1223,6 +1223,17 @@ ], "description": "Summary table for SSP2-4.5." }, + "ssp370": { + "oneOf": [ + { + "type": "null" + }, + { + "$ref": "https://datapackage.org/profiles/2.0/dataresource.json" + } + ], + "description": "Summary table for SSP3-7.0." + }, "ssp585": { "oneOf": [ {