muse2/model/

parameters.rs

//! Read and validate model parameters from `model.toml`.
//!
//! This module defines the `ModelParameters` struct and helpers for loading and validating the
//! `model.toml` configuration used by the model. Validation functions ensure sensible numeric
//! ranges and invariants for runtime use.
use crate::asset::check_capacity_valid_for_asset;
use crate::commodity::PricingStrategy;
use crate::input::{
    deserialise_proportion_nonzero, input_err_msg, is_sorted_and_unique, read_toml,
};
use crate::units::{Capacity, Dimensionless, Flow, MoneyPerFlow};
use anyhow::{Context, Result, ensure};
use itertools::Itertools;
use log::warn;
use serde::{Deserialize, Deserializer};
use std::path::Path;
use std::sync::OnceLock;
use toml::Table;

const MODEL_PARAMETERS_FILE_NAME: &str = "model.toml";

/// The key in `model.toml` which enables potentially dangerous model options.
///
/// If this option is present and true, the model will permit certain experimental or unsafe
/// behaviours that are normally disallowed.
pub const ALLOW_DANGEROUS_OPTION_NAME: &str = "please_give_me_broken_results";

/// Global flag indicating whether potentially dangerous model options have been enabled.
///
/// This is stored in a `OnceLock` and must be set exactly once during startup (see
/// [`set_dangerous_model_options_flag`]).
static DANGEROUS_OPTIONS_ENABLED: OnceLock<bool> = OnceLock::new();

/// The default value for the `remaining_demand_absolute_tolerance` parameter
const DEFAULT_REMAINING_DEMAND_ABSOLUTE_TOLERANCE: Flow = Flow(1e-12);

/// Whether potentially dangerous model options were enabled by the loaded config.
///
/// # Panics
///
/// Panics if the global flag has not been set yet (the flag should be set by
/// [`ModelParameters::from_path`] during program initialisation).
pub fn dangerous_model_options_enabled() -> bool {
    *DANGEROUS_OPTIONS_ENABLED
        .get()
        .expect("Dangerous options flag not set")
}

/// Set the global flag indicating whether potentially dangerous model options are enabled.
///
/// Can only be called once; subsequent calls will panic (except in tests, where it can be called
/// multiple times so long as the value is the same).
fn set_dangerous_model_options_flag(enabled: bool) {
    let result = DANGEROUS_OPTIONS_ENABLED.set(enabled);
    if result.is_err() {
        if cfg!(test) {
            // Sanity check
            assert_eq!(enabled, dangerous_model_options_enabled());
        } else {
            panic!("Attempted to set DANGEROUS_OPTIONS_ENABLED twice");
        }
    }
}

/// Model parameters as defined in the `model.toml` file.
///
/// NOTE: If you add or change a field in this struct, you must also update the schema in
/// `schemas/input/model.yaml`.
#[derive(Deserialize)]
#[serde(default)]
pub struct ModelParameters {
    /// Milestone years
    pub milestone_years: Vec<u32>,
    /// Allow potentially dangerous options to be enabled.
    #[serde(rename = "please_give_me_broken_results")] // Can't use constant here :-(
    pub allow_dangerous_options: bool,
    /// The (small) value of capacity given to candidate assets.
    ///
    /// Don't change unless you know what you're doing.
    pub candidate_asset_capacity: Capacity,
    /// The epsilon added to commodity balance lower bounds to force candidate dispatch.
    ///
    /// Don't change unless you know what you're doing.
    pub commodity_balance_epsilon: Flow,
    /// Affects the maximum capacity that can be given to a newly created asset.
    ///
    /// It is the proportion of maximum capacity that could be required across time slices.
    #[serde(deserialize_with = "deserialise_proportion_nonzero")]
    pub capacity_limit_factor: Dimensionless,
    /// The pricing strategy used to calculate fallback prices for the mini dispatch optimisation
    /// during investment appraisal.
    ///
    /// If set to `unpriced`, a fallback price of zero is used, which reverts to the
    /// pure shadow-price formulation.
    pub fallback_pricing_strategy: PricingStrategy,
    /// The cost applied to unmet demand.
    ///
    /// Currently this only applies to the LCOX appraisal.
    pub value_of_lost_load: MoneyPerFlow,
    /// The maximum number of iterations to run the "ironing out" step of agent investment for
    pub max_ironing_out_iterations: u32,
    /// The relative tolerance for price convergence in the ironing out loop
    pub price_tolerance: Dimensionless,
    /// Slack applied during cycle balancing, allowing newly selected assets to flex their capacity
    /// by this proportion.
    ///
    /// Existing assets remain fixed; this gives newly selected assets the wiggle-room to absorb
    /// small demand changes before we would otherwise need to break for re-investment.
    pub capacity_margin: Dimensionless,
    /// Number of years an asset can remain unused before being decommissioned
    pub mothball_years: u32,
    /// Absolute tolerance when checking if remaining demand is close enough to zero
    pub remaining_demand_absolute_tolerance: Flow,
    /// Options for the HiGHS solver.
    ///
    /// For a full list of options, see [the HiGHS documentation].
    ///
    /// [the HiGHS documentation]: https://ergo-code.github.io/HiGHS/stable/options/definitions/
    pub highs: HighsOptions,
}

impl Default for ModelParameters {
    fn default() -> Self {
        Self {
            // Required parameters.
            // milestone_years cannot be empty and we validate this when loading model.toml files.
            milestone_years: Vec::default(),

            // Default values for optional parameters
            allow_dangerous_options: false,
            candidate_asset_capacity: Capacity(1e-4),
            commodity_balance_epsilon: Flow(1e-6),
            capacity_limit_factor: Dimensionless(0.05),
            fallback_pricing_strategy: PricingStrategy::FullCostAverage,
            value_of_lost_load: MoneyPerFlow(1e9),
            max_ironing_out_iterations: 1,
            price_tolerance: Dimensionless(1e-6),
            capacity_margin: Dimensionless(0.2),
            mothball_years: 0,
            remaining_demand_absolute_tolerance: DEFAULT_REMAINING_DEMAND_ABSOLUTE_TOLERANCE,
            highs: HighsOptions::default(),
        }
    }
}

/// Defines the TOML table holding the sub-tables to define HiGHS options
#[derive(Default)]
pub struct HighsOptions {
    /// HiGHS options applied to dispatch optimisation
    pub dispatch_options: Table,
    /// HiGHS options applied to appraisal optimisation
    pub appraisal_options: Table,
}

impl<'de> Deserialize<'de> for HighsOptions {
    fn deserialize<D>(deserializer: D) -> std::result::Result<Self, D::Error>
    where
        D: Deserializer<'de>,
    {
        #[derive(Default, Deserialize)]
        #[serde(default)]
        #[allow(clippy::struct_field_names)]
        struct RawHighsOptions {
            global_options: Table,
            dispatch_options: Table,
            appraisal_options: Table,
        }

        let RawHighsOptions {
            global_options,
            mut dispatch_options,
            mut appraisal_options,
        } = RawHighsOptions::deserialize(deserializer)?;

        let append_global_options = |options: &mut Table| {
            for (option, value) in &global_options {
                options
                    .entry(option.clone())
                    .or_insert_with(|| value.clone());
            }
        };
        append_global_options(&mut dispatch_options);
        append_global_options(&mut appraisal_options);

        Ok(Self {
            dispatch_options,
            appraisal_options,
        })
    }
}

impl HighsOptions {
    /// Log custom HiGHS options set by user, if any
    fn log_options(&self) {
        fn log_highs_options(name: &str, options: &Table) {
            if options.is_empty() {
                return;
            }

            let options_str = options
                .iter()
                .format_with("\n  - ", |(opt, val), f| f(&format_args!("{opt} = {val}")))
                .to_string();
            warn!("Using custom HiGHS options for {name}:\n  - {options_str}");
        }

        log_highs_options("dispatch", &self.dispatch_options);
        log_highs_options("appraisal", &self.appraisal_options);
    }

    /// Check whether any options have been set
    pub fn is_empty(&self) -> bool {
        self.dispatch_options.is_empty() && self.appraisal_options.is_empty()
    }
}

/// Check that the `milestone_years` parameter is valid
fn check_milestone_years(years: &[u32]) -> Result<()> {
    ensure!(
        !years.is_empty(),
        "`milestone_years` must be provided and non-empty"
    );

    ensure!(
        is_sorted_and_unique(years),
        "`milestone_years` must be composed of unique values in order"
    );

    Ok(())
}

/// Check that the `value_of_lost_load` parameter is valid
fn check_value_of_lost_load(value: MoneyPerFlow) -> Result<()> {
    ensure!(
        value.is_finite() && value > MoneyPerFlow(0.0),
        "value_of_lost_load must be a finite number greater than zero"
    );

    Ok(())
}

/// Check that the `max_ironing_out_iterations` parameter is valid
fn check_max_ironing_out_iterations(value: u32) -> Result<()> {
    ensure!(value > 0, "max_ironing_out_iterations cannot be zero");

    Ok(())
}

/// Check the `price_tolerance` parameter is valid
fn check_price_tolerance(value: Dimensionless) -> Result<()> {
    ensure!(
        value.is_finite() && value >= Dimensionless(0.0),
        "price_tolerance must be a finite number greater than or equal to zero"
    );

    Ok(())
}

fn check_remaining_demand_absolute_tolerance(
    dangerous_options_enabled: bool,
    value: Flow,
) -> Result<()> {
    ensure!(
        value.is_finite() && value >= Flow(0.0),
        "remaining_demand_absolute_tolerance must be a finite number greater than or equal to zero"
    );

    if !dangerous_options_enabled {
        ensure!(
            value == DEFAULT_REMAINING_DEMAND_ABSOLUTE_TOLERANCE,
            "Setting a remaining_demand_absolute_tolerance different from the default value of \
            {:e} is potentially dangerous, set {ALLOW_DANGEROUS_OPTION_NAME} to true if you want \
            to allow this.",
            DEFAULT_REMAINING_DEMAND_ABSOLUTE_TOLERANCE.value()
        );
    }

    Ok(())
}

/// Check that the `capacity_margin` parameter is valid
fn check_capacity_margin(value: Dimensionless) -> Result<()> {
    ensure!(
        value.is_finite() && value >= Dimensionless(0.0),
        "capacity_margin must be a finite number greater than or equal to zero"
    );

    Ok(())
}

/// Check the custom HiGHS options are valid.
///
/// Note that we cannot know whether the options specified exist and are of the correct type until
/// we attempt to use them. We could check for types that are never valid (e.g. an array), but as
/// we're checking later anyway, we don't bother.
fn check_highs_options(dangerous_options_enabled: bool, highs: &HighsOptions) -> Result<()> {
    ensure!(
        dangerous_options_enabled || highs.is_empty(),
        "Cannot set custom HiGHS options without enabling {ALLOW_DANGEROUS_OPTION_NAME}"
    );

    Ok(())
}

impl ModelParameters {
    /// Read a model file from the specified directory.
    ///
    /// # Arguments
    ///
    /// * `model_dir` - Folder containing model configuration files
    ///
    /// # Returns
    ///
    /// The model file contents as a [`ModelParameters`] struct or an error if the file is invalid
    pub fn from_path<P: AsRef<Path>>(model_dir: P) -> Result<ModelParameters> {
        let file_path = model_dir.as_ref().join(MODEL_PARAMETERS_FILE_NAME);
        let model_params: ModelParameters = read_toml(&file_path)?;

        set_dangerous_model_options_flag(model_params.allow_dangerous_options);

        model_params
            .validate()
            .with_context(|| input_err_msg(file_path))?;

        model_params.highs.log_options();

        Ok(model_params)
    }

    /// Validate parameters after reading in file
    fn validate(&self) -> Result<()> {
        if self.allow_dangerous_options {
            warn!(
                "!!! You've enabled the {ALLOW_DANGEROUS_OPTION_NAME} option. !!!\n\
                I see you like to live dangerously 😈. This option should ONLY be used by \
                developers as it can cause peculiar behaviour that breaks things. NEVER enable it \
                for results you actually care about or want to publish. You have been warned!"
            );
        }

        // milestone_years
        check_milestone_years(&self.milestone_years)?;

        // capacity_limit_factor already validated with deserialise_proportion_nonzero

        // fallback_pricing_strategy already validated by deserialisation

        // candidate_asset_capacity
        check_capacity_valid_for_asset(self.candidate_asset_capacity)
            .context("Invalid value for candidate_asset_capacity")?;

        // commodity_balance_epsilon
        ensure!(
            self.commodity_balance_epsilon.is_finite()
                && self.commodity_balance_epsilon >= Flow(0.0),
            "commodity_balance_epsilon must be a finite number greater than or equal to zero"
        );

        // value_of_lost_load
        check_value_of_lost_load(self.value_of_lost_load)?;

        // max_ironing_out_iterations
        check_max_ironing_out_iterations(self.max_ironing_out_iterations)?;

        // price_tolerance
        check_price_tolerance(self.price_tolerance)?;

        // capacity_margin
        check_capacity_margin(self.capacity_margin)?;

        // remaining_demand_absolute_tolerance
        check_remaining_demand_absolute_tolerance(
            self.allow_dangerous_options,
            self.remaining_demand_absolute_tolerance,
        )?;

        check_highs_options(self.allow_dangerous_options, &self.highs)?;

        Ok(())
    }
}

#[cfg(test)]
mod tests {
    use super::*;
    use rstest::rstest;
    use std::fmt::Display;
    use std::fs::File;
    use std::io::Write;
    use tempfile::tempdir;

    /// Helper function to assert validation result based on expected validity
    fn assert_validation_result<T, U: Display>(
        result: Result<T>,
        expected_valid: bool,
        value: U,
        expected_error_fragment: &str,
    ) {
        if expected_valid {
            assert!(
                result.is_ok(),
                "Expected value {} to be valid, but got error: {:?}",
                value,
                result.err()
            );
        } else {
            assert!(
                result.is_err(),
                "Expected value {value} to be invalid, but it was accepted",
            );
            let error_message = result.err().unwrap().to_string();
            assert!(
                error_message.contains(expected_error_fragment),
                "Error message should mention the validation constraint, got: {error_message}",
            );
        }
    }

    #[test]
    fn check_milestone_years_works() {
        // Valid
        check_milestone_years(&[1]).unwrap();
        check_milestone_years(&[1, 2]).unwrap();

        // Invalid
        assert!(check_milestone_years(&[]).is_err());
        assert!(check_milestone_years(&[1, 1]).is_err());
        assert!(check_milestone_years(&[2, 1]).is_err());
    }

    #[test]
    fn model_params_from_path() {
        let dir = tempdir().unwrap();
        {
            let mut file = File::create(dir.path().join(MODEL_PARAMETERS_FILE_NAME)).unwrap();
            writeln!(file, "milestone_years = [2020, 2100]").unwrap();
        }

        let model_params = ModelParameters::from_path(dir.path()).unwrap();
        assert_eq!(model_params.milestone_years, [2020, 2100]);
    }

    #[test]
    fn model_params_deserialisation_copies_highs_global_options() {
        let model_params: ModelParameters = toml::from_str(
            "
            milestone_years = [2020, 2100]

            [highs.global_options]
            output_flag = true

            [highs.dispatch_options]
            log_to_console = false
            ",
        )
        .unwrap();

        assert_eq!(
            model_params.highs.dispatch_options["output_flag"],
            toml::Value::Boolean(true)
        );
        assert_eq!(
            model_params.highs.dispatch_options["log_to_console"],
            toml::Value::Boolean(false)
        );
        assert_eq!(
            model_params.highs.appraisal_options["output_flag"],
            toml::Value::Boolean(true)
        );
    }

    #[test]
    fn highs_options_deserialisation_copies_global_options() {
        let highs: HighsOptions = toml::from_str(
            "
            [global_options]
            output_flag = true
            log_to_console = true

            [dispatch_options]
            primal_feasibility_tolerance = 1e-5

            [appraisal_options]
            optimality_tolerance = 1e-5
            ",
        )
        .unwrap();

        assert_eq!(
            highs.dispatch_options["output_flag"],
            toml::Value::Boolean(true)
        );
        assert_eq!(
            highs.dispatch_options["log_to_console"],
            toml::Value::Boolean(true)
        );
        assert_eq!(
            highs.appraisal_options["output_flag"],
            toml::Value::Boolean(true)
        );
        assert_eq!(
            highs.appraisal_options["log_to_console"],
            toml::Value::Boolean(true)
        );
    }

    #[test]
    fn highs_options_deserialisation_preserves_specific_options() {
        let highs: HighsOptions = toml::from_str(
            "
            [global_options]
            output_flag = true
            log_to_console = true

            [dispatch_options]
            output_flag = false

            [appraisal_options]
            log_to_console = false
            ",
        )
        .unwrap();

        assert_eq!(
            highs.dispatch_options["output_flag"],
            toml::Value::Boolean(false)
        );
        assert_eq!(
            highs.dispatch_options["log_to_console"],
            toml::Value::Boolean(true)
        );
        assert_eq!(
            highs.appraisal_options["output_flag"],
            toml::Value::Boolean(true)
        );
        assert_eq!(
            highs.appraisal_options["log_to_console"],
            toml::Value::Boolean(false)
        );
    }

    #[rstest]
    #[case(1.0, true)] // Valid positive value
    #[case(1e-10, true)] // Valid very small positive value
    #[case(1e9, true)] // Valid large value (default)
    #[case(f64::MAX, true)] // Valid maximum finite value
    #[case(0.0, false)] // Invalid: exactly zero
    #[case(-1.0, false)] // Invalid: negative value
    #[case(-1e-10, false)] // Invalid: very small negative value
    #[case(f64::INFINITY, false)] // Invalid: infinite value
    #[case(f64::NEG_INFINITY, false)] // Invalid: negative infinite value
    #[case(f64::NAN, false)] // Invalid: NaN value
    fn check_value_of_lost_load_works(#[case] value: f64, #[case] expected_valid: bool) {
        let money_per_flow = MoneyPerFlow::new(value);
        let result = check_value_of_lost_load(money_per_flow);

        assert_validation_result(
            result,
            expected_valid,
            value,
            "value_of_lost_load must be a finite number greater than zero",
        );
    }

    #[rstest]
    #[case(1, true)] // Valid minimum value
    #[case(10, true)] // Valid default value
    #[case(100, true)] // Valid large value
    #[case(u32::MAX, true)] // Valid maximum value
    #[case(0, false)] // Invalid: zero
    fn check_max_ironing_out_iterations_works(#[case] value: u32, #[case] expected_valid: bool) {
        let result = check_max_ironing_out_iterations(value);

        assert_validation_result(
            result,
            expected_valid,
            value,
            "max_ironing_out_iterations cannot be zero",
        );
    }

    #[rstest]
    #[case(0.0, true)] // Valid minimum value (exactly zero)
    #[case(1e-10, true)] // Valid very small positive value
    #[case(1e-6, true)] // Valid default value
    #[case(1.0, true)] // Valid larger value
    #[case(f64::MAX, true)] // Valid maximum finite value
    #[case(-1e-10, false)] // Invalid: negative value
    #[case(-1.0, false)] // Invalid: negative value
    #[case(f64::INFINITY, false)] // Invalid: infinite value
    #[case(f64::NEG_INFINITY, false)] // Invalid: negative infinite value
    #[case(f64::NAN, false)] // Invalid: NaN value
    fn check_price_tolerance_works(#[case] value: f64, #[case] expected_valid: bool) {
        let dimensionless = Dimensionless::new(value);
        let result = check_price_tolerance(dimensionless);

        assert_validation_result(
            result,
            expected_valid,
            value,
            "price_tolerance must be a finite number greater than or equal to zero",
        );
    }

    #[rstest]
    #[case(true, 0.0, true)] // Valid minimum value dangerous options allowed
    #[case(true, 1e-10, true)] // Valid value with dangerous options allowed
    #[case(true, 1e-15, true)] // Valid value with dangerous options allowed
    #[case(false, 1e-12, true)] // Valid value same as default, no dangerous options needed
    #[case(true, 1.0, true)] // Valid larger value with dangerous options allowed
    #[case(true, f64::MAX, true)] // Valid maximum finite value with dangerous options allowed
    #[case(true, -1e-10, false)] // Invalid: negative value
    #[case(true, f64::INFINITY, false)] // Invalid: positive infinity
    #[case(true, f64::NEG_INFINITY, false)] // Invalid: negative infinity
    #[case(true, f64::NAN, false)] // Invalid: NaN
    #[case(false, -1e-10, false)] // Invalid: negative value
    #[case(false, f64::INFINITY, false)] // Invalid: positive infinity
    #[case(false, f64::NEG_INFINITY, false)] // Invalid: negative infinity
    #[case(false, f64::NAN, false)] // Invalid: NaN
    fn check_remaining_demand_absolute_tolerance_works(
        #[case] allow_dangerous_options: bool,
        #[case] value: f64,
        #[case] expected_valid: bool,
    ) {
        let flow = Flow::new(value);
        let result = check_remaining_demand_absolute_tolerance(allow_dangerous_options, flow);

        assert_validation_result(
            result,
            expected_valid,
            value,
            "remaining_demand_absolute_tolerance must be a finite number greater than or equal to zero",
        );
    }

    #[rstest]
    #[case(0.0)] // smaller than default
    #[case(1e-10)] // Larger than default (1e-12)
    #[case(1.0)] // Well above default
    #[case(f64::MAX)] // Maximum finite value
    fn check_remaining_demand_absolute_tolerance_requires_dangerous_options_if_non_default(
        #[case] value: f64,
    ) {
        let flow = Flow::new(value);
        let result = check_remaining_demand_absolute_tolerance(false, flow);
        assert_validation_result(
            result,
            false,
            value,
            "Setting a remaining_demand_absolute_tolerance different from the default value of \
            1e-12 is potentially dangerous, set please_give_me_broken_results to true if you want \
            to allow this.",
        );
    }

    #[rstest]
    #[case(0.0, true)] // Valid minimum value
    #[case(0.2, true)] // Valid default value
    #[case(10.0, true)] // Valid large value
    #[case(-1e-6, false)] // Invalid: negative margin
    #[case(f64::INFINITY, false)] // Invalid: infinite value
    #[case(f64::NEG_INFINITY, false)] // Invalid: negative infinite value
    #[case(f64::NAN, false)] // Invalid: NaN value
    fn check_capacity_margin_works(#[case] value: f64, #[case] expected_valid: bool) {
        let result = check_capacity_margin(Dimensionless(value));

        assert_validation_result(
            result,
            expected_valid,
            value,
            "capacity_margin must be a finite number greater than or equal to zero",
        );
    }
}