Battery Electrolyte Benchmarks (Inter-Intra, Volume Scans)

docs/source/user_guide/benchmarks/electrolytes.rst

@@ -0,0 +1,83 @@
+====================
+Electrolytes
+====================
+
+LIB Electrolyte Inter-Intra Properties
+======================================
+
+Summary
+-------
+
+These tests examine the model's ability to accurately predict inter- and intra- molecular properties of a common LIB battery electrolyte and solvent.
+Inter-molecular forces heavily influence the prediction of density and diffusivity for liquids. Evaluate models on a mix of 200 LIB full electrolyte
+and neat solvent configs across a range of densities. The following predicted properties will be tested against PBE DFT:
+
+Intra-forces
+Inter-forces
+Inter-energy
+Intra-virial
+Inter-virial
+
+
+Metrics
+-------
+
+1. RMSE (PBE)
+
+Root mean square errors for each predicted property compared to PBE data.
+
+All properties listed above are calculated for each structure. The intra decomposition for a frame is achieved by isolating each molecule and evaluating it with PBE.
+The intra properties are then calculated by subtracting the intra property from the total property prediction. The D3 correction is applied both on the models and the PBE functional.
+
+Computational cost
+------------------
+
+Small: tests are likely to take seconds to 10 minutes of GPU time per model.
+
+
+Data availability
+-----------------
+
+Input structures:
+
+* Built from LIB ful electrolyte (LiPF6 EC:EMC) and neat solvent (EC:EMC) configs.
+
+Reference data:
+
+* DFT data
+
+  * PBE-D3(BJ)
+
+
+LIB Electrolyte Volume Scans
+============================
+
+Summary
+-------
+
+Evaluate model energy predictions across battery solvent and battery electrolyte Volume Scans.
+
+Metrics
+-------
+
+(1) Energy RMSE
+
+Root mean square error (RMSE) between predicted and reference energy values for each volume scan config.
+
+Volume scans consist of an initial config, where the molecules are frozen and the volume isotropically expanded or contracted.
+The resulting set of configurations represent a scan across different electrolyte densities with all intra properties remaining unchanged.
+The relative energy difference between densities is fully dependent on inter-molecular interactions, which heavily influence the density and diffusivity of an electrolyte.
+The D3 correction is applied both on the models and the PBE functional.
+
+Computational cost
+------------------
+
+Small: tests are likely to take seconds to 10 minutes of GPU time per model.
+
+Data availability
+-----------------
+
+Input structures:
+
+* Constructed using the aseMolec package https://github.com/imagdau/aseMolec.git
+* PBE

docs/source/user_guide/benchmarks/index.rst

@@ -8,6 +8,7 @@ Benchmarks
     bulk_crystal
     conformers
     defect
+    electrolytes
     f_block
     molecular
     molecular_crystal

ml_peg/analysis/electrolytes/LIB_electrolyte_inter_intra/analyse_LIB_electrolyte_inter_intra.py

@@ -0,0 +1,222 @@
+"""Analyse LIB electrolyte inter intra benchmark."""
+
+from __future__ import annotations
+
+from pathlib import Path
+
+from ase.io import read
+import numpy as np
+import pytest
+
+from ml_peg.analysis.utils.decorators import (
+    build_table,
+    plot_density_scatter,
+    plot_parity,
+)
+from ml_peg.analysis.utils.utils import get_struct_info, load_metrics_config, rmse
+from ml_peg.app import APP_ROOT
+from ml_peg.calcs import CALCS_ROOT
+from ml_peg.models import current_models
+from ml_peg.models.get_models import get_model_names
+
+MODELS = get_model_names(current_models)
+
+CALC_PATH = CALCS_ROOT / "electrolytes" / "LIB_electrolyte_inter_intra" / "outputs"
+REF_PATH = CALC_PATH / "ref"
+OUT_PATH = APP_ROOT / "data" / "electrolytes" / "LIB_electrolyte_inter_intra"
+METRICS_CONFIG_PATH = Path(__file__).with_name("metrics.yml")
+DEFAULT_THRESHOLDS, DEFAULT_TOOLTIPS, _ = load_metrics_config(METRICS_CONFIG_PATH)
+
+SYSTEM_INFO = get_struct_info(
+    calc_path=CALC_PATH,
+    glob_pattern="*.xyz",
+    index=":",
+    include_filenames=True,
+    write_structs=False,
+    out_path=OUT_PATH,
+    info_keys=["sys_formula"],
+)
+
+property_metadata = {
+    "Intra-Forces": ["arrays", "forces_intram"],
+    "Inter-Forces": ["arrays", "forces_interm"],
+    "Inter-Energy": ["info", "energy_interm"],
+    "Intra-Virial": ["info", "virial_intram"],
+    "Inter-Virial": ["info", "virial_interm"],
+}
+
+
+def get_property_results(prop_key: str) -> dict[str, float]:
+    """
+    Get inter-intra results for a specific property.
+
+    Parameters
+    ----------
+    prop_key
+        String of property name.
+
+    Returns
+    -------
+    dict[str, list]
+        Dictionary of reference and predicted inter-intra property.
+    """
+    results = {"ref": []} | {mlip: [] for mlip in MODELS}
+
+    stored, property = property_metadata[prop_key]
+
+    for model in results.keys():
+        if model == "ref":
+            configs = read(REF_PATH / "intrainter_PBED3.xyz", ":")
+
+        else:
+            configs = read(CALC_PATH / model / f"intrainter_{model}_D3.xyz", ":")
+
+        for frame in configs:
+            frame_data = getattr(frame, stored)
+            property_data = frame_data[property]
+            results[model].append(property_data.tolist())
+
+        if "Forces" in prop_key:
+            results[model] = np.concatenate(results[model]).flatten()
+            results[model] = results[model].tolist()
+
+        if "Virial" in prop_key:
+            results[model] = np.array(results[model]).flatten()
+            results[model] = results[model].tolist()
+
+    return results
+
+
+def plot_parity_results(prop_key: str, results: dict[str, float]) -> None:
+    """
+    Plot inter-intra property parity plots.
+
+    Parameters
+    ----------
+    prop_key
+        Name of inter-intra property to be plotted.
+    results
+        Results from all models for a single property.
+    """
+
+    @plot_parity(
+        filename=OUT_PATH / f"{prop_key.lower()}_parity.json",
+        title=prop_key,
+        x_label=f"Predicted {prop_key} / {DEFAULT_THRESHOLDS[prop_key]['unit']}",
+        y_label=f"DFT {prop_key} / {DEFAULT_THRESHOLDS[prop_key]['unit']}",
+        plot_combined=False,
+    )
+    def plot_parity_result() -> dict[str, list[float]]:
+        """
+        Plot the inter-intra propery parity plots.
+
+        Returns
+        -------
+        dict[str, tuple[list[float]]]
+            Dictionary of reference and predicted inter-intra property.
+        """
+        return results
+
+    plot_parity_result()
+
+
+def plot_density_parity_results(prop_key: str, results: dict[str, float]) -> None:
+    """
+    Plot inter-intra property parity density plots.
+
+    Parameters
+    ----------
+    prop_key
+        Name of inter-intra property to be plotted.
+    results
+        Results from all models for a single property.
+    """
+
+    @plot_density_scatter(
+        filename=OUT_PATH / f"{prop_key.lower()}_density_parity.json",
+        title=prop_key,
+        x_label=f"Predicted {prop_key} / {DEFAULT_THRESHOLDS[prop_key]['unit']}",
+        y_label=f"DFT {prop_key} / {DEFAULT_THRESHOLDS[prop_key]['unit']}",
+    )
+    def plot_density_parity_result() -> dict[str, list[float]]:
+        """
+        Plot the inter-intra propery density parity plots.
+
+        Returns
+        -------
+        dict[str, tuple[list[float]]]
+            Dictionary of reference and predicted inter-intra property.
+        """
+        results_formatted: dict[str, dict] = {}
+        ref_vals = results["ref"]
+        for model, model_pred in results.items():
+            if model != "ref":
+                results_formatted[model] = {
+                    "ref": ref_vals,
+                    "pred": model_pred,
+                }
+        return results_formatted
+
+    plot_density_parity_result()
+
+
+@pytest.fixture
+def get_property_rmses() -> dict[str, dict]:
+    """
+    Get model prediction RMSEs for all inter-intra properties.
+
+    Returns
+    -------
+    dict[str, dict]
+        Dictionary of inter-intra properties and the respective RMSE per model.
+    """
+    property_rmse = {prop_key: {} for prop_key in property_metadata.keys()}
+
+    for prop_key in property_metadata.keys():
+        results = get_property_results(prop_key)
+        if "Forces" in prop_key:
+            plot_density_parity_results(prop_key, results)
+        else:
+            plot_parity_results(prop_key, results)
+        for model in MODELS:
+            model_rmse = rmse(results["ref"], results[model])
+            property_rmse[prop_key][model] = model_rmse
+
+    return property_rmse
+
+
+@pytest.fixture
+@build_table(
+    filename=OUT_PATH / "inter_intra_metrics_table.json",
+    metric_tooltips=DEFAULT_TOOLTIPS,
+    thresholds=DEFAULT_THRESHOLDS,
+)
+def rmse_metrics(get_property_rmses: dict[str, dict]) -> dict[str, dict]:
+    """
+    Get all inter intra RMSE metrics.
+
+    Parameters
+    ----------
+    get_property_rmses
+        Dictionary for every property containing each model's RMSE.
+
+    Returns
+    -------
+    dict[str, dict]
+        Dictionary for every property containing each model's RMSE.
+    """
+    return get_property_rmses
+
+
+def test_rmse_metrics(
+    rmse_metrics: dict[str, dict],
+) -> None:
+    """
+    Run inter-intra property test.
+
+    Parameters
+    ----------
+    rmse_metrics
+        All inter-intra metrics.
+    """
+    return

ml_peg/analysis/electrolytes/LIB_electrolyte_inter_intra/metrics.yml

@@ -0,0 +1,36 @@
+metrics:
+  Intra-Forces:
+    good: 0.1
+    bad: 0.5
+    unit: eV/Å
+    tooltip: "Weighted Root Mean Square Deviation of Intra-Forces"
+    level_of_theory: PBE
+    weight: 1
+  Inter-Forces:
+    good: 0.1
+    bad: 0.5
+    unit: eV/Å
+    tooltip: "Weighted Root Mean Square Deviation of Inter-Forces"
+    level_of_theory: PBE
+    weight: 1
+  Inter-Energy:
+    good: 5
+    bad: 60.0
+    unit: meV/atom
+    tooltip: "Weighted Root Mean Square Deviation of Inter-Energy"
+    level_of_theory: PBE
+    weight: 1
+  Intra-Virial:
+    good: 10.0
+    bad: 50.0
+    unit: meV
+    tooltip: "Weighted Root Mean Square Deviation of Intra-Virial"
+    level_of_theory: PBE
+    weight: 1
+  Inter-Virial:
+    good: 1.0
+    bad: 50.0
+    unit: meV
+    tooltip: "Weighted Root Mean Square Deviation of Inter-Virial"
+    level_of_theory: PBE
+    weight: 1