From 93ffdae8e793bde5b2fa4c76c4793e4eb82c1504 Mon Sep 17 00:00:00 2001 From: Tomasini Luca Date: Wed, 4 Dec 2024 14:42:15 +0100 Subject: [PATCH] Finalize second stage output --- docs/_build/doctrees/environment.pickle | Bin 29738 -> 29794 bytes .../doctrees/first_model_baseline.doctree | Bin 61513 -> 61593 bytes .../_sources/first_model_baseline.rst.txt | 14 +- docs/first_model_baseline.rst | 14 +- settings.toml | 3 +- .../first_stage_optimization_plots.py | 6 +- src/optimization/first_model_baseline.py | 24 - src/optimization/optimizaztion_pipeline.py | 38 - src/pyomo_models/baseline/baseline_input.py | 52 + .../first_stage}/constraints/basin_volume.py | 0 .../baseline/first_stage}/constraints/pump.py | 0 .../first_stage}/constraints/turbine.py | 0 .../baseline/first_stage}/objective.py | 0 .../baseline/first_stage}/parameters.py | 0 .../baseline/first_stage/pipeline.py | 137 ++ .../baseline/first_stage}/sets.py | 0 .../baseline/first_stage}/variables.py | 0 .../second_stage/constraints/basin_volume.py | 152 ++ .../baseline/second_stage/constraints/pump.py | 60 + .../second_stage/constraints/turbine.py | 59 + .../baseline/second_stage/objective.py | 26 + .../baseline/second_stage/parameters.py | 24 + .../baseline/second_stage/pipeline.py | 137 ++ .../baseline/second_stage/sets.py | 45 + .../baseline/second_stage/variables.py | 20 + .../input_data_preprocessing.py} | 195 +- src/utility/general_function.py | 14 + src/utility/pyomo_preprocessing.py | 138 +- tests/base_line.ipynb | 1784 +++++++---------- 29 files changed, 1633 insertions(+), 1309 deletions(-) delete mode 100644 src/optimization/first_model_baseline.py delete mode 100644 src/optimization/optimizaztion_pipeline.py create mode 100644 src/pyomo_models/baseline/baseline_input.py rename src/{optimization/pyomo_models => pyomo_models/baseline/first_stage}/constraints/basin_volume.py (100%) rename src/{optimization/pyomo_models => pyomo_models/baseline/first_stage}/constraints/pump.py (100%) rename src/{optimization/pyomo_models => pyomo_models/baseline/first_stage}/constraints/turbine.py (100%) rename src/{optimization/pyomo_models => pyomo_models/baseline/first_stage}/objective.py (100%) rename src/{optimization/pyomo_models => pyomo_models/baseline/first_stage}/parameters.py (100%) create mode 100644 src/pyomo_models/baseline/first_stage/pipeline.py rename src/{optimization/pyomo_models => pyomo_models/baseline/first_stage}/sets.py (100%) rename src/{optimization/pyomo_models => pyomo_models/baseline/first_stage}/variables.py (100%) create mode 100644 src/pyomo_models/baseline/second_stage/constraints/basin_volume.py create mode 100644 src/pyomo_models/baseline/second_stage/constraints/pump.py create mode 100644 src/pyomo_models/baseline/second_stage/constraints/turbine.py create mode 100644 src/pyomo_models/baseline/second_stage/objective.py create mode 100644 src/pyomo_models/baseline/second_stage/parameters.py create mode 100644 src/pyomo_models/baseline/second_stage/pipeline.py create mode 100644 src/pyomo_models/baseline/second_stage/sets.py create mode 100644 src/pyomo_models/baseline/second_stage/variables.py rename src/{optimization/input_data_prepocessing.py => pyomo_models/input_data_preprocessing.py} (51%) diff --git a/docs/_build/doctrees/environment.pickle b/docs/_build/doctrees/environment.pickle index d02600d87ce61500355dddcc7f04fe3c059345ec..4fb143fface51d3951431d61dc4097404d6e6e62 100644 GIT binary patch delta 1541 zcma)*T}TvB6vuaT-PJTpTg%dx1WQfbRkDl{q_kGsK2-e3a+Ps+T<7lW&N4H8MYTaB zeJn)X33~B42!dkYDkzAew_2fss0WKgp`fIoAf+?+j+YD)U;cZ}?|;vpb3f+x4EJG% zvz|-4(RI(7Y{*CxBLQJlk2`S7paah}v=mIHS}f=dNIZjhE=AS!3woQ5lGxW+kV*Wv z#-e26e4{gUlL0QSr1R$SH-#%qrhC+_#;arj(YE}rsDammD!-atyaF|KzNyg@g=A9kSd z+rgROyQ$+dUgv`UFznD9C;|-!3TjD7>bM?X1J6v1REGJ{N_b(Kpd(^g|FXbdwD6-q zA^gL;fU|}yv%|90+K|!@?=(<3AjuC^pdm<^sZC~`bF9SS`-e(jN#Eoo5eD&qiOgdM?6>$+00^wW~u#|fiDz% zW#AhH-x>Hp!A}N$QSh6Ac>-d{VmF&24kGAlWgv}$bOtgg$YQ`8d<_HUsJRT7qvkPS zj+#$F44Rv>7O*AetWE~#tf+_qbJSu6%u!1iFh|{P0P?w%Lj=2XEdp}JRiuFU)zq(9 P^pROmOF*JRUaozHY@d<)-(V+>@K!cwg!q zjC9DlmtHp5x%!*`fZKi;f&DTNi$D2`kboPApqo=N*kEb}-}N~!Jtz#j<%e&c`}sa6 z&vVXmZhkLIXUmdr)_>~FAAQB`QWJhR)l}`del4r0AK^RO$J>f4Dk{*gAZ`tX5!150 z&%FH}D&ff;ZFSVYLXt{_lRMU}XeHpQK)S7c;bB)5D$%Fa8_5&?d3d;+*xz^|cQ6u( zTIO`r$k|#(`&hO$BOA?~Ffzte#!yt}B|Ky3PDb@m5icmMkatyS zJxufbkQZcmwqeV9*s^7tB$A3%V7|_eUI`!7jpHxb0k9ihvPpq4wIxqi*N~qk^H%?5 zZ7rB}FPOYy9pHM7bH;JiIQpck|W(6wkEsDa)A{SI}r8}err z`~mn5z+2^g?zuJMogq73M4gb@k|sKoUJt6IH-nx@4}o4y_klL(y`Wq4ZS)ShYw(ob zj8E$@N4%hK04?i-0eS;_12`Ug0}p3{KqoRudYI@2)SvUvE%>6dNo|9t^3oF`?E&~p zfWH~|M}XT5EDCUsf%^hHVBoO;PbkQviU2Q)B#5AlPXIpy)dJKoP%i*)e5C-q)l~xU zR#ywaTWw|_55;+}Z6byDx=sMrE9ww{x7sNHZ?#JR-f9m4{N(BGB1?dtaUWek{Y_xN S?)#T~)XTsC1qH=;>U$2p?=hhO diff --git a/docs/_build/doctrees/first_model_baseline.doctree b/docs/_build/doctrees/first_model_baseline.doctree index 671d288f3262db27304ac6b1741a65661c72f783..6befff179a570becb7078e53680d9c239f37adf2 100644 GIT binary patch delta 445 zcmX^4fO+OaW|julseKz+*2ytWp4_m)ETW(?KQ})Ta z`Br`X&DzPLpKTp4)GCwyzJ~uxlHK$m= zIJKl$ub|Q@B|o{iq$o2lT_HbB0a-av5}|yvqr6`yV=u@|Ut~q7X67elr6!kTmZg$l zu=2_;%#*9mnM>j{c5?n&EfiT)m+e}sBFwlD~u O(m5qOMu?toQ3L>_j-S^6 diff --git a/docs/_build/html/_sources/first_model_baseline.rst.txt b/docs/_build/html/_sources/first_model_baseline.rst.txt index b2f1b83..987048d 100644 --- a/docs/_build/html/_sources/first_model_baseline.rst.txt +++ b/docs/_build/html/_sources/first_model_baseline.rst.txt @@ -8,13 +8,13 @@ Sets :file: /tables/sets_docs.csv :header-rows: 1 -.. automodule:: optimization.pyomo_models.sets +.. automodule:: pyomo_models.baseline.first_stage.sets :no-index: Variables ---------- -.. automodule:: optimization.pyomo_models.variables +.. automodule:: pyomo_models.baseline.first_stage.variables :no-index: .. csv-table:: @@ -24,7 +24,7 @@ Variables Parameters ---------- -.. automodule:: optimization.pyomo_models.parameters +.. automodule:: pyomo_models.baseline.first_stage.parameters :no-index: .. csv-table:: @@ -34,18 +34,18 @@ Parameters Objective ---------- -.. automodule:: optimization.pyomo_models.objective +.. automodule:: pyomo_models.baseline.first_stage.objective :no-index: Constraints ------------ -.. automodule:: optimization.pyomo_models.constraints.basin_volume +.. automodule:: pyomo_models.baseline.first_stage.constraints.basin_volume :no-index: -.. automodule:: optimization.pyomo_models.constraints.turbine +.. automodule:: pyomo_models.baseline.first_stage.constraints.turbine :no-index: -.. automodule:: optimization.pyomo_models.constraints.pump +.. automodule:: pyomo_models.baseline.first_stage.constraints.pump :no-index: \ No newline at end of file diff --git a/docs/first_model_baseline.rst b/docs/first_model_baseline.rst index b2f1b83..987048d 100644 --- a/docs/first_model_baseline.rst +++ b/docs/first_model_baseline.rst @@ -8,13 +8,13 @@ Sets :file: /tables/sets_docs.csv :header-rows: 1 -.. automodule:: optimization.pyomo_models.sets +.. automodule:: pyomo_models.baseline.first_stage.sets :no-index: Variables ---------- -.. automodule:: optimization.pyomo_models.variables +.. automodule:: pyomo_models.baseline.first_stage.variables :no-index: .. csv-table:: @@ -24,7 +24,7 @@ Variables Parameters ---------- -.. automodule:: optimization.pyomo_models.parameters +.. automodule:: pyomo_models.baseline.first_stage.parameters :no-index: .. csv-table:: @@ -34,18 +34,18 @@ Parameters Objective ---------- -.. automodule:: optimization.pyomo_models.objective +.. automodule:: pyomo_models.baseline.first_stage.objective :no-index: Constraints ------------ -.. automodule:: optimization.pyomo_models.constraints.basin_volume +.. automodule:: pyomo_models.baseline.first_stage.constraints.basin_volume :no-index: -.. automodule:: optimization.pyomo_models.constraints.turbine +.. automodule:: pyomo_models.baseline.first_stage.constraints.turbine :no-index: -.. automodule:: optimization.pyomo_models.constraints.pump +.. automodule:: pyomo_models.baseline.first_stage.constraints.pump :no-index: \ No newline at end of file diff --git a/settings.toml b/settings.toml index 4c44bc9..f1b2cad 100644 --- a/settings.toml +++ b/settings.toml @@ -1,2 +1,3 @@ INPUT_FILE_NAMES = "data/input_file_names.json" -OUTPUT_FILE_NAMES = "data/output_file_names.json" \ No newline at end of file +OUTPUT_FILE_NAMES = "data/output_file_names.json" +LOG_LEVEL = "INFO" \ No newline at end of file diff --git a/src/data_display/first_stage_optimization_plots.py b/src/data_display/first_stage_optimization_plots.py index ec30f68..cf08c4d 100644 --- a/src/data_display/first_stage_optimization_plots.py +++ b/src/data_display/first_stage_optimization_plots.py @@ -95,11 +95,11 @@ def plot_powered_volume( go.Bar( x=results["T"].to_list(), y=(factor*results[col_name]).to_list(), showlegend=True, marker=dict(color=COLORS[c_nb]), width=1, name= col_name.replace("_", " ") + " " + var_name.replace("_", " "), - legendgroup="volume_powered" + legendgroup="volume_flowed" ), row=row, col=1 ) c_nb += 1 - fig.update_traces(selector=dict(legendgroup="volume_powered"), legendgrouptitle_text="Powered water volume") + fig.update_traces(selector=dict(legendgroup="volume_flowed"), legendgrouptitle_text="Powered water flow") return fig def plot_result_summarized( @@ -107,7 +107,7 @@ def plot_result_summarized( fig: Figure = make_subplots( rows=3, cols = 1, shared_xaxes=True, vertical_spacing=0.02, x_title="Weeks", - row_titles= ["DA price [EUR/MWh]", "Basin water volume [%]", "Powered volume [Mm3]"]) + row_titles= ["DA price [EUR/MWh]", "Basin water volume [%]", "Avg powered flow [Mm3/s]"]) kwargs: dict = { "model_instance": model_instance, diff --git a/src/optimization/first_model_baseline.py b/src/optimization/first_model_baseline.py deleted file mode 100644 index 5e1aee2..0000000 --- a/src/optimization/first_model_baseline.py +++ /dev/null @@ -1,24 +0,0 @@ -import pyomo.environ as pyo -from optimization.pyomo_models.sets import baseline_sets -from optimization.pyomo_models.parameters import baseline_parameters -from optimization.pyomo_models.variables import baseline_variables -from optimization.pyomo_models.objective import baseline_objective -from optimization.pyomo_models.constraints.basin_volume import basin_volume_constraints -from optimization.pyomo_models.constraints.turbine import turbine_constraints -from optimization.pyomo_models.constraints.pump import pump_constraints - -def generate_baseline_model() -> pyo.AbstractModel: - - model: pyo.AbstractModel = pyo.AbstractModel() - - model = baseline_sets(model) - model = baseline_parameters(model) - model = baseline_variables(model) - - model = baseline_objective(model) - model = basin_volume_constraints(model) - model = turbine_constraints(model) - model = pump_constraints(model) - - - return model diff --git a/src/optimization/optimizaztion_pipeline.py b/src/optimization/optimizaztion_pipeline.py deleted file mode 100644 index ec5ce51..0000000 --- a/src/optimization/optimizaztion_pipeline.py +++ /dev/null @@ -1,38 +0,0 @@ -import pyomo.environ as pyo -import json -from datetime import timedelta - - -from optimization_model.first_model_baseline import generate_baseline_model -from optimization_model.input_data_prepocessing import generate_first_problem_input_data -from data_display.first_stage_optimization_plots import plot_first_stage_summarized -from data_federation.input_model import SmallflexInputSchema - -from config import settings - -def first_stage_pipeline(year: int, nb_days: int, max_flow_factor: float, min_flow_factor: float, n_segments: int): - - output_file_names: dict[str, str] = json.load(open(settings.OUTPUT_FILE_NAMES)) - - small_flex_input_schema: SmallflexInputSchema = SmallflexInputSchema()\ - .duckdb_to_schema(file_path=output_file_names["duckdb_input"]) - - model: pyo.AbstractModel = generate_baseline_model() - data = generate_first_problem_input_data( - small_flex_input_schema=small_flex_input_schema, - hydro_power_plant_name = "Aegina hydro", - year = year, - n_segments = n_segments, - max_flow_factor=max_flow_factor, - min_flow_factor=min_flow_factor, - first_time_delta = timedelta(days=nb_days) - ) - - solver = pyo.SolverFactory('gurobi') - - model_instance = model.create_instance({None: data}) - res = solver.solve(model_instance, load_solutions=True, tee=False) - - plot_first_stage_summarized(model_instance=model_instance,max_flow_factor=max_flow_factor, min_flow_factor=min_flow_factor, nb_days=nb_days, year=year) - - return model_instance \ No newline at end of file diff --git a/src/pyomo_models/baseline/baseline_input.py b/src/pyomo_models/baseline/baseline_input.py new file mode 100644 index 0000000..7d1e83e --- /dev/null +++ b/src/pyomo_models/baseline/baseline_input.py @@ -0,0 +1,52 @@ +from typing import Optional +from datetime import datetime, timedelta, timezone +import polars as pl +from polars import col as c +import pyomo.environ as pyo +import tqdm + +from data_federation.input_model import SmallflexInputSchema +from pyomo_models.input_data_preprocessing import ( + generate_baseline_index, generate_clean_timeseries, generate_water_flow_factor, generate_basin_volume_table, + clean_hydro_power_performance_table, generate_hydro_power_state +) +from utility.general_function import pl_to_dict, pl_to_dict_with_tuple, generate_log + +from pyomo_models.baseline.first_stage.sets import baseline_sets +from pyomo_models.baseline.first_stage.parameters import baseline_parameters +from pyomo_models.baseline.first_stage.variables import baseline_variables +from pyomo_models.baseline.first_stage.objective import baseline_objective +from pyomo_models.baseline.first_stage.constraints.basin_volume import basin_volume_constraints +from pyomo_models.baseline.first_stage.constraints.turbine import turbine_constraints +from pyomo_models.baseline.first_stage.constraints.pump import pump_constraints + +class BaseLineInput(): + def __init__( + self, input_schema_file_name: str, real_time_step: timedelta, year: int, market_country: str = "CH", + market: str = "DA", hydro_power_mask: Optional[pl.Expr] = None, max_alpha_error: float = 1.3, + solver_name: str = 'gurobi'): + + self.real_time_step = real_time_step + self.min_datetime = datetime(year, 1, 1, tzinfo=timezone.utc) + self.max_datetime = datetime(year + 1, 1, 1, tzinfo=timezone.utc) + self.small_flex_input_schema: SmallflexInputSchema = SmallflexInputSchema()\ + .duckdb_to_schema(file_path=input_schema_file_name) + self.year = year + self.market_country = market_country + self.market = market + self.hydro_power_mask = hydro_power_mask + self.max_alpha_error = max_alpha_error + self.solver= pyo.SolverFactory(solver_name) + + self.discharge_flow_measurement: pl.DataFrame = self.small_flex_input_schema.discharge_flow_measurement\ + .filter(c("river") == "Griessee")\ + .with_columns( + (c("value") * real_time_step.total_seconds()).alias("discharge_volume"), + pl.lit(0).alias("B") + ) + + self.market_price_measurement:pl.DataFrame = self.small_flex_input_schema.market_price_measurement\ + .filter(c("country") == self.market_country)\ + .filter(c("market") == self.market) + + \ No newline at end of file diff --git a/src/optimization/pyomo_models/constraints/basin_volume.py b/src/pyomo_models/baseline/first_stage/constraints/basin_volume.py similarity index 100% rename from src/optimization/pyomo_models/constraints/basin_volume.py rename to src/pyomo_models/baseline/first_stage/constraints/basin_volume.py diff --git a/src/optimization/pyomo_models/constraints/pump.py b/src/pyomo_models/baseline/first_stage/constraints/pump.py similarity index 100% rename from src/optimization/pyomo_models/constraints/pump.py rename to src/pyomo_models/baseline/first_stage/constraints/pump.py diff --git a/src/optimization/pyomo_models/constraints/turbine.py b/src/pyomo_models/baseline/first_stage/constraints/turbine.py similarity index 100% rename from src/optimization/pyomo_models/constraints/turbine.py rename to src/pyomo_models/baseline/first_stage/constraints/turbine.py diff --git a/src/optimization/pyomo_models/objective.py b/src/pyomo_models/baseline/first_stage/objective.py similarity index 100% rename from src/optimization/pyomo_models/objective.py rename to src/pyomo_models/baseline/first_stage/objective.py diff --git a/src/optimization/pyomo_models/parameters.py b/src/pyomo_models/baseline/first_stage/parameters.py similarity index 100% rename from src/optimization/pyomo_models/parameters.py rename to src/pyomo_models/baseline/first_stage/parameters.py diff --git a/src/pyomo_models/baseline/first_stage/pipeline.py b/src/pyomo_models/baseline/first_stage/pipeline.py new file mode 100644 index 0000000..1d996ec --- /dev/null +++ b/src/pyomo_models/baseline/first_stage/pipeline.py @@ -0,0 +1,137 @@ +import re +from typing import Optional +from datetime import datetime, timedelta, timezone +import polars as pl +from polars import col as c +import pyomo.environ as pyo +import tqdm + +from data_federation.input_model import SmallflexInputSchema +from pyomo_models.input_data_preprocessing import ( + generate_baseline_index, generate_clean_timeseries, generate_water_flow_factor, generate_basin_volume_table, + clean_hydro_power_performance_table, generate_hydro_power_state +) +from utility.general_function import pl_to_dict, pl_to_dict_with_tuple, generate_log +from pyomo_models.baseline.baseline_input import BaseLineInput +from pyomo_models.baseline.first_stage.sets import baseline_sets +from pyomo_models.baseline.first_stage.parameters import baseline_parameters +from pyomo_models.baseline.first_stage.variables import baseline_variables +from pyomo_models.baseline.first_stage.objective import baseline_objective +from pyomo_models.baseline.first_stage.constraints.basin_volume import basin_volume_constraints +from pyomo_models.baseline.first_stage.constraints.turbine import turbine_constraints +from pyomo_models.baseline.first_stage.constraints.pump import pump_constraints + +log = generate_log(name=__name__) + +class BaselineFirstStage(BaseLineInput): + def __init__(self, input_instance: BaseLineInput, first_time_step: timedelta): + self.subset_mapping = { + "T": "T", "H": "H", "B": "B", "BS": ["B", "S_B"], "HS": ["H", "S_H"], "S_BH": {"H", "H", "S_B", "S_H"} + } + self.first_time_step = first_time_step + self.retrieve_input(input_instance) + self.model_instance: pyo.Model = pyo.ConcreteModel() + self.generate_index() + self.process_timeseries() + self.generate_model() + + def retrieve_input(self, input_instance): + for name, value in input_instance.__dict__.items(): + setattr(self, name, value) + + + def generate_index(self): + self.index : dict[str, pl.DataFrame]= generate_baseline_index( + small_flex_input_schema=self.small_flex_input_schema, + year=self.year, + timestep=self.first_time_step, + real_timestep=self.real_time_step, + hydro_power_mask=self.hydro_power_mask + ) + + self.water_flow_factor: pl.DataFrame = generate_water_flow_factor(index=self.index) + basin_volume_table: dict[int, Optional[pl.DataFrame]] = generate_basin_volume_table( + small_flex_input_schema=self.small_flex_input_schema, index=self.index) + + self.power_performance_table: list[dict] = clean_hydro_power_performance_table( + small_flex_input_schema=self.small_flex_input_schema, index=self.index, + basin_volume_table=basin_volume_table) + + self.index: dict[str, pl.DataFrame] = generate_hydro_power_state( + power_performance_table=self.power_performance_table, index=self.index, error_percent=1.3 + ) + + def process_timeseries(self): + ### Discharge_flow ############################################################################################## + self.discharge_volume: pl.DataFrame = generate_clean_timeseries( + data=self.discharge_flow_measurement, + col_name="discharge_volume", + min_datetime=self.min_datetime, + max_datetime=self.max_datetime, + timestep=self.first_time_step , + agg_type="sum" + ).with_columns( + pl.concat_list(["T", pl.lit(0).alias("B")]).alias("TB") + ) + ### Market price ############################################################################################### + self.market_price: pl.DataFrame = generate_clean_timeseries( + data=self.market_price_measurement, + col_name="avg", + min_datetime=self.min_datetime, + max_datetime=self.max_datetime, + timestep=self.first_time_step, + agg_type="mean" + ) + + + def generate_model(self): + self.model: pyo.AbstractModel = pyo.AbstractModel() + self.model = baseline_sets(self.model) + self.model = baseline_parameters(self.model) + self.model = baseline_variables(self.model) + + self.model = baseline_objective(self.model) + self.model = basin_volume_constraints(self.model) + self.model = turbine_constraints(self.model) + self.model = pump_constraints(self.model) + + def create_model_instance(self): + hydropower_state: pl.DataFrame = self.index["state"].drop_nulls("H") + data: dict = {} + + data["T"] = {None: self.index["datetime"]["T"].to_list()} + data["H"] = {None: self.index["hydro_power_plant"]["H"].to_list()} + data["B"] = {None: self.index["water_basin"]["B"].to_list()} + data["S_b"] = pl_to_dict(self.index["state"].group_by("B", maintain_order=True).agg("S")) + data["S_h"] = pl_to_dict(self.index["state"].drop_nulls("H").group_by("H", maintain_order=True).agg("S")) + data["S_BH"] = {None: list(map(tuple, self.index["state"].drop_nulls("H")["S_BH"].to_list()))} + + data["start_basin_volume"] = pl_to_dict(self.index["water_basin"][["B", "start_volume"]]) + data["water_pumped_factor"] = pl_to_dict_with_tuple(self.water_flow_factor["BH", "pumped_factor"]) + data["water_turbined_factor"] = pl_to_dict_with_tuple(self.water_flow_factor["BH", "turbined_factor"]) + data["min_basin_volume"] = pl_to_dict_with_tuple( + self.index["state"].select("BS", c("volume").struct.field("min"))) + data["max_basin_volume"] = pl_to_dict_with_tuple( + self.index["state"].select("BS", c("volume").struct.field("max"))) + data["max_flow_turbined"] = pl_to_dict_with_tuple( + hydropower_state.select("HS", c("flow_turbined").struct.field("min"))) + data["max_flow_pumped"] = pl_to_dict_with_tuple( + hydropower_state.select("HS", c("flow_pumped").struct.field("min"))) + data["alpha_turbined"] = pl_to_dict_with_tuple( + hydropower_state.select("HS", c("alpha_turbined").struct.field("min"))) + data["alpha_pumped"] = pl_to_dict_with_tuple( + hydropower_state.select("HS", c("alpha_pumped").struct.field("min"))) + data["discharge_volume"] = pl_to_dict_with_tuple(self.discharge_volume[["TB", "discharge_volume"]]) + data["market_price"] = pl_to_dict(self.market_price[["T", "avg"]]) + data["max_market_price"] = pl_to_dict(self.market_price[["T", "max_avg"]]) + data["min_market_price"] = pl_to_dict(self.market_price[["T", "min_avg"]]) + data["nb_hours"] = pl_to_dict(self.index["datetime"][["T", "n_index"]]) + + self.model_instance: pyo.Model = self.model.create_instance({None: data}) + + def solve_model(self): + with tqdm.tqdm(total=1, desc="Solving first stage optimization problem") as pbar: + + _ = self.solver.solve(self.model_instance) + pbar.update() + \ No newline at end of file diff --git a/src/optimization/pyomo_models/sets.py b/src/pyomo_models/baseline/first_stage/sets.py similarity index 100% rename from src/optimization/pyomo_models/sets.py rename to src/pyomo_models/baseline/first_stage/sets.py diff --git a/src/optimization/pyomo_models/variables.py b/src/pyomo_models/baseline/first_stage/variables.py similarity index 100% rename from src/optimization/pyomo_models/variables.py rename to src/pyomo_models/baseline/first_stage/variables.py diff --git a/src/pyomo_models/baseline/second_stage/constraints/basin_volume.py b/src/pyomo_models/baseline/second_stage/constraints/basin_volume.py new file mode 100644 index 0000000..42f81a5 --- /dev/null +++ b/src/pyomo_models/baseline/second_stage/constraints/basin_volume.py @@ -0,0 +1,152 @@ +""" +Water basin volume evolution +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +.. math:: + :label: basin-volume-evolution + :nowrap: + + \\begin{align} + V_\\text{BAS}^{t,~b} = + \\begin{cases} + V_\\text{BAS, START}^{b} & \\text{if } t = t_0 \\\\ + V_\\text{BAS}^{t - 1,~b} + V_\\text{DIS}^{t - 1,~b} - V_\\text{SPIL}^{t - 1,~b} + + nb_\\text{SEC} \cdot nb_\\text{HOUR}^{t-1} \cdot + \sum_{h \in H} \left( + F_\\text{TUR}^{b,~h} \cdot Q_\\text{TUR}^{t-1,~h} + + F_\\text{PUM}^{b,~h} \cdot Q_\\text{PUM}^{t-1,~h} + \\right) \quad & \\text{if } t \\neq t_0 + \end{cases} \qquad \\forall \{t\in T, b \in B \} + \\end{align} + +.. math:: + :label: basin-end-volume + :nowrap: + + \\begin{align} + V_\\text{BAS, START}^{b} = V_\\text{BAS}^{t_{end},~b} + V_\\text{DIS}^{t_{end},~b} - V_\\text{SPIL}^{t_{end},~b} + + nb_\\text{SEC} \cdot nb_\\text{HOUR}^{t_{end}} \cdot + \sum_{h \in H} \left( + F_\\text{TUR}^{b,~h} \cdot Q_\\text{TUR}^{t_{end},~h} + + F_\\text{PUM}^{b,~h} \cdot Q_\\text{PUM}^{t_{end},~h} + \\right) \qquad \\forall \{b \in B \} + \\end{align} + + +Water basin state +~~~~~~~~~~~~~~~~~~ + +.. math:: + :label: basin-max-state + :nowrap: + + \\begin{align} + V_\\text{BAS}^{t,~b} \leq V_\\text{BAS, MAX}^{b,~s_b} + V_\\text{BAS, MAX}^{b,~S_B^\\text{END}\{b\}} + \cdot \left(1 -S_\\text{BAS}^{t,~b,~s_b} \\right) + \qquad \\forall \{t\in T~\\vert~b \in B~\\vert~ s_b \in S_B\{b\} \} + + \\end{align} + +.. math:: + :label: basin-min-state + :nowrap: + + \\begin{align} + V_\\text{BAS}^{t,~b} \geq V_\\text{BAS, MIN}^{b,~s_b} \cdot S_\\text{BAS}^{t,~b,~s_b} + \qquad \\forall \{t\in T~\\vert~b \in B \} + \\end{align} +.. math:: + :label: basin-total-state + :nowrap: + + \\begin{align} + \sum_{s \in S_B\{b\}} S_\\text{BAS}^{t,~b,~s} = 1 \qquad \\forall \{t\in T~\\vert~b \in B \} + \\end{align} + +""" + + + +def basin_volume_constraints(model): + #################################################################################################################### + ### Basin volume evolution constraints ############################################################################# + #################################################################################################################### + @model.Constraint(model.T, model.B) # type: ignore + def basin_volume_evolution(model, t, b): + if t == model.T.first(): + return model.basin_volume[t, b] == model.start_basin_volume[b] + else: + return model.basin_volume[t, b] == ( + model.basin_volume[t - 1, b] + model.discharge_volume[t - 1, b] - model.spilled_volume[t - 1, b] + + sum( + model.nb_hours * model.nb_sec * model.water_factor[b, h] * model.flow[t - 1, h] + ) for h in model.H + ) + + + @model.Constraint(model.B) # type: ignore + def basin_end_volume_constraint(model, b): + t_max = model.T.last() + return model.start_basin_volume[b] == ( + model.basin_volume[t_max, b] + model.discharge_volume[t_max, b] - model.spilled_volume[t_max, b] + +sum( + model.nb_hours * model.nb_sec * model.water_factor[b, h] * model.flow[t_max, h] + for h in model.H + ) + ) + #################################################################################################################### + ### Basin volume boundary constraints used to determine the state of each basin #################################### + #################################################################################################################### + @model.Constraint(model.T, model.BS) # type: ignore + def basin_max_state_constraint(model, t, b, s_b): + return ( + model.basin_volume[t, b] <= model.max_basin_volume[b, s_b] + + model.max_basin_volume[b, model.S_b[b].last()] * + (1 - model.basin_state[t, b, s_b]) + ) + + @model.Constraint(model.T, model.BS) # type: ignore + def basin_min_state_constraint(model, t, b, s_b): + return model.basin_volume[t, b] >= model.min_basin_volume[b, s_b] * model.basin_state[t, b, s_b] + + @model.Constraint(model.T, model.B) # type: ignore + def basin_state_constraint(model, t, b): + return sum(model.basin_state[t, b, s] for s in model.S_b[b]) == 1 + + ################################################################################################################### + ## Basin volume state constraints used to determine the state of each basin ####################################### + ################################################################################################################### + + @model.Constraint(model.T, model.BS) # type: ignore + def basin_state_max_active_constraint(model, t, b, s_b): + return ( + model.basin_volume_by_state[t, b, s_b] <= + model.max_basin_volume[b, model.S_b[b].last()] * model.basin_state[t, b, s_b] + ) + + @model.Constraint(model.T, model.BS) # type: ignore + def basin_state_max_inactive_constraint(model, t, b, s_b): + return ( + model.basin_volume_by_state[t, b, s_b] >= + model.basin_volume[t, b] - + model.max_basin_volume[b, model.S_b[b].last()] * (1 - model.basin_state[t, b, s_b]) + ) + + @model.Constraint(model.T, model.BS) # type: ignore + def basin_state_min_active_constraint(model, t, b, s_b): + return ( + model.basin_volume_by_state[t, b, s_b] >= + model.min_basin_volume[b, model.S_b[b].first()] * model.basin_state[t, b, s_b] + ) + + + + @model.Constraint(model.T, model.BS) # type: ignore + def basin_state_min_inactive_constraint(model, t, b, s_b): + return ( + model.basin_volume_by_state[t, b, s_b] <= + model.basin_volume[t, b] - + model.min_basin_volume[b, model.S_b[b].first()] * (1 - model.basin_state[t, b, s_b]) + ) + + return model \ No newline at end of file diff --git a/src/pyomo_models/baseline/second_stage/constraints/pump.py b/src/pyomo_models/baseline/second_stage/constraints/pump.py new file mode 100644 index 0000000..c0c8853 --- /dev/null +++ b/src/pyomo_models/baseline/second_stage/constraints/pump.py @@ -0,0 +1,60 @@ +""" +Water pumped +~~~~~~~~~~~~~~~ + +.. math:: + :label: pumped-flow-state + :nowrap: + + \\begin{align} + Q_\\text{PUM, S}^{t,~h,~s_h} \leq Q_\\text{PUM, MAX}^{h,~s_h} \cdot S_\\text{BAS}^{t,~b,~s_b} + \qquad \\forall \{t\in T~\\vert~b,~h,~s_h ~s_h \in S_{BH} \} + \\end{align} +.. math:: + :label: pumped-flow + :nowrap: + + \\begin{align} + Q_\\text{PUM}^{t,~h} = \sum_{s \in S_H\{h\}} Q_\\text{PUM, S}^{t,~h,~s} + \qquad \\forall \{t\in T~\\vert~h \in H~\} + \\end{align} + +.. math:: + :label: pumped-power + :nowrap: + + \\begin{align} + P_\\text{PUM}^{t,~h} = \sum_{s \in S_H\{h\}} \\alpha_\\text{PUM, AVG}^{h,~s} \cdot Q_\\text{PUM, S}^{t,~h,~s} + \qquad \\forall \{t\in T~\\vert~h \in H\} + \\end{align} + +The constraint :eq:`pumped-flow-state` takes the set :math:`S\_BH` as argument, enabling the connection between the +basin set :math:`B` and the hydro powerplant set :math:`H`. +""" + + +def pump_constraints(model): + #################################################################################################################### + ### basin volume per state constraints used to determine the state of each basin ################################### + #################################################################################################################### + @model.Constraint(model.T, model.S_BH) # type: ignore + def pumped_flow_by_state_constraint(model, t, h, b, s_h, s_b): + return ( + model.pumped_flow_by_state[t, h, s_h] <= model.max_flow_pumped[h, s_h] * model.basin_state[t, b, s_b] + ) + @model.Constraint(model.T, model.H) # type: ignore + def pumped_flow_constraint(model, t, h): + return ( + model.pumped_flow[t, h] == + sum(model.pumped_flow_by_state[t, h, s_h] for s_h in model.S_h[h]) + ) + + @model.Constraint(model.T, model.H) # type: ignore + def pumped_power_constraint(model, t, h): + return ( + model.pumped_power[t, h] == + sum( + model.pumped_flow_by_state[t, h, s_h] * model.alpha_pumped[h, s_h] + for s_h in model.S_h[h]) + ) + return model diff --git a/src/pyomo_models/baseline/second_stage/constraints/turbine.py b/src/pyomo_models/baseline/second_stage/constraints/turbine.py new file mode 100644 index 0000000..f6ecd0a --- /dev/null +++ b/src/pyomo_models/baseline/second_stage/constraints/turbine.py @@ -0,0 +1,59 @@ +""" +Water turbined +~~~~~~~~~~~~~~~ + +.. math:: + :label: turbined-flow-state + :nowrap: + + \\begin{align} + Q_\\text{TUR, S}^{t,~h,~s_h} \leq Q_\\text{TUR, MAX}^{h,~s_h} \cdot S_\\text{BAS}^{t,~b,~s_b} + \qquad \\forall \{t\in T~\\vert~b,~h,~s_h ~s_h \in S_{BH} \} + \\end{align} +.. math:: + :label: turbined-flow + :nowrap: + + \\begin{align} + Q_\\text{TUR}^{t,~h} = \sum_{s \in S_H\{h\}} Q_\\text{TUR, S}^{t,~h,~s} + \qquad \\forall \{t\in T~\\vert~h \in H~\} + \\end{align} + +.. math:: + :label: turbined-power + :nowrap: + + \\begin{align} + P_\\text{TUR}^{t,~h} = \sum_{s \in S_H\{h\}} \\alpha_\\text{TUR, AVG}^{h,~s} \cdot Q_\\text{TUR, S}^{t,~h,~s} + \qquad \\forall \{t\in T~\\vert~h \in H\} + \\end{align} + +The constraint :eq:`turbined-flow-state` takes the set :math:`S\_BH` as argument, enabling the connection between the +basin set :math:`B` and the hydro powerplant set :math:`H`. +""" + +def turbine_constraints(model): + #################################################################################################################### + ### basin volume per state constraints used to determine the state of each basin ################################### + #################################################################################################################### + @model.Constraint(model.T, model.S_BH) # type: ignore + def turbined_flow_by_state_constraint(model, t, h, b, s_h, s_b): + return ( + model.turbined_flow_by_state[t, h, s_h] <= model.max_flow_turbined[h, s_h] * model.basin_state[t, b, s_b] + ) + @model.Constraint(model.T, model.H) # type: ignore + def turbined_flow_constraint(model, t, h): + return ( + model.turbined_flow[t, h] == + sum(model.turbined_flow_by_state[t, h, s_h] for s_h in model.S_h[h]) + ) + + @model.Constraint(model.T, model.H) # type: ignore + def turbined_power_constraint(model, t, h): + return ( + model.turbined_power[t, h] == + sum( + model.turbined_flow_by_state[t, h, s_h] * model.alpha_turbined[h, s_h] + for s_h in model.S_h[h]) + ) + return model diff --git a/src/pyomo_models/baseline/second_stage/objective.py b/src/pyomo_models/baseline/second_stage/objective.py new file mode 100644 index 0000000..9c5cf09 --- /dev/null +++ b/src/pyomo_models/baseline/second_stage/objective.py @@ -0,0 +1,26 @@ +""" +.. math:: + :label: first-objective + :nowrap: + + \\begin{align} + \max \sum_{t \in T} c_\\text{DA}^{t} \cdot nb_\\text{HOUR}^{t} \cdot + \sum_{h \in H} \left( P_\\text{TUR}^{t,~h} - P_\\text{PUM}^{t,~h} \\right) + \\end{align} + +""" + +import pyomo.environ as pyo +def baseline_objective(model): + + @model.Objective(sense=pyo.maximize) # type: ignore + def selling_income(model): + return sum( + model.market_price[t] * model.nb_hours[t] * + sum( + model.power[t, h] + for h in model.H + ) for t in model.T + ) + + return model \ No newline at end of file diff --git a/src/pyomo_models/baseline/second_stage/parameters.py b/src/pyomo_models/baseline/second_stage/parameters.py new file mode 100644 index 0000000..544d024 --- /dev/null +++ b/src/pyomo_models/baseline/second_stage/parameters.py @@ -0,0 +1,24 @@ +import pyomo.environ as pyo + +def baseline_parameters(model): + + model.max_market_price = pyo.Param(model.T) + model.nb_hours = pyo.Param(default=1) + model.nb_sec = pyo.Param(default=3600) # s + + model.start_basin_volume = pyo.Param(model.B, default=0) # m^3 + + model.min_basin_volume = pyo.Param(model.BS, default=0) # m^3 + model.max_basin_volume = pyo.Param(model.BS, default=0) # m^3 + model.discharge_volume = pyo.Param(model.T, model.B, default=0) # m^3 + + model.water_factor = pyo.Param(model.B, model.H, default=0) # m^3 + + + model.min_flow = pyo.Param(model.HS, default=0) #m^3/s + model.d_flow = pyo.Param(model.HS, default=0) #m^3/s + model.min_power = pyo.Param(model.HS, default=0) #MW/(m^3/s) + model.d_power = pyo.Param(model.HS, default=0) #MW/(m^3/s) + + + return model \ No newline at end of file diff --git a/src/pyomo_models/baseline/second_stage/pipeline.py b/src/pyomo_models/baseline/second_stage/pipeline.py new file mode 100644 index 0000000..1d996ec --- /dev/null +++ b/src/pyomo_models/baseline/second_stage/pipeline.py @@ -0,0 +1,137 @@ +import re +from typing import Optional +from datetime import datetime, timedelta, timezone +import polars as pl +from polars import col as c +import pyomo.environ as pyo +import tqdm + +from data_federation.input_model import SmallflexInputSchema +from pyomo_models.input_data_preprocessing import ( + generate_baseline_index, generate_clean_timeseries, generate_water_flow_factor, generate_basin_volume_table, + clean_hydro_power_performance_table, generate_hydro_power_state +) +from utility.general_function import pl_to_dict, pl_to_dict_with_tuple, generate_log +from pyomo_models.baseline.baseline_input import BaseLineInput +from pyomo_models.baseline.first_stage.sets import baseline_sets +from pyomo_models.baseline.first_stage.parameters import baseline_parameters +from pyomo_models.baseline.first_stage.variables import baseline_variables +from pyomo_models.baseline.first_stage.objective import baseline_objective +from pyomo_models.baseline.first_stage.constraints.basin_volume import basin_volume_constraints +from pyomo_models.baseline.first_stage.constraints.turbine import turbine_constraints +from pyomo_models.baseline.first_stage.constraints.pump import pump_constraints + +log = generate_log(name=__name__) + +class BaselineFirstStage(BaseLineInput): + def __init__(self, input_instance: BaseLineInput, first_time_step: timedelta): + self.subset_mapping = { + "T": "T", "H": "H", "B": "B", "BS": ["B", "S_B"], "HS": ["H", "S_H"], "S_BH": {"H", "H", "S_B", "S_H"} + } + self.first_time_step = first_time_step + self.retrieve_input(input_instance) + self.model_instance: pyo.Model = pyo.ConcreteModel() + self.generate_index() + self.process_timeseries() + self.generate_model() + + def retrieve_input(self, input_instance): + for name, value in input_instance.__dict__.items(): + setattr(self, name, value) + + + def generate_index(self): + self.index : dict[str, pl.DataFrame]= generate_baseline_index( + small_flex_input_schema=self.small_flex_input_schema, + year=self.year, + timestep=self.first_time_step, + real_timestep=self.real_time_step, + hydro_power_mask=self.hydro_power_mask + ) + + self.water_flow_factor: pl.DataFrame = generate_water_flow_factor(index=self.index) + basin_volume_table: dict[int, Optional[pl.DataFrame]] = generate_basin_volume_table( + small_flex_input_schema=self.small_flex_input_schema, index=self.index) + + self.power_performance_table: list[dict] = clean_hydro_power_performance_table( + small_flex_input_schema=self.small_flex_input_schema, index=self.index, + basin_volume_table=basin_volume_table) + + self.index: dict[str, pl.DataFrame] = generate_hydro_power_state( + power_performance_table=self.power_performance_table, index=self.index, error_percent=1.3 + ) + + def process_timeseries(self): + ### Discharge_flow ############################################################################################## + self.discharge_volume: pl.DataFrame = generate_clean_timeseries( + data=self.discharge_flow_measurement, + col_name="discharge_volume", + min_datetime=self.min_datetime, + max_datetime=self.max_datetime, + timestep=self.first_time_step , + agg_type="sum" + ).with_columns( + pl.concat_list(["T", pl.lit(0).alias("B")]).alias("TB") + ) + ### Market price ############################################################################################### + self.market_price: pl.DataFrame = generate_clean_timeseries( + data=self.market_price_measurement, + col_name="avg", + min_datetime=self.min_datetime, + max_datetime=self.max_datetime, + timestep=self.first_time_step, + agg_type="mean" + ) + + + def generate_model(self): + self.model: pyo.AbstractModel = pyo.AbstractModel() + self.model = baseline_sets(self.model) + self.model = baseline_parameters(self.model) + self.model = baseline_variables(self.model) + + self.model = baseline_objective(self.model) + self.model = basin_volume_constraints(self.model) + self.model = turbine_constraints(self.model) + self.model = pump_constraints(self.model) + + def create_model_instance(self): + hydropower_state: pl.DataFrame = self.index["state"].drop_nulls("H") + data: dict = {} + + data["T"] = {None: self.index["datetime"]["T"].to_list()} + data["H"] = {None: self.index["hydro_power_plant"]["H"].to_list()} + data["B"] = {None: self.index["water_basin"]["B"].to_list()} + data["S_b"] = pl_to_dict(self.index["state"].group_by("B", maintain_order=True).agg("S")) + data["S_h"] = pl_to_dict(self.index["state"].drop_nulls("H").group_by("H", maintain_order=True).agg("S")) + data["S_BH"] = {None: list(map(tuple, self.index["state"].drop_nulls("H")["S_BH"].to_list()))} + + data["start_basin_volume"] = pl_to_dict(self.index["water_basin"][["B", "start_volume"]]) + data["water_pumped_factor"] = pl_to_dict_with_tuple(self.water_flow_factor["BH", "pumped_factor"]) + data["water_turbined_factor"] = pl_to_dict_with_tuple(self.water_flow_factor["BH", "turbined_factor"]) + data["min_basin_volume"] = pl_to_dict_with_tuple( + self.index["state"].select("BS", c("volume").struct.field("min"))) + data["max_basin_volume"] = pl_to_dict_with_tuple( + self.index["state"].select("BS", c("volume").struct.field("max"))) + data["max_flow_turbined"] = pl_to_dict_with_tuple( + hydropower_state.select("HS", c("flow_turbined").struct.field("min"))) + data["max_flow_pumped"] = pl_to_dict_with_tuple( + hydropower_state.select("HS", c("flow_pumped").struct.field("min"))) + data["alpha_turbined"] = pl_to_dict_with_tuple( + hydropower_state.select("HS", c("alpha_turbined").struct.field("min"))) + data["alpha_pumped"] = pl_to_dict_with_tuple( + hydropower_state.select("HS", c("alpha_pumped").struct.field("min"))) + data["discharge_volume"] = pl_to_dict_with_tuple(self.discharge_volume[["TB", "discharge_volume"]]) + data["market_price"] = pl_to_dict(self.market_price[["T", "avg"]]) + data["max_market_price"] = pl_to_dict(self.market_price[["T", "max_avg"]]) + data["min_market_price"] = pl_to_dict(self.market_price[["T", "min_avg"]]) + data["nb_hours"] = pl_to_dict(self.index["datetime"][["T", "n_index"]]) + + self.model_instance: pyo.Model = self.model.create_instance({None: data}) + + def solve_model(self): + with tqdm.tqdm(total=1, desc="Solving first stage optimization problem") as pbar: + + _ = self.solver.solve(self.model_instance) + pbar.update() + \ No newline at end of file diff --git a/src/pyomo_models/baseline/second_stage/sets.py b/src/pyomo_models/baseline/second_stage/sets.py new file mode 100644 index 0000000..0ce75e2 --- /dev/null +++ b/src/pyomo_models/baseline/second_stage/sets.py @@ -0,0 +1,45 @@ +""" +The sets :math:`T`, :math:`S_H` and :math:`S_B` are arranged in a specific order such that indexing a variable with +the first or last element of a set (i.e :math:`t^{0}` and :math:`t^{\\text{END}} \in T`) corresponds respectively to +the lowest/first or highest/final element of that variable. + +:math:`S_H` and :math:`S_B` include subset to specify the corresponding hydro powerplants and basins that the +state is associated with, respectively. The sets are constructed as demonstrated in the following example. To collect +all states associated with a basin :math:`b` we can use the notation :math:`S_B\{b\}` + +:math:`S_B =\\begin{cases} 1: \left[1, 2, 3\\right] \\\\ 2: \left[4, 5\\right] \\\\ 3: \left[6\\right] \\end{cases}` + +In a Pyomo model, it is not possible to directly index variables and parameters using sets that contain subsets, +such as :math:`S\_B`. To handle this limitation, we need to create new sets (i.e. :math:`BS`) which will explicitly +represent the deployment of these subsets. These new sets will be structured to map the relationships required for +indexing variables and parameters in the model effectively. + +:math:`SB \in \{b, s_b\}=\{(1,~1),~(1,~2),~(2,~3),~(2,~4),~(2,~5),~(3,~6)\}` + + +The set :math:`S\_BH` defines the connections between each basin, its corresponding state, and the hydro powerplants. +This link is established solely between a hydro powerplant and its associated upstream basin. It is assumed that the +water level in downstream basins does not affect the behavior of turbined or pumped energy. + +:math:`S_{BH} \in \{b, h\, s_b, s_h\}=\{(1, ~1, ~1, ~1),~(1, ~1, ~2, ~2),~(1, ~1, ~3, ~3)\}` + +""" + +import pyomo.environ as pyo + +def baseline_sets(model): + model.T = pyo.Set() + model.H = pyo.Set() + model.B = pyo.Set() + # index gathering the state per basin and the hydro powerplants + model.S_B = pyo.Set(model.B) + model.S_H = pyo.Set(model.H) + model.F = pyo.Set(model.H) + # index gathering the state of every basin and hydro powerplants + model.BS = pyo.Set(dimen=2, initialize=lambda model: [(b, s_b) for b in model.B for s_b in model.S_B[b]]) + model.HS = pyo.Set(dimen=2, initialize=lambda model: [(h, s_h) for h in model.H for s_h in model.S_H[h]]) + model.HF = pyo.Set(dimen=2, initialize=lambda model: [(h, s_f) for h in model.H for s_f in model.F[h]]) + # index (gathering h, b, s_h, s_b) to make the correspondence between the state of basin and hydro powerplants + model.S_BH = pyo.Set(dimen=4) + + return model diff --git a/src/pyomo_models/baseline/second_stage/variables.py b/src/pyomo_models/baseline/second_stage/variables.py new file mode 100644 index 0000000..45c8fc6 --- /dev/null +++ b/src/pyomo_models/baseline/second_stage/variables.py @@ -0,0 +1,20 @@ +import pyomo.environ as pyo + +def baseline_variables(model): + + model.basin_volume = pyo.Var(model.T, model.B, within=pyo.NonNegativeReals) + model.spilled_volume = pyo.Var(model.T, model.B, within=pyo.NonNegativeReals) + + model.basin_state = pyo.Var(model.T, model.BS, within=pyo.Binary) + + model.flow_state = pyo.Var(model.T, model.FS, within=pyo.Binary) + + model.basin_volume_by_state = pyo.Var(model.T, model.BS, within=pyo.NonNegativeReals) # m^3 + + model.flow = pyo.Var(model.T, model.H, within=pyo.NonNegativeReals) # m^3 + model.power= pyo.Var(model.T, model.H, within=pyo.NonNegativeReals) # MWh + + model.flow_by_state = pyo.Var(model.T, model.HS, within=pyo.NonNegativeReals) # m^3 + # model.flow_by_state = pyo.Var(model.T, model.HS, within=pyo.NonNegativeReals) # m^3 + + return model \ No newline at end of file diff --git a/src/optimization/input_data_prepocessing.py b/src/pyomo_models/input_data_preprocessing.py similarity index 51% rename from src/optimization/input_data_prepocessing.py rename to src/pyomo_models/input_data_preprocessing.py index 6f05c56..69458dd 100644 --- a/src/optimization/input_data_prepocessing.py +++ b/src/pyomo_models/input_data_preprocessing.py @@ -7,13 +7,18 @@ from data_federation.input_model import SmallflexInputSchema from typing_extensions import Optional from utility.pyomo_preprocessing import ( - generate_datetime_index, generate_segments, generate_clean_timeseries, arange_float, - linear_interpolation_for_bound, arange_float, linear_interpolation_using_cols ) -from utility.general_function import pl_to_dict + generate_datetime_index, generate_segments, generate_clean_timeseries, arange_float, filter_data_with_next, + linear_interpolation_for_bound, arange_float, linear_interpolation_using_cols,generate_state_index_using_errors, + filter_by_index, get_min_avg_max_diff, define_state) +from utility.general_function import pl_to_dict, generate_log + + +log = generate_log(name=__name__) def generate_baseline_index( - small_flex_input_schema: SmallflexInputSchema, year: int, first_time_delta: timedelta, - second_time_delta: timedelta=timedelta(hours=1), hydro_power_mask: Optional[pl.Expr]= None + small_flex_input_schema: SmallflexInputSchema, year: int, + real_timestep: timedelta, timestep: Optional[timedelta] = None, + hydro_power_mask: Optional[pl.Expr]= None ): min_datetime: datetime = datetime(year, 1, 1, tzinfo=timezone.utc) @@ -21,8 +26,10 @@ def generate_baseline_index( if hydro_power_mask is None: hydro_power_mask= pl.lit(True) index: dict[str, pl.DataFrame] = {} - index["first_datetime"], index["second_datetime"] = generate_datetime_index( - min_datetime=min_datetime, max_datetime=max_datetime, first_time_delta=first_time_delta, second_time_delta=second_time_delta) + index["datetime"] = generate_datetime_index( + min_datetime=min_datetime, max_datetime=max_datetime, model_timestep=timestep, + real_timestep=real_timestep + ) index["hydro_power_plant"] = small_flex_input_schema.hydro_power_plant\ .filter(hydro_power_mask).with_row_index(name="H") index["water_basin"] = small_flex_input_schema.water_basin\ @@ -132,23 +139,23 @@ def clean_hydro_power_performance_table( }) return power_performance_table - def generate_hydro_power_state( - power_performance_table: list[dict], index: dict[str, pl.DataFrame] + power_performance_table: list[dict], index: dict[str, pl.DataFrame], error_percent: float ) -> dict[str, pl.DataFrame]: state_index: pl.DataFrame = pl.DataFrame() for data in power_performance_table: - power_performance: pl.DataFrame = data["power_performance"] - # power_plant_data = index["hydro_power_plant"] .filter(c("H") == data["H"]).to_dicts()[0] - # upstream_basin = index["water_basin"].filter(c("uuid") == power_plant_data["upstream_basin_fk"]).to_dicts()[0] - # y_cols: list[str] = power_performance.select(cs.starts_with("flow"), cs.starts_with("electrical_power")).columns + power_performance: pl.DataFrame = data["power_performance"] y_cols: list[str] = power_performance.select(cs.starts_with("alpha")).columns - state_performance_table: pl.DataFrame = generate_segments( - data=power_performance, - x_col= "volume", y_cols=y_cols, - n_segments=5) + + segments = generate_state_index_using_errors(data=power_performance, column_list=y_cols, error_percent=error_percent) + if len(segments) > 10: + log.warning(f"{len(segments)} states found in {data["H"]} hydro power plant") + state_performance_table = filter_by_index(data=power_performance, index_list=segments)\ + .with_columns( + c(col).abs().pipe(get_min_avg_max_diff).alias(col) for col in power_performance.columns + ).slice(offset=0, length=len(segments)-1) state_index = pl.concat([ state_index , @@ -157,10 +164,15 @@ def generate_hydro_power_state( pl.lit(data["B"]).alias("B")) ], how="diagonal") # Add every downstream basin - missing_basin: pl.DataFrame = index["water_basin"].filter(~c("B").is_in(state_index["B"])).select( - c("B"), - pl.concat_list(c("volume_min").fill_null(0.0), c("volume_max").fill_null(0.0)).alias("volume") - ) + missing_basin: pl.DataFrame = index["water_basin"].filter(~c("B").is_in(state_index["B"])).select( + c("B"), + pl.struct( + c("volume_min").fill_null(0.0).alias("min"), + c("volume_max").fill_null(0.0).alias("max"), + (c("volume_max").fill_null(0.0) + c("volume_min").fill_null(0.0)/2).alias("avg"), + (c("volume_max").fill_null(0.0) - c("volume_min").fill_null(0.0)).alias("diff"), + ).alias("volume") + ) state_index = pl.concat([state_index, missing_basin], how="diagonal_relaxed") index["state"] = state_index.with_row_index(name="S").with_columns( pl.concat_list("H", "B", "S", "S").alias("S_BH"), @@ -170,106 +182,55 @@ def generate_hydro_power_state( return index -# def generate_first_problem_input_data( -# small_flex_input_schema: SmallflexInputSchema, max_flow_factor: float, min_flow_factor: float, -# hydro_power_plant_name: str, year: int, first_time_delta: timedelta, add_wind_production: bool = False, -# with_pumping: bool = False, second_time_delta: timedelta = timedelta(minutes=60), market_country = "CH", -# market = "DA", n_segments = 5 -# ) -> tuple[dict, dict]: - -# min_datetime = datetime(year, 1, 1, tzinfo=timezone.utc) -# max_datetime = datetime(year + 1, 1, 1, tzinfo=timezone.utc) +def split_timestamps_per_sim(data: pl.DataFrame, divisors: int, col_name: str = "T") -> pl.DataFrame: + return( + data.with_columns( + (c(col_name)//divisors).alias("sim_nb") + ).with_columns( + c(col_name).cum_count().over("sim_nb").alias(col_name) + ) + ) -# # turbined_table_per_volume, turbined_table_per_state = process_performance_table( -# # small_flex_input_schema=small_flex_input_schema, power_plant_name=hydro_power_plant_name, state=[True, False, False]) +def generate_second_stage_state( + index: dict[str, pl.DataFrame], power_performance_table: list[dict], + start_volume_dict: dict, discharge_volume:dict, + timestep: timedelta, error_threshold: float, + ) -> pl.DataFrame: -# # pumped_table_per_volume, pumped_table_per_state = process_performance_table( -# # small_flex_input_schema=small_flex_input_schema, power_plant_name=hydro_power_plant_name, state=[False, True, True]) - -# first_datetime_index, second_datetime_index = generate_datetime_index( -# min_datetime=min_datetime, max_datetime=max_datetime, first_time_delta=first_time_delta, second_time_delta=second_time_delta) - -# market_price_measurement:pl.DataFrame = small_flex_input_schema.market_price_measurement\ -# .filter(c("country") == market_country).filter(c("market") == market) - -# discharge_flow_measurement: pl.DataFrame = small_flex_input_schema.discharge_flow_measurement\ -# .filter(c("river") == "Griessee")\ -# .with_columns( -# (c("value") * timedelta(hours=1).total_seconds()).alias("discharge_volume") -# ) - -# wind_production_measurement: pl.DataFrame = small_flex_input_schema.power_production_measurement.select( -# "avg_active_power", -# c("timestamp").dt.year().alias("year"), -# c("timestamp").dt.to_string(format="%m-%d %H:%M").alias("date_str"), -# ).sort("year").pivot(on="year", values="avg_active_power", index="date_str").sort("date_str")\ -# .with_columns( -# pl.coalesce("2021", "2024").alias("wind_data") -# ).select( -# (str(year) + "-" + c("date_str")).str.to_datetime(format="%Y-%m-%d %H:%M", time_zone="UTC").alias("timestamp"), -# -c("wind_data") * timedelta(minutes=15) / timedelta(hours=1) # form MW per 15 minutes to MWh -# ) + rated_flow_dict = pl_to_dict(index["hydro_power_plant"][["H", "rated_flow"]]) + state_index: pl.DataFrame = pl.DataFrame() + for performance_table in power_performance_table: + start_volume = start_volume_dict[performance_table["B"]] + rated_flow = rated_flow_dict[performance_table["H"]] * timestep.total_seconds() -# discharge_volume: pl.DataFrame = generate_clean_timeseries( -# data=discharge_flow_measurement, datetime_index=first_datetime_index, -# col_name="discharge_volume", min_datetime=min_datetime, -# max_datetime=max_datetime, time_delta=first_time_delta, agg_type="sum") + boundaries = ( + start_volume - rated_flow, start_volume + rated_flow + discharge_volume[performance_table["B"]] + ) -# market_price: pl.DataFrame = generate_clean_timeseries( -# data=market_price_measurement, datetime_index=first_datetime_index, -# col_name="avg", min_datetime=min_datetime, -# max_datetime=max_datetime, time_delta=first_time_delta, agg_type="mean") - -# wind_production: pl.DataFrame = generate_clean_timeseries( -# data=wind_production_measurement, datetime_index=first_datetime_index, -# col_name="wind_data", min_datetime=min_datetime, -# max_datetime=max_datetime, time_delta=first_time_delta, agg_type="sum") + data: pl.DataFrame = filter_data_with_next(data=performance_table["power_performance"], col="volume", boundaries=boundaries) + y_cols = data.select(cs.starts_with(name) for name in ["flow", "electrical"]).columns + state_name_list = set(map(lambda x : x.split("_")[-1], y_cols)) -# start_basin_volume: float = turbined_table_per_volume["volume"][-2] -# turbined_rated_flow: float = turbined_table_per_state["flow"][0][0]*3600 # m3/h -# pumped_rated_flow: float = -pumped_table_per_state["flow"][0][0]*3600 # m3/h - -# data: dict = {} + data = define_state(data=data, x_col="volume", y_cols=y_cols, error_threshold=error_threshold)\ + .with_columns( + pl.lit(performance_table["H"]).alias("H"), + pl.lit(performance_table["B"]).alias("B") + ).with_columns( + pl.struct(cs.ends_with(col_name)).name.map_fields(lambda x: "_".join(x.split("_")[:-1])).alias(col_name) + for col_name in list(state_name_list) + ).unpivot( + on=list(state_name_list), index= ["volume", "H", "B"], value_name="data", variable_name="state" + ).unnest("data").drop("state") + + state_index = pl.concat([state_index, data], how="diagonal") + state_index = state_index.with_row_index(name="F") + missing_basin: pl.DataFrame = index["water_basin"].filter(~c("B").is_in(state_index["B"])).select( + c("B"), + pl.struct( + c("volume_min").fill_null(0.0).alias("min"), + c("volume_max").fill_null(0.0).alias("max"), + ).alias("volume") + ) -# sets: dict = { -# "T": first_datetime_index["index"].to_list(), -# "H": turbined_table_per_state["index"].to_list(), -# } -# constant_params: dict = { -# "t_max": first_datetime_index.height - 1, -# "h_max": n_segments - 1, -# "max_turbined_flow": turbined_rated_flow*max_flow_factor, -# "min_turbined_flow": turbined_rated_flow*min_flow_factor, -# "max_pumped_flow": pumped_rated_flow*max_flow_factor if with_pumping else 0, -# "min_pumped_flow": pumped_rated_flow*min_flow_factor if with_pumping else 0, -# "start_basin_volume": start_basin_volume, -# } - -# set_params: dict = { -# "max_basin_volume": pl_to_dict(turbined_table_per_state.select("index", c("volume").list.get(1))), -# "min_basin_volume": pl_to_dict(turbined_table_per_state.select("index", c("volume").list.get(0))), -# "alpha_turbined": pl_to_dict(turbined_table_per_state[["index", "mean_alpha"]]), -# "alpha_pumped": pl_to_dict(pumped_table_per_state[["index", "mean_alpha"]]), -# "market_price": pl_to_dict(market_price[["index", "avg"]]), -# "max_market_price": pl_to_dict(market_price[["index", "max_avg"]]), -# "min_market_price": pl_to_dict(market_price[["index", "min_avg"]]), -# "nb_hours": pl_to_dict(first_datetime_index[["index", "n_index"]]), -# "discharge_volume": pl_to_dict(discharge_volume[["index", "discharge_volume"]]) -# } -# if add_wind_production: -# set_params["wind_energy"] = pl_to_dict(wind_production[["index", "wind_data"]]) -# data.update(dict(map(lambda set: (set[0], {None: set[1]}), sets.items()))) -# data.update(dict(map(lambda constant_param: (constant_param[0], {None: constant_param[1]}), constant_params.items()))) -# data.update(set_params) - -# input_table: dict[str, pl.DataFrame] = { -# "first_datetime_index": first_datetime_index, -# "second_datetime_index": second_datetime_index, -# "turbined_table_per_volume": turbined_table_per_volume, -# "turbined_table_per_state": turbined_table_per_state, -# "pumped_table_per_volume": pumped_table_per_volume, -# "pumped_table_per_state": pumped_table_per_state, -# } - -# return data, input_table \ No newline at end of file + return pl.concat([state_index, missing_basin], how="diagonal_relaxed") \ No newline at end of file diff --git a/src/utility/general_function.py b/src/utility/general_function.py index 1e13131..0b585ee 100644 --- a/src/utility/general_function.py +++ b/src/utility/general_function.py @@ -11,9 +11,23 @@ import owncloud import tqdm import duckdb +import logging +import coloredlogs from config import settings +def generate_log(name: str): + """ + Load configurations from .env file and set up logging. + + Returns: + log (logging.Logger): Logger object. + config (dict): Dictionary containing the loaded configurations. + """ + log = logging.getLogger(name) + coloredlogs.install(level=settings.LOG_LEVEL) + return log + def scan_switch_directory(oc: owncloud.Client, local_folder_path: str, switch_folder_path: str, download_anyway: bool) -> list[str]: file_list = [] diff --git a/src/utility/pyomo_preprocessing.py b/src/utility/pyomo_preprocessing.py index 57e6585..f5af287 100644 --- a/src/utility/pyomo_preprocessing.py +++ b/src/utility/pyomo_preprocessing.py @@ -58,14 +58,15 @@ def optimal_segments(x, y, n_segments: int): return segments def generate_clean_timeseries( - data: pl.DataFrame, datetime_index: pl.DataFrame, - col_name : str, min_datetime: datetime, max_datetime: datetime, time_delta: timedelta, + data: pl.DataFrame, col_name : str, min_datetime: datetime, max_datetime: datetime, timestep: timedelta, max_value: Optional[int] = None, min_value: Optional[int] = None, agg_type: Literal["mean", "sum", "first"] = "first", timestamp_col: str = "timestamp" ) -> pl.DataFrame: - - cleaned_data = data\ + datetime_index = generate_datetime_index( + min_datetime=min_datetime, max_datetime=max_datetime, real_timestep=timestep) + + cleaned_data: pl.DataFrame = data\ .filter(c(timestamp_col).ge(min_datetime).and_(c(timestamp_col).lt(max_datetime)))\ .sort(timestamp_col)\ .with_columns( @@ -74,7 +75,7 @@ def generate_clean_timeseries( cleaned_data = cleaned_data\ .group_by_dynamic( - index_column=timestamp_col, start_by="datapoint", every=time_delta, closed="left" + index_column=timestamp_col, start_by="datapoint", every=timestep, closed="left" ).agg( c(col_name).mean() if agg_type=="mean" else c(col_name).sum() if agg_type=="sum" else c(col_name).first(), c(col_name).max().name.prefix("max_"), @@ -103,27 +104,40 @@ def generate_segments( n_segments=n_segments )) # Get the mean of the segments index founded for each segmentation - segments = list(np.array(segments).mean(axis=0, dtype=int)) + return list(np.array(segments).mean(axis=0, dtype=int)) - return( - data.with_columns( - cs.exclude(cs.starts_with("alpha")).abs() - ).with_row_index(name= "val_index").filter(pl.col("val_index").is_in(segments)) - .drop("val_index") - .with_columns( - # Define max and min values for each segment - pl.concat_list(c(col), c(col).shift(-1)).alias(col) for col in data.columns - ).slice(offset=0, length=n_segments) +def define_state(data: pl.DataFrame, x_col: str, y_cols: Union[str, list[str]], error_threshold: float= 0.1): + nb = 1 + if isinstance(y_cols, str): + y_cols = [y_cols] + + data = data[[x_col] + y_cols].with_row_index(name="index") + while True: + row_idx: list = generate_segments(data=data, x_col="volume", y_cols=y_cols, n_segments=nb) + new_data: pl.DataFrame = data.with_columns( + pl.when(c("index").is_in(row_idx)) + .then(c(col)).otherwise(pl.lit(None)) + for col in y_cols) + + new_data = linear_interpolation_using_cols(new_data, x_col="volume", y_col=y_cols ) + + error: float = ((new_data[y_cols] - data[y_cols])/ data[y_cols])\ + .select( + pl.max_horizontal(pl.all().abs()*100).alias("max") + )["max"].mean() # type: ignore + if error < error_threshold : + break + nb += 1 + return ( + data.filter(c("index").is_in(row_idx)).drop("index") .with_columns( - c(col).list.mean().alias("avg_" + col) for col in data.columns + (c(col).diff()/c(x_col).diff()).shift(-1).alias("d_" + col) for col in y_cols ).with_columns( - # Calculate the slope of the segments - (cs.starts_with("alpha") - .list.eval(pl.element().get(1) - pl.element().get(0)).list.get(0) / - c(x_col).list.eval(pl.element().get(1) - pl.element().get(0)).list.get(0)).name.prefix("d_"), - ) + pl.struct(c(x_col).alias("min"), c(x_col).shift(-1).alias("max")).alias(x_col), + ).slice(0, -1) ) + def limit_column( col: pl.Expr, lower_bound: Optional[int | datetime | float | date] = None, upper_bound: Optional[int | datetime | float | date] = None @@ -136,23 +150,23 @@ def limit_column( def generate_datetime_index( min_datetime: datetime, max_datetime: datetime, - first_time_delta: timedelta, second_time_delta: timedelta - ) -> tuple[pl.DataFrame, pl.DataFrame]: + real_timestep: timedelta, model_timestep: Optional[timedelta] = None, + ) -> pl.DataFrame: - second_datetime_index: pl.DataFrame = pl.datetime_range( + datetime_index: pl.DataFrame = pl.datetime_range( start=min_datetime, end=max_datetime, - interval= second_time_delta, eager=True, closed="left", time_zone="UTC" + interval= real_timestep, eager=True, closed="left", time_zone="UTC" ).to_frame(name="timestamp").with_row_index(name="T") + if model_timestep: + datetime_index: pl.DataFrame = datetime_index.group_by_dynamic( + every=model_timestep, index_column="timestamp", start_by="datapoint", closed="left" + ).agg( + c("T").count().alias("n_index") + ).with_row_index(name="T") + else: + datetime_index = datetime_index.with_columns(pl.lit(1).alias("n_index")) - first_datetime_index: pl.DataFrame = second_datetime_index.group_by_dynamic( - every=first_time_delta, index_column="timestamp", start_by="datapoint", closed="left" - ).agg( - c("T").min().alias("T_min"), - c("T").max().alias("T_max"), - c("T").count().alias("n_index") - ).with_row_index(name="T") - - return first_datetime_index, second_datetime_index + return datetime_index def extract_optimization_results(model_instance: pyo.Model, var_name: str, subset_mapping: dict) -> pl.DataFrame: index_list = list(chain(*map(lambda x: subset_mapping[x.name], getattr(model_instance, var_name).index_set().subsets()))) @@ -249,7 +263,59 @@ def calculate_curviness(data: pl.DataFrame, x_col: str, y_col_list: list[str]): return data def max_curviness(df: pl.DataFrame, x_col: str, y_col_list: list[str]): - result: pl.DataFrame = calculate_curviness(df=df, x_col=x_col, y_col_list=y_col_list) + result: pl.DataFrame = calculate_curviness(data=df, x_col=x_col, y_col_list=y_col_list) return ( result.select(pl.max_horizontal(cs.starts_with("k_")).alias("max")).max().row(0)[0] - ) \ No newline at end of file + ) + +def get_nb_states(col: pl.Expr, error_percent: float) -> pl.Expr: + return ( + 100 / error_percent * (col.max() - col.min()) / (col.max() + col.min()) + ) + +def digitize_col(col: pl.Expr, data: pl.DataFrame, nb_state: int): + bin = np.linspace(data.select(col.min()).item(), data.select(col.max()).item(), nb_state + 1) + return ( + col.map_elements(lambda x: np.digitize(x, bin), return_dtype=pl.Int64) + ) +def generate_state_index_using_errors( + data: pl.DataFrame, column_list: list[str], error_percent: float + ) -> list[int]: + nb_state: int = int(np.ceil( + data.select( + pl.max_horizontal(c(col).pipe(get_nb_states, error_percent=error_percent).alias(col) + for col in column_list + )).rows()[0][0] + )) + + segments: list[int] = data.select( + c(cols).pipe(digitize_col,data=data, nb_state=nb_state).alias(cols + "_bin") for cols in column_list + ).with_row_index(name="index")\ + .select( + c("index").first().over(cols + "_bin").alias(cols).unique() for cols in column_list + ).select( + pl.mean_horizontal(pl.all()).cast(pl.UInt32).alias("state_index"), + )["state_index"].to_list() + + return segments + +def filter_by_index(data: pl.DataFrame, index_list: list[int]) -> pl.DataFrame: + return data.with_row_index(name= "val_index").filter(pl.col("val_index").is_in(index_list)).drop("val_index") + +def get_min_avg_max_diff(col: pl.Expr) -> pl.Expr: + return ( + pl.concat_list(col.min(), (col + col.shift(-1))/2, col.max(), col.diff().shift(-1)) + .list.to_struct(fields=["min", "avg", "max", "diff"]) + ) + + + +def filter_data_with_next(data: pl.DataFrame, col: str, boundaries: tuple[float, float]) -> pl.DataFrame: + lower = data.filter(c(col).lt(boundaries[0]))[col].max() + larger = data.filter(c(col).gt(boundaries[1]))[col].min() + if lower is None: + lower = data.min() + if larger is None: + larger = data.max() + + return data.filter(c(col).ge(lower).and_(c(col).le(larger))) \ No newline at end of file diff --git a/tests/base_line.ipynb b/tests/base_line.ipynb index 72c4020..87a0428 100644 --- a/tests/base_line.ipynb +++ b/tests/base_line.ipynb @@ -26,17 +26,23 @@ "from data_federation.input_model import SmallflexInputSchema\n", "from utility.pyomo_preprocessing import *\n", "from config import settings\n", - "from utility.general_function import pl_to_dict, pl_to_dict_with_tuple, build_non_existing_dirs\n", + "from utility.general_function import pl_to_dict, pl_to_dict_with_tuple, build_non_existing_dirs, generate_log\n", + "from pyomo_models.input_data_preprocessing import (\n", + " generate_baseline_index, generate_clean_timeseries, generate_water_flow_factor, generate_basin_volume_table,\n", + " clean_hydro_power_performance_table, generate_hydro_power_state, split_timestamps_per_sim, generate_second_stage_state\n", + ")\n", "from plotly.subplots import make_subplots\n", "import plotly.express as px\n", "import plotly.graph_objs as go\n", "from plotly.graph_objects import Figure\n", "\n", "from plotly.subplots import make_subplots\n", - "from optimization.first_model_baseline import generate_baseline_model\n", - "from optimization.input_data_prepocessing import *\n", + "\n", + "from pyomo_models.baseline.baseline_input import BaseLineInput\n", + "from pyomo_models.baseline.first_stage.pipeline import BaselineFirstStage\n", "COLORS = px.colors.qualitative.Plotly\n", - "\n" + "\n", + "log = generate_log(name=\"test\")" ] }, { @@ -58,160 +64,10 @@ "name": "stderr", "output_type": "stream", "text": [ - "Read and validate tables from small_flex_input_data.db file: 100%|████████████████████████████████████████████████████| 12/12 [00:01<00:00, 9.26it/s]\n" + "Read and validate tables from small_flex_input_data.db file: 100%|████████████████████████████████████████████████████| 12/12 [00:02<00:00, 5.04it/s]\n", + "Solving first stage optimization problem: 100%|██████████| 1/1 [00:00<00:00, 1.23it/s]\n" ] - } - ], - "source": [ - "output_file_names: dict[str, str] = json.load(open(settings.OUTPUT_FILE_NAMES))\n", - "\n", - "small_flex_input_schema: SmallflexInputSchema = SmallflexInputSchema()\\\n", - ".duckdb_to_schema(file_path=output_file_names[\"duckdb_input\"])" - ] - }, - { - "cell_type": "code", - "execution_count": 4, - "metadata": {}, - "outputs": [], - "source": [ - "\n", - "nb_days = 1\n", - "year = 2020\n", - "min_datetime = datetime(year, 1, 1, tzinfo=timezone.utc)\n", - "max_datetime = datetime(year + 1, 1, 1, tzinfo=timezone.utc)\n", - "market_country = \"CH\"\n", - "market = \"DA\"\n", - "\n", - "hydro_power_mask: pl.Expr = c(\"name\").is_in([\"Aegina hydro\"])\n", - "\n", - "first_time_delta=timedelta(days=nb_days)\n", - "\n", - "index : dict[str, pl.DataFrame]= generate_baseline_index(\n", - " small_flex_input_schema=small_flex_input_schema,\n", - " year=year,\n", - " first_time_delta=first_time_delta, \n", - " hydro_power_mask=hydro_power_mask)\n", - "\n", - "water_flow_factor: pl.DataFrame = generate_water_flow_factor(index=index)\n", - "basin_volume_table: dict[int, Optional[pl.DataFrame]] = generate_basin_volume_table(\n", - " small_flex_input_schema=small_flex_input_schema, index=index)\n", - "\n", - "power_performance_table: list[dict] = clean_hydro_power_performance_table(\n", - " small_flex_input_schema= small_flex_input_schema, index=index, \n", - " basin_volume_table=basin_volume_table)\n", - "\n", - "index: dict[str, pl.DataFrame] = generate_hydro_power_state(\n", - " power_performance_table=power_performance_table, index=index,\n", - " )\n", - "\n", - "\n", - "market_price_measurement:pl.DataFrame = small_flex_input_schema.market_price_measurement\\\n", - " .filter(c(\"country\") == market_country).filter(c(\"market\") == market)\n", - " \n", - "discharge_flow_measurement: pl.DataFrame = small_flex_input_schema.discharge_flow_measurement\\\n", - " .filter(c(\"river\") == \"Griessee\")\\\n", - " .with_columns(\n", - " (c(\"value\") * timedelta(hours=1).total_seconds()).alias(\"discharge_volume\"),\n", - " pl.lit(0).alias(\"B\")\n", - " )\n", - " \n", - "wind_production_measurement: pl.DataFrame = small_flex_input_schema.power_production_measurement.select(\n", - " \"avg_active_power\", \n", - " c(\"timestamp\").dt.year().alias(\"year\"),\n", - " c(\"timestamp\").dt.to_string(format=\"%m-%d %H:%M\").alias(\"date_str\"),\n", - ").sort(\"year\").pivot(on=\"year\", values=\"avg_active_power\", index=\"date_str\").sort(\"date_str\")\\\n", - ".with_columns(\n", - " pl.coalesce(\"2021\", \"2024\").alias(\"wind_data\")\n", - ").select(\n", - " (str(year) + \"-\" + c(\"date_str\")).str.to_datetime(format=\"%Y-%m-%d %H:%M\", time_zone=\"UTC\").alias(\"timestamp\"),\n", - " -c(\"wind_data\") * timedelta(minutes=15) / timedelta(hours=1) # form MW per 15 minutes to MWh\n", - ")\n", - " \n", - "discharge_volume: pl.DataFrame = generate_clean_timeseries(\n", - " data=discharge_flow_measurement, datetime_index=index[\"first_datetime\"],\n", - " col_name=\"discharge_volume\", min_datetime=min_datetime,\n", - " max_datetime=max_datetime, time_delta=first_time_delta, agg_type=\"sum\")\n", - "discharge_volume = discharge_volume.with_columns(pl.concat_list([\"T\", pl.lit(0).alias(\"B\")]).alias(\"TB\"))\n", - "\n", - "market_price: pl.DataFrame = generate_clean_timeseries(\n", - " data=market_price_measurement, datetime_index=index[\"first_datetime\"],\n", - " col_name=\"avg\", min_datetime=min_datetime, \n", - " max_datetime=max_datetime, time_delta=first_time_delta, agg_type=\"mean\")\n", - "\n", - "wind_production: pl.DataFrame = generate_clean_timeseries(\n", - " data=wind_production_measurement, datetime_index=index[\"first_datetime\"],\n", - " col_name=\"wind_data\", min_datetime=min_datetime,\n", - " max_datetime=max_datetime, time_delta=first_time_delta, agg_type=\"sum\")\n", - " " - ] - }, - { - "cell_type": "code", - "execution_count": 5, - "metadata": {}, - "outputs": [], - "source": [ - "data: dict = {}\n", - "subset_mapping = {\n", - " \"T\": \"T\", \"H\": \"H\", \"B\": \"B\", \"BS\": [\"B\", \"S_B\"], \"HS\": [\"H\", \"S_H\"], \"S_BH\": {\"H\", \"H\", \"S_B\", \"S_H\"}\n", - "}\n", - "\n", - "s_bh = list(map(tuple, index[\"state\"].drop_nulls(\"H\")[\"S_BH\"].to_list()))\n", - "\n", - "hydropower_state: pl.DataFrame = index[\"state\"].drop_nulls(\"H\")\n", - "sets: dict = {\n", - " \"T\": {None: index[\"first_datetime\"][\"T\"].to_list()},\n", - " \"H\": {None: index[\"hydro_power_plant\"][\"H\"].to_list()},\n", - " \"B\": {None: index[\"water_basin\"][\"B\"].to_list()},\n", - " \"S_b\": pl_to_dict(index[\"state\"].group_by(\"B\", maintain_order=True).agg(\"S\")),\n", - " \"S_h\": pl_to_dict(index[\"state\"].drop_nulls(\"H\").group_by(\"H\", maintain_order=True).agg(\"S\")),\n", - " \"S_BH\": {None: s_bh}\n", - "}\n", - "\n", - "set_params: dict = {\n", - " \"start_basin_volume\": pl_to_dict(index[\"water_basin\"][[\"B\", \"start_volume\"]]),\n", - " \"water_pumped_factor\": pl_to_dict_with_tuple(water_flow_factor[\"BH\", \"pumped_factor\"]),\n", - " \"water_turbined_factor\": pl_to_dict_with_tuple(water_flow_factor[\"BH\", \"turbined_factor\"]),\n", - " \"min_basin_volume\": pl_to_dict_with_tuple(index[\"state\"].select(\"BS\", c(\"volume\").list.get(0))),\n", - " \"max_basin_volume\": pl_to_dict_with_tuple(index[\"state\"].select(\"BS\", c(\"volume\").list.get(1))),\n", - "\n", - " \"max_flow_turbined\": pl_to_dict_with_tuple(hydropower_state.select(\"HS\", c(\"flow_turbined\").list.get(0))),\n", - " \"max_flow_pumped\": pl_to_dict_with_tuple(hydropower_state.select(\"HS\", c(\"flow_pumped\").list.get(0))),\n", - " \n", - " \"alpha_turbined\": pl_to_dict_with_tuple(hydropower_state.select(\"HS\", c(\"avg_alpha_turbined\"))),\n", - " \"alpha_pumped\": pl_to_dict_with_tuple(hydropower_state.select(\"HS\", c(\"avg_alpha_pumped\"))),\n", - "\n", - " \n", - " \"discharge_volume\": pl_to_dict_with_tuple(discharge_volume[[\"TB\", \"discharge_volume\"]]),\n", - " \n", - " \"market_price\": pl_to_dict(market_price[[\"T\", \"avg\"]]),\n", - " \"max_market_price\": pl_to_dict(market_price[[\"T\", \"max_avg\"]]),\n", - " \"min_market_price\": pl_to_dict(market_price[[\"T\", \"min_avg\"]]),\n", - " \"nb_hours\": pl_to_dict(index[\"first_datetime\"][[\"T\", \"n_index\"]]), \n", - "} \n" - ] - }, - { - "cell_type": "code", - "execution_count": 6, - "metadata": {}, - "outputs": [], - "source": [ - "data.update(sets)\n", - "data.update(set_params)\n", - "solver = pyo.SolverFactory('gurobi')\n", - "\n", - "model = generate_baseline_model()\n", - "model_instance = model.create_instance({None: data})\n", - "_ = solver.solve(model_instance, load_solutions=True, tee=False)\n" - ] - }, - { - "cell_type": "code", - "execution_count": 8, - "metadata": {}, - "outputs": [ + }, { "data": { "text/html": [ @@ -2885,311 +2741,311 @@ "y": [ 81.06696702621393, 81.14506083842456, - 80.05140662669352, - 78.95588753980766, - 77.95200967722563, - 76.8535056353706, - 75.75397966502517, - 74.6536280688372, - 73.55216216644219, - 72.45027479071496, - 71.34714608942028, - 70.24282802502378, - 69.13833097880125, - 68.05986993217492, - 66.97980382272183, - 65.89849638770131, - 64.81618434501225, - 63.73299471401502, - 62.64904874046275, - 61.56431178271169, - 60.4788127087983, - 59.39256883954444, - 58.30559172216467, - 57.21786980944445, - 56.12943196942023, - 55.040341711772165, - 53.95064522535862, - 52.86034828378685, - 51.770114851895286, - 50.680458780732764, - 49.590537123634896, - 48.50016512516836, - 47.40922731318736, - 47.461380307841985, - 46.37039053339537, - 45.93575918316988, - 44.84450959639519, - 43.752601818389394, - 42.66053815298677, - 41.56846294036954, - 41.618953136124524, - 41.66866966850258, - 41.71802823722863, - 41.76657272732673, - 40.67096406704776, - 39.575961635534284, - 39.62538371394053, - 39.67675727161095, - 39.73039985694652, - 39.78356323287698, - 38.69052960144954, - 37.59597751130472, - 37.643973508710225, - 37.69187135479179, - 38.992077324107875, - 39.05485375617694, - 39.112099072461746, - 39.16496799442039, - 38.072471308470966, - 38.12235527546045, - 38.170934407202296, - 39.459714407690576, - 39.26683752145308, - 38.170934407202296, - 37.07428649761105, - 35.99160943097306, - 36.03770013797272, - 36.08313265374126, - 37.014939423312306, - 35.93001642343834, - 35.9736244793031, - 36.017388422564714, - 36.06820771393524, - 36.11991036722122, - 36.167473344079944, - 37.453845750328114, - 36.368622522875036, - 36.30011042436344, - 36.343839725981304, - 36.38686464750973, - 36.429046622373754, - 37.71168927831229, - 38.993771894343666, - 39.034042805194545, - 39.07370748727993, - 39.1128409974946, - 39.15153571355519, - 38.046273581886595, - 39.32511720422802, - 40.5682744238435, - 40.60536985068464, - 39.498849072626705, - 39.082148765074884, - 39.11793935666839, - 38.00983083660559, - 38.04478425514197, - 39.32046971429553, - 39.36459739481643, - 39.41654831321589, - 38.320801086361946, - 37.22317743713862, - 37.271150340114964, - 37.367344411181136, - 38.743432901115604, - 40.045278838838506, - 40.11703323024422, - 40.184307302392526, - 39.116181012816085, - 38.061149264574404, - 38.170934407202296, - 39.54917645265609, - 40.877107548117245, - 42.17723872447489, - 43.46342663922611, - 43.5452213337099, - 43.63563602387834, - 43.73327927037442, - 43.84224456076707, - 42.80662945384315, - 41.759825095990344, - 40.69585580366304, - 40.766161019638844, - 42.037222083289066, - 43.30619310110015, - 44.57465604146969, - 43.50016146305188, - 42.538130981044745, - 42.79666156829649, - 41.910605984624375, - 41.12147772014433, - 41.50444109171983, - 43.271016227465594, - 44.98633905129404, - 45.32314820618969, - 45.07151968602132, - 44.718679830051364, - 45.220689210348645, - 47.029009264385216, - 49.11904497497111, - 49.86568786977203, - 50.6380521850519, - 50.23429488638747, - 51.12163481724934, - 52.53048737383564, - 55.62734204455199, - 58.096034015674725, - 58.76078406466792, - 58.282460627847335, - 58.86568156829208, - 59.415109585265824, - 58.705495440389846, - 60.24756963025377, - 61.754522966969894, - 63.37317828393435, - 63.98009410869909, - 64.6008931825465, - 64.94543580397536, - 65.21245620031534, - 67.14549598145935, - 69.8660788914498, - 70.86069677872302, - 70.25886153042164, - 69.49601192200417, - 69.83669516739171, - 70.24282802502381, - 71.82577935450412, - 73.46032964328346, - 74.49606282872446, - 74.30946541675218, - 73.96385295139106, - 74.8376826411933, - 76.0574264645882, - 78.12591530371044, - 80.48258220352088, - 81.69993575349756, - 82.9030342670741, - 83.10712531323195, - 83.27502161528597, - 83.3040973036484, - 84.44032012397969, - 86.98130943653771, - 88.7037439258902, - 90.22250287085748, - 90.3747931124054, - 90.69235863624974, - 90.74679778143913, - 92.83946071428647, - 95.06976644493814, - 96.22694168612645, - 95.99114736643224, - 95.73426783291332, - 95.72690628566573, - 95.8086892349827, - 97.00782128035071, - 97.89514966399801, - 97.48414945746784, - 96.95781567170364, - 96.35731990029363, - 95.72540415800897, - 95.37571388729607, - 96.30061690718962, - 97.12374277776712, - 96.74360827581167, - 96.37500366761526, - 96.07705646543934, - 95.86476072741729, - 95.40216179253235, - 96.00295759356922, - 97.76297991598462, - 98.40442191234625, - 99.14786045510097, - 99.0336659700947, - 98.65045990906074, - 98.26628965560926, - 99.17084070980397, - 100, - 99.7111289084847, - 99.05114188755388, - 99.51766667744405, - 98.94716479590792, + 80.11646482135578, + 79.0860039291322, + 78.05408808787143, + 77.0206422406787, + 75.98617446499554, + 74.95088106346985, + 73.91447335573712, + 72.87764417467217, + 71.83957366803978, + 70.80031379830555, + 69.76087494674533, + 68.72138990632679, + 67.6802998030815, + 66.63796837426875, + 65.59463233778747, + 64.55041871299804, + 63.50544874565357, + 62.459687794110295, + 61.41316472640469, + 60.365896863358614, + 59.31789575218666, + 58.26914984567422, + 57.21968801185777, + 56.169573760417514, + 55.118853280211745, + 54.06753234484778, + 53.016274919164, + 51.965594854209264, + 50.914649203319186, + 49.86325321106043, + 48.81129140528722, + 48.863444399941855, + 47.81143063170303, + 46.759353353784014, + 45.70707977321712, + 44.65414800141911, + 43.601060342224265, + 42.54796113581483, + 42.598451331569805, + 42.64816786394787, + 42.69752643267392, + 41.64159238508488, + 40.58495973101369, + 39.528933305708, + 39.578355384114246, + 39.62972894178467, + 39.68337152712024, + 39.736534903050696, + 38.682477277831055, + 37.62690119389402, + 37.67489719129953, + 37.72279503738109, + 37.78188790799884, + 37.8446643400679, + 37.90190965635271, + 36.85030004062421, + 35.79677936088258, + 34.74218479018493, + 34.79076392192677, + 35.91955416314478, + 34.86262128149449, + 33.80569417345149, + 32.74802227006803, + 31.689859610064875, + 31.735950317064532, + 31.78138283283308, + 32.907228534332965, + 31.846819941093823, + 31.890427996958593, + 31.934191940220195, + 31.985011231590732, + 32.036713884876704, + 32.084276861735425, + 33.21065950871333, + 32.17330588615416, + 32.21691394201893, + 32.260643243636785, + 32.30366816516522, + 32.34585014002924, + 33.46850303669751, + 34.590595893458605, + 34.63086680430949, + 34.67053148639488, + 34.70966499660955, + 34.74835971267014, + 33.68207358720932, + 34.80092745028047, + 35.91933674559025, + 35.95643217243139, + 34.888887400581254, + 34.92523803208192, + 34.961028623675425, + 34.0086721011158, + 34.043625519652174, + 35.15932121953545, + 35.20344890005636, + 35.25539981845582, + 34.19862859780966, + 33.139980954794105, + 33.18795385777045, + 33.284147928836624, + 34.50024665950082, + 35.67734467291802, + 35.74909906432373, + 35.81637313647204, + 34.83393324962156, + 33.817877507587646, + 33.92766265021554, + 35.145914936399066, + 36.349098107554525, + 37.52448135960647, + 38.68592135005196, + 38.76771604453576, + 38.8581307347042, + 38.955773981200274, + 39.06473927159293, + 38.068100170876804, + 37.06027181923178, + 36.035278533112255, + 36.105583749088055, + 37.251896888432576, + 38.396119981937964, + 39.53983499800179, + 38.50431642579178, + 37.93122604365379, + 38.18975663090553, + 37.3426770534412, + 36.59252479516894, + 36.97548816674445, + 38.61731537818451, + 40.20789027770725, + 40.544699432602904, + 40.33204691864232, + 40.018183068880155, + 40.520192449177436, + 42.47289377156445, + 44.43818155784465, + 45.184824452645564, + 45.957188767925444, + 45.592407475468804, + 46.479747406330674, + 47.88859996291698, + 50.860706709327616, + 53.204650756144645, + 53.869400805137836, + 54.53453191221218, + 55.117752852656935, + 55.66718086963067, + 54.99654273096248, + 56.4138689965207, + 57.79607440893113, + 59.28998180158986, + 59.896897626354615, + 60.6425301398797, + 61.02604876751635, + 61.378755566582264, + 63.18704742342057, + 65.78288240910533, + 65.79419003547969, + 65.25741298184057, + 65.66410010577253, + 66.00478335116009, + 66.41091620879216, + 67.86911961396676, + 69.44363811161477, + 70.47937129705578, + 70.35783207974579, + 70.00072005130929, + 70.87454974111151, + 72.09429356450643, + 74.10275061249732, + 76.3993857211764, + 77.61673927115307, + 78.81983778472961, + 79.89221305336714, + 80.12516755008346, + 80.21930143310819, + 81.35552425343947, + 83.83648177486614, + 85.55891626421862, + 87.07767520918591, + 87.29502364539613, + 87.67764736390275, + 87.79714470375441, + 89.82977584547042, + 92.00004978499072, + 93.15722502617905, + 94.09096743883427, + 93.89914609997764, + 93.95684274739236, + 94.10368389137162, + 95.30281593673963, + 96.19014432038693, + 95.84420230851906, + 95.38292671741713, + 94.84748914066941, + 94.28063159304706, + 93.99599951699645, + 94.92090253688998, + 95.7440284074675, + 95.42895210017434, + 95.12540568664022, + 94.89251667912663, + 94.74527913576685, + 94.3477383955442, + 94.94853419658106, + 96.64852472786511, + 97.28996672422674, + 98.03340526698146, + 98.88260474808263, + 98.56445688171095, + 98.2453448229218, + 99.14989587711649, + 99.97905516731251, + 99.7552422704595, + 99.16031344419098, + 99.62683823408115, + 99.1213945472073, 98.66033835318244, 99.32892785102378, 100, - 99.47027710675633, - 99.0198257756264, - 98.50654611594591, - 98.07045381213145, - 97.60450041141075, - 98.20764029700754, - 98.75658910457618, - 98.1442805326499, - 97.4780060533798, - 96.78381040925308, - 97.31114283112628, - 97.8653571685438, - 98.40971595831654, + 99.5353353014186, + 99.07035882804351, + 98.6221373630253, + 98.25110325387311, + 97.85020804781472, + 98.45334793341152, + 99.00229674098016, + 98.45504636371616, + 97.85383007910832, + 97.22469262964388, + 97.37117462225761, + 97.92538895967515, + 98.46974774944788, 100, - 99.2057997039954, - 98.40385122700701, - 98.87120741634703, - 98.19782562650238, - 97.53041374659605, + 99.27085789865768, + 98.53396761633158, + 99.0013238056716, + 98.39300021048923, + 97.7906465252452, 98.19030898366695, 100, - 99.57767774957324, - 98.87352840647779, - 98.12002049465627, - 98.50330141463276, - 99.03226199376188, - 99.53049543489121, - 99.93859131900203, - 99.08267693466982, - 98.22515748751086, - 98.56301743893337, - 97.76702732466873, - 96.96458809106065, - 97.30780595004872, - 97.66374883950492, - 96.82721068123946, - 95.97109422065209, - 95.10578040365102, - 94.2240002586576, - 93.32112335262487, - 93.55847634833523, - 93.77059290657905, - 92.81179861393831, - 91.86052733159704, - 90.9642554323046, - 90.05237169889875, - 90.40521416124, - 90.60499252070413, - 90.74958674168614, - 89.70769869292126, - 88.66676328935931, - 87.709885834339, - 86.7733257033738, - 87.53398691667593, - 92.49181401462336, - 95.03795338488446, - 95.62008888715862, - 95.97494056483693, - 96.21536511962583, - 96.40980866595358, - 96.62775656756075, - 96.84215947429159, - 98.16093385953438, - 97.14899928539235, - 96.12711101228159, - 96.26793506770294, - 95.23403769142804, - 94.1958678457582, - 94.32329713226572, - 94.44715255586044, - 93.39809946044812, - 93.54337496709041, - 93.70407755241278, - 92.6719295791469, - 91.6962474851816, - 92.91105032144145, - 94.206164464008, - 94.34003709625165, - 94.46203919190614, - 94.57876998410444, - 94.69214053685971, - 94.82157326509699, - 95.06755780370659, - 96.56623779211978, + 99.64273594423554, + 99.00364479580234, + 98.31519507864311, + 98.56471009563073, + 99.09367067475984, + 99.59190411588918, + 100, + 99.20914381033008, + 98.4166825578334, + 98.51029529856797, + 97.77936337896563, + 97.04198234001986, + 97.3852001990079, + 97.74114308846411, + 96.96966312486094, + 96.17860485893586, + 95.37834923659709, + 94.56162728626596, + 93.72380857489553, + 93.9611615706059, + 94.17327812884972, + 93.27954203087128, + 92.39332894319229, + 91.56211523856213, + 90.71528969981858, + 91.06813216215983, + 91.26791052162395, + 91.41250474260596, + 90.43567488850336, + 89.45979767960368, + 88.56797841924566, + 87.69647648294273, + 88.45713769624486, + 93.35493300306096, + 95.84104058219071, + 96.42317608446487, + 96.77802776214318, + 97.0184523169321, + 97.21289586325983, + 97.430843764867, + 97.64524667159785, + 98.90398926570928, + 97.95711288622955, + 97.00028280778108, + 97.14110686320242, + 96.17226768158982, + 95.19915603058227, + 95.3265853170898, + 95.45044074068451, + 94.46644583993448, + 94.61172134657677, + 94.77242393189914, + 93.80533415329553, + 92.89471025399254, + 93.0911456948355, + 94.32622804627069, + 94.46010067851435, + 94.58210277416883, + 94.69883356636714, + 94.8122041191224, + 94.94163684735969, + 95.18762138596928, + 96.62626958325113, 98.1712964948595, 98.66597916750521, 99.08955986636904, @@ -3197,12 +3053,12 @@ 99.64009064233524, 99.83243836921658, 100, - 99.83320500528863, - 98.80939412095013, - 97.7776329793727, - 97.90791442788169, - 98.03574786689954, - 98.16667018581778, + 99.76312040709539, + 98.8043677174192, + 97.83766477050405, + 97.96794621901303, + 98.09577965803088, + 98.22670197694912, 99.44143813629277, 99.57082467567174, 99.69584636793913, @@ -3214,41 +3070,41 @@ 99.88159486168747, 100, 100, - 98.93529465522329, - 97.86652469091408, - 97.9640697860862, - 98.05883777615165, - 96.98155751469632, - 95.90194471589564, - 94.82018990879021, - 93.73640856552584, - 92.65262144865414, - 91.57017958228117, - 91.65903539848118, - 90.57554273558132, - 89.49077410547028, - 88.40414637381171, - 87.31568263503479, - 86.2253828891395, - 85.13335683466435, - 85.20927977053377, - 85.28357454914726, - 84.18671598537001, - 83.08828122680248, - 81.98833955673214, - 80.88694871123188, - 79.78414910555277, - 79.8495409817244, - 78.74413747915729, - 77.6378794827112, - 77.70018825258973, - 78.41609131845215, - 79.61719958711559, - 80.81712426628448, - 82.01585380874425, - 82.0724698018344, - 82.12812737611434, - 82.18280343715489 + 99.00035284988556, + 97.99664108023863, + 98.09418617541078, + 98.18895416547622, + 97.17673209868317, + 96.16217749454479, + 95.14548088210164, + 94.12675773349955, + 93.10802881129015, + 92.09064513957945, + 92.17950095577947, + 91.1610664875419, + 90.14135605209313, + 89.11978651509685, + 88.0963809709822, + 87.0711394197492, + 86.04417155993633, + 86.12009449580574, + 86.19438927441925, + 85.16258890530428, + 84.12921234139904, + 83.09432886599097, + 82.05799621515301, + 81.02025480413619, + 81.0856466803078, + 80.04530137240297, + 79.00410157061916, + 79.0664103404977, + 79.12748933202334, + 80.26856580955543, + 81.40845869759298, + 82.5471564489214, + 82.60377244201153, + 82.65943001629147, + 82.1177452424926 ], "yaxis": "y2" }, @@ -3636,8 +3492,6 @@ "y": [ 0, 0, - 21.26943359999992, - 42.53886719999984, 0, 0, 0, @@ -3655,11 +3509,6 @@ 0, 0, 0, - 0.752318490122608, - 3.8006590062603354, - 9.247928176087642, - 13.656203562112035, - 6.789445955781057, 0, 0, 0, @@ -3679,151 +3528,155 @@ 0, 0, 0, - 16.262314599999712, - 16.262314599999712, - 16.262314599999712, - 16.262314599999712, - 16.262314599999712, - 37.057412199999845, - 57.85250979999997, - 57.85250979999997, - 57.85250979999997, - 35.281373799999976, - 35.281373799999976, - 35.281373799999976, - 35.281373799999976, - 56.07647139999998, - 56.07647139999998, - 56.07647139999998, - 33.50533539999998, - 37.8777088, - 58.6728064, - 79.467904, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 18.805427199999983, + 38.891699199999984, + 38.891699199999984, + 19.23016959999999, + 39.31644159999999, + 59.40271359999999, + 79.48898559999999, + 99.5752576, + 99.5752576, + 99.5752576, + 79.913728, 100, 100, 100, - 79.467904, 100, 100, 100, + 80.3384704, 100, 100, 100, - 77.42886399999986, - 97.96096, 100, 100, + 80.33847039999989, + 60.6769407999999, + 60.6769407999999, + 60.6769407999999, + 60.6769407999999, + 60.6769407999999, + 80.7632128, + 61.101683200000004, + 41.44015360000001, + 41.44015360000001, + 61.5264256, + 61.5264256, + 61.5264256, + 79.48898559999988, + 79.48898559999988, + 59.82745599999989, + 59.82745599999989, + 59.82745599999989, + 79.913728, 100, 100, - 77.42886399999986, - 54.857727999999724, - 54.857727999999724, - 54.857727999999724, - 54.857727999999724, - 54.857727999999724, - 75.65282559999972, - 53.08168959999957, - 31.15146879999958, - 31.15146879999958, - 51.94656639999975, - 60.185843199999866, - 60.185843199999866, - 80.9809408, - 80.9809408, - 58.40980479999986, - 58.40980479999986, - 58.40980479999986, - 79.2049024, 100, + 80.3384704, + 60.676940800000004, + 60.676940800000004, + 60.676940800000004, + 79.91372799999988, 100, 100, - 77.42886399999986, - 55.49864319999986, - 55.49864319999986, - 55.49864319999986, - 76.29374080000004, - 97.08883840000003, - 97.08883840000003, - 74.51770240000003, - 52.58748160000003, - 30.657260799999897, - 8.727039999999755, - 8.727039999999755, - 8.727039999999755, - 8.727039999999755, - 8.727039999999755, - 29.522137599999898, - 50.317235200000034, - 71.11233280000003, - 71.11233280000003, - 49.182112000000025, - 27.251891200000024, - 5.321670399999822, - 26.116767999999986, - 46.91186560000013, - 46.91186560000013, - 67.70696320000027, - 88.5020608000004, - 88.5020608000004, - 66.57184000000038, - 44.641619200000214, - 44.641619200000214, - 65.43671680000035, - 86.2318144000005, - 86.2318144000005, - 69.19603200000064, - 47.26581120000046, - 47.26581120000046, - 47.26581120000046, - 68.06090880000063, - 68.06090880000063, - 68.06090880000063, - 46.13068800000043, - 24.200467200000233, - 24.200467200000233, - 44.99556480000037, - 44.99556480000037, - 44.99556480000037, - 65.79066240000054, - 43.860441600000335, - 21.930220800000157, + 80.3384704, + 60.6769407999999, + 41.0154111999999, + 21.3538815999999, + 21.3538815999999, + 21.3538815999999, + 21.3538815999999, + 21.3538815999999, + 41.44015360000001, + 61.526425600000124, + 81.61269760000025, + 81.61269760000025, + 61.95116800000024, + 42.28963840000024, + 22.62810880000024, + 42.71438080000024, + 56.43613439999975, + 56.43613439999975, + 76.52240639999987, + 96.60867839999997, + 96.60867839999997, + 76.94714879999988, + 57.28561919999977, + 57.28561919999977, + 77.3718911999999, + 97.45816319999999, + 97.45816319999999, + 77.79663359999999, + 58.135103999999885, + 58.135103999999885, + 58.135103999999885, + 78.22137599999999, + 78.22137599999999, + 78.22137599999999, + 58.559846399999905, + 38.8983167999999, + 38.8983167999999, + 38.8983167999999, + 38.8983167999999, + 38.8983167999999, + 58.984588800000004, + 39.323059199999996, + 19.661529599999998, 0, 0, - 2.2702464000000386, - 23.06534400000018, - 43.86044160000032, - 21.930220800000143, 0, - 1.354106200000085, - 22.623539800000085, - 43.89297340000009, - 43.89297340000009, - 43.89297340000009, - 21.962752600000083, - 1.209472600000081, - 1.209472600000081, - 22.47890620000021, - 41.506560000000285, - 41.506560000000285, - 41.506560000000285, - 20.753280000000142, + 20.086272, + 39.3230592000001, + 19.661529599999998, 0, + 19.2367872, + 39.323059199999996, + 39.323059199999996, + 39.323059199999996, + 39.323059199999996, + 19.661529599999998, 0, 0, + 20.086272, + 39.323059199999996, + 39.323059199999996, + 39.323059199999996, + 19.661529599999998, 0, 0, - 20.75328, - 20.75328, 0, 0, 0, + 19.661529599999998, + 19.661529599999998, 0, - 20.237126400000157, - 41.506560000000285, - 20.753280000000142, 0, 0, 0, + 19.2367872, + 39.323059199999996, + 19.661529599999998, 0, 0, 0, @@ -3840,11 +3693,11 @@ 0, 0, 0, - 20.75328, - 20.75328, 0, 0, 0, + 19.661529599999998, + 19.661529599999998, 0, 0, 0, @@ -3866,15 +3719,14 @@ 0, 0, 0, - 20.75328, - 20.75328, - 20.75328, - 20.75328, 0, 0, 0, + 12.735305999999857, + 19.661529599999998, + 19.661529599999998, + 19.661529599999998, 0, - 0.5309908083965537, 0, 0, 0, @@ -3893,67 +3745,77 @@ 0, 0, 0, - 7.450135178750974, - 2.46614798280528, 0, 0, 0, 0, - 3.6662820294517444, - 5.906117061212321, 0, 0, 0, 0, 0, 0, - 19.720972800000013, - 40.99040640000003, - 62.25984, - 62.25984, - 41.50656, - 20.75328, - 20.75328, - 20.75328, - 20.75328, - 20.75328, - 20.75328, - 20.75328, 0, 0, 0, 0, - 14.922265599999973, - 36.191699199999974, - 36.191699199999974, - 36.191699199999974, - 57.4611328, - 57.4611328, - 57.4611328, - 78.7305664, - 100, - 81.47976279999968, - 60.72648279999967, - 60.72648279999967, - 60.72648279999967, - 60.72648279999967, - 60.72648279999967, - 60.72648279999967, - 60.72648279999967, - 20.75328, 0, 0, 0, 0, + 18.812044799999995, + 38.898316799999996, + 58.984588800000004, + 58.984588800000004, + 39.323059199999996, + 19.661529599999998, + 19.661529599999998, + 19.661529599999998, + 19.661529599999998, + 19.661529599999998, + 19.661529599999998, + 19.661529599999998, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 18.812044799999995, + 18.812044799999995, + 18.812044799999995, + 38.898316799999996, + 58.984588800000004, + 58.984588800000004, + 39.323059199999996, + 39.323059199999996, + 39.323059199999996, + 39.323059199999996, + 39.323059199999996, + 39.323059199999996, + 39.323059199999996, + 19.661529599999998, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 0, + 19.661529599999998, + 19.661529599999998, + 19.661529599999998, + 19.661529599999998, + 0, 0, 0, 0, 0, 0, - 20.75328000000038, - 20.75328000000038, - 20.75328000000038, - 20.75328000000038, 0, 0, 0, @@ -3966,36 +3828,30 @@ 0, 0, 0, - 4.032368751798663, 0, 0, 0, 0, - 8.51854653548163, - 2.643638790918875, 0, 0, 0, 0, - 8.019875364846538, - 21.238722641353693, - 32.445579750700716, - 25.93963679905554, - 17.87198065625158, - 5.44478999433947, 0, 0, 0, 0, - 10.360303399999637, - 31.62973699999964, - 31.62973699999964, - 52.899170599999636, - 74.16860419999965, - 74.16860419999965, - 62.25984000000011, - 41.50656, - 20.75328, + 0, + 0, + 0, + 0, + 18.812044799999757, + 18.812044799999757, + 38.898316799999826, + 58.98458879999983, + 58.98458879999983, + 58.98458879999983, + 39.3230592, + 19.661529599999813, 0, 0, 0, @@ -4004,9 +3860,9 @@ "yaxis": "y2" }, { - "legendgroup": "volume_powered", + "legendgroup": "volume_flowed", "legendgrouptitle": { - "text": "Powered water volume" + "text": "Powered water flow" }, "marker": { "color": "#636EFA" @@ -4072,66 +3928,66 @@ ], "xaxis": "x3", "y": [ - 14.574483287037136, - 17.122380000000007, - 16.847880000000107, - 16.84788000000011, - 13.059347499999653, - 9.627360000000046, - 4.813680000000032, - 4.813680000000031, - 5.319741736111114, - 7.15964, - 2.6124000000000165, - 0, - 4.81368000000002, - 3.3604600000000295, - 4.813680000000016, - 4.81368000000002, - 4.813680000000032, - 7.220520000000036, - 4.813680000000032, - 4.813680000000033, - 4.813680000000035, - 2.406840000000019, - 7.694905254629678, - 4.923480000000016, - 2.2022747453703793, - 7.385220000000048, - 9.846960000000061, - 12.308700000000078, - 12.308700000000034, - 7.385220000000009, - 11.874793240740628, - 9.175772708333382, - 12.30870000000001, - 7.385220000000006, - 10.91139583333337, - 7.385220000000006, - 9.846960000000005, - 12.308700000000018, - 9.846960000000008, - 4.923480000000006, - 4.923480000000006, - 7.385220000000006, - 4.9234800000000005, - 0, - 3.1332322222222313, - 2.461740000000011, + 13.9488, + 16.273600000000002, + 16.273600000000002, + 16.273600000000002, + 13.9488, + 9.2992, + 6.974400000000001, + 4.6496, + 11.624, + 6.974400000000001, + 2.2756399999999997, + 0, + 4.649600000000013, + 2.0789999999999864, + 4.649600000000013, + 4.551279999999998, + 4.649600000000025, + 6.237765925925881, + 4.649600000000025, + 4.649600000000025, + 2.3248000000000126, + 2.3248000000000126, + 9.102560000000011, + 2.3248, + 2.2264799999999996, + 5.2837018055559195, + 6.974400000000001, + 11.624, + 11.624, + 6.974400000000001, + 9.2992, + 9.2992, + 11.624, + 7.776046250000016, + 10.911395833333362, + 7.255961805555529, + 9.81331222222244, + 11.624, + 9.2992, + 4.6496, + 4.6496, + 6.974400000000001, + 4.6496, + 0, + 3.1438122222222598, + 2.3248, 1.3178350694444358, - 10.079296805555563, - 14.77044000000004, - 12.308700000000004, - 14.770440000000006, + 9.531536805555538, + 13.9488, + 11.624, + 13.9488, 0, - 2.461740000000011 + 3.580070810185181 ], "yaxis": "y3" }, { - "legendgroup": "volume_powered", + "legendgroup": "volume_flowed", "legendgrouptitle": { - "text": "Powered water volume" + "text": "Powered water flow" }, "marker": { "color": "#EF553B" @@ -4197,6 +4053,7 @@ ], "xaxis": "x3", "y": [ + 1.5487613108807297e-14, 0, 0, 0, @@ -4204,51 +4061,50 @@ 0, 0, 0, - -2.6124, - -2.6124, - -1.867210416666661, - -2.6124000000000165, - -5.224800000000033, - -5.150620000000017, - -2.6124000000000165, - -5.150620000000017, - -7.688840000000017, - -2.5382200000000164, - -7.614660000000024, - -5.07644000000002, - -4.509954074074078, - -5.076440000000044, - -7.614660000000063, - -5.076440000000037, - -4.94022, - -4.804000000000034, - -2.402, - -4.804000000000033, - 0, + -2.27564, + -2.27564, + -2.27564, + -4.551280000000012, + -4.55128, + -2.27564, + -4.55128, + -6.826920000000012, + -2.27564, + -6.82692, + -4.551280000000023, + -4.551280000000012, + -4.551280000000012, + -6.82692, + -4.551280000000012, + -4.55128, + -4.55128, + -2.27564, + -4.55128, 0, - -2.4020000000000064, 0, + -2.27564, 0, 0, - -2.4020000000000103, 0, + -2.27564, 0, 0, 0, 0, - -4.804000000000005, - -2.4020000000000024, 0, - -4.54554597222226, - -4.804, + -4.55128, + -2.27564, 0, - -2.402000000000026, + -2.27564, + -4.55128, 0, + -2.27564, 0, 0, 0, 0, - -8.584329189814772, + -3.6415315207705135e-14, + -6.826919999999991, 0 ], "yaxis": "y3" @@ -4289,7 +4145,7 @@ "size": 16 }, "showarrow": false, - "text": "Powered volume [Mm3]", + "text": "Avg powered flow [Mm3/s]", "textangle": 90, "x": 0.98, "xanchor": "left", @@ -5193,9 +5049,9 @@ } }, "text/html": [ - "