From ba69f14f48c4f0ae64e7d388930ab855edf70d5e Mon Sep 17 00:00:00 2001
From: leufen1 <l.leufen@fz-juelich.de>
Date: Tue, 31 May 2022 14:55:46 +0200
Subject: [PATCH] new packages in reqs, halve data loading in make prediction
---
HPC_setup/requirements_HDFML_additionals.txt | 4 +++-
HPC_setup/requirements_JUWELS_additionals.txt | 4 +++-
mlair/data_handler/default_data_handler.py | 5 +++--
mlair/run_modules/post_processing.py | 5 +++--
requirements.txt | 4 +++-
5 files changed, 15 insertions(+), 7 deletions(-)
diff --git a/HPC_setup/requirements_HDFML_additionals.txt b/HPC_setup/requirements_HDFML_additionals.txt
index dadd8403..e811df8b 100644
--- a/HPC_setup/requirements_HDFML_additionals.txt
+++ b/HPC_setup/requirements_HDFML_additionals.txt
@@ -2,9 +2,11 @@ astropy==4.1
bottleneck==1.3.2
cached-property==1.5.2
iniconfig==1.1.1
-multiprocess==0.70.12.2
+multiprocess==0.70.11
ordered-set==4.0.2
pathos==0.2.7
+pox==0.2.9
+ppft==1.6.6.3
pyshp==2.1.3
pytest-html==3.1.1
pytest-lazy-fixture==0.6.3
diff --git a/HPC_setup/requirements_JUWELS_additionals.txt b/HPC_setup/requirements_JUWELS_additionals.txt
index dadd8403..e811df8b 100644
--- a/HPC_setup/requirements_JUWELS_additionals.txt
+++ b/HPC_setup/requirements_JUWELS_additionals.txt
@@ -2,9 +2,11 @@ astropy==4.1
bottleneck==1.3.2
cached-property==1.5.2
iniconfig==1.1.1
-multiprocess==0.70.12.2
+multiprocess==0.70.11
ordered-set==4.0.2
pathos==0.2.7
+pox==0.2.9
+ppft==1.6.6.3
pyshp==2.1.3
pytest-html==3.1.1
pytest-lazy-fixture==0.6.3
diff --git a/mlair/data_handler/default_data_handler.py b/mlair/data_handler/default_data_handler.py
index 1bcc1491..300e0435 100644
--- a/mlair/data_handler/default_data_handler.py
+++ b/mlair/data_handler/default_data_handler.py
@@ -125,8 +125,9 @@ class DefaultDataHandler(AbstractDataHandler):
def get_data(self, upsampling=False, as_numpy=True):
self._load()
- X = self.get_X(upsampling, as_numpy)
- Y = self.get_Y(upsampling, as_numpy)
+ as_numpy_X, as_numpy_Y = as_numpy if isinstance(as_numpy, tuple) else (as_numpy, as_numpy)
+ X = self.get_X(upsampling, as_numpy_X)
+ Y = self.get_Y(upsampling, as_numpy_Y)
self._reset_data()
return X, Y
diff --git a/mlair/run_modules/post_processing.py b/mlair/run_modules/post_processing.py
index ed541a66..2be09830 100644
--- a/mlair/run_modules/post_processing.py
+++ b/mlair/run_modules/post_processing.py
@@ -1329,8 +1329,9 @@ def f_proc_make_prediction(data, frequency, time_dimension, forecast_indicator,
prediction_maker = MakePrediction(model, window_lead_time, ols_model)
- input_data = data.get_X()
- target_data = data.get_Y(as_numpy=False)
+ input_data, target_data = data.get_data(as_numpy=(True, False))
+ # input_data = data.get_X()
+ # target_data = data.get_Y(as_numpy=False)
observation_data = data.get_observation()
# get scaling parameters
diff --git a/requirements.txt b/requirements.txt
index 3a669e60..1ba41d28 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -9,12 +9,14 @@ keras_nightly==2.5.0.dev2021032900
locket==0.2.1
matplotlib==3.3.4
mock==4.0.3
-multiprocess==0.70.12.2
+multiprocess==0.70.11
netcdf4==1.5.8
numpy==1.19.5
pandas==1.1.5
partd==1.2.0
pathos==0.2.7
+pox==0.2.9
+ppft==1.6.6.3
psutil==5.8.0
pydot==1.4.2
pytest==6.2.2
--
GitLab