Skip to content
GitLab
Explore
Sign in
Primary navigation
Search or go to…
Project
MLAir
Manage
Activity
Members
Labels
Plan
Issues
Issue boards
Milestones
Wiki
Code
Merge requests
Repository
Branches
Commits
Tags
Repository graph
Compare revisions
Snippets
Build
Pipelines
Jobs
Pipeline schedules
Artifacts
Deploy
Releases
Container registry
Model registry
Operate
Environments
Monitor
Incidents
Analyze
Value stream analytics
Contributor analytics
CI/CD analytics
Repository analytics
Model experiments
Help
Help
Support
GitLab documentation
Compare GitLab plans
GitLab community forum
Contribute to GitLab
Provide feedback
Keyboard shortcuts
?
Snippets
Groups
Projects
Show more breadcrumbs
esde
machine-learning
MLAir
Commits
d17ebf8b
Commit
d17ebf8b
authored
May 30, 2022
by
leufen1
Browse files
Options
Downloads
Patches
Plain Diff
introduced new class method to set data handler fir pos
parent
4efdbbe1
No related branches found
No related tags found
5 merge requests
!432
IOA works now also with xarray and on identical data, IOA is included in...
,
!431
Resolve "release v2.1.0"
,
!430
update recent developments
,
!428
Develop
,
!425
Resolve "BUG: on transformation for climate and fir clim mix"
Pipeline
#101591
passed
May 30, 2022
Stage: test
Stage: docs
Stage: pages
Stage: deploy
Changes
1
Pipelines
2
Show whitespace changes
Inline
Side-by-side
Showing
1 changed file
mlair/data_handler/data_handler_mixed_sampling.py
+39
-21
39 additions, 21 deletions
mlair/data_handler/data_handler_mixed_sampling.py
with
39 additions
and
21 deletions
mlair/data_handler/data_handler_mixed_sampling.py
+
39
−
21
View file @
d17ebf8b
...
@@ -316,10 +316,17 @@ class DataHandlerMixedSamplingWithClimateAndFirFilter(DataHandlerMixedSamplingWi
...
@@ -316,10 +316,17 @@ class DataHandlerMixedSamplingWithClimateAndFirFilter(DataHandlerMixedSamplingWi
@classmethod
@classmethod
def
_split_chem_and_meteo_variables
(
cls
,
**
kwargs
):
def
_split_chem_and_meteo_variables
(
cls
,
**
kwargs
):
"""
Select all used variables and split them into categories chem and other.
Chemical variables are indicated by `cls.data_handler_climate_fir.chem_vars`. To indicate used variables, this
method uses 1) parameter `variables`, 2) keys from `statistics_per_var`, 3) keys from
`cls.data_handler_climate_fir.DEFAULT_VAR_ALL_DICT`. Option 3) is also applied if 1) or 2) are given but None.
"""
if
"
variables
"
in
kwargs
:
if
"
variables
"
in
kwargs
:
variables
=
kwargs
.
get
(
"
variables
"
)
variables
=
kwargs
.
get
(
"
variables
"
)
elif
"
statistics_per_var
"
in
kwargs
:
elif
"
statistics_per_var
"
in
kwargs
:
variables
=
kwargs
.
get
(
"
statistics_per_var
"
)
variables
=
kwargs
.
get
(
"
statistics_per_var
"
)
.
keys
()
else
:
else
:
variables
=
None
variables
=
None
if
variables
is
None
:
if
variables
is
None
:
...
@@ -348,14 +355,7 @@ class DataHandlerMixedSamplingWithClimateAndFirFilter(DataHandlerMixedSamplingWi
...
@@ -348,14 +355,7 @@ class DataHandlerMixedSamplingWithClimateAndFirFilter(DataHandlerMixedSamplingWi
cls
.
prepare_build
(
sp_keys
,
chem_vars
,
cls
.
chem_indicator
)
cls
.
prepare_build
(
sp_keys
,
chem_vars
,
cls
.
chem_indicator
)
sp_chem_unfiltered
=
cls
.
data_handler_unfiltered
(
station
,
**
sp_keys
)
sp_chem_unfiltered
=
cls
.
data_handler_unfiltered
(
station
,
**
sp_keys
)
if
len
(
meteo_vars
)
>
0
:
if
len
(
meteo_vars
)
>
0
:
if
cls
.
data_handler_fir_pos
is
None
:
cls
.
set_data_handler_fir_pos
(
**
kwargs
)
if
"
extend_length_opts
"
in
kwargs
:
if
isinstance
(
kwargs
[
"
extend_length_opts
"
],
dict
)
and
cls
.
meteo_indicator
not
in
kwargs
[
"
extend_length_opts
"
].
keys
():
cls
.
data_handler_fir_pos
=
0
# use faster fir version without climate estimate
else
:
cls
.
data_handler_fir_pos
=
1
# use slower fir version with climate estimate
else
:
cls
.
data_handler_fir_pos
=
0
# use faster fir version without climate estimate
sp_keys
=
{
k
:
copy
.
deepcopy
(
kwargs
[
k
])
for
k
in
cls
.
data_handler_fir
[
cls
.
data_handler_fir_pos
].
requirements
()
if
k
in
kwargs
}
sp_keys
=
{
k
:
copy
.
deepcopy
(
kwargs
[
k
])
for
k
in
cls
.
data_handler_fir
[
cls
.
data_handler_fir_pos
].
requirements
()
if
k
in
kwargs
}
sp_keys
=
cls
.
build_update_transformation
(
sp_keys
,
dh_type
=
"
filtered_meteo
"
)
sp_keys
=
cls
.
build_update_transformation
(
sp_keys
,
dh_type
=
"
filtered_meteo
"
)
cls
.
prepare_build
(
sp_keys
,
meteo_vars
,
cls
.
meteo_indicator
)
cls
.
prepare_build
(
sp_keys
,
meteo_vars
,
cls
.
meteo_indicator
)
...
@@ -369,8 +369,36 @@ class DataHandlerMixedSamplingWithClimateAndFirFilter(DataHandlerMixedSamplingWi
...
@@ -369,8 +369,36 @@ class DataHandlerMixedSamplingWithClimateAndFirFilter(DataHandlerMixedSamplingWi
dp_args
=
{
k
:
copy
.
deepcopy
(
kwargs
[
k
])
for
k
in
cls
.
own_args
(
"
id_class
"
)
if
k
in
kwargs
}
dp_args
=
{
k
:
copy
.
deepcopy
(
kwargs
[
k
])
for
k
in
cls
.
own_args
(
"
id_class
"
)
if
k
in
kwargs
}
return
cls
(
sp_chem
,
sp_meteo
,
sp_chem_unfiltered
,
sp_meteo_unfiltered
,
chem_vars
,
meteo_vars
,
**
dp_args
)
return
cls
(
sp_chem
,
sp_meteo
,
sp_chem_unfiltered
,
sp_meteo_unfiltered
,
chem_vars
,
meteo_vars
,
**
dp_args
)
@classmethod
def
set_data_handler_fir_pos
(
cls
,
**
kwargs
):
"""
Set position of fir data handler to use either faster FIR version or slower climate FIR.
This method will set data handler indicator to 0 if either no parameter
"
extend_length_opts
"
is given or the
parameter is of type dict but has no entry for the meteo_indicator. In all other cases, indicator is set to 1.
"""
p_name
=
"
extend_length_opts
"
if
cls
.
data_handler_fir_pos
is
None
:
if
p_name
in
kwargs
:
if
isinstance
(
kwargs
[
p_name
],
dict
)
and
cls
.
meteo_indicator
not
in
kwargs
[
p_name
].
keys
():
cls
.
data_handler_fir_pos
=
0
# use faster fir version without climate estimate
else
:
cls
.
data_handler_fir_pos
=
1
# use slower fir version with climate estimate
else
:
cls
.
data_handler_fir_pos
=
0
# use faster fir version without climate estimate
@classmethod
@classmethod
def
prepare_build
(
cls
,
kwargs
,
var_list
,
var_type
):
def
prepare_build
(
cls
,
kwargs
,
var_list
,
var_type
):
"""
Prepares for build of class.
`variables` parameter is updated by `var_list`, which should only include variables of a specific type (e.g.
only chemical variables) indicated by `var_type`. Furthermore, this method cleans the `kwargs` dictionary as
follows: For all parameters provided as dict to separate between chem and meteo options (dict must have keys
from `cls.chem_indicator` and/or `cls.meteo_indicator`), this parameter is removed from kwargs and its value
related to `var_type` added again. In case there is no value for given `var_type`, the parameter is not added
at all (as this parameter is assumed to affect only other types of variables).
"""
kwargs
.
update
({
"
variables
"
:
var_list
})
kwargs
.
update
({
"
variables
"
:
var_list
})
for
k
in
list
(
kwargs
.
keys
()):
for
k
in
list
(
kwargs
.
keys
()):
v
=
kwargs
[
k
]
v
=
kwargs
[
k
]
...
@@ -382,17 +410,6 @@ class DataHandlerMixedSamplingWithClimateAndFirFilter(DataHandlerMixedSamplingWi
...
@@ -382,17 +410,6 @@ class DataHandlerMixedSamplingWithClimateAndFirFilter(DataHandlerMixedSamplingWi
except
KeyError
:
except
KeyError
:
pass
pass
@staticmethod
def
adjust_window_opts
(
key
:
str
,
parameter_name
:
str
,
kwargs
:
dict
):
try
:
if
parameter_name
in
kwargs
:
window_opt
=
kwargs
.
pop
(
parameter_name
)
if
isinstance
(
window_opt
,
dict
):
window_opt
=
window_opt
[
key
]
kwargs
[
parameter_name
]
=
window_opt
except
KeyError
:
pass
def
_create_collection
(
self
):
def
_create_collection
(
self
):
collection
=
super
().
_create_collection
()
collection
=
super
().
_create_collection
()
if
self
.
id_class_other
is
not
None
:
if
self
.
id_class_other
is
not
None
:
...
@@ -419,9 +436,10 @@ class DataHandlerMixedSamplingWithClimateAndFirFilter(DataHandlerMixedSamplingWi
...
@@ -419,9 +436,10 @@ class DataHandlerMixedSamplingWithClimateAndFirFilter(DataHandlerMixedSamplingWi
# meteo transformation
# meteo transformation
if
len
(
meteo_vars
)
>
0
:
if
len
(
meteo_vars
)
>
0
:
cls
.
set_data_handler_fir_pos
(
**
kwargs
)
kwargs_meteo
=
copy
.
deepcopy
(
kwargs
)
kwargs_meteo
=
copy
.
deepcopy
(
kwargs
)
cls
.
prepare_build
(
kwargs_meteo
,
meteo_vars
,
cls
.
meteo_indicator
)
cls
.
prepare_build
(
kwargs_meteo
,
meteo_vars
,
cls
.
meteo_indicator
)
dh_transformation
=
(
cls
.
data_handler_fir
[
cls
.
data_handler_fir_pos
or
0
],
cls
.
data_handler_unfiltered
)
dh_transformation
=
(
cls
.
data_handler_fir
[
cls
.
data_handler_fir_pos
],
cls
.
data_handler_unfiltered
)
transformation_meteo
=
super
().
transformation
(
set_stations
,
tmp_path
=
tmp_path
,
transformation_meteo
=
super
().
transformation
(
set_stations
,
tmp_path
=
tmp_path
,
dh_transformation
=
dh_transformation
,
**
kwargs_meteo
)
dh_transformation
=
dh_transformation
,
**
kwargs_meteo
)
...
...
This diff is collapsed.
Click to expand it.
Preview
0%
Loading
Try again
or
attach a new file
.
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Save comment
Cancel
Please
sign in
to comment