Skip to content
GitLab
Explore
Sign in
Primary navigation
Search or go to…
Project
MLAir
Manage
Activity
Members
Labels
Plan
Issues
Issue boards
Milestones
Wiki
Code
Merge requests
Repository
Branches
Commits
Tags
Repository graph
Compare revisions
Snippets
Build
Pipelines
Jobs
Pipeline schedules
Artifacts
Deploy
Releases
Container registry
Model registry
Operate
Environments
Monitor
Incidents
Analyze
Value stream analytics
Contributor analytics
CI/CD analytics
Repository analytics
Model experiments
Help
Help
Support
GitLab documentation
Compare GitLab plans
GitLab community forum
Contribute to GitLab
Provide feedback
Keyboard shortcuts
?
Snippets
Groups
Projects
Show more breadcrumbs
esde
machine-learning
MLAir
Commits
a6cd5f5c
Commit
a6cd5f5c
authored
4 years ago
by
leufen1
Browse files
Options
Downloads
Patches
Plain Diff
all filter data handlers can now return either filter portions as dimension or distinct branches
parent
e872a388
No related branches found
No related tags found
5 merge requests
!319
add all changes of dev into release v1.4.0 branch
,
!318
Resolve "release v1.4.0"
,
!317
enabled window_lead_time=1
,
!295
Resolve "data handler FIR filter"
,
!259
Draft: Resolve "WRF-Datahandler should inherit from SingleStationDatahandler"
Changes
2
Show whitespace changes
Inline
Side-by-side
Showing
2 changed files
mlair/data_handler/data_handler_mixed_sampling.py
+10
-6
10 additions, 6 deletions
mlair/data_handler/data_handler_mixed_sampling.py
mlair/data_handler/data_handler_with_filter.py
+57
-9
57 additions, 9 deletions
mlair/data_handler/data_handler_with_filter.py
with
67 additions
and
15 deletions
mlair/data_handler/data_handler_mixed_sampling.py
+
10
−
6
View file @
a6cd5f5c
...
@@ -4,6 +4,8 @@ __date__ = '2020-11-05'
...
@@ -4,6 +4,8 @@ __date__ = '2020-11-05'
from
mlair.data_handler.data_handler_single_station
import
DataHandlerSingleStation
from
mlair.data_handler.data_handler_single_station
import
DataHandlerSingleStation
from
mlair.data_handler.data_handler_with_filter
import
DataHandlerKzFilterSingleStation
,
\
from
mlair.data_handler.data_handler_with_filter
import
DataHandlerKzFilterSingleStation
,
\
DataHandlerFirFilterSingleStation
,
DataHandlerFilterSingleStation
,
DataHandlerClimateFirFilterSingleStation
DataHandlerFirFilterSingleStation
,
DataHandlerFilterSingleStation
,
DataHandlerClimateFirFilterSingleStation
from
mlair.data_handler.data_handler_with_filter
import
DataHandlerClimateFirFilter
,
DataHandlerFirFilter
,
\
DataHandlerKzFilter
from
mlair.data_handler
import
DefaultDataHandler
from
mlair.data_handler
import
DefaultDataHandler
from
mlair
import
helpers
from
mlair
import
helpers
from
mlair.helpers
import
remove_items
from
mlair.helpers
import
remove_items
...
@@ -176,11 +178,12 @@ class DataHandlerMixedSamplingWithKzFilterSingleStation(DataHandlerMixedSampling
...
@@ -176,11 +178,12 @@ class DataHandlerMixedSamplingWithKzFilterSingleStation(DataHandlerMixedSampling
return
int
(
self
.
kz_filter_length
[
0
]
*
np
.
sqrt
(
self
.
kz_filter_iter
[
0
])
*
2
)
return
int
(
self
.
kz_filter_length
[
0
]
*
np
.
sqrt
(
self
.
kz_filter_iter
[
0
])
*
2
)
def
_extract_lazy
(
self
,
lazy_data
):
def
_extract_lazy
(
self
,
lazy_data
):
_data
,
_meta
,
_input_data
,
_target_data
,
self
.
cutoff_period
,
self
.
cutoff_period_days
=
lazy_data
_data
,
_meta
,
_input_data
,
_target_data
,
self
.
cutoff_period
,
self
.
cutoff_period_days
,
\
self
.
filter_dim_order
=
lazy_data
super
(
__class__
,
self
).
_extract_lazy
((
_data
,
_meta
,
_input_data
,
_target_data
))
super
(
__class__
,
self
).
_extract_lazy
((
_data
,
_meta
,
_input_data
,
_target_data
))
class
DataHandlerMixedSamplingWithKzFilter
(
Default
DataHandler
):
class
DataHandlerMixedSamplingWithKzFilter
(
DataHandler
KzFilter
):
"""
Data handler using mixed sampling for input and target. Inputs are temporal filtered.
"""
"""
Data handler using mixed sampling for input and target. Inputs are temporal filtered.
"""
data_handler
=
DataHandlerMixedSamplingWithKzFilterSingleStation
data_handler
=
DataHandlerMixedSamplingWithKzFilterSingleStation
...
@@ -199,7 +202,7 @@ class DataHandlerMixedSamplingWithFirFilterSingleStation(DataHandlerMixedSamplin
...
@@ -199,7 +202,7 @@ class DataHandlerMixedSamplingWithFirFilterSingleStation(DataHandlerMixedSamplin
return
max
(
self
.
filter_order
)
return
max
(
self
.
filter_order
)
def
_extract_lazy
(
self
,
lazy_data
):
def
_extract_lazy
(
self
,
lazy_data
):
_data
,
_meta
,
_input_data
,
_target_data
,
self
.
fir_coeff
=
lazy_data
_data
,
_meta
,
_input_data
,
_target_data
,
self
.
fir_coeff
,
self
.
filter_dim_order
=
lazy_data
super
(
__class__
,
self
).
_extract_lazy
((
_data
,
_meta
,
_input_data
,
_target_data
))
super
(
__class__
,
self
).
_extract_lazy
((
_data
,
_meta
,
_input_data
,
_target_data
))
@staticmethod
@staticmethod
...
@@ -214,7 +217,7 @@ class DataHandlerMixedSamplingWithFirFilterSingleStation(DataHandlerMixedSamplin
...
@@ -214,7 +217,7 @@ class DataHandlerMixedSamplingWithFirFilterSingleStation(DataHandlerMixedSamplin
raise
ValueError
(
f
"
Unknown sampling rate
{
sampling
}
. Only daily and hourly resolution is supported.
"
)
raise
ValueError
(
f
"
Unknown sampling rate
{
sampling
}
. Only daily and hourly resolution is supported.
"
)
class
DataHandlerMixedSamplingWithFirFilter
(
Default
DataHandler
):
class
DataHandlerMixedSamplingWithFirFilter
(
DataHandler
FirFilter
):
"""
Data handler using mixed sampling for input and target. Inputs are temporal filtered.
"""
"""
Data handler using mixed sampling for input and target. Inputs are temporal filtered.
"""
data_handler
=
DataHandlerMixedSamplingWithFirFilterSingleStation
data_handler
=
DataHandlerMixedSamplingWithFirFilterSingleStation
...
@@ -236,7 +239,8 @@ class DataHandlerMixedSamplingWithClimateFirFilterSingleStation(DataHandlerMixed
...
@@ -236,7 +239,8 @@ class DataHandlerMixedSamplingWithClimateFirFilterSingleStation(DataHandlerMixed
super
().
__init__
(
*
args
,
**
kwargs
)
super
().
__init__
(
*
args
,
**
kwargs
)
def
_extract_lazy
(
self
,
lazy_data
):
def
_extract_lazy
(
self
,
lazy_data
):
_data
,
_meta
,
_input_data
,
_target_data
,
self
.
climate_filter_coeff
,
self
.
apriori
,
self
.
all_apriori
=
lazy_data
_data
,
_meta
,
_input_data
,
_target_data
,
self
.
climate_filter_coeff
,
self
.
apriori
,
self
.
all_apriori
,
\
self
.
filter_dim_order
=
lazy_data
DataHandlerMixedSamplingWithFilterSingleStation
.
_extract_lazy
(
self
,
(
_data
,
_meta
,
_input_data
,
_target_data
))
DataHandlerMixedSamplingWithFilterSingleStation
.
_extract_lazy
(
self
,
(
_data
,
_meta
,
_input_data
,
_target_data
))
@staticmethod
@staticmethod
...
@@ -251,7 +255,7 @@ class DataHandlerMixedSamplingWithClimateFirFilterSingleStation(DataHandlerMixed
...
@@ -251,7 +255,7 @@ class DataHandlerMixedSamplingWithClimateFirFilterSingleStation(DataHandlerMixed
raise
ValueError
(
f
"
Unknown sampling rate
{
sampling
}
. Only daily and hourly resolution is supported.
"
)
raise
ValueError
(
f
"
Unknown sampling rate
{
sampling
}
. Only daily and hourly resolution is supported.
"
)
class
DataHandlerMixedSamplingWithClimateFirFilter
(
Default
DataHandler
):
class
DataHandlerMixedSamplingWithClimateFirFilter
(
DataHandler
ClimateFirFilter
):
"""
Data handler using mixed sampling for input and target. Inputs are temporal filtered.
"""
"""
Data handler using mixed sampling for input and target. Inputs are temporal filtered.
"""
data_handler
=
DataHandlerMixedSamplingWithClimateFirFilterSingleStation
data_handler
=
DataHandlerMixedSamplingWithClimateFirFilterSingleStation
...
...
This diff is collapsed.
Click to expand it.
mlair/data_handler/data_handler_with_filter.py
+
57
−
9
View file @
a6cd5f5c
...
@@ -47,6 +47,7 @@ class DataHandlerFilterSingleStation(DataHandlerSingleStation):
...
@@ -47,6 +47,7 @@ class DataHandlerFilterSingleStation(DataHandlerSingleStation):
def
__init__
(
self
,
*
args
,
filter_dim
=
DEFAULT_FILTER_DIM
,
**
kwargs
):
def
__init__
(
self
,
*
args
,
filter_dim
=
DEFAULT_FILTER_DIM
,
**
kwargs
):
# self.original_data = None # ToDo: implement here something to store unfiltered data
# self.original_data = None # ToDo: implement here something to store unfiltered data
self
.
filter_dim
=
filter_dim
self
.
filter_dim
=
filter_dim
self
.
filter_dim_order
=
None
super
().
__init__
(
*
args
,
**
kwargs
)
super
().
__init__
(
*
args
,
**
kwargs
)
def
setup_transformation
(
self
,
transformation
:
Union
[
None
,
dict
,
Tuple
])
->
Tuple
[
Optional
[
dict
],
Optional
[
dict
]]:
def
setup_transformation
(
self
,
transformation
:
Union
[
None
,
dict
,
Tuple
])
->
Tuple
[
Optional
[
dict
],
Optional
[
dict
]]:
...
@@ -104,6 +105,38 @@ class DataHandlerFilterSingleStation(DataHandlerSingleStation):
...
@@ -104,6 +105,38 @@ class DataHandlerFilterSingleStation(DataHandlerSingleStation):
self
.
_data
,
self
.
input_data
,
self
.
target_data
=
list
(
map
(
f_prep
,
[
_data
,
_input_data
,
_target_data
]))
self
.
_data
,
self
.
input_data
,
self
.
target_data
=
list
(
map
(
f_prep
,
[
_data
,
_input_data
,
_target_data
]))
class
DataHandlerFilter
(
DefaultDataHandler
):
"""
Data handler using FIR filtered data.
"""
data_handler
=
DataHandlerFilterSingleStation
data_handler_transformation
=
DataHandlerFilterSingleStation
_requirements
=
data_handler
.
requirements
()
def
__init__
(
self
,
*
args
,
use_filter_branches
=
False
,
**
kwargs
):
self
.
use_filter_branches
=
use_filter_branches
super
().
__init__
(
*
args
,
**
kwargs
)
@classmethod
def
own_args
(
cls
,
*
args
):
"""
Return all arguments (including kwonlyargs).
"""
super_own_args
=
DefaultDataHandler
.
own_args
(
*
args
)
arg_spec
=
inspect
.
getfullargspec
(
cls
)
list_of_args
=
arg_spec
.
args
+
arg_spec
.
kwonlyargs
+
super_own_args
return
remove_items
(
list_of_args
,
[
"
self
"
]
+
list
(
args
))
def
get_X_original
(
self
):
if
self
.
use_filter_branches
is
True
:
X
=
[]
for
data
in
self
.
_collection
:
X_total
=
data
.
get_X
()
filter_dim
=
data
.
filter_dim
for
filter_name
in
data
.
filter_dim_order
:
X
.
append
(
X_total
.
sel
({
filter_dim
:
filter_name
},
drop
=
True
))
return
X
else
:
return
super
().
get_X_original
()
class
DataHandlerFirFilterSingleStation
(
DataHandlerFilterSingleStation
):
class
DataHandlerFirFilterSingleStation
(
DataHandlerFilterSingleStation
):
"""
Data handler for a single station to be used by a superior data handler. Inputs are FIR filtered.
"""
"""
Data handler for a single station to be used by a superior data handler. Inputs are FIR filtered.
"""
...
@@ -203,17 +236,18 @@ class DataHandlerFirFilterSingleStation(DataHandlerFilterSingleStation):
...
@@ -203,17 +236,18 @@ class DataHandlerFirFilterSingleStation(DataHandlerFilterSingleStation):
band_num
+=
1
band_num
+=
1
if
self
.
_add_unfiltered
:
if
self
.
_add_unfiltered
:
index
.
append
(
"
unfiltered
"
)
index
.
append
(
"
unfiltered
"
)
self
.
filter_dim_order
=
index
return
pd
.
Index
(
index
,
name
=
self
.
filter_dim
)
return
pd
.
Index
(
index
,
name
=
self
.
filter_dim
)
def
_create_lazy_data
(
self
):
def
_create_lazy_data
(
self
):
return
[
self
.
_data
,
self
.
meta
,
self
.
input_data
,
self
.
target_data
,
self
.
fir_coeff
]
return
[
self
.
_data
,
self
.
meta
,
self
.
input_data
,
self
.
target_data
,
self
.
fir_coeff
,
self
.
filter_dim_order
]
def
_extract_lazy
(
self
,
lazy_data
):
def
_extract_lazy
(
self
,
lazy_data
):
_data
,
_meta
,
_input_data
,
_target_data
,
self
.
fir_coeff
=
lazy_data
_data
,
_meta
,
_input_data
,
_target_data
,
self
.
fir_coeff
,
self
.
filter_dim_order
=
lazy_data
super
(
__class__
,
self
).
_extract_lazy
((
_data
,
_meta
,
_input_data
,
_target_data
))
super
(
__class__
,
self
).
_extract_lazy
((
_data
,
_meta
,
_input_data
,
_target_data
))
class
DataHandlerFirFilter
(
Default
DataHandler
):
class
DataHandlerFirFilter
(
DataHandler
Filter
):
"""
Data handler using FIR filtered data.
"""
"""
Data handler using FIR filtered data.
"""
data_handler
=
DataHandlerFirFilterSingleStation
data_handler
=
DataHandlerFirFilterSingleStation
...
@@ -262,17 +296,20 @@ class DataHandlerKzFilterSingleStation(DataHandlerFilterSingleStation):
...
@@ -262,17 +296,20 @@ class DataHandlerKzFilterSingleStation(DataHandlerFilterSingleStation):
f
=
lambda
x
:
int
(
np
.
round
(
x
))
if
x
>=
10
else
np
.
round
(
x
,
1
)
f
=
lambda
x
:
int
(
np
.
round
(
x
))
if
x
>=
10
else
np
.
round
(
x
,
1
)
index
=
list
(
map
(
f
,
index
.
tolist
()))
index
=
list
(
map
(
f
,
index
.
tolist
()))
index
=
list
(
map
(
lambda
x
:
str
(
x
)
+
"
d
"
,
index
))
+
[
"
res
"
]
index
=
list
(
map
(
lambda
x
:
str
(
x
)
+
"
d
"
,
index
))
+
[
"
res
"
]
self
.
filter_dim_order
=
index
return
pd
.
Index
(
index
,
name
=
self
.
filter_dim
)
return
pd
.
Index
(
index
,
name
=
self
.
filter_dim
)
def
_create_lazy_data
(
self
):
def
_create_lazy_data
(
self
):
return
[
self
.
_data
,
self
.
meta
,
self
.
input_data
,
self
.
target_data
,
self
.
cutoff_period
,
self
.
cutoff_period_days
]
return
[
self
.
_data
,
self
.
meta
,
self
.
input_data
,
self
.
target_data
,
self
.
cutoff_period
,
self
.
cutoff_period_days
,
self
.
filter_dim_order
]
def
_extract_lazy
(
self
,
lazy_data
):
def
_extract_lazy
(
self
,
lazy_data
):
_data
,
_meta
,
_input_data
,
_target_data
,
self
.
cutoff_period
,
self
.
cutoff_period_days
=
lazy_data
_data
,
_meta
,
_input_data
,
_target_data
,
self
.
cutoff_period
,
self
.
cutoff_period_days
,
\
self
.
filter_dim_order
=
lazy_data
super
(
__class__
,
self
).
_extract_lazy
((
_data
,
_meta
,
_input_data
,
_target_data
))
super
(
__class__
,
self
).
_extract_lazy
((
_data
,
_meta
,
_input_data
,
_target_data
))
class
DataHandlerKzFilter
(
Default
DataHandler
):
class
DataHandlerKzFilter
(
DataHandler
Filter
):
"""
Data handler using kz filtered data.
"""
"""
Data handler using kz filtered data.
"""
data_handler
=
DataHandlerKzFilterSingleStation
data_handler
=
DataHandlerKzFilterSingleStation
...
@@ -372,14 +409,16 @@ class DataHandlerClimateFirFilterSingleStation(DataHandlerFirFilterSingleStation
...
@@ -372,14 +409,16 @@ class DataHandlerClimateFirFilterSingleStation(DataHandlerFirFilterSingleStation
index
=
list
(
map
(
lambda
x
:
str
(
x
)
+
"
d
"
,
index
))
+
[
"
res
"
]
index
=
list
(
map
(
lambda
x
:
str
(
x
)
+
"
d
"
,
index
))
+
[
"
res
"
]
if
self
.
_add_unfiltered
:
if
self
.
_add_unfiltered
:
index
.
append
(
"
unfiltered
"
)
index
.
append
(
"
unfiltered
"
)
self
.
filter_dim_order
=
index
return
pd
.
Index
(
index
,
name
=
self
.
filter_dim
)
return
pd
.
Index
(
index
,
name
=
self
.
filter_dim
)
def
_create_lazy_data
(
self
):
def
_create_lazy_data
(
self
):
return
[
self
.
_data
,
self
.
meta
,
self
.
input_data
,
self
.
target_data
,
self
.
climate_filter_coeff
,
return
[
self
.
_data
,
self
.
meta
,
self
.
input_data
,
self
.
target_data
,
self
.
climate_filter_coeff
,
self
.
apriori
,
self
.
all_apriori
]
self
.
apriori
,
self
.
all_apriori
,
self
.
filter_dim_order
]
def
_extract_lazy
(
self
,
lazy_data
):
def
_extract_lazy
(
self
,
lazy_data
):
_data
,
_meta
,
_input_data
,
_target_data
,
self
.
climate_filter_coeff
,
self
.
apriori
,
self
.
all_apriori
=
lazy_data
_data
,
_meta
,
_input_data
,
_target_data
,
self
.
climate_filter_coeff
,
self
.
apriori
,
self
.
all_apriori
,
\
self
.
filter_dim_order
=
lazy_data
DataHandlerSingleStation
.
_extract_lazy
(
self
,
(
_data
,
_meta
,
_input_data
,
_target_data
))
DataHandlerSingleStation
.
_extract_lazy
(
self
,
(
_data
,
_meta
,
_input_data
,
_target_data
))
@staticmethod
@staticmethod
...
@@ -440,10 +479,19 @@ class DataHandlerClimateFirFilterSingleStation(DataHandlerFirFilterSingleStation
...
@@ -440,10 +479,19 @@ class DataHandlerClimateFirFilterSingleStation(DataHandlerFirFilterSingleStation
self
.
_transformation
=
(
opts_input
,
opts_target
)
self
.
_transformation
=
(
opts_input
,
opts_target
)
class
DataHandlerClimateFirFilter
(
Default
DataHandler
):
class
DataHandlerClimateFirFilter
(
DataHandler
Filter
):
"""
Data handler using climatic adjusted FIR filtered data.
"""
"""
Data handler using climatic adjusted FIR filtered data.
"""
data_handler
=
DataHandlerClimateFirFilterSingleStation
data_handler
=
DataHandlerClimateFirFilterSingleStation
data_handler_transformation
=
DataHandlerClimateFirFilterSingleStation
data_handler_transformation
=
DataHandlerClimateFirFilterSingleStation
_requirements
=
data_handler
.
requirements
()
_requirements
=
data_handler
.
requirements
()
_store_attributes
=
data_handler
.
store_attributes
()
_store_attributes
=
data_handler
.
store_attributes
()
# def get_X_original(self):
# X = []
# for data in self._collection:
# X_total = data.get_X()
# filter_dim = data.filter_dim
# for filter in data.filter_dim_order:
# X.append(X_total.sel({filter_dim: filter}, drop=True))
# return X
This diff is collapsed.
Click to expand it.
Preview
0%
Loading
Try again
or
attach a new file
.
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Save comment
Cancel
Please
register
or
sign in
to comment