Skip to content
Snippets Groups Projects
Select Git revision
  • ad302db1fb194e46731af79dca057e881c73e965
  • master default protected
  • enxhi_issue460_remove_TOAR-I_access
  • michael_issue459_preprocess_german_stations
  • sh_pollutants
  • develop protected
  • release_v2.4.0
  • michael_issue450_feat_load-ifs-data
  • lukas_issue457_feat_set-config-paths-as-parameter
  • lukas_issue454_feat_use-toar-statistics-api-v2
  • lukas_issue453_refac_advanced-retry-strategy
  • lukas_issue452_bug_update-proj-version
  • lukas_issue449_refac_load-era5-data-from-toar-db
  • lukas_issue451_feat_robust-apriori-estimate-for-short-timeseries
  • lukas_issue448_feat_load-model-from-path
  • lukas_issue447_feat_store-and-load-local-clim-apriori-data
  • lukas_issue445_feat_data-insight-plot-monthly-distribution
  • lukas_issue442_feat_bias-free-evaluation
  • lukas_issue444_feat_choose-interp-method-cams
  • 414-include-crps-analysis-and-other-ens-verif-methods-or-plots
  • lukas_issue384_feat_aqw-data-handler
  • v2.4.0 protected
  • v2.3.0 protected
  • v2.2.0 protected
  • v2.1.0 protected
  • Kleinert_etal_2022_initial_submission
  • v2.0.0 protected
  • v1.5.0 protected
  • v1.4.0 protected
  • v1.3.0 protected
  • v1.2.1 protected
  • v1.2.0 protected
  • v1.1.0 protected
  • IntelliO3-ts-v1.0_R1-submit
  • v1.0.0 protected
  • v0.12.2 protected
  • v0.12.1 protected
  • v0.12.0 protected
  • v0.11.0 protected
  • v0.10.0 protected
  • IntelliO3-ts-v1.0_initial-submit
41 results

experiment_setup.py

Blame
  • DataPreprocess_dev.sh 3.21 KiB
    #!/bin/bash -x
    #SBATCH --account=deepacf
    #SBATCH --nodes=1
    #SBATCH --ntasks=12
    ##SBATCH --ntasks-per-node=12
    #SBATCH --cpus-per-task=1
    #SBATCH --output=DataPreprocess-out.%j
    #SBATCH --error=DataPreprocess-err.%j
    #SBATCH --time=00:20:00
    #SBATCH --partition=devel
    #SBATCH --mail-type=ALL
    #SBATCH --mail-user=m.langguth@fz-juelich.de
    
    module --force purge
    module use  $OTHERSTAGES
    module load Stages/2019a
    module load Intel/2019.3.199-GCC-8.3.0  ParaStationMPI/5.2.2-1
    module load h5py/2.9.0-Python-3.6.8
    module load mpi4py/3.0.1-Python-3.6.8
    
    source_dir=/p/scratch/deepacf/video_prediction_shared_folder/extractedData
    destination_dir=/p/scratch/deepacf/video_prediction_shared_folder/preprocessedData/era5-Y2015toY2017M01to12-128x160-74d00N71d00E-T_MSL_gph500/hickle
    declare -a years=("2015" 
                     "2016" 
                     "2017"
                      )
    
    
    
    for year in "${years[@]}"; 
        do 
            echo "Year $year"
    	echo "source_dir ${source_dir}/${year}"
    	srun python ../../workflow_parallel_frame_prediction/DataPreprocess/mpi_stager_v2_process_netCDF.py \
             --source_dir ${source_dir}/${year}/ \
             --destination_dir ${destination_dir}/${year}/ --vars T2 MSL gph500 --lat_s 74 --lat_e 202 --lon_s 550 --lon_e 710
        done
    
    
    srun python ../../workflow_parallel_frame_prediction/DataPreprocess/mpi_split_data_multi_years.py --destination_dir ${destination_dir}
    
    
    
    
    #srun python ../../workflow_parallel_frame_prediction/DataPreprocess/mpi_stager_v2_process_netCDF.py \
    # --source_dir /p/scratch/deepacf/video_prediction_shared_folder/extractedData/2015/ \
    # --destination_dir /p/scratch/deepacf/video_prediction_shared_folder/preprocessedData/era5-Y2015toY2017M01to12-128x160-74d00N71d00E-T_MSL_gph500/2015/ \
    # --vars T2 MSL gph500 --lat_s 74 --lat_e 202 --lon_s 550 --lon_e 710
    
    #srun python ../../workflow_parallel_frame_prediction/DataPreprocess/mpi_stager_v2_process_netCDF.py \
    # --source_dir /p/scratch/deepacf/video_prediction_shared_folder/extractedData/2016/ \
    # --destination_dir /p/scratch/deepacf/video_prediction_shared_folder/preprocessedData/era5-Y2015toY2017M01to12-128x160-74d00N71d00E-T_MSL_gph500/2016/ \
    # --vars T2 MSL gph500 --lat_s 74 --lat_e 202 --lon_s 550 --lon_e 710
    
    #srun python ../../workflow_parallel_frame_prediction/DataPreprocess/mpi_stager_v2_process_netCDF.py \
    # --source_dir /p/scratch/deepacf/video_prediction_shared_folder/extractedData/2017/ \
    # --destination_dir /p/scratch/deepacf/video_prediction_shared_folder/preprocessedData/era5-Y2015toY2017M01to12-128x160-74d00N71d00E-T_MSL_gph500/2017/ \
    # --vars T2 MSL gph500 --lat_s 74 --lat_e 202 --lon_s 550 --lon_e 710
    
    #srun python ../../workflow_parallel_frame_prediction/DataPreprocess/mpi_split_data_multi_years.py \
    #--destination_dir /p/scratch/deepacf/video_prediction_shared_folder/preprocessedData/era5-Y2015toY2017M01to12-128x160-74d00N71d00E-#T_MSL_gph500/
    
    
    
    #srun python ../../workflow_parallel_frame_prediction/DataPreprocess/mpi_stager_v2_process_netCDF.py \
    # --source_dir /p/scratch/deepacf/video_prediction_shared_folder/extractedData/2017 \
    # --destination_dir /p/scratch/deepacf/video_prediction_shared_folder/preprocessedData/Y2016toY2017M01to12-128x160-74d00N71d0E-T_MSL_gph500/2017 \
    # --vars T2 MSL gph500 --lat_s 74 --lat_e 202 --lon_s 550 --lon_e 710