Skip to content
Snippets Groups Projects
Select Git revision
  • 5687e99f85ca80dfabe5db8a3cd820794bc2f54a
  • master default
  • bing_issues#190_tf2
  • bing_tf2_convert
  • bing_issue#189_train_modular
  • simon_#172_integrate_weatherbench
  • develop
  • bing_issue#188_restructure_ambs
  • yan_issue#100_extract_prcp_data
  • bing_issue#170_data_preprocess_training_tf1
  • Gong2022_temperature_forecasts
  • bing_issue#186_clean_GMD1_tag
  • yan_issue#179_integrate_GZAWS_data_onfly
  • bing_issue#178_runscript_bug_postprocess
  • michael_issue#187_bugfix_setup_runscript_template
  • bing_issue#180_bugs_postprpocess_meta_postprocess
  • yan_issue#177_repo_for_CLGAN_gmd
  • bing_issue#176_integrate_weather_bench
  • michael_issue#181_eval_era5_forecasts
  • michael_issue#182_eval_subdomain
  • michael_issue#119_warmup_Horovod
  • bing_issue#160_test_zam347
  • ambs_v1
  • ambs_gmd_nowcasting_v1.0
  • GMD1
  • modular_booster_20210203
  • new_structure_20201004_v1.0
  • old_structure_20200930
28 results

sna_model.py

Blame
  • DataPreprocess_dev.sh 3.21 KiB
    #!/bin/bash -x
    #SBATCH --account=deepacf
    #SBATCH --nodes=1
    #SBATCH --ntasks=12
    ##SBATCH --ntasks-per-node=12
    #SBATCH --cpus-per-task=1
    #SBATCH --output=DataPreprocess-out.%j
    #SBATCH --error=DataPreprocess-err.%j
    #SBATCH --time=00:20:00
    #SBATCH --partition=devel
    #SBATCH --mail-type=ALL
    #SBATCH --mail-user=m.langguth@fz-juelich.de
    
    module --force purge
    module use  $OTHERSTAGES
    module load Stages/2019a
    module load Intel/2019.3.199-GCC-8.3.0  ParaStationMPI/5.2.2-1
    module load h5py/2.9.0-Python-3.6.8
    module load mpi4py/3.0.1-Python-3.6.8
    
    source_dir=/p/scratch/deepacf/video_prediction_shared_folder/extractedData
    destination_dir=/p/scratch/deepacf/video_prediction_shared_folder/preprocessedData/era5-Y2015toY2017M01to12-128x160-74d00N71d00E-T_MSL_gph500/hickle
    declare -a years=("2015" 
                     "2016" 
                     "2017"
                      )
    
    
    
    for year in "${years[@]}"; 
        do 
            echo "Year $year"
    	echo "source_dir ${source_dir}/${year}"
    	srun python ../../workflow_parallel_frame_prediction/DataPreprocess/mpi_stager_v2_process_netCDF.py \
             --source_dir ${source_dir}/${year}/ \
             --destination_dir ${destination_dir}/${year}/ --vars T2 MSL gph500 --lat_s 74 --lat_e 202 --lon_s 550 --lon_e 710
        done
    
    
    srun python ../../workflow_parallel_frame_prediction/DataPreprocess/mpi_split_data_multi_years.py --destination_dir ${destination_dir}
    
    
    
    
    #srun python ../../workflow_parallel_frame_prediction/DataPreprocess/mpi_stager_v2_process_netCDF.py \
    # --source_dir /p/scratch/deepacf/video_prediction_shared_folder/extractedData/2015/ \
    # --destination_dir /p/scratch/deepacf/video_prediction_shared_folder/preprocessedData/era5-Y2015toY2017M01to12-128x160-74d00N71d00E-T_MSL_gph500/2015/ \
    # --vars T2 MSL gph500 --lat_s 74 --lat_e 202 --lon_s 550 --lon_e 710
    
    #srun python ../../workflow_parallel_frame_prediction/DataPreprocess/mpi_stager_v2_process_netCDF.py \
    # --source_dir /p/scratch/deepacf/video_prediction_shared_folder/extractedData/2016/ \
    # --destination_dir /p/scratch/deepacf/video_prediction_shared_folder/preprocessedData/era5-Y2015toY2017M01to12-128x160-74d00N71d00E-T_MSL_gph500/2016/ \
    # --vars T2 MSL gph500 --lat_s 74 --lat_e 202 --lon_s 550 --lon_e 710
    
    #srun python ../../workflow_parallel_frame_prediction/DataPreprocess/mpi_stager_v2_process_netCDF.py \
    # --source_dir /p/scratch/deepacf/video_prediction_shared_folder/extractedData/2017/ \
    # --destination_dir /p/scratch/deepacf/video_prediction_shared_folder/preprocessedData/era5-Y2015toY2017M01to12-128x160-74d00N71d00E-T_MSL_gph500/2017/ \
    # --vars T2 MSL gph500 --lat_s 74 --lat_e 202 --lon_s 550 --lon_e 710
    
    #srun python ../../workflow_parallel_frame_prediction/DataPreprocess/mpi_split_data_multi_years.py \
    #--destination_dir /p/scratch/deepacf/video_prediction_shared_folder/preprocessedData/era5-Y2015toY2017M01to12-128x160-74d00N71d00E-#T_MSL_gph500/
    
    
    
    #srun python ../../workflow_parallel_frame_prediction/DataPreprocess/mpi_stager_v2_process_netCDF.py \
    # --source_dir /p/scratch/deepacf/video_prediction_shared_folder/extractedData/2017 \
    # --destination_dir /p/scratch/deepacf/video_prediction_shared_folder/preprocessedData/Y2016toY2017M01to12-128x160-74d00N71d0E-T_MSL_gph500/2017 \
    # --vars T2 MSL gph500 --lat_s 74 --lat_e 202 --lon_s 550 --lon_e 710