Skip to content
Snippets Groups Projects
Select Git revision
  • 837c510d4eaffa4b3f96bdd7bf79bcb8f2c9d965
  • master default protected
  • staggered-max
  • unifed-object
  • no_Amalgamated
  • standard_stb
  • parallel_doc_fix
  • release_0.9.3
  • cmake_mpich_tests
  • update_install
  • external_VORO
  • cmake_add_soversion
  • iterative_method
  • cmake_install
  • fixes_042023
  • tensor_max
  • personal/schulz3/tensor_max
  • releases/v0.9
  • ForceBasedDevel
  • refactor
  • feature/simple_test_cases
  • v0.9.3
  • v0.9.2
  • v0.9.1
  • v0.9.0
  • v0.9.0-rc2
26 results

ALL_fortran.cpp

Blame
  • DataPreprocess_dev.sh 3.21 KiB
    #!/bin/bash -x
    #SBATCH --account=deepacf
    #SBATCH --nodes=1
    #SBATCH --ntasks=12
    ##SBATCH --ntasks-per-node=12
    #SBATCH --cpus-per-task=1
    #SBATCH --output=DataPreprocess-out.%j
    #SBATCH --error=DataPreprocess-err.%j
    #SBATCH --time=00:20:00
    #SBATCH --partition=devel
    #SBATCH --mail-type=ALL
    #SBATCH --mail-user=m.langguth@fz-juelich.de
    
    module --force purge
    module use  $OTHERSTAGES
    module load Stages/2019a
    module load Intel/2019.3.199-GCC-8.3.0  ParaStationMPI/5.2.2-1
    module load h5py/2.9.0-Python-3.6.8
    module load mpi4py/3.0.1-Python-3.6.8
    
    source_dir=/p/scratch/deepacf/video_prediction_shared_folder/extractedData
    destination_dir=/p/scratch/deepacf/video_prediction_shared_folder/preprocessedData/era5-Y2015toY2017M01to12-128x160-74d00N71d00E-T_MSL_gph500/hickle
    declare -a years=("2015" 
                     "2016" 
                     "2017"
                      )
    
    
    
    for year in "${years[@]}"; 
        do 
            echo "Year $year"
    	echo "source_dir ${source_dir}/${year}"
    	srun python ../../workflow_parallel_frame_prediction/DataPreprocess/mpi_stager_v2_process_netCDF.py \
             --source_dir ${source_dir}/${year}/ \
             --destination_dir ${destination_dir}/${year}/ --vars T2 MSL gph500 --lat_s 74 --lat_e 202 --lon_s 550 --lon_e 710
        done
    
    
    srun python ../../workflow_parallel_frame_prediction/DataPreprocess/mpi_split_data_multi_years.py --destination_dir ${destination_dir}
    
    
    
    
    #srun python ../../workflow_parallel_frame_prediction/DataPreprocess/mpi_stager_v2_process_netCDF.py \
    # --source_dir /p/scratch/deepacf/video_prediction_shared_folder/extractedData/2015/ \
    # --destination_dir /p/scratch/deepacf/video_prediction_shared_folder/preprocessedData/era5-Y2015toY2017M01to12-128x160-74d00N71d00E-T_MSL_gph500/2015/ \
    # --vars T2 MSL gph500 --lat_s 74 --lat_e 202 --lon_s 550 --lon_e 710
    
    #srun python ../../workflow_parallel_frame_prediction/DataPreprocess/mpi_stager_v2_process_netCDF.py \
    # --source_dir /p/scratch/deepacf/video_prediction_shared_folder/extractedData/2016/ \
    # --destination_dir /p/scratch/deepacf/video_prediction_shared_folder/preprocessedData/era5-Y2015toY2017M01to12-128x160-74d00N71d00E-T_MSL_gph500/2016/ \
    # --vars T2 MSL gph500 --lat_s 74 --lat_e 202 --lon_s 550 --lon_e 710
    
    #srun python ../../workflow_parallel_frame_prediction/DataPreprocess/mpi_stager_v2_process_netCDF.py \
    # --source_dir /p/scratch/deepacf/video_prediction_shared_folder/extractedData/2017/ \
    # --destination_dir /p/scratch/deepacf/video_prediction_shared_folder/preprocessedData/era5-Y2015toY2017M01to12-128x160-74d00N71d00E-T_MSL_gph500/2017/ \
    # --vars T2 MSL gph500 --lat_s 74 --lat_e 202 --lon_s 550 --lon_e 710
    
    #srun python ../../workflow_parallel_frame_prediction/DataPreprocess/mpi_split_data_multi_years.py \
    #--destination_dir /p/scratch/deepacf/video_prediction_shared_folder/preprocessedData/era5-Y2015toY2017M01to12-128x160-74d00N71d00E-#T_MSL_gph500/
    
    
    
    #srun python ../../workflow_parallel_frame_prediction/DataPreprocess/mpi_stager_v2_process_netCDF.py \
    # --source_dir /p/scratch/deepacf/video_prediction_shared_folder/extractedData/2017 \
    # --destination_dir /p/scratch/deepacf/video_prediction_shared_folder/preprocessedData/Y2016toY2017M01to12-128x160-74d00N71d0E-T_MSL_gph500/2017 \
    # --vars T2 MSL gph500 --lat_s 74 --lat_e 202 --lon_s 550 --lon_e 710