Skip to content
Snippets Groups Projects
Select Git revision
  • 1c7f2433c5f73f4a83c14e790d03e11caad2f761
  • 2023 default protected
2 results

jax.py

Blame
  • DataPreprocess_dev.sh 3.21 KiB
    #!/bin/bash -x
    #SBATCH --account=deepacf
    #SBATCH --nodes=1
    #SBATCH --ntasks=12
    ##SBATCH --ntasks-per-node=12
    #SBATCH --cpus-per-task=1
    #SBATCH --output=DataPreprocess-out.%j
    #SBATCH --error=DataPreprocess-err.%j
    #SBATCH --time=00:20:00
    #SBATCH --partition=devel
    #SBATCH --mail-type=ALL
    #SBATCH --mail-user=m.langguth@fz-juelich.de
    
    module --force purge
    module use  $OTHERSTAGES
    module load Stages/2019a
    module load Intel/2019.3.199-GCC-8.3.0  ParaStationMPI/5.2.2-1
    module load h5py/2.9.0-Python-3.6.8
    module load mpi4py/3.0.1-Python-3.6.8
    
    source_dir=/p/scratch/deepacf/video_prediction_shared_folder/extractedData
    destination_dir=/p/scratch/deepacf/video_prediction_shared_folder/preprocessedData/era5-Y2015toY2017M01to12-128x160-74d00N71d00E-T_MSL_gph500/hickle
    declare -a years=("2015" 
                     "2016" 
                     "2017"
                      )
    
    
    
    for year in "${years[@]}"; 
        do 
            echo "Year $year"
    	echo "source_dir ${source_dir}/${year}"
    	srun python ../../workflow_parallel_frame_prediction/DataPreprocess/mpi_stager_v2_process_netCDF.py \
             --source_dir ${source_dir}/${year}/ \
             --destination_dir ${destination_dir}/${year}/ --vars T2 MSL gph500 --lat_s 74 --lat_e 202 --lon_s 550 --lon_e 710
        done
    
    
    srun python ../../workflow_parallel_frame_prediction/DataPreprocess/mpi_split_data_multi_years.py --destination_dir ${destination_dir}
    
    
    
    
    #srun python ../../workflow_parallel_frame_prediction/DataPreprocess/mpi_stager_v2_process_netCDF.py \
    # --source_dir /p/scratch/deepacf/video_prediction_shared_folder/extractedData/2015/ \
    # --destination_dir /p/scratch/deepacf/video_prediction_shared_folder/preprocessedData/era5-Y2015toY2017M01to12-128x160-74d00N71d00E-T_MSL_gph500/2015/ \
    # --vars T2 MSL gph500 --lat_s 74 --lat_e 202 --lon_s 550 --lon_e 710
    
    #srun python ../../workflow_parallel_frame_prediction/DataPreprocess/mpi_stager_v2_process_netCDF.py \
    # --source_dir /p/scratch/deepacf/video_prediction_shared_folder/extractedData/2016/ \
    # --destination_dir /p/scratch/deepacf/video_prediction_shared_folder/preprocessedData/era5-Y2015toY2017M01to12-128x160-74d00N71d00E-T_MSL_gph500/2016/ \
    # --vars T2 MSL gph500 --lat_s 74 --lat_e 202 --lon_s 550 --lon_e 710
    
    #srun python ../../workflow_parallel_frame_prediction/DataPreprocess/mpi_stager_v2_process_netCDF.py \
    # --source_dir /p/scratch/deepacf/video_prediction_shared_folder/extractedData/2017/ \
    # --destination_dir /p/scratch/deepacf/video_prediction_shared_folder/preprocessedData/era5-Y2015toY2017M01to12-128x160-74d00N71d00E-T_MSL_gph500/2017/ \
    # --vars T2 MSL gph500 --lat_s 74 --lat_e 202 --lon_s 550 --lon_e 710
    
    #srun python ../../workflow_parallel_frame_prediction/DataPreprocess/mpi_split_data_multi_years.py \
    #--destination_dir /p/scratch/deepacf/video_prediction_shared_folder/preprocessedData/era5-Y2015toY2017M01to12-128x160-74d00N71d00E-#T_MSL_gph500/
    
    
    
    #srun python ../../workflow_parallel_frame_prediction/DataPreprocess/mpi_stager_v2_process_netCDF.py \
    # --source_dir /p/scratch/deepacf/video_prediction_shared_folder/extractedData/2017 \
    # --destination_dir /p/scratch/deepacf/video_prediction_shared_folder/preprocessedData/Y2016toY2017M01to12-128x160-74d00N71d0E-T_MSL_gph500/2017 \
    # --vars T2 MSL gph500 --lat_s 74 --lat_e 202 --lon_s 550 --lon_e 710