Skip to content
Snippets Groups Projects
Select Git revision
  • master
  • pytorch-2.0
2 results

Makefile

Blame
    • start_spark_cluster.sh 2.16 KiB
    1
    2
    3
    4
    5
    6
    7
    8
    9
    10
    11
    12
    13
    14
    15
    16
    17
    18
    19
    20
    21
    22
    23
    24
    25
    26
    27
    28
    29
    30
    31
    32
    33
    34
    35
    36
    37
    38
    39
    40
    41
    42
    43
    44
    45
    46
    47
    48
    49
    50
    51
    52
    53
    54
    55
    56
    57
    58
    59
    60
    61
    62
    63
    64
    65
    66
    67
    68
    69
    70
    71
    72
    73 
    

    

    

    #!/bin/bash

#SBATCH --partition=develbooster
#SBATCH --account=atmlaml
#SBATCH --nodes=1
#SBATCH --tasks-per-node=9
#SBATCH --time=00:60:00
#SBATCH --gres gpu
#SBATCH --job-name spark-cluster
 
module load Stages/2023  GCC  OpenMPI Spark

JOB="$SLURM_JOB_NAME-$SLURM_JOB_ID"
export SPARK_WORKER_DIR="$SLURM_SUBMIT_DIR/$JOB/worker"
export SPARK_LOG_DIR="$SLURM_SUBMIT_DIR/$JOB/log"
export SPARK_MASTER_HOST=`hostname`
export SPARK_MASTER_PORT="4124"
export SPARK_WORKER_CORES=7
export SPARK_WORKER_INSTANCES=5 ## This is not working yet
export MASTER_URL="spark://${SPARK_MASTER_HOST}:${SPARK_MASTER_PORT}"
export > ${SPARK_LOG_DIR}/env.txt

echo "------------ Starting Spark Server -------------"
echo "MASTER_URL: $MASTER_URL"
echo "------------------------------------------------"
start-master.sh

echo "------------ Starting Spark Workers ------------"
echo "MASTER_URL: $MASTER_URL"
echo "------------------------------------------------"

#srun -n 14 spark-class org.apache.spark.deploy.worker.Worker $MASTER_URL
#srun -n 1 start-worker.sh $MASTER_URL
start-worker.sh $MASTER_URL

. wait-worker.sh

echo "----------- How to test with spark shell ------"
echo "sgoto $SLURM_JOB_ID 0"
echo "module load Stages/2023  GCC  OpenMPI Spark"
echo "export MASTER_URL=$MASTER_URL"
echo "spark-shell --master \$MASTER_URL"
echo "# Now your shell should connect to the spark master"
echo
echo "In the shell you can now execute the following test script"
echo
echo ">>>>"
echo "val NUM_SAMPLES=1000000"
echo "val count = sc.parallelize(1 to NUM_SAMPLES).filter { _ =>"
echo "  val x = math.random"
echo "  val y = math.random"
echo "  x*x + y*y < 1"
echo "}.count()"
echo "println(s\"Pi is roughly \${4.0 * count / NUM_SAMPLES}\")"
echo "<<<<"
echo "------------------------------------------------"
echo
echo "----------- How to test with pyspark ------"
echo "sgoto $SLURM_JOB_ID 0"
echo "module load Stages/2023  GCC  OpenMPI Spark"
echo "export MASTER_URL=$MASTER_URL"
echo "python pyspark_pi.py"
echo "------------------------------------------------"
echo 
echo "---------- How to kill -------------------------"
echo scancel $SLURM_JOB_ID
echo "------------------------------------------------"


sleep infinity