start_spark_cluster.sh

#!/bin/bash

#SBATCH --partition=develbooster
#SBATCH --account=atmlaml
#SBATCH --nodes=1
#SBATCH --tasks-per-node=9
#SBATCH --time=00:60:00
#SBATCH --gres gpu
#SBATCH --job-name spark-cluster
 
module load Stages/2023  GCC  OpenMPI Spark

JOB="$SLURM_JOB_NAME-$SLURM_JOB_ID"
export SPARK_WORKER_DIR="$SLURM_SUBMIT_DIR/$JOB/worker"
export SPARK_LOG_DIR="$SLURM_SUBMIT_DIR/$JOB/log"
export SPARK_MASTER_HOST=`hostname`
export SPARK_MASTER_PORT="4124"
export SPARK_WORKER_CORES=7
export SPARK_WORKER_INSTANCES=5 ## This is not working yet
export MASTER_URL="spark://${SPARK_MASTER_HOST}:${SPARK_MASTER_PORT}"
export > ${SPARK_LOG_DIR}/env.txt

echo "------------ Starting Spark Server -------------"
echo "MASTER_URL: $MASTER_URL"
echo "------------------------------------------------"
start-master.sh

echo "------------ Starting Spark Workers ------------"
echo "MASTER_URL: $MASTER_URL"
echo "------------------------------------------------"

#srun -n 14 spark-class org.apache.spark.deploy.worker.Worker $MASTER_URL
#srun -n 1 start-worker.sh $MASTER_URL
start-worker.sh $MASTER_URL

. wait-worker.sh

echo "----------- How to test with spark shell ------"
echo "sgoto $SLURM_JOB_ID 0"
echo "module load Stages/2023  GCC  OpenMPI Spark"
echo "export MASTER_URL=$MASTER_URL"
echo "spark-shell --master \$MASTER_URL"
echo "# Now your shell should connect to the spark master"
echo
echo "In the shell you can now execute the following test script"
echo
echo ">>>>"
echo "val NUM_SAMPLES=1000000"
echo "val count = sc.parallelize(1 to NUM_SAMPLES).filter { _ =>"
echo "  val x = math.random"
echo "  val y = math.random"
echo "  x*x + y*y < 1"
echo "}.count()"
echo "println(s\"Pi is roughly \${4.0 * count / NUM_SAMPLES}\")"
echo "<<<<"
echo "------------------------------------------------"
echo
echo "----------- How to test with pyspark ------"
echo "sgoto $SLURM_JOB_ID 0"
echo "module load Stages/2023  GCC  OpenMPI Spark"
echo "export MASTER_URL=$MASTER_URL"
echo "python pyspark_pi.py"
echo "------------------------------------------------"
echo 
echo "---------- How to kill -------------------------"
echo scancel $SLURM_JOB_ID
echo "------------------------------------------------"


sleep infinity