added example notebook

073ba868 · Stefan Kesselheim · 58053a3a · 073ba868
Commit 073ba868 authored 2 years ago by Stefan Kesselheim
--- a/PySpark_PI.ipynb
+++ b/PySpark_PI.ipynb
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": 1,
+   "id": "0d4c4bff-43ef-4574-8252-d309f872de3a",
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "             JOBID PARTITION     NAME     USER ST       TIME  NODES NODELIST(REASON)\n",
+      "           6526639 develboos spark-cl kesselhe  R      24:48      2 jwb[0097,0117]\n"
+     ]
+    }
+   ],
+   "source": [
+    "!squeue --me"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "67b9bb6e-6dbb-4c0f-80fa-3aea2afcce28",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "from pyspark.sql import SparkSession\n",
+    "import os\n",
+    "import random\n",
+    "home = os.environ[\"HOME\"]\n",
+    "spark_master=\"spark://jwb0097i.juwels\"\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "5b2ee080-70e8-42b4-935f-3c8fa9634c85",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "\n",
+    "# This is required to add a \"i\" to the hostname\n",
+    "tmp=os.environ[\"HOSTNAME\"].split(\".\"); tmp[0]+=\"i\"; spark_driver_hostname=\".\".join(tmp)\n",
+    "\n",
+    "spark = SparkSession \\\n",
+    "    .builder \\\n",
+    "    .appName(\"My SparkSession\") \\\n",
+    "    .config(\"spark.master\", spark_master) \\\n",
+    "    .config(\"spark.driver.memory\", \"10g\") \\\n",
+    "    .config(\"spark.driver.host\", spark_driver_hostname) \\\n",
+    "    .config(\"spark.executor.memory\", \"400g\") \\\n",
+    "    .getOrCreate()\n",
+    "\n",
+    "sc=spark.sparkContext\n",
+    "\n",
+    "def inside(p):\n",
+    "    x, y = random.random(), random.random()\n",
+    "    return x*x + y*y < 1\n",
+    "\n",
+    "NUM_SAMPLES=10000000\n",
+    "count = sc.parallelize(range(0, NUM_SAMPLES)) \\\n",
+    "             .filter(inside).count()\n",
+    "print(\"Pi is roughly %f\" % (4.0 * count / NUM_SAMPLES))"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "spark_env",
+   "language": "python",
+   "name": "spark_env"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.9.6"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}
+%% Cell type:code id:0d4c4bff-43ef-4574-8252-d309f872de3a tags:
+``` python
+!squeue --me
+```
+%% Output
+                 JOBID PARTITION     NAME     USER ST       TIME  NODES NODELIST(REASON)
+               6526639 develboos spark-cl kesselhe  R      24:48      2 jwb[0097,0117]
+%% Cell type:code id:67b9bb6e-6dbb-4c0f-80fa-3aea2afcce28 tags:
+``` python
+from pyspark.sql import SparkSession
+import os
+import random
+home = os.environ["HOME"]
+spark_master="spark://jwb0097i.juwels"
+```
+%% Cell type:code id:5b2ee080-70e8-42b4-935f-3c8fa9634c85 tags:
+``` python
+# This is required to add a "i" to the hostname
+tmp=os.environ["HOSTNAME"].split("."); tmp[0]+="i"; spark_driver_hostname=".".join(tmp)
+spark = SparkSession \
+    .builder \
+    .appName("My SparkSession") \
+    .config("spark.master", spark_master) \
+    .config("spark.driver.memory", "10g") \
+    .config("spark.driver.host", spark_driver_hostname) \
+    .config("spark.executor.memory", "400g") \
+    .getOrCreate()
+sc=spark.sparkContext
+def inside(p):
+    x, y = random.random(), random.random()
+    return x*x + y*y < 1
+NUM_SAMPLES=10000000
+count = sc.parallelize(range(0, NUM_SAMPLES)) \
+             .filter(inside).count()
+print("Pi is roughly %f" % (4.0 * count / NUM_SAMPLES))
+```