From 5c39dd1a802dad82f38e52fc847768a2dac2a33e Mon Sep 17 00:00:00 2001
From: Tim Kreuzer <t.kreuzer@fz-juelich.de>
Date: Fri, 26 Nov 2021 12:02:41 +0000
Subject: [PATCH] update cronjobs
---
cronjob/cleanup_job_archive.sh | 2 ++
cronjob/cleanup_jupyterhub_metrics.sh | 2 ++
cronjob/metrics_7day_hifis.sh | 3 ++-
cronjob/metrics_7day_hifis_csv.sh | 31 +++++++++++++++++++++++++++
4 files changed, 37 insertions(+), 1 deletion(-)
create mode 100644 cronjob/cleanup_job_archive.sh
create mode 100644 cronjob/cleanup_jupyterhub_metrics.sh
create mode 100644 cronjob/metrics_7day_hifis_csv.sh
diff --git a/cronjob/cleanup_job_archive.sh b/cronjob/cleanup_job_archive.sh
new file mode 100644
index 00000000..8cfd69dd
--- /dev/null
+++ b/cronjob/cleanup_job_archive.sh
@@ -0,0 +1,2 @@
+#!/bin/bash
+sudo find /nfs/jupyter-jsc-live/common/jobs-archive/ -maxdepth 1 -type d -ctime +30 -exec rm -rf {} \;
diff --git a/cronjob/cleanup_jupyterhub_metrics.sh b/cronjob/cleanup_jupyterhub_metrics.sh
new file mode 100644
index 00000000..bd99abbb
--- /dev/null
+++ b/cronjob/cleanup_jupyterhub_metrics.sh
@@ -0,0 +1,2 @@
+#!/bin/bash
+sudo find /nfs/jupyter-jsc-live/logs/jupyterhub/ -maxdepth 1 -type f -name "metrics.log*" -mtime +30 -exec rm -rf {} \;
diff --git a/cronjob/metrics_7day_hifis.sh b/cronjob/metrics_7day_hifis.sh
index 352abcaf..eee6bca1 100644
--- a/cronjob/metrics_7day_hifis.sh
+++ b/cronjob/metrics_7day_hifis.sh
@@ -1,5 +1,6 @@
#!/bin/bash
-cd /nfs/jupyter-jsc-live/HIFIS-metrics/FZJ
+# cd /nfs/jupyter-jsc-live/HIFIS-metrics/FZJ
+cd /nfs/jupyter-jsc-live/metrics/FZJ-Jupyter/stats
git pull origin master
d_1=$(date -d 'today - 1 days' +%Y_%m_%d)
d_2=$(date -d 'today - 2 days' +%Y_%m_%d)
diff --git a/cronjob/metrics_7day_hifis_csv.sh b/cronjob/metrics_7day_hifis_csv.sh
new file mode 100644
index 00000000..063be9d1
--- /dev/null
+++ b/cronjob/metrics_7day_hifis_csv.sh
@@ -0,0 +1,31 @@
+#!/bin/bash
+cd /nfs/jupyter-jsc-live/metrics/FZJ-Jupyter/stats
+git pull origin master
+d_1=$(date -d 'today - 1 days' +%Y_%m_%d)
+d_2=$(date -d 'today - 2 days' +%Y_%m_%d)
+d_3=$(date -d 'today - 3 days' +%Y_%m_%d)
+d_4=$(date -d 'today - 4 days' +%Y_%m_%d)
+d_5=$(date -d 'today - 5 days' +%Y_%m_%d)
+d_6=$(date -d 'today - 6 days' +%Y_%m_%d)
+d_7=$(date -d 'today - 7 days' +%Y_%m_%d)
+
+declare -a days=("${d_7}" "${d_6}" "${d_5}" "${d_4}" "${d_3}" "${d_2}" "${d_1}")
+
+successful_total=$(grep -r -E "${d_1}|${d_2}|${d_3}|${d_4}|${d_5}|${d_6}|${d_7}" /nfs/jupyter-jsc-live/logs/jupyterhub/metrics.log* | grep "action=successful" | wc -l)
+successful_systems=$(awk -v OFS=";" '{print $2,$1}' <(grep -r -E "${d_1}|${d_2}|${d_3}|${d_4}|${d_5}|${d_6}|${d_7}" /nfs/jupyter-jsc-live/logs/jupyterhub/metrics.log* | grep "action=successful" | sed -e 's/.*system_input=\([^;]*\);.*/\1/' | sort | uniq -c))
+
+user_total=$(grep -r -E "${d_1}|${d_2}|${d_3}|${d_4}|${d_5}|${d_6}|${d_7}" /nfs/jupyter-jsc-live/logs/jupyterhub/metrics.log* | sed -e 's/.*userid=\([^;]*\);.*/\1/' | sort | uniq | wc -l)
+logins=$(awk -v OFS=";" '{print $2,$1}' <( grep -r -E "${d_1}|${d_2}|${d_3}|${d_4}|${d_5}|${d_6}|${d_7}" /nfs/jupyter-jsc-live/logs/jupyterhub/metrics.log* | grep "action=login" | sed -e 's/.*authenticator=\([^;]*\).*/\1/' | sort | uniq -c))
+
+used_nodes=$(grep -r -E "${d_1}|${d_2}|${d_3}|${d_4}|${d_5}|${d_6}|${d_7}" /nfs/jupyter-jsc-live/logs/jupyterhub/metrics.log* | grep "action=successful" | grep "resource_Nodes" | sed -e 's/.*resource_Nodes=\([^;]*\).*/\1/' | sort | uniq -c | awk '{s+=$1*$2} END {print s}')
+
+used_gpus=$(grep -r -E "${d_1}|${d_2}|${d_3}|${d_4}|${d_5}|${d_6}|${d_7}" /nfs/jupyter-jsc-live/logs/jupyterhub/metrics.log* | grep "action=successful" | grep "resource_GPUS" | sed -e 's/.*resource_GPUS=\([^;]*\).*/\1/' | sort | uniq -c | awk '{s+=$1*$2} END {print s}')
+
+datetime=`date --rfc-3339=seconds -d 'today - 7 days'`
+echo "$datetime,$successful_total,$user_total,$used_nodes,$used_gpus" >> usage-stats-jupyter-jsc-weekly.csv
+
+git add usage-stats-jupyter-jsc-weekly.csv
+git commit -m "update Jupyter-JSC metrics"
+git push origin master
+
+find /nfs/jupyter-jsc-live/logs/jupyterhub -type f -name "metrics.log*" -mtime +30 -delete
--
GitLab