diff --git a/unicorespawner/spawner.py b/unicorespawner/spawner.py
index 7115cde862f06f14192b21193eb93efcb5531dbd..5305a1de3892c654946b7bcdcd673ab7cb471420 100644
--- a/unicorespawner/spawner.py
+++ b/unicorespawner/spawner.py
@@ -3,22 +3,17 @@ import html
import json
import re
import time
-import uuid
from datetime import datetime
-from datetime import timedelta
import pyunicore.client as pyunicore
from forwardbasespawner import ForwardBaseSpawner
-from jupyterhub.utils import AnyTimeoutError
from jupyterhub.utils import maybe_future
from jupyterhub.utils import url_escape_path
from jupyterhub.utils import url_path_join
from pyunicore.forwarder import Forwarder
from requests.exceptions import HTTPError
-from tornado import gen
from traitlets import Any
from traitlets import Bool
-from traitlets import Dict
from traitlets import Integer
@@ -441,7 +436,7 @@ class UnicoreSpawner(ForwardBaseSpawner):
logs_s = "<br>".join(log_list_short_escaped)
return f"<details><summary>    {summary}(click here to expand):</summary>{logs_s}</details>"
- async def download_file(self, job, file):
+ def download_file(self, job, file):
self.log.info(f"{self._log_name} - Download {file}")
try:
file_path = job.working_dir.stat(file)
@@ -460,29 +455,22 @@ class UnicoreSpawner(ForwardBaseSpawner):
async def unicore_stop_event(self, spawner):
job = await self._get_job()
- timeout = 3
- download_future = self.download_file(job, "stdout")
+ timeout = 10
+ unicore_stdout = unicore_stderr = unicore_logs = "does not exist"
try:
- unicore_stdout = await gen.with_timeout(
- timedelta(seconds=timeout), download_future
+ unicore_stdout, unicore_stderr, unicore_logs = await asyncio.wait_for(
+ asyncio.gather(
+ asyncio.to_thread(self.download_file, job, "stdout"),
+ asyncio.to_thread(self.download_file, job, "stderr"),
+ asyncio.to_thread(job.properties.get, "log", []),
+ ),
+ timeout=timeout,
)
- except AnyTimeoutError:
+ except asyncio.TimeoutError:
self.log.exception(f"{self._log_name} - Timeout while downloading stdout")
- unicore_stdout = "Could not download file"
except:
self.log.exception(f"{self._log_name} - Error while downloading stdout")
- unicore_stdout = "Could not download file"
- download_future = self.download_file(job, "stderr")
- try:
- unicore_stderr = await gen.with_timeout(
- timedelta(seconds=timeout), download_future
- )
- except AnyTimeoutError:
- self.log.exception(f"{self._log_name} - Timeout while downloading stderr")
- unicore_stderr = "Could not download file"
- except:
- self.log.exception(f"{self._log_name} - Error while downloading stderr")
- unicore_stderr = "Could not download file"
+
now = datetime.now().strftime("%Y-%m-%d %H:%M:%S.%f")[:-3]
summary = f"UNICORE Job stopped with exitCode: {job.properties.get('exitCode', 'unknown exitCode')}"
@@ -491,7 +479,7 @@ class UnicoreSpawner(ForwardBaseSpawner):
"statusMessage", "unknown statusMessage"
)
unicore_logs_details = self._prettify_error_logs(
- job.properties.get("log", []), 20, "UNICORE logs"
+ unicore_logs, 20, "UNICORE logs"
)
unicore_stdout_details = self._prettify_error_logs(
@@ -634,22 +622,24 @@ class UnicoreSpawner(ForwardBaseSpawner):
await asyncio.sleep(5)
continue
# Download stderr to receive port + address
- timeout = 3
- download_future = self.download_file(unicore_job, "stderr")
+ timeout = 10
+ unicore_stderr = "does not exist"
try:
- unicore_stderr = await gen.with_timeout(
- timedelta(seconds=timeout), download_future
+ unicore_stderr = await asyncio.wait_for(
+ asyncio.gather(
+ asyncio.to_thread(self.download_file, job, "stderr")
+ ),
+ timeout=timeout,
)
- except AnyTimeoutError:
+ except asyncio.TimeoutError:
self.log.exception(
- f"{self._log_name} - Timeout while downloading stderr"
+ f"{self._log_name} - Timeout while downloading stdout"
)
- unicore_stderr = "Could not download file"
except:
self.log.exception(
- f"{self._log_name} - Error while downloading stderr"
+ f"{self._log_name} - Error while downloading stdout"
)
- unicore_stderr = "Could not download file"
+
if type(unicore_stderr) == str:
unicore_stderr = unicore_stderr.split("\n")
log_line = [
@@ -749,31 +739,24 @@ class UnicoreSpawner(ForwardBaseSpawner):
job = await self._get_job()
- timeout = 3
- download_future = self.download_file(job, "stdout")
+ timeout = 10
+ unicore_stdout = unicore_stderr = unicore_logs = "does not exist"
try:
- unicore_stdout = await gen.with_timeout(
- timedelta(seconds=timeout), download_future
+ unicore_stdout, unicore_stderr, unicore_logs = await asyncio.wait_for(
+ asyncio.gather(
+ asyncio.to_thread(self.download_file, job, "stdout"),
+ asyncio.to_thread(self.download_file, job, "stderr"),
+ asyncio.to_thread(job.properties.get, "log", []),
+ ),
+ timeout=timeout,
)
- except AnyTimeoutError:
+ except asyncio.TimeoutError:
self.log.exception(f"{self._log_name} - Timeout while downloading stdout")
- unicore_stdout = "Could not download file"
except:
self.log.exception(f"{self._log_name} - Error while downloading stdout")
- unicore_stdout = "Could not download file"
- download_future = self.download_file(job, "stderr")
- try:
- unicore_stderr = await gen.with_timeout(
- timedelta(seconds=timeout), download_future
- )
- except AnyTimeoutError:
- self.log.exception(f"{self._log_name} - Timeout while downloading stderr")
- unicore_stderr = "Could not download file"
- except:
- self.log.exception(f"{self._log_name} - Error while downloading stderr")
- unicore_stderr = "Could not download file"
+
self.log.debug(f"{self._log_name} - File download complete")
- unicore_logs = job.properties.get("log", [])
+
self.log.info(
f"{self._log_name} - Stop job. unicore log:\n{self.short_logs(unicore_logs, 20)}"
)