Select Git revision
simple_unicore.py 2.06 KiB
import json
import pyunicore.client as uc_client
import pyunicore.credentials as uc_credentials
from hidden_settings import (
UNICORE_BASE,
UNICORE_USER, UNICORE_PASSWORD
)
base_url = f"{UNICORE_BASE}JURECA/rest/core"
#base_url = f"{UNICORE_BASE}JUWELS/rest/core"
# Authentification/Authorization
credentials = uc_credentials.UsernamePassword(UNICORE_USER, UNICORE_PASSWORD)
# examples for transfering files
# remote base directory that we want to access (why is it automatically already in my one and only project
# what if I had two projects?)
#base_directory = "cjicg21/schroeder5/DATA/TEST/CTM/201807/18"
base_directory = "schroeder5/DATA/TEST/CTM/201807/18"
download_filename = "ctmout_wrf_cutcropped_199_h09.nc"
transport = uc_client.Transport(credentials)
#home = uc_client.Storage(transport, f"{base_url}/storages/HOME")
#print(home.contents())
scratch = uc_client.Storage(transport, f"{base_url}/storages/SCRATCH")
dfile = scratch.stat(f"{base_directory}/{download_filename}")
dfile.download(download_filename)
# examples for unicore (submitting commands/scripts/bash scripts)
client = uc_client.Client(credentials, base_url)
#job_description = {'Executable': "/p/project/deepacf/schroeder5/mlair/run_hdfml_batch.bash", 'Arguments' :[], }
#job_description = {'Executable': "/bin/ls", 'Arguments' :["-lisa", "$HOME"], }
#job_description = {'Executable': "/p/project/cjicg21/schroeder5/Destine_AQ/start_destine_demonstrator.sh", "Job type": "ON_LOGIN_NODE", 'Arguments':[], }
job_description = {'Executable': "/bin/cd /p/project/cjicg21/schroeder5/Destine_AQ;/bin/ls", "Job type": "ON_LOGIN_NODE", 'Arguments':[], }
job = client.new_job(job_description)
print("Submitted (on login node!): %s" % job)
# let's wait while the job is still running
job.poll()
print(json.dumps(job.properties, sort_keys=True, indent=4))
working_dir = job.working_dir
print (json.dumps(working_dir.properties, sort_keys=True, indent=4))
for f in working_dir.listdir("."):
print(f)
stdout_content = working_dir.stat("stdout").raw().readlines()
for line in stdout_content:
print(line)