ssh connection setup and test

README.md

@@ -17,7 +17,19 @@ docker-compose -f dockers/docker-compose.yaml --project-directory . up -d
 ```
 
 ## Setup connection
+
+### B2Share connection 
+Here we use testing instance (check hostname)
+
 ```
-curl -X POST -u creds -H "Content-Type: application/json"  --data '{"connection_id": "default_b2share","conn_type":"https", "host": "b2share-testing.fz-juelich.de", "schema":""}' localhost:7001/api/v1/connections
+curl -X POST -u creds -H "Content-Type: application/json"  --data '{"connection_id": "default_b2share","conn_type":"https", "host": "b2share-testing.fz-juelich.de", "schema":""}' airflow:7001/api/v1/connections
 ```
 
+### SSH 
+Copy to target goes through scp (example with username/pass)
+
+```
+curl -X POST -u creds -H "Content-Type: application/json"  --data '{"connection_id": "default_ssh", "conn_type": "ssh", "host": "ssh", "login": "user", "port": 2222, "password": "pass"}' airflow:7001/api/v1/connections
+```
+
+

dags/taskflow.py

@@ -23,7 +23,7 @@ def taskflow_example(**kwargs):
         print(f"Rereiving data from {server}")
 
         params = kwargs['params']
-        if 'oid' not in params:
+        if 'oid' not in params: #{"oid":"b38609df2b334ea296ea1857e568dbea"}
             print(f"Missing object id in pipeline parameters")
             lst = get_objects(server=server)
             flist = {o['id']: [f['key'] for f in o['files']] for o in lst}
@@ -52,8 +52,8 @@ def taskflow_example(**kwargs):
         ssh_hook = SSHHook(ssh_conn_id='default_ssh')
         with ssh_hook.get_conn() as ssh_client:
             sftp_client = ssh_client.open_sftp()
-            for [local, remote] in files.items():
-                sftp_client.put(local, f"/tmp/{remote}")
+            for [truename, local] in files.items():
+                sftp_client.put(local, f"/tmp/{truename}")