diff --git a/README.md b/README.md index 323709ec3608f63bfbb93fdaef1312529e9450ad..fe0d6923d7b0c9dc2cdc42cff5c23dfa6559d778 100644 --- a/README.md +++ b/README.md @@ -17,7 +17,19 @@ docker-compose -f dockers/docker-compose.yaml --project-directory . up -d ``` ## Setup connection + +### B2Share connection +Here we use testing instance (check hostname) + ``` -curl -X POST -u creds -H "Content-Type: application/json" --data '{"connection_id": "default_b2share","conn_type":"https", "host": "b2share-testing.fz-juelich.de", "schema":""}' localhost:7001/api/v1/connections +curl -X POST -u creds -H "Content-Type: application/json" --data '{"connection_id": "default_b2share","conn_type":"https", "host": "b2share-testing.fz-juelich.de", "schema":""}' airflow:7001/api/v1/connections ``` +### SSH +Copy to target goes through scp (example with username/pass) + +``` +curl -X POST -u creds -H "Content-Type: application/json" --data '{"connection_id": "default_ssh", "conn_type": "ssh", "host": "ssh", "login": "user", "port": 2222, "password": "pass"}' airflow:7001/api/v1/connections +``` + + diff --git a/dags/taskflow.py b/dags/taskflow.py index e45ed47a3e2b830c426f1b27e02b3b2e9ce2c15b..9b7b4deedb2e2134bf49c1c2c8f6bf25125b8188 100644 --- a/dags/taskflow.py +++ b/dags/taskflow.py @@ -23,7 +23,7 @@ def taskflow_example(**kwargs): print(f"Rereiving data from {server}") params = kwargs['params'] - if 'oid' not in params: + if 'oid' not in params: #{"oid":"b38609df2b334ea296ea1857e568dbea"} print(f"Missing object id in pipeline parameters") lst = get_objects(server=server) flist = {o['id']: [f['key'] for f in o['files']] for o in lst} @@ -52,8 +52,8 @@ def taskflow_example(**kwargs): ssh_hook = SSHHook(ssh_conn_id='default_ssh') with ssh_hook.get_conn() as ssh_client: sftp_client = ssh_client.open_sftp() - for [local, remote] in files.items(): - sftp_client.put(local, f"/tmp/{remote}") + for [truename, local] in files.items(): + sftp_client.put(local, f"/tmp/{truename}")