From b3d0f9621ce1dc19dd9ae67f05efce8042fcc272 Mon Sep 17 00:00:00 2001
From: jrybicki-jsc <j.rybicki@fz-juelich.de>
Date: Fri, 27 Aug 2021 11:25:49 +0200
Subject: [PATCH] urllib fix
---
dags/b2shareoperator.py | 4 +++-
dags/taskflow.py | 2 +-
2 files changed, 4 insertions(+), 2 deletions(-)
diff --git a/dags/b2shareoperator.py b/dags/b2shareoperator.py
index 4ef4a9c..628fd9b 100644
--- a/dags/b2shareoperator.py
+++ b/dags/b2shareoperator.py
@@ -3,6 +3,7 @@ from airflow.models.connection import Connection
import requests
from urllib.parse import urljoin
import tempfile
+import urllib
def get_objects(server):
lst = requests.get(urljoin(server, 'api/records')).json()
@@ -19,7 +20,8 @@ def get_object_md(server, oid):
return obj
def download_file(url: str, target_dir: str):
- _, fname = tempfile.mkstemp(dir=target_dir)
+
+ fname = tempfile.mkstemp(dir=target_dir)
urllib.request.urlretrieve(url=url, filename=fname)
return fname
diff --git a/dags/taskflow.py b/dags/taskflow.py
index 8e80f63..24d5672 100644
--- a/dags/taskflow.py
+++ b/dags/taskflow.py
@@ -30,7 +30,7 @@ def taskflow_example():
name_mappings = {}
for fname, url in flist.items():
print(f"Processing: {fname} --> {url}")
- tmpname = download_file(url=url, target_dir='/tmp/downs/')
+ tmpname = download_file(url=url, target_dir='/tmp/')
name_mappings[fname]=tmpname
return name_mappings
--
GitLab