diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 9fd074e02b00444a87f188f101dfe5d5cb836ed6..7bd4a0e711b96ba12c915c420f970745599784c3 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -30,6 +30,24 @@ test: - airflow dags test testdag 2021-08-18 - nosetests +build-custom-image: + stage: build + image: docker:latest + services: + - docker:dind + when: manual + variables: + IMAGE_COMMIT_TAG: $CI_REGISTRY_IMAGE/eflows-airflow:$CI_COMMIT_SHORT_SHA + IMAGE_LATEST_TAG: $CI_REGISTRY_IMAGE/eflows-airflow:latest + + script: + - docker login -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD $CI_REGISTRY + - docker build --no-cache=true --pull -t $IMAGE_COMMIT_TAG -f dockers/eflows-airflow.docker . + - docker push $IMAGE_COMMIT_TAG + - docker tag $IMAGE_COMMIT_TAG $IMAGE_LATEST_TAG + - docker push $IMAGE_LATEST_TAG + + deploy-test: stage: deploy environment: Testing @@ -40,6 +58,9 @@ deploy-test: OS_IDENTITY_API_VERSION: 3 OS_REGION_NAME: "HDFCloud" OS_INTERFACE: public + # TODO set proepr values for env variable - need to pass those values to the machine (alternatively edit airflow.cfg only inside the machine) + AIRFLOW__SECRETS__BACKEND_KWARGS: '{\"url\" : \"https://zam10036.zam.kfa-juelich.de\", \"user\" : \"${DATACAT_TESTING_USERNAME}\", \"password\" : \"${DATACAT_TESTING_PASSWORD}\"}\' + AIRFLOW__SECRETS__BACKEND: datacat_integration.secrets.DatacatSecretsBackend FLOATING_IP: 134.94.199.220 script: - echo "Starting the full testing deployment of airflows example." diff --git a/config/airflow.cfg b/config/airflow.cfg index ad90566620303d47e3c2391f55e4af5ddb8b7aed..e5dba33dc3655a56c33841e4935812ece961c238 100644 --- a/config/airflow.cfg +++ b/config/airflow.cfg @@ -347,13 +347,13 @@ statsd_datadog_tags = [secrets] # Full class name of secrets backend to enable (will precede env vars and metastore in search path) # Example: backend = airflow.providers.amazon.aws.secrets.systems_manager.SystemsManagerParameterStoreBackend -backend = datacat_integration.secrets.DatacatSecretsBackend +backend = # The backend_kwargs param is loaded into a dictionary and passed to __init__ of secrets backend class. # See documentation for the secrets backend you are using. JSON is expected. # Example for AWS Systems Manager ParameterStore: # ``{"connections_prefix": "/airflow/connections", "profile_name": "default"}`` -backend_kwargs = {"url" : "https://zam10036.zam.kfa-juelich.de", "user" : "dls-testing", "password" : "dls-testing-pass"} +backend_kwargs = [cli] # In what way should the cli access the API. The LocalClient will use the diff --git a/dags/test_secrets_backend.py b/dags/test_secrets_backend.py new file mode 100644 index 0000000000000000000000000000000000000000..f8409669f8b02486f0f49743df0876f560f452f4 --- /dev/null +++ b/dags/test_secrets_backend.py @@ -0,0 +1,21 @@ + +from airflow.decorators import dag, task +from airflow.utils.dates import days_ago +from airflow.hooks.base import BaseHook + +default_args = { + 'owner': 'airflow', +} + + +@dag(default_args=default_args, schedule_interval=None, start_date=days_ago(2), tags=['example']) +def test_secrets_backend(): + @task() + def get_print_and_return_conenction(): + conn = BaseHook.get_connection('860355e9-975f-4253-9421-1815e20c879b') + print(conn.get_extra()) + + get_print_and_return_conenction() + + +dag = test_secrets_backend() diff --git a/dockers/docker-compose.yaml b/dockers/docker-compose.yaml index 24c78d4b44dc381c2382b39ca5ffea88c6c44a14..1bd2a0fadc3a0bc4c74d1d4533da87613fa81c34 100644 --- a/dockers/docker-compose.yaml +++ b/dockers/docker-compose.yaml @@ -56,6 +56,8 @@ x-airflow-common: AIRFLOW__CORE__DAGS_ARE_PAUSED_AT_CREATION: 'true' AIRFLOW__CORE__LOAD_EXAMPLES: 'false' AIRFLOW__API__AUTH_BACKEND: 'airflow.api.auth.backend.basic_auth' + AIRFLOW__SECRETS__BACKEND_KWARGS: ${AIRFLOW__SECRETS__BACKEND_KWARGS} + AIRFLOW__SECRETS__BACKEND: ${AIRFLOW__SECRETS__BACKEND} _PIP_ADDITIONAL_REQUIREMENTS: ${_PIP_ADDITIONAL_REQUIREMENTS:-} volumes: - ./dags:/opt/airflow/dags diff --git a/requirements.txt b/requirements.txt index 0b4ee447adaaaef3371d8d4177499e2810f9ab6d..2881e27f51283a5279f8ec3d78ec236b2a9e5c97 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,8 +1,8 @@ requests -urllib3==1.26.6 +urllib3 plyvel apache-airflow-providers-ssh apache-airflow-providers-http apache-airflow-providers-sftp --index-url https://gitlab.jsc.fz-juelich.de/api/v4/projects/4405/packages/pypi/simple -airflow-datacat-integration>=0.0.4 +airflow-datacat-integration>=0.0.10