diff --git a/.gitignore b/.gitignore
index 003d1453b38ecde5e0088b5af73a6fc74db71345..a63696d55a0014c022e5b159a15164cd64d86626 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1,5 +1,6 @@
**__pycache__/
.vscode/*
+*.pyc
# contains data for local tests
app/
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 1d4760d508646d6167a8cb457d21ee29595a81a1..a44a95e4bfa63af8ed1276c75871d519e717efb4 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -16,6 +16,10 @@ test:
script:
- pip install -r testing_requirements.txt
- nosetests --with-coverage --cover-package=apiserver
+ artifacts:
+ reports:
+ cobertura: coverage.xml
+
build:sites:
cache: {}
@@ -61,8 +65,20 @@ deploy:
# - docker build --no-cache=true --pull -f ./apiserver/Dockerfile -t api-test . #pull from $CI_REGISTRY
# - sudo docker run --name api-test-cloud --network net -e VIRTUAL_HOST="zam10028.zam.kfa-juelich.de" -e LETSENCRYPT_HOST="zam10028.zam.kfa-juelich.de" -d api-test
# - openstack server destroy pipeline-inst #old
+
+publishgit-do:
+ image: python:3-slim
+ stage: publish
only:
- - mptest
+ - tags
+ tags: [stable]
+ script:
+ - apt-get update
+ - apt-get install -y git
+ - git remote set-url gith "https://${GITHUB_USER}:${GITHUB_TOKEN}@github.com/eflows4hpc/datacatalog.git"
+ - git remote -v
+ - git show-ref
+ - git push gith $CI_COMMIT_REF_NAME
# This should be an automatic push of the docker image into gitLab container repository
diff --git a/README.md b/README.md
index d61686c05bb93a25e717276407fc8e5badc67a69..d641e4ae0b5a978fa9f8622d528133f564504193 100644
--- a/README.md
+++ b/README.md
@@ -24,6 +24,7 @@ For development (and only for development), an easy way to deploay a local serve
```shell
python -m http.server <localport> --directory site/
```
+The python http.server package should **not** be used for deployment, as it does not ensure that current security standards are met, and is only intended for local testing.
## API-Server for the Data Catalog
@@ -31,7 +32,21 @@ python -m http.server <localport> --directory site/
It is implemented via [fastAPI](https://fastapi.tiangolo.com/) and provides an api documentation via openAPI.
-For deployment via [docker](https://www.docker.com/), a docker image is included.
+For deployment via [docker](https://www.docker.com/), a docker image is included.
+
+### Configuration
+
+Some server settings can be changed. This can either be used during testing, so that a test api server can be launched with testing data, or for deployment, if the appdata or the userdb is not in the default location.
+
+These settings can be set either via environment variables, changed in the `apiserver/config.env` file, or a different `.env` file can be configured via the `DATACATALOG_API_DOTENV_FILE_PATH` environment variable.
+
+At the moment, the settings are considered at launch, and can not be updated while the server is running.
+
+| Variable Name | Default Value | Description |
+|-----------------------------------------|------------------------|--------------------------------------------------------|
+| DATACATALOG_API_DOTENV_FILE_PATH | `apiserver/config.env` | Location of the `.env` file considered at launch |
+| DATACATALOG_APISERVER_JSON_STORAGE_PATH | `./app/data` | Directory where the data (i.e. dataset info) is stored |
+| DATACATALOG_APISERVER_USERDB_PATH | `./app/userdb.json` | Location of the `.json` file containing the accounts |
### Security
diff --git a/apiserver/main.py b/apiserver/main.py
index c4c97d83035a7a68023298d241ac7c929ad6fb57..48f306cf8ce68dd51843b01010bc466dd83abb29 100644
--- a/apiserver/main.py
+++ b/apiserver/main.py
@@ -2,20 +2,23 @@
Main module of data catalog api
"""
import logging
+import os
from datetime import timedelta
from enum import Enum
-from typing import List, Tuple
+from typing import List
from fastapi import FastAPI, HTTPException, Request, status
from fastapi.param_functions import Depends
from fastapi.responses import JSONResponse
from fastapi.security import OAuth2PasswordBearer, OAuth2PasswordRequestForm
+from pydantic import UUID4
+
from .config import ApiserverSettings
from .security import (ACCESS_TOKEN_EXPIRES_MINUTES, JsonDBInterface, Token,
User, authenticate_user, create_access_token,
get_current_user)
-from .storage import JsonFileStorageAdapter, LocationData, LocationDataType
+from .storage import JsonFileStorageAdapter, LocationData, LocationDataType, verify_oid
class ReservedPaths(str, Enum):
@@ -24,13 +27,17 @@ class ReservedPaths(str, Enum):
AUTH = 'auth'
ME = 'me'
+DOTENV_FILE_PATH_VARNAME = "DATACATALOG_API_DOTENV_FILE_PATH"
+DOTENV_FILE_PATH_DEFAULT = "apiserver/config.env"
app = FastAPI(
title="API-Server for the Data Catalog"
)
+# if env variable is set, get config .env filepath from it, else use default
+dotenv_file_path = os.getenv(DOTENV_FILE_PATH_VARNAME, DOTENV_FILE_PATH_DEFAULT)
-settings = ApiserverSettings()
+settings = ApiserverSettings(_env_file=dotenv_file_path)
adapter = JsonFileStorageAdapter(settings)
userdb = JsonDBInterface(settings)
oauth2_scheme = OAuth2PasswordBearer(tokenUrl=ReservedPaths.TOKEN)
@@ -73,18 +80,18 @@ async def get_types():
"""
return [{element.value: "/" + element.value} for element in LocationDataType]
-@app.get("/{location_data_type}")
+@app.get("/{location_data_type}")
async def list_datasets(location_data_type: LocationDataType):
"""list id and name of every registered dataset for the specified type"""
return adapter.get_list(location_data_type)
@app.get("/{location_data_type}/{dataset_id}", response_model=LocationData)
-async def get_specific_dataset(location_data_type: LocationDataType, dataset_id: str):
+async def get_specific_dataset(location_data_type: LocationDataType, dataset_id: UUID4):
"""returns all information about a specific dataset, identified by id"""
- return adapter.get_details(location_data_type, dataset_id)
+ return adapter.get_details(location_data_type, str(dataset_id))
-@app.post("/{location_data_type}")
+@app.post("/{location_data_type}")
async def add_dataset(location_data_type: LocationDataType,
dataset: LocationData,
user: User = Depends(my_user)):
@@ -94,19 +101,19 @@ async def add_dataset(location_data_type: LocationDataType,
@app.put("/{location_data_type}/{dataset_id}")
async def update_specific_dataset(location_data_type: LocationDataType,
- dataset_id: str, dataset: LocationData,
+ dataset_id: UUID4, dataset: LocationData,
user: User = Depends(my_user)):
"""update the information about a specific dataset, identified by id"""
- return adapter.update_details(location_data_type, dataset_id, dataset, user.username)
+ return adapter.update_details(location_data_type, str(dataset_id), dataset, user.username)
@app.delete("/{location_data_type}/{dataset_id}")
async def delete_specific_dataset(location_data_type: LocationDataType,
- dataset_id: str,
+ dataset_id: UUID4,
user: str = Depends(my_user)):
"""delete a specific dataset"""
# TODO: 404 is the right answer? 204 could also be the right one
- return adapter.delete(location_data_type, dataset_id, user.username)
+ return adapter.delete(location_data_type, str(dataset_id), user.username)
@app.exception_handler(FileNotFoundError)
@@ -114,4 +121,4 @@ async def not_found_handler(request: Request, ex: FileNotFoundError):
_ =request.path_params.get('dataset_id', '')
logging.error("File not found translated %s", ex)
return JSONResponse(status_code=status.HTTP_404_NOT_FOUND,
- content={'message':f"Object does not exist"})
+ content={'message':'Object does not exist'})
diff --git a/apiserver/storage/JsonFileStorageAdapter.py b/apiserver/storage/JsonFileStorageAdapter.py
index 45106946ada5fc1274e11d5dd05a582ebdddaada..ff7dcaefa34711bee11e983ce863896ab7f3e4c4 100644
--- a/apiserver/storage/JsonFileStorageAdapter.py
+++ b/apiserver/storage/JsonFileStorageAdapter.py
@@ -2,6 +2,7 @@ import json
import os
import uuid
from typing import List
+import logging
from pydantic import BaseModel
@@ -24,6 +25,18 @@ def get_unique_id(path: str) -> str:
oid = str(uuid.uuid4())
return oid
+
+def verify_oid(oid: str, version=4):
+ """ Ensure thatthe oid is formatted as a valid oid (i.e. UUID v4).
+ If it isn't, the corresponding request could theoretically be
+ an attempted path traversal attack (or a regular typo).
+ """
+ try:
+ uuid_obj = uuid.UUID(oid, version=version)
+ return str(uuid_obj) == oid
+ except:
+ return False
+
class JsonFileStorageAdapter(AbstractLocationDataStorageAdapter):
""" This stores LocationData via the StoredData Object as json files
@@ -53,9 +66,9 @@ class JsonFileStorageAdapter(AbstractLocationDataStorageAdapter):
full_path = os.path.join(localpath, oid)
common = os.path.commonprefix((os.path.realpath(full_path),os.path.realpath(self.data_dir)))
if common != os.path.realpath(self.data_dir):
- print(f"Escaping the data dir! {common} {full_path}")
+ logging.error(f"Escaping the data dir! {common} {full_path}")
raise FileNotFoundError()
-
+
if not os.path.isfile(full_path):
raise FileNotFoundError(
f"The requested object ({oid}) {full_path} does not exist.")
diff --git a/apiserver/storage/__init__.py b/apiserver/storage/__init__.py
index 3d63565e7b7cc56a1939e1b1371edffbfdfd95e9..8c48a896dd34600a875076a9603eeb6f52574e5c 100644
--- a/apiserver/storage/__init__.py
+++ b/apiserver/storage/__init__.py
@@ -1,3 +1,3 @@
-from .JsonFileStorageAdapter import JsonFileStorageAdapter
+from .JsonFileStorageAdapter import JsonFileStorageAdapter, verify_oid
from .LocationStorage import LocationDataType, LocationData, AbstractLocationDataStorageAdapter
\ No newline at end of file
diff --git a/client/1.json b/client/1.json
index fa4df1bd0a80eeafcb8702ba2e8eca8a6d7ee087..9b4ebcddb1f37ab172eea3c665ae8071f65c86fb 100644
--- a/client/1.json
+++ b/client/1.json
@@ -30,5 +30,36 @@
"Division": "IAFES - Impacts on Agriculture, Forests and Ecosystem Services.",
"author": "CMCC Foundation"
}
- }
+ },
+ {
+ "name":"ArcSWAT 2012 Global Weather Database",
+ "url":"https://swat.tamu.edu/media/99082/cfsr_world.zip",
+ "metadata": {
+ "author": "CFSR",
+ "format": "zip"
+ }
+},
+ {
+ "name":"The China Meteorological Assimilation Driving Datasets for the SWAT model (CMADS)",
+ "url": "https://pan.baidu.com/s/1OYWe7ejyZUeSKE8T5Gg4jg#list/path=%2F",
+ "metadata": {
+ "author": "Meng, X.Y.; Wang, H.; Chen, J. "
+ }
+},
+ {
+ "name": "he CMCC Eddy-permitting Global Ocean Physical Reanalysis",
+ "url": "https://doi.pangaea.de/10.1594/PANGAEA.857995?format=textfile",
+ "metadata": {
+ "author": "Storto, Andrea; Masina, Simona",
+ "organization": "PANGAEA"
+ }
+},
+{
+ "name": "Multilayer-HySEA model",
+ "url": "https://drive.google.com/file/d/1DlIoWs1vmyzZz9H_5iPp4O78k2Wezg1t/view?usp=sharing",
+ "metadata": {
+ "author": "Jorge Macías"
+ }
+}
+
]
diff --git a/frontend/createStatic.py b/frontend/createStatic.py
index b3f390ed668049bb3fd6fa4e7bd8ac7f600f01db..452156130608dff1e9dd6471d35977c14d5de0b5 100644
--- a/frontend/createStatic.py
+++ b/frontend/createStatic.py
@@ -1,8 +1,10 @@
from jinja2 import Environment, FileSystemLoader
-import os
-import shutil
+import os, argparse, shutil
-def render_template_to_site():
+API_URL_ENV_VARNAME = "DATACATALOG_API_URL"
+API_URL_DEFAULT_VALUE = "http://localhost:8000/"
+
+def render_template_to_site(api_url=API_URL_DEFAULT_VALUE):
## copy javascript files to site folder
src_files = os.listdir('frontend/js')
dest = 'site/js'
@@ -23,6 +25,16 @@ def render_template_to_site():
if os.path.isfile(full_name):
shutil.copy(full_name, dest)
+ ## replace {{API_URL}} tag with actual api url from env
+ apicalls_file_path = 'site/js/apicalls.js'
+ api_tag = '{{API_URL}}'
+ with open(apicalls_file_path, 'r') as file:
+ filedata = file.read()
+ filedata = filedata.replace(api_tag, api_url)
+ with open(apicalls_file_path, 'w') as file:
+ file.write(filedata)
+
+
## render templates to site folder
file_loader = FileSystemLoader('frontend/templates')
env = Environment(loader=file_loader)
@@ -52,4 +64,17 @@ def render_template_to_site():
f.write(html[file])
if __name__ == "__main__":
- render_template_to_site()
\ No newline at end of file
+ # priority for setting the API URL from most to least important: commandline argument >>> environment variable >>> default value
+
+ # if env variable is set, get api url from it, else use default
+ api_url = os.getenv(API_URL_ENV_VARNAME, API_URL_DEFAULT_VALUE)
+ # if argument is there, set api url
+ parser = argparse.ArgumentParser("createStatic.py", description="Generates the static files for the web frontend to the site/ folder.")
+ parser.add_argument("-u", "--api-url", help="The url for the backend API. This must include protocol (usually http or https). Defaults to {} or the environment variable {} (if set).".format(API_URL_DEFAULT_VALUE, API_URL_ENV_VARNAME))
+ args = parser.parse_args()
+ if args.api_url:
+ api_url = args.api_url
+
+ print(api_url)
+
+ render_template_to_site(api_url)
\ No newline at end of file
diff --git a/frontend/js/apicalls.js b/frontend/js/apicalls.js
index 4ac79d0608499a261ce96397b806f748525ed8d2..57872fc8bff43364a7296354a62422828de7b7b9 100644
--- a/frontend/js/apicalls.js
+++ b/frontend/js/apicalls.js
@@ -1,5 +1,5 @@
// This file contains the api calls, as well as transform the data into html-text
-var apiUrl = "http://zam024.fritz.box/api/"; // TODO switch out with real url, ideally during deployment
+var apiUrl = "{{API_URL}}";
var allowedTypesList = [];
// get data from url query variables
@@ -30,10 +30,18 @@ function setTypeText() {
// return the dataset id
function getId() {
var id = getUrlVars()["oid"];
- console.log("ID: " + id);
return id;
}
+function escapeHtml(unsafe) {
+ return unsafe
+ .replace(/&/g, "&")
+ .replace(/</g, "<")
+ .replace(/>/g, ">")
+ .replace(/"/g, """)
+ .replace(/'/g, "'");
+ }
+
// get option-html string from typename suffix
function getTypeHTMLString(name) {
return '<li><a class="dropdown-item" href="?type=' + name + '">' + name + '</a></li>';
@@ -41,7 +49,8 @@ function getTypeHTMLString(name) {
// get tableentry-html for a dataset
function getDatasetHTMLString(dataset) {
- return '<tr><th scope="row">'+ dataset[0] + '</th><td><a href="?type=' + getType() + "&oid=" + dataset[1] + '">' + dataset[1] + '</a></td></tr>'
+ var safename = escapeHtml(dataset[0]);
+ return '<tr><th scope="row">'+ safename + '</th><td><a href="?type=' + getType() + "&oid=" + dataset[1] + '">' + dataset[1] + '</a></td></tr>'
}
/*
@@ -50,7 +59,9 @@ The value field is editable, but the edit is blocked by default
authenticated users should be able to edit and submit
*/
function getPropertyHTMLString(property, value, readonly=true) {
- return '<tr><th scope="row">' + property + '</th><td colspan="2"><input class="form-control" type="text" id="' + property + 'Input" value="' + value + (readonly ? '" readonly' : '"') + '></td></tr>';
+ var safekey = escapeHtml(property);
+ var safeval = escapeHtml(value);
+ return '<tr><th scope="row">' + safekey + '</th><td colspan="2"><input class="form-control" type="text" id="' + safekey + 'Input" value="' + safeval + (readonly ? '" readonly' : '"') + '></td></tr>';
}
/*
@@ -71,7 +82,9 @@ authenticated users should be able to edit and submit
*/
function getMetadataPropertyHTMLString(property, value, readonly=true) {
var randID = getRandomID();
- return '<tr id="' + randID + 'Row"><th scope="row"><input class="form-control dynamic-metadata" type="text" id="' + randID + '" value="' + property + (readonly ? '" readonly' : '"') + '></th><td><input class="form-control" type="text" id="' + randID + 'Input" value="' + value + (readonly ? '" readonly' : '"') + '></td><th><button type="button" class="btn btn-danger dynamic-metadata-button" onclick="removeMetadataRow(\'' + randID + '\')" id="' + randID + 'Button">-</button></th></tr>';
+ var safekey = escapeHtml(property);
+ var safeval = escapeHtml(value);
+ return '<tr id="' + randID + 'Row"><th scope="row"><input class="form-control dynamic-metadata" type="text" id="' + randID + '" value="' + safekey + (readonly ? '" readonly' : '"') + '></th><td><input class="form-control" type="text" id="' + randID + 'Input" value="' + safeval + (readonly ? '" readonly' : '"') + '></td><th><button type="button" class="btn btn-danger dynamic-metadata-button" onclick="removeMetadataRow(\'' + randID + '\')" id="' + randID + 'Button">-</button></th></tr>';
}
/**
@@ -117,14 +130,14 @@ function fillDatasetTable(table, dataset, readonly=true, id=getId()) {
// XMLHttpRequest EVENTLISTENER: if a dropdown-menu (a <ul> element) with the dropdownOptions id is present, update it with the available types
function setTypeData() {
- console.log("GET " + this.responseUrl + ": " + this.responseText);
+ console.log("Response to list available types GET: " + this.responseText);
var types = JSON.parse(this.responseText);
allowedTypesList = [];
// types is now a list of {name : url} elements, where url starts with a slash, and is relative to the root
var keyName = "";
types.forEach(element => {
keyName = Object.keys(element)[0];
- console.log("Detected location type: " + keyName);
+ console.debug("Detected location type: " + keyName);
allowedTypesList.push(keyName);
$('#dropdownOptions').append(getTypeHTMLString(keyName));
});
@@ -132,18 +145,24 @@ function setTypeData() {
// XMLHttpRequest EVENTLISTENER: append to the list of datasets
function setDatasetList() {
- console.log("GET " + this.responseUrl + ": " + this.responseText);
+ console.log("Response to list datasets GET: " + this.responseText);
var datasets = JSON.parse(this.responseText);
datasets.forEach(element => {
- console.log("Found Dataset: " + element)
+ console.debug("Found Dataset: " + element)
$('#datasetTableBody').append(getDatasetHTMLString(element));
});
}
// XMLHttpRequest EVENTLISTENER: show banner with new dataset id
function showNewDatasetID() {
- console.log("POST " + this.responseUrl + ": " + this.responseText);
- // TODO http status check
+ console.log("Response to create new Dataset POST: " + this.responseText);
+ if (this.status >= 400) {
+ // some error occured while getting the data
+ // show an alert and don't do anything else
+ var alertHTML = '<div class="alert alert-danger" role="alert">Invalid response from server! Either the API server is down, or the dataset creation failed. Response code: ' + this.status + '<hr>Please try agagin later, and if the error persists, contact the server administrator.</div>';
+ $('#storageTypeChooser').after(alertHTML);
+ return;
+ }
var data = JSON.parse(this.responseText);
var id = data[0];
var alertHTML = '<div class="alert alert-success" role="alert">Dataset created! OID is: <a href="?type=' + getType() + '&oid=' + id + '">' + id + '</a></div>';
@@ -153,8 +172,14 @@ function showNewDatasetID() {
// XMLHttpRequest EVENTLISTENER: show banner with success message for change
function showSuccessfullyChangedDataset() {
- console.log("PUT " + this.responseUrl + ": " + this.responseText);
- // TODO http status check
+ console.log("Response to modify dataset PUT: " + this.responseText);
+ if (this.status >= 400) {
+ // some error occured while getting the data
+ // show an alert and don't do anything else
+ var alertHTML = '<div class="alert alert-danger" role="alert">Invalid response from server! Either the API server is down, or the dataset modification failed. Response code: ' + this.status + '<hr>Please try agagin later, and if the error persists, contact the server administrator.</div>';
+ $('#storageTypeChooser').after(alertHTML);
+ return;
+ }
var alertHTML = '<div class="alert alert-success" role="alert">Dataset was successfully changed!</div>';
$('#storageTypeChooser').after(alertHTML);
$('#spinner').remove();
@@ -162,19 +187,27 @@ function showSuccessfullyChangedDataset() {
// XMLHttpRequest EVENTLISTENER: show banner with success message for deletion
function showSuccessfullyDeletedDataset() {
- console.log("DELETE " + this.responseUrl + ": " + this.responseText);
- // TODO http status check
+ console.log("Response to DELETE dataset: " + this.responseText);
+ if (this.status >= 400) {
+ // some error occured while getting the data
+ // show an alert and don't do anything else
+ var alertHTML = '<div class="alert alert-danger" role="alert">Invalid response from server! Either the API server is down, or the dataset deletion failed. Response code: ' + this.status + '<hr>Please try agagin later, and if the error persists, contact the server administrator.</div>';
+ $('#storageTypeChooser').after(alertHTML);
+ return;
+ }
var alertHTML = '<div class="alert alert-danger" role="alert">Dataset was successfully deleted!</div>';
$('#storageTypeChooser').after(alertHTML);
$('#spinner').remove();
}
// XMLHttpRequest EVENTLISTENER: show dataset in table
-function setDatasetView() {
- console.log("GET " + this.responseUrl + ": " + this.responseText);
+async function setDatasetView() {
+ console.log("Response to show dataset GET: " + this.responseText);
var dataset = JSON.parse(this.responseText);
if (this.status >= 300) {
- alert(getId() + " does not exists for this storage type!");
+ var alertHTML = '<div class="alert alert-danger" role="alert">Invalid id was requested. Redirecting to list of elements with the same type.<div class="spinner-border" role="status"><span class="sr-only">Loading...</span></div></div>';
+ $('#storageTypeChooser').before(alertHTML);
+ await new Promise(resolve => setTimeout(resolve, 3000));
window.location.href = "?type=" + getType();
return;
}
@@ -203,7 +236,7 @@ function getTypes() {
// get listing of datasets of the given type, put them in the list element (via listener)
function listDatasets(datatype) {
var fullUrl = apiUrl + datatype;
- console.log("Full url for listing request is " + fullUrl)
+ console.log("Sending GET request to " + fullUrl + " for listing datasets.")
var xmlhttp = new XMLHttpRequest();
xmlhttp.addEventListener("loadend", setDatasetList);
xmlhttp.open("GET", fullUrl);
@@ -213,7 +246,7 @@ function listDatasets(datatype) {
// get details about given dataset, put them in the view element (via listener)
function showDataset(datatype, dataset_id) {
var fullUrl = apiUrl + datatype + "/" + dataset_id;
- console.log("Full url for showing request is " + fullUrl)
+ console.log("Sending GET request to " + fullUrl + " for showing dataset.")
var xmlhttp = new XMLHttpRequest();
xmlhttp.addEventListener("loadend", setDatasetView);
xmlhttp.open("GET", fullUrl);
@@ -225,8 +258,13 @@ async function showListingOrSingleDataset() {
while (allowedTypesList.length == 0) {
await new Promise(resolve => setTimeout(resolve, 10));
}
- if (!getType() ||!allowedTypesList.includes(getType())) {
- // no type or invalid type: reload page with first allowed type TODO add some alert?
+ if (!getType() || !allowedTypesList.includes(getType())) {
+ if (getType) {
+ // an invalid type was provided, give some alert
+ var alertHTML = '<div class="alert alert-danger" role="alert">Invalid type was requested. Redirecting to default type.<div class="spinner-border" role="status"><span class="sr-only">Loading...</span></div></div>';
+ $('#storageTypeChooser').before(alertHTML);
+ await new Promise(resolve => setTimeout(resolve, 3000));
+ }
window.location.href = "?type=" + allowedTypesList[0];
}
if (!getId()) { // no id given, so list all elements
@@ -259,8 +297,8 @@ async function showListingOrSingleDataset() {
function createNewDataset(datatype, name, url, metadata) {
var dataset = {"name" : name, "url" : url, "metadata" : metadata};
var fullUrl = apiUrl + datatype;
- console.log("Full url for creating new dataset is " + fullUrl)
- console.log("New Dataset is " + dataset)
+ console.log("Sending POST request to " + fullUrl + " for creating dataset.")
+ console.debug("New Dataset is " + dataset)
var xmlhttp = new XMLHttpRequest();
xmlhttp.addEventListener("loadend", showNewDatasetID);
xmlhttp.open("POST", fullUrl);
@@ -276,8 +314,8 @@ function createNewDataset(datatype, name, url, metadata) {
function updateDataset(oid, datatype, name, url, metadata) {
var dataset = {"name" : name, "url" : url, "metadata" : metadata};
var fullUrl = apiUrl + datatype + "/" + oid;
- console.log("Full url for editing dataset is " + fullUrl)
- console.log("New Dataset data is " + dataset)
+ console.log("Sending PUT request to " + fullUrl + " for editing dataset.")
+ console.debug("New Dataset data is " + dataset)
var xmlhttp = new XMLHttpRequest();
xmlhttp.addEventListener("loadend", showSuccessfullyChangedDataset);
xmlhttp.open("PUT", fullUrl);
@@ -292,7 +330,7 @@ function updateDataset(oid, datatype, name, url, metadata) {
// DELETE existing Dataset (get bearer token from session storage)
function deleteDataset(oid, datatype) {
var fullUrl = apiUrl + datatype + "/" + oid;
- console.log("Full url for deleting dataset is " + fullUrl)
+ console.log("Sending DELETE request to " + fullUrl + " for deleting dataset.")
var xmlhttp = new XMLHttpRequest();
xmlhttp.addEventListener("loadend", showSuccessfullyDeletedDataset);
xmlhttp.open("DELETE", fullUrl);
diff --git a/frontend/js/auth.js b/frontend/js/auth.js
index 11d03ca14e1b92e8f1ba5bb0441f7539f2c86ee9..40b3b7b3f2de45f74c74333680462f6f36f9a9f8 100644
--- a/frontend/js/auth.js
+++ b/frontend/js/auth.js
@@ -1,14 +1,12 @@
// This file will contain functions to manage authentication (including the token storage and access)
-// TODO function to rewrite Login as username
-
/************************************************
* Event Listeners for XMLHttpRequests
************************************************/
// XMLHttpRequest EVENTLISTENER: if the call was successful, store the token locally and reload login page
function setLoginToken() {
- console.log("POST " + this.responseUrl + ": " + this.responseText);
+ console.log("Response to login POST: " + this.responseText);
if (this.status >= 400) {
alert("The username and/ or the password is invalid!");
logout();
@@ -22,12 +20,12 @@ function setLoginToken() {
// To be called by an inline XMLHttpRequest EVENTLISTENER: if the call was successful, update the userdata
function setUserdata(data, updateView) {
- console.log("GET " + data.responseUrl + ": " + data.responseText);
- if (this.status >= 400) {
- logout();
+ console.log("Response to auth verification GET: " + data.responseText + " with status " + data.status);
+ if (data.status >= 400) {
+ console.log("Auth verification returned a " + data.status + " status. Reattempt login.");
+ relog();
} else {
var userdata = JSON.parse(data.responseText);
- console.log("Userdata: " + userdata.username + " - " + userdata.email);
// store username and email in sessionData (blind overwrite if exists)
window.sessionStorage.username = userdata.username;
window.sessionStorage.email = userdata.email;
@@ -43,7 +41,7 @@ function setUserdata(data, updateView) {
*/
function loginPOST(username, password) {
var fullUrl = apiUrl + "token";
- console.log("Full url for token request is " + fullUrl)
+ console.log("Sending POST request to " + fullUrl + " for login.")
var xmlhttp = new XMLHttpRequest();
xmlhttp.addEventListener("loadend", setLoginToken);
xmlhttp.open("POST", fullUrl);
@@ -51,6 +49,14 @@ function loginPOST(username, password) {
xmlhttp.send("username=" + encodeURIComponent(username) + "&password=" + encodeURIComponent(password));
}
+/**
+ * logs out, and then redirects to the login page.
+ */
+function relog() {
+ logout();
+ window.location.href = "/login.html?relog=true";
+}
+
/**
* checks the textfields for username and password, gives an error message if not given, then calls the loginPOST(username, password) function
*/
@@ -88,7 +94,7 @@ function getInfo(updateView=false) {
// start GET /me, pass wether an update of the user labels is needed
var fullUrl = apiUrl + "me";
- console.log("Full url for /me request is " + fullUrl)
+ console.log("Sending GET request to " + fullUrl + " for verifying authentication.")
var xmlhttp = new XMLHttpRequest();
xmlhttp.open("GET", fullUrl);
xmlhttp.setRequestHeader('Authorization', 'Bearer ' + window.sessionStorage.auth_token);
diff --git a/frontend/templates/login_content.html.jinja b/frontend/templates/login_content.html.jinja
index 19dde7a3787a4a6b7df57342693f3d51a2c46af9..59628d7b430e71d082b0faef5a6d6fa0c2fb7c37 100644
--- a/frontend/templates/login_content.html.jinja
+++ b/frontend/templates/login_content.html.jinja
@@ -9,6 +9,14 @@
<div class="container">
<div class="row">
<div class="mt-5 col-sm-12 mb-5">
+ <div class="alert alert-warning alert-dismissible fade show" role="alert" id="authTokenExpiredAlert">
+ Your stored authentication token expired, or is otherwise invalid.
+ <hr>
+ Please try to log in again.
+ <button type="button" class="close" data-dismiss="alert" aria-label="Close">
+ <span aria-hidden="true">×</span>
+ </button>
+ </div>
<div id="loginForm">
<!-- Tabs Titles -->
@@ -68,5 +76,10 @@ $("form").on('submit',function(e){
</script>
<script>
showElementsDependingOnLoginStatus(getInfo(true));
+if (getUrlVars()["relog"] == "true") {
+ $('#authTokenExpiredAlert').show();
+} else {
+ $('#authTokenExpiredAlert').hide();
+}
</script>
{% endblock %}
\ No newline at end of file
diff --git a/tests/apiserver_tests/test_apiwithauth.py b/tests/apiserver_tests/test_apiwithauth.py
index 73ff1acb2d3ff8f0f4175a8a64166443cd4f5b4a..0d51267a17c481701fb8046c87830cc6f8f7a9df 100644
--- a/tests/apiserver_tests/test_apiwithauth.py
+++ b/tests/apiserver_tests/test_apiwithauth.py
@@ -5,6 +5,9 @@ from context import apiserver, storage
from unittest import TestCase
from apiserver.security.user import User
+# a properly formatted uuidv4 to ensure the proper errors are returned by the api; the exact value is irrelevant
+proper_uuid = "3a33262e-276e-4de8-87bc-f2d5a0195faf"
+
def myfunc():
return User(username='foo', email='bar')
@@ -32,7 +35,7 @@ class UserTests(TestCase):
def test_create(self):
my_data = {
- 'name': 'some dataset',
+ 'name': 'some datase1t',
'url': 'http://loc.me/1',
'metadata': {'key': 'value'}
}
@@ -47,11 +50,11 @@ class UserTests(TestCase):
def test_delete(self):
- rsp = self.client.delete("/dataset/foo")
+ rsp = self.client.delete(f"/dataset/{proper_uuid}")
self.assertEqual(rsp.status_code, 404, 'deleted called on non-existing')
rsp = self.client.post('/dataset', json={
- 'name': 'some dataset',
+ 'name': 'some dataset2',
'url': 'http://loc.me/1'}
)
self.assertEqual(rsp.status_code, 200)
@@ -60,6 +63,10 @@ class UserTests(TestCase):
rsp = self.client.delete(f"/dataset/{oid}")
self.assertEqual(rsp.status_code, 200)
+ def test_delete_invalid_uuid(self):
+ rsp = self.client.delete("/dataset/invalid-uuid")
+ self.assertEqual(rsp.status_code, 422, 'deleted called on invalid uuid')
+
def test_create_and_get(self):
@@ -82,7 +89,7 @@ class UserTests(TestCase):
def test_create_and_delete(self):
lst = self.client.get('/dataset').json()
- self.assertEqual(len(lst), 0)
+ self.assertEqual(len(lst), 0, f"{lst}")
self.client.post('/dataset', json={
'name': 'new_obj',
@@ -129,3 +136,14 @@ class UserTests(TestCase):
self.client.delete(f"/dataset/{oid}")
+ def test_update_invalid_uuid(self):
+ oid = "invalid_uuid"
+ rsp = self.client.put(f"/dataset/{oid}", json={
+ 'name': 'new_name',
+ 'url': 'new_url',
+ 'metadata': {
+ 'key': 'value'
+ }
+ }
+ )
+ self.assertEqual(rsp.status_code, 422)
diff --git a/tests/apiserver_tests/test_responsiveness.py b/tests/apiserver_tests/test_responsiveness.py
index 7a65933ffba5f6fec4cf977e36c4557468c69a68..b8e62e9ead99ba65a16277e2ff62c5c6d15830d8 100644
--- a/tests/apiserver_tests/test_responsiveness.py
+++ b/tests/apiserver_tests/test_responsiveness.py
@@ -4,6 +4,9 @@ from context import apiserver, storage
import unittest
+# a properly formatted uuidv4 to ensure the proper errors are returned by the api; the exact value is irrelevant
+proper_uuid = "3a33262e-276e-4de8-87bc-f2d5a0195faf"
+
class NonAuthTests(unittest.TestCase):
def setUp(self):
#TODO: we should do better here (cleanup or use some testing dir)
@@ -35,12 +38,18 @@ class NonAuthTests(unittest.TestCase):
def test_token(self):
rsp = self.client.post('/token', data={'username': 'foo', 'password': 'bar'})
- self.assertEqual(rsp.status_code, 401, 'Ath')
+ self.assertEqual(rsp.status_code, 401, 'Auth required')
def test_get_non_existing(self):
- rsp = self.client.get('/dataset/foo')
+ rsp = self.client.get(f'/dataset/{proper_uuid}')
self.assertEqual(404, rsp.status_code)
j = rsp.json()
self.assertTrue('message' in j, f"{j} should contain message")
- self.assertFalse('foo' in j['message'], f"error message should contain object id (foo)")
+ self.assertFalse('foo' in j['message'], f"error message should not contain object id (foo)")
+
+ def test_get_invalid_oid(self):
+ rsp = self.client.get('/dataset/invalid-uuid')
+ self.assertEqual(422, rsp.status_code)
+ j = rsp.json()
+ self.assertTrue('detail' in j, f"{j} should contain message")
diff --git a/tests/storage_tests/test_jsonbackend.py b/tests/storage_tests/test_jsonbackend.py
index 2abc405e5e8109b927388e88458cb510c5917e37..52dc17c6cbbbcb55a8212cb9ab7b585bc95c3202 100644
--- a/tests/storage_tests/test_jsonbackend.py
+++ b/tests/storage_tests/test_jsonbackend.py
@@ -1,6 +1,6 @@
import unittest
-from apiserver.storage.JsonFileStorageAdapter import JsonFileStorageAdapter, StoredData
+from apiserver.storage.JsonFileStorageAdapter import JsonFileStorageAdapter, StoredData, verify_oid, get_unique_id
from apiserver.storage import LocationDataType, LocationData
from collections import namedtuple
import os
@@ -100,4 +100,10 @@ class SomeTests(unittest.TestCase):
print(details)
self.assertIsNone(details)
-
+ def test_oid_veirfication(self):
+ oid = get_unique_id(path='/tmp/')
+ self.assertTrue(verify_oid(oid=oid))
+ self.assertTrue(verify_oid(oid=oid.replace('5', '7')))
+ self.assertFalse(verify_oid(oid='random strawberry'))
+ self.assertFalse(verify_oid(oid=None))
+ self.assertFalse(verify_oid(oid=1))