Commit 7e48c0dd authored by Mehdi Cherti's avatar Mehdi Cherti 💬
Browse files

add evaluation_script

parent b5413f40
from .main import evaluate
import random
import numpy as np
import pandas as pd
from sklearn.metrics import accuracy_score
def evaluate(test_annotation_file, user_submission_file, phase_codename, **kwargs):
print("Starting Evaluation.....")
"""
Evaluates the submission for a particular challenge phase and returns score
Arguments:
`test_annotations_file`: Path to test_annotation_file on the server
`user_submission_file`: Path to file submitted by the user
`phase_codename`: Phase to which submission is made
`**kwargs`: keyword arguments that contains additional submission
metadata that challenge hosts can use to send slack notification.
You can access the submission metadata
with kwargs['submission_metadata']
Example: A sample submission metadata can be accessed like this:
>>> print(kwargs['submission_metadata'])
{
'status': u'running',
'when_made_public': None,
'participant_team': 5,
'input_file': 'https://abc.xyz/path/to/submission/file.json',
'execution_time': u'123',
'publication_url': u'ABC',
'challenge_phase': 1,
'created_by': u'ABC',
'stdout_file': 'https://abc.xyz/path/to/stdout/file.json',
'method_name': u'Test',
'stderr_file': 'https://abc.xyz/path/to/stderr/file.json',
'participant_team_name': u'Test Team',
'project_url': u'http://foo.bar',
'method_description': u'ABC',
'is_public': False,
'submission_result_file': 'https://abc.xyz/path/result/file.json',
'id': 123,
'submitted_at': u'2017-03-20T19:22:03.880652Z'
}
"""
output = {}
target = read_annotation(test_annotation_file)
pred = read_annotation(user_submission_file)
split = phase_codename
print(f"Evaluating for {phase_codename} Phase")
output["result"] = [
{
split: {
# Please add your metrics here
"accuracy": accuracy_score(target, pred),
}
}
]
# To display the results in the result file
output["submission_result"] = output["result"][0][split]
print(f"Completed evaluation for {phase_codename} Phase")
return output
def read_annotation(path):
df = pd.read_csv(path)
output_cols = ["image", "label"]
assert set(output_cols).issubset(set(df.columns))
df = df[output_cols]
return df
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment