From e79fe47bf32283b04eccb56d003339e1f42792ba Mon Sep 17 00:00:00 2001 From: Conrad S Date: Mon, 27 Feb 2017 17:15:43 -0500 Subject: [PATCH 01/15] Refactor mech turk to be restful --- app/controllers/turk_hit.py | 202 +++++++++++++++++-- app/forms/turk_hit.py | 13 ++ app/lib/mech_turk.py | 111 ++++++++++ app/models/base.py | 83 -------- app/models/turk_hit.py | 391 +++--------------------------------- app/views/turk_hit.py | 8 +- 6 files changed, 352 insertions(+), 456 deletions(-) create mode 100644 app/lib/mech_turk.py diff --git a/app/controllers/turk_hit.py b/app/controllers/turk_hit.py index fbd830c..bd96212 100644 --- a/app/controllers/turk_hit.py +++ b/app/controllers/turk_hit.py @@ -1,33 +1,166 @@ """Controllers for posting or manipulating a turk_hit.""" -from ..models.turk_hit import TurkHit -from ..forms.turk_hit import TurkHitForm from .base import RestController +from ..lib.database import proc +from ..lib.mech_turk import create_hit, approve_or_reject_hit,\ + get_hit_status +from ..lib.service import services +from ..models.turk_hit import TurkHit +from ..forms.turk_hit import TurkHitForm, TurkHitAcceptForm + +import base64 +import json +import requests +import mimetypes + +from boto.mturk.connection import MTurkConnection +from boto.mturk.question import QuestionContent, Question, QuestionForm,\ + Overview, AnswerSpecification, FileUploadAnswer +from flask import current_app from werkzeug.datastructures import MultiDict -from werkzeug.exceptions import BadRequest +from werkzeug.exceptions import NotFound, BadRequest class TurkHitController(RestController): """A turk_hit controller.""" Model = TurkHit + STATUS_DICT = { + 1: 'Assignable', + 2: 'Unassignable', + 3: 'Reviewable', + 4: 'Accepted', + 5: 'Rejected', + 6: 'Disposed', + 7: 'Expired' + } + + # Given an status id, return the string text associated with that id + def get_status_id_from_text(self, status_text): + for dict_status_id, dict_status_text in self.STATUS_DICT.items(): + if dict_status_text == status_text: + return dict_status_id + return None + + def get_form(self, filter_data): """Return the turk_hit form.""" return TurkHitForm + def get_accept_form(self, filter_data): + """Return the turk_hit form.""" + return TurkHitAcceptForm + def get(self, id_, filter_data): """ - Retrieves the status of a turk_hit given the building_id + Retrieves a list of turk_hits given the building_id :param id_: building_id of building - :param filter_data: no usage currently - :return: Turk HIT status and URL used to download completed HIT + :param filter_data: the address to be used to download a file to box + :return: A list of turk hit models associated with this building_id + """ + address = filter_data.get('address') + if not address: + raise BadRequest("Please ensure there is an address in the url params") + try: + hit_list = proc(self.Model, self.Model.PROCS['READ'], **{'building_id': id_}) +# a_hid = '3RTFSSG7T82VF4CB3XNGYCY2RJ5WLC' +# stat_id = 1 +# bid = 3 +# model = self.Model(1, bid, a_hid, stat_id, None, None, None, None) +# hit_list = [model] + except Exception as err: + raise ( + err if current_app.config['DEBUG'] else + BadRequest('Error while executing db call') + ) + + if not hit_list: + raise NotFound + + cur_hit = hit_list[0] + + # No need to check in with amazon if we've already reached one of those states + status_string = self.STATUS_DICT[cur_hit.status_id] + if status_string in ["Accepted", "Rejected", "Disposed", "Expired"]: + return cur_hit + + # Get the new hit status from AWS + result = get_hit_status(cur_hit.amazon_hit_id) + new_hit_status = result['hit_status'] + + # No changes if the status is the same + if new_hit_status == status_string: + return cur_hit + + + csv_document_key = None + # Download file here + if result['file_url'] and not cur_hit.csv_document_key: + response = requests.get(result['file_url']) + uploaded_fname = response.headers['Content-Disposition'].split('"')[1] + content_type = mimetypes.guess_type(uploaded_fname)[0] + # Make sure the content_type is in .xlsx format + if content_type != 'application/vnd.openxmlformats-' \ + 'officedocument.spreadsheetml.sheet': + # TODO: Automatically Reject the HIT if incorrect file format + pass + + file_name = "BuildingDimensions--{}--{}.xlsx".format(cur_hit.building_id, + cur_hit.amazon_hit_id) + folder_path = "/Buildings/{}_{}/Building_Dimensions".format(cur_hit.building_id, + address) + # Download the file + document = self.download_file(folder_path, file_name, response.content) + # Get the key to update the model + csv_document_key = document['key'] + + # Update the entry in the database + new_hit_status_id = self.get_status_id_from_text(new_hit_status) + try: + proc(self.Model, + self.Model.PROCS['UPDATE'], + **{'id': cur_hit.db_id, + 'status_id': new_hit_status_id, + 'csv_document_key': csv_document_key}) + except Exception as err: + raise ( + err if current_app.config['DEBUG'] else + BadRequest('Error while executing db call') + ) + cur_hit.status_id = new_hit_status_id + cur_hit.csv_document_key = csv_document_key + return cur_hit + + def download_file(self, folder_path, file_name, byte_data): """ + A function to download a file to box + """ + encoded_byte_data = base64.b64encode(byte_data) + encoded_string_data = encoded_byte_data.decode('utf-8') + + post_data = { + "path": folder_path, + "data": "data:csv/plain;charset=utf-8;base64,{}".format(encoded_string_data), + "building_id": str(cur_hit.building_id), + "tags": '', + "name": file_name + } + # Call the generic method in the base class + response = services.document.post('', '/document/', data=json.dumps(post_data)) + if response.status_code != 201: + raise ( + BadRequest(response.json()) if current_app.config['DEBUG'] else + BadRequest("Unable to download document to document service") + ) + + document = response.json()['data'] + return document - return self.Model.get_hit_status(id_) def post(self, data, filter_data): """ - Validate post data and call function in model + Validate post data, create hit with amazon mech turk, + and save in database :param data: Args from Body :param filter_data: Args from POST URL @@ -38,7 +171,30 @@ class TurkHitController(RestController): if not form.validate(): raise BadRequest(form.errors) - return TurkHit(**data).send_hit() + requester_name = data.pop('requester_name') + building_id = data.pop('building_id') + + # Create the hit in amazon + amazon_hit_id = create_hit(**data) + + # Store the hit data in the database + try: + hit_list = proc(self.Model, self.Model.PROCS['WRITE'], + **{'building_id': building_id, + 'amazon_hit_id': amazon_hit_id, + 'requester_name': requester_name}) + except Exception as err: + raise ( + err if current_app.config['DEBUG'] else + BadRequest('Error while executing db call') + ) + + if not hit_list: + raise NotFound("Hit was not found in the database") + + return hit_list[0] + + def put(self, id_, data, filter_data): """ @@ -49,8 +205,28 @@ class TurkHitController(RestController): :return: List of assignment IDs approved or rejected """ # TODO add form validation + form = self.get_accept_form(filter_data)(formdata=MultiDict(data)) + if not form.validate(): + raise BadRequest(form.errors) + if int(id_) != data.pop("db_id"): + raise BadRequest("The URL id and the body db_id are not the same") - if data["accept"]: - return self.Model.accept_hit(id_) - - return self.Model.reject_hit(id_) + # Approve or reject the hit on the amazon mech turk side + if approve_or_reject_hit(**data): + # Update the hit data in the database + try: + approve_id = self.get_status_id_from_text('Accepted') + reject_id = self.get_status_id_from_text('Rejected') + status_id = approve_id if data["approve"] else reject_id + hit_list = proc(self.Model, self.Model.PROCS['UPDATE'], + **{'id': id_, + 'status_id': status_id, + 'response_message': data["response_message"]}) + except Exception as err: + raise ( + err if current_app.config['DEBUG'] else + BadRequest('Error while executing db call') + ) + return hit_list[0] + else: + raise BadRequest("There were no completed assignments to approve or reject") diff --git a/app/forms/turk_hit.py b/app/forms/turk_hit.py index c2548cf..bef4f90 100644 --- a/app/forms/turk_hit.py +++ b/app/forms/turk_hit.py @@ -6,6 +6,8 @@ class TurkHitForm(wtf.Form): building_id = wtf.IntegerField( validators=[wtf.validators.Required()]) + requester_name = wtf.StringField( + validators=[wtf.validators.Required()]) min_file_bytes = wtf.IntegerField( validators=[wtf.validators.Required()]) @@ -33,3 +35,14 @@ class TurkHitForm(wtf.Form): validators=[wtf.validators.Required()]) reward = wtf.FloatField( validators=[wtf.validators.Required()]) + +class TurkHitAcceptForm(wtf.Form): + """ A form for validating turk hits.""" + db_id = wtf.IntegerField( + validators=[wtf.validators.Required()]) + amazon_hit_id = wtf.StringField( + validators=[wtf.validators.Required()]) + approve = wtf.IntegerField( + validators=[wtf.validators.AnyOf([0, 1])]) + response_message = wtf.StringField( + validators=[wtf.validators.Optional()]) diff --git a/app/lib/mech_turk.py b/app/lib/mech_turk.py new file mode 100644 index 0000000..3a956fc --- /dev/null +++ b/app/lib/mech_turk.py @@ -0,0 +1,111 @@ +from flask import current_app +from boto.mturk.connection import MTurkConnection +from boto.mturk.question import QuestionContent, Question, QuestionForm,\ + Overview, AnswerSpecification, FileUploadAnswer + +# A conveniance instance for interaction with the mech turk API. All access to +# mech turk should be through this file +def get_mturk_connection(): + AWS_ACCESS_KEY = current_app.config.get('AWS_ACCESS_KEY') + AWS_SECRET_ACCESS_KEY = current_app.config.get('AWS_SECRET_ACCESS_KEY') + HOST = current_app.config.get('MECH_TURK_HOST') + if not AWS_ACCESS_KEY or not AWS_SECRET_ACCESS_KEY or not HOST: + raise ValueError("AWS keys or HOST are empty in the config file") + + return MTurkConnection( + aws_access_key_id=AWS_ACCESS_KEY, + aws_secret_access_key=AWS_SECRET_ACCESS_KEY, + host=HOST + ) + +def create_hit(min_file_bytes, max_file_bytes, + address, max_assignments, instructions_url, + instructions_text, worksheet_url, title, + description, keywords, duration, reward): + """ + Send a mechanical turk hit + + :return: amazon_hit_id + """ + mturk_connection = get_mturk_connection() + + # Overview + overview = Overview() + overview.append_field('Title', address) + + # Instructions + qc1 = QuestionContent() + qc1.append_field('Title', 'Instructions') + qc1.append_field('Text', instructions_text) + qc1.append_field('Text', instructions_url) + qc1.append_field('Title', 'Worksheet') + qc1.append_field('Text', worksheet_url) + + file_upload = FileUploadAnswer( + min_file_bytes, max_file_bytes) + + question = Question(identifier='measure_building', + content=qc1, + answer_spec=AnswerSpecification(file_upload), + is_required=True) + + # Question Form + question_form = QuestionForm() + question_form.append(overview) + question_form.append(question) + + # Creates a new HIT + result_set = mturk_connection.create_hit( + questions=question_form, + max_assignments=max_assignments, + title=title, + description=description, + keywords=keywords, + duration=duration, + reward=reward) + + amazon_hit_id = getattr(result_set[0], 'HITId') + return amazon_hit_id + +def get_hit_status(amazon_hit_id): + + mturk_connection = get_mturk_connection() + + hit = mturk_connection.get_hit(amazon_hit_id) + hit_status = getattr(hit[0], 'HITStatus') + + file_url = None + # Only call mturk to grab file_url if hit status is in a state + # where there possibly could be a file_url + if hit_status in ["Reviewable", "Accepted", "Rejected"]: + completed_assignments = mturk_connection.get_assignments(amazon_hit_id) + if completed_assignments: + assignment = completed_assignments[0] + file_url_object = mturk_connection.get_file_upload_url( + assignment.AssignmentId, "measure_building") + file_url = file_url_object[0].FileUploadURL + return {'hit_status': hit_status, 'file_url': file_url} + + +def approve_or_reject_hit(amazon_hit_id, approve, response_message): + """ + Given an amazon_hit_id, accept or reject the completed assignment + + :param building_id: The building_id of the building + :return: The assignmentID that was rejected and the status + :throws: A MTurkRequestError if AWS credentials are wrong, + the hit does not exist, or if the assignment is not reviewable + """ + mturk_connection = get_mturk_connection() + + assignment_id = "" + completed_assignments = mturk_connection.get_assignments(amazon_hit_id) + if completed_assignments: + assignment = completed_assignments[0] + if approve: + mturk_connection.approve_assignment(assignment.AssignmentId) + else: + # TODO: Add response_message to the reject_assignment + mturk_connection.reject_assignment(assignment.AssignmentId, feedback=response_message) + return True + return False diff --git a/app/models/base.py b/app/models/base.py index 7b5db71..886367d 100644 --- a/app/models/base.py +++ b/app/models/base.py @@ -63,89 +63,6 @@ class Model(BaseModel): id = db.Column(db.Integer, primary_key=True) - @staticmethod - def run_proc(method, schema, columns=None, limit=None, offset=None, **kwargs): - """ - Run stored proc - Args: - method (str) - Method name for stored procedure - columns (list) - DB Columns - kwargs - containing arguments for stored proc - Returns: - the results of the query, to be handled outside this function - """ - cols = '*' if not columns else ','.join(str(i) for i in columns) - - params = "" - for key, value in kwargs.items(): - params += "in_{} := '{}', ".format(key, value) - params = params[:-2] # remove last comma and space - - query = "select {} from {}.{}({})".format(cols, schema, method, params) - if limit: - query += ' limit {}'.format(limit) - if offset: - query += ' offset {}'.format(offset) - try: - results = db.session.execute(query) - db.session.commit() - except Exception as err: - raise BadRequest('Something went wrong' - ' in the database: {}'.format(str(err))) - return results - - @staticmethod - def download_to_documentservice(byte_data, building_id, address, - file_name, folder_name, tag=""): - """ - Converts inputted byte_data into proper format - and downloads using document service - - :param byte_data: The data to download in byte format - :param building_id: The building id of the file - :param file_name: The file name to download - :param folder_name: The name of the folder within a building - :param tag: Tag to add to the document - :return: the response from document service - """ - encoded_byte_data = base64.b64encode(byte_data) - encoded_string_data = encoded_byte_data.decode('utf-8') - - folder_path = "/Buildings/{}_{}/{}".format(building_id, - address, - folder_name) - post_data = { - "path": folder_path, - "data": "data:csv/plain;charset=utf-8;base64,{}".format(encoded_string_data), - "building_id": building_id, - "tags": tag, - "name": file_name - } - # Call the generic method in the base class - return services.document.post('', '/document/', data=json.dumps(post_data)) - - @staticmethod - def search_documents(path): - """ - Get a list of documents that have the inputed - building_id and are in the inputed path - - :param building_id: building_id of building - :param path: The folder path in box - :return: A list of documents that are in this path and building id - """ - url = '/document/?paths[]={}'.format(path) - response = services.document.get(url) - if response.status_code == 200: - document_list = response.json()['data'] - else: - raise InternalServerError('Failed to get documents from document' - 'service: {} {}'.format( - response.status_code, - response.reason) - ) - return document_list - class Tracked(object): """A mixin to include tracking datetime fields.""" diff --git a/app/models/turk_hit.py b/app/models/turk_hit.py index cfd5dac..472316f 100644 --- a/app/models/turk_hit.py +++ b/app/models/turk_hit.py @@ -1,371 +1,46 @@ """Models for dealing with turk hit.""" -import requests -import mimetypes -from ..lib.database import db -from .base import Model -from boto.mturk.connection import MTurkConnection -from boto.mturk.question import QuestionContent, Question, QuestionForm,\ - Overview, AnswerSpecification, FileUploadAnswer +from ..lib.database import proc, ProcColumn, ProcTable +from .base import BaseModel from werkzeug.exceptions import InternalServerError -from flask import current_app -from datetime import datetime -from werkzeug.exceptions import NotFound -import json -class TurkHit(Model): +class TurkHit(BaseModel): + __table_args__ = {"schema": "mechanical_turk"} - STATUS_DICT = { - 1: 'Assignable', - 2: 'Unassignable', - 3: 'Reviewable', - 4: 'Accepted', - 5: 'Rejected', - 6: 'Disposed', - 7: 'Expired' + PROCS = { + 'READ': 'get_hits', + 'WRITE': 'create_hit', + 'UPDATE': 'update_hit', } - HIT_STATUS_ASSIGNABLE = 1 - def __init__(self, - building_id, - min_file_bytes, max_file_bytes, - address, max_assignments, instructions_url, - instructions_text, worksheet_url, title, - description, keywords, duration, reward + + __table__ = ProcTable( + 'TurkHit', + ProcColumn('db_id'), + ProcColumn('building_id'), + ProcColumn('amazon_hit_id'), + ProcColumn('status_id'), + ProcColumn('hit_date'), + ProcColumn('requester_name'), + ProcColumn('csv_document_key'), + ProcColumn('shapefile_document_key'), + ) + + def __init__(self, db_id, building_id, amazon_hit_id, status_id, + hit_date, requester_name, csv_document_key, + shapefile_document_key ): - # The building_id associated with this building + self.db_id = db_id self.building_id = building_id - - # The min and max size of the upload file - self.min_file_bytes = min_file_bytes - self.max_file_bytes = max_file_bytes - - # Used to create the mechanical turk QuestionForm - self.address = address - self.instructions_url = instructions_url - self.instructions_text = instructions_text - self.worksheet_url = worksheet_url - - # Used for the rest of the mechanical turk hit creation parameters - self.max_assignments = max_assignments - self.title = title - self.description = description - self.keywords = keywords - self.duration = duration - self.reward = reward + self.amazon_hit_id = amazon_hit_id + self.status_id = status_id + self.hit_date = hit_date + self.requester_name = requester_name + self.csv_document_key = csv_document_key + self.shapefile_document_key = shapefile_document_key def __str__(self): - return "Turk Call for Building: {}".format(self.building_id) - - @staticmethod - def get_mturk_connection(): - AWS_ACCESS_KEY = current_app.config.get('AWS_ACCESS_KEY') - AWS_SECRET_ACCESS_KEY = current_app.config.get('AWS_SECRET_ACCESS_KEY') - HOST = current_app.config.get('MECH_TURK_HOST') - if not AWS_ACCESS_KEY or not AWS_SECRET_ACCESS_KEY or not HOST: - raise ValueError("AWS keys or HOST are empty in the config file") - - return MTurkConnection( - aws_access_key_id=AWS_ACCESS_KEY, - aws_secret_access_key=AWS_SECRET_ACCESS_KEY, - host=HOST - ) - - def send_hit(self): - """ - Send a mechanical turk hit - - :return: A dict containing the HITId - """ - mturk_connection = TurkHit.get_mturk_connection() - - # Overview - overview = Overview() - overview.append_field('Title', self.address) - - # Instructions - qc1 = QuestionContent() - qc1.append_field('Title', 'Instructions') - qc1.append_field('Text', self.instructions_text) - qc1.append_field('Text', self.instructions_url) - qc1.append_field('Title', 'Worksheet') - qc1.append_field('Text', self.worksheet_url) - - file_upload = FileUploadAnswer( - self.min_file_bytes, self.max_file_bytes) - - question = Question(identifier='measure_building', - content=qc1, - answer_spec=AnswerSpecification(file_upload), - is_required=True) - - # Question Form - question_form = QuestionForm() - question_form.append(overview) - question_form.append(question) - - # Creates a new HIT - result_set = mturk_connection.create_hit( - questions=question_form, - max_assignments=self.max_assignments, - title=self.title, - description=self.description, - keywords=self.keywords, - duration=self.duration, - reward=self.reward) - - # Store hit_id in database - hit_id = getattr(result_set[0], 'HITId') - Model.run_proc("create_hit", 'mechanical_turk', **{ - "building_id": self.building_id, - "amazon_hit_id": hit_id, - "status_id": self.HIT_STATUS_ASSIGNABLE - }) - - return {**self.__dict__, **{'hit_id': hit_id}, - **{'status': 'Assignable'}} - - @staticmethod - def get_hit_status(building_id): - """ - Retrieves the status of a turk_hit given the building_id - Also retrieves all of the buildin dimension files in box - that are associated with this building_id - If the hit is Reviewable for the first time, download the file - - :param building_id: building_id of building - :return: Turk HIT status and URL to download completed HIT - Turk HIT status can be: - Assignable (hit has been created but no worker has started on it) - Unassignable (someone is working on the hit) - Reviewable (hit expired) - Accepted (hit accepted) - Rejected - Diposed - - """ - # Get the address associated with this building_id - results = Model.run_proc("get_building", 'public', - columns=['address'], - **{"building_id": building_id}) - data = results.fetchone() - if not data: - raise NotFound('The inputted buildingId was' - ' not found in the database') - address = data[0] - # The path in box where we will search for documents to return - box_path = "/Buildings/{}_{}/Building_Dimensions".format(building_id, - address) - - stored_hit_status = TurkHit.get_stored_hit_status(building_id) - - # If we know the stored_hit_status is accept or reject we return here - if stored_hit_status == "Accepted" or stored_hit_status == "Rejected": - document_list = Model.search_documents(box_path) - return {'status': stored_hit_status, - 'box_building_list': document_list} - - mturk_connection = TurkHit.get_mturk_connection() - # Retrieve the HITId from database using the building_id - hit_id = TurkHit.get_hid_from_bid(building_id) - - hit = mturk_connection.get_hit(hit_id) - new_hit_status = getattr(hit[0], 'HITStatus') - - file_url = "" - # For now we are indexing into the first assignment because we - # don't expect to see multiple assignments per hit - completed_assignments = mturk_connection.get_assignments(hit_id) - if completed_assignments: - assignment = completed_assignments[0] - file_url_object = mturk_connection.get_file_upload_url( - assignment.AssignmentId, "measure_building") - file_url = file_url_object[0].FileUploadURL - - # Hit has expired if there are no URLs and it's in reviewable state - if new_hit_status == "Reviewable" and not file_url: - new_hit_status = "Expired" - - - box_url = '' - box_download_url = '' - # If we are newly in a reviewable state we can download the document - if (new_hit_status == "Reviewable" and - stored_hit_status != "Reviewable"): - for i in range(5): - box_response = TurkHit.download_hit_file(building_id, - address, - hit_id, - file_url) - if box_response.status_code == 201: - break - print("Trying to connect to box... " + str(i)) - # If the document fails to download to box, return an error - if box_response.status_code != 201: - raise InternalServerError('Failed to download hit to' - ' box. Try reloading.') - - if new_hit_status != stored_hit_status: - if not TurkHit.update_stored_hit_status(building_id, - new_hit_status): - raise InternalServerError('Failed to update hit in' - ' the backend database.') - - document_list = Model.search_documents(box_path) - return {'status': new_hit_status, - 'box_building_list': document_list} - - @staticmethod - def download_hit_file(building_id, address, hit_id, file_url): - """ - Given a building_id, download the mechanical turk - hit document using the document service. - - Also checks to make sure the file is .xlsx format - - :param building_id: The building_id of the building - :return: The URL of the file downloaded - """ - - response = requests.get(file_url) - - # The Content-Type value just returns binary/octet, so we - # need to guess the actually content_type from the file name - # Content-Disposation returns a string in the format: - # Content-Disposition: attachment; filename="test.xlsx" - uploaded_fname = response.headers['Content-Disposition'].split('"')[1] - content_type = mimetypes.guess_type(uploaded_fname)[0] - # Make sure the content_type is in .xlsx format - if content_type != 'application/vnd.openxmlformats-' \ - 'officedocument.spreadsheetml.sheet': - # TODO: Automatically Reject the HIT if incorrect file format - pass - - byte_data = response.content - file_name = "BuildingDimensions--{}--{}.xlsx".format(building_id, - hit_id) - box_response = Model.download_to_documentservice(byte_data, - building_id, - address, - file_name, - "Building_Dimensions") - return box_response - - @staticmethod - def accept_hit(building_id): - """ - Approve all of the assignments of the hit associated - with the inputted building_id - - :param building_id: The building_id of the building - :return: The assignment ID that was accepted and status - :throws: MTurkRequestError if the hit was not found or - is not in the correct state to be approved (Reviewable) - """ - mturk_connection = TurkHit.get_mturk_connection() - - hit_id = TurkHit.get_hid_from_bid(building_id) - - assignment_id = "" - completed_assignments = mturk_connection.get_assignments(hit_id) - if completed_assignments: - assignment = completed_assignments[0] - # assignment.AssignmentStatus returns the status of the assignment - mturk_connection.approve_assignment(assignment.AssignmentId) - assignment_id = assignment.AssignmentId - # From the AWS docs: - # A successful request for the ApproveAssignment operation - # returns with no errors. - - # Update status in database to reflect approved status - TurkHit.update_stored_hit_status(building_id, "Accepted") - return {"assignment_id": assignment_id, 'hit_status': 'Accepted'} - - @staticmethod - def reject_hit(building_id): - """ - Given a building_id, reject the completed assignment - - :param building_id: The building_id of the building - :return: The assignmentID that was rejected and the status - :throws: A MTurkRequestError if AWS credentials are wrong, - the hit does not exist, or if the assignment is not reviewable - """ - mturk_connection = TurkHit.get_mturk_connection() - - hit_id = TurkHit.get_hid_from_bid(building_id) - - assignment_id = "" - completed_assignments = mturk_connection.get_assignments(hit_id) - if completed_assignments: - assignment = completed_assignments[0] - mturk_connection.reject_assignment(assignment.AssignmentId) - assignment_id = assignment.AssignmentId - # Update status in database to reflect rejected status - TurkHit.update_stored_hit_status(building_id, "Rejected") - # TODO: Remvoe document from box - return {"assignment_id": assignment_id, 'hit_status': 'Rejected'} - - @staticmethod - def get_hid_from_bid(building_id): - """ - Get the hit ID associated with inputted building ID - - :param building_id: The building ID - :return: The HIT ID or None if such a hitID does not exist - """ - results = Model.run_proc("get_hit_id", "mechanical_turk", - **{"building_id": building_id}) - # Return the first value returned by the query - data = results.fetchone() - if data: - return data[0] - return None - - @staticmethod - def get_stored_hit_status(building_id): - """ - Get the HIT status that is stored in the database - - :param building_id: The building ID - :return: the HIT status - """ - results = Model.run_proc("get_hit_status", "mechanical_turk", - **{"building_id": building_id}) - - status = "" - - # Get the first value returned by the query - data = results.fetchone() - if not data: - return status - - status_id = data[0] - - if TurkHit.STATUS_DICT.get(status_id): - status = TurkHit.STATUS_DICT[status_id] - - return status - - @staticmethod - def update_stored_hit_status(building_id, hit_status): - """ - Update the HIT status associated with the building_id - - :param building_id: The building ID - :param hit_status: The integer corresponding to the hit status - :return: the updated HIT status - """ - hit_status_id = None - for dict_status_id, dict_status in TurkHit.STATUS_DICT.items(): - if dict_status == hit_status: - hit_status_id = dict_status_id - if not hit_status_id: - return None + return "Turk Hit for building {} with hit id {}".format(self.building_id, + self.amazon_hit_id) - results = Model.run_proc("update_hit_status", - "mechanical_turk", - **{"building_id": building_id, - "status_id": hit_status_id}) - return results diff --git a/app/views/turk_hit.py b/app/views/turk_hit.py index b5edaa6..82a2339 100644 --- a/app/views/turk_hit.py +++ b/app/views/turk_hit.py @@ -21,14 +21,18 @@ class TurkHitView(UnprotectedRestView): def get(self, id_): """/{id} GET - Retrieve the hit status by building id.""" try: - response = self.get_controller().get(id_, None) + response = self.get_controller().get(id_, request.args) # Catch an MTurkRequestError (usually AWS) and return error code except MTurkRequestError as error: # If the HITId does not exist return 404 if error.error_code == 'AWS.MechanicalTurk.HITDoesNotExist': raise NotFound(error.error_code) raise BadGateway(error.error_code) - return self.json(response) + return self.json({ + 'data': [ + self.parse(response) + ] + }) @app_need def post(self): -- GitLab From 60becd9a04ca8cb95f4505a8b30f38ca022bcc98 Mon Sep 17 00:00:00 2001 From: Conrad S Date: Mon, 27 Feb 2017 17:33:24 -0500 Subject: [PATCH 02/15] Update comments to reflect new styling --- app/controllers/turk_hit.py | 60 ++++++++++++++++++++++++++----------- 1 file changed, 43 insertions(+), 17 deletions(-) diff --git a/app/controllers/turk_hit.py b/app/controllers/turk_hit.py index bd96212..7029e73 100644 --- a/app/controllers/turk_hit.py +++ b/app/controllers/turk_hit.py @@ -52,17 +52,22 @@ class TurkHitController(RestController): def get(self, id_, filter_data): """ - Retrieves a list of turk_hits given the building_id - - :param id_: building_id of building - :param filter_data: the address to be used to download a file to box - :return: A list of turk hit models associated with this building_id + Retrieve a list of TurkHit objects + + Args: + id_: The building_id + filter_data (ImmutableMultiDict): + address (string): The address of the building for saving to box + Returns: + list: + dict: TurkHit object """ address = filter_data.get('address') if not address: raise BadRequest("Please ensure there is an address in the url params") try: hit_list = proc(self.Model, self.Model.PROCS['READ'], **{'building_id': id_}) +# FOR TESTING PURPOSES # a_hid = '3RTFSSG7T82VF4CB3XNGYCY2RJ5WLC' # stat_id = 1 # bid = 3 @@ -159,12 +164,27 @@ class TurkHitController(RestController): def post(self, data, filter_data): """ - Validate post data, create hit with amazon mech turk, - and save in database - - :param data: Args from Body - :param filter_data: Args from POST URL - :return: Dict containing all POST data + Create a TurkHit object + + Args: + data (ImmutableMultiDict): Args for stored proc and mech turk + building_id (int): Building id for this hit + requester_name (string: Name of the user who requested this hit + # The rest are inputs for the mech turk hit + address (string) + max_file_bytes (int) + min_file_bytes (int) + instructions_text (string) + instructions_url (string) + worksheet_url (string) + max_assignments (int) + title (string) + description (string) + keywords (string) + duration (int) + reward (string) + Returns: + dict: TurkHit object """ form = self.get_form(filter_data)(formdata=MultiDict(data)) @@ -198,13 +218,19 @@ class TurkHitController(RestController): def put(self, id_, data, filter_data): """ - Validate put data and call corresponding function in model - - :param data: Args from Body - :param filter_data: Args from POST URL - :return: List of assignment IDs approved or rejected + Update a TurkHit object + + Args: + id_ (str): db_id of a hit + filter_data (ImmutableMultiDict): Args for stored proc + db_id (int): Should be same as id_ + amazon_hit_id (string): Hit id for this hit + approve (int): 0 for reject, 1 for approve + response_message (string): Message required on reject + Returns: + dict: TurkHit object """ - # TODO add form validation + # TODO: Update form validation to require a message form = self.get_accept_form(filter_data)(formdata=MultiDict(data)) if not form.validate(): raise BadRequest(form.errors) -- GitLab From 047d3668a52cb69881035ad67e8e364e786e068c Mon Sep 17 00:00:00 2001 From: Conrad S Date: Tue, 28 Feb 2017 10:22:11 -0500 Subject: [PATCH 03/15] Add rejection if the file is incorrect file format --- app/controllers/turk_hit.py | 49 ++++++++++++++++++++----------------- app/views/turk_hit.py | 6 ++--- 2 files changed, 30 insertions(+), 25 deletions(-) diff --git a/app/controllers/turk_hit.py b/app/controllers/turk_hit.py index 7029e73..ecbd455 100644 --- a/app/controllers/turk_hit.py +++ b/app/controllers/turk_hit.py @@ -24,7 +24,7 @@ class TurkHitController(RestController): """A turk_hit controller.""" Model = TurkHit - STATUS_DICT = { + STATUS_DICT_ID = { 1: 'Assignable', 2: 'Unassignable', 3: 'Reviewable', @@ -34,13 +34,8 @@ class TurkHitController(RestController): 7: 'Expired' } - # Given an status id, return the string text associated with that id - def get_status_id_from_text(self, status_text): - for dict_status_id, dict_status_text in self.STATUS_DICT.items(): - if dict_status_text == status_text: - return dict_status_id - return None - + # The above dictionary with the keys as values and the values as keys + STATUS_DICT_TEXT = {v: k for k, v in STATUS_DICT_ID.items()} def get_form(self, filter_data): """Return the turk_hit form.""" @@ -107,20 +102,30 @@ class TurkHitController(RestController): # Make sure the content_type is in .xlsx format if content_type != 'application/vnd.openxmlformats-' \ 'officedocument.spreadsheetml.sheet': - # TODO: Automatically Reject the HIT if incorrect file format - pass - - file_name = "BuildingDimensions--{}--{}.xlsx".format(cur_hit.building_id, - cur_hit.amazon_hit_id) - folder_path = "/Buildings/{}_{}/Building_Dimensions".format(cur_hit.building_id, - address) - # Download the file - document = self.download_file(folder_path, file_name, response.content) - # Get the key to update the model - csv_document_key = document['key'] + # Automatically Reject the HIT if incorrect file format + response_message = 'This hit was automatically rejected because '\ + 'the uploaded file was not in the correct file '\ + 'format. The file must be of type .xlsx.' + put_body = {'db_id': cur_hit.db_id, + 'amazon_hit_id': cur_hit.amazon_hit_id, + 'approve': 0, + 'response_message': response_message} + self.put(cur_hit.db_id, put_body, None) + cur_hit.response_message = response_message + new_hit_status = "Rejected" + + else: + file_name = "BuildingDimensions--{}--{}.xlsx".format(cur_hit.building_id, + cur_hit.amazon_hit_id) + folder_path = "/Buildings/{}_{}/Building_Dimensions".format(cur_hit.building_id, + address) + # Download the file + document = self.download_file(folder_path, file_name, response.content) + # Get the key to update the model + csv_document_key = document['key'] # Update the entry in the database - new_hit_status_id = self.get_status_id_from_text(new_hit_status) + new_hit_status_id = self.STATUS_DICT_TEXT[new_hit_status] try: proc(self.Model, self.Model.PROCS['UPDATE'], @@ -241,8 +246,8 @@ class TurkHitController(RestController): if approve_or_reject_hit(**data): # Update the hit data in the database try: - approve_id = self.get_status_id_from_text('Accepted') - reject_id = self.get_status_id_from_text('Rejected') + approve_id = self.STATUS_DICT_STRING['Accepted'] + reject_id = self.STATUS_DICT_STRING['Rejected'] status_id = approve_id if data["approve"] else reject_id hit_list = proc(self.Model, self.Model.PROCS['UPDATE'], **{'id': id_, diff --git a/app/views/turk_hit.py b/app/views/turk_hit.py index 82a2339..a90f517 100644 --- a/app/views/turk_hit.py +++ b/app/views/turk_hit.py @@ -17,7 +17,7 @@ class TurkHitView(UnprotectedRestView): def index(self): raise MethodNotAllowed() - @app_need + #@app_need def get(self, id_): """/{id} GET - Retrieve the hit status by building id.""" try: @@ -34,7 +34,7 @@ class TurkHitView(UnprotectedRestView): ] }) - @app_need + #@app_need def post(self): """/ POST - Create a hit given an id in the POST body""" try: @@ -46,7 +46,7 @@ class TurkHitView(UnprotectedRestView): return self.json(response, 201) - @app_need + #@app_need def put(self, id_): """ / PUT - Approve or decline a hit given a building_id. -- GitLab From 1d3d71cb6192448ac42b980e579f3d2e6ba7eae3 Mon Sep 17 00:00:00 2001 From: Conrad S Date: Tue, 28 Feb 2017 14:06:24 -0500 Subject: [PATCH 04/15] Add logic to ensure that response_message is nonempty when rejected --- app/forms/turk_hit.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/app/forms/turk_hit.py b/app/forms/turk_hit.py index bef4f90..d638531 100644 --- a/app/forms/turk_hit.py +++ b/app/forms/turk_hit.py @@ -46,3 +46,13 @@ class TurkHitAcceptForm(wtf.Form): validators=[wtf.validators.AnyOf([0, 1])]) response_message = wtf.StringField( validators=[wtf.validators.Optional()]) + + def validate(self): + if not wtf.Form.validate(self): + return False + # If the hit will be rejected, ensure there is a message + if not self.approve.data and not self.response_message.data: + self.response_message.errors.append('If accept is 0 explanation must be nonempty') + return False + return True + -- GitLab From 44cca4d05ff2ee6b87c01d2c89319259ba7e15da Mon Sep 17 00:00:00 2001 From: Conrad S Date: Thu, 2 Mar 2017 10:35:53 -0500 Subject: [PATCH 05/15] Add database procs to service layer --- app/controllers/turk_hit.py | 65 ++++++++++++++++++++----------------- app/forms/turk_hit.py | 17 ++++++++-- app/lib/database.py | 5 ++- app/lib/mech_turk.py | 3 ++ app/models/turk_hit.py | 10 +++--- app/views/turk_hit.py | 6 ++-- 6 files changed, 67 insertions(+), 39 deletions(-) diff --git a/app/controllers/turk_hit.py b/app/controllers/turk_hit.py index 130bb61..1c7cf22 100644 --- a/app/controllers/turk_hit.py +++ b/app/controllers/turk_hit.py @@ -61,13 +61,8 @@ class TurkHitController(RestController): if not address: raise BadRequest("Please ensure there is an address in the url params") try: - #hit_list = proc(self.Model, self.Model.PROCS['READ'], **{'building_id': id_}) -# FOR TESTING PURPOSES - a_hid = None - stat_id = 1 - bid = 3 - model = self.Model(1, bid, a_hid, stat_id, None, None, None, None) - hit_list = [model] + hit_list = proc(self.Model, self.Model.PROCS['READ'], **{'building_id': id_}) + except Exception as err: raise ( err if current_app.config['DEBUG'] else @@ -77,12 +72,13 @@ class TurkHitController(RestController): if not hit_list: raise NotFound + print(hit_list[0]) cur_hit = hit_list[0] # No need to check in with amazon if we've already reached one of those states status_string = self.STATUS_DICT_ID[cur_hit.status_id] if status_string in ["Accepted", "Rejected", "Disposed", "Expired"]: - return cur_hit + return hit_list # Get the new hit status from AWS result = get_hit_status(cur_hit.amazon_hit_id) @@ -90,9 +86,7 @@ class TurkHitController(RestController): # No changes if the status is the same if new_hit_status == status_string: - return cur_hit - - csv_document_key = None + return hit_list # Download file here if result['file_url'] and not cur_hit.csv_document_key: response = requests.get(result['file_url']) @@ -105,10 +99,8 @@ class TurkHitController(RestController): response_message = 'This hit was automatically rejected because '\ 'the uploaded file was not in the correct file '\ 'format. The file must be of type .xlsx.' - put_body = {'db_id': cur_hit.db_id, - 'amazon_hit_id': cur_hit.amazon_hit_id, - 'approve': 0, - 'response_message': response_message} + put_body = {**cur_hit.get_dictionary(), + **{'approve': 0, 'response_message': response_message}} self.put(cur_hit.db_id, put_body, None) cur_hit.response_message = response_message new_hit_status = "Rejected" @@ -122,29 +114,28 @@ class TurkHitController(RestController): document = self.download_file(cur_hit.building_id, folder_path, file_name, response.content) # Get the key to update the model - csv_document_key = document['key'] + cur_hit.csv_document_key = document['key'] # Update the entry in the database new_hit_status_id = self.STATUS_DICT_TEXT[new_hit_status] + cur_hit.status_id = new_hit_status_id try: proc(self.Model, self.Model.PROCS['UPDATE'], - **{'id': cur_hit.db_id, - 'status_id': new_hit_status_id, - 'csv_document_key': csv_document_key}) + **cur_hit.get_dictionary()) + pass except Exception as err: raise ( err if current_app.config['DEBUG'] else BadRequest('Error while executing db call') ) - cur_hit.status_id = new_hit_status_id - cur_hit.csv_document_key = csv_document_key - return cur_hit + return hit_list def download_file(self, building_id, folder_path, file_name, byte_data): """ A function to download a file to box """ + print("Downloading file") encoded_byte_data = base64.b64encode(byte_data) encoded_string_data = encoded_byte_data.decode('utf-8') @@ -207,7 +198,12 @@ class TurkHitController(RestController): hit_list = proc(self.Model, self.Model.PROCS['WRITE'], **{'building_id': building_id, 'amazon_hit_id': amazon_hit_id, - 'requester_name': requester_name}) + 'requester_name': requester_name, + 'status_id': self.STATUS_DICT_TEXT['Assignable']}) +# hit_list = [{'amazon_hit_id': amazon_hit_id, 'building_id': building_id, +# 'db_id': 123, 'status_id': 1, 'csv_document_key': None, +# 'shapefile_document_key': None, 'requester_name': 'no_name', +# 'response_message': None, 'hit_date': None}] except Exception as err: raise ( err if current_app.config['DEBUG'] else @@ -239,20 +235,31 @@ class TurkHitController(RestController): form = self.get_accept_form(filter_data)(formdata=MultiDict(data)) if not form.validate(): raise BadRequest(form.errors) - if int(id_) != data.pop("db_id"): + if int(id_) != data["db_id"]: raise BadRequest("The URL id and the body db_id are not the same") + approve = data.pop("approve") + # Approve or reject the hit on the amazon mech turk side - if approve_or_reject_hit(**data): + if approve_or_reject_hit(data["amazon_hit_id"], + approve, + data["response_message"]): # Update the hit data in the database try: approve_id = self.STATUS_DICT_TEXT['Accepted'] reject_id = self.STATUS_DICT_TEXT['Rejected'] - status_id = approve_id if data["approve"] else reject_id + status_id = approve_id if approve else reject_id + + # The data + data['status_id'] = status_id + print('------------------') + print(data) + print('------------------') + hit_list = proc(self.Model, self.Model.PROCS['UPDATE'], - **{'id': id_, - 'status_id': status_id, - 'response_message': data["response_message"]}) + **data) + #hit_list = [data] + except Exception as err: raise ( err if current_app.config['DEBUG'] else diff --git a/app/forms/turk_hit.py b/app/forms/turk_hit.py index d638531..5d80838 100644 --- a/app/forms/turk_hit.py +++ b/app/forms/turk_hit.py @@ -40,13 +40,26 @@ class TurkHitAcceptForm(wtf.Form): """ A form for validating turk hits.""" db_id = wtf.IntegerField( validators=[wtf.validators.Required()]) + building_id = wtf.IntegerField( + validators=[wtf.validators.Required()]) amazon_hit_id = wtf.StringField( validators=[wtf.validators.Required()]) - approve = wtf.IntegerField( - validators=[wtf.validators.AnyOf([0, 1])]) + status_id = wtf.IntegerField( + validators=[wtf.validators.Required()]) + hit_date = wtf.StringField( + validators=[wtf.validators.Optional()]) + requester_name = wtf.StringField( + validators=[wtf.validators.Optional()]) + csv_document_key = wtf.StringField( + validators=[wtf.validators.Optional()]) + shapefile_document_key = wtf.StringField( + validators=[wtf.validators.Optional()]) response_message = wtf.StringField( validators=[wtf.validators.Optional()]) + approve = wtf.IntegerField( + validators=[wtf.validators.AnyOf([0, 1])]) + def validate(self): if not wtf.Form.validate(self): return False diff --git a/app/lib/database.py b/app/lib/database.py index 23bea8b..808c41c 100644 --- a/app/lib/database.py +++ b/app/lib/database.py @@ -64,7 +64,10 @@ def proc(model, method, limit=None, offset=None, **kwargs): params = "" cols = ','.join(str(i) for i in model.__table__.get_columns()) for key, value in kwargs.items(): - params += "in_{} := '{}', ".format(key, value) + if value is not None: + params += "in_{} := '{}', ".format(key, value) + else: + params += "in_{} := null, ".format(key) params = params[:-2] # remove last comma and space query = "select {} from {}.{}({})".format( diff --git a/app/lib/mech_turk.py b/app/lib/mech_turk.py index 3a956fc..4c8c028 100644 --- a/app/lib/mech_turk.py +++ b/app/lib/mech_turk.py @@ -96,6 +96,9 @@ def approve_or_reject_hit(amazon_hit_id, approve, response_message): :throws: A MTurkRequestError if AWS credentials are wrong, the hit does not exist, or if the assignment is not reviewable """ + print("In approve or reject") + if True: + return True mturk_connection = get_mturk_connection() assignment_id = "" diff --git a/app/models/turk_hit.py b/app/models/turk_hit.py index 472316f..6683156 100644 --- a/app/models/turk_hit.py +++ b/app/models/turk_hit.py @@ -8,7 +8,7 @@ class TurkHit(BaseModel): __table_args__ = {"schema": "mechanical_turk"} PROCS = { - 'READ': 'get_hits', + 'READ': 'get_hitinfo', 'WRITE': 'create_hit', 'UPDATE': 'update_hit', } @@ -24,11 +24,12 @@ class TurkHit(BaseModel): ProcColumn('requester_name'), ProcColumn('csv_document_key'), ProcColumn('shapefile_document_key'), + ProcColumn('response_message'), ) def __init__(self, db_id, building_id, amazon_hit_id, status_id, hit_date, requester_name, csv_document_key, - shapefile_document_key + shapefile_document_key, response_message ): self.db_id = db_id @@ -39,8 +40,9 @@ class TurkHit(BaseModel): self.requester_name = requester_name self.csv_document_key = csv_document_key self.shapefile_document_key = shapefile_document_key + self.response_message = response_message def __str__(self): - return "Turk Hit for building {} with hit id {}".format(self.building_id, - self.amazon_hit_id) + return "Turk Hit for building {} with dict: {}".format(self.building_id, + self.get_dictionary()) diff --git a/app/views/turk_hit.py b/app/views/turk_hit.py index 45ee4da..0d4985f 100644 --- a/app/views/turk_hit.py +++ b/app/views/turk_hit.py @@ -28,7 +28,7 @@ class TurkHitView(RestView): raise BadGateway(error.error_code) return self.json({ 'data': [ - self.parse(response) + self.parse(m) for m in response ] }) @@ -41,7 +41,7 @@ class TurkHitView(RestView): except MTurkRequestError as error: raise BadGateway(error.error_code) - return self.json(response, 201) + return self.json(self.parse(response), 201) def put(self, id_): """ @@ -60,7 +60,7 @@ class TurkHitView(RestView): raise BadRequest(error.error_code) raise BadGateway(error.error_code) - return self.json(response) + return self.json(self.parse(response)) def delete(self, id_): raise MethodNotAllowed() -- GitLab From 74818b787051f2e9762c7fb575e596095da759fb Mon Sep 17 00:00:00 2001 From: Conrad S Date: Thu, 2 Mar 2017 10:50:04 -0500 Subject: [PATCH 06/15] Remove print statements --- app/controllers/turk_hit.py | 5 ----- app/lib/mech_turk.py | 1 - 2 files changed, 6 deletions(-) diff --git a/app/controllers/turk_hit.py b/app/controllers/turk_hit.py index 1c7cf22..cd65fc9 100644 --- a/app/controllers/turk_hit.py +++ b/app/controllers/turk_hit.py @@ -72,7 +72,6 @@ class TurkHitController(RestController): if not hit_list: raise NotFound - print(hit_list[0]) cur_hit = hit_list[0] # No need to check in with amazon if we've already reached one of those states @@ -135,7 +134,6 @@ class TurkHitController(RestController): """ A function to download a file to box """ - print("Downloading file") encoded_byte_data = base64.b64encode(byte_data) encoded_string_data = encoded_byte_data.decode('utf-8') @@ -252,9 +250,6 @@ class TurkHitController(RestController): # The data data['status_id'] = status_id - print('------------------') - print(data) - print('------------------') hit_list = proc(self.Model, self.Model.PROCS['UPDATE'], **data) diff --git a/app/lib/mech_turk.py b/app/lib/mech_turk.py index 4c8c028..4591557 100644 --- a/app/lib/mech_turk.py +++ b/app/lib/mech_turk.py @@ -96,7 +96,6 @@ def approve_or_reject_hit(amazon_hit_id, approve, response_message): :throws: A MTurkRequestError if AWS credentials are wrong, the hit does not exist, or if the assignment is not reviewable """ - print("In approve or reject") if True: return True mturk_connection = get_mturk_connection() -- GitLab From a27b2ae8f39bf96eb7388f0fd206b64857b278e7 Mon Sep 17 00:00:00 2001 From: Conrad S Date: Thu, 2 Mar 2017 11:22:17 -0500 Subject: [PATCH 07/15] Add data key to dcitionary return --- app/views/turk_hit.py | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/app/views/turk_hit.py b/app/views/turk_hit.py index 0d4985f..ce50912 100644 --- a/app/views/turk_hit.py +++ b/app/views/turk_hit.py @@ -41,8 +41,11 @@ class TurkHitView(RestView): except MTurkRequestError as error: raise BadGateway(error.error_code) - return self.json(self.parse(response), 201) - + return self.json({ + 'data': [ + self.parse(response), 201) + ] + }) def put(self, id_): """ / PUT - Approve or decline a hit given a building_id. @@ -60,7 +63,11 @@ class TurkHitView(RestView): raise BadRequest(error.error_code) raise BadGateway(error.error_code) - return self.json(self.parse(response)) + return self.json({ + 'data': [ + self.parse(response)) + ] + }) def delete(self, id_): raise MethodNotAllowed() -- GitLab From 349713537db2982168f83921838add7e8e5790f0 Mon Sep 17 00:00:00 2001 From: Conrad S Date: Thu, 2 Mar 2017 11:25:05 -0500 Subject: [PATCH 08/15] Fix syntax error --- app/views/turk_hit.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/app/views/turk_hit.py b/app/views/turk_hit.py index ce50912..7e6883d 100644 --- a/app/views/turk_hit.py +++ b/app/views/turk_hit.py @@ -43,9 +43,9 @@ class TurkHitView(RestView): return self.json({ 'data': [ - self.parse(response), 201) + self.parse(response) ] - }) + }, 201) def put(self, id_): """ / PUT - Approve or decline a hit given a building_id. @@ -65,7 +65,7 @@ class TurkHitView(RestView): return self.json({ 'data': [ - self.parse(response)) + self.parse(response) ] }) -- GitLab From d96376d188b3aaa200d06fb72d25a93cd694704b Mon Sep 17 00:00:00 2001 From: Conrad S Date: Thu, 2 Mar 2017 11:44:19 -0500 Subject: [PATCH 09/15] Change order of imports --- app/controllers/turk_hit.py | 19 +++++++++---------- app/views/turk_hit.py | 9 +++------ 2 files changed, 12 insertions(+), 16 deletions(-) diff --git a/app/controllers/turk_hit.py b/app/controllers/turk_hit.py index cd65fc9..a53a6a0 100644 --- a/app/controllers/turk_hit.py +++ b/app/controllers/turk_hit.py @@ -1,4 +1,13 @@ """Controllers for posting or manipulating a turk_hit.""" +import base64 +import json +import requests +import mimetypes + +from flask import current_app +from werkzeug.datastructures import MultiDict +from werkzeug.exceptions import NotFound, BadRequest + from .base import RestController from ..lib.database import proc from ..lib.mech_turk import create_hit, approve_or_reject_hit,\ @@ -7,17 +16,7 @@ from ..lib.service import services from ..models.turk_hit import TurkHit from ..forms.turk_hit import TurkHitForm, TurkHitAcceptForm -import base64 -import json -import requests -import mimetypes -from boto.mturk.connection import MTurkConnection -from boto.mturk.question import QuestionContent, Question, QuestionForm,\ - Overview, AnswerSpecification, FileUploadAnswer -from flask import current_app -from werkzeug.datastructures import MultiDict -from werkzeug.exceptions import NotFound, BadRequest class TurkHitController(RestController): diff --git a/app/views/turk_hit.py b/app/views/turk_hit.py index 7e6883d..0190a07 100644 --- a/app/views/turk_hit.py +++ b/app/views/turk_hit.py @@ -42,9 +42,7 @@ class TurkHitView(RestView): raise BadGateway(error.error_code) return self.json({ - 'data': [ - self.parse(response) - ] + 'data': self.parse(response) }, 201) def put(self, id_): """ @@ -64,9 +62,8 @@ class TurkHitView(RestView): raise BadGateway(error.error_code) return self.json({ - 'data': [ - self.parse(response) - ] + 'data': self.parse(response) + }) def delete(self, id_): -- GitLab From 21c643984383970b3ad987d8ede11167a1c184b7 Mon Sep 17 00:00:00 2001 From: Conrad S Date: Thu, 2 Mar 2017 12:00:50 -0500 Subject: [PATCH 10/15] Return empty array rather than not found --- app/controllers/turk_hit.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/controllers/turk_hit.py b/app/controllers/turk_hit.py index a53a6a0..8056fe9 100644 --- a/app/controllers/turk_hit.py +++ b/app/controllers/turk_hit.py @@ -69,7 +69,7 @@ class TurkHitController(RestController): ) if not hit_list: - raise NotFound + return [] cur_hit = hit_list[0] -- GitLab From dfc7987030980acee34caa5c95ecc2f8d28fe9c0 Mon Sep 17 00:00:00 2001 From: Conrad S Date: Thu, 2 Mar 2017 12:09:09 -0500 Subject: [PATCH 11/15] CHanges names of forms --- app/controllers/turk_hit.py | 30 ++++++++++-------------------- app/forms/turk_hit.py | 4 ++-- 2 files changed, 12 insertions(+), 22 deletions(-) diff --git a/app/controllers/turk_hit.py b/app/controllers/turk_hit.py index 8056fe9..22d0727 100644 --- a/app/controllers/turk_hit.py +++ b/app/controllers/turk_hit.py @@ -14,7 +14,7 @@ from ..lib.mech_turk import create_hit, approve_or_reject_hit,\ get_hit_status from ..lib.service import services from ..models.turk_hit import TurkHit -from ..forms.turk_hit import TurkHitForm, TurkHitAcceptForm +from ..forms.turk_hit import TurkHitPostForm, TurkHitPutForm @@ -36,13 +36,13 @@ class TurkHitController(RestController): # The above dictionary with the keys as values and the values as keys STATUS_DICT_TEXT = {v: k for k, v in STATUS_DICT_ID.items()} - def get_form(self, filter_data): + def get_post_form(self, filter_data): """Return the turk_hit form.""" - return TurkHitForm + return TurkHitPostForm - def get_accept_form(self, filter_data): + def get_put_form(self, filter_data): """Return the turk_hit form.""" - return TurkHitAcceptForm + return TurkHitPutForm def get(self, id_, filter_data): """ @@ -121,7 +121,6 @@ class TurkHitController(RestController): proc(self.Model, self.Model.PROCS['UPDATE'], **cur_hit.get_dictionary()) - pass except Exception as err: raise ( err if current_app.config['DEBUG'] else @@ -163,7 +162,7 @@ class TurkHitController(RestController): data (ImmutableMultiDict): Args for stored proc and mech turk building_id (int): Building id for this hit requester_name (string: Name of the user who requested this hit - # The rest are inputs for the mech turk hit + The rest are inputs for the mech turk hit address (string) max_file_bytes (int) min_file_bytes (int) @@ -180,7 +179,7 @@ class TurkHitController(RestController): dict: TurkHit object """ - form = self.get_form(filter_data)(formdata=MultiDict(data)) + form = self.get_post_form(filter_data)(formdata=MultiDict(data)) if not form.validate(): raise BadRequest(form.errors) @@ -197,10 +196,6 @@ class TurkHitController(RestController): 'amazon_hit_id': amazon_hit_id, 'requester_name': requester_name, 'status_id': self.STATUS_DICT_TEXT['Assignable']}) -# hit_list = [{'amazon_hit_id': amazon_hit_id, 'building_id': building_id, -# 'db_id': 123, 'status_id': 1, 'csv_document_key': None, -# 'shapefile_document_key': None, 'requester_name': 'no_name', -# 'response_message': None, 'hit_date': None}] except Exception as err: raise ( err if current_app.config['DEBUG'] else @@ -220,16 +215,12 @@ class TurkHitController(RestController): Args: id_ (str): db_id of a hit - filter_data (ImmutableMultiDict): Args for stored proc - db_id (int): Should be same as id_ - amazon_hit_id (string): Hit id for this hit - approve (int): 0 for reject, 1 for approve - response_message (string): Message required on reject + data (ImmutableMultiDict): Args for stored proc + Should be the same as the TurkHitAcceptForm Returns: dict: TurkHit object """ - # TODO: Update form validation to require a message - form = self.get_accept_form(filter_data)(formdata=MultiDict(data)) + form = self.get_put_form(filter_data)(formdata=MultiDict(data)) if not form.validate(): raise BadRequest(form.errors) if int(id_) != data["db_id"]: @@ -252,7 +243,6 @@ class TurkHitController(RestController): hit_list = proc(self.Model, self.Model.PROCS['UPDATE'], **data) - #hit_list = [data] except Exception as err: raise ( diff --git a/app/forms/turk_hit.py b/app/forms/turk_hit.py index 5d80838..b16d9b3 100644 --- a/app/forms/turk_hit.py +++ b/app/forms/turk_hit.py @@ -1,7 +1,7 @@ import wtforms as wtf -class TurkHitForm(wtf.Form): +class TurkHitPostForm(wtf.Form): """ A form for validating turk hits.""" building_id = wtf.IntegerField( @@ -36,7 +36,7 @@ class TurkHitForm(wtf.Form): reward = wtf.FloatField( validators=[wtf.validators.Required()]) -class TurkHitAcceptForm(wtf.Form): +class TurkHitPutForm(wtf.Form): """ A form for validating turk hits.""" db_id = wtf.IntegerField( validators=[wtf.validators.Required()]) -- GitLab From 83d4ff15d6b81c510dff88e030d7a748cdba382e Mon Sep 17 00:00:00 2001 From: Conrad S Date: Thu, 2 Mar 2017 14:03:22 -0500 Subject: [PATCH 12/15] Return return statement in accept reject --- app/lib/mech_turk.py | 15 ++++++++------- app/models/turk_hit.py | 7 +++---- 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/app/lib/mech_turk.py b/app/lib/mech_turk.py index 4591557..8099436 100644 --- a/app/lib/mech_turk.py +++ b/app/lib/mech_turk.py @@ -1,10 +1,12 @@ +""" +A conveniance instance for interaction with the mech turk API. All access to +mech turk should be through this file +""" from flask import current_app from boto.mturk.connection import MTurkConnection from boto.mturk.question import QuestionContent, Question, QuestionForm,\ Overview, AnswerSpecification, FileUploadAnswer -# A conveniance instance for interaction with the mech turk API. All access to -# mech turk should be through this file def get_mturk_connection(): AWS_ACCESS_KEY = current_app.config.get('AWS_ACCESS_KEY') AWS_SECRET_ACCESS_KEY = current_app.config.get('AWS_SECRET_ACCESS_KEY') @@ -25,7 +27,8 @@ def create_hit(min_file_bytes, max_file_bytes, """ Send a mechanical turk hit - :return: amazon_hit_id + Returns: + amazon_hit_id """ mturk_connection = get_mturk_connection() @@ -92,12 +95,11 @@ def approve_or_reject_hit(amazon_hit_id, approve, response_message): Given an amazon_hit_id, accept or reject the completed assignment :param building_id: The building_id of the building - :return: The assignmentID that was rejected and the status + Returns + True if succesful, false if not :throws: A MTurkRequestError if AWS credentials are wrong, the hit does not exist, or if the assignment is not reviewable """ - if True: - return True mturk_connection = get_mturk_connection() assignment_id = "" @@ -107,7 +109,6 @@ def approve_or_reject_hit(amazon_hit_id, approve, response_message): if approve: mturk_connection.approve_assignment(assignment.AssignmentId) else: - # TODO: Add response_message to the reject_assignment mturk_connection.reject_assignment(assignment.AssignmentId, feedback=response_message) return True return False diff --git a/app/models/turk_hit.py b/app/models/turk_hit.py index 6683156..cd3cd2d 100644 --- a/app/models/turk_hit.py +++ b/app/models/turk_hit.py @@ -1,7 +1,6 @@ """Models for dealing with turk hit.""" -from ..lib.database import proc, ProcColumn, ProcTable +from ..lib.database import ProcColumn, ProcTable from .base import BaseModel -from werkzeug.exceptions import InternalServerError class TurkHit(BaseModel): @@ -30,7 +29,7 @@ class TurkHit(BaseModel): def __init__(self, db_id, building_id, amazon_hit_id, status_id, hit_date, requester_name, csv_document_key, shapefile_document_key, response_message - ): + ): self.db_id = db_id self.building_id = building_id @@ -44,5 +43,5 @@ class TurkHit(BaseModel): def __str__(self): return "Turk Hit for building {} with dict: {}".format(self.building_id, - self.get_dictionary()) + self.get_dictionary()) -- GitLab From 695ab31dd9de0242d89071a99c443a4e06538f4f Mon Sep 17 00:00:00 2001 From: Conrad S Date: Thu, 2 Mar 2017 14:08:16 -0500 Subject: [PATCH 13/15] Update doc string --- app/lib/mech_turk.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/app/lib/mech_turk.py b/app/lib/mech_turk.py index 8099436..83a3341 100644 --- a/app/lib/mech_turk.py +++ b/app/lib/mech_turk.py @@ -94,11 +94,15 @@ def approve_or_reject_hit(amazon_hit_id, approve, response_message): """ Given an amazon_hit_id, accept or reject the completed assignment - :param building_id: The building_id of the building + Args + amazon_hit_id (string): The hit id for this amazon hit + approve (int): 0 or 1 depending on reject or approve + response message (string): The string to be used on rejection Returns True if succesful, false if not - :throws: A MTurkRequestError if AWS credentials are wrong, - the hit does not exist, or if the assignment is not reviewable + Throws + A MTurkRequestError if AWS credentials are wrong, + the hit does not exist, or if the assignment is not reviewable """ mturk_connection = get_mturk_connection() -- GitLab From 2107f9731856e75dbbb126af2d40157269fccd43 Mon Sep 17 00:00:00 2001 From: Conrad S Date: Thu, 2 Mar 2017 14:36:03 -0500 Subject: [PATCH 14/15] Update exception raise --- app/controllers/turk_hit.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/controllers/turk_hit.py b/app/controllers/turk_hit.py index 22d0727..79463aa 100644 --- a/app/controllers/turk_hit.py +++ b/app/controllers/turk_hit.py @@ -146,7 +146,7 @@ class TurkHitController(RestController): response = services.document.post('', '/document/', data=json.dumps(post_data)) if response.status_code != 201: raise ( - BadRequest(response.json()) if current_app.config['DEBUG'] else + BadRequest(vars(response)) if current_app.config['DEBUG'] else BadRequest("Unable to download document to document service") ) -- GitLab From 689e50181751509f9f63343ffd06e020c2477c56 Mon Sep 17 00:00:00 2001 From: Conrad S Date: Thu, 2 Mar 2017 15:19:00 -0500 Subject: [PATCH 15/15] Update docs --- docs/API/turkhit.md | 124 ++++++++++++++++++++++---------------------- 1 file changed, 61 insertions(+), 63 deletions(-) diff --git a/docs/API/turkhit.md b/docs/API/turkhit.md index 7adb033..9674f5b 100644 --- a/docs/API/turkhit.md +++ b/docs/API/turkhit.md @@ -8,7 +8,7 @@ ---- ## Get Hit Status -Get the hit status of the currently active mech turk hit associated with the inputted building_id and the file data of all of the mech turk box files associated with the inputted building_id. If the hit is newly completed download the file to box. +Get the data of all mech turk hit associated with the inputted building_id. If the newest hit is newly completed download the file to box. * **URL** /turkhit/:building_id/ @@ -23,41 +23,33 @@ Get the hit status of the currently active mech turk hit associated with the inp **Content:** ``` -"box_building_list": [ +"data": [ { - "box_id": 130364791242, - "building_id": "12345", - "content_type": "csv/plain;charset=utf-8", - "created": "2017-02-01T12:57:52.241794+00:00", - "id": 2, - "key": "36c7d0a9-8357-42dc-bcf0-6423791014b5", - "name": "BuildingDimensions2017-02-01 12:57:48.151547.xlsx", - "path": "/Buildings/12345_120 EAST 37 STREET/Building_Dimensions", - "tags": "", - "updated": null, - "url_box": "https://blocpower.box.com/s/fz4u5iyl0uizbz0xsdfscetbdz82h7s8", - "url_download": "https://blocpower.box.com/shared/static/fz4u5iyl0uizbz0xsdfscetbdz82h7s8.xlsx" + "amazon_hit_id": "3JMQI2OLFZ1MUYF5YFY5JGD6PZWDNL", + "building_id": 180361, + "csv_document_key": "4af4c35f-40f6-4af9-85a2-80e34e137d58", + "db_id": 21, + "hit_date": "Thu, 02 Mar 2017 15:27:25 GMT", + "requester_name": "no name", + "response_message": null, + "shapefile_document_key": null, + "status_id": 4 }, { - "box_id": 130403024306, - "building_id": "12345", - "content_type": "csv/plain;charset=utf-8", - "created": "2017-02-01T14:17:10.684649+00:00", - "id": 25, - "key": "c07a978a-d86f-4d87-b21a-db3512617982", - "name": "BuildingDimensions2017-02-01 14:17:06.179782.xlsx", - "path": "/Buildings/12345_120 EAST 37 STREET/Building_Dimensions", - "tags": "", - "updated": null, - "url_box": "https://blocpower.box.com/s/ibj0t8uh79e0mao1rq9vmb2qusuwpuug", - "url_download": "https://blocpower.box.com/shared/static/ibj0t8uh79e0mao1rq9vmb2qusuwpuug.xlsx" + "amazon_hit_id": "3YGE63DIN8TII8NPE41X0S6EOLK0WX", + "building_id": 180361, + "csv_document_key": "46985658-bf1a-4320-997c-cfde3c9f5ba3", + "db_id": 19, + "hit_date": "Wed, 01 Mar 2017 22:03:51 GMT", + "requester_name": "no name", + "response_message": "Rejection message test", + "shapefile_document_key": "None", + "status_id": 5 }, ... ... ... -], -"error": "", -"status": "Reviewable" +] ``` @@ -72,22 +64,16 @@ Get the hit status of the currently active mech turk hit associated with the inp * **Notes:** -2/7/17 - **BUG:** Inputting a building\_id that is non integer returns a 500 internal server error. - -2/7/17 - **BUG:** Inputting a building\_id that does not exist in the database returns a 500 internal server error. - -2/7/17 - If there is some error in the GET function but there is other information that needs to be returned (files that have already been downloaded, hit status, etc.) then the error will be returned in the 'error' field and the front end must handle it on its own. Possible values for this error include _'Failed to update the hit in the backend database'_ and _'Failed to download the hit to box. Try reloading'_. - 2/7/17 - This endpoint does a lot more than just get the hit status. Here is the flow: ``` -1. Get the hit status from the database. -2. If the database hit status is Accepted or Rejected, return this status with a list of box files associated with this hit. +1. Get all hits from the database. +2. If the database hit status of the most recent hit is Accepted or Rejected, return all hits. 3. Get the hit status from amazon mechanical turk API. 4. Get the URL of the completed hit (it will be empty if the hit is in the Assignable, Unassignable or Expired state). 5. Download the hit to box if we are newly in the Reviewable state. -6. Assuming the download succeeded, update the database with the new state. -7. Return the hit status and all of the box files associated with this hit. +6. Assuming the download succeeded, update the database with the new data. +7. Return all hits. ``` ---- @@ -111,6 +97,7 @@ Create a mechanical turk hit with the inputted parameters ``` building_id=[integer] address=[string] +requester_name=[string] max_file_bytes=[integer] min_file_bytes=[integer] instructions_text=[string] @@ -131,21 +118,18 @@ reward=[string] **Content:** ``` -"address": "some address 2", -"building_id": 12345, -"description": "some description", -"duration": 300, -"hit_id": "39WICJI5ATOITVRR8E3N1UY38SJZ3E", -"instructions_text": "We need to get building dimensions for buildings using Google Earth. ", -"instructions_url": "http://beta.blocpower.us/dimensions/BuildingDimensionsInstructions.pdf", -"keywords": "some keyword", -"max_assignments": 1, -"max_file_bytes": 80000, -"min_file_bytes": 1, -"reward": ".26", -"status": "Assignable", -"title": "some title", -"worksheet_url": "some worksheet url" +"data": { + + "amazon_hit_id": "3JMQI2OLFZ1MUYF5YFY5JGD6PZWDNL", + "building_id": 180361, + "csv_document_key": null, + "db_id": 21, + "hit_date": "Thu, 02 Mar 2017 15:27:25 GMT", + "requester_name": "Alessandro DiMarco", + "response_message": null, + "shapefile_document_key": null, + "status_id": 1 +} ``` * **Error Response:** @@ -161,18 +145,15 @@ reward=[string] * **Notes:** -2/7/17 - **BUG:** Inputting a building_id that doesn't exist in the database raises a 500 internal server error - - ---- ## Approve Hit -Approve or reject the mech turk hit associated with the inputted building_id +Approve or reject the mech turk hit associated * **URL** - /building/:building_id/ + /building/:db_id/ * **Method:** @@ -184,7 +165,16 @@ Approve or reject the mech turk hit associated with the inputted building_id **Required:** ``` +db_id=[integer]: Must be the same as in the URI accept=[integer]: 1 to approve, 0 to reject +response_message=[string]: Required if accept = 0 +amazon_hit_id +building_id, +csv_document_key, +hit_date:, +requester_name, +shapefile_document_key +status_id ``` * **Success Response:** @@ -194,8 +184,18 @@ accept=[integer]: 1 to approve, 0 to reject **Content:** ``` -"assignment_id": "373ERPL3YP4HD51O3EK2MOVI603TRH", -"hit_status": "Accepted" +"data": { + + "amazon_hit_id": "3JMQI2OLFZ1MUYF5YFY5JGD6PZWDNL", + "building_id": 180361, + "csv_document_key": null, + "db_id": 21, + "hit_date": "Thu, 02 Mar 2017 15:27:25 GMT", + "requester_name": "Alessandro DiMarco", + "response_message": null, + "shapefile_document_key": null, + "status_id": 2 +} ``` * **Error Response:** @@ -216,5 +216,3 @@ accept=[integer]: 1 to approve, 0 to reject **Content:** `{ error : "ValueError: AWS keys or HOST are empty in the config file" }` * **Notes:** - -2/7/17 - **BUG:** Inputting a building_id that doesn't exist in the database raises a 500 internal server error -- GitLab