From 52d3bdac36bec6bce356348a6c02c9b4443f8308 Mon Sep 17 00:00:00 2001 From: Conrad Date: Tue, 16 May 2017 11:36:04 -0400 Subject: [PATCH 1/4] Bump boto version --- app/config/local.default.py | 3 +- app/controllers/turk_hit.py | 1 - app/lib/mech_turk.py | 142 ++++++++++++++++++++++-------------- app/views/turk_hit.py | 48 ++++-------- requirements.txt | 4 +- 5 files changed, 106 insertions(+), 92 deletions(-) diff --git a/app/config/local.default.py b/app/config/local.default.py index ca502b4..e0e7c4c 100644 --- a/app/config/local.default.py +++ b/app/config/local.default.py @@ -32,4 +32,5 @@ HEADER_AUTH_TOKEN = 'x-blocpower-auth-token' # Mechanical Turk credentials AWS_ACCESS_KEY = '$AWS_KEY' AWS_SECRET_ACCESS_KEY = '$AWS_SECRET_KEY' -MECH_TURK_HOST = '$MECH_TURK_HOST' +MECH_TURK_ENDPOINT_URL = '$MECH_TURK_ENDPOINT_URL' +MECH_TURK_REGION = '$MECH_TURK_REGION' diff --git a/app/controllers/turk_hit.py b/app/controllers/turk_hit.py index 1809a41..3986c1c 100644 --- a/app/controllers/turk_hit.py +++ b/app/controllers/turk_hit.py @@ -20,7 +20,6 @@ from ..lib.service import services from ..models.turk_hit import TurkHit from ..forms.turk_hit import TurkHitPostForm, TurkHitPutForm, TurkHitDeleteForm - class TurkHitController(RestController): """A turk_hit controller.""" Model = TurkHit diff --git a/app/lib/mech_turk.py b/app/lib/mech_turk.py index 2c8983a..47c4fdc 100644 --- a/app/lib/mech_turk.py +++ b/app/lib/mech_turk.py @@ -3,21 +3,23 @@ A conveniance instance for interaction with the mech turk API. All access to mech turk should be through this file """ from flask import current_app -from boto.mturk.connection import MTurkConnection -from boto.mturk.question import QuestionContent, Question, QuestionForm,\ - Overview, AnswerSpecification, FileUploadAnswer +import boto3 +from datetime import datetime def get_mturk_connection(): AWS_ACCESS_KEY = current_app.config.get('AWS_ACCESS_KEY') AWS_SECRET_ACCESS_KEY = current_app.config.get('AWS_SECRET_ACCESS_KEY') - HOST = current_app.config.get('MECH_TURK_HOST') - if not AWS_ACCESS_KEY or not AWS_SECRET_ACCESS_KEY or not HOST: - raise ValueError("AWS keys or HOST are empty in the config file") + ENDPOINT_URL = current_app.config.get('MECH_TURK_ENDPOINT_URL') + REGION = current_app.config.get('MECH_TURK_REGION') + if not AWS_ACCESS_KEY or not AWS_SECRET_ACCESS_KEY or not ENDPOINT_URL: + raise ValueError("AWS keys or endpoint url are empty in the config file") - return MTurkConnection( + return boto3.client( + 'mturk', aws_access_key_id=AWS_ACCESS_KEY, aws_secret_access_key=AWS_SECRET_ACCESS_KEY, - host=HOST + endpoint_url=ENDPOINT_URL, + region_name=REGION, ) def create_hit(min_file_bytes, max_file_bytes, @@ -32,62 +34,82 @@ def create_hit(min_file_bytes, max_file_bytes, """ mturk_connection = get_mturk_connection() - # Overview - overview = Overview() - overview.append_field('Title', address) - - # Instructions - qc1 = QuestionContent() - qc1.append_field('Title', 'Instructions') - qc1.append_field('Text', instructions_text) - qc1.append_field('Text', instructions_url) - qc1.append_field('Title', 'Worksheet') - qc1.append_field('Text', worksheet_url) - - file_upload = FileUploadAnswer( - min_file_bytes, max_file_bytes) - - question = Question(identifier='measure_building', - content=qc1, - answer_spec=AnswerSpecification(file_upload), - is_required=True) - - # Question Form - question_form = QuestionForm() - question_form.append(overview) - question_form.append(question) + question_form = """ + + + {} + + + measure_building + true + + Instructions + {} + {} + Worksheet + {} + + + + {} + {} + + + + + """.format( + address, + instructions_text, + instructions_url, + worksheet_url, + max_file_bytes, + min_file_bytes, + ).strip() + + THIRTY_DAYS_SECONDS = 60 * 60 * 24 * 30 # Creates a new HIT result_set = mturk_connection.create_hit( - questions=question_form, - max_assignments=max_assignments, - title=title, - description=description, - keywords=keywords, - duration=duration, - reward=reward) - - amazon_hit_id = getattr(result_set[0], 'HITId') + Question=question_form, + MaxAssignments=int(max_assignments), + Title=title, + Description=description, + Keywords=keywords, + AssignmentDurationInSeconds=int(duration), + LifetimeInSeconds=THIRTY_DAYS_SECONDS, + Reward=reward, + ) + + amazon_hit_id = result_set['HIT']['HITId'] return amazon_hit_id def get_hit_status(amazon_hit_id): mturk_connection = get_mturk_connection() - hit = mturk_connection.get_hit(amazon_hit_id) - hit_status = getattr(hit[0], 'HITStatus') + hit = mturk_connection.get_hit( + HITId=amazon_hit_id, + ) + hit_status = hit['HIT']['HITStatus'] file_url = None # Only call mturk to grab file_url if hit status is in a state # where there possibly could be a file_url if hit_status in ["Reviewable", "Accepted", "Rejected"]: - completed_assignments = mturk_connection.get_assignments(amazon_hit_id) - if completed_assignments: - assignment = completed_assignments[0] + completed_assignments = mturk_connection.list_assignments_for_hit( + HITId=amazon_hit_id, + ) + if completed_assignments['Assignments']: + assignment = completed_assignments['Assignments'][0] file_url_object = mturk_connection.get_file_upload_url( - assignment.AssignmentId, "measure_building") - file_url = file_url_object[0].FileUploadURL - return {'hit_status': hit_status, 'file_url': file_url} + AssignmentId=assignment['AssignmentId'], + QuestionIdentifier="measure_building", + ) + file_url = file_url_object['FileUploadURL'] + return { + 'hit_status': hit_status, + 'file_url': file_url, + } def approve_or_reject_hit(amazon_hit_id, approve, response_message): @@ -107,13 +129,20 @@ def approve_or_reject_hit(amazon_hit_id, approve, response_message): mturk_connection = get_mturk_connection() assignment_id = "" - completed_assignments = mturk_connection.get_assignments(amazon_hit_id) - if completed_assignments: - assignment = completed_assignments[0] + completed_assignments = mturk_connection.list_assignments_for_hit( + HITId=amazon_hit_id, + ) + if completed_assignments['Assignments']: + assignment = completed_assignments['Assignments'][0] if approve: - mturk_connection.approve_assignment(assignment.AssignmentId) + mturk_connection.approve_assignment( + AssignmentId=assignment['AssignmentId'], + ) else: - mturk_connection.reject_assignment(assignment.AssignmentId, feedback=response_message) + mturk_connection.reject_assignment( + AssignmentId=assignment['AssignmentId'], + RequesterFeedback=response_message, + ) return True return False @@ -122,5 +151,8 @@ def expire_hit(amazon_hit_id): Expire the hit associated with the amazon hit id """ mturk_connection = get_mturk_connection() - mturk_connection.expire_hit(amazon_hit_id) + mturk_connection.update_expiration_for_hit( + HITId=amazon_hit_id, + ExpireAt=datetime(1980, 1, 1) + ) return True diff --git a/app/views/turk_hit.py b/app/views/turk_hit.py index 8b15862..d2e9094 100644 --- a/app/views/turk_hit.py +++ b/app/views/turk_hit.py @@ -2,7 +2,6 @@ from flask import request from werkzeug.exceptions import (BadRequest, NotFound, MethodNotAllowed, BadGateway) -from boto.mturk.connection import MTurkRequestError from ..controllers.turk_hit import TurkHitController from .base import RestView @@ -18,26 +17,17 @@ class TurkHitView(RestView): def get(self, id_): """/{id} GET - Retrieve the hit status by building id.""" - try: - response = self.get_controller().get(id_, request.args) - # Catch an MTurkRequestError (usually AWS) and return error code - except MTurkRequestError as error: - # If the HITId does not exist return 404 - if error.error_code == 'AWS.MechanicalTurk.HITDoesNotExist': - raise NotFound(error.error_code) - raise BadGateway(error.error_code) + response = self.get_controller().get(id_, request.args) return self.json( [self.parse(m) for m in response] ) def post(self): """/ POST - Create a hit given an id in the POST body""" - try: - response = self.get_controller().post( - self.request_json(), request.args) - # Catch an MTurkRequestError (usually AWS) and return error code - except MTurkRequestError as error: - raise BadGateway(error.error_code) + response = self.get_controller().post( + self.request_json(), + request.args, + ) return self.json(self.parse(response), 201) def put(self, id_): @@ -47,27 +37,19 @@ class TurkHitView(RestView): if accept = 1, accept if accept = 0, reject """ - try: - response = self.get_controller().put( - id_, self.request_json(), request.args) - # Catch an MTurkRequestError and return error code - except MTurkRequestError as error: - # If the assignment is not in the proper state to accept or reject - if error.error_code == 'AWS.MechanicalTurk.InvalidAssignmentState': - raise BadRequest(error.error_code) - raise BadGateway(error.error_code) + response = self.get_controller().put( + id_, + self.request_json(), + request.args, + ) return self.json(self.parse(response)) def delete(self, id_): - try: - response = self.get_controller().delete( - id_, - self.request_json(), - request.args - ) - # Catch an MTurkRequestError (usually AWS) and return error code - except MTurkRequestError as error: - raise BadGateway(error.error_code) + response = self.get_controller().delete( + id_, + self.request_json(), + request.args + ) return self.json(self.parse(response), 200) diff --git a/requirements.txt b/requirements.txt index 1e5d92d..649a48a 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,8 +1,8 @@ arrow==0.7.0 beautifulsoup4==4.4.1 blessed==1.9.5 -boto==2.45.0 -botocore==1.3.28 +boto3==1.4.4 +botocore==1.5.48 cement==2.4.0 colorama==0.3.3 docker-py==1.1.0 -- GitLab From 66ba56399e67918b3c3a18d590a6f34829a77b0e Mon Sep 17 00:00:00 2001 From: Conrad Date: Tue, 16 May 2017 11:40:00 -0400 Subject: [PATCH 2/4] Update config file defaults --- app/config/development.default.py | 3 ++- app/config/production.default.py | 3 ++- app/config/staging.default.py | 3 ++- app/config/test.default.py | 3 ++- 4 files changed, 8 insertions(+), 4 deletions(-) diff --git a/app/config/development.default.py b/app/config/development.default.py index e72d5aa..185c56b 100644 --- a/app/config/development.default.py +++ b/app/config/development.default.py @@ -35,4 +35,5 @@ HEADER_AUTH_TOKEN = 'x-blocpower-auth-token' # Mechanical Turk credentials AWS_ACCESS_KEY = os.environ['AWS_ACCESS_KEY'] AWS_SECRET_ACCESS_KEY = os.environ['AWS_SECRET_ACCESS_KEY'] -MECH_TURK_HOST = os.environ['MECH_TURK_HOST'] +MECH_TURK_ENDPOINT_URL = os.environ['MECH_TURK_ENDPOINT_URL'] +MECH_TURK_REGION = os.environ['MECH_TURK_REGION'] diff --git a/app/config/production.default.py b/app/config/production.default.py index cfc8b88..f33db91 100644 --- a/app/config/production.default.py +++ b/app/config/production.default.py @@ -35,4 +35,5 @@ HEADER_AUTH_TOKEN = 'x-blocpower-auth-token' # Mechanical Turk credentials AWS_ACCESS_KEY = os.environ['AWS_ACCESS_KEY'] AWS_SECRET_ACCESS_KEY = os.environ['AWS_SECRET_ACCESS_KEY'] -MECH_TURK_HOST = os.environ['MECH_TURK_HOST'] +MECH_TURK_ENDPOINT_URL = os.environ['MECH_TURK_ENDPOINT_URL'] +MECH_TURK_REGION = os.environ['MECH_TURK_REGION'] diff --git a/app/config/staging.default.py b/app/config/staging.default.py index 82561ac..49f9fe8 100644 --- a/app/config/staging.default.py +++ b/app/config/staging.default.py @@ -35,4 +35,5 @@ HEADER_AUTH_TOKEN = 'x-blocpower-auth-token' # Mechanical Turk credentials AWS_ACCESS_KEY = os.environ['AWS_ACCESS_KEY'] AWS_SECRET_ACCESS_KEY = os.environ['AWS_SECRET_ACCESS_KEY'] -MECH_TURK_HOST = os.environ['MECH_TURK_HOST'] +MECH_TURK_ENDPOINT_URL = os.environ['MECH_TURK_ENDPOINT_URL'] +MECH_TURK_REGION = os.environ['MECH_TURK_REGION'] diff --git a/app/config/test.default.py b/app/config/test.default.py index d6cdbbb..e7f5b22 100644 --- a/app/config/test.default.py +++ b/app/config/test.default.py @@ -32,7 +32,8 @@ HEADER_AUTH_TOKEN = 'x-blocpower-auth-token' # Mechanical Turk credentials AWS_ACCESS_KEY = '$AWS_KEY' AWS_SECRET_ACCESS_KEY = '$AWS_SECRET_KEY' -MECH_TURK_HOST = '$MECH_TURK_HOST' +MECH_TURK_ENDPOINT_URL = '$MECH_TURK_ENDPOINT_URL' +MECH_TURK_REGION = '$MECH_TURK_REGION' # HACK Fix issue where raising an exception in a test context creates an extra # request. -- GitLab From ec51e2c08b75a23299482365922ebe7474540828 Mon Sep 17 00:00:00 2001 From: Conrad Date: Tue, 16 May 2017 11:58:06 -0400 Subject: [PATCH 3/4] Remove document keys from model --- app/controllers/turk_hit.py | 47 ++----------------------------------- app/forms/turk_hit.py | 4 ---- app/models/turk_hit.py | 21 ++++++++++------- 3 files changed, 14 insertions(+), 58 deletions(-) diff --git a/app/controllers/turk_hit.py b/app/controllers/turk_hit.py index 3986c1c..a86e47d 100644 --- a/app/controllers/turk_hit.py +++ b/app/controllers/turk_hit.py @@ -107,8 +107,8 @@ class TurkHitController(RestController): cur_hit.dimensions = dimensions return hit_list - # Download file here - if result['file_url'] and not cur_hit.csv_document_key: + # Download data here + if result['file_url'] and not dimensions['building_dimensions']: response = requests.get(result['file_url']) uploaded_fname = response.headers['Content-Disposition'].split('"')[1] content_type = mimetypes.guess_type(uploaded_fname)[0] @@ -154,24 +154,6 @@ class TurkHitController(RestController): None, ) - # Save the file to box - file_name = "BuildingDimensions--{}--{}.xlsx".format( - cur_hit.building_id, - cur_hit.amazon_hit_id, - ) - folder_path = "/Buildings/{}_{}/Building_Dimensions".format( - cur_hit.building_id, - address, - ) - document = self.download_file( - cur_hit.building_id, - folder_path, - file_name, - response.content, - ) - # Get the key to update the model - cur_hit.csv_document_key = document['key'] - # Update the entry in the database new_hit_status_id = self.STATUS_DICT_TEXT[new_hit_status] cur_hit.status_id = new_hit_status_id @@ -201,31 +183,6 @@ class TurkHitController(RestController): self.put(hit.db_id, put_body, None) return "Rejected" - def download_file(self, building_id, folder_path, file_name, byte_data): - """ - A function to download a file to box - """ - encoded_byte_data = base64.b64encode(byte_data) - encoded_string_data = encoded_byte_data.decode('utf-8') - - post_data = { - "path": folder_path, - "data": "data:csv/plain;charset=utf-8;base64,{}".format(encoded_string_data), - "building_id": str(building_id), - "tags": '', - "name": file_name, - } - # Call the generic method in the base class - response = services.document.post('', '/document/', data=json.dumps(post_data)) - if response.status_code != 201: - raise ( - BadRequest(vars(response)) if current_app.config['DEBUG'] else - BadRequest("Unable to download document to document service") - ) - - document = response.json()['data'] - return document - def parse_dimensions_data(self, contents, hit_id, building_id): """ A function to parse dimensions data from the completed mech turk hit diff --git a/app/forms/turk_hit.py b/app/forms/turk_hit.py index c8990f3..88a587d 100644 --- a/app/forms/turk_hit.py +++ b/app/forms/turk_hit.py @@ -51,10 +51,6 @@ class TurkHitDeleteForm(wtf.Form): validators=[wtf.validators.Optional()]) requester_name = wtf.StringField( validators=[wtf.validators.Optional()]) - csv_document_key = wtf.StringField( - validators=[wtf.validators.Optional()]) - shapefile_document_key = wtf.StringField( - validators=[wtf.validators.Optional()]) response_message = wtf.StringField( validators=[wtf.validators.Optional()]) diff --git a/app/models/turk_hit.py b/app/models/turk_hit.py index b993dcb..4b8e5df 100644 --- a/app/models/turk_hit.py +++ b/app/models/turk_hit.py @@ -23,15 +23,18 @@ class TurkHit(BaseModel): ProcColumn('status_id'), ProcColumn('hit_date'), ProcColumn('requester_name'), - ProcColumn('csv_document_key'), - ProcColumn('shapefile_document_key'), ProcColumn('response_message'), ) def __init__( - self, db_id=None, building_id=None, amazon_hit_id=None, status_id=None, - hit_date=None, requester_name=None, csv_document_key=None, - shapefile_document_key=None, response_message=None, + self, + db_id=None, + building_id=None, + amazon_hit_id=None, + status_id=None, + hit_date=None, + requester_name=None, + response_message=None, ): self.db_id = db_id self.building_id = building_id @@ -39,11 +42,11 @@ class TurkHit(BaseModel): self.status_id = status_id self.hit_date = hit_date self.requester_name = requester_name - self.csv_document_key = csv_document_key - self.shapefile_document_key = shapefile_document_key self.response_message = response_message def __str__(self): - return "Turk Hit for building {} with dict: {}".format(self.building_id, - self.get_dictionary()) + return "Turk Hit for building {} with dict: {}".format( + self.building_id, + self.get_dictionary() + ) -- GitLab From ce9841ab7fc2a0f82b608ce57a885f11c586e0fa Mon Sep 17 00:00:00 2001 From: Conrad Date: Tue, 16 May 2017 15:25:45 -0400 Subject: [PATCH 4/4] Remove documentservice from config files --- app/config/development.default.py | 2 +- app/config/local.default.py | 4 ++-- app/config/production.default.py | 4 ++-- app/config/staging.default.py | 4 ++-- 4 files changed, 7 insertions(+), 7 deletions(-) diff --git a/app/config/development.default.py b/app/config/development.default.py index 185c56b..3877816 100644 --- a/app/config/development.default.py +++ b/app/config/development.default.py @@ -18,7 +18,7 @@ SERVICE_CONFIG = { 'urls': { 'app': 'http://dev.appservice.blocpower.io/', 'user': 'http://dev.userservice.blocpower.io/', - 'document': 'http://dev.document.s.blocpower.io/'} + } } # AppService diff --git a/app/config/local.default.py b/app/config/local.default.py index e0e7c4c..87bf9f5 100644 --- a/app/config/local.default.py +++ b/app/config/local.default.py @@ -14,8 +14,8 @@ SERVICE_CONFIG = { 'app_secret': 'x-blocpower-app-secret'}, 'urls': { 'app': 'http://127.0.0.1:5400', - 'document': 'http://127.0.0.1:5403', - 'user': 'http://127.0.0.1:5401'} + 'user': 'http://127.0.0.1:5401', + } } # AppService diff --git a/app/config/production.default.py b/app/config/production.default.py index f33db91..0e23120 100644 --- a/app/config/production.default.py +++ b/app/config/production.default.py @@ -17,8 +17,8 @@ SERVICE_CONFIG = { 'app_secret': 'x-blocpower-app-secret'}, 'urls': { 'app': 'http://app.s.blocpower.us', - 'document': 'http://documentservice.blocpower.io/', - 'user': 'http://user.s.blocpower.us'} + 'user': 'http://user.s.blocpower.us', + } } # AppService diff --git a/app/config/staging.default.py b/app/config/staging.default.py index 49f9fe8..04bbebf 100644 --- a/app/config/staging.default.py +++ b/app/config/staging.default.py @@ -17,8 +17,8 @@ SERVICE_CONFIG = { 'app_secret': 'x-blocpower-app-secret'}, 'urls': { 'app': 'http://staging.app.s.blocpower.us/', - 'document': 'http://staging.documentservice.blocpower.io/', - 'user': 'http://staging.user.s.blocpower.us/'} + 'user': 'http://staging.user.s.blocpower.us/', + } } # AppService -- GitLab