From 5879d18656595ddf9b15c3b6c50f4ac941930e3d Mon Sep 17 00:00:00 2001 From: Conrad S Date: Tue, 14 Mar 2017 16:23:11 -0400 Subject: [PATCH 01/14] Add unapproved_point endpoint --- .../unapproved_building_dimensions.py | 62 +++++++++++++++++++ app/controllers/unapproved_point.py | 62 +++++++++++++++++++ app/controllers/unapproved_window_door.py | 62 +++++++++++++++++++ app/forms/unapproved_building_dimensions.py | 16 +++++ app/forms/unapproved_point.py | 16 +++++ app/forms/unapproved_window_door.py | 16 +++++ app/models/unapproved_building_dimensions.py | 26 ++++++++ app/models/unapproved_point.py | 26 ++++++++ app/models/unapproved_window_door.py | 26 ++++++++ app/views/__init__.py | 3 +- app/views/base.py | 2 +- app/views/building.py | 14 ++--- app/views/turk_hit.py | 17 ++--- app/views/unapproved_building_dimensions.py | 21 +++++++ app/views/unapproved_point.py | 21 +++++++ app/views/unapproved_window_door.py | 21 +++++++ 16 files changed, 389 insertions(+), 22 deletions(-) create mode 100644 app/controllers/unapproved_building_dimensions.py create mode 100644 app/controllers/unapproved_point.py create mode 100644 app/controllers/unapproved_window_door.py create mode 100644 app/forms/unapproved_building_dimensions.py create mode 100644 app/forms/unapproved_point.py create mode 100644 app/forms/unapproved_window_door.py create mode 100644 app/models/unapproved_building_dimensions.py create mode 100644 app/models/unapproved_point.py create mode 100644 app/models/unapproved_window_door.py create mode 100644 app/views/unapproved_building_dimensions.py create mode 100644 app/views/unapproved_point.py create mode 100644 app/views/unapproved_window_door.py diff --git a/app/controllers/unapproved_building_dimensions.py b/app/controllers/unapproved_building_dimensions.py new file mode 100644 index 0000000..a85e5a0 --- /dev/null +++ b/app/controllers/unapproved_building_dimensions.py @@ -0,0 +1,62 @@ +"""Controllers for managing top-level unapproved_points.""" +from sqlalchemy import and_ +from werkzeug.exceptions import BadRequest +from flask import current_app + +from app.lib.database import db, proc +from app.controllers.base import RestController +from app.models.unapproved_point import UnapprovedPoint +from app.forms.unapproved_point import UnapprovedPointForm + + +class UnapprovedPointController(RestController): + """The unapproved_point controller.""" + Model = UnapprovedPoint + filters = { + 'hit_id[]': \ + lambda d: UnapprovedPoint.hit_id.in_(d.getlist('hit_id[]')) + } + + def get_form(self, filter_data): + """Return the unapproved_point form.""" + return UnapprovedPointForm + + def post(self, data, filter_data): + """Post a new model. + + """ + # Will want to loop through the inputted unapproved_points + try: + #self.delete(filter_data["account_id"], None) + model_list = [] + for point in data: + model = super(UnapprovedPointController, self).post(point, filter_data) + model_list.append(model) + return model_list + except Exception as e: + raise ( + e if current_app.config["DEBUG"] else + BadRequest("Error while posting unapproved_point data") + ) + +# def delete(self, id_, filter_data): +# +# try: +# query = "select * from public.{}({})".format( +# 'delete_unapproved_point', +# 'in_account_id := {}'.format(id_) +# ) +# results = db.session.execute(query) +# db.session.commit() +# except Exception as e: +# raise ( +# e if current_app.config['DEBUG'] else +# BadRequest('Error while executing db call') +# ) +# return_list = [] +# for row in results: +# print(row) +# d = dict(row.items()) +# return_list.append(d) +# print(return_list) +# return return_list diff --git a/app/controllers/unapproved_point.py b/app/controllers/unapproved_point.py new file mode 100644 index 0000000..a85e5a0 --- /dev/null +++ b/app/controllers/unapproved_point.py @@ -0,0 +1,62 @@ +"""Controllers for managing top-level unapproved_points.""" +from sqlalchemy import and_ +from werkzeug.exceptions import BadRequest +from flask import current_app + +from app.lib.database import db, proc +from app.controllers.base import RestController +from app.models.unapproved_point import UnapprovedPoint +from app.forms.unapproved_point import UnapprovedPointForm + + +class UnapprovedPointController(RestController): + """The unapproved_point controller.""" + Model = UnapprovedPoint + filters = { + 'hit_id[]': \ + lambda d: UnapprovedPoint.hit_id.in_(d.getlist('hit_id[]')) + } + + def get_form(self, filter_data): + """Return the unapproved_point form.""" + return UnapprovedPointForm + + def post(self, data, filter_data): + """Post a new model. + + """ + # Will want to loop through the inputted unapproved_points + try: + #self.delete(filter_data["account_id"], None) + model_list = [] + for point in data: + model = super(UnapprovedPointController, self).post(point, filter_data) + model_list.append(model) + return model_list + except Exception as e: + raise ( + e if current_app.config["DEBUG"] else + BadRequest("Error while posting unapproved_point data") + ) + +# def delete(self, id_, filter_data): +# +# try: +# query = "select * from public.{}({})".format( +# 'delete_unapproved_point', +# 'in_account_id := {}'.format(id_) +# ) +# results = db.session.execute(query) +# db.session.commit() +# except Exception as e: +# raise ( +# e if current_app.config['DEBUG'] else +# BadRequest('Error while executing db call') +# ) +# return_list = [] +# for row in results: +# print(row) +# d = dict(row.items()) +# return_list.append(d) +# print(return_list) +# return return_list diff --git a/app/controllers/unapproved_window_door.py b/app/controllers/unapproved_window_door.py new file mode 100644 index 0000000..a85e5a0 --- /dev/null +++ b/app/controllers/unapproved_window_door.py @@ -0,0 +1,62 @@ +"""Controllers for managing top-level unapproved_points.""" +from sqlalchemy import and_ +from werkzeug.exceptions import BadRequest +from flask import current_app + +from app.lib.database import db, proc +from app.controllers.base import RestController +from app.models.unapproved_point import UnapprovedPoint +from app.forms.unapproved_point import UnapprovedPointForm + + +class UnapprovedPointController(RestController): + """The unapproved_point controller.""" + Model = UnapprovedPoint + filters = { + 'hit_id[]': \ + lambda d: UnapprovedPoint.hit_id.in_(d.getlist('hit_id[]')) + } + + def get_form(self, filter_data): + """Return the unapproved_point form.""" + return UnapprovedPointForm + + def post(self, data, filter_data): + """Post a new model. + + """ + # Will want to loop through the inputted unapproved_points + try: + #self.delete(filter_data["account_id"], None) + model_list = [] + for point in data: + model = super(UnapprovedPointController, self).post(point, filter_data) + model_list.append(model) + return model_list + except Exception as e: + raise ( + e if current_app.config["DEBUG"] else + BadRequest("Error while posting unapproved_point data") + ) + +# def delete(self, id_, filter_data): +# +# try: +# query = "select * from public.{}({})".format( +# 'delete_unapproved_point', +# 'in_account_id := {}'.format(id_) +# ) +# results = db.session.execute(query) +# db.session.commit() +# except Exception as e: +# raise ( +# e if current_app.config['DEBUG'] else +# BadRequest('Error while executing db call') +# ) +# return_list = [] +# for row in results: +# print(row) +# d = dict(row.items()) +# return_list.append(d) +# print(return_list) +# return return_list diff --git a/app/forms/unapproved_building_dimensions.py b/app/forms/unapproved_building_dimensions.py new file mode 100644 index 0000000..c27ff33 --- /dev/null +++ b/app/forms/unapproved_building_dimensions.py @@ -0,0 +1,16 @@ +import wtforms as wtf + + +class UnapprovedPointForm(wtf.Form): + """ A form for validating unapproved_point requests.""" + + hit_id = wtf.IntegerField( + validators=[wtf.validators.Required()]) + latitude = wtf.DecimalField() + longitude = wtf.DecimalField() + elevation = wtf.IntegerField() + corresponding_height = wtf.IntegerField() + adjacent = wtf.BooleanField() + feature = wtf.IntegerField() + building_id = wtf.IntegerField() + diff --git a/app/forms/unapproved_point.py b/app/forms/unapproved_point.py new file mode 100644 index 0000000..c27ff33 --- /dev/null +++ b/app/forms/unapproved_point.py @@ -0,0 +1,16 @@ +import wtforms as wtf + + +class UnapprovedPointForm(wtf.Form): + """ A form for validating unapproved_point requests.""" + + hit_id = wtf.IntegerField( + validators=[wtf.validators.Required()]) + latitude = wtf.DecimalField() + longitude = wtf.DecimalField() + elevation = wtf.IntegerField() + corresponding_height = wtf.IntegerField() + adjacent = wtf.BooleanField() + feature = wtf.IntegerField() + building_id = wtf.IntegerField() + diff --git a/app/forms/unapproved_window_door.py b/app/forms/unapproved_window_door.py new file mode 100644 index 0000000..c27ff33 --- /dev/null +++ b/app/forms/unapproved_window_door.py @@ -0,0 +1,16 @@ +import wtforms as wtf + + +class UnapprovedPointForm(wtf.Form): + """ A form for validating unapproved_point requests.""" + + hit_id = wtf.IntegerField( + validators=[wtf.validators.Required()]) + latitude = wtf.DecimalField() + longitude = wtf.DecimalField() + elevation = wtf.IntegerField() + corresponding_height = wtf.IntegerField() + adjacent = wtf.BooleanField() + feature = wtf.IntegerField() + building_id = wtf.IntegerField() + diff --git a/app/models/unapproved_building_dimensions.py b/app/models/unapproved_building_dimensions.py new file mode 100644 index 0000000..6b14651 --- /dev/null +++ b/app/models/unapproved_building_dimensions.py @@ -0,0 +1,26 @@ +"""Models for dealing with an unapproved_point.""" +from sqlalchemy.ext.declarative import declared_attr + +from ..lib.database import ProcColumn, ProcTable, db +from .base import Model + + +class UnapprovedPoint(Model, db.Model): + + @declared_attr + def __tablename__(cls): + return "unapproved_point" + + __table_args__ = {"schema": "mechanical_turk"} + + latitude = db.Column(db.Float) + longitude = db.Column(db.Float) + elevation = db.Column(db.Integer) + corresponding_height = db.Column(db.Integer) + adjacent = db.Column(db.Boolean) + feature = db.Column(db.Integer) + hit_id = db.Column(db.Integer, nullable=False) + building_id = db.Column(db.Integer) + + def __str__(self): + return "unapproved_point data for account {}".format(self.hit_id) diff --git a/app/models/unapproved_point.py b/app/models/unapproved_point.py new file mode 100644 index 0000000..6b14651 --- /dev/null +++ b/app/models/unapproved_point.py @@ -0,0 +1,26 @@ +"""Models for dealing with an unapproved_point.""" +from sqlalchemy.ext.declarative import declared_attr + +from ..lib.database import ProcColumn, ProcTable, db +from .base import Model + + +class UnapprovedPoint(Model, db.Model): + + @declared_attr + def __tablename__(cls): + return "unapproved_point" + + __table_args__ = {"schema": "mechanical_turk"} + + latitude = db.Column(db.Float) + longitude = db.Column(db.Float) + elevation = db.Column(db.Integer) + corresponding_height = db.Column(db.Integer) + adjacent = db.Column(db.Boolean) + feature = db.Column(db.Integer) + hit_id = db.Column(db.Integer, nullable=False) + building_id = db.Column(db.Integer) + + def __str__(self): + return "unapproved_point data for account {}".format(self.hit_id) diff --git a/app/models/unapproved_window_door.py b/app/models/unapproved_window_door.py new file mode 100644 index 0000000..6b14651 --- /dev/null +++ b/app/models/unapproved_window_door.py @@ -0,0 +1,26 @@ +"""Models for dealing with an unapproved_point.""" +from sqlalchemy.ext.declarative import declared_attr + +from ..lib.database import ProcColumn, ProcTable, db +from .base import Model + + +class UnapprovedPoint(Model, db.Model): + + @declared_attr + def __tablename__(cls): + return "unapproved_point" + + __table_args__ = {"schema": "mechanical_turk"} + + latitude = db.Column(db.Float) + longitude = db.Column(db.Float) + elevation = db.Column(db.Integer) + corresponding_height = db.Column(db.Integer) + adjacent = db.Column(db.Boolean) + feature = db.Column(db.Integer) + hit_id = db.Column(db.Integer, nullable=False) + building_id = db.Column(db.Integer) + + def __str__(self): + return "unapproved_point data for account {}".format(self.hit_id) diff --git a/app/views/__init__.py b/app/views/__init__.py index 0248c90..1718e2a 100644 --- a/app/views/__init__.py +++ b/app/views/__init__.py @@ -1,5 +1,5 @@ """Flask-classy views for the flask application.""" -from . import building, turk_hit +from . import building, turk_hit, unapproved_point def register(app): @@ -10,3 +10,4 @@ def register(app): """ building.BuildingView.register(app) turk_hit.TurkHitView.register(app) + unapproved_point.UnapprovedPointView.register(app) diff --git a/app/views/base.py b/app/views/base.py index 452a5bb..1a175c5 100644 --- a/app/views/base.py +++ b/app/views/base.py @@ -15,7 +15,7 @@ class View(FlaskView): def json(self, obj, status=200): """A wrapper for Flask's jsonify().""" - response = jsonify(obj) + response = jsonify(data=obj) return response, status def parse(self, model): diff --git a/app/views/building.py b/app/views/building.py index 8e56151..15fc763 100644 --- a/app/views/building.py +++ b/app/views/building.py @@ -15,21 +15,19 @@ class BuildingView(RestView): """/ GET - Retrieve a list of resources.""" # TODO: Add data key back to self.json # return super(BuildingView, self).index(request.args) - return self.json({ - 'data': [ - self.parse(m) for m in self.get_controller().index(request.args) - ] - }) + return self.json( + [self.parse(m) for m in self.get_controller().index(request.args)] + ) def get(self, id_): """/{id} GET - Retrieve a resource by id.""" # TODO: Add data key back in self.json # return super(BuildingView, self).get(id_, request.args) - return self.json({ - 'data': self.parse( + return self.json( + self.parse( self.get_controller().get(id_, request.args) ) - }) + ) def post(self): raise MethodNotAllowed() diff --git a/app/views/turk_hit.py b/app/views/turk_hit.py index 0190a07..f2e0f3a 100644 --- a/app/views/turk_hit.py +++ b/app/views/turk_hit.py @@ -26,11 +26,9 @@ class TurkHitView(RestView): if error.error_code == 'AWS.MechanicalTurk.HITDoesNotExist': raise NotFound(error.error_code) raise BadGateway(error.error_code) - return self.json({ - 'data': [ - self.parse(m) for m in response - ] - }) + return self.json( + [self.parse(m) for m in response] + ) def post(self): """/ POST - Create a hit given an id in the POST body""" @@ -41,9 +39,7 @@ class TurkHitView(RestView): except MTurkRequestError as error: raise BadGateway(error.error_code) - return self.json({ - 'data': self.parse(response) - }, 201) + return self.json(self.parse(response), 201) def put(self, id_): """ / PUT - Approve or decline a hit given a building_id. @@ -61,10 +57,7 @@ class TurkHitView(RestView): raise BadRequest(error.error_code) raise BadGateway(error.error_code) - return self.json({ - 'data': self.parse(response) - - }) + return self.json(self.parse(response)) def delete(self, id_): raise MethodNotAllowed() diff --git a/app/views/unapproved_building_dimensions.py b/app/views/unapproved_building_dimensions.py new file mode 100644 index 0000000..dafc94d --- /dev/null +++ b/app/views/unapproved_building_dimensions.py @@ -0,0 +1,21 @@ +"""Views for managing unapproved_point data.""" +from werkzeug.exceptions import MethodNotAllowed +from flask import request + +from app.views.base import UnprotectedRestView +from app.controllers.unapproved_point import UnapprovedPointController + + +class UnapprovedPointView(UnprotectedRestView): + """The unapproved_point view.""" + def get_controller(self): + """Return an instance of the unapproved_point controller.""" + return UnapprovedPointController() + + def post(self): + """Post a list of unapproved_point rows""" + result = self.get_controller().post(self.request_json(), request.args) + parsed_result = [] + for model in result: + parsed_result.append(self.parse(model)) + return self.json(parsed_result, 201) diff --git a/app/views/unapproved_point.py b/app/views/unapproved_point.py new file mode 100644 index 0000000..dafc94d --- /dev/null +++ b/app/views/unapproved_point.py @@ -0,0 +1,21 @@ +"""Views for managing unapproved_point data.""" +from werkzeug.exceptions import MethodNotAllowed +from flask import request + +from app.views.base import UnprotectedRestView +from app.controllers.unapproved_point import UnapprovedPointController + + +class UnapprovedPointView(UnprotectedRestView): + """The unapproved_point view.""" + def get_controller(self): + """Return an instance of the unapproved_point controller.""" + return UnapprovedPointController() + + def post(self): + """Post a list of unapproved_point rows""" + result = self.get_controller().post(self.request_json(), request.args) + parsed_result = [] + for model in result: + parsed_result.append(self.parse(model)) + return self.json(parsed_result, 201) diff --git a/app/views/unapproved_window_door.py b/app/views/unapproved_window_door.py new file mode 100644 index 0000000..dafc94d --- /dev/null +++ b/app/views/unapproved_window_door.py @@ -0,0 +1,21 @@ +"""Views for managing unapproved_point data.""" +from werkzeug.exceptions import MethodNotAllowed +from flask import request + +from app.views.base import UnprotectedRestView +from app.controllers.unapproved_point import UnapprovedPointController + + +class UnapprovedPointView(UnprotectedRestView): + """The unapproved_point view.""" + def get_controller(self): + """Return an instance of the unapproved_point controller.""" + return UnapprovedPointController() + + def post(self): + """Post a list of unapproved_point rows""" + result = self.get_controller().post(self.request_json(), request.args) + parsed_result = [] + for model in result: + parsed_result.append(self.parse(model)) + return self.json(parsed_result, 201) -- GitLab From 4f2544c9de3295882a707b9f4cb3294f19e01f33 Mon Sep 17 00:00:00 2001 From: Conrad S Date: Tue, 14 Mar 2017 17:54:56 -0400 Subject: [PATCH 02/14] Add endpoints for window_door and building_dimensions --- .../unapproved_building_dimensions.py | 51 +++++-------------- app/controllers/unapproved_window_door.py | 49 +++++------------- app/forms/unapproved_building_dimensions.py | 14 +++-- app/forms/unapproved_window_door.py | 14 ++--- app/models/unapproved_building_dimensions.py | 18 +++---- app/models/unapproved_window_door.py | 18 +++---- app/views/__init__.py | 8 ++- app/views/unapproved_building_dimensions.py | 21 ++++---- app/views/unapproved_window_door.py | 14 ++--- 9 files changed, 78 insertions(+), 129 deletions(-) diff --git a/app/controllers/unapproved_building_dimensions.py b/app/controllers/unapproved_building_dimensions.py index a85e5a0..2566047 100644 --- a/app/controllers/unapproved_building_dimensions.py +++ b/app/controllers/unapproved_building_dimensions.py @@ -1,62 +1,35 @@ -"""Controllers for managing top-level unapproved_points.""" +"""Controllers for managing top-level unapproved_building_dimensions.""" from sqlalchemy import and_ from werkzeug.exceptions import BadRequest from flask import current_app from app.lib.database import db, proc from app.controllers.base import RestController -from app.models.unapproved_point import UnapprovedPoint -from app.forms.unapproved_point import UnapprovedPointForm +from app.models.unapproved_building_dimensions import UnapprovedBuildingDimensions +from app.forms.unapproved_building_dimensions import UnapprovedBuildingDimensionsForm -class UnapprovedPointController(RestController): - """The unapproved_point controller.""" - Model = UnapprovedPoint +class UnapprovedBuildingDimensionsController(RestController): + """The unapproved_building_dimensions controller.""" + Model = UnapprovedBuildingDimensions filters = { 'hit_id[]': \ - lambda d: UnapprovedPoint.hit_id.in_(d.getlist('hit_id[]')) + lambda d: UnapprovedBuildingDimensions.hit_id.in_(d.getlist('hit_id[]')) } def get_form(self, filter_data): - """Return the unapproved_point form.""" - return UnapprovedPointForm + """Return the unapproved_building_dimensions form.""" + return UnapprovedBuildingDimensionsForm def post(self, data, filter_data): """Post a new model. """ - # Will want to loop through the inputted unapproved_points try: - #self.delete(filter_data["account_id"], None) - model_list = [] - for point in data: - model = super(UnapprovedPointController, self).post(point, filter_data) - model_list.append(model) - return model_list + model = super(UnapprovedBuildingDimensionsController, self).post(data, filter_data) + return model except Exception as e: raise ( e if current_app.config["DEBUG"] else - BadRequest("Error while posting unapproved_point data") + BadRequest("Error while posting unapproved_building_dimensions data") ) - -# def delete(self, id_, filter_data): -# -# try: -# query = "select * from public.{}({})".format( -# 'delete_unapproved_point', -# 'in_account_id := {}'.format(id_) -# ) -# results = db.session.execute(query) -# db.session.commit() -# except Exception as e: -# raise ( -# e if current_app.config['DEBUG'] else -# BadRequest('Error while executing db call') -# ) -# return_list = [] -# for row in results: -# print(row) -# d = dict(row.items()) -# return_list.append(d) -# print(return_list) -# return return_list diff --git a/app/controllers/unapproved_window_door.py b/app/controllers/unapproved_window_door.py index a85e5a0..250810c 100644 --- a/app/controllers/unapproved_window_door.py +++ b/app/controllers/unapproved_window_door.py @@ -1,62 +1,39 @@ -"""Controllers for managing top-level unapproved_points.""" +"""Controllers for managing top-level unapproved_window_doors.""" from sqlalchemy import and_ from werkzeug.exceptions import BadRequest from flask import current_app from app.lib.database import db, proc from app.controllers.base import RestController -from app.models.unapproved_point import UnapprovedPoint -from app.forms.unapproved_point import UnapprovedPointForm +from app.models.unapproved_window_door import UnapprovedWindowDoor +from app.forms.unapproved_window_door import UnapprovedWindowDoorForm -class UnapprovedPointController(RestController): - """The unapproved_point controller.""" - Model = UnapprovedPoint +class UnapprovedWindowDoorController(RestController): + """The unapproved_window_door controller.""" + Model = UnapprovedWindowDoor filters = { 'hit_id[]': \ - lambda d: UnapprovedPoint.hit_id.in_(d.getlist('hit_id[]')) + lambda d: UnapprovedWindowDoor.hit_id.in_(d.getlist('hit_id[]')) } def get_form(self, filter_data): - """Return the unapproved_point form.""" - return UnapprovedPointForm + """Return the unapproved_window_door form.""" + return UnapprovedWindowDoorForm def post(self, data, filter_data): """Post a new model. """ - # Will want to loop through the inputted unapproved_points + # Will want to loop through the inputted unapproved_window_doors try: - #self.delete(filter_data["account_id"], None) model_list = [] - for point in data: - model = super(UnapprovedPointController, self).post(point, filter_data) + for item in data: + model = super(UnapprovedWindowDoorController, self).post(item, filter_data) model_list.append(model) return model_list except Exception as e: raise ( e if current_app.config["DEBUG"] else - BadRequest("Error while posting unapproved_point data") + BadRequest("Error while posting unapproved_window_door data") ) - -# def delete(self, id_, filter_data): -# -# try: -# query = "select * from public.{}({})".format( -# 'delete_unapproved_point', -# 'in_account_id := {}'.format(id_) -# ) -# results = db.session.execute(query) -# db.session.commit() -# except Exception as e: -# raise ( -# e if current_app.config['DEBUG'] else -# BadRequest('Error while executing db call') -# ) -# return_list = [] -# for row in results: -# print(row) -# d = dict(row.items()) -# return_list.append(d) -# print(return_list) -# return return_list diff --git a/app/forms/unapproved_building_dimensions.py b/app/forms/unapproved_building_dimensions.py index c27ff33..0e08e8f 100644 --- a/app/forms/unapproved_building_dimensions.py +++ b/app/forms/unapproved_building_dimensions.py @@ -1,16 +1,14 @@ import wtforms as wtf -class UnapprovedPointForm(wtf.Form): - """ A form for validating unapproved_point requests.""" +class UnapprovedBuildingDimensionsForm(wtf.Form): + """ A form for validating unapproved_building_dimensions requests.""" hit_id = wtf.IntegerField( validators=[wtf.validators.Required()]) - latitude = wtf.DecimalField() - longitude = wtf.DecimalField() - elevation = wtf.IntegerField() - corresponding_height = wtf.IntegerField() - adjacent = wtf.BooleanField() - feature = wtf.IntegerField() + perimeter = wtf.DecimalField() + area = wtf.DecimalField() + num_floors = wtf.IntegerField() + ground_elevation = wtf.IntegerField() building_id = wtf.IntegerField() diff --git a/app/forms/unapproved_window_door.py b/app/forms/unapproved_window_door.py index c27ff33..2d9650a 100644 --- a/app/forms/unapproved_window_door.py +++ b/app/forms/unapproved_window_door.py @@ -1,16 +1,16 @@ import wtforms as wtf -class UnapprovedPointForm(wtf.Form): - """ A form for validating unapproved_point requests.""" +class UnapprovedWindowDoorForm(wtf.Form): + """ A form for validating unapproved_window_door requests.""" hit_id = wtf.IntegerField( validators=[wtf.validators.Required()]) - latitude = wtf.DecimalField() - longitude = wtf.DecimalField() - elevation = wtf.IntegerField() - corresponding_height = wtf.IntegerField() - adjacent = wtf.BooleanField() + height = wtf.DecimalField() + width = wtf.DecimalField() + quantity = wtf.IntegerField() + orientation = wtf.IntegerField() + direction = wtf.IntegerField() feature = wtf.IntegerField() building_id = wtf.IntegerField() diff --git a/app/models/unapproved_building_dimensions.py b/app/models/unapproved_building_dimensions.py index 6b14651..ae95e71 100644 --- a/app/models/unapproved_building_dimensions.py +++ b/app/models/unapproved_building_dimensions.py @@ -1,26 +1,24 @@ -"""Models for dealing with an unapproved_point.""" +"""Models for dealing with an unapproved_building_dimensions.""" from sqlalchemy.ext.declarative import declared_attr from ..lib.database import ProcColumn, ProcTable, db from .base import Model -class UnapprovedPoint(Model, db.Model): +class UnapprovedBuildingDimensions(Model, db.Model): @declared_attr def __tablename__(cls): - return "unapproved_point" + return "unapproved_building_dimensions" __table_args__ = {"schema": "mechanical_turk"} - latitude = db.Column(db.Float) - longitude = db.Column(db.Float) - elevation = db.Column(db.Integer) - corresponding_height = db.Column(db.Integer) - adjacent = db.Column(db.Boolean) - feature = db.Column(db.Integer) + perimeter = db.Column(db.Float) + area = db.Column(db.Float) + num_floors = db.Column(db.Integer) + ground_elevation = db.Column(db.Integer) hit_id = db.Column(db.Integer, nullable=False) building_id = db.Column(db.Integer) def __str__(self): - return "unapproved_point data for account {}".format(self.hit_id) + return "unapproved_building_dimensions data for account {}".format(self.hit_id) diff --git a/app/models/unapproved_window_door.py b/app/models/unapproved_window_door.py index 6b14651..a3e4344 100644 --- a/app/models/unapproved_window_door.py +++ b/app/models/unapproved_window_door.py @@ -1,26 +1,26 @@ -"""Models for dealing with an unapproved_point.""" +"""Models for dealing with an unapproved_window_door.""" from sqlalchemy.ext.declarative import declared_attr from ..lib.database import ProcColumn, ProcTable, db from .base import Model -class UnapprovedPoint(Model, db.Model): +class UnapprovedWindowDoor(Model, db.Model): @declared_attr def __tablename__(cls): - return "unapproved_point" + return "unapproved_window_door" __table_args__ = {"schema": "mechanical_turk"} - latitude = db.Column(db.Float) - longitude = db.Column(db.Float) - elevation = db.Column(db.Integer) - corresponding_height = db.Column(db.Integer) - adjacent = db.Column(db.Boolean) + height = db.Column(db.Float) + width = db.Column(db.Float) + quantity = db.Column(db.Integer) + orientation = db.Column(db.Integer) + direction = db.Column(db.Integer) feature = db.Column(db.Integer) hit_id = db.Column(db.Integer, nullable=False) building_id = db.Column(db.Integer) def __str__(self): - return "unapproved_point data for account {}".format(self.hit_id) + return "unapproved_window_door data for account {}".format(self.hit_id) diff --git a/app/views/__init__.py b/app/views/__init__.py index 1718e2a..3afb4b1 100644 --- a/app/views/__init__.py +++ b/app/views/__init__.py @@ -1,5 +1,9 @@ """Flask-classy views for the flask application.""" -from . import building, turk_hit, unapproved_point +from . import (building, + turk_hit, + unapproved_point, + unapproved_window_door, + unapproved_building_dimensions) def register(app): @@ -11,3 +15,5 @@ def register(app): building.BuildingView.register(app) turk_hit.TurkHitView.register(app) unapproved_point.UnapprovedPointView.register(app) + unapproved_window_door.UnapprovedWindowDoorView.register(app) + unapproved_building_dimensions.UnapprovedBuildingDimensionsView.register(app) diff --git a/app/views/unapproved_building_dimensions.py b/app/views/unapproved_building_dimensions.py index dafc94d..6283440 100644 --- a/app/views/unapproved_building_dimensions.py +++ b/app/views/unapproved_building_dimensions.py @@ -1,21 +1,18 @@ -"""Views for managing unapproved_point data.""" +"""Views for managing unapproved_building_dimensions data.""" from werkzeug.exceptions import MethodNotAllowed from flask import request from app.views.base import UnprotectedRestView -from app.controllers.unapproved_point import UnapprovedPointController +from app.controllers.unapproved_building_dimensions import UnapprovedBuildingDimensionsController -class UnapprovedPointView(UnprotectedRestView): - """The unapproved_point view.""" +class UnapprovedBuildingDimensionsView(UnprotectedRestView): + """The unapproved_building_dimensions view.""" def get_controller(self): - """Return an instance of the unapproved_point controller.""" - return UnapprovedPointController() + """Return an instance of the unapproved_building_dimensions controller.""" + return UnapprovedBuildingDimensionsController() def post(self): - """Post a list of unapproved_point rows""" - result = self.get_controller().post(self.request_json(), request.args) - parsed_result = [] - for model in result: - parsed_result.append(self.parse(model)) - return self.json(parsed_result, 201) + """Post an unapproved_building_dimensions row""" + model = self.get_controller().post(self.request_json(), request.args) + return self.json(self.parse(model), 201) diff --git a/app/views/unapproved_window_door.py b/app/views/unapproved_window_door.py index dafc94d..5cb81cd 100644 --- a/app/views/unapproved_window_door.py +++ b/app/views/unapproved_window_door.py @@ -1,19 +1,19 @@ -"""Views for managing unapproved_point data.""" +"""Views for managing unapproved_window_door data.""" from werkzeug.exceptions import MethodNotAllowed from flask import request from app.views.base import UnprotectedRestView -from app.controllers.unapproved_point import UnapprovedPointController +from app.controllers.unapproved_window_door import UnapprovedWindowDoorController -class UnapprovedPointView(UnprotectedRestView): - """The unapproved_point view.""" +class UnapprovedWindowDoorView(UnprotectedRestView): + """The unapproved_window_door view.""" def get_controller(self): - """Return an instance of the unapproved_point controller.""" - return UnapprovedPointController() + """Return an instance of the unapproved_window_door controller.""" + return UnapprovedWindowDoorController() def post(self): - """Post a list of unapproved_point rows""" + """Post a list of unapproved_window_door rows""" result = self.get_controller().post(self.request_json(), request.args) parsed_result = [] for model in result: -- GitLab From 4c2c32f3278793bd88b324da161bc61076c33c8f Mon Sep 17 00:00:00 2001 From: Conrad S Date: Wed, 15 Mar 2017 13:20:29 -0400 Subject: [PATCH 03/14] Save unapproved dimensions data to database --- app/controllers/base.py | 2 + app/controllers/turk_hit.py | 178 +++++++++++++++++++++++++++- app/controllers/unapproved_point.py | 22 ---- app/models/turk_hit.py | 3 + requirements.txt | 3 +- 5 files changed, 183 insertions(+), 25 deletions(-) diff --git a/app/controllers/base.py b/app/controllers/base.py index 8d5ad7a..71137a5 100644 --- a/app/controllers/base.py +++ b/app/controllers/base.py @@ -83,11 +83,13 @@ class RestController(object): raise BadRequest(form.errors) form.populate_obj(model) + return model def post(self, data, filter_data): """Post a new model from a dictionary.""" model = self.create(data, filter_data) + db.session.add(model) try: commit() diff --git a/app/controllers/turk_hit.py b/app/controllers/turk_hit.py index 79463aa..b08cbfa 100644 --- a/app/controllers/turk_hit.py +++ b/app/controllers/turk_hit.py @@ -3,13 +3,17 @@ import base64 import json import requests import mimetypes +import xlrd from flask import current_app from werkzeug.datastructures import MultiDict from werkzeug.exceptions import NotFound, BadRequest from .base import RestController -from ..lib.database import proc +from .unapproved_point import UnapprovedPointController +from .unapproved_window_door import UnapprovedWindowDoorController +from .unapproved_building_dimensions import UnapprovedBuildingDimensionsController +from ..lib.database import proc, db from ..lib.mech_turk import create_hit, approve_or_reject_hit,\ get_hit_status from ..lib.service import services @@ -36,6 +40,20 @@ class TurkHitController(RestController): # The above dictionary with the keys as values and the values as keys STATUS_DICT_TEXT = {v: k for k, v in STATUS_DICT_ID.items()} + DIRECTION_DICT = { + 'South': 1, 'front': 1, + 'East': 2, 'right': 2, + 'West': 3, 'left': 3, + 'North': 4, 'back': 4 + } + + FEATURE_DICT = { + 'Door': 1, + 'Window': 2, + 'Building Point': 3, + 'Roof Point': 4 + } + def get_post_form(self, filter_data): """Return the turk_hit form.""" return TurkHitPostForm @@ -71,11 +89,21 @@ class TurkHitController(RestController): if not hit_list: return [] + # Get the dimensions for all of the previous hits + for hit in hit_list[1:]: + dimen = self.get_dimensions_data(hit.db_id) + if (dimen['building_dimensions']): + hit.dimensions = dimen + cur_hit = hit_list[0] + # Get the dimensions for the cur hit if they exist + dimensions = self.get_dimensions_data(cur_hit.db_id) + # No need to check in with amazon if we've already reached one of those states status_string = self.STATUS_DICT_ID[cur_hit.status_id] if status_string in ["Accepted", "Rejected", "Disposed", "Expired"]: + cur_hit.dimensions = dimensions return hit_list # Get the new hit status from AWS @@ -84,7 +112,10 @@ class TurkHitController(RestController): # No changes if the status is the same if new_hit_status == status_string: + if (dimensions['building_dimensions']): + cur_hit.dimensions = dimensions return hit_list + # Download file here if result['file_url'] and not cur_hit.csv_document_key: response = requests.get(result['file_url']) @@ -111,6 +142,9 @@ class TurkHitController(RestController): # Download the file document = self.download_file(cur_hit.building_id, folder_path, file_name, response.content) + dimensions = self.save_dimensions_data(response.content, + cur_hit.db_id, + cur_hit.building_id) # Get the key to update the model cur_hit.csv_document_key = document['key'] @@ -121,11 +155,14 @@ class TurkHitController(RestController): proc(self.Model, self.Model.PROCS['UPDATE'], **cur_hit.get_dictionary()) + pass except Exception as err: raise ( err if current_app.config['DEBUG'] else BadRequest('Error while executing db call') ) + if (dimensions['building_dimensions']): + cur_hit.dimensions = dimensions return hit_list def download_file(self, building_id, folder_path, file_name, byte_data): @@ -153,6 +190,142 @@ class TurkHitController(RestController): document = response.json()['data'] return document + def save_dimensions_data(self, contents, hit_db_id, building_id): + """ + A function to save dimensions data from the completed mech turk hit + + Relies on a specific formatting of excel documents given to the worker + + """ + workbook = xlrd.open_workbook(file_contents=contents) + worksheet = workbook.sheet_by_index(0) + + perimeter = worksheet.cell(4, 1).value + area = worksheet.cell(5, 1).value + num_floors = worksheet.cell(4, 3).value + ground_elevation = worksheet.cell(5, 3).value + + building_dimensions = {'perimeter': float(perimeter), + 'area': float(area), + 'num_floors': int(num_floors), + 'ground_elevation': int(ground_elevation), + 'hit_id': hit_db_id, + 'building_id': building_id} + + point_list = [] + # building and roof points have seperate starting places on the excel spreadsheet, + start_points = [{'row_start': 8, 'column_start': 1, + 'feature': self.FEATURE_DICT['Building Point']}, + {'row_start': 14, 'column_start': 8, + 'feature': self.FEATURE_DICT['Roof Point']}] + # Get the building and roof points + for start_point in start_points: + row_start = start_point['row_start'] + column_start = start_point['column_start'] + row_counter = 0 + lat = worksheet.cell(row_start, column_start).value + while lat: + column_counter = 1 + longt = worksheet.cell(row_start + row_counter, column_start + column_counter).value + column_counter += 1 + elevation = worksheet.cell(row_start + row_counter, column_start + column_counter).value + column_counter += 1 + height = worksheet.cell(row_start + row_counter, column_start + column_counter).value + column_counter += 1 + adjacent = worksheet.cell(row_start + row_counter, column_start + column_counter).value + # Roof points don't have an adjacent field, this cell will be an empty string + if not adjacent: + adjacent = None + else: + if (adjacent == 'yes' + or adjacent == 'Yes' + or adjacent == 'YES'): + adjacent = 'true' + else: + # wtforms expects a json object, thus we can't use python bools + adjacent = 'false' + point_list.append({'latitude': float(lat), 'longitude': float(longt), + 'elevation': int(elevation), + 'corresponding_height': int(height), + 'adjacent': adjacent, 'feature': start_point['feature'], + 'hit_id': hit_db_id, 'building_id': building_id}) + row_counter += 1 + lat = worksheet.cell(row_start + row_counter, column_start).value + + windows_and_doors = [] + # Window and door data has many start points + start_points = [{'row_start': 6, 'column_start': 13, + 'direction': self.DIRECTION_DICT['front']}, + {'row_start': 6, 'column_start': 19, + 'direction': self.DIRECTION_DICT['back']}, + {'row_start': 20, 'column_start': 13, + 'direction': self.DIRECTION_DICT['left']}, + {'row_start': 20, 'column_start': 19, + 'direction': self.DIRECTION_DICT['right']}] + # Get Window & Door data + for start_point in start_points: + row_start = start_point['row_start'] + column_start = start_point['column_start'] + row_counter = 0 + feature_string = worksheet.cell(row_start, column_start).value + while feature_string: + # The cell telling us whether or not it is a door or a window is 1 before height + if feature_string.startswith('Window'): + feature = self.FEATURE_DICT['Window'] + elif feature_string.startswith('Door'): + feature = self.FEATURE_DICT['Door'] + else: + raise + # We shouldn't be in this row + break + column_counter = 1 + height = worksheet.cell(row_start + row_counter, column_start + column_counter).value + column_counter += 1 + width = worksheet.cell(row_start + row_counter, column_start + column_counter).value + column_counter += 1 + quantity = worksheet.cell(row_start + row_counter, column_start + column_counter).value + column_counter += 1 + orientation = worksheet.cell(row_start + row_counter, column_start + column_counter).value + if (height): + windows_and_doors.append({'height': float(height), + 'width': float(width), + 'quantity': int(quantity), + 'orientation': self.DIRECTION_DICT[orientation], + 'feature': feature, + 'direction': start_point['direction'], + 'hit_id': hit_db_id, + 'building_id': building_id}) + row_counter += 1 + feature_string = worksheet.cell(row_start + row_counter, column_start).value + + # Post new dimensions data to the database + UnapprovedWindowDoorController().post(windows_and_doors, None) + UnapprovedPointController().post(point_list, None) + UnapprovedBuildingDimensionsController().post(building_dimensions, None) + + return {'windows_doors': windows_and_doors, + 'points': point_list, + 'building_dimensions': building_dimensions} + + def get_dimensions_data(self, hit_id): + """ + Get the unapproved dimensions for a hit + """ + args = MultiDict([('hit_id[]', hit_id)]) + windows_doors = UnapprovedPointController().index(args) + points = UnapprovedWindowDoorController().index(args) + building_dimensions = UnapprovedBuildingDimensionsController().index(args) + + dimensions = {} + dimensions["windows_doors"] = [model.get_dictionary() + for model in windows_doors] + dimensions["points"] = [model.get_dictionary() + for model in points] + dimensions["building_dimensions"] = building_dimensions[0].get_dictionary() if building_dimensions else [] + return dimensions + + + def post(self, data, filter_data): """ @@ -204,7 +377,6 @@ class TurkHitController(RestController): if not hit_list: raise NotFound("Hit was not found in the database") - return hit_list[0] @@ -227,6 +399,7 @@ class TurkHitController(RestController): raise BadRequest("The URL id and the body db_id are not the same") approve = data.pop("approve") + dimensions = data.pop("dimensions") # Approve or reject the hit on the amazon mech turk side if approve_or_reject_hit(data["amazon_hit_id"], @@ -249,6 +422,7 @@ class TurkHitController(RestController): err if current_app.config['DEBUG'] else BadRequest('Error while executing db call') ) + hit_list[0].dimensions = dimensions return hit_list[0] else: raise BadRequest("There were no completed assignments to approve or reject") diff --git a/app/controllers/unapproved_point.py b/app/controllers/unapproved_point.py index a85e5a0..dc4598b 100644 --- a/app/controllers/unapproved_point.py +++ b/app/controllers/unapproved_point.py @@ -38,25 +38,3 @@ class UnapprovedPointController(RestController): e if current_app.config["DEBUG"] else BadRequest("Error while posting unapproved_point data") ) - -# def delete(self, id_, filter_data): -# -# try: -# query = "select * from public.{}({})".format( -# 'delete_unapproved_point', -# 'in_account_id := {}'.format(id_) -# ) -# results = db.session.execute(query) -# db.session.commit() -# except Exception as e: -# raise ( -# e if current_app.config['DEBUG'] else -# BadRequest('Error while executing db call') -# ) -# return_list = [] -# for row in results: -# print(row) -# d = dict(row.items()) -# return_list.append(d) -# print(return_list) -# return return_list diff --git a/app/models/turk_hit.py b/app/models/turk_hit.py index cd3cd2d..97fd8b4 100644 --- a/app/models/turk_hit.py +++ b/app/models/turk_hit.py @@ -12,6 +12,9 @@ class TurkHit(BaseModel): 'UPDATE': 'update_hit', } + def get_dictionary(self): + return self.__dict__ + __table__ = ProcTable( 'TurkHit', diff --git a/requirements.txt b/requirements.txt index ef551a0..6e8a437 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,6 +1,7 @@ arrow==0.7.0 beautifulsoup4==4.4.1 blessed==1.9.5 +boto==2.45.0 botocore==1.3.28 cement==2.4.0 colorama==0.3.3 @@ -40,4 +41,4 @@ wcwidth==0.1.6 websocket-client==0.35.0 Werkzeug==0.11.4 WTForms==2.1 -boto==2.45.0 +xlrd==1.0.0 -- GitLab From 7e1219721f150de363cae0fdd99f8a9b0229d331 Mon Sep 17 00:00:00 2001 From: Conrad S Date: Wed, 15 Mar 2017 13:56:33 -0400 Subject: [PATCH 04/14] Switch to restview --- app/views/unapproved_building_dimensions.py | 4 ++-- app/views/unapproved_point.py | 4 ++-- app/views/unapproved_window_door.py | 4 ++-- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/app/views/unapproved_building_dimensions.py b/app/views/unapproved_building_dimensions.py index 6283440..2313281 100644 --- a/app/views/unapproved_building_dimensions.py +++ b/app/views/unapproved_building_dimensions.py @@ -2,11 +2,11 @@ from werkzeug.exceptions import MethodNotAllowed from flask import request -from app.views.base import UnprotectedRestView +from app.views.base import RestView from app.controllers.unapproved_building_dimensions import UnapprovedBuildingDimensionsController -class UnapprovedBuildingDimensionsView(UnprotectedRestView): +class UnapprovedBuildingDimensionsView(RestView): """The unapproved_building_dimensions view.""" def get_controller(self): """Return an instance of the unapproved_building_dimensions controller.""" diff --git a/app/views/unapproved_point.py b/app/views/unapproved_point.py index dafc94d..6459a31 100644 --- a/app/views/unapproved_point.py +++ b/app/views/unapproved_point.py @@ -2,11 +2,11 @@ from werkzeug.exceptions import MethodNotAllowed from flask import request -from app.views.base import UnprotectedRestView +from app.views.base import RestView from app.controllers.unapproved_point import UnapprovedPointController -class UnapprovedPointView(UnprotectedRestView): +class UnapprovedPointView(RestView): """The unapproved_point view.""" def get_controller(self): """Return an instance of the unapproved_point controller.""" diff --git a/app/views/unapproved_window_door.py b/app/views/unapproved_window_door.py index 5cb81cd..1a257eb 100644 --- a/app/views/unapproved_window_door.py +++ b/app/views/unapproved_window_door.py @@ -2,11 +2,11 @@ from werkzeug.exceptions import MethodNotAllowed from flask import request -from app.views.base import UnprotectedRestView +from app.views.base import RestView from app.controllers.unapproved_window_door import UnapprovedWindowDoorController -class UnapprovedWindowDoorView(UnprotectedRestView): +class UnapprovedWindowDoorView(RestView): """The unapproved_window_door view.""" def get_controller(self): """Return an instance of the unapproved_window_door controller.""" -- GitLab From 72d345295609908b214f2ec1185feddc8e15e393 Mon Sep 17 00:00:00 2001 From: Conrad S Date: Tue, 21 Mar 2017 11:59:15 -0400 Subject: [PATCH 05/14] Update return value for building_dimensions --- app/controllers/turk_hit.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/app/controllers/turk_hit.py b/app/controllers/turk_hit.py index b08cbfa..041d873 100644 --- a/app/controllers/turk_hit.py +++ b/app/controllers/turk_hit.py @@ -312,8 +312,8 @@ class TurkHitController(RestController): Get the unapproved dimensions for a hit """ args = MultiDict([('hit_id[]', hit_id)]) - windows_doors = UnapprovedPointController().index(args) - points = UnapprovedWindowDoorController().index(args) + windows_doors = UnapprovedWindowDoorController().index(args) + points = UnapprovedPointController().index(args) building_dimensions = UnapprovedBuildingDimensionsController().index(args) dimensions = {} @@ -321,7 +321,7 @@ class TurkHitController(RestController): for model in windows_doors] dimensions["points"] = [model.get_dictionary() for model in points] - dimensions["building_dimensions"] = building_dimensions[0].get_dictionary() if building_dimensions else [] + dimensions["building_dimensions"] = building_dimensions[0].get_dictionary() if building_dimensions else {} return dimensions -- GitLab From 858522cc7c9e4a21af3de1c3b68550c5fd4cb429 Mon Sep 17 00:00:00 2001 From: Conrad S Date: Tue, 21 Mar 2017 12:18:55 -0400 Subject: [PATCH 06/14] Add new parameters to PUT call in GET --- app/controllers/turk_hit.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/app/controllers/turk_hit.py b/app/controllers/turk_hit.py index 041d873..8064a8a 100644 --- a/app/controllers/turk_hit.py +++ b/app/controllers/turk_hit.py @@ -129,7 +129,15 @@ class TurkHitController(RestController): 'the uploaded file was not in the correct file '\ 'format. The file must be of type .xlsx.' put_body = {**cur_hit.get_dictionary(), - **{'approve': 0, 'response_message': response_message}} + **{'approve': 0, + 'response_message': response_message, + 'dimensions': { + 'windows_doors': [], + 'points': [], + 'building_dimensions': {} + } + } + } self.put(cur_hit.db_id, put_body, None) cur_hit.response_message = response_message new_hit_status = "Rejected" -- GitLab From bb8fadb01a642a3062cdf1acccbbc58e0bc37f10 Mon Sep 17 00:00:00 2001 From: Conrad S Date: Tue, 21 Mar 2017 17:38:41 -0400 Subject: [PATCH 07/14] Add all cases for cardinal directions --- app/controllers/turk_hit.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/app/controllers/turk_hit.py b/app/controllers/turk_hit.py index 8064a8a..865958d 100644 --- a/app/controllers/turk_hit.py +++ b/app/controllers/turk_hit.py @@ -41,10 +41,10 @@ class TurkHitController(RestController): STATUS_DICT_TEXT = {v: k for k, v in STATUS_DICT_ID.items()} DIRECTION_DICT = { - 'South': 1, 'front': 1, - 'East': 2, 'right': 2, - 'West': 3, 'left': 3, - 'North': 4, 'back': 4 + 'South': 1, 'south': 1, 'front': 1, + 'East': 2, 'east': 2, 'right': 2, + 'West': 3, 'west': 3, 'left': 3, + 'North': 4, 'north': 4, 'back': 4, } FEATURE_DICT = { -- GitLab From 0bb6513b6e8b791da47652baf473c10d7467a5b5 Mon Sep 17 00:00:00 2001 From: Conrad S Date: Wed, 22 Mar 2017 15:01:06 -0400 Subject: [PATCH 08/14] Update to fit new tech spec and database --- app/controllers/turk_hit.py | 51 +++++++++----------- app/forms/unapproved_building_dimensions.py | 4 ++ app/forms/unapproved_point.py | 1 - app/models/unapproved_building_dimensions.py | 4 ++ app/models/unapproved_point.py | 1 - 5 files changed, 30 insertions(+), 31 deletions(-) diff --git a/app/controllers/turk_hit.py b/app/controllers/turk_hit.py index 865958d..8616bf3 100644 --- a/app/controllers/turk_hit.py +++ b/app/controllers/turk_hit.py @@ -131,11 +131,6 @@ class TurkHitController(RestController): put_body = {**cur_hit.get_dictionary(), **{'approve': 0, 'response_message': response_message, - 'dimensions': { - 'windows_doors': [], - 'points': [], - 'building_dimensions': {} - } } } self.put(cur_hit.db_id, put_body, None) @@ -150,9 +145,15 @@ class TurkHitController(RestController): # Download the file document = self.download_file(cur_hit.building_id, folder_path, file_name, response.content) - dimensions = self.save_dimensions_data(response.content, + try: + dimensions = self.save_dimensions_data(response.content, cur_hit.db_id, cur_hit.building_id) + except Exception as e: + if current_app.config['DEBUG']: + raise e + print("Failed to parse the dimensions file") + print(vars(e)) # Get the key to update the model cur_hit.csv_document_key = document['key'] @@ -163,7 +164,6 @@ class TurkHitController(RestController): proc(self.Model, self.Model.PROCS['UPDATE'], **cur_hit.get_dictionary()) - pass except Exception as err: raise ( err if current_app.config['DEBUG'] else @@ -212,19 +212,27 @@ class TurkHitController(RestController): area = worksheet.cell(5, 1).value num_floors = worksheet.cell(4, 3).value ground_elevation = worksheet.cell(5, 3).value + north_adjacency = worksheet.cell(5, 16).value + south_adjacency = worksheet.cell(5, 22).value + west_adjacency = worksheet.cell(20, 16).value + east_adjacency = worksheet.cell(20, 22).value building_dimensions = {'perimeter': float(perimeter), 'area': float(area), 'num_floors': int(num_floors), 'ground_elevation': int(ground_elevation), 'hit_id': hit_db_id, - 'building_id': building_id} + 'building_id': building_id, + 'north_adjacency': north_adjacency, + 'south_adjacency': south_adjacency, + 'west_adjacency': west_adjacency, + 'east_adjacency': east_adjacency} point_list = [] # building and roof points have seperate starting places on the excel spreadsheet, start_points = [{'row_start': 8, 'column_start': 1, 'feature': self.FEATURE_DICT['Building Point']}, - {'row_start': 14, 'column_start': 8, + {'row_start': 14, 'column_start': 7, 'feature': self.FEATURE_DICT['Roof Point']}] # Get the building and roof points for start_point in start_points: @@ -239,36 +247,23 @@ class TurkHitController(RestController): elevation = worksheet.cell(row_start + row_counter, column_start + column_counter).value column_counter += 1 height = worksheet.cell(row_start + row_counter, column_start + column_counter).value - column_counter += 1 - adjacent = worksheet.cell(row_start + row_counter, column_start + column_counter).value - # Roof points don't have an adjacent field, this cell will be an empty string - if not adjacent: - adjacent = None - else: - if (adjacent == 'yes' - or adjacent == 'Yes' - or adjacent == 'YES'): - adjacent = 'true' - else: - # wtforms expects a json object, thus we can't use python bools - adjacent = 'false' point_list.append({'latitude': float(lat), 'longitude': float(longt), 'elevation': int(elevation), 'corresponding_height': int(height), - 'adjacent': adjacent, 'feature': start_point['feature'], + 'feature': start_point['feature'], 'hit_id': hit_db_id, 'building_id': building_id}) row_counter += 1 lat = worksheet.cell(row_start + row_counter, column_start).value windows_and_doors = [] # Window and door data has many start points - start_points = [{'row_start': 6, 'column_start': 13, + start_points = [{'row_start': 7, 'column_start': 12, 'direction': self.DIRECTION_DICT['front']}, - {'row_start': 6, 'column_start': 19, + {'row_start': 7, 'column_start': 18, 'direction': self.DIRECTION_DICT['back']}, - {'row_start': 20, 'column_start': 13, + {'row_start': 22, 'column_start': 12, 'direction': self.DIRECTION_DICT['left']}, - {'row_start': 20, 'column_start': 19, + {'row_start': 22, 'column_start': 18, 'direction': self.DIRECTION_DICT['right']}] # Get Window & Door data for start_point in start_points: @@ -407,7 +402,6 @@ class TurkHitController(RestController): raise BadRequest("The URL id and the body db_id are not the same") approve = data.pop("approve") - dimensions = data.pop("dimensions") # Approve or reject the hit on the amazon mech turk side if approve_or_reject_hit(data["amazon_hit_id"], @@ -430,7 +424,6 @@ class TurkHitController(RestController): err if current_app.config['DEBUG'] else BadRequest('Error while executing db call') ) - hit_list[0].dimensions = dimensions return hit_list[0] else: raise BadRequest("There were no completed assignments to approve or reject") diff --git a/app/forms/unapproved_building_dimensions.py b/app/forms/unapproved_building_dimensions.py index 0e08e8f..d852c1e 100644 --- a/app/forms/unapproved_building_dimensions.py +++ b/app/forms/unapproved_building_dimensions.py @@ -11,4 +11,8 @@ class UnapprovedBuildingDimensionsForm(wtf.Form): num_floors = wtf.IntegerField() ground_elevation = wtf.IntegerField() building_id = wtf.IntegerField() + north_adjacency = wtf.DecimalField() + south_adjacency = wtf.DecimalField() + west_adjacency = wtf.DecimalField() + east_adjacency = wtf.DecimalField() diff --git a/app/forms/unapproved_point.py b/app/forms/unapproved_point.py index c27ff33..eabf729 100644 --- a/app/forms/unapproved_point.py +++ b/app/forms/unapproved_point.py @@ -10,7 +10,6 @@ class UnapprovedPointForm(wtf.Form): longitude = wtf.DecimalField() elevation = wtf.IntegerField() corresponding_height = wtf.IntegerField() - adjacent = wtf.BooleanField() feature = wtf.IntegerField() building_id = wtf.IntegerField() diff --git a/app/models/unapproved_building_dimensions.py b/app/models/unapproved_building_dimensions.py index ae95e71..e4a59c9 100644 --- a/app/models/unapproved_building_dimensions.py +++ b/app/models/unapproved_building_dimensions.py @@ -19,6 +19,10 @@ class UnapprovedBuildingDimensions(Model, db.Model): ground_elevation = db.Column(db.Integer) hit_id = db.Column(db.Integer, nullable=False) building_id = db.Column(db.Integer) + north_adjacency = db.Column(db.Float) + south_adjacency = db.Column(db.Float) + west_adjacency = db.Column(db.Float) + east_adjacency = db.Column(db.Float) def __str__(self): return "unapproved_building_dimensions data for account {}".format(self.hit_id) diff --git a/app/models/unapproved_point.py b/app/models/unapproved_point.py index 6b14651..e44674d 100644 --- a/app/models/unapproved_point.py +++ b/app/models/unapproved_point.py @@ -17,7 +17,6 @@ class UnapprovedPoint(Model, db.Model): longitude = db.Column(db.Float) elevation = db.Column(db.Integer) corresponding_height = db.Column(db.Integer) - adjacent = db.Column(db.Boolean) feature = db.Column(db.Integer) hit_id = db.Column(db.Integer, nullable=False) building_id = db.Column(db.Integer) -- GitLab From 30ecb83d7464c67eb54cc557c820bc1ef48b55b7 Mon Sep 17 00:00:00 2001 From: Conrad S Date: Wed, 22 Mar 2017 15:09:04 -0400 Subject: [PATCH 09/14] Fix pep8 errors --- app/controllers/turk_hit.py | 25 +++++++++------------ app/forms/turk_hit.py | 2 +- app/forms/unapproved_building_dimensions.py | 1 - app/forms/unapproved_point.py | 1 - app/forms/unapproved_window_door.py | 1 - app/models/turk_hit.py | 3 +-- 6 files changed, 12 insertions(+), 21 deletions(-) diff --git a/app/controllers/turk_hit.py b/app/controllers/turk_hit.py index 8616bf3..6cf53bf 100644 --- a/app/controllers/turk_hit.py +++ b/app/controllers/turk_hit.py @@ -21,8 +21,6 @@ from ..models.turk_hit import TurkHit from ..forms.turk_hit import TurkHitPostForm, TurkHitPutForm - - class TurkHitController(RestController): """A turk_hit controller.""" Model = TurkHit @@ -78,7 +76,9 @@ class TurkHitController(RestController): if not address: raise BadRequest("Please ensure there is an address in the url params") try: - hit_list = proc(self.Model, self.Model.PROCS['READ'], **{'building_id': id_}) + hit_list = proc(self.Model, + self.Model.PROCS['READ'], + **{'building_id': id_}) except Exception as err: raise ( @@ -100,7 +100,7 @@ class TurkHitController(RestController): # Get the dimensions for the cur hit if they exist dimensions = self.get_dimensions_data(cur_hit.db_id) - # No need to check in with amazon if we've already reached one of those states + # No need to check in with amazon if we've already reached these states status_string = self.STATUS_DICT_ID[cur_hit.status_id] if status_string in ["Accepted", "Rejected", "Disposed", "Expired"]: cur_hit.dimensions = dimensions @@ -130,9 +130,7 @@ class TurkHitController(RestController): 'format. The file must be of type .xlsx.' put_body = {**cur_hit.get_dictionary(), **{'approve': 0, - 'response_message': response_message, - } - } + 'response_message': response_message}} self.put(cur_hit.db_id, put_body, None) cur_hit.response_message = response_message new_hit_status = "Rejected" @@ -147,8 +145,8 @@ class TurkHitController(RestController): file_name, response.content) try: dimensions = self.save_dimensions_data(response.content, - cur_hit.db_id, - cur_hit.building_id) + cur_hit.db_id, + cur_hit.building_id) except Exception as e: if current_app.config['DEBUG']: raise e @@ -324,12 +322,11 @@ class TurkHitController(RestController): for model in windows_doors] dimensions["points"] = [model.get_dictionary() for model in points] - dimensions["building_dimensions"] = building_dimensions[0].get_dictionary() if building_dimensions else {} + dimensions["building_dimensions"] = {} + if building_dimensions: + dimensions["building_dimensions"] = building_dimensions[0].get_dictionary() return dimensions - - - def post(self, data, filter_data): """ Create a TurkHit object @@ -382,8 +379,6 @@ class TurkHitController(RestController): raise NotFound("Hit was not found in the database") return hit_list[0] - - def put(self, id_, data, filter_data): """ Update a TurkHit object diff --git a/app/forms/turk_hit.py b/app/forms/turk_hit.py index b16d9b3..1699e91 100644 --- a/app/forms/turk_hit.py +++ b/app/forms/turk_hit.py @@ -36,6 +36,7 @@ class TurkHitPostForm(wtf.Form): reward = wtf.FloatField( validators=[wtf.validators.Required()]) + class TurkHitPutForm(wtf.Form): """ A form for validating turk hits.""" db_id = wtf.IntegerField( @@ -68,4 +69,3 @@ class TurkHitPutForm(wtf.Form): self.response_message.errors.append('If accept is 0 explanation must be nonempty') return False return True - diff --git a/app/forms/unapproved_building_dimensions.py b/app/forms/unapproved_building_dimensions.py index d852c1e..ef46a94 100644 --- a/app/forms/unapproved_building_dimensions.py +++ b/app/forms/unapproved_building_dimensions.py @@ -15,4 +15,3 @@ class UnapprovedBuildingDimensionsForm(wtf.Form): south_adjacency = wtf.DecimalField() west_adjacency = wtf.DecimalField() east_adjacency = wtf.DecimalField() - diff --git a/app/forms/unapproved_point.py b/app/forms/unapproved_point.py index eabf729..c7fe218 100644 --- a/app/forms/unapproved_point.py +++ b/app/forms/unapproved_point.py @@ -12,4 +12,3 @@ class UnapprovedPointForm(wtf.Form): corresponding_height = wtf.IntegerField() feature = wtf.IntegerField() building_id = wtf.IntegerField() - diff --git a/app/forms/unapproved_window_door.py b/app/forms/unapproved_window_door.py index 2d9650a..9864a61 100644 --- a/app/forms/unapproved_window_door.py +++ b/app/forms/unapproved_window_door.py @@ -13,4 +13,3 @@ class UnapprovedWindowDoorForm(wtf.Form): direction = wtf.IntegerField() feature = wtf.IntegerField() building_id = wtf.IntegerField() - diff --git a/app/models/turk_hit.py b/app/models/turk_hit.py index 97fd8b4..0542b1c 100644 --- a/app/models/turk_hit.py +++ b/app/models/turk_hit.py @@ -15,7 +15,6 @@ class TurkHit(BaseModel): def get_dictionary(self): return self.__dict__ - __table__ = ProcTable( 'TurkHit', ProcColumn('db_id'), @@ -32,7 +31,7 @@ class TurkHit(BaseModel): def __init__(self, db_id, building_id, amazon_hit_id, status_id, hit_date, requester_name, csv_document_key, shapefile_document_key, response_message - ): + ): self.db_id = db_id self.building_id = building_id -- GitLab From 569c9518ffde9377ab06aa9be5fc9686f7515ede Mon Sep 17 00:00:00 2001 From: Conrad S Date: Thu, 23 Mar 2017 14:17:38 -0400 Subject: [PATCH 10/14] Add bpeng and fix styling --- app/controllers/turk_hit.py | 227 +++++++++++++----------------------- requirements.txt | 3 +- 2 files changed, 81 insertions(+), 149 deletions(-) diff --git a/app/controllers/turk_hit.py b/app/controllers/turk_hit.py index 6cf53bf..9c0f7e8 100644 --- a/app/controllers/turk_hit.py +++ b/app/controllers/turk_hit.py @@ -3,7 +3,7 @@ import base64 import json import requests import mimetypes -import xlrd +from bpeng import ParseDimensions from flask import current_app from werkzeug.datastructures import MultiDict @@ -32,26 +32,12 @@ class TurkHitController(RestController): 4: 'Accepted', 5: 'Rejected', 6: 'Disposed', - 7: 'Expired' + 7: 'Expired', } # The above dictionary with the keys as values and the values as keys STATUS_DICT_TEXT = {v: k for k, v in STATUS_DICT_ID.items()} - DIRECTION_DICT = { - 'South': 1, 'south': 1, 'front': 1, - 'East': 2, 'east': 2, 'right': 2, - 'West': 3, 'west': 3, 'left': 3, - 'North': 4, 'north': 4, 'back': 4, - } - - FEATURE_DICT = { - 'Door': 1, - 'Window': 2, - 'Building Point': 3, - 'Roof Point': 4 - } - def get_post_form(self, filter_data): """Return the turk_hit form.""" return TurkHitPostForm @@ -70,15 +56,17 @@ class TurkHitController(RestController): address (string): The address of the building for saving to box Returns: list: - dict: TurkHit object + dict: TurkHit object, with additional dimensions data if applicable """ address = filter_data.get('address') if not address: raise BadRequest("Please ensure there is an address in the url params") try: - hit_list = proc(self.Model, - self.Model.PROCS['READ'], - **{'building_id': id_}) + hit_list = proc( + self.Model, + self.Model.PROCS['READ'], + **{'building_id': id_}, + ) except Exception as err: raise ( @@ -128,25 +116,39 @@ class TurkHitController(RestController): response_message = 'This hit was automatically rejected because '\ 'the uploaded file was not in the correct file '\ 'format. The file must be of type .xlsx.' - put_body = {**cur_hit.get_dictionary(), - **{'approve': 0, - 'response_message': response_message}} + put_body = { + **cur_hit.get_dictionary(), + **{ + 'approve': 0, + 'response_message': response_message, + }, + } self.put(cur_hit.db_id, put_body, None) cur_hit.response_message = response_message new_hit_status = "Rejected" else: - file_name = "BuildingDimensions--{}--{}.xlsx".format(cur_hit.building_id, - cur_hit.amazon_hit_id) - folder_path = "/Buildings/{}_{}/Building_Dimensions".format(cur_hit.building_id, - address) + file_name = "BuildingDimensions--{}--{}.xlsx".format( + cur_hit.building_id, + cur_hit.amazon_hit_id, + ) + folder_path = "/Buildings/{}_{}/Building_Dimensions".format( + cur_hit.building_id, + address, + ) # Download the file - document = self.download_file(cur_hit.building_id, folder_path, - file_name, response.content) + document = self.download_file( + cur_hit.building_id, + folder_path, + file_name, + response.content, + ) try: - dimensions = self.save_dimensions_data(response.content, - cur_hit.db_id, - cur_hit.building_id) + dimensions = self.save_dimensions_data( + response.content, + cur_hit.db_id, + cur_hit.building_id, + ) except Exception as e: if current_app.config['DEBUG']: raise e @@ -159,9 +161,11 @@ class TurkHitController(RestController): new_hit_status_id = self.STATUS_DICT_TEXT[new_hit_status] cur_hit.status_id = new_hit_status_id try: - proc(self.Model, - self.Model.PROCS['UPDATE'], - **cur_hit.get_dictionary()) + proc( + self.Model, + self.Model.PROCS['UPDATE'], + **cur_hit.get_dictionary(), + ) except Exception as err: raise ( err if current_app.config['DEBUG'] else @@ -183,7 +187,7 @@ class TurkHitController(RestController): "data": "data:csv/plain;charset=utf-8;base64,{}".format(encoded_string_data), "building_id": str(building_id), "tags": '', - "name": file_name + "name": file_name, } # Call the generic method in the base class response = services.document.post('', '/document/', data=json.dumps(post_data)) @@ -196,117 +200,35 @@ class TurkHitController(RestController): document = response.json()['data'] return document - def save_dimensions_data(self, contents, hit_db_id, building_id): + def save_dimensions_data(self, contents, hit_id, building_id): """ A function to save dimensions data from the completed mech turk hit Relies on a specific formatting of excel documents given to the worker """ - workbook = xlrd.open_workbook(file_contents=contents) - worksheet = workbook.sheet_by_index(0) - - perimeter = worksheet.cell(4, 1).value - area = worksheet.cell(5, 1).value - num_floors = worksheet.cell(4, 3).value - ground_elevation = worksheet.cell(5, 3).value - north_adjacency = worksheet.cell(5, 16).value - south_adjacency = worksheet.cell(5, 22).value - west_adjacency = worksheet.cell(20, 16).value - east_adjacency = worksheet.cell(20, 22).value - - building_dimensions = {'perimeter': float(perimeter), - 'area': float(area), - 'num_floors': int(num_floors), - 'ground_elevation': int(ground_elevation), - 'hit_id': hit_db_id, - 'building_id': building_id, - 'north_adjacency': north_adjacency, - 'south_adjacency': south_adjacency, - 'west_adjacency': west_adjacency, - 'east_adjacency': east_adjacency} - - point_list = [] - # building and roof points have seperate starting places on the excel spreadsheet, - start_points = [{'row_start': 8, 'column_start': 1, - 'feature': self.FEATURE_DICT['Building Point']}, - {'row_start': 14, 'column_start': 7, - 'feature': self.FEATURE_DICT['Roof Point']}] - # Get the building and roof points - for start_point in start_points: - row_start = start_point['row_start'] - column_start = start_point['column_start'] - row_counter = 0 - lat = worksheet.cell(row_start, column_start).value - while lat: - column_counter = 1 - longt = worksheet.cell(row_start + row_counter, column_start + column_counter).value - column_counter += 1 - elevation = worksheet.cell(row_start + row_counter, column_start + column_counter).value - column_counter += 1 - height = worksheet.cell(row_start + row_counter, column_start + column_counter).value - point_list.append({'latitude': float(lat), 'longitude': float(longt), - 'elevation': int(elevation), - 'corresponding_height': int(height), - 'feature': start_point['feature'], - 'hit_id': hit_db_id, 'building_id': building_id}) - row_counter += 1 - lat = worksheet.cell(row_start + row_counter, column_start).value - - windows_and_doors = [] - # Window and door data has many start points - start_points = [{'row_start': 7, 'column_start': 12, - 'direction': self.DIRECTION_DICT['front']}, - {'row_start': 7, 'column_start': 18, - 'direction': self.DIRECTION_DICT['back']}, - {'row_start': 22, 'column_start': 12, - 'direction': self.DIRECTION_DICT['left']}, - {'row_start': 22, 'column_start': 18, - 'direction': self.DIRECTION_DICT['right']}] - # Get Window & Door data - for start_point in start_points: - row_start = start_point['row_start'] - column_start = start_point['column_start'] - row_counter = 0 - feature_string = worksheet.cell(row_start, column_start).value - while feature_string: - # The cell telling us whether or not it is a door or a window is 1 before height - if feature_string.startswith('Window'): - feature = self.FEATURE_DICT['Window'] - elif feature_string.startswith('Door'): - feature = self.FEATURE_DICT['Door'] - else: - raise - # We shouldn't be in this row - break - column_counter = 1 - height = worksheet.cell(row_start + row_counter, column_start + column_counter).value - column_counter += 1 - width = worksheet.cell(row_start + row_counter, column_start + column_counter).value - column_counter += 1 - quantity = worksheet.cell(row_start + row_counter, column_start + column_counter).value - column_counter += 1 - orientation = worksheet.cell(row_start + row_counter, column_start + column_counter).value - if (height): - windows_and_doors.append({'height': float(height), - 'width': float(width), - 'quantity': int(quantity), - 'orientation': self.DIRECTION_DICT[orientation], - 'feature': feature, - 'direction': start_point['direction'], - 'hit_id': hit_db_id, - 'building_id': building_id}) - row_counter += 1 - feature_string = worksheet.cell(row_start + row_counter, column_start).value + windows_doors, points, building_dimensions = ParseDimensions.parse(contents) + for point in points: + point['building_id'] = building_id + point['hit_id'] = hit_id + + for window_door in windows_doors: + window_door['building_id'] = building_id + window_door['hit_id'] = hit_id + + building_dimensions['building_id'] = building_id + building_dimensions['hit_id'] = hit_id # Post new dimensions data to the database - UnapprovedWindowDoorController().post(windows_and_doors, None) - UnapprovedPointController().post(point_list, None) + UnapprovedWindowDoorController().post(windows_doors, None) + UnapprovedPointController().post(points, None) UnapprovedBuildingDimensionsController().post(building_dimensions, None) - return {'windows_doors': windows_and_doors, - 'points': point_list, - 'building_dimensions': building_dimensions} + return { + 'windows_doors': windows_doors, + 'points': points, + 'building_dimensions': building_dimensions, + } def get_dimensions_data(self, hit_id): """ @@ -341,7 +263,6 @@ class TurkHitController(RestController): min_file_bytes (int) instructions_text (string) instructions_url (string) - worksheet_url (string) max_assignments (int) title (string) description (string) @@ -364,11 +285,16 @@ class TurkHitController(RestController): # Store the hit data in the database try: - hit_list = proc(self.Model, self.Model.PROCS['WRITE'], - **{'building_id': building_id, - 'amazon_hit_id': amazon_hit_id, - 'requester_name': requester_name, - 'status_id': self.STATUS_DICT_TEXT['Assignable']}) + hit_list = proc( + self.Model, + self.Model.PROCS['WRITE'], + **{ + 'building_id': building_id, + 'amazon_hit_id': amazon_hit_id, + 'requester_name': requester_name, + 'status_id': self.STATUS_DICT_TEXT['Assignable'], + } + ) except Exception as err: raise ( err if current_app.config['DEBUG'] else @@ -399,9 +325,11 @@ class TurkHitController(RestController): approve = data.pop("approve") # Approve or reject the hit on the amazon mech turk side - if approve_or_reject_hit(data["amazon_hit_id"], - approve, - data["response_message"]): + if approve_or_reject_hit( + data["amazon_hit_id"], + approve, + data["response_message"], + ): # Update the hit data in the database try: approve_id = self.STATUS_DICT_TEXT['Accepted'] @@ -411,8 +339,11 @@ class TurkHitController(RestController): # The data data['status_id'] = status_id - hit_list = proc(self.Model, self.Model.PROCS['UPDATE'], - **data) + hit_list = proc( + self.Model, + self.Model.PROCS['UPDATE'], + **data, + ) except Exception as err: raise ( diff --git a/requirements.txt b/requirements.txt index 6e8a437..641a43c 100644 --- a/requirements.txt +++ b/requirements.txt @@ -41,4 +41,5 @@ wcwidth==0.1.6 websocket-client==0.35.0 Werkzeug==0.11.4 WTForms==2.1 -xlrd==1.0.0 + +git+ssh://git@github.com/Blocp/bpeng.git -- GitLab From 24b29ecb0e40ed5e7ad5ef89455485a91baca2df Mon Sep 17 00:00:00 2001 From: Conrad S Date: Mon, 27 Mar 2017 12:14:30 -0400 Subject: [PATCH 11/14] Remove new lines --- app/controllers/base.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/app/controllers/base.py b/app/controllers/base.py index 71137a5..8d5ad7a 100644 --- a/app/controllers/base.py +++ b/app/controllers/base.py @@ -83,13 +83,11 @@ class RestController(object): raise BadRequest(form.errors) form.populate_obj(model) - return model def post(self, data, filter_data): """Post a new model from a dictionary.""" model = self.create(data, filter_data) - db.session.add(model) try: commit() -- GitLab From 3050e64b40f87de4f1830b33a5e4b2b0b47e28be Mon Sep 17 00:00:00 2001 From: Conrad S Date: Mon, 27 Mar 2017 16:36:59 -0400 Subject: [PATCH 12/14] Remove commented delete --- app/controllers/unapproved_point.py | 1 - 1 file changed, 1 deletion(-) diff --git a/app/controllers/unapproved_point.py b/app/controllers/unapproved_point.py index dc4598b..042facf 100644 --- a/app/controllers/unapproved_point.py +++ b/app/controllers/unapproved_point.py @@ -27,7 +27,6 @@ class UnapprovedPointController(RestController): """ # Will want to loop through the inputted unapproved_points try: - #self.delete(filter_data["account_id"], None) model_list = [] for point in data: model = super(UnapprovedPointController, self).post(point, filter_data) -- GitLab From cef1d25a81ad79bcaaca4dbfc2d9912f4229cd2d Mon Sep 17 00:00:00 2001 From: Conrad S Date: Mon, 27 Mar 2017 18:34:25 -0400 Subject: [PATCH 13/14] Fix accept and reject bug --- app/controllers/turk_hit.py | 11 +++++++---- app/models/turk_hit.py | 10 +++++----- 2 files changed, 12 insertions(+), 9 deletions(-) diff --git a/app/controllers/turk_hit.py b/app/controllers/turk_hit.py index 9c0f7e8..8ba9d8a 100644 --- a/app/controllers/turk_hit.py +++ b/app/controllers/turk_hit.py @@ -322,8 +322,7 @@ class TurkHitController(RestController): if int(id_) != data["db_id"]: raise BadRequest("The URL id and the body db_id are not the same") - approve = data.pop("approve") - + approve = data["approve"] # Approve or reject the hit on the amazon mech turk side if approve_or_reject_hit( data["amazon_hit_id"], @@ -337,12 +336,16 @@ class TurkHitController(RestController): status_id = approve_id if approve else reject_id # The data - data['status_id'] = status_id + clean_model = self.Model() + form.populate_obj(clean_model) + clean_data = clean_model.get_dictionary() + clean_data.pop("approve") + clean_data['status_id'] = status_id hit_list = proc( self.Model, self.Model.PROCS['UPDATE'], - **data, + **clean_data, ) except Exception as err: diff --git a/app/models/turk_hit.py b/app/models/turk_hit.py index 0542b1c..b993dcb 100644 --- a/app/models/turk_hit.py +++ b/app/models/turk_hit.py @@ -28,11 +28,11 @@ class TurkHit(BaseModel): ProcColumn('response_message'), ) - def __init__(self, db_id, building_id, amazon_hit_id, status_id, - hit_date, requester_name, csv_document_key, - shapefile_document_key, response_message - ): - + def __init__( + self, db_id=None, building_id=None, amazon_hit_id=None, status_id=None, + hit_date=None, requester_name=None, csv_document_key=None, + shapefile_document_key=None, response_message=None, + ): self.db_id = db_id self.building_id = building_id self.amazon_hit_id = amazon_hit_id -- GitLab From ef1a81a19fffb7316fe485637eb4a564b65778e0 Mon Sep 17 00:00:00 2001 From: Conrad S Date: Tue, 28 Mar 2017 10:07:55 -0400 Subject: [PATCH 14/14] Automatically reject the hit if it fails to parse --- app/controllers/turk_hit.py | 92 ++++++++++++++++++++++--------------- 1 file changed, 55 insertions(+), 37 deletions(-) diff --git a/app/controllers/turk_hit.py b/app/controllers/turk_hit.py index 8ba9d8a..69d6e0b 100644 --- a/app/controllers/turk_hit.py +++ b/app/controllers/turk_hit.py @@ -116,46 +116,58 @@ class TurkHitController(RestController): response_message = 'This hit was automatically rejected because '\ 'the uploaded file was not in the correct file '\ 'format. The file must be of type .xlsx.' - put_body = { - **cur_hit.get_dictionary(), - **{ - 'approve': 0, - 'response_message': response_message, - }, - } - self.put(cur_hit.db_id, put_body, None) + new_hit_status = self.reject_hit(cur_hit, response_message) cur_hit.response_message = response_message - new_hit_status = "Rejected" else: - file_name = "BuildingDimensions--{}--{}.xlsx".format( - cur_hit.building_id, - cur_hit.amazon_hit_id, - ) - folder_path = "/Buildings/{}_{}/Building_Dimensions".format( - cur_hit.building_id, - address, - ) - # Download the file - document = self.download_file( - cur_hit.building_id, - folder_path, - file_name, - response.content, - ) + # Parse the excel file try: - dimensions = self.save_dimensions_data( + dimensions = self.parse_dimensions_data( response.content, cur_hit.db_id, cur_hit.building_id, ) + # Automatically Reject the HIT if it failed to parse except Exception as e: if current_app.config['DEBUG']: - raise e - print("Failed to parse the dimensions file") - print(vars(e)) - # Get the key to update the model - cur_hit.csv_document_key = document['key'] + print("Failed to parse the dimensions file, rejecting now") + response_message = 'This hit was automatically rejected because '\ + 'the uploaded file was not parseable. Did '\ + 'you try uploading the file to our testing site?' + new_hit_status = self.reject_hit(cur_hit, response_message) + cur_hit.response_message = response_message + else: + # Save new dimensions data to the database + UnapprovedWindowDoorController().post( + dimensions['windows_doors'], + None, + ) + UnapprovedPointController().post( + dimensions['points'], + None, + ) + UnapprovedBuildingDimensionsController().post( + dimensions['building_dimensions'], + None, + ) + + # Save the file to box + file_name = "BuildingDimensions--{}--{}.xlsx".format( + cur_hit.building_id, + cur_hit.amazon_hit_id, + ) + folder_path = "/Buildings/{}_{}/Building_Dimensions".format( + cur_hit.building_id, + address, + ) + document = self.download_file( + cur_hit.building_id, + folder_path, + file_name, + response.content, + ) + # Get the key to update the model + cur_hit.csv_document_key = document['key'] # Update the entry in the database new_hit_status_id = self.STATUS_DICT_TEXT[new_hit_status] @@ -175,6 +187,17 @@ class TurkHitController(RestController): cur_hit.dimensions = dimensions return hit_list + def reject_hit(self, hit, response_message): + put_body = { + **hit.get_dictionary(), + **{ + 'approve': 0, + 'response_message': response_message, + }, + } + self.put(hit.db_id, put_body, None) + return "Rejected" + def download_file(self, building_id, folder_path, file_name, byte_data): """ A function to download a file to box @@ -200,9 +223,9 @@ class TurkHitController(RestController): document = response.json()['data'] return document - def save_dimensions_data(self, contents, hit_id, building_id): + def parse_dimensions_data(self, contents, hit_id, building_id): """ - A function to save dimensions data from the completed mech turk hit + A function to parse dimensions data from the completed mech turk hit Relies on a specific formatting of excel documents given to the worker @@ -219,11 +242,6 @@ class TurkHitController(RestController): building_dimensions['building_id'] = building_id building_dimensions['hit_id'] = hit_id - # Post new dimensions data to the database - UnapprovedWindowDoorController().post(windows_doors, None) - UnapprovedPointController().post(points, None) - UnapprovedBuildingDimensionsController().post(building_dimensions, None) - return { 'windows_doors': windows_doors, 'points': points, -- GitLab