From 0511c0acaa5dd248aa7c98dab828b9c685815bfe Mon Sep 17 00:00:00 2001 From: Conrad Date: Fri, 1 Dec 2017 16:17:31 -0500 Subject: [PATCH 1/3] Retrieve data from gateway endppoint --- app/__init__.py | 5 +++-- app/controllers/base.py | 3 ++- app/controllers/gateway.py | 39 ++++++++++++++++++++++++++++++++++++ app/lib/__init__.py | 1 + app/lib/redshift_database.py | 26 ++++++++++++++++++++++++ 5 files changed, 71 insertions(+), 3 deletions(-) create mode 100644 app/lib/redshift_database.py diff --git a/app/__init__.py b/app/__init__.py index dc103fa..6d91b7d 100644 --- a/app/__init__.py +++ b/app/__init__.py @@ -27,11 +27,12 @@ def create_app(config): from . import views views.register(app) - from app.lib import database, red, exceptions, service, session, auth0 - services = (database, red, exceptions, service, session, auth0) + from app.lib import database, red, exceptions, service, session, auth0, redshift_database + services = (database, red, exceptions, service, session, auth0, redshift_database) for service in services: service.register(app) + # with app.app_context(): # database.db.create_all() diff --git a/app/controllers/base.py b/app/controllers/base.py index cf86692..2e60fb7 100644 --- a/app/controllers/base.py +++ b/app/controllers/base.py @@ -1,6 +1,6 @@ """Flask RestController wrapper""" from bpvalve.flask.controllers import RestController as BaseRestController -from app.lib import db, redis, auth0_ +from app.lib import db, redis, auth0_, redshift class RestController(BaseRestController): @@ -10,3 +10,4 @@ class RestController(BaseRestController): db = db redis = redis auth0 = auth0_ + redshift = redshift diff --git a/app/controllers/gateway.py b/app/controllers/gateway.py index eefece4..141a0ae 100644 --- a/app/controllers/gateway.py +++ b/app/controllers/gateway.py @@ -1,6 +1,9 @@ from .base import RestController from ..forms.gateway import GatewayForm from ..models.gateway import Gateway +import datetime +import json +import psycopg2 class GatewayController(RestController): """A sensor controller.""" @@ -13,3 +16,39 @@ class GatewayController(RestController): def get_form(self, filter_data): """Return the sensor form.""" return GatewayForm + + def index(self, filter_data): + result = super().index(filter_data) + # If the user is also requesting data, get data + if 'data' in filter_data and filter_data.get('from') and result: + date = filter_data.get('from') + cur = self.redshift.db.cursor(cursor_factory=psycopg2.extras.DictCursor) + # Add a from date + where_statement = 'WHERE ts>%s AND (' + where_tuple = (date,) + # Add WHERE STATEMENTS + for gateway in result: + where_statement += "sn=%s OR " + where_tuple = where_tuple + (gateway['gateway_serial'],) + where_statement = where_statement[:-4] + ')' + sql = "SELECT * FROM data {}".format(where_statement) + cur.execute(sql, where_tuple) + + # Map data to gateway serial numbers + data_mapping = {} + for record in cur: + record_dict = dict(record) + if record_dict['sn'] not in data_mapping: + data_mapping[record_dict['sn']] = [] + data_mapping[record_dict['sn']].append(record_dict) + # Add the data to the gateways + for gateway in result: + if gateway['gateway_serial'] in data_mapping: + gateway['data'] = data_mapping[gateway['gateway_serial']] + else: + gateway['data'] = [] + + print('Number of data points', len(gateway['data'])) + print(sql) + + return result diff --git a/app/lib/__init__.py b/app/lib/__init__.py index 3fac200..90e9f86 100644 --- a/app/lib/__init__.py +++ b/app/lib/__init__.py @@ -1,3 +1,4 @@ from .database import db from .red import redis from .auth0 import auth0_ +from .redshift_database import redshift diff --git a/app/lib/redshift_database.py b/app/lib/redshift_database.py new file mode 100644 index 0000000..ba2fb79 --- /dev/null +++ b/app/lib/redshift_database.py @@ -0,0 +1,26 @@ +import psycopg2 + +class RedshiftWrapper(object): + """A wrapper for Redshift.""" + db = None + + def init_app(self, app): + """Stores information about the redis database from the URI.""" + DB_NAME = 'senseware' + USER = 'blocpower' + PASSWORD = 't106DZpVcCO9' + HOST = 'senseware.cz6saalxbzqr.us-east-1.redshift.amazonaws.com' + conn_string = """ + dbname='{}' + port='5439' + user='{}' + password='{}' + host='{}' + """.format(DB_NAME, USER, PASSWORD, HOST) + self.db = psycopg2.connect(conn_string) + +redshift = RedshiftWrapper() + +def register(app): + redshift.init_app(app) + -- GitLab From c4ded12c8831bf57b2518d890e96a52ec5bf23b3 Mon Sep 17 00:00:00 2001 From: Conrad Date: Mon, 4 Dec 2017 10:50:56 -0500 Subject: [PATCH 2/3] Add data endpoint and call it in gateway --- app/controllers/data.py | 37 +++++++++++++++++++++++++++++++++++++ app/controllers/gateway.py | 34 ++++++++++++++-------------------- app/views/__init__.py | 2 ++ app/views/data.py | 25 +++++++++++++++++++++++++ app/views/gateway.py | 4 ++-- 5 files changed, 80 insertions(+), 22 deletions(-) create mode 100644 app/controllers/data.py create mode 100644 app/views/data.py diff --git a/app/controllers/data.py b/app/controllers/data.py new file mode 100644 index 0000000..f582f12 --- /dev/null +++ b/app/controllers/data.py @@ -0,0 +1,37 @@ +from .base import RestController +import datetime +import json +import psycopg2 +from werkzeug.exceptions import BadRequest + +class DataController(RestController): + """A data controller.""" + + def index(self, filter_data): + if 'from' not in filter_data: + raise BadRequest("'from' is a required field in the query params") + date = filter_data.get('from') + gateway_serials = filter_data.getlist('gateway_serial[]') + node_module_numbers = filter_data.getlist('node_id[]') + + # Set up the WHERE statement with date info + where_statement = 'WHERE ts>%s AND (' + where_tuple = (date,) + + # Add the gateway serials to hte WHERE statement + for gateway_serial in gateway_serials: + where_statement += "sn=%s OR " + where_tuple = where_tuple + (gateway_serial,) + + # Add the node_module_ids to hte WHERE statement + for node_module_number in node_module_numbers: + where_statement += "mod=%s OR " + where_tuple = where_tuple + (node_module_number,) + + where_statement = where_statement[:-4] + ')' + sql = "SELECT * FROM data {}".format(where_statement) + + cur = self.redshift.db.cursor(cursor_factory=psycopg2.extras.DictCursor) + cur.execute(sql, where_tuple) + + return [dict(record) for record in cur] diff --git a/app/controllers/gateway.py b/app/controllers/gateway.py index 141a0ae..f668e1b 100644 --- a/app/controllers/gateway.py +++ b/app/controllers/gateway.py @@ -1,9 +1,11 @@ from .base import RestController from ..forms.gateway import GatewayForm from ..models.gateway import Gateway +from ..controllers.data import DataController import datetime import json import psycopg2 +from werkzeug.datastructures import MultiDict class GatewayController(RestController): """A sensor controller.""" @@ -18,29 +20,24 @@ class GatewayController(RestController): return GatewayForm def index(self, filter_data): + """ + Retrieve a list of gateways + """ result = super().index(filter_data) # If the user is also requesting data, get data - if 'data' in filter_data and filter_data.get('from') and result: + if 'data' in filter_data and 'from' in filter_data and result: date = filter_data.get('from') - cur = self.redshift.db.cursor(cursor_factory=psycopg2.extras.DictCursor) - # Add a from date - where_statement = 'WHERE ts>%s AND (' - where_tuple = (date,) - # Add WHERE STATEMENTS - for gateway in result: - where_statement += "sn=%s OR " - where_tuple = where_tuple + (gateway['gateway_serial'],) - where_statement = where_statement[:-4] + ')' - sql = "SELECT * FROM data {}".format(where_statement) - cur.execute(sql, where_tuple) + args = [('gateway_serial[]', gateway['gateway_serial']) for gateway in result] + args.append(('from', date)) + multi_dict_args = MultiDict(args) + data_result = DataController().index(multi_dict_args) # Map data to gateway serial numbers data_mapping = {} - for record in cur: - record_dict = dict(record) - if record_dict['sn'] not in data_mapping: - data_mapping[record_dict['sn']] = [] - data_mapping[record_dict['sn']].append(record_dict) + for record in data_result: + if record['sn'] not in data_mapping: + data_mapping[record['sn']] = [] + data_mapping[record['sn']].append(record) # Add the data to the gateways for gateway in result: if gateway['gateway_serial'] in data_mapping: @@ -48,7 +45,4 @@ class GatewayController(RestController): else: gateway['data'] = [] - print('Number of data points', len(gateway['data'])) - print(sql) - return result diff --git a/app/views/__init__.py b/app/views/__init__.py index 8cb25c8..80bafc5 100644 --- a/app/views/__init__.py +++ b/app/views/__init__.py @@ -5,6 +5,7 @@ from . import ( apartment_node, repeater_node, sensor_image, + data, ) def register(app): @@ -19,3 +20,4 @@ def register(app): apartment_node.ApartmentNodeView.register(app) repeater_node.RepeaterNodeView.register(app) sensor_image.SensorImageView.register(app) + data.DataView.register(app) diff --git a/app/views/data.py b/app/views/data.py new file mode 100644 index 0000000..3d4ef79 --- /dev/null +++ b/app/views/data.py @@ -0,0 +1,25 @@ +"""Views for working with accounts.""" +from .base import UnprotectedRestView +from ..controllers.data import DataController + +from flask import request +from werkzeug.exceptions import MethodNotAllowed + +class DataView(UnprotectedRestView): + """The data view.""" + + def get_controller(self): + """Return an instance of the data controller.""" + return DataController() + + def get(self, id_): + raise MethodNotAllowed() + + def put(self, id_): + raise MethodNotAllowed() + + def delete(self, id_): + raise MethodNotAllowed() + + def post(self): + raise MethodNotAllowed() diff --git a/app/views/gateway.py b/app/views/gateway.py index 7f506a5..b00bff8 100644 --- a/app/views/gateway.py +++ b/app/views/gateway.py @@ -1,10 +1,10 @@ """Views for working with accounts.""" -from .base import RestView +from .base import UnprotectedRestView from ..controllers.gateway import GatewayController from flask import request -class GatewayView(RestView): +class GatewayView(UnprotectedRestView): """The sensor view.""" def get_controller(self): -- GitLab From 3fee33e15e4af0db456a0e033faf17fc70ef59c3 Mon Sep 17 00:00:00 2001 From: Conrad Date: Mon, 4 Dec 2017 11:21:18 -0500 Subject: [PATCH 3/3] Handle errors in sql execute --- app/controllers/data.py | 8 ++++++-- app/views/data.py | 4 ++-- app/views/gateway.py | 4 ++-- 3 files changed, 10 insertions(+), 6 deletions(-) diff --git a/app/controllers/data.py b/app/controllers/data.py index f582f12..04ad7bf 100644 --- a/app/controllers/data.py +++ b/app/controllers/data.py @@ -30,8 +30,12 @@ class DataController(RestController): where_statement = where_statement[:-4] + ')' sql = "SELECT * FROM data {}".format(where_statement) - cur = self.redshift.db.cursor(cursor_factory=psycopg2.extras.DictCursor) - cur.execute(sql, where_tuple) + try: + cur.execute(sql, where_tuple) + except Exception as e: + cur.close() + self.redshift.db.rollback() + raise e return [dict(record) for record in cur] diff --git a/app/views/data.py b/app/views/data.py index 3d4ef79..c50e6d3 100644 --- a/app/views/data.py +++ b/app/views/data.py @@ -1,11 +1,11 @@ """Views for working with accounts.""" -from .base import UnprotectedRestView +from .base import RestView from ..controllers.data import DataController from flask import request from werkzeug.exceptions import MethodNotAllowed -class DataView(UnprotectedRestView): +class DataView(RestView): """The data view.""" def get_controller(self): diff --git a/app/views/gateway.py b/app/views/gateway.py index b00bff8..7f506a5 100644 --- a/app/views/gateway.py +++ b/app/views/gateway.py @@ -1,10 +1,10 @@ """Views for working with accounts.""" -from .base import UnprotectedRestView +from .base import RestView from ..controllers.gateway import GatewayController from flask import request -class GatewayView(UnprotectedRestView): +class GatewayView(RestView): """The sensor view.""" def get_controller(self): -- GitLab