From 341ceb289dd1d2fc057ba1dfd6da8cb1fa1e589a Mon Sep 17 00:00:00 2001 From: Conrad Date: Mon, 14 May 2018 12:36:27 -0400 Subject: [PATCH 1/3] Add event endpoint --- app/controllers/event.py | 147 +++++++++++++++++++++++++++++++++++++++ app/views/__init__.py | 2 + app/views/event.py | 22 ++++++ 3 files changed, 171 insertions(+) create mode 100644 app/controllers/event.py create mode 100644 app/views/event.py diff --git a/app/controllers/event.py b/app/controllers/event.py new file mode 100644 index 0000000..9bb95cd --- /dev/null +++ b/app/controllers/event.py @@ -0,0 +1,147 @@ +from flask import current_app +from werkzeug.datastructures import MultiDict +from werkzeug.exceptions import BadRequest + +from .base import RestController + +class EventController(RestController): + """An event controller.""" + + def get(self, id_, filter_data): + ''' Get an event by id ''' + sql = ''' + SELECT + event.id, + event.created, + event.ts, + event.type, + event_type.description as type_description, + event.metadata_id, + temperature.indoor_temperature, + temperature.outdoor_temperature + FROM iot.event as event + INNER JOIN iot.event_type as event_type on event_type.id = event.type + LEFT JOIN iot.event_temperature as temperature on temperature.event_id = event.id + WHERE event.id = :event_id + ''' + res = self.db.session.execute(sql, {'event_id': id_}) + row = res.fetchone() + # Update this list for future temperature events + if row[4] in ['Illegal Underheating']: + event_specific_data = { + 'indoor_temperature': row[6], + 'outdoor_temperature': row[7], + } + # For future non temperature events, add them here + return { + 'id': row[0], + 'created': row[1], + 'ts': row[2], + 'type': row[3], + 'type_description': row[4], + 'metadata_id': row[5], + 'event_specific': event_specific_data, + } + + def index(self, filter_data): + """ + Retrieve a list of events by building id or + """ + events = [] + if 'building_id[]' in filter_data: + building_ids = filter_data.getlist('building_id[]') + building_to_metadata_ids = self.get_metadata_ids_for_buildings(building_ids) + for building_id, metadata_ids in building_to_metadata_ids.items(): + building_events = self.get_events_for_metadata_ids(metadata_ids) + for event in building_events: + event['building_id'] = building_id + events.extend(building_events) + + elif 'metadata_id[]' in filter_data: + metadata_ids = filter_data.getlist('metadata_id[]') + events = self.get_events_for_metadata_ids(metadata_ids) + else: + raise BadRequest( + 'An index request should have a list of building_id[] or a list of metadata_id[]' + ) + + return events + + def get_events_for_metadata_ids(self, metadata_ids): + ''' A function that takes in a list of metadata ids and returns events ''' + # To stop sql injection we have to insert this way + metadata_id_insert_dict = { + 'val' + str(index): val for index, val in enumerate(metadata_ids) + } + sql = ''' + SELECT + event.id, + event.created, + event.ts, + event.type, + event_type.description as type_description, + event.metadata_id, + temperature.indoor_temperature, + temperature.outdoor_temperature + FROM iot.event as event + INNER JOIN iot.event_type as event_type on event_type.id = event.type + LEFT JOIN iot.event_temperature as temperature on temperature.event_id = event.id + WHERE event.metadata_id in ({}) + '''.format(', '.join( + [':' + key for key in metadata_id_insert_dict.keys()] + )) + + res = self.db.session.execute(sql, metadata_id_insert_dict) + return_list = [] + for row in res.fetchall(): + # Update this list for future temperature events + if row[4] in ['Illegal Underheating']: + event_specific_data = { + 'indoor_temperature': row[6], + 'outdoor_temperature': row[7], + } + # For future non temperature events, add them here + return_list.append({ + 'id': row[0], + 'created': row[1], + 'ts': row[2], + 'type': row[3], + 'type_description': row[4], + 'metadata_id': row[5], + 'event_specific': event_specific_data, + }) + return return_list + + def get_metadata_ids_for_buildings(self, building_ids): + ''' A function that takes in a list of building ids and returns metadata ids for those buildings ''' + # To stop sql injection we have to insert this way + building_id_insert_dict = { + 'val' + str(index): val for index, val in enumerate(building_ids) + } + sql = ''' + SELECT + meta.id AS metadata_id, + gateway.building_id + FROM sensor.gateway as gateway + LEFT JOIN sensor.senseware_node AS senseware_node ON senseware_node.gateway_id = gateway.id + LEFT JOIN iot.metadata_awair AS awair ON gateway.gateway_id = awair.device_id::text + LEFT JOIN iot.metadata_senseware AS sense ON ( + gateway.gateway_serial = sense.sn AND + senseware_node.node_id = sense.mod::text + ) + INNER JOIN iot.metadata as meta ON ( + meta.id = awair.metadata_id OR + meta.id = sense.metadata_id + ) + WHERE gateway.building_id IN ({}) + '''.format(', '.join( + [':' + key for key in building_id_insert_dict.keys()] + )) + res = self.db.session.execute(sql, building_id_insert_dict) + return_dict = {} + for row in res.fetchall(): + if row[1] in return_dict: + return_dict[row[1]].append(row[0]) + else: + return_dict[row[1]] = [row[0]] + return return_dict diff --git a/app/views/__init__.py b/app/views/__init__.py index 31c75cc..caa8735 100644 --- a/app/views/__init__.py +++ b/app/views/__init__.py @@ -4,6 +4,7 @@ from . import ( senseware_node, data, sensor_image, + event, ) def register(app): @@ -17,3 +18,4 @@ def register(app): senseware_node.SensewareNodeView.register(app) data.DataView.register(app) sensor_image.SensorImageView.register(app) + event.EventView.register(app) diff --git a/app/views/event.py b/app/views/event.py new file mode 100644 index 0000000..651aac4 --- /dev/null +++ b/app/views/event.py @@ -0,0 +1,22 @@ +"""Views for working with events.""" +from .base import UnprotectedRestView +from ..controllers.event import EventController + +from werkzeug.exceptions import MethodNotAllowed +from flask import request + +class EventView(UnprotectedRestView): + """The event view.""" + + def post(self): + raise MethodNotAllowed('Events are created with lambda functions') + + def delete(self, id_): + raise MethodNotAllowed('Events cannot be deleted') + + def put(self, id_): + raise MethodNotAllowed('Events cannot be updated') + + def get_controller(self): + """Return an instance of the event controller.""" + return EventController() -- GitLab From 3905ce7e9a6248cfe50ed6b7ea472cbcfa1ce09d Mon Sep 17 00:00:00 2001 From: Conrad Date: Mon, 14 May 2018 12:37:00 -0400 Subject: [PATCH 2/3] Make the endpoint protected --- app/views/event.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/app/views/event.py b/app/views/event.py index 651aac4..38ff600 100644 --- a/app/views/event.py +++ b/app/views/event.py @@ -1,11 +1,11 @@ """Views for working with events.""" -from .base import UnprotectedRestView +from .base import RestView from ..controllers.event import EventController from werkzeug.exceptions import MethodNotAllowed from flask import request -class EventView(UnprotectedRestView): +class EventView(RestView): """The event view.""" def post(self): -- GitLab From aa2b37fe9f261a25dae2c6384c3b2003e2e0acdd Mon Sep 17 00:00:00 2001 From: Conrad Date: Tue, 15 May 2018 13:53:28 -0400 Subject: [PATCH 3/3] Add more filter params to events endpoint --- app/controllers/data.py | 19 ++++++----- app/controllers/event.py | 67 +++++++++++++++++++++++++++++++++----- app/controllers/gateway.py | 6 ++-- 3 files changed, 74 insertions(+), 18 deletions(-) diff --git a/app/controllers/data.py b/app/controllers/data.py index ded3ccb..a2c1a20 100644 --- a/app/controllers/data.py +++ b/app/controllers/data.py @@ -21,10 +21,6 @@ class DataController(RestController): Takes in device data like device[]=$DEVICE_TYPE::$DEVICE_SPECIFIC_ID_1::$DEVICE_SPECIFIC_ID_2_OPTIONAL ''' - import time - start = time.clock() - if 'from' not in filter_data: - raise BadRequest("'from' is a required field in the query params") if 'device[]' not in filter_data: raise BadRequest("'device[]' is a required field in the query params") unit_id = filter_data.get('unit_id') @@ -106,6 +102,12 @@ class DataController(RestController): # No metadata found for the specified sensor if len(metadata_id_dict.keys()) == 0: return [] + from_clause = '' + if date: + from_clause = ''' + AND ts > %s + ''' + sql = ''' SELECT da.ts, da.value, @@ -114,16 +116,17 @@ class DataController(RestController): iot.data as da WHERE da.metadata_id in ({}) - AND - ts > %s + {} ORDER BY ts ASC '''.format( - ('%s,' * len(metadata_id_dict.keys()))[:-1] + ('%s,' * len(metadata_id_dict.keys()))[:-1], + from_clause, ) insert_tuple = tuple() for metadata_id in metadata_id_dict.keys(): insert_tuple += (metadata_id,) - insert_tuple += (date,) + if date: + insert_tuple += (date,) redshift_db = get_redshift_db(current_app) cur = redshift_db.cursor(cursor_factory=psycopg2.extras.DictCursor) diff --git a/app/controllers/event.py b/app/controllers/event.py index 9bb95cd..029f654 100644 --- a/app/controllers/event.py +++ b/app/controllers/event.py @@ -46,20 +46,49 @@ class EventController(RestController): def index(self, filter_data): """ Retrieve a list of events by building id or + + building_id[] - filter by building id + metadata_id[] - filter by metadata id + limit - limit the query + from - from a ts + to -to a ts + order - desc or asc """ - events = [] if 'building_id[]' in filter_data: building_ids = filter_data.getlist('building_id[]') building_to_metadata_ids = self.get_metadata_ids_for_buildings(building_ids) - for building_id, metadata_ids in building_to_metadata_ids.items(): - building_events = self.get_events_for_metadata_ids(metadata_ids) - for event in building_events: - event['building_id'] = building_id - events.extend(building_events) + metadata_ids = [ + metadata_id + for sublist in building_to_metadata_ids.values() + for metadata_id in sublist + ] + if not len(metadata_ids): + return [] + events = self.get_events_for_metadata_ids( + metadata_ids, + limit=filter_data.get('limit'), + from_ts=filter_data.get('from'), + to_ts=filter_data.get('to'), + order=filter_data.get('order'), + ) + # A dictionary that links metadata ids with their building id + building_id_dict = { + metadata_id: building_id + for building_id, sublist in building_to_metadata_ids.items() + for metadata_id in sublist + } + for event in events: + event['building_id'] = building_id_dict[event['metadata_id']] elif 'metadata_id[]' in filter_data: metadata_ids = filter_data.getlist('metadata_id[]') - events = self.get_events_for_metadata_ids(metadata_ids) + events = self.get_events_for_metadata_ids( + metadata_ids, + limit=filter_data.get('limit'), + from_ts=filter_data.get('from'), + to_ts=filter_data.get('to'), + order=filter_data.get('order'), + ) else: raise BadRequest( 'An index request should have a list of building_id[] or a list of metadata_id[]' @@ -67,7 +96,7 @@ class EventController(RestController): return events - def get_events_for_metadata_ids(self, metadata_ids): + def get_events_for_metadata_ids(self, metadata_ids, order=None, limit=None, from_ts=None, to_ts=None): ''' A function that takes in a list of metadata ids and returns events ''' # To stop sql injection we have to insert this way metadata_id_insert_dict = { @@ -90,6 +119,28 @@ class EventController(RestController): '''.format(', '.join( [':' + key for key in metadata_id_insert_dict.keys()] )) + if from_ts: + sql += ''' + AND event.ts >= :from + ''' + metadata_id_insert_dict['from'] = from_ts + if to_ts: + sql += ''' + AND event.ts < :to + ''' + metadata_id_insert_dict['to'] = to_ts + if order: + if order.lower() == 'desc': + sql += ''' + ORDER BY ts DESC + ''' + elif order.lower() == 'asc': + sql += ''' + ORDER BY ts ASC + ''' + if limit: + sql += 'LIMIT :limit' + metadata_id_insert_dict['limit'] = limit res = self.db.session.execute(sql, metadata_id_insert_dict) return_list = [] diff --git a/app/controllers/gateway.py b/app/controllers/gateway.py index 17faf62..04affbc 100644 --- a/app/controllers/gateway.py +++ b/app/controllers/gateway.py @@ -28,9 +28,11 @@ class GatewayController(RestController): """ gateway_result = super().index(filter_data) """ New way of accessing data """ - if 'data' in filter_data and 'from' in filter_data and gateway_result: + if 'data' in filter_data and gateway_result: # Access data for gateways that don't have seperate nodes - args = MultiDict([('from', filter_data['from'])]) + args = MultiDict([]) + if 'from' in filter_data: + args.add('from', filter_data.get('from')) if 'unit_id' in filter_data: args.add('unit_id', filter_data.get('unit_id')) for gateway in gateway_result: -- GitLab