From 072b96fd8b40594100231c2f7de9755497eaa1f7 Mon Sep 17 00:00:00 2001 From: Jose Contreras Date: Fri, 24 Aug 2018 21:42:37 -0400 Subject: [PATCH 01/15] Add to date to redshift query --- app/controllers/data.py | 13 ++++++++++--- app/controllers/gateway.py | 12 +++++++++--- 2 files changed, 19 insertions(+), 6 deletions(-) diff --git a/app/controllers/data.py b/app/controllers/data.py index a2c1a20..9519b95 100644 --- a/app/controllers/data.py +++ b/app/controllers/data.py @@ -20,13 +20,15 @@ class DataController(RestController): ''' Takes in device data like device[]=$DEVICE_TYPE::$DEVICE_SPECIFIC_ID_1::$DEVICE_SPECIFIC_ID_2_OPTIONAL + filter_data can contain unit_id, from, to, device[] ''' if 'device[]' not in filter_data: raise BadRequest("'device[]' is a required field in the query params") unit_id = filter_data.get('unit_id') date = filter_data.get('from') - devices = filter_data.getlist('device[]') - metadata_id_dict = {} + to_date = filter_data.get('to') + devices = filter_data.getlist('device[]') # Devices for which we need to find metadata IDs + metadata_id_dict = {} # Metadata IDs associated to sensors for device in devices: device_info = device.split('::') device_type = device_info[0] @@ -99,13 +101,17 @@ class DataController(RestController): 'data': [], } - # No metadata found for the specified sensor + ''' + No metadata id found for the specified sensor + can't fetch data from redshift + ''' if len(metadata_id_dict.keys()) == 0: return [] from_clause = '' if date: from_clause = ''' AND ts > %s + AND ts < %s ''' sql = ''' @@ -127,6 +133,7 @@ class DataController(RestController): insert_tuple += (metadata_id,) if date: insert_tuple += (date,) + insert_tuple += (to_date,) redshift_db = get_redshift_db(current_app) cur = redshift_db.cursor(cursor_factory=psycopg2.extras.DictCursor) diff --git a/app/controllers/gateway.py b/app/controllers/gateway.py index 04affbc..7c9222c 100644 --- a/app/controllers/gateway.py +++ b/app/controllers/gateway.py @@ -25,14 +25,17 @@ class GatewayController(RestController): def index(self, filter_data): """ Retrieve a list of gateways + return data if data is present in query params """ gateway_result = super().index(filter_data) - """ New way of accessing data """ + # New way of accessing data if 'data' in filter_data and gateway_result: # Access data for gateways that don't have seperate nodes args = MultiDict([]) if 'from' in filter_data: args.add('from', filter_data.get('from')) + if 'to' in filter_data: + args.add('to', filter_data.get('to')) if 'unit_id' in filter_data: args.add('unit_id', filter_data.get('unit_id')) for gateway in gateway_result: @@ -49,8 +52,11 @@ class GatewayController(RestController): )) node['data'] = [] gateway['nodes'] = node_result - data_result_list = DataController().index(args) - # Parse through the data response and add it to the correct gateways and nodes + data_result_list = DataController().index(args) + ''' + Parse through the data response and add it to the correct gateways and nodes + Loop over results in data from data controller + ''' for data_result in data_result_list: for gateway in gateway_result: -- GitLab From 75df3dd64d6a1f1fbb3372a5edf78c2c32aef883 Mon Sep 17 00:00:00 2001 From: Jose Contreras Date: Fri, 24 Aug 2018 21:49:48 -0400 Subject: [PATCH 02/15] Make doc string clearer --- app/controllers/data.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/app/controllers/data.py b/app/controllers/data.py index 9519b95..2218807 100644 --- a/app/controllers/data.py +++ b/app/controllers/data.py @@ -18,9 +18,12 @@ class DataController(RestController): def index(self, filter_data): ''' - Takes in device data like + filter_data values device[]=$DEVICE_TYPE::$DEVICE_SPECIFIC_ID_1::$DEVICE_SPECIFIC_ID_2_OPTIONAL - filter_data can contain unit_id, from, to, device[] + unit_id + from + to + date format = Wed, 15 Aug 2018 01:44:58 GMT ''' if 'device[]' not in filter_data: raise BadRequest("'device[]' is a required field in the query params") -- GitLab From 83e547d924eadeeb6e1bbeba99fdb9de2264669c Mon Sep 17 00:00:00 2001 From: Jose Contreras Date: Fri, 24 Aug 2018 21:56:34 -0400 Subject: [PATCH 03/15] Seperate to and from --- app/controllers/data.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/app/controllers/data.py b/app/controllers/data.py index 2218807..08ecd42 100644 --- a/app/controllers/data.py +++ b/app/controllers/data.py @@ -114,6 +114,9 @@ class DataController(RestController): if date: from_clause = ''' AND ts > %s + ''' + if to_date: + from_clause += ''' AND ts < %s ''' @@ -136,6 +139,7 @@ class DataController(RestController): insert_tuple += (metadata_id,) if date: insert_tuple += (date,) + if to_date: insert_tuple += (to_date,) redshift_db = get_redshift_db(current_app) -- GitLab From 1cbd27a26e39362c9d0526ca0b4ed5eaa142e37a Mon Sep 17 00:00:00 2001 From: Jose Contreras Date: Sun, 26 Aug 2018 19:45:35 -0400 Subject: [PATCH 04/15] Fetch MachineQ data. --- app/controllers/data.py | 31 +++++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/app/controllers/data.py b/app/controllers/data.py index 08ecd42..c062926 100644 --- a/app/controllers/data.py +++ b/app/controllers/data.py @@ -103,6 +103,37 @@ class DataController(RestController): 'metadata_id': metadata_id, 'data': [], } + elif device_type == 'machineq': + if len(device_info) < 2: + raise BadRequest("deveui is a required field in the device[] query param if the device is a machineq") + deveui = device_info[1] + query = ''' + SELECT meta.id as metadata_id, unit.description as unit FROM + iot.metadata as meta + INNER JOIN iot.unit ON unit.id = meta.unit_id + INNER JOIN iot.metadata_machineq as machineq on machineq.metadata_id = meta.id + WHERE machineq.deveui=:param + ''' + params = {'param': deveui} + if unit_id: + query += ''' + AND unit.id=:unit_param + ''' + params['unit_param'] = unit_id + + res = db.session.execute(query, params) + + metadata_id_rows = res.fetchall() + + for metadata_id_row in metadata_id_rows: + metadata_id = metadata_id_row[0] + metadata_id_dict[metadata_id] = { + 'type': device_type, + 'deveui': deveui, + 'unit': metadata_id_row[1], + 'metadata_id': metadata_id, + 'data': [], + } ''' No metadata id found for the specified sensor -- GitLab From 713701ee9a6726f920dea20bc7d8fa1efe4cdc18 Mon Sep 17 00:00:00 2001 From: Jose Contreras Date: Sun, 26 Aug 2018 20:03:57 -0400 Subject: [PATCH 05/15] Adjust gateway controller to pull machineq data. --- app/controllers/gateway.py | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/app/controllers/gateway.py b/app/controllers/gateway.py index 7c9222c..b85fe37 100644 --- a/app/controllers/gateway.py +++ b/app/controllers/gateway.py @@ -52,13 +52,21 @@ class GatewayController(RestController): )) node['data'] = [] gateway['nodes'] = node_result + if gateway['sensor_type'] == GATEWAY_TYPES['machineq']: + node_args = MultiDict([('gateway_id', gateway['id'])]) + node_result = MachineqNodeController().index(node_args) + for node in node_result: + args.add('device[]', 'machineq::{}'.format( + node['deveui'], + )) + node['data'] = [] + gateway['nodes'] = node_result data_result_list = DataController().index(args) ''' Parse through the data response and add it to the correct gateways and nodes Loop over results in data from data controller ''' for data_result in data_result_list: - for gateway in gateway_result: if GATEWAY_TYPES[data_result['type']] == gateway['sensor_type']: if data_result['type'] == 'awair': @@ -71,6 +79,10 @@ class GatewayController(RestController): data_result['sn'] == gateway['gateway_serial'] ): node['data'].extend(data_result['data']) + elif data_result['type'] == 'machineq': + for node in gateway['nodes']: + if data_result['deveui'] == node['deveui']: + node['data'].extend(data_result['data']) elif 'nodes' in filter_data and gateway_result: for gateway in gateway_result: @@ -78,4 +90,8 @@ class GatewayController(RestController): node_args = MultiDict([('gateway_id', gateway['id'])]) node_result = SensewareNodeController().index(node_args) gateway['nodes'] = node_result + elif gateway['sensor_type'] == GATEWAY_TYPES['machineq']: + node_args = MultiDict([('gateway_id', gateway['id'])]) + node_result = MachineqNodeController().index(node_args) + gateway['nodes'] = node_result return gateway_result -- GitLab From dd941495f12a91fe72126753eedb002f92c897a4 Mon Sep 17 00:00:00 2001 From: Jose Contreras Date: Sun, 26 Aug 2018 20:27:14 -0400 Subject: [PATCH 06/15] Create view, controller, form, and model for machineq_node. --- app/controllers/machineq_node.py | 14 ++++++++++ app/forms/machineq_node.py | 44 ++++++++++++++++++++++++++++++++ app/models/machineq_node.py | 19 ++++++++++++++ app/views/machineq_node.py | 12 +++++++++ 4 files changed, 89 insertions(+) create mode 100644 app/controllers/machineq_node.py create mode 100644 app/forms/machineq_node.py create mode 100644 app/models/machineq_node.py create mode 100644 app/views/machineq_node.py diff --git a/app/controllers/machineq_node.py b/app/controllers/machineq_node.py new file mode 100644 index 0000000..2f96f90 --- /dev/null +++ b/app/controllers/machineq_node.py @@ -0,0 +1,14 @@ +from .base import RestController +from ..forms.machineq_node import MachineqNodeForm +from ..models.machineq_node import MachineqNode + +class MachineqNodeController(RestController): + """A machineq node controller.""" + Model = MachineqNode + filters = { + 'gateway_id': lambda d: MachineqNode.gateway_id == d['gateway_id'], + } + + def get_form(self, filter_data): + """Return the machineq node form.""" + return MachineqNodeForm \ No newline at end of file diff --git a/app/forms/machineq_node.py b/app/forms/machineq_node.py new file mode 100644 index 0000000..9521296 --- /dev/null +++ b/app/forms/machineq_node.py @@ -0,0 +1,44 @@ +import wtforms as wtf +from .base import Form, UserForm + +class NullableIntegerField(wtf.IntegerField): + """ + An IntegerField where the field can be null if the input data is an empty + string. + """ + + def process_formdata(self, valuelist): + if valuelist: + if valuelist[0] == '' or valuelist[0] is None: + self.data = None + else: + try: + self.data = int(valuelist[0]) + except ValueError: + self.data = None + raise ValueError(self.gettext('Not a valid integer value')) + +class MachinqNodeForm(Form, UserForm): + """ A form for validating machineq node requests.""" + + id = wtf.IntegerField( + validators=[wtf.validators.Optional()] + ) + gateway_id = wtf.IntegerField( + validators=[wtf.validators.Required()] + ) + deveui = wtf.StringField( + validators=[wtf.validators.Optional()] + ) + space_id = NullableIntegerField( + validators=[wtf.validators.Optional()] + ) + node_type = NullableIntegerField( + validators=[wtf.validators.Optional()] + ) + surface_type = wtf.StringField( + validators=[wtf.validators.Optional()] + ) + notes = wtf.StringField( + validators=[wtf.validators.Optional()] + ) diff --git a/app/models/machineq_node.py b/app/models/machineq_node.py new file mode 100644 index 0000000..f8db649 --- /dev/null +++ b/app/models/machineq_node.py @@ -0,0 +1,19 @@ +from .base import Model, Tracked, User +from ..lib.database import db + +from sqlalchemy.ext.declarative import declared_attr + +class MachineQNode(Model, User, Tracked, db.Model): + + @declared_attr + def __tablename__(cls): + return "senseware_node" + + __table_args__ = {"schema": "sensor"} + + gateway_id = db.Column(db.Integer) + deveui = db.Column(db.Unicode(30)) + space_id = db.Column(db.Integer) + node_type = db.Column(db.Integer()) + surface_type = db.Column(db.Unicode(50)) + notes = db.Column(db.Unicode(500)) \ No newline at end of file diff --git a/app/views/machineq_node.py b/app/views/machineq_node.py new file mode 100644 index 0000000..b1bc0b6 --- /dev/null +++ b/app/views/machineq_node.py @@ -0,0 +1,12 @@ +"""Views for working with machineq nodes.""" +from .base import RestView +from ..controllers.machineq_node import MachineqNodeController + +from flask import request + +class MachineqNodeView(RestView): + """The machineq node view.""" + + def get_controller(self): + """Return an instance of the machineq node controller.""" + return MachineQNodeController() \ No newline at end of file -- GitLab From e33d7d777f97d20fdc060efd22b311d933f53b2e Mon Sep 17 00:00:00 2001 From: Jose Contreras Date: Sun, 26 Aug 2018 20:28:27 -0400 Subject: [PATCH 07/15] Fix tablename for machineq node. --- app/models/machineq_node.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/models/machineq_node.py b/app/models/machineq_node.py index f8db649..9c700fe 100644 --- a/app/models/machineq_node.py +++ b/app/models/machineq_node.py @@ -7,7 +7,7 @@ class MachineQNode(Model, User, Tracked, db.Model): @declared_attr def __tablename__(cls): - return "senseware_node" + return "machineq_node" __table_args__ = {"schema": "sensor"} -- GitLab From a4a6cc4aa741bad74ce683215b88dd1d3489914b Mon Sep 17 00:00:00 2001 From: Jose Contreras Date: Sun, 26 Aug 2018 20:33:03 -0400 Subject: [PATCH 08/15] Add eof newline. --- app/controllers/machineq_node.py | 2 +- app/models/machineq_node.py | 2 +- app/views/machineq_node.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/app/controllers/machineq_node.py b/app/controllers/machineq_node.py index 2f96f90..650dfb1 100644 --- a/app/controllers/machineq_node.py +++ b/app/controllers/machineq_node.py @@ -11,4 +11,4 @@ class MachineqNodeController(RestController): def get_form(self, filter_data): """Return the machineq node form.""" - return MachineqNodeForm \ No newline at end of file + return MachineqNodeForm diff --git a/app/models/machineq_node.py b/app/models/machineq_node.py index 9c700fe..cb5f81d 100644 --- a/app/models/machineq_node.py +++ b/app/models/machineq_node.py @@ -16,4 +16,4 @@ class MachineQNode(Model, User, Tracked, db.Model): space_id = db.Column(db.Integer) node_type = db.Column(db.Integer()) surface_type = db.Column(db.Unicode(50)) - notes = db.Column(db.Unicode(500)) \ No newline at end of file + notes = db.Column(db.Unicode(500)) diff --git a/app/views/machineq_node.py b/app/views/machineq_node.py index b1bc0b6..04a8b24 100644 --- a/app/views/machineq_node.py +++ b/app/views/machineq_node.py @@ -9,4 +9,4 @@ class MachineqNodeView(RestView): def get_controller(self): """Return an instance of the machineq node controller.""" - return MachineQNodeController() \ No newline at end of file + return MachineQNodeController() -- GitLab From b5be16f9ac361b9448f1e71ca0fe3202eb4bfa68 Mon Sep 17 00:00:00 2001 From: Jose Contreras Date: Mon, 27 Aug 2018 23:46:15 -0400 Subject: [PATCH 09/15] Add missing gatewaytype. --- app/controllers/constants.py | 1 + 1 file changed, 1 insertion(+) diff --git a/app/controllers/constants.py b/app/controllers/constants.py index dc2a0a1..47bfc47 100644 --- a/app/controllers/constants.py +++ b/app/controllers/constants.py @@ -2,4 +2,5 @@ GATEWAY_TYPES = { 'senseware': 1, 'awair': 2, 'other': 3, + 'machineq': 4, } -- GitLab From 3c34e685ec8cf19f730b69bcfae1d8b80f451d65 Mon Sep 17 00:00:00 2001 From: Jose Contreras Date: Tue, 28 Aug 2018 00:34:15 -0400 Subject: [PATCH 10/15] Import machineq node controller. --- app/controllers/gateway.py | 1 + 1 file changed, 1 insertion(+) diff --git a/app/controllers/gateway.py b/app/controllers/gateway.py index b85fe37..99b7db9 100644 --- a/app/controllers/gateway.py +++ b/app/controllers/gateway.py @@ -6,6 +6,7 @@ from ..forms.gateway import GatewayForm from ..models.gateway import Gateway from ..controllers.data import DataController from ..controllers.senseware_node import SensewareNodeController +from ..controllers.machineq_node import MachineqNodeController from .constants import ( GATEWAY_TYPES, ) -- GitLab From 39124030362bd30cbdb21c84d6d77831c184271e Mon Sep 17 00:00:00 2001 From: Jose Contreras Date: Tue, 28 Aug 2018 01:39:40 -0400 Subject: [PATCH 11/15] Fix form name mispelling. --- app/forms/machineq_node.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/forms/machineq_node.py b/app/forms/machineq_node.py index 9521296..ce8df95 100644 --- a/app/forms/machineq_node.py +++ b/app/forms/machineq_node.py @@ -18,7 +18,7 @@ class NullableIntegerField(wtf.IntegerField): self.data = None raise ValueError(self.gettext('Not a valid integer value')) -class MachinqNodeForm(Form, UserForm): +class MachineqNodeForm(Form, UserForm): """ A form for validating machineq node requests.""" id = wtf.IntegerField( -- GitLab From 430d25c79cde5ca593e47ce4ee7a344d368c2260 Mon Sep 17 00:00:00 2001 From: Jose Contreras Date: Tue, 28 Aug 2018 11:24:52 -0400 Subject: [PATCH 12/15] Add machineq node to views. --- app/views/__init__.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/app/views/__init__.py b/app/views/__init__.py index a8534ae..39203cf 100644 --- a/app/views/__init__.py +++ b/app/views/__init__.py @@ -2,6 +2,7 @@ from . import ( gateway, senseware_node, + machineq_node, data, sensor_image, event, @@ -17,6 +18,7 @@ def register(app): """ gateway.GatewayView.register(app) senseware_node.SensewareNodeView.register(app) + machineq_node.MachineqNodeView(app) data.DataView.register(app) sensor_image.SensorImageView.register(app) event.EventView.register(app) -- GitLab From e539387367cd770381891bea19dbfbb40266025a Mon Sep 17 00:00:00 2001 From: Jose Contreras Date: Tue, 28 Aug 2018 13:31:29 -0400 Subject: [PATCH 13/15] Fix mispelling in machineq_node controller. --- app/views/machineq_node.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/views/machineq_node.py b/app/views/machineq_node.py index 04a8b24..b2d7d85 100644 --- a/app/views/machineq_node.py +++ b/app/views/machineq_node.py @@ -9,4 +9,4 @@ class MachineqNodeView(RestView): def get_controller(self): """Return an instance of the machineq node controller.""" - return MachineQNodeController() + return MachineqNodeController() -- GitLab From 5dcb59429d36b6b12c461dfccc3501685d3397ca Mon Sep 17 00:00:00 2001 From: Jose Contreras Date: Tue, 28 Aug 2018 14:42:34 -0400 Subject: [PATCH 14/15] Correct spelling again... --- app/models/machineq_node.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/models/machineq_node.py b/app/models/machineq_node.py index cb5f81d..994a1ff 100644 --- a/app/models/machineq_node.py +++ b/app/models/machineq_node.py @@ -3,7 +3,7 @@ from ..lib.database import db from sqlalchemy.ext.declarative import declared_attr -class MachineQNode(Model, User, Tracked, db.Model): +class MachineqNode(Model, User, Tracked, db.Model): @declared_attr def __tablename__(cls): -- GitLab From 4e1c22b01983c84c7673b0f358cd68f068ac3d6c Mon Sep 17 00:00:00 2001 From: Jose Contreras Date: Tue, 28 Aug 2018 15:13:37 -0400 Subject: [PATCH 15/15] Register machineq_node_view correctly. --- app/views/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/views/__init__.py b/app/views/__init__.py index 39203cf..2636c87 100644 --- a/app/views/__init__.py +++ b/app/views/__init__.py @@ -18,7 +18,7 @@ def register(app): """ gateway.GatewayView.register(app) senseware_node.SensewareNodeView.register(app) - machineq_node.MachineqNodeView(app) + machineq_node.MachineqNodeView.register(app) data.DataView.register(app) sensor_image.SensorImageView.register(app) event.EventView.register(app) -- GitLab