diff --git a/app/controllers/constants.py b/app/controllers/constants.py new file mode 100644 index 0000000000000000000000000000000000000000..dc2a0a187695b4cd85c101c7c9a3681a4a891fbb --- /dev/null +++ b/app/controllers/constants.py @@ -0,0 +1,5 @@ +GATEWAY_TYPES = { + 'senseware': 1, + 'awair': 2, + 'other': 3, +} diff --git a/app/controllers/data.py b/app/controllers/data.py index 92eb723544040d855a30ea6b461d3f08be8457f2..ded3ccbc1f26b6324ef9d70c407a6eff3e492435 100644 --- a/app/controllers/data.py +++ b/app/controllers/data.py @@ -5,43 +5,135 @@ import psycopg2 from werkzeug.exceptions import BadRequest from app.lib import get_redshift_db +from app.lib.database import db + from .base import RestController +from .constants import ( + GATEWAY_TYPES, +) + class DataController(RestController): """A data controller.""" def index(self, filter_data): + ''' + Takes in device data like + device[]=$DEVICE_TYPE::$DEVICE_SPECIFIC_ID_1::$DEVICE_SPECIFIC_ID_2_OPTIONAL + ''' + import time + start = time.clock() if 'from' not in filter_data: raise BadRequest("'from' is a required field in the query params") + if 'device[]' not in filter_data: + raise BadRequest("'device[]' is a required field in the query params") + unit_id = filter_data.get('unit_id') date = filter_data.get('from') - gateway_serials = filter_data.getlist('gateway_serial[]') - node_module_numbers = filter_data.getlist('node_id[]') - - # Set up the WHERE statement with date info - where_statement = 'WHERE ts>%s AND (' - where_tuple = (date,) - - # Add the gateway serials to hte WHERE statement - for gateway_serial in gateway_serials: - where_statement += "sn=%s OR " - where_tuple = where_tuple + (gateway_serial,) - - # Add the node_module_ids to hte WHERE statement - for node_module_number in node_module_numbers: - where_statement += "mod=%s OR " - where_tuple = where_tuple + (node_module_number,) - - where_statement = where_statement[:-4] + ')' - - if 'unit_id' in filter_data: - where_statement += 'AND unit_id=%s' - where_tuple = where_tuple + (filter_data['unit_id'],) - - sql = "SELECT ts, value, unit, unit_id, name, sn, mod, channel_id FROM data {} ORDER BY ts".format(where_statement) - db = get_redshift_db(current_app) - cur = db.cursor(cursor_factory=psycopg2.extras.DictCursor) - cur.execute(sql, where_tuple) - results = [dict(record) for record in cur] - cur.close() + devices = filter_data.getlist('device[]') + metadata_id_dict = {} + for device in devices: + device_info = device.split('::') + device_type = device_info[0] + + if device_type == 'awair': + if len(device_info) < 2: + raise BadRequest("device_id is a required field in the device[] query param if the device is awair") + device_id = device_info[1] + query = ''' + SELECT meta.id as metadata_id, unit.description as unit FROM + iot.metadata as meta + INNER JOIN iot.unit ON unit.id = meta.unit_id + INNER JOIN iot.metadata_awair as awair on awair.metadata_id = meta.id + WHERE awair.device_id=:param + ''' + params = {'param': device_id} + if unit_id: + query += ''' + AND unit.id=:unit_param + ''' + params['unit_param'] = unit_id + + res = db.session.execute(query, params) + + metadata_id_rows = res.fetchall() + for metadata_id_row in metadata_id_rows: + metadata_id = metadata_id_row[0] + metadata_id_dict[metadata_id] = { + 'type': device_type, + 'device_id': device_id, + 'unit': metadata_id_row[1], + 'metadata_id': metadata_id, + 'data': [], + } - return results + elif device_type == 'senseware': + if len(device_info) < 3: + raise BadRequest("'mod' and 'sn' are required fields in the device[] query param if the device is senseware") + mod = device_info[1] + sn = device_info[2] + query = ''' + SELECT meta.id as metadata_id, unit.description as unit FROM + iot.metadata as meta + INNER JOIN iot.unit ON unit.id = meta.unit_id + INNER JOIN iot.metadata_senseware as sense on sense.metadata_id = meta.id + WHERE sense.mod=:mod_param + AND sense.sn=:sn_param + ''' + params = { + 'mod_param': mod, + 'sn_param': sn, + } + if unit_id: + query += ''' + AND unit.id=:unit_param + ''' + params['unit_param'] = unit_id + + res = db.session.execute(query, params) + + metadata_id_rows = res.fetchall() + for metadata_id_row in metadata_id_rows: + metadata_id = metadata_id_row[0] + metadata_id_dict[metadata_id] = { + 'type': device_type, + 'sn': sn, + 'mod': mod, + 'unit': metadata_id_row[1], + 'metadata_id': metadata_id, + 'data': [], + } + + # No metadata found for the specified sensor + if len(metadata_id_dict.keys()) == 0: + return [] + sql = ''' + SELECT + da.ts, da.value, + da.metadata_id + FROM + iot.data as da + WHERE + da.metadata_id in ({}) + AND + ts > %s + ORDER BY ts ASC + '''.format( + ('%s,' * len(metadata_id_dict.keys()))[:-1] + ) + insert_tuple = tuple() + for metadata_id in metadata_id_dict.keys(): + insert_tuple += (metadata_id,) + insert_tuple += (date,) + + redshift_db = get_redshift_db(current_app) + cur = redshift_db.cursor(cursor_factory=psycopg2.extras.DictCursor) + cur.execute(sql, insert_tuple) + for record in cur: + row = dict(record) + return_device = metadata_id_dict[row['metadata_id']] + row['unit'] = return_device['unit'] + row.pop('metadata_id') + return_device['data'].append(row) + + cur.close() + return metadata_id_dict.values() diff --git a/app/controllers/gateway.py b/app/controllers/gateway.py index 105cda2ecf0e5225d4872b234051e58a512566b1..17faf623bce2603ed5841c5634e706a24c74ebab 100644 --- a/app/controllers/gateway.py +++ b/app/controllers/gateway.py @@ -1,12 +1,14 @@ +from flask import current_app +from werkzeug.datastructures import MultiDict + from .base import RestController from ..forms.gateway import GatewayForm from ..models.gateway import Gateway from ..controllers.data import DataController from ..controllers.senseware_node import SensewareNodeController -import datetime -import json -import psycopg2 -from werkzeug.datastructures import MultiDict +from .constants import ( + GATEWAY_TYPES, +) class GatewayController(RestController): """A sensor controller.""" @@ -24,42 +26,48 @@ class GatewayController(RestController): """ Retrieve a list of gateways """ - result = super().index(filter_data) - # If the user is also requesting data, get data - if 'nodes' in filter_data and result: - node_ids = [] - for gateway in result: - multi_dict_args = MultiDict([('gateway_id', gateway['id'])]) - node_result = SensewareNodeController().index(multi_dict_args) - for node in node_result: - if (node['node_id']): - node_ids.append(node['node_id']) - gateway['nodes'] = node_result - - if 'data' in filter_data and 'from' in filter_data and result: - args = [('node_id[]', node_id) for node_id in node_ids] - data_mapping = {} - # Continue if any of the gateways have serials - if args: - args.append(('from', filter_data.get('from'))) - - if 'unit_id' in filter_data: - args.append(('unit_id', filter_data.get('unit_id'))) + gateway_result = super().index(filter_data) + """ New way of accessing data """ + if 'data' in filter_data and 'from' in filter_data and gateway_result: + # Access data for gateways that don't have seperate nodes + args = MultiDict([('from', filter_data['from'])]) + if 'unit_id' in filter_data: + args.add('unit_id', filter_data.get('unit_id')) + for gateway in gateway_result: + if gateway['sensor_type'] == GATEWAY_TYPES['awair']: + args.add('device[]', 'awair::{}'.format(gateway['gateway_id'])) + gateway['data'] = [] + if gateway['sensor_type'] == GATEWAY_TYPES['senseware']: + node_args = MultiDict([('gateway_id', gateway['id'])]) + node_result = SensewareNodeController().index(node_args) + for node in node_result: + args.add('device[]', 'senseware::{}::{}'.format( + node['node_id'], + gateway['gateway_serial'], + )) + node['data'] = [] + gateway['nodes'] = node_result + data_result_list = DataController().index(args) + # Parse through the data response and add it to the correct gateways and nodes + for data_result in data_result_list: - multi_dict_args = MultiDict(args) - data_result = DataController().index(multi_dict_args) - # Map data to gateway serial numbers - for record in data_result: - key = str(record['mod']) - if key not in data_mapping: - data_mapping[key] = [] - data_mapping[key].append(record) - # Add the data to the nodes - for gateway in result: - for node in gateway['nodes']: - if node['node_id'] in data_mapping: - node['data'] = data_mapping[node['node_id']] - else: - node['data'] = [] + for gateway in gateway_result: + if GATEWAY_TYPES[data_result['type']] == gateway['sensor_type']: + if data_result['type'] == 'awair': + if data_result['device_id'] == gateway['gateway_id']: + gateway['data'].extend(data_result['data']) + elif data_result['type'] == 'senseware': + for node in gateway['nodes']: + if ( + data_result['mod'] == node['node_id'] and + data_result['sn'] == gateway['gateway_serial'] + ): + node['data'].extend(data_result['data']) - return result + elif 'nodes' in filter_data and gateway_result: + for gateway in gateway_result: + if gateway['sensor_type'] == GATEWAY_TYPES['senseware']: + node_args = MultiDict([('gateway_id', gateway['id'])]) + node_result = SensewareNodeController().index(node_args) + gateway['nodes'] = node_result + return gateway_result diff --git a/app/forms/gateway.py b/app/forms/gateway.py index b1279c2be54ed18cb8d49d67e4afaa755933b9ec..794bd351580aa1f4127641953ed0368671c5d45b 100644 --- a/app/forms/gateway.py +++ b/app/forms/gateway.py @@ -1,6 +1,23 @@ import wtforms as wtf from .base import Form, UserForm +class NullableIntegerField(wtf.IntegerField): + """ + An IntegerField where the field can be null if the input data is an empty + string. + """ + + def process_formdata(self, valuelist): + if valuelist: + if valuelist[0] == '' or valuelist[0] is None: + self.data = None + else: + try: + self.data = int(valuelist[0]) + except ValueError: + self.data = None + raise ValueError(self.gettext('Not a valid integer value')) + class GatewayForm(Form, UserForm): """ A form for validating building sensor requests.""" @@ -31,6 +48,9 @@ class GatewayForm(Form, UserForm): placement_type = wtf.IntegerField( validators=[wtf.validators.Optional()] ) + space_id = NullableIntegerField( + validators=[wtf.validators.Optional()] + ) notes = wtf.StringField( validators=[wtf.validators.Optional()] ) diff --git a/app/models/gateway.py b/app/models/gateway.py index 1d3de69a10edb1f2256f40eae173cc9dbbfea58c..8bc2956ce57a55283f13e98ac45e37aed0827e49 100644 --- a/app/models/gateway.py +++ b/app/models/gateway.py @@ -19,4 +19,5 @@ class Gateway(Model, User, Tracked, db.Model): gateway_id = db.Column(db.Integer) installer_name = db.Column(db.Unicode(50)) placement_type = db.Column(db.Integer) + space_id = db.Column(db.Integer) notes = db.Column(db.Unicode(500))