From c566793a606ba1f4b1f7ce271b59a7522c8e54bc Mon Sep 17 00:00:00 2001 From: mchlburton Date: Mon, 13 Mar 2017 20:02:06 +0000 Subject: [PATCH 01/31] Dummy API for Philips Hue light, pushes to Cassandra, working on pushing to influx instead --- .../devicediscovery/agent.py | 13 +- Agents/LightingAgent/lighting/agent.py | 2 + .../LightingAgent/lightingagent.launch.json | 2 +- .../classAPI/classAPI_DummyPhilipsHue.py | 317 ++++++++++++++++++ 4 files changed, 329 insertions(+), 5 deletions(-) create mode 100755 DeviceAPI/classAPI/classAPI_DummyPhilipsHue.py diff --git a/Agents/DeviceDiscoveryAgent/devicediscovery/agent.py b/Agents/DeviceDiscoveryAgent/devicediscovery/agent.py index 1b1fd1e..f561c02 100755 --- a/Agents/DeviceDiscoveryAgent/devicediscovery/agent.py +++ b/Agents/DeviceDiscoveryAgent/devicediscovery/agent.py @@ -258,8 +258,8 @@ def DeviceDiscoveryAgent(config_path, **kwargs): discovered_address = discovery_module.discover(discovery_type) if discovered_address == None: discovered_address = list() - - + if discovery_type == 'Philips': + discovered_address.append("http://10.0.2.56") print discovered_address for address in discovered_address: @@ -269,14 +269,19 @@ def DeviceDiscoveryAgent(config_path, **kwargs): ip_address = address try: macaddress = discovery_module.getMACaddress(discovery_type, ip_address) + print "macaddress: !!!!!!!!!!!! {}".format(macaddress) if macaddress is not None: _valid_macaddress = True else: _valid_macaddress = False except Exception as er: - print "exception: ",er - _valid_macaddress = False + if ip_address == "http://10.0.2.56": + macaddress = "00:17:88:1a:2b:3c" + _valid_macaddress = True + else: + print "exception: ",er + _valid_macaddress = False else: # this device does not return IP, wait until it get the right mac try: diff --git a/Agents/LightingAgent/lighting/agent.py b/Agents/LightingAgent/lighting/agent.py index 8970aa3..d3030f6 100755 --- a/Agents/LightingAgent/lighting/agent.py +++ b/Agents/LightingAgent/lighting/agent.py @@ -211,6 +211,7 @@ def LightingAgent(config_path, **kwargs): print("device connection is not successful") self.changed_variables = dict() + for v in log_variables: if v in Light.variables: if not v in self.variables or self.variables[v] != Light.variables[v]: @@ -253,6 +254,7 @@ def LightingAgent(config_path, **kwargs): (self.get_variable('hexcolor'), agent_id)) self.con.commit() try: + if self.get_variable('status') == "ON": multiple_on_off_status = "" for dummyvar in range(self.get_variable('number_lights')): diff --git a/Agents/LightingAgent/lightingagent.launch.json b/Agents/LightingAgent/lightingagent.launch.json index 34e661d..80a4f51 100755 --- a/Agents/LightingAgent/lightingagent.launch.json +++ b/Agents/LightingAgent/lightingagent.launch.json @@ -10,7 +10,7 @@ "model": "Philips Hue", "type": "lighting", "api": "classAPI_DummyPhilipsHue", - "address": "http://38.68.232.161/api/newdeveloper/groups/0/", + "address": "http://10.0.2.56", "topic": "datalogger/log/building1/zone1/lightingload/PhilipsHue", "db_host": "localhost", "db_port": "5432", diff --git a/DeviceAPI/classAPI/classAPI_DummyPhilipsHue.py b/DeviceAPI/classAPI/classAPI_DummyPhilipsHue.py new file mode 100755 index 0000000..6276e10 --- /dev/null +++ b/DeviceAPI/classAPI/classAPI_DummyPhilipsHue.py @@ -0,0 +1,317 @@ +# -*- coding: utf-8 -*- +''' +Copyright (c) 2016, Virginia Tech +All rights reserved. + +Redistribution and use in source and binary forms, with or without modification, are permitted provided that the + following conditions are met: +1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following +disclaimer. +2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following +disclaimer in the documentation and/or other materials provided with the distribution. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, +INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, +WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +The views and conclusions contained in the software and documentation are those of the authors and should not be +interpreted as representing official policies, either expressed or implied, of the FreeBSD Project. + +This material was prepared as an account of work sponsored by an agency of the United States Government. Neither the +United States Government nor the United States Department of Energy, nor Virginia Tech, nor any of their employees, +nor any jurisdiction or organization that has cooperated in the development of these materials, makes any warranty, +express or implied, or assumes any legal liability or responsibility for the accuracy, completeness, or usefulness or +any information, apparatus, product, software, or process disclosed, or represents that its use would not infringe +privately owned rights. + +Reference herein to any specific commercial product, process, or service by trade name, trademark, manufacturer, or +otherwise does not necessarily constitute or imply its endorsement, recommendation, favoring by the United States +Government or any agency thereof, or Virginia Tech - Advanced Research Institute. The views and opinions of authors +expressed herein do not necessarily state or reflect those of the United States Government or any agency thereof. + +VIRGINIA TECH – ADVANCED RESEARCH INSTITUTE +under Contract DE-EE0006352 + +#__author__ = "BEMOSS Team" +#__credits__ = "" +#__version__ = "2.0" +#__maintainer__ = "BEMOSS Team" +#__email__ = "aribemoss@gmail.com" +#__website__ = "www.bemoss.org" +#__created__ = "2014-09-12 12:04:50" +#__lastUpdated__ = "2016-03-14 11:23:33" +''' + +import time +import urllib2 +import json +from random import randint +from bemoss_lib.utils import rgb_cie +class API: + # 1. constructor : gets call every time when create a new class + # requirements for instantiation1. model, 2.type, 3.api, 4. address + def __init__(self,**kwargs): # default color is white + # Initialized common attributes + self.variables = kwargs + self.debug = True + self.set_variable('offline_count',0) + self.set_variable('connection_renew_interval',6000) #nothing to renew, right now + self.only_white_bulb = None + # to initialize the only white bulb value + self.getDeviceStatus() + def renewConnection(self): + pass + + def set_variable(self,k,v): # k=key, v=value + self.variables[k] = v + + def get_variable(self,k): + return self.variables.get(k, None) # default of get_variable is none + + # 2. Attributes from Attributes table + ''' + Attributes: + ------------------------------------------------------------------------------------------ + status GET POST Philips Hue ON/OFF status + brightness GET POST brightness percentage + effect GET POST Hue light effect 'none' or 'colorloop' + color GET POST temporary target heat setpoint (floating point in deg F) + ------------------------------------------------------------------------------------------ + + ''' + + # 3. Capabilites (methods) from Capabilities table + ''' + API3 available methods: + 1. getDeviceStatus() GET + 2. setDeviceStatus(postmsg) PUT + 3. identifyDevice() + ''' + + # ---------------------------------------------------------------------- + # getDeviceStatus(), getDeviceStatusJson(data), printDeviceStatus() + def getDeviceStatus(self): + getDeviceStatusResult = True + _hue_username = self.get_variable("username") + _url_append = '/api/'+_hue_username+'/groups/0/' + _urlData = self.get_variable("address").replace(':80', _url_append) + try: + '''_deviceUrl = urllib2.urlopen(_urlData, timeout=20) + print(" {0}Agent is querying its current status (status:{1}) please wait ...". + format(self.variables.get('agent_id', None), _deviceUrl.getcode())) + if (_deviceUrl.getcode() == 200): + self.getDeviceStatusJson(_deviceUrl.read().decode("utf-8")) #convert string data to JSON object then interpret it + if self.debug is True: + self.printDeviceStatus() + else: + print (" Received an error from server, cannot retrieve results " + str(_deviceUrl.getcode())) + getDeviceStatusResult = False''' + onoff = randint(0, 1) + if onoff == 1: + self.set_variable('status', "ON") + else: + self.set_variable('status', "OFF") + bri = randint(0, 255) + hue = randint(0, 255) + xy = [randint(0, 255), randint(0,255)] + ct = randint(0, 255) + sat = randint(0, 255) + effect = "effect" + mode = "warm" + name = "DummyHue" + # 2. brightness convert to % + self.set_variable('brightness',int(round(float(bri)*100/255,0))) + # update only white variable every round is necessary in case user add a/take away all color bulb(s). + self.only_white_bulb = False if hue else True + if self.only_white_bulb is False: + # 3. color convert to RGB 0-255 + self.set_variable('hue', hue) + self.set_variable('xy', xy) + self.set_variable('ct', ct) + x=xy[0] + y=xy[1] + self.set_variable('color', rgb_cie.ColorHelper.getRGBFromXYAndBrightness(x,y,bri)) + self.set_variable('hexcolor', '#%02x%02x%02x' % self.get_variable('color')) + # 4. saturation convert to % + self.set_variable('saturation',int(round(float(sat)*100/255,0))) + self.set_variable('effect',effect) + self.set_variable('colormode',mode) + self.set_variable('number_lights', [2]) + self.set_variable('name',name) + + + # Check the connectivity + if getDeviceStatusResult==True: + self.set_variable('offline_count', 0) + else: + self.set_variable('offline_count', self.get_variable('offline_count')+1) + except Exception as er: + print er + print('ERROR: classAPI_PhilipsHue failed to getDeviceStatus') + self.set_variable('offline_count',self.get_variable('offline_count')+1) + + ''' + def getDeviceStatusJson(self,data): + # Use the json module to load the string data into a dictionary + _theJSON = json.loads(data) + # 1. status + if _theJSON["action"]["on"] == True: + self.set_variable('status',"ON") + else: + self.set_variable('status',"OFF") + # 2. brightness convert to % + self.set_variable('brightness',int(round(float(_theJSON["action"]["bri"])*100/255,0))) + # update only white variable every round is necessary in case user add a/take away all color bulb(s). + self.only_white_bulb = False if 'hue' in _theJSON["action"].keys() else True + if self.only_white_bulb is False: + # 3. color convert to RGB 0-255 + self.set_variable('hue', _theJSON["action"]["hue"]) + self.set_variable('xy', _theJSON["action"]["xy"]) + self.set_variable('ct', _theJSON["action"]["ct"]) + x=_theJSON["action"]["xy"][0] + y=_theJSON["action"]["xy"][1] + self.set_variable('color', rgb_cie.ColorHelper.getRGBFromXYAndBrightness(x,y,_theJSON["action"]["bri"])) + self.set_variable('hexcolor', '#%02x%02x%02x' % self.get_variable('color')) + # 4. saturation convert to % + self.set_variable('saturation',int(round(float(_theJSON["action"]["sat"])*100/255,0))) + self.set_variable('effect',_theJSON["action"]["effect"]) + self.set_variable('colormode',_theJSON["action"]["colormode"]) + for k in _theJSON["lights"]: + self.set_variable("lights{}".format(k), k) + self.set_variable('number_lights', len(_theJSON["lights"])) + self.set_variable('name',_theJSON["name"]) + ''' + def printDeviceStatus(self): + # now we can access the contents of the JSON like any other Python object + print(" the current status is as follows:") + print(" name = {}".format(self.get_variable('name'))) + #print(" number_lights = {}".format(self.get_variable('number_lights'))) + print(" status = {}".format(self.get_variable('status'))) + print(" brightness = {}".format(self.get_variable('brightness'))) + if self.only_white_bulb is False: + print(" hue = {}".format(self.get_variable('hue'))) + print(" color = {}".format(self.get_variable('color'))) + print(" saturation = {}".format(self.get_variable('saturation'))) + print(" xy= {}".format(self.get_variable('xy'))) + print(" ct = {}".format(self.get_variable('ct'))) + print(" effect = {}".format(self.get_variable('effect'))) + print(" colormode = {}\n".format(self.get_variable('colormode'))) + # ---------------------------------------------------------------------- + # setDeviceStatus(postmsg), isPostmsgValid(postmsg), convertPostMsg(postmsg) + def setDeviceStatus(self, postmsg): + setDeviceStatusResult = True + ''' + #Ex. postmsg = {"on":True,"bri":100,"hue":50260,"sat":200} + _hue_username = self.get_variable("username") + _url_append = '/api/'+_hue_username+'/groups/0/' + _urlData = self.get_variable("address").replace(':80', _url_append) + if self.isPostMsgValid(postmsg) == True: #check if the data is valid + _data = json.dumps(self.convertPostMsg(postmsg)) + _data = _data.encode(encoding='utf_8') + _request = urllib2.Request(_urlData+'action') + _request.add_header('Content-Type','application/json') + _request.get_method = lambda: 'PUT' + try: + _f = urllib2.urlopen(_request, _data, timeout=20) #when include data this become a POST command + print(" {0}Agent for {1} is changing its status with {2} please wait ..." + .format(self.variables.get('agent_id', None), self.variables.get('model', None), postmsg)) + print(" after send a POST request: {}".format(_f.read().decode('utf-8'))) + except: + print("ERROR: classAPI_PhilipsHue connection failure! @ setDeviceStatus") + setDeviceStatusResult = False + else: + print("The POST message is invalid, try again\n") + ''' + return setDeviceStatusResult + + def isPostMsgValid(self,postmsg): #check validity of postmsg + dataValidity = True + #TODO algo to check whether postmsg is valid + return dataValidity + + + def convertPostMsg(self,postmsg): + msgToDevice = {} + datacontainsRGB=False + if 'color' in postmsg.keys(): + datacontainsRGB=True + + for k,v in postmsg.items(): + if k == 'status': + if postmsg.get('status') == "ON": + msgToDevice['on'] = True + elif postmsg.get('status') == "OFF": + msgToDevice['on'] = False + elif k == 'brightness': + msgToDevice['bri'] = int(round(float(postmsg.get('brightness'))*255.0/100.0,0)) + elif k == 'color': + if self.only_white_bulb is False: + print(type(postmsg['color'])) + _red = postmsg['color'][0] + _green = postmsg['color'][1] + _blue = postmsg['color'][2] + _xyY = rgb_cie.ColorHelper.getXYPointFromRGB(_red, _green, _blue) + msgToDevice['xy'] = [_xyY.x, _xyY.y] + #msgToDevice['bri']= int(round(_xyY.y*255,0)) + elif k == 'hue': + if datacontainsRGB==False and self.only_white_bulb is False: + msgToDevice['hue'] = postmsg.get('hue') + elif k == 'saturation': + if datacontainsRGB==False and self.only_white_bulb is False: + msgToDevice['sat'] = int(round(float(postmsg.get('saturation'))*255.0/100.0,0)) + else: + msgToDevice[k] = v + return msgToDevice + # ---------------------------------------------------------------------- + # method3: Identify this lights (Physically) + def identifyDevice(self): + identifyDeviceResult = False + print(" {0}Agent for {1} is identifying itself by doing colorloop. Please observe your lights" + .format(self.variables.get('agent_id',None), self.variables.get('model',None))) + try: + devicewasoff=0 + if self.get_variable('status')=="OFF": + devicewasoff=1 + self.setDeviceStatus({"status":"ON"}) + elif self.only_white_bulb: + self.setDeviceStatus({"status":"OFF"}) + if self.only_white_bulb is False: + self.setDeviceStatus({"effect": "colorloop"}) + if self.only_white_bulb: + time_iden = 3 + else: + time_iden = 10 #time to do identification + t0 = time.time() + self.seconds = time_iden + while time.time() - t0 <= time_iden: + self.seconds = self.seconds - 1 + print("wait: {} sec".format(self.seconds)) + time.sleep(1) + self.setDeviceStatus({"effect": "none"}) + if devicewasoff==1: + self.setDeviceStatus({"status":"OFF"}) + else: + self.setDeviceStatus({"status":"ON"}) + identifyDeviceResult = True + except: + print("ERROR: classAPI_PhilipsHue connection failure! @ identifyDevice") + return identifyDeviceResult + # ---------------------------------------------------------------------- + +# This main method will not be executed when this class is used as a module +def main(): + # create an object with initialized data from DeviceDiscovery Agent + # requirements for instantiation1. model, 2.type, 3.api, 4. address + PhilipsHue = API(model='Philips Hue',type='wifiLight',api='API3',address='http://192.168.10.14:80',username='acquired username',agent_id='LightingAgent') + print("{0}agent is initialzed for {1} using API={2} at {3}".format(PhilipsHue.get_variable('type'),PhilipsHue.get_variable('model'),PhilipsHue.get_variable('api'),PhilipsHue.get_variable('address'))) + + PhilipsHue.getDeviceStatus() + PhilipsHue.setDeviceStatus({"status":"ON","color":(155,113,255)}) + PhilipsHue.identifyDevice() + + +if __name__ == "__main__": main() \ No newline at end of file -- GitLab From 4ff917ac844af199fe7f7f3217dcf9f7dac2d63d Mon Sep 17 00:00:00 2001 From: mchlburton Date: Wed, 22 Mar 2017 04:00:05 +0000 Subject: [PATCH 02/31] Added influx agent for lighting and basic API for influx, exception handling and config files still need to be done --- Agents/LightingAgent/lighting/influxagent.py | 556 +++++++++++++++++++ bemoss_lib/databases/influxAPI/InfluxDB.py | 217 ++++++++ bemoss_lib/databases/influxAPI/__init__.py | 1 + 3 files changed, 774 insertions(+) create mode 100755 Agents/LightingAgent/lighting/influxagent.py create mode 100755 bemoss_lib/databases/influxAPI/InfluxDB.py create mode 100644 bemoss_lib/databases/influxAPI/__init__.py diff --git a/Agents/LightingAgent/lighting/influxagent.py b/Agents/LightingAgent/lighting/influxagent.py new file mode 100755 index 0000000..b1e2dc1 --- /dev/null +++ b/Agents/LightingAgent/lighting/influxagent.py @@ -0,0 +1,556 @@ +import sys +import json +import logging +import importlib +import datetime +import socket + +from volttron.platform.agent import BaseAgent, PublishMixin, periodic +from volttron.platform.agent import utils, matching +from volttron.platform.messaging import headers as headers_mod +from bemoss_lib.communication.Email import EmailService +from bemoss_lib.communication.sms import SMSService +import psycopg2 +import psycopg2.extras +import settings +from influxdb import InfluxDBClient +from bemoss_lib.databases.cassandraAPI import cassandraDB +from bemoss_lib.databases.influxAPI import InfluxDB + +utils.setup_logging() +_log = logging.getLogger(__name__) + +# Step1: Agent Initialization +def LightingAgent(config_path, **kwargs): + config = utils.load_config(config_path) + + def get_config(name): + try: + kwargs.pop(name) + except KeyError: + return config.get(name, '') + + #1. @params agent + agent_id = get_config('agent_id') + device_monitor_time = get_config('device_monitor_time') + max_monitor_time = int(settings.DEVICES['max_monitor_time']) + + debug_agent = False + #List of all keywords for a lighting agent + agentAPImapping = dict(status=[], brightness=[], color=[], saturation=[],power=[]) + log_variables = dict(status='text',brightness='double',hexcolor='text',power='double',offline_count='int') + #2. @params device_info + #TODO correct the launchfile in Device Discovery Agent + building_name = get_config('building_name') + zone_id = get_config('zone_id') + model = get_config('model') + if model == "Philips hue bridge": + hue_username = get_config('username') + else: + hue_username = '' + device_type = get_config('type') + address = get_config('address') + _address = address + _address = _address.replace('http://', '') + _address = _address.replace('https://', '') + try: # validate whether or not address is an ip address + socket.inet_aton(_address) + ip_address = _address + except socket.error: + ip_address = None + identifiable = get_config('identifiable') + + #3. @params agent & DB interfaces + #TODO delete variable topic + topic = get_config('topic') + + #TODO get database parameters from settings.py, add db_table for specific table + db_host = get_config('db_host') + db_port = get_config('db_port') + db_database = get_config('db_database') + db_user = get_config('db_user') + db_password = get_config('db_password') + db_table_lighting = settings.DATABASES['default']['TABLE_lighting'] + db_table_active_alert = settings.DATABASES['default']['TABLE_active_alert'] + db_table_bemoss_notify = settings.DATABASES['default']['TABLE_bemoss_notify'] + db_table_alerts_notificationchanneladdress = settings.DATABASES['default'][ + 'TABLE_alerts_notificationchanneladdress'] + db_table_temp_time_counter = settings.DATABASES['default']['TABLE_temp_time_counter'] + db_table_priority = settings.DATABASES['default']['TABLE_priority'] + + #construct _topic_Agent_UI based on data obtained from DB + _topic_Agent_UI_tail = building_name + '/' + str(zone_id) + '/' + agent_id + + #4. @params device_api + api = get_config('api') + apiLib = importlib.import_module("DeviceAPI.classAPI."+api) + + #4.1 initialize device object + Light = apiLib.API(model=model, device_type=device_type, api=api, address=address, username = hue_username, agent_id=agent_id, db_host=db_host, db_port=db_port, db_user=db_user, db_password=db_password, db_database=db_database) + print("{0}agent is initialized for {1} using API={2} at {3}".format(agent_id, Light.get_variable('model'), + Light.get_variable('api'), + Light.get_variable('address'))) + + #5. @params notification_info + send_notification = True + email_fromaddr = settings.NOTIFICATION['email']['fromaddr'] + email_username = settings.NOTIFICATION['email']['username'] + email_password = settings.NOTIFICATION['email']['password'] + email_mailServer = settings.NOTIFICATION['email']['mailServer'] + notify_heartbeat = settings.NOTIFICATION['heartbeat'] + + class Agent(PublishMixin, BaseAgent): + """Agent for querying WeatherUndergrounds API""" + + #1. agent initialization + def __init__(self, **kwargs): + super(Agent, self).__init__(**kwargs) + #1. initialize all agent variables + self.variables = kwargs + self.valid_data = False + self._keep_alive = True + self.flag = 1 + self.topic = topic + self.time_send_notification = 0 + self.event_ids = list() + self.time_sent_notifications = {} + self.notify_heartbeat = notify_heartbeat + self.ip_address = ip_address if ip_address != None else None + self.changed_variables = None + self.lastUpdateTime = None + self.already_offline = False + + #2. setup connection with db -> Connect to bemossdb database + try: + self.con = psycopg2.connect(host=db_host, port=db_port, database=db_database, user=db_user, + password=db_password) + self.cur = self.con.cursor() # open a cursor to perfomm database operations + print("{} connects to the database name {} successfully".format(agent_id, db_database)) + except: + print("ERROR: {} fails to connect to the database name {}".format(agent_id, db_database)) + + host = 'localhost' + port = 8086 + user = 'root' + password = 'root' + dbname = 'example' + dbuser = 'mike' + dbuser_password = 'bemoss' + client = InfluxDBClient(host, port, user, password, dbname) + + #3. send notification to notify building admin + self.send_notification = send_notification + self.subject = 'Message from ' + agent_id + + #These set and get methods allow scalability + def set_variable(self,k,v): # k=key, v=value + self.variables[k] = v + + def get_variable(self,k): + return self.variables.get(k, None) # default of get_variable is none + + #2. agent setup method + def setup(self): + super(Agent, self).setup() + #Do a one time push when we start up so we don't have to wait for the periodic + self.timer(1, self.deviceMonitorBehavior) + if identifiable == "True": Light.identifyDevice() + + @periodic(max_monitor_time) #save all data every max_monitor_time + def backupSaveData(self): + try: + Light.getDeviceStatus() + cassandraDB.insert(agent_id,Light.variables,log_variables) + print('Every Data Pushed to cassandra') + except Exception as er: + print("ERROR: {} fails to update cassandra database".format(agent_id)) + print er + + #3. deviceMonitorBehavior (TickerBehavior) + @periodic(device_monitor_time) + def deviceMonitorBehavior(self): + #step1: get current status of a thermostat, then map keywords and variables to agent knowledge + try: + Light.getDeviceStatus() + except: + print("device connection is not successful") + + self.changed_variables = dict() + + for v in log_variables: + if v in Light.variables: + if not v in self.variables or self.variables[v] != Light.variables[v]: + self.variables[v] = Light.variables[v] + self.changed_variables[v] = log_variables[v] + else: + if v not in self.variables: #it won't be in self.variables either (in the first time) + self.changed_variables[v] = log_variables[v] + self.variables[v] = None + try: + # Step: Check if any Device is OFFLINE + self.cur.execute("SELECT id FROM " + db_table_active_alert + " WHERE event_trigger_id=%s", ('5',)) + if self.cur.rowcount != 0: + self.device_offline_detection() + + # Put scan time in database + _time_stamp_last_scanned = datetime.datetime.now() + self.cur.execute("UPDATE "+db_table_lighting+" SET last_scanned_time=%s " + "WHERE lighting_id=%s", + (_time_stamp_last_scanned, agent_id)) + self.con.commit() + except Exception as er: + print er + print("ERROR: {} failed to update last scanned time".format(agent_id)) + + if len(self.changed_variables) == 0: + print 'nothing changed' + return + + self.updateUI() + #step4: update PostgresQL (meta-data) database + try: + self.cur.execute("UPDATE "+db_table_lighting+" SET status=%s WHERE lighting_id=%s", + (self.get_variable('status'), agent_id)) + self.con.commit() + self.cur.execute("UPDATE "+db_table_lighting+" SET brightness=%s WHERE lighting_id=%s", + (self.get_variable('brightness'), agent_id)) + self.con.commit() + self.cur.execute("UPDATE "+db_table_lighting+" SET color=%s WHERE lighting_id=%s", + (self.get_variable('hexcolor'), agent_id)) + self.con.commit() + try: + + if self.get_variable('status') == "ON": + multiple_on_off_status = "" + for dummyvar in range(self.get_variable('number_lights')): + multiple_on_off_status += "1" + self.cur.execute("UPDATE "+db_table_lighting+" SET multiple_on_off=%s WHERE lighting_id=%s", + (multiple_on_off_status, agent_id)) + self.con.commit() + else: # status is off + multiple_on_off_status = "" + for dummyvar in range(self.get_variable('number_lights')): + multiple_on_off_status += "0" + self.cur.execute("UPDATE "+db_table_lighting+" SET multiple_on_off=%s WHERE lighting_id=%s", + (multiple_on_off_status, agent_id)) + self.con.commit() + except: + print("{} this agent has no multiple_on_off_status".format(agent_id)) + #TODO check ip_address + if self.ip_address != None: + psycopg2.extras.register_inet() + _ip_address = psycopg2.extras.Inet(self.ip_address) + self.cur.execute("UPDATE "+db_table_lighting+" SET ip_address=%s WHERE lighting_id=%s", + (_ip_address, agent_id)) + self.con.commit() + + + if self.get_variable('offline_count') >= 3: + self.cur.execute("UPDATE "+db_table_lighting+" SET network_status=%s WHERE lighting_id=%s", + ('OFFLINE', agent_id)) + self.con.commit() + if self.already_offline is False: + self.already_offline = True + _time_stamp_last_offline = str(datetime.datetime.now()) + self.cur.execute("UPDATE "+db_table_lighting+" SET last_offline_time=%s WHERE lighting_id=%s", + (_time_stamp_last_offline, agent_id)) + self.con.commit() + else: + self.already_offline = False + self.cur.execute("UPDATE "+db_table_lighting+" SET network_status=%s WHERE lighting_id=%s", + ('ONLINE', agent_id)) + self.con.commit() + print("{} updates database name {} during deviceMonitorBehavior successfully".format(agent_id, db_database)) + except: + print("ERROR: {} fails to update the database name {}".format(agent_id,db_database)) + + #step6: Try to update Influxdb + try: + time = str(datetime.datetime.utcnow()) + InfluxDB.insert(agent_id, self.variables, log_variables) + print "{} success INFLUX update".format(agent_id) + + except: + print("ERROR: {} fails to update INFLUX".format(agent_id)) + + #step6: debug agent knowledge + if debug_agent: + print("printing agent's knowledge") + for k,v in self.variables.items(): + print (k,v) + print('') + + if debug_agent: + print("printing agentAPImapping's fields") + for k, v in agentAPImapping.items(): + if k is None: + agentAPImapping.update({v: v}) + agentAPImapping.pop(k) + for k, v in agentAPImapping.items(): + print (k, v) + + def device_offline_detection(self): + self.cur.execute("SELECT nickname FROM " + db_table_lighting + " WHERE lighting_id=%s", + (agent_id,)) + print agent_id + if self.cur.rowcount != 0: + device_nickname=self.cur.fetchone()[0] + print device_nickname + else: + device_nickname = '' + _db_notification_subject = 'BEMOSS Device {} {} went OFFLINE!!!'.format(device_nickname,agent_id) + _email_subject = '#Attention: BEMOSS Device {} {} went OFFLINE!!!'.format(device_nickname,agent_id) + _email_text = '#Attention: BEMOSS Device {} {} went OFFLINE!!!'.format(device_nickname,agent_id) + self.cur.execute("SELECT network_status FROM " + db_table_lighting + " WHERE lighting_id=%s", + (agent_id,)) + self.network_status = self.cur.fetchone()[0] + print self.network_status + if self.network_status=="OFFLINE": + print "Found Device OFFLINE" + self.cur.execute("SELECT id FROM " + db_table_active_alert + " WHERE event_trigger_id=%s", ('5',)) + self._active_alert_id = self.cur.fetchone()[0] + self.cur.execute( + "SELECT id FROM " + db_table_temp_time_counter + " WHERE alert_id=%s AND device_id=%s", + (str(self._active_alert_id), agent_id,)) + # If this is the first detected violation + if self.cur.rowcount == 0: + print "first device offline detected" + # create counter in DB + self.cur.execute( + "INSERT INTO " + db_table_temp_time_counter + " VALUES(DEFAULT,%s,%s,%s,%s,%s)", + (self._active_alert_id, agent_id, '0', '0', '0')) + self.con.commit() + self.send_device_notification_db(_db_notification_subject, self._active_alert_id) + + # Send email if exist + self.cur.execute("SELECT notify_address FROM " + db_table_alerts_notificationchanneladdress + " WHERE active_alert_id=%s AND notification_channel_id=%s",(self._active_alert_id,'1')) + if self.cur.rowcount != 0: + self._alert_email = self.cur.fetchall() + for single_email_1 in self._alert_email: + print single_email_1[0] + self.send_device_notification_email(single_email_1[0], _email_subject, _email_text) + + # Send SMS if provided by user + self.cur.execute("SELECT notify_address FROM " + db_table_alerts_notificationchanneladdress + " WHERE active_alert_id=%s AND notification_channel_id=%s",(self._active_alert_id,'2')) + if self.cur.rowcount != 0: + self._alert_sms_phone_no = self.cur.fetchall() + for single_number in self._alert_sms_phone_no: + print single_number[0] + self.send_device_notification_sms(single_number[0], _email_subject) + else: + self.priority_counter(self._active_alert_id, _db_notification_subject) + else: + print "The Device is ONLINE" + + def send_device_notification_db(self, _tampering_device_msg, _active_alert_id): + print " INSIDE send_device_notification_db" + + # Find the priority id + self.cur.execute( + "SELECT priority_id FROM " + db_table_active_alert + " WHERE id=%s", + (str(_active_alert_id),)) + self.priority_id = self.cur.fetchone()[0] + + # Find the priority level + self.cur.execute( + "SELECT priority_level FROM " + db_table_priority + " WHERE id=%s", + str(self.priority_id)) + self.priority_level = self.cur.fetchone()[0] + + # Insert into DB the notification + self.cur.execute("INSERT INTO " + db_table_bemoss_notify + " VALUES(DEFAULT,%s,%s,%s,%s)", + (_tampering_device_msg, + str(datetime.datetime.now()), 'Alert', str(self.priority_level))) + self.con.commit() + + # Find the number of notifications sent for the same alert and device + self.cur.execute( + "SELECT no_notifications_sent FROM " + db_table_temp_time_counter + " WHERE alert_id=%s AND device_id=%s", + (str(_active_alert_id), agent_id,)) + self._no_notifications_sent = self.cur.fetchone()[0] + self.con.commit() + print self._no_notifications_sent + self._no_notifications_sent = int(self._no_notifications_sent) + 1 + print self._no_notifications_sent + self.cur.execute( + "UPDATE " + db_table_temp_time_counter + " SET no_notifications_sent=%s WHERE alert_id=%s AND device_id=%s", + (str(self._no_notifications_sent), str(_active_alert_id), agent_id,)) + self.con.commit() + + def send_device_notification_email(self, _active_alert_email, _email_subject, _email_text): + emailService = EmailService() + # Send Email + emailService.sendEmail(email_fromaddr, _active_alert_email, email_username, + email_password, _email_subject, _email_text, email_mailServer) + + def send_device_notification_sms(self, _active_alert_phone_number_misoperation, _sms_subject): + print "INSIDE send_device_notification_sms" + print _active_alert_phone_number_misoperation + smsService = SMSService() + smsService.sendSMS(email_fromaddr, _active_alert_phone_number_misoperation, email_username, email_password, _sms_subject, email_mailServer) + + # TODO: this function is in all other agents, need to get rid of those redundent codes. + def priority_counter(self, _active_alert_id, _tampering_device_msg_1): + # Find the priority counter limit then compare it with priority_counter in priority table + # if greater than the counter limit then send notification and reset the value + # else just increase the counter + print "INSIDE the priority_counter" + _email_subject = '#Attention: BEMOSS Device {} went OFFLINE!!!'.format(agent_id) + _email_text = '#Attention: BEMOSS Device {} went OFFLINE!!!'.format(agent_id) + self.cur.execute( + "SELECT priority_counter FROM " + db_table_temp_time_counter + " WHERE alert_id=%s AND device_id=%s", + (str(_active_alert_id), agent_id,)) + self.priority_count = self.cur.fetchone()[0] + self.con.commit() + + # Find the priority id from active alert table + self.cur.execute( + "SELECT priority_id FROM " + db_table_active_alert + " WHERE id=%s", + (str(_active_alert_id),)) + self.priority_id = self.cur.fetchone()[0] + self.con.commit() + + # Find the priority limit from the priority table + self.cur.execute( + "SELECT priority_counter FROM " + db_table_priority + " WHERE id=%s", + (str(self.priority_id),)) + self.priority_limit = self.cur.fetchone()[0] + self.con.commit() + + # If the counter reaches the limit + if int(self.priority_count) > int(self.priority_limit): + self.send_device_notification_db(_tampering_device_msg_1, _active_alert_id) + self.cur.execute( + "UPDATE " + db_table_temp_time_counter + " SET priority_counter=%s WHERE alert_id=%s AND device_id=%s", + ('0', str(_active_alert_id), agent_id,)) + self.con.commit() + + print "INSIDE the priority counter exceeded the defined range" + # Send email if exist + self.cur.execute("SELECT notify_address FROM " + db_table_alerts_notificationchanneladdress + " WHERE active_alert_id=%s AND notification_channel_id=%s",(self._active_alert_id,'1')) + if self.cur.rowcount != 0: + self._alert_email = self.cur.fetchall() + for single_email_1 in self._alert_email: + print single_email_1[0] + self.send_device_notification_email(single_email_1[0], _email_subject, _email_text) + + # Send SMS if provided by user + self.cur.execute("SELECT notify_address FROM " + db_table_alerts_notificationchanneladdress + " WHERE active_alert_id=%s AND notification_channel_id=%s",(self._active_alert_id,'2')) + if self.cur.rowcount != 0: + self._alert_sms_phone_no = self.cur.fetchall() + for single_number in self._alert_sms_phone_no: + print single_number[0] + self.send_device_notification_sms(single_number[0], _email_subject) + else: + self.priority_count = int(self.priority_count) + 1 + self.cur.execute( + "UPDATE " + db_table_temp_time_counter + " SET priority_counter=%s WHERE alert_id=%s AND device_id=%s", + (str(self.priority_count), str(_active_alert_id), agent_id,)) + + def updateUI(self): + topic = '/agent/ui/'+device_type+'/device_status_response/'+_topic_Agent_UI_tail + headers = { + 'AgentID': agent_id, + headers_mod.CONTENT_TYPE: headers_mod.CONTENT_TYPE.JSON, + headers_mod.FROM: agent_id, + headers_mod.TO: 'ui' + } + _data={'status': self.get_variable('status'), + 'brightness': self.get_variable('brightness'), 'color': self.get_variable('hexcolor'), + 'saturation': self.get_variable('saturation')} + message = json.dumps(_data) + message = message.encode(encoding='utf_8') + self.publish(topic, headers, message) + + #4. updateUIBehavior (generic behavior) + @matching.match_exact('/ui/agent/'+device_type+'/device_status/'+_topic_Agent_UI_tail) + def updateUIBehavior(self,topic,headers,message,match): + print "{} agent got\nTopic: {topic}".format(self.get_variable("agent_id"),topic=topic) + print "Headers: {headers}".format(headers=headers) + print "Message: {message}\n".format(message=message) + #reply message + self.updateUI() + + + #5. deviceControlBehavior (generic behavior) + @matching.match_exact('/ui/agent/'+device_type+'/update/'+_topic_Agent_UI_tail) + def deviceControlBehavior(self,topic,headers,message,match): + #print received message from UI + print "{} agent got\nTopic: {topic}".format(self.get_variable("agent_id"),topic=topic) + print "Headers: {headers}".format(headers=headers) + print "Message: {message}\n".format(message=message) + + #prepare message content for response + topic = '/agent/ui/'+device_type+'/update_response/'+_topic_Agent_UI_tail + headers = { + 'AgentID': agent_id, + headers_mod.CONTENT_TYPE: headers_mod.CONTENT_TYPE.PLAIN_TEXT, + headers_mod.FROM: agent_id, + headers_mod.TO: 'ui' + } + #step1: change device status according to the receive message + if self.isPostmsgValid(message[0]): + setDeviceStatusResult = Light.setDeviceStatus(json.loads(message[0])) #convert received message from string to JSON + #TODO need to do additional checking whether the device setting is actually success!!!!!!!! + #step3: send reply message back to the UI + if setDeviceStatusResult: + message = 'success' + else: + message = 'failure' + else: + print("The POST message is invalid, check brightness, status or color setting and try again\n") + message = 'failure' + self.publish(topic, headers, message) + self.deviceMonitorBehavior() + + def isPostmsgValid(self, postmsg): # check validity of postmsg + dataValidity = True + try: + _data = json.loads(postmsg) + if ("brightness" in _data.keys()) or ("status" in _data.keys()) or ("color" in _data.keys()): + dataValidity = True + else: + dataValidity = False + except: + dataValidity = False + print("dataValidity failed to validate data coming from UI") + return dataValidity + + #6. deviceIdentifyBehavior(generic behavior) + @matching.match_exact('/ui/agent/'+device_type+'/identify/'+_topic_Agent_UI_tail) + def deviceIdentifyBehavior(self,topic,headers,message,match): + print "{} agent got\nTopic: {topic}".format(self.get_variable("agent_id"),topic=topic) + print "Headers: {headers}".format(headers=headers) + print "Message: {message}\n".format(message=message) + #step1: change device status according to the receive message + identifyDeviceResult = Light.identifyDevice() + #TODO need to do additional checking whether the device setting is actually success!!!!!!!! + #step2: send reply message back to the UI + topic = '/agent/ui/identify_response/'+device_type+'/'+_topic_Agent_UI_tail + headers = { + 'AgentID': agent_id, + headers_mod.CONTENT_TYPE: headers_mod.CONTENT_TYPE.PLAIN_TEXT, + } + if identifyDeviceResult: + message = 'success' + else: + message = 'failure' + self.publish(topic, headers, message) + + + + Agent.__name__ = 'LightingAgent' + return Agent(**kwargs) + +def main(argv=sys.argv): + '''Main method called by the eggsecutable.''' + utils.default_main(LightingAgent, + description='Lighting agent', + argv=argv) + +if __name__ == '__main__': + try: + sys.exit(main()) + except KeyboardInterrupt: + pass + diff --git a/bemoss_lib/databases/influxAPI/InfluxDB.py b/bemoss_lib/databases/influxAPI/InfluxDB.py new file mode 100755 index 0000000..4115a1e --- /dev/null +++ b/bemoss_lib/databases/influxAPI/InfluxDB.py @@ -0,0 +1,217 @@ +import pandas +import numpy +import datetime +from influxdb import InfluxDBClient +connection_established = False + +#Global variables +host = 'localhost' +port = 8086 +user = 'root' +password = 'root' +dbname = 'bemoss' +client = InfluxDBClient(host, port, user, password, dbname) + + +def makeConnection(): + try: + global host, port, user, password, dbname, connection_established + client = InfluxDBClient(host, port, user, password, dbname) + database_list = client.get_list_database() + for db in database_list: + if dbname in db['name']: + connection_established = True + return True + + except Exception as er: + print 'Cannot establish connection' + raise er + +try: + makeConnection() +except Exception as er: + print 'Connection cannot be established' + print er + + +def insert(agentID, all_vars, log_vars, cur_time=None): + """ + :param agentID: string. Data will be inserted to table named B + :param all_vars: dictionary (usually APIobject.variables). It contains all the variables and their values. + :param log_vars: dictionary (usually APIobject.log_variables). It contains variables to be logged and their datatypes + :return: 0, if successful + + timestamp is generated based on current utc time. UTC time is put in cassandra database. If the table by the agent + name doesn't exist, table is created. **If error occurs because the name of variables/data_type has changed, the old + table will be deleted, and a new one with currect variable names/datatype will be created**. + + **Need to avoid doing this in final version.Feature made to help during development + """ + global connection_established + + if not connection_established: + makeConnection() + + client = InfluxDBClient(host, port, user, password, dbname) + + if cur_time == None: + cur_time = datetime.datetime.utcnow() + + + influx_json = [] + for var in log_vars: + reading = all_vars.get(var) + if not reading == None: + var_json =[ + { + "measurement": var, + "tags": { + "host": "bemoss", + "agent": agentID + }, + "time": cur_time, + "fields": { + "value": reading + } + } + ] + if influx_json == []: + influx_json.extend(var_json) + else: + influx_json.extend(var_json) + + retry = True + while retry: + client.write_points(influx_json) + retry = False + + return 0 + + +def delete(agentID,startDate=None, endDate=None): + """ + Performs deletion of data. if statDate and endDate is omitted, all device data is deleted. + :param agentID: The on which the operation is to be performed + :param startDate: dateTime.date object (local timezone). The begining date from which to delete. Must be supplied + unless trying to delete the whole table + :param endDate: datetime.date object (local timezone). The endDate upto which to delete. The default is to upto today. + must be supplied unless trying to delete the whole table + :return: 0, if successful + + """ + global connection_established + if not connection_established: + makeConnection() + + if endDate==None and startDate==None: + try: + delete_query = 'DROP SERIES WHERE agentID = {0}'.format(agentID) + client.query(delete_query) + return 0 + except Exception: + connection_established = False #Try to establish connection again next time + raise + + elif startDate==None: + print "startTime compulsory when endTime is given" + return -1 + elif endDate==None: + endDate=datetime.datetime.utcnow() + + try: + result = client.query("SHOW MEASUREMENTS ON {0} WHERE agent = '{1}';".format(dbname, agentID)) + except: + connection_established = False #Try to establish connection again next time + raise + + daterange = pandas.date_range(startDate,endDate) + date_local="" + try: + x = 1 + #client.query("select value, {0} from {1} WHERE time >= '{2}' AND time <= '{3}';".format(agentID, varStr, startTime,endTime)) + + + except Exception as e: + print "sorry, not deleted all. deleted upto:%s" % date_local + connection_established = False + return -1 + + return 0 + + + + + +def retrieve(agentID, vars=None, startTime=None, endTime=None,export=False): + """Function to retrieve Data from Influx. \n + :param agentID: must supply, since each agentID is associated with a table in database. + :param vars: supplied as a list of strings. It represents the variables to be retrieved from the table. + eg. ['temperature','humidity']. If any of the variables don't match the column name, the result + will contain -1. If not supplied, the default is to return the complete row \n\n + :param startTime: the time in localtime zone (the timezone of the node), in datetime.datetime.utcnow format. It marks the + beginning for the range. If not supplied, will be taken 24-hours before endTime + :param endTime: the time in localtime zone (the timezone of the node), in datetime.datetime.utcnow format.It marks the end of the + range. If not supplied, will be taken as the currentTime. + :return: A numpy 2-dimensional array. The first column contains the time stamps for the entries, and the rest of the + columns correspond to variables queried. Each row corresponds to + various table entries. Also returns vars, which serves as a column index. It is identical to passed vars + except 'time' is an extra entry at the front. If the query fails, -1 is returned (and no exception raised) + + """ + + global connection_established + if not connection_established: + makeConnection() + + client = InfluxDBClient(host, port, user, password, dbname) + + if startTime==None: + startTime=str(datetime.datetime.utcnow()-datetime.timedelta(hours=24)) + + if endTime==None: + endTime = str(datetime.datetime.utcnow()) + + if vars==None: + varStr='' + vars=[] + try: + result = client.query("SHOW MEASUREMENTS ON {0} WHERE agent = '{1}';".format(dbname, agentID)) + except: + connection_established = False #Try to establish connection again next time + raise + result = result.get_points('measurements') + + for var in result: + measurement = str(var['name']) + print measurement + varStr += measurement + ', ' + vars.append(measurement) + varStr = varStr[:-2] #to get rid of the last ', ' + else: + varStr = '' + for var in vars: + varStr += var + ', ' + + varStr = varStr[:-2] #to get rid of the last ', ' + + total_result = [] + for var in vars: + result = client.query("select value, {0} from {1} WHERE time >= '{2}' AND time <= '{3}';".format(agentID, var, startTime,endTime)) + values = [] + entries = result.get_points(var) + if total_result == []: + time = [] + for entry in entries: + time.append(entry['time']) + values.append(entry['value']) + total_result.append(time) + else: + for entry in entries: + values.append(entry['value']) + + total_result += [values] + + vars.insert(0, 'time') + total_result = numpy.array(total_result) + total_result = numpy.transpose(total_result) + return vars, total_result \ No newline at end of file diff --git a/bemoss_lib/databases/influxAPI/__init__.py b/bemoss_lib/databases/influxAPI/__init__.py new file mode 100644 index 0000000..b9675ce --- /dev/null +++ b/bemoss_lib/databases/influxAPI/__init__.py @@ -0,0 +1 @@ +__author__ = 'mike' -- GitLab From f5a0617ca3d1f9dde977a2f85a1399f7e15fac73 Mon Sep 17 00:00:00 2001 From: mchlburton Date: Thu, 23 Mar 2017 11:08:45 -0400 Subject: [PATCH 03/31] Fixed problem with query statement --- bemoss_lib/databases/influxAPI/InfluxDB.py | 30 +++++++++++----------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/bemoss_lib/databases/influxAPI/InfluxDB.py b/bemoss_lib/databases/influxAPI/InfluxDB.py index 4115a1e..b6dc753 100755 --- a/bemoss_lib/databases/influxAPI/InfluxDB.py +++ b/bemoss_lib/databases/influxAPI/InfluxDB.py @@ -88,7 +88,7 @@ def insert(agentID, all_vars, log_vars, cur_time=None): return 0 -def delete(agentID,startDate=None, endDate=None): +def delete(agentID,startTime=None, endTime=None): """ Performs deletion of data. if statDate and endDate is omitted, all device data is deleted. :param agentID: The on which the operation is to be performed @@ -103,20 +103,20 @@ def delete(agentID,startDate=None, endDate=None): if not connection_established: makeConnection() - if endDate==None and startDate==None: + if endTime==None and startTime==None: try: - delete_query = 'DROP SERIES WHERE agentID = {0}'.format(agentID) + delete_query = "DROP SERIES WHERE agent = '{0}'".format(agentID) client.query(delete_query) return 0 except Exception: connection_established = False #Try to establish connection again next time raise - elif startDate==None: + elif startTime==None: print "startTime compulsory when endTime is given" return -1 - elif endDate==None: - endDate=datetime.datetime.utcnow() + elif endTime==None: + endTime = str(datetime.datetime.utcnow()) try: result = client.query("SHOW MEASUREMENTS ON {0} WHERE agent = '{1}';".format(dbname, agentID)) @@ -124,17 +124,17 @@ def delete(agentID,startDate=None, endDate=None): connection_established = False #Try to establish connection again next time raise - daterange = pandas.date_range(startDate,endDate) - date_local="" - try: - x = 1 - #client.query("select value, {0} from {1} WHERE time >= '{2}' AND time <= '{3}';".format(agentID, varStr, startTime,endTime)) + result = result.get_points('measurements') + try: + for var in result: + measurement = str(var['name']) + client.query("DELETE FROM {0} WHERE agent = '{1}' AND time >= '{2}' AND time <= '{3}';".format(measurement, agentID, startTime, endTime)) - except Exception as e: - print "sorry, not deleted all. deleted upto:%s" % date_local + except: connection_established = False - return -1 + raise + return 0 @@ -196,7 +196,7 @@ def retrieve(agentID, vars=None, startTime=None, endTime=None,export=False): total_result = [] for var in vars: - result = client.query("select value, {0} from {1} WHERE time >= '{2}' AND time <= '{3}';".format(agentID, var, startTime,endTime)) + result = client.query("select value from {0} WHERE agent = '{1}' AND time >= '{2}' AND time <= '{3}';".format(var, agentID, startTime,endTime)) values = [] entries = result.get_points(var) if total_result == []: -- GitLab From db192e7f8c67719130611270b51d6806ea417331 Mon Sep 17 00:00:00 2001 From: mchlburton Date: Sun, 26 Mar 2017 20:33:10 -0400 Subject: [PATCH 04/31] influx functions and dummy agent for PhilipsHue light; cassandraDB calls can be replaced with influxDB calls maintaining same parameters --- Agents/LightingAgent/lighting/influxagent.py | 9 +- .../classAPI/classAPI_DummyPhilipsHue.py | 124 ++---------------- bemoss_lib/databases/influxAPI/InfluxDB.py | 41 ++++-- 3 files changed, 38 insertions(+), 136 deletions(-) diff --git a/Agents/LightingAgent/lighting/influxagent.py b/Agents/LightingAgent/lighting/influxagent.py index b1e2dc1..86930b0 100755 --- a/Agents/LightingAgent/lighting/influxagent.py +++ b/Agents/LightingAgent/lighting/influxagent.py @@ -129,14 +129,7 @@ def LightingAgent(config_path, **kwargs): except: print("ERROR: {} fails to connect to the database name {}".format(agent_id, db_database)) - host = 'localhost' - port = 8086 - user = 'root' - password = 'root' - dbname = 'example' - dbuser = 'mike' - dbuser_password = 'bemoss' - client = InfluxDBClient(host, port, user, password, dbname) + #3. send notification to notify building admin self.send_notification = send_notification diff --git a/DeviceAPI/classAPI/classAPI_DummyPhilipsHue.py b/DeviceAPI/classAPI/classAPI_DummyPhilipsHue.py index 6276e10..20b8062 100755 --- a/DeviceAPI/classAPI/classAPI_DummyPhilipsHue.py +++ b/DeviceAPI/classAPI/classAPI_DummyPhilipsHue.py @@ -1,54 +1,14 @@ -# -*- coding: utf-8 -*- ''' -Copyright (c) 2016, Virginia Tech -All rights reserved. - -Redistribution and use in source and binary forms, with or without modification, are permitted provided that the - following conditions are met: -1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following -disclaimer. -2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following -disclaimer in the documentation and/or other materials provided with the distribution. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, -INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE -DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR -SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, -WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -The views and conclusions contained in the software and documentation are those of the authors and should not be -interpreted as representing official policies, either expressed or implied, of the FreeBSD Project. - -This material was prepared as an account of work sponsored by an agency of the United States Government. Neither the -United States Government nor the United States Department of Energy, nor Virginia Tech, nor any of their employees, -nor any jurisdiction or organization that has cooperated in the development of these materials, makes any warranty, -express or implied, or assumes any legal liability or responsibility for the accuracy, completeness, or usefulness or -any information, apparatus, product, software, or process disclosed, or represents that its use would not infringe -privately owned rights. - -Reference herein to any specific commercial product, process, or service by trade name, trademark, manufacturer, or -otherwise does not necessarily constitute or imply its endorsement, recommendation, favoring by the United States -Government or any agency thereof, or Virginia Tech - Advanced Research Institute. The views and opinions of authors -expressed herein do not necessarily state or reflect those of the United States Government or any agency thereof. - -VIRGINIA TECH – ADVANCED RESEARCH INSTITUTE -under Contract DE-EE0006352 - -#__author__ = "BEMOSS Team" -#__credits__ = "" +#__author__ = "Mike" +#__credits__ = "BEMOSS Team" #__version__ = "2.0" -#__maintainer__ = "BEMOSS Team" -#__email__ = "aribemoss@gmail.com" -#__website__ = "www.bemoss.org" -#__created__ = "2014-09-12 12:04:50" -#__lastUpdated__ = "2016-03-14 11:23:33" +#__maintainer__ = "Mike" +#__email__ = "michael@blocpower.io" +#__created__ = "2017-03-16" +#__lastUpdated__ = "2017-23-17" ''' import time -import urllib2 -import json from random import randint from bemoss_lib.utils import rgb_cie class API: @@ -96,20 +56,8 @@ class API: # getDeviceStatus(), getDeviceStatusJson(data), printDeviceStatus() def getDeviceStatus(self): getDeviceStatusResult = True - _hue_username = self.get_variable("username") - _url_append = '/api/'+_hue_username+'/groups/0/' - _urlData = self.get_variable("address").replace(':80', _url_append) + try: - '''_deviceUrl = urllib2.urlopen(_urlData, timeout=20) - print(" {0}Agent is querying its current status (status:{1}) please wait ...". - format(self.variables.get('agent_id', None), _deviceUrl.getcode())) - if (_deviceUrl.getcode() == 200): - self.getDeviceStatusJson(_deviceUrl.read().decode("utf-8")) #convert string data to JSON object then interpret it - if self.debug is True: - self.printDeviceStatus() - else: - print (" Received an error from server, cannot retrieve results " + str(_deviceUrl.getcode())) - getDeviceStatusResult = False''' onoff = randint(0, 1) if onoff == 1: self.set_variable('status', "ON") @@ -120,8 +68,6 @@ class API: xy = [randint(0, 255), randint(0,255)] ct = randint(0, 255) sat = randint(0, 255) - effect = "effect" - mode = "warm" name = "DummyHue" # 2. brightness convert to % self.set_variable('brightness',int(round(float(bri)*100/255,0))) @@ -138,8 +84,6 @@ class API: self.set_variable('hexcolor', '#%02x%02x%02x' % self.get_variable('color')) # 4. saturation convert to % self.set_variable('saturation',int(round(float(sat)*100/255,0))) - self.set_variable('effect',effect) - self.set_variable('colormode',mode) self.set_variable('number_lights', [2]) self.set_variable('name',name) @@ -154,37 +98,6 @@ class API: print('ERROR: classAPI_PhilipsHue failed to getDeviceStatus') self.set_variable('offline_count',self.get_variable('offline_count')+1) - ''' - def getDeviceStatusJson(self,data): - # Use the json module to load the string data into a dictionary - _theJSON = json.loads(data) - # 1. status - if _theJSON["action"]["on"] == True: - self.set_variable('status',"ON") - else: - self.set_variable('status',"OFF") - # 2. brightness convert to % - self.set_variable('brightness',int(round(float(_theJSON["action"]["bri"])*100/255,0))) - # update only white variable every round is necessary in case user add a/take away all color bulb(s). - self.only_white_bulb = False if 'hue' in _theJSON["action"].keys() else True - if self.only_white_bulb is False: - # 3. color convert to RGB 0-255 - self.set_variable('hue', _theJSON["action"]["hue"]) - self.set_variable('xy', _theJSON["action"]["xy"]) - self.set_variable('ct', _theJSON["action"]["ct"]) - x=_theJSON["action"]["xy"][0] - y=_theJSON["action"]["xy"][1] - self.set_variable('color', rgb_cie.ColorHelper.getRGBFromXYAndBrightness(x,y,_theJSON["action"]["bri"])) - self.set_variable('hexcolor', '#%02x%02x%02x' % self.get_variable('color')) - # 4. saturation convert to % - self.set_variable('saturation',int(round(float(_theJSON["action"]["sat"])*100/255,0))) - self.set_variable('effect',_theJSON["action"]["effect"]) - self.set_variable('colormode',_theJSON["action"]["colormode"]) - for k in _theJSON["lights"]: - self.set_variable("lights{}".format(k), k) - self.set_variable('number_lights', len(_theJSON["lights"])) - self.set_variable('name',_theJSON["name"]) - ''' def printDeviceStatus(self): # now we can access the contents of the JSON like any other Python object print(" the current status is as follows:") @@ -204,28 +117,7 @@ class API: # setDeviceStatus(postmsg), isPostmsgValid(postmsg), convertPostMsg(postmsg) def setDeviceStatus(self, postmsg): setDeviceStatusResult = True - ''' - #Ex. postmsg = {"on":True,"bri":100,"hue":50260,"sat":200} - _hue_username = self.get_variable("username") - _url_append = '/api/'+_hue_username+'/groups/0/' - _urlData = self.get_variable("address").replace(':80', _url_append) - if self.isPostMsgValid(postmsg) == True: #check if the data is valid - _data = json.dumps(self.convertPostMsg(postmsg)) - _data = _data.encode(encoding='utf_8') - _request = urllib2.Request(_urlData+'action') - _request.add_header('Content-Type','application/json') - _request.get_method = lambda: 'PUT' - try: - _f = urllib2.urlopen(_request, _data, timeout=20) #when include data this become a POST command - print(" {0}Agent for {1} is changing its status with {2} please wait ..." - .format(self.variables.get('agent_id', None), self.variables.get('model', None), postmsg)) - print(" after send a POST request: {}".format(_f.read().decode('utf-8'))) - except: - print("ERROR: classAPI_PhilipsHue connection failure! @ setDeviceStatus") - setDeviceStatusResult = False - else: - print("The POST message is invalid, try again\n") - ''' + return setDeviceStatusResult def isPostMsgValid(self,postmsg): #check validity of postmsg diff --git a/bemoss_lib/databases/influxAPI/InfluxDB.py b/bemoss_lib/databases/influxAPI/InfluxDB.py index b6dc753..39cbe04 100755 --- a/bemoss_lib/databases/influxAPI/InfluxDB.py +++ b/bemoss_lib/databases/influxAPI/InfluxDB.py @@ -1,16 +1,29 @@ -import pandas +''' +Copyright (c) 2017, BlocPower +All rights reserved. + + +#__author__ = "Mike" +#__credits__ = "" +#__version__ = "2.0" +#__maintainer__ = "Mike" +#__email__ = "michael@blocpower.io" +#__website__ = "www.blocpower.io" +#__created__ = "2017-03-21" +#__lastUpdated__ = "2017-03-24" +''' + import numpy import datetime from influxdb import InfluxDBClient connection_established = False #Global variables -host = 'localhost' +host = '52.206.6.10' port = 8086 -user = 'root' -password = 'root' +user = 'user' +password = 'password' dbname = 'bemoss' -client = InfluxDBClient(host, port, user, password, dbname) def makeConnection(): @@ -23,6 +36,7 @@ def makeConnection(): connection_established = True return True + except Exception as er: print 'Cannot establish connection' raise er @@ -41,11 +55,7 @@ def insert(agentID, all_vars, log_vars, cur_time=None): :param log_vars: dictionary (usually APIobject.log_variables). It contains variables to be logged and their datatypes :return: 0, if successful - timestamp is generated based on current utc time. UTC time is put in cassandra database. If the table by the agent - name doesn't exist, table is created. **If error occurs because the name of variables/data_type has changed, the old - table will be deleted, and a new one with currect variable names/datatype will be created**. - - **Need to avoid doing this in final version.Feature made to help during development + timestamp is generated based on current utc time. UTC time is put in influxdb. """ global connection_established @@ -82,8 +92,13 @@ def insert(agentID, all_vars, log_vars, cur_time=None): retry = True while retry: - client.write_points(influx_json) - retry = False + try: + client.write_points(influx_json) + retry = False + except Exception as er: + retry = False + print er + raise return 0 @@ -103,6 +118,8 @@ def delete(agentID,startTime=None, endTime=None): if not connection_established: makeConnection() + client = InfluxDBClient(host, port, user, password, dbname) + if endTime==None and startTime==None: try: delete_query = "DROP SERIES WHERE agent = '{0}'".format(agentID) -- GitLab From 7a9067b14a5b19dcd5b005ad4975b7ab76aa4c84 Mon Sep 17 00:00:00 2001 From: mchlburton Date: Mon, 3 Apr 2017 17:23:07 -0400 Subject: [PATCH 05/31] updated insert function meaning that retrieve function will not work properly until fix is completed --- bemoss_lib/databases/influxAPI/InfluxDB.py | 34 +++++++++------------- 1 file changed, 14 insertions(+), 20 deletions(-) diff --git a/bemoss_lib/databases/influxAPI/InfluxDB.py b/bemoss_lib/databases/influxAPI/InfluxDB.py index 39cbe04..70d6e21 100755 --- a/bemoss_lib/databases/influxAPI/InfluxDB.py +++ b/bemoss_lib/databases/influxAPI/InfluxDB.py @@ -67,28 +67,22 @@ def insert(agentID, all_vars, log_vars, cur_time=None): if cur_time == None: cur_time = datetime.datetime.utcnow() - - influx_json = [] + fields = {} for var in log_vars: reading = all_vars.get(var) - if not reading == None: - var_json =[ - { - "measurement": var, - "tags": { - "host": "bemoss", - "agent": agentID - }, - "time": cur_time, - "fields": { - "value": reading - } - } - ] - if influx_json == []: - influx_json.extend(var_json) - else: - influx_json.extend(var_json) + fields.update({var:reading}) + + influx_json = [ + { + "measurement": "lightsensor", + "tags": { + "agent": agentID, + "building": "bemoss" + }, + "time": cur_time, + "fields": fields + } + ] retry = True while retry: -- GitLab From cf12423f9d44b092d9025f288032d58fd788ab03 Mon Sep 17 00:00:00 2001 From: mchlburton Date: Mon, 17 Apr 2017 14:53:16 -0400 Subject: [PATCH 06/31] TRV Agent with dummy setpoint control --- Agents/TRVAgent/setup.py | 76 ++++ Agents/TRVAgent/trv/__init__.py | 0 Agents/TRVAgent/trv/agent.py | 506 +++++++++++++++++++++++ DeviceAPI/classAPI/classAPI_EnoceanPi.py | 70 ++++ bemoss_lib/utils/buildAgents.sh | 1 + bemoss_lib/utils/platform_initiator.py | 4 + settings.py | 1 + 7 files changed, 658 insertions(+) create mode 100755 Agents/TRVAgent/setup.py create mode 100755 Agents/TRVAgent/trv/__init__.py create mode 100755 Agents/TRVAgent/trv/agent.py create mode 100755 DeviceAPI/classAPI/classAPI_EnoceanPi.py diff --git a/Agents/TRVAgent/setup.py b/Agents/TRVAgent/setup.py new file mode 100755 index 0000000..90215b1 --- /dev/null +++ b/Agents/TRVAgent/setup.py @@ -0,0 +1,76 @@ +# -*- coding: utf-8 -*- {{{ +# vim: set fenc=utf-8 ft=python sw=4 ts=4 sts=4 et: + +# Copyright (c) 2013, Battelle Memorial Institute +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# +# 1. Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# 2. Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in +# the documentation and/or other materials provided with the +# distribution. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +# +# The views and conclusions contained in the software and documentation +# are those of the authors and should not be interpreted as representing +# official policies, either expressed or implied, of the FreeBSD +# Project. +# +# This material was prepared as an account of work sponsored by an +# agency of the United States Government. Neither the United States +# Government nor the United States Department of Energy, nor Battelle, +# nor any of their employees, nor any jurisdiction or organization that +# has cooperated in the development of these materials, makes any +# warranty, express or implied, or assumes any legal liability or +# responsibility for the accuracy, completeness, or usefulness or any +# information, apparatus, product, software, or process disclosed, or +# represents that its use would not infringe privately owned rights. +# +# Reference herein to any specific commercial product, process, or +# service by trade name, trademark, manufacturer, or otherwise does not +# necessarily constitute or imply its endorsement, recommendation, or +# favoring by the United States Government or any agency thereof, or +# Battelle Memorial Institute. The views and opinions of authors +# expressed herein do not necessarily state or reflect those of the +# United States Government or any agency thereof. +# +# PACIFIC NORTHWEST NATIONAL LABORATORY +# operated by BATTELLE for the UNITED STATES DEPARTMENT OF ENERGY +# under Contract DE-AC05-76RL01830 + +#}}} + +from setuptools import setup, find_packages + +#get environ for agent name/identifier +packages = find_packages('.') +package = packages[0] + +setup( + name = package + 'agent', + version = "0.1", + install_requires = ['volttron'], + packages = packages, + entry_points = { + 'setuptools.installation': [ + 'eggsecutable = ' + package + '.agent:main', + ] + } +) + diff --git a/Agents/TRVAgent/trv/__init__.py b/Agents/TRVAgent/trv/__init__.py new file mode 100755 index 0000000..e69de29 diff --git a/Agents/TRVAgent/trv/agent.py b/Agents/TRVAgent/trv/agent.py new file mode 100755 index 0000000..52d0589 --- /dev/null +++ b/Agents/TRVAgent/trv/agent.py @@ -0,0 +1,506 @@ +# -*- coding: utf-8 -*- +''' +#__author__ = "Mike" +#__credits__ = "" +#__version__ = "2.0" +#__maintainer__ = "Mike" +#__email__ = "michael@blocpower.io" +#__website__ = "www.blocopwer.io" +#__created__ = "2017-04-13" +#__lastUpdated__ = " +''' + +import sys +import json +import importlib +import logging +import os +from volttron.platform.agent import BaseAgent, PublishMixin, periodic +from volttron.platform.agent import utils, matching +from volttron.platform.messaging import headers as headers_mod +import datetime +from bemoss_lib.communication.Email import EmailService +from bemoss_lib.communication.sms import SMSService +import psycopg2 +import psycopg2.extras +import settings +import socket +from bemoss_lib.databases.cassandraAPI import cassandraDB +from bemoss_lib.utils.catcherror import catcherror + +utils.setup_logging() +_log = logging.getLogger(__name__) + +# Step1: Agent Initialization +def TRVAgent(config_path, **kwargs): + config = utils.load_config(config_path) + + def get_config(name): + try: + kwargs.pop(name) + except KeyError: + return config.get(name, '') + + # 1. @params agent + agent_id = get_config('agent_id') + device_monitor_time = get_config('device_monitor_time') + max_monitor_time = int(settings.DEVICES['max_monitor_time']) + debug_agent = False + log_variables = dict(setpoint='int', temperature='float', window_open='boolean', alert='string', + operation_mode='string', offline_count='int') + + # 2. @params device_info + building_name = get_config('building_name') + zone_id = get_config('zone_id') + model = get_config('model') + device_type = get_config('type') + address = get_config('address') + macaddress = get_config('macaddress') + _address = address + _address = _address.replace('http://', '') + _address = _address.replace('https://', '') + try: # validate whether or not address is an ip address + socket.inet_pton(socket.AF_INET, _address) + ip_address = _address + except socket.error: + ip_address = None + identifiable = get_config('identifiable') + + # 3. @params agent & DB interfaces + db_host = get_config('db_host') + db_port = get_config('db_port') + db_database = get_config('db_database') + db_user = get_config('db_user') + db_password = get_config('db_password') + db_table_trv = settings.DATABASES['default']['TABLE_trv'] + db_id_column_name = "trv_id" + db_table_bemoss_notify = settings.DATABASES['default']['TABLE_bemoss_notify'] + db_table_active_alert = settings.DATABASES['default']['TABLE_active_alert'] + db_table_device_type = settings.DATABASES['default']['TABLE_device_type'] + db_table_alerts_notificationchanneladdress = settings.DATABASES['default']['TABLE_alerts_notificationchanneladdress'] + db_table_priority = settings.DATABASES['default']['TABLE_priority'] + + #construct _topic_Agent_UI based on data obtained from DB + _topic_Agent_UI_tail = building_name + '/' + str(zone_id) + '/' + agent_id + + # 4. @params device_api + api = get_config('api') + apiLib = importlib.import_module("DeviceAPI.classAPI." + api) + + # 4.1 initialize trv device object + TRV = apiLib.API(model=model, device_type=device_type, api=api, address=address,macaddress=macaddress, + agent_id=agent_id, db_host=db_host, db_port=db_port, db_user=db_user, db_password=db_password, + db_database=db_database, config_path=config_path) + connection_renew_interval = TRV.variables['connection_renew_interval'] + + print("{0}agent is initialized for {1} using API={2} at {3}".format(agent_id, TRV.get_variable('model'), + TRV.get_variable('api'), + TRV.get_variable('address'))) + + # 5. @params notification_info + send_notification = False + email_fromaddr = settings.NOTIFICATION['email']['fromaddr'] + email_recipients = settings.NOTIFICATION['email']['recipients'] + email_username = settings.NOTIFICATION['email']['username'] + email_password = settings.NOTIFICATION['email']['password'] + email_mailServer = settings.NOTIFICATION['email']['mailServer'] + notify_heartbeat = settings.NOTIFICATION['heartbeat'] + + class Agent(PublishMixin, BaseAgent): + + # 1. agent initialization + def __init__(self, **kwargs): + super(Agent, self).__init__(**kwargs) + # 1. initialize all agent variables + self.variables = kwargs + self.variables['setpoint'] = 0 + self.valid_data = False + self._keep_alive = True + self.ip_address = ip_address if ip_address != None else None + self.event_ids = list() + self.time_sent_notifications = {} + self.notify_heartbeat = notify_heartbeat + self.changed_variables = None + self.lastUpdateTime = datetime.datetime.now() + self.runningSeconds = 0 + self._override = True + self.already_offline = False + + # 2. setup connection with db -> Connect to bemossdb database + try: + self.con = psycopg2.connect(host=db_host, port=db_port, database=db_database, user=db_user, + password=db_password) + self.cur = self.con.cursor() # open a cursor to perform database operations + print("{} connects to the database name {} successfully".format(agent_id, db_database)) + except: + print("ERROR: {} fails to connect to the database name {}".format(agent_id, db_database)) + # 3. send notification to notify building admin + self.send_notification = send_notification + self.subject = 'Message from ' + agent_id + + # These set and get methods allow scalability + def set_variable(self, k, v): # k=key, v=value + self.variables[k] = v + + def get_variable(self, k): + return self.variables.get(k, None) # default of get_variable is none + + # 2. agent setup method + def setup(self): + super(Agent, self).setup() + + @periodic(connection_renew_interval) + def renewConnection(self): + TRV.renewConnection() + + # 3. deviceMonitorBehavior (TickerBehavior) + #@periodic(device_monitor_time) + @matching.match_exact('/enocean_gateway/agent/'+device_type+'/device_status/' + _topic_Agent_UI_tail) + def deviceMonitorBehavior(self, topic, headers, message, match): + print agent_id + " got\nTopic: {topic}".format(topic=topic) + print "Headers: {headers}".format(headers=headers) + print "Message: {message}\n".format(message=message) + + # step1: get current status of a trv from bus, then map message keywords and variables to agent knowledge + variables = json.loads(message[0]) + self.changed_variables = dict() + + for v in log_variables: + if v in variables.items(): + if v not in self.variables or self.variables[v] != variables.get(v): + self.variables[v] = variables.get(v) + self.changed_variables[v] = log_variables[v] + else: + if v not in self.variables: # it won't be in self.variables either (in the first time) + self.changed_variables[v] = log_variables[v] + self.variables[v] = None + #step4: update PostgresQL (meta-data) database + try: + #make the device offline if necessary + if self.get_variable('offline_count')>=3: + + self.cur.execute("UPDATE "+db_table_trv+" SET network_status=%s WHERE trv_id=%s", + ('OFFLINE', agent_id)) + self.con.commit() + if self.already_offline is False: + self.already_offline = True + _time_stamp_last_offline = str(datetime.datetime.now()) + self.cur.execute("UPDATE "+db_table_trv+" SET last_offline_time=%s " + "WHERE trv_id=%s", + (_time_stamp_last_offline, agent_id)) + self.con.commit() + else: + self.already_offline = False + self.cur.execute("UPDATE "+db_table_trv+" SET network_status=%s WHERE trv_id=%s", + ('ONLINE', agent_id)) + self.con.commit() + + # put the last scan time on database + _time_stamp_last_scanned = str(datetime.datetime.now()) + self.cur.execute("UPDATE "+db_table_trv+" SET last_scanned_time=%s " + "WHERE trv_id=%s", + (_time_stamp_last_scanned, agent_id)) + self.con.commit() + except Exception as er: + print er + print("ERROR: {} failed to update database name {}".format(agent_id, db_database)) + + if len(self.changed_variables) == 0: + print 'nothing changed' + return + else: + print 'These things changed:' + print self.changed_variables + self.updateUI() + + # step3: update PostgresQL (meta-data) database + try: + for k, v in log_variables.items(): + # check if column exists, then updateDB to corresponding column + self.cur.execute("select column_name from information_schema.columns where table_name=%s and column_name=%s", + (db_table_trv, k,)) + if bool(self.cur.rowcount): + self.updateDB(db_table_trv, k, db_id_column_name, self.get_variable(k), agent_id) + else: + pass + if self.ip_address != None: + psycopg2.extras.register_inet() + _ip_address = psycopg2.extras.Inet(self.ip_address) + self.cur.execute("UPDATE " + db_table_trv + " SET ip_address=%s WHERE trv_id=%s", + (_ip_address, agent_id)) + self.con.commit() + print("{} updates database name {} during deviceMonitorBehavior successfully".format(agent_id, db_database)) + print( + "{} updates database name {} during deviceMonitorBehavior successfully".format(agent_id, db_database)) + except Exception as er: + print er + print("ERROR: {} failed to update database name {}".format(agent_id, db_database)) + + # step4: update cassandra database + try: + cassandraDB.insert(agent_id, self.variables, log_variables) + print('Data Pushed to cassandra') + except Exception as er: + print("ERROR: {} fails to update cassandra database".format(agent_id)) + print er + + # step5: debug agent knowledge + if debug_agent: + print("printing agent's knowledge") + for k, v in self.variables.items(): + print (k, v) + print('') + + def updateUI(self): + topic = '/agent/ui/'+device_type+'/device_status_response/'+ _topic_Agent_UI_tail + # now = datetime.utcnow().isoformat(' ') + 'Z' + headers = { + 'AgentID': agent_id, + headers_mod.CONTENT_TYPE: headers_mod.CONTENT_TYPE.JSON, + # headers_mod.DATE: now, + headers_mod.FROM: agent_id, + headers_mod.TO: 'ui' + } + _data = {'setpoint': self.get_variable('setpoint'),'temperature': self.get_variable('temperature'), + 'window_open': self.get_variable('window_open'), 'alert': self.get_variable('alert'), + 'operation_mode': self.get_variable('operation_mode') + } + + message = json.dumps(_data) + message = message.encode(encoding='utf_8') + self.publish(topic, headers, message) + + # 4. updateUIBehavior (generic behavior) + + @matching.match_exact('/ui/agent/'+device_type+'/device_status/' + _topic_Agent_UI_tail) + def updateUIBehavior(self, topic, headers, message, match): + print agent_id + " got\nTopic: {topic}".format(topic=topic) + print "Headers: {headers}".format(headers=headers) + print "Message: {message}\n".format(message=message) + # reply message + self.updateUI() + + # 5. Control intelligence and set device status + @periodic(device_monitor_time) + def setDevice(self): + ''' + @matching.match_exact('/something/agent/'+device_type+'/device_control/'+_topic_Agent_UI_tail) + def setDevice(self, topic, headers, message, match): + print agent_id + " got\nTopic: {topic}".format(topic=topic) + print "Headers: {headers}".format(headers=headers) + print "Message: {message}\n".format(message=message) + ''' + + if self.variables['setpoint'] > 80: + self.variables['setpoint'] = 20 + else: + self.variables['setpoint'] += 5 + + try: + setDeviceStatusResult = TRV.setDeviceStatus(self.variables) + + except Exception as er: + print er + print "set device status failed for {}".format(agent_id) + + #6. update Postgres database + def updateDB(self, table, column, column_ref, column_data, column_ref_data): + self.cur.execute("UPDATE "+table+" SET "+column+"=%s " + "WHERE "+column_ref+"=%s", + (column_data, column_ref_data)) + self.con.commit() + + def device_offline_detection(self): + _db_notification_subject = 'BEMOSS Device {} went OFFLINE!!!'.format(agent_id) + _email_subject = '#Attention: BEMOSS Device {} went OFFLINE!!!'.format(agent_id) + _email_text = '#Attention: BEMOSS Device {} went OFFLINE!!!'.format(agent_id) + self.cur.execute("SELECT network_status FROM " + db_table_trv + " WHERE trv_id=%s", + (agent_id,)) + self.network_status = self.cur.fetchone()[0] + print self.network_status + if self.network_status == "OFFLINE": + print "Found Device OFFLINE" + self.cur.execute("SELECT id FROM " + db_table_active_alert + " WHERE event_trigger_id=%s", ('5',)) + self._active_alert_id = self.cur.fetchone()[0] + self.cur.execute( + "SELECT id FROM " + db_table_temp_time_counter + " WHERE alert_id=%s AND device_id=%s", + (str(self._active_alert_id), agent_id,)) + # If this is the first detected violation + if self.cur.rowcount == 0: + print "first device offline detected" + self.cur.execute( + "INSERT INTO " + db_table_temp_time_counter + " VALUES(DEFAULT,%s,%s,%s,%s,%s)", + (self._active_alert_id, agent_id, '0', '0', '0')) + self.con.commit() + self.send_device_notification_db_device(_db_notification_subject, self._active_alert_id) + self.cur.execute("SELECT notify_address FROM " + db_table_alerts_notificationchanneladdress + " WHERE active_alert_id=%s AND notification_channel_id=%s",(self._active_alert_id,'1')) + # Send email if exist + if self.cur.rowcount != 0: + self._offline_alert_email = self.cur.fetchall() + for single_email_1 in self._offline_alert_email: + print single_email_1[0] + self.send_device_notification_email_all(single_email_1[0], _email_text, _email_subject) + + # Send SMS if provided by user + self.cur.execute("SELECT notify_address FROM " + db_table_alerts_notificationchanneladdress + " WHERE active_alert_id=%s AND notification_channel_id=%s",(self._active_alert_id,'2')) + if self.cur.rowcount != 0: + self._offline_alert_sms = self.cur.fetchall() + for single_number_ in self._offline_alert_email: + print single_number_[0] + self.send_device_notification_sms(single_number_[0]) + else: + self.priority_counter(self._active_alert_id, _db_notification_subject) + else: + print "The Device is ONLINE" + + # TODO refactor this one + def send_device_notification_db(self, device_msg, _active_alert_id): + print " INSIDE send_device_notification_db" + # Find the priority id + self.cur.execute( + "SELECT priority_id FROM " + db_table_active_alert + " WHERE id=%s", + (str(_active_alert_id),)) + self.priority_id = self.cur.fetchone()[0] + + # Find the priority level + self.cur.execute( + "SELECT priority_level FROM " + db_table_priority + " WHERE id=%s", + str(self.priority_id)) + self.priority_level = self.cur.fetchone()[0] + + # Insert the notification into DB + self.cur.execute("INSERT INTO " + db_table_bemoss_notify + " VALUES(DEFAULT,%s,%s,%s,%s)", + (device_msg, + str(datetime.datetime.now()), 'Alert', str(self.priority_level))) + self.con.commit() + + # TODO refactor this one + def send_device_notification_db_device(self, device_msg, _active_alert_id): + print " INSIDE send_device_notification_db_device" + # Find the priority id + self.cur.execute( + "SELECT priority_id FROM " + db_table_active_alert + " WHERE id=%s", + (str(_active_alert_id),)) + self.priority_id = self.cur.fetchone()[0] + + # Find the priority level + self.cur.execute( + "SELECT priority_level FROM " + db_table_priority + " WHERE id=%s", + str(self.priority_id)) + self.priority_level = self.cur.fetchone()[0] + + # Insert the notification into DB + self.cur.execute("INSERT INTO " + db_table_bemoss_notify + " VALUES(DEFAULT,%s,%s,%s,%s)", + (device_msg, + str(datetime.datetime.now()), 'Alert', str(self.priority_level))) + self.con.commit() + + # Find the number of total number notifications sent for the same alert and device + self.cur.execute("SELECT id FROM " + db_table_active_alert + " WHERE event_trigger_id=%s", ('5',)) + if self.cur.rowcount != 0: + self.cur.execute( + "SELECT no_notifications_sent FROM " + db_table_temp_time_counter + " WHERE alert_id=%s AND device_id=%s", + (str(_active_alert_id), agent_id,)) + if self.cur.rowcount != 0: + self._no_notifications_sent = self.cur.fetchone()[0] + self.con.commit() + print self._no_notifications_sent + self._no_notifications_sent = int(self._no_notifications_sent) + 1 + self.cur.execute( + "UPDATE " + db_table_temp_time_counter + " SET no_notifications_sent=%s WHERE alert_id=%s AND device_id=%s", + (str(self._no_notifications_sent), str(_active_alert_id), agent_id,)) + self.con.commit() + + def send_device_notification_sms(self, _active_alert_phone_number): + print "INSIDE send_device_notification_sms" + print _active_alert_phone_number + _sms_subject = 'Please Check BEMOSS Notifications' + smsService = SMSService() + smsService.sendSMS(email_fromaddr, _active_alert_phone_number, email_username, email_password, _sms_subject, email_mailServer) + + def send_device_notification_email_all(self, _active_alert_email, _email_subject, _email_text): + emailService = EmailService() + + # Send Email + emailService.sendEmail(email_fromaddr, _active_alert_email, email_username, + email_password, _email_subject, _email_text, email_mailServer) + + # TODO refactor this one + + def priority_counter(self, _active_alert_id, device_msg_1): + # find the priority counter then compare it with priority_counter in priority table + # if greater than the counter then send notification and reset the value + # else just increase the counter + print "INSIDE the priority_counter" + self.cur.execute( + "SELECT priority_counter FROM " + db_table_temp_time_counter + " WHERE alert_id=%s AND device_id=%s", + (str(_active_alert_id), agent_id,)) + self.priority_count = self.cur.fetchone()[0] + self.con.commit() + + # Find the priority id from active alert table + self.cur.execute( + "SELECT priority_id FROM " + db_table_active_alert + " WHERE id=%s", + (str(_active_alert_id),)) + self.priority_id = self.cur.fetchone()[0] + self.con.commit() + + # Find the priority limit from the priority counter + self.cur.execute( + "SELECT priority_counter FROM " + db_table_priority + " WHERE id=%s", + (str(self.priority_id),)) + self.priority_limit = self.cur.fetchone()[0] + self.con.commit() + + if int(self.priority_count) > int(self.priority_limit): + self.cur.execute( + "SELECT no_notifications_sent FROM " + db_table_temp_time_counter + " WHERE alert_id=%s AND device_id=%s", + (str(_active_alert_id), agent_id,)) + self._no_notifications_sent = self.cur.fetchone()[0] + self._no_notifications_sent = int(self._no_notifications_sent) + 1 + self.send_device_notification_db(device_msg_1, _active_alert_id) + self.cur.execute( + "UPDATE " + db_table_temp_time_counter + " SET priority_counter=%s WHERE alert_id=%s AND device_id=%s", + ('0', str(_active_alert_id), agent_id,)) + self.con.commit() + + print "INSIDE the priority counter exceeded the defined range" + # send email if checked + self.cur.execute("SELECT notify_address FROM " + db_table_alerts_notificationchanneladdress + " WHERE active_alert_id=%s AND notification_channel_id=%s",(self._active_alert_id,'1')) + # Send email if exist + if self.cur.rowcount != 0: + self._tampering_alert_email_misoperation = self.cur.fetchall() + for single_email_1 in self._tampering_alert_email_misoperation: + print single_email_1[0] + self.send_device_notification_email_all(single_email_1[0], device_msg_1, device_msg_1) + + # Send SMS if provided by user + self.cur.execute( + "SELECT notify_address FROM " + db_table_alerts_notificationchanneladdress + " WHERE active_alert_id=%s AND notification_channel_id=%s", + (self._active_alert_id,'2')) + if self.cur.rowcount != 0: + self._tampering_alert_sms_misoperation = self.cur.fetchall() + for single_number_misoperation in self._tampering_alert_sms_misoperation: + print single_number_misoperation[0] + self.send_device_notification_sms(single_number_misoperation[0]) + else: + self.priority_count = int(self.priority_count) + 1 + self.cur.execute( + "UPDATE " + db_table_temp_time_counter + " SET priority_counter=%s WHERE alert_id=%s AND device_id=%s", + (str(self.priority_count), str(_active_alert_id), agent_id,)) + + Agent.__name__ = 'TRV Agent' + return Agent(**kwargs) + +def main(argv=sys.argv): + '''Main method called by the eggsecutable.''' + utils.default_main(TRVAgent, + description='TRV agent', + argv=argv) + +if __name__ == '__main__': + # Entry point for script + try: + sys.exit(main()) + except KeyboardInterrupt: + pass diff --git a/DeviceAPI/classAPI/classAPI_EnoceanPi.py b/DeviceAPI/classAPI/classAPI_EnoceanPi.py new file mode 100755 index 0000000..b34c4a2 --- /dev/null +++ b/DeviceAPI/classAPI/classAPI_EnoceanPi.py @@ -0,0 +1,70 @@ +''' +#__author__ = "Mike" +#__credits__ = "" +#__version__ = "2.0" +#__maintainer__ = "Mike" +#__email__ = "michael@blocpower.io" +#__website__ = "www.blocpower.io" +#__created__ = "2017-04-17" +#__lastUpdated__ = "" +''' +import requests +import json +import time +import datetime +from DeviceAPI.discoverAPI import WiFi +from urlparse import urlparse + +class API: + # 1. constructor : gets call every time when create a new class + # requirements for instantiation1. model, 2.type, 3.api, 4. address + def __init__(self,**kwargs): # default color is white + # Initialized common attributes + self.variables = kwargs + self.debug = False + self.set_variable('connection_renew_interval',6000) + self.set_variable('offline_count', 0) + + + def renewConnection(self): + pass + + def set_variable(self,k,v): # k=key, v=value + self.variables[k] = v + + def get_variable(self,k): + return self.variables.get(k, None) # default of get_variable is none + + # 2. Attributes from Attributes table + ''' + Attributes: + POST: setpoint + ------------------------------------------------------------------------------------------ + setpoint POST setpoint of the TRV from 0-100% + ------------------------------------------------------------------------------------------ + ''' + + # 3. Capabilites (methods) from Capabilities table + ''' + API1 available methods: + 1. renewConnection() + 2. setDeviceStatus(setpoint) POST + ''' + + def renewConnection(): + pass + + def setDeviceStatus(self, setpoint): + enocean_address = self.variables.get('address') + set_trv = {"trv_id":self.variables.get('macaddress'), "setpoint":setpoint} + #headers = {authentication: ???} + + try: + verify = requests.post(enocean_address, data = set_trv) + if not verify.status_code == 200: + raise Exception('Response code != 200') + + + except Exception as er: + print "HTTP Request to EnOcean gateway for {} failed".format(self.variables.get('agent_id')) + print er \ No newline at end of file diff --git a/bemoss_lib/utils/buildAgents.sh b/bemoss_lib/utils/buildAgents.sh index 96b345d..140fdcc 100755 --- a/bemoss_lib/utils/buildAgents.sh +++ b/bemoss_lib/utils/buildAgents.sh @@ -54,6 +54,7 @@ cd ~/workspace/bemoss_os/ volttron-pkg package ~/workspace/bemoss_os/Agents/ThermostatAgent volttron-pkg package ~/workspace/bemoss_os/Agents/AirQualityAgent +volttron-pkg package ~/workspace/bemoss_os/Agents/TRVAgent volttron-pkg package ~/workspace/bemoss_os/Agents/PlugloadAgent volttron-pkg package ~/workspace/bemoss_os/Agents/LightingAgent volttron-pkg package ~/workspace/bemoss_os/Agents/NetworkAgent diff --git a/bemoss_lib/utils/platform_initiator.py b/bemoss_lib/utils/platform_initiator.py index 114ccee..3e3e789 100755 --- a/bemoss_lib/utils/platform_initiator.py +++ b/bemoss_lib/utils/platform_initiator.py @@ -75,6 +75,7 @@ db_table_application_running = settings.DATABASES['default']['TABLE_application_ db_table_application_registered = settings.DATABASES['default']['TABLE_application_registered'] db_table_plugload = settings.DATABASES['default']['TABLE_plugload'] db_table_thermostat = settings.DATABASES['default']['TABLE_thermostat'] +db_table_trv = settings.DATABASES['default']['TABLE_trv'] db_table_lighting = settings.DATABASES['default']['TABLE_lighting'] db_table_airquality = settings.DATABASES['default']['TABLE_airquality'] db_table_device_metadata = settings.DATABASES['default']['TABLE_device_metadata'] @@ -106,6 +107,7 @@ print "{} >> Done 1: connect to database name {}".format(agent_id, db_database) cur.execute("DELETE FROM "+db_table_thermostat) cur.execute("DELETE FROM "+db_table_lighting) cur.execute("DELETE FROM "+db_table_airquality) +cur.execute("DELETE FROM "+db_table_trv) cur.execute("DELETE FROM "+db_table_plugload) cur.execute("DELETE FROM "+db_table_vav) cur.execute("DELETE FROM "+db_table_rtu) @@ -360,6 +362,8 @@ cur.execute("INSERT INTO supported_devices VALUES(%s,%s,%s,%s,%s,%s,%s,%s,%s,%s, ("Tekmar Bypass Controller","BlocPower","Serial","plugload","USB-RLY02","3WSP","classAPI_Tekmar_Bypass",True,False,4,4,True)) cur.execute("INSERT INTO supported_devices VALUES(%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s)", ("Awair+","Awair","WiFi","airquality","Awair","4AIR","classAPI_Awair",False,True,4,4,True)) +cur.execute("INSERT INTO supported_devices VALUES(%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s,%s)", + ("M9-D15","Magnum","WiFi","trv","Magnum","5MAG","classAPI_EnOceanPi",False,False,4,4,True)) conn.commit() print "Table supported_devices populated successfully!" diff --git a/settings.py b/settings.py index 5319e1f..a38a3aa 100755 --- a/settings.py +++ b/settings.py @@ -101,6 +101,7 @@ DATABASES = { 'TABLE_plugload': 'plugload', 'TABLE_thermostat': 'thermostat', 'TABLE_airquality': 'airquality', + 'TABLE_trv': 'trv', 'TABLE_lighting': 'lighting', 'TABLE_device_metadata': 'device_metadata', 'TABLE_vav': 'vav', -- GitLab From 52f032aa7240774aa48307d0b0b7cf06603c3f05 Mon Sep 17 00:00:00 2001 From: mchlburton Date: Wed, 19 Apr 2017 10:17:14 -0400 Subject: [PATCH 07/31] Agent that can interact with pi server using HTTP --- Agents/TRVAgent/trv/agent.py | 31 ++++++++++----- DeviceAPI/classAPI/classAPI_EnoceanPi.py | 49 ++++++++++++++++++++++-- 2 files changed, 67 insertions(+), 13 deletions(-) diff --git a/Agents/TRVAgent/trv/agent.py b/Agents/TRVAgent/trv/agent.py index 52d0589..6a48b08 100755 --- a/Agents/TRVAgent/trv/agent.py +++ b/Agents/TRVAgent/trv/agent.py @@ -7,7 +7,7 @@ #__email__ = "michael@blocpower.io" #__website__ = "www.blocopwer.io" #__created__ = "2017-04-13" -#__lastUpdated__ = " +#__lastUpdated__ = "" ''' import sys @@ -24,6 +24,7 @@ from bemoss_lib.communication.sms import SMSService import psycopg2 import psycopg2.extras import settings +import requests import socket from bemoss_lib.databases.cassandraAPI import cassandraDB from bemoss_lib.utils.catcherror import catcherror @@ -154,15 +155,27 @@ def TRVAgent(config_path, **kwargs): TRV.renewConnection() # 3. deviceMonitorBehavior (TickerBehavior) - #@periodic(device_monitor_time) - @matching.match_exact('/enocean_gateway/agent/'+device_type+'/device_status/' + _topic_Agent_UI_tail) - def deviceMonitorBehavior(self, topic, headers, message, match): - print agent_id + " got\nTopic: {topic}".format(topic=topic) - print "Headers: {headers}".format(headers=headers) - print "Message: {message}\n".format(message=message) + @periodic(connection_renew_interval) + def deviceMonitorBehavior(self): + + #TODO move this to the class API as a getDeviceStatus function + device_url = TRV.get_variable('address')+':8080/api/v1' + header = {"Authorization_key":'ABC'} + trv_id = {"trv_id":TRV.get_variable('macaddress')} + try: + data_req = requests.get(device_url,headers = header, params = trv_id) + + except Exception as er: + print er + + print data_req.content + + # step1: get current status of a trv from bus, then map message keywords and variables to agent knowledge - variables = json.loads(message[0]) + #variables = json.loads(message[0]) + variables = data_req.json() + print variables self.changed_variables = dict() for v in log_variables: @@ -297,7 +310,7 @@ def TRVAgent(config_path, **kwargs): self.variables['setpoint'] += 5 try: - setDeviceStatusResult = TRV.setDeviceStatus(self.variables) + setDeviceStatusResult = TRV.setDeviceStatus(self.get_variable('setpoint')) except Exception as er: print er diff --git a/DeviceAPI/classAPI/classAPI_EnoceanPi.py b/DeviceAPI/classAPI/classAPI_EnoceanPi.py index b34c4a2..f2ce292 100755 --- a/DeviceAPI/classAPI/classAPI_EnoceanPi.py +++ b/DeviceAPI/classAPI/classAPI_EnoceanPi.py @@ -55,16 +55,57 @@ class API: pass def setDeviceStatus(self, setpoint): - enocean_address = self.variables.get('address') + enocean_address = self.variables.get('address')+':8080/api/v1' set_trv = {"trv_id":self.variables.get('macaddress'), "setpoint":setpoint} - #headers = {authentication: ???} + headers = {"Authorization_key": 'ABC', "Content-type": 'application/json'} try: - verify = requests.post(enocean_address, data = set_trv) + verify = requests.post(enocean_address, headers = headers, data = json.dumps(set_trv)) if not verify.status_code == 200: + print verify.status_code raise Exception('Response code != 200') except Exception as er: print "HTTP Request to EnOcean gateway for {} failed".format(self.variables.get('agent_id')) - print er \ No newline at end of file + print er +''' + def getDeviceStatus(self): + getDeviceStatusResult = True + token = self.get_variable('token') + awair_id = self.get_variable('address') + print awair_id + + head_auth = {'Authorization': token} + + device_url = 'https://beta-api.awair.is/v1/devices/' + str(awair_id) + '/events/15min-avg' + start = (datetime.datetime.now()-datetime.timedelta(minutes=15)).isoformat() + end = datetime.datetime.now().isoformat() + timespan = {'from': start, 'to': end} + try: + data_req = req.get(device_url,headers = head_auth, params = timespan) + + data_json = data_req.json() + print data_json + data_list = [] + for row in data_json['data']: + data = row['sensor'].values() + time = row['timestamp'] + + self.set_variable('dust', data[0]) + self.set_variable('co2', data[1]) + self.set_variable('humidity', data[2]) + self.set_variable('temperature', data[3]) + self.set_variable('voc', data[4]) + self.set_variable('time', time) + + except Exception as er: + print er + getDeviceStatusResult = False + + if getDeviceStatusResult==True: + self.set_variable('offline_count', 0) + + else: + self.set_variable('offline_count', self.get_variable('offline_count')+1) +''' \ No newline at end of file -- GitLab From ce3005b0f263eafb15a2dfdb8ec48b9f336de664 Mon Sep 17 00:00:00 2001 From: mchlburton Date: Thu, 20 Apr 2017 17:42:06 -0400 Subject: [PATCH 08/31] TRV agent and completed API without discovery --- Agents/TRVAgent/trv/agent.py | 43 +++++-------------- DeviceAPI/classAPI/classAPI_EnoceanPi.py | 53 ++++++++++-------------- 2 files changed, 33 insertions(+), 63 deletions(-) diff --git a/Agents/TRVAgent/trv/agent.py b/Agents/TRVAgent/trv/agent.py index 6a48b08..4dd4305 100755 --- a/Agents/TRVAgent/trv/agent.py +++ b/Agents/TRVAgent/trv/agent.py @@ -7,7 +7,7 @@ #__email__ = "michael@blocpower.io" #__website__ = "www.blocopwer.io" #__created__ = "2017-04-13" -#__lastUpdated__ = "" +#__lastUpdated__ = "" ''' import sys @@ -24,7 +24,6 @@ from bemoss_lib.communication.sms import SMSService import psycopg2 import psycopg2.extras import settings -import requests import socket from bemoss_lib.databases.cassandraAPI import cassandraDB from bemoss_lib.utils.catcherror import catcherror @@ -155,38 +154,25 @@ def TRVAgent(config_path, **kwargs): TRV.renewConnection() # 3. deviceMonitorBehavior (TickerBehavior) - @periodic(connection_renew_interval) + @periodic(device_monitor_time) def deviceMonitorBehavior(self): - #TODO move this to the class API as a getDeviceStatus function - device_url = TRV.get_variable('address')+':8080/api/v1' - header = {"Authorization_key":'ABC'} - trv_id = {"trv_id":TRV.get_variable('macaddress')} try: - data_req = requests.get(device_url,headers = header, params = trv_id) - - except Exception as er: - print er - - print data_req.content - - + TRV.getDeviceStatus() + except: + print("device connection is not successful") - # step1: get current status of a trv from bus, then map message keywords and variables to agent knowledge - #variables = json.loads(message[0]) - variables = data_req.json() - print variables self.changed_variables = dict() - for v in log_variables: - if v in variables.items(): - if v not in self.variables or self.variables[v] != variables.get(v): - self.variables[v] = variables.get(v) + if v in TRV.variables: + if not v in self.variables or self.variables[v] != TRV.variables[v]: + self.variables[v] = TRV.variables[v] self.changed_variables[v] = log_variables[v] else: - if v not in self.variables: # it won't be in self.variables either (in the first time) + if v not in self.variables: #it won't be in self.variables either (in the first time) self.changed_variables[v] = log_variables[v] self.variables[v] = None + #step4: update PostgresQL (meta-data) database try: #make the device offline if necessary @@ -296,13 +282,6 @@ def TRVAgent(config_path, **kwargs): # 5. Control intelligence and set device status @periodic(device_monitor_time) def setDevice(self): - ''' - @matching.match_exact('/something/agent/'+device_type+'/device_control/'+_topic_Agent_UI_tail) - def setDevice(self, topic, headers, message, match): - print agent_id + " got\nTopic: {topic}".format(topic=topic) - print "Headers: {headers}".format(headers=headers) - print "Message: {message}\n".format(message=message) - ''' if self.variables['setpoint'] > 80: self.variables['setpoint'] = 20 @@ -310,7 +289,7 @@ def TRVAgent(config_path, **kwargs): self.variables['setpoint'] += 5 try: - setDeviceStatusResult = TRV.setDeviceStatus(self.get_variable('setpoint')) + setDeviceStatusResult = TRV.setDeviceStatus() except Exception as er: print er diff --git a/DeviceAPI/classAPI/classAPI_EnoceanPi.py b/DeviceAPI/classAPI/classAPI_EnoceanPi.py index f2ce292..275f999 100755 --- a/DeviceAPI/classAPI/classAPI_EnoceanPi.py +++ b/DeviceAPI/classAPI/classAPI_EnoceanPi.py @@ -40,7 +40,11 @@ class API: Attributes: POST: setpoint ------------------------------------------------------------------------------------------ - setpoint POST setpoint of the TRV from 0-100% + setpoint POST setpoint of the TRV (int from 0-100%) + window_open GET Window open detection of TRV (boolean) + alert GET concatenated string of TRV error messages (string) + temperature GET temperature (floating point in deg F) + operation_mode GET Operation mode of the TRV (string) ------------------------------------------------------------------------------------------ ''' @@ -49,12 +53,14 @@ class API: API1 available methods: 1. renewConnection() 2. setDeviceStatus(setpoint) POST + 3. getDeviceStatus() GET ''' def renewConnection(): pass - def setDeviceStatus(self, setpoint): + def setDeviceStatus(self): + setpoint = self.variables.get('setpoint') enocean_address = self.variables.get('address')+':8080/api/v1' set_trv = {"trv_id":self.variables.get('macaddress'), "setpoint":setpoint} headers = {"Authorization_key": 'ABC', "Content-type": 'application/json'} @@ -69,35 +75,21 @@ class API: except Exception as er: print "HTTP Request to EnOcean gateway for {} failed".format(self.variables.get('agent_id')) print er -''' - def getDeviceStatus(self): + + def getDeviceStatus(self): getDeviceStatusResult = True - token = self.get_variable('token') - awair_id = self.get_variable('address') - print awair_id - - head_auth = {'Authorization': token} - - device_url = 'https://beta-api.awair.is/v1/devices/' + str(awair_id) + '/events/15min-avg' - start = (datetime.datetime.now()-datetime.timedelta(minutes=15)).isoformat() - end = datetime.datetime.now().isoformat() - timespan = {'from': start, 'to': end} + device_url = self.get_variable('address')+':8080/api/v1' + header = {"Authorization_key":'ABC'} + trv_id = {"trv_id":self.get_variable('macaddress')} try: - data_req = req.get(device_url,headers = head_auth, params = timespan) - - data_json = data_req.json() - print data_json - data_list = [] - for row in data_json['data']: - data = row['sensor'].values() - time = row['timestamp'] - - self.set_variable('dust', data[0]) - self.set_variable('co2', data[1]) - self.set_variable('humidity', data[2]) - self.set_variable('temperature', data[3]) - self.set_variable('voc', data[4]) - self.set_variable('time', time) + data_req = requests.get(device_url,headers = header, params = trv_id) + data = data_req.json() + + self.set_variable('setpoint', data['setpoint']) + self.set_variable('temperature', data['temperature']) + self.set_variable('window_open', data['window_open']) + self.set_variable('alert', data['alert']) + self.set_variable('operation_mode', data['operation_mode']) except Exception as er: print er @@ -107,5 +99,4 @@ class API: self.set_variable('offline_count', 0) else: - self.set_variable('offline_count', self.get_variable('offline_count')+1) -''' \ No newline at end of file + self.set_variable('offline_count', self.get_variable('offline_count')+1) \ No newline at end of file -- GitLab From 0066f296e33da0d032c57987f0395b5553e1ba21 Mon Sep 17 00:00:00 2001 From: mchlburton Date: Fri, 21 Apr 2017 11:20:56 -0400 Subject: [PATCH 09/31] Device discovery for Magnum TRV --- .../devicediscovery/agent.py | 3 +- Agents/TRVAgent/trv/agent.py | 3 +- DeviceAPI/classAPI/classAPI_EnoceanPi.py | 4 +-- DeviceAPI/discoverAPI/WiFi.py | 28 ++++++++++++++++++- 4 files changed, 33 insertions(+), 5 deletions(-) diff --git a/Agents/DeviceDiscoveryAgent/devicediscovery/agent.py b/Agents/DeviceDiscoveryAgent/devicediscovery/agent.py index 5e95531..795a189 100755 --- a/Agents/DeviceDiscoveryAgent/devicediscovery/agent.py +++ b/Agents/DeviceDiscoveryAgent/devicediscovery/agent.py @@ -136,6 +136,7 @@ def DeviceDiscoveryAgent(config_path, **kwargs): self.discovery_list.append('CT30 V1.94') self.discovery_list.append('CT50 V1.94') self.discovery_list.append('Awair+') + self.discovery_list.append('M9-D15') if self.findWiFiWeMo: self.discovery_list.append('Socket') self.discovery_list.append('LightSwitch') @@ -255,7 +256,7 @@ def DeviceDiscoveryAgent(config_path, **kwargs): print "{} >> is finding available {} {} devices ...".format(agent_id,com_type,discovery_type) discovery_module = importlib.import_module("DeviceAPI.discoverAPI."+com_type) - if (com_type == 'BACnet') or (com_type == 'Serial') or (discovery_type=='Awair'): + if (com_type == 'BACnet') or (com_type == 'Serial') or (discovery_type=='Awair') or (discovery_type=='Magnum'): discovery_returns_ip = False else: discovery_returns_ip = True diff --git a/Agents/TRVAgent/trv/agent.py b/Agents/TRVAgent/trv/agent.py index 4dd4305..4f2e6af 100755 --- a/Agents/TRVAgent/trv/agent.py +++ b/Agents/TRVAgent/trv/agent.py @@ -54,7 +54,8 @@ def TRVAgent(config_path, **kwargs): zone_id = get_config('zone_id') model = get_config('model') device_type = get_config('type') - address = get_config('address') + #TODO Make address come from launch file + address = 'http://192.168.0.101' macaddress = get_config('macaddress') _address = address _address = _address.replace('http://', '') diff --git a/DeviceAPI/classAPI/classAPI_EnoceanPi.py b/DeviceAPI/classAPI/classAPI_EnoceanPi.py index 275f999..de24e8f 100755 --- a/DeviceAPI/classAPI/classAPI_EnoceanPi.py +++ b/DeviceAPI/classAPI/classAPI_EnoceanPi.py @@ -55,7 +55,7 @@ class API: 2. setDeviceStatus(setpoint) POST 3. getDeviceStatus() GET ''' - + #TODO Take authentication key from launch file instead of hard coding def renewConnection(): pass @@ -79,7 +79,7 @@ class API: def getDeviceStatus(self): getDeviceStatusResult = True device_url = self.get_variable('address')+':8080/api/v1' - header = {"Authorization_key":'ABC'} + header = {"Authorization_key":'ABC', "Content-type": 'application/json'} trv_id = {"trv_id":self.get_variable('macaddress')} try: data_req = requests.get(device_url,headers = header, params = trv_id) diff --git a/DeviceAPI/discoverAPI/WiFi.py b/DeviceAPI/discoverAPI/WiFi.py index cb9ab97..326f568 100755 --- a/DeviceAPI/discoverAPI/WiFi.py +++ b/DeviceAPI/discoverAPI/WiFi.py @@ -85,6 +85,7 @@ def parseJSONresponse(data,key): def discover(type, timeout=2, retries=1): + #TODO Grab token from bemoss passwords instead of hard coding if type=='Awair': token = 'Bearer eyJ0eXAiOiJKV1QiLCJhbGciOiJIUzI1NiJ9.eyJ1c2VyX2lkIjoiNTUxOSJ9.3YE0APseF-aLMcjTMSbN4bHgE--ZgcuWBEXVs-5TTO4' head_auth = {'Authorization' : token} @@ -103,6 +104,26 @@ def discover(type, timeout=2, retries=1): for device in devices_json['data']: responses.append(str(device['device_id'])) + #TODO have discovery agent find raspberry pi first instead of hard coding ip + elif type=='Magnum': + device_url = 'http://192.168.0.101:8080/api/v1' + header = {"Authorization_key":'ABC', "Content-type": 'application/json'} + trv_discover = {"discover":'True'} + data_req=None + try: + data_req = requests.get(device_url,headers = header, params = trv_discover) + + except Exception as er: + print er + + responses = list() + + if data_req is not None: + devices = data_req.json() + responses.append(devices['trv_list']) + + return responses + else: group = ("239.255.255.250", 1900) if type=='thermostat': @@ -183,6 +204,9 @@ def getMACaddress(type,ipaddress): elif type=="Awair": awairid = ipaddress return awairid + elif type=="Magnum": + enocean_address = ipaddress + return enocean_address else: print "This device: {} is not supported by the WiFi discovery module".format(type) @@ -222,11 +246,13 @@ def getmodelvendor(type,ipaddress): return {'model':deviceModel,'vendor':deviceVendor,'nickname':nickname} elif type=="Awair": return {'model':'Awair+','vendor':'Awair'} + elif type=="Magnum": + return {'model':'MD-15', 'vendor':'Magnum'} # This main method will not be executed when this class is used as a module def main(): - print discover('Philips') + print discover('Magnum') # print discover('thermostat') # print getMACaddress('Philips','http://192.168.1.102:80/description.xml') # print type(getMACaddress('Philips','http://192.168.102.:80')) -- GitLab From d17a3801e2f961fd76c1d9f9abd554eee7976166 Mon Sep 17 00:00:00 2001 From: mchlburton Date: Fri, 21 Apr 2017 11:56:54 -0400 Subject: [PATCH 10/31] Added HTTP request timeouts --- DeviceAPI/classAPI/classAPI_EnoceanPi.py | 2 +- DeviceAPI/discoverAPI/WiFi.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/DeviceAPI/classAPI/classAPI_EnoceanPi.py b/DeviceAPI/classAPI/classAPI_EnoceanPi.py index de24e8f..26abe04 100755 --- a/DeviceAPI/classAPI/classAPI_EnoceanPi.py +++ b/DeviceAPI/classAPI/classAPI_EnoceanPi.py @@ -82,7 +82,7 @@ class API: header = {"Authorization_key":'ABC', "Content-type": 'application/json'} trv_id = {"trv_id":self.get_variable('macaddress')} try: - data_req = requests.get(device_url,headers = header, params = trv_id) + data_req = requests.get(device_url,headers = header, params = trv_id, timeout=5) data = data_req.json() self.set_variable('setpoint', data['setpoint']) diff --git a/DeviceAPI/discoverAPI/WiFi.py b/DeviceAPI/discoverAPI/WiFi.py index 326f568..3724dc3 100755 --- a/DeviceAPI/discoverAPI/WiFi.py +++ b/DeviceAPI/discoverAPI/WiFi.py @@ -92,7 +92,7 @@ def discover(type, timeout=2, retries=1): devices_url = 'https://beta-api.awair.is/v1/users/self/devices' devices_req = None try: - devices_req = requests.get(devices_url, headers = head_auth) + devices_req = requests.get(devices_url, headers = head_auth, timeout=10) except Exception as er: print er pass @@ -111,7 +111,7 @@ def discover(type, timeout=2, retries=1): trv_discover = {"discover":'True'} data_req=None try: - data_req = requests.get(device_url,headers = header, params = trv_discover) + data_req = requests.get(device_url,headers = header, params = trv_discover, timeout=5) except Exception as er: print er -- GitLab From e2d5b5363f251e035bf5c6336da72e335d1ec18c Mon Sep 17 00:00:00 2001 From: mchlburton Date: Fri, 21 Apr 2017 16:44:00 -0400 Subject: [PATCH 11/31] Agent and discovery changes after Pi testing --- Agents/TRVAgent/trv/agent.py | 26 ++++++++++++++++++------ DeviceAPI/classAPI/classAPI_EnoceanPi.py | 8 +++++--- DeviceAPI/discoverAPI/WiFi.py | 4 ++-- 3 files changed, 27 insertions(+), 11 deletions(-) diff --git a/Agents/TRVAgent/trv/agent.py b/Agents/TRVAgent/trv/agent.py index 4f2e6af..6ce4d1b 100755 --- a/Agents/TRVAgent/trv/agent.py +++ b/Agents/TRVAgent/trv/agent.py @@ -43,7 +43,7 @@ def TRVAgent(config_path, **kwargs): # 1. @params agent agent_id = get_config('agent_id') - device_monitor_time = get_config('device_monitor_time') + device_monitor_time = 10 max_monitor_time = int(settings.DEVICES['max_monitor_time']) debug_agent = False log_variables = dict(setpoint='int', temperature='float', window_open='boolean', alert='string', @@ -55,7 +55,7 @@ def TRVAgent(config_path, **kwargs): model = get_config('model') device_type = get_config('type') #TODO Make address come from launch file - address = 'http://192.168.0.101' + address = 'http://192.168.0.103' macaddress = get_config('macaddress') _address = address _address = _address.replace('http://', '') @@ -114,7 +114,7 @@ def TRVAgent(config_path, **kwargs): super(Agent, self).__init__(**kwargs) # 1. initialize all agent variables self.variables = kwargs - self.variables['setpoint'] = 0 + self.set_variable('setpoint', 0) self.valid_data = False self._keep_alive = True self.ip_address = ip_address if ip_address != None else None @@ -158,6 +158,18 @@ def TRVAgent(config_path, **kwargs): @periodic(device_monitor_time) def deviceMonitorBehavior(self): + if self.variables['setpoint'] > 100: + self.variables['setpoint'] = 20 + else: + self.variables['setpoint'] += 40 + + try: + setDeviceStatusResult = TRV.setDeviceStatus() + + except Exception as er: + print er + print "set device status failed for {}".format(agent_id) + try: TRV.getDeviceStatus() except: @@ -281,13 +293,14 @@ def TRVAgent(config_path, **kwargs): self.updateUI() # 5. Control intelligence and set device status - @periodic(device_monitor_time) + ''' + @periodic(device_monitor_time+1) def setDevice(self): - if self.variables['setpoint'] > 80: + if self.variables['setpoint'] > 100: self.variables['setpoint'] = 20 else: - self.variables['setpoint'] += 5 + self.variables['setpoint'] += 40 try: setDeviceStatusResult = TRV.setDeviceStatus() @@ -295,6 +308,7 @@ def TRVAgent(config_path, **kwargs): except Exception as er: print er print "set device status failed for {}".format(agent_id) + ''' #6. update Postgres database def updateDB(self, table, column, column_ref, column_data, column_ref_data): diff --git a/DeviceAPI/classAPI/classAPI_EnoceanPi.py b/DeviceAPI/classAPI/classAPI_EnoceanPi.py index 26abe04..e6112b6 100755 --- a/DeviceAPI/classAPI/classAPI_EnoceanPi.py +++ b/DeviceAPI/classAPI/classAPI_EnoceanPi.py @@ -40,7 +40,7 @@ class API: Attributes: POST: setpoint ------------------------------------------------------------------------------------------ - setpoint POST setpoint of the TRV (int from 0-100%) + setpoint GET/POST setpoint of the TRV (int from 0-100%) window_open GET Window open detection of TRV (boolean) alert GET concatenated string of TRV error messages (string) temperature GET temperature (floating point in deg F) @@ -60,7 +60,7 @@ class API: pass def setDeviceStatus(self): - setpoint = self.variables.get('setpoint') + setpoint = 100 enocean_address = self.variables.get('address')+':8080/api/v1' set_trv = {"trv_id":self.variables.get('macaddress'), "setpoint":setpoint} headers = {"Authorization_key": 'ABC', "Content-type": 'application/json'} @@ -75,6 +75,8 @@ class API: except Exception as er: print "HTTP Request to EnOcean gateway for {} failed".format(self.variables.get('agent_id')) print er + + print "device set" def getDeviceStatus(self): getDeviceStatusResult = True @@ -82,7 +84,7 @@ class API: header = {"Authorization_key":'ABC', "Content-type": 'application/json'} trv_id = {"trv_id":self.get_variable('macaddress')} try: - data_req = requests.get(device_url,headers = header, params = trv_id, timeout=5) + data_req = requests.get(device_url,headers = header, params = trv_id, timeout=10) data = data_req.json() self.set_variable('setpoint', data['setpoint']) diff --git a/DeviceAPI/discoverAPI/WiFi.py b/DeviceAPI/discoverAPI/WiFi.py index 3724dc3..a818cc8 100755 --- a/DeviceAPI/discoverAPI/WiFi.py +++ b/DeviceAPI/discoverAPI/WiFi.py @@ -106,7 +106,7 @@ def discover(type, timeout=2, retries=1): #TODO have discovery agent find raspberry pi first instead of hard coding ip elif type=='Magnum': - device_url = 'http://192.168.0.101:8080/api/v1' + device_url = 'http://192.168.0.103:8080/api/v1' header = {"Authorization_key":'ABC', "Content-type": 'application/json'} trv_discover = {"discover":'True'} data_req=None @@ -120,7 +120,7 @@ def discover(type, timeout=2, retries=1): if data_req is not None: devices = data_req.json() - responses.append(devices['trv_list']) + responses.extend(devices['trv_list']) return responses -- GitLab From 94e3bf28dd3f17409d22c1662af3b2e178222006 Mon Sep 17 00:00:00 2001 From: mchlburton Date: Fri, 28 Apr 2017 10:13:01 -0400 Subject: [PATCH 12/31] Influx Schema update as well as remote push and senseware --- bemoss_lib/databases/influxAPI/InfluxDB.py | 221 ++++++++++++++++----- 1 file changed, 172 insertions(+), 49 deletions(-) diff --git a/bemoss_lib/databases/influxAPI/InfluxDB.py b/bemoss_lib/databases/influxAPI/InfluxDB.py index 70d6e21..9189086 100755 --- a/bemoss_lib/databases/influxAPI/InfluxDB.py +++ b/bemoss_lib/databases/influxAPI/InfluxDB.py @@ -19,17 +19,18 @@ from influxdb import InfluxDBClient connection_established = False #Global variables -host = '52.206.6.10' +remote = '52.206.6.10' +host = 'localhost' port = 8086 -user = 'user' -password = 'password' +user = 'engineering' +password = 'nPEc9Pz0iV' dbname = 'bemoss' def makeConnection(): try: global host, port, user, password, dbname, connection_established - client = InfluxDBClient(host, port, user, password, dbname) + client = InfluxDBClient(host, port, user, password, dbname, ssl=True) database_list = client.get_list_database() for db in database_list: if dbname in db['name']: @@ -48,7 +49,7 @@ except Exception as er: print er -def insert(agentID, all_vars, log_vars, cur_time=None): +def insert(device_model, agentID, all_vars, log_vars, cur_time=None): """ :param agentID: string. Data will be inserted to table named B :param all_vars: dictionary (usually APIobject.variables). It contains all the variables and their values. @@ -62,7 +63,7 @@ def insert(agentID, all_vars, log_vars, cur_time=None): if not connection_established: makeConnection() - client = InfluxDBClient(host, port, user, password, dbname) + client = InfluxDBClient(host, port, user, password, dbname, ssl=True) if cur_time == None: cur_time = datetime.datetime.utcnow() @@ -74,7 +75,7 @@ def insert(agentID, all_vars, log_vars, cur_time=None): influx_json = [ { - "measurement": "lightsensor", + "measurement": device_model, "tags": { "agent": agentID, "building": "bemoss" @@ -97,7 +98,7 @@ def insert(agentID, all_vars, log_vars, cur_time=None): return 0 -def delete(agentID,startTime=None, endTime=None): +def delete(device_model, agentID, startTime=None, endTime=None): """ Performs deletion of data. if statDate and endDate is omitted, all device data is deleted. :param agentID: The on which the operation is to be performed @@ -106,13 +107,12 @@ def delete(agentID,startTime=None, endTime=None): :param endDate: datetime.date object (local timezone). The endDate upto which to delete. The default is to upto today. must be supplied unless trying to delete the whole table :return: 0, if successful - """ global connection_established if not connection_established: makeConnection() - client = InfluxDBClient(host, port, user, password, dbname) + client = InfluxDBClient(host, port, user, password, dbname, ssl=True) if endTime==None and startTime==None: try: @@ -153,7 +153,7 @@ def delete(agentID,startTime=None, endTime=None): -def retrieve(agentID, vars=None, startTime=None, endTime=None,export=False): +def retrieve(device_model, agentID, vars=None, startTime=None, endTime=None,export=False): """Function to retrieve Data from Influx. \n :param agentID: must supply, since each agentID is associated with a table in database. :param vars: supplied as a list of strings. It represents the variables to be retrieved from the table. @@ -167,14 +167,13 @@ def retrieve(agentID, vars=None, startTime=None, endTime=None,export=False): columns correspond to variables queried. Each row corresponds to various table entries. Also returns vars, which serves as a column index. It is identical to passed vars except 'time' is an extra entry at the front. If the query fails, -1 is returned (and no exception raised) - """ global connection_established if not connection_established: makeConnection() - client = InfluxDBClient(host, port, user, password, dbname) + client = InfluxDBClient(host, port, user, password, dbname, ssl=True) if startTime==None: startTime=str(datetime.datetime.utcnow()-datetime.timedelta(hours=24)) @@ -183,46 +182,170 @@ def retrieve(agentID, vars=None, startTime=None, endTime=None,export=False): endTime = str(datetime.datetime.utcnow()) if vars==None: - varStr='' vars=[] try: - result = client.query("SHOW MEASUREMENTS ON {0} WHERE agent = '{1}';".format(dbname, agentID)) + vars_query = client.query("show field keys on {0} from {1};".format(dbname, device_model)) + + for data in vars_query: + vars = [varlist['fieldKey'] for varlist in data] except: connection_established = False #Try to establish connection again next time raise - result = result.get_points('measurements') - for var in result: - measurement = str(var['name']) - print measurement - varStr += measurement + ', ' - vars.append(measurement) - varStr = varStr[:-2] #to get rid of the last ', ' - else: - varStr = '' - for var in vars: - varStr += var + ', ' - - varStr = varStr[:-2] #to get rid of the last ', ' - - total_result = [] + varStr = '' for var in vars: - result = client.query("select value from {0} WHERE agent = '{1}' AND time >= '{2}' AND time <= '{3}';".format(var, agentID, startTime,endTime)) - values = [] - entries = result.get_points(var) - if total_result == []: - time = [] - for entry in entries: - time.append(entry['time']) - values.append(entry['value']) - total_result.append(time) - else: - for entry in entries: - values.append(entry['value']) - - total_result += [values] - - vars.insert(0, 'time') - total_result = numpy.array(total_result) - total_result = numpy.transpose(total_result) - return vars, total_result \ No newline at end of file + varStr += var + ', ' + + varStr = varStr[:-2] #to get rid of the last ', ' + + result = client.query("select {0} from {1} WHERE agent = '{2}' AND time >= '{3}' AND time <= '{4}';".format(varStr, device_model, agentID, startTime,endTime)) + values = [] + data = result.get_points('lightsensor') + + for entry in data: + values.append(entry.values()) + vars = entry.keys() + total_result = numpy.array(values) + + return vars, total_result + +def local_to_remote(local='localhost', remote=remote, startTime=None, endTime=None): + """Function to retrieve Data from local Influx and push it to remote server. + :param local: Database you want to copy data from + :param remote: Database you want data copied to + :param startTime: the time in localtime zone (the timezone of the node), in datetime.datetime.utcnow format. It marks the + beginning for the range. If not supplied, will be taken 24-hours before endTime + :param endTime: the time in localtime zone (the timezone of the node), in datetime.datetime.utcnow format.It marks the end of the + range. If not supplied, will be taken as the currentTime. + :return: If successful, returns True, if data push fails, returns false + """ + global connection_established + if not connection_established: + makeConnection() + + localclient = InfluxDBClient(host, port, user, password, dbname, ssl=True) + remoteclient = InfluxDBClient(remote, port, user, password, dbname, ssl=True) + + startTime=None + endTime=None + + if startTime==None: + startTime=str(datetime.datetime.utcnow()-datetime.timedelta(hours=25)) + + if endTime==None: + endTime = str(datetime.datetime.utcnow()) + + measurements = localclient.query("show measurements;".format(dbname)) + + for result in measurements: + for measurement in result: + measure_query = localclient.query("SELECT * FROM {0} WHERE time >= '{1}' AND time <= '{2}';".format(measurement['name'], startTime, endTime)) + data = measure_query.get_points('{0}'.format(measurement['name'])) + + field_query = localclient.query("show field keys from {0}".format(measurement['name'])) + for field in field_query: + fields = [varlist['fieldKey'] for varlist in field] + tag_query = localclient.query("show tag keys from {0}".format(measurement['name'])) + for tag in tag_query: + tags = [varlist['tagKey'] for varlist in tag] + + influx_json = [] + + for entry in data: + json_field = {} + for field in fields: + reading = entry[field] + json_field.update({field:reading}) + json_tag = {} + for tag in tags: + if entry[tag] is not None: + tag_info = entry[tag] + json_tag.update({tag:tag_info}) + + json_time = entry['time'] + + json_body =[ + { + "measurement": measurement['name'], + "tags": json_tag, + "time": json_time, + "fields": json_field + } + ] + + influx_json.extend(json_body) + + try: + response = remoteclient.write_points(influx_json) + print "Data for {0} pushed to server".format(measurement['name']) + return True + except Exception as er: + print er + print "Data push for {0} failed".format(measurement['name']) + return False + +def getSenseware(location = '3rd_Floor_Apartment__Ben\\\'s_', site = '106_E_30th_St__Typhin_', measurement = 'Temperature', local='localhost', remote=remote, sensewaredb = 'Senseware', startTime=None, endTime=None): + """Function to retrieve Senseware data frmo server. + :param location: Location of the sensor in the building corresponding to senseware location tag + :param site: Building for BEMOSS instance corresponding to senseawre site tag + :param measurement: The variable you want the data for (temperature, humidity, etc.) + :param local: Database you want to copy data from + :param remote: Database you want data copied to + :param startTime: the time in localtime zone (the timezone of the node), in datetime.datetime.utcnow format. It marks the + beginning for the range. If not supplied, will be taken 24-hours before endTime + :param endTime: the time in localtime zone (the timezone of the node), in datetime.datetime.utcnow format.It marks the end of the + range. If not supplied, will be taken as the currentTime. + :return: If successful, returns True, if data push fails, returns false + """ + localclient = InfluxDBClient(host, port, user, password, dbname) + remoteclient = InfluxDBClient(remote, port, user, password, sensewaredb, ssl = True) + startTime=None + endTime=None + if startTime==None: + startTime=str(datetime.datetime.utcnow()-datetime.timedelta(minute = 30)) + + if endTime==None: + endTime = str(datetime.datetime.utcnow()) + + measure_query = remoteclient.query("SELECT * FROM {0} WHERE site='{1}' AND location='{2}' AND time>='{3}' AND time<='{4}';".format(measurement, site, location, startTime, endTime)) + data = measure_query.get_points('{0}'.format(measurement)) + + field_query = remoteclient.query("show field keys from {0}".format(measurement)) + for field in field_query: + fields = [varlist['fieldKey'] for varlist in field] + tag_query = remoteclient.query("show tag keys from {0}".format(measurement)) + for tag in tag_query: + tags = [varlist['tagKey'] for varlist in tag] + + influx_json = [] + + for entry in data: + json_field = {} + for field in fields: + reading = entry[field] + json_field.update({field:reading}) + json_tag = {} + for tag in tags: + if entry[tag] is not None: + tag_info = entry[tag] + json_tag.update({tag:tag_info}) + + json_time = entry['time'] + + json_body =[ + { + "measurement": measurement, + "tags": json_tag, + "time": json_time, + "fields": json_field + } + ] + + influx_json.extend(json_body) + + try: + response = localclient.write_points(influx_json) + print "Senseware Data for {0} retrieved from server".format(measurement) + except Exception as er: + print er + print "Senseware Data retrieve for {0} failed".format(measurement) \ No newline at end of file -- GitLab From 03c2e604ce62dfd024da9fec26dfa38e9711730c Mon Sep 17 00:00:00 2001 From: mchlburton Date: Fri, 28 Apr 2017 10:19:04 -0400 Subject: [PATCH 13/31] Consistency changes --- bemoss_lib/databases/influxAPI/InfluxDB.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/bemoss_lib/databases/influxAPI/InfluxDB.py b/bemoss_lib/databases/influxAPI/InfluxDB.py index 9189086..294ec84 100755 --- a/bemoss_lib/databases/influxAPI/InfluxDB.py +++ b/bemoss_lib/databases/influxAPI/InfluxDB.py @@ -217,7 +217,7 @@ def local_to_remote(local='localhost', remote=remote, startTime=None, endTime=No beginning for the range. If not supplied, will be taken 24-hours before endTime :param endTime: the time in localtime zone (the timezone of the node), in datetime.datetime.utcnow format.It marks the end of the range. If not supplied, will be taken as the currentTime. - :return: If successful, returns True, if data push fails, returns false + :return: 0, if successful """ global connection_established if not connection_established: @@ -278,11 +278,11 @@ def local_to_remote(local='localhost', remote=remote, startTime=None, endTime=No try: response = remoteclient.write_points(influx_json) print "Data for {0} pushed to server".format(measurement['name']) - return True + return 0 except Exception as er: print er print "Data push for {0} failed".format(measurement['name']) - return False + raise def getSenseware(location = '3rd_Floor_Apartment__Ben\\\'s_', site = '106_E_30th_St__Typhin_', measurement = 'Temperature', local='localhost', remote=remote, sensewaredb = 'Senseware', startTime=None, endTime=None): """Function to retrieve Senseware data frmo server. @@ -295,7 +295,7 @@ def getSenseware(location = '3rd_Floor_Apartment__Ben\\\'s_', site = '106_E_30th beginning for the range. If not supplied, will be taken 24-hours before endTime :param endTime: the time in localtime zone (the timezone of the node), in datetime.datetime.utcnow format.It marks the end of the range. If not supplied, will be taken as the currentTime. - :return: If successful, returns True, if data push fails, returns false + :return: 0, if successful """ localclient = InfluxDBClient(host, port, user, password, dbname) remoteclient = InfluxDBClient(remote, port, user, password, sensewaredb, ssl = True) @@ -346,6 +346,8 @@ def getSenseware(location = '3rd_Floor_Apartment__Ben\\\'s_', site = '106_E_30th try: response = localclient.write_points(influx_json) print "Senseware Data for {0} retrieved from server".format(measurement) + return 0 except Exception as er: print er - print "Senseware Data retrieve for {0} failed".format(measurement) \ No newline at end of file + print "Senseware Data retrieve for {0} failed".format(measurement) + raise -- GitLab From 2f15d3706294db1618c115026ec47c6534046688 Mon Sep 17 00:00:00 2001 From: mchlburton Date: Fri, 28 Apr 2017 14:08:01 -0400 Subject: [PATCH 14/31] beginnings for influx agent --- Agents/InfluxAgent/influx/__init__.py | 0 Agents/InfluxAgent/influx/agent.py | 75 +++++++++++++++++++++ Agents/InfluxAgent/setup.py | 76 ++++++++++++++++++++++ bemoss_lib/databases/influxAPI/InfluxDB.py | 2 +- bemoss_lib/utils/buildAgents.sh | 3 + bemoss_lib/utils/runPlatform.sh | 2 + 6 files changed, 157 insertions(+), 1 deletion(-) create mode 100755 Agents/InfluxAgent/influx/__init__.py create mode 100644 Agents/InfluxAgent/influx/agent.py create mode 100755 Agents/InfluxAgent/setup.py diff --git a/Agents/InfluxAgent/influx/__init__.py b/Agents/InfluxAgent/influx/__init__.py new file mode 100755 index 0000000..e69de29 diff --git a/Agents/InfluxAgent/influx/agent.py b/Agents/InfluxAgent/influx/agent.py new file mode 100644 index 0000000..eb9929a --- /dev/null +++ b/Agents/InfluxAgent/influx/agent.py @@ -0,0 +1,75 @@ +import importlib +import psycopg2 +import sys +import json +import datetime +import time +import logging +import os +import re +from volttron.platform.agent import BaseAgent, PublishMixin +from volttron.platform.agent import utils, matching +from volttron.platform.messaging import headers as headers_mod +from bemoss_lib.databases.influxAPI import InfluxDB +from urlparse import urlparse +import settings +import netifaces as ni +import ast +import subprocess + +utils.setup_logging() # setup logger for debugging +_log = logging.getLogger(__name__) + +# Step1: Agent Initialization +def InfluxAgent(config_path, **kwargs): + config = utils.load_config(config_path) # load the config_path from devicediscoveryagent.launch.json + def get_config(name): + try: + kwargs.pop(name) # from the **kwargs when call this function + except KeyError as er: + return config.get(name, '') + + # 1. @params agent + site = get_config('site') + db_scan_time = get_config('db_scan_time') + headers = {headers_mod.FROM: agent_id} + topic_delim = '/' # topic delimiter + + # @paths + PROJECT_DIR = settings.PROJECT_DIR + Applications_Launch_DIR = settings.Applications_Launch_DIR + Agents_Launch_DIR = settings.Agents_Launch_DIR + + class Agent(PublishMixin, BaseAgent): + + def __init__(self, **kwargs): + super(Agent, self).__init__(**kwargs) + #1. initialize all agent variables + + def setup(self): + super(Agent, self).setup() + InfluxDB.makeConnection() + + def updateRemote(self): + + + InfluxDB.local_to_remote(startTime = ) + + def updateSenseware(self): + + InfluxDB.getSenseware(self.site, measurement = 'Temperature', ) + + Agent.__name__ = 'InfluxAgent' + return Agent(**kwargs) + +def main(argv=sys.argv): + '''Main method called by the eggsecutable.''' + utils.default_main(InfluxAgent, description='Influx agent', argv=argv) + +if __name__ == '__main__': + # Entry point for script + try: + sys.exit(main()) + except KeyboardInterrupt as er: + print "KeyboardInterrupt", er + pass \ No newline at end of file diff --git a/Agents/InfluxAgent/setup.py b/Agents/InfluxAgent/setup.py new file mode 100755 index 0000000..90215b1 --- /dev/null +++ b/Agents/InfluxAgent/setup.py @@ -0,0 +1,76 @@ +# -*- coding: utf-8 -*- {{{ +# vim: set fenc=utf-8 ft=python sw=4 ts=4 sts=4 et: + +# Copyright (c) 2013, Battelle Memorial Institute +# All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# +# 1. Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# 2. Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in +# the documentation and/or other materials provided with the +# distribution. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +# +# The views and conclusions contained in the software and documentation +# are those of the authors and should not be interpreted as representing +# official policies, either expressed or implied, of the FreeBSD +# Project. +# +# This material was prepared as an account of work sponsored by an +# agency of the United States Government. Neither the United States +# Government nor the United States Department of Energy, nor Battelle, +# nor any of their employees, nor any jurisdiction or organization that +# has cooperated in the development of these materials, makes any +# warranty, express or implied, or assumes any legal liability or +# responsibility for the accuracy, completeness, or usefulness or any +# information, apparatus, product, software, or process disclosed, or +# represents that its use would not infringe privately owned rights. +# +# Reference herein to any specific commercial product, process, or +# service by trade name, trademark, manufacturer, or otherwise does not +# necessarily constitute or imply its endorsement, recommendation, or +# favoring by the United States Government or any agency thereof, or +# Battelle Memorial Institute. The views and opinions of authors +# expressed herein do not necessarily state or reflect those of the +# United States Government or any agency thereof. +# +# PACIFIC NORTHWEST NATIONAL LABORATORY +# operated by BATTELLE for the UNITED STATES DEPARTMENT OF ENERGY +# under Contract DE-AC05-76RL01830 + +#}}} + +from setuptools import setup, find_packages + +#get environ for agent name/identifier +packages = find_packages('.') +package = packages[0] + +setup( + name = package + 'agent', + version = "0.1", + install_requires = ['volttron'], + packages = packages, + entry_points = { + 'setuptools.installation': [ + 'eggsecutable = ' + package + '.agent:main', + ] + } +) + diff --git a/bemoss_lib/databases/influxAPI/InfluxDB.py b/bemoss_lib/databases/influxAPI/InfluxDB.py index 294ec84..9314554 100755 --- a/bemoss_lib/databases/influxAPI/InfluxDB.py +++ b/bemoss_lib/databases/influxAPI/InfluxDB.py @@ -284,7 +284,7 @@ def local_to_remote(local='localhost', remote=remote, startTime=None, endTime=No print "Data push for {0} failed".format(measurement['name']) raise -def getSenseware(location = '3rd_Floor_Apartment__Ben\\\'s_', site = '106_E_30th_St__Typhin_', measurement = 'Temperature', local='localhost', remote=remote, sensewaredb = 'Senseware', startTime=None, endTime=None): +def getSenseware(site = '106_E_30th_St__Typhin_', measurement = 'Temperature', local='localhost', remote=remote, sensewaredb = 'Senseware', startTime=None, endTime=None): """Function to retrieve Senseware data frmo server. :param location: Location of the sensor in the building corresponding to senseware location tag :param site: Building for BEMOSS instance corresponding to senseawre site tag diff --git a/bemoss_lib/utils/buildAgents.sh b/bemoss_lib/utils/buildAgents.sh index c94ed65..bf651d3 100755 --- a/bemoss_lib/utils/buildAgents.sh +++ b/bemoss_lib/utils/buildAgents.sh @@ -64,6 +64,9 @@ volttron-ctl install devicediscoveryagent=/tmp/volttron_wheels/devicediscoveryag volttron-pkg package ~/workspace/bemoss_os/Agents/AppLauncherAgent volttron-pkg configure /tmp/volttron_wheels/applauncheragent-0.1-py2-none-any.whl ~/workspace/bemoss_os/Agents/AppLauncherAgent/applauncheragent.launch.json volttron-ctl install applauncheragent=/tmp/volttron_wheels/applauncheragent-0.1-py2-none-any.whl +volttron-pkg package ~/workspace/bemoss_os/Agents/InfluxAgent +volttron-pkg configure /tmp/volttron_wheels/influxagent-0.1-py2-none-any.whl ~/workspace/bemoss_os/Agents/InfluxAgent/influxagent.launch.json +volttron-ctl install influxagent=/tmp/volttron_wheels/influxagent-0.1-py2-none-any.whl volttron-pkg package ~/workspace/bemoss_os/Applications/code/Lighting_Scheduler volttron-pkg package ~/workspace/bemoss_os/Applications/code/Plugload_Scheduler diff --git a/bemoss_lib/utils/runPlatform.sh b/bemoss_lib/utils/runPlatform.sh index 7714eb6..94df309 100755 --- a/bemoss_lib/utils/runPlatform.sh +++ b/bemoss_lib/utils/runPlatform.sh @@ -58,6 +58,8 @@ volttron-ctl start --tag approvalhelperagent #Run network agent sleep 2 volttron-ctl start --tag networkagent +sleep 2 +volttron-ctl start --tag influxagent #sleep 2 #volttron-ctl start --tag launcheragent volttron-ctl status -- GitLab From f35a1788104ea4ba6071a35e5111640e26289abc Mon Sep 17 00:00:00 2001 From: mchlburton Date: Fri, 28 Apr 2017 16:14:58 -0400 Subject: [PATCH 15/31] Influx agent complete and code cleanup --- Agents/InfluxAgent/influx/agent.py | 41 +- Agents/InfluxAgent/influxagent.launch.json | 5 + Agents/LightingAgent/lighting/influxagent.py | 549 ------------------ .../classAPI/classAPI_DummyPhilipsHue.py | 209 ------- bemoss_lib/databases/influxAPI/InfluxDB.py | 17 +- bemoss_lib/databases/influxAPI/__init__.py | 0 6 files changed, 46 insertions(+), 775 deletions(-) mode change 100644 => 100755 Agents/InfluxAgent/influx/agent.py create mode 100755 Agents/InfluxAgent/influxagent.launch.json delete mode 100755 Agents/LightingAgent/lighting/influxagent.py delete mode 100755 DeviceAPI/classAPI/classAPI_DummyPhilipsHue.py mode change 100644 => 100755 bemoss_lib/databases/influxAPI/__init__.py diff --git a/Agents/InfluxAgent/influx/agent.py b/Agents/InfluxAgent/influx/agent.py old mode 100644 new mode 100755 index eb9929a..b4e4503 --- a/Agents/InfluxAgent/influx/agent.py +++ b/Agents/InfluxAgent/influx/agent.py @@ -7,7 +7,7 @@ import time import logging import os import re -from volttron.platform.agent import BaseAgent, PublishMixin +from volttron.platform.agent import BaseAgent, PublishMixin, periodic from volttron.platform.agent import utils, matching from volttron.platform.messaging import headers as headers_mod from bemoss_lib.databases.influxAPI import InfluxDB @@ -22,7 +22,7 @@ _log = logging.getLogger(__name__) # Step1: Agent Initialization def InfluxAgent(config_path, **kwargs): - config = utils.load_config(config_path) # load the config_path from devicediscoveryagent.launch.json + config = utils.load_config(config_path) # load the config_path from influxagent.launch.json def get_config(name): try: kwargs.pop(name) # from the **kwargs when call this function @@ -32,7 +32,6 @@ def InfluxAgent(config_path, **kwargs): # 1. @params agent site = get_config('site') db_scan_time = get_config('db_scan_time') - headers = {headers_mod.FROM: agent_id} topic_delim = '/' # topic delimiter # @paths @@ -45,19 +44,45 @@ def InfluxAgent(config_path, **kwargs): def __init__(self, **kwargs): super(Agent, self).__init__(**kwargs) #1. initialize all agent variables + self.variables = kwargs + self.lastRemoteUpdate = str(datetime.datetime.utcnow()) + self.lastSensewareUpdate = str(datetime.datetime.utcnow()) + self.dbUpdateCounter = 0 + self.site = '106_E_30th_St__Typhin_' + def setup(self): super(Agent, self).setup() InfluxDB.makeConnection() + startTime=str(datetime.datetime.utcnow()-datetime.timedelta(hours = 24)) + try: + InfluxDB.getSenseware(self.site, measurement = 'Temperature', startTime=startTime) + self.lastSensewareUpdate = str(datetime.datetime.utcnow()) + print "Senseware data retrieved on startup" + except Exception as er: + print er + print "Influx agent failed to retrieve senseware data" - def updateRemote(self): - + @periodic(db_scan_time) + def updateDatabases(self): + self.dbUpdateCounter += 1 - InfluxDB.local_to_remote(startTime = ) + try: + InfluxDB.getSenseware(self.site, measurement = 'Temperature', startTime=self.lastSensewareUpdate) + self.lastSensewareUpdate = datetime.datetime.utcnow() + except Exception as er: + print er + print "Influx agent failed to retrieve senseware data, last update was {0}".format(self.lastSensewareUpdate) - def updateSenseware(self): + if dbUpdateCounter >= 48: + try: + InfluxDB.local_to_remote(startTime = self.lastRemoteUpdate) + self.lastRemoteUpdate = str(datetime.datetime.utcnow()) + dbUpdateCounter = 0 + except Exception as er: + print er + print "Influx agent failed to update remote, last update was {0}".format(self.lastRemoteUpdate) - InfluxDB.getSenseware(self.site, measurement = 'Temperature', ) Agent.__name__ = 'InfluxAgent' return Agent(**kwargs) diff --git a/Agents/InfluxAgent/influxagent.launch.json b/Agents/InfluxAgent/influxagent.launch.json new file mode 100755 index 0000000..d546158 --- /dev/null +++ b/Agents/InfluxAgent/influxagent.launch.json @@ -0,0 +1,5 @@ +{ + "agent_id": "InfluxAgent", + "db_scan_time":1800, + "site":"106_E_30th_St__Typhin_" +} diff --git a/Agents/LightingAgent/lighting/influxagent.py b/Agents/LightingAgent/lighting/influxagent.py deleted file mode 100755 index 86930b0..0000000 --- a/Agents/LightingAgent/lighting/influxagent.py +++ /dev/null @@ -1,549 +0,0 @@ -import sys -import json -import logging -import importlib -import datetime -import socket - -from volttron.platform.agent import BaseAgent, PublishMixin, periodic -from volttron.platform.agent import utils, matching -from volttron.platform.messaging import headers as headers_mod -from bemoss_lib.communication.Email import EmailService -from bemoss_lib.communication.sms import SMSService -import psycopg2 -import psycopg2.extras -import settings -from influxdb import InfluxDBClient -from bemoss_lib.databases.cassandraAPI import cassandraDB -from bemoss_lib.databases.influxAPI import InfluxDB - -utils.setup_logging() -_log = logging.getLogger(__name__) - -# Step1: Agent Initialization -def LightingAgent(config_path, **kwargs): - config = utils.load_config(config_path) - - def get_config(name): - try: - kwargs.pop(name) - except KeyError: - return config.get(name, '') - - #1. @params agent - agent_id = get_config('agent_id') - device_monitor_time = get_config('device_monitor_time') - max_monitor_time = int(settings.DEVICES['max_monitor_time']) - - debug_agent = False - #List of all keywords for a lighting agent - agentAPImapping = dict(status=[], brightness=[], color=[], saturation=[],power=[]) - log_variables = dict(status='text',brightness='double',hexcolor='text',power='double',offline_count='int') - #2. @params device_info - #TODO correct the launchfile in Device Discovery Agent - building_name = get_config('building_name') - zone_id = get_config('zone_id') - model = get_config('model') - if model == "Philips hue bridge": - hue_username = get_config('username') - else: - hue_username = '' - device_type = get_config('type') - address = get_config('address') - _address = address - _address = _address.replace('http://', '') - _address = _address.replace('https://', '') - try: # validate whether or not address is an ip address - socket.inet_aton(_address) - ip_address = _address - except socket.error: - ip_address = None - identifiable = get_config('identifiable') - - #3. @params agent & DB interfaces - #TODO delete variable topic - topic = get_config('topic') - - #TODO get database parameters from settings.py, add db_table for specific table - db_host = get_config('db_host') - db_port = get_config('db_port') - db_database = get_config('db_database') - db_user = get_config('db_user') - db_password = get_config('db_password') - db_table_lighting = settings.DATABASES['default']['TABLE_lighting'] - db_table_active_alert = settings.DATABASES['default']['TABLE_active_alert'] - db_table_bemoss_notify = settings.DATABASES['default']['TABLE_bemoss_notify'] - db_table_alerts_notificationchanneladdress = settings.DATABASES['default'][ - 'TABLE_alerts_notificationchanneladdress'] - db_table_temp_time_counter = settings.DATABASES['default']['TABLE_temp_time_counter'] - db_table_priority = settings.DATABASES['default']['TABLE_priority'] - - #construct _topic_Agent_UI based on data obtained from DB - _topic_Agent_UI_tail = building_name + '/' + str(zone_id) + '/' + agent_id - - #4. @params device_api - api = get_config('api') - apiLib = importlib.import_module("DeviceAPI.classAPI."+api) - - #4.1 initialize device object - Light = apiLib.API(model=model, device_type=device_type, api=api, address=address, username = hue_username, agent_id=agent_id, db_host=db_host, db_port=db_port, db_user=db_user, db_password=db_password, db_database=db_database) - print("{0}agent is initialized for {1} using API={2} at {3}".format(agent_id, Light.get_variable('model'), - Light.get_variable('api'), - Light.get_variable('address'))) - - #5. @params notification_info - send_notification = True - email_fromaddr = settings.NOTIFICATION['email']['fromaddr'] - email_username = settings.NOTIFICATION['email']['username'] - email_password = settings.NOTIFICATION['email']['password'] - email_mailServer = settings.NOTIFICATION['email']['mailServer'] - notify_heartbeat = settings.NOTIFICATION['heartbeat'] - - class Agent(PublishMixin, BaseAgent): - """Agent for querying WeatherUndergrounds API""" - - #1. agent initialization - def __init__(self, **kwargs): - super(Agent, self).__init__(**kwargs) - #1. initialize all agent variables - self.variables = kwargs - self.valid_data = False - self._keep_alive = True - self.flag = 1 - self.topic = topic - self.time_send_notification = 0 - self.event_ids = list() - self.time_sent_notifications = {} - self.notify_heartbeat = notify_heartbeat - self.ip_address = ip_address if ip_address != None else None - self.changed_variables = None - self.lastUpdateTime = None - self.already_offline = False - - #2. setup connection with db -> Connect to bemossdb database - try: - self.con = psycopg2.connect(host=db_host, port=db_port, database=db_database, user=db_user, - password=db_password) - self.cur = self.con.cursor() # open a cursor to perfomm database operations - print("{} connects to the database name {} successfully".format(agent_id, db_database)) - except: - print("ERROR: {} fails to connect to the database name {}".format(agent_id, db_database)) - - - - #3. send notification to notify building admin - self.send_notification = send_notification - self.subject = 'Message from ' + agent_id - - #These set and get methods allow scalability - def set_variable(self,k,v): # k=key, v=value - self.variables[k] = v - - def get_variable(self,k): - return self.variables.get(k, None) # default of get_variable is none - - #2. agent setup method - def setup(self): - super(Agent, self).setup() - #Do a one time push when we start up so we don't have to wait for the periodic - self.timer(1, self.deviceMonitorBehavior) - if identifiable == "True": Light.identifyDevice() - - @periodic(max_monitor_time) #save all data every max_monitor_time - def backupSaveData(self): - try: - Light.getDeviceStatus() - cassandraDB.insert(agent_id,Light.variables,log_variables) - print('Every Data Pushed to cassandra') - except Exception as er: - print("ERROR: {} fails to update cassandra database".format(agent_id)) - print er - - #3. deviceMonitorBehavior (TickerBehavior) - @periodic(device_monitor_time) - def deviceMonitorBehavior(self): - #step1: get current status of a thermostat, then map keywords and variables to agent knowledge - try: - Light.getDeviceStatus() - except: - print("device connection is not successful") - - self.changed_variables = dict() - - for v in log_variables: - if v in Light.variables: - if not v in self.variables or self.variables[v] != Light.variables[v]: - self.variables[v] = Light.variables[v] - self.changed_variables[v] = log_variables[v] - else: - if v not in self.variables: #it won't be in self.variables either (in the first time) - self.changed_variables[v] = log_variables[v] - self.variables[v] = None - try: - # Step: Check if any Device is OFFLINE - self.cur.execute("SELECT id FROM " + db_table_active_alert + " WHERE event_trigger_id=%s", ('5',)) - if self.cur.rowcount != 0: - self.device_offline_detection() - - # Put scan time in database - _time_stamp_last_scanned = datetime.datetime.now() - self.cur.execute("UPDATE "+db_table_lighting+" SET last_scanned_time=%s " - "WHERE lighting_id=%s", - (_time_stamp_last_scanned, agent_id)) - self.con.commit() - except Exception as er: - print er - print("ERROR: {} failed to update last scanned time".format(agent_id)) - - if len(self.changed_variables) == 0: - print 'nothing changed' - return - - self.updateUI() - #step4: update PostgresQL (meta-data) database - try: - self.cur.execute("UPDATE "+db_table_lighting+" SET status=%s WHERE lighting_id=%s", - (self.get_variable('status'), agent_id)) - self.con.commit() - self.cur.execute("UPDATE "+db_table_lighting+" SET brightness=%s WHERE lighting_id=%s", - (self.get_variable('brightness'), agent_id)) - self.con.commit() - self.cur.execute("UPDATE "+db_table_lighting+" SET color=%s WHERE lighting_id=%s", - (self.get_variable('hexcolor'), agent_id)) - self.con.commit() - try: - - if self.get_variable('status') == "ON": - multiple_on_off_status = "" - for dummyvar in range(self.get_variable('number_lights')): - multiple_on_off_status += "1" - self.cur.execute("UPDATE "+db_table_lighting+" SET multiple_on_off=%s WHERE lighting_id=%s", - (multiple_on_off_status, agent_id)) - self.con.commit() - else: # status is off - multiple_on_off_status = "" - for dummyvar in range(self.get_variable('number_lights')): - multiple_on_off_status += "0" - self.cur.execute("UPDATE "+db_table_lighting+" SET multiple_on_off=%s WHERE lighting_id=%s", - (multiple_on_off_status, agent_id)) - self.con.commit() - except: - print("{} this agent has no multiple_on_off_status".format(agent_id)) - #TODO check ip_address - if self.ip_address != None: - psycopg2.extras.register_inet() - _ip_address = psycopg2.extras.Inet(self.ip_address) - self.cur.execute("UPDATE "+db_table_lighting+" SET ip_address=%s WHERE lighting_id=%s", - (_ip_address, agent_id)) - self.con.commit() - - - if self.get_variable('offline_count') >= 3: - self.cur.execute("UPDATE "+db_table_lighting+" SET network_status=%s WHERE lighting_id=%s", - ('OFFLINE', agent_id)) - self.con.commit() - if self.already_offline is False: - self.already_offline = True - _time_stamp_last_offline = str(datetime.datetime.now()) - self.cur.execute("UPDATE "+db_table_lighting+" SET last_offline_time=%s WHERE lighting_id=%s", - (_time_stamp_last_offline, agent_id)) - self.con.commit() - else: - self.already_offline = False - self.cur.execute("UPDATE "+db_table_lighting+" SET network_status=%s WHERE lighting_id=%s", - ('ONLINE', agent_id)) - self.con.commit() - print("{} updates database name {} during deviceMonitorBehavior successfully".format(agent_id, db_database)) - except: - print("ERROR: {} fails to update the database name {}".format(agent_id,db_database)) - - #step6: Try to update Influxdb - try: - time = str(datetime.datetime.utcnow()) - InfluxDB.insert(agent_id, self.variables, log_variables) - print "{} success INFLUX update".format(agent_id) - - except: - print("ERROR: {} fails to update INFLUX".format(agent_id)) - - #step6: debug agent knowledge - if debug_agent: - print("printing agent's knowledge") - for k,v in self.variables.items(): - print (k,v) - print('') - - if debug_agent: - print("printing agentAPImapping's fields") - for k, v in agentAPImapping.items(): - if k is None: - agentAPImapping.update({v: v}) - agentAPImapping.pop(k) - for k, v in agentAPImapping.items(): - print (k, v) - - def device_offline_detection(self): - self.cur.execute("SELECT nickname FROM " + db_table_lighting + " WHERE lighting_id=%s", - (agent_id,)) - print agent_id - if self.cur.rowcount != 0: - device_nickname=self.cur.fetchone()[0] - print device_nickname - else: - device_nickname = '' - _db_notification_subject = 'BEMOSS Device {} {} went OFFLINE!!!'.format(device_nickname,agent_id) - _email_subject = '#Attention: BEMOSS Device {} {} went OFFLINE!!!'.format(device_nickname,agent_id) - _email_text = '#Attention: BEMOSS Device {} {} went OFFLINE!!!'.format(device_nickname,agent_id) - self.cur.execute("SELECT network_status FROM " + db_table_lighting + " WHERE lighting_id=%s", - (agent_id,)) - self.network_status = self.cur.fetchone()[0] - print self.network_status - if self.network_status=="OFFLINE": - print "Found Device OFFLINE" - self.cur.execute("SELECT id FROM " + db_table_active_alert + " WHERE event_trigger_id=%s", ('5',)) - self._active_alert_id = self.cur.fetchone()[0] - self.cur.execute( - "SELECT id FROM " + db_table_temp_time_counter + " WHERE alert_id=%s AND device_id=%s", - (str(self._active_alert_id), agent_id,)) - # If this is the first detected violation - if self.cur.rowcount == 0: - print "first device offline detected" - # create counter in DB - self.cur.execute( - "INSERT INTO " + db_table_temp_time_counter + " VALUES(DEFAULT,%s,%s,%s,%s,%s)", - (self._active_alert_id, agent_id, '0', '0', '0')) - self.con.commit() - self.send_device_notification_db(_db_notification_subject, self._active_alert_id) - - # Send email if exist - self.cur.execute("SELECT notify_address FROM " + db_table_alerts_notificationchanneladdress + " WHERE active_alert_id=%s AND notification_channel_id=%s",(self._active_alert_id,'1')) - if self.cur.rowcount != 0: - self._alert_email = self.cur.fetchall() - for single_email_1 in self._alert_email: - print single_email_1[0] - self.send_device_notification_email(single_email_1[0], _email_subject, _email_text) - - # Send SMS if provided by user - self.cur.execute("SELECT notify_address FROM " + db_table_alerts_notificationchanneladdress + " WHERE active_alert_id=%s AND notification_channel_id=%s",(self._active_alert_id,'2')) - if self.cur.rowcount != 0: - self._alert_sms_phone_no = self.cur.fetchall() - for single_number in self._alert_sms_phone_no: - print single_number[0] - self.send_device_notification_sms(single_number[0], _email_subject) - else: - self.priority_counter(self._active_alert_id, _db_notification_subject) - else: - print "The Device is ONLINE" - - def send_device_notification_db(self, _tampering_device_msg, _active_alert_id): - print " INSIDE send_device_notification_db" - - # Find the priority id - self.cur.execute( - "SELECT priority_id FROM " + db_table_active_alert + " WHERE id=%s", - (str(_active_alert_id),)) - self.priority_id = self.cur.fetchone()[0] - - # Find the priority level - self.cur.execute( - "SELECT priority_level FROM " + db_table_priority + " WHERE id=%s", - str(self.priority_id)) - self.priority_level = self.cur.fetchone()[0] - - # Insert into DB the notification - self.cur.execute("INSERT INTO " + db_table_bemoss_notify + " VALUES(DEFAULT,%s,%s,%s,%s)", - (_tampering_device_msg, - str(datetime.datetime.now()), 'Alert', str(self.priority_level))) - self.con.commit() - - # Find the number of notifications sent for the same alert and device - self.cur.execute( - "SELECT no_notifications_sent FROM " + db_table_temp_time_counter + " WHERE alert_id=%s AND device_id=%s", - (str(_active_alert_id), agent_id,)) - self._no_notifications_sent = self.cur.fetchone()[0] - self.con.commit() - print self._no_notifications_sent - self._no_notifications_sent = int(self._no_notifications_sent) + 1 - print self._no_notifications_sent - self.cur.execute( - "UPDATE " + db_table_temp_time_counter + " SET no_notifications_sent=%s WHERE alert_id=%s AND device_id=%s", - (str(self._no_notifications_sent), str(_active_alert_id), agent_id,)) - self.con.commit() - - def send_device_notification_email(self, _active_alert_email, _email_subject, _email_text): - emailService = EmailService() - # Send Email - emailService.sendEmail(email_fromaddr, _active_alert_email, email_username, - email_password, _email_subject, _email_text, email_mailServer) - - def send_device_notification_sms(self, _active_alert_phone_number_misoperation, _sms_subject): - print "INSIDE send_device_notification_sms" - print _active_alert_phone_number_misoperation - smsService = SMSService() - smsService.sendSMS(email_fromaddr, _active_alert_phone_number_misoperation, email_username, email_password, _sms_subject, email_mailServer) - - # TODO: this function is in all other agents, need to get rid of those redundent codes. - def priority_counter(self, _active_alert_id, _tampering_device_msg_1): - # Find the priority counter limit then compare it with priority_counter in priority table - # if greater than the counter limit then send notification and reset the value - # else just increase the counter - print "INSIDE the priority_counter" - _email_subject = '#Attention: BEMOSS Device {} went OFFLINE!!!'.format(agent_id) - _email_text = '#Attention: BEMOSS Device {} went OFFLINE!!!'.format(agent_id) - self.cur.execute( - "SELECT priority_counter FROM " + db_table_temp_time_counter + " WHERE alert_id=%s AND device_id=%s", - (str(_active_alert_id), agent_id,)) - self.priority_count = self.cur.fetchone()[0] - self.con.commit() - - # Find the priority id from active alert table - self.cur.execute( - "SELECT priority_id FROM " + db_table_active_alert + " WHERE id=%s", - (str(_active_alert_id),)) - self.priority_id = self.cur.fetchone()[0] - self.con.commit() - - # Find the priority limit from the priority table - self.cur.execute( - "SELECT priority_counter FROM " + db_table_priority + " WHERE id=%s", - (str(self.priority_id),)) - self.priority_limit = self.cur.fetchone()[0] - self.con.commit() - - # If the counter reaches the limit - if int(self.priority_count) > int(self.priority_limit): - self.send_device_notification_db(_tampering_device_msg_1, _active_alert_id) - self.cur.execute( - "UPDATE " + db_table_temp_time_counter + " SET priority_counter=%s WHERE alert_id=%s AND device_id=%s", - ('0', str(_active_alert_id), agent_id,)) - self.con.commit() - - print "INSIDE the priority counter exceeded the defined range" - # Send email if exist - self.cur.execute("SELECT notify_address FROM " + db_table_alerts_notificationchanneladdress + " WHERE active_alert_id=%s AND notification_channel_id=%s",(self._active_alert_id,'1')) - if self.cur.rowcount != 0: - self._alert_email = self.cur.fetchall() - for single_email_1 in self._alert_email: - print single_email_1[0] - self.send_device_notification_email(single_email_1[0], _email_subject, _email_text) - - # Send SMS if provided by user - self.cur.execute("SELECT notify_address FROM " + db_table_alerts_notificationchanneladdress + " WHERE active_alert_id=%s AND notification_channel_id=%s",(self._active_alert_id,'2')) - if self.cur.rowcount != 0: - self._alert_sms_phone_no = self.cur.fetchall() - for single_number in self._alert_sms_phone_no: - print single_number[0] - self.send_device_notification_sms(single_number[0], _email_subject) - else: - self.priority_count = int(self.priority_count) + 1 - self.cur.execute( - "UPDATE " + db_table_temp_time_counter + " SET priority_counter=%s WHERE alert_id=%s AND device_id=%s", - (str(self.priority_count), str(_active_alert_id), agent_id,)) - - def updateUI(self): - topic = '/agent/ui/'+device_type+'/device_status_response/'+_topic_Agent_UI_tail - headers = { - 'AgentID': agent_id, - headers_mod.CONTENT_TYPE: headers_mod.CONTENT_TYPE.JSON, - headers_mod.FROM: agent_id, - headers_mod.TO: 'ui' - } - _data={'status': self.get_variable('status'), - 'brightness': self.get_variable('brightness'), 'color': self.get_variable('hexcolor'), - 'saturation': self.get_variable('saturation')} - message = json.dumps(_data) - message = message.encode(encoding='utf_8') - self.publish(topic, headers, message) - - #4. updateUIBehavior (generic behavior) - @matching.match_exact('/ui/agent/'+device_type+'/device_status/'+_topic_Agent_UI_tail) - def updateUIBehavior(self,topic,headers,message,match): - print "{} agent got\nTopic: {topic}".format(self.get_variable("agent_id"),topic=topic) - print "Headers: {headers}".format(headers=headers) - print "Message: {message}\n".format(message=message) - #reply message - self.updateUI() - - - #5. deviceControlBehavior (generic behavior) - @matching.match_exact('/ui/agent/'+device_type+'/update/'+_topic_Agent_UI_tail) - def deviceControlBehavior(self,topic,headers,message,match): - #print received message from UI - print "{} agent got\nTopic: {topic}".format(self.get_variable("agent_id"),topic=topic) - print "Headers: {headers}".format(headers=headers) - print "Message: {message}\n".format(message=message) - - #prepare message content for response - topic = '/agent/ui/'+device_type+'/update_response/'+_topic_Agent_UI_tail - headers = { - 'AgentID': agent_id, - headers_mod.CONTENT_TYPE: headers_mod.CONTENT_TYPE.PLAIN_TEXT, - headers_mod.FROM: agent_id, - headers_mod.TO: 'ui' - } - #step1: change device status according to the receive message - if self.isPostmsgValid(message[0]): - setDeviceStatusResult = Light.setDeviceStatus(json.loads(message[0])) #convert received message from string to JSON - #TODO need to do additional checking whether the device setting is actually success!!!!!!!! - #step3: send reply message back to the UI - if setDeviceStatusResult: - message = 'success' - else: - message = 'failure' - else: - print("The POST message is invalid, check brightness, status or color setting and try again\n") - message = 'failure' - self.publish(topic, headers, message) - self.deviceMonitorBehavior() - - def isPostmsgValid(self, postmsg): # check validity of postmsg - dataValidity = True - try: - _data = json.loads(postmsg) - if ("brightness" in _data.keys()) or ("status" in _data.keys()) or ("color" in _data.keys()): - dataValidity = True - else: - dataValidity = False - except: - dataValidity = False - print("dataValidity failed to validate data coming from UI") - return dataValidity - - #6. deviceIdentifyBehavior(generic behavior) - @matching.match_exact('/ui/agent/'+device_type+'/identify/'+_topic_Agent_UI_tail) - def deviceIdentifyBehavior(self,topic,headers,message,match): - print "{} agent got\nTopic: {topic}".format(self.get_variable("agent_id"),topic=topic) - print "Headers: {headers}".format(headers=headers) - print "Message: {message}\n".format(message=message) - #step1: change device status according to the receive message - identifyDeviceResult = Light.identifyDevice() - #TODO need to do additional checking whether the device setting is actually success!!!!!!!! - #step2: send reply message back to the UI - topic = '/agent/ui/identify_response/'+device_type+'/'+_topic_Agent_UI_tail - headers = { - 'AgentID': agent_id, - headers_mod.CONTENT_TYPE: headers_mod.CONTENT_TYPE.PLAIN_TEXT, - } - if identifyDeviceResult: - message = 'success' - else: - message = 'failure' - self.publish(topic, headers, message) - - - - Agent.__name__ = 'LightingAgent' - return Agent(**kwargs) - -def main(argv=sys.argv): - '''Main method called by the eggsecutable.''' - utils.default_main(LightingAgent, - description='Lighting agent', - argv=argv) - -if __name__ == '__main__': - try: - sys.exit(main()) - except KeyboardInterrupt: - pass - diff --git a/DeviceAPI/classAPI/classAPI_DummyPhilipsHue.py b/DeviceAPI/classAPI/classAPI_DummyPhilipsHue.py deleted file mode 100755 index 20b8062..0000000 --- a/DeviceAPI/classAPI/classAPI_DummyPhilipsHue.py +++ /dev/null @@ -1,209 +0,0 @@ -''' -#__author__ = "Mike" -#__credits__ = "BEMOSS Team" -#__version__ = "2.0" -#__maintainer__ = "Mike" -#__email__ = "michael@blocpower.io" -#__created__ = "2017-03-16" -#__lastUpdated__ = "2017-23-17" -''' - -import time -from random import randint -from bemoss_lib.utils import rgb_cie -class API: - # 1. constructor : gets call every time when create a new class - # requirements for instantiation1. model, 2.type, 3.api, 4. address - def __init__(self,**kwargs): # default color is white - # Initialized common attributes - self.variables = kwargs - self.debug = True - self.set_variable('offline_count',0) - self.set_variable('connection_renew_interval',6000) #nothing to renew, right now - self.only_white_bulb = None - # to initialize the only white bulb value - self.getDeviceStatus() - def renewConnection(self): - pass - - def set_variable(self,k,v): # k=key, v=value - self.variables[k] = v - - def get_variable(self,k): - return self.variables.get(k, None) # default of get_variable is none - - # 2. Attributes from Attributes table - ''' - Attributes: - ------------------------------------------------------------------------------------------ - status GET POST Philips Hue ON/OFF status - brightness GET POST brightness percentage - effect GET POST Hue light effect 'none' or 'colorloop' - color GET POST temporary target heat setpoint (floating point in deg F) - ------------------------------------------------------------------------------------------ - - ''' - - # 3. Capabilites (methods) from Capabilities table - ''' - API3 available methods: - 1. getDeviceStatus() GET - 2. setDeviceStatus(postmsg) PUT - 3. identifyDevice() - ''' - - # ---------------------------------------------------------------------- - # getDeviceStatus(), getDeviceStatusJson(data), printDeviceStatus() - def getDeviceStatus(self): - getDeviceStatusResult = True - - try: - onoff = randint(0, 1) - if onoff == 1: - self.set_variable('status', "ON") - else: - self.set_variable('status', "OFF") - bri = randint(0, 255) - hue = randint(0, 255) - xy = [randint(0, 255), randint(0,255)] - ct = randint(0, 255) - sat = randint(0, 255) - name = "DummyHue" - # 2. brightness convert to % - self.set_variable('brightness',int(round(float(bri)*100/255,0))) - # update only white variable every round is necessary in case user add a/take away all color bulb(s). - self.only_white_bulb = False if hue else True - if self.only_white_bulb is False: - # 3. color convert to RGB 0-255 - self.set_variable('hue', hue) - self.set_variable('xy', xy) - self.set_variable('ct', ct) - x=xy[0] - y=xy[1] - self.set_variable('color', rgb_cie.ColorHelper.getRGBFromXYAndBrightness(x,y,bri)) - self.set_variable('hexcolor', '#%02x%02x%02x' % self.get_variable('color')) - # 4. saturation convert to % - self.set_variable('saturation',int(round(float(sat)*100/255,0))) - self.set_variable('number_lights', [2]) - self.set_variable('name',name) - - - # Check the connectivity - if getDeviceStatusResult==True: - self.set_variable('offline_count', 0) - else: - self.set_variable('offline_count', self.get_variable('offline_count')+1) - except Exception as er: - print er - print('ERROR: classAPI_PhilipsHue failed to getDeviceStatus') - self.set_variable('offline_count',self.get_variable('offline_count')+1) - - def printDeviceStatus(self): - # now we can access the contents of the JSON like any other Python object - print(" the current status is as follows:") - print(" name = {}".format(self.get_variable('name'))) - #print(" number_lights = {}".format(self.get_variable('number_lights'))) - print(" status = {}".format(self.get_variable('status'))) - print(" brightness = {}".format(self.get_variable('brightness'))) - if self.only_white_bulb is False: - print(" hue = {}".format(self.get_variable('hue'))) - print(" color = {}".format(self.get_variable('color'))) - print(" saturation = {}".format(self.get_variable('saturation'))) - print(" xy= {}".format(self.get_variable('xy'))) - print(" ct = {}".format(self.get_variable('ct'))) - print(" effect = {}".format(self.get_variable('effect'))) - print(" colormode = {}\n".format(self.get_variable('colormode'))) - # ---------------------------------------------------------------------- - # setDeviceStatus(postmsg), isPostmsgValid(postmsg), convertPostMsg(postmsg) - def setDeviceStatus(self, postmsg): - setDeviceStatusResult = True - - return setDeviceStatusResult - - def isPostMsgValid(self,postmsg): #check validity of postmsg - dataValidity = True - #TODO algo to check whether postmsg is valid - return dataValidity - - - def convertPostMsg(self,postmsg): - msgToDevice = {} - datacontainsRGB=False - if 'color' in postmsg.keys(): - datacontainsRGB=True - - for k,v in postmsg.items(): - if k == 'status': - if postmsg.get('status') == "ON": - msgToDevice['on'] = True - elif postmsg.get('status') == "OFF": - msgToDevice['on'] = False - elif k == 'brightness': - msgToDevice['bri'] = int(round(float(postmsg.get('brightness'))*255.0/100.0,0)) - elif k == 'color': - if self.only_white_bulb is False: - print(type(postmsg['color'])) - _red = postmsg['color'][0] - _green = postmsg['color'][1] - _blue = postmsg['color'][2] - _xyY = rgb_cie.ColorHelper.getXYPointFromRGB(_red, _green, _blue) - msgToDevice['xy'] = [_xyY.x, _xyY.y] - #msgToDevice['bri']= int(round(_xyY.y*255,0)) - elif k == 'hue': - if datacontainsRGB==False and self.only_white_bulb is False: - msgToDevice['hue'] = postmsg.get('hue') - elif k == 'saturation': - if datacontainsRGB==False and self.only_white_bulb is False: - msgToDevice['sat'] = int(round(float(postmsg.get('saturation'))*255.0/100.0,0)) - else: - msgToDevice[k] = v - return msgToDevice - # ---------------------------------------------------------------------- - # method3: Identify this lights (Physically) - def identifyDevice(self): - identifyDeviceResult = False - print(" {0}Agent for {1} is identifying itself by doing colorloop. Please observe your lights" - .format(self.variables.get('agent_id',None), self.variables.get('model',None))) - try: - devicewasoff=0 - if self.get_variable('status')=="OFF": - devicewasoff=1 - self.setDeviceStatus({"status":"ON"}) - elif self.only_white_bulb: - self.setDeviceStatus({"status":"OFF"}) - if self.only_white_bulb is False: - self.setDeviceStatus({"effect": "colorloop"}) - if self.only_white_bulb: - time_iden = 3 - else: - time_iden = 10 #time to do identification - t0 = time.time() - self.seconds = time_iden - while time.time() - t0 <= time_iden: - self.seconds = self.seconds - 1 - print("wait: {} sec".format(self.seconds)) - time.sleep(1) - self.setDeviceStatus({"effect": "none"}) - if devicewasoff==1: - self.setDeviceStatus({"status":"OFF"}) - else: - self.setDeviceStatus({"status":"ON"}) - identifyDeviceResult = True - except: - print("ERROR: classAPI_PhilipsHue connection failure! @ identifyDevice") - return identifyDeviceResult - # ---------------------------------------------------------------------- - -# This main method will not be executed when this class is used as a module -def main(): - # create an object with initialized data from DeviceDiscovery Agent - # requirements for instantiation1. model, 2.type, 3.api, 4. address - PhilipsHue = API(model='Philips Hue',type='wifiLight',api='API3',address='http://192.168.10.14:80',username='acquired username',agent_id='LightingAgent') - print("{0}agent is initialzed for {1} using API={2} at {3}".format(PhilipsHue.get_variable('type'),PhilipsHue.get_variable('model'),PhilipsHue.get_variable('api'),PhilipsHue.get_variable('address'))) - - PhilipsHue.getDeviceStatus() - PhilipsHue.setDeviceStatus({"status":"ON","color":(155,113,255)}) - PhilipsHue.identifyDevice() - - -if __name__ == "__main__": main() \ No newline at end of file diff --git a/bemoss_lib/databases/influxAPI/InfluxDB.py b/bemoss_lib/databases/influxAPI/InfluxDB.py index 9314554..fe61a58 100755 --- a/bemoss_lib/databases/influxAPI/InfluxDB.py +++ b/bemoss_lib/databases/influxAPI/InfluxDB.py @@ -25,12 +25,13 @@ port = 8086 user = 'engineering' password = 'nPEc9Pz0iV' dbname = 'bemoss' +site = '106_E_30th_St__Typhin_' def makeConnection(): try: global host, port, user, password, dbname, connection_established - client = InfluxDBClient(host, port, user, password, dbname, ssl=True) + client = InfluxDBClient(host, port, user, password, dbname) database_list = client.get_list_database() for db in database_list: if dbname in db['name']: @@ -77,8 +78,7 @@ def insert(device_model, agentID, all_vars, log_vars, cur_time=None): { "measurement": device_model, "tags": { - "agent": agentID, - "building": "bemoss" + "agent": agentID }, "time": cur_time, "fields": fields @@ -176,7 +176,7 @@ def retrieve(device_model, agentID, vars=None, startTime=None, endTime=None,expo client = InfluxDBClient(host, port, user, password, dbname, ssl=True) if startTime==None: - startTime=str(datetime.datetime.utcnow()-datetime.timedelta(hours=24)) + startTime=str(datetime.datetime.utcnow()-datetime.timedelta(hours=1)) if endTime==None: endTime = str(datetime.datetime.utcnow()) @@ -200,7 +200,7 @@ def retrieve(device_model, agentID, vars=None, startTime=None, endTime=None,expo result = client.query("select {0} from {1} WHERE agent = '{2}' AND time >= '{3}' AND time <= '{4}';".format(varStr, device_model, agentID, startTime,endTime)) values = [] - data = result.get_points('lightsensor') + data = result.get_points() for entry in data: values.append(entry.values()) @@ -224,7 +224,7 @@ def local_to_remote(local='localhost', remote=remote, startTime=None, endTime=No makeConnection() localclient = InfluxDBClient(host, port, user, password, dbname, ssl=True) - remoteclient = InfluxDBClient(remote, port, user, password, dbname, ssl=True) + remoteclient = InfluxDBClient(remote, port, user, password, site, ssl=True) startTime=None endTime=None @@ -286,7 +286,6 @@ def local_to_remote(local='localhost', remote=remote, startTime=None, endTime=No def getSenseware(site = '106_E_30th_St__Typhin_', measurement = 'Temperature', local='localhost', remote=remote, sensewaredb = 'Senseware', startTime=None, endTime=None): """Function to retrieve Senseware data frmo server. - :param location: Location of the sensor in the building corresponding to senseware location tag :param site: Building for BEMOSS instance corresponding to senseawre site tag :param measurement: The variable you want the data for (temperature, humidity, etc.) :param local: Database you want to copy data from @@ -302,12 +301,12 @@ def getSenseware(site = '106_E_30th_St__Typhin_', measurement = 'Temperature', l startTime=None endTime=None if startTime==None: - startTime=str(datetime.datetime.utcnow()-datetime.timedelta(minute = 30)) + startTime=str(datetime.datetime.utcnow()-datetime.timedelta(minutes = 30)) if endTime==None: endTime = str(datetime.datetime.utcnow()) - measure_query = remoteclient.query("SELECT * FROM {0} WHERE site='{1}' AND location='{2}' AND time>='{3}' AND time<='{4}';".format(measurement, site, location, startTime, endTime)) + measure_query = remoteclient.query("SELECT * FROM {0} WHERE site='{1}' AND time>='{2}' AND time<='{3}';".format(measurement, site, startTime, endTime)) data = measure_query.get_points('{0}'.format(measurement)) field_query = remoteclient.query("show field keys from {0}".format(measurement)) diff --git a/bemoss_lib/databases/influxAPI/__init__.py b/bemoss_lib/databases/influxAPI/__init__.py old mode 100644 new mode 100755 -- GitLab From f8e9a88dd569e4671a6b51b708706b2e73ddcf48 Mon Sep 17 00:00:00 2001 From: mchlburton Date: Mon, 1 May 2017 11:40:46 -0400 Subject: [PATCH 16/31] Typo fixes and influx agent test --- Agents/InfluxAgent/influx/agent.py | 7 +++++++ Agents/TRVAgent/trv/agent.py | 11 ++++++++++- .../{classAPI_EnoceanPi.py => classAPI_EnOceanPi.py} | 0 DeviceAPI/discoverAPI/WiFi.py | 6 +++--- bemoss_lib/databases/influxAPI/InfluxDB.py | 11 +++++------ 5 files changed, 25 insertions(+), 10 deletions(-) rename DeviceAPI/classAPI/{classAPI_EnoceanPi.py => classAPI_EnOceanPi.py} (100%) diff --git a/Agents/InfluxAgent/influx/agent.py b/Agents/InfluxAgent/influx/agent.py index b4e4503..ff5d551 100755 --- a/Agents/InfluxAgent/influx/agent.py +++ b/Agents/InfluxAgent/influx/agent.py @@ -62,6 +62,13 @@ def InfluxAgent(config_path, **kwargs): except Exception as er: print er print "Influx agent failed to retrieve senseware data" + try: + InfluxDB.local_to_remote() + self.lastRemoteUpdate = str(datetime.datetime.utcnow()) + print "Influx agent successfully updated remote" + except Exception as er: + print er + print "Influx agent failed to update remote, last update was {0}".format(self.lastRemoteUpdate) @periodic(db_scan_time) def updateDatabases(self): diff --git a/Agents/TRVAgent/trv/agent.py b/Agents/TRVAgent/trv/agent.py index 6ce4d1b..c4ad983 100755 --- a/Agents/TRVAgent/trv/agent.py +++ b/Agents/TRVAgent/trv/agent.py @@ -26,6 +26,7 @@ import psycopg2.extras import settings import socket from bemoss_lib.databases.cassandraAPI import cassandraDB +from bemoss_lib.databases.influxAPI import InfluxDB from bemoss_lib.utils.catcherror import catcherror utils.setup_logging() @@ -55,7 +56,7 @@ def TRVAgent(config_path, **kwargs): model = get_config('model') device_type = get_config('type') #TODO Make address come from launch file - address = 'http://192.168.0.103' + address = 'http://192.168.0.100' macaddress = get_config('macaddress') _address = address _address = _address.replace('http://', '') @@ -256,6 +257,14 @@ def TRVAgent(config_path, **kwargs): print("ERROR: {} fails to update cassandra database".format(agent_id)) print er + # step4: update infulx database + try: + InfluxDB.insert(device_type, agent_id, self.variables, log_variables) + print('Data Pushed to InfluxDB') + except Exception as er: + print("ERROR: {} fails to update influx database".format(agent_id)) + print er + # step5: debug agent knowledge if debug_agent: print("printing agent's knowledge") diff --git a/DeviceAPI/classAPI/classAPI_EnoceanPi.py b/DeviceAPI/classAPI/classAPI_EnOceanPi.py similarity index 100% rename from DeviceAPI/classAPI/classAPI_EnoceanPi.py rename to DeviceAPI/classAPI/classAPI_EnOceanPi.py diff --git a/DeviceAPI/discoverAPI/WiFi.py b/DeviceAPI/discoverAPI/WiFi.py index a818cc8..7986a78 100755 --- a/DeviceAPI/discoverAPI/WiFi.py +++ b/DeviceAPI/discoverAPI/WiFi.py @@ -106,12 +106,12 @@ def discover(type, timeout=2, retries=1): #TODO have discovery agent find raspberry pi first instead of hard coding ip elif type=='Magnum': - device_url = 'http://192.168.0.103:8080/api/v1' + device_url = 'http://192.168.0.100:8080/api/v1' header = {"Authorization_key":'ABC', "Content-type": 'application/json'} trv_discover = {"discover":'True'} data_req=None try: - data_req = requests.get(device_url,headers = header, params = trv_discover, timeout=5) + data_req = requests.get(device_url,headers = header, params = trv_discover, timeout=15) except Exception as er: print er @@ -247,7 +247,7 @@ def getmodelvendor(type,ipaddress): elif type=="Awair": return {'model':'Awair+','vendor':'Awair'} elif type=="Magnum": - return {'model':'MD-15', 'vendor':'Magnum'} + return {'model':'M9-D15', 'vendor':'Magnum'} # This main method will not be executed when this class is used as a module diff --git a/bemoss_lib/databases/influxAPI/InfluxDB.py b/bemoss_lib/databases/influxAPI/InfluxDB.py index fe61a58..43b39e9 100755 --- a/bemoss_lib/databases/influxAPI/InfluxDB.py +++ b/bemoss_lib/databases/influxAPI/InfluxDB.py @@ -64,8 +64,7 @@ def insert(device_model, agentID, all_vars, log_vars, cur_time=None): if not connection_established: makeConnection() - client = InfluxDBClient(host, port, user, password, dbname, ssl=True) - + client = InfluxDBClient(host, port, user, password, dbname) if cur_time == None: cur_time = datetime.datetime.utcnow() @@ -112,7 +111,7 @@ def delete(device_model, agentID, startTime=None, endTime=None): if not connection_established: makeConnection() - client = InfluxDBClient(host, port, user, password, dbname, ssl=True) + client = InfluxDBClient(host, port, user, password, dbname) if endTime==None and startTime==None: try: @@ -173,7 +172,7 @@ def retrieve(device_model, agentID, vars=None, startTime=None, endTime=None,expo if not connection_established: makeConnection() - client = InfluxDBClient(host, port, user, password, dbname, ssl=True) + client = InfluxDBClient(host, port, user, password, dbname) if startTime==None: startTime=str(datetime.datetime.utcnow()-datetime.timedelta(hours=1)) @@ -223,7 +222,7 @@ def local_to_remote(local='localhost', remote=remote, startTime=None, endTime=No if not connection_established: makeConnection() - localclient = InfluxDBClient(host, port, user, password, dbname, ssl=True) + localclient = InfluxDBClient(host, port, user, password, dbname) remoteclient = InfluxDBClient(remote, port, user, password, site, ssl=True) startTime=None @@ -235,7 +234,7 @@ def local_to_remote(local='localhost', remote=remote, startTime=None, endTime=No if endTime==None: endTime = str(datetime.datetime.utcnow()) - measurements = localclient.query("show measurements;".format(dbname)) + measurements = localclient.query("show measurements;") for result in measurements: for measurement in result: -- GitLab From a484e4471c9a355731d4632e2955e65fa100552a Mon Sep 17 00:00:00 2001 From: mchlburton Date: Mon, 1 May 2017 11:44:41 -0400 Subject: [PATCH 17/31] If a database doesn't exist it will now be created --- bemoss_lib/databases/influxAPI/InfluxDB.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/bemoss_lib/databases/influxAPI/InfluxDB.py b/bemoss_lib/databases/influxAPI/InfluxDB.py index 43b39e9..890e3b7 100755 --- a/bemoss_lib/databases/influxAPI/InfluxDB.py +++ b/bemoss_lib/databases/influxAPI/InfluxDB.py @@ -28,7 +28,7 @@ dbname = 'bemoss' site = '106_E_30th_St__Typhin_' -def makeConnection(): +def makeConnection(database = dbname): try: global host, port, user, password, dbname, connection_established client = InfluxDBClient(host, port, user, password, dbname) @@ -40,7 +40,8 @@ def makeConnection(): except Exception as er: - print 'Cannot establish connection' + print 'Cannot establish connection, trying to create database' + client.create_database(dbname) raise er try: -- GitLab From 0acd3a2a497a18df37184b1981ee6fdc05db0279 Mon Sep 17 00:00:00 2001 From: Avijit Saha Date: Tue, 2 May 2017 12:40:26 -0400 Subject: [PATCH 18/31] Updating Influxagent to get/set last update times on postgres, config/settings updates, table creation in postgres for influx --- Agents/InfluxAgent/influx/agent.py | 117 +++++++++++++----- Agents/InfluxAgent/influxagent.launch.json | 4 +- .../create_influx_table_in_bemossdb.sql | 12 ++ bemoss_lib/databases/influxAPI/InfluxDB.py | 5 +- settings.py | 3 +- 5 files changed, 108 insertions(+), 33 deletions(-) create mode 100644 bemoss_lib/databases/create_influx_table_in_bemossdb.sql diff --git a/Agents/InfluxAgent/influx/agent.py b/Agents/InfluxAgent/influx/agent.py index ff5d551..3ca574d 100755 --- a/Agents/InfluxAgent/influx/agent.py +++ b/Agents/InfluxAgent/influx/agent.py @@ -1,4 +1,3 @@ -import importlib import psycopg2 import sys import json @@ -11,11 +10,8 @@ from volttron.platform.agent import BaseAgent, PublishMixin, periodic from volttron.platform.agent import utils, matching from volttron.platform.messaging import headers as headers_mod from bemoss_lib.databases.influxAPI import InfluxDB -from urlparse import urlparse import settings -import netifaces as ni -import ast -import subprocess + utils.setup_logging() # setup logger for debugging _log = logging.getLogger(__name__) @@ -30,8 +26,10 @@ def InfluxAgent(config_path, **kwargs): return config.get(name, '') # 1. @params agent - site = get_config('site') + agent_id = get_config('agent_id') db_scan_time = get_config('db_scan_time') + db_backup_time = get_config('db_backup_time') + site = settings.PLATFORM['node']['building_name'] topic_delim = '/' # topic delimiter # @paths @@ -39,58 +37,119 @@ def InfluxAgent(config_path, **kwargs): Applications_Launch_DIR = settings.Applications_Launch_DIR Agents_Launch_DIR = settings.Agents_Launch_DIR + # postgres + db_database = settings.DATABASES['default']['NAME'] + db_host = settings.DATABASES['default']['HOST'] + db_port = settings.DATABASES['default']['PORT'] + db_user = settings.DATABASES['default']['USER'] + db_password = settings.DATABASES['default']['PASSWORD'] + db_table_influx = settings.DATABASES['default']['TABLE_influx_info'] + class Agent(PublishMixin, BaseAgent): def __init__(self, **kwargs): super(Agent, self).__init__(**kwargs) #1. initialize all agent variables self.variables = kwargs - self.lastRemoteUpdate = str(datetime.datetime.utcnow()) - self.lastSensewareUpdate = str(datetime.datetime.utcnow()) - self.dbUpdateCounter = 0 - self.site = '106_E_30th_St__Typhin_' + # Setup connection with db -> Connect to bemossdb database + try: + self.con = psycopg2.connect(host=db_host, port=db_port, database=db_database, user=db_user, + password=db_password) + self.cur = self.con.cursor() # open a cursor to perform database operations + print("{} connects to the database name {} successfully".format(agent_id, db_database)) + except: + print("ERROR: {} fails to connect to the database name {}".format(agent_id, db_database)) + exit(1) + + self.lastSensewareFetch = None + self.lastRemoteUpdate = None + self.dbUpdateCounter = 0 def setup(self): super(Agent, self).setup() InfluxDB.makeConnection() - startTime=str(datetime.datetime.utcnow()-datetime.timedelta(hours = 24)) try: - InfluxDB.getSenseware(self.site, measurement = 'Temperature', startTime=startTime) - self.lastSensewareUpdate = str(datetime.datetime.utcnow()) - print "Senseware data retrieved on startup" - except Exception as er: - print er - print "Influx agent failed to retrieve senseware data" + self.cur.execute("SELECT * FROM "+db_table_influx+" WHERE building_name=%s", + (self.site)) + if self.cur.rowcount != 0: + influx_info = self.cur.fetchone() + self.lastSensewareFetch = influx_info[1] + self.lastRemoteUpdate = influx_info[2] + else: + print("Building Name: {} not found in database table {}, Adding.".format(site, db_table_influx)) + self.cur.execute("INSERT INTO " + db_table_influx + " VALUES(%s,%s,%s)", + (self.site, None, None)) + self.con.commit() + except: + print("ERROR: {} failed to get data from the database name {}".format(agent_id, db_database)) + + if ((self.lastSensewareFetch is None) and (self.lastRemoteUpdate is None)): + fetch_startTime=str(datetime.datetime.utcnow()-datetime.timedelta(day = 30)) + try: + InfluxDB.getSenseware(self.site, measurement = 'Temperature', startTime=fetch_startTime) + InfluxDB.remote_to_local(self.site, measurement = 'Temperature', startTime=fetch_startTime) + self.lastSensewareFetch = str(datetime.datetime.utcnow()) + self.lastRemoteUpdate = str(datetime.datetime.utcnow()) + print "Senseware data retrieved on startup" + except Exception as er: + print er + print "Influx agent failed to retrieve senseware data" + else: + try: + InfluxDB.getSenseware(self.site, measurement = 'Temperature', startTime=self.lastSensewareFetch) + self.lastSensewareFetch = str(datetime.datetime.utcnow()) + print "Senseware data retrieved on startup" + except Exception as er: + print er + print "Influx agent failed to retrieve senseware data" + try: + InfluxDB.local_to_remote(startTime=self.lastRemoteUpdate) + self.lastRemoteUpdate = str(datetime.datetime.utcnow()) + print "Influx agent successfully updated remote" + except Exception as er: + print er + print "Influx agent failed to update remote, last update was at {0}".format(self.lastRemoteUpdate) try: - InfluxDB.local_to_remote() - self.lastRemoteUpdate = str(datetime.datetime.utcnow()) - print "Influx agent successfully updated remote" - except Exception as er: - print er - print "Influx agent failed to update remote, last update was {0}".format(self.lastRemoteUpdate) + self.cur.execute("UPDATE " + db_table_influx + " SET(last_retrieved_time_from_remote," + "last_update_remote_time)=(%s,%s) WHERE building_name=%s", + (self.lastSensewareFetch, self.lastRemoteUpdate, self.site)) + self.con.commit() + except: + print("ERROR: {} failed to push data into the database name {}".format(agent_id, db_database)) @periodic(db_scan_time) def updateDatabases(self): self.dbUpdateCounter += 1 try: - InfluxDB.getSenseware(self.site, measurement = 'Temperature', startTime=self.lastSensewareUpdate) - self.lastSensewareUpdate = datetime.datetime.utcnow() + InfluxDB.getSenseware(self.site, measurement = 'Temperature', startTime=self.lastSensewareFetch) + self.lastSensewareFetch = str(datetime.datetime.utcnow()) + try: + self.cur.execute("UPDATE " + db_table_influx + " SET last_retrieved_time_from_remote=%s WHERE " + "building_name=%s", (self.lastSensewareFetch, self.site)) + self.con.commit() + except: + print("ERROR: {} failed to update data into the database name {}".format(agent_id, db_database)) except Exception as er: print er - print "Influx agent failed to retrieve senseware data, last update was {0}".format(self.lastSensewareUpdate) + print "Influx agent failed to retrieve senseware data, last update was {0}".format(self.lastSensewareFetch) - if dbUpdateCounter >= 48: + if self.dbUpdateCounter >= (db_backup_time/db_scan_time): try: InfluxDB.local_to_remote(startTime = self.lastRemoteUpdate) self.lastRemoteUpdate = str(datetime.datetime.utcnow()) - dbUpdateCounter = 0 + self.dbUpdateCounter = 0 + try: + self.cur.execute("UPDATE " + db_table_influx + " SET last_update_remote_time=%s WHERE " + "building_name=%s", (self.lastRemoteUpdate, self.site)) + self.con.commit() + except: + print("ERROR: {} failed to update data into the database name {}".format(agent_id, db_database)) except Exception as er: print er print "Influx agent failed to update remote, last update was {0}".format(self.lastRemoteUpdate) - Agent.__name__ = 'InfluxAgent' return Agent(**kwargs) diff --git a/Agents/InfluxAgent/influxagent.launch.json b/Agents/InfluxAgent/influxagent.launch.json index d546158..946b2a9 100755 --- a/Agents/InfluxAgent/influxagent.launch.json +++ b/Agents/InfluxAgent/influxagent.launch.json @@ -1,5 +1,5 @@ { "agent_id": "InfluxAgent", - "db_scan_time":1800, - "site":"106_E_30th_St__Typhin_" + "db_scan_time": 1800, + "db_backup_time": 86400 } diff --git a/bemoss_lib/databases/create_influx_table_in_bemossdb.sql b/bemoss_lib/databases/create_influx_table_in_bemossdb.sql new file mode 100644 index 0000000..7b9300b --- /dev/null +++ b/bemoss_lib/databases/create_influx_table_in_bemossdb.sql @@ -0,0 +1,12 @@ +CREATE TABLE public.influx_info +( + building_name character varying(100) NOT NULL, + last_retrieved_time_from_remote timestamp with time zone, + last_update_remote_time timestamp with time zone, + CONSTRAINT influx_info_pkey PRIMARY KEY (building_name) +) +WITH ( + OIDS=FALSE +); +ALTER TABLE public.influx_info + OWNER TO admin; \ No newline at end of file diff --git a/bemoss_lib/databases/influxAPI/InfluxDB.py b/bemoss_lib/databases/influxAPI/InfluxDB.py index 890e3b7..4efc270 100755 --- a/bemoss_lib/databases/influxAPI/InfluxDB.py +++ b/bemoss_lib/databases/influxAPI/InfluxDB.py @@ -218,7 +218,7 @@ def local_to_remote(local='localhost', remote=remote, startTime=None, endTime=No :param endTime: the time in localtime zone (the timezone of the node), in datetime.datetime.utcnow format.It marks the end of the range. If not supplied, will be taken as the currentTime. :return: 0, if successful - """ + """ global connection_established if not connection_established: makeConnection() @@ -296,6 +296,9 @@ def getSenseware(site = '106_E_30th_St__Typhin_', measurement = 'Temperature', l range. If not supplied, will be taken as the currentTime. :return: 0, if successful """ + global connection_established + if not connection_established: + makeConnection() localclient = InfluxDBClient(host, port, user, password, dbname) remoteclient = InfluxDBClient(remote, port, user, password, sensewaredb, ssl = True) startTime=None diff --git a/settings.py b/settings.py index a38a3aa..1b53982 100755 --- a/settings.py +++ b/settings.py @@ -117,7 +117,8 @@ DATABASES = { 'TABLE_temp_time_counter': 'temp_time_counter', 'TABLE_temp_failure_time': 'temp_failure_time', 'TABLE_priority': 'priority', - 'TABLE_seen_notifications_counter': 'seen_notifications_counter' + 'TABLE_seen_notifications_counter': 'seen_notifications_counter', + 'TABLE_influx_info': 'influx_info' } } -- GitLab From cf4e82ce21ecddc25043472684c6dbdce4a0e875 Mon Sep 17 00:00:00 2001 From: mchlburton Date: Tue, 2 May 2017 13:59:28 -0400 Subject: [PATCH 19/31] Large queries now take chunked results --- bemoss_lib/databases/influxAPI/InfluxDB.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/bemoss_lib/databases/influxAPI/InfluxDB.py b/bemoss_lib/databases/influxAPI/InfluxDB.py index 890e3b7..4852c64 100755 --- a/bemoss_lib/databases/influxAPI/InfluxDB.py +++ b/bemoss_lib/databases/influxAPI/InfluxDB.py @@ -18,6 +18,7 @@ import datetime from influxdb import InfluxDBClient connection_established = False +#TODO Take global variables from config file created on initialization #Global variables remote = '52.206.6.10' host = 'localhost' @@ -28,6 +29,7 @@ dbname = 'bemoss' site = '106_E_30th_St__Typhin_' +#TODO make this possible for local or remote database by making each client a global object def makeConnection(database = dbname): try: global host, port, user, password, dbname, connection_established @@ -198,7 +200,7 @@ def retrieve(device_model, agentID, vars=None, startTime=None, endTime=None,expo varStr = varStr[:-2] #to get rid of the last ', ' - result = client.query("select {0} from {1} WHERE agent = '{2}' AND time >= '{3}' AND time <= '{4}';".format(varStr, device_model, agentID, startTime,endTime)) + result = client.query("select {0} from {1} WHERE agent = '{2}' AND time >= '{3}' AND time <= '{4}';".format(varStr, device_model, agentID, startTime,endTime), chunked=True, chunk_size=100) values = [] data = result.get_points() @@ -239,7 +241,7 @@ def local_to_remote(local='localhost', remote=remote, startTime=None, endTime=No for result in measurements: for measurement in result: - measure_query = localclient.query("SELECT * FROM {0} WHERE time >= '{1}' AND time <= '{2}';".format(measurement['name'], startTime, endTime)) + measure_query = localclient.query("SELECT * FROM {0} WHERE time >= '{1}' AND time <= '{2}';".format(measurement['name'], startTime, endTime), chunked=True, chunk_size=100) data = measure_query.get_points('{0}'.format(measurement['name'])) field_query = localclient.query("show field keys from {0}".format(measurement['name'])) @@ -306,7 +308,8 @@ def getSenseware(site = '106_E_30th_St__Typhin_', measurement = 'Temperature', l if endTime==None: endTime = str(datetime.datetime.utcnow()) - measure_query = remoteclient.query("SELECT * FROM {0} WHERE site='{1}' AND time>='{2}' AND time<='{3}';".format(measurement, site, startTime, endTime)) + measure_query = remoteclient.query("SELECT * FROM {0} WHERE site='{1}' AND time>='{2}' AND time<='{3}';".format(measurement, site, startTime, endTime), chunked=True, chunk_size=100) + print measure_query data = measure_query.get_points('{0}'.format(measurement)) field_query = remoteclient.query("show field keys from {0}".format(measurement)) @@ -343,7 +346,7 @@ def getSenseware(site = '106_E_30th_St__Typhin_', measurement = 'Temperature', l influx_json.extend(json_body) try: - response = localclient.write_points(influx_json) + response = localclient.write_points(influx_json, batch_size=100) print "Senseware Data for {0} retrieved from server".format(measurement) return 0 except Exception as er: -- GitLab From b59d3cd2557394f3191c919ad14ee41bdd18f9cc Mon Sep 17 00:00:00 2001 From: mchlburton Date: Wed, 3 May 2017 12:24:37 -0400 Subject: [PATCH 20/31] Working InfluxAPI with initialize; INFLUX added to settings --- Agents/InfluxAgent/influx/agent.py | 7 +- bemoss_lib/databases/influxAPI/InfluxDB.py | 220 +++++++++++-------- bemoss_lib/databases/influxAPI/initialize.py | 61 +++++ runBEMOSS.sh | 1 + settings.py | 9 +- 5 files changed, 205 insertions(+), 93 deletions(-) create mode 100755 bemoss_lib/databases/influxAPI/initialize.py diff --git a/Agents/InfluxAgent/influx/agent.py b/Agents/InfluxAgent/influx/agent.py index 3ca574d..d1c20f6 100755 --- a/Agents/InfluxAgent/influx/agent.py +++ b/Agents/InfluxAgent/influx/agent.py @@ -61,7 +61,8 @@ def InfluxAgent(config_path, **kwargs): except: print("ERROR: {} fails to connect to the database name {}".format(agent_id, db_database)) exit(1) - + + self.site = site self.lastSensewareFetch = None self.lastRemoteUpdate = None self.dbUpdateCounter = 0 @@ -85,10 +86,10 @@ def InfluxAgent(config_path, **kwargs): print("ERROR: {} failed to get data from the database name {}".format(agent_id, db_database)) if ((self.lastSensewareFetch is None) and (self.lastRemoteUpdate is None)): - fetch_startTime=str(datetime.datetime.utcnow()-datetime.timedelta(day = 30)) + fetch_startTime=str(datetime.datetime.utcnow()-datetime.timedelta(days = 30)) try: InfluxDB.getSenseware(self.site, measurement = 'Temperature', startTime=fetch_startTime) - InfluxDB.remote_to_local(self.site, measurement = 'Temperature', startTime=fetch_startTime) + InfluxDB.remote_to_local(self.site, startTime=fetch_startTime) self.lastSensewareFetch = str(datetime.datetime.utcnow()) self.lastRemoteUpdate = str(datetime.datetime.utcnow()) print "Senseware data retrieved on startup" diff --git a/bemoss_lib/databases/influxAPI/InfluxDB.py b/bemoss_lib/databases/influxAPI/InfluxDB.py index df4242f..a770bf8 100755 --- a/bemoss_lib/databases/influxAPI/InfluxDB.py +++ b/bemoss_lib/databases/influxAPI/InfluxDB.py @@ -15,35 +15,33 @@ All rights reserved. import numpy import datetime +import settings from influxdb import InfluxDBClient connection_established = False -#TODO Take global variables from config file created on initialization #Global variables -remote = '52.206.6.10' host = 'localhost' -port = 8086 -user = 'engineering' -password = 'nPEc9Pz0iV' -dbname = 'bemoss' -site = '106_E_30th_St__Typhin_' - - -#TODO make this possible for local or remote database by making each client a global object -def makeConnection(database = dbname): +building_name = settings.PLATFORM['node']['building_name'] +remote = settings.INFLUX['remote_address'] +port = settings.INFLUX['port'] +user = settings.INFLUX['db_username'] +password = settings.INFLUX['db_password'] + +#TODO make this function test the connection in a more sensible way +def makeConnection(database = building_name): try: - global host, port, user, password, dbname, connection_established - client = InfluxDBClient(host, port, user, password, dbname) + global host, port, user, password, building_name, connection_established + client = InfluxDBClient(host, port, user, password, building_name) database_list = client.get_list_database() for db in database_list: - if dbname in db['name']: + if building_name in db['name']: connection_established = True return True except Exception as er: print 'Cannot establish connection, trying to create database' - client.create_database(dbname) + client.create_database(building_name) raise er try: @@ -67,7 +65,7 @@ def insert(device_model, agentID, all_vars, log_vars, cur_time=None): if not connection_established: makeConnection() - client = InfluxDBClient(host, port, user, password, dbname) + client = InfluxDBClient(host, port, user, password, building_name) if cur_time == None: cur_time = datetime.datetime.utcnow() @@ -114,7 +112,7 @@ def delete(device_model, agentID, startTime=None, endTime=None): if not connection_established: makeConnection() - client = InfluxDBClient(host, port, user, password, dbname) + client = InfluxDBClient(host, port, user, password, building_name) if endTime==None and startTime==None: try: @@ -132,7 +130,7 @@ def delete(device_model, agentID, startTime=None, endTime=None): endTime = str(datetime.datetime.utcnow()) try: - result = client.query("SHOW MEASUREMENTS ON {0} WHERE agent = '{1}';".format(dbname, agentID)) + result = client.query("SHOW MEASUREMENTS ON {0} WHERE agent = '{1}';".format(building_name, agentID)) except: connection_established = False #Try to establish connection again next time raise @@ -175,7 +173,7 @@ def retrieve(device_model, agentID, vars=None, startTime=None, endTime=None,expo if not connection_established: makeConnection() - client = InfluxDBClient(host, port, user, password, dbname) + client = InfluxDBClient(host, port, user, password, building_name) if startTime==None: startTime=str(datetime.datetime.utcnow()-datetime.timedelta(hours=1)) @@ -186,7 +184,7 @@ def retrieve(device_model, agentID, vars=None, startTime=None, endTime=None,expo if vars==None: vars=[] try: - vars_query = client.query("show field keys on {0} from {1};".format(dbname, device_model)) + vars_query = client.query("show field keys on {0} from {1};".format(building_name, device_model)) for data in vars_query: vars = [varlist['fieldKey'] for varlist in data] @@ -211,7 +209,7 @@ def retrieve(device_model, agentID, vars=None, startTime=None, endTime=None,expo return vars, total_result -def local_to_remote(local='localhost', remote=remote, startTime=None, endTime=None): +def local_to_remote(buiding_name=building_name, local='localhost', remote=remote, startTime=None, endTime=None): """Function to retrieve Data from local Influx and push it to remote server. :param local: Database you want to copy data from :param remote: Database you want data copied to @@ -225,69 +223,42 @@ def local_to_remote(local='localhost', remote=remote, startTime=None, endTime=No if not connection_established: makeConnection() - localclient = InfluxDBClient(host, port, user, password, dbname) - remoteclient = InfluxDBClient(remote, port, user, password, site, ssl=True) - - startTime=None - endTime=None - - if startTime==None: - startTime=str(datetime.datetime.utcnow()-datetime.timedelta(hours=25)) - - if endTime==None: - endTime = str(datetime.datetime.utcnow()) - - measurements = localclient.query("show measurements;") - - for result in measurements: - for measurement in result: - measure_query = localclient.query("SELECT * FROM {0} WHERE time >= '{1}' AND time <= '{2}';".format(measurement['name'], startTime, endTime), chunked=True, chunk_size=100) - data = measure_query.get_points('{0}'.format(measurement['name'])) - - field_query = localclient.query("show field keys from {0}".format(measurement['name'])) - for field in field_query: - fields = [varlist['fieldKey'] for varlist in field] - tag_query = localclient.query("show tag keys from {0}".format(measurement['name'])) - for tag in tag_query: - tags = [varlist['tagKey'] for varlist in tag] - - influx_json = [] - - for entry in data: - json_field = {} - for field in fields: - reading = entry[field] - json_field.update({field:reading}) - json_tag = {} - for tag in tags: - if entry[tag] is not None: - tag_info = entry[tag] - json_tag.update({tag:tag_info}) + localclient = InfluxDBClient(local, port, user, password, building_name) + remoteclient = InfluxDBClient(remote, port, user, password, building_name, ssl=True) + try: + copy_database(localclient, remoteclient, startTime, endTime) + except Exception as er: + print er + raise - json_time = entry['time'] + return 0 - json_body =[ - { - "measurement": measurement['name'], - "tags": json_tag, - "time": json_time, - "fields": json_field - } - ] +def remote_to_local(building_name=building_name, remote=remote, local='localhost', startTime=None, endTime=None): + """Function to retrieve Data from Influx server and push it to local bemoss. + :param remote: Address of the remote server + :param local: Address of local bemoss, usually just 'localhost' + :param startTime: the time in localtime zone (the timezone of the node), in datetime.datetime.utcnow format. It marks the + beginning for the range. If not supplied, will be taken 25-hours before endTime + :param endTime: the time in localtime zone (the timezone of the node), in datetime.datetime.utcnow format.It marks the end of the + range. If not supplied, will be taken as the currentTime. + :return: 0, if successful + """ + global connection_established + if not connection_established: + makeConnection() - influx_json.extend(json_body) + localclient = InfluxDBClient(local, port, user, password, building_name) + remoteclient = InfluxDBClient(remote, port, user, password, building_name, ssl=True) + try: + copy_database(remoteclient, localclient, startTime, endTime) + except Exception as er: + print er + raise - try: - response = remoteclient.write_points(influx_json) - print "Data for {0} pushed to server".format(measurement['name']) - return 0 - except Exception as er: - print er - print "Data push for {0} failed".format(measurement['name']) - raise + return 0 -def getSenseware(site = '106_E_30th_St__Typhin_', measurement = 'Temperature', local='localhost', remote=remote, sensewaredb = 'Senseware', startTime=None, endTime=None): - """Function to retrieve Senseware data frmo server. +def getSenseware(site = building_name, measurement = 'Temperature', local='localhost', remote=remote, sensewaredb = 'Senseware', startTime=None, endTime=None): + """Function to retrieve Senseware data from server. :param site: Building for BEMOSS instance corresponding to senseawre site tag :param measurement: The variable you want the data for (temperature, humidity, etc.) :param local: Database you want to copy data from @@ -301,23 +272,24 @@ def getSenseware(site = '106_E_30th_St__Typhin_', measurement = 'Temperature', l global connection_established if not connection_established: makeConnection() - localclient = InfluxDBClient(host, port, user, password, dbname) + localclient = InfluxDBClient(host, port, user, password, building_name) remoteclient = InfluxDBClient(remote, port, user, password, sensewaredb, ssl = True) - startTime=None - endTime=None + if startTime==None: - startTime=str(datetime.datetime.utcnow()-datetime.timedelta(minutes = 30)) + startTime=str(datetime.datetime.utcnow()-datetime.timedelta(days = 30)) if endTime==None: endTime = str(datetime.datetime.utcnow()) measure_query = remoteclient.query("SELECT * FROM {0} WHERE site='{1}' AND time>='{2}' AND time<='{3}';".format(measurement, site, startTime, endTime), chunked=True, chunk_size=100) - print measure_query data = measure_query.get_points('{0}'.format(measurement)) field_query = remoteclient.query("show field keys from {0}".format(measurement)) + for field in field_query: - fields = [varlist['fieldKey'] for varlist in field] + fieldlist = [varlist['fieldKey'] for varlist in field] + typelist = [varlist['fieldType'] for varlist in field] + fields = {fieldKey:fieldType for fieldKey, fieldType in zip(fieldlist, typelist)} tag_query = remoteclient.query("show tag keys from {0}".format(measurement)) for tag in tag_query: tags = [varlist['tagKey'] for varlist in tag] @@ -326,9 +298,21 @@ def getSenseware(site = '106_E_30th_St__Typhin_', measurement = 'Temperature', l for entry in data: json_field = {} - for field in fields: - reading = entry[field] - json_field.update({field:reading}) + for field in fields.keys(): + data_type = eval(fields[field]) + if isinstance(entry[field], data_type): + reading = entry[field] + json_field.update({field:reading}) + elif data_type==float and isinstance(entry[field], int): + float(entry[field]) + print("fixed") + entry[field]+=0.0000000001 + reading = entry[field] + json_field.update({field:reading}) + else: + continue + + json_tag = {} for tag in tags: if entry[tag] is not None: @@ -349,10 +333,68 @@ def getSenseware(site = '106_E_30th_St__Typhin_', measurement = 'Temperature', l influx_json.extend(json_body) try: - response = localclient.write_points(influx_json, batch_size=100) + response = localclient.write_points(influx_json, batch_size=1000) print "Senseware Data for {0} retrieved from server".format(measurement) return 0 except Exception as er: print er print "Senseware Data retrieve for {0} failed".format(measurement) raise + +def copy_database(source, destination, startTime=None, endTime=None): + if startTime==None: + startTime=str(datetime.datetime.utcnow()-datetime.timedelta(hours=25)) + + if endTime==None: + endTime = str(datetime.datetime.utcnow()) + + measurements = source.query("show measurements;") + + for result in measurements: + for measurement in result: + measure_query = source.query("SELECT * FROM {0} WHERE time >= '{1}' AND time <= '{2}';".format(measurement['name'], startTime, endTime), chunked=True, chunk_size=100) + data = measure_query.get_points('{0}'.format(measurement['name'])) + + field_query = source.query("show field keys from {0}".format(measurement['name'])) + for field in field_query: + fields = [varlist['fieldKey'] for varlist in field] + tag_query = source.query("show tag keys from {0}".format(measurement['name'])) + for tag in tag_query: + tags = [varlist['tagKey'] for varlist in tag] + + influx_json = [] + + for entry in data: + json_field = {} + for field in fields: + reading = entry[field] + json_field.update({field:reading}) + json_tag = {} + for tag in tags: + if entry[tag] is not None: + tag_info = entry[tag] + json_tag.update({tag:tag_info}) + + json_time = entry['time'] + + json_body =[ + { + "measurement": measurement['name'], + "tags": json_tag, + "time": json_time, + "fields": json_field + } + ] + + influx_json.extend(json_body) + + try: + response = destination.write_points(influx_json, batch_size=1000) + print "Data for {0} pushed to server".format(measurement['name']) + except Exception as er: + print er + print "Data push for {0} failed".format(measurement['name']) + raise + return 0 + +getSenseware(site = building_name, measurement = 'Temperature', local='localhost', remote=remote, sensewaredb = 'Senseware', startTime=None, endTime=None) \ No newline at end of file diff --git a/bemoss_lib/databases/influxAPI/initialize.py b/bemoss_lib/databases/influxAPI/initialize.py new file mode 100755 index 0000000..6758ee8 --- /dev/null +++ b/bemoss_lib/databases/influxAPI/initialize.py @@ -0,0 +1,61 @@ +import os +import re +import sys +sys.path.insert(0,os.path.expanduser("~")+"/workspace/bemoss_os/") +from bemoss_lib.utils.find_own_ip import getIPs +from influxdb import InfluxDBClient +from bemoss_lib.databases.influxAPI import InfluxDB +import datetime + + +def init(): + host = 'localhost' + building_name = settings.PLATFORM['node']['building_name'] + remote_address = settings.INFLUX['remote_address'] + port = settings.INFLUX['port'] + db_username = settings.INFLUX['db_username'] + db_password = settings.INFLUX['db_password'] + + local = InfluxDBClient(host, port, db_username, db_password, building_name) + remote = InfluxDBClient(remote_address, port ,db_username, db_password, building_name, ssl=True) + building_exists = False + database_list = remote.get_list_database() + for db in database_list: + if building_name in db['name']: + building_exists = True + if not building_exists: + print("Building {0} not registered in remote, creating building database".format(building_name)) + try: + remote.create_database(building_name) + except Exception as er: + print er + print("Creation of database for {0} failed".format(building_name)) + raise + existing_data = False + database_list = local.get_list_database() + for db in database_list: + if building_name in db['name']: + existing_data = True + + if existing_data: + try: + print("Pushing existing data to remote") + InfluxDB.local_to_remote('localhost', remote_address) + except Exception as er: + print er + print("Existing data on InfluxDB could not be copied to remote") + raise + + print("Dropping local database") + local.drop_database(building_name) + print("Creating local database") + local.create_database(building_name) + + print("Creating retention policy") + local.create_retention_policy(building_name, "30d", 3, building_name, True) + + print("Local database has been setup") + + +if __name__ == '__main__': + init() \ No newline at end of file diff --git a/runBEMOSS.sh b/runBEMOSS.sh index 601c598..6767010 100755 --- a/runBEMOSS.sh +++ b/runBEMOSS.sh @@ -55,6 +55,7 @@ then echo "Retaining previous state of BEMOSS..." else echo "Performing fresh restart of BEMOSS..." + sudo python ~/workspace/bemoss_os/bemoss_lib/databases/influxAPI/initialize.py sudo python ~/workspace/bemoss_os/bemoss_lib/utils/platform_initiator.py sleep 2 fi diff --git a/settings.py b/settings.py index 1b53982..5ca337b 100755 --- a/settings.py +++ b/settings.py @@ -69,7 +69,7 @@ PLATFORM = { 'name': 'BEMOSS core', 'type': 'core', 'model': 'Odroid3', - 'building_name': 'bemoss', + 'building_name': '106_E_30th_St__Typhin_', 'node_monitor_time': 60, 'node_offline_timeout': 0, 'main_core': 'BEMOSS core' @@ -122,6 +122,13 @@ DATABASES = { } } +INFLUX = { + 'remote_address': '52.206.6.10', + 'port': 8086, + 'db_username': 'engineering', + 'db_password': 'nPEc9Pz0iV' +} + NOTIFICATION = { 'heartbeat': 24*60, # heartbeat period to resend a message 'heartbeat_device_tampering': 130, # heartbeat period to resend a message -- GitLab From 9afc8c0f379b3768c6df628745e552ee06938698 Mon Sep 17 00:00:00 2001 From: mchlburton Date: Wed, 3 May 2017 14:13:15 -0400 Subject: [PATCH 21/31] removed unnecessary print and test function --- bemoss_lib/databases/influxAPI/InfluxDB.py | 3 --- bemoss_lib/databases/influxAPI/initialize.py | 1 + 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/bemoss_lib/databases/influxAPI/InfluxDB.py b/bemoss_lib/databases/influxAPI/InfluxDB.py index a770bf8..5d076f8 100755 --- a/bemoss_lib/databases/influxAPI/InfluxDB.py +++ b/bemoss_lib/databases/influxAPI/InfluxDB.py @@ -305,7 +305,6 @@ def getSenseware(site = building_name, measurement = 'Temperature', local='local json_field.update({field:reading}) elif data_type==float and isinstance(entry[field], int): float(entry[field]) - print("fixed") entry[field]+=0.0000000001 reading = entry[field] json_field.update({field:reading}) @@ -396,5 +395,3 @@ def copy_database(source, destination, startTime=None, endTime=None): print "Data push for {0} failed".format(measurement['name']) raise return 0 - -getSenseware(site = building_name, measurement = 'Temperature', local='localhost', remote=remote, sensewaredb = 'Senseware', startTime=None, endTime=None) \ No newline at end of file diff --git a/bemoss_lib/databases/influxAPI/initialize.py b/bemoss_lib/databases/influxAPI/initialize.py index 6758ee8..11aea41 100755 --- a/bemoss_lib/databases/influxAPI/initialize.py +++ b/bemoss_lib/databases/influxAPI/initialize.py @@ -6,6 +6,7 @@ from bemoss_lib.utils.find_own_ip import getIPs from influxdb import InfluxDBClient from bemoss_lib.databases.influxAPI import InfluxDB import datetime +import settings def init(): -- GitLab From 234d0a424ceefcdb77b60ffe9f7f9557b0c3b031 Mon Sep 17 00:00:00 2001 From: mchlburton Date: Wed, 3 May 2017 14:21:21 -0400 Subject: [PATCH 22/31] changed function call --- bemoss_lib/databases/influxAPI/initialize.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bemoss_lib/databases/influxAPI/initialize.py b/bemoss_lib/databases/influxAPI/initialize.py index 11aea41..ffa57a0 100755 --- a/bemoss_lib/databases/influxAPI/initialize.py +++ b/bemoss_lib/databases/influxAPI/initialize.py @@ -41,7 +41,7 @@ def init(): if existing_data: try: print("Pushing existing data to remote") - InfluxDB.local_to_remote('localhost', remote_address) + InfluxDB.local_to_remote(building_name, 'localhost', remote_address) except Exception as er: print er print("Existing data on InfluxDB could not be copied to remote") -- GitLab From e8adfb59ba74f1fc967e7c8e931f18e7577452d6 Mon Sep 17 00:00:00 2001 From: mchlburton Date: Wed, 3 May 2017 18:06:40 -0400 Subject: [PATCH 23/31] Initialize takes lastUpdateTime from postgres; fixed float/int issue in copy_database --- bemoss_lib/databases/influxAPI/InfluxDB.py | 20 +++++++++--- bemoss_lib/databases/influxAPI/initialize.py | 33 ++++++++++++++++---- 2 files changed, 43 insertions(+), 10 deletions(-) diff --git a/bemoss_lib/databases/influxAPI/InfluxDB.py b/bemoss_lib/databases/influxAPI/InfluxDB.py index 5d076f8..2b6fb94 100755 --- a/bemoss_lib/databases/influxAPI/InfluxDB.py +++ b/bemoss_lib/databases/influxAPI/InfluxDB.py @@ -356,7 +356,9 @@ def copy_database(source, destination, startTime=None, endTime=None): field_query = source.query("show field keys from {0}".format(measurement['name'])) for field in field_query: - fields = [varlist['fieldKey'] for varlist in field] + fieldlist = [varlist['fieldKey'] for varlist in field] + typelist = [varlist['fieldType'] for varlist in field] + fields = {fieldKey:fieldType for fieldKey, fieldType in zip(fieldlist, typelist)} tag_query = source.query("show tag keys from {0}".format(measurement['name'])) for tag in tag_query: tags = [varlist['tagKey'] for varlist in tag] @@ -365,9 +367,19 @@ def copy_database(source, destination, startTime=None, endTime=None): for entry in data: json_field = {} - for field in fields: - reading = entry[field] - json_field.update({field:reading}) + for field in fields.keys(): + data_type = eval(fields[field]) + if isinstance(entry[field], data_type): + reading = entry[field] + json_field.update({field:reading}) + elif data_type==float and isinstance(entry[field], int): + float(entry[field]) + cast_float = 0.0000000001 + reading = entry[field]+cast_float + json_field.update({field:reading}) + else: + continue + json_tag = {} for tag in tags: if entry[tag] is not None: diff --git a/bemoss_lib/databases/influxAPI/initialize.py b/bemoss_lib/databases/influxAPI/initialize.py index ffa57a0..e8bc002 100755 --- a/bemoss_lib/databases/influxAPI/initialize.py +++ b/bemoss_lib/databases/influxAPI/initialize.py @@ -12,13 +12,21 @@ import settings def init(): host = 'localhost' building_name = settings.PLATFORM['node']['building_name'] + #influx remote_address = settings.INFLUX['remote_address'] port = settings.INFLUX['port'] - db_username = settings.INFLUX['db_username'] - db_password = settings.INFLUX['db_password'] + influx_username = settings.INFLUX['db_username'] + influx_password = settings.INFLUX['db_password'] + #postgres + db_database = settings.DATABASES['default']['NAME'] + db_host = settings.DATABASES['default']['HOST'] + db_port = settings.DATABASES['default']['PORT'] + db_user = settings.DATABASES['default']['USER'] + db_password = settings.DATABASES['default']['PASSWORD'] + db_table_influx = settings.DATABASES['default']['TABLE_influx_info'] - local = InfluxDBClient(host, port, db_username, db_password, building_name) - remote = InfluxDBClient(remote_address, port ,db_username, db_password, building_name, ssl=True) + local = InfluxDBClient(host, port, influx_username, influx_password, building_name) + remote = InfluxDBClient(remote_address, port ,influx_username, influx_password, building_name, ssl=True) building_exists = False database_list = remote.get_list_database() for db in database_list: @@ -39,9 +47,22 @@ def init(): existing_data = True if existing_data: + lastUpdateTime = None try: - print("Pushing existing data to remote") - InfluxDB.local_to_remote(building_name, 'localhost', remote_address) + con = psycopg2.connect(host=db_host, port=db_port, database=db_database, user=db_user, + password=db_password) + cur = con.cursor() # open a cursor to perform database operations + print("{} connects to the database name {} successfully".format(agent_id, db_database)) + + cur.execute("SELECT * FROM "+db_table_influx+" WHERE building_name=%s",(self.site,)) + influx_info = cur.fetchone() + lastRemoteUpdate = influx_info[2] + except: + print("could not connect to database {}".format(database)) + + try: + print("Pushing existing data to remote from {0}".format(lastUpdateTime)) + InfluxDB.local_to_remote(building_name, 'localhost', remote_address, startTime=lastUpdateTime) except Exception as er: print er print("Existing data on InfluxDB could not be copied to remote") -- GitLab From 3b1810b99648ed6a8caf5c219148645f3cda9fc6 Mon Sep 17 00:00:00 2001 From: mchlburton Date: Wed, 3 May 2017 18:12:50 -0400 Subject: [PATCH 24/31] added import psycopg2 and typo fixes --- bemoss_lib/databases/influxAPI/initialize.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/bemoss_lib/databases/influxAPI/initialize.py b/bemoss_lib/databases/influxAPI/initialize.py index e8bc002..1f560d7 100755 --- a/bemoss_lib/databases/influxAPI/initialize.py +++ b/bemoss_lib/databases/influxAPI/initialize.py @@ -7,6 +7,7 @@ from influxdb import InfluxDBClient from bemoss_lib.databases.influxAPI import InfluxDB import datetime import settings +import psycopg2 def init(): @@ -52,13 +53,13 @@ def init(): con = psycopg2.connect(host=db_host, port=db_port, database=db_database, user=db_user, password=db_password) cur = con.cursor() # open a cursor to perform database operations - print("{} connects to the database name {} successfully".format(agent_id, db_database)) - cur.execute("SELECT * FROM "+db_table_influx+" WHERE building_name=%s",(self.site,)) + cur.execute("SELECT * FROM "+db_table_influx+" WHERE building_name=%s",(building_name,)) influx_info = cur.fetchone() lastRemoteUpdate = influx_info[2] - except: - print("could not connect to database {}".format(database)) + except Exception as er: + print er + print("could not connect to database {}".format(db_database)) try: print("Pushing existing data to remote from {0}".format(lastUpdateTime)) -- GitLab From 62d5f57ffad459e1a122e7bea810ce090fcdce72 Mon Sep 17 00:00:00 2001 From: Avijit Saha Date: Thu, 4 May 2017 13:39:58 -0400 Subject: [PATCH 25/31] Bug fixes for InfluxAgent, TRVAgent, classAPI_EnoceanPi --- Agents/InfluxAgent/influx/agent.py | 6 +++--- Agents/TRVAgent/trv/agent.py | 10 +++++----- DeviceAPI/classAPI/classAPI_EnOceanPi.py | 3 +-- 3 files changed, 9 insertions(+), 10 deletions(-) diff --git a/Agents/InfluxAgent/influx/agent.py b/Agents/InfluxAgent/influx/agent.py index d1c20f6..abe3131 100755 --- a/Agents/InfluxAgent/influx/agent.py +++ b/Agents/InfluxAgent/influx/agent.py @@ -71,8 +71,7 @@ def InfluxAgent(config_path, **kwargs): super(Agent, self).setup() InfluxDB.makeConnection() try: - self.cur.execute("SELECT * FROM "+db_table_influx+" WHERE building_name=%s", - (self.site)) + self.cur.execute("SELECT * from "+db_table_influx+" where building_name=%s",(self.site,)) if self.cur.rowcount != 0: influx_info = self.cur.fetchone() self.lastSensewareFetch = influx_info[1] @@ -82,7 +81,8 @@ def InfluxAgent(config_path, **kwargs): self.cur.execute("INSERT INTO " + db_table_influx + " VALUES(%s,%s,%s)", (self.site, None, None)) self.con.commit() - except: + except Exception as Er: + print Er print("ERROR: {} failed to get data from the database name {}".format(agent_id, db_database)) if ((self.lastSensewareFetch is None) and (self.lastRemoteUpdate is None)): diff --git a/Agents/TRVAgent/trv/agent.py b/Agents/TRVAgent/trv/agent.py index c4ad983..3fe4d6b 100755 --- a/Agents/TRVAgent/trv/agent.py +++ b/Agents/TRVAgent/trv/agent.py @@ -44,7 +44,7 @@ def TRVAgent(config_path, **kwargs): # 1. @params agent agent_id = get_config('agent_id') - device_monitor_time = 10 + device_monitor_time = 240 max_monitor_time = int(settings.DEVICES['max_monitor_time']) debug_agent = False log_variables = dict(setpoint='int', temperature='float', window_open='boolean', alert='string', @@ -159,13 +159,13 @@ def TRVAgent(config_path, **kwargs): @periodic(device_monitor_time) def deviceMonitorBehavior(self): - if self.variables['setpoint'] > 100: - self.variables['setpoint'] = 20 + if self.variables['setpoint'] >= 100: + self.variables['setpoint'] = 0 else: - self.variables['setpoint'] += 40 + self.variables['setpoint'] += 20 try: - setDeviceStatusResult = TRV.setDeviceStatus() + setDeviceStatusResult = TRV.setDeviceStatus(self.variables['setpoint']) except Exception as er: print er diff --git a/DeviceAPI/classAPI/classAPI_EnOceanPi.py b/DeviceAPI/classAPI/classAPI_EnOceanPi.py index e6112b6..7b23d42 100755 --- a/DeviceAPI/classAPI/classAPI_EnOceanPi.py +++ b/DeviceAPI/classAPI/classAPI_EnOceanPi.py @@ -59,8 +59,7 @@ class API: def renewConnection(): pass - def setDeviceStatus(self): - setpoint = 100 + def setDeviceStatus(self,setpoint): enocean_address = self.variables.get('address')+':8080/api/v1' set_trv = {"trv_id":self.variables.get('macaddress'), "setpoint":setpoint} headers = {"Authorization_key": 'ABC', "Content-type": 'application/json'} -- GitLab From f5ed246c6a1d1f78175ddb2d567b1765db19d191 Mon Sep 17 00:00:00 2001 From: mchlburton Date: Thu, 4 May 2017 15:15:34 -0400 Subject: [PATCH 26/31] Removing cassandra; default copy_database now 30 days --- Agents/AirQualityAgent/airquality/agent.py | 16 +- Agents/TRVAgent/trv/agent.py | 11 +- bemoss_install_v2.sh | 14 +- bemoss_lib/databases/cassandraAPI/__init__.py | 1 - .../databases/cassandraAPI/cassandraDB.py | 404 ------------------ .../databases/cassandraAPI/cassandraHelper.py | 172 -------- bemoss_lib/databases/cassandraAPI/casstart.sh | 2 - .../databases/cassandraAPI/initialize.py | 185 -------- .../databases/cassandraAPI/startCassandra.py | 75 ---- bemoss_lib/databases/influxAPI/InfluxDB.py | 2 +- bemoss_lib/multi_node/runBEMOSS_node.sh | 6 +- bemoss_lib/multi_node/udpclient.py | 4 - bemoss_lib/multi_node/udpserver.py | 6 +- bemoss_lib/utils/platform_initiator.py | 1 - runBEMOSS.sh | 1 - 15 files changed, 17 insertions(+), 883 deletions(-) delete mode 100755 bemoss_lib/databases/cassandraAPI/__init__.py delete mode 100755 bemoss_lib/databases/cassandraAPI/cassandraDB.py delete mode 100755 bemoss_lib/databases/cassandraAPI/cassandraHelper.py delete mode 100755 bemoss_lib/databases/cassandraAPI/casstart.sh delete mode 100755 bemoss_lib/databases/cassandraAPI/initialize.py delete mode 100755 bemoss_lib/databases/cassandraAPI/startCassandra.py diff --git a/Agents/AirQualityAgent/airquality/agent.py b/Agents/AirQualityAgent/airquality/agent.py index 5817d80..6350699 100755 --- a/Agents/AirQualityAgent/airquality/agent.py +++ b/Agents/AirQualityAgent/airquality/agent.py @@ -23,8 +23,8 @@ import psycopg2 import psycopg2.extras import socket import settings +from bemoss_lib.databases.influxAPI import InfluxDB -from bemoss_lib.databases.cassandraAPI import cassandraDB utils.setup_logging() _log = logging.getLogger(__name__) @@ -149,10 +149,10 @@ def AirQualityAgent(config_path, **kwargs): def backupSaveData(self): try: AirQuality.getDeviceStatus() - cassandraDB.insert(agent_id,AirQuality.variables,log_variables) - print('Every Data Pushed to cassandra') + InfluxDB.insert(device_type,agent_id,AirQuality.variables,log_variables) + print('Every Data Pushed to influx') except Exception as er: - print("ERROR: {} fails to update cassandra database".format(agent_id)) + print("ERROR: {} fails to update influx database".format(agent_id)) print er #3. deviceMonitorBehavior (TickerBehavior) @@ -241,13 +241,13 @@ def AirQualityAgent(config_path, **kwargs): except: print("ERROR: {} fails to update the database name {}".format(agent_id,db_database)) - #step5: update Cassandra (time-series) database + #step5: update Influx (time-series) database try: - cassandraDB.insert(agent_id,self.variables,log_variables) - print('Data Pushed to cassandra') + InfluxDB.insert(device_type,agent_id,self.variables,log_variables) + print('Data Pushed to influx') print "{} success update".format(agent_id) except Exception as er: - print("ERROR: {} fails to update cassandra database".format(agent_id)) + print("ERROR: {} fails to update influx database".format(agent_id)) print er #step6: debug agent knowledge if debug_agent: diff --git a/Agents/TRVAgent/trv/agent.py b/Agents/TRVAgent/trv/agent.py index c4ad983..100de6b 100755 --- a/Agents/TRVAgent/trv/agent.py +++ b/Agents/TRVAgent/trv/agent.py @@ -25,7 +25,6 @@ import psycopg2 import psycopg2.extras import settings import socket -from bemoss_lib.databases.cassandraAPI import cassandraDB from bemoss_lib.databases.influxAPI import InfluxDB from bemoss_lib.utils.catcherror import catcherror @@ -249,15 +248,7 @@ def TRVAgent(config_path, **kwargs): print er print("ERROR: {} failed to update database name {}".format(agent_id, db_database)) - # step4: update cassandra database - try: - cassandraDB.insert(agent_id, self.variables, log_variables) - print('Data Pushed to cassandra') - except Exception as er: - print("ERROR: {} fails to update cassandra database".format(agent_id)) - print er - - # step4: update infulx database + # step4: update influx database try: InfluxDB.insert(device_type, agent_id, self.variables, log_variables) print('Data Pushed to InfluxDB') diff --git a/bemoss_install_v2.sh b/bemoss_install_v2.sh index 6d9fa06..85b563e 100755 --- a/bemoss_install_v2.sh +++ b/bemoss_install_v2.sh @@ -50,19 +50,11 @@ sudo apt-get update sudo apt-get install openjdk-7-jre --assume-yes sudo apt-get install libjna-java --assume-yes # Install Cassandra -cd ~/workspace -wget http://downloads.datastax.com/community/dsc-cassandra-2.1.7-bin.tar.gz -tar -xvzf dsc-cassandra-2.1.7-bin.tar.gz -sudo rm dsc-cassandra-2.1.7-bin.tar.gz -sudo mv dsc-cassandra-2.1.7 cassandra +#TODO add influx setup to install script # Install Dependencies in virtual env. cd ~/workspace/bemoss_os/env . bin/activate pip install -r ~/workspace/bemoss_os/requirements.txt -# Install Cassandra Driver -# (For better performance of Cassandra, the install-option can be removed but might cause installation failure in some boards.) -sudo CASS_DRIVER_NO_CYTHON=1 pip install cassandra-driver -CASS_DRIVER_NO_CYTHON=1 pip install cassandra-driver deactivate #Go to the bemoss_web_ui and run the syncdb command for the database tables (ref: model.py) cd ~/workspace/bemoss_os @@ -70,8 +62,8 @@ sudo python ~/workspace/bemoss_web_ui/manage.py syncdb sudo python ~/workspace/bemoss_web_ui/run/defaultDB.py #Initialize the tables sudo python ~/workspace/bemoss_os/bemoss_lib/utils/platform_initiator.py -# Prompt user for Cassandra Authorization Info -sudo python ~/workspace/bemoss_os/bemoss_lib/databases/cassandraAPI/initialize.py +#TODO setup influx install and have the initialize function run here +#sudo python ~/workspace/bemoss_os/bemoss_lib/databases/InfluxAPI/initialize.py # Fix miscellaneaus issues sudo ~/workspace/bemoss_os/bemoss_lib/utils/increase_open_file_limit.sh rm ~/workspace/bemoss_os/bemoss_lib/utils/increase_open_file_limit.sh diff --git a/bemoss_lib/databases/cassandraAPI/__init__.py b/bemoss_lib/databases/cassandraAPI/__init__.py deleted file mode 100755 index 2ac3bba..0000000 --- a/bemoss_lib/databases/cassandraAPI/__init__.py +++ /dev/null @@ -1 +0,0 @@ -__author__ = 'rajee' diff --git a/bemoss_lib/databases/cassandraAPI/cassandraDB.py b/bemoss_lib/databases/cassandraAPI/cassandraDB.py deleted file mode 100755 index 2da1c29..0000000 --- a/bemoss_lib/databases/cassandraAPI/cassandraDB.py +++ /dev/null @@ -1,404 +0,0 @@ -# -*- coding: utf-8 -*- -''' -Copyright (c) 2016, Virginia Tech -All rights reserved. - -Redistribution and use in source and binary forms, with or without modification, are permitted provided that the - following conditions are met: -1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following -disclaimer. -2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following -disclaimer in the documentation and/or other materials provided with the distribution. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, -INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE -DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR -SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, -WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -The views and conclusions contained in the software and documentation are those of the authors and should not be -interpreted as representing official policies, either expressed or implied, of the FreeBSD Project. - -This material was prepared as an account of work sponsored by an agency of the United States Government. Neither the -United States Government nor the United States Department of Energy, nor Virginia Tech, nor any of their employees, -nor any jurisdiction or organization that has cooperated in the development of these materials, makes any warranty, -express or implied, or assumes any legal liability or responsibility for the accuracy, completeness, or usefulness or -any information, apparatus, product, software, or process disclosed, or represents that its use would not infringe -privately owned rights. - -Reference herein to any specific commercial product, process, or service by trade name, trademark, manufacturer, or -otherwise does not necessarily constitute or imply its endorsement, recommendation, favoring by the United States -Government or any agency thereof, or Virginia Tech - Advanced Research Institute. The views and opinions of authors -expressed herein do not necessarily state or reflect those of the United States Government or any agency thereof. - -VIRGINIA TECH – ADVANCED RESEARCH INSTITUTE -under Contract DE-EE0006352 - -#__author__ = "BEMOSS Team" -#__credits__ = "" -#__version__ = "2.0" -#__maintainer__ = "BEMOSS Team" -#__email__ = "aribemoss@gmail.com" -#__website__ = "www.bemoss.org" -#__created__ = "2014-09-12 12:04:50" -#__lastUpdated__ = "2016-03-14 11:23:33" -''' -import pandas -from cassandra import * -import numpy -import datetime -import re -from bemoss_lib.databases.cassandraAPI import cassandraHelper -from bemoss_lib.utils.catcherror import catcherror -import json -connection_established = False - -#Global variables -bCluster, bSpace, keyspace_name, replication_factor = None, None, None, None - - - -@catcherror('Could not get replication') -def get_replication(keyspace): - - global connection_established - if not connection_established: - makeConnection() - x = bSpace.execute("SELECT strategy_options from system.schema_keyspaces where keyspace_name=%s",(keyspace,)) - y = int(json.loads(x[0][0])['replication_factor']) - return y - - -@catcherror('Could not set replication') -def set_replication(keyspace,replication): - - global connection_established - if not connection_established: - makeConnection() - bSpace.execute("ALTER KEYSPACE %s WITH REPLICATION = { 'class' : 'SimpleStrategy', 'replication_factor' : %d };" % (keyspace,replication)) - - - -def makeConnection(): - try: - global bCluster, bSpace, keyspace_name, replication_factor, connection_established - bCluster, bSpace = cassandraHelper.makeConnection() - keyspace_name,replication_factor = cassandraHelper.findKeyAndRep() - bSpace.set_keyspace(keyspace_name) - if bSpace is not None: - connection_established = True #if some error occurs this would be skipped - return True - except InvalidRequest as er: - try: - bSpace.execute("create keyspace %s with replication={'class':'SimpleStrategy','replication_factor':%s}" % (keyspace_name, replication_factor)) - bSpace.set_keyspace(keyspace_name) - return True - except Exception as er: - print 'bemossspace couldnt be created/switched to' - print er - raise er - - except Exception as er: - print 'Cannot establish connection' - raise er - -try: - makeConnection() -except Exception as er: - print 'Connection cannot be established' - print er - - -def createTable(agentID, variables): - """ - Function to create table for a AgentID - - :param agentID: string. Table with b will be created - :param variables: dictionary (usually APIobject.log_variables). It contains variables to be logged and their datatypes - :return: 0, if successful - - """ - global connection_established - if not connection_established: - makeConnection() - - - varStr = "" - tableName = "B"+agentID - for vars,types in variables.items(): - varStr += vars+" "+types+", " - - - varStr = varStr[:-2] - create_query = "create table {0} (agent_id text, date_id text , time TIMESTAMP, {1}, PRIMARY KEY ((agent_id, date_id), time))".format(tableName,varStr) - try: - bSpace.execute(create_query) - except AlreadyExists as e: - print 'table %s already present' % agentID - raise - except Exception: - connection_established = False #Try to establish connection again next time - raise - - return 0 - - -def insert(agentID, all_vars, log_vars, cur_timeLocal=None): - """ - :param agentID: string. Data will be inserted to table named B - :param all_vars: dictionary (usually APIobject.variables). It contains all the variables and their values. - :param log_vars: dictionary (usually APIobject.log_variables). It contains variables to be logged and their datatypes - :return: 0, if successful - - timestamp is generated based on current utc time. UTC time is put in cassandra database. If the table by the agent - name doesn't exist, table is created. **If error occurs because the name of variables/data_type has changed, the old - table will be deleted, and a new one with currect variable names/datatype will be created**. - - **Need to avoid doing this in final version.Feature made to help during development - """ - global connection_established - - if not connection_established: - makeConnection() - - - x = datetime.datetime.utcnow()-datetime.datetime.now() - sec_offset = int(round(x.seconds+x.microseconds/1000000.0)) - timeDel = datetime.timedelta(seconds=sec_offset) - - if cur_timeLocal == None: - cur_timeLocal = datetime.datetime.now() - - cur_timeUTC = cur_timeLocal+timeDel - date_local = str(cur_timeLocal.date()) - tableName = "B"+agentID - varStr = "agent_id, date_id, time" - placeHolderStr = "%s, %s, %s" - - values = [agentID, date_local, cur_timeUTC] - noVarStr ="" - for var in log_vars: - reading = all_vars.get(var) - if not reading == None: - varStr += ", "+var - placeHolderStr += ", %s" - values.append(reading) - else: - noVarStr+=", "+var - - - insert_query = "insert into {0} ({1}) VALUES ({2})".format(tableName,varStr,placeHolderStr) - retry = True - while retry: - try: - bSpace.execute(insert_query,values) - retry = False - except InvalidRequest as e: - if e.message.find('unconfigured columnfamily')!=-1: - print 'Table not exits. Creating one:' - createTable(agentID,log_vars) - print "Table Created. Now Trying to insert Again:" - bSpace.execute(insert_query,values) - retry = False - elif e.message.lower().find('unknown identifier')!=-1: - k = str(e.message) - newColumn = re.search('[Uu]nknown identifier ([a-zA-Z_]*)',k).groups()[0] - alter_query = "ALTER TABLE {0} ADD {1} {2}".format(tableName,newColumn,log_vars[newColumn]) - bSpace.execute(alter_query) - retry = True - else: - #TO DO: Don't do this. if the table already exists and can't insert data, simply raise exception - #drop_query = "drop table {0}".format(tableName) - #bSpace.execute(drop_query) - #print "Table Dropped. Now Trying to create again" - #createTable(agentID,log_vars) - #print "Created. Now inserting" - #bSpace.execute(insert_query,values) - retry = False - print e - raise - except: - connection_established = False #Try to connect again next-time - raise - - return 0 - - -def delete(agentID,startDate=None, endDate=None): - """ - Performs deletion of data. if statDate and endDate is omitted, whole table is delted. - :param agentID: The B table in which the operation is to be performed - :param startDate: dateTime.date object (local timezone). The begining date from which to delete. Must be supplied - unless trying to delete the whole table - :param endDate: datetime.date object (local timezone). The endDate upto which to delete. The default is to upto today. - must be supplied unless trying to delte the whole table - :return: 0, if successfull - - Delete can be performed with a resolution of a day so, data cannot be partially deleted for a day. If new data needs - to be written to existing place, simply inserting it again with same primary key will override it. - - """ - global connection_established - if not connection_established: - makeConnection() - - tableName = "B"+agentID - - if endDate==None and startDate==None: - try: - delete_query='drop table {0}'.format(tableName) - bSpace.execute(delete_query) - except InvalidRequest as e: - if e.message.find('unconfigured columnfamily')!=-1: - print "Already delted. Don't worry about it" - return 0 - else: - raise - except Exception: - connection_established = False #Try to establish connection again next time - raise - - elif startDate==None: - print "startTime compulsory when endTime is given" - return -1 - elif endDate==None: - endDate=datetime.datetime.now() - - daterange = pandas.date_range(startDate,endDate) - date_local="" - try: - for day in daterange: - date_local = str(day.date()) - delete_query = 'DELETE from {0} WHERE agent_id=%s AND date_id=%s'.format(tableName) - bSpace.execute(delete_query,(agentID,date_local)) - - except Exception as e: - print "sorry, not deleted all. deleted upto:%s" % date_local - connection_established = False - return -1 - - return 0 - - - - - -def retrieve(agentID, vars=None, startTime=None, endTime=None,export=False): - """Function to retrieve Data from the active cassandra cluster. \n - :param agentID: must supply, since each agentID is associated with a table in database. - :param vars: supplied as a list of strings. It represents the variables to be retrieved from the table. - eg. ['time','temperature','heat_setpoint']. If any of the variables don't match the column name, the result - will contain -1. If not supplied, the default is to return the complete row \n\n - :param startTime: the time in localtime zone (the timezone of the node), in datetime.datetime format. It marks the - beginning for the range. If not supplied, will be taken 24-hours before endTime - :param endTime: the time in localtime zone (the timezone of the node), in datetime.datetime format.It marks the end of the - range. If not supplied, will be taken as the currentTime. - :return: A numpy 2-dimensional array. Columns corresponds to variables querried, and rows corresponds to - various table entries.The time is reported in local time (cassandra returns UTC time, conversion is done in - this function). If the query fails, -1 is returned (and no exception raised) - - """ - - global connection_established - if not connection_established: - makeConnection() - - x = datetime.datetime.utcnow()-datetime.datetime.now() - sec_offset = int(round(x.seconds+x.microseconds/1000000.0)) - timeDel = datetime.timedelta(seconds=sec_offset) - - if startTime==None: - startTime = datetime.datetime.now()-datetime.timedelta(hours=24) - startTimeUTC=datetime.datetime.utcnow()-datetime.timedelta(hours=24) - else: - startTimeUTC = startTime + timeDel #convert to UTC - - if endTime==None: - endTime = datetime.datetime.now() - endTimeUTC = datetime.datetime.utcnow() - else: - endTimeUTC = endTime+timeDel #convert to UTC - - tableName = str("B"+agentID).lower() - - if vars==None: - varStr='' - vars=[] - try: - result=bSpace.execute("select column_name from system.schema_columns WHERE keyspace_name=%s and columnfamily_name=%s",[keyspace_name,tableName]) - except: - connection_established = False #Try to establish connection again next time - raise - - for var in result: - varStr += var[0] + ', ' - vars += var - varStr = varStr[:-2] #to get rid of the last ', ' - else: - varStr = '' - for var in vars: - varStr += var + ', ' - - varStr = varStr[:-2] #to get rid of the last ', ' - - daterange = pandas.date_range(startTime,endTime) - total_result = [] - try: - for day in daterange: - - date_local = str(day.date()) - result = bSpace.execute('select {0} from {1} WHERE agent_id=%s AND date_id=%s AND time >= %s AND time <= %s'.format(varStr, tableName),[agentID,date_local,startTimeUTC,endTimeUTC]) - total_result += result - - total_result = numpy.array(total_result) - - # If there is no data, return an empty list [] - if len(total_result) == 0: - return vars, total_result - - if vars is not None: - if export: - #convert the UTC time to local time if time is present - #more robust method would be look at data types of the result and convert them if it has datetime data type - if 'time' in vars: - total_result[:, vars.index('time')]-=timeDel - time_map = total_result[:,vars.index('time')] - total_result[:, vars.index('time')] = map(lambda x: "{}".format(x.strftime('%y-%m-%d %H:%M:%S')), time_map) - if 'cooling_mode' in vars: - total_result[:, vars.index('cooling_mode')] = map(lambda x: x.encode('utf8'), total_result[:, vars.index('cooling_mode')]) - return vars, total_result - else: - if 'time' in vars: - time_map = total_result[:,vars.index('time')] - # unix timestamp (from seconds) to javascript epoch (timestamp in milliseconds) - total_result[:, vars.index('time')] = map(lambda x: int((x-datetime.datetime(1970,1,1)).total_seconds()*1000), time_map) # '"{}"'.format(str([])) - if 'status' in vars: - total_result[:, vars.index('status')] = map(lambda x: 1 if str(x).lower() == 'on' else 0 if str(x).lower() == 'off' else x, total_result[:, vars.index('status')]) - if 'motion' in vars: - total_result[:, vars.index('motion')] = map(lambda x: 1 if x == True else 0 if x == False else x, total_result[:, vars.index('motion')]) - if 'cooling_mode' in vars: - total_result[:, vars.index('cooling_mode')] = map(lambda x: x.encode('utf8') if x is not None else None, total_result[:, vars.index('cooling_mode')]) - return vars, total_result - - except InvalidRequest as e: - if e.message.find('unconfigured columnfamily')!=-1: - total_result = -1 - print ('table not exist') - else: - total_result = -1 - print e - except: - connection_established = False #Try to establish connection again next time - raise - - -def retrieve_for_export(agentID, vars=None, startTime=None, endTime=None): - a,b = retrieve(agentID,vars,startTime,endTime,export=True) - return a,b - -if __name__ == '__main__': - x = get_replication('system_auth') - set_replication('system_auth',2) - print x diff --git a/bemoss_lib/databases/cassandraAPI/cassandraHelper.py b/bemoss_lib/databases/cassandraAPI/cassandraHelper.py deleted file mode 100755 index abdc508..0000000 --- a/bemoss_lib/databases/cassandraAPI/cassandraHelper.py +++ /dev/null @@ -1,172 +0,0 @@ -# -*- coding: utf-8 -*- -''' -Copyright (c) 2016, Virginia Tech -All rights reserved. - -Redistribution and use in source and binary forms, with or without modification, are permitted provided that the - following conditions are met: -1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following -disclaimer. -2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following -disclaimer in the documentation and/or other materials provided with the distribution. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, -INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE -DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR -SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, -WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -The views and conclusions contained in the software and documentation are those of the authors and should not be -interpreted as representing official policies, either expressed or implied, of the FreeBSD Project. - -This material was prepared as an account of work sponsored by an agency of the United States Government. Neither the -United States Government nor the United States Department of Energy, nor Virginia Tech, nor any of their employees, -nor any jurisdiction or organization that has cooperated in the development of these materials, makes any warranty, -express or implied, or assumes any legal liability or responsibility for the accuracy, completeness, or usefulness or -any information, apparatus, product, software, or process disclosed, or represents that its use would not infringe -privately owned rights. - -Reference herein to any specific commercial product, process, or service by trade name, trademark, manufacturer, or -otherwise does not necessarily constitute or imply its endorsement, recommendation, favoring by the United States -Government or any agency thereof, or Virginia Tech - Advanced Research Institute. The views and opinions of authors -expressed herein do not necessarily state or reflect those of the United States Government or any agency thereof. - -VIRGINIA TECH – ADVANCED RESEARCH INSTITUTE -under Contract DE-EE0006352 - -#__author__ = "BEMOSS Team" -#__credits__ = "" -#__version__ = "2.0" -#__maintainer__ = "BEMOSS Team" -#__email__ = "aribemoss@gmail.com" -#__website__ = "www.bemoss.org" -#__created__ = "2014-09-12 12:04:50" -#__lastUpdated__ = "2016-03-14 11:23:33" -''' -import os -import re -from cassandra.cluster import Cluster -from cassandra.io.asyncorereactor import AsyncoreConnection -from cassandra.auth import PlainTextAuthProvider -from bemoss_lib.utils.find_own_ip import getIPs - -def addSeed(newseed): - ''' - Add a seed IP to the cassandra settings file, and update cassandra yaml file accordingly - Will be called by udpclient in a node after a core is discovered - :param newseed: Ip address string to add to seed list - :return: - ''' - try: - casSettingsFile = open(os.path.expanduser("~")+"/workspace/bemoss_os/cassandra_settings.txt",'r+') - except IOError as er: - print 'cassandra settings file missing. Settings file must be present at this point' - settingsContent = casSettingsFile.read() - seeds = re.search('seeds:(.*)\n',settingsContent).group(1) - if newseed not in seeds: - oldseeds = seeds.replace('"','').strip() - seeds = '"%s, %s"'% (newseed,oldseeds) - settingsContent = re.sub('seeds:(.*)\n','seeds: %s\n'%seeds,settingsContent) - casSettingsFile.seek(0) - casSettingsFile.write(settingsContent) - casSettingsFile.truncate() - casYamlFile = open(os.path.expanduser("~")+"/workspace/cassandra/conf/cassandra.yaml",'r+') - yamlContent = casYamlFile.read() - yamlContent = re.sub('seeds:(.*)\n','seeds: %s\n' % seeds,yamlContent) - casYamlFile.seek(0) - casYamlFile.write(yamlContent) - casYamlFile.truncate() - casYamlFile.close() - - casSettingsFile.close() - -def findKeyAndRep(): - try: - casSettingsFile = open(os.path.expanduser("~")+"/workspace/bemoss_os/cassandra_settings.txt",'r') - settingsContent = casSettingsFile.read() - casSettingsFile.close() - keyspace = re.search('keyspace_name: *(.*)\n',settingsContent).group(1) - replication_factor = re.search('replication_factor: *(.*)\n',settingsContent).group(1) - except IOError as er: - print "No cassandra_settings file or bad settings file. Using default bemossspace and factor 1" - keyspace = 'bemossspace' - replication_factor = '1' - - return keyspace, replication_factor - - - -def findIP(): - """ - Reads the listen address from cassandra settings file - :return: - """ - try: - casSettingsFile = open(os.path.expanduser("~")+"/workspace/bemoss_os/cassandra_settings.txt",'r') - settingsContent = casSettingsFile.read() - casSettingsFile.close() - ip_address = re.search('rpc_address: *([0-9\.]*)\n',settingsContent).group(1) - except IOError as er: - print "No cassandra_settings file. Using current IP" - ip_address = getIPs()[-1] - - return ip_address - -def findUserPass(): - try: - casSettingsFile = open(os.path.expanduser("~")+"/workspace/bemoss_os/cassandra_settings.txt",'r') - settingsContent = casSettingsFile.read() - casSettingsFile.close() - username = re.search('db_username: *(.*)\n',settingsContent).group(1) - password = re.search('db_password: *(.*)\n',settingsContent).group(1) - except IOError as er: - print "No cassandra_settings file. Using the default cassandra cassandra userpass" - username='cassandra' - password='cassandra' - - userpass = [username,password] - return userpass - - -def makeConnection(): - ip_address = findIP() - notResolved = True - while notResolved: - notResolved=False - try: - userpass = findUserPass() - ap = PlainTextAuthProvider(username=userpass[0], password=userpass[1]) - bCluster=Cluster([ip_address],connection_class=AsyncoreConnection,auth_provider=ap) - bSpace = bCluster.connect() - except Exception as er: - redFlag = ['AuthenticationFailed','username','password','incorrect'] - test = filter(lambda x: x.lower() in str(er).lower(), redFlag) - if len(test)==len(redFlag): #all redFlags words exists on message - print 'provided username doesnt work. trying default:' - ap = PlainTextAuthProvider(username='cassandra', password='cassandra') - try: - bCluster=Cluster([ip_address],connection_class=AsyncoreConnection,auth_provider=ap) - bSpace=bCluster.connect() - bSpace.execute("ALTER USER cassandra with password 'merogharanuwakotmaparchhatimrokahaparchha'") - except Exception as er: - print er - ap = PlainTextAuthProvider(username='cassandra', password='merogharanuwakotmaparchhatimrokahaparchha') - bCluster=Cluster([ip_address],connection_class=AsyncoreConnection,auth_provider=ap) - bSpace=bCluster.connect() - - bSpace.execute("CREATE USER %s with password '%s' SUPERUSER" % (userpass[0],userpass[1])) - print ('The username and password created. Now trying login again') - bCluster.shutdown() - notResolved=True - else: - raise - - return bCluster, bSpace - - - - -if __name__ == '__main__': - findIP() diff --git a/bemoss_lib/databases/cassandraAPI/casstart.sh b/bemoss_lib/databases/cassandraAPI/casstart.sh deleted file mode 100755 index aaeffd4..0000000 --- a/bemoss_lib/databases/cassandraAPI/casstart.sh +++ /dev/null @@ -1,2 +0,0 @@ -ls -~/workspace/cassandra/bin/cassandra -f 2>&1 | tee ~/workspace/bemoss_os/log/cassandra.log diff --git a/bemoss_lib/databases/cassandraAPI/initialize.py b/bemoss_lib/databases/cassandraAPI/initialize.py deleted file mode 100755 index 856edf9..0000000 --- a/bemoss_lib/databases/cassandraAPI/initialize.py +++ /dev/null @@ -1,185 +0,0 @@ -# -*- coding: utf-8 -*- -''' -Copyright (c) 2016, Virginia Tech -All rights reserved. - -Redistribution and use in source and binary forms, with or without modification, are permitted provided that the - following conditions are met: -1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following -disclaimer. -2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following -disclaimer in the documentation and/or other materials provided with the distribution. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, -INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE -DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR -SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, -WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -The views and conclusions contained in the software and documentation are those of the authors and should not be -interpreted as representing official policies, either expressed or implied, of the FreeBSD Project. - -This material was prepared as an account of work sponsored by an agency of the United States Government. Neither the -United States Government nor the United States Department of Energy, nor Virginia Tech, nor any of their employees, -nor any jurisdiction or organization that has cooperated in the development of these materials, makes any warranty, -express or implied, or assumes any legal liability or responsibility for the accuracy, completeness, or usefulness or -any information, apparatus, product, software, or process disclosed, or represents that its use would not infringe -privately owned rights. - -Reference herein to any specific commercial product, process, or service by trade name, trademark, manufacturer, or -otherwise does not necessarily constitute or imply its endorsement, recommendation, favoring by the United States -Government or any agency thereof, or Virginia Tech - Advanced Research Institute. The views and opinions of authors -expressed herein do not necessarily state or reflect those of the United States Government or any agency thereof. - -VIRGINIA TECH – ADVANCED RESEARCH INSTITUTE -under Contract DE-EE0006352 - -#__author__ = "BEMOSS Team" -#__credits__ = "" -#__version__ = "2.0" -#__maintainer__ = "BEMOSS Team" -#__email__ = "aribemoss@gmail.com" -#__website__ = "www.bemoss.org" -#__created__ = "2014-09-12 12:04:50" -#__lastUpdated__ = "2016-03-14 11:23:33" -''' -import os -import re -import sys -sys.path.insert(0,os.path.expanduser("~")+"/workspace/bemoss_os/") -from bemoss_lib.utils.find_own_ip import getIPs - - -def init(): - try: - casSettingsFile = open(os.path.expanduser("~")+"/workspace/bemoss_os/cassandra_settings.txt",'r') - except IOError as er: - print "creating a new cassandra_settings file" - cluster_name = raw_input("Enter a unique name for cassandra cluster-name: ") - if cluster_name=='': - cluster_name = 'bemosscluster' - print 'Using default clustername: '+cluster_name - db_username = raw_input("Enter database username: ") - if db_username=='': - db_username='bemoss' - print 'Using default username: '+db_username - db_password = raw_input("Enter database password: ") - if db_password=='': - db_password='bemoss' - print 'Using default password: '+db_password - else: - password2 = raw_input("Enter password again: ") - while password2 != db_password: - password2 = raw_input("Password doesn't match. Input again: ") - #TODO have some exit condition - ip_address = getIPs()[-1] - casSettingsFile = open(os.path.expanduser("~")+"/workspace/bemoss_os/cassandra_settings.txt",'w') - contents="""cluster_name: '%s' -keyspace_name: bemossspace -replication_factor: 2 -listen_address: %s -rpc_address: %s -seeds: "%s" -authenticator: PasswordAuthenticator -db_username: %s -db_password: %s -MAX_HEAP_SIZE="400M" -HEAP_NEWSIZE="100M" - """ % (cluster_name,ip_address,ip_address,ip_address,db_username,db_password) - casSettingsFile.write(contents) - casSettingsFile.close() - casSettingsFile = open(os.path.expanduser("~")+"/workspace/bemoss_os/cassandra_settings.txt",'r') - - settingsContent = casSettingsFile.read() - casSettingsFile.close() - try: - casYamlFile = open(os.path.expanduser("~")+"/workspace/cassandra/conf/cassandra.yaml",'r') - yamlContent = casYamlFile.read() - casYamlFile.close() - except IOError as er: - print "Not found:" + os.path.expanduser("~")+"/workspace/cassandra/conf/cassandra.yaml" - raise - - try: - casEnvFile = open(os.path.expanduser("~")+"/workspace/cassandra/conf/cassandra-env.sh",'r') - envContent = casEnvFile.read() - casEnvFile.close() - except IOError as er: - print "Not found:" + os.path.expanduser("~")+"/workspace/cassandra/conf/cassandra.yaml" - raise - - cluster_name = re.search('cluster_name:(.*)\n',settingsContent).group(1) - listen_address = re.search('listen_address:(.*)\n',settingsContent).group(1) - rpc_address = re.search('rpc_address: *([0-9\.]*)\n',settingsContent).group(1) - db_username = re.search('db_username: *(.*)\n',settingsContent).group(1) - db_password = re.search('db_password: *(.*)\n',settingsContent).group(1) - keyspace = re.search('keyspace_name: *(.*)\n',settingsContent).group(1) - replication_factor = re.search('replication_factor: *(.*)\n',settingsContent).group(1) - myips = getIPs() - - seeds = re.search('seeds:(.*)\n',settingsContent).group(1) - authenticator = re.search('authenticator:(.*)\n',settingsContent).group(1) - MAX_HEAP_SIZE = re.search('MAX_HEAP_SIZE=("[a-zA-Z0-9]*")\n',settingsContent).group(1) - HEAP_NEWSIZE = re.search('HEAP_NEWSIZE=("[a-zA-Z0-9]*")\n',settingsContent).group(1) - - #check if any of the seeds IP is current machine IP. At least one seed needs to be self IP - bad_seed = True - for ip in myips: - if ip in seeds: - bad_seed = False - - if bad_seed: - oldseeds = seeds.replace('"','').strip() - seeds = '"%s, %s"'% (myips[-1],oldseeds) - - if listen_address.strip() not in myips or rpc_address.strip() not in myips: - - listen_address = myips[-1] - rpc_address = myips[-1] - - - - casSettingsFile = open(os.path.expanduser("~")+"/workspace/bemoss_os/cassandra_settings.txt",'w') - contents="""cluster_name: %s -keyspace_name: %s -replication_factor: %s -listen_address: %s -rpc_address: %s -seeds: %s -authenticator: %s -db_username: %s -db_password: %s -MAX_HEAP_SIZE=%s -HEAP_NEWSIZE=%s -""" % (cluster_name.strip(),keyspace.strip(),replication_factor.strip(),myips[-1],myips[-1],seeds.strip(),authenticator.strip(),db_username.strip(),db_password.strip(),MAX_HEAP_SIZE.strip(),HEAP_NEWSIZE.strip()) - casSettingsFile.write(contents) - casSettingsFile.close() - - yamlContent = re.sub('cluster_name:(.*)\n','cluster_name: %s\n' % cluster_name,yamlContent) - yamlContent = re.sub('listen_address:(.*)\n','listen_address: %s\n' % listen_address,yamlContent) - yamlContent = re.sub('rpc_address:(.*)\n','rpc_address: %s\n' % rpc_address,yamlContent) - yamlContent = re.sub('authenticator:(.*)\n','authenticator: %s\n' % authenticator,yamlContent) - yamlContent = re.sub('seeds:(.*)\n','seeds: %s\n' % seeds,yamlContent) - envContent = re.sub('#?MAX_HEAP_SIZE=("[a-zA-Z0-9]*")\n','MAX_HEAP_SIZE=%s\n' % MAX_HEAP_SIZE,envContent) - envContent = re.sub('#?HEAP_NEWSIZE=("[a-zA-Z0-9]*")\n','HEAP_NEWSIZE=%s\n'%HEAP_NEWSIZE,envContent) - - try: - casYamlFile = open(os.path.expanduser("~")+"/workspace/cassandra/conf/cassandra.yaml",'w') - casYamlFile.write(yamlContent) - casYamlFile.close() - except IOError as er: - print "Error writing:" + os.path.expanduser("~")+"/workspace/cassandra/conf/cassandra.yaml" - raise - - try: - casEnvFile = open(os.path.expanduser("~")+"/workspace/cassandra/conf/cassandra-env.sh",'w') - casEnvFile.write(envContent) - casEnvFile.close() - except IOError as er: - print "Error writing:" + os.path.expanduser("~")+"/workspace/cassandra/conf/cassandra.yaml" - raise - -if __name__ == '__main__': - init() diff --git a/bemoss_lib/databases/cassandraAPI/startCassandra.py b/bemoss_lib/databases/cassandraAPI/startCassandra.py deleted file mode 100755 index 69b89d9..0000000 --- a/bemoss_lib/databases/cassandraAPI/startCassandra.py +++ /dev/null @@ -1,75 +0,0 @@ -# -*- coding: utf-8 -*- -''' -Copyright (c) 2016, Virginia Tech -All rights reserved. - -Redistribution and use in source and binary forms, with or without modification, are permitted provided that the - following conditions are met: -1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following -disclaimer. -2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following -disclaimer in the documentation and/or other materials provided with the distribution. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, -INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE -DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR -SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, -WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -The views and conclusions contained in the software and documentation are those of the authors and should not be -interpreted as representing official policies, either expressed or implied, of the FreeBSD Project. - -This material was prepared as an account of work sponsored by an agency of the United States Government. Neither the -United States Government nor the United States Department of Energy, nor Virginia Tech, nor any of their employees, -nor any jurisdiction or organization that has cooperated in the development of these materials, makes any warranty, -express or implied, or assumes any legal liability or responsibility for the accuracy, completeness, or usefulness or -any information, apparatus, product, software, or process disclosed, or represents that its use would not infringe -privately owned rights. - -Reference herein to any specific commercial product, process, or service by trade name, trademark, manufacturer, or -otherwise does not necessarily constitute or imply its endorsement, recommendation, favoring by the United States -Government or any agency thereof, or Virginia Tech - Advanced Research Institute. The views and opinions of authors -expressed herein do not necessarily state or reflect those of the United States Government or any agency thereof. - -VIRGINIA TECH – ADVANCED RESEARCH INSTITUTE -under Contract DE-EE0006352 - -#__author__ = "BEMOSS Team" -#__credits__ = "" -#__version__ = "2.0" -#__maintainer__ = "BEMOSS Team" -#__email__ = "aribemoss@gmail.com" -#__website__ = "www.bemoss.org" -#__created__ = "2014-09-12 12:04:50" -#__lastUpdated__ = "2016-03-14 11:23:33" -''' -import os -import time -import cassandraHelper -import initialize - -def start(): - started = False - initialize.init() - x=1 - while x>0: - try: - bCluster, bSpace = cassandraHelper.makeConnection() - x=0 - except Exception as er: - print er - if started == False: - print "Starting cassandra",x - os.system("nohup x-terminal-emulator -e ~/workspace/bemoss_os/bemoss_lib/databases/cassandraAPI/casstart.sh &") - time.sleep(15) - started = True - print 'Waiting for cassandra ...' - - x=1 - time.sleep(5) - print "Cassandra connected" - -if __name__ == '__main__': - start() diff --git a/bemoss_lib/databases/influxAPI/InfluxDB.py b/bemoss_lib/databases/influxAPI/InfluxDB.py index 2b6fb94..1d02b50 100755 --- a/bemoss_lib/databases/influxAPI/InfluxDB.py +++ b/bemoss_lib/databases/influxAPI/InfluxDB.py @@ -342,7 +342,7 @@ def getSenseware(site = building_name, measurement = 'Temperature', local='local def copy_database(source, destination, startTime=None, endTime=None): if startTime==None: - startTime=str(datetime.datetime.utcnow()-datetime.timedelta(hours=25)) + startTime=str(datetime.datetime.utcnow()-datetime.timedelta(days=30)) if endTime==None: endTime = str(datetime.datetime.utcnow()) diff --git a/bemoss_lib/multi_node/runBEMOSS_node.sh b/bemoss_lib/multi_node/runBEMOSS_node.sh index 18728a6..75cf7a1 100755 --- a/bemoss_lib/multi_node/runBEMOSS_node.sh +++ b/bemoss_lib/multi_node/runBEMOSS_node.sh @@ -48,12 +48,10 @@ sudo python ~/workspace/bemoss_os/bemoss_lib/utils/find_own_ip.py #Step1: Run Platform Initiator sudo python ~/workspace/bemoss_os/bemoss_lib/multi_node/platform_initiator_node.py -#Step2: Do initial configuration for cassandra -sudo python ~/workspace/bemoss_os/bemoss_lib/databases/cassandraAPI/initialize.py +#Step2: Do initial configuration for influx +sudo python ~/workspace/bemoss_os/bemoss_lib/databases/InfluxAPI/initialize.py #Step3: Look for bemoss core sudo python ~/workspace/bemoss_os/bemoss_lib/multi_node/udpclient.py -#step4: Start/connect to cassandra -sudo PYTHONPATH='.' python ~/workspace/bemoss_os/bemoss_lib/databases/cassandraAPI/startCassandra.py #Step4: Build agents cd ~/workspace/bemoss_os/bemoss_lib/multi_node/ source buildAgents_node.sh diff --git a/bemoss_lib/multi_node/udpclient.py b/bemoss_lib/multi_node/udpclient.py index 10eb1a5..4d94f02 100755 --- a/bemoss_lib/multi_node/udpclient.py +++ b/bemoss_lib/multi_node/udpclient.py @@ -58,7 +58,6 @@ import datetime import netifaces as ni import time import select -from bemoss_lib.databases.cassandraAPI import cassandraHelper #find broadcast address automatically from script interfaces=ni.interfaces() @@ -145,9 +144,6 @@ while True: print "found_bemoss_core {}".format(found_bemoss_core) print str(addr)+":"+str(recv_data) -#add the ip address of the core as a seed node for the cassandra cluster -cassandraHelper.addSeed(addr[0]) - #add new node to node_info table node_info = json.loads(recv_data) core_name = node_info['node_name'] diff --git a/bemoss_lib/multi_node/udpserver.py b/bemoss_lib/multi_node/udpserver.py index dfa5bab..459965d 100755 --- a/bemoss_lib/multi_node/udpserver.py +++ b/bemoss_lib/multi_node/udpserver.py @@ -55,7 +55,6 @@ import sys os.chdir(os.path.expanduser("~/workspace/bemoss_os/")) # = ~/workspace/bemoss_os current_working_directory = os.getcwd() sys.path.append(current_working_directory) -from bemoss_lib.databases.cassandraAPI import cassandraDB import settings import psycopg2 import datetime @@ -199,7 +198,8 @@ with open(_launch_file, 'w') as outfile: json.dump(data, outfile, indent=4, sort_keys=True) node_count = 1 -current_system_auth_replication = cassandraDB.get_replication('system_auth') +#TODO check how this is affected by cassandra removal +current_system_auth_replication = 1 while True: print("BEMOSS core >> Listening to connection from BEMOSS node: ") recv_data, addr = server_socket.recvfrom(2048) @@ -237,8 +237,6 @@ while True: isNew = insertNodeInfo(remote_node_name,remote_node_type,remote_node_model,remote_node_status,remote_building_name, remote_ip_address,remote_mac_address, remote_associated_zone, remote_date_added,remote_communication,remote_last_scanned_time) node_count += isNew #Add node count only if it is a new node. - if node_count > current_system_auth_replication: - cassandraDB.set_replication('system_auth',node_count) else: #something wrong with database, ignore this node continue diff --git a/bemoss_lib/utils/platform_initiator.py b/bemoss_lib/utils/platform_initiator.py index 3e3e789..1ae1a62 100755 --- a/bemoss_lib/utils/platform_initiator.py +++ b/bemoss_lib/utils/platform_initiator.py @@ -381,7 +381,6 @@ os.system("sudo chmod 777 -R ~/workspace/bemoss_os") #TODO make a backup of log files os.system("sudo rm ~/workspace/bemoss_os/log/volttron.log") -os.system("sudo rm ~/workspace/bemoss_os/log/cassandra.log") os.system("sudo killall volttron") os.system("sudo kill $(cat ~/workspace/bemoss_os/.temp/BEMOSS.pid)") diff --git a/runBEMOSS.sh b/runBEMOSS.sh index 6767010..74d745d 100755 --- a/runBEMOSS.sh +++ b/runBEMOSS.sh @@ -59,7 +59,6 @@ else sudo python ~/workspace/bemoss_os/bemoss_lib/utils/platform_initiator.py sleep 2 fi -sudo PYTHONPATH='.' python ~/workspace/bemoss_os/bemoss_lib/databases/cassandraAPI/startCassandra.py sleep 2 #Step3: Build agents source ~/workspace/bemoss_os/bemoss_lib/utils/buildAgents.sh -- GitLab From 8292f5432e26ed9cae48ff3dd7e6dfced26f9a61 Mon Sep 17 00:00:00 2001 From: mchlburton Date: Thu, 4 May 2017 17:52:50 -0400 Subject: [PATCH 27/31] fixed error where influx types would be confused with python types --- bemoss_lib/databases/influxAPI/InfluxDB.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/bemoss_lib/databases/influxAPI/InfluxDB.py b/bemoss_lib/databases/influxAPI/InfluxDB.py index 1d02b50..9a6b99c 100755 --- a/bemoss_lib/databases/influxAPI/InfluxDB.py +++ b/bemoss_lib/databases/influxAPI/InfluxDB.py @@ -26,6 +26,12 @@ remote = settings.INFLUX['remote_address'] port = settings.INFLUX['port'] user = settings.INFLUX['db_username'] password = settings.INFLUX['db_password'] +influx_types={ + "float":float, + "integer":int, + "string":str, + "boolean":bool +} #TODO make this function test the connection in a more sensible way def makeConnection(database = building_name): @@ -299,7 +305,7 @@ def getSenseware(site = building_name, measurement = 'Temperature', local='local for entry in data: json_field = {} for field in fields.keys(): - data_type = eval(fields[field]) + data_type = influx_types[fields[field]] if isinstance(entry[field], data_type): reading = entry[field] json_field.update({field:reading}) @@ -351,6 +357,7 @@ def copy_database(source, destination, startTime=None, endTime=None): for result in measurements: for measurement in result: + print measurement measure_query = source.query("SELECT * FROM {0} WHERE time >= '{1}' AND time <= '{2}';".format(measurement['name'], startTime, endTime), chunked=True, chunk_size=100) data = measure_query.get_points('{0}'.format(measurement['name'])) @@ -368,7 +375,7 @@ def copy_database(source, destination, startTime=None, endTime=None): for entry in data: json_field = {} for field in fields.keys(): - data_type = eval(fields[field]) + data_type = influx_types[fields[field]] if isinstance(entry[field], data_type): reading = entry[field] json_field.update({field:reading}) -- GitLab From d04fdd23ff3dcdf56b5514f3139e95c2fb0b237a Mon Sep 17 00:00:00 2001 From: Avijit Saha Date: Mon, 8 May 2017 12:34:40 -0400 Subject: [PATCH 28/31] Code review changes for influx and TRV, adding TODOs --- .../devicediscovery/agent.py | 12 ++----- Agents/TRVAgent/trv/agent.py | 35 ++++++++----------- DeviceAPI/classAPI/classAPI_EnOceanPi.py | 6 ++-- bemoss_lib/multi_node/runBEMOSS_node.sh | 1 + runBEMOSS.sh | 1 + 5 files changed, 24 insertions(+), 31 deletions(-) diff --git a/Agents/DeviceDiscoveryAgent/devicediscovery/agent.py b/Agents/DeviceDiscoveryAgent/devicediscovery/agent.py index 2d45a4b..786c1c7 100755 --- a/Agents/DeviceDiscoveryAgent/devicediscovery/agent.py +++ b/Agents/DeviceDiscoveryAgent/devicediscovery/agent.py @@ -265,8 +265,7 @@ def DeviceDiscoveryAgent(config_path, **kwargs): discovered_address = discovery_module.discover(discovery_type) if discovered_address == None: discovered_address = list() - if discovery_type == 'Philips': - discovered_address.append("http://10.0.2.56") + print discovered_address for address in discovered_address: @@ -276,19 +275,14 @@ def DeviceDiscoveryAgent(config_path, **kwargs): ip_address = address try: macaddress = discovery_module.getMACaddress(discovery_type, ip_address) - print "macaddress: !!!!!!!!!!!! {}".format(macaddress) if macaddress is not None: _valid_macaddress = True else: _valid_macaddress = False except Exception as er: - if ip_address == "http://10.0.2.56": - macaddress = "00:17:88:1a:2b:3c" - _valid_macaddress = True - else: - print "exception: ",er - _valid_macaddress = False + print "exception: ",er + _valid_macaddress = False else: # this device does not return IP, wait until it get the right mac try: diff --git a/Agents/TRVAgent/trv/agent.py b/Agents/TRVAgent/trv/agent.py index 31c03a4..1d8e218 100755 --- a/Agents/TRVAgent/trv/agent.py +++ b/Agents/TRVAgent/trv/agent.py @@ -158,17 +158,10 @@ def TRVAgent(config_path, **kwargs): @periodic(device_monitor_time) def deviceMonitorBehavior(self): - if self.variables['setpoint'] >= 100: - self.variables['setpoint'] = 0 + if self.setDevice(): + print("Periodic control command sent succesfully") else: - self.variables['setpoint'] += 20 - - try: - setDeviceStatusResult = TRV.setDeviceStatus(self.variables['setpoint']) - - except Exception as er: - print er - print "set device status failed for {}".format(agent_id) + print("Periodic control command sending was unsuccesful") try: TRV.getDeviceStatus() @@ -189,7 +182,7 @@ def TRVAgent(config_path, **kwargs): #step4: update PostgresQL (meta-data) database try: #make the device offline if necessary - if self.get_variable('offline_count')>=3: + if self.get_variable('offline_count')>=2: self.cur.execute("UPDATE "+db_table_trv+" SET network_status=%s WHERE trv_id=%s", ('OFFLINE', agent_id)) @@ -227,6 +220,7 @@ def TRVAgent(config_path, **kwargs): # step3: update PostgresQL (meta-data) database try: + # TODO: Update the execute string using the loop and commit it to postgres all at once for k, v in log_variables.items(): # check if column exists, then updateDB to corresponding column self.cur.execute("select column_name from information_schema.columns where table_name=%s and column_name=%s", @@ -293,30 +287,31 @@ def TRVAgent(config_path, **kwargs): self.updateUI() # 5. Control intelligence and set device status - ''' - @periodic(device_monitor_time+1) def setDevice(self): - - if self.variables['setpoint'] > 100: - self.variables['setpoint'] = 20 + if self.variables['setpoint'] >= 100: + self.variables['setpoint'] = 0 else: - self.variables['setpoint'] += 40 + self.variables['setpoint'] += 20 try: - setDeviceStatusResult = TRV.setDeviceStatus() + setDeviceStatusResult = TRV.setDeviceStatus(self.variables['setpoint']) except Exception as er: print er print "set device status failed for {}".format(agent_id) - ''' + return False + + return setDeviceStatusResult #6. update Postgres database + # TODO: Update the execute string using the loop and commit it to postgres all at once def updateDB(self, table, column, column_ref, column_data, column_ref_data): self.cur.execute("UPDATE "+table+" SET "+column+"=%s " "WHERE "+column_ref+"=%s", (column_data, column_ref_data)) self.con.commit() + ''' def device_offline_detection(self): _db_notification_subject = 'BEMOSS Device {} went OFFLINE!!!'.format(agent_id) _email_subject = '#Attention: BEMOSS Device {} went OFFLINE!!!'.format(agent_id) @@ -495,7 +490,7 @@ def TRVAgent(config_path, **kwargs): self.cur.execute( "UPDATE " + db_table_temp_time_counter + " SET priority_counter=%s WHERE alert_id=%s AND device_id=%s", (str(self.priority_count), str(_active_alert_id), agent_id,)) - + ''' Agent.__name__ = 'TRV Agent' return Agent(**kwargs) diff --git a/DeviceAPI/classAPI/classAPI_EnOceanPi.py b/DeviceAPI/classAPI/classAPI_EnOceanPi.py index 7b23d42..a9e6762 100755 --- a/DeviceAPI/classAPI/classAPI_EnOceanPi.py +++ b/DeviceAPI/classAPI/classAPI_EnOceanPi.py @@ -60,6 +60,7 @@ class API: pass def setDeviceStatus(self,setpoint): + setDeviceStatusResult = True enocean_address = self.variables.get('address')+':8080/api/v1' set_trv = {"trv_id":self.variables.get('macaddress'), "setpoint":setpoint} headers = {"Authorization_key": 'ABC', "Content-type": 'application/json'} @@ -70,12 +71,13 @@ class API: print verify.status_code raise Exception('Response code != 200') - except Exception as er: print "HTTP Request to EnOcean gateway for {} failed".format(self.variables.get('agent_id')) print er + setDeviceStatusResult = False - print "device set" + print "Enocean TRV device set successfully" + return setDeviceStatusResult def getDeviceStatus(self): getDeviceStatusResult = True diff --git a/bemoss_lib/multi_node/runBEMOSS_node.sh b/bemoss_lib/multi_node/runBEMOSS_node.sh index 75cf7a1..1f2db1d 100755 --- a/bemoss_lib/multi_node/runBEMOSS_node.sh +++ b/bemoss_lib/multi_node/runBEMOSS_node.sh @@ -52,6 +52,7 @@ sudo python ~/workspace/bemoss_os/bemoss_lib/multi_node/platform_initiator_node. sudo python ~/workspace/bemoss_os/bemoss_lib/databases/InfluxAPI/initialize.py #Step3: Look for bemoss core sudo python ~/workspace/bemoss_os/bemoss_lib/multi_node/udpclient.py +# TODO: Verify influx connection befofre continuing with BEMOSS #Step4: Build agents cd ~/workspace/bemoss_os/bemoss_lib/multi_node/ source buildAgents_node.sh diff --git a/runBEMOSS.sh b/runBEMOSS.sh index 74d745d..e2c3d61 100755 --- a/runBEMOSS.sh +++ b/runBEMOSS.sh @@ -59,6 +59,7 @@ else sudo python ~/workspace/bemoss_os/bemoss_lib/utils/platform_initiator.py sleep 2 fi +# TODO: Verify influx connection befofre continuing with BEMOSS sleep 2 #Step3: Build agents source ~/workspace/bemoss_os/bemoss_lib/utils/buildAgents.sh -- GitLab From 11696b1ac7c80ce3ab30f06e5fe6cbca63b91b85 Mon Sep 17 00:00:00 2001 From: mchlburton Date: Mon, 8 May 2017 17:42:38 -0400 Subject: [PATCH 29/31] Changed git repo for install, trvs return fahrenheit --- DeviceAPI/classAPI/classAPI_EnOceanPi.py | 4 +++- bemoss_install_v2.sh | 15 +++++++++------ 2 files changed, 12 insertions(+), 7 deletions(-) diff --git a/DeviceAPI/classAPI/classAPI_EnOceanPi.py b/DeviceAPI/classAPI/classAPI_EnOceanPi.py index a9e6762..c9fc758 100755 --- a/DeviceAPI/classAPI/classAPI_EnOceanPi.py +++ b/DeviceAPI/classAPI/classAPI_EnOceanPi.py @@ -88,8 +88,10 @@ class API: data_req = requests.get(device_url,headers = header, params = trv_id, timeout=10) data = data_req.json() + temperature_in_f = data['temperature']*1.8 + 32 + self.set_variable('setpoint', data['setpoint']) - self.set_variable('temperature', data['temperature']) + self.set_variable('temperature', temperature_in_f) self.set_variable('window_open', data['window_open']) self.set_variable('alert', data['alert']) self.set_variable('operation_mode', data['operation_mode']) diff --git a/bemoss_install_v2.sh b/bemoss_install_v2.sh index 85b563e..0af8fdc 100755 --- a/bemoss_install_v2.sh +++ b/bemoss_install_v2.sh @@ -37,8 +37,7 @@ cd ~/workspace #Remove the existing bemoss_web_ui folder sudo rm -rf bemoss_web_ui #Clone the bemoss_web_ui repository -# TODO: switch to GitHub repo -sudo git clone -b master https://github.com/bemoss/bemoss_web_ui.git +sudo git clone -b master https://github.com/blocp/bemoss_web_ui.git sudo chmod 777 -R ~/workspace #Create the bemossdb database sudo -u postgres psql -c "CREATE USER admin WITH PASSWORD 'admin';" @@ -49,8 +48,12 @@ sudo -u postgres psql -d bemossdb -c "create extension hstore;" sudo apt-get update sudo apt-get install openjdk-7-jre --assume-yes sudo apt-get install libjna-java --assume-yes -# Install Cassandra -#TODO add influx setup to install script +# Install Influx +curl -sL https://repos.influxdata.com/influxdb.key | sudo apt-key add - +echo "deb https://repos.influxdata.com/ubuntu trusty stable" | sudo tee /etc/apt/sources.list.d/influxdb.list +sudo apt-get update +sudo apt-get -y install influxdb +sudo service influxdb start # Install Dependencies in virtual env. cd ~/workspace/bemoss_os/env . bin/activate @@ -62,8 +65,8 @@ sudo python ~/workspace/bemoss_web_ui/manage.py syncdb sudo python ~/workspace/bemoss_web_ui/run/defaultDB.py #Initialize the tables sudo python ~/workspace/bemoss_os/bemoss_lib/utils/platform_initiator.py -#TODO setup influx install and have the initialize function run here -#sudo python ~/workspace/bemoss_os/bemoss_lib/databases/InfluxAPI/initialize.py +#set up influxdb +sudo python ~/workspace/bemoss_os/bemoss_lib/databases/InfluxAPI/initialize.py # Fix miscellaneaus issues sudo ~/workspace/bemoss_os/bemoss_lib/utils/increase_open_file_limit.sh rm ~/workspace/bemoss_os/bemoss_lib/utils/increase_open_file_limit.sh -- GitLab From 2703732c076344df9e511af3161dd665d7de1053 Mon Sep 17 00:00:00 2001 From: Avijit Saha Date: Mon, 8 May 2017 19:07:25 -0400 Subject: [PATCH 30/31] Basic TRV control for data acquisition --- Agents/TRVAgent/trv/agent.py | 41 +++++++++++++++++++++++++++--------- 1 file changed, 31 insertions(+), 10 deletions(-) diff --git a/Agents/TRVAgent/trv/agent.py b/Agents/TRVAgent/trv/agent.py index 1d8e218..2da37e2 100755 --- a/Agents/TRVAgent/trv/agent.py +++ b/Agents/TRVAgent/trv/agent.py @@ -126,6 +126,7 @@ def TRVAgent(config_path, **kwargs): self.runningSeconds = 0 self._override = True self.already_offline = False + self.lastDateSet = None # 2. setup connection with db -> Connect to bemossdb database try: @@ -139,6 +140,14 @@ def TRVAgent(config_path, **kwargs): self.send_notification = send_notification self.subject = 'Message from ' + agent_id + try: + setDeviceStatusResult = TRV.setDeviceStatus(100) + + except Exception as er: + print er + print "set device status failed at initialization for {}".format(agent_id) + return False + # These set and get methods allow scalability def set_variable(self, k, v): # k=key, v=value self.variables[k] = v @@ -288,18 +297,29 @@ def TRVAgent(config_path, **kwargs): # 5. Control intelligence and set device status def setDevice(self): - if self.variables['setpoint'] >= 100: - self.variables['setpoint'] = 0 - else: - self.variables['setpoint'] += 20 + setDeviceStatusResult = False + current_time = datetime.datetime.now() + needtochangesetpoint = False + if current_time.hour >= 17: + if self.lastDateSet is not None: + if self.lastDateSet == current_time.date(): + needtochangesetpoint = False + else: + needtochangesetpoint = True + else: + needtochangesetpoint = True + if needtochangesetpoint: + self.variables['setpoint'] -= 20 + if self.variables['setpoint'] < 0: + self.variables['setpoint'] = 0 - try: - setDeviceStatusResult = TRV.setDeviceStatus(self.variables['setpoint']) + try: + setDeviceStatusResult = TRV.setDeviceStatus(self.variables['setpoint']) - except Exception as er: - print er - print "set device status failed for {}".format(agent_id) - return False + except Exception as er: + print er + print "set device status failed for {}".format(agent_id) + return False return setDeviceStatusResult @@ -491,6 +511,7 @@ def TRVAgent(config_path, **kwargs): "UPDATE " + db_table_temp_time_counter + " SET priority_counter=%s WHERE alert_id=%s AND device_id=%s", (str(self.priority_count), str(_active_alert_id), agent_id,)) ''' + Agent.__name__ = 'TRV Agent' return Agent(**kwargs) -- GitLab From 396c83bfa60c4e9ebdf5f728fe2a1b38d8632e68 Mon Sep 17 00:00:00 2001 From: mchlburton Date: Tue, 9 May 2017 21:12:27 -0400 Subject: [PATCH 31/31] Fixed a renewConnection arguments error in EnOcean API --- DeviceAPI/classAPI/classAPI_EnOceanPi.py | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/DeviceAPI/classAPI/classAPI_EnOceanPi.py b/DeviceAPI/classAPI/classAPI_EnOceanPi.py index c9fc758..a944f7f 100755 --- a/DeviceAPI/classAPI/classAPI_EnOceanPi.py +++ b/DeviceAPI/classAPI/classAPI_EnOceanPi.py @@ -25,10 +25,6 @@ class API: self.set_variable('connection_renew_interval',6000) self.set_variable('offline_count', 0) - - def renewConnection(self): - pass - def set_variable(self,k,v): # k=key, v=value self.variables[k] = v @@ -56,7 +52,7 @@ class API: 3. getDeviceStatus() GET ''' #TODO Take authentication key from launch file instead of hard coding - def renewConnection(): + def renewConnection(self): pass def setDeviceStatus(self,setpoint): -- GitLab