From 6bbe221ede3524487b6d59debf0587a42adc91be Mon Sep 17 00:00:00 2001 From: Alessandro DiMarco Date: Tue, 19 Sep 2017 17:20:19 -0400 Subject: [PATCH 01/16] Add limit and order_by in base view --- app/controllers/base.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/app/controllers/base.py b/app/controllers/base.py index 8d5ad7a..7d1e1b3 100644 --- a/app/controllers/base.py +++ b/app/controllers/base.py @@ -54,6 +54,13 @@ class RestController(object): for k, f in self.filters.items(): if k in filter_data: q = q.filter(f(filter_data)) + + if 'order_by' in filter_data: + q = q.order_by(getattr(self.Model, filter_data['order_by'])) + + if 'limit' in filter_data: + q = q.limit(int(filter_data['limit'])) + return q def get_form(self, filter_data): -- GitLab From 19d7991c2137d433535ccd8ffa7ef975574390f3 Mon Sep 17 00:00:00 2001 From: Alessandro DiMarco Date: Thu, 21 Sep 2017 14:46:59 -0400 Subject: [PATCH 02/16] Add sort param to filter --- app/controllers/base.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/app/controllers/base.py b/app/controllers/base.py index 7d1e1b3..546e7de 100644 --- a/app/controllers/base.py +++ b/app/controllers/base.py @@ -56,7 +56,13 @@ class RestController(object): q = q.filter(f(filter_data)) if 'order_by' in filter_data: - q = q.order_by(getattr(self.Model, filter_data['order_by'])) + order_by = getattr(self.Model, filter_data['order_by']) + sort = '' + + if 'sort' in filter_data: + sort = filter_data['sort'] + + q = q.order_by('{} {}'.format(order_by, sort)) if 'limit' in filter_data: q = q.limit(int(filter_data['limit'])) -- GitLab From 0ec524151bd3636cbf61f71448cdcb79c92b5d23 Mon Sep 17 00:00:00 2001 From: Alessandro DiMarco Date: Mon, 25 Sep 2017 17:20:39 -0400 Subject: [PATCH 03/16] Add user to flask.g --- app/permissions/auth.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/app/permissions/auth.py b/app/permissions/auth.py index fb6bfbd..de95310 100644 --- a/app/permissions/auth.py +++ b/app/permissions/auth.py @@ -1,5 +1,5 @@ """Permissions to check authentication.""" -from flask import current_app +from flask import current_app, g from werkzeug.exceptions import Unauthorized from jose import jwt import json @@ -49,6 +49,7 @@ class AuthNeed(Permission): audience=API_AUDIENCE, issuer="https://"+AUTH0_DOMAIN+"/" ) + g.sub = payload['sub'] # For now we will print and return unauthorized. In the future # we will log these errors and the requester except jwt.ExpiredSignatureError: -- GitLab From 31d6b1464b2fdd06e10d625ec5dff5c44dc85604 Mon Sep 17 00:00:00 2001 From: Conrad Date: Tue, 26 Sep 2017 11:19:49 -0400 Subject: [PATCH 04/16] Raise error when scraping bad NG accounts --- app/lib/utilitybillscraper | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app/lib/utilitybillscraper b/app/lib/utilitybillscraper index cb511fb..3618f7d 160000 --- a/app/lib/utilitybillscraper +++ b/app/lib/utilitybillscraper @@ -1 +1 @@ -Subproject commit cb511fb53225ee655e72b72a9d2698ee31700c4d +Subproject commit 3618f7d2ab0ab5e54ba7666cb00ecf86d492fd4b -- GitLab From 7a19e118a4b4cbc04b6edc0e13610549cd6cf585 Mon Sep 17 00:00:00 2001 From: Conrad Date: Tue, 26 Sep 2017 14:28:24 -0400 Subject: [PATCH 05/16] Add new columns to account endpoint and refactor to use model correctly --- app/controllers/account.py | 143 +++++++++++++++++++-- app/forms/account.py | 19 +-- app/models/account.py | 256 ++++++------------------------------- app/views/account.py | 20 +-- 4 files changed, 185 insertions(+), 253 deletions(-) diff --git a/app/controllers/account.py b/app/controllers/account.py index 389e8d9..e8b0924 100644 --- a/app/controllers/account.py +++ b/app/controllers/account.py @@ -1,4 +1,5 @@ """Controllers for posting or manipulating a account.""" +from flask import g from .base import RestController from ..lib.database import db from ..models.account import Account @@ -9,11 +10,35 @@ from ..forms.account import AccountForm from werkzeug.datastructures import MultiDict from werkzeug.exceptions import BadRequest from datetime import datetime +from email.mime.multipart import MIMEMultipart +from email.mime.text import MIMEText +import smtplib class AccountController(RestController): """A account controller.""" Model = Account + filters = { + 'building_id': lambda d: Account.building_id == d['building_id'], + } + + UTILITY_PROVIDERS = { + 'national_grid': 1, + 'con_edison': 2, + 'other': 3, + 'con_edison_detail': 11, + } + # The above dictionary with the keys as values and the values as keys + UTILITY_PROVIDERS_INV = {v: k for k, v in UTILITY_PROVIDERS.items()} + + UTILITY_TYPES = { + 'electric': 1, + 'gas': 2, + 'oil': 3, + 'water': 4, + } + # The above dictionary with the keys as values and the values as keys + UTILITY_TYPES_INV = {v: k for k, v in UTILITY_TYPES.items()} def get_form(self, filter_data): """Return the account form.""" @@ -28,6 +53,44 @@ class AccountController(RestController): :return: Account data """ + + account_provider = '' + account_type = '' + utility = data.pop('utility') + if utility == 'con_edison_electric': + account_provider = self.UTILITY_PROVIDERS['con_edison'] + account_type = self.UTILITY_TYPES['electric'] + elif utility == 'con_edison_gas': + account_provider = self.UTILITY_PROVIDERS['con_edison'] + account_type = self.UTILITY_TYPES['gas'] + elif utility == 'national_grid_gas': + account_provider = self.UTILITY_PROVIDERS['national_grid'] + account_type = self.UTILITY_TYPES['gas'] + elif utility == 'other_electric': + account_provider = self.UTILITY_PROVIDERS['other'] + account_type = self.UTILITY_TYPES['electric'] + elif utility == 'other_gas': + account_provider = self.UTILITY_PROVIDERS['other'] + account_type = self.UTILITY_TYPES['gas'] + elif utility == 'other_oil': + account_provider = self.UTILITY_PROVIDERS['other'] + account_type = self.UTILITY_TYPES['oil'] + elif utility == 'other_water': + account_provider = self.UTILITY_PROVIDERS['other'] + account_type = self.UTILITY_TYPES['water'] + elif utility == 'con_edison_detail_electric': + account_provider = self.UTILITY_PROVIDERS['con_edison_detail'] + account_type = self.UTILITY_TYPES['electric'] + elif utility == 'con_edison_detail_gas': + account_provider = self.UTILITY_PROVIDERS['con_edison_detail'] + account_type = self.UTILITY_TYPES['gas'] + else: + raise BadRequest('Invalid utility {} given'.format(self.utility)) + if not data['usage_type']: + data['usage_type'] = 1 + data['account_provider'] = account_provider + data['account_type'] = account_type + data['created_by_id'] = g.get('sub', None) form = self.get_form(filter_data)(formdata=MultiDict(data)) if not form.validate(): raise BadRequest(form.errors) @@ -35,17 +98,77 @@ class AccountController(RestController): email_address = None if 'email_address' in data: email_address = data.pop('email_address') - return Account(**data).create_account(email_address) - def get(self, id_, filter_data): - """ - Retrieves utility bill information + if ( + not data['username'] and + not data['password'] and + data['access_code'] and + utility == "national_grid_gas" + ): + self.utility_email_notification(email_address) + return super().post(data, filter_data) - :param id_: - :param filter_data: - :return: - """ - return Account.get_accounts(id_) + def utility_email_notification(self, to_address=None): + """ send out email notification to request for manual input """ + from_address = current_app.config.get("EMAIL_SENDING_ADDRESS") + from_password = current_app.config.get("EMAIL_SENDING_PASSWORD") + if not to_address: + to_address = current_app.config.get("EMAIL_RECEIVING_ADDRESS") + smtp = current_app.config.get("EMAIL_SENDING_SMTP") + port = current_app.config.get("EMAIL_SENDING_PORT") + msg = MIMEMultipart() + msg['From'] = from_address + msg['To'] = to_address + msg['Subject'] = "Notification for NationalGrid Utility Account " + + body = 'In order for the scraper to work, the account number and '\ + 'access code need to be added to blocpower national grid ' \ + 'account. Please add the folllowing account number ' \ + 'and access code into the NationalGrid account.\n\n' \ + 'account number: {}\n' \ + 'access_code: {}\n\n' \ + 'To add the accout number and access code to the blocpower '\ + 'account navigate to https://online.nationalgridus.com/login'\ + '/LoginActivate?applicurl=aHR0cHM6Ly9vbmxpbmUubmF0aW9uY'\ + 'WxncmlkdXMuY29tL2VzZXJ2aWNlX2VudQ==&auth_method=0 '\ + 'and login with user BlocpowerTest (ask a developer for '\ + 'the password). Once logged, click "Add Account" on the right '\ + 'side.'.format(self.account_number, self.access_code) + msg.attach(MIMEText(body, 'plain')) + + try: + server = smtplib.SMTP(smtp, port) + server.starttls() + server.login(from_address, from_password) + text = msg.as_string() + server.sendmail(from_address, to_address, text) + server.quit() + except smtplib.SMTPAuthenticationError as e: + if current_app.config['DEBUG']: + raise e + raise Unauthorized('The username or password does not match for'\ + ' the email account being sent a notification.'\ + ' Please contact the development team.') + + def index(self, filter_data): + data = super().index(filter_data) + # Accounts with both bill and disaggregated bill + full_data = [] + for account in data: + account.type = '{}_{}'.format( + self.UTILITY_PROVIDERS_INV[account.account_provider], + self.UTILITY_TYPES_INV[account.account_type], + ) + args = MultiDict([('account_id[]', account.id)]) + # Also get the disaggregated data for the account + disaggregated_database_output = DisaggregatedBillController().index(args) + bill_database_output = BillController().index(args) + full_data.append({ + 'account': account, + 'disaggregated_database_output': disaggregated_database_output, + 'bill_database_output': bill_database_output, + }) + return full_data def delete(self, id_, filter_data): """ @@ -69,7 +192,7 @@ class AccountController(RestController): None, ) # Then delete the account - response = Account.delete_account(account_id) + response = super().delete(account_id, filter_data) return_list = [] for row in response: d = dict(row.items()) diff --git a/app/forms/account.py b/app/forms/account.py index 8ec4688..ab36a31 100644 --- a/app/forms/account.py +++ b/app/forms/account.py @@ -6,22 +6,9 @@ class AccountForm(wtf.Form): building_id = wtf.StringField( validators=[wtf.validators.Required()]) - utility = wtf.StringField( - validators=[ - wtf.validators.Required(), - wtf.validators.AnyOf(values=[ - 'con_edison_electric', - 'con_edison_gas', - 'con_edison_detail_electric', - 'con_edison_detail_gas', - 'national_grid_gas', - 'other_electric', - 'other_gas', - 'other_oil', - 'other_water', - ]) - ] - ) + account_provider = wtf.IntegerField() + account_type = wtf.IntegerField() + usage_type = wtf.IntegerField() account_number = wtf.StringField( validators=[wtf.validators.Optional()]) username = wtf.StringField( diff --git a/app/models/account.py b/app/models/account.py index 8efdaee..f256942 100644 --- a/app/models/account.py +++ b/app/models/account.py @@ -3,227 +3,45 @@ from flask import current_app from sqlalchemy.exc import IntegrityError from werkzeug.exceptions import Unauthorized, BadRequest from werkzeug.datastructures import MultiDict -import smtplib -from email.mime.multipart import MIMEMultipart -from email.mime.text import MIMEText from ..lib.database import db from ..controllers.disaggregated_bill import DisaggregatedBillController from ..controllers.bill import BillController -from .base import Model - - -class Account(Model): - - UTILITY_PROVIDERS = { - 'national_grid': 1, - 'con_edison': 2, - 'other': 3, - 'con_edison_detail': 11, - } - # The above dictionary with the keys as values and the values as keys - UTILITY_PROVIDERS_INV = {v: k for k, v in UTILITY_PROVIDERS.items()} - - UTILITY_TYPES = { - 'electric': 1, - 'gas': 2, - 'oil': 3, - 'water': 4, - } - # The above dictionary with the keys as values and the values as keys - UTILITY_TYPES_INV = {v: k for k, v in UTILITY_TYPES.items()} - - SCHEMA = 'public' - - def __init__( - self, - building_id, - utility, - account_number, - username, - password, - access_code, - usage_type, - ): - self.building_id = building_id - self.utility = utility - self.account_number = account_number - self.username = username - self.password = password - self.access_code = access_code - # Default to 1, or unkown - self.usage_type = 1 - if usage_type: - self.usage_type = usage_type - - def __str__(self): - return "Account model for building {}".format(self.building_id) - - def create_account(self, email_address=None): - """ Create new utility account """ - account_provider = '' - account_type = '' - if self.utility == 'con_edison_electric': - account_provider = self.UTILITY_PROVIDERS['con_edison'] - account_type = self.UTILITY_TYPES['electric'] - elif self.utility == 'con_edison_gas': - account_provider = self.UTILITY_PROVIDERS['con_edison'] - account_type = self.UTILITY_TYPES['gas'] - elif self.utility == 'national_grid_gas': - account_provider = self.UTILITY_PROVIDERS['national_grid'] - account_type = self.UTILITY_TYPES['gas'] - elif self.utility == 'other_electric': - account_provider = self.UTILITY_PROVIDERS['other'] - account_type = self.UTILITY_TYPES['electric'] - elif self.utility == 'other_gas': - account_provider = self.UTILITY_PROVIDERS['other'] - account_type = self.UTILITY_TYPES['gas'] - elif self.utility == 'other_oil': - account_provider = self.UTILITY_PROVIDERS['other'] - account_type = self.UTILITY_TYPES['oil'] - elif self.utility == 'other_water': - account_provider = self.UTILITY_PROVIDERS['other'] - account_type = self.UTILITY_TYPES['water'] - elif self.utility == 'con_edison_detail_electric': - account_provider = self.UTILITY_PROVIDERS['con_edison_detail'] - account_type = self.UTILITY_TYPES['electric'] - elif self.utility == 'con_edison_detail_gas': - account_provider = self.UTILITY_PROVIDERS['con_edison_detail'] - account_type = self.UTILITY_TYPES['gas'] - else: - raise BadRequest('Invalid utility {} given'.format(self.utility)) - account_id = -1 - try: - res = Model.run_proc( - 'create_account', - Account.SCHEMA, - **{ - 'building_id': self.building_id, - 'account_number': self.account_number, - 'account_provider': account_provider, - 'account_type': account_type, - 'login': self.username, - 'pass': self.password, - 'access_code': self.access_code, - 'usage_type': self.usage_type, - } - ) - account_id = res.fetchone()[0] - except IntegrityError as e: - if str(e).startswith( - '(psycopg2.IntegrityError) duplicate key value violates unique constraint "account_type_unique"' - ): - raise BadRequest('There is already an account of that utility type for this building id') - raise e - # Get the account_id from the response - if ( - not self.username and - not self.password and - self.access_code and - self.utility == "national_grid_gas" - ): - self.utility_email_notification(email_address) - +from .base import Model, Tracked +import datetime + +class Account(Model, Tracked, db.Model): + + __table_args__ = {"schema": "public"} + + account_number = db.Column(db.Unicode(50)) + account_provider = db.Column(db.Integer) + account_type = db.Column(db.Integer) + building_id = db.Column(db.Integer) + access_code = db.Column(db.Unicode(50)) + login = db.Column(db.Unicode(25)) + password = db.Column(db.Unicode(25)) + usage_type = db.Column(db.Integer) + r_squared = db.Column(db.Float) + created_by_id = db.Column(db.Unicode(50)) + created_by_name = db.Column(db.Unicode(50)) + updated_by_id = db.Column(db.Unicode(50)) + updated_by_name = db.Column(db.Unicode(50)) + label = db.Column(db.Unicode(50)) + label_created_by_id = db.Column(db.Unicode(50)) + label_created_by_name = db.Column(db.Unicode(50)) + scrape_date= db.Column(db.Date) + type = None + + # Parse datetime into a string + def get_dictionary(self): + d = super(Account, self).get_dictionary() + d['type'] = self.type return { - 'building_id': self.building_id, - 'account_number': self.account_number, - 'type': self.utility, - 'login': self.username, - 'pass': self.password, - 'access_code': self.access_code, - 'id': account_id, - 'usage_type': self.usage_type, + key: ( + val.strftime( + '%m/%d/%Y' + ) if isinstance( + val, datetime.date + ) else val + ) for key,val in d.items() } - - def utility_email_notification(self, to_address=None): - """ send out email notification to request for manual input """ - from_address = current_app.config.get("EMAIL_SENDING_ADDRESS") - from_password = current_app.config.get("EMAIL_SENDING_PASSWORD") - if not to_address: - to_address = current_app.config.get("EMAIL_RECEIVING_ADDRESS") - smtp = current_app.config.get("EMAIL_SENDING_SMTP") - port = current_app.config.get("EMAIL_SENDING_PORT") - msg = MIMEMultipart() - msg['From'] = from_address - msg['To'] = to_address - msg['Subject'] = "Notification for NationalGrid Utility Account " - - body = 'In order for the scraper to work, the account number and '\ - 'access code need to be added to blocpower national grid ' \ - 'account. Please add the folllowing account number ' \ - 'and access code into the NationalGrid account.\n\n' \ - 'account number: {}\n' \ - 'access_code: {}\n\n' \ - 'To add the accout number and access code to the blocpower '\ - 'account navigate to https://online.nationalgridus.com/login'\ - '/LoginActivate?applicurl=aHR0cHM6Ly9vbmxpbmUubmF0aW9uY'\ - 'WxncmlkdXMuY29tL2VzZXJ2aWNlX2VudQ==&auth_method=0 '\ - 'and login with user BlocpowerTest (ask a developer for '\ - 'the password). Once logged, click "Add Account" on the right '\ - 'side.'.format(self.account_number, self.access_code) - msg.attach(MIMEText(body, 'plain')) - - try: - server = smtplib.SMTP(smtp, port) - server.starttls() - server.login(from_address, from_password) - text = msg.as_string() - server.sendmail(from_address, to_address, text) - server.quit() - except smtplib.SMTPAuthenticationError as e: - if current_app.config['DEBUG']: - raise e - raise Unauthorized('The username or password does not match for'\ - ' the email account being sent a notification.'\ - ' Please contact the development team.') - - @staticmethod - def delete_account(account_id): - """ - Delete utility account - Args: - account_number (str) - The util account number - Returns: - Deleted object from DB - """ - return Model.run_proc( - 'delete_account', - Account.SCHEMA, - **{ - 'account_id': account_id, - }, - ) - - @staticmethod - def get_accounts(building_id): - """ - Retrieve utility accounts - Args: - building_id (int) - The id of the building - Returns: - List of accounts for the building - """ - response = Model.run_proc( - 'get_account', - Account.SCHEMA, - **{'building_id': building_id}, - ) - utility_list = [] - for row in response: - d = dict(row.items()) - d['type'] = '{}_{}'.format( - Account.UTILITY_PROVIDERS_INV[d['account_provider']], - Account.UTILITY_TYPES_INV[d['account_type']], - ) - args = MultiDict([('account_id[]', d['id'])]) - # Also get the disaggregated data for the account - disaggregated_data = DisaggregatedBillController().index(args) - d["disaggregated_database_output"] = [ - model.get_dictionary() for model in disaggregated_data - ] - bill_data = BillController().index(args) - d["bill_database_output"] = [ - model.get_dictionary() for model in bill_data - ] - utility_list.append(d) - - return {"utilities": utility_list} diff --git a/app/views/account.py b/app/views/account.py index a186d68..ed8f9c4 100644 --- a/app/views/account.py +++ b/app/views/account.py @@ -12,20 +12,24 @@ class AccountView(RestView): def get_controller(self): """Return an instance of the account controller.""" return AccountController() - def index(self): - raise MethodNotAllowed() + """/{id} GET - Retrieve a resource by id.""" + return_list = [] + for m in self.get_controller().index(request.args): + return_dict = {} + for k, v in m.items(): + if type(v) == list: + return_dict[k] = [self.parse(l) for l in v] + else: + return_dict[k] = self.parse(v) + return_list.append(return_dict) + + return self.json(return_list) def get(self, id_): """/{id} GET - Retrieve a resource by id.""" return self.json(self.get_controller().get(id_, request.args)) - def post(self): - """/ POST - Save utility account data given in POST body""" - response = self.get_controller().post( - self.request_json(), request.args) - return self.json(response) - def put(self, id_): raise MethodNotAllowed() -- GitLab From 5c15ae5bc8372ee746c654657306d553a4251373 Mon Sep 17 00:00:00 2001 From: Conrad Date: Tue, 26 Sep 2017 17:36:18 -0400 Subject: [PATCH 06/16] Seperate scrape and disaggregate --- app/controllers/disaggregate.py | 221 +++++++++++++++++++++++++ app/controllers/scrape.py | 276 +++++++++++++++++++++++++------- app/models/scrape.py | 200 +---------------------- app/views/__init__.py | 3 +- app/views/disaggregate.py | 35 ++++ 5 files changed, 480 insertions(+), 255 deletions(-) create mode 100644 app/controllers/disaggregate.py create mode 100644 app/views/disaggregate.py diff --git a/app/controllers/disaggregate.py b/app/controllers/disaggregate.py new file mode 100644 index 0000000..5ce8fa8 --- /dev/null +++ b/app/controllers/disaggregate.py @@ -0,0 +1,221 @@ +"""Controllers for posting or manipulating a disaggregate.""" +from .base import RestController +from ..controllers.disaggregated_bill import DisaggregatedBillController +from ..controllers.bill import BillController +from ..forms.account import AccountForm +from ..lib.database import db +from ..models.account import Account +from ..lib.service import services + +import base64 +import csv +import json +from io import StringIO +from datetime import datetime, timedelta +from io import BytesIO +import pandas +from werkzeug.datastructures import MultiDict +from werkzeug.exceptions import BadRequest +from bpeng.bill import BillDisaggregation + + +class DisaggregateController(RestController): + """A disaggregate controller.""" + + USAGE_TYPE = { + 1: 'Unknown', + 2: 'Heating', + 3: 'Cooling', + 4: 'Both', + } + + COLUMN_TITLES = 'Bill From Date,Bill To Date,'\ + 'Days In Bill,Usage,Supply Charge,'\ + 'Delivery Charge,Total Charge' + + def get_form(self, filter_data): + """Return the account form.""" + return AccountForm + + def post(self, data, filter_data): + """ + Call function in model to run scraper + + :param data: Args from Body + :param filter_data: Args from POST URL + :return: Disaggregate data + """ + + form = self.get_form(filter_data)(formdata=MultiDict(data)) + if not form.validate(): + raise BadRequest(form.errors) + + account_id = data['account_id'] + usage_type = data['usage_type'] + args = MultiDict([('account_id[]', account_id)]) + bill_database_output = BillController().index(args) + bill_csv_output = self.convert_database_to_csv(bill_database_output) + + r_squared, disaggregated_bill = self.disaggregate( + bill_csv_output, + account_id, + usage_type, + ) + DisaggregateController.update_r_squared(r_squared, account_id) + + # Save the disaggregated bill data in the database + DisaggregatedBillController().post( + disaggregated_bill['disaggregate_database_output'], + {'account_id': account_id}, + ) + + return {**disaggregated_bill, 'r_squared': r_squared} + + def put(self, id_, data, filter_data): + """ + Upload a new disaggregated bill + """ + account_id = id_ + file_content = data.pop('file_content') + buffer_object = BytesIO(base64.b64decode(bytes(file_content, 'utf-8'))) + file_object = pandas.read_csv(buffer_object) + json_content = file_object.to_json(orient="records") + + disaggregated_bill = self.save_disaggregated_data( + json_content, + account_id, + ) + # Save the disaggregated bill data in the database + DisaggregatedBillController().post( + disaggregated_bill['disaggregate_database_output'], + {'account_id': account_id}, + ) + + # Set r_squared to null since we're not actually calculating + # disaggregation here + DisaggregateController.update_r_squared('null', account_id) + return {**disaggregated_bill, 'r_squared': None} + + def convert_database_to_csv(self, bill_database_output): + row_list = [] + for bill_object in bill_database_output: + bill = bill_object.get_dictionary() + time_format = "%m/%d/%Y" + start_date = datetime.strptime( + bill['bill_from_date'], + time_format, + ) + end_date = datetime.strptime( + bill['bill_to_date'], + time_format, + ) + time_difference_in_days = (end_date - start_date) / timedelta(days=1) + row_list.append([ + bill['id'], + bill['bill_from_date'], + bill['bill_to_date'], + time_difference_in_days, + bill['usage'], + bill['delivery_charge'], + bill['supply_charge'], + bill['total_charge_bill'], + ]) + csv_output = StringIO() + csv_writer = csv.writer(csv_output) + column_titles_list = self.COLUMN_TITLES.split(",") + csv_writer.writerow(column_titles_list) + for row in row_list: + csv_writer.writerow(row) + return csv_output.getvalue() + + def save_disaggregated_data(self, file_json, account_id): + rows = json.loads(file_json) + # Convert the keys to reflect what is in the database + new_output = [] + try: + for row in rows: + new_row = {} + new_row["account_id"] = account_id + new_row["bill_from_date"] = row["Bill From Date"] + new_row["bill_to_date"] = row["Bill To Date"] + new_row["usage"] = row["Usage"] if row["Usage"] else 0 + new_row["heating_usage"] = row["Heating Usage"] if row["Heating Usage"] else 0 + new_row["cooling_usage"] = row["Cooling Usage"] if row["Cooling Usage"] else 0 + new_row["other_usage"] = row["Other Usage"] if row["Other Usage"] else 0 + new_output.append(new_row) + except KeyError as e: + raise BadRequest('The disaggregate CSV is missing a column. KeyError: ' + str(e)) + + # Return the json of disaggregate + return { + "disaggregate_database_output": sorted( + new_output, + key=DisaggregateController.sort_by_bill_from_date, + ) + } + + def disaggregate(self, csv_content, account_id, usage_type): + # How the date is formatted in influx + INFLUX_DATE_FORMAT_STRING = '%Y-%m-%dT%H:%M:%SZ' + # Get weather data + response = services.weather.get( + '/weather?'\ + 'measurement=temperature&'\ + 'interval=daily', + data=json.dumps({}), + ) + weather_list = [] + for day in response.json()['data']: + date = datetime.strptime( + day['time'], + INFLUX_DATE_FORMAT_STRING, + ) + # This is the format bill disaggregation expects + date_string = '{dt.month}/{dt.day}/{dt.year}'.format( + dt = date + ) + weather_list.append({ + 'date': date_string, + 'temperature': day['fields']['value'], + }) + + scraped_weather_data = pandas.DataFrame(weather_list) + bd = BillDisaggregation( + pandas.read_csv(StringIO(csv_content)), + scraped_weather_data, + ) + file_json = '[]' + r_squared = None + try: + if not usage_type: + bd.optimize() + else: + usage_type_string = self.USAGE_TYPE[int(usage_type)] + bd.optimize(usage_type_string) + r_squared = bd.r_squared_of_fit + file_json = bd.to_json() + except AssertionError as e: + # Skip over this error + if str(e) != 'No sufficient months for regression.': + raise e + except ArithmeticError as e: + # Less than a years worth of data, return empty disaggregation + raise BadRequest('Less than a years worth of data. Cannot disaggregate') + + return r_squared, self.save_disaggregated_data(file_json, account_id) + + @staticmethod + def update_r_squared(r_squared, account_id): + """Commit the r_squared value to the database""" + if r_squared: + query = "UPDATE public.account SET r_squared={} WHERE id={}".format(r_squared, account_id) + results = db.session.execute(query) + db.session.commit() + + # Sorting function ensure that disaggregated data is properly sorted + @staticmethod + def sort_by_bill_from_date(L): + splitup = L['bill_from_date'].split('/') + return splitup[2], splitup[0], splitup[1] + + diff --git a/app/controllers/scrape.py b/app/controllers/scrape.py index 2e4ddfa..2644c44 100644 --- a/app/controllers/scrape.py +++ b/app/controllers/scrape.py @@ -5,19 +5,27 @@ from ..controllers.bill import BillController from ..forms.account import AccountForm from ..lib.database import db from ..models.account import Account -from ..models.scrape import Scrape +from ..lib.utilitybillscraper.scraper_api.scrape import main as run_scraper import base64 -from datetime import datetime +from io import StringIO +from datetime import datetime, timedelta +import csv +import json from io import BytesIO -import pandas from werkzeug.datastructures import MultiDict -from werkzeug.exceptions import BadRequest +from types import SimpleNamespace +from werkzeug.exceptions import BadGateway, ServiceUnavailable, NotFound, BadRequest +from requests.exceptions import ConnectionError +import pandas class ScrapeController(RestController): """A scrape controller.""" - Model = Scrape + COLUMN_TITLES = 'Bill From Date,Bill To Date,'\ + 'Days In Bill,Usage,Supply Charge,'\ + 'Delivery Charge,Total Charge,,'\ + ',Usage Units,{},Account#,{},Supplier,{},Address,{}' def get_form(self, filter_data): """Return the account form.""" @@ -36,79 +44,237 @@ class ScrapeController(RestController): if not form.validate(): raise BadRequest(form.errors) - scrape = Scrape(**data) - scraped_bill = scrape.scrape() - - r_squared, disaggregated_bill = scrape.disaggregate(scraped_bill["bill_csv_output"]) - ScrapeController.update_r_squared(r_squared, scrape.account_id) + scraped_bill = self.scrape( + data['utility'], + data['account_number'], + data['username'], + data['password'], + data['account_id'] + ) # Save the scraped bill data in the database BillController().post( scraped_bill['bill_database_output'], - {'account_id': scrape.account_id}, + {'account_id': data['account_id']}, ) + return {**scraped_bill} - # Save the disaggregated bill data in the database - DisaggregatedBillController().post( - disaggregated_bill['disaggregate_database_output'], - {'account_id': scrape.account_id}, + def scrape( + self, + utility, + account_number, + username, + password, + account_id + ): + """ + Call the scraper function in the utility bill scraper + + :return: The raw data returned from utility bill scraper + """ + + # A simple object that allows us to set attributes for the + # argv argument in the utility bill scraper + arguments = SimpleNamespace() + setattr(arguments, "utility", utility) + setattr(arguments, "account_numbers", account_number) + setattr(arguments, "username", username) + setattr(arguments, "password", password) + setattr(arguments, "verbose", "") + try: + results = run_scraper(arguments) + if not results: + current_app.logger.info('No bill found with with utility {} and account number {}'.format( + utility, + account_number, + )) + raise NotFound('The utility bill was not found. Are you sure the account number is correct?') + except ConnectionError as e: + current_app.logger.info('Connection error with utility {} and account number {}'.format(utility, account_number)) + raise ServiceUnavailable( + 'Failed to scrape the utility data due' + ' to a connection error on the side of' + ' the utility company.' + ) + + account_address, database_format = self.format_in_csv( + results, + account_id, + account_number, ) + ScrapeController.update_account_address(account_address, account_id) + + return { + "account_address": account_address, + "utility_bills": results, + "bill_database_output": database_format, + } + + def format_in_csv(self, scraper_results, account_id, account_number): + """ + Parses the bill into a CSV format that can be used + for the disaggregation tool + + :param scraper_results: The results from scraping + :return: the StringIO object that contains the CSV + """ + # The results are returned from scraper as a list with 1 element + # So we make 'result' the first element in the list + [result] = scraper_results + + account_address = result["service_address"] + # The meter unit that we will display in the CSV + # This will be grabbed from the actual bill data + usage_unit = None + # List of data that will be inputted into the + # database + database_output = [] + + bill_list = result['line_items'] + for bill in bill_list: + # For now we are only using metered data in con edison + # in national grid all data is fixed, check if esimated + if bill.get("category") == "metered" or bill.get("estimated"): + if not usage_unit: + # Slight difference in syntax between nat-grid and con-ed + if bill.get("usage_unit"): + usage_unit = bill.get("usage_unit") + else: + usage_unit = bill.get("usage_units") + # This is the format the bill gives the date in + bill_time_format = "%Y-%m-%dT%H:%M:%S" + # This is te format we want the date to be in + new_time_format = "%m/%d/%Y" + start_date = datetime.strptime(bill.get("start"), + bill_time_format) + end_date = datetime.strptime(bill.get("end"), + bill_time_format) + new_start_date = start_date.strftime(new_time_format) + new_end_date = end_date.strftime(new_time_format) + + time_difference = end_date - start_date + time_difference_in_days = time_difference / timedelta(days=1) - return {**scraped_bill, **disaggregated_bill, 'r_squared': r_squared} + delivery_charge = float(bill.get('delivery_charge')) + supply_charge = float(bill.get('supply_charge')) + + database_output.append({ + 'account_id': account_id, + 'bill_from_date': new_start_date, + 'bill_to_date': new_end_date, + 'usage': float(bill.get('actual_usage')), + 'delivery_charge': delivery_charge, + 'supply_charge': supply_charge, + 'total_charge_bill': float(bill.get("charge")), + }) + # Reverse the order of the list as the scraper returns it in reverse chronological order + database_output = database_output[::-1] + return account_address, database_output def put(self, id_, data, filter_data): """ - Upload a new disaggregated bill + Upload a bill """ + account_id = id_ file_content = data.pop('file_content') buffer_object = BytesIO(base64.b64decode(bytes(file_content, 'utf-8'))) file_object = pandas.read_csv(buffer_object) json_content = file_object.to_json(orient="records") data_type = data.pop('data_type', '') - if data_type == 'disaggregated_bill': - scrape = Scrape(**data) - disaggregated_bill = scrape.save_disaggregated_data(json_content) - # Save the disaggregated bill data in the database - DisaggregatedBillController().post( - disaggregated_bill['disaggregate_database_output'], - {'account_id': scrape.account_id}, - ) - # Set r_squared to null since we're not actually calculating - # disaggregation here - ScrapeController.update_r_squared('null', scrape.account_id) - return {**disaggregated_bill, 'r_squared': None} - elif data_type == 'bill': - scrape = Scrape(**data) - bill = scrape.update_bill(json_content) - # Save the disaggregated bill data in the database - new_bill_database_output = BillController().post( - bill['bill_database_output'], - {'account_id': scrape.account_id}, - ) - bill['bill_database_output'] = [val.get_dictionary() for val in new_bill_database_output] + bill = self.update_bill(json_content, account_id) + # Save the disaggregated bill data in the database + new_bill_database_output = BillController().post( + bill['bill_database_output'], + {'account_id': account_id}, + ) + bill['bill_database_output'] = [val.get_dictionary() for val in new_bill_database_output] - r_squared, disaggregated_bill = scrape.disaggregate(bill["bill_csv_output"]) - ScrapeController.update_r_squared(r_squared, scrape.account_id) + return {**bill} - # Save the disaggregated bill data in the database - new_disaggregate_database_output = DisaggregatedBillController().post( - disaggregated_bill['disaggregate_database_output'], - {'account_id': scrape.account_id}, - ) - disaggregated_bill['disaggregate_database_output'] = [val.get_dictionary() for val in new_disaggregate_database_output] + def update_bill(self, file_json, account_id): + """ + Update a bill - return {**bill, **disaggregated_bill, 'r_squared': r_squared} - else: - raise BadRequest( - "Put must include data_type which must be one of disaggregated_bill or bill" - ) + Args + file_json (json reprentation of file) + + """ + rows = json.loads(file_json) + # Convert the keys to reflect what is in the database + new_output = [] + + # Also create the object that is expected for disaggregation + csv_output = StringIO() + csv_writer = csv.writer(csv_output) + column_titles = self.COLUMN_TITLES + column_titles_list = column_titles.split(",") + csv_writer.writerow(column_titles_list) + + try: + for row in rows: + usage = ScrapeController.parse_float(row["Usage"]) + supply_charge = ScrapeController.parse_float(row["Supply Charge"]) + delivery_charge = ScrapeController.parse_float(row["Delivery Charge"]) + total_charge = ScrapeController.parse_float(row["Total Charge"]) + if not row["Bill From Date"] or not row["Bill To Date"] or not row["Days In Bill"]: + raise BadRequest('Missing some data in the Bill From Date, Bill To Date, or Days In Bill column') + from_date = datetime.strptime(row["Bill From Date"], '%m/%d/%Y') + to_date = datetime.strptime(row["Bill To Date"], '%m/%d/%Y') + days_in_bill = (to_date - from_date).days + new_row = {} + new_csv_row = [ + row["Bill From Date"], + row["Bill To Date"], + days_in_bill, + usage, + supply_charge, + delivery_charge, + total_charge, + ] + csv_writer.writerow(new_csv_row) + new_output.append({ + "account_id": account_id, + "bill_from_date": row["Bill From Date"], + "bill_to_date": row["Bill To Date"], + "usage": usage, + "supply_charge": supply_charge, + "delivery_charge": delivery_charge, + "total_charge_bill": total_charge, + }) + except ValueError as e: + current_app.logger.info('Unable to parse some value for this bill. account # {}'.format(account_id)) + raise BadRequest('Unable to parse some value. ValueError: ' + str(e)) + except KeyError as e: + current_app.logger.info('Missing a column from the raw bill. account # {}'.format(account_id)) + raise BadRequest('The raw bill CSV is missing a column. KeyError: ' + str(e)) + + # Return the json of disaggregate + return { + "bill_database_output": sorted( + new_output, + key=ScrapeController.sort_by_bill_from_date + ), + "bill_csv_output": csv_output.getvalue(), + } @staticmethod - def update_r_squared(r_squared, account_id): + def parse_float(val): + return float(str(val).replace('$', '').replace(',', '')) + + @staticmethod + def sort_by_bill_from_date(L): + splitup = L['bill_from_date'].split('/') + return splitup[2], splitup[0], splitup[1] + + @staticmethod + def update_account_address(account_address, account_id): """Commit the r_squared value to the database""" - if r_squared: - query = "UPDATE public.account SET r_squared={} WHERE id={}".format(r_squared, account_id) - results = db.session.execute(query) + if account_address: + query = 'UPDATE public.account SET account_address = :account_address WHERE id={}'.format(account_id) + results = db.session.execute( + query, + {'account_address': account_address}, + ) db.session.commit() diff --git a/app/models/scrape.py b/app/models/scrape.py index 2d7608c..e5c706c 100644 --- a/app/models/scrape.py +++ b/app/models/scrape.py @@ -2,7 +2,6 @@ from flask import current_app from io import StringIO from datetime import datetime, timedelta -from decimal import Decimal import csv import json from types import SimpleNamespace @@ -14,15 +13,10 @@ from bpeng.bill import BillDisaggregation from ..lib.database import db from ..lib.service import services from .base import Model -from ..lib.utilitybillscraper.scraper_api.scrape import main as run_scraper class Scrape(Model): - COLUMN_TITLES = 'Bill From Date,Bill To Date,'\ - 'Days In Bill,Usage,Supply Charge,'\ - 'Delivery Charge,Total Charge,,'\ - ',Usage Units,{},Account#,{},Supplier,{},Address,{}' USAGE_TYPE = { 1: 'Unknown', 2: 'Heating', @@ -53,200 +47,8 @@ class Scrape(Model): def __str__(self): return "Scrape model for building {}".format(self.building_id) - def scrape(self): - """ - Call the scraper function in the utility bill scraper - :return: The raw data returned from utility bill scraper - """ - # A simple object that allows us to set attributes for the - # argv argument in the utility bill scraper - arguments = SimpleNamespace() - setattr(arguments, "utility", self.utility) - setattr(arguments, "account_numbers", self.account_number) - setattr(arguments, "username", self.username) - setattr(arguments, "password", self.password) - setattr(arguments, "verbose", "") - try: - results = run_scraper(arguments) - if not results: - current_app.logger.info('No bill found with with utility {} and account number {}'.format( - self.utility, - self.account_number, - )) - raise NotFound('The utility bill was not found. Are you sure the account number is correct?') - except ConnectionError as e: - current_app.logger.info('Connection error with utility {} and account number {}'.format(self.utility, self.account_number)) - raise ServiceUnavailable('Failed to scrape the utility data due' - ' to a connection error on the side of' - ' the utility company.') - - # Save the file in BOX - csv_format, database_format = self.format_in_csv(results) - - return { - "utility_bills": results, - "bill_csv_output": csv_format.getvalue(), - "bill_database_output": database_format, - } - - def format_in_csv(self, scraper_results): - """ - Parses the bill into a CSV format that can be used - for the disaggregation tool - - :param scraper_results: The results from scraping - :return: the StringIO object that contains the CSV - """ - # The results are returned from scraper as a list with 1 element - # So we make 'result' the first element in the list - - [result] = scraper_results - - # The company name that will be displayed in the CSV - supplier = result["company"] - # The address that will be displayed in the CSV - address = result["service_address"] - # The meter unit that we will display in the CSV - # This will be grabbed from the actual bill data - usage_unit = None - # List of rows that contain data about the bill - row_list = [] - # List of data that will be inputted into the - # database - database_output = [] - - bill_list = result['line_items'] - for bill in bill_list: - # For now we are only using metered data in con edison - # in national grid all data is fixed, check if esimated - if bill.get("category") == "metered" or bill.get("estimated"): - if not usage_unit: - # Slight difference in syntax between nat-grid and con-ed - if bill.get("usage_unit"): - usage_unit = bill.get("usage_unit") - else: - usage_unit = bill.get("usage_units") - # This is the format the bill gives the date in - bill_time_format = "%Y-%m-%dT%H:%M:%S" - # This is te format we want the date to be in - new_time_format = "%m/%d/%Y" - start_date = datetime.strptime(bill.get("start"), - bill_time_format) - end_date = datetime.strptime(bill.get("end"), - bill_time_format) - new_start_date = start_date.strftime(new_time_format) - new_end_date = end_date.strftime(new_time_format) - - time_difference = end_date - start_date - time_difference_in_days = time_difference / timedelta(days=1) - - delivery_charge = float(bill.get('delivery_charge')) - supply_charge = float(bill.get('supply_charge')) - - row = [] - row.append(new_start_date) - row.append(new_end_date) - row.append(time_difference_in_days) - row.append(bill.get("actual_usage")) - # Scraper doesn't get delivery/supply charge - row.append(supply_charge) - row.append(delivery_charge) - row.append(bill.get("charge")) - database_output.append({ - 'account_id': self.account_id, - 'bill_from_date': new_start_date, - 'bill_to_date': new_end_date, - 'usage': float(bill.get('actual_usage')), - 'delivery_charge': delivery_charge, - 'supply_charge': supply_charge, - 'total_charge_bill': float(bill.get("charge")), - }) - row_list.append(row) - csv_output = StringIO() - csv_writer = csv.writer(csv_output) - # Populate column titles with the values associated with this data - column_titles = Scrape.COLUMN_TITLES.format( - usage_unit, - self.account_number, - supplier, - address, - ) - column_titles_list = column_titles.split(",") - csv_writer.writerow(column_titles_list) - # Reverse the order of the list as the scraper returns it in reverse chronological order - row_list = row_list[::-1] - database_output = database_output[::-1] - for row in row_list: - csv_writer.writerow(row) - return csv_output, database_output - - def update_bill(self, file_json): - """ - Update a bill - - Args - file_json (json reprentation of file) - - """ - rows = json.loads(file_json) - # Convert the keys to reflect what is in the database - new_output = [] - - # Also create the object that is expected for disaggregation - csv_output = StringIO() - csv_writer = csv.writer(csv_output) - column_titles = Scrape.COLUMN_TITLES - column_titles_list = column_titles.split(",") - csv_writer.writerow(column_titles_list) - - try: - for row in rows: - usage = self.parse_float(row["Usage"]) - supply_charge = self.parse_float(row["Supply Charge"]) - delivery_charge = self.parse_float(row["Delivery Charge"]) - total_charge = self.parse_float(row["Total Charge"]) - if not row["Bill From Date"] or not row["Bill To Date"] or not row["Days In Bill"]: - raise BadRequest('Missing some data in the Bill From Date, Bill To Date, or Days In Bill column') - from_date = datetime.strptime(row["Bill From Date"], '%m/%d/%Y') - to_date = datetime.strptime(row["Bill To Date"], '%m/%d/%Y') - days_in_bill = (to_date - from_date).days - new_row = {} - new_csv_row = [ - row["Bill From Date"], - row["Bill To Date"], - days_in_bill, - usage, - supply_charge, - delivery_charge, - total_charge, - ] - csv_writer.writerow(new_csv_row) - new_output.append({ - "account_id": self.account_id, - "bill_from_date": row["Bill From Date"], - "bill_to_date": row["Bill To Date"], - "usage": usage, - "supply_charge": supply_charge, - "delivery_charge": delivery_charge, - "total_charge_bill": total_charge, - }) - except ValueError as e: - current_app.logger.info('Unable to parse some value for this bill. account # {}'.format(self.account_id)) - raise BadRequest('Unable to parse some value. ValueError: ' + str(e)) - except KeyError as e: - current_app.logger.info('Missing a column from the raw bill. account # {}'.format(self.account_id)) - raise BadRequest('The raw bill CSV is missing a column. KeyError: ' + str(e)) - - # Return the json of disaggregate - return { - "bill_database_output": sorted(new_output, key=Scrape.sort_by_bill_from_date), - "bill_csv_output": csv_output.getvalue(), - } - - def parse_float(self, val): - return float(str(val).replace('$', '').replace(',', '')) def disaggregate(self, csv_content): @@ -309,7 +111,7 @@ class Scrape(Model): new_row["account_id"] = self.account_id new_row["bill_from_date"] = row["Bill From Date"] new_row["bill_to_date"] = row["Bill To Date"] - new_row["usage"] = row["Usage"] if row["Usage"] else 0 + new_row["usage"] = row["Usage"] if row["Usage"u] else 0 new_row["heating_usage"] = row["Heating Usage"] if row["Heating Usage"] else 0 new_row["cooling_usage"] = row["Cooling Usage"] if row["Cooling Usage"] else 0 new_row["other_usage"] = row["Other Usage"] if row["Other Usage"] else 0 diff --git a/app/views/__init__.py b/app/views/__init__.py index dc52343..d6eee70 100644 --- a/app/views/__init__.py +++ b/app/views/__init__.py @@ -1,5 +1,5 @@ """Flask-classy views for the flask application.""" -from . import scrape, account, disaggregated_bill, bill +from . import disaggregate, scrape, account, disaggregated_bill, bill def register(app): @@ -9,6 +9,7 @@ def register(app): in the application. """ scrape.ScrapeView.register(app) + disaggregate.DisaggregateView.register(app) account.AccountView.register(app) disaggregated_bill.DisaggregatedBillView.register(app) bill.BillView.register(app) diff --git a/app/views/disaggregate.py b/app/views/disaggregate.py new file mode 100644 index 0000000..dac0511 --- /dev/null +++ b/app/views/disaggregate.py @@ -0,0 +1,35 @@ +"""Views for working with disaggregate.""" +from flask import request +from werkzeug.exceptions import MethodNotAllowed +from ..controllers.disaggregate import DisaggregateController +from .base import RestView + + +class DisaggregateView(RestView): + """The disaggregate view.""" + + def get_controller(self): + """Return an instance of the disaggregate controller.""" + return DisaggregateController() + + def index(self): + raise MethodNotAllowed() + + def get(self, id_): + """/{id} GET - Retrieve a resource by id.""" + raise MethodNotAllowed() + + def post(self): + """/ POST - Disaggregate and save data""" + response = self.get_controller().post( + self.request_json(), request.args) + return self.json(response) + + def put(self, id_): + """/{id} PUT - Upload a new disaggregated bill """ + response = self.get_controller().put( + id_, self.request_json(), request.args) + return self.json(response) + + def delete(self, id_): + raise MethodNotAllowed() -- GitLab From e4bea04c2b0885f8688d3dc6ac68d7649e687b3f Mon Sep 17 00:00:00 2001 From: Conrad Date: Tue, 26 Sep 2017 17:36:58 -0400 Subject: [PATCH 07/16] Add account_address to model --- app/models/account.py | 1 + 1 file changed, 1 insertion(+) diff --git a/app/models/account.py b/app/models/account.py index f256942..e421118 100644 --- a/app/models/account.py +++ b/app/models/account.py @@ -30,6 +30,7 @@ class Account(Model, Tracked, db.Model): label_created_by_id = db.Column(db.Unicode(50)) label_created_by_name = db.Column(db.Unicode(50)) scrape_date= db.Column(db.Date) + account_address = db.Column(db.Unicode(50)) type = None # Parse datetime into a string -- GitLab From 4eec871348eb08356151c1461842f86095f419d4 Mon Sep 17 00:00:00 2001 From: Conrad Date: Wed, 27 Sep 2017 11:36:12 -0400 Subject: [PATCH 08/16] Update scrape address after scraping --- app/controllers/scrape.py | 20 ++++++++++++++++---- 1 file changed, 16 insertions(+), 4 deletions(-) diff --git a/app/controllers/scrape.py b/app/controllers/scrape.py index 2644c44..c8eff70 100644 --- a/app/controllers/scrape.py +++ b/app/controllers/scrape.py @@ -102,10 +102,14 @@ class ScrapeController(RestController): account_id, account_number, ) - ScrapeController.update_account_address(account_address, account_id) + scrape_date = datetime.utcnow() + ScrapeController.update_scrape_address_and_date(account_address, scrape_date, account_id) + time_format = "%m/%d/%Y" + scrape_date_string = scrape_date.strftime(time_format) return { "account_address": account_address, + "scrape_date": scrape_date_string, "utility_bills": results, "bill_database_output": database_format, } @@ -189,6 +193,7 @@ class ScrapeController(RestController): {'account_id': account_id}, ) bill['bill_database_output'] = [val.get_dictionary() for val in new_bill_database_output] + ScrapeController.update_scrape_address_and_date(None, None, account_id) return {**bill} @@ -268,13 +273,20 @@ class ScrapeController(RestController): return splitup[2], splitup[0], splitup[1] @staticmethod - def update_account_address(account_address, account_id): + def update_scrape_address_and_date(account_address, scrape_date, account_id): """Commit the r_squared value to the database""" if account_address: - query = 'UPDATE public.account SET account_address = :account_address WHERE id={}'.format(account_id) + query = 'UPDATE public.account SET {}, {} WHERE id={}'.format( + 'account_address = :account_address', + 'scrape_date = :scrape_date', + account_id, + ) results = db.session.execute( query, - {'account_address': account_address}, + { + 'account_address': account_address, + 'scrape_date': scrape_date, + }, ) db.session.commit() -- GitLab From 126edd36afaab1bfb0428ebe47d5afd98d6656c0 Mon Sep 17 00:00:00 2001 From: Conrad Date: Wed, 27 Sep 2017 16:14:22 -0400 Subject: [PATCH 09/16] Add put for account endpoint --- app/controllers/account.py | 175 +++++++++++++++++++++---------------- app/controllers/scrape.py | 1 + app/forms/account.py | 12 +++ app/models/account.py | 2 +- app/views/account.py | 3 - 5 files changed, 116 insertions(+), 77 deletions(-) diff --git a/app/controllers/account.py b/app/controllers/account.py index e8b0924..a2564d6 100644 --- a/app/controllers/account.py +++ b/app/controllers/account.py @@ -1,5 +1,5 @@ """Controllers for posting or manipulating a account.""" -from flask import g +from flask import g, current_app from .base import RestController from ..lib.database import db from ..models.account import Account @@ -53,39 +53,8 @@ class AccountController(RestController): :return: Account data """ - - account_provider = '' - account_type = '' utility = data.pop('utility') - if utility == 'con_edison_electric': - account_provider = self.UTILITY_PROVIDERS['con_edison'] - account_type = self.UTILITY_TYPES['electric'] - elif utility == 'con_edison_gas': - account_provider = self.UTILITY_PROVIDERS['con_edison'] - account_type = self.UTILITY_TYPES['gas'] - elif utility == 'national_grid_gas': - account_provider = self.UTILITY_PROVIDERS['national_grid'] - account_type = self.UTILITY_TYPES['gas'] - elif utility == 'other_electric': - account_provider = self.UTILITY_PROVIDERS['other'] - account_type = self.UTILITY_TYPES['electric'] - elif utility == 'other_gas': - account_provider = self.UTILITY_PROVIDERS['other'] - account_type = self.UTILITY_TYPES['gas'] - elif utility == 'other_oil': - account_provider = self.UTILITY_PROVIDERS['other'] - account_type = self.UTILITY_TYPES['oil'] - elif utility == 'other_water': - account_provider = self.UTILITY_PROVIDERS['other'] - account_type = self.UTILITY_TYPES['water'] - elif utility == 'con_edison_detail_electric': - account_provider = self.UTILITY_PROVIDERS['con_edison_detail'] - account_type = self.UTILITY_TYPES['electric'] - elif utility == 'con_edison_detail_gas': - account_provider = self.UTILITY_PROVIDERS['con_edison_detail'] - account_type = self.UTILITY_TYPES['gas'] - else: - raise BadRequest('Invalid utility {} given'.format(self.utility)) + account_provider, account_type = self.utility_to_provider_type(utility) if not data['usage_type']: data['usage_type'] = 1 data['account_provider'] = account_provider @@ -108,47 +77,16 @@ class AccountController(RestController): self.utility_email_notification(email_address) return super().post(data, filter_data) - def utility_email_notification(self, to_address=None): - """ send out email notification to request for manual input """ - from_address = current_app.config.get("EMAIL_SENDING_ADDRESS") - from_password = current_app.config.get("EMAIL_SENDING_PASSWORD") - if not to_address: - to_address = current_app.config.get("EMAIL_RECEIVING_ADDRESS") - smtp = current_app.config.get("EMAIL_SENDING_SMTP") - port = current_app.config.get("EMAIL_SENDING_PORT") - msg = MIMEMultipart() - msg['From'] = from_address - msg['To'] = to_address - msg['Subject'] = "Notification for NationalGrid Utility Account " - - body = 'In order for the scraper to work, the account number and '\ - 'access code need to be added to blocpower national grid ' \ - 'account. Please add the folllowing account number ' \ - 'and access code into the NationalGrid account.\n\n' \ - 'account number: {}\n' \ - 'access_code: {}\n\n' \ - 'To add the accout number and access code to the blocpower '\ - 'account navigate to https://online.nationalgridus.com/login'\ - '/LoginActivate?applicurl=aHR0cHM6Ly9vbmxpbmUubmF0aW9uY'\ - 'WxncmlkdXMuY29tL2VzZXJ2aWNlX2VudQ==&auth_method=0 '\ - 'and login with user BlocpowerTest (ask a developer for '\ - 'the password). Once logged, click "Add Account" on the right '\ - 'side.'.format(self.account_number, self.access_code) - msg.attach(MIMEText(body, 'plain')) + def put(self, id_, data, filter_data): + utility = data.pop('utility') + account_provider, account_type = self.utility_to_provider_type(utility) + if not data['usage_type']: + data['usage_type'] = 1 + data['account_provider'] = account_provider + data['account_type'] = account_type + data['updated_by_id'] = g.get('sub', None) + return super().put(id_, data, filter_data) - try: - server = smtplib.SMTP(smtp, port) - server.starttls() - server.login(from_address, from_password) - text = msg.as_string() - server.sendmail(from_address, to_address, text) - server.quit() - except smtplib.SMTPAuthenticationError as e: - if current_app.config['DEBUG']: - raise e - raise Unauthorized('The username or password does not match for'\ - ' the email account being sent a notification.'\ - ' Please contact the development team.') def index(self, filter_data): data = super().index(filter_data) @@ -202,3 +140,94 @@ class AccountController(RestController): 'deleted_disaggregated_data': deleted_disaggregated_data, 'deleted_bill_data': deleted_bill_data, } + + def utility_to_provider_type(self, utility): + if utility == 'con_edison_electric': + return ( + self.UTILITY_PROVIDERS['con_edison'], + self.UTILITY_TYPES['electric'], + ) + elif utility == 'con_edison_gas': + return ( + self.UTILITY_PROVIDERS['con_edison'], + self.UTILITY_TYPES['gas'], + ) + elif utility == 'national_grid_gas': + return ( + self.UTILITY_PROVIDERS['national_grid'], + self.UTILITY_TYPES['gas'], + ) + elif utility == 'other_electric': + return ( + self.UTILITY_PROVIDERS['other'], + self.UTILITY_TYPES['electric'], + ) + elif utility == 'other_gas': + return ( + self.UTILITY_PROVIDERS['other'], + self.UTILITY_TYPES['gas'], + ) + elif utility == 'other_oil': + return ( + self.UTILITY_PROVIDERS['other'], + self.UTILITY_TYPES['oil'], + ) + elif utility == 'other_water': + return ( + self.UTILITY_PROVIDERS['other'], + self.UTILITY_TYPES['water'], + ) + elif utility == 'con_edison_detail_electric': + return ( + self.UTILITY_PROVIDERS['con_edison_detail'], + self.UTILITY_TYPES['electric'], + ) + elif utility == 'con_edison_detail_gas': + return ( + self.UTILITY_PROVIDERS['con_edison_detail'], + self.UTILITY_TYPES['gas'], + ) + else: + raise BadRequest('Invalid utility {} given'.format(self.utility)) + + def utility_email_notification(self, to_address=None): + """ send out email notification to request for manual input """ + from_address = current_app.config.get("EMAIL_SENDING_ADDRESS") + from_password = current_app.config.get("EMAIL_SENDING_PASSWORD") + if not to_address: + to_address = current_app.config.get("EMAIL_RECEIVING_ADDRESS") + smtp = current_app.config.get("EMAIL_SENDING_SMTP") + port = current_app.config.get("EMAIL_SENDING_PORT") + msg = MIMEMultipart() + msg['From'] = from_address + msg['To'] = to_address + msg['Subject'] = "Notification for NationalGrid Utility Account " + + body = 'In order for the scraper to work, the account number and '\ + 'access code need to be added to blocpower national grid ' \ + 'account. Please add the folllowing account number ' \ + 'and access code into the NationalGrid account.\n\n' \ + 'account number: {}\n' \ + 'access_code: {}\n\n' \ + 'To add the accout number and access code to the blocpower '\ + 'account navigate to https://online.nationalgridus.com/login'\ + '/LoginActivate?applicurl=aHR0cHM6Ly9vbmxpbmUubmF0aW9uY'\ + 'WxncmlkdXMuY29tL2VzZXJ2aWNlX2VudQ==&auth_method=0 '\ + 'and login with user BlocpowerTest (ask a developer for '\ + 'the password). Once logged, click "Add Account" on the right '\ + 'side.'.format(self.account_number, self.access_code) + msg.attach(MIMEText(body, 'plain')) + + try: + server = smtplib.SMTP(smtp, port) + server.starttls() + server.login(from_address, from_password) + text = msg.as_string() + server.sendmail(from_address, to_address, text) + server.quit() + except smtplib.SMTPAuthenticationError as e: + if current_app.config['DEBUG']: + raise e + raise Unauthorized('The username or password does not match for'\ + ' the email account being sent a notification.'\ + ' Please contact the development team.') diff --git a/app/controllers/scrape.py b/app/controllers/scrape.py index c8eff70..c927ac0 100644 --- a/app/controllers/scrape.py +++ b/app/controllers/scrape.py @@ -1,4 +1,5 @@ """Controllers for posting or manipulating a scrape.""" +from flask import current_app from .base import RestController from ..controllers.disaggregated_bill import DisaggregatedBillController from ..controllers.bill import BillController diff --git a/app/forms/account.py b/app/forms/account.py index ab36a31..1f993e9 100644 --- a/app/forms/account.py +++ b/app/forms/account.py @@ -4,6 +4,9 @@ import wtforms as wtf class AccountForm(wtf.Form): """ A form for validating account requests.""" + id = wtf.IntegerField( + validators=[wtf.validators.Optional()] + ) building_id = wtf.StringField( validators=[wtf.validators.Required()]) account_provider = wtf.IntegerField() @@ -20,3 +23,12 @@ class AccountForm(wtf.Form): email_address = wtf.StringField( validators=[wtf.validators.Optional()] ) + updated_by_id = wtf.StringField( + validators=[wtf.validators.Optional()] + ) + created_by_id = wtf.StringField( + validators=[wtf.validators.Optional()] + ) + label = wtf.StringField( + validators=[wtf.validators.Optional()] + ) diff --git a/app/models/account.py b/app/models/account.py index e421118..48c3587 100644 --- a/app/models/account.py +++ b/app/models/account.py @@ -26,7 +26,7 @@ class Account(Model, Tracked, db.Model): created_by_name = db.Column(db.Unicode(50)) updated_by_id = db.Column(db.Unicode(50)) updated_by_name = db.Column(db.Unicode(50)) - label = db.Column(db.Unicode(50)) + label = db.Column(db.Unicode(75)) label_created_by_id = db.Column(db.Unicode(50)) label_created_by_name = db.Column(db.Unicode(50)) scrape_date= db.Column(db.Date) diff --git a/app/views/account.py b/app/views/account.py index ed8f9c4..086d8d7 100644 --- a/app/views/account.py +++ b/app/views/account.py @@ -30,9 +30,6 @@ class AccountView(RestView): """/{id} GET - Retrieve a resource by id.""" return self.json(self.get_controller().get(id_, request.args)) - def put(self, id_): - raise MethodNotAllowed() - def delete(self, id_): response = self.get_controller().delete( id_, request.args) -- GitLab From 476ca225c3c785dc35af1bae495a597a738a540f Mon Sep 17 00:00:00 2001 From: Conrad Date: Fri, 29 Sep 2017 10:24:29 -0400 Subject: [PATCH 10/16] Reflect new database tables --- app/controllers/account.py | 10 +++++++--- app/controllers/scrape.py | 26 +++++++++++++------------- app/forms/bill.py | 3 +++ app/lib/utilitybillscraper | 2 +- app/models/account.py | 4 ---- app/models/bill.py | 3 +++ 6 files changed, 27 insertions(+), 21 deletions(-) diff --git a/app/controllers/account.py b/app/controllers/account.py index a2564d6..7d1c8f9 100644 --- a/app/controllers/account.py +++ b/app/controllers/account.py @@ -74,7 +74,11 @@ class AccountController(RestController): data['access_code'] and utility == "national_grid_gas" ): - self.utility_email_notification(email_address) + self.utility_email_notification( + data['account_number'], + data['access_code'], + email_address, + ) return super().post(data, filter_data) def put(self, id_, data, filter_data): @@ -190,7 +194,7 @@ class AccountController(RestController): else: raise BadRequest('Invalid utility {} given'.format(self.utility)) - def utility_email_notification(self, to_address=None): + def utility_email_notification(self, account_number, access_code, to_address=None): """ send out email notification to request for manual input """ from_address = current_app.config.get("EMAIL_SENDING_ADDRESS") from_password = current_app.config.get("EMAIL_SENDING_PASSWORD") @@ -215,7 +219,7 @@ class AccountController(RestController): 'WxncmlkdXMuY29tL2VzZXJ2aWNlX2VudQ==&auth_method=0 '\ 'and login with user BlocpowerTest (ask a developer for '\ 'the password). Once logged, click "Add Account" on the right '\ - 'side.'.format(self.account_number, self.access_code) + 'side.'.format(account_number, access_code) msg.attach(MIMEText(body, 'plain')) try: diff --git a/app/controllers/scrape.py b/app/controllers/scrape.py index c927ac0..4d0c354 100644 --- a/app/controllers/scrape.py +++ b/app/controllers/scrape.py @@ -52,7 +52,6 @@ class ScrapeController(RestController): data['password'], data['account_id'] ) - # Save the scraped bill data in the database BillController().post( scraped_bill['bill_database_output'], @@ -89,7 +88,7 @@ class ScrapeController(RestController): utility, account_number, )) - raise NotFound('The utility bill was not found. Are you sure the account number is correct?') + raise NotFound('The utility bill was not found. Are you sure the account number is correct and that you\'ve added it to our national grid account?') except ConnectionError as e: current_app.logger.info('Connection error with utility {} and account number {}'.format(utility, account_number)) raise ServiceUnavailable( @@ -162,6 +161,9 @@ class ScrapeController(RestController): delivery_charge = float(bill.get('delivery_charge')) supply_charge = float(bill.get('supply_charge')) + peak_demand = bill.get('peak_demand', 0) + delivery_tax = bill.get('delivery_tax', 0) + esco_charge = bill.get('esco_charge', 0) database_output.append({ 'account_id': account_id, @@ -169,8 +171,11 @@ class ScrapeController(RestController): 'bill_to_date': new_end_date, 'usage': float(bill.get('actual_usage')), 'delivery_charge': delivery_charge, + 'delivery_tax': delivery_tax, 'supply_charge': supply_charge, + 'esco_charge': esco_charge, 'total_charge_bill': float(bill.get("charge")), + 'demand': peak_demand, }) # Reverse the order of the list as the scraper returns it in reverse chronological order database_output = database_output[::-1] @@ -221,7 +226,9 @@ class ScrapeController(RestController): for row in rows: usage = ScrapeController.parse_float(row["Usage"]) supply_charge = ScrapeController.parse_float(row["Supply Charge"]) + esco_charge = ScrapeController.parse_float(row["ESCO Charge"]) delivery_charge = ScrapeController.parse_float(row["Delivery Charge"]) + delivery_tax = ScrapeController.parse_float(row["Delivery Tax"]) total_charge = ScrapeController.parse_float(row["Total Charge"]) if not row["Bill From Date"] or not row["Bill To Date"] or not row["Days In Bill"]: raise BadRequest('Missing some data in the Bill From Date, Bill To Date, or Days In Bill column') @@ -229,23 +236,15 @@ class ScrapeController(RestController): to_date = datetime.strptime(row["Bill To Date"], '%m/%d/%Y') days_in_bill = (to_date - from_date).days new_row = {} - new_csv_row = [ - row["Bill From Date"], - row["Bill To Date"], - days_in_bill, - usage, - supply_charge, - delivery_charge, - total_charge, - ] - csv_writer.writerow(new_csv_row) new_output.append({ "account_id": account_id, "bill_from_date": row["Bill From Date"], "bill_to_date": row["Bill To Date"], "usage": usage, "supply_charge": supply_charge, + "esco_charge": esco_charge, "delivery_charge": delivery_charge, + "delivery_tax": delivery_tax, "total_charge_bill": total_charge, }) except ValueError as e: @@ -261,11 +260,12 @@ class ScrapeController(RestController): new_output, key=ScrapeController.sort_by_bill_from_date ), - "bill_csv_output": csv_output.getvalue(), } @staticmethod def parse_float(val): + if val is None: + return None return float(str(val).replace('$', '').replace(',', '')) @staticmethod diff --git a/app/forms/bill.py b/app/forms/bill.py index 1a0b034..5be0a66 100644 --- a/app/forms/bill.py +++ b/app/forms/bill.py @@ -10,5 +10,8 @@ class BillForm(wtf.Form): bill_to_date = wtf.DateField(format='%m/%d/%Y') usage = wtf.DecimalField() delivery_charge = wtf.DecimalField() + delivery_tax = wtf.DecimalField() supply_charge = wtf.DecimalField() + esco_charge = wtf.DecimalField() + demand = wtf.DecimalField() total_charge_bill = wtf.DecimalField() diff --git a/app/lib/utilitybillscraper b/app/lib/utilitybillscraper index 3618f7d..a57b867 160000 --- a/app/lib/utilitybillscraper +++ b/app/lib/utilitybillscraper @@ -1 +1 @@ -Subproject commit 3618f7d2ab0ab5e54ba7666cb00ecf86d492fd4b +Subproject commit a57b867a07ae522b36a4fd33ee8b7daf07dc53e6 diff --git a/app/models/account.py b/app/models/account.py index 48c3587..77520d3 100644 --- a/app/models/account.py +++ b/app/models/account.py @@ -23,12 +23,8 @@ class Account(Model, Tracked, db.Model): usage_type = db.Column(db.Integer) r_squared = db.Column(db.Float) created_by_id = db.Column(db.Unicode(50)) - created_by_name = db.Column(db.Unicode(50)) updated_by_id = db.Column(db.Unicode(50)) - updated_by_name = db.Column(db.Unicode(50)) label = db.Column(db.Unicode(75)) - label_created_by_id = db.Column(db.Unicode(50)) - label_created_by_name = db.Column(db.Unicode(50)) scrape_date= db.Column(db.Date) account_address = db.Column(db.Unicode(50)) type = None diff --git a/app/models/bill.py b/app/models/bill.py index 1b032e1..c78a21a 100644 --- a/app/models/bill.py +++ b/app/models/bill.py @@ -19,7 +19,10 @@ class Bill(Model, db.Model): bill_to_date = db.Column(db.Unicode(255)) usage = db.Column(db.Float) delivery_charge = db.Column(db.Float) + delivery_tax = db.Column(db.Float) supply_charge = db.Column(db.Float) + esco_charge = db.Column(db.Float) + demand = db.Column(db.Float) total_charge_bill = db.Column(db.Float) # Parse datetime into a string -- GitLab From 1d20c278586a42937ac46871f1d71c02bcddd41f Mon Sep 17 00:00:00 2001 From: Conrad Date: Mon, 2 Oct 2017 16:27:25 -0400 Subject: [PATCH 11/16] Remove standard_login_need for POST endpoint for account --- app/views/account.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/app/views/account.py b/app/views/account.py index a186d68..2a2e3a4 100644 --- a/app/views/account.py +++ b/app/views/account.py @@ -1,12 +1,15 @@ """Views for working with accounts.""" from werkzeug.exceptions import MethodNotAllowed, BadRequest from ..controllers.account import AccountController -from .base import RestView +from .base import UnprotectedRestView from ..permissions.application import app_need +from ..permissions.auth import standard_login_need from flask import request -class AccountView(RestView): +class AccountView(UnprotectedRestView): + decorators = (app_need,) + """The account view.""" def get_controller(self): @@ -16,6 +19,7 @@ class AccountView(RestView): def index(self): raise MethodNotAllowed() + @standard_login_need def get(self, id_): """/{id} GET - Retrieve a resource by id.""" return self.json(self.get_controller().get(id_, request.args)) @@ -26,9 +30,11 @@ class AccountView(RestView): self.request_json(), request.args) return self.json(response) + @standard_login_need def put(self, id_): raise MethodNotAllowed() + @standard_login_need def delete(self, id_): response = self.get_controller().delete( id_, request.args) -- GitLab From 288c3a3af2ca8ce20c3f3cfdec5f0020bd790657 Mon Sep 17 00:00:00 2001 From: Conrad Date: Tue, 3 Oct 2017 10:56:44 -0400 Subject: [PATCH 12/16] Remove post function in view --- app/views/account.py | 6 ------ 1 file changed, 6 deletions(-) diff --git a/app/views/account.py b/app/views/account.py index 5f30f88..5353345 100644 --- a/app/views/account.py +++ b/app/views/account.py @@ -36,12 +36,6 @@ class AccountView(UnprotectedRestView): """/{id} GET - Retrieve a resource by id.""" return self.json(self.get_controller().get(id_, request.args)) - def post(self): - """/ POST - Save utility account data given in POST body""" - response = self.get_controller().post( - self.request_json(), request.args) - return self.json(response) - @standard_login_need def put(self, id_): super().put(id_) -- GitLab From a516d56185f6904ff920686f4d345d7d796196ff Mon Sep 17 00:00:00 2001 From: Conrad Date: Thu, 5 Oct 2017 16:56:43 -0400 Subject: [PATCH 13/16] Add automate filter params for account and scrape endpoints to trigger the next step --- app/controllers/account.py | 30 +++++++++++++++++++++++++++++- app/controllers/scrape.py | 30 ++++++++++++++++++++++-------- 2 files changed, 51 insertions(+), 9 deletions(-) diff --git a/app/controllers/account.py b/app/controllers/account.py index 7d1c8f9..fb7cdc5 100644 --- a/app/controllers/account.py +++ b/app/controllers/account.py @@ -5,6 +5,7 @@ from ..lib.database import db from ..models.account import Account from .disaggregated_bill import DisaggregatedBillController from .bill import BillController +from .scrape import ScrapeController from ..forms.account import AccountForm from werkzeug.datastructures import MultiDict @@ -13,6 +14,7 @@ from datetime import datetime from email.mime.multipart import MIMEMultipart from email.mime.text import MIMEText import smtplib +from threading import Thread class AccountController(RestController): @@ -79,7 +81,33 @@ class AccountController(RestController): data['access_code'], email_address, ) - return super().post(data, filter_data) + + + result = super().post(data, filter_data) + if filter_data.get('automate'): + post_data = { + 'account_id': result.id, + 'utility': utility, + 'account_number': result.account_number, + 'username': result.username, + 'password': result.password, + 'building_id': result.building_id, + 'usage_type': result.usage_type, + } + thread = Thread( + target=self.automate_scrape, + args=( + current_app._get_current_object(), + post_data, + filter_data, + ) + ) + thread.start() + return result + + def automate_scrape(self, app, post_data, filter_data): + with app.app_context(): + ScrapeController().post(post_data, filter_data) def put(self, id_, data, filter_data): utility = data.pop('utility') diff --git a/app/controllers/scrape.py b/app/controllers/scrape.py index 4d0c354..a105077 100644 --- a/app/controllers/scrape.py +++ b/app/controllers/scrape.py @@ -1,24 +1,24 @@ """Controllers for posting or manipulating a scrape.""" from flask import current_app from .base import RestController -from ..controllers.disaggregated_bill import DisaggregatedBillController from ..controllers.bill import BillController +from .disaggregate import DisaggregateController from ..forms.account import AccountForm from ..lib.database import db -from ..models.account import Account from ..lib.utilitybillscraper.scraper_api.scrape import main as run_scraper +from ..models.account import Account import base64 -from io import StringIO -from datetime import datetime, timedelta import csv +from datetime import datetime, timedelta +from io import StringIO, BytesIO import json -from io import BytesIO -from werkzeug.datastructures import MultiDict +import pandas +from requests.exceptions import ConnectionError +from threading import Thread from types import SimpleNamespace from werkzeug.exceptions import BadGateway, ServiceUnavailable, NotFound, BadRequest -from requests.exceptions import ConnectionError -import pandas +from werkzeug.datastructures import MultiDict class ScrapeController(RestController): @@ -57,8 +57,22 @@ class ScrapeController(RestController): scraped_bill['bill_database_output'], {'account_id': data['account_id']}, ) + if filter_data.get('automate'): + thread = Thread( + target=self.automate_disaggregate, + args=( + current_app._get_current_object(), + data, + filter_data, + ) + ) + thread.start() return {**scraped_bill} + def automate_disaggregate(self, app, post_data, filter_data): + with app.app_context(): + DisaggregateController().post(post_data, filter_data) + def scrape( self, utility, -- GitLab From ca9e6b9ea0e659508eac16a90eb8a83ddf90503c Mon Sep 17 00:00:00 2001 From: Conrad Date: Fri, 6 Oct 2017 14:21:05 -0400 Subject: [PATCH 14/16] Send an email when account automate flag is set --- app/config/development.default.py | 8 ++++ app/config/local.default.py | 18 ++++++--- app/config/production.default.py | 8 ++++ app/config/staging.default.py | 8 ++++ app/config/test.default.py | 8 ++++ app/controllers/account.py | 67 ++++++++++++++++++++++++++----- 6 files changed, 103 insertions(+), 14 deletions(-) diff --git a/app/config/development.default.py b/app/config/development.default.py index 6fd0056..21124a3 100644 --- a/app/config/development.default.py +++ b/app/config/development.default.py @@ -40,3 +40,11 @@ EMAIL_RECEIVING_ADDRESS = os.environ['EMAIL_RECEIVING_ADDRESS'] EMAIL_SENDING_PASSWORD = os.environ['EMAIL_SENDING_PASSWORD'] EMAIL_SENDING_SMTP = os.environ['EMAIL_SENDING_SMTP'] EMAIL_SENDING_PORT = os.environ['EMAIL_SENDING_PORT'] + +# Automate config +AUTOMATE_EMAIL_SENDING_ADDRESS = os.environ['AUTOMATE_EMAIL_SENDING_ADDRESS'] +# comma seperated list of emails +AUTOMATE_EMAIL_RECEIVING_ADDRESS = os.environ['AUTOMATE_EMAIL_RECEIVING_ADDRESS'] +AUTOMATE_EMAIL_SENDING_PASSWORD = os.environ['AUTOMATE_EMAIL_SENDING_PASSWORD'] +AUTOMATE_EMAIL_SENDING_SMTP = os.environ['AUTOMATE_EMAIL_SENDING_SMTP'] +AUTOMATE_EMAIL_SENDING_PORT = os.environ['AUTOMATE_EMAIL_SENDING_PORT'] diff --git a/app/config/local.default.py b/app/config/local.default.py index b6ff77e..d2b7acd 100644 --- a/app/config/local.default.py +++ b/app/config/local.default.py @@ -35,8 +35,16 @@ HEADER_AUTH_KEY = 'x-blocpower-auth-key' HEADER_AUTH_TOKEN = 'x-blocpower-auth-token' # Email config -EMAIL_SENDING_ADDRESS = os.environ['EMAIL_SENDING_ADDRESS'] -EMAIL_RECEIVING_ADDRESS = os.environ['EMAIL_RECEIVING_ADDRESS'] -EMAIL_SENDING_PASSWORD = os.environ['EMAIL_SENDING_PASSWORD'] -EMAIL_SENDING_SMTP = os.environ['EMAIL_SENDING_SMTP'] -EMAIL_SENDING_PORT = os.environ['EMAIL_SENDING_PORT'] +EMAIL_SENDING_ADDRESS = '$EMAIL_SENDING_ADDRESS' +EMAIL_RECEIVING_ADDRESS = '$EMAIL_RECEIVING_ADDRESS' +EMAIL_SENDING_PASSWORD = '$EMAIL_SENDING_PASSWORD' +EMAIL_SENDING_SMTP = '$EMAIL_SENDING_SMTP' +EMAIL_SENDING_PORT = '$EMAIL_SENDING_PORT' + +# Automate config +AUTOMATE_EMAIL_SENDING_ADDRESS = '$AUTOMATE_EMAIL_SENDING_ADDRESS' +# comma seperated list of emails +AUTOMATE_EMAIL_RECEIVING_ADDRESS = '$AUTOMATE_EMAIL_RECEIVING_ADDRESS' +AUTOMATE_EMAIL_SENDING_PASSWORD = '$AUTOMATE_EMAIL_SENDING_PASSWORD' +AUTOMATE_EMAIL_SENDING_SMTP = '$AUTOMATE_EMAIL_SENDING_SMTP' +AUTOMATE_EMAIL_SENDING_PORT = '$AUTOMATE_EMAIL_SENDING_PORT' diff --git a/app/config/production.default.py b/app/config/production.default.py index b770f04..46da8c5 100644 --- a/app/config/production.default.py +++ b/app/config/production.default.py @@ -40,3 +40,11 @@ EMAIL_RECEIVING_ADDRESS = os.environ['EMAIL_RECEIVING_ADDRESS'] EMAIL_SENDING_PASSWORD = os.environ['EMAIL_SENDING_PASSWORD'] EMAIL_SENDING_SMTP = os.environ['EMAIL_SENDING_SMTP'] EMAIL_SENDING_PORT = os.environ['EMAIL_SENDING_PORT'] + +# Automate config +AUTOMATE_EMAIL_SENDING_ADDRESS = os.environ['AUTOMATE_EMAIL_SENDING_ADDRESS'] +# comma seperated list of emails +AUTOMATE_EMAIL_RECEIVING_ADDRESS = os.environ['AUTOMATE_EMAIL_RECEIVING_ADDRESS'] +AUTOMATE_EMAIL_SENDING_PASSWORD = os.environ['AUTOMATE_EMAIL_SENDING_PASSWORD'] +AUTOMATE_EMAIL_SENDING_SMTP = os.environ['AUTOMATE_EMAIL_SENDING_SMTP'] +AUTOMATE_EMAIL_SENDING_PORT = os.environ['AUTOMATE_EMAIL_SENDING_PORT'] diff --git a/app/config/staging.default.py b/app/config/staging.default.py index d3a80a1..f0c2450 100644 --- a/app/config/staging.default.py +++ b/app/config/staging.default.py @@ -40,3 +40,11 @@ EMAIL_RECEIVING_ADDRESS = os.environ['EMAIL_RECEIVING_ADDRESS'] EMAIL_SENDING_PASSWORD = os.environ['EMAIL_SENDING_PASSWORD'] EMAIL_SENDING_SMTP = os.environ['EMAIL_SENDING_SMTP'] EMAIL_SENDING_PORT = os.environ['EMAIL_SENDING_PORT'] + +# Automate config +AUTOMATE_EMAIL_SENDING_ADDRESS = os.environ['AUTOMATE_EMAIL_SENDING_ADDRESS'] +# comma seperated list of emails +AUTOMATE_EMAIL_RECEIVING_ADDRESS = os.environ['AUTOMATE_EMAIL_RECEIVING_ADDRESS'] +AUTOMATE_EMAIL_SENDING_PASSWORD = os.environ['AUTOMATE_EMAIL_SENDING_PASSWORD'] +AUTOMATE_EMAIL_SENDING_SMTP = os.environ['AUTOMATE_EMAIL_SENDING_SMTP'] +AUTOMATE_EMAIL_SENDING_PORT = os.environ['AUTOMATE_EMAIL_SENDING_PORT'] diff --git a/app/config/test.default.py b/app/config/test.default.py index abe7786..11ce4d4 100644 --- a/app/config/test.default.py +++ b/app/config/test.default.py @@ -39,6 +39,14 @@ HEADER_AUTH_TOKEN = 'x-blocpower-auth-token' UTILITY_SCRAPER_KEY = os.environ['UTILITY_SCRAPER_KEY'] UTILITY_SCRAPER_REFERRER = os.environ['UTILITY_SCRAPER_REFERRER'] +# Automate config +AUTOMATE_EMAIL_SENDING_ADDRESS = os.environ['AUTOMATE_EMAIL_SENDING_ADDRESS'] +# comma seperated list of emails +AUTOMATE_EMAIL_RECEIVING_ADDRESS = os.environ['AUTOMATE_EMAIL_RECEIVING_ADDRESS'] +AUTOMATE_EMAIL_SENDING_PASSWORD = os.environ['AUTOMATE_EMAIL_SENDING_PASSWORD'] +AUTOMATE_EMAIL_SENDING_SMTP = os.environ['AUTOMATE_EMAIL_SENDING_SMTP'] +AUTOMATE_EMAIL_SENDING_PORT = os.environ['AUTOMATE_EMAIL_SENDING_PORT'] + # HACK Fix issue where raising an exception in a test context creates an extra # request. PRESERVE_CONTEXT_ON_EXCEPTION = False diff --git a/app/controllers/account.py b/app/controllers/account.py index fb7cdc5..a13dc69 100644 --- a/app/controllers/account.py +++ b/app/controllers/account.py @@ -76,7 +76,7 @@ class AccountController(RestController): data['access_code'] and utility == "national_grid_gas" ): - self.utility_email_notification( + self.national_grid_email_notification( data['account_number'], data['access_code'], email_address, @@ -94,7 +94,7 @@ class AccountController(RestController): 'building_id': result.building_id, 'usage_type': result.usage_type, } - thread = Thread( + scrape_thread = Thread( target=self.automate_scrape, args=( current_app._get_current_object(), @@ -102,7 +102,12 @@ class AccountController(RestController): filter_data, ) ) - thread.start() + scrape_thread.start() + self.automate_email_notification( + result.account_number, + utility, + result.building_id, + ) return result def automate_scrape(self, app, post_data, filter_data): @@ -222,7 +227,7 @@ class AccountController(RestController): else: raise BadRequest('Invalid utility {} given'.format(self.utility)) - def utility_email_notification(self, account_number, access_code, to_address=None): + def national_grid_email_notification(self, account_number, access_code, to_address=None): """ send out email notification to request for manual input """ from_address = current_app.config.get("EMAIL_SENDING_ADDRESS") from_password = current_app.config.get("EMAIL_SENDING_PASSWORD") @@ -230,10 +235,7 @@ class AccountController(RestController): to_address = current_app.config.get("EMAIL_RECEIVING_ADDRESS") smtp = current_app.config.get("EMAIL_SENDING_SMTP") port = current_app.config.get("EMAIL_SENDING_PORT") - msg = MIMEMultipart() - msg['From'] = from_address - msg['To'] = to_address - msg['Subject'] = "Notification for NationalGrid Utility Account " + subject = "Notification for NationalGrid Utility Account " body = 'In order for the scraper to work, the account number and '\ 'access code need to be added to blocpower national grid ' \ @@ -248,8 +250,55 @@ class AccountController(RestController): 'and login with user BlocpowerTest (ask a developer for '\ 'the password). Once logged, click "Add Account" on the right '\ 'side.'.format(account_number, access_code) - msg.attach(MIMEText(body, 'plain')) + self.send_email( + from_address, + from_password, + to_address, + smtp, + port, + subject, + body, + ) + + def automate_email_notification(self, account_number, utility, building_id): + from_address = current_app.config.get("AUTOMATE_EMAIL_SENDING_ADDRESS") + from_password = current_app.config.get("AUTOMATE_EMAIL_SENDING_PASSWORD") + to_address_list = current_app.config.get("AUTOMATE_EMAIL_RECEIVING_ADDRESS") + smtp = current_app.config.get("AUTOMATE_EMAIL_SENDING_SMTP") + port = current_app.config.get("AUTOMATE_EMAIL_SENDING_PORT") + subject = "Notification for BlocMaps automated input " + + body = 'An account for {} (account number {}) has been automatically '\ + 'added to the dashboard, probably from blocmaps. Please navigate to '\ + 'dashboard.blocpower.io/buildings/{}/utilities to '\ + 'view the bill'.format(utility, account_number, building_id) + for to_address in to_address_list.split(','): + self.send_email( + from_address, + from_password, + to_address, + smtp, + port, + subject, + body, + ) + + def send_email( + self, + from_address, + from_password, + to_address, + smtp, + port, + subject, + body, + ): + msg = MIMEMultipart() + msg['From'] = from_address + msg['To'] = to_address + msg['Subject'] = subject + msg.attach(MIMEText(body, 'plain')) try: server = smtplib.SMTP(smtp, port) server.starttls() -- GitLab From ca9737a947044897a3a96e16e7c3b5bebbed7b09 Mon Sep 17 00:00:00 2001 From: Conrad Date: Mon, 9 Oct 2017 16:32:39 -0400 Subject: [PATCH 15/16] Email notification chagnes --- app/config/development.default.py | 4 ++ app/config/local.default.py | 4 ++ app/config/production.default.py | 4 ++ app/config/staging.default.py | 4 ++ app/config/test.default.py | 4 ++ app/controllers/account.py | 62 ++++++++++++++++++++++++++----- app/controllers/scrape.py | 14 ------- 7 files changed, 72 insertions(+), 24 deletions(-) diff --git a/app/config/development.default.py b/app/config/development.default.py index 21124a3..e5fab21 100644 --- a/app/config/development.default.py +++ b/app/config/development.default.py @@ -48,3 +48,7 @@ AUTOMATE_EMAIL_RECEIVING_ADDRESS = os.environ['AUTOMATE_EMAIL_RECEIVING_ADDRESS' AUTOMATE_EMAIL_SENDING_PASSWORD = os.environ['AUTOMATE_EMAIL_SENDING_PASSWORD'] AUTOMATE_EMAIL_SENDING_SMTP = os.environ['AUTOMATE_EMAIL_SENDING_SMTP'] AUTOMATE_EMAIL_SENDING_PORT = os.environ['AUTOMATE_EMAIL_SENDING_PORT'] + +# URLs for sending emails about accounts +BLOCLINK_URL = os.environ['BLOCLINK_URL'] +DASHBOARD_URL = os.environ['DASHBOARD_URL'] diff --git a/app/config/local.default.py b/app/config/local.default.py index d2b7acd..6a9be78 100644 --- a/app/config/local.default.py +++ b/app/config/local.default.py @@ -48,3 +48,7 @@ AUTOMATE_EMAIL_RECEIVING_ADDRESS = '$AUTOMATE_EMAIL_RECEIVING_ADDRESS' AUTOMATE_EMAIL_SENDING_PASSWORD = '$AUTOMATE_EMAIL_SENDING_PASSWORD' AUTOMATE_EMAIL_SENDING_SMTP = '$AUTOMATE_EMAIL_SENDING_SMTP' AUTOMATE_EMAIL_SENDING_PORT = '$AUTOMATE_EMAIL_SENDING_PORT' + +# URLs for sending emails about accounts +BLOCLINK_URL = '$BLOCLINK_URL' +DASHBOARD_URL = '$DASHBOARD_URL' diff --git a/app/config/production.default.py b/app/config/production.default.py index 46da8c5..039ab14 100644 --- a/app/config/production.default.py +++ b/app/config/production.default.py @@ -48,3 +48,7 @@ AUTOMATE_EMAIL_RECEIVING_ADDRESS = os.environ['AUTOMATE_EMAIL_RECEIVING_ADDRESS' AUTOMATE_EMAIL_SENDING_PASSWORD = os.environ['AUTOMATE_EMAIL_SENDING_PASSWORD'] AUTOMATE_EMAIL_SENDING_SMTP = os.environ['AUTOMATE_EMAIL_SENDING_SMTP'] AUTOMATE_EMAIL_SENDING_PORT = os.environ['AUTOMATE_EMAIL_SENDING_PORT'] + +# URLs for sending emails about accounts +BLOCLINK_URL = os.environ['BLOCLINK_URL'] +DASHBOARD_URL = os.environ['DASHBOARD_URL'] diff --git a/app/config/staging.default.py b/app/config/staging.default.py index f0c2450..88db037 100644 --- a/app/config/staging.default.py +++ b/app/config/staging.default.py @@ -48,3 +48,7 @@ AUTOMATE_EMAIL_RECEIVING_ADDRESS = os.environ['AUTOMATE_EMAIL_RECEIVING_ADDRESS' AUTOMATE_EMAIL_SENDING_PASSWORD = os.environ['AUTOMATE_EMAIL_SENDING_PASSWORD'] AUTOMATE_EMAIL_SENDING_SMTP = os.environ['AUTOMATE_EMAIL_SENDING_SMTP'] AUTOMATE_EMAIL_SENDING_PORT = os.environ['AUTOMATE_EMAIL_SENDING_PORT'] + +# URLs for sending emails about accounts +BLOCLINK_URL = os.environ['BLOCLINK_URL'] +DASHBOARD_URL = os.environ['DASHBOARD_URL'] diff --git a/app/config/test.default.py b/app/config/test.default.py index 11ce4d4..9ed5722 100644 --- a/app/config/test.default.py +++ b/app/config/test.default.py @@ -47,6 +47,10 @@ AUTOMATE_EMAIL_SENDING_PASSWORD = os.environ['AUTOMATE_EMAIL_SENDING_PASSWORD'] AUTOMATE_EMAIL_SENDING_SMTP = os.environ['AUTOMATE_EMAIL_SENDING_SMTP'] AUTOMATE_EMAIL_SENDING_PORT = os.environ['AUTOMATE_EMAIL_SENDING_PORT'] +# URLs for sending emails about accounts +BLOCLINK_URL = os.environ['BLOCLINK_URL'] +DASHBOARD_URL = os.environ['DASHBOARD_URL'] + # HACK Fix issue where raising an exception in a test context creates an extra # request. PRESERVE_CONTEXT_ON_EXCEPTION = False diff --git a/app/controllers/account.py b/app/controllers/account.py index a13dc69..7f22cce 100644 --- a/app/controllers/account.py +++ b/app/controllers/account.py @@ -5,6 +5,7 @@ from ..lib.database import db from ..models.account import Account from .disaggregated_bill import DisaggregatedBillController from .bill import BillController +from .disaggregate import DisaggregateController from .scrape import ScrapeController from ..forms.account import AccountForm @@ -74,7 +75,8 @@ class AccountController(RestController): not data['username'] and not data['password'] and data['access_code'] and - utility == "national_grid_gas" + utility == "national_grid_gas" and + not filter_data.get('automate') ): self.national_grid_email_notification( data['account_number'], @@ -100,19 +102,50 @@ class AccountController(RestController): current_app._get_current_object(), post_data, filter_data, + utility ) ) scrape_thread.start() + return result + + def automate_scrape(self, app, post_data, filter_data, utility): + with app.app_context(): + additional_text = '' + try: + ScrapeController().post(post_data, filter_data) + additional_text += '\nScraping was successful' + except Exception as e: + if utility == 'national_grid_gas': + additional_text += '\nScraping was not succesful. '\ + 'Please add this national grid account to our '\ + 'account and run the scraper again' + else: + additional_text += '\nScraping was not successful' + + additional_text += '\n' + try: + result = DisaggregateController().post(post_data, filter_data) + if (result['disaggregate_database_output']): + additional_text += '\nDisaggregation was successful' + else: + additional_text += '\nDisaggregation was not successful due to no bill data' + except Exception as e: + try: + if e.description == 'Less than a years worth of data. Cannot disaggregate': + additional_text += '\nDisaggregation was not successful because there is less than a year of data' + else: + additional_text += '\nDisaggregation was not successful' + except: + pass + additional_text += '\nDisaggregation was not successful' + self.automate_email_notification( - result.account_number, + post_data['account_number'], utility, - result.building_id, + post_data['building_id'], + additional_text, ) - return result - def automate_scrape(self, app, post_data, filter_data): - with app.app_context(): - ScrapeController().post(post_data, filter_data) def put(self, id_, data, filter_data): utility = data.pop('utility') @@ -261,7 +294,7 @@ class AccountController(RestController): ) - def automate_email_notification(self, account_number, utility, building_id): + def automate_email_notification(self, account_number, utility, building_id, additional_text): from_address = current_app.config.get("AUTOMATE_EMAIL_SENDING_ADDRESS") from_password = current_app.config.get("AUTOMATE_EMAIL_SENDING_PASSWORD") to_address_list = current_app.config.get("AUTOMATE_EMAIL_RECEIVING_ADDRESS") @@ -271,8 +304,17 @@ class AccountController(RestController): body = 'An account for {} (account number {}) has been automatically '\ 'added to the dashboard, probably from blocmaps. Please navigate to '\ - 'dashboard.blocpower.io/buildings/{}/utilities to '\ - 'view the bill'.format(utility, account_number, building_id) + '{}/buildings/{}/utilities to '\ + 'view the bill.\n\n{}\n\nA simulation has also started. Check the status at '\ + '{}/buildings/{}/envelope'.format( + utility, + account_number, + current_app.config['DASHBOARD_URL'], + building_id, + additional_text, + current_app.config['BLOCLINK_URL'], + building_id, + ) for to_address in to_address_list.split(','): self.send_email( from_address, diff --git a/app/controllers/scrape.py b/app/controllers/scrape.py index a105077..ad5acf9 100644 --- a/app/controllers/scrape.py +++ b/app/controllers/scrape.py @@ -57,22 +57,8 @@ class ScrapeController(RestController): scraped_bill['bill_database_output'], {'account_id': data['account_id']}, ) - if filter_data.get('automate'): - thread = Thread( - target=self.automate_disaggregate, - args=( - current_app._get_current_object(), - data, - filter_data, - ) - ) - thread.start() return {**scraped_bill} - def automate_disaggregate(self, app, post_data, filter_data): - with app.app_context(): - DisaggregateController().post(post_data, filter_data) - def scrape( self, utility, -- GitLab From aef92f9626aaf6aab5a6b16ad68286493f62e09e Mon Sep 17 00:00:00 2001 From: Conrad Date: Thu, 12 Oct 2017 14:33:05 -0400 Subject: [PATCH 16/16] Remove unused imports --- app/controllers/scrape.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/app/controllers/scrape.py b/app/controllers/scrape.py index 3f2093a..2c24f60 100644 --- a/app/controllers/scrape.py +++ b/app/controllers/scrape.py @@ -1,7 +1,6 @@ """Controllers for posting or manipulating a scrape.""" from .base import RestController from ..controllers.bill import BillController -from .disaggregate import DisaggregateController from ..forms.account import AccountForm from ..lib.database import db from ..lib.utilitybillscraper.scraper_api.scrape import main as run_scraper @@ -15,7 +14,6 @@ from io import StringIO, BytesIO import json import pandas from requests.exceptions import ConnectionError -from threading import Thread from types import SimpleNamespace from werkzeug.datastructures import MultiDict from werkzeug.exceptions import BadGateway, ServiceUnavailable, NotFound, BadRequest -- GitLab