From e33146760a18ebd3769d464186f21d65d5ae383c Mon Sep 17 00:00:00 2001 From: unknown Date: Wed, 14 Feb 2018 18:08:47 -0500 Subject: [PATCH 1/8] Convert single quote to double quote for docstring --- bpeng/bill/awesome_disaggregate.py | 62 +++++++++++++++--------------- 1 file changed, 31 insertions(+), 31 deletions(-) diff --git a/bpeng/bill/awesome_disaggregate.py b/bpeng/bill/awesome_disaggregate.py index 1f070c0..f7e08aa 100644 --- a/bpeng/bill/awesome_disaggregate.py +++ b/bpeng/bill/awesome_disaggregate.py @@ -60,7 +60,7 @@ class BillDisaggregation(): self.annual_usage = None def weather_cleaning(self, raw_daily_temp): - ''' + """ Format the daily temperature data from influx query Args: @@ -70,7 +70,7 @@ class BillDisaggregation(): Returns: pd.DateFrame: Returns formatted daily temperature - ''' + """ raw_daily_temp.rename( columns={'time': 'date', 'value': 'temperature'}, inplace=True) @@ -85,7 +85,7 @@ class BillDisaggregation(): return self.daily_temp def bill_period_weather(self, bill_from_date, bill_end_date): - ''' + """ get the outdoor temperature date between two date, return a list Args: @@ -96,7 +96,7 @@ class BillDisaggregation(): Returns: list: Returns a list of outdoor temperature for a period - ''' + """ end_date_id = self.daily_temp[self.daily_temp.date == bill_end_date].index[0] start_date_id = self.daily_temp[self.daily_temp.date == @@ -105,7 +105,7 @@ class BillDisaggregation(): @staticmethod def cdd(indoor_set_point, outdoor_temp): - ''' + """ CDD Assumption: cooling setting point shall always higher than 55 F, @@ -114,7 +114,7 @@ class BillDisaggregation(): ?? set_point is for indoor temperature - ''' + """ if indoor_set_point > 55: if indoor_set_point < outdoor_temp: @@ -123,12 +123,12 @@ class BillDisaggregation(): @staticmethod def hdd(indoor_set_point, outdoor_temp): - ''' + """ HDD Assumption: Only if the outdoor temperature drop below 60'F, then the heating system may be able to be turn on - ''' + """ if (outdoor_temp < 60) & (indoor_set_point > outdoor_temp): hdd = indoor_set_point - outdoor_temp else: @@ -137,14 +137,14 @@ class BillDisaggregation(): @staticmethod def threshold(data, set_point): - '''If data is less the set_point, return 0''' + """If data is less the set_point, return 0""" if data <= set_point: data = 0 return data @staticmethod def outliers_iqr(ys): - ''' + """ Find outlier using IQR method Args: @@ -157,7 +157,7 @@ class BillDisaggregation(): True: Outliner False: Not Outliner - ''' + """ quartile_1, quartile_3 = np.percentile(ys, [25, 75]) iqr = quartile_3 - quartile_1 lower_bound = quartile_1 - (iqr * 1.5) @@ -166,7 +166,7 @@ class BillDisaggregation(): @staticmethod def anomaly_point(alist, thresholds): - ''' + """ Find outlier and return its index Args: @@ -179,7 +179,7 @@ class BillDisaggregation(): list: Returns a list the index of the outliner - ''' + """ amean = [] for x in range(len(alist)): temp = np.hstack((alist[:(x)], alist[(x + 1):])) @@ -193,7 +193,7 @@ class BillDisaggregation(): @staticmethod def num_month_dates(last_date_bill, first_date_bill): - '''Return number of month in between two date ''' + """Return number of month in between two date """ lastdate = last_date_bill - timedelta(last_date_bill.day) firstdate = first_date_bill + timedelta(days=32) firstdate = firstdate.replace(day=1) @@ -202,7 +202,7 @@ class BillDisaggregation(): return (num_month) def bill_formating(self, raw_bill): - ''' + """ Bill Cleaning Step 1: 1. format each column of the raw bill @@ -223,7 +223,7 @@ class BillDisaggregation(): boolean: True - Length of the bill has changed during bill cleaning step 1 - ''' + """ bill_copy = raw_bill.copy() bill_copy['Bill From Date'] = pd.to_datetime( bill_copy['Bill From Date']) @@ -261,7 +261,7 @@ class BillDisaggregation(): return bill_formatted, bill_shape_change def bill_quality(self, bill_formatted): - ''' + """ Bill Cleaning Step 2: 1. Check each billing period to find a bill is too short or too long; @@ -273,7 +273,7 @@ class BillDisaggregation(): pd.DataFrame: a dataframe with columns: 'index': the index of the billing period which is identified as an outlier 'flag': to indicate either it is too long or too short - ''' + """ bill = bill_formatted.copy() bill = pd.DataFrame(bill) @@ -312,7 +312,7 @@ class BillDisaggregation(): return bill_quality def short_bill_consolidate(self, bill_formatted, bill_quality): - ''' + """ Bill Cleaning Step 3: consolidation of the bills that are too short compare to others @@ -325,7 +325,7 @@ class BillDisaggregation(): Returns: pd.DataFrame: the cleaned bill and ready for analysis - ''' + """ bill_quality_short = bill_quality[bill_quality['flag'] == 'short'] bill_consi = bill_formatted.copy() # consolidate the billing period that is too short compare to others @@ -391,7 +391,7 @@ class BillDisaggregation(): return bill_consi def regression_1(self, hp, cp, processed_bill): - ''' + """ A linear regression model with heating and cooling set fixed Args: @@ -406,7 +406,7 @@ class BillDisaggregation(): float: r-squared of the linear regression model 2d-array: a 2D numpy array of normalized billing period average daily HDDs and CDDs - ''' + """ bill = processed_bill.copy() @@ -446,7 +446,7 @@ class BillDisaggregation(): return regr_model, score, regression_temp def summer_dhw(self, hp, abill): - ''' + """ This funcion uses summer month gas usage as base consumption for the year A linear regression of weather-related consumption and a fixed heating system set point NOTE: USUALLY ERROR @@ -464,7 +464,7 @@ class BillDisaggregation(): 2d-array: a 2D numpy array of normalized billing period HDDs sum pd.DataFrame - ''' + """ bill = abill.copy() ahdd = [[BillDisaggregation.hdd(hp, xx) for xx in x] @@ -805,9 +805,9 @@ class BillDisaggregation(): self.annual_usage = self.annual_usage_costs(self.recent_year_bill_breakdown, non_weather_related_end_use) def benchmarking_output(self): - ''' + """ output perimeters that related with evaluating the bills - ''' + """ test = self.output_table.copy() bill_start_date = pd.to_datetime(test['Bill From Date']).iloc[0] @@ -929,7 +929,7 @@ class BillDisaggregation(): return monthly_output def non_weahter_related_breakdown(self, end_uses, monthly_output_table): - ''' + """ breakdown the non_weather_related_usage Args: @@ -941,7 +941,7 @@ class BillDisaggregation(): Returns: pd.DataFrame: bill breakdown of all end-use - ''' + """ monthly_usages = monthly_output_table.copy() eu = pd.DataFrame( @@ -959,7 +959,7 @@ class BillDisaggregation(): return monthly_usages def annual_usage_costs(self, annual_bill_breakdown, end_uses): - ''' + """ Calcuate annual usage and costs for each end use Args: @@ -970,7 +970,7 @@ class BillDisaggregation(): Return: pd.DataFrame: annual usage, costs for each end uses - ''' + """ annual_usage = pd.DataFrame(columns=['End Use', 'Usage', 'Costs']) @@ -1013,7 +1013,7 @@ class BillDisaggregation(): # @staticmethod # def projection_figure(bill): - # '''ploat the disaggregated bill''' + # """ploat the disaggregated bill""" # plt.figure(figsize=(10, 5)) # x = pd.to_datetime(bill['Bill From Date']) -- GitLab From c6a8e40e1af727ffdfa61ac5629d923ecf7ef445 Mon Sep 17 00:00:00 2001 From: unknown Date: Wed, 14 Feb 2018 18:16:36 -0500 Subject: [PATCH 2/8] change & to and --- bpeng/bill/awesome_disaggregate.py | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/bpeng/bill/awesome_disaggregate.py b/bpeng/bill/awesome_disaggregate.py index f7e08aa..ad08242 100644 --- a/bpeng/bill/awesome_disaggregate.py +++ b/bpeng/bill/awesome_disaggregate.py @@ -129,7 +129,7 @@ class BillDisaggregation(): Only if the outdoor temperature drop below 60'F, then the heating system may be able to be turn on """ - if (outdoor_temp < 60) & (indoor_set_point > outdoor_temp): + if (outdoor_temp < 60) and (indoor_set_point > outdoor_temp): hdd = indoor_set_point - outdoor_temp else: hdd = 0 @@ -335,7 +335,7 @@ class BillDisaggregation(): if bill_quality['flag'].iloc[xxx] == 'short': row_index = bill_quality_short['index'].iloc[xxx] - if (row_index != 0) & (row_index != bill_consi.index[-1]): + if (row_index != 0) and (row_index != bill_consi.index[-1]): if bill_consi['Days In Bill'][int( row_index - 1)] <= bill_consi['Days In Bill'][int( @@ -560,13 +560,13 @@ class BillDisaggregation(): heating_coef, cooling_coef = regr_model.coef_ if -opt.fun > 0.5: - if (heating_coef > 0) & (cooling_coef <= 0): + if (heating_coef > 0) and (cooling_coef <= 0): weather_related_usage = 'Heating' - elif (heating_coef <= 0) & (cooling_coef > 0): + elif (heating_coef <= 0) and (cooling_coef > 0): weather_related_usage = 'Cooling' - elif (heating_coef <= 0) & (cooling_coef <= 0): + elif (heating_coef <= 0) and (cooling_coef <= 0): weather_related_usage = 'Both Not' - elif (heating_coef >= 0) & (cooling_coef >= 0): + elif (heating_coef >= 0) and (cooling_coef >= 0): weather_related_usage = 'Both' else: weather_related_usage = 'Both Not' @@ -588,13 +588,13 @@ class BillDisaggregation(): # change accordingly for JOENYC buildings - if (heating_coef > 0) & (cooling_coef < 0): + if (heating_coef > 0) and (cooling_coef < 0): weather_related_usage = 'Heating' cooling_coef = 0 - elif (heating_coef <= 0) & (cooling_coef > 0): + elif (heating_coef <= 0) and (cooling_coef > 0): weather_related_usage = 'Cooling' heating_coef = 0 - elif (heating_coef <= 0) & (cooling_coef <= 0): + elif (heating_coef <= 0) and (cooling_coef <= 0): weather_related_usage = 'Both Not' heating_coef = 0 cooling_coef = 0 @@ -602,7 +602,7 @@ class BillDisaggregation(): # changes on Jan 17th 2018 # please futher check with more bills - elif (heating_coef > 0) & (cooling_coef > 0): + elif (heating_coef > 0) and (cooling_coef > 0): if heating_coef / cooling_coef > 5: weather_related_usage = 'Heating' cooling_coef = 0 @@ -624,17 +624,17 @@ class BillDisaggregation(): heating_set_point = opt.x[0] cooling_set_point = opt.x[1] - if (heating_coef > 0) & (cooling_coef < 0): + if (heating_coef > 0) and (cooling_coef < 0): weather_related_usage = 'Heating' cooling_coef = 0 - elif (heating_coef <= 0) & (cooling_coef > 0): + elif (heating_coef <= 0) and (cooling_coef > 0): weather_related_usage = 'Cooling' heating_coef = 0 - elif (heating_coef <= 0) & (cooling_coef <= 0): + elif (heating_coef <= 0) and (cooling_coef <= 0): weather_related_usage = 'Both Not' heating_coef = 0 cooling_coef = 0 - elif (heating_coef > 0) & (cooling_coef > 0): + elif (heating_coef > 0) and (cooling_coef > 0): if heating_coef / cooling_coef > 5: weather_related_usage = 'Heating' cooling_coef = 0 -- GitLab From abe59bf0f936f4867082173e37c82b09fd63465a Mon Sep 17 00:00:00 2001 From: unknown Date: Wed, 14 Feb 2018 20:21:38 -0500 Subject: [PATCH 3/8] Add r_squared unit test for bill dissagregation --- tests/bill/__init__.py | 0 tests/bill/bill_sample.py | 115 +++++++++++++++++++++++++++++++++++ tests/bill/test_bill_disa.py | 56 +++++++++++++++++ 3 files changed, 171 insertions(+) create mode 100644 tests/bill/__init__.py create mode 100644 tests/bill/bill_sample.py create mode 100644 tests/bill/test_bill_disa.py diff --git a/tests/bill/__init__.py b/tests/bill/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/bill/bill_sample.py b/tests/bill/bill_sample.py new file mode 100644 index 0000000..69f861e --- /dev/null +++ b/tests/bill/bill_sample.py @@ -0,0 +1,115 @@ +TEST_BILL = [{ + 'Bill From Date': '05/31/2016', + 'Bill To Date': '06/30/2016', + 'Days In Bill': 30, + 'Total Charge': 78.81, + 'Usage': 116.0 +}, { + 'Bill From Date': '06/30/2016', + 'Bill To Date': '08/01/2016', + 'Days In Bill': 32, + 'Total Charge': 74.76, + 'Usage': 96.0 +}, { + 'Bill From Date': '08/01/2016', + 'Bill To Date': '08/30/2016', + 'Days In Bill': 29, + 'Total Charge': 75.63, + 'Usage': 111.0 +}, { + 'Bill From Date': '08/30/2016', + 'Bill To Date': '09/29/2016', + 'Days In Bill': 30, + 'Total Charge': 66.11, + 'Usage': 78.0 +}, { + 'Bill From Date': '09/29/2016', + 'Bill To Date': '10/28/2016', + 'Days In Bill': 29, + 'Total Charge': 121.26, + 'Usage': 232.0 +}, { + 'Bill From Date': '10/28/2016', + 'Bill To Date': '11/30/2016', + 'Days In Bill': 33, + 'Total Charge': 358.77, + 'Usage': 941.0 +}, { + 'Bill From Date': '11/30/2016', + 'Bill To Date': '12/29/2016', + 'Days In Bill': 29, + 'Total Charge': 482.06, + 'Usage': 1438.0 +}, { + 'Bill From Date': '12/29/2016', + 'Bill To Date': '01/30/2017', + 'Days In Bill': 32, + 'Total Charge': 508.43, + 'Usage': 1104.0 +}, { + 'Bill From Date': '01/30/2017', + 'Bill To Date': '02/28/2017', + 'Days In Bill': 29, + 'Total Charge': 305.04, + 'Usage': 604.0 +}, { + 'Bill From Date': '02/28/2017', + 'Bill To Date': '03/30/2017', + 'Days In Bill': 30, + 'Total Charge': 643.39, + 'Usage': 1870.0 +}, { + 'Bill From Date': '03/30/2017', + 'Bill To Date': '05/09/2017', + 'Days In Bill': 40, + 'Total Charge': 236.68, + 'Usage': 432.0 +}, { + 'Bill From Date': '05/09/2017', + 'Bill To Date': '05/31/2017', + 'Days In Bill': 22, + 'Total Charge': 264.24, + 'Usage': 670.0 +}, { + 'Bill From Date': '05/31/2017', + 'Bill To Date': '07/31/2017', + 'Days In Bill': 61, + 'Total Charge': 167.14, + 'Usage': 219.0 +}, { + 'Bill From Date': '07/31/2017', + 'Bill To Date': '08/30/2017', + 'Days In Bill': 30, + 'Total Charge': 107.02, + 'Usage': 172.0 +}, { + 'Bill From Date': '08/30/2017', + 'Bill To Date': '09/28/2017', + 'Days In Bill': 29, + 'Total Charge': 57.6, + 'Usage': 48.0 +}, { + 'Bill From Date': '09/28/2017', + 'Bill To Date': '10/30/2017', + 'Days In Bill': 32, + 'Total Charge': 129.85, + 'Usage': 160.0 +}, { + 'Bill From Date': '10/30/2017', + 'Bill To Date': '11/30/2017', + 'Days In Bill': 31, + 'Total Charge': 467.7, + 'Usage': 1171.0 +}, { + 'Bill From Date': '11/30/2017', + 'Bill To Date': '01/09/2018', + 'Days In Bill': 40, + 'Total Charge': 398.76, + 'Usage': 957.0 +}, { + 'Bill From Date': '01/09/2018', + 'Bill To Date': '01/30/2018', + 'Days In Bill': 21, + 'Total Charge': 480.34, + 'Usage': 1046.0 +}] diff --git a/tests/bill/test_bill_disa.py b/tests/bill/test_bill_disa.py new file mode 100644 index 0000000..1dcf58c --- /dev/null +++ b/tests/bill/test_bill_disa.py @@ -0,0 +1,56 @@ +import requests +import pandas as pd + +from bpeng.bill.awesome_disaggregate import BillDisaggregation +from .bill_sample import TEST_BILL + + +class TestBillDisaggregtion: + bill_d = None + weather = None + START_DATE = '2016-01-01' + END_DATE = '2018-02-14' + + @staticmethod + def get_weather_data(start_date, end_date): + WEATHER_SERVICE_URL = ( + '' + 'measurement=temperature&' + 'interval=daily&' + f'date_start={start_date}&' + f'date_end={end_date}' + ) + + HEADERS = { + 'x-blocpower-app-key': '', + 'x-blocpower-app-secret': '', + } + + res = requests.get(WEATHER_SERVICE_URL, headers=HEADERS) + results = res.json() + + pretty_weather_data = [] + for i in results['data']: + pretty_weather_data.append({ + 'time': i['time'], + 'interval': i['tags']['interval'], + 'location': i['tags']['location'], + 'value': i['fields']['value'] + }) + + return pretty_weather_data + + def setup_class(self): + + self.weather = TestBillDisaggregtion.get_weather_data(self.START_DATE, self.END_DATE) + + self.bill_d = BillDisaggregation( + pd.DataFrame(TEST_BILL), + pd.DataFrame(self.weather) + ) + + def test_bill_disaggre(self): + self.bill_d.optimize_setpoints() + r_squared = self.bill_d.r_squared_of_fit + + assert r_squared > 0.71, 'r_squared is not greater than .71' -- GitLab From f3e2360bde0c3d955b87b79fb78e6557356f967e Mon Sep 17 00:00:00 2001 From: Alessandro DiMarco Date: Wed, 14 Feb 2018 20:35:05 -0500 Subject: [PATCH 4/8] Add environment variables to test --- tests/bill/test_bill_disa.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/tests/bill/test_bill_disa.py b/tests/bill/test_bill_disa.py index 1dcf58c..5442e15 100644 --- a/tests/bill/test_bill_disa.py +++ b/tests/bill/test_bill_disa.py @@ -1,5 +1,6 @@ import requests import pandas as pd +import os from bpeng.bill.awesome_disaggregate import BillDisaggregation from .bill_sample import TEST_BILL @@ -13,8 +14,10 @@ class TestBillDisaggregtion: @staticmethod def get_weather_data(start_date, end_date): + weather_service = os.environ['WEATHER_SERVICE'] + WEATHER_SERVICE_URL = ( - '' + f'{weather_service}/weather?' 'measurement=temperature&' 'interval=daily&' f'date_start={start_date}&' @@ -22,8 +25,8 @@ class TestBillDisaggregtion: ) HEADERS = { - 'x-blocpower-app-key': '', - 'x-blocpower-app-secret': '', + 'x-blocpower-app-key': os.environ['APP_KEY'], + 'x-blocpower-app-secret': os.environ['APP_SECRET'], } res = requests.get(WEATHER_SERVICE_URL, headers=HEADERS) -- GitLab From 949994505dde28425a5dddbf5f433c9d8b37cf86 Mon Sep 17 00:00:00 2001 From: Alessandro DiMarco Date: Wed, 14 Feb 2018 20:36:25 -0500 Subject: [PATCH 5/8] Add pytest-env to requirements --- requirements-dev.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/requirements-dev.txt b/requirements-dev.txt index 020a6d8..0d9d1eb 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -4,4 +4,5 @@ pydocstyle>=1.1.1 pylint>=1.7.0 pytest>=3.0.7 pytest-cov>=2.4.0 +pytest-env>=0.6.2 yapf>=0.16.1 -- GitLab From e0ceca7b13764990ed13a9ce1fd4a37b7098f65c Mon Sep 17 00:00:00 2001 From: Alessandro DiMarco Date: Thu, 15 Feb 2018 09:50:42 -0500 Subject: [PATCH 6/8] Add pytest.default ini file --- .gitignore | 1 + pytest.default.ini | 5 +++++ tests/bill/test_bill_disa.py | 4 ++-- 3 files changed, 8 insertions(+), 2 deletions(-) create mode 100644 pytest.default.ini diff --git a/.gitignore b/.gitignore index e2a9431..0243490 100644 --- a/.gitignore +++ b/.gitignore @@ -10,6 +10,7 @@ bpeng.egg-info pylint.log pycodestyle.log .coverage +pytest.ini # mac .DS_Store diff --git a/pytest.default.ini b/pytest.default.ini new file mode 100644 index 0000000..98b708a --- /dev/null +++ b/pytest.default.ini @@ -0,0 +1,5 @@ +[pytest] +env = + WEATHER_SERVICE= + API_KEY= + API_SECRET= diff --git a/tests/bill/test_bill_disa.py b/tests/bill/test_bill_disa.py index 5442e15..e57ac35 100644 --- a/tests/bill/test_bill_disa.py +++ b/tests/bill/test_bill_disa.py @@ -25,8 +25,8 @@ class TestBillDisaggregtion: ) HEADERS = { - 'x-blocpower-app-key': os.environ['APP_KEY'], - 'x-blocpower-app-secret': os.environ['APP_SECRET'], + 'x-blocpower-app-key': os.environ['API_KEY'], + 'x-blocpower-app-secret': os.environ['API_SECRET'], } res = requests.get(WEATHER_SERVICE_URL, headers=HEADERS) -- GitLab From 44f8907a3cdbd9106805c10dbf3b34caef0c1b23 Mon Sep 17 00:00:00 2001 From: Alessandro DiMarco Date: Thu, 15 Feb 2018 09:53:11 -0500 Subject: [PATCH 7/8] Update readme --- README.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 4e1629f..c014f07 100644 --- a/README.md +++ b/README.md @@ -13,7 +13,7 @@ pip install -r requirements-dev.txt ``` #### Run Tests and Coverage +- Copy `pytest.default.ini` --> `pytest.ini` + - `pytest bpeng tests` -- `pytest bpeng tests` - -- `pytest --cov bpeng tests` + - `pytest --cov bpeng tests` -- GitLab From 1107c5076fe23c3274d0bdce2a1de2ca29b2284d Mon Sep 17 00:00:00 2001 From: unknown Date: Thu, 15 Feb 2018 16:22:29 -0500 Subject: [PATCH 8/8] change r_square np.nan to 0 --- bpeng/bill/awesome_disaggregate.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bpeng/bill/awesome_disaggregate.py b/bpeng/bill/awesome_disaggregate.py index ad08242..3523553 100644 --- a/bpeng/bill/awesome_disaggregate.py +++ b/bpeng/bill/awesome_disaggregate.py @@ -781,7 +781,7 @@ class BillDisaggregation(): bill_cp['Other Usage'] = self.others_consumption_pred if self.usage == 'Both Not': - self.r_squared_of_fit = np.NaN + self.r_squared_of_fit = 0 # self.h = np.NaN else: self.r_squared_of_fit = regr[1] -- GitLab