diff --git a/.gitignore b/.gitignore index e2a943145ad7a8cf812a6d1af5b6926978e0d1b0..02434905630e31f164eef2a32c702de7c38d754d 100644 --- a/.gitignore +++ b/.gitignore @@ -10,6 +10,7 @@ bpeng.egg-info pylint.log pycodestyle.log .coverage +pytest.ini # mac .DS_Store diff --git a/README.md b/README.md index 4e1629f53fd2607c74de6550c7d700dd94c42ac2..c014f071260199b77cdea2ce571bb91305e63485 100644 --- a/README.md +++ b/README.md @@ -13,7 +13,7 @@ pip install -r requirements-dev.txt ``` #### Run Tests and Coverage +- Copy `pytest.default.ini` --> `pytest.ini` + - `pytest bpeng tests` -- `pytest bpeng tests` - -- `pytest --cov bpeng tests` + - `pytest --cov bpeng tests` diff --git a/bpeng/bill/awesome_disaggregate.py b/bpeng/bill/awesome_disaggregate.py index 1f070c068473acc0210a90f3bb1413d9ea8dd600..35235531316ae6f7a1d3c27869ac9de764ae724f 100644 --- a/bpeng/bill/awesome_disaggregate.py +++ b/bpeng/bill/awesome_disaggregate.py @@ -60,7 +60,7 @@ class BillDisaggregation(): self.annual_usage = None def weather_cleaning(self, raw_daily_temp): - ''' + """ Format the daily temperature data from influx query Args: @@ -70,7 +70,7 @@ class BillDisaggregation(): Returns: pd.DateFrame: Returns formatted daily temperature - ''' + """ raw_daily_temp.rename( columns={'time': 'date', 'value': 'temperature'}, inplace=True) @@ -85,7 +85,7 @@ class BillDisaggregation(): return self.daily_temp def bill_period_weather(self, bill_from_date, bill_end_date): - ''' + """ get the outdoor temperature date between two date, return a list Args: @@ -96,7 +96,7 @@ class BillDisaggregation(): Returns: list: Returns a list of outdoor temperature for a period - ''' + """ end_date_id = self.daily_temp[self.daily_temp.date == bill_end_date].index[0] start_date_id = self.daily_temp[self.daily_temp.date == @@ -105,7 +105,7 @@ class BillDisaggregation(): @staticmethod def cdd(indoor_set_point, outdoor_temp): - ''' + """ CDD Assumption: cooling setting point shall always higher than 55 F, @@ -114,7 +114,7 @@ class BillDisaggregation(): ?? set_point is for indoor temperature - ''' + """ if indoor_set_point > 55: if indoor_set_point < outdoor_temp: @@ -123,13 +123,13 @@ class BillDisaggregation(): @staticmethod def hdd(indoor_set_point, outdoor_temp): - ''' + """ HDD Assumption: Only if the outdoor temperature drop below 60'F, then the heating system may be able to be turn on - ''' - if (outdoor_temp < 60) & (indoor_set_point > outdoor_temp): + """ + if (outdoor_temp < 60) and (indoor_set_point > outdoor_temp): hdd = indoor_set_point - outdoor_temp else: hdd = 0 @@ -137,14 +137,14 @@ class BillDisaggregation(): @staticmethod def threshold(data, set_point): - '''If data is less the set_point, return 0''' + """If data is less the set_point, return 0""" if data <= set_point: data = 0 return data @staticmethod def outliers_iqr(ys): - ''' + """ Find outlier using IQR method Args: @@ -157,7 +157,7 @@ class BillDisaggregation(): True: Outliner False: Not Outliner - ''' + """ quartile_1, quartile_3 = np.percentile(ys, [25, 75]) iqr = quartile_3 - quartile_1 lower_bound = quartile_1 - (iqr * 1.5) @@ -166,7 +166,7 @@ class BillDisaggregation(): @staticmethod def anomaly_point(alist, thresholds): - ''' + """ Find outlier and return its index Args: @@ -179,7 +179,7 @@ class BillDisaggregation(): list: Returns a list the index of the outliner - ''' + """ amean = [] for x in range(len(alist)): temp = np.hstack((alist[:(x)], alist[(x + 1):])) @@ -193,7 +193,7 @@ class BillDisaggregation(): @staticmethod def num_month_dates(last_date_bill, first_date_bill): - '''Return number of month in between two date ''' + """Return number of month in between two date """ lastdate = last_date_bill - timedelta(last_date_bill.day) firstdate = first_date_bill + timedelta(days=32) firstdate = firstdate.replace(day=1) @@ -202,7 +202,7 @@ class BillDisaggregation(): return (num_month) def bill_formating(self, raw_bill): - ''' + """ Bill Cleaning Step 1: 1. format each column of the raw bill @@ -223,7 +223,7 @@ class BillDisaggregation(): boolean: True - Length of the bill has changed during bill cleaning step 1 - ''' + """ bill_copy = raw_bill.copy() bill_copy['Bill From Date'] = pd.to_datetime( bill_copy['Bill From Date']) @@ -261,7 +261,7 @@ class BillDisaggregation(): return bill_formatted, bill_shape_change def bill_quality(self, bill_formatted): - ''' + """ Bill Cleaning Step 2: 1. Check each billing period to find a bill is too short or too long; @@ -273,7 +273,7 @@ class BillDisaggregation(): pd.DataFrame: a dataframe with columns: 'index': the index of the billing period which is identified as an outlier 'flag': to indicate either it is too long or too short - ''' + """ bill = bill_formatted.copy() bill = pd.DataFrame(bill) @@ -312,7 +312,7 @@ class BillDisaggregation(): return bill_quality def short_bill_consolidate(self, bill_formatted, bill_quality): - ''' + """ Bill Cleaning Step 3: consolidation of the bills that are too short compare to others @@ -325,7 +325,7 @@ class BillDisaggregation(): Returns: pd.DataFrame: the cleaned bill and ready for analysis - ''' + """ bill_quality_short = bill_quality[bill_quality['flag'] == 'short'] bill_consi = bill_formatted.copy() # consolidate the billing period that is too short compare to others @@ -335,7 +335,7 @@ class BillDisaggregation(): if bill_quality['flag'].iloc[xxx] == 'short': row_index = bill_quality_short['index'].iloc[xxx] - if (row_index != 0) & (row_index != bill_consi.index[-1]): + if (row_index != 0) and (row_index != bill_consi.index[-1]): if bill_consi['Days In Bill'][int( row_index - 1)] <= bill_consi['Days In Bill'][int( @@ -391,7 +391,7 @@ class BillDisaggregation(): return bill_consi def regression_1(self, hp, cp, processed_bill): - ''' + """ A linear regression model with heating and cooling set fixed Args: @@ -406,7 +406,7 @@ class BillDisaggregation(): float: r-squared of the linear regression model 2d-array: a 2D numpy array of normalized billing period average daily HDDs and CDDs - ''' + """ bill = processed_bill.copy() @@ -446,7 +446,7 @@ class BillDisaggregation(): return regr_model, score, regression_temp def summer_dhw(self, hp, abill): - ''' + """ This funcion uses summer month gas usage as base consumption for the year A linear regression of weather-related consumption and a fixed heating system set point NOTE: USUALLY ERROR @@ -464,7 +464,7 @@ class BillDisaggregation(): 2d-array: a 2D numpy array of normalized billing period HDDs sum pd.DataFrame - ''' + """ bill = abill.copy() ahdd = [[BillDisaggregation.hdd(hp, xx) for xx in x] @@ -560,13 +560,13 @@ class BillDisaggregation(): heating_coef, cooling_coef = regr_model.coef_ if -opt.fun > 0.5: - if (heating_coef > 0) & (cooling_coef <= 0): + if (heating_coef > 0) and (cooling_coef <= 0): weather_related_usage = 'Heating' - elif (heating_coef <= 0) & (cooling_coef > 0): + elif (heating_coef <= 0) and (cooling_coef > 0): weather_related_usage = 'Cooling' - elif (heating_coef <= 0) & (cooling_coef <= 0): + elif (heating_coef <= 0) and (cooling_coef <= 0): weather_related_usage = 'Both Not' - elif (heating_coef >= 0) & (cooling_coef >= 0): + elif (heating_coef >= 0) and (cooling_coef >= 0): weather_related_usage = 'Both' else: weather_related_usage = 'Both Not' @@ -588,13 +588,13 @@ class BillDisaggregation(): # change accordingly for JOENYC buildings - if (heating_coef > 0) & (cooling_coef < 0): + if (heating_coef > 0) and (cooling_coef < 0): weather_related_usage = 'Heating' cooling_coef = 0 - elif (heating_coef <= 0) & (cooling_coef > 0): + elif (heating_coef <= 0) and (cooling_coef > 0): weather_related_usage = 'Cooling' heating_coef = 0 - elif (heating_coef <= 0) & (cooling_coef <= 0): + elif (heating_coef <= 0) and (cooling_coef <= 0): weather_related_usage = 'Both Not' heating_coef = 0 cooling_coef = 0 @@ -602,7 +602,7 @@ class BillDisaggregation(): # changes on Jan 17th 2018 # please futher check with more bills - elif (heating_coef > 0) & (cooling_coef > 0): + elif (heating_coef > 0) and (cooling_coef > 0): if heating_coef / cooling_coef > 5: weather_related_usage = 'Heating' cooling_coef = 0 @@ -624,17 +624,17 @@ class BillDisaggregation(): heating_set_point = opt.x[0] cooling_set_point = opt.x[1] - if (heating_coef > 0) & (cooling_coef < 0): + if (heating_coef > 0) and (cooling_coef < 0): weather_related_usage = 'Heating' cooling_coef = 0 - elif (heating_coef <= 0) & (cooling_coef > 0): + elif (heating_coef <= 0) and (cooling_coef > 0): weather_related_usage = 'Cooling' heating_coef = 0 - elif (heating_coef <= 0) & (cooling_coef <= 0): + elif (heating_coef <= 0) and (cooling_coef <= 0): weather_related_usage = 'Both Not' heating_coef = 0 cooling_coef = 0 - elif (heating_coef > 0) & (cooling_coef > 0): + elif (heating_coef > 0) and (cooling_coef > 0): if heating_coef / cooling_coef > 5: weather_related_usage = 'Heating' cooling_coef = 0 @@ -781,7 +781,7 @@ class BillDisaggregation(): bill_cp['Other Usage'] = self.others_consumption_pred if self.usage == 'Both Not': - self.r_squared_of_fit = np.NaN + self.r_squared_of_fit = 0 # self.h = np.NaN else: self.r_squared_of_fit = regr[1] @@ -805,9 +805,9 @@ class BillDisaggregation(): self.annual_usage = self.annual_usage_costs(self.recent_year_bill_breakdown, non_weather_related_end_use) def benchmarking_output(self): - ''' + """ output perimeters that related with evaluating the bills - ''' + """ test = self.output_table.copy() bill_start_date = pd.to_datetime(test['Bill From Date']).iloc[0] @@ -929,7 +929,7 @@ class BillDisaggregation(): return monthly_output def non_weahter_related_breakdown(self, end_uses, monthly_output_table): - ''' + """ breakdown the non_weather_related_usage Args: @@ -941,7 +941,7 @@ class BillDisaggregation(): Returns: pd.DataFrame: bill breakdown of all end-use - ''' + """ monthly_usages = monthly_output_table.copy() eu = pd.DataFrame( @@ -959,7 +959,7 @@ class BillDisaggregation(): return monthly_usages def annual_usage_costs(self, annual_bill_breakdown, end_uses): - ''' + """ Calcuate annual usage and costs for each end use Args: @@ -970,7 +970,7 @@ class BillDisaggregation(): Return: pd.DataFrame: annual usage, costs for each end uses - ''' + """ annual_usage = pd.DataFrame(columns=['End Use', 'Usage', 'Costs']) @@ -1013,7 +1013,7 @@ class BillDisaggregation(): # @staticmethod # def projection_figure(bill): - # '''ploat the disaggregated bill''' + # """ploat the disaggregated bill""" # plt.figure(figsize=(10, 5)) # x = pd.to_datetime(bill['Bill From Date']) diff --git a/pytest.default.ini b/pytest.default.ini new file mode 100644 index 0000000000000000000000000000000000000000..98b708a0271287ae636390178f9e6722d1c5929d --- /dev/null +++ b/pytest.default.ini @@ -0,0 +1,5 @@ +[pytest] +env = + WEATHER_SERVICE= + API_KEY= + API_SECRET= diff --git a/requirements-dev.txt b/requirements-dev.txt index 020a6d8d36d8d31a5de11e6e917c7dc99113fb73..0d9d1ebdff5dee3d6dadda28d4a24aa6373f9470 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -4,4 +4,5 @@ pydocstyle>=1.1.1 pylint>=1.7.0 pytest>=3.0.7 pytest-cov>=2.4.0 +pytest-env>=0.6.2 yapf>=0.16.1 diff --git a/tests/bill/__init__.py b/tests/bill/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/tests/bill/bill_sample.py b/tests/bill/bill_sample.py new file mode 100644 index 0000000000000000000000000000000000000000..69f861e1a14293d7893e5eeb81342ae1eb0d78f9 --- /dev/null +++ b/tests/bill/bill_sample.py @@ -0,0 +1,115 @@ +TEST_BILL = [{ + 'Bill From Date': '05/31/2016', + 'Bill To Date': '06/30/2016', + 'Days In Bill': 30, + 'Total Charge': 78.81, + 'Usage': 116.0 +}, { + 'Bill From Date': '06/30/2016', + 'Bill To Date': '08/01/2016', + 'Days In Bill': 32, + 'Total Charge': 74.76, + 'Usage': 96.0 +}, { + 'Bill From Date': '08/01/2016', + 'Bill To Date': '08/30/2016', + 'Days In Bill': 29, + 'Total Charge': 75.63, + 'Usage': 111.0 +}, { + 'Bill From Date': '08/30/2016', + 'Bill To Date': '09/29/2016', + 'Days In Bill': 30, + 'Total Charge': 66.11, + 'Usage': 78.0 +}, { + 'Bill From Date': '09/29/2016', + 'Bill To Date': '10/28/2016', + 'Days In Bill': 29, + 'Total Charge': 121.26, + 'Usage': 232.0 +}, { + 'Bill From Date': '10/28/2016', + 'Bill To Date': '11/30/2016', + 'Days In Bill': 33, + 'Total Charge': 358.77, + 'Usage': 941.0 +}, { + 'Bill From Date': '11/30/2016', + 'Bill To Date': '12/29/2016', + 'Days In Bill': 29, + 'Total Charge': 482.06, + 'Usage': 1438.0 +}, { + 'Bill From Date': '12/29/2016', + 'Bill To Date': '01/30/2017', + 'Days In Bill': 32, + 'Total Charge': 508.43, + 'Usage': 1104.0 +}, { + 'Bill From Date': '01/30/2017', + 'Bill To Date': '02/28/2017', + 'Days In Bill': 29, + 'Total Charge': 305.04, + 'Usage': 604.0 +}, { + 'Bill From Date': '02/28/2017', + 'Bill To Date': '03/30/2017', + 'Days In Bill': 30, + 'Total Charge': 643.39, + 'Usage': 1870.0 +}, { + 'Bill From Date': '03/30/2017', + 'Bill To Date': '05/09/2017', + 'Days In Bill': 40, + 'Total Charge': 236.68, + 'Usage': 432.0 +}, { + 'Bill From Date': '05/09/2017', + 'Bill To Date': '05/31/2017', + 'Days In Bill': 22, + 'Total Charge': 264.24, + 'Usage': 670.0 +}, { + 'Bill From Date': '05/31/2017', + 'Bill To Date': '07/31/2017', + 'Days In Bill': 61, + 'Total Charge': 167.14, + 'Usage': 219.0 +}, { + 'Bill From Date': '07/31/2017', + 'Bill To Date': '08/30/2017', + 'Days In Bill': 30, + 'Total Charge': 107.02, + 'Usage': 172.0 +}, { + 'Bill From Date': '08/30/2017', + 'Bill To Date': '09/28/2017', + 'Days In Bill': 29, + 'Total Charge': 57.6, + 'Usage': 48.0 +}, { + 'Bill From Date': '09/28/2017', + 'Bill To Date': '10/30/2017', + 'Days In Bill': 32, + 'Total Charge': 129.85, + 'Usage': 160.0 +}, { + 'Bill From Date': '10/30/2017', + 'Bill To Date': '11/30/2017', + 'Days In Bill': 31, + 'Total Charge': 467.7, + 'Usage': 1171.0 +}, { + 'Bill From Date': '11/30/2017', + 'Bill To Date': '01/09/2018', + 'Days In Bill': 40, + 'Total Charge': 398.76, + 'Usage': 957.0 +}, { + 'Bill From Date': '01/09/2018', + 'Bill To Date': '01/30/2018', + 'Days In Bill': 21, + 'Total Charge': 480.34, + 'Usage': 1046.0 +}] diff --git a/tests/bill/test_bill_disa.py b/tests/bill/test_bill_disa.py new file mode 100644 index 0000000000000000000000000000000000000000..e57ac3504b4664ae5e21337fb3a74c3af5e737b1 --- /dev/null +++ b/tests/bill/test_bill_disa.py @@ -0,0 +1,59 @@ +import requests +import pandas as pd +import os + +from bpeng.bill.awesome_disaggregate import BillDisaggregation +from .bill_sample import TEST_BILL + + +class TestBillDisaggregtion: + bill_d = None + weather = None + START_DATE = '2016-01-01' + END_DATE = '2018-02-14' + + @staticmethod + def get_weather_data(start_date, end_date): + weather_service = os.environ['WEATHER_SERVICE'] + + WEATHER_SERVICE_URL = ( + f'{weather_service}/weather?' + 'measurement=temperature&' + 'interval=daily&' + f'date_start={start_date}&' + f'date_end={end_date}' + ) + + HEADERS = { + 'x-blocpower-app-key': os.environ['API_KEY'], + 'x-blocpower-app-secret': os.environ['API_SECRET'], + } + + res = requests.get(WEATHER_SERVICE_URL, headers=HEADERS) + results = res.json() + + pretty_weather_data = [] + for i in results['data']: + pretty_weather_data.append({ + 'time': i['time'], + 'interval': i['tags']['interval'], + 'location': i['tags']['location'], + 'value': i['fields']['value'] + }) + + return pretty_weather_data + + def setup_class(self): + + self.weather = TestBillDisaggregtion.get_weather_data(self.START_DATE, self.END_DATE) + + self.bill_d = BillDisaggregation( + pd.DataFrame(TEST_BILL), + pd.DataFrame(self.weather) + ) + + def test_bill_disaggre(self): + self.bill_d.optimize_setpoints() + r_squared = self.bill_d.r_squared_of_fit + + assert r_squared > 0.71, 'r_squared is not greater than .71'