From 929d2f10e37fc7e80e8091ae8dccfaea133e1965 Mon Sep 17 00:00:00 2001 From: unknown Date: Thu, 1 Mar 2018 16:46:07 -0500 Subject: [PATCH 1/8] change to self.avg unit price; add unit price column to bills --- bpeng/bill/awesome_disaggregate.py | 28 +++++++++++++++++----------- 1 file changed, 17 insertions(+), 11 deletions(-) diff --git a/bpeng/bill/awesome_disaggregate.py b/bpeng/bill/awesome_disaggregate.py index 8511839..f8131ab 100644 --- a/bpeng/bill/awesome_disaggregate.py +++ b/bpeng/bill/awesome_disaggregate.py @@ -54,7 +54,7 @@ class BillDisaggregation(): self.output_table = None self.output_table_monthly = None self.most_recent_monthly_output = None - self.unit_price = None + self.avg_unit_price = None self.bill_breakdown = None self.recent_year_bill_breakdown = None self.annual_usage = None @@ -215,8 +215,6 @@ class BillDisaggregation(): 'Bill To Date' 'Days In Bill' 'Usage' - 'Delivery Charge' - 'Supply Charge' 'Total Charge' Returns: pd.DataFrame: a formatted raw_bill @@ -255,7 +253,7 @@ class BillDisaggregation(): bill_formatted = bill_copy2 - self.unit_price = (sum(bill_formatted['Total Charge'])) / ( + self.avg_unit_price = (sum(bill_formatted['Total Charge'])) / ( sum(bill_formatted['Usage'])) return bill_formatted, bill_shape_change @@ -348,6 +346,10 @@ class BillDisaggregation(): row_index - 1)] = bill_consi['Usage'][int( row_index - 1)] + bill_consi['Usage'][int( row_index)] + bill_consi['Total Charge'][int( + row_index - 1)] = bill_consi['Total Charge'][int( + row_index - 1)] + bill_consi['Total Charge'][int( + row_index)] bill_consi['Days In Bill'][int( row_index - 1)] = bill_consi['Days In Bill'][int( row_index - 1 @@ -360,6 +362,10 @@ class BillDisaggregation(): row_index + 1)] = bill_consi['Usage'][int( row_index + 1)] + bill_consi['Usage'][int( row_index)] + bill_consi['Total Charge'][int( + row_index + 1)] = bill_consi['Total Charge'][int( + row_index + 1)] + bill_consi['Total Charge'][int( + row_index)] bill_consi['Days In Bill'][int( row_index + 1)] = bill_consi['Days In Bill'][int( row_index + 1 @@ -370,6 +376,8 @@ class BillDisaggregation(): 'Bill From Date'][0] bill_consi['Usage'][ 1] = bill_consi['Usage'][0] + bill_consi['Usage'][1] + bill_consi['Total Charge'][ + 1] = bill_consi['Total Charge'][0] + bill_consi['Total Charge'][1] bill_consi['Days In Bill'][ 1] = bill_consi['Days In Bill'][0] + bill_consi['Days In Bill'][1] @@ -378,6 +386,8 @@ class BillDisaggregation(): 'Bill To Date'].iloc[-1] bill_consi['Usage'].iloc[ -2] = bill_consi['Usage'].iloc[-2] + bill_consi['Usage'].iloc[-1] + bill_consi['Total Charge'].iloc[ + -2] = bill_consi['Total Charge'].iloc[-2] + bill_consi['Total Charge'].iloc[-1] bill_consi['Days In Bill'].iloc[ -2] = bill_consi['Days In Bill'].iloc[-1] + bill_consi['Days In Bill'].iloc[-2] @@ -385,7 +395,6 @@ class BillDisaggregation(): bill_consi = bill_consi.drop( bill_consi.index[list(bill_quality_short['index'])]) - # bill_consi = bill_consi.reset_index(inplace = True) bill_consi = bill_consi.reset_index(drop=False) return bill_consi @@ -773,21 +782,18 @@ class BillDisaggregation(): bill_cp = self.processed_bill.copy() bill_cp = self.processed_bill[[ - 'Bill From Date', 'Bill To Date', 'Days In Bill', 'Usage' + 'Bill From Date', 'Bill To Date', 'Days In Bill', 'Usage', 'Total Charge' ]] - + bill_cp['Unit Price'] = bill_cp['Total Charge'] / bill_cp['Usage'] bill_cp['Heating Usage'] = self.heating_consumption_pred bill_cp['Cooling Usage'] = self.cooling_consumption_pred bill_cp['Other Usage'] = self.others_consumption_pred if self.usage == 'Both Not': self.r_squared_of_fit = 0 - # self.h = np.NaN else: self.r_squared_of_fit = regr[1] - # self.set_points = opt.x - # update 2018/01/17 self.heating_set_point = heating_set_point self.cooling_set_point = cooling_set_point self.output_table = bill_cp @@ -996,7 +1002,7 @@ class BillDisaggregation(): temp_usage = sum(temp) annual_usage['Usage'].iloc[j] = temp_usage - annual_usage['Costs'] = annual_usage['Usage'] * (self.unit_price) + annual_usage['Costs'] = annual_usage['Usage'] * (self.avg_unit_price) return annual_usage -- GitLab From 747b4433113d8347f99ba1d03c7cf7993b53f5a4 Mon Sep 17 00:00:00 2001 From: unknown Date: Mon, 5 Mar 2018 17:12:20 -0500 Subject: [PATCH 2/8] rearrange the monthly output in ascending order --- bpeng/bill/awesome_disaggregate.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bpeng/bill/awesome_disaggregate.py b/bpeng/bill/awesome_disaggregate.py index f8131ab..1370635 100644 --- a/bpeng/bill/awesome_disaggregate.py +++ b/bpeng/bill/awesome_disaggregate.py @@ -931,7 +931,7 @@ class BillDisaggregation(): monthly_output = monthly_output_table[['Month', 'Bill From Date', 'Bill To Date', 'Days In Bill', 'Heating Usage', 'Cooling Usage', 'Other Usage', 'HDD', 'CDD']] - + monthly_output = monthly_output.sort('Bill From Date').reset_index(drop=True) return monthly_output def non_weahter_related_breakdown(self, end_uses, monthly_output_table): -- GitLab From 8fa301fc67af95c22781c5e472b64da0f0a772db Mon Sep 17 00:00:00 2001 From: unknown Date: Mon, 5 Mar 2018 17:15:42 -0500 Subject: [PATCH 3/8] formating --- bpeng/bill/awesome_disaggregate.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/bpeng/bill/awesome_disaggregate.py b/bpeng/bill/awesome_disaggregate.py index 1370635..ec559bf 100644 --- a/bpeng/bill/awesome_disaggregate.py +++ b/bpeng/bill/awesome_disaggregate.py @@ -807,7 +807,8 @@ class BillDisaggregation(): self.most_recent_monthly_output = self.output_to_month(last_bill_date, self.heating_set_point, self.cooling_set_point, 12) self.bill_breakdown = self.non_weahter_related_breakdown(non_weather_related_end_use, self.output_table_monthly) - self.recent_year_bill_breakdown = self.non_weahter_related_breakdown(non_weather_related_end_use, self.most_recent_monthly_output) + self.recent_year_bill_breakdown = self.non_weahter_related_breakdown(non_weather_related_end_use, + self.most_recent_monthly_output) self.annual_usage = self.annual_usage_costs(self.recent_year_bill_breakdown, non_weather_related_end_use) def benchmarking_output(self): @@ -1076,4 +1077,3 @@ class BillDisaggregation(): return self.bill_breakdown.to_dict(orient="records") return self.output_table_monthly.to_dict(orient="records") - -- GitLab From b8519ba7fc0b0f2bd61f685f040a27c89400ed98 Mon Sep 17 00:00:00 2001 From: unknown Date: Mon, 5 Mar 2018 18:17:32 -0500 Subject: [PATCH 4/8] add the normalized unit price to monthly output table; test not working --- bpeng/bill/awesome_disaggregate.py | 67 ++++++++++++++++++++++++++++-- 1 file changed, 64 insertions(+), 3 deletions(-) diff --git a/bpeng/bill/awesome_disaggregate.py b/bpeng/bill/awesome_disaggregate.py index ec559bf..4eb228c 100644 --- a/bpeng/bill/awesome_disaggregate.py +++ b/bpeng/bill/awesome_disaggregate.py @@ -210,14 +210,14 @@ class BillDisaggregation(): Args: - raw_bill (pd.DataFrame): a raw bill with columns of + self.output_table (pd.DataFrame): a raw bill with columns of 'Bill From Date' 'Bill To Date' 'Days In Bill' 'Usage' 'Total Charge' Returns: - pd.DataFrame: a formatted raw_bill + pd.DataFrame: a formatted self.output_table boolean: True - Length of the bill has changed during bill cleaning step 1 @@ -802,8 +802,9 @@ class BillDisaggregation(): first_bill_date = self.processed_bill['Bill From Date'].iloc[0] billing_months = self.num_month_dates(last_bill_date, first_bill_date) - self.output_table_monthly = self.output_to_month(last_bill_date, self.heating_set_point, + output_monthly_initial = self.output_to_month(last_bill_date, self.heating_set_point, self.cooling_set_point, billing_months) + self.output_table_monthly = self.normalized_unit_price(self.output_table, output_monthly_initial) self.most_recent_monthly_output = self.output_to_month(last_bill_date, self.heating_set_point, self.cooling_set_point, 12) self.bill_breakdown = self.non_weahter_related_breakdown(non_weather_related_end_use, self.output_table_monthly) @@ -1077,3 +1078,63 @@ class BillDisaggregation(): return self.bill_breakdown.to_dict(orient="records") return self.output_table_monthly.to_dict(orient="records") + + def find_index_in_first_raw_biil(self, norm_bill_date): + for index, bill in self.output_table.iterrows(): + if bill['Bill From Date'] <= norm_bill_date < bill['Bill To Date']: + return index + return None + + def days_in_raw_bill_period(self, norm_bill_date, norm_bill_date_respected_index, flag): + if flag == 'start': + days = (self.output_table['Bill To Date'][norm_bill_date_respected_index] - norm_bill_date).days + if flag == 'end': + days = (norm_bill_date - self.output_table['Bill From Date'][norm_bill_date_respected_index]).days + return days + + def weighted_unit_price(self, index_numdays): + weights = [] + total_days = [] + for ind in range(len(index_numdays)): + unit_price = self.output_table['Unit Price'][int(index_numdays[ind]['index'])] + days_in_that_period = int(index_numdays[ind]['num_days']) + weights.append(unit_price * days_in_that_period) + total_days.append(days_in_that_period) + weighted_unit_price = sum(weights)/sum(total_days) + return weighted_unit_price + + def find_bills_in_raw(self, norm_bill_from, norm_bill_to): + + norm_bill_days = (norm_bill_to - norm_bill_from).days + results = [] + + index_start = self.find_index_in_first_raw_biil(norm_bill_from) + index_end = self.find_index_in_first_raw_biil(norm_bill_to) + + if index_start == index_end: + results.append({'index': index_start, 'num_days': norm_bill_days}) + + elif index_end - index_start >= 1: + days_in_start_period = self.days_in_raw_bill_period(norm_bill_from, index_start, 'start') + results.append({'index': index_start, 'num_days': days_in_start_period}) + days_in_end_period = self.days_in_raw_bill_period(norm_bill_to, index_end, 'end') + results.append({'index': index_end, 'num_days': days_in_end_period}) + + if index_end - index_start >= 2: + for p in range(index_end - index_start - 1): + days_in_period = self.output_table['Days In Bill'][index_start+p+1] + index_of_this_period = index_start+p+1 + results.append({'index': index_of_this_period, 'num_days': days_in_period}) + + return results + + def normalized_unit_price(self, rawbill, mbill): + normalized_unit_price = [] + for m in range(len(mbill)): + from_date = mbill['Bill From Date'].iloc[m] + to_date = mbill['Bill To Date'].iloc[m] + index_numdays = self.find_bills_in_raw(from_date, to_date) + weighted_unit_price_for_this_month = self.weighted_unit_price(index_numdays) + normalized_unit_price.append(weighted_unit_price_for_this_month) + mbill['Unit Price'] = normalized_unit_price + return mbill -- GitLab From 5bef931546cd55d6a04c6689d3cbe0f2581efa3e Mon Sep 17 00:00:00 2001 From: unknown Date: Tue, 6 Mar 2018 14:28:22 -0500 Subject: [PATCH 5/8] add formatted bill/ unit price --- bpeng/bill/awesome_disaggregate.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/bpeng/bill/awesome_disaggregate.py b/bpeng/bill/awesome_disaggregate.py index 4eb228c..cc3ebf9 100644 --- a/bpeng/bill/awesome_disaggregate.py +++ b/bpeng/bill/awesome_disaggregate.py @@ -58,6 +58,7 @@ class BillDisaggregation(): self.bill_breakdown = None self.recent_year_bill_breakdown = None self.annual_usage = None + self.formatted_bill = None def weather_cleaning(self, raw_daily_temp): """ @@ -554,6 +555,10 @@ class BillDisaggregation(): self.processed_bill = self.processed_bill.sort_values('Bill From Date') + formatted_bill = formatted_bill.sort_values('Bill From Date') + formatted_bill['Unit Price'] = formatted_bill['Total Charge']/ formatted_bill['Usage'] + self.formatted_bill = formatted_bill + regression_method = 1 if weather_related_usage == 'Unknown': -- GitLab From cbe8d991f12707cb058ba89c8591f7e92591f4b6 Mon Sep 17 00:00:00 2001 From: unknown Date: Tue, 6 Mar 2018 14:40:04 -0500 Subject: [PATCH 6/8] replace the input for unit price from ouput table to formatted bill --- bpeng/bill/awesome_disaggregate.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/bpeng/bill/awesome_disaggregate.py b/bpeng/bill/awesome_disaggregate.py index cc3ebf9..e847375 100644 --- a/bpeng/bill/awesome_disaggregate.py +++ b/bpeng/bill/awesome_disaggregate.py @@ -556,7 +556,7 @@ class BillDisaggregation(): self.processed_bill = self.processed_bill.sort_values('Bill From Date') formatted_bill = formatted_bill.sort_values('Bill From Date') - formatted_bill['Unit Price'] = formatted_bill['Total Charge']/ formatted_bill['Usage'] + formatted_bill['Unit Price'] = formatted_bill['Total Charge'] / formatted_bill['Usage'] self.formatted_bill = formatted_bill regression_method = 1 @@ -1085,23 +1085,23 @@ class BillDisaggregation(): return self.output_table_monthly.to_dict(orient="records") def find_index_in_first_raw_biil(self, norm_bill_date): - for index, bill in self.output_table.iterrows(): + for index, bill in self.formatted_bill.iterrows(): if bill['Bill From Date'] <= norm_bill_date < bill['Bill To Date']: return index return None def days_in_raw_bill_period(self, norm_bill_date, norm_bill_date_respected_index, flag): if flag == 'start': - days = (self.output_table['Bill To Date'][norm_bill_date_respected_index] - norm_bill_date).days + days = (self.formatted_bill['Bill To Date'][norm_bill_date_respected_index] - norm_bill_date).days if flag == 'end': - days = (norm_bill_date - self.output_table['Bill From Date'][norm_bill_date_respected_index]).days + days = (norm_bill_date - self.formatted_bill['Bill From Date'][norm_bill_date_respected_index]).days return days def weighted_unit_price(self, index_numdays): weights = [] total_days = [] for ind in range(len(index_numdays)): - unit_price = self.output_table['Unit Price'][int(index_numdays[ind]['index'])] + unit_price = self.formatted_bill['Unit Price'][int(index_numdays[ind]['index'])] days_in_that_period = int(index_numdays[ind]['num_days']) weights.append(unit_price * days_in_that_period) total_days.append(days_in_that_period) @@ -1127,7 +1127,7 @@ class BillDisaggregation(): if index_end - index_start >= 2: for p in range(index_end - index_start - 1): - days_in_period = self.output_table['Days In Bill'][index_start+p+1] + days_in_period = self.formatted_bill['Days In Bill'][index_start+p+1] index_of_this_period = index_start+p+1 results.append({'index': index_of_this_period, 'num_days': days_in_period}) -- GitLab From 49a4ebdaad28ac38a104f3d4d33d27b5e67942fb Mon Sep 17 00:00:00 2001 From: unknown Date: Tue, 6 Mar 2018 14:50:13 -0500 Subject: [PATCH 7/8] add narrative for each new function --- bpeng/bill/awesome_disaggregate.py | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/bpeng/bill/awesome_disaggregate.py b/bpeng/bill/awesome_disaggregate.py index e847375..f35640f 100644 --- a/bpeng/bill/awesome_disaggregate.py +++ b/bpeng/bill/awesome_disaggregate.py @@ -1085,12 +1085,19 @@ class BillDisaggregation(): return self.output_table_monthly.to_dict(orient="records") def find_index_in_first_raw_biil(self, norm_bill_date): + ''' + Return the index of the row of raw bill contains the bill date from a normalized bill + ''' for index, bill in self.formatted_bill.iterrows(): if bill['Bill From Date'] <= norm_bill_date < bill['Bill To Date']: return index return None def days_in_raw_bill_period(self, norm_bill_date, norm_bill_date_respected_index, flag): + ''' + Return how many days from a normalized bill within a raw bill billing period + ''' + if flag == 'start': days = (self.formatted_bill['Bill To Date'][norm_bill_date_respected_index] - norm_bill_date).days if flag == 'end': @@ -1098,6 +1105,9 @@ class BillDisaggregation(): return days def weighted_unit_price(self, index_numdays): + ''' + Return the weighted average of unit price + ''' weights = [] total_days = [] for ind in range(len(index_numdays)): @@ -1109,6 +1119,9 @@ class BillDisaggregation(): return weighted_unit_price def find_bills_in_raw(self, norm_bill_from, norm_bill_to): + ''' + Return the index / number of days in each raw bill billing period for a normalized billing period + ''' norm_bill_days = (norm_bill_to - norm_bill_from).days results = [] @@ -1134,6 +1147,9 @@ class BillDisaggregation(): return results def normalized_unit_price(self, rawbill, mbill): + ''' + calculate the unit price for each nomralized billing period + ''' normalized_unit_price = [] for m in range(len(mbill)): from_date = mbill['Bill From Date'].iloc[m] -- GitLab From d52ecb01dfdf928df05ad84d29448d5ef057b812 Mon Sep 17 00:00:00 2001 From: unknown Date: Tue, 6 Mar 2018 15:05:32 -0500 Subject: [PATCH 8/8] triple double --- bpeng/bill/awesome_disaggregate.py | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/bpeng/bill/awesome_disaggregate.py b/bpeng/bill/awesome_disaggregate.py index f35640f..35940e4 100644 --- a/bpeng/bill/awesome_disaggregate.py +++ b/bpeng/bill/awesome_disaggregate.py @@ -1085,18 +1085,18 @@ class BillDisaggregation(): return self.output_table_monthly.to_dict(orient="records") def find_index_in_first_raw_biil(self, norm_bill_date): - ''' + """ Return the index of the row of raw bill contains the bill date from a normalized bill - ''' + """ for index, bill in self.formatted_bill.iterrows(): if bill['Bill From Date'] <= norm_bill_date < bill['Bill To Date']: return index return None def days_in_raw_bill_period(self, norm_bill_date, norm_bill_date_respected_index, flag): - ''' + """ Return how many days from a normalized bill within a raw bill billing period - ''' + """ if flag == 'start': days = (self.formatted_bill['Bill To Date'][norm_bill_date_respected_index] - norm_bill_date).days @@ -1105,9 +1105,9 @@ class BillDisaggregation(): return days def weighted_unit_price(self, index_numdays): - ''' + """ Return the weighted average of unit price - ''' + """ weights = [] total_days = [] for ind in range(len(index_numdays)): @@ -1119,9 +1119,9 @@ class BillDisaggregation(): return weighted_unit_price def find_bills_in_raw(self, norm_bill_from, norm_bill_to): - ''' + """ Return the index / number of days in each raw bill billing period for a normalized billing period - ''' + """ norm_bill_days = (norm_bill_to - norm_bill_from).days results = [] @@ -1147,9 +1147,9 @@ class BillDisaggregation(): return results def normalized_unit_price(self, rawbill, mbill): - ''' + """ calculate the unit price for each nomralized billing period - ''' + """ normalized_unit_price = [] for m in range(len(mbill)): from_date = mbill['Bill From Date'].iloc[m] -- GitLab