From 1453abecaade25d9d48c3140481e3724c96cd207 Mon Sep 17 00:00:00 2001 From: gspasov Date: Wed, 13 Mar 2019 14:55:28 +0200 Subject: [PATCH 1/6] Implemented API module for exporting p2p conversation to a '.csv' file --- apps/roster/src/rest/rest_chat_csv.erl | 185 +++++++++++++++++++++++++ apps/roster/src/rest/rest_handler.erl | 1 + apps/roster/static/rest_var.hrl | 3 + 3 files changed, 189 insertions(+) create mode 100644 apps/roster/src/rest/rest_chat_csv.erl diff --git a/apps/roster/src/rest/rest_chat_csv.erl b/apps/roster/src/rest/rest_chat_csv.erl new file mode 100644 index 000000000..cf547af29 --- /dev/null +++ b/apps/roster/src/rest/rest_chat_csv.erl @@ -0,0 +1,185 @@ +%%%----------------------------------------------------------------------------- +%%% @doc API module for converting chat history to CSV file. +%%% +%%% Here we expose a POST request for generating csv file from a private chat room. +%%% The POST request requires phone numbers + ids of both users. +%%% +%%% @see rest_handler:handle_request/3 +%%% @author Georgi Spasov +%%% @end +%%%----------------------------------------------------------------------------- +-module(rest_chat_csv). +% -compile({parse_transform, lager_transform}). +-include("roster.hrl"). +-include_lib("roster/static/rest_var.hrl"). +-include_lib("kvs/include/metainfo.hrl"). + +-export([ + handle_request/3 +]). + +%% Custom data types +-type type_chat_element() :: {Date::list(), Time::string(), Username::string(), Message::string()}. +-type type_chat() :: [type_chat_element()]. +-type type_chat_history() :: list(#'Message'{}). +-type type_io_lib_format() :: list(string() | non_neg_integer() | list()). + +%%%============================================================================= +%%% API +%%%============================================================================= + +handle_request('POST', _Path, Req) -> + case parse_incomming_data(Req:parse_post()) of + {ok, {From, To}} -> + FromUsername = get_username_by_id(extract_user_id(From)), + ToUsername = get_username_by_id(extract_user_id(To)), + CsvText = conversation_to_csv({From, FromUsername}, {To, ToUsername}), + Filename = FromUsername ++ "-" ++ ToUsername, + + {ResponseStatus, ResponseData} = {?HTTP_CODE_200, CsvText}, + ResponseHeader = [{"Content-Type", "text/csv"}, {"Content-Disposition", "attachment; filename=" ++ Filename ++ ".csv"}], + Req:respond({ResponseStatus, ResponseHeader, ResponseData}); + + {error, wrong_data} -> + rest_response_helper:error_405_response(Req) + end; + +handle_request(_, _, Req) -> + rest_response_helper:error_405_response(Req). + +%%------------------------------------------------------------------------------ +%% @doc Converts conversation to csv file in few steps. +%% +%% The format of From and To is <<"359111111111_58">>. +%% Where: +%% - "359" is contry code, +%% - "111111111" is user phone number, +%% - "58" is user ID. +%% @end +%%------------------------------------------------------------------------------ +-spec conversation_to_csv(FromData::tuple(), ToData::tuple()) -> list(binary()). +conversation_to_csv({From, _} = FromData, {To, ToUsername} = ToData) -> + ChatHistory = roster_db:get_chain('Message', {p2p, From, To}), + ChatData = extract_chat_data(ChatHistory, FromData, ToData), + ChatSpan = get_chat_span(ChatData), + CsvColumns = build_csv_columns(ChatSpan, ToUsername), + build_csv_text(ChatData, CsvColumns). + +%%%============================================================================= +%%% Internal functions +%%%============================================================================= + +%%------------------------------------------------------------------------------ +%% @doc Parse incomming body if it fits the required pattern and format data to binary. +%% +%% @end +%%------------------------------------------------------------------------------ +-spec parse_incomming_data(list(tuple())) -> {ok, {binary(), binary()}} | {error, wrong_data}. +parse_incomming_data([{"from", From}, {"to", To}]) -> {ok, {list_to_binary(From), list_to_binary(To)}}; +parse_incomming_data(_) -> {error, wrong_data}. + +%%------------------------------------------------------------------------------ +%% @doc Concatenates the csv 'Header' columns with each formatted chat message. +%% +%% @end +%%------------------------------------------------------------------------------ +% -spec build_csv_text(Chat::list(tuple())) -> list(binary()). +% build_csv_text(Chat) -> +% build_csv_text(Chat, build_csv_columns(Chat)). + +-spec build_csv_text(type_chat(), Acc::list(binary())) -> list(binary()). +build_csv_text([{Date, Time, Username, Message} | T], Acc) -> + build_csv_text(T, [Acc, [<<"\n">>, Date, <<",">>, Time, <<",">>, Username, <<",">>, Message]]); +build_csv_text([], Acc) -> + lists:flatten(Acc). + +%%------------------------------------------------------------------------------ +%% @doc Builds the csv `Header` columns. +%% +%% @end +%%------------------------------------------------------------------------------ +-spec build_csv_columns({FirstDate::binary(), LastDate::binary()}, ToUsername::type_io_lib_format()) -> type_io_lib_format(). +build_csv_columns({FirstDate, LastDate}, ToUsername) -> + io_lib:format("NYNJA Chat Export\n\nFrom Date:,~s\nTo Date:,~s\n\nChat(s):,~s\n\nIncluded:,Text\n,Photos\n,Locations\n,Videos\n,Files\n\nSave Media as:,Links\n\nDate,Time,People,Message", + [FirstDate, LastDate, ToUsername]). + +%%------------------------------------------------------------------------------ +%% @doc Extract the needed data from each message between the users. +%% +%% Needed data: Date, Time, UserId, PayloadType, Payload. +%% @end +%%------------------------------------------------------------------------------ +-spec extract_chat_data(ChatHistory::type_chat_history(), tuple(), tuple()) -> type_chat(). +extract_chat_data(ChatHistory, {From, FromUsername}, {To, ToUsername}) -> + [begin + [Date, Time] = string:split(roster:timestamp_to_datetime(TimeCreated), "T"), + FormattedDate = format_date(Date), + Message = format_message(PayloadType, Payload), + Username = case User of + From -> FromUsername; + To -> ToUsername + end, + {FormattedDate, Time, Username, Message} + end + || + #'Message'{created = TimeCreated, + from = User, + files = [#'Desc'{mime = PayloadType, payload = Payload}]} <- ChatHistory + ]. + +%%------------------------------------------------------------------------------ +%% @doc Extracts the user id and converts it to integer. +%% +%% @end +%%------------------------------------------------------------------------------ +-spec extract_user_id(binary()) -> non_neg_integer(). +extract_user_id(PhoneAndId) -> + [_Phone, UserId] = string:split(PhoneAndId, "_"), + element(1, string:to_integer(UserId)). + +%%------------------------------------------------------------------------------ +%% @doc Formats the date in the format dd/mm/yyyy. +%% +%% @end +%%------------------------------------------------------------------------------ +-spec format_date(Date::string()) -> type_io_lib_format(). +format_date(Date) -> + [Year, Month, Day] = string:split(Date, "-", all), + io_lib:format("~s/~s/~s", [Day, Month, Year]). + +%%------------------------------------------------------------------------------ +%% @doc Append information about the payload if it's not plain-text. +%% +%% @end +%%------------------------------------------------------------------------------ +-spec format_message(PayloadType::binary(), Payload::binary()) -> binary() | list(binary()). +format_message(<<"text">>, Payload) -> Payload; +format_message(<<"file">>, Payload) -> [<<"File: ">>, Payload]; +format_message(<<"link">>, Payload) -> [<<"Link: ">>, Payload]; +format_message(<<"photo">>, Payload) -> [<<"Photo: ">>, Payload]; +format_message(<<"video">>, Payload) -> [<<"Video: ">>, Payload]; +format_message(<<"location">>, Payload) -> [<<"Location: ">>, Payload]. + +%%------------------------------------------------------------------------------ +%% @doc Gets First and Last name of User from db, "no name" if user_id is not found. +%% +%% @end +%%------------------------------------------------------------------------------ +-spec get_username_by_id(Id::non_neg_integer()) -> type_io_lib_format(). +get_username_by_id(Id) -> + case kvs:get('Roster', Id) of + {ok, #'Roster'{names = Name, surnames = Surname}} -> + io_lib:format("~s ~s", [Name, Surname]); + {error, _} -> + "no name" + end. +%%------------------------------------------------------------------------------ +%% @doc Returns First and Last name of User, "no name" if user_id is not found. +%% +%% @end +%%------------------------------------------------------------------------------ +-spec get_chat_span(Chat::type_chat_element()) -> {type_io_lib_format(), type_io_lib_format()}. +get_chat_span(Chat) -> + [{FirstDate, _, _, _} | _] = Chat, + [{LastDate, _, _, _} | _] = lists:reverse(Chat), + {FirstDate, LastDate}. \ No newline at end of file diff --git a/apps/roster/src/rest/rest_handler.erl b/apps/roster/src/rest/rest_handler.erl index 33cd88f4b..55275db3c 100644 --- a/apps/roster/src/rest/rest_handler.erl +++ b/apps/roster/src/rest/rest_handler.erl @@ -87,6 +87,7 @@ handle_request(Method, Path, Req) case ?CHECK_FAKE_NUMBERS of true -> rest_fnawm:handle_request(Method, Path, Req); _ -> handle_request_404(Method, Path, Req) end; +handle_request(Method, ?CHAT_CSV_ENDPOINT = Path, Req) -> rest_chat_csv:handle_request(Method, Path, Req); handle_request(Method, ?MSG_PUSH_ENDPOINT = Path, Req) -> rest_push:handle_request(Method, Path, Req); handle_request(Method, ?ROOM_ENDPOINT = Path, Req) -> rest_deeplink:handle_request(Method, Path, Req); handle_request(Method, ?PUBLISH_ENDPOINT = Path, Req) -> rest_publish:handle_request(Method, Path, Req); diff --git a/apps/roster/static/rest_var.hrl b/apps/roster/static/rest_var.hrl index 2c13ed749..a1063ba4c 100644 --- a/apps/roster/static/rest_var.hrl +++ b/apps/roster/static/rest_var.hrl @@ -18,6 +18,9 @@ -define(USERS_ENDPOINT, "/users"). -define(METRICS_ENDPOINT, "/metrics"). +% Chat to Csv file +-define(CHAT_CSV_ENDPOINT, "/csv/chat"). + -define(RCI_ROOM_TYPE_ENDPOINT, "/cri/rooms/type"). -define(RCI_ROOM_ENDPOINT, "/cri/rooms"). -define(RCI_ROOM_MEMBERS_ENDPOINT, "/cri/rooms/members"). -- GitLab From 53827934525460592a5d4d6d74ca219586b2c95b Mon Sep 17 00:00:00 2001 From: gspasov Date: Wed, 13 Mar 2019 20:41:24 +0200 Subject: [PATCH 2/6] Changed io_lib for IO list, changed endpoint, changed error response, fixed types --- apps/roster/src/rest/rest_chat_csv.erl | 58 ++++++++++++++++---------- apps/roster/src/rest/rest_handler.erl | 3 +- apps/roster/static/rest_var.hrl | 5 ++- 3 files changed, 40 insertions(+), 26 deletions(-) diff --git a/apps/roster/src/rest/rest_chat_csv.erl b/apps/roster/src/rest/rest_chat_csv.erl index cf547af29..d121dc3ef 100644 --- a/apps/roster/src/rest/rest_chat_csv.erl +++ b/apps/roster/src/rest/rest_chat_csv.erl @@ -9,7 +9,6 @@ %%% @end %%%----------------------------------------------------------------------------- -module(rest_chat_csv). -% -compile({parse_transform, lager_transform}). -include("roster.hrl"). -include_lib("roster/static/rest_var.hrl"). -include_lib("kvs/include/metainfo.hrl"). @@ -22,26 +21,29 @@ -type type_chat_element() :: {Date::list(), Time::string(), Username::string(), Message::string()}. -type type_chat() :: [type_chat_element()]. -type type_chat_history() :: list(#'Message'{}). --type type_io_lib_format() :: list(string() | non_neg_integer() | list()). %%%============================================================================= %%% API %%%============================================================================= -handle_request('POST', _Path, Req) -> - case parse_incomming_data(Req:parse_post()) of - {ok, {From, To}} -> - FromUsername = get_username_by_id(extract_user_id(From)), - ToUsername = get_username_by_id(extract_user_id(To)), - CsvText = conversation_to_csv({From, FromUsername}, {To, ToUsername}), - Filename = FromUsername ++ "-" ++ ToUsername, +handle_request('POST', ?CSV_P2P_ENDPOINT, Req) -> + ReqData = Req:parse_post(), + roster:info(?MODULE, "~p:Request:~p:~p", [Req:get(method), Req:get(path), ReqData]), - {ResponseStatus, ResponseData} = {?HTTP_CODE_200, CsvText}, - ResponseHeader = [{"Content-Type", "text/csv"}, {"Content-Disposition", "attachment; filename=" ++ Filename ++ ".csv"}], - Req:respond({ResponseStatus, ResponseHeader, ResponseData}); + case parse_incomming_data(ReqData) of + {ok, {From, To}} -> + FromUsername = get_username_by_id(extract_user_id(From)), + ToUsername = get_username_by_id(extract_user_id(To)), + CsvText = conversation_to_csv({From, FromUsername}, {To, ToUsername}), + Filename = [FromUsername, "-", ToUsername, ".csv"], + + {ResponseStatus, ResponseData} = {?HTTP_CODE_200, CsvText}, + ResponseHeader = [{"Content-Type", "text/csv"}, {"Content-Disposition", "attachment; filename=" ++ Filename}], + roster:info(?MODULE, "ResponseData:~p", [ResponseData]), + Req:respond({ResponseStatus, ResponseHeader, ResponseData}); {error, wrong_data} -> - rest_response_helper:error_405_response(Req) + rest_response_helper:error_400_response(Req) end; handle_request(_, _, Req) -> @@ -89,7 +91,7 @@ parse_incomming_data(_) -> {error, wrong_data}. -spec build_csv_text(type_chat(), Acc::list(binary())) -> list(binary()). build_csv_text([{Date, Time, Username, Message} | T], Acc) -> - build_csv_text(T, [Acc, [<<"\n">>, Date, <<",">>, Time, <<",">>, Username, <<",">>, Message]]); + build_csv_text(T, [Acc, [$\n, Date, $,, Time, $,, Username, $,, Message]]); build_csv_text([], Acc) -> lists:flatten(Acc). @@ -98,10 +100,20 @@ build_csv_text([], Acc) -> %% %% @end %%------------------------------------------------------------------------------ --spec build_csv_columns({FirstDate::binary(), LastDate::binary()}, ToUsername::type_io_lib_format()) -> type_io_lib_format(). +-spec build_csv_columns({FirstDate::binary(), LastDate::binary()}, ToUsername::string()) -> string(). build_csv_columns({FirstDate, LastDate}, ToUsername) -> - io_lib:format("NYNJA Chat Export\n\nFrom Date:,~s\nTo Date:,~s\n\nChat(s):,~s\n\nIncluded:,Text\n,Photos\n,Locations\n,Videos\n,Files\n\nSave Media as:,Links\n\nDate,Time,People,Message", - [FirstDate, LastDate, ToUsername]). + [ + "NYNJA Chat Export", $\n, $\n, + "From Date:,", FirstDate, $\n, + "To Date:,", LastDate, $\n, $\n, + "Chat(s):,", ToUsername, $\n, $\n, + "Included:,Text", $\n, + ",Photos", $\n, + ",Locations", $\n, + ",Files", $\n, $\n, + "Save Media as:,Links", $\n, $\n, + "Date,Time,People,Message" + ]. %%------------------------------------------------------------------------------ %% @doc Extract the needed data from each message between the users. @@ -142,10 +154,10 @@ extract_user_id(PhoneAndId) -> %% %% @end %%------------------------------------------------------------------------------ --spec format_date(Date::string()) -> type_io_lib_format(). +-spec format_date(Date::string()) -> string(). format_date(Date) -> [Year, Month, Day] = string:split(Date, "-", all), - io_lib:format("~s/~s/~s", [Day, Month, Year]). + [Day, $/, Month, $/, Year]. %%------------------------------------------------------------------------------ %% @doc Append information about the payload if it's not plain-text. @@ -165,11 +177,11 @@ format_message(<<"location">>, Payload) -> [<<"Location: ">>, Payload]. %% %% @end %%------------------------------------------------------------------------------ --spec get_username_by_id(Id::non_neg_integer()) -> type_io_lib_format(). +-spec get_username_by_id(Id::non_neg_integer()) -> string(). get_username_by_id(Id) -> case kvs:get('Roster', Id) of - {ok, #'Roster'{names = Name, surnames = Surname}} -> - io_lib:format("~s ~s", [Name, Surname]); + {ok, #'Roster'{names = Name, surnames = Surname}} -> + [Name, $ , Surname]; {error, _} -> "no name" end. @@ -178,7 +190,7 @@ get_username_by_id(Id) -> %% %% @end %%------------------------------------------------------------------------------ --spec get_chat_span(Chat::type_chat_element()) -> {type_io_lib_format(), type_io_lib_format()}. +-spec get_chat_span(Chat::type_chat_element()) -> {string(), string()}. get_chat_span(Chat) -> [{FirstDate, _, _, _} | _] = Chat, [{LastDate, _, _, _} | _] = lists:reverse(Chat), diff --git a/apps/roster/src/rest/rest_handler.erl b/apps/roster/src/rest/rest_handler.erl index 55275db3c..a428a881b 100644 --- a/apps/roster/src/rest/rest_handler.erl +++ b/apps/roster/src/rest/rest_handler.erl @@ -87,7 +87,8 @@ handle_request(Method, Path, Req) case ?CHECK_FAKE_NUMBERS of true -> rest_fnawm:handle_request(Method, Path, Req); _ -> handle_request_404(Method, Path, Req) end; -handle_request(Method, ?CHAT_CSV_ENDPOINT = Path, Req) -> rest_chat_csv:handle_request(Method, Path, Req); +handle_request(Method, ?CSV_P2P_ENDPOINT = Path, Req) -> rest_chat_csv:handle_request(Method, Path, Req); +handle_request(Method, ?CSV_MUC_ENDPOINT = Path, Req) -> rest_chat_csv:handle_request(Method, Path, Req); handle_request(Method, ?MSG_PUSH_ENDPOINT = Path, Req) -> rest_push:handle_request(Method, Path, Req); handle_request(Method, ?ROOM_ENDPOINT = Path, Req) -> rest_deeplink:handle_request(Method, Path, Req); handle_request(Method, ?PUBLISH_ENDPOINT = Path, Req) -> rest_publish:handle_request(Method, Path, Req); diff --git a/apps/roster/static/rest_var.hrl b/apps/roster/static/rest_var.hrl index a1063ba4c..b31bbc5cf 100644 --- a/apps/roster/static/rest_var.hrl +++ b/apps/roster/static/rest_var.hrl @@ -18,8 +18,9 @@ -define(USERS_ENDPOINT, "/users"). -define(METRICS_ENDPOINT, "/metrics"). -% Chat to Csv file --define(CHAT_CSV_ENDPOINT, "/csv/chat"). +% p2p and muc Chat to Csv file +-define(CSV_P2P_ENDPOINT, "/csv/p2p"). +-define(CSV_MUC_ENDPOINT, "/csv/muc"). -define(RCI_ROOM_TYPE_ENDPOINT, "/cri/rooms/type"). -define(RCI_ROOM_ENDPOINT, "/cri/rooms"). -- GitLab From 7a4b8c90942661fd7f9fe34ad271c6e07c67530f Mon Sep 17 00:00:00 2001 From: gspasov Date: Wed, 13 Mar 2019 21:54:58 +0200 Subject: [PATCH 3/6] Changed func for getting date, added func for formatting time, removed unnecessary docs, fixed docs --- apps/roster/src/rest/rest_chat_csv.erl | 25 +++++++++++-------------- 1 file changed, 11 insertions(+), 14 deletions(-) diff --git a/apps/roster/src/rest/rest_chat_csv.erl b/apps/roster/src/rest/rest_chat_csv.erl index d121dc3ef..3b3b2a4a3 100644 --- a/apps/roster/src/rest/rest_chat_csv.erl +++ b/apps/roster/src/rest/rest_chat_csv.erl @@ -72,7 +72,7 @@ conversation_to_csv({From, _} = FromData, {To, ToUsername} = ToData) -> %%%============================================================================= %%------------------------------------------------------------------------------ -%% @doc Parse incomming body if it fits the required pattern and format data to binary. +%% @doc Parse incomming body if it fits the required pattern and parse data to binary. %% %% @end %%------------------------------------------------------------------------------ @@ -124,14 +124,15 @@ build_csv_columns({FirstDate, LastDate}, ToUsername) -> -spec extract_chat_data(ChatHistory::type_chat_history(), tuple(), tuple()) -> type_chat(). extract_chat_data(ChatHistory, {From, FromUsername}, {To, ToUsername}) -> [begin - [Date, Time] = string:split(roster:timestamp_to_datetime(TimeCreated), "T"), + {Date, Time} = roster:msToUT(TimeCreated), FormattedDate = format_date(Date), + FormattedTime = format_time() Message = format_message(PayloadType, Payload), Username = case User of From -> FromUsername; To -> ToUsername end, - {FormattedDate, Time, Username, Message} + {FormattedDate, FormattedTime, Username, Message} end || #'Message'{created = TimeCreated, @@ -149,15 +150,11 @@ extract_user_id(PhoneAndId) -> [_Phone, UserId] = string:split(PhoneAndId, "_"), element(1, string:to_integer(UserId)). -%%------------------------------------------------------------------------------ -%% @doc Formats the date in the format dd/mm/yyyy. -%% -%% @end -%%------------------------------------------------------------------------------ --spec format_date(Date::string()) -> string(). -format_date(Date) -> - [Year, Month, Day] = string:split(Date, "-", all), - [Day, $/, Month, $/, Year]. +-spec format_date(tuple()) -> string(). +format_date({Year, Month, Day}}) -> [Day, $/, Month, $/, Year]. + +-spec format_time(tuple()) -> string(). +format_time({Hour, Min, Sec}}) -> [Hour, $:, Min, $:, Sec]. %%------------------------------------------------------------------------------ %% @doc Append information about the payload if it's not plain-text. @@ -181,12 +178,12 @@ format_message(<<"location">>, Payload) -> [<<"Location: ">>, Payload]. get_username_by_id(Id) -> case kvs:get('Roster', Id) of {ok, #'Roster'{names = Name, surnames = Surname}} -> - [Name, $ , Surname]; + [Name, $ , Surname]; %% Empty space after the '$' sign is important here! {error, _} -> "no name" end. %%------------------------------------------------------------------------------ -%% @doc Returns First and Last name of User, "no name" if user_id is not found. +%% @doc Returns First and Last date of the messages in the chat. %% %% @end %%------------------------------------------------------------------------------ -- GitLab From f4abc39ce157cc5263110f230a4b25628202e4f1 Mon Sep 17 00:00:00 2001 From: gspasov Date: Thu, 14 Mar 2019 14:57:31 +0200 Subject: [PATCH 4/6] Added functionality for extracting group chat, removed unecessary docs --- apps/roster/src/rest/rest_chat_csv.erl | 177 +++++++++++-------------- 1 file changed, 80 insertions(+), 97 deletions(-) diff --git a/apps/roster/src/rest/rest_chat_csv.erl b/apps/roster/src/rest/rest_chat_csv.erl index 3b3b2a4a3..a76fd156d 100644 --- a/apps/roster/src/rest/rest_chat_csv.erl +++ b/apps/roster/src/rest/rest_chat_csv.erl @@ -5,7 +5,6 @@ %%% The POST request requires phone numbers + ids of both users. %%% %%% @see rest_handler:handle_request/3 -%%% @author Georgi Spasov %%% @end %%%----------------------------------------------------------------------------- -module(rest_chat_csv). @@ -14,33 +13,42 @@ -include_lib("kvs/include/metainfo.hrl"). -export([ - handle_request/3 + handle_request/3, + get_group_chat_members/1, + match_member_to_phone_id/2 ]). %% Custom data types --type type_chat_element() :: {Date::list(), Time::string(), Username::string(), Message::string()}. +-type type_chat_element() :: {Date::string(), Time::string(), Username::string(), Message::string()}. -type type_chat() :: [type_chat_element()]. -type type_chat_history() :: list(#'Message'{}). +-type type_user_data() :: {binary(), string()}. %%%============================================================================= %%% API %%%============================================================================= -handle_request('POST', ?CSV_P2P_ENDPOINT, Req) -> +handle_request('POST', Path, Req) when Path == ?CSV_P2P_ENDPOINT; Path == ?CSV_MUC_ENDPOINT -> ReqData = Req:parse_post(), roster:info(?MODULE, "~p:Request:~p:~p", [Req:get(method), Req:get(path), ReqData]), case parse_incomming_data(ReqData) of - {ok, {From, To}} -> - FromUsername = get_username_by_id(extract_user_id(From)), - ToUsername = get_username_by_id(extract_user_id(To)), - CsvText = conversation_to_csv({From, FromUsername}, {To, ToUsername}), + {ok, {FromPhoneId, ToPhoneId}} -> + ChatHistory = roster_db:get_chain('Message', {p2p, FromPhoneId, ToPhoneId}), + FromUsername = get_username_by_id(extract_user_id(FromPhoneId)), + ToUsername = get_username_by_id(extract_user_id(ToPhoneId)), + Members = [{FromPhoneId, FromUsername}, {ToPhoneId, ToUsername}], + CsvText = conversation_to_csv(ChatHistory, Members, ToUsername), Filename = [FromUsername, "-", ToUsername, ".csv"], + send_response(CsvText, Filename, Req); - {ResponseStatus, ResponseData} = {?HTTP_CODE_200, CsvText}, - ResponseHeader = [{"Content-Type", "text/csv"}, {"Content-Disposition", "attachment; filename=" ++ Filename}], - roster:info(?MODULE, "ResponseData:~p", [ResponseData]), - Req:respond({ResponseStatus, ResponseHeader, ResponseData}); + {ok, RoomId} -> + ChatHistory = roster_db:get_chain('Message', {muc, RoomId}), + Members = get_group_chat_members(RoomId), + GroupChatName = get_group_chat_name(RoomId), + CsvText = conversation_to_csv(ChatHistory, Members, GroupChatName), + Filename = ["Group_chat-", GroupChatName, ".csv"], + send_response(CsvText, Filename, Req); {error, wrong_data} -> rest_response_helper:error_400_response(Req) @@ -49,57 +57,34 @@ handle_request('POST', ?CSV_P2P_ENDPOINT, Req) -> handle_request(_, _, Req) -> rest_response_helper:error_405_response(Req). -%%------------------------------------------------------------------------------ -%% @doc Converts conversation to csv file in few steps. -%% -%% The format of From and To is <<"359111111111_58">>. -%% Where: -%% - "359" is contry code, -%% - "111111111" is user phone number, -%% - "58" is user ID. -%% @end -%%------------------------------------------------------------------------------ --spec conversation_to_csv(FromData::tuple(), ToData::tuple()) -> list(binary()). -conversation_to_csv({From, _} = FromData, {To, ToUsername} = ToData) -> - ChatHistory = roster_db:get_chain('Message', {p2p, From, To}), - ChatData = extract_chat_data(ChatHistory, FromData, ToData), - ChatSpan = get_chat_span(ChatData), - CsvColumns = build_csv_columns(ChatSpan, ToUsername), - build_csv_text(ChatData, CsvColumns). - %%%============================================================================= %%% Internal functions %%%============================================================================= -%%------------------------------------------------------------------------------ -%% @doc Parse incomming body if it fits the required pattern and parse data to binary. -%% -%% @end -%%------------------------------------------------------------------------------ --spec parse_incomming_data(list(tuple())) -> {ok, {binary(), binary()}} | {error, wrong_data}. -parse_incomming_data([{"from", From}, {"to", To}]) -> {ok, {list_to_binary(From), list_to_binary(To)}}; -parse_incomming_data(_) -> {error, wrong_data}. +send_response(CsvText, Filename, Req) -> + {ResponseStatus, ResponseData} = {?HTTP_CODE_200, CsvText}, + ResponseHeader = [{"Content-Type", "text/csv"}, {"Content-Disposition", "attachment; filename=" ++ Filename}], + roster:info(?MODULE, "ResponseData:~p", [ResponseData]), + Req:respond({ResponseStatus, ResponseHeader, ResponseData}). + +-spec conversation_to_csv(ChatHistory::type_chat_history(), Members::list(type_user_data()), CsvColumnsData::string()) -> string(). +conversation_to_csv(ChatHistory, Members, CsvColumnsData) -> + ChatData = extract_chat_data(ChatHistory, Members), + ChatSpan = get_chat_time_span(ChatData), + CsvColumns = build_csv_columns(ChatSpan, CsvColumnsData), + build_csv_text(ChatData, CsvColumns). -%%------------------------------------------------------------------------------ -%% @doc Concatenates the csv 'Header' columns with each formatted chat message. -%% -%% @end -%%------------------------------------------------------------------------------ -% -spec build_csv_text(Chat::list(tuple())) -> list(binary()). -% build_csv_text(Chat) -> -% build_csv_text(Chat, build_csv_columns(Chat)). +-spec parse_incomming_data(list(tuple())) -> {ok, term()} | {error, wrong_data}. +parse_incomming_data([{"from", FromPhoneId}, {"to", ToPhoneId}]) -> {ok, {list_to_binary(FromPhoneId), list_to_binary(ToPhoneId)}}; +parse_incomming_data([{"muc", RoomId}]) -> {ok, list_to_binary(RoomId)}; +parse_incomming_data(_) -> {error, wrong_data}. --spec build_csv_text(type_chat(), Acc::list(binary())) -> list(binary()). +-spec build_csv_text(type_chat(), Acc::string()) -> string(). build_csv_text([{Date, Time, Username, Message} | T], Acc) -> build_csv_text(T, [Acc, [$\n, Date, $,, Time, $,, Username, $,, Message]]); build_csv_text([], Acc) -> lists:flatten(Acc). -%%------------------------------------------------------------------------------ -%% @doc Builds the csv `Header` columns. -%% -%% @end -%%------------------------------------------------------------------------------ -spec build_csv_columns({FirstDate::binary(), LastDate::binary()}, ToUsername::string()) -> string(). build_csv_columns({FirstDate, LastDate}, ToUsername) -> [ @@ -115,52 +100,39 @@ build_csv_columns({FirstDate, LastDate}, ToUsername) -> "Date,Time,People,Message" ]. -%%------------------------------------------------------------------------------ -%% @doc Extract the needed data from each message between the users. -%% -%% Needed data: Date, Time, UserId, PayloadType, Payload. -%% @end -%%------------------------------------------------------------------------------ --spec extract_chat_data(ChatHistory::type_chat_history(), tuple(), tuple()) -> type_chat(). -extract_chat_data(ChatHistory, {From, FromUsername}, {To, ToUsername}) -> +-spec extract_chat_data(ChatHistory::type_chat_history(), Members::list(type_user_data())) -> type_chat(). +extract_chat_data(ChatHistory, Members) -> [begin {Date, Time} = roster:msToUT(TimeCreated), FormattedDate = format_date(Date), - FormattedTime = format_time() + FormattedTime = format_time(Time), + Username = match_member_to_phone_id(Members, PhoneId), Message = format_message(PayloadType, Payload), - Username = case User of - From -> FromUsername; - To -> ToUsername - end, {FormattedDate, FormattedTime, Username, Message} end || #'Message'{created = TimeCreated, - from = User, - files = [#'Desc'{mime = PayloadType, payload = Payload}]} <- ChatHistory + from = PhoneId, + files = [#'Desc'{mime = PayloadType, payload = Payload}]} = M <- ChatHistory ]. -%%------------------------------------------------------------------------------ -%% @doc Extracts the user id and converts it to integer. -%% -%% @end -%%------------------------------------------------------------------------------ +match_member_to_phone_id(Members, PhoneId) -> + case lists:dropwhile(fun({Id, _}) -> Id /= PhoneId end, Members) of + [] -> "no name"; %% TODO: Figure what result should be returned here + [{_, Username} | _] -> Username + end. + -spec extract_user_id(binary()) -> non_neg_integer(). -extract_user_id(PhoneAndId) -> - [_Phone, UserId] = string:split(PhoneAndId, "_"), +extract_user_id(PhoneId) -> + [_Phone, UserId] = string:split(PhoneId, "_"), element(1, string:to_integer(UserId)). -spec format_date(tuple()) -> string(). -format_date({Year, Month, Day}}) -> [Day, $/, Month, $/, Year]. +format_date({Year, Month, Day}) -> io_lib:format("~2..0w/~2..0w/~w", [Day, Month, Year]). -spec format_time(tuple()) -> string(). -format_time({Hour, Min, Sec}}) -> [Hour, $:, Min, $:, Sec]. +format_time({Hour, Min, Sec}) -> io_lib:format("~2..0w:~2..0w:~2..0w", [Hour, Min, Sec]). -%%------------------------------------------------------------------------------ -%% @doc Append information about the payload if it's not plain-text. -%% -%% @end -%%------------------------------------------------------------------------------ -spec format_message(PayloadType::binary(), Payload::binary()) -> binary() | list(binary()). format_message(<<"text">>, Payload) -> Payload; format_message(<<"file">>, Payload) -> [<<"File: ">>, Payload]; @@ -169,26 +141,37 @@ format_message(<<"photo">>, Payload) -> [<<"Photo: ">>, Payload]; format_message(<<"video">>, Payload) -> [<<"Video: ">>, Payload]; format_message(<<"location">>, Payload) -> [<<"Location: ">>, Payload]. -%%------------------------------------------------------------------------------ -%% @doc Gets First and Last name of User from db, "no name" if user_id is not found. -%% -%% @end -%%------------------------------------------------------------------------------ -spec get_username_by_id(Id::non_neg_integer()) -> string(). get_username_by_id(Id) -> case kvs:get('Roster', Id) of - {ok, #'Roster'{names = Name, surnames = Surname}} -> - [Name, $ , Surname]; %% Empty space after the '$' sign is important here! - {error, _} -> - "no name" + {ok, #'Roster'{names = Name, surnames = Surname}} -> io_lib:format("~s ~s", [Name, Surname]); + {error, _} -> "no name" + end. + +-spec get_group_chat_members(RoomId::binary()) -> list(#'Member'{}). +get_group_chat_members(RoomId) -> + case roster:members({muc, RoomId}) of + [] -> + "room doesn't exist"; %% TODO: Figure what result should be returned here + Members -> + [ + {PhoneId, io_lib:format("~s ~s", [Name, Surname])} + || + #'Member'{names = Name, + surnames = Surname, + phone_id = PhoneId} <- Members + ] end. -%%------------------------------------------------------------------------------ -%% @doc Returns First and Last date of the messages in the chat. -%% -%% @end -%%------------------------------------------------------------------------------ --spec get_chat_span(Chat::type_chat_element()) -> {string(), string()}. -get_chat_span(Chat) -> + +-spec get_group_chat_name(RoomId::binary()) -> string(). +get_group_chat_name(RoomId) -> + case kvs:get('Room', RoomId) of + {ok, #'Room'{name = RoomName}} -> RoomName; + {error, _} -> "no name" + end. + +-spec get_chat_time_span(Chat::type_chat_element()) -> {string(), string()}. +get_chat_time_span(Chat) -> [{FirstDate, _, _, _} | _] = Chat, [{LastDate, _, _, _} | _] = lists:reverse(Chat), {FirstDate, LastDate}. \ No newline at end of file -- GitLab From 2c40d475ae7899522175125beae85572f379b35c Mon Sep 17 00:00:00 2001 From: gspasov Date: Fri, 15 Mar 2019 09:49:05 +0200 Subject: [PATCH 5/6] Add handling of errors, add more methods to the rest_response_helper --- .../src/rest/helpers/rest_response_helper.erl | 26 +++++++- apps/roster/src/rest/rest_chat_csv.erl | 60 ++++++++++++------- apps/roster/static/rest_text.hrl | 1 + apps/roster/static/rest_var.hrl | 3 +- 4 files changed, 64 insertions(+), 26 deletions(-) diff --git a/apps/roster/src/rest/helpers/rest_response_helper.erl b/apps/roster/src/rest/helpers/rest_response_helper.erl index 5a83b41b8..b37e074d2 100644 --- a/apps/roster/src/rest/helpers/rest_response_helper.erl +++ b/apps/roster/src/rest/helpers/rest_response_helper.erl @@ -5,7 +5,8 @@ -export([ description/0, response/3, response/4, - error_405/0, error_404/0, error_401/0, error_405_response/1, + error_400/0, error_401/0, error_404/0, error_405/0, error_422/0, + error_400_response/1, error_404_response/1, error_405_response/1, error_422_response/1, error_response/0, error_response/1, error_response/2, success_response/0, success_response/1, success_response/2, success_200/0 @@ -25,14 +26,20 @@ response(Req, Status, Body, ContentType) -> %% Static +error_400() -> + error_response(?HTTP_CODE_400, ?ERROR_400). + +error_401() -> + error_response(?HTTP_CODE_401, ?ERROR_401). + error_404() -> error_response(?HTTP_CODE_404, ?ERROR_404). error_405() -> error_response(?HTTP_CODE_405, ?ERROR_405). -error_401() -> - error_response(?HTTP_CODE_401, ?ERROR_401). +error_422() -> + error_response(?HTTP_CODE_422, ?ERROR_422). %% Dynamic @@ -47,10 +54,23 @@ error_response(Status, []) -> error_response(Status, Msg) -> response_json(Status, jsx:encode([{<<"message">>, nitro:to_binary(Msg)}])). + +error_400_response(Req) -> + roster:info(?MODULE, "BadRequest:~p:~p:~p", [Req:get(method), Req:get(path), Req:parse_post()]), + response(Req, ?HTTP_CODE_400, rest_response_helper:error_400()). + +error_404_response(Req) -> + roster:info(?MODULE, "NotFound:~p:~p:~p", [Req:get(method), Req:get(path), Req:parse_post()]), + response(Req, ?HTTP_CODE_404, rest_response_helper:error_404()). + error_405_response(Req) -> roster:info(?MODULE, "MethodNotAllowed:~p:~p", [Req:get(method), Req:get(path)]), response(Req, ?HTTP_CODE_405, rest_response_helper:error_405()). +error_422_response(Req) -> + roster:info(?MODULE, "UnprocessableEntity:~p:~p:~p", [Req:get(method), Req:get(path), Req:parse_post()]), + response(Req, ?HTTP_CODE_422, rest_response_helper:error_422()). + %% - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - %% SUCCESS %% - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/apps/roster/src/rest/rest_chat_csv.erl b/apps/roster/src/rest/rest_chat_csv.erl index a76fd156d..60ba4d2e3 100644 --- a/apps/roster/src/rest/rest_chat_csv.erl +++ b/apps/roster/src/rest/rest_chat_csv.erl @@ -13,9 +13,7 @@ -include_lib("kvs/include/metainfo.hrl"). -export([ - handle_request/3, - get_group_chat_members/1, - match_member_to_phone_id/2 + handle_request/3 ]). %% Custom data types @@ -34,20 +32,29 @@ handle_request('POST', Path, Req) when Path == ?CSV_P2P_ENDPOINT; Path == ?CSV_M case parse_incomming_data(ReqData) of {ok, {FromPhoneId, ToPhoneId}} -> - ChatHistory = roster_db:get_chain('Message', {p2p, FromPhoneId, ToPhoneId}), - FromUsername = get_username_by_id(extract_user_id(FromPhoneId)), - ToUsername = get_username_by_id(extract_user_id(ToPhoneId)), - Members = [{FromPhoneId, FromUsername}, {ToPhoneId, ToUsername}], - CsvText = conversation_to_csv(ChatHistory, Members, ToUsername), - Filename = [FromUsername, "-", ToUsername, ".csv"], - send_response(CsvText, Filename, Req); + try + ChatHistory = get_chat_history({p2p, FromPhoneId, ToPhoneId}), + FromUsername = get_username_by_id(extract_user_id(FromPhoneId)), + ToUsername = get_username_by_id(extract_user_id(ToPhoneId)), + Members = [{FromPhoneId, FromUsername}, {ToPhoneId, ToUsername}], + CsvText = conversation_to_csv(ChatHistory, Members, ToUsername), + Filename = ["NynjaPrivateChat-", FromUsername, "-", ToUsername, ".csv"], + send_response(CsvText, Filename, Req) + catch + throw:no_username_match -> rest_response_helper:error_422_response(Req); + throw:no_chat_history -> rest_response_helper:error_404_response(Req); + throw:no_username -> rest_response_helper:error_422_response(Req); + throw:no_muc_name -> rest_response_helper:error_404_response(Req); + throw:no_room -> rest_response_helper:error_404_response(Req); + _:_ -> rest_response_helper:error_404_response(Req) + end; {ok, RoomId} -> - ChatHistory = roster_db:get_chain('Message', {muc, RoomId}), + ChatHistory = get_chat_history({muc, RoomId}), Members = get_group_chat_members(RoomId), GroupChatName = get_group_chat_name(RoomId), CsvText = conversation_to_csv(ChatHistory, Members, GroupChatName), - Filename = ["Group_chat-", GroupChatName, ".csv"], + Filename = ["NynjaGroupChat-", GroupChatName, ".csv"], send_response(CsvText, Filename, Req); {error, wrong_data} -> @@ -113,20 +120,22 @@ extract_chat_data(ChatHistory, Members) -> || #'Message'{created = TimeCreated, from = PhoneId, - files = [#'Desc'{mime = PayloadType, payload = Payload}]} = M <- ChatHistory + files = [#'Desc'{mime = PayloadType, payload = Payload}]} <- ChatHistory ]. -match_member_to_phone_id(Members, PhoneId) -> - case lists:dropwhile(fun({Id, _}) -> Id /= PhoneId end, Members) of - [] -> "no name"; %% TODO: Figure what result should be returned here - [{_, Username} | _] -> Username - end. - -spec extract_user_id(binary()) -> non_neg_integer(). extract_user_id(PhoneId) -> [_Phone, UserId] = string:split(PhoneId, "_"), element(1, string:to_integer(UserId)). +-spec match_member_to_phone_id(Members::list(type_user_data()), PhoneId::binary()) -> string(). +match_member_to_phone_id(Members, PhoneId) -> + case lists:dropwhile(fun({Id, _}) -> Id /= PhoneId end, Members) of + [] -> throw(no_username_match); + [{_, Username} | _] -> Username; + _ -> throw(no_username_match) + end. + -spec format_date(tuple()) -> string(). format_date({Year, Month, Day}) -> io_lib:format("~2..0w/~2..0w/~w", [Day, Month, Year]). @@ -145,14 +154,14 @@ format_message(<<"location">>, Payload) -> [<<"Location: ">>, Payload]. get_username_by_id(Id) -> case kvs:get('Roster', Id) of {ok, #'Roster'{names = Name, surnames = Surname}} -> io_lib:format("~s ~s", [Name, Surname]); - {error, _} -> "no name" + {error, _} -> throw(no_username) end. -spec get_group_chat_members(RoomId::binary()) -> list(#'Member'{}). get_group_chat_members(RoomId) -> case roster:members({muc, RoomId}) of [] -> - "room doesn't exist"; %% TODO: Figure what result should be returned here + throw(no_room); Members -> [ {PhoneId, io_lib:format("~s ~s", [Name, Surname])} @@ -167,7 +176,14 @@ get_group_chat_members(RoomId) -> get_group_chat_name(RoomId) -> case kvs:get('Room', RoomId) of {ok, #'Room'{name = RoomName}} -> RoomName; - {error, _} -> "no name" + {error, _} -> throw(no_muc_name) + end. + +-spec get_chat_history(Feed::tuple()) -> list(#'Message'{}). +get_chat_history(Feed) -> + case roster_db:get_chain('Message', Feed) of + [] -> throw(no_chat_history); + History -> History end. -spec get_chat_time_span(Chat::type_chat_element()) -> {string(), string()}. diff --git a/apps/roster/static/rest_text.hrl b/apps/roster/static/rest_text.hrl index 692e23675..4f46c4a84 100644 --- a/apps/roster/static/rest_text.hrl +++ b/apps/roster/static/rest_text.hrl @@ -11,6 +11,7 @@ -define(ERROR_403, <<"Forbidden">>). -define(ERROR_404, <<"Not Found">>). -define(ERROR_405, <<"Method Not Allowed">>). +-define(ERROR_422, <<"Unprocessable Entity">>). -define(ERROR_INVALID_JSON, <<"Invalid Request Json">>). -define(ERROR_MISSING_PARAM, <<"Missing Request Parameter(s)">>). -define(ERROR_INVALID_REQUEST_PARAM, <<"Invalid Request Parameter(s)">>). diff --git a/apps/roster/static/rest_var.hrl b/apps/roster/static/rest_var.hrl index b31bbc5cf..22c1afb79 100644 --- a/apps/roster/static/rest_var.hrl +++ b/apps/roster/static/rest_var.hrl @@ -46,4 +46,5 @@ -define(HTTP_CODE_401, 401). -define(HTTP_CODE_403, 403). -define(HTTP_CODE_404, 404). --define(HTTP_CODE_405, 405). \ No newline at end of file +-define(HTTP_CODE_405, 405). +-define(HTTP_CODE_422, 422). \ No newline at end of file -- GitLab From 0845c0e881200637f0e697bf6e2d46ae5c807703 Mon Sep 17 00:00:00 2001 From: gspasov Date: Fri, 15 Mar 2019 13:53:56 +0200 Subject: [PATCH 6/6] Fix merge conflit --- apps/roster/src/rest/rest_handler.erl | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/apps/roster/src/rest/rest_handler.erl b/apps/roster/src/rest/rest_handler.erl index abb26f786..987856a31 100644 --- a/apps/roster/src/rest/rest_handler.erl +++ b/apps/roster/src/rest/rest_handler.erl @@ -78,14 +78,12 @@ handle_request(Method, "/admin_whitelist" = Path, Req) -> case ?CHECK_WHITELIST handle_request(Method, Path, Req) when Path == "/fake_numbers"; Path == "/fn" -> case ?CHECK_FAKE_NUMBERS of true -> rest_fake_numbers:handle_request(Method, Path, Req); _ -> handle_request_404(Method, Path, Req) end; -<<<<<<< HEAD -handle_request(Method, ?CSV_P2P_ENDPOINT = Path, Req) -> rest_chat_csv:handle_request(Method, Path, Req); -handle_request(Method, ?CSV_MUC_ENDPOINT = Path, Req) -> rest_chat_csv:handle_request(Method, Path, Req); -======= + %% NOTE! uncomment the row under to turn on call bubbles %% handle_request(_, ?RCI_BUBBLE_ENDPOINT, Req) -> rest_cri :handle_request(Req); +handle_request(Method, ?CSV_P2P_ENDPOINT = Path, Req) -> rest_chat_csv:handle_request(Method, Path, Req); +handle_request(Method, ?CSV_MUC_ENDPOINT = Path, Req) -> rest_chat_csv:handle_request(Method, Path, Req); handle_request(Method, "/metrics" = Path, Req) -> rest_metric :handle_request(Method, Path, Req); ->>>>>>> cluster handle_request(Method, ?MSG_PUSH_ENDPOINT = Path, Req) -> rest_push:handle_request(Method, Path, Req); handle_request(Method, ?ROOM_ENDPOINT = Path, Req) -> rest_deeplink:handle_request(Method, Path, Req); handle_request(Method, ?PUBLISH_ENDPOINT = Path, Req) -> rest_publish:handle_request(Method, Path, Req); -- GitLab