From e9af9911de7487c08fdba72e3078e167cb329c13 Mon Sep 17 00:00:00 2001 From: qomputer Date: Mon, 22 Apr 2019 18:02:15 +0300 Subject: [PATCH 01/25] Update roster_db:unsubscribe --- apps/roster/src/roster_db.erl | 23 ++++++++++++++--------- 1 file changed, 14 insertions(+), 9 deletions(-) diff --git a/apps/roster/src/roster_db.erl b/apps/roster/src/roster_db.erl index d8febe3ea..2b8eb026a 100644 --- a/apps/roster/src/roster_db.erl +++ b/apps/roster/src/roster_db.erl @@ -605,17 +605,22 @@ remigrate_default_contact_settings() -> %% Subscriptions control -unsub(<<"emqttd_", _/binary>>, _)-> ok; -unsub(<<"sys_", _/binary>>, _)-> ok; -unsub(_, <<"events/+/", _/binary>>)-> ok; -unsub(ClientId, T)->n2o_vnode:unsubscribe(ClientId,T). +unsubs(<<"emqttd_", _/binary>>, _) -> []; +unsubs(<<"sys_", _/binary>>, _) -> []; +unsubs(_, <<"events/+/", _/binary>>) -> []; +unsubs(ClientId, T) -> n2o_vnode:unsubscribe(ClientId,T), {ClientId, T}. unsubscribe({all,Topic})-> - [n2o_vnode:unsubscribe(ClientId,T)|| - #mqtt_subproperty{key = {T, ClientId}} <- kvs:all(mqtt_subproperty), T==Topic]; + unsubscribe(Topic, [n2o_vnode ,unsubscribe]); -unsubscribe(Topic)-> - [unsub(ClientId,T)|| - #mqtt_subproperty{key = {T, ClientId}} <- kvs:all(mqtt_subproperty), T==Topic]. +unsubscribe({f,Fun})-> + lists:flatten([?MODULE:Fun(Cli, T) || + #mqtt_subproperty{key = {<<"ac",_/binary>> = T, Cli }} <- kvs:all(mqtt_subproperty)]); +unsubscribe(Topic) -> + unsubscribe(Topic, [?MODULE, unsubs]). + +unsubscribe(Topic, [M,Fun])-> + lists:flatten([M:Fun(Cli, T) || + #mqtt_subproperty{key = {T, Cli}} <- kvs:all(mqtt_subproperty),T==Topic]). -- GitLab From ce7540977882d751f05613df696d47a87ef4bb4b Mon Sep 17 00:00:00 2001 From: "radostin.dimitrov" Date: Mon, 22 Apr 2019 18:08:59 +0300 Subject: [PATCH 02/25] fixed updating, removing link functionality;fixed test; --- .gitignore | 9 ++ apps/roster/src/protocol/roster_link.erl | 184 +++++++++++++++-------- apps/roster/src/roster.erl | 2 +- apps/roster/src/test/room_test.erl | 144 +++++++++++------- 4 files changed, 220 insertions(+), 119 deletions(-) diff --git a/.gitignore b/.gitignore index 2726ae357..727e3341c 100644 --- a/.gitignore +++ b/.gitignore @@ -44,3 +44,12 @@ priv/protobuf/service_room/jobType.proto priv/protobuf/service_room/messageEvent.proto priv/protobuf/service_room/messageStatus.proto priv/protobuf/service_room/messageType.proto +apps/roster/priv/macbert/Model/FakeNumbers.swift +apps/roster/priv/macbert/Model/Presence.swift +apps/roster/priv/macbert/Model/cx.swift +apps/roster/priv/macbert/Model/load.swift +apps/roster/priv/macbert/Spec/FakeNumbers_Spec.swift +apps/roster/priv/macbert/Spec/Presence_Spec.swift +apps/roster/priv/macbert/Spec/cx_Spec.swift +apps/roster/priv/macbert/Spec/load_Spec.swift +apps/service/priv/ diff --git a/apps/roster/src/protocol/roster_link.erl b/apps/roster/src/protocol/roster_link.erl index 74be0db6f..87f933d54 100644 --- a/apps/roster/src/protocol/roster_link.erl +++ b/apps/roster/src/protocol/roster_link.erl @@ -4,38 +4,30 @@ -include_lib("roster/include/static/roster_var.hrl"). -include_lib("roster/include/static/main_text.hrl"). -include_lib("n2o/include/n2o.hrl"). + +-import(lists, [map/2, member/2, keydelete/3, keymember/3, keystore/4, keyreplace/4]). + -compile(export_all). +% ====================================== +% API +% ====================================== info(#'Link'{id = LinkId, type = group, status = join = LStatus} = RequestData, Req, #cx{params = ClientId} = State) when LinkId /= [] -> roster:info(?MODULE, "~p:Link/~p:~p", [ClientId, LStatus, RequestData]), -Res = case get_room_by_link_Id(LinkId) of - #ok{code = #'Room'{id = RoomId} = Room}-> - #'Member'{id = MemberId, reader = MemberReader} = - Member = case roster:muc_member(ClientId, RoomId) of - #'Member'{} = M -> M; - _ -> - PhoneID = roster:phone_id(ClientId), - Member0 = #'Member'{ - feed_id = #muc{name = RoomId}, - phone_id = PhoneID, - reader = 0, update = 0, - presence = roster:is_online(PhoneID), - status = member}, - #ok{code = Member1} = roster:to_member(PhoneID), - roster:patch_member(Member0, Member1) - end, - Messages = case roster:member_lastmsg(Member) of - {error, _} -> []; - {_, LM} -> LM - end, + Res = case get_room_by_link_id(LinkId) of + #ok{code = #'Room'{id = RoomId} = Room} -> + #'Member'{id = MemberId, reader = MemberReader} = + Member = build_membership_card(ClientId, RoomId), + Messages = member_last_messages(Member), {UnreadMsg, LastMsg, _} = roster:unread_msg({'Message', Messages}, MemberReader, MemberId), {#'Member'{}, [], #'Room'{} = Room1} = roster:add_member(Room, Member, {no_muc_message, Room#'Room'.readers}), {As, Ms} = roster:split_members(roster:members(#muc{name = Room1#'Room'.id}, presence)), - roster:reader_cache(Room1#'Room'{status = join, last_msg = LastMsg, unread = UnreadMsg, members = Ms, admins = As}); - #error{code = not_found} = Err -> Err - end, + #'Room'{} = Room2 = roster:reader_cache(Room1#'Room'{status = join, last_msg = LastMsg, unread = UnreadMsg, members = Ms, admins = As}), + ok_code(Room2); + #error{code = not_found} = Err -> Err + end, roster:info(?MODULE, "Link/~p.Response:~p", [Res, LStatus]), {reply, {bert, Res}, Req, State}; @@ -43,12 +35,13 @@ info(#'Link'{id = LinkId, name = RoomId, type = group, status = update = LStatus Req, #cx{params = ClientId} = State) when RoomId /= [], LinkId /= [] -> roster:info(?MODULE, "~p:Link/~p:~p", [ClientId, LStatus, RequestData]), Res = case roster:muc_member(ClientId, RoomId) of - #'Member'{status = admin} = M-> - update_link(RoomId, RequestData); - #'Member'{status = member} -> - error_code(permission_denied); - _-> - error_code(invalide_data) + #'Member'{status = Status} when + Status == owner; Status == admin -> + update_link(RoomId, RequestData); + #'Member'{status = member} -> + error_code(permission_denied); + _ -> + error_code(invalid_data) end, roster:info(?MODULE, "Link/~p.Response:~p", [Res, LStatus]), {reply, {bert, Res}, Req, State}; @@ -56,71 +49,130 @@ info(#'Link'{id = LinkId, name = RoomId, type = group, status = update = LStatus info(#'Link'{id = LinkId, type = group, status = get = LStatus} = RequestData, Req, #cx{params = ClientId} = State) when LinkId /= [] -> roster:info(?MODULE, "~p:Link/~p:~p", [ClientId, LStatus, RequestData]), - Res = case get_room_by_link_Id(LinkId) of - #ok{code = #'Room'{id = RoomId} = Room } -> #ok{code = Room#'Room'{status = info}}; - #error{code = not_found} = Err -> Err - end, + Res = case get_room_by_link_id(LinkId) of + #ok{code = #'Room'{} = Room} -> + #ok{code = Room#'Room'{status = info}}; + #error{code = not_found} = Err -> Err + end, roster:info(?MODULE, "Link/~p.Response:~p", [Res, LStatus]), {reply, {bert, Res}, Req, State}; -info(#'Link'{status = delete = LStatus} = RequestData, Req, #cx{params = ClientId} = State) -> +info(#'Link'{type = group, status = delete = LStatus} = RequestData, + Req, #cx{params = ClientId} = State) -> roster:info(?MODULE, "~p:Link/~p:~p", [ClientId, LStatus, RequestData]), + % Res = delete_link(ClientId, RequestData), Res = error_code(permission_denied), roster:info(?MODULE, "Link/~p.Response:~p", [Res, LStatus]), {reply, {bert, Res}, Req, State}; info(#'Link'{} = RequestData, Req, #cx{params = ClientId} = State) -> roster:info(?MODULE, "~p:Link/unknown:~p", [ClientId, RequestData]), - {reply, {bert, io_error_code(invalid_data)}, Req, State}. + {reply, {bert, io_error(invalid_data)}, Req, State}. +% ====================================== +% Helper functions +% ====================================== -gen_link(RoomId)-> +gen_link(RoomId) -> TmpLinkId = hash(RoomId), LinkId = case kvs:get('Link', TmpLinkId) of - #ok{code = #'Link'{id = TmpLinkId}}-> gen_link(RoomId); - #error{code = not_found} -> TmpLinkId - end, - -#'Link'{ + #ok{code = #'Link'{id = TmpLinkId}} -> gen_link(RoomId); + #error{code = not_found} -> TmpLinkId + end, + + #'Link'{ % id = <>, id = LinkId, name = RoomId, - type = group + type = group, + status = get }. -update_link(RoomId, #'Link'{id = LinkId} = Link) -> - Res = case kvs:get('Room', RoomId) of +update_link(#'Link'{name = RoomId} = Link) -> + update_link(RoomId, Link). +update_link(RoomId, #'Link'{id = LinkId}) -> + case kvs:get('Room', RoomId) of #ok{code = #'Room'{links = Links0} = Room} -> - NewLink = gen_link(RoomId), - % Links1 = lists:keyreplace(LinkId, #'Link'.id, Links0, NewLink), - ok = kvs:put(Room#'Room'{links = [NewLink]}), - ok = kvs:delete('Link', LinkId), - ok = kvs:put(NewLink), - NewLink; - #error{code = not_found} = Err -> Err - end, - Res. + % remove case statement for multiple links + case keymember(LinkId, #'Link'.id, Links0) of + true -> + NewLink = gen_link(RoomId), + Links1 = keystore(LinkId, #'Link'.id, Links0, NewLink), + ok = kvs:put(Room#'Room'{links = Links1}), + ok = kvs:delete('Link', LinkId), + ok = kvs:put(NewLink), + ok_code(NewLink); + false -> #error{code = invalid_data} + end; + #error{code = not_found} = Err -> Err + end. + +delete_link(ClientId, #'Link'{id = LinkId, name = RoomId}) -> + map( + fun(#'Member'{status = member}) -> + error_code(permission_denied); + (#'Member'{status = Status}) when + Status == owner; Status == admin -> + case kvs:get('Room', RoomId) of + #ok{code = #'Room'{id = RoomId, links = Links} = Room} -> + ok = kvs:put(Room#'Room'{links = keydelete(LinkId, #'Link'.id, Links)}), + ok = kvs:delete('Link', LinkId); + #error{code = not_found} = Err -> Err + end; + (_) -> error_code(invalid_data) + end, roster:muc_member(ClientId, RoomId)). -purge_link(RoomId)-> - [kvs:delete('Link', Lid) || #'Link'{id = Lid,name = RoomId}<- kvs:all('Link')], +purge_room_links(RoomId) -> + %% TODO: Improve error handling + case kvs:get('Room', RoomId) of + #ok{code = #'Room'{links = Links}} when + is_list(Links), Links /= [] -> + [kvs:delete('Link', Lid) || #'Link'{id = Lid, name = RoomId} <- Links]; + _ -> [] + end, ok. hash(Data) -> {_, <>} = roster:gen_token([], Data), Hash. -get_room_by_link_Id(<>)-> - get_room_by_link_Id(LinkId); -get_room_by_link_Id(Id) -> - case kvs:get('Link',Id) of - #ok{code = #'Link'{name = RoomId}}-> - case kvs:get('Room',RoomId) of + +get_room_by_link_id(<>) -> + get_room_by_link_id(LinkId); +get_room_by_link_id(Id) -> + case kvs:get('Link', Id) of + #ok{code = #'Link'{name = RoomId}} -> + case kvs:get('Room', RoomId) of #ok{code = #'Room'{}} = Ok -> Ok; #error{code = not_found} = Err -> Err end; #error{code = not_found} = Err -> Err - end. -io_error_code(Payload) -> io_code(error_code(Payload)). -io_ok_code(Payload) -> io_code(ok_code(Payload)). + end. + +build_membership_card(ClientId, RoomId) -> + case roster:muc_member(ClientId, RoomId) of + #'Member'{} = M -> M; + _ -> + PhoneID = roster:phone_id(ClientId), + Member0 = #'Member'{ + feed_id = #muc{name = RoomId}, + phone_id = PhoneID, + reader = 0, update = 0, + presence = roster:is_online(PhoneID), + status = member}, + #ok{code = Member1} = roster:to_member(PhoneID), + roster:patch_member(Member0, Member1) + end. + +member_last_messages(Member) -> + case roster:member_lastmsg(Member) of + {error, _} -> []; + {_, LM} -> LM + end. + +ok_code(Payload) -> #ok{code = Payload}. +error_code(Payload) -> #error{code = Payload}. + +io_ok(Payload) -> io_code(ok_code(Payload)). +io_error(Payload) -> io_code(error_code(Payload)). + io_code(Payload) -> #io{code = Payload}. -error_code(Payload) -> #error{code = Payload}. -ok_code(Payload) -> #ok{code =Payload}. diff --git a/apps/roster/src/roster.erl b/apps/roster/src/roster.erl index 8efcefd88..46f60f057 100644 --- a/apps/roster/src/roster.erl +++ b/apps/roster/src/roster.erl @@ -779,7 +779,7 @@ validate(Payload, ClientId) -> %% MUC API purge_room(false) -> []; purge_room(Room) -> - roster_link:purge_link(Room), + roster_link:purge_room_links(Room), [begin kvs:remove('Member', Id), kvs:delete(reader, R), unsubscribe_room(M), remove_rooms(PhoneId, [Room]) end || #'Member'{id = Id, phone_id = PhoneId, reader = R} = M <- members(#muc{name = Room})], kvs:delete(writer, {muc, Room}), delete_msg(#muc{name = Room}), diff --git a/apps/roster/src/test/room_test.erl b/apps/roster/src/test/room_test.erl index 0b5a3d8db..96f1a2de5 100644 --- a/apps/roster/src/test/room_test.erl +++ b/apps/roster/src/test/room_test.erl @@ -125,15 +125,12 @@ link() -> roster_client:send_receive(AClientId, Counter + 1, #'Room'{status = add, id = RoomID, admins = [EM]}), true = is_integer(U), %% Get the room link by RoomId - #ok{code = #'Room'{links = [#'Link'{id = LinkId}]}} = kvs:get('Room', Room), + #ok{code = #'Room'{links = [#'Link'{id = LinkId}]}} = kvs:get('Room', Room), %% Assert linkId by Link get request - #ok{code = #'Room'{links = [#'Link'{id = LinkId}], readers = [_ | _]}} = -roster_client:send_receive(AClientId, 1, #'Link'{type = group, status = get, id = LinkId}), + #ok{code = #'Room'{links = [#'Link'{id = LinkId}], readers = [_ | _]}} = + roster_client:send_receive(AClientId, 1, #'Link'{id = LinkId, type = group, status = get}), %% Update the link from the administrator and compare it with the old one. - % #'Room'{links = LinkId} =/= - X = roster_client:send_receive(AClientId, 1, #'Link'{id = LinkId, name = Room, type = group, status = update}), -io:format("~nXXXXXXXXXXXXXXXXXXXXX~p~n",[X]), - + #ok{code = #'Link'{id = LinkId}} /= roster_client:send_receive(AClientId, 1, #'Link'{id = LinkId, name = Room, type = group, status = update}), %%add member with message size roster:purge_user(FPhone = <<"6789000">>), {FClientId, FToken} = roster_client:reg_fake_user(FPhone), @@ -171,59 +168,102 @@ reg_by_phone(Phones, #'muc'{} = Feed) -> test_joinlink() -> +%%% Initializing test state. RoomName = RoomId = <<"test_room_joinlink">>, roster:purge_room(RoomId), Feed = #muc{name = RoomId}, - GroupAMembers = [{_APhone = <<"9000000001">>, admin}, - {_BPhone = <<"9000000002">>, member}], - GroupACounter = length(GroupAMembers), - [{APhoneId, AClientId, _, _}, {BPhoneId, BClientId, _, _}] = - GroupAPCs = reg_by_phone(GroupAMembers, Feed), - GroupAClientIds = [ClientId || {_, ClientId, _, _} <- GroupAPCs], - roster_client:set_filer(GroupAClientIds, filter), - {[Owner | _], Members} = roster:split_members([M || {_, _, M, _} <- GroupAPCs]), - RoomInit = #'Room'{status = create, type = group, id = RoomId, name = RoomName, - admins = [Owner#'Member'{alias = <<"Radostin">>}], members = [hd(Members)]}, - #'Room'{id = RoomId, links = [#'Link'{id = LinkId}], - admins = [_RoomOwner = #'Member'{phone_id = APhoneId}], - members = [#'Member'{phone_id = BPhoneId}]} = - roster_client:send_receive(AClientId, GroupACounter, RoomInit), - -%% Test: get room info with fake room link - roster_link:io_error_code(room_not_found) == roster_client:send_receive(BClientId, 1, #'Link'{id = ?BLANK_LINK, type = group, status = get}), -%% Test: get room info with correct room link - kvs:get('Room', RoomId) == #ok{code = roster_client:send_receive(BClientId, 1, #'Link'{id = LinkId, type = group, status = get})}, - GroupBMembers = [{<<"9000000003">>, member}, {<<"9000000004">>, member}], -% GroupBCounter = length(GroupBMembers), - [{CPhoneId, CClientId, _, _}, {_, DClientId, _, _}] = - _GroupBPCs = reg_by_phone(GroupBMembers, Feed), - -%% Test: join room - #'Room'{members = Room2Members} = roster_client:send_receive(CClientId, 1, #'Link'{id = LinkId, type = group, status = join}), - true == lists:keymember(CPhoneId, #'Member'.phone_id, Room2Members), -%% Test: update old link and try to join with new one - % #'Link'{name = LinkId1} = -roster_client:send_receive(AClientId, 1, #'Link'{id = LinkId, name = RoomId, type = group, status = update}). -% , -% roster_client:send_receive(DClientId, 1, #'Link'{id = LinkId1, type = group, status = join}), -% %%% Test: Delete joinlink -% %% Non admin user -% roster_link:io_error_code(invalid_data) == roster_client:send_receive(DClientId, 1, #'Link'{id = LinkId, name = RoomId, type = group, status = delete}), -% %% Admin user, unavailable link (old) -% roster_link:io_error_code(invalid_data) == -% roster_client:send_receive(AClientId, 1, #'Link'{id = LinkId, name = RoomId, type = group, status = delete}), -% %% Test join: unexisting link -% [{_, EClientId, _, _}] = reg_by_phone([{<<"9000000005">>, member}], #muc{}), -% roster_link:io_error_code(room_not_found) == roster_client:send_receive(EClientId, 1, #'Link'{id = LinkId1, type = group, status = join}), -% %% Disconnect users -% [roster_client:stop_client(CId) || CId <- GroupAClientIds ++ [CClientId, DClientId, EClientId]], -% ok. + Phones = [ + {_APhone = <<"9000000001">>, admin}, + {_BPhone = <<"9000000002">>, admin}, + {_CPhone = <<"9000000003">>, member}, + {_DPhone = <<"9000000004">>, member}, + {_EPhone = <<"9000000005">>, member} + ], + + PCMT = [ + {_APhoneId, AClientId, _AMember, _AToken}, + {_BPhoneId, BClientId, _BMember, _BToken}, + {CPhoneId, _CClientId, _CMember, _CToken}, + {DPhoneId, DClientId, _DMember, _DToken}, + {EPhoneId, EClientId, _EMember, _EToken} + ] = reg_by_phone(Phones, Feed), + + { + _PhoneIds, + ClientIds, + MembersList, + _Tokens + } = lists:foldr(fun({P, C, M, T}, {Ps, Cs, Ms, Ts}) -> + {[P | Ps], [C | Cs], [M | Ms], [T | Ts]} end, {[], [], [], []}, PCMT), + roster_client:set_filer(ClientIds, filter), + + {[Owner, Admin], RoomMembers} = roster:split_members(MembersList), + +%%% Initializing Room state. + RoomInit = #'Room'{id = RoomId, name = RoomName, + members = [hd(RoomMembers)], + admins = [ + Owner#'Member'{alias = <<"Radostin">>, status = admin}, + Admin#'Member'{alias = <<"Admin">>, status = admin} + ], + type = group, + status = create}, + +%%% Create and join first three clients to the 'Room'. + #'Room'{id = RoomId, links = [#'Link'{id = LinkId} = Link], + admins = RoomAdmins0, + members = [#'Member'{phone_id = CPhoneId}]} = + roster_client:send_receive(AClientId, 3, RoomInit), + lists:all(fun(true)-> true end, [lists:keymember(CId, #'Member'.phone_id, RoomAdmins0) + || CId <- [Owner#'Member'.phone_id, Admin#'Member'.phone_id]]), + +%%% Start tests: +%% Test: try to get room info; fake room link; admin + {error, not_found} = roster_client:send_receive(BClientId, 1, Link#'Link'{id = ?BLANK_LINK, status = get}), + +%% Test: try to get room info; correct room link; admin + {ok, #'Room'{id = RoomId, links = [#'Link'{id = LinkId}], status = info}} = + roster_client:send_receive(BClientId, 1, Link#'Link'{status = get}), + +%% Test: try to get room info; correct room link; not room member + {ok, #'Room'{id = RoomId, links = [#'Link'{id = LinkId}], status = info}} = + roster_client:send_receive(DClientId, 1, Link#'Link'{status = get}), + +%% Test: join room + {ok, #'Room'{id = RoomId, status = join} = Room1} = + roster_client:send_receive(DClientId, 1, Link#'Link'{status = join}), +% Check if new member is added + true = lists:keymember(DPhoneId, #'Member'.phone_id, Room1#'Room'.members), +%% Test: try to update room link; room member, non admin + {error, permission_denied} = roster_client:send_receive(DClientId, 1, Link#'Link'{status = update}), +%% Test: try to update room link; non room member, non admin, fake link + {error, invalid_data} = roster_client:send_receive(EClientId, 1, Link#'Link'{id = ?BLANK_LINK, status = update}), +%% Test: try to update room link; non room member + {error, invalid_data} = roster_client:send_receive(EClientId, 1, Link#'Link'{status = update}), +%% Test: try to update room link; room member, admin + {ok, #'Link'{name = RoomId} = Link1} = roster_client:send_receive(BClientId, 1, Link#'Link'{status = update}), +%% Test: try to get room info; correct room link; + {ok, #'Room'{id = RoomId, links = [Link1], status = info}} = + roster_client:send_receive(EClientId, 1, Link1#'Link'{status = get}), +%% Test: try to update room link; non-existent link, room member, admin + {error, invalid_data} = roster_client:send_receive(BClientId, 1, Link#'Link'{status = update}), +%% Test: join room; non-existent link, non room member + {error, not_found} = roster_client:send_receive(EClientId, 1, Link#'Link'{status = join}), +%% Test: join room; non room member + {ok, #'Room'{id = RoomId, links = [Link1], status = join} = Room2} = + roster_client:send_receive(EClientId, 1, Link1#'Link'{status = join}), +% Check if new member is added + true = lists:keymember(EPhoneId, #'Member'.phone_id, Room2#'Room'.members), + %%% Disconnect Clients + [roster_client:stop_client(CId) || CId <- ClientIds], + ok. suite() -> [ check_alias_increment(), - add_many_members() + add_many_members(), + test_joinlink() ]. check() -> roster:check(?MODULE). \ No newline at end of file -- GitLab From 614c98ce2d3dd06a54d3af5252d27b5fb34e629c Mon Sep 17 00:00:00 2001 From: Namdak Tonpa Date: Mon, 22 Apr 2019 20:15:02 +0300 Subject: [PATCH 03/25] Revert "fixed updating, removing link functionality;fixed test;" This reverts commit ce7540977882d751f05613df696d47a87ef4bb4b. --- .gitignore | 9 -- apps/roster/src/protocol/roster_link.erl | 184 ++++++++--------------- apps/roster/src/roster.erl | 2 +- apps/roster/src/test/room_test.erl | 144 +++++++----------- 4 files changed, 119 insertions(+), 220 deletions(-) diff --git a/.gitignore b/.gitignore index 727e3341c..2726ae357 100644 --- a/.gitignore +++ b/.gitignore @@ -44,12 +44,3 @@ priv/protobuf/service_room/jobType.proto priv/protobuf/service_room/messageEvent.proto priv/protobuf/service_room/messageStatus.proto priv/protobuf/service_room/messageType.proto -apps/roster/priv/macbert/Model/FakeNumbers.swift -apps/roster/priv/macbert/Model/Presence.swift -apps/roster/priv/macbert/Model/cx.swift -apps/roster/priv/macbert/Model/load.swift -apps/roster/priv/macbert/Spec/FakeNumbers_Spec.swift -apps/roster/priv/macbert/Spec/Presence_Spec.swift -apps/roster/priv/macbert/Spec/cx_Spec.swift -apps/roster/priv/macbert/Spec/load_Spec.swift -apps/service/priv/ diff --git a/apps/roster/src/protocol/roster_link.erl b/apps/roster/src/protocol/roster_link.erl index 87f933d54..74be0db6f 100644 --- a/apps/roster/src/protocol/roster_link.erl +++ b/apps/roster/src/protocol/roster_link.erl @@ -4,30 +4,38 @@ -include_lib("roster/include/static/roster_var.hrl"). -include_lib("roster/include/static/main_text.hrl"). -include_lib("n2o/include/n2o.hrl"). - --import(lists, [map/2, member/2, keydelete/3, keymember/3, keystore/4, keyreplace/4]). - -compile(export_all). -% ====================================== -% API -% ====================================== info(#'Link'{id = LinkId, type = group, status = join = LStatus} = RequestData, Req, #cx{params = ClientId} = State) when LinkId /= [] -> roster:info(?MODULE, "~p:Link/~p:~p", [ClientId, LStatus, RequestData]), - Res = case get_room_by_link_id(LinkId) of - #ok{code = #'Room'{id = RoomId} = Room} -> - #'Member'{id = MemberId, reader = MemberReader} = - Member = build_membership_card(ClientId, RoomId), - Messages = member_last_messages(Member), +Res = case get_room_by_link_Id(LinkId) of + #ok{code = #'Room'{id = RoomId} = Room}-> + #'Member'{id = MemberId, reader = MemberReader} = + Member = case roster:muc_member(ClientId, RoomId) of + #'Member'{} = M -> M; + _ -> + PhoneID = roster:phone_id(ClientId), + Member0 = #'Member'{ + feed_id = #muc{name = RoomId}, + phone_id = PhoneID, + reader = 0, update = 0, + presence = roster:is_online(PhoneID), + status = member}, + #ok{code = Member1} = roster:to_member(PhoneID), + roster:patch_member(Member0, Member1) + end, + Messages = case roster:member_lastmsg(Member) of + {error, _} -> []; + {_, LM} -> LM + end, {UnreadMsg, LastMsg, _} = roster:unread_msg({'Message', Messages}, MemberReader, MemberId), {#'Member'{}, [], #'Room'{} = Room1} = roster:add_member(Room, Member, {no_muc_message, Room#'Room'.readers}), {As, Ms} = roster:split_members(roster:members(#muc{name = Room1#'Room'.id}, presence)), - #'Room'{} = Room2 = roster:reader_cache(Room1#'Room'{status = join, last_msg = LastMsg, unread = UnreadMsg, members = Ms, admins = As}), - ok_code(Room2); - #error{code = not_found} = Err -> Err - end, + roster:reader_cache(Room1#'Room'{status = join, last_msg = LastMsg, unread = UnreadMsg, members = Ms, admins = As}); + #error{code = not_found} = Err -> Err + end, roster:info(?MODULE, "Link/~p.Response:~p", [Res, LStatus]), {reply, {bert, Res}, Req, State}; @@ -35,13 +43,12 @@ info(#'Link'{id = LinkId, name = RoomId, type = group, status = update = LStatus Req, #cx{params = ClientId} = State) when RoomId /= [], LinkId /= [] -> roster:info(?MODULE, "~p:Link/~p:~p", [ClientId, LStatus, RequestData]), Res = case roster:muc_member(ClientId, RoomId) of - #'Member'{status = Status} when - Status == owner; Status == admin -> - update_link(RoomId, RequestData); - #'Member'{status = member} -> - error_code(permission_denied); - _ -> - error_code(invalid_data) + #'Member'{status = admin} = M-> + update_link(RoomId, RequestData); + #'Member'{status = member} -> + error_code(permission_denied); + _-> + error_code(invalide_data) end, roster:info(?MODULE, "Link/~p.Response:~p", [Res, LStatus]), {reply, {bert, Res}, Req, State}; @@ -49,130 +56,71 @@ info(#'Link'{id = LinkId, name = RoomId, type = group, status = update = LStatus info(#'Link'{id = LinkId, type = group, status = get = LStatus} = RequestData, Req, #cx{params = ClientId} = State) when LinkId /= [] -> roster:info(?MODULE, "~p:Link/~p:~p", [ClientId, LStatus, RequestData]), - Res = case get_room_by_link_id(LinkId) of - #ok{code = #'Room'{} = Room} -> - #ok{code = Room#'Room'{status = info}}; - #error{code = not_found} = Err -> Err - end, + Res = case get_room_by_link_Id(LinkId) of + #ok{code = #'Room'{id = RoomId} = Room } -> #ok{code = Room#'Room'{status = info}}; + #error{code = not_found} = Err -> Err + end, roster:info(?MODULE, "Link/~p.Response:~p", [Res, LStatus]), {reply, {bert, Res}, Req, State}; -info(#'Link'{type = group, status = delete = LStatus} = RequestData, - Req, #cx{params = ClientId} = State) -> +info(#'Link'{status = delete = LStatus} = RequestData, Req, #cx{params = ClientId} = State) -> roster:info(?MODULE, "~p:Link/~p:~p", [ClientId, LStatus, RequestData]), - % Res = delete_link(ClientId, RequestData), Res = error_code(permission_denied), roster:info(?MODULE, "Link/~p.Response:~p", [Res, LStatus]), {reply, {bert, Res}, Req, State}; info(#'Link'{} = RequestData, Req, #cx{params = ClientId} = State) -> roster:info(?MODULE, "~p:Link/unknown:~p", [ClientId, RequestData]), - {reply, {bert, io_error(invalid_data)}, Req, State}. + {reply, {bert, io_error_code(invalid_data)}, Req, State}. -% ====================================== -% Helper functions -% ====================================== -gen_link(RoomId) -> +gen_link(RoomId)-> TmpLinkId = hash(RoomId), LinkId = case kvs:get('Link', TmpLinkId) of - #ok{code = #'Link'{id = TmpLinkId}} -> gen_link(RoomId); - #error{code = not_found} -> TmpLinkId - end, - - #'Link'{ + #ok{code = #'Link'{id = TmpLinkId}}-> gen_link(RoomId); + #error{code = not_found} -> TmpLinkId + end, + +#'Link'{ % id = <>, id = LinkId, name = RoomId, - type = group, - status = get + type = group }. -update_link(#'Link'{name = RoomId} = Link) -> - update_link(RoomId, Link). -update_link(RoomId, #'Link'{id = LinkId}) -> - case kvs:get('Room', RoomId) of +update_link(RoomId, #'Link'{id = LinkId} = Link) -> + Res = case kvs:get('Room', RoomId) of #ok{code = #'Room'{links = Links0} = Room} -> - % remove case statement for multiple links - case keymember(LinkId, #'Link'.id, Links0) of - true -> - NewLink = gen_link(RoomId), - Links1 = keystore(LinkId, #'Link'.id, Links0, NewLink), - ok = kvs:put(Room#'Room'{links = Links1}), - ok = kvs:delete('Link', LinkId), - ok = kvs:put(NewLink), - ok_code(NewLink); - false -> #error{code = invalid_data} - end; - #error{code = not_found} = Err -> Err - end. - -delete_link(ClientId, #'Link'{id = LinkId, name = RoomId}) -> - map( - fun(#'Member'{status = member}) -> - error_code(permission_denied); - (#'Member'{status = Status}) when - Status == owner; Status == admin -> - case kvs:get('Room', RoomId) of - #ok{code = #'Room'{id = RoomId, links = Links} = Room} -> - ok = kvs:put(Room#'Room'{links = keydelete(LinkId, #'Link'.id, Links)}), - ok = kvs:delete('Link', LinkId); - #error{code = not_found} = Err -> Err - end; - (_) -> error_code(invalid_data) - end, roster:muc_member(ClientId, RoomId)). - -purge_room_links(RoomId) -> - %% TODO: Improve error handling - case kvs:get('Room', RoomId) of - #ok{code = #'Room'{links = Links}} when - is_list(Links), Links /= [] -> - [kvs:delete('Link', Lid) || #'Link'{id = Lid, name = RoomId} <- Links]; - _ -> [] + NewLink = gen_link(RoomId), + % Links1 = lists:keyreplace(LinkId, #'Link'.id, Links0, NewLink), + ok = kvs:put(Room#'Room'{links = [NewLink]}), + ok = kvs:delete('Link', LinkId), + ok = kvs:put(NewLink), + NewLink; + #error{code = not_found} = Err -> Err end, + Res. + +purge_link(RoomId)-> + [kvs:delete('Link', Lid) || #'Link'{id = Lid,name = RoomId}<- kvs:all('Link')], ok. hash(Data) -> {_, <>} = roster:gen_token([], Data), Hash. - -get_room_by_link_id(<>) -> - get_room_by_link_id(LinkId); -get_room_by_link_id(Id) -> - case kvs:get('Link', Id) of - #ok{code = #'Link'{name = RoomId}} -> - case kvs:get('Room', RoomId) of +get_room_by_link_Id(<>)-> + get_room_by_link_Id(LinkId); +get_room_by_link_Id(Id) -> + case kvs:get('Link',Id) of + #ok{code = #'Link'{name = RoomId}}-> + case kvs:get('Room',RoomId) of #ok{code = #'Room'{}} = Ok -> Ok; #error{code = not_found} = Err -> Err end; #error{code = not_found} = Err -> Err - end. - -build_membership_card(ClientId, RoomId) -> - case roster:muc_member(ClientId, RoomId) of - #'Member'{} = M -> M; - _ -> - PhoneID = roster:phone_id(ClientId), - Member0 = #'Member'{ - feed_id = #muc{name = RoomId}, - phone_id = PhoneID, - reader = 0, update = 0, - presence = roster:is_online(PhoneID), - status = member}, - #ok{code = Member1} = roster:to_member(PhoneID), - roster:patch_member(Member0, Member1) - end. - -member_last_messages(Member) -> - case roster:member_lastmsg(Member) of - {error, _} -> []; - {_, LM} -> LM - end. - -ok_code(Payload) -> #ok{code = Payload}. -error_code(Payload) -> #error{code = Payload}. - -io_ok(Payload) -> io_code(ok_code(Payload)). -io_error(Payload) -> io_code(error_code(Payload)). - + end. +io_error_code(Payload) -> io_code(error_code(Payload)). +io_ok_code(Payload) -> io_code(ok_code(Payload)). io_code(Payload) -> #io{code = Payload}. +error_code(Payload) -> #error{code = Payload}. +ok_code(Payload) -> #ok{code =Payload}. diff --git a/apps/roster/src/roster.erl b/apps/roster/src/roster.erl index 46f60f057..8efcefd88 100644 --- a/apps/roster/src/roster.erl +++ b/apps/roster/src/roster.erl @@ -779,7 +779,7 @@ validate(Payload, ClientId) -> %% MUC API purge_room(false) -> []; purge_room(Room) -> - roster_link:purge_room_links(Room), + roster_link:purge_link(Room), [begin kvs:remove('Member', Id), kvs:delete(reader, R), unsubscribe_room(M), remove_rooms(PhoneId, [Room]) end || #'Member'{id = Id, phone_id = PhoneId, reader = R} = M <- members(#muc{name = Room})], kvs:delete(writer, {muc, Room}), delete_msg(#muc{name = Room}), diff --git a/apps/roster/src/test/room_test.erl b/apps/roster/src/test/room_test.erl index 96f1a2de5..0b5a3d8db 100644 --- a/apps/roster/src/test/room_test.erl +++ b/apps/roster/src/test/room_test.erl @@ -125,12 +125,15 @@ link() -> roster_client:send_receive(AClientId, Counter + 1, #'Room'{status = add, id = RoomID, admins = [EM]}), true = is_integer(U), %% Get the room link by RoomId - #ok{code = #'Room'{links = [#'Link'{id = LinkId}]}} = kvs:get('Room', Room), + #ok{code = #'Room'{links = [#'Link'{id = LinkId}]}} = kvs:get('Room', Room), %% Assert linkId by Link get request - #ok{code = #'Room'{links = [#'Link'{id = LinkId}], readers = [_ | _]}} = - roster_client:send_receive(AClientId, 1, #'Link'{id = LinkId, type = group, status = get}), + #ok{code = #'Room'{links = [#'Link'{id = LinkId}], readers = [_ | _]}} = +roster_client:send_receive(AClientId, 1, #'Link'{type = group, status = get, id = LinkId}), %% Update the link from the administrator and compare it with the old one. - #ok{code = #'Link'{id = LinkId}} /= roster_client:send_receive(AClientId, 1, #'Link'{id = LinkId, name = Room, type = group, status = update}), + % #'Room'{links = LinkId} =/= + X = roster_client:send_receive(AClientId, 1, #'Link'{id = LinkId, name = Room, type = group, status = update}), +io:format("~nXXXXXXXXXXXXXXXXXXXXX~p~n",[X]), + %%add member with message size roster:purge_user(FPhone = <<"6789000">>), {FClientId, FToken} = roster_client:reg_fake_user(FPhone), @@ -168,102 +171,59 @@ reg_by_phone(Phones, #'muc'{} = Feed) -> test_joinlink() -> -%%% Initializing test state. RoomName = RoomId = <<"test_room_joinlink">>, roster:purge_room(RoomId), Feed = #muc{name = RoomId}, - Phones = [ - {_APhone = <<"9000000001">>, admin}, - {_BPhone = <<"9000000002">>, admin}, - {_CPhone = <<"9000000003">>, member}, - {_DPhone = <<"9000000004">>, member}, - {_EPhone = <<"9000000005">>, member} - ], - - PCMT = [ - {_APhoneId, AClientId, _AMember, _AToken}, - {_BPhoneId, BClientId, _BMember, _BToken}, - {CPhoneId, _CClientId, _CMember, _CToken}, - {DPhoneId, DClientId, _DMember, _DToken}, - {EPhoneId, EClientId, _EMember, _EToken} - ] = reg_by_phone(Phones, Feed), - - { - _PhoneIds, - ClientIds, - MembersList, - _Tokens - } = lists:foldr(fun({P, C, M, T}, {Ps, Cs, Ms, Ts}) -> - {[P | Ps], [C | Cs], [M | Ms], [T | Ts]} end, {[], [], [], []}, PCMT), - roster_client:set_filer(ClientIds, filter), - - {[Owner, Admin], RoomMembers} = roster:split_members(MembersList), - -%%% Initializing Room state. - RoomInit = #'Room'{id = RoomId, name = RoomName, - members = [hd(RoomMembers)], - admins = [ - Owner#'Member'{alias = <<"Radostin">>, status = admin}, - Admin#'Member'{alias = <<"Admin">>, status = admin} - ], - type = group, - status = create}, - -%%% Create and join first three clients to the 'Room'. - #'Room'{id = RoomId, links = [#'Link'{id = LinkId} = Link], - admins = RoomAdmins0, - members = [#'Member'{phone_id = CPhoneId}]} = - roster_client:send_receive(AClientId, 3, RoomInit), - lists:all(fun(true)-> true end, [lists:keymember(CId, #'Member'.phone_id, RoomAdmins0) - || CId <- [Owner#'Member'.phone_id, Admin#'Member'.phone_id]]), - -%%% Start tests: -%% Test: try to get room info; fake room link; admin - {error, not_found} = roster_client:send_receive(BClientId, 1, Link#'Link'{id = ?BLANK_LINK, status = get}), - -%% Test: try to get room info; correct room link; admin - {ok, #'Room'{id = RoomId, links = [#'Link'{id = LinkId}], status = info}} = - roster_client:send_receive(BClientId, 1, Link#'Link'{status = get}), - -%% Test: try to get room info; correct room link; not room member - {ok, #'Room'{id = RoomId, links = [#'Link'{id = LinkId}], status = info}} = - roster_client:send_receive(DClientId, 1, Link#'Link'{status = get}), - -%% Test: join room - {ok, #'Room'{id = RoomId, status = join} = Room1} = - roster_client:send_receive(DClientId, 1, Link#'Link'{status = join}), -% Check if new member is added - true = lists:keymember(DPhoneId, #'Member'.phone_id, Room1#'Room'.members), -%% Test: try to update room link; room member, non admin - {error, permission_denied} = roster_client:send_receive(DClientId, 1, Link#'Link'{status = update}), -%% Test: try to update room link; non room member, non admin, fake link - {error, invalid_data} = roster_client:send_receive(EClientId, 1, Link#'Link'{id = ?BLANK_LINK, status = update}), -%% Test: try to update room link; non room member - {error, invalid_data} = roster_client:send_receive(EClientId, 1, Link#'Link'{status = update}), -%% Test: try to update room link; room member, admin - {ok, #'Link'{name = RoomId} = Link1} = roster_client:send_receive(BClientId, 1, Link#'Link'{status = update}), -%% Test: try to get room info; correct room link; - {ok, #'Room'{id = RoomId, links = [Link1], status = info}} = - roster_client:send_receive(EClientId, 1, Link1#'Link'{status = get}), -%% Test: try to update room link; non-existent link, room member, admin - {error, invalid_data} = roster_client:send_receive(BClientId, 1, Link#'Link'{status = update}), -%% Test: join room; non-existent link, non room member - {error, not_found} = roster_client:send_receive(EClientId, 1, Link#'Link'{status = join}), -%% Test: join room; non room member - {ok, #'Room'{id = RoomId, links = [Link1], status = join} = Room2} = - roster_client:send_receive(EClientId, 1, Link1#'Link'{status = join}), -% Check if new member is added - true = lists:keymember(EPhoneId, #'Member'.phone_id, Room2#'Room'.members), - %%% Disconnect Clients - [roster_client:stop_client(CId) || CId <- ClientIds], - ok. + GroupAMembers = [{_APhone = <<"9000000001">>, admin}, + {_BPhone = <<"9000000002">>, member}], + GroupACounter = length(GroupAMembers), + [{APhoneId, AClientId, _, _}, {BPhoneId, BClientId, _, _}] = + GroupAPCs = reg_by_phone(GroupAMembers, Feed), + GroupAClientIds = [ClientId || {_, ClientId, _, _} <- GroupAPCs], + roster_client:set_filer(GroupAClientIds, filter), + {[Owner | _], Members} = roster:split_members([M || {_, _, M, _} <- GroupAPCs]), + RoomInit = #'Room'{status = create, type = group, id = RoomId, name = RoomName, + admins = [Owner#'Member'{alias = <<"Radostin">>}], members = [hd(Members)]}, + #'Room'{id = RoomId, links = [#'Link'{id = LinkId}], + admins = [_RoomOwner = #'Member'{phone_id = APhoneId}], + members = [#'Member'{phone_id = BPhoneId}]} = + roster_client:send_receive(AClientId, GroupACounter, RoomInit), + +%% Test: get room info with fake room link + roster_link:io_error_code(room_not_found) == roster_client:send_receive(BClientId, 1, #'Link'{id = ?BLANK_LINK, type = group, status = get}), +%% Test: get room info with correct room link + kvs:get('Room', RoomId) == #ok{code = roster_client:send_receive(BClientId, 1, #'Link'{id = LinkId, type = group, status = get})}, + GroupBMembers = [{<<"9000000003">>, member}, {<<"9000000004">>, member}], +% GroupBCounter = length(GroupBMembers), + [{CPhoneId, CClientId, _, _}, {_, DClientId, _, _}] = + _GroupBPCs = reg_by_phone(GroupBMembers, Feed), + +%% Test: join room + #'Room'{members = Room2Members} = roster_client:send_receive(CClientId, 1, #'Link'{id = LinkId, type = group, status = join}), + true == lists:keymember(CPhoneId, #'Member'.phone_id, Room2Members), +%% Test: update old link and try to join with new one + % #'Link'{name = LinkId1} = +roster_client:send_receive(AClientId, 1, #'Link'{id = LinkId, name = RoomId, type = group, status = update}). +% , +% roster_client:send_receive(DClientId, 1, #'Link'{id = LinkId1, type = group, status = join}), +% %%% Test: Delete joinlink +% %% Non admin user +% roster_link:io_error_code(invalid_data) == roster_client:send_receive(DClientId, 1, #'Link'{id = LinkId, name = RoomId, type = group, status = delete}), +% %% Admin user, unavailable link (old) +% roster_link:io_error_code(invalid_data) == +% roster_client:send_receive(AClientId, 1, #'Link'{id = LinkId, name = RoomId, type = group, status = delete}), +% %% Test join: unexisting link +% [{_, EClientId, _, _}] = reg_by_phone([{<<"9000000005">>, member}], #muc{}), +% roster_link:io_error_code(room_not_found) == roster_client:send_receive(EClientId, 1, #'Link'{id = LinkId1, type = group, status = join}), +% %% Disconnect users +% [roster_client:stop_client(CId) || CId <- GroupAClientIds ++ [CClientId, DClientId, EClientId]], +% ok. suite() -> [ check_alias_increment(), - add_many_members(), - test_joinlink() + add_many_members() ]. check() -> roster:check(?MODULE). \ No newline at end of file -- GitLab From f06285af5479f1f36c0da208cfecd6b6526362cc Mon Sep 17 00:00:00 2001 From: qomputer Date: Wed, 24 Apr 2019 18:44:57 +0300 Subject: [PATCH 04/25] Fix acl --- apps/roster/src/protocol/roster_acl.erl | 2 +- apps/roster/src/roster.erl | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/apps/roster/src/protocol/roster_acl.erl b/apps/roster/src/protocol/roster_acl.erl index 570bea6ef..b47d1398a 100644 --- a/apps/roster/src/protocol/roster_acl.erl +++ b/apps/roster/src/protocol/roster_acl.erl @@ -72,7 +72,7 @@ check_acl({#mqtt_client{client_id = <<"emqttd_", _/binary>> = Client}, publish, allow; check_acl({#mqtt_client{client_id = <<"emqttd_", _/binary>>}, publish, <<"auth/", _/binary>>}, #state{}) -> deny; -check_acl({#mqtt_client{client_id = <<"emqttd_", _/binary>> = Client}, publish, <<"events/", _/binary>>} = Who, #state{}) -> +check_acl({#mqtt_client{client_id = <<"emqttd_", _/binary>> = Client}, publish, Topic} = Who, #state{}) -> case kvs:get('Auth', Client) of {ok, #'Auth'{type = expired}} -> deny; {ok, _} -> allow; diff --git a/apps/roster/src/roster.erl b/apps/roster/src/roster.erl index 8efcefd88..8c8774ba0 100644 --- a/apps/roster/src/roster.erl +++ b/apps/roster/src/roster.erl @@ -51,6 +51,7 @@ start(_, _) -> atoms(), catch load([]), application:stop(n2o), application:start(n2o), X = try begin A = supervisor:start_link({local, roster}, roster, []), + emqttd_access_control:register_mod(auth, n2o_auth, [[]], 9997), emqttd_access_control:register_mod(auth, roster_auth, [[]], 9998), emqttd_access_control:register_mod(auth, micro_auth, [[]], 9996), register_acl_mod(), -- GitLab From f922aafafb25df98ed72b7f7007592864fedfcf3 Mon Sep 17 00:00:00 2001 From: qomputer Date: Wed, 24 Apr 2019 19:26:04 +0300 Subject: [PATCH 05/25] Revert roster_channel_helper.erl --- apps/roster/src/roster_channel_helper.erl | 278 ++++++++++++++++++++++ 1 file changed, 278 insertions(+) create mode 100644 apps/roster/src/roster_channel_helper.erl diff --git a/apps/roster/src/roster_channel_helper.erl b/apps/roster/src/roster_channel_helper.erl new file mode 100644 index 000000000..c51cf8283 --- /dev/null +++ b/apps/roster/src/roster_channel_helper.erl @@ -0,0 +1,278 @@ +-module(roster_channel_helper). +-include("roster.hrl"). +-include_lib("roster/include/static/roster_text.hrl"). +-include_lib("roster/include/static/roster_var.hrl"). +-include_lib("roster/include/static/main_text.hrl"). +-compile(export_all). + +%% --------------------------------------------------------------------------------------------------------------------- +%% Errors Management +%% --------------------------------------------------------------------------------------------------------------------- + +%% TODO replace error_response section to another helper in future +error_response(ListOfCodes, Model) when not is_list(ListOfCodes) -> + error_response([ListOfCodes], Model); +error_response(ListOfCodes, Model) -> + #errors{code = ListOfCodes, data = Model}. + +error_response_403(Model) -> +%% permissions denied error response + error_response(?ERROR_PERMISSION_DENIED, Model). + +error_response_400(Model) -> +%% invalid data error response + error_response(?ERROR_INVALID_DATA, Model). + +%% --------------------------------------------------------------------------------------------------------------------- +%% Channel Management +%% --------------------------------------------------------------------------------------------------------------------- + +is_channel(FeedId) -> + case FeedId of #muc{name = RoomId} -> case kvs:get('Room', RoomId) of {ok, #'Room'{type = channel}} -> true; _ -> false end; _ -> false end. + +%% TODO request. Make it "get_room_by_id" and move to db helpers section +get_channel_by_id(ChannelId) -> +%% Check is channel exists +%% Params => ChannelId :: binary() +%% Return => Succes -> {ok, ChannelObject}; Error -> {error, ChannelNotFoundMsg} + case kvs:get('Room', ChannelId) of + {error, _} -> {error, ?ERROR_CHANNEL_NOT_FOUND}; + {ok, Channel} -> {ok, Channel} + end. + +get_channel_by_link(Link) when is_list(Link) -> + get_channel_by_link(list_to_binary(Link)); +get_channel_by_link(Link) -> + case kvs:get('Index', {?LINK_INDEX_KEYWORD, string:lowercase(Link)}) of + {ok, #'Index'{roster = [RoomId]}} -> + case kvs:get('Room', RoomId) of + {ok, Room} -> channel_response_object(Room); + {error, _} -> [] + end; + _ -> [] + end. + +get_room_avatar(#'Room'{data = Descs}) -> + case lists:keyfind(?ROOM_AVATAR_MIME, #'Desc'.mime, Descs) of #'Desc'{payload = LinkToAvatar} -> LinkToAvatar; _ -> [] end. + +get_last_msg(MsgId) -> + case kvs:get('Message', MsgId) of + {ok, Msg} -> Msg; + {error, _} -> [] + end. + +channel_response_object(#'Room'{id = RoomId, links = Links, last_msg = LastMsgId} = Channel) -> +%% add extra-processing for links and last msg + Channel#'Room'{links = [Link#'Link'{id = RoomId} || Link <- Links], last_msg = get_last_msg(LastMsgId)}. + +%% TODO check permissions function: check is member exists / is member owner / is member admin with OK permittions to make changes + +%% Broadcast functions + +send_channel(ClientPid, ChannelId, Msg) -> + roster:send_room(ClientPid, ChannelId, Msg). + +send_channel(ClientPid, Channel) -> + roster:send_room(ClientPid, channel_response_object(Channel)). + +send_admin(ClientPid, #'Room'{id = ChannelId} = Channel) -> +%% notify all channel admins and owner + [roster:send_ses(ClientPid, MemberPhoneId, channel_response_object(Channel)) || #'Member'{phone_id = MemberPhoneId} <- roster:members(#muc{name = ChannelId}, admin)]. + +%% --------------------------------------------------------------------------------------------------------------------- +%% Member Management +%% --------------------------------------------------------------------------------------------------------------------- + +member_get_feed(MemberId) -> +%% get member feed by member id +%% return member feed + case kvs:get('Member', MemberId) of + {ok, #'Member'{feed_id = FeedId}} -> FeedId; + _ -> [] + end. + +check_user_permissions(ChannelId, ClientId) -> + check_user_permissions(ChannelId, ClientId, []). +check_user_permissions(ChannelId, ClientId, PermissionKey) -> +%% Check acted user permissions +%% Params => ChannelId :: binary(); ClientId :: binary(); PermissionKey :: binary() +%% Return => Success -> {ok, MemberObject}; Error -> {error, PermissionsDeniedMsg} +%% get Acted user member in this Channel + ResponseError = {error, ?ERROR_PERMISSION_DENIED}, + case roster:muc_member(ClientId, ChannelId) of + [] -> ResponseError; + #'Member'{status = ActedMemberStatus, settings = PermList} = ActedMember -> + ResponseSuccess = {ok, ActedMember}, +%% check Acted user member status + case ActedMemberStatus of + owner -> ResponseSuccess; + admin -> + case PermissionKey of + [] -> ResponseSuccess; + _ -> +%% check admin permission list + case lists:keyfind(PermissionKey, #'Feature'.key, PermList) of + #'Feature'{value = true} -> ResponseSuccess; + _ -> ResponseError + end + end; + _ -> ResponseError + end + end. + +add_members(#'Room'{id = ChannelId, readers = ChannelReaders} = Channel, Members) -> +%% Proceed Members to be added + [begin +%% check is this member already added + case roster:muc_member(MemberPhoneId, ChannelId) of + [] -> +%% add new member + roster:add_member(Channel, #'Member'{id = [], feed_id = #muc{name = ChannelId}, status = member, + phone_id = MemberPhoneId}); + #'Member'{status = MemberStatus} = ExistingMember -> +%% check was member removed. if true - change his status from removed to member + case MemberStatus of + removed -> + PrePatchedMember = roster:patch_member(ExistingMember#'Member'{status = member}, ExistingMember), + {PatchedMember, _, _} = roster:add_member(Channel, PrePatchedMember, {no_muc_message, ChannelReaders}), + kvs:put(PatchedMember); + _ -> skip + end + end + end || #'Member'{phone_id = MemberPhoneId} <- Members]. + +count_members(ChannelId, Status) -> +%% count channel members filtered by statuses +%% active: member, owner, admin +%% admin: member, owner + length(roster:members(#muc{name = ChannelId}, Status)). + +update_members_roster(#'Room'{id = ChannelId} = Channel) -> + [roster:update_rooms(MemberPhoneId, Channel) || #'Member'{phone_id = MemberPhoneId} <- roster:members(#muc{name = ChannelId})]. + +%% --------------------------------------------------------------------------------------------------------------------- +%% Feature Management +%% --------------------------------------------------------------------------------------------------------------------- + +update_feature(FeatureKey, FeatureValue, ExistingFeatures) -> + (case lists:keyfind(FeatureKey, #'Feature'.key, ExistingFeatures) of + false -> #'Feature'{id = iolist_to_binary([<<"srv_">>, i2b(kvs:next_id('Desc', 1)), "_", i2b(roster:now_msec())]), + key = FeatureKey, group = ?FGC_INFO}; + ExistingFeature -> ExistingFeature + end)#'Feature'{value = FeatureValue}. + +set_count_features(#'Room'{settings = ExistingFeatures, id = ChannelId} = Channel) -> +%% NOTE! Should update Channel Subscribers Count on: Room/add, Room/remove, Room/join, Room/leave +%% NOTE! Active member means that Member.status is not *removed* + SubscribersCountFeature = update_feature(?FKC_SUBSCRIBERS_COUNT, i2b(count_members(ChannelId, active)), ExistingFeatures), +%% NOTE! Admin member means that Member.status is *admin* or *owner* + AdminsCountFeature = update_feature(?FKC_ADMINS_COUNT, i2b(count_members(ChannelId, admin)), ExistingFeatures), + SettingsUpdSubscribersCountFeature = lists:keystore(?FKC_SUBSCRIBERS_COUNT, #'Feature'.key, ExistingFeatures, SubscribersCountFeature), + SettingsUpdAdminsCountFeature = lists:keystore(?FKC_ADMINS_COUNT, #'Feature'.key, SettingsUpdSubscribersCountFeature, AdminsCountFeature), + Channel#'Room'{settings = SettingsUpdAdminsCountFeature}. + +%% --------------------------------------------------------------------------------------------------------------------- +%% Link Management +%% --------------------------------------------------------------------------------------------------------------------- + +generate_link_id(ChannelId) -> + iolist_to_binary([ChannelId, <<"_">>, integer_to_list(kvs:next_id('Link', 1))]). + +generate_link() -> +%% 1. Create random string based on timestamp +%% 2. Delete not allowed symbols + list_to_binary(lists:filter(fun(Char) -> lists:member(Char, binary_to_list(?LINK_ALLOWED_CHARS)) end, + binary_to_list(base64:encode(crypto:hmac(sha256, i2b(roster:now_msec()), i2b(roster:now_msec())))))). + +validate_link_format(Link) -> + case regexp_api:validate_string(Link, lists:flatten(io_lib:format(?LINK_REGEXP, [?LINK_LEN_MIN - 1, ?LINK_LEN_MAX - 1]))) of + {ok, _} -> {ok, Link}; + {error, _} -> {error, ?LINK_INVALID_FORMAT} + end. + +check_link_availability(Link) -> + check_link_availability(Link, []). +check_link_availability(Link, ChannelId) -> + ResponseError = {error, ?LINK_NOT_AVAILABLE}, + ResponseSuccess = {ok, []}, + case kvs:get('Index', {?LINK_INDEX_KEYWORD, string:lowercase(Link)}) of + {error, _} -> ResponseSuccess; + {ok, #'Index'{roster = EntityList}} -> + case lists:member(ChannelId, EntityList) of + false -> ResponseError; + _ -> ResponseSuccess + end + end. + +add_link_index(Link, RoomId) -> + kvs:put(#'Index'{id = {?LINK_INDEX_KEYWORD, string:lowercase(Link)}, roster = [RoomId]}). + +delete_link_index(Link) -> + kvs:delete('Index', {?LINK_INDEX_KEYWORD, string:lowercase(Link)}). + +%% lists:keyfind(LinkId, #'Link'.id, StoredLinks) +get_link_by_id(LinkId, LinksList) -> +%% Get is Link with requested id exists in Stored Links List +%% Params => LinkId :: binary(), LinksList :: list of #'Link'{} objects +%% Return => Success -> {ok, #'Link'{}}; Error -> {error, LinkNotFoundError} + case lists:keyfind(LinkId, #'Link'.id, LinksList) of false -> {error, ?ERROR_LINK_NOT_FOUND}; #'Link'{} = StoredLink -> {ok, StoredLink} end. + +get_link_count(#'Room'{links = LinkList}) -> +%% Get links count in Channel +%% Params => Channel :: ErlangRecord +%% Return => Success -> LinksCountInteger + length(LinkList). + +check_link_add_availability(Channel) -> +%% Check existing links count in Channel +%% Params => Channel :: ErlangRecord +%% Return => Success -> {ok, []}; Error -> {error, MaxNumberOfLinksReachedMsg} + case get_link_count(Channel) of LinksCount when LinksCount >= ?MAX_LINKS_NUM -> {error, ?MAX_LINKS_NUM_REACHED}; _ -> {ok, []} end. + +%% --------------------------------------------------------------------------------------------------------------------- +%% Non-grouped helpers +%% --------------------------------------------------------------------------------------------------------------------- + +i2b(X) -> integer_to_binary(X). + +paginator(List, Limit, Offset) -> +%% protect from empty list slicing + case length(List) of + 0 -> []; + _ -> +%% protect from invalid Offset value + ValidatedOffset = case Offset of + O when O > length(List) -> length(List); + O when O == []; O == 0 -> 1; + _ -> Offset end, +%% protect from invalid Limit value + ValidatedLimit = case Limit of + L when L < 0 -> 1; + _ -> Limit end, + lists:sublist(List, ValidatedOffset, ValidatedLimit) + end. + +startswith(String, Head) when is_binary(String) -> + startswith(binary_to_list(String), Head); +startswith(String, Head) when is_binary(Head) -> + startswith(String, binary_to_list(Head)); +startswith(String, Head) -> + string:str(string:lowercase(String), string:lowercase(Head)) > 0. + +%% TODO - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - +%% TODO delete debug helpers after development +%% TODO - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + +delete_test_channel() -> + TestChannelId = <<"channel_1">>, +%% delete members + [begin + kvs:delete('Member', MemId) + end || #'Member'{id = MemId} <- roster:members(#muc{name = TestChannelId})], +%% delete messages + [begin + kvs:delete('Message', MsgId) + end || #'Message'{id = MsgId} <- kvs:index('Message', to, TestChannelId)], +%% delete channel + kvs:delete('Room', TestChannelId), + ok. \ No newline at end of file -- GitLab From 4e96b3bb738dc41d33c50b5ff492fe57062d9ec7 Mon Sep 17 00:00:00 2001 From: Namdak Tonpa Date: Thu, 25 Apr 2019 15:18:49 +0300 Subject: [PATCH 06/25] TOKEN protocol change AES/CBC-128 for prehistoric clients --- apps/roster/src/protocol/roster_auth.erl | 2 +- apps/roster/src/roster.erl | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/apps/roster/src/protocol/roster_auth.erl b/apps/roster/src/protocol/roster_auth.erl index 9d979e937..4d6f829cf 100644 --- a/apps/roster/src/protocol/roster_auth.erl +++ b/apps/roster/src/protocol/roster_auth.erl @@ -36,7 +36,7 @@ check(#mqtt_client{client_id = <<"reg_", _/binary>> = ClientId, check(#mqtt_client{client_id = <<"emqttd_", _/binary>> = ClientId, username = <<"api">>, client_pid = ClientPid, will_topic = WT, peername = {Ip, _}, ws_initial_headers = Headers} = MC, Token, State) -> roster:info(?MODULE, "[WS Headers]: ~p", [Headers]), - roster:info(?MODULE, "~p:Auth:auth/check:~p~p", [ClientId, lists:sublist(binary_to_list(Token), 16),WT]), + roster:info(?MODULE, "~p:Auth:auth/check:~p~n", [ClientId,WT]), case case State of #'Auth'{} -> {ok, State}; _ -> kvs:get('Auth', ClientId) end of {ok, #'Auth'{type = Type, token = Token, settings = Settings, last_online = LO} = Auth} when Type == verified; Type == []; Type == expired -> diff --git a/apps/roster/src/roster.erl b/apps/roster/src/roster.erl index 8c8774ba0..1e61740f2 100644 --- a/apps/roster/src/roster.erl +++ b/apps/roster/src/roster.erl @@ -708,7 +708,8 @@ feed_key(#p2p{from = From, to = To}) -> feed_key(p2p, From, To); feed_key(#muc{} = Feed) -> Feed. ttl() -> application:get_env(roster, auth_ttl, 60 * 15). -depicle(Token) -> case n2o_secret:depickle(Token) of <<>> -> <<>>; T -> binary_to_term(T,[safe]) end. +depicle(Token) -> case n2o_secret:depickle(Token) of <<>> -> <<>>; T -> + try binary_to_term(T,[safe]) catch _:_ -> <<>> end end. gen_token([], Data) -> {'Token', n2o_secret:pickle(term_to_binary({now_msec() + ttl() * 1000, Data}))}; gen_token(Token, Data) -> -- GitLab From 668c9f2b60c869bc7659529dd36548ea8a816142 Mon Sep 17 00:00:00 2001 From: Namdak Tonpa Date: Thu, 25 Apr 2019 16:19:27 +0300 Subject: [PATCH 07/25] fix auth bug --- apps/roster/src/protocol/roster_auth.erl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/roster/src/protocol/roster_auth.erl b/apps/roster/src/protocol/roster_auth.erl index 4d6f829cf..424d0ea14 100644 --- a/apps/roster/src/protocol/roster_auth.erl +++ b/apps/roster/src/protocol/roster_auth.erl @@ -53,7 +53,7 @@ check(#mqtt_client{client_id = <<"emqttd_", _/binary>> = ClientId, username = << {_, expired} -> AuthPid ! Auth2, ok; {{error, token_expired}, _} -> kvs:put(Auth2#'Auth'{type = expired}), AuthPid ! Auth2, ok; - {{error, _}, _} = E -> E; + {{error, _}=E, _} -> E; {_, _} -> OldIpBin = case FindIp = lists:keyfind(?IP_KEY, #'Feature'.key, Settings) of false -> IpBin; #'Feature'{value = V} -> V end, -- GitLab From 5dc9af386bba3c2246983ef6d78b04bef8465ad7 Mon Sep 17 00:00:00 2001 From: qomputer Date: Fri, 26 Apr 2019 12:57:51 +0300 Subject: [PATCH 08/25] Fix roster_test:bpe() and sms sending --- apps/roster/src/protocol/roster_auth.erl | 2 +- apps/roster/src/protocol/roster_presence.erl | 10 ++++++---- apps/roster/src/rest/rest_metric.erl | 2 +- apps/roster/src/roster.erl | 7 +++++-- apps/roster/src/test/roster_test.erl | 6 +++--- 5 files changed, 16 insertions(+), 11 deletions(-) diff --git a/apps/roster/src/protocol/roster_auth.erl b/apps/roster/src/protocol/roster_auth.erl index 424d0ea14..9a0cb05dd 100644 --- a/apps/roster/src/protocol/roster_auth.erl +++ b/apps/roster/src/protocol/roster_auth.erl @@ -318,7 +318,7 @@ proc({sms, Phone, SmsCode}, #handler{} = H) -> Half = size(SmsCode) - round(size(SmsCode) / 2), [Codes1, Codes2] = [binary:part(SmsCode, S, Half) || S <- [0, Half]], ClientSms = <<"Welcome to Nynja! Your verification code: ", Codes1/binary, " ", Codes2/binary>>, - telesign_api:send_sms(Phone, ClientSms, fun(_) -> ok end), + spawn(fun () -> telesign_api:send_sms(Phone, ClientSms, fun(_) -> ok end) end), {reply, [], H}; proc({voice, Phone, SmsCode, Lang}, #handler{} = H) -> diff --git a/apps/roster/src/protocol/roster_presence.erl b/apps/roster/src/protocol/roster_presence.erl index 1c3c60048..604de499e 100644 --- a/apps/roster/src/protocol/roster_presence.erl +++ b/apps/roster/src/protocol/roster_presence.erl @@ -22,12 +22,13 @@ on_connect(Phone, ClientId, C, Ver) -> case send_presence(online, Phone, C, ClientId) of {error, profile_not_found} -> roster:info(?MODULE, "~p:~p:Connect:ProfileNotFound", [Phone, ClientId]), roster:delete_sessions(Phone); - #'Profile'{} = P when Ver==?VERSION -> ok; + #'Profile'{} = P when Ver==?VERSION -> + ok; #'Profile'{} = P-> roster:send_profile(P#'Profile'{settings = amazon_settings(P), status = init}, [], 0, ClientId, C) end catch Err:Rea -> - n2o:error(?MODULE, "Catch:~p~n", [n2o:stack_trace(Err, Rea)]) + roster:info(?MODULE, "Catch:~p", [n2o:stack_trace(Err, Rea)]) end. on_disconnect(#'Auth'{type = logout, phone = Phone, client_id = ClientId, user_id = PhoneId}, C) -> @@ -56,8 +57,9 @@ send_presence(Status, Phone, C, ClientId) when Status == online; Status == offli [roster:Send(C, PhoneId, #'Contact'{phone_id = PhoneId, presence = Status, update = Now, status = internal}) || Send <- [send_muc, send_ac]]; #error{} -> skip - end || AccId <- Rosters], P2; - _ -> P end; + end || AccId <- Rosters], + P2; + _ -> roster:info(?MODULE, "~p:~p:~p",[Phone, ClientId,P]), P end; _ -> {error, profile_not_found} end. diff --git a/apps/roster/src/rest/rest_metric.erl b/apps/roster/src/rest/rest_metric.erl index efd80c294..96058662c 100644 --- a/apps/roster/src/rest/rest_metric.erl +++ b/apps/roster/src/rest/rest_metric.erl @@ -11,7 +11,7 @@ handle_request('GET', _, Req) -> prometheus_gauge:set(?METRIC_CHATS_TOTAL, [?METRIC_LABEL_P2P_CHAT], roster_db:p2p_stats()), prometheus_gauge:set(?METRIC_CHATS_TOTAL, [?METRIC_LABEL_GROUP_CHAT], roster_db:room_stats()), [prometheus_gauge:set(?METRIC_USERS_PER_COUNTRY, [Country], Value) || {Country, Value} <- roster_db:country_user_stats()], - [prometheus_gauge:set(?METRIC_USERS_PER_COUNTRY_ONLINE, [Country], Value) || {Country, Value} <- roster_db:country_user_stats(online)], +% [prometheus_gauge:set(?METRIC_USERS_PER_COUNTRY_ONLINE, [Country], Value) || {Country, Value} <- roster_db:country_user_stats(online)], [prometheus_gauge:set(?METRIC_MSGS_BY_TYPE_NMBR, [Mime], Value) || {Mime, Value} <- roster_db:msg_stats()], [prometheus_gauge:set(?METRIC_SESSIONS_PER_COUNTRY_ONLINE, [Country], Value) || {Country, Value} <- roster_db:country_session_stats(online)], rest_response_helper:response(Req, 200, prometheus_text_format:format(), "text/plain"); diff --git a/apps/roster/src/roster.erl b/apps/roster/src/roster.erl index 1e61740f2..a811e8e18 100644 --- a/apps/roster/src/roster.erl +++ b/apps/roster/src/roster.erl @@ -152,10 +152,12 @@ list_rosters(Phone, Fun) -> {ok, #'Profile'{rosters = Rosters}} -> ?MODULE:Fun(Phone, Rosters) end. -send_profile(#'Profile'{rosters = []}, _N, _LastSync, _ClientId, _C) -> ok; +send_profile(#'Profile'{rosters = []}, _N, _LastSync, _ClientId, _C) -> + ok; send_profile(#'Profile'{rosters = [RosterId|TRosterIds], phone = Phone, status = Status} = P, N, LastSync, ClientId, C) -> case kvs:get('Roster', RosterId) of - {error, _} -> roster:error(?MODULE, "roster ~p not found in profile ~p", [RosterId, Phone]); + {error, _} -> + roster:info(?MODULE, "roster ~p not found in profile ~p", [RosterId, Phone]); {ok, #'Roster'{} = R} -> Rosters = split_roster(R, N, LastSync, []), [roster:send_action(C, ClientId, P#'Profile'{rosters = [Roster]}) || Roster<-Rosters], @@ -1013,6 +1015,7 @@ feed(#'Star'{message = #'Message'{feed_id = Feed}}, _) -> Feed. last_upd(#'Room'{id = Room}) -> {ok, #'Room'{update = Update}} = kvs:get('Room', Room), Update; %% get update field from Room table! + last_upd(#'Contact'{phone_id = PhoneId, update = Update}) -> case roster:is_online2(phone(PhoneId)) of {online, _} -> []; _-> Update end; last_upd(#'Star'{message = #'Message'{created = Update}}) -> Update. diff --git a/apps/roster/src/test/roster_test.erl b/apps/roster/src/test/roster_test.erl index 018c30cca..5ccd44fc6 100644 --- a/apps/roster/src/test/roster_test.erl +++ b/apps/roster/src/test/roster_test.erl @@ -1462,8 +1462,8 @@ suite() -> muc_remove_test(), test_call_room(), pagination_test(), - event_securty_test(), - test_messages() + event_securty_test() +% test_messages() ]. check() -> case lists:all(fun(X) -> X == ok end, suite()) of @@ -1507,7 +1507,7 @@ bpe() -> PhoneA = <<"773">>, Phones = [<<"5333">>, <<"377">>], [kvs:put(#'Whitelist'{phone = Phone, created = roster:now_msec()}) || Phone <- Phones ++ [PhoneA]], - [roster:purge_user(Phone) || Phone <- [PhoneA | Phones]], + [ begin try roster:purge_user(Phone) catch _:_ -> skip end end || Phone <- [PhoneA | Phones]], [{B, BClientId, _}, {C, _, _}, {A, AClientId, _}] = [begin C = roster_client:gen_name_reg(Phone), {ClientId, Token} = roster_client:reg_fake_user(Phone), -- GitLab From caf61e66323ef82fddbcf320ac2fc5cc7dc3e6d3 Mon Sep 17 00:00:00 2001 From: qomputer Date: Fri, 26 Apr 2019 13:11:53 +0300 Subject: [PATCH 09/25] Add info to send_profile --- apps/roster/src/roster.erl | 2 ++ 1 file changed, 2 insertions(+) diff --git a/apps/roster/src/roster.erl b/apps/roster/src/roster.erl index a811e8e18..9f81c0d2e 100644 --- a/apps/roster/src/roster.erl +++ b/apps/roster/src/roster.erl @@ -153,8 +153,10 @@ list_rosters(Phone, Fun) -> send_profile(#'Profile'{rosters = []}, _N, _LastSync, _ClientId, _C) -> + roster:info(?MODULE, "SEND PROFILE []", []), ok; send_profile(#'Profile'{rosters = [RosterId|TRosterIds], phone = Phone, status = Status} = P, N, LastSync, ClientId, C) -> + roster:info(?MODULE, "SEND PROFILE [~p]", [length([RosterId|TRosterIds])]), case kvs:get('Roster', RosterId) of {error, _} -> roster:info(?MODULE, "roster ~p not found in profile ~p", [RosterId, Phone]); -- GitLab From ab166c4cf01702f9ac8ecafaf02e814fb4cf4152 Mon Sep 17 00:00:00 2001 From: firebuilder22 Date: Fri, 26 Apr 2019 13:37:32 +0300 Subject: [PATCH 10/25] fix split_roster infinite loop and crashes + db tests --- apps/roster/src/roster.erl | 28 ++++++++++++++++++++-------- apps/roster/src/test/roster_test.erl | 9 ++++++++- 2 files changed, 28 insertions(+), 9 deletions(-) diff --git a/apps/roster/src/roster.erl b/apps/roster/src/roster.erl index 1e61740f2..359ef6007 100644 --- a/apps/roster/src/roster.erl +++ b/apps/roster/src/roster.erl @@ -163,9 +163,10 @@ send_profile(#'Profile'{rosters = [RosterId|TRosterIds], phone = Phone, status = end. split_roster(#'Roster'{} = R, N, LastSync, Rosters) -> - case roster:roster(R, N, LastSync) of + case roster(R, N, LastSync) of #'Roster'{userlist = {[], _} , roomlist = {[], _} , favorite = {[], _}} -> Rosters; - #'Roster'{userlist = {Users, TUsers}, roomlist = {Rooms, TRooms}, favorite = {Stars, TStars}} = Roster -> + #'Roster'{userlist = {Users, TUsers}, roomlist = {Rooms, TRooms}, favorite = {Stars, TStars}} = Roster + when is_list(Users), is_list(Rooms), is_list(Stars) -> split_roster(R#'Roster'{userlist = TUsers, roomlist = TRooms, favorite = TStars}, N, LastSync, Rosters++[Roster#'Roster'{userlist = Users, roomlist = Rooms, favorite = Stars}]) end. @@ -482,7 +483,7 @@ send_event(C, ClientId, Token, Term, VNode) -> n2o_vnode:send(C, event_topic(ClientId, Token, VNode), term_to_binary(Term)). send_action(C, ClientId, Term) -> send_action(C, ClientId, Term, "1"). send_action(C, ClientId, Term, VSN) -> - n2o_vnode:send(C, action_topic(ClientId, VSN), term_to_binary(Term)). + send(C, action_topic(ClientId, VSN), Term). % ROSTER TOPICS @@ -1012,7 +1013,10 @@ feed(#'Contact'{phone_id = P} , PhoneId) -> feed_key(#p2p{from = feed(#'Star'{message = #'Message'{feed_id = Feed}}, _) -> Feed. last_upd(#'Room'{id = Room}) -> - {ok, #'Room'{update = Update}} = kvs:get('Room', Room), Update; %% get update field from Room table! + case kvs:get('Room', Room) of + {ok, #'Room'{update = Update}} -> Update; + {error, _} -> 0 + end; %% get update field from Room table! last_upd(#'Contact'{phone_id = PhoneId, update = Update}) -> case roster:is_online2(phone(PhoneId)) of {online, _} -> []; _-> Update end; last_upd(#'Star'{message = #'Message'{created = Update}}) -> Update. @@ -1044,8 +1048,11 @@ split_objlist([], _Roster, _LastSync, _N, Acc) -> Acc; split_objlist([_|_] = List, #'Roster'{}=Roster, LastSync, N, Acc) -> {Objs, TObjs} = objlist(List, Roster, LastSync, N), split_objlist(TObjs, Roster, LastSync, N, Acc++[Objs]); -split_objlist(Index, Id, LastSync, N, Acc) -> - case kvs:get('Roster', Id) of {ok, R} -> split_objlist(Index, R, LastSync, N, Acc); _ -> {error, roster_not_found} end. +split_objlist(Index, Id, LastSync, N, Acc) + when Index == #'Roster'.userlist; Index == #'Roster'.roomlist; Index == #'Roster'.favorite -> + case kvs:get('Roster', Id) of + {ok, R} -> split_objlist(Index, R, LastSync, N, Acc); + _ -> {error, roster_not_found} end. objlist(Index, Roster) -> objlist(Index, Roster, 0). @@ -1064,8 +1071,13 @@ objlist([Obj|_] = List, #'Roster'{id = Id, phone = Phone}=Roster, LastSync, N) #'Star'{} -> star(Roster, Obj, W, LastSync) end || {Obj, W} <- SubList], {Objs, TList}; -objlist(Index, Id, LastSync, N) -> - case kvs:get('Roster', Id) of {ok, R} -> objlist(Index, R, LastSync, N); _ -> {error, roster_not_found} end. +objlist([Obj|TObjs], #'Roster'{id = Id}=Roster, LastSync, N) -> + roster:info(?MODULE, "invalid object in roster ~p: ~p", [Id, Obj]), + objlist(TObjs, #'Roster'{}=Roster, LastSync, N); +objlist(Index, Id, LastSync, N) when Index == #'Roster'.userlist; Index == #'Roster'.roomlist; Index == #'Roster'.favorite -> + case kvs:get('Roster', Id) of + {ok, R} -> objlist(Index, R, LastSync, N); + _ -> {error, roster_not_found} end. userlist(R) -> hd(split_objlist(#'Roster'.userlist, R)). diff --git a/apps/roster/src/test/roster_test.erl b/apps/roster/src/test/roster_test.erl index 018c30cca..abbe845e1 100644 --- a/apps/roster/src/test/roster_test.erl +++ b/apps/roster/src/test/roster_test.erl @@ -2340,4 +2340,11 @@ test_security(Host) -> end, %% #'History'{data = [_, _]} %% only two message in history %% = roster_client:send_receive(AClientId, #'History'{feed = Feed, roster_id = APhoneId, size = [], entity_id = [], status = get}), - roster_client:stop_client([AClientId, BClientId, CClientId]). \ No newline at end of file + roster_client:stop_client([AClientId, BClientId, CClientId]). + +test_send_profile() -> + [ begin + io:format("Phone: ~p",[P#'Profile'.phone]), + {Time,_} = timer:tc(fun() -> roster:send_profile(P,[],0,[],[]) end), + io:format("done in ~p.~n",[Time]) + end || P <- kvs:all('Profile')]. \ No newline at end of file -- GitLab From 55f3a08ad0be4b905edf51e63883c72e56069411 Mon Sep 17 00:00:00 2001 From: Namdak Tonpa Date: Fri, 26 Apr 2019 13:56:04 +0300 Subject: [PATCH 11/25] us --- apps/roster/src/test/roster_test.erl | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/apps/roster/src/test/roster_test.erl b/apps/roster/src/test/roster_test.erl index abbe845e1..3c7bad262 100644 --- a/apps/roster/src/test/roster_test.erl +++ b/apps/roster/src/test/roster_test.erl @@ -2343,8 +2343,9 @@ test_security(Host) -> roster_client:stop_client([AClientId, BClientId, CClientId]). test_send_profile() -> - [ begin + lists:reverse(lists:keysort(1,[ begin io:format("Phone: ~p",[P#'Profile'.phone]), {Time,_} = timer:tc(fun() -> roster:send_profile(P,[],0,[],[]) end), - io:format("done in ~p.~n",[Time]) - end || P <- kvs:all('Profile')]. \ No newline at end of file + io:format("done in ~p uc.~n",[Time]), + {Time,P#'Profile'.phone} + end || P <- kvs:all('Profile')])). \ No newline at end of file -- GitLab From 1d9336ced3d16ed5065734bb2e6839acec6b6222 Mon Sep 17 00:00:00 2001 From: firebuilder22 Date: Fri, 26 Apr 2019 15:38:25 +0300 Subject: [PATCH 12/25] fix sticker pack case clause with size=[] --- apps/roster/src/protocol/roster_history.erl | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/apps/roster/src/protocol/roster_history.erl b/apps/roster/src/protocol/roster_history.erl index 8afdf1bf4..cd10f08c0 100644 --- a/apps/roster/src/protocol/roster_history.erl +++ b/apps/roster/src/protocol/roster_history.erl @@ -15,7 +15,8 @@ info(#'History'{roster_id = RosterId, feed = #'StickerPack'{} = Feed, size = N, #cx{params = ClientId} = State) -> roster:info(?MODULE, "~p:~p:History/get:~p", [RosterId, ClientId, Feed]), Stickers = case kvs:get('Roster', roster:roster_id(RosterId)) of - {ok, #'Roster'{}} -> lists:sublist(kvs:all('StickerPack'), N); %%TODO Temporary all sticker packs + {ok, #'Roster'{}} -> S = kvs:all('StickerPack'), + case N of [] -> S; _ -> lists:sublist(S, N) end; %%TODO Temporary all sticker packs {error, _} -> [] end, {reply, {bert, Data#'History'{data = Stickers}}, Req, State}; -- GitLab From 499476995591fe4fa1c5071e0b771e5490bfa749 Mon Sep 17 00:00:00 2001 From: qomputer Date: Fri, 26 Apr 2019 15:57:56 +0300 Subject: [PATCH 13/25] Remove vox and payload in log of Message --- apps/roster/src/protocol/roster_auth.erl | 4 ++-- apps/roster/src/protocol/roster_message.erl | 6 +++--- sys.config | 2 +- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/apps/roster/src/protocol/roster_auth.erl b/apps/roster/src/protocol/roster_auth.erl index 9a0cb05dd..48625ce24 100644 --- a/apps/roster/src/protocol/roster_auth.erl +++ b/apps/roster/src/protocol/roster_auth.erl @@ -180,7 +180,7 @@ info(#'Auth'{type = verify, phone = Phone, dev_key = DevKey, settings = Settings || #'Feature'{key = Key} = F <-Settings], kvs:put(P = #'Profile'{phone = Phone, rosters = [Roster], settings = NewSettings, update = roster:now_msec()}), - roster:n2o_pid(?MODULE) ! {vox, P, client(RegClientId)}, +% roster:n2o_pid(?MODULE) ! {vox, P, client(RegClientId)}, {roster, Roster} end, UserId = roster:phone_id(Phone, RosterId), ClientId = client(RegClientId), [kvs:delete('Auth', ClId) || #'Auth'{client_id = ClId} <- kvs:index('Auth', dev_key, DevKey)], @@ -323,7 +323,7 @@ proc({sms, Phone, SmsCode}, #handler{} = H) -> proc({voice, Phone, SmsCode, Lang}, #handler{} = H) -> roster:info(?MODULE, "~p:telesign:~p:~p", [Phone, SmsCode, Lang]), - telesign_api:telesign_voice_call(Phone, SmsCode, [], Lang), +% telesign_api:telesign_voice_call(Phone, SmsCode, [], Lang), {reply, [], H}; proc({mqttc, C, connected}, State = #handler{state = C, seq = S}) -> {ok, State#handler{seq = S + 1}}; diff --git a/apps/roster/src/protocol/roster_message.erl b/apps/roster/src/protocol/roster_message.erl index ce20676f8..4c7896c82 100644 --- a/apps/roster/src/protocol/roster_message.erl +++ b/apps/roster/src/protocol/roster_message.erl @@ -95,7 +95,7 @@ info(#'Message'{status = [], id = [], feed_id = F, from=From0, to = To, created kvs:put(LnkRes2), LnkRes2; _ -> LnkRes end, roster:send_feed(C, F, Msg2 = M#'Message'{link = LnkRes3}), - roster:info(?MODULE, "~p:~p:Message/new:~P", [From, To, Payload, 100]), + roster:info(?MODULE, "~p:Message/new:~p", [From, To]), %% have to skip push notifications for call bubbles case [lists:keyfind(BubbleContentType, #'Desc'.mime, Descs) || BubbleContentType <- [?CONTENT_TYPE_VIDEOCALL, ?CONTENT_TYPE_AUDIOCALL]] of [false, false] -> n2o_async:pid(system, ?MODULE) ! {send_push, From, To, Msg2, []}; @@ -112,7 +112,7 @@ info(#'Message'{status = edit, id = Id, msg_id = ClMID, feed_id = Feed, from = F #cx{params = ClientId, client_pid = C, state=ack} = State) -> MSG_LTNCY = os:system_time(), PhoneId = roster:phone_id(ClientId, From), - roster:info(?MODULE, "~p:~p:Message/edit:~P", [PhoneId, To, Payload, 100]), + roster:info(?MODULE, "~p:Message/edit:~p", [PhoneId, To]), DV = length([D || D = #'Desc'{id = ID} <- Descs, is_binary(ID), ID /= <<>>]) == length(Descs), Data = case kvs:get('Message', Id) of @@ -250,7 +250,7 @@ info(#'Message'{status = update, id = Id, feed_id = Feed, from = From, to = To, #cx{params = ClientId, client_pid = C, state=ack} = State) when is_integer(Id) -> MSG_LTNCY = os:system_time(), PhoneId = case ClientId of <<"sys_bpe">> -> From; <<"emqttd_", _/binary>> -> roster:phone_id(ClientId) end, - roster:info(?MODULE, "~p:~p:Message/update:~P", [PhoneId, To, Payload, 100]), + roster:info(?MODULE, "~p:Message/update:~p", [PhoneId, To]), Lang = roster:get_data_val(?LANG_KEY, Data), IO = case kvs:get('Message', Id) of {ok, #'Message'{feed_id = Feed, files = Descs} = Msg} -> diff --git a/sys.config b/sys.config index b8b352c6a..7546512cd 100644 --- a/sys.config +++ b/sys.config @@ -69,7 +69,7 @@ {formatter,n2o_bert}, {auth_ttl, 900}, %% 15 mins {protocols,[n2o_nitro,n2o_ftp,roster_proto,kvs_stream]}, - {log_modules,[roster]}, + {log_modules,[roster,n2o_vnode]}, {log_level,roster}, {vnode, {roster, get_vnode}}, {validate, {roster, validate}} -- GitLab From 2fe3d6ea13f42db32c5821b7948cdf421f1bf90c Mon Sep 17 00:00:00 2001 From: qomputer Date: Fri, 26 Apr 2019 16:11:38 +0300 Subject: [PATCH 14/25] Remove info Invalid object --- apps/roster/src/roster.erl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/roster/src/roster.erl b/apps/roster/src/roster.erl index 6c831b9c2..7df923219 100644 --- a/apps/roster/src/roster.erl +++ b/apps/roster/src/roster.erl @@ -1076,7 +1076,7 @@ objlist([Obj|_] = List, #'Roster'{id = Id, phone = Phone}=Roster, LastSync, N) end || {Obj, W} <- SubList], {Objs, TList}; objlist([Obj|TObjs], #'Roster'{id = Id}=Roster, LastSync, N) -> - roster:info(?MODULE, "invalid object in roster ~p: ~p", [Id, Obj]), +% roster:info(?MODULE, "invalid object in roster ~p: ~p", [Id, Obj]), objlist(TObjs, #'Roster'{}=Roster, LastSync, N); objlist(Index, Id, LastSync, N) when Index == #'Roster'.userlist; Index == #'Roster'.roomlist; Index == #'Roster'.favorite -> case kvs:get('Roster', Id) of -- GitLab From f1ef5b8f32f0ab50a2c8bb01f78771d2e47acef3 Mon Sep 17 00:00:00 2001 From: Namdak Tonpa Date: Fri, 26 Apr 2019 16:35:19 +0300 Subject: [PATCH 15/25] fix bug in History/get --- apps/roster/src/protocol/roster_history.erl | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/apps/roster/src/protocol/roster_history.erl b/apps/roster/src/protocol/roster_history.erl index cd10f08c0..5abb31378 100644 --- a/apps/roster/src/protocol/roster_history.erl +++ b/apps/roster/src/protocol/roster_history.erl @@ -195,12 +195,13 @@ info(#'History'{feed = Feed, status = delete}, Req, #cx{client_pid = C, params = roster:info(?MODULE, "~p:History/delete:~p", [PhoneId, Feed]), IO = case clean_history(Feed, PhoneId) of #error{} = Res -> #io{code = Res}; - {_Unread, Internal} -> + {_Unread, #'Message'{}=Internal} -> %% D=case is_integer(Unread) of true -> roster:send_ses(C, From, roster:update_field(FList,Record,unread,Unread)), <<>>; false -> Unread end, roster:send_feed(C, Feed, roster:readmsgs(Internal#'Message'{next = []}, PhoneId)), %% send push if no history clean error occured n2o_async:pid(system, ?MODULE) ! {send_push, PhoneId, Feed, ?HISTORY_DELETE_ACTION}, - <<>> end, + <<>>; + {I,_} -> I end, {reply, {bert, IO}, Req, State}; info(#'History'{} = Data, Req, State) -> -- GitLab From 3e3a3653cf2e0511e8299e5e849bdd6e1d5cc330 Mon Sep 17 00:00:00 2001 From: Namdak Tonpa Date: Fri, 26 Apr 2019 17:51:45 +0300 Subject: [PATCH 16/25] autoupdate legal but old tokens --- apps/roster/src/protocol/roster_auth.erl | 4 ++++ apps/roster/src/roster.erl | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/apps/roster/src/protocol/roster_auth.erl b/apps/roster/src/protocol/roster_auth.erl index 48625ce24..25e9e69ad 100644 --- a/apps/roster/src/protocol/roster_auth.erl +++ b/apps/roster/src/protocol/roster_auth.erl @@ -50,6 +50,10 @@ check(#mqtt_client{client_id = <<"emqttd_", _/binary>> = ClientId, username = << LastOnline = case Type of verified -> Now; _ -> LO end, Auth2 = Auth#'Auth'{type = update, settings = Settings2, created = Now, last_online = LastOnline}, case {roster:parse_token(Token), Type} of + {bad_token,_} -> Auth3 = Auth2#'Auth'{type = expired, token = element(2,roster:gen_token([],[])) }, + roster:info(?MODULE, "Auth:auth/roamToken:~p", [Auth3]), + kvs:put(Auth3), + AuthPid ! Auth3#'Auth'{ type = update }, ok; {_, expired} -> AuthPid ! Auth2, ok; {{error, token_expired}, _} -> kvs:put(Auth2#'Auth'{type = expired}), AuthPid ! Auth2, ok; diff --git a/apps/roster/src/roster.erl b/apps/roster/src/roster.erl index 7df923219..d26dfa463 100644 --- a/apps/roster/src/roster.erl +++ b/apps/roster/src/roster.erl @@ -714,7 +714,7 @@ feed_key(#muc{} = Feed) -> Feed. ttl() -> application:get_env(roster, auth_ttl, 60 * 15). depicle(Token) -> case n2o_secret:depickle(Token) of <<>> -> <<>>; T -> - try binary_to_term(T,[safe]) catch _:_ -> <<>> end end. + try binary_to_term(T,[safe]) catch _:_ -> {error,bad_token} end end. gen_token([], Data) -> {'Token', n2o_secret:pickle(term_to_binary({now_msec() + ttl() * 1000, Data}))}; gen_token(Token, Data) -> -- GitLab From 15d4dc3e7b55e470b0930f1601c34590e1c27660 Mon Sep 17 00:00:00 2001 From: qomputer Date: Sat, 27 Apr 2019 12:00:20 +0300 Subject: [PATCH 17/25] Fix Version notification for IOS --- apps/roster/src/api/push/ios.erl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/apps/roster/src/api/push/ios.erl b/apps/roster/src/api/push/ios.erl index ef79bee09..924e1ef5b 100644 --- a/apps/roster/src/api/push/ios.erl +++ b/apps/roster/src/api/push/ios.erl @@ -30,11 +30,11 @@ notify(A, C, T, DeviceId, SessionSettings) -> %% create aps json Aps = jsx:encode([{<<"model">>, Custom}, {<<"type">>, Type}, {<<"title">>, Alert}, - {<<"dns">>, get_data_from_feature(SessionSettings, ?FKPN_SERVER_DNS)}, {<<"version">>, ?VERSION}]), + {<<"dns">>, get_data_from_feature(SessionSettings, ?FKPN_SERVER_DNS)}, {<<"version">>, <>}]), %% create ios payload string PayloadString = binary_to_list(iolist_to_binary(["{\"aps\": {\"nynja\": ", Aps, "}}"])), - roster:info(?MODULE, "PayloadString ~p~n~n", [PayloadString]), + roster:info(roster, "PayloadString ~p~n~n", [PayloadString]), %% prepare push data Payload = list_to_binary(PayloadString), -- GitLab From 7a248341369188ee35b477c1b3a57ad832c5c333 Mon Sep 17 00:00:00 2001 From: firebuilder22 Date: Tue, 30 Apr 2019 18:02:22 +0300 Subject: [PATCH 18/25] add "call" type to 'Room' model spec #7337 --- apps/roster/include/roster.hrl | 2 +- apps/roster/src/roster_validator.erl | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/apps/roster/include/roster.hrl b/apps/roster/include/roster.hrl index 435a06db9..b3162a6dc 100644 --- a/apps/roster/include/roster.hrl +++ b/apps/roster/include/roster.hrl @@ -151,7 +151,7 @@ members = [] :: list(#'Member'{}), admins = [] :: list(#'Member'{}), data = [] :: list(#'Desc'{}), - type = [] :: [] | group | channel, + type = [] :: [] | group | channel | call, tos = [] :: [] | binary(), tos_update = 0 :: [] | integer(), unread = 0 :: [] | integer(), diff --git a/apps/roster/src/roster_validator.erl b/apps/roster/src/roster_validator.erl index 6285b5c53..e339d24f7 100644 --- a/apps/roster/src/roster_validator.erl +++ b/apps/roster/src/roster_validator.erl @@ -520,7 +520,7 @@ validate(D = #'Room'{id = Id, name = Name, links = Links, description = Descript lists:foldl(fun(Tmp, Acc3) when is_record(Tmp,'Member') -> validate(Tmp, Acc3, CM); (Tmp, Acc3) -> [{admins, D}|Acc3] end, Acc2, Admins); {data,_} when is_list(Data) -> lists:foldl(fun(Tmp, Acc3) when is_record(Tmp,'Desc') -> validate(Tmp, Acc3, CM); (Tmp, Acc3) -> [{data, D}|Acc3] end, Acc2, Data); - {type,_} when Type==[] orelse Type=='group' orelse Type=='channel' -> Acc2; + {type,_} when Type==[] orelse Type=='group' orelse Type=='channel' orelse Type=='call' -> Acc2; {tos,_} when Tos==[] orelse is_binary(Tos) -> Acc2; {tos_update,_} when Tos_update==[] orelse is_integer(Tos_update) -> Acc2; {unread,_} when Unread==[] orelse is_integer(Unread) -> Acc2; -- GitLab From 6cc18e78e09a0db93e19c6c87c0dacf3ae3e6be5 Mon Sep 17 00:00:00 2001 From: firebuilder22 Date: Thu, 2 May 2019 17:51:59 +0300 Subject: [PATCH 19/25] add cache logic for leave room --- apps/roster/src/protocol/roster_auth.erl | 14 +++++++------- apps/roster/src/protocol/roster_presence.erl | 6 +++--- apps/roster/src/protocol/roster_room.erl | 3 ++- apps/roster/src/roster.erl | 15 ++++++++------- apps/roster/src/test/roster_test.erl | 20 ++++++++++++++++---- sys.config | 2 +- 6 files changed, 37 insertions(+), 23 deletions(-) diff --git a/apps/roster/src/protocol/roster_auth.erl b/apps/roster/src/protocol/roster_auth.erl index 25e9e69ad..6483a9498 100644 --- a/apps/roster/src/protocol/roster_auth.erl +++ b/apps/roster/src/protocol/roster_auth.erl @@ -269,8 +269,8 @@ info(#'Auth'{phone = Phone} = Auth, Req, #cx{params = ClientId} = State) -> %% roster_auth handlers proc(init, #handler{name = roster_auth} = Async) -> {ok, C} = emqttc:start_link([{client_id, <<"roster_auth">>}, {logger, {console, error}}, {reconnect, 5}]), - _ = locus:start_loader(city, application:get_env(locus, url, ?GEO_URL)), - case locus:wait_for_loader(city, 10000) of {ok, _} -> ok; + _ = (catch locus:start_loader(city, application:get_env(locus, url, ?GEO_URL))), + case catch locus:wait_for_loader(city, 10000) of {ok, _} -> ok; Err -> roster:error(?MODULE, "error:~p:locus", [Err]) end, roster:info(?MODULE, "ASYNC AUTH PROC:~p started", [C]), {ok, Async#handler{state = C, seq = 0}}; @@ -282,26 +282,26 @@ proc(#'Auth'{type = {reg,Ver}, client_id = <<"emqttd_", Post/binary>> = ClientId RegClientId = <<"reg_", Post/binary>>, n2o_vnode:unsubscribe(RegClientId, roster:action_topic(RegClientId)), kvs:delete(mqtt_session, RegClientId), - roster_presence:on_verify(ClientId, UserId), Reply; + catch roster_presence:on_verify(ClientId, UserId), Reply; %%proc(#'Auth'{type = [], token = Token, phone = Phone, client_id = <<"emqttd_", _/binary>> = ClientId}=Auth,H)-> %% proc(Auth#'Auth'{type = {ver,?VERSION}}, H); proc(#'Auth'{type = {ver,Ver}, token = Token, phone = Phone, client_id = <<"emqttd_", _/binary>> = ClientId}, #handler{state = C} = H) -> NewC=roster:restart_module(C,roster_auth), roster:info(?MODULE, "~p:~p:Auth/login:~p", [Phone, ClientId, binary:part(Token, 0, 16)]), - roster_presence:on_connect(Phone, ClientId, NewC, Ver), + catch roster_presence:on_connect(Phone, ClientId, NewC, Ver), {reply, [], H#handler{state = NewC}}; proc(#'Auth'{type = update, client_id = <<"emqttd_", _/binary>> = ClientId} = Auth, #handler{state = C} = H) -> {ok, #'Auth'{token = T}} = kvs:get('Auth', ClientId), NewC=roster:restart_module(C,roster_auth), {'Token', Token2} = roster:gen_token([], binary:part(T, 0, 10)), roster:info(?MODULE, "~p:Auth/update:~p", [ClientId, binary:part(Token2, 0, 16)]), - n2o_vnode:send(NewC, roster:action_topic(ClientId), term_to_binary(Auth#'Auth'{token = Token2}), [{qos, 0}]), + catch n2o_vnode:send(NewC, roster:action_topic(ClientId), term_to_binary(Auth#'Auth'{token = Token2}), [{qos, 0}]), {reply, [], H#handler{state = NewC}}; proc({disconnect, #'Auth'{client_id = <<"emqttd_", _/binary>>} = Auth}, #handler{state = C} = H) -> % NewC=roster:restart_client(C,<<"roster_auth">>), % spawn(roster, client_pid, [C,<<"roster_auth">>]), - roster_presence:on_disconnect(Auth, C), + catch roster_presence:on_disconnect(Auth, C), {reply, [], H#handler{state = C}}; proc({vox, #'Profile'{phone = Phone, services = Services} = Profile, Cli}, #handler{state = C} = H) -> {VoxStatus, VoxResponse} = vox_api:create_voximplant_user(Phone), @@ -313,7 +313,7 @@ proc({vox, #'Profile'{phone = Phone, services = Services} = Profile, Cli}, #hand _ER -> [] end, kvs:put(Profile2 = #'Profile'{rosters = Rosters} = Profile#'Profile'{services = lists:flatten([Service | Services])}), case emqttd_cm:lookup(Cli) of undefined -> skip; - _ -> roster:send_action(C, Cli, + _ -> catch roster:send_action(C, Cli, Profile2#'Profile'{presence = online, rosters = lists:map(fun roster:roster/1, Rosters)}) end, {reply, [], H}; diff --git a/apps/roster/src/protocol/roster_presence.erl b/apps/roster/src/protocol/roster_presence.erl index 604de499e..df9f1847f 100644 --- a/apps/roster/src/protocol/roster_presence.erl +++ b/apps/roster/src/protocol/roster_presence.erl @@ -22,10 +22,10 @@ on_connect(Phone, ClientId, C, Ver) -> case send_presence(online, Phone, C, ClientId) of {error, profile_not_found} -> roster:info(?MODULE, "~p:~p:Connect:ProfileNotFound", [Phone, ClientId]), roster:delete_sessions(Phone); - #'Profile'{} = P when Ver==?VERSION -> - ok; + #'Profile'{} = P when Ver==?VERSION -> roster:send_cache(ClientId, C); #'Profile'{} = P-> - roster:send_profile(P#'Profile'{settings = amazon_settings(P), status = init}, [], 0, ClientId, C) + roster:send_profile(P#'Profile'{settings = amazon_settings(P), status = init}, [], 0, ClientId, C), + roster:send_cache(ClientId, C) end catch Err:Rea -> roster:info(?MODULE, "Catch:~p", [n2o:stack_trace(Err, Rea)]) diff --git a/apps/roster/src/protocol/roster_room.erl b/apps/roster/src/protocol/roster_room.erl index e69babdee..7dc9d3dea 100644 --- a/apps/roster/src/protocol/roster_room.erl +++ b/apps/roster/src/protocol/roster_room.erl @@ -283,7 +283,6 @@ info(#'Room'{status = leave, id = Room}, Req, #cx{params = ClientId, client_pid #'Member'{phone_id = PhoneId, status = Status, reader = Reader} = M -> roster:remove_rooms(ClientId, [Room]), kvs:delete(reader, Reader), - kvs:put(M#'Member'{reader = 0, update = roster:now_msec(), status = removed}), Room2 = roster:update_field(roster:sort_readers(StoredRoom), [{case Status of admin -> admins;_-> members end, [M]}]), roster:send_ses(C, roster:phone(PhoneId), @@ -294,10 +293,12 @@ info(#'Room'{status = leave, id = Room}, Req, #cx{params = ClientId, client_pid from = PhoneId, msg_id = roster:generate_server_id(), to = Room, status = [], type = [sys], files = [#'Desc'{payload = <>}]}), roster:send_room(C, R), + [roster:cache_ses(PId, R)|| #'Member'{phone_id = PId} <-roster:members(#muc{name = Room}, active)], roster:send_room(C, Room, Msg), n2o_async:pid(system, ?MODULE) ! {send_push, [], Msg, leave}, Room2#'Room'{update = roster:now_msec(), last_msg = Msg#'Message'.id} end, + kvs:put(M#'Member'{reader = 0, update = roster:now_msec(), status = removed}), kvs:put(UpdRoom2), <<>>; _ -> #io{code = #error{code = member_not_found}} end; diff --git a/apps/roster/src/roster.erl b/apps/roster/src/roster.erl index d26dfa463..bb273873e 100644 --- a/apps/roster/src/roster.erl +++ b/apps/roster/src/roster.erl @@ -2088,14 +2088,15 @@ delete_call_bubbles(Msg, _) -> io:format("FoundCallBubbleMsg:~p~n", [Msg]), Msg#'Message'{seenby = [-1]}. -cashe_ses(PhoneId, Term) -> - [begin - Now = integer_to_binary(now_msec()), - K = <<"cashe_", Now/binary>>, - F = #'Feature'{id = <<>>, key = K, value = Term, group = <<"CACHE_GROUP">>}, - kvs:put(Auth#'Auth'{settings = lists:keystore(K, #'Feature'.key, Settings, F)}) +cache_ses(PhoneId, Term) -> + [begin Now = integer_to_binary(now_msec()), + F = #'Feature'{id = <<>>, key = K = <<"cache_", Now/binary>>, value = Term, group = <<"CACHE_GROUP">>}, + kvs:put(Auth#'Auth'{settings = lists:keystore(K, #'Feature'.key, Settings, F)}) end||#'Auth'{client_id = ClientId, settings = Settings} = Auth <-kvs:index('Auth', user_id, PhoneId), emqttd_cm:lookup_proc(ClientId) == undefined]. + send_cache(#'Auth'{client_id = ClientId, settings = Settings} = Auth, C) -> [send_action(C, ClientId, V) || #'Feature'{key = <<"cache_", _/binary>>, value = V} <- Settings], - kvs:put(Auth#'Auth'{settings = [F||#'Feature'{group = Group} = F<-Settings, Group /= <<"CACHE_GROUP">>]}). \ No newline at end of file + case [F||#'Feature'{group = Group} = F <- Settings, Group /= <<"CACHE_GROUP">>] of + Settings -> ok; S -> kvs:put(Auth#'Auth'{settings = S}) end; +send_cache(ClientId, C) -> send_cache(element(2, kvs:get('Auth', ClientId)), C). \ No newline at end of file diff --git a/apps/roster/src/test/roster_test.erl b/apps/roster/src/test/roster_test.erl index 49df6e284..bcf8d29ec 100644 --- a/apps/roster/src/test/roster_test.erl +++ b/apps/roster/src/test/roster_test.erl @@ -623,12 +623,12 @@ get_room_id(Phone, RoomName) -> {error, _} -> false end. test_multi_muc() -> - RoomNames = [<<"test_room_A">>, <<"test_room_B">>, <<"test_room_C">>, + RoomNames = [ARoom = <<"test_room_A">>, <<"test_room_B">>, <<"test_room_C">>, <<"test_room_D">>, <<"test_room_E">>, <<"test_room_F">>], - Phones = [{APhone = <<"001">>, admin}, {<<"002">>, admin}, {<<"003">>, member}, {<<"004">>, member}], + Phones = [{APhone = <<"001">>, admin}, {<<"002">>, admin}, {CPhone = <<"003">>, member}, {<<"004">>, member}], [roster:purge_room(R) || R <- RoomNames], Counter = length(Phones), - PCs = [{APhoneId, AClientId, _, _} | [{_, _, _, _}, {_, _, _, _}, {_, _, _, _}]] = + PCs = [{APhoneId, AClientId, _, _} | [{_, _, _, _}, {CPhoneId, CClientId, _, _}, {_, _, _, _}]] = [begin roster:purge_user(Phone), {ClientId, Token} = roster_client:reg_fake_user(Phone), roster_client:start_cli_receive(ClientId, Token), @@ -652,7 +652,19 @@ test_multi_muc() -> %% = roster:roster(roster:roster_id(APhoneId)), %% ?MAX_LAST_ROOMS = length(Rooms2), TODO uncomment in the future for the last rooms - roster_client:stop_client(AClientId2), + {CClientId2, CToken2} = roster_client:reg_fake_user(CPhone, <<"DevKey_003C">>), + roster_client:stop_client(roster_client:gen_name_reg(CPhone, <<"DevKey_003C">>)), + roster_client:start_cli_receive(CClientId2, CToken2), + roster_client:stop_client(CClientId2), + timer:sleep(100), + roster_client:send_receive(CClientId, Counter+1, #'Room'{id = ARoom, status = leave, type = group}), + {ok, #'Auth'{settings = [_, _, #'Feature'{key = <<"cache_", _/binary>>}]}} = kvs:get('Auth', CClientId2), + roster_client:start_cli_receive(CClientId2, CToken2, 1), + + [#'Profile'{}, #'Room'{status = leave}] = roster_client:send_receive(CClientId2, last_res), %% + roster_client:stop_client(CClientId), + #'Profile'{} = roster_client:start_cli_receive(CClientId, CToken2), + [roster_client:stop_client(ClientId)|| ClientId <- [AClientId2, CClientId2]], [roster_client:stop_client(ClientId) || {_, ClientId, _, _} <- PCs], ok. diff --git a/sys.config b/sys.config index 7546512cd..d4fab7b58 100644 --- a/sys.config +++ b/sys.config @@ -69,7 +69,7 @@ {formatter,n2o_bert}, {auth_ttl, 900}, %% 15 mins {protocols,[n2o_nitro,n2o_ftp,roster_proto,kvs_stream]}, - {log_modules,[roster,n2o_vnode]}, + {log_modules,[roster, n2o_vnode]}, {log_level,roster}, {vnode, {roster, get_vnode}}, {validate, {roster, validate}} -- GitLab From 95525a54322289b780a595f7562dcc165f1b2eec Mon Sep 17 00:00:00 2001 From: firebuilder22 Date: Thu, 2 May 2019 18:17:44 +0300 Subject: [PATCH 20/25] fix delete_p2p_history_test --- apps/roster/src/test/roster_test.erl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/roster/src/test/roster_test.erl b/apps/roster/src/test/roster_test.erl index bcf8d29ec..50b9aabdf 100644 --- a/apps/roster/src/test/roster_test.erl +++ b/apps/roster/src/test/roster_test.erl @@ -1871,7 +1871,7 @@ delete_p2p_history_test() -> %%unban user [#'Contact'{reader = RdrB3, status = friend} | _] = - roster_client:receive_last(BClientId, 2, #'Friend'{phone_id = BPhoneId, friend_id = APhoneId, status = unban}), + roster_client:receive_last(BClientId, 3, #'Friend'{phone_id = BPhoneId, friend_id = APhoneId, status = unban}), true = RdrB3 /= [0, 0], #'Contact'{reader = RdrB3} = roster:get_contact(roster:roster(roster:roster_id(BPhoneId)), APhoneId), roster_client:stop_client(AClientId), -- GitLab From 26863452c7954ff95b003e3a9035a03d27ddde77 Mon Sep 17 00:00:00 2001 From: qomputer Date: Fri, 3 May 2019 10:35:50 +0300 Subject: [PATCH 21/25] Fix created time in Message --- apps/roster/src/api/push/ios.erl | 2 +- apps/roster/src/protocol/roster_message.erl | 4 ++-- apps/roster/src/protocol/roster_presence.erl | 6 +++--- apps/roster/src/rest/rest_handler.erl | 2 +- apps/roster/src/test/roster_test.erl | 7 +++---- 5 files changed, 10 insertions(+), 11 deletions(-) diff --git a/apps/roster/src/api/push/ios.erl b/apps/roster/src/api/push/ios.erl index 924e1ef5b..b6197e2b7 100644 --- a/apps/roster/src/api/push/ios.erl +++ b/apps/roster/src/api/push/ios.erl @@ -34,7 +34,7 @@ notify(A, C, T, DeviceId, SessionSettings) -> %% create ios payload string PayloadString = binary_to_list(iolist_to_binary(["{\"aps\": {\"nynja\": ", Aps, "}}"])), - roster:info(roster, "PayloadString ~p~n~n", [PayloadString]), +% roster:info(roster, "PayloadString ~p~n~n", [PayloadString]), %% prepare push data Payload = list_to_binary(PayloadString), diff --git a/apps/roster/src/protocol/roster_message.erl b/apps/roster/src/protocol/roster_message.erl index 4c7896c82..c17c5646d 100644 --- a/apps/roster/src/protocol/roster_message.erl +++ b/apps/roster/src/protocol/roster_message.erl @@ -28,7 +28,7 @@ info(#'Message'{type = T} = M, Req, #cx{state=[]}=State)-> info(M#'Message'{type = T}, Req, State#cx{state=ack}); info(#'Message'{feed_id = #muc{name = To}, to = []} = RequestData, Req, State) when To /= []-> info(RequestData#'Message'{to = To}, Req, #cx{state=ack}=State); -info(#'Message'{status = [], id = [], feed_id = F, from=From0, to = To, created = [], +info(#'Message'{status = [], id = [], feed_id = F, from=From0, to = To, type = Type, files = [#'Desc'{payload = Payload} | _] = Descs} = Msg, Req, #cx{client_pid = C, params = ClientId, state=ack} = State) -> MSG_LTNCY = os:system_time(), @@ -74,7 +74,7 @@ info(#'Message'{status = [], id = [], feed_id = F, from=From0, to = To, created %% create message amd save to db Writer = #writer{cache = #'Message'{id = MsgId} = M} = kvs_stream:save(kvs_stream:add(PrevWriter2#writer{args = Msg#'Message'{container = chain, - feed_id = Feed, created = case Created of [] -> roster:now_msec(); _ -> Created end}})), + feed_id = Feed, created = roster:now_msec()}})), case Feed of #muc{name = Room} -> {ok, Room2} = kvs:get('Room', Room), diff --git a/apps/roster/src/protocol/roster_presence.erl b/apps/roster/src/protocol/roster_presence.erl index df9f1847f..f5d96b0cf 100644 --- a/apps/roster/src/protocol/roster_presence.erl +++ b/apps/roster/src/protocol/roster_presence.erl @@ -22,10 +22,10 @@ on_connect(Phone, ClientId, C, Ver) -> case send_presence(online, Phone, C, ClientId) of {error, profile_not_found} -> roster:info(?MODULE, "~p:~p:Connect:ProfileNotFound", [Phone, ClientId]), roster:delete_sessions(Phone); - #'Profile'{} = P when Ver==?VERSION -> roster:send_cache(ClientId, C); + #'Profile'{} = P when Ver==?VERSION -> ok; % TODO 1 roster:send_cache(ClientId, C); #'Profile'{} = P-> - roster:send_profile(P#'Profile'{settings = amazon_settings(P), status = init}, [], 0, ClientId, C), - roster:send_cache(ClientId, C) + roster:send_profile(P#'Profile'{settings = amazon_settings(P), status = init}, [], 0, ClientId, C) + %TODO 2 uncomment after tests roster:send_cache(ClientId, C) end catch Err:Rea -> roster:info(?MODULE, "Catch:~p", [n2o:stack_trace(Err, Rea)]) diff --git a/apps/roster/src/rest/rest_handler.erl b/apps/roster/src/rest/rest_handler.erl index a985c53ef..80a6e4214 100644 --- a/apps/roster/src/rest/rest_handler.erl +++ b/apps/roster/src/rest/rest_handler.erl @@ -51,7 +51,7 @@ c_tpl([File | Files], Opts) -> %% ------------------------------------------------------------------ handle_request(Req) -> - "/" ++ Path = Req:get(path), + Path= case Req:get(path) of "/" ++ P -> P; _ ->"/" end, %% NOTE extra check for "favicon.ico" file on rest requests without html body case Path of "favicon.ico" -> Req:serve_file("assets/img/favicon.ico", docroot()); diff --git a/apps/roster/src/test/roster_test.erl b/apps/roster/src/test/roster_test.erl index 50b9aabdf..553cc8cbd 100644 --- a/apps/roster/src/test/roster_test.erl +++ b/apps/roster/src/test/roster_test.erl @@ -658,10 +658,9 @@ test_multi_muc() -> roster_client:stop_client(CClientId2), timer:sleep(100), roster_client:send_receive(CClientId, Counter+1, #'Room'{id = ARoom, status = leave, type = group}), - {ok, #'Auth'{settings = [_, _, #'Feature'{key = <<"cache_", _/binary>>}]}} = kvs:get('Auth', CClientId2), - roster_client:start_cli_receive(CClientId2, CToken2, 1), - - [#'Profile'{}, #'Room'{status = leave}] = roster_client:send_receive(CClientId2, last_res), %% +% {ok, #'Auth'{settings = [_, _, #'Feature'{key = <<"cache_", _/binary>>}]}} = kvs:get('Auth', CClientId2), +% roster_client:start_cli_receive(CClientId2, CToken2, 1), TODO uncomment for send_cache test +% [#'Profile'{}, #'Room'{status = leave}] = roster_client:send_receive(CClientId2, last_res), %% roster_client:stop_client(CClientId), #'Profile'{} = roster_client:start_cli_receive(CClientId, CToken2), [roster_client:stop_client(ClientId)|| ClientId <- [AClientId2, CClientId2]], -- GitLab From 0e420217e4b6636a8da9a4b1e9ebda3ef315a378 Mon Sep 17 00:00:00 2001 From: firebuilder22 Date: Fri, 3 May 2019 16:10:16 +0300 Subject: [PATCH 22/25] fix get history crash #178 --- apps/roster/src/protocol/roster_history.erl | 36 +++++++++++++-------- apps/roster/src/roster.erl | 21 ++++++------ apps/roster/src/test/roster_test.erl | 2 +- 3 files changed, 34 insertions(+), 25 deletions(-) diff --git a/apps/roster/src/protocol/roster_history.erl b/apps/roster/src/protocol/roster_history.erl index 5abb31378..46cd49d24 100644 --- a/apps/roster/src/protocol/roster_history.erl +++ b/apps/roster/src/protocol/roster_history.erl @@ -110,21 +110,29 @@ info(#'History'{status = get, roster_id = Roster0, feed = Feed, size = N, entity {Msgs, _} = roster:fold(FilterFun, {[], roster:start_reader(Reader)}, 'Message', StartId, FId, #kvs{mod = store_mnesia}, Iter, StopFun), - Msgs2 = case is_integer(MId) of + Msgs2 = + case is_integer(MId) of true when MId > 0, Mime2 == [] -> - {ok, #'Message'{} = EntityMsg} = kvs:get('Message', MId), - lists:reverse(lists:ukeymerge(#'Message'.id, lists:reverse(Msgs), - [EntityMsg#'Message'{files = []}])); - _ -> Msgs - end, - ResponseMsgs = case MId of - 0 -> -%% nullify Message.next for last message in query - LastMsgObject = lists:last(Msgs2), - lists:sublist(Msgs2, length(Msgs2) -1) ++ [LastMsgObject#'Message'{next = []}]; - _ -> Msgs2 end, - CheckedResponseMsgs = [roster:check_message(Msg) || Msg <- ResponseMsgs], - History#'History'{data = ResponseMsgs, size = length(ResponseMsgs), status = InitialStatus} + case kvs:get('Message', MId) of + {ok, #'Message'{} = EntityMsg} -> + lists:reverse(lists:ukeymerge(#'Message'.id, lists:reverse(Msgs), + [EntityMsg#'Message'{files = []}])); + {error, _} -> + roster:info(?MODULE, "message ~p not found", [MId]), + #error{code = invalid_data} + end; + _ -> Msgs + end, +%% ResponseMsgs = case MId of %%TODO remove this after testing +%% 0 -> +%%%% nullify Message.next for last message in query +%%%% [LastMsgObject, ] = lists:reverse(Msgs2), +%% LastMsgObject = lists:last(Msgs2), +%% lists:sublist(Msgs2, length(Msgs2) -1) ++ [LastMsgObject#'Message'{next = []}]; +%% _ -> Msgs2 end, +%% CheckedResponseMsgs = [roster:check_message(Msg) || Msg <- ResponseMsgs], + case Msgs2 of #error{} = E -> #io{code = E}; _ -> + History#'History'{data = Msgs2, size = length(Msgs2), status = InitialStatus} end end, {reply, {bert, IO}, Req, State}; diff --git a/apps/roster/src/roster.erl b/apps/roster/src/roster.erl index bb273873e..f1212f71c 100644 --- a/apps/roster/src/roster.erl +++ b/apps/roster/src/roster.erl @@ -2062,18 +2062,19 @@ link_thumb(ParentType, ChildType, Payload) -> generate_server_id() -> iolist_to_binary(["srv_", nitro:to_binary(roster:now_msec())]). %% TODO add test for the func below +%% TODO remove this after testing %% Set Message.next to [] if it is message about history deletion %% This messages payload is <<"History was removed">> or/and message Msg.id starts with <<"rmv_history_">> -check_message(MsgId) when is_integer(MsgId) -> - {ok, Msg} = kvs:get('Message', MsgId), - check_message(Msg); -check_message(#'Message'{msg_id = <<"rmv_history_",_/binary>>} = Msg) -> - Msg#'Message'{next = []}; -check_message(#'Message'{files = Descs} = Msg) -> - case lists:keyfind(<<"History was removed">>, #'Desc'.payload, Descs) of - false -> Msg; - _ -> Msg#'Message'{next = []} - end. +%%check_message(MsgId) when is_integer(MsgId) -> +%% {ok, Msg} = kvs:get('Message', MsgId), +%% check_message(Msg); +%%check_message(#'Message'{msg_id = <<"rmv_history_",_/binary>>} = Msg) -> +%% Msg#'Message'{next = []}; +%%check_message(#'Message'{files = Descs} = Msg) -> +%% case lists:keyfind(<<"History was removed">>, #'Desc'.payload, Descs) of +%% false -> Msg; +%% _ -> Msg#'Message'{next = []} +%% end. delete_call_bubbles() -> backup(), diff --git a/apps/roster/src/test/roster_test.erl b/apps/roster/src/test/roster_test.erl index 553cc8cbd..2a5b0cb40 100644 --- a/apps/roster/src/test/roster_test.erl +++ b/apps/roster/src/test/roster_test.erl @@ -264,7 +264,7 @@ test_roster() -> #'Profile'{status = get} = roster_client:send_receive(AClientId, #'Profile'{status = get}), #'Message'{id = IdLast} = roster_client:send_receive(BClientId, 2, #'Message'{feed_id = Feed, from = BPhoneId, to = APhoneId, files = [#'Desc'{id = <<"7">>, payload = <<"TestB2222!">>}], status = []}), - #'History'{status = get, data = [_, #'Message'{status = clear}|_]} = %% if msg_id < clear_id then clear message doesn't stop history + #'History'{status = get, data = [_, #'Message'{status = clear, next = []}|_]} = %% if msg_id < clear_id then clear message doesn't stop history roster_client:send_receive(AClientId, #'History'{feed = Feed, roster_id = APhoneId, entity_id = IdA, size = 5, status = get}), #'Message'{id = IdEd, status = edit} = roster_client:send_receive(BClientId, 2, #'Message'{id = IdLast, feed_id = Feed, from = BPhoneId, to = APhoneId, files = [#'Desc'{id = <<"7">>, payload = <<"Edit TestB!">>}], status = edit}), -- GitLab From d4404ae3bbf2ef9f56709115913c0afdcc1864ad Mon Sep 17 00:00:00 2001 From: firebuilder22 Date: Fri, 3 May 2019 16:13:54 +0300 Subject: [PATCH 23/25] uncomment send_cache in roster_presence:on_connect --- apps/roster/src/protocol/roster_presence.erl | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/apps/roster/src/protocol/roster_presence.erl b/apps/roster/src/protocol/roster_presence.erl index f5d96b0cf..df9f1847f 100644 --- a/apps/roster/src/protocol/roster_presence.erl +++ b/apps/roster/src/protocol/roster_presence.erl @@ -22,10 +22,10 @@ on_connect(Phone, ClientId, C, Ver) -> case send_presence(online, Phone, C, ClientId) of {error, profile_not_found} -> roster:info(?MODULE, "~p:~p:Connect:ProfileNotFound", [Phone, ClientId]), roster:delete_sessions(Phone); - #'Profile'{} = P when Ver==?VERSION -> ok; % TODO 1 roster:send_cache(ClientId, C); + #'Profile'{} = P when Ver==?VERSION -> roster:send_cache(ClientId, C); #'Profile'{} = P-> - roster:send_profile(P#'Profile'{settings = amazon_settings(P), status = init}, [], 0, ClientId, C) - %TODO 2 uncomment after tests roster:send_cache(ClientId, C) + roster:send_profile(P#'Profile'{settings = amazon_settings(P), status = init}, [], 0, ClientId, C), + roster:send_cache(ClientId, C) end catch Err:Rea -> roster:info(?MODULE, "Catch:~p", [n2o:stack_trace(Err, Rea)]) -- GitLab From d833b2240d34f372b9fbbe8d4576eed1af65374d Mon Sep 17 00:00:00 2001 From: qomputer Date: Mon, 6 May 2019 16:21:01 +0300 Subject: [PATCH 24/25] Update the specs for Job, Schedule and roster_bpe --- apps/roster/include/roster.hrl | 15 +++++++-------- apps/roster/src/protocol/roster_bpe.erl | 19 +++---------------- apps/roster/src/protocol/roster_profile.erl | 2 +- apps/roster/src/roster.erl | 16 ---------------- apps/roster/src/roster_db.erl | 16 ++++++++++++++++ apps/roster/src/roster_validator.erl | 10 +++++----- 6 files changed, 32 insertions(+), 46 deletions(-) diff --git a/apps/roster/include/roster.hrl b/apps/roster/include/roster.hrl index b3162a6dc..58bd0ef22 100644 --- a/apps/roster/include/roster.hrl +++ b/apps/roster/include/roster.hrl @@ -47,7 +47,6 @@ -define(GEO_URL , "http://geolite.maxmind.com/download/geoip/database/GeoLite2-City.tar.gz"). -define(AUTH_GROUP , <<"AUTH_DATA">>). - -record(chain, {?CONTAINER, aclver = [], unread = {[],[]}}). -record(push, {model = [] :: [] | term(), @@ -246,6 +245,7 @@ -record(act, {name= <<"publish">> :: [] | binary(), data=[]:: binary() | integer() | list(term())}). + -record('Job', {id = [] :: [] | integer(), container = chain :: chain | [], feed_id = [] :: #act{}, @@ -254,10 +254,10 @@ context = [] :: [] | integer() | binary(), proc = [] :: [] | integer() | #process{}, time = [] :: [] | integer(), - data = [] :: list(#'Message'{}), + data = [] :: [] | binary() | list(#'Message'{}), events = [] :: [] | list(#messageEvent{}), settings = [] :: [] | list(#'Feature'{}), - status = [] :: [] | init | update | delete | pending | stop | complete}). + status = [] :: [] | init | update | delete | pending | stop | complete | restart}). -record('History', {roster_id = [] :: binary(), feed = [] :: #p2p{} | #muc{} | #act{} | #'StickerPack'{} | [], @@ -267,11 +267,10 @@ status = [] :: updated | get | update | last_loaded | last_msg | get_reply | double_get | delete | image | video | file | link | audio | contact | location | text}). - --record('Schedule', {id = [] :: [] | integer(), - proc = [] :: [] | integer(), - data = [] :: list(term()), - state =[] :: [] | term()}). +-record('Schedule', {id = [] :: [] | integer() | {integer(), {integer(),integer(),integer()}}, + proc = [] :: [] | integer() | binary(), + data = [] :: binary() | list(term()), + state =[] :: [] | term()}). %% Index.id - term for indexing in format {keyword_atom, value_binary} %% Index.roster - entity id, list of Parent elements' id. Unique for Nick and Link indexing. diff --git a/apps/roster/src/protocol/roster_bpe.erl b/apps/roster/src/protocol/roster_bpe.erl index 18819011c..6917bcd1d 100644 --- a/apps/roster/src/protocol/roster_bpe.erl +++ b/apps/roster/src/protocol/roster_bpe.erl @@ -149,7 +149,7 @@ info(#'Job'{id = Id, time = T0, feed_id = {act, <<"publish">>, PhoneId} = Feed, {reply, {bert, Replay}, R, S}; -info(#'Job'{id = Id, proc = [Proc], data = <<"stop">>, status = update} = M, R, S) -> +info(#'Job'{id = Id, proc = Proc, data = <<"stop">>, status = update} = M, R, S) -> roster:info(?MODULE, "STOP:~w", [M]), Docs = #'act'{data = <<"stop">>}, {reply, {bert, {io, bpe:amend(Proc, Docs, noflow), <<>>}}, R, S}; @@ -182,18 +182,6 @@ info(#'Job'{id = Id, feed_id = {act, <<"publish">>, UID} = Feed, status = delete %timer:apply_after(Time, bpe, cleanup, [Id]), {reply, {bert, D}, R, S}; -info(#'Job'{id = Proc, status = history} = M, R, S) -> - roster:info(?MODULE, "history:~w", [M]), - {reply, {bert, {io, bpe:hist(Proc), <<>>}}, R, S}; - -info(#'Job'{id = Proc, status = proc} = M, R, S) -> - roster:info(?MODULE, "proc:~w", [M]), - {reply, {bert, {io, bpe:process(Proc), <<>>}}, R, S}; - -info(#'Job'{id = Proc, status = complete} = M, R, S) -> - roster:info(?MODULE, "complete:~w", [M]), - {reply, {bert, {io, bpe:complete(Proc), <<>>}}, R, S}; - info(#'Job'{feed_id = Feed} = RequestData, R, S) -> roster:info(?MODULE, "~p::Job/unknown", [Feed]), {reply, {bert, roster_channel_helper:error_response_400(RequestData) @@ -229,9 +217,9 @@ proc(init, #handler{name = roster_bpe} = Async) -> _ -> {ok, Id0} = bpe:start(job:def(#'Schedule'{id = {0, {0, 0, 1}}, proc = <<"publish">>}), []), bpe:complete(Id0), Id0 end, - % bpe:amend(Proc,{send_pid, C},noflow), + % bpe:amend(Proc,{send_pid, C},noflow), roster:info(?MODULE, "ASYNC BPE started: ~p; ~p", [C, Proc]), - % Proc is transfered throuw state + % Proc is transfered through state {ok, Async#handler{state = {C, Proc}, seq = 0}}; proc({update, #'Job'{id = Id0, time = Time, feed_id = {act, <<"publish">>, PhoneId} = Feed, data = D} = J, {OldTime, Limit}, @@ -290,7 +278,6 @@ proc({update, #'Job'{id = Id0, time = Time, feed_id = {act, <<"publish">>, Phone end end; - proc({update, #'Job'{id = Id0, time = Time, feed_id = {act, <<"publish">>, PhoneId} = Feed, data = D} = J, {OldTime, Limit}, #cx{params = ClientId} = S}, #handler{state = {C, Proc}} = H) -> roster:info(?MODULE, "BPE update limit ~p; ~p", [C, Proc]), diff --git a/apps/roster/src/protocol/roster_profile.erl b/apps/roster/src/protocol/roster_profile.erl index 4cad2101d..5b63acf1b 100644 --- a/apps/roster/src/protocol/roster_profile.erl +++ b/apps/roster/src/protocol/roster_profile.erl @@ -133,7 +133,7 @@ del_roster(UID,RosterId) -> roster:update_chains({'Message',from},{#'Message'.from,PId},{#'Message'.files,[#'Desc'{payload = "Deleted"}]}), kvs:delete('Index', {nick, string:lowercase(Nick)}), case roster:is_online(user_id,PId) of offline -> roster:delete_sessions(user_id, PId); - online -> roster:force_logout(user_id,PId) end, + online -> roster:force_logout(user_id,PId) end, % TODO Add force_logout/2 kvs:put(#'Roster'{id=RosterId, phone=UID, nick= <<"Deleted user">>, status=del}), RosterId; {error, _} -> [] diff --git a/apps/roster/src/roster.erl b/apps/roster/src/roster.erl index f1212f71c..d73382dee 100644 --- a/apps/roster/src/roster.erl +++ b/apps/roster/src/roster.erl @@ -1968,22 +1968,6 @@ check(Module) -> case lists:all(fun(X) -> X == ok end, Module:suite()) of true -> roster:info("ALL TESTS PASSED", []), true; false -> roster:info("TEST ERRORS", []), false end. -%% tables storage -save_db(Path) -> - Data = lists:append([kvs:all(B) || B <- [Name || {table,Name} <- kvs:dir()-- - [{table,schema},{table,mqtt_session},{table,mqtt_admin}]]]), - kvs:save(Path, Data). - -load_db(Path) -> - kvs:add_seq_ids(), - AllEntries= try kvs:load(Path) catch _:_ -> [] end, - [begin case element(1,E) of - mqtt_trie=Tab -> write_trie(Tab,setelement(1,E,trie)); - mqtt_trie_node=Tab -> write_trie(Tab,setelement(1,E,trie_node)); - _ -> kvs:put(E) end end || E <- lists:filter(fun(E) -> is_tuple(E) end ,AllEntries)]. - -write_trie(Tab,Trie) ->mnesia:dirty_write(Tab, Trie). - n2o_pid(Name) -> case n2o_async:pid(system, Name) of diff --git a/apps/roster/src/roster_db.erl b/apps/roster/src/roster_db.erl index 2b8eb026a..a3ac9c535 100644 --- a/apps/roster/src/roster_db.erl +++ b/apps/roster/src/roster_db.erl @@ -603,6 +603,22 @@ remigrate_default_contact_settings() -> kvs:delete(schema_migrations, 20181008144317), roster:migrate(). +%% tables storage +save_db(Path) -> + Data = lists:append([kvs:all(B) || B <- [Name || {table,Name} <- kvs:dir()-- + [{table,schema},{table,mqtt_session},{table,mqtt_admin}]]]), + kvs:save(Path, Data). + +load_db(Path) -> + kvs:add_seq_ids(), + AllEntries= try kvs:load(Path) catch _:_ -> [] end, + [begin case element(1,E) of + mqtt_trie=Tab -> write_trie(Tab,setelement(1,E,trie)); + mqtt_trie_node=Tab -> write_trie(Tab,setelement(1,E,trie_node)); + _ -> kvs:put(E) end end || E <- lists:filter(fun(E) -> is_tuple(E) end ,AllEntries)]. + +write_trie(Tab,Trie) ->mnesia:dirty_write(Tab, Trie). + %% Subscriptions control unsubs(<<"emqttd_", _/binary>>, _) -> []; diff --git a/apps/roster/src/roster_validator.erl b/apps/roster/src/roster_validator.erl index e339d24f7..615a234e3 100644 --- a/apps/roster/src/roster_validator.erl +++ b/apps/roster/src/roster_validator.erl @@ -698,13 +698,13 @@ validate(D = #'Job'{id = Id, container = Container, feed_id = Feed_id, prev = Pr {context,_} when Context==[] orelse is_integer(Context) orelse is_binary(Context) -> Acc2; {proc,_} when Proc==[] orelse is_integer(Proc) orelse is_record(Proc,'process') -> Acc2; {time,_} when Time==[] orelse is_integer(Time) -> Acc2; - {data,_} when is_list(Data) -> + {data,_} when Data==[] orelse is_binary(Data) orelse is_list(Data) -> lists:foldl(fun(Tmp, Acc3) when is_record(Tmp,'Message') -> validate(Tmp, Acc3, CM); (Tmp, Acc3) -> [{data, D}|Acc3] end, Acc2, Data); {events,_} when Events==[] orelse is_list(Events) -> lists:foldl(fun(Tmp, Acc3) when is_record(Tmp,'messageEvent') -> validate(Tmp, Acc3, CM); (Tmp, Acc3) -> [{events, D}|Acc3] end, Acc2, Events); {settings,_} when Settings==[] orelse is_list(Settings) -> lists:foldl(fun(Tmp, Acc3) when is_record(Tmp,'Feature') -> validate(Tmp, Acc3, CM); (Tmp, Acc3) -> [{settings, D}|Acc3] end, Acc2, Settings); - {status,_} when Status==[] orelse Status=='init' orelse Status=='update' orelse Status=='delete' orelse Status=='pending' orelse Status=='stop' orelse Status=='complete' -> Acc2; + {status,_} when Status==[] orelse Status=='init' orelse Status=='update' orelse Status=='delete' orelse Status=='pending' orelse Status=='stop' orelse Status=='complete' orelse Status=='restart' -> Acc2; _ -> [{RecField, D}|Acc2] end end, Acc, lists:zip(record_info(fields, 'Job'), tl(tuple_to_list(D)))), ErrFields++case ErrFields of [] -> CustomValidateModule:ValidateFun(D); _ -> [] end; @@ -724,9 +724,9 @@ validate(D = #'History'{roster_id = Roster_id, feed = Feed, size = Size, entity_ validate(D = #'Schedule'{id = Id, proc = Proc, data = Data, state = State}, Acc, {CustomValidateModule, ValidateFun} = CM) -> ErrFields = lists:foldl(fun ({RecField, F}, Acc2) -> case {RecField, F} of - {id,_} when Id==[] orelse is_integer(Id) -> Acc2; - {proc,_} when Proc==[] orelse is_integer(Proc) -> Acc2; - {data,_} when is_list(Data) -> + {id,_} when Id==[] orelse is_integer(Id) orelse is_tuple(Id) -> Acc2; + {proc,_} when Proc==[] orelse is_integer(Proc) orelse is_binary(Proc) -> Acc2; + {data,_} when is_binary(Data) orelse is_list(Data) -> lists:foldl(fun(Tmp, Acc3) when true -> validate(Tmp, Acc3, CM); (Tmp, Acc3) -> [{data, D}|Acc3] end, Acc2, Data); {state,_} when State==[] orelse true -> Acc2; _ -> [{RecField, D}|Acc2] -- GitLab From e532c64eb9c4df7ee88c67fc60de523809d229fc Mon Sep 17 00:00:00 2001 From: qomputer Date: Tue, 7 May 2019 14:38:46 +0300 Subject: [PATCH 25/25] Set auth_ttl on 24h --- etc/certs/cert.pem | 20 -------------------- sys.config | 4 ++-- 2 files changed, 2 insertions(+), 22 deletions(-) delete mode 100644 etc/certs/cert.pem diff --git a/etc/certs/cert.pem b/etc/certs/cert.pem deleted file mode 100644 index 276ba49ac..000000000 --- a/etc/certs/cert.pem +++ /dev/null @@ -1,20 +0,0 @@ ------BEGIN CERTIFICATE----- -MIIDWjCCAkICCQDPVIdsxF6HMTANBgkqhkiG9w0BAQsFADBvMQswCQYDVQQGEwJV -QTENMAsGA1UECAwES3lpdjENMAsGA1UEBwwES3lpdjEhMB8GA1UECgwYSW50ZXJu -ZXQgV2lkZ2l0cyBQdHkgTHRkMR8wHQYDVQQDDBZ0cmFuc2xhdGUuY2kubnluamEu -bmV0MB4XDTE5MDExNTE0NTAyOVoXDTI5MDExMjE0NTAyOVowbzELMAkGA1UEBhMC -VUExDTALBgNVBAgMBEt5aXYxDTALBgNVBAcMBEt5aXYxITAfBgNVBAoMGEludGVy -bmV0IFdpZGdpdHMgUHR5IEx0ZDEfMB0GA1UEAwwWdHJhbnNsYXRlLmNpLm55bmph -Lm5ldDCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBANLs7ZQ4qi8oQZ51 -0QqR6Rgie8KFyauYYr1kp8VOzDEXHTm3ntbCa3qo9yoGdZZh38Nx/CeSUm5hf70d -nYE7FYFvzZpqMq+7CpnxsMiEIzL1815kPB/XGTAZzPvT0CGTjlekwBjTM/AkJV57 -+cgrbYu7KUJ+7L+UvOrodF1O43d1+uJEz0sJintpQoLCyO0MYyyiAPozifaOI5c6 -UkNz0k73tdwbnjequAFvPm0gMmJ3FvCk7Y0uEWaqR2yh9YXVaHqWMb4raJdwCQXe -vYgO8z7ptDnJYnQvPAy9M/WLFOygZjLfKJVh3//tP9z/yFGjATkft2j6BmDJnsuK -Hf9W9B8CAwEAATANBgkqhkiG9w0BAQsFAAOCAQEAotACgSqQE1M7nhOOsrhMvi5r -nRydS9td7Tx9RyHGW0NpfFZavr14J652NzbO/By97kqB6JaTjedfAVl0+OS5+9o5 -oc9gxVtJa8p6MXTfoPC6dbX2KfjD1dwVwKkKQo4JNz/etvWfEwXj9pmNVBRBac9T -Ty6WCxiN5Qzh1QJ+I3Iy4AoF6zaHI/NtTNaoB2I3bH2XBHClCVXJreHQzpvG1l9r -8WM3xNz/OMYTe7Ahe5HYHijwD8k3I2uP/yrirOfFBvj+xlrhhvA99/Z1tBN1+qGY -fROPlVQmFIiyekz/gRIsP0qOn1cNUcP/pRi2isfY5QeKqarY7vQIL/eX+IPl2w== ------END CERTIFICATE----- diff --git a/sys.config b/sys.config index d4fab7b58..25cbdab71 100644 --- a/sys.config +++ b/sys.config @@ -3,7 +3,7 @@ {bert,[{js,"apps/roster/priv/macbert/"}, {erl,"apps/roster/src"}, {google,"apps/service/priv/"}, - {disallowed,['feed', 'Whitelist', 'FakeNumbers']}, + {disallowed,['feed', 'Whitelist', 'FakeNumbers', 'Schedule','Index']}, {allowed_hrl, ["roster"]}, {custom_validate, {roster, validate}}, {swift,"apps/roster/priv/macbert/"}]}, @@ -67,7 +67,7 @@ {tables,[ cookies, file, caching, ring, async, nick, names, surnames, system]}, {pickler,nitro_pickle}, {formatter,n2o_bert}, - {auth_ttl, 900}, %% 15 mins + {auth_ttl, 86400}, %% 24 h {protocols,[n2o_nitro,n2o_ftp,roster_proto,kvs_stream]}, {log_modules,[roster, n2o_vnode]}, {log_level,roster}, -- GitLab