From c40ea24fd86ba2b3b9221d01316f959c8866cb94 Mon Sep 17 00:00:00 2001 From: qomputer Date: Fri, 13 Oct 2017 17:26:01 +0300 Subject: [PATCH 1/3] Add simple indexing for special fields such as example nick --- apps/roster/include/roster.hrl | 9 ++++--- apps/roster/src/protocol/roster_roster.erl | 28 ++++++++++++---------- apps/roster/src/protocol/roster_search.erl | 21 ++++++++++------ apps/roster/src/roster.erl | 3 ++- 4 files changed, 37 insertions(+), 24 deletions(-) diff --git a/apps/roster/include/roster.hrl b/apps/roster/include/roster.hrl index 3d8f6cde3..60c29242a 100644 --- a/apps/roster/include/roster.hrl +++ b/apps/roster/include/roster.hrl @@ -25,8 +25,8 @@ names =[] :: [] | binary(), surnames =[] :: [] | binary(), query =[] :: [] | binary(), - status =[] :: [] | contact | qrcode | nick - | names | phone | phonebook }). + status =[] :: [] | contact | qrcode }). + -record(p2p, {from =[] :: [] | binary(), to =[] :: [] | binary() }). @@ -215,6 +215,9 @@ status =[] :: [] | contact | simple | list | common_groups | ban | unban | mute | unmute - | request | confirm | revoke }). + | request | confirm | revoke }). +-record('Index', {id = [] :: [] | term(), + roster = [] :: [] | list(integer()), + tag = [] :: [] | list(integer())}). -endif. diff --git a/apps/roster/src/protocol/roster_roster.erl b/apps/roster/src/protocol/roster_roster.erl index 2f83a54c0..d4bd977e4 100644 --- a/apps/roster/src/protocol/roster_roster.erl +++ b/apps/roster/src/protocol/roster_roster.erl @@ -10,24 +10,26 @@ info(#'Roster'{id=RosterId, status=patch}=Data, Req, n2o:info(?MODULE, "~p:~p:Roster/patch:~s",[RosterId,ClientId,io_lib:format("~p",[Data])]), case kvs:get('Roster',RosterId) of {error,Reason} -> {io,{error,Reason},<<>>}; - {ok, Roster=#'Roster'{phone=Phone}} -> + {ok, Roster=#'Roster'{phone=Phone, nick=OldNick}} -> %% Data preprocessing. Validate nickname - #'Roster'{nick = Nick} = Data, + OldLnick=string:lowercase(OldNick), + Lnick=string:lowercase(Nick), n2o:info(?MODULE, "Debug. Nick ~p",[Nick]), ValidationError = case Nick of [] -> false; - Nick -> - case kvs:index('Roster', nick, Nick) of - [] -> false; - Val -> - #'Roster'{id = TakenRosterId} = lists:last(Val), - n2o:info(?MODULE, "Debug. TakenRosterId ~p",[TakenRosterId]), - case TakenRosterId of - RosterId -> false; - _ -> true - end + _ -> + case kvs:get('Index', {nick, Lnick}) of + {ok,#'Index'{roster=[TakenRosterId]}} when TakenRosterId==RosterId-> + %#'Roster'{id = TakenRosterId} = lists:last(Val), + n2o:info(?MODULE, "Debug. TakenRosterId ~p",[TakenRosterId]), + case Lnick of + OldLnick when Nick==OldNick -> true; + _ -> false + end; + {ok,#'Index'{roster=[TakenRosterId]}} -> true; + _ -> false end end, n2o:info(?MODULE, "Debug. ValidationError ~p",[ValidationError]), @@ -40,7 +42,7 @@ info(#'Roster'{id=RosterId, status=patch}=Data, Req, A = list_to_tuple([case D of[] -> R; _->D end || {R,D} <- lists:zip(tuple_to_list(Roster),tuple_to_list(Data)) ]), LastUpdated = A#'Roster'{update=roster:now_msec()}, - kvs:put(LastUpdated), + kvs:put(LastUpdated), kvs:put(#'Index'{id={nick,Lnick},roster=[RosterId]}), [NewC] = roster:to_contact([RosterId],friend), PhoneId = roster:phone_id(Phone,RosterId), emqttc:publish(C, <<"ac/", PhoneId/binary>>, term_to_binary(NewC)), diff --git a/apps/roster/src/protocol/roster_search.erl b/apps/roster/src/protocol/roster_search.erl index fe8be89f2..4803e88ed 100644 --- a/apps/roster/src/protocol/roster_search.erl +++ b/apps/roster/src/protocol/roster_search.erl @@ -6,16 +6,23 @@ start() -> n2o_async:start(#handler{module=?MODULE,class=system,group=roster,name=?MODULE,state=[]}). % @TODO: Need update this code after add new field 'nick' into protocol SEARCH for search user by nick -info(#'Search'{names = Names, surnames = Surnames, phone=[], status = Status}, Req, #cx{params = <<"emqttd_",_/binary>>} = State) -> - L = [[begin - N = binary_to_list(V) ++ F, [#'Contact'{avatar = Av, names = Na, surnames = Sn} - || #'Roster'{avatar = Av, names = Na, surnames = Sn} <- kvs:index('Roster', K, list_to_binary(N))] - end || F <- ets:match(K, {binary_to_list(V) ++ '$1'})] || {K, V}<-[{names, Names}, {surnames, Surnames}, {nick, Names}], is_binary(V)], - Data = #'Roster'{userlist=lists:flatten(L), status = search}, +%%info(#'Search'{names = Names, surnames = Surnames, phone=[], status = Status}, Req, #cx{params = <<"emqttd_",_/binary>>} = State) -> +%% L = [[begin +%% N = binary_to_list(V) ++ F, [#'Contact'{avatar = Av, names = Na, surnames = Sn} +%% || #'Roster'{avatar = Av, names = Na, surnames = Sn} <- kvs:index('Roster', K, list_to_binary(N))] +%% end || F <- ets:match(K, {binary_to_list(V) ++ '$1'})] || {K, V}<-[{names, Names}, {surnames, Surnames}, {nick, Names}], is_binary(V)], +%% Data = #'Roster'{userlist=lists:flatten(L), status = search}, +%% {reply, {bert, Data}, Req, State}; + +info(#'Search'{id=From, phone=[], query =Q, status = contact}, Req, #cx{params = <<"emqttd_",_/binary>>= ClientId} = State) -> + n2o:info(?MODULE, "~p:~p:Search/contact:~p",[From,ClientId,Q]), + L = [begin Rs=case kvs:get('Index',{K, string:lowercase(V)}) of {ok,#'Index'{roster=IDs}}-> IDs; _ ->[] end, + [ roster:to_contact([RosterId],[]) || RosterId <-Rs ] end || {K, V}<-Q], + Data = #'Roster'{id=From, userlist=lists:flatten(L), status = search}, {reply, {bert, Data}, Req, State}; info(#'Search'{id=From, phone=Phones, query = [], status=Status},Req, - #cx{params = <<"emqttd_",_/binary>> = ClientId, session= Token} =State) when is_list(Phones) -> + #cx{params = <<"emqttd_",_/binary>> = ClientId} =State) when is_list(Phones) -> n2o:info(?MODULE, "~p:~p:Search/contact:~p",[From,ClientId,Phones]), Result=[ begin case roster:parts_phone_id(Phone) of diff --git a/apps/roster/src/roster.erl b/apps/roster/src/roster.erl index 4d476059c..69bbe259c 100644 --- a/apps/roster/src/roster.erl +++ b/apps/roster/src/roster.erl @@ -42,7 +42,8 @@ tables() -> [ #table{name='Auth',fields=record_info(fields,'Auth'), keys=[dev_k #table{name=reader,fields=record_info(fields,reader), keys=[feed]}, #table{name='Room',fields=record_info(fields,'Room')}, #table{name='Member',container=chain, fields=record_info(fields,'Member')}, - #table{name='Profile',fields=record_info(fields,'Profile')} ]. + #table{name='Profile',fields=record_info(fields,'Profile')}, + #table{name='Index',fields=record_info(fields,'Index')}]. % PROFILE MANAGEMENT -- GitLab From af513c86795207b7a56a1395e3b7d594f9b5c460 Mon Sep 17 00:00:00 2001 From: qomputer Date: Tue, 17 Oct 2017 15:26:00 +0300 Subject: [PATCH 2/3] Update indexing and search --- apps/roster/include/roster.hrl | 9 ++---- apps/roster/src/protocol/roster_roster.erl | 17 +++++++---- apps/roster/src/protocol/roster_search.erl | 23 ++++++-------- apps/roster/src/roster.erl | 2 ++ apps/roster/src/roster_test.erl | 35 +++++++++++----------- 5 files changed, 44 insertions(+), 42 deletions(-) diff --git a/apps/roster/include/roster.hrl b/apps/roster/include/roster.hrl index 60c29242a..7069adc2a 100644 --- a/apps/roster/include/roster.hrl +++ b/apps/roster/include/roster.hrl @@ -22,10 +22,8 @@ -record('Search', {id =[] :: [] | integer(), phone =[] :: [] | list(binary()), - names =[] :: [] | binary(), - surnames =[] :: [] | binary(), - query =[] :: [] | binary(), - status =[] :: [] | contact | qrcode }). + query =[] :: [] | list(term()), % [{nick, _}] + status =[] :: [] | contact | qrcode | names}). -record(p2p, {from =[] :: [] | binary(), @@ -218,6 +216,5 @@ | request | confirm | revoke }). -record('Index', {id = [] :: [] | term(), - roster = [] :: [] | list(integer()), - tag = [] :: [] | list(integer())}). + roster = [] :: [] | list(integer())}). -endif. diff --git a/apps/roster/src/protocol/roster_roster.erl b/apps/roster/src/protocol/roster_roster.erl index d4bd977e4..88933cfad 100644 --- a/apps/roster/src/protocol/roster_roster.erl +++ b/apps/roster/src/protocol/roster_roster.erl @@ -9,7 +9,7 @@ info(#'Roster'{id=RosterId, status=patch}=Data, Req, #cx{params = ClientId, client_pid = C }=State) -> n2o:info(?MODULE, "~p:~p:Roster/patch:~s",[RosterId,ClientId,io_lib:format("~p",[Data])]), case kvs:get('Roster',RosterId) of - {error,Reason} -> {io,{error,Reason},<<>>}; + {error,Reason} -> {reply, {bert, {io,{error,Reason}},<<>>},Req, State}; {ok, Roster=#'Roster'{phone=Phone, nick=OldNick}} -> %% Data preprocessing. Validate nickname @@ -25,11 +25,13 @@ info(#'Roster'{id=RosterId, status=patch}=Data, Req, %#'Roster'{id = TakenRosterId} = lists:last(Val), n2o:info(?MODULE, "Debug. TakenRosterId ~p",[TakenRosterId]), case Lnick of - OldLnick when Nick==OldNick -> true; - _ -> false + OldLnick when Nick==OldNick -> true; + _ -> n2o_async:send(system,?MODULE,{update_index,#'Roster'{id=RosterId,nick=Lnick}}), + false end; {ok,#'Index'{roster=[TakenRosterId]}} -> true; - _ -> false + _ -> n2o_async:send(system,?MODULE,{update_index,#'Roster'{id=RosterId,nick=Lnick}}), + false end end, n2o:info(?MODULE, "Debug. ValidationError ~p",[ValidationError]), @@ -42,7 +44,7 @@ info(#'Roster'{id=RosterId, status=patch}=Data, Req, A = list_to_tuple([case D of[] -> R; _->D end || {R,D} <- lists:zip(tuple_to_list(Roster),tuple_to_list(Data)) ]), LastUpdated = A#'Roster'{update=roster:now_msec()}, - kvs:put(LastUpdated), kvs:put(#'Index'{id={nick,Lnick},roster=[RosterId]}), + kvs:put(LastUpdated), [NewC] = roster:to_contact([RosterId],friend), PhoneId = roster:phone_id(Phone,RosterId), emqttc:publish(C, <<"ac/", PhoneId/binary>>, term_to_binary(NewC)), @@ -102,4 +104,9 @@ proc({update_ac,#'Contact'{phone_id=PhoneId}=C}, #handler{}=H) -> roster:update_contact(FriendId, C#'Contact'{reader=R, presence=[], update=UPT}); E -> E end end || #'Contact'{phone_id=PId} <-roster:get_on_status(RosterId,friend)], + {reply, [], H}; + +proc({update_index,#'Roster'{id=RosterId, nick=Nick}}, #handler{}=H) -> + kvs:put(#'Index'{id={nick,Nick},roster=[RosterId]}), +%% @TODO: Have to be code for adding new nick into B-tree for nicks {reply, [], H}. \ No newline at end of file diff --git a/apps/roster/src/protocol/roster_search.erl b/apps/roster/src/protocol/roster_search.erl index 4803e88ed..4f1d40960 100644 --- a/apps/roster/src/protocol/roster_search.erl +++ b/apps/roster/src/protocol/roster_search.erl @@ -5,23 +5,15 @@ start() -> n2o_async:start(#handler{module=?MODULE,class=system,group=roster,name=?MODULE,state=[]}). -% @TODO: Need update this code after add new field 'nick' into protocol SEARCH for search user by nick -%%info(#'Search'{names = Names, surnames = Surnames, phone=[], status = Status}, Req, #cx{params = <<"emqttd_",_/binary>>} = State) -> -%% L = [[begin -%% N = binary_to_list(V) ++ F, [#'Contact'{avatar = Av, names = Na, surnames = Sn} -%% || #'Roster'{avatar = Av, names = Na, surnames = Sn} <- kvs:index('Roster', K, list_to_binary(N))] -%% end || F <- ets:match(K, {binary_to_list(V) ++ '$1'})] || {K, V}<-[{names, Names}, {surnames, Surnames}, {nick, Names}], is_binary(V)], -%% Data = #'Roster'{userlist=lists:flatten(L), status = search}, -%% {reply, {bert, Data}, Req, State}; - info(#'Search'{id=From, phone=[], query =Q, status = contact}, Req, #cx{params = <<"emqttd_",_/binary>>= ClientId} = State) -> n2o:info(?MODULE, "~p:~p:Search/contact:~p",[From,ClientId,Q]), L = [begin Rs=case kvs:get('Index',{K, string:lowercase(V)}) of {ok,#'Index'{roster=IDs}}-> IDs; _ ->[] end, - [ roster:to_contact([RosterId],[]) || RosterId <-Rs ] end || {K, V}<-Q], + [ roster:to_contact([RosterId],[]) || RosterId <-Rs ] end || {K, V}<-Q ], Data = #'Roster'{id=From, userlist=lists:flatten(L), status = search}, {reply, {bert, Data}, Req, State}; -info(#'Search'{id=From, phone=Phones, query = [], status=Status},Req, + +info(#'Search'{id=From, phone=Phones, query = [], status = contact},Req, #cx{params = <<"emqttd_",_/binary>> = ClientId} =State) when is_list(Phones) -> n2o:info(?MODULE, "~p:~p:Search/contact:~p",[From,ClientId,Phones]), Result=[ begin @@ -32,14 +24,17 @@ info(#'Search'{id=From, phone=Phones, query = [], status=Status},Req, _ -> roster:to_contact([FriendId], []) end; E -> [] end end || Phone<-lists:usort(Phones)], - Res = #'Roster'{id=From, userlist=lists:flatten(Result),status=Status}, + Res = #'Roster'{id=From, userlist=lists:flatten(Result),status=search}, {reply, {bert, Res}, Req, State}; +info(#'Search'{id=From, phone=[], query =Q, status = names}, Req, #cx{params = <<"emqttd_",_/binary>>= ClientId} = State) -> + n2o:info(?MODULE, "~p:~p:Search/names:~p",[From,ClientId,Q]), + %% @TODO: Need to add code for search into B-tree for all matches of registred names to Q-template + {reply, {bert, <<>>}, Req, State}; + info(#'Search'{}=Data, Req, State) -> {reply, {bert, Data}, Req, State}. proc(init,#handler{name=roster_search} = Async) -> n2o:info(?MODULE, "Search ASYNC started", []), -%% [[ets:insert(K, [{binary_to_list(V)}]) || {K, V}<-[{names, Names}, {surnames, Surnames}, {nick, Nick}], is_binary(V)] -%% || #'Roster'{names = Names, surnames = Surnames, nick = Nick} <- kvs:all('Roster')], {ok,Async}. \ No newline at end of file diff --git a/apps/roster/src/roster.erl b/apps/roster/src/roster.erl index 69bbe259c..133459555 100644 --- a/apps/roster/src/roster.erl +++ b/apps/roster/src/roster.erl @@ -146,6 +146,8 @@ remove_roster(Phone, RosterId) -> [begin [_P,Id]=parts_phone_id(PhoneId), kvs:delete(writer,roster:feed_key(p2p, PId, PhoneId)), kvs:delete(reader,R), remove_contacts(Id, [PId]), remove_member(PhoneId) end || #'Contact'{phone_id=PhoneId, reader=R}<-get_contacts(RosterId)], + {ok,#'Roster'{nick=Nick}}=kvs:get('Roster',RosterId), + kvs:delete('Index',{nick,string:lowercase(Nick)}), kvs:delete('Roster',RosterId), kvs:put(Profile#'Profile'{rosters=Roster, update=now_msec()}), case Roster of diff --git a/apps/roster/src/roster_test.erl b/apps/roster/src/roster_test.erl index b06ed2993..d11aa264d 100644 --- a/apps/roster/src/roster_test.erl +++ b/apps/roster/src/roster_test.erl @@ -38,6 +38,8 @@ receive_drop() -> receive _ -> receive_drop() after ?DRP_TMOUT -> skip end. roster_case(Term,State) -> case Term of + #'Roster'{} = Roster -> %n2o:info(?MODULE, "Roster/get:~p",[Roster]), + State#state{last_res = Roster}; #'Person'{id=_Id} = P -> %n2o:info(?MODULE, "~p", [P]), State#state{last_res = P}; #'Profile'{phone=_Phone} = P -> %n2o:info(?MODULE, "Profile/get:~p",[P]), @@ -459,23 +461,22 @@ test_loaded()-> #'Roster'{} = send_sync(ClientId,R) end || _Cont<-lists:seq(1,10)], stop_client(ClientId). -test_public_search() -> - Phone = <<"753">>, - {ClientId, Token} = reg_fake_user(Phone, <<"DevKey_", Phone/binary>>, 0), - start_client(ClientId, Token), - receive_drop(), - kvs:put(#'Roster'{id = 99999, names = <<"John">>, surnames = <<"Doe">>, avatar = <<"avatar">>, nick = <<"maximilian">>}), - ets:insert(nick, [{"maximilian"}]), - ets:insert(names, [{"John"}]), - ets:insert(surnames, [{"Doe"}]), - #'Roster'{userlist = [#'Contact'{names = <<"John">>}]} = send_sync(ClientId, #'Search'{names = <<"m">>, status = nick}), - #'Roster'{userlist = [#'Contact'{names = <<"John">>}]} = send_sync(ClientId, #'Search'{names = <<"ma">>, status = nick}), - #'Roster'{userlist = [#'Contact'{names = <<"John">>}]} = send_sync(ClientId, #'Search'{names = <<"max">>, status = nick}), - #'Roster'{userlist = [#'Contact'{names = <<"John">>}]} = send_sync(ClientId, #'Search'{names = <<"J">>, status = names}), - #'Roster'{userlist = [#'Contact'{names = <<"John">>}]} = send_sync(ClientId, #'Search'{names = <<"Jo">>, status = names}), - #'Roster'{userlist = [#'Contact'{names = <<"John">>}]} = send_sync(ClientId, #'Search'{surnames = <<"D">>, status = names}), - #'Roster'{userlist = [#'Contact'{names = <<"John">>}]} = send_sync(ClientId, #'Search'{surnames = <<"Do">>, status = names}), - stop_client(ClientId). +test_search() -> + Phones= [_PhoneA,PhoneB] = [<<"717">>,<<"313">>], + L=[{FromId, AClientId}, {ToId, BClientId}] = [begin + roster:purge_user(Phone), + {ClientId, Token} = reg_fake_user(Phone), + start_client(ClientId , Token), + receive_test(ClientId, #'Auth'{}, 0), + [Roster] = list_rosters(ClientId, Phone), + {roster:roster_id(Roster), ClientId} + end || Phone<-Phones], + %receive_drop(), timer:sleep(1000), + #'Roster'{} = send_sync(AClientId,#'Roster'{id=FromId, nick= <<"Test">>, status = patch}), + #'Roster'{} = send_sync(BClientId,#'Roster'{id=ToId, nick= <<"Maximilian">>, status = patch}), + #'Roster'{userlist=[#'Contact'{}]} = send_sync(AClientId, #'Search'{id=FromId, query=[{nick, <<"maxiMilian">>}], status = contact}), + #'Roster'{userlist=[]} = send_sync(BClientId, #'Search'{id=ToId, query= [{nick,<<"Tes">>}], status = contact}), + [stop_client(ClientId) || {_, ClientId}<-L]. client_test() -> Phone = <<"753">>, -- GitLab From e2fcf3ad25f40fee80c3305d0e4373926a0083dc Mon Sep 17 00:00:00 2001 From: qomputer Date: Tue, 17 Oct 2017 15:52:50 +0300 Subject: [PATCH 3/3] Fix test_search --- apps/roster/src/roster_test.erl | 1 - 1 file changed, 1 deletion(-) diff --git a/apps/roster/src/roster_test.erl b/apps/roster/src/roster_test.erl index d11aa264d..3529cd5f0 100644 --- a/apps/roster/src/roster_test.erl +++ b/apps/roster/src/roster_test.erl @@ -471,7 +471,6 @@ test_search() -> [Roster] = list_rosters(ClientId, Phone), {roster:roster_id(Roster), ClientId} end || Phone<-Phones], - %receive_drop(), timer:sleep(1000), #'Roster'{} = send_sync(AClientId,#'Roster'{id=FromId, nick= <<"Test">>, status = patch}), #'Roster'{} = send_sync(BClientId,#'Roster'{id=ToId, nick= <<"Maximilian">>, status = patch}), #'Roster'{userlist=[#'Contact'{}]} = send_sync(AClientId, #'Search'{id=FromId, query=[{nick, <<"maxiMilian">>}], status = contact}), -- GitLab