From e2cdc628342f79bb17df98214ca2057c4ca2e3e2 Mon Sep 17 00:00:00 2001 From: gspasov Date: Wed, 8 May 2019 10:46:29 +0300 Subject: [PATCH] Merging with master branch --- apps/roster/include/roster.hrl | 17 +- apps/roster/src/api/push/ios.erl | 4 +- apps/roster/src/protocol/roster_acl.erl | 2 +- apps/roster/src/protocol/roster_auth.erl | 14 +- apps/roster/src/protocol/roster_bpe.erl | 19 +- apps/roster/src/protocol/roster_history.erl | 44 +-- apps/roster/src/protocol/roster_message.erl | 10 +- apps/roster/src/protocol/roster_presence.erl | 6 +- apps/roster/src/protocol/roster_profile.erl | 2 +- apps/roster/src/rest/rest_handler.erl | 2 +- apps/roster/src/rest/rest_metric.erl | 2 +- apps/roster/src/roster.erl | 77 ++--- apps/roster/src/roster_channel_helper.erl | 278 +++++++++++++++++++ apps/roster/src/roster_db.erl | 16 ++ apps/roster/src/roster_validator.erl | 12 +- apps/roster/src/test/roster_test.erl | 27 +- etc/certs/cert.pem | 20 -- sys.config | 4 +- 18 files changed, 420 insertions(+), 136 deletions(-) create mode 100644 apps/roster/src/roster_channel_helper.erl delete mode 100644 etc/certs/cert.pem diff --git a/apps/roster/include/roster.hrl b/apps/roster/include/roster.hrl index cdf1c5126..b0d5fe7b5 100644 --- a/apps/roster/include/roster.hrl +++ b/apps/roster/include/roster.hrl @@ -47,7 +47,6 @@ -define(GEO_URL , "http://geolite.maxmind.com/download/geoip/database/GeoLite2-City.tar.gz"). -define(AUTH_GROUP , <<"AUTH_DATA">>). - -record(chain, {?CONTAINER, aclver = [], unread = {[],[]}}). -record(push, {model = [] :: [] | term(), @@ -154,7 +153,7 @@ members = [] :: list(#'Member'{}), admins = [] :: list(#'Member'{}), data = [] :: list(#'Desc'{}), - type = [] :: [] | group | channel, + type = [] :: [] | group | channel | call, tos = [] :: [] | binary(), tos_update = 0 :: [] | integer(), unread = 0 :: [] | integer(), @@ -249,6 +248,7 @@ -record(act, {name= <<"publish">> :: [] | binary(), data=[]:: binary() | integer() | list(term())}). + -record('Job', {id = [] :: [] | integer(), container = chain :: chain | [], feed_id = [] :: #act{}, @@ -257,10 +257,10 @@ context = [] :: [] | integer() | binary(), proc = [] :: [] | integer() | #process{}, time = [] :: [] | integer(), - data = [] :: list(#'Message'{}), + data = [] :: [] | binary() | list(#'Message'{}), events = [] :: [] | list(#messageEvent{}), settings = [] :: [] | list(#'Feature'{}), - status = [] :: [] | init | update | delete | pending | stop | complete}). + status = [] :: [] | init | update | delete | pending | stop | complete | restart}). -record('History', {roster_id = [] :: binary(), feed = [] :: #p2p{} | #muc{} | #act{} | #'StickerPack'{} | [], @@ -270,11 +270,10 @@ status = [] :: updated | get | update | last_loaded | last_msg | get_reply | double_get | delete | image | video | file | link | audio | contact | location | text}). - --record('Schedule', {id = [] :: [] | integer(), - proc = [] :: [] | integer(), - data = [] :: list(term()), - state =[] :: [] | term()}). +-record('Schedule', {id = [] :: [] | integer() | {integer(), {integer(),integer(),integer()}}, + proc = [] :: [] | integer() | binary(), + data = [] :: binary() | list(term()), + state =[] :: [] | term()}). %% Index.id - term for indexing in format {keyword_atom, value_binary} %% Index.roster - entity id, list of Parent elements' id. Unique for Nick and Link indexing. diff --git a/apps/roster/src/api/push/ios.erl b/apps/roster/src/api/push/ios.erl index ef79bee09..b6197e2b7 100644 --- a/apps/roster/src/api/push/ios.erl +++ b/apps/roster/src/api/push/ios.erl @@ -30,11 +30,11 @@ notify(A, C, T, DeviceId, SessionSettings) -> %% create aps json Aps = jsx:encode([{<<"model">>, Custom}, {<<"type">>, Type}, {<<"title">>, Alert}, - {<<"dns">>, get_data_from_feature(SessionSettings, ?FKPN_SERVER_DNS)}, {<<"version">>, ?VERSION}]), + {<<"dns">>, get_data_from_feature(SessionSettings, ?FKPN_SERVER_DNS)}, {<<"version">>, <>}]), %% create ios payload string PayloadString = binary_to_list(iolist_to_binary(["{\"aps\": {\"nynja\": ", Aps, "}}"])), - roster:info(?MODULE, "PayloadString ~p~n~n", [PayloadString]), +% roster:info(roster, "PayloadString ~p~n~n", [PayloadString]), %% prepare push data Payload = list_to_binary(PayloadString), diff --git a/apps/roster/src/protocol/roster_acl.erl b/apps/roster/src/protocol/roster_acl.erl index 570bea6ef..b47d1398a 100644 --- a/apps/roster/src/protocol/roster_acl.erl +++ b/apps/roster/src/protocol/roster_acl.erl @@ -72,7 +72,7 @@ check_acl({#mqtt_client{client_id = <<"emqttd_", _/binary>> = Client}, publish, allow; check_acl({#mqtt_client{client_id = <<"emqttd_", _/binary>>}, publish, <<"auth/", _/binary>>}, #state{}) -> deny; -check_acl({#mqtt_client{client_id = <<"emqttd_", _/binary>> = Client}, publish, <<"events/", _/binary>>} = Who, #state{}) -> +check_acl({#mqtt_client{client_id = <<"emqttd_", _/binary>> = Client}, publish, Topic} = Who, #state{}) -> case kvs:get('Auth', Client) of {ok, #'Auth'{type = expired}} -> deny; {ok, _} -> allow; diff --git a/apps/roster/src/protocol/roster_auth.erl b/apps/roster/src/protocol/roster_auth.erl index 615219742..6483a9498 100644 --- a/apps/roster/src/protocol/roster_auth.erl +++ b/apps/roster/src/protocol/roster_auth.erl @@ -36,7 +36,7 @@ check(#mqtt_client{client_id = <<"reg_", _/binary>> = ClientId, check(#mqtt_client{client_id = <<"emqttd_", _/binary>> = ClientId, username = <<"api">>, client_pid = ClientPid, will_topic = WT, peername = {Ip, _}, ws_initial_headers = Headers} = MC, Token, State) -> roster:info(?MODULE, "[WS Headers]: ~p", [Headers]), - roster:info(?MODULE, "~p:Auth:auth/check:~p~p", [ClientId, lists:sublist(binary_to_list(Token), 16),WT]), + roster:info(?MODULE, "~p:Auth:auth/check:~p~n", [ClientId,WT]), case case State of #'Auth'{} -> {ok, State}; _ -> kvs:get('Auth', ClientId) end of {ok, #'Auth'{type = Type, token = Token, settings = Settings, last_online = LO} = Auth} when Type == verified; Type == []; Type == expired -> @@ -50,10 +50,14 @@ check(#mqtt_client{client_id = <<"emqttd_", _/binary>> = ClientId, username = << LastOnline = case Type of verified -> Now; _ -> LO end, Auth2 = Auth#'Auth'{type = update, settings = Settings2, created = Now, last_online = LastOnline}, case {roster:parse_token(Token), Type} of + {bad_token,_} -> Auth3 = Auth2#'Auth'{type = expired, token = element(2,roster:gen_token([],[])) }, + roster:info(?MODULE, "Auth:auth/roamToken:~p", [Auth3]), + kvs:put(Auth3), + AuthPid ! Auth3#'Auth'{ type = update }, ok; {_, expired} -> AuthPid ! Auth2, ok; {{error, token_expired}, _} -> kvs:put(Auth2#'Auth'{type = expired}), AuthPid ! Auth2, ok; - {{error, _}, _} = E -> E; + {{error, _}=E, _} -> E; {_, _} -> OldIpBin = case FindIp = lists:keyfind(?IP_KEY, #'Feature'.key, Settings) of false -> IpBin; #'Feature'{value = V} -> V end, @@ -180,7 +184,7 @@ info(#'Auth'{type = verify, phone = Phone, dev_key = DevKey, settings = Settings || #'Feature'{key = Key} = F <-Settings], kvs:put(P = #'Profile'{phone = Phone, rosters = [Roster], settings = NewSettings, update = roster:now_msec()}), - roster:n2o_pid(?MODULE) ! {vox, P, client(RegClientId)}, +% roster:n2o_pid(?MODULE) ! {vox, P, client(RegClientId)}, {roster, Roster} end, UserId = roster:phone_id(Phone, RosterId), ClientId = client(RegClientId), [kvs:delete('Auth', ClId) || #'Auth'{client_id = ClId} <- kvs:index('Auth', dev_key, DevKey)], @@ -318,12 +322,12 @@ proc({sms, Phone, SmsCode}, #handler{} = H) -> Half = size(SmsCode) - round(size(SmsCode) / 2), [Codes1, Codes2] = [binary:part(SmsCode, S, Half) || S <- [0, Half]], ClientSms = <<"Welcome to Nynja! Your verification code: ", Codes1/binary, " ", Codes2/binary>>, - telesign_api:send_sms(Phone, ClientSms, fun(_) -> ok end), + spawn(fun () -> telesign_api:send_sms(Phone, ClientSms, fun(_) -> ok end) end), {reply, [], H}; proc({voice, Phone, SmsCode, Lang}, #handler{} = H) -> roster:info(?MODULE, "~p:telesign:~p:~p", [Phone, SmsCode, Lang]), - telesign_api:telesign_voice_call(Phone, SmsCode, [], Lang), +% telesign_api:telesign_voice_call(Phone, SmsCode, [], Lang), {reply, [], H}; proc({mqttc, C, connected}, State = #handler{state = C, seq = S}) -> {ok, State#handler{seq = S + 1}}; diff --git a/apps/roster/src/protocol/roster_bpe.erl b/apps/roster/src/protocol/roster_bpe.erl index 18819011c..6917bcd1d 100644 --- a/apps/roster/src/protocol/roster_bpe.erl +++ b/apps/roster/src/protocol/roster_bpe.erl @@ -149,7 +149,7 @@ info(#'Job'{id = Id, time = T0, feed_id = {act, <<"publish">>, PhoneId} = Feed, {reply, {bert, Replay}, R, S}; -info(#'Job'{id = Id, proc = [Proc], data = <<"stop">>, status = update} = M, R, S) -> +info(#'Job'{id = Id, proc = Proc, data = <<"stop">>, status = update} = M, R, S) -> roster:info(?MODULE, "STOP:~w", [M]), Docs = #'act'{data = <<"stop">>}, {reply, {bert, {io, bpe:amend(Proc, Docs, noflow), <<>>}}, R, S}; @@ -182,18 +182,6 @@ info(#'Job'{id = Id, feed_id = {act, <<"publish">>, UID} = Feed, status = delete %timer:apply_after(Time, bpe, cleanup, [Id]), {reply, {bert, D}, R, S}; -info(#'Job'{id = Proc, status = history} = M, R, S) -> - roster:info(?MODULE, "history:~w", [M]), - {reply, {bert, {io, bpe:hist(Proc), <<>>}}, R, S}; - -info(#'Job'{id = Proc, status = proc} = M, R, S) -> - roster:info(?MODULE, "proc:~w", [M]), - {reply, {bert, {io, bpe:process(Proc), <<>>}}, R, S}; - -info(#'Job'{id = Proc, status = complete} = M, R, S) -> - roster:info(?MODULE, "complete:~w", [M]), - {reply, {bert, {io, bpe:complete(Proc), <<>>}}, R, S}; - info(#'Job'{feed_id = Feed} = RequestData, R, S) -> roster:info(?MODULE, "~p::Job/unknown", [Feed]), {reply, {bert, roster_channel_helper:error_response_400(RequestData) @@ -229,9 +217,9 @@ proc(init, #handler{name = roster_bpe} = Async) -> _ -> {ok, Id0} = bpe:start(job:def(#'Schedule'{id = {0, {0, 0, 1}}, proc = <<"publish">>}), []), bpe:complete(Id0), Id0 end, - % bpe:amend(Proc,{send_pid, C},noflow), + % bpe:amend(Proc,{send_pid, C},noflow), roster:info(?MODULE, "ASYNC BPE started: ~p; ~p", [C, Proc]), - % Proc is transfered throuw state + % Proc is transfered through state {ok, Async#handler{state = {C, Proc}, seq = 0}}; proc({update, #'Job'{id = Id0, time = Time, feed_id = {act, <<"publish">>, PhoneId} = Feed, data = D} = J, {OldTime, Limit}, @@ -290,7 +278,6 @@ proc({update, #'Job'{id = Id0, time = Time, feed_id = {act, <<"publish">>, Phone end end; - proc({update, #'Job'{id = Id0, time = Time, feed_id = {act, <<"publish">>, PhoneId} = Feed, data = D} = J, {OldTime, Limit}, #cx{params = ClientId} = S}, #handler{state = {C, Proc}} = H) -> roster:info(?MODULE, "BPE update limit ~p; ~p", [C, Proc]), diff --git a/apps/roster/src/protocol/roster_history.erl b/apps/roster/src/protocol/roster_history.erl index 8afdf1bf4..46cd49d24 100644 --- a/apps/roster/src/protocol/roster_history.erl +++ b/apps/roster/src/protocol/roster_history.erl @@ -15,7 +15,8 @@ info(#'History'{roster_id = RosterId, feed = #'StickerPack'{} = Feed, size = N, #cx{params = ClientId} = State) -> roster:info(?MODULE, "~p:~p:History/get:~p", [RosterId, ClientId, Feed]), Stickers = case kvs:get('Roster', roster:roster_id(RosterId)) of - {ok, #'Roster'{}} -> lists:sublist(kvs:all('StickerPack'), N); %%TODO Temporary all sticker packs + {ok, #'Roster'{}} -> S = kvs:all('StickerPack'), + case N of [] -> S; _ -> lists:sublist(S, N) end; %%TODO Temporary all sticker packs {error, _} -> [] end, {reply, {bert, Data#'History'{data = Stickers}}, Req, State}; @@ -109,21 +110,29 @@ info(#'History'{status = get, roster_id = Roster0, feed = Feed, size = N, entity {Msgs, _} = roster:fold(FilterFun, {[], roster:start_reader(Reader)}, 'Message', StartId, FId, #kvs{mod = store_mnesia}, Iter, StopFun), - Msgs2 = case is_integer(MId) of + Msgs2 = + case is_integer(MId) of true when MId > 0, Mime2 == [] -> - {ok, #'Message'{} = EntityMsg} = kvs:get('Message', MId), - lists:reverse(lists:ukeymerge(#'Message'.id, lists:reverse(Msgs), - [EntityMsg#'Message'{files = []}])); - _ -> Msgs - end, - ResponseMsgs = case MId of - 0 -> -%% nullify Message.next for last message in query - LastMsgObject = lists:last(Msgs2), - lists:sublist(Msgs2, length(Msgs2) -1) ++ [LastMsgObject#'Message'{next = []}]; - _ -> Msgs2 end, - CheckedResponseMsgs = [roster:check_message(Msg) || Msg <- ResponseMsgs], - History#'History'{data = ResponseMsgs, size = length(ResponseMsgs), status = InitialStatus} + case kvs:get('Message', MId) of + {ok, #'Message'{} = EntityMsg} -> + lists:reverse(lists:ukeymerge(#'Message'.id, lists:reverse(Msgs), + [EntityMsg#'Message'{files = []}])); + {error, _} -> + roster:info(?MODULE, "message ~p not found", [MId]), + #error{code = invalid_data} + end; + _ -> Msgs + end, +%% ResponseMsgs = case MId of %%TODO remove this after testing +%% 0 -> +%%%% nullify Message.next for last message in query +%%%% [LastMsgObject, ] = lists:reverse(Msgs2), +%% LastMsgObject = lists:last(Msgs2), +%% lists:sublist(Msgs2, length(Msgs2) -1) ++ [LastMsgObject#'Message'{next = []}]; +%% _ -> Msgs2 end, +%% CheckedResponseMsgs = [roster:check_message(Msg) || Msg <- ResponseMsgs], + case Msgs2 of #error{} = E -> #io{code = E}; _ -> + History#'History'{data = Msgs2, size = length(Msgs2), status = InitialStatus} end end, {reply, {bert, IO}, Req, State}; @@ -194,12 +203,13 @@ info(#'History'{feed = Feed, status = delete}, Req, #cx{client_pid = C, params = roster:info(?MODULE, "~p:History/delete:~p", [PhoneId, Feed]), IO = case clean_history(Feed, PhoneId) of #error{} = Res -> #io{code = Res}; - {_Unread, Internal} -> + {_Unread, #'Message'{}=Internal} -> %% D=case is_integer(Unread) of true -> roster:send_ses(C, From, roster:update_field(FList,Record,unread,Unread)), <<>>; false -> Unread end, roster:send_feed(C, Feed, roster:readmsgs(Internal#'Message'{next = []}, PhoneId)), %% send push if no history clean error occured n2o_async:pid(system, ?MODULE) ! {send_push, PhoneId, Feed, ?HISTORY_DELETE_ACTION}, - <<>> end, + <<>>; + {I,_} -> I end, {reply, {bert, IO}, Req, State}; info(#'History'{} = Data, Req, State) -> diff --git a/apps/roster/src/protocol/roster_message.erl b/apps/roster/src/protocol/roster_message.erl index ce20676f8..c17c5646d 100644 --- a/apps/roster/src/protocol/roster_message.erl +++ b/apps/roster/src/protocol/roster_message.erl @@ -28,7 +28,7 @@ info(#'Message'{type = T} = M, Req, #cx{state=[]}=State)-> info(M#'Message'{type = T}, Req, State#cx{state=ack}); info(#'Message'{feed_id = #muc{name = To}, to = []} = RequestData, Req, State) when To /= []-> info(RequestData#'Message'{to = To}, Req, #cx{state=ack}=State); -info(#'Message'{status = [], id = [], feed_id = F, from=From0, to = To, created = [], +info(#'Message'{status = [], id = [], feed_id = F, from=From0, to = To, type = Type, files = [#'Desc'{payload = Payload} | _] = Descs} = Msg, Req, #cx{client_pid = C, params = ClientId, state=ack} = State) -> MSG_LTNCY = os:system_time(), @@ -74,7 +74,7 @@ info(#'Message'{status = [], id = [], feed_id = F, from=From0, to = To, created %% create message amd save to db Writer = #writer{cache = #'Message'{id = MsgId} = M} = kvs_stream:save(kvs_stream:add(PrevWriter2#writer{args = Msg#'Message'{container = chain, - feed_id = Feed, created = case Created of [] -> roster:now_msec(); _ -> Created end}})), + feed_id = Feed, created = roster:now_msec()}})), case Feed of #muc{name = Room} -> {ok, Room2} = kvs:get('Room', Room), @@ -95,7 +95,7 @@ info(#'Message'{status = [], id = [], feed_id = F, from=From0, to = To, created kvs:put(LnkRes2), LnkRes2; _ -> LnkRes end, roster:send_feed(C, F, Msg2 = M#'Message'{link = LnkRes3}), - roster:info(?MODULE, "~p:~p:Message/new:~P", [From, To, Payload, 100]), + roster:info(?MODULE, "~p:Message/new:~p", [From, To]), %% have to skip push notifications for call bubbles case [lists:keyfind(BubbleContentType, #'Desc'.mime, Descs) || BubbleContentType <- [?CONTENT_TYPE_VIDEOCALL, ?CONTENT_TYPE_AUDIOCALL]] of [false, false] -> n2o_async:pid(system, ?MODULE) ! {send_push, From, To, Msg2, []}; @@ -112,7 +112,7 @@ info(#'Message'{status = edit, id = Id, msg_id = ClMID, feed_id = Feed, from = F #cx{params = ClientId, client_pid = C, state=ack} = State) -> MSG_LTNCY = os:system_time(), PhoneId = roster:phone_id(ClientId, From), - roster:info(?MODULE, "~p:~p:Message/edit:~P", [PhoneId, To, Payload, 100]), + roster:info(?MODULE, "~p:Message/edit:~p", [PhoneId, To]), DV = length([D || D = #'Desc'{id = ID} <- Descs, is_binary(ID), ID /= <<>>]) == length(Descs), Data = case kvs:get('Message', Id) of @@ -250,7 +250,7 @@ info(#'Message'{status = update, id = Id, feed_id = Feed, from = From, to = To, #cx{params = ClientId, client_pid = C, state=ack} = State) when is_integer(Id) -> MSG_LTNCY = os:system_time(), PhoneId = case ClientId of <<"sys_bpe">> -> From; <<"emqttd_", _/binary>> -> roster:phone_id(ClientId) end, - roster:info(?MODULE, "~p:~p:Message/update:~P", [PhoneId, To, Payload, 100]), + roster:info(?MODULE, "~p:Message/update:~p", [PhoneId, To]), Lang = roster:get_data_val(?LANG_KEY, Data), IO = case kvs:get('Message', Id) of {ok, #'Message'{feed_id = Feed, files = Descs} = Msg} -> diff --git a/apps/roster/src/protocol/roster_presence.erl b/apps/roster/src/protocol/roster_presence.erl index d6dce1f37..df9f1847f 100644 --- a/apps/roster/src/protocol/roster_presence.erl +++ b/apps/roster/src/protocol/roster_presence.erl @@ -28,7 +28,7 @@ on_connect(Phone, ClientId, C, Ver) -> roster:send_cache(ClientId, C) end catch Err:Rea -> - n2o:error(?MODULE, "Catch:~p~n", [n2o:stack_trace(Err, Rea)]) + roster:info(?MODULE, "Catch:~p", [n2o:stack_trace(Err, Rea)]) end. on_disconnect(#'Auth'{type = logout, phone = Phone, client_id = ClientId, user_id = PhoneId}, C) -> @@ -58,8 +58,8 @@ send_presence(Status, Phone, C, ClientId) when Status == online; Status == offli presence = Status, update = Now, status = internal}) || Send <- [send_muc, send_ac]]; #error{} -> skip end || AccId <- Rosters], - P2; - _ -> P end; + P2; + _ -> roster:info(?MODULE, "~p:~p:~p",[Phone, ClientId,P]), P end; _ -> {error, profile_not_found} end. diff --git a/apps/roster/src/protocol/roster_profile.erl b/apps/roster/src/protocol/roster_profile.erl index 4cad2101d..5b63acf1b 100644 --- a/apps/roster/src/protocol/roster_profile.erl +++ b/apps/roster/src/protocol/roster_profile.erl @@ -133,7 +133,7 @@ del_roster(UID,RosterId) -> roster:update_chains({'Message',from},{#'Message'.from,PId},{#'Message'.files,[#'Desc'{payload = "Deleted"}]}), kvs:delete('Index', {nick, string:lowercase(Nick)}), case roster:is_online(user_id,PId) of offline -> roster:delete_sessions(user_id, PId); - online -> roster:force_logout(user_id,PId) end, + online -> roster:force_logout(user_id,PId) end, % TODO Add force_logout/2 kvs:put(#'Roster'{id=RosterId, phone=UID, nick= <<"Deleted user">>, status=del}), RosterId; {error, _} -> [] diff --git a/apps/roster/src/rest/rest_handler.erl b/apps/roster/src/rest/rest_handler.erl index a985c53ef..80a6e4214 100644 --- a/apps/roster/src/rest/rest_handler.erl +++ b/apps/roster/src/rest/rest_handler.erl @@ -51,7 +51,7 @@ c_tpl([File | Files], Opts) -> %% ------------------------------------------------------------------ handle_request(Req) -> - "/" ++ Path = Req:get(path), + Path= case Req:get(path) of "/" ++ P -> P; _ ->"/" end, %% NOTE extra check for "favicon.ico" file on rest requests without html body case Path of "favicon.ico" -> Req:serve_file("assets/img/favicon.ico", docroot()); diff --git a/apps/roster/src/rest/rest_metric.erl b/apps/roster/src/rest/rest_metric.erl index efd80c294..96058662c 100644 --- a/apps/roster/src/rest/rest_metric.erl +++ b/apps/roster/src/rest/rest_metric.erl @@ -11,7 +11,7 @@ handle_request('GET', _, Req) -> prometheus_gauge:set(?METRIC_CHATS_TOTAL, [?METRIC_LABEL_P2P_CHAT], roster_db:p2p_stats()), prometheus_gauge:set(?METRIC_CHATS_TOTAL, [?METRIC_LABEL_GROUP_CHAT], roster_db:room_stats()), [prometheus_gauge:set(?METRIC_USERS_PER_COUNTRY, [Country], Value) || {Country, Value} <- roster_db:country_user_stats()], - [prometheus_gauge:set(?METRIC_USERS_PER_COUNTRY_ONLINE, [Country], Value) || {Country, Value} <- roster_db:country_user_stats(online)], +% [prometheus_gauge:set(?METRIC_USERS_PER_COUNTRY_ONLINE, [Country], Value) || {Country, Value} <- roster_db:country_user_stats(online)], [prometheus_gauge:set(?METRIC_MSGS_BY_TYPE_NMBR, [Mime], Value) || {Mime, Value} <- roster_db:msg_stats()], [prometheus_gauge:set(?METRIC_SESSIONS_PER_COUNTRY_ONLINE, [Country], Value) || {Country, Value} <- roster_db:country_session_stats(online)], rest_response_helper:response(Req, 200, prometheus_text_format:format(), "text/plain"); diff --git a/apps/roster/src/roster.erl b/apps/roster/src/roster.erl index 4556e3d08..126ed6acd 100644 --- a/apps/roster/src/roster.erl +++ b/apps/roster/src/roster.erl @@ -51,6 +51,7 @@ start(_, _) -> atoms(), catch load([]), application:stop(n2o), application:start(n2o), X = try begin A = supervisor:start_link({local, roster}, roster, []), + emqttd_access_control:register_mod(auth, n2o_auth, [[]], 9997), emqttd_access_control:register_mod(auth, roster_auth, [[]], 9998), emqttd_access_control:register_mod(auth, micro_auth, [[]], 9996), register_acl_mod(), @@ -151,10 +152,14 @@ list_rosters(Phone, Fun) -> {ok, #'Profile'{rosters = Rosters}} -> ?MODULE:Fun(Phone, Rosters) end. -send_profile(#'Profile'{rosters = []}, _N, _LastSync, _ClientId, _C) -> ok; +send_profile(#'Profile'{rosters = []}, _N, _LastSync, _ClientId, _C) -> + roster:info(?MODULE, "SEND PROFILE []", []), + ok; send_profile(#'Profile'{rosters = [RosterId|TRosterIds], phone = Phone, status = Status} = P, N, LastSync, ClientId, C) -> + roster:info(?MODULE, "SEND PROFILE [~p]", [length([RosterId|TRosterIds])]), case kvs:get('Roster', RosterId) of - {error, _} -> roster:error(?MODULE, "roster ~p not found in profile ~p", [RosterId, Phone]); + {error, _} -> + roster:info(?MODULE, "roster ~p not found in profile ~p", [RosterId, Phone]); {ok, #'Roster'{} = R} -> Rosters = split_roster(R, N, LastSync, []), [roster:send_action(C, ClientId, P#'Profile'{rosters = [Roster]}) || Roster<-Rosters], @@ -162,9 +167,10 @@ send_profile(#'Profile'{rosters = [RosterId|TRosterIds], phone = Phone, status = end. split_roster(#'Roster'{} = R, N, LastSync, Rosters) -> - case roster:roster(R, N, LastSync) of + case roster(R, N, LastSync) of #'Roster'{userlist = {[], _} , roomlist = {[], _} , favorite = {[], _}} -> Rosters; - #'Roster'{userlist = {Users, TUsers}, roomlist = {Rooms, TRooms}, favorite = {Stars, TStars}} = Roster -> + #'Roster'{userlist = {Users, TUsers}, roomlist = {Rooms, TRooms}, favorite = {Stars, TStars}} = Roster + when is_list(Users), is_list(Rooms), is_list(Stars) -> split_roster(R#'Roster'{userlist = TUsers, roomlist = TRooms, favorite = TStars}, N, LastSync, Rosters++[Roster#'Roster'{userlist = Users, roomlist = Rooms, favorite = Stars}]) end. @@ -481,7 +487,7 @@ send_event(C, ClientId, Token, Term, VNode) -> n2o_vnode:send(C, event_topic(ClientId, Token, VNode), term_to_binary(Term)). send_action(C, ClientId, Term) -> send_action(C, ClientId, Term, "1"). send_action(C, ClientId, Term, VSN) -> - n2o_vnode:send(C, action_topic(ClientId, VSN), term_to_binary(Term)). + send(C, action_topic(ClientId, VSN), Term). % ROSTER TOPICS @@ -707,7 +713,8 @@ feed_key(#p2p{from = From, to = To}) -> feed_key(p2p, From, To); feed_key(#muc{} = Feed) -> Feed. ttl() -> application:get_env(roster, auth_ttl, 60 * 15). -depicle(Token) -> case n2o_secret:depickle(Token) of <<>> -> <<>>; T -> binary_to_term(T,[safe]) end. +depicle(Token) -> case n2o_secret:depickle(Token) of <<>> -> <<>>; T -> + try binary_to_term(T,[safe]) catch _:_ -> {error,bad_token} end end. gen_token([], Data) -> {'Token', n2o_secret:pickle(term_to_binary({now_msec() + ttl() * 1000, Data}))}; gen_token(Token, Data) -> @@ -1010,7 +1017,10 @@ feed(#'Contact'{phone_id = P} , PhoneId) -> feed_key(#p2p{from = feed(#'Star'{message = #'Message'{feed_id = Feed}}, _) -> Feed. last_upd(#'Room'{id = Room}) -> - {ok, #'Room'{update = Update}} = kvs:get('Room', Room), Update; %% get update field from Room table! + case kvs:get('Room', Room) of + {ok, #'Room'{update = Update}} -> Update; + {error, _} -> 0 + end; %% get update field from Room table! last_upd(#'Contact'{phone_id = PhoneId, update = Update}) -> case roster:is_online2(phone(PhoneId)) of {online, _} -> []; _-> Update end; last_upd(#'Star'{message = #'Message'{created = Update}}) -> Update. @@ -1042,8 +1052,11 @@ split_objlist([], _Roster, _LastSync, _N, Acc) -> Acc; split_objlist([_|_] = List, #'Roster'{}=Roster, LastSync, N, Acc) -> {Objs, TObjs} = objlist(List, Roster, LastSync, N), split_objlist(TObjs, Roster, LastSync, N, Acc++[Objs]); -split_objlist(Index, Id, LastSync, N, Acc) -> - case kvs:get('Roster', Id) of {ok, R} -> split_objlist(Index, R, LastSync, N, Acc); _ -> {error, roster_not_found} end. +split_objlist(Index, Id, LastSync, N, Acc) + when Index == #'Roster'.userlist; Index == #'Roster'.roomlist; Index == #'Roster'.favorite -> + case kvs:get('Roster', Id) of + {ok, R} -> split_objlist(Index, R, LastSync, N, Acc); + _ -> {error, roster_not_found} end. objlist(Index, Roster) -> objlist(Index, Roster, 0). @@ -1062,8 +1075,13 @@ objlist([Obj|_] = List, #'Roster'{id = Id, phone = Phone}=Roster, LastSync, N) #'Star'{} -> star(Roster, Obj, W, LastSync) end || {Obj, W} <- SubList], {Objs, TList}; -objlist(Index, Id, LastSync, N) -> - case kvs:get('Roster', Id) of {ok, R} -> objlist(Index, R, LastSync, N); _ -> {error, roster_not_found} end. +objlist([Obj|TObjs], #'Roster'{id = Id}=Roster, LastSync, N) -> +% roster:info(?MODULE, "invalid object in roster ~p: ~p", [Id, Obj]), + objlist(TObjs, #'Roster'{}=Roster, LastSync, N); +objlist(Index, Id, LastSync, N) when Index == #'Roster'.userlist; Index == #'Roster'.roomlist; Index == #'Roster'.favorite -> + case kvs:get('Roster', Id) of + {ok, R} -> objlist(Index, R, LastSync, N); + _ -> {error, roster_not_found} end. userlist(R) -> hd(split_objlist(#'Roster'.userlist, R)). @@ -1950,22 +1968,6 @@ check(Module) -> case lists:all(fun(X) -> X == ok end, Module:suite()) of true -> roster:info("ALL TESTS PASSED", []), true; false -> roster:info("TEST ERRORS", []), false end. -%% tables storage -save_db(Path) -> - Data = lists:append([kvs:all(B) || B <- [Name || {table,Name} <- kvs:dir()-- - [{table,schema},{table,mqtt_session},{table,mqtt_admin}]]]), - kvs:save(Path, Data). - -load_db(Path) -> - kvs:add_seq_ids(), - AllEntries= try kvs:load(Path) catch _:_ -> [] end, - [begin case element(1,E) of - mqtt_trie=Tab -> write_trie(Tab,setelement(1,E,trie)); - mqtt_trie_node=Tab -> write_trie(Tab,setelement(1,E,trie_node)); - _ -> kvs:put(E) end end || E <- lists:filter(fun(E) -> is_tuple(E) end ,AllEntries)]. - -write_trie(Tab,Trie) ->mnesia:dirty_write(Tab, Trie). - n2o_pid(Name) -> case n2o_async:pid(system, Name) of @@ -2044,18 +2046,19 @@ link_thumb(ParentType, ChildType, Payload) -> generate_server_id() -> iolist_to_binary(["srv_", nitro:to_binary(roster:now_msec())]). %% TODO add test for the func below +%% TODO remove this after testing %% Set Message.next to [] if it is message about history deletion %% This messages payload is <<"History was removed">> or/and message Msg.id starts with <<"rmv_history_">> -check_message(MsgId) when is_integer(MsgId) -> - {ok, Msg} = kvs:get('Message', MsgId), - check_message(Msg); -check_message(#'Message'{msg_id = <<"rmv_history_",_/binary>>} = Msg) -> - Msg#'Message'{next = []}; -check_message(#'Message'{files = Descs} = Msg) -> - case lists:keyfind(<<"History was removed">>, #'Desc'.payload, Descs) of - false -> Msg; - _ -> Msg#'Message'{next = []} - end. +%%check_message(MsgId) when is_integer(MsgId) -> +%% {ok, Msg} = kvs:get('Message', MsgId), +%% check_message(Msg); +%%check_message(#'Message'{msg_id = <<"rmv_history_",_/binary>>} = Msg) -> +%% Msg#'Message'{next = []}; +%%check_message(#'Message'{files = Descs} = Msg) -> +%% case lists:keyfind(<<"History was removed">>, #'Desc'.payload, Descs) of +%% false -> Msg; +%% _ -> Msg#'Message'{next = []} +%% end. delete_call_bubbles() -> backup(), diff --git a/apps/roster/src/roster_channel_helper.erl b/apps/roster/src/roster_channel_helper.erl new file mode 100644 index 000000000..c51cf8283 --- /dev/null +++ b/apps/roster/src/roster_channel_helper.erl @@ -0,0 +1,278 @@ +-module(roster_channel_helper). +-include("roster.hrl"). +-include_lib("roster/include/static/roster_text.hrl"). +-include_lib("roster/include/static/roster_var.hrl"). +-include_lib("roster/include/static/main_text.hrl"). +-compile(export_all). + +%% --------------------------------------------------------------------------------------------------------------------- +%% Errors Management +%% --------------------------------------------------------------------------------------------------------------------- + +%% TODO replace error_response section to another helper in future +error_response(ListOfCodes, Model) when not is_list(ListOfCodes) -> + error_response([ListOfCodes], Model); +error_response(ListOfCodes, Model) -> + #errors{code = ListOfCodes, data = Model}. + +error_response_403(Model) -> +%% permissions denied error response + error_response(?ERROR_PERMISSION_DENIED, Model). + +error_response_400(Model) -> +%% invalid data error response + error_response(?ERROR_INVALID_DATA, Model). + +%% --------------------------------------------------------------------------------------------------------------------- +%% Channel Management +%% --------------------------------------------------------------------------------------------------------------------- + +is_channel(FeedId) -> + case FeedId of #muc{name = RoomId} -> case kvs:get('Room', RoomId) of {ok, #'Room'{type = channel}} -> true; _ -> false end; _ -> false end. + +%% TODO request. Make it "get_room_by_id" and move to db helpers section +get_channel_by_id(ChannelId) -> +%% Check is channel exists +%% Params => ChannelId :: binary() +%% Return => Succes -> {ok, ChannelObject}; Error -> {error, ChannelNotFoundMsg} + case kvs:get('Room', ChannelId) of + {error, _} -> {error, ?ERROR_CHANNEL_NOT_FOUND}; + {ok, Channel} -> {ok, Channel} + end. + +get_channel_by_link(Link) when is_list(Link) -> + get_channel_by_link(list_to_binary(Link)); +get_channel_by_link(Link) -> + case kvs:get('Index', {?LINK_INDEX_KEYWORD, string:lowercase(Link)}) of + {ok, #'Index'{roster = [RoomId]}} -> + case kvs:get('Room', RoomId) of + {ok, Room} -> channel_response_object(Room); + {error, _} -> [] + end; + _ -> [] + end. + +get_room_avatar(#'Room'{data = Descs}) -> + case lists:keyfind(?ROOM_AVATAR_MIME, #'Desc'.mime, Descs) of #'Desc'{payload = LinkToAvatar} -> LinkToAvatar; _ -> [] end. + +get_last_msg(MsgId) -> + case kvs:get('Message', MsgId) of + {ok, Msg} -> Msg; + {error, _} -> [] + end. + +channel_response_object(#'Room'{id = RoomId, links = Links, last_msg = LastMsgId} = Channel) -> +%% add extra-processing for links and last msg + Channel#'Room'{links = [Link#'Link'{id = RoomId} || Link <- Links], last_msg = get_last_msg(LastMsgId)}. + +%% TODO check permissions function: check is member exists / is member owner / is member admin with OK permittions to make changes + +%% Broadcast functions + +send_channel(ClientPid, ChannelId, Msg) -> + roster:send_room(ClientPid, ChannelId, Msg). + +send_channel(ClientPid, Channel) -> + roster:send_room(ClientPid, channel_response_object(Channel)). + +send_admin(ClientPid, #'Room'{id = ChannelId} = Channel) -> +%% notify all channel admins and owner + [roster:send_ses(ClientPid, MemberPhoneId, channel_response_object(Channel)) || #'Member'{phone_id = MemberPhoneId} <- roster:members(#muc{name = ChannelId}, admin)]. + +%% --------------------------------------------------------------------------------------------------------------------- +%% Member Management +%% --------------------------------------------------------------------------------------------------------------------- + +member_get_feed(MemberId) -> +%% get member feed by member id +%% return member feed + case kvs:get('Member', MemberId) of + {ok, #'Member'{feed_id = FeedId}} -> FeedId; + _ -> [] + end. + +check_user_permissions(ChannelId, ClientId) -> + check_user_permissions(ChannelId, ClientId, []). +check_user_permissions(ChannelId, ClientId, PermissionKey) -> +%% Check acted user permissions +%% Params => ChannelId :: binary(); ClientId :: binary(); PermissionKey :: binary() +%% Return => Success -> {ok, MemberObject}; Error -> {error, PermissionsDeniedMsg} +%% get Acted user member in this Channel + ResponseError = {error, ?ERROR_PERMISSION_DENIED}, + case roster:muc_member(ClientId, ChannelId) of + [] -> ResponseError; + #'Member'{status = ActedMemberStatus, settings = PermList} = ActedMember -> + ResponseSuccess = {ok, ActedMember}, +%% check Acted user member status + case ActedMemberStatus of + owner -> ResponseSuccess; + admin -> + case PermissionKey of + [] -> ResponseSuccess; + _ -> +%% check admin permission list + case lists:keyfind(PermissionKey, #'Feature'.key, PermList) of + #'Feature'{value = true} -> ResponseSuccess; + _ -> ResponseError + end + end; + _ -> ResponseError + end + end. + +add_members(#'Room'{id = ChannelId, readers = ChannelReaders} = Channel, Members) -> +%% Proceed Members to be added + [begin +%% check is this member already added + case roster:muc_member(MemberPhoneId, ChannelId) of + [] -> +%% add new member + roster:add_member(Channel, #'Member'{id = [], feed_id = #muc{name = ChannelId}, status = member, + phone_id = MemberPhoneId}); + #'Member'{status = MemberStatus} = ExistingMember -> +%% check was member removed. if true - change his status from removed to member + case MemberStatus of + removed -> + PrePatchedMember = roster:patch_member(ExistingMember#'Member'{status = member}, ExistingMember), + {PatchedMember, _, _} = roster:add_member(Channel, PrePatchedMember, {no_muc_message, ChannelReaders}), + kvs:put(PatchedMember); + _ -> skip + end + end + end || #'Member'{phone_id = MemberPhoneId} <- Members]. + +count_members(ChannelId, Status) -> +%% count channel members filtered by statuses +%% active: member, owner, admin +%% admin: member, owner + length(roster:members(#muc{name = ChannelId}, Status)). + +update_members_roster(#'Room'{id = ChannelId} = Channel) -> + [roster:update_rooms(MemberPhoneId, Channel) || #'Member'{phone_id = MemberPhoneId} <- roster:members(#muc{name = ChannelId})]. + +%% --------------------------------------------------------------------------------------------------------------------- +%% Feature Management +%% --------------------------------------------------------------------------------------------------------------------- + +update_feature(FeatureKey, FeatureValue, ExistingFeatures) -> + (case lists:keyfind(FeatureKey, #'Feature'.key, ExistingFeatures) of + false -> #'Feature'{id = iolist_to_binary([<<"srv_">>, i2b(kvs:next_id('Desc', 1)), "_", i2b(roster:now_msec())]), + key = FeatureKey, group = ?FGC_INFO}; + ExistingFeature -> ExistingFeature + end)#'Feature'{value = FeatureValue}. + +set_count_features(#'Room'{settings = ExistingFeatures, id = ChannelId} = Channel) -> +%% NOTE! Should update Channel Subscribers Count on: Room/add, Room/remove, Room/join, Room/leave +%% NOTE! Active member means that Member.status is not *removed* + SubscribersCountFeature = update_feature(?FKC_SUBSCRIBERS_COUNT, i2b(count_members(ChannelId, active)), ExistingFeatures), +%% NOTE! Admin member means that Member.status is *admin* or *owner* + AdminsCountFeature = update_feature(?FKC_ADMINS_COUNT, i2b(count_members(ChannelId, admin)), ExistingFeatures), + SettingsUpdSubscribersCountFeature = lists:keystore(?FKC_SUBSCRIBERS_COUNT, #'Feature'.key, ExistingFeatures, SubscribersCountFeature), + SettingsUpdAdminsCountFeature = lists:keystore(?FKC_ADMINS_COUNT, #'Feature'.key, SettingsUpdSubscribersCountFeature, AdminsCountFeature), + Channel#'Room'{settings = SettingsUpdAdminsCountFeature}. + +%% --------------------------------------------------------------------------------------------------------------------- +%% Link Management +%% --------------------------------------------------------------------------------------------------------------------- + +generate_link_id(ChannelId) -> + iolist_to_binary([ChannelId, <<"_">>, integer_to_list(kvs:next_id('Link', 1))]). + +generate_link() -> +%% 1. Create random string based on timestamp +%% 2. Delete not allowed symbols + list_to_binary(lists:filter(fun(Char) -> lists:member(Char, binary_to_list(?LINK_ALLOWED_CHARS)) end, + binary_to_list(base64:encode(crypto:hmac(sha256, i2b(roster:now_msec()), i2b(roster:now_msec())))))). + +validate_link_format(Link) -> + case regexp_api:validate_string(Link, lists:flatten(io_lib:format(?LINK_REGEXP, [?LINK_LEN_MIN - 1, ?LINK_LEN_MAX - 1]))) of + {ok, _} -> {ok, Link}; + {error, _} -> {error, ?LINK_INVALID_FORMAT} + end. + +check_link_availability(Link) -> + check_link_availability(Link, []). +check_link_availability(Link, ChannelId) -> + ResponseError = {error, ?LINK_NOT_AVAILABLE}, + ResponseSuccess = {ok, []}, + case kvs:get('Index', {?LINK_INDEX_KEYWORD, string:lowercase(Link)}) of + {error, _} -> ResponseSuccess; + {ok, #'Index'{roster = EntityList}} -> + case lists:member(ChannelId, EntityList) of + false -> ResponseError; + _ -> ResponseSuccess + end + end. + +add_link_index(Link, RoomId) -> + kvs:put(#'Index'{id = {?LINK_INDEX_KEYWORD, string:lowercase(Link)}, roster = [RoomId]}). + +delete_link_index(Link) -> + kvs:delete('Index', {?LINK_INDEX_KEYWORD, string:lowercase(Link)}). + +%% lists:keyfind(LinkId, #'Link'.id, StoredLinks) +get_link_by_id(LinkId, LinksList) -> +%% Get is Link with requested id exists in Stored Links List +%% Params => LinkId :: binary(), LinksList :: list of #'Link'{} objects +%% Return => Success -> {ok, #'Link'{}}; Error -> {error, LinkNotFoundError} + case lists:keyfind(LinkId, #'Link'.id, LinksList) of false -> {error, ?ERROR_LINK_NOT_FOUND}; #'Link'{} = StoredLink -> {ok, StoredLink} end. + +get_link_count(#'Room'{links = LinkList}) -> +%% Get links count in Channel +%% Params => Channel :: ErlangRecord +%% Return => Success -> LinksCountInteger + length(LinkList). + +check_link_add_availability(Channel) -> +%% Check existing links count in Channel +%% Params => Channel :: ErlangRecord +%% Return => Success -> {ok, []}; Error -> {error, MaxNumberOfLinksReachedMsg} + case get_link_count(Channel) of LinksCount when LinksCount >= ?MAX_LINKS_NUM -> {error, ?MAX_LINKS_NUM_REACHED}; _ -> {ok, []} end. + +%% --------------------------------------------------------------------------------------------------------------------- +%% Non-grouped helpers +%% --------------------------------------------------------------------------------------------------------------------- + +i2b(X) -> integer_to_binary(X). + +paginator(List, Limit, Offset) -> +%% protect from empty list slicing + case length(List) of + 0 -> []; + _ -> +%% protect from invalid Offset value + ValidatedOffset = case Offset of + O when O > length(List) -> length(List); + O when O == []; O == 0 -> 1; + _ -> Offset end, +%% protect from invalid Limit value + ValidatedLimit = case Limit of + L when L < 0 -> 1; + _ -> Limit end, + lists:sublist(List, ValidatedOffset, ValidatedLimit) + end. + +startswith(String, Head) when is_binary(String) -> + startswith(binary_to_list(String), Head); +startswith(String, Head) when is_binary(Head) -> + startswith(String, binary_to_list(Head)); +startswith(String, Head) -> + string:str(string:lowercase(String), string:lowercase(Head)) > 0. + +%% TODO - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - +%% TODO delete debug helpers after development +%% TODO - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + +delete_test_channel() -> + TestChannelId = <<"channel_1">>, +%% delete members + [begin + kvs:delete('Member', MemId) + end || #'Member'{id = MemId} <- roster:members(#muc{name = TestChannelId})], +%% delete messages + [begin + kvs:delete('Message', MsgId) + end || #'Message'{id = MsgId} <- kvs:index('Message', to, TestChannelId)], +%% delete channel + kvs:delete('Room', TestChannelId), + ok. \ No newline at end of file diff --git a/apps/roster/src/roster_db.erl b/apps/roster/src/roster_db.erl index 2b8eb026a..a3ac9c535 100644 --- a/apps/roster/src/roster_db.erl +++ b/apps/roster/src/roster_db.erl @@ -603,6 +603,22 @@ remigrate_default_contact_settings() -> kvs:delete(schema_migrations, 20181008144317), roster:migrate(). +%% tables storage +save_db(Path) -> + Data = lists:append([kvs:all(B) || B <- [Name || {table,Name} <- kvs:dir()-- + [{table,schema},{table,mqtt_session},{table,mqtt_admin}]]]), + kvs:save(Path, Data). + +load_db(Path) -> + kvs:add_seq_ids(), + AllEntries= try kvs:load(Path) catch _:_ -> [] end, + [begin case element(1,E) of + mqtt_trie=Tab -> write_trie(Tab,setelement(1,E,trie)); + mqtt_trie_node=Tab -> write_trie(Tab,setelement(1,E,trie_node)); + _ -> kvs:put(E) end end || E <- lists:filter(fun(E) -> is_tuple(E) end ,AllEntries)]. + +write_trie(Tab,Trie) ->mnesia:dirty_write(Tab, Trie). + %% Subscriptions control unsubs(<<"emqttd_", _/binary>>, _) -> []; diff --git a/apps/roster/src/roster_validator.erl b/apps/roster/src/roster_validator.erl index b866916ad..527b4c6dc 100644 --- a/apps/roster/src/roster_validator.erl +++ b/apps/roster/src/roster_validator.erl @@ -522,7 +522,7 @@ validate(D = #'Room'{id = Id, name = Name, links = Links, description = Descript lists:foldl(fun(Tmp, Acc3) when is_record(Tmp,'Member') -> validate(Tmp, Acc3, CM); (Tmp, Acc3) -> [{admins, D}|Acc3] end, Acc2, Admins); {data,_} when is_list(Data) -> lists:foldl(fun(Tmp, Acc3) when is_record(Tmp,'Desc') -> validate(Tmp, Acc3, CM); (Tmp, Acc3) -> [{data, D}|Acc3] end, Acc2, Data); - {type,_} when Type==[] orelse Type=='group' orelse Type=='channel' -> Acc2; + {type,_} when Type==[] orelse Type=='group' orelse Type=='channel' orelse Type=='call' -> Acc2; {tos,_} when Tos==[] orelse is_binary(Tos) -> Acc2; {tos_update,_} when Tos_update==[] orelse is_integer(Tos_update) -> Acc2; {unread,_} when Unread==[] orelse is_integer(Unread) -> Acc2; @@ -700,13 +700,13 @@ validate(D = #'Job'{id = Id, container = Container, feed_id = Feed_id, prev = Pr {context,_} when Context==[] orelse is_integer(Context) orelse is_binary(Context) -> Acc2; {proc,_} when Proc==[] orelse is_integer(Proc) orelse is_record(Proc,'process') -> Acc2; {time,_} when Time==[] orelse is_integer(Time) -> Acc2; - {data,_} when is_list(Data) -> + {data,_} when Data==[] orelse is_binary(Data) orelse is_list(Data) -> lists:foldl(fun(Tmp, Acc3) when is_record(Tmp,'Message') -> validate(Tmp, Acc3, CM); (Tmp, Acc3) -> [{data, D}|Acc3] end, Acc2, Data); {events,_} when Events==[] orelse is_list(Events) -> lists:foldl(fun(Tmp, Acc3) when is_record(Tmp,'messageEvent') -> validate(Tmp, Acc3, CM); (Tmp, Acc3) -> [{events, D}|Acc3] end, Acc2, Events); {settings,_} when Settings==[] orelse is_list(Settings) -> lists:foldl(fun(Tmp, Acc3) when is_record(Tmp,'Feature') -> validate(Tmp, Acc3, CM); (Tmp, Acc3) -> [{settings, D}|Acc3] end, Acc2, Settings); - {status,_} when Status==[] orelse Status=='init' orelse Status=='update' orelse Status=='delete' orelse Status=='pending' orelse Status=='stop' orelse Status=='complete' -> Acc2; + {status,_} when Status==[] orelse Status=='init' orelse Status=='update' orelse Status=='delete' orelse Status=='pending' orelse Status=='stop' orelse Status=='complete' orelse Status=='restart' -> Acc2; _ -> [{RecField, D}|Acc2] end end, Acc, lists:zip(record_info(fields, 'Job'), tl(tuple_to_list(D)))), ErrFields++case ErrFields of [] -> CustomValidateModule:ValidateFun(D); _ -> [] end; @@ -726,9 +726,9 @@ validate(D = #'History'{roster_id = Roster_id, feed = Feed, size = Size, entity_ validate(D = #'Schedule'{id = Id, proc = Proc, data = Data, state = State}, Acc, {CustomValidateModule, ValidateFun} = CM) -> ErrFields = lists:foldl(fun ({RecField, F}, Acc2) -> case {RecField, F} of - {id,_} when Id==[] orelse is_integer(Id) -> Acc2; - {proc,_} when Proc==[] orelse is_integer(Proc) -> Acc2; - {data,_} when is_list(Data) -> + {id,_} when Id==[] orelse is_integer(Id) orelse is_tuple(Id) -> Acc2; + {proc,_} when Proc==[] orelse is_integer(Proc) orelse is_binary(Proc) -> Acc2; + {data,_} when is_binary(Data) orelse is_list(Data) -> lists:foldl(fun(Tmp, Acc3) when true -> validate(Tmp, Acc3, CM); (Tmp, Acc3) -> [{data, D}|Acc3] end, Acc2, Data); {state,_} when State==[] orelse true -> Acc2; _ -> [{RecField, D}|Acc2] diff --git a/apps/roster/src/test/roster_test.erl b/apps/roster/src/test/roster_test.erl index 079adc575..2a5b0cb40 100644 --- a/apps/roster/src/test/roster_test.erl +++ b/apps/roster/src/test/roster_test.erl @@ -264,7 +264,7 @@ test_roster() -> #'Profile'{status = get} = roster_client:send_receive(AClientId, #'Profile'{status = get}), #'Message'{id = IdLast} = roster_client:send_receive(BClientId, 2, #'Message'{feed_id = Feed, from = BPhoneId, to = APhoneId, files = [#'Desc'{id = <<"7">>, payload = <<"TestB2222!">>}], status = []}), - #'History'{status = get, data = [_, #'Message'{status = clear}|_]} = %% if msg_id < clear_id then clear message doesn't stop history + #'History'{status = get, data = [_, #'Message'{status = clear, next = []}|_]} = %% if msg_id < clear_id then clear message doesn't stop history roster_client:send_receive(AClientId, #'History'{feed = Feed, roster_id = APhoneId, entity_id = IdA, size = 5, status = get}), #'Message'{id = IdEd, status = edit} = roster_client:send_receive(BClientId, 2, #'Message'{id = IdLast, feed_id = Feed, from = BPhoneId, to = APhoneId, files = [#'Desc'{id = <<"7">>, payload = <<"Edit TestB!">>}], status = edit}), @@ -658,10 +658,9 @@ test_multi_muc() -> roster_client:stop_client(CClientId2), timer:sleep(100), roster_client:send_receive(CClientId, Counter+1, #'Room'{id = ARoom, status = leave, type = group}), - {ok, #'Auth'{settings = [_, _, #'Feature'{key = <<"cache_", _/binary>>}]}} = kvs:get('Auth', CClientId2), - roster_client:start_cli_receive(CClientId2, CToken2, 1), - - [#'Profile'{}, #'Room'{status = leave}] = roster_client:send_receive(CClientId2, last_res), %% +% {ok, #'Auth'{settings = [_, _, #'Feature'{key = <<"cache_", _/binary>>}]}} = kvs:get('Auth', CClientId2), +% roster_client:start_cli_receive(CClientId2, CToken2, 1), TODO uncomment for send_cache test +% [#'Profile'{}, #'Room'{status = leave}] = roster_client:send_receive(CClientId2, last_res), %% roster_client:stop_client(CClientId), #'Profile'{} = roster_client:start_cli_receive(CClientId, CToken2), [roster_client:stop_client(ClientId)|| ClientId <- [AClientId2, CClientId2]], @@ -1474,8 +1473,8 @@ suite() -> muc_remove_test(), test_call_room(), pagination_test(), - event_securty_test(), - test_messages() + event_securty_test() +% test_messages() ]. check() -> case lists:all(fun(X) -> X == ok end, suite()) of @@ -1519,7 +1518,7 @@ bpe() -> PhoneA = <<"773">>, Phones = [<<"5333">>, <<"377">>], [kvs:put(#'Whitelist'{phone = Phone, created = roster:now_msec()}) || Phone <- Phones ++ [PhoneA]], - [roster:purge_user(Phone) || Phone <- [PhoneA | Phones]], + [ begin try roster:purge_user(Phone) catch _:_ -> skip end end || Phone <- [PhoneA | Phones]], [{B, BClientId, _}, {C, _, _}, {A, AClientId, _}] = [begin C = roster_client:gen_name_reg(Phone), {ClientId, Token} = roster_client:reg_fake_user(Phone), @@ -1871,7 +1870,7 @@ delete_p2p_history_test() -> %%unban user [#'Contact'{reader = RdrB3, status = friend} | _] = - roster_client:receive_last(BClientId, 2, #'Friend'{phone_id = BPhoneId, friend_id = APhoneId, status = unban}), + roster_client:receive_last(BClientId, 3, #'Friend'{phone_id = BPhoneId, friend_id = APhoneId, status = unban}), true = RdrB3 /= [0, 0], #'Contact'{reader = RdrB3} = roster:get_contact(roster:roster(roster:roster_id(BPhoneId)), APhoneId), roster_client:stop_client(AClientId), @@ -2352,4 +2351,12 @@ test_security(Host) -> end, %% #'History'{data = [_, _]} %% only two message in history %% = roster_client:send_receive(AClientId, #'History'{feed = Feed, roster_id = APhoneId, size = [], entity_id = [], status = get}), - roster_client:stop_client([AClientId, BClientId, CClientId]). \ No newline at end of file + roster_client:stop_client([AClientId, BClientId, CClientId]). + +test_send_profile() -> + lists:reverse(lists:keysort(1,[ begin + io:format("Phone: ~p",[P#'Profile'.phone]), + {Time,_} = timer:tc(fun() -> roster:send_profile(P,[],0,[],[]) end), + io:format("done in ~p uc.~n",[Time]), + {Time,P#'Profile'.phone} + end || P <- kvs:all('Profile')])). \ No newline at end of file diff --git a/etc/certs/cert.pem b/etc/certs/cert.pem deleted file mode 100644 index 276ba49ac..000000000 --- a/etc/certs/cert.pem +++ /dev/null @@ -1,20 +0,0 @@ ------BEGIN CERTIFICATE----- -MIIDWjCCAkICCQDPVIdsxF6HMTANBgkqhkiG9w0BAQsFADBvMQswCQYDVQQGEwJV -QTENMAsGA1UECAwES3lpdjENMAsGA1UEBwwES3lpdjEhMB8GA1UECgwYSW50ZXJu -ZXQgV2lkZ2l0cyBQdHkgTHRkMR8wHQYDVQQDDBZ0cmFuc2xhdGUuY2kubnluamEu -bmV0MB4XDTE5MDExNTE0NTAyOVoXDTI5MDExMjE0NTAyOVowbzELMAkGA1UEBhMC -VUExDTALBgNVBAgMBEt5aXYxDTALBgNVBAcMBEt5aXYxITAfBgNVBAoMGEludGVy -bmV0IFdpZGdpdHMgUHR5IEx0ZDEfMB0GA1UEAwwWdHJhbnNsYXRlLmNpLm55bmph -Lm5ldDCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBANLs7ZQ4qi8oQZ51 -0QqR6Rgie8KFyauYYr1kp8VOzDEXHTm3ntbCa3qo9yoGdZZh38Nx/CeSUm5hf70d -nYE7FYFvzZpqMq+7CpnxsMiEIzL1815kPB/XGTAZzPvT0CGTjlekwBjTM/AkJV57 -+cgrbYu7KUJ+7L+UvOrodF1O43d1+uJEz0sJintpQoLCyO0MYyyiAPozifaOI5c6 -UkNz0k73tdwbnjequAFvPm0gMmJ3FvCk7Y0uEWaqR2yh9YXVaHqWMb4raJdwCQXe -vYgO8z7ptDnJYnQvPAy9M/WLFOygZjLfKJVh3//tP9z/yFGjATkft2j6BmDJnsuK -Hf9W9B8CAwEAATANBgkqhkiG9w0BAQsFAAOCAQEAotACgSqQE1M7nhOOsrhMvi5r -nRydS9td7Tx9RyHGW0NpfFZavr14J652NzbO/By97kqB6JaTjedfAVl0+OS5+9o5 -oc9gxVtJa8p6MXTfoPC6dbX2KfjD1dwVwKkKQo4JNz/etvWfEwXj9pmNVBRBac9T -Ty6WCxiN5Qzh1QJ+I3Iy4AoF6zaHI/NtTNaoB2I3bH2XBHClCVXJreHQzpvG1l9r -8WM3xNz/OMYTe7Ahe5HYHijwD8k3I2uP/yrirOfFBvj+xlrhhvA99/Z1tBN1+qGY -fROPlVQmFIiyekz/gRIsP0qOn1cNUcP/pRi2isfY5QeKqarY7vQIL/eX+IPl2w== ------END CERTIFICATE----- diff --git a/sys.config b/sys.config index d4fab7b58..25cbdab71 100644 --- a/sys.config +++ b/sys.config @@ -3,7 +3,7 @@ {bert,[{js,"apps/roster/priv/macbert/"}, {erl,"apps/roster/src"}, {google,"apps/service/priv/"}, - {disallowed,['feed', 'Whitelist', 'FakeNumbers']}, + {disallowed,['feed', 'Whitelist', 'FakeNumbers', 'Schedule','Index']}, {allowed_hrl, ["roster"]}, {custom_validate, {roster, validate}}, {swift,"apps/roster/priv/macbert/"}]}, @@ -67,7 +67,7 @@ {tables,[ cookies, file, caching, ring, async, nick, names, surnames, system]}, {pickler,nitro_pickle}, {formatter,n2o_bert}, - {auth_ttl, 900}, %% 15 mins + {auth_ttl, 86400}, %% 24 h {protocols,[n2o_nitro,n2o_ftp,roster_proto,kvs_stream]}, {log_modules,[roster, n2o_vnode]}, {log_level,roster}, -- GitLab