diff --git a/apps/roster/src/protocol/roster_message.erl b/apps/roster/src/protocol/roster_message.erl index 99119a529f893d498d40b867e306f11b23300db5..e1350816cafb5b4063c97c43ff9e905099375ccb 100644 --- a/apps/roster/src/protocol/roster_message.erl +++ b/apps/roster/src/protocol/roster_message.erl @@ -309,7 +309,11 @@ info(#'Message'{id = Id, msg_id = ClMID, feed_id = Feed, from = From0, seenby = info(#'Message'{status = update, type = [draft], feed_id = Feed0, from = From, to = To, files = File}=M, Req, #cx{params = ClientId, client_pid = C, state=ack} = State) -> - PhoneId = case ClientId of ?SYS_SCHED_CLIENT -> From; <<"emqttd_", _/binary>> -> roster:phone_id(ClientId) end, + PhoneId = + case ClientId of + ?SYS_SCHED_CLIENT -> From; + <<"emqttd_", _/binary>> -> roster:phone_id(ClientId) + end, ?ROSTER_LOG_REQ('Message', update, ClientId, "Type=draft, Feed=~p, Files=~p", [Feed0, [PL || #'Desc'{payload = PL} <- File]]), {Feed,M1,D}=case File of @@ -338,25 +342,34 @@ info(#'Message'{status = update, type = [draft], feed_id = Feed0, from = From, t end, {reply, {bert, IO}, Req, State}; -info(#'Message'{status = update, id = Id, files = [#'Desc'{mime = <<"transcribe">>, payload = [], data = Data} = Desc | _]} = Msg, Req, - #cx{params = ClientId, state = ack} = State) - when is_integer(Id), not is_tuple(Req) -> +info(#'Message'{status = update, id = Id, files = [#'Desc'{mime = <<"transcribe">>, payload = [], data = Data} = Desc | _]} = Msg, + Req, + #cx{params = ClientId, state = ack} = State) when is_integer(Id), not is_tuple(Req) -> ?ROSTER_LOG_REQ('Message', update, ClientId, "Type=transcribe, Id=~p", [Id]), - Type = case roster:get_data_val(<<"TYPE">>, Data) of [] -> short; T -> binary_to_atom(T, utf8) end, + Type = + case roster:get_data_val(<<"TYPE">>, Data) of + [] -> <<"short">>; + T -> string:lowercase(T) + end, Lang = roster:get_data_val(?LANG_KEY, Data), Pid = n2o_async:pid(system, roster_message), case kvs:get('Message', Id) of {ok, #'Message'{feed_id = Feed, from = From, to = To, files = [#'Desc'{mime = <<"audio">>, payload = Uri} | _]}} -> - ?LOG_DEBUG("enter ~p transcribe process with ~p, (Lang=~p)", [Type, Uri, Lang]), + PhoneId = + case ClientId of + ?SYS_SCHED_CLIENT -> From; + <<"emqttd_", _/binary>> -> roster:phone_id(ClientId) + end, + ?LOG_DEBUG("enter ~s transcribe process with ~p, (Lang=~p) from ~p", [Type, Uri, Lang, PhoneId]), ErrMsg = #'Message'{id = Id, feed_id = Feed, from = From, to = To}, case Type of - short -> + <<"short">> -> spawn(fun() -> Res = case google_api:convert_ffmpeg(binary_to_list(Uri)) of {file, FileOut, FileIn} -> {ok, Binary} = file:read_file(FileOut), - R = case google_api:transcribe(Type, base64:encode(Binary), Lang) of + R = case google_api:transcribe(short, base64:encode(Binary), Lang) of {error, _} = E -> #io{code = E, data = ErrMsg}; Text -> @@ -369,15 +382,15 @@ info(#'Message'{status = update, id = Id, files = [#'Desc'{mime = <<"transcribe" ?LOG_ERROR("invalid url for transcribe: ~p ~p", [Uri, ErrInfo]), #io{code = {error, invalid_data}, data = ErrMsg} end, - Pid ! {Res, ClientId} + Pid ! {transcribed, Res, ClientId, PhoneId, Lang} end), {reply, {bert, #io{code = #ok{code = transcribe}, data = Id}}, Req, State}; - long -> + <<"long">> -> spawn(fun() -> Res = case google_api:gs_upload(binary_to_list(Uri)) of {gs, GsUri} -> - R = case google_api:transcribe(Type, GsUri, Lang) of + R = case google_api:transcribe(long, GsUri, Lang) of {error, _} = E -> #io{code = E, data = ErrMsg}; Text -> @@ -390,11 +403,11 @@ info(#'Message'{status = update, id = Id, files = [#'Desc'{mime = <<"transcribe" ?LOG_ERROR("invalid url for transcribe: ~p ~p", [Uri, ErrInfo]), #io{code = {error, invalid_data}, data = ErrMsg} end, - Pid ! {Res, ClientId} + Pid ! {transcribed, Res, ClientId, PhoneId, Lang} end), {reply, {bert, #io{code = #ok{code = transcribe}, data = Id}}, Req, State}; _ -> - ?LOG_ERROR("invalid transcribe type ~p", [Type]), + ?LOG_ERROR("invalid transcribe type ~s", [Type]), {reply, {bert, #io{code = #error{code = invalid_data}, data = ErrMsg}}, Req, State} end; {ok, InvalidMsg} -> @@ -406,57 +419,19 @@ info(#'Message'{status = update, id = Id, files = [#'Desc'{mime = <<"transcribe" end; -info(#'Message'{status = update, id = Id, feed_id = Feed, from = From, - files = [#'Desc'{id = ID, payload = Payload, data = Data, mime = DMime} = ND | _]}, Req, - #cx{params = ClientId, client_pid = C, state=ack} = State) when is_integer(Id) -> +info(#'Message'{status = update, id = Id, from = From, + files = [#'Desc'{data = Data} | _]} = Msg, Req, + #cx{params = ClientId, client_pid = C, state = ack} = State) when is_integer(Id) -> PhoneId = case ClientId of ?SYS_SCHED_CLIENT -> From; <<"emqttd_", _/binary>> -> roster:phone_id(ClientId) end, ?ROSTER_LOG_REQ('Message', update, ClientId, "Id=~p", [Id]), Lang = roster:get_data_val(?LANG_KEY, Data), - IO = case kvs:get('Message', Id) of - {ok, #'Message'{feed_id = Feed, files = Descs} = Msg} -> - case roster:is_visible_msg(Msg, PhoneId) of - 1 -> - Descs2 = lists:flatten(lists:foldr( - fun (#'Desc'{data = NData, id = DescId} = D, [Acc, _]) when Lang == [], ID /= [] -> - #'Feature'{value = I} = - case roster:get_data(?USERS_KEY, NData) of - [] -> #'Feature'{key = ?USERS_KEY, value = []}; FU -> FU end, - [[case DescId of - ID -> roster:set_data(#'Feature'{key = ?USERS_KEY, value = binary_delpart(I, PhoneId)}, D); - _-> D end | Acc], []]; - (#'Desc'{data = NData, mime = TMime} = D, [Acc, ND2]) - when TMime =:= DMime -> - FUsers = #'Feature'{value = I} = - case roster:get_data(?USERS_KEY, NData) of - [] -> #'Feature'{key = ?USERS_KEY, value = []}; FU -> FU end, - case roster:get_data(?LANG_KEY, NData) of - #'Feature'{value = <<_:8, _/binary>> = TLang} -> - case binary_delpart(I, PhoneId) of - [] when I == [] -> [Acc, []]; - [] -> [Acc, ND2]; - I2 when TLang == Lang -> - [[roster:set_data(FUsers#'Feature'{key = ?USERS_KEY, value = <>}, - D#'Desc'{payload = Payload}) | Acc], []]; - I2 -> - [[roster:set_data(FUsers#'Feature'{key = ?USERS_KEY, value = I2}, - D#'Desc'{}) | Acc], ND2] - end; - _ -> [[D | Acc], ND2] - end; - (D, [Acc, ND2]) -> [[D | Acc], ND2] - end, [[], [roster:set_data(#'Feature'{key = ?USERS_KEY, value = PhoneId}, ND)]], - [DD#'Desc'{data = lists:keysort(#'Feature'.key, DData)} || #'Desc'{data = DData} = DD<-Descs])), - kvs:put(NMsg = Msg#'Message'{files = Descs2}), - case kvs_stream:load_writer(Feed) of - #writer{cache = #'Message'{id = Id}} = W -> kvs_stream:save(W#writer{cache = NMsg}); - _ -> skip - end, - roster:send_feed(C, Feed, NMsg#'Message'{status = update}), <<>>; - _ -> #io{code = #error{code = invalid_data}} - end; - _ -> #io{code = #error{code = message_not_found}} end, - {reply, {bert, IO}, Req, State}; - + case msg_update(Msg, ClientId, PhoneId, Lang) of + {updated, #'Message'{feed_id = Feed} = NMsg, _} -> + roster:send_feed(C, Feed, NMsg#'Message'{status = update}), + {reply, {bert, <<>>}, Req, State}; + {update, #io{} = IO, _} -> + {reply, {bert, IO}, Req, State} + end; info(#'Message'{from = From, to = To, status = Status}, Req, #cx{params = ClientId} = State) -> ?ROSTER_LOG_REQ('Message', Status, ClientId, "Unknown request, From=~p, To=~p", [From, To]), @@ -483,15 +458,19 @@ proc({#io{} = IO, ClientId}, #handler{state = C} = H) -> ?LOG_ERROR("~p", [IO]), roster:send_action(C, ClientId, IO), {reply, [], H}; -proc({#'Message'{status = update} = Msg, ClientId}, #handler{state = C} = H) -> - ?LOG_INFO("UPDATE TRANSCRIBE: ~p", [Msg]), - %% As a dirty hack, we let sys_bpe post the messaage update. The client itself may have disconnected - try {reply, {bert, IO}, _, _} = info(Msg, {[], handled}, #cx{params = <<"sys_bpe">>, client_pid = C, state = ack}), - roster:send_action(C, ClientId, IO) - catch Err:Rea:Sta -> - ?LOG_ERROR("~p:~p:~p", [Err, Rea, Sta]) +proc({transcribed, #'Message'{} = Msg, ClientId, PhoneId, Lang}, #handler{state = C} = H) -> + case msg_update(Msg, ClientId, PhoneId, Lang) of + {updated, #'Message'{feed_id = Feed} = NMsg, _} -> + ?LOG_INFO("Updated ~p", [NMsg]), + roster:send_feed(C, Feed, NMsg#'Message'{status = update}); + {update, #io{} = IO, _} -> + ?LOG_ERROR("transcribe ~p update error ~p", [Msg, IO]), + roster:send_action(C, ClientId, IO) end, {reply, [], H}; +proc({transcribed, #io{} = IO, ClientId, _PhoneId, _Lang}, #handler{state = C} = H) -> + roster:send_action(C, ClientId, IO), + {reply, [], H}; proc({mqttc, C, connected}, State = #handler{state = C, seq = S}) -> {ok, State#handler{seq = S + 1}}; proc({mqttc, _C, disconnected}, State) -> {ok, State}; proc(Unknown, #handler{} = H) -> @@ -562,14 +541,110 @@ msg_preview(#'Message'{files = Attachments}) -> iolist_to_binary([string:to_upper(FirstLetter), LastText]) end. -binary_delpart([], _Part) -> []; +msg_update(#'Message'{id = Id, files = [#'Desc'{id = UpdateDescId} = UpdateDesc | _]}, ClientId, PhoneId, Lang) -> + ?ROSTER_LOG_REQ('Message', update, ClientId, "Id=~p Lang=~p", [Id, Lang]), + %% find most recent version of message... + %% another update could have been ongoing in parallel + %% The 'Feature' with ?USERS_KEY tells the client whether or not to display the update + case kvs:get('Message', Id) of + {ok, #'Message'{feed_id = Feed, files = Descs} = Msg} -> + case roster:is_visible_msg(Msg, PhoneId) of + 1 -> + %% There is no clear reason why we should sort features, but client may rely on it. + DescsFS = + [ Desc#'Desc'{data = lists:keysort(#'Feature'.key, DData)} || + #'Desc'{data = DData} = Desc <- Descs ], + NewDescs = + case Lang == [] andalso UpdateDescId /= [] of + true -> + descs_removing_update(DescsFS, UpdateDescId, PhoneId); + false -> + descs_changing_update(DescsFS, + roster:set_data(#'Feature'{key = ?USERS_KEY, + value = PhoneId}, UpdateDesc), + Lang, PhoneId) + end, + NewMsg = Msg#'Message'{files = NewDescs}, + kvs:put(NewMsg), + case kvs_stream:load_writer(Feed) of + #writer{cache = #'Message'{id = Id}} = W -> kvs_stream:save(W#writer{cache = NewMsg}); + _ -> skip + end, + {updated, NewMsg, ClientId}; + _ -> + {updated, #io{code = #error{code = invalid_data}}, ClientId} + end; + _ -> + {updated, #io{code = #error{code = message_not_found}}, ClientId} + end. + +descs_removing_update(Descs, UpdateDescId, PhoneId) -> + lists:foldr( + fun (#'Desc'{data = DescData, id = DescId} = D, Acc) -> + UsersFeature = roster:get_data(?USERS_KEY, DescData), + case {DescId == UpdateDescId, UsersFeature} of + {true, []} -> + Acc; + {true, #'Feature'{value = PhoneIds}} -> + case binary_delpart(PhoneIds, PhoneId) of + [] -> Acc; + RemainingUsers -> + ?LOG_DEBUG("Other users using this Desc ~p (~p)", [D, Descs]), + [roster:set_data(#'Feature'{key = ?USERS_KEY, + value = RemainingUsers}, D) | Acc] + end; + {false, _} -> + [D | Acc] + end + end, [], Descs). + +descs_changing_update(Descs, #'Desc'{payload = Payload, mime = DMime} = UpdateDesc, Lang, PhoneId) -> + {NewDescs, NewDesc2} = + lists:foldr( + fun (#'Desc'{data = DescData, mime = TMime} = D, {Acc, ND2}) + when TMime =:= DMime -> + UsersFeature = roster:get_data(?USERS_KEY, DescData), + LanguageFeature = roster:get_data(?LANG_KEY, DescData), + case {LanguageFeature, UsersFeature} of + {#'Feature'{}, []} -> + ?LOG_ERROR("Message contains Desc with LANGUAGE but no USERS viewing it ~p (~p)", [D, Descs]), + %% This indicates database polution or unknown feature + {Acc, ND2}; + {#'Feature'{value = <<_:8, _/binary>> = TLang}, #'Feature'{value = PhoneIds}} -> + case binary_delpart(PhoneIds, PhoneId) of + [] -> + ?LOG_DEBUG("No other users for this Desc, remove it ~p (~p)", [D, Descs]), + {Acc, ND2}; + RemainingUsers when TLang == Lang -> + %% We keep the same language (re-done a translate/transcribe), update result for all + %% We could also just keep it, without change of payload, but client may have better + %% translation available? + ?LOG_DEBUG("Add user to other users for this Desc ~p (~p)", [D, Descs]), + {[roster:set_data(UsersFeature#'Feature'{key = ?USERS_KEY, + value = <>}, + D#'Desc'{payload = Payload}) | Acc], []}; + RemainingUsers -> + ?LOG_DEBUG("Remove user from this Desc ~p (~p)", [D, Descs]), + %% New language added for this user, remove old translation for this user. + %% but keep the new as ND2 + %% Note that RemainingUsers /= [] and hence Desc /= [] + {[roster:set_data(UsersFeature#'Feature'{key = ?USERS_KEY, + value = RemainingUsers}, D) | Acc], ND2} + end; + _ -> + {[D | Acc], ND2} + end; + (D, {Acc, ND2}) -> + {[D | Acc], ND2} + end, {[], [UpdateDesc]}, Descs), + NewDescs ++ NewDesc2. + +binary_delpart([], _Part) -> + []; binary_delpart(Bin, Part) -> - List = binary:split(Bin, [<<",">>], [global]), - case lists:reverse(List--[Part]) of + case string:split(Bin, ",", all) -- [Part] of [] -> []; - [O] -> <>; - [H | L] -> T = lists:foldl(fun(A, B) -> <> end, <<>>, L), - <> + RemainingParts -> iolist_to_binary(lists:join(",", RemainingParts)) end. has_flag(Descs, message_ack) -> lists:keymember(<<"ack">>, #'Desc'.mime, Descs). diff --git a/doc/release-notes/next/NY8653-transcribe b/doc/release-notes/next/NY8653-transcribe index b8251c7271906febe1473a5f572e9d1bac6da1f3..ffed3b5ac9f149bcf32a4bb11bff46116bc202dc 100644 --- a/doc/release-notes/next/NY8653-transcribe +++ b/doc/release-notes/next/NY8653-transcribe @@ -2,10 +2,13 @@ ### Highlights -* Fixed bugs in transcribing long messages -* -- +* Fixed bugs in transcribing long messages NY-8653 ### List of changes * Added tests for transcribe +* Added tests for translate * fixed enenra package to not fail on token refresh +* fixed bug such that closing tab while transcribe/translate in progress results in transcription +* partly addresses NY-10444: longer messages may hit google api time limit for translation and google does not always translate/transcribes right (or actually almost never). + diff --git a/test/nynja.erl b/test/nynja.erl index 6b5289235c6e7f2a1a98128229865b55a82f150d..9f7ce8f6c284a12b238aa75b177cc482e7211a82 100644 --- a/test/nynja.erl +++ b/test/nynja.erl @@ -262,11 +262,28 @@ transcribe_msg_room(User = #user{ client = ClientId, roster_id = PhoneId}, RoomN N = if Ack -> 2; true -> 1 end, ws_send(User, mqtt_publish(ClientId, MsgS), N, 30000). +translate_msg_room(User = #user{ client = ClientId, roster_id = PhoneId}, RoomName, IdMessage, Lang, Ack) -> + RoomId = typed_uuid(<<"room">>, RoomName), + MsgS = make_translate_message(ClientId, PhoneId, IdMessage, #muc{ name = RoomId }, Lang, Ack), + N = if Ack -> 2; true -> 1 end, + ws_send(User, mqtt_publish(ClientId, MsgS), N, 30000). + +untranslate_msg_room(User = #user{ client = ClientId, roster_id = PhoneId}, RoomName, IdMessage, DescId, false = Ack) -> + RoomId = typed_uuid(<<"room">>, RoomName), + MsgS = make_untranslate_message(ClientId, PhoneId, IdMessage, #muc{ name = RoomId }, DescId), + N = if Ack -> 2; true -> 1 end, + ws_send(User, mqtt_publish(ClientId, MsgS), N, 10000). + transcribe_msg_chat(User = #user{ client = ClientId, roster_id = PhoneId}, FeedId, IdMessage, Type, Ack) -> MsgS = make_transcribe_message(ClientId, PhoneId, IdMessage, FeedId, Type, true), N = if Ack -> 2; true -> 1 end, ws_send(User, mqtt_publish(ClientId, MsgS), N, 30000). +untranscribe_msg_chat(User = #user{ client = ClientId, roster_id = PhoneId}, FeedId, IdMessage, DescId, Ack) -> + MsgS = make_untranscribe_message(ClientId, PhoneId, IdMessage, FeedId, DescId), + N = if Ack -> 2; true -> 1 end, + ws_send(User, mqtt_publish(ClientId, MsgS), N, 10000). + make_room_message(User, RoomName, Msg) -> make_room_message(User, RoomName, Msg, false). @@ -350,6 +367,51 @@ make_transcribe_message(ClientId, PhoneId, IdMessage, Feed, Type, Ack) -> files = PayloadDesc ++ AckDesc, status = update }. +%% NO ack in client... hard to see what Id of ack should be. +make_untranscribe_message(_ClientId, PhoneId, IdMessage, Feed, DescId) -> + %% Lookup USERS feature for this PhoneId in the Features of Msgs and use the ID of + %% that description to put it to [] + PayloadDesc = [ #'Desc'{ id = DescId, mime = [] } ], + #'Message'{ id = IdMessage, + feed_id = Feed, + container = chain, + from = PhoneId, + to = [], + files = PayloadDesc, + status = update }. + +%% This is done client side, we don't really know the format +make_translate_message(ClientId, PhoneId, IdMessage, Feed, Lang, Ack) -> + TS = integer_to_binary(uniq()), + ID = <>, + Data = + [#'Feature'{id = <>, + key = <<"LANGUAGE">>,value = Lang, + group = <<"FILE_DATA">>}, + #'Feature'{id = <>, + key = <<"USERS">>, + value = PhoneId, + group = <<"FILE_DATA">>}, + #'Feature'{id = <>, + key = <<"TRANSLATION">>, + value = iolist_to_binary(["This is the text translated to ", Lang]), + group = <<"FILE_DATA">>}], + PayloadDesc = [ #'Desc'{ id = ID, mime = <<"translate">>, + payload = <<"This is the original text">>, + data = Data } ], + AckDesc = [ #'Desc'{ id = <<"ack", TS/binary>>, mime = <<"ack">> } || Ack ], %% no ack in client + #'Message'{ id = IdMessage, + feed_id = Feed, %% not really needed + container = chain, %% not really needed + from = PhoneId, + to = [], + files = PayloadDesc ++ AckDesc, + status = update }. + +%% NO ack in client... hard to see what Id of ack should be. +make_untranslate_message(ClientId, PhoneId, IdMessage, Feed, DescId) -> + %% same as untranscribe + make_untranscribe_message(ClientId, PhoneId, IdMessage, Feed, DescId). make_member(#{ phone_id := PhoneId } = Args) -> Settings = diff --git a/test/transcribe_SUITE.erl b/test/transcribe_SUITE.erl index 40317adf3112971f10cbd937f1d39d7b0068c5d7..a36e4805876aa171134d9f80479c0bbbb5cf5d70 100644 --- a/test/transcribe_SUITE.erl +++ b/test/transcribe_SUITE.erl @@ -1,6 +1,16 @@ %% Test transcription and translation of messages %% Which uses google API +%% This is how it used to work (May 2020, on staging). +%% Both parties can transcribe an audio message, but the result is only +%% visible to them. The other party may possibly receive that message, +%% but the transcribe and untranscribe are not made visible in the web client. +%% This is a client property, to make a transcription visible. +%% +%% This visibility issue cannot be tested without the real clients +%% What happens is the addition of a feature with a USERS key +%% {'Feature', <<"..._users">>, <<"USERS">>, Key,<<"DUMMY_GROUP">>} + -module(transcribe_SUITE). -include_lib("stdlib/include/assert.hrl"). -include_lib("emqttc/include/emqttc_packet.hrl"). @@ -22,10 +32,11 @@ , short_transcribe/1 , long_transcribe/1 , chat_transcribe/1 + , translate/1 ]). all() -> - [ %% {group, translate}, + [{group, translate}, {group, transcribe} ]. @@ -110,9 +121,10 @@ short_transcribe(Cfg) -> ct:log("User notified ~p", [UserNotify]), %% Admin sends audio - [#mqtt_packet{ payload = #'Message'{ id = IdMessage } } | _Ack ] = - nynja:audio_msg_room(AdminUser, RoomName, - <<"https://nynja-defaults.s3.us-west-2.amazonaws.com/58f1b135-2c43-48b6-b83a-129254e567f2.mpeg">>, true), + IdMessage = + ack_and_msgid( + nynja:audio_msg_room(AdminUser, RoomName, + <<"https://nynja-defaults.s3.us-west-2.amazonaws.com/58f1b135-2c43-48b6-b83a-129254e567f2.mpeg">>, true)), ct:log("Id for 1st audio msg ~p", [IdMessage]), _ = nynja:ws_receive(User), %% User gets this msg @@ -127,9 +139,10 @@ short_transcribe(Cfg) -> _ = nynja:ws_receive(User, any, 10), %% User sends audio - [#mqtt_packet{ payload = #'Message'{ id = IdMessage2 } } | _ ] = - nynja:audio_msg_room(User, RoomName, - <<"https://nynja-defaults.s3.us-west-2.amazonaws.com/5b3b3a41-0439-43ab-9bdf-c847e9c72780.mpeg">>, true), + IdMessage2 = + ack_and_msgid( + nynja:audio_msg_room(User, RoomName, + <<"https://nynja-defaults.s3.us-west-2.amazonaws.com/5b3b3a41-0439-43ab-9bdf-c847e9c72780.mpeg">>, true)), ct:log("Id for 2nd audio msg ~p", [IdMessage2]), _ = nynja:ws_receive(AdminUser), %% AdminUser gets this msg @@ -164,9 +177,10 @@ long_transcribe(Cfg) -> ct:log("User notified ~p", [UserNotify]), %% Admin sends audio - [#mqtt_packet{ payload = #'Message'{ id = IdMessage } } | _Ack ] = - nynja:audio_msg_room(AdminUser, RoomName, - <<"https://nynja-defaults.s3.us-west-2.amazonaws.com/58f1b135-2c43-48b6-b83a-129254e567f2.mpeg">>, true), + IdMessage = + ack_and_msgid( + nynja:audio_msg_room(AdminUser, RoomName, + <<"https://nynja-defaults.s3.us-west-2.amazonaws.com/58f1b135-2c43-48b6-b83a-129254e567f2.mpeg">>, true)), ct:log("Id for 1st audio msg ~p", [IdMessage]), _ = nynja:ws_receive(User), %% User gets this msg @@ -203,12 +217,130 @@ chat_transcribe(Cfg) -> nynja:audio_msg_chat(User, Friend, <<"https://nynja-defaults.s3.us-west-2.amazonaws.com/58f1b135-2c43-48b6-b83a-129254e567f2.mpeg">>, FeedId), ct:log("Audio Msg ~p: ~p", [IdMessage, Audio]), + [#mqtt_packet{ payload = #'Message'{ id = FriendIdMessage } }] = nynja:ws_receive(Friend), %% Friend gets this msg + ct:log("Friend audio msg ~p", [FriendIdMessage]), [#mqtt_packet{ payload = {io, {ok, transcribe}, IdMessage} }, #mqtt_packet{ payload = #'Message'{id = IdMessage, status = update, files = Data}}] = nynja:transcribe_msg_chat(User, FeedId, IdMessage, <<"short">>, true), ct:log("Transcribe result = ~p", [Data]), #'Desc'{mime = <<"transcribe">>, payload = <<"1 2 3 4 5">>} = lists:last(Data), + + %% Both parties get the result, but client filters what is shown to them. + FM = nynja:ws_receive(Friend, any, 10), + ct:log("Transcribe result for friend = ~p", [FM]), + UM = nynja:ws_receive(User, any, 10), + ct:log("Transcribe result for user = ~p", [UM]), + + %% Friend sends audio + #mqtt_packet{ payload = #'Message'{} } = + nynja:audio_msg_chat(Friend, User, + <<"https://nynja-defaults.s3.us-west-2.amazonaws.com/5b3b3a41-0439-43ab-9bdf-c847e9c72780.mpeg">>, FeedId), + [#mqtt_packet{ payload = #'Message'{ id = IdMessage2 } }] = nynja:ws_receive(User), %% User gets this msg + ct:log("Id for 2nd audio msg ~p", [IdMessage2]), + + %% Friend disconnects, we still want to be able to transcribe + nynja:ws_close(Friend), + + %% User transcribes Friend's audio + [#mqtt_packet{ payload = {io, {ok, transcribe}, IdMessage2} }, + #mqtt_packet{ payload = #'Message'{id = IdMessage2, status = update, files = UserSeesTranscribe2}}] = + nynja:transcribe_msg_chat(User, FeedId, IdMessage2, <<"short">>, true), + + ct:log("User sees transcribe = ~p", [UserSeesTranscribe2]), + #'Desc'{id = TranscribeId, mime = <<"transcribe">>, payload = <<"this is a test">>} = lists:last(UserSeesTranscribe2), + + %% Now untranscribe this message + #mqtt_packet{ payload = #'Message'{status = update, files = UnTranscribe}} = + nynja:untranscribe_msg_chat(User, FeedId, IdMessage2, TranscribeId, false), + ct:log("User UnTranscribe = ~p", [UnTranscribe]), + ?assertEqual([], [ D || #'Desc'{mime = <<"transcribe">>} = D <- UnTranscribe]), + ok. + +translate(Cfg) -> + Phone = proplists:get_value(phone, Cfg, <<"000000038">>), + MemberPhone1 = <<"001002003">>, + MemberPhone2 = <<"001002004">>, + + + register_user([{phone, Phone}]), %% may fail if already registered + register_user([{phone, MemberPhone1}, {fname, <<"Kalle">>}]), + register_user([{phone, MemberPhone2}, {fname, <<"Lisa">>}]), + + AdminUser = nynja:connect_user(Phone), + User1 = nynja:connect_user(MemberPhone1), + User2 = nynja:connect_user(MemberPhone2), + RoomName = iolist_to_binary(["Translate-", integer_to_list(os:system_time())]), + + ct:log("Creating room ~p", [RoomName]), + nynja:create_room(AdminUser, #{name => RoomName}), + nynja:add_to_room(AdminUser, User1, RoomName, 1), + nynja:add_to_room(AdminUser, User2, RoomName, 1), + + %% cleanup all unreceived messages + nynja:ws_receive(User1, any, 10), + nynja:ws_receive(User2, any, 10), + + #mqtt_packet{ payload = #'Message'{} } = + nynja:msg_room(AdminUser, RoomName, <<"Welcome to our nynja room">>), + [ #mqtt_packet{ payload = #'Message'{id = IdMessage}} = WelcomeMsg] = nynja:ws_receive(User1), %% User1 gets this msg + [ #mqtt_packet{ payload = #'Message'{id = IdMessage}} ] = nynja:ws_receive(User2), %% User2 gets this msg + + ct:log("User sees ~p", [WelcomeMsg]), + + #mqtt_packet{ payload = #'Message'{id = IdMessage, status = update, files = Descs1}} = + nynja:translate_msg_room(User1, RoomName, IdMessage, <<"sv">>, false), + ct:log("User sees one translation ~p in sv", [Descs1]), + [ TRDesc1 ] = [ D || #'Desc'{mime = <<"translate">>} = D <- Descs1], + ?assertEqual(<<"sv">>, get_desc_value(<<"LANGUAGE">>, TRDesc1)), + + #mqtt_packet{ payload = #'Message'{id = IdMessage, status = update, files = Descs2}} = + nynja:translate_msg_room(User1, RoomName, IdMessage, <<"fr">>, false), + ct:log("User only sees second translation ~p in fr", [Descs2]), + [ TRDesc2 ] = [ D || #'Desc'{mime = <<"translate">>} = D <- Descs2], + ?assertEqual(<<"fr">>, get_desc_value(<<"LANGUAGE">>, TRDesc2)), + + _ = nynja:ws_receive(User2, any, 10), %% Second user and Admin have received translate messages + _ = nynja:ws_receive(AdminUser, any, 10), + + #mqtt_packet{ payload = #'Message'{id = IdMessage, status = update, files = Descs3}} = + nynja:translate_msg_room(User2, RoomName, IdMessage, <<"fr">>, false), + ct:log("User2 also reads it in french ~p in fr", [Descs3]), + [ #'Desc'{id = FrenchId} = TRDesc3 ] = [ D || #'Desc'{mime = <<"translate">>} = D <- Descs3], + ?assertEqual(<<"fr">>, get_desc_value(<<"LANGUAGE">>, TRDesc3)), + ?assertEqual(2, length(string:split(get_desc_value(<<"USERS">>, TRDesc3), ",", all))), + + _ = nynja:ws_receive(User1, any, 10), %% User and Admin have received translate messages + _ = nynja:ws_receive(AdminUser, any, 10), + + #mqtt_packet{ payload = #'Message'{id = IdMessage, status = update, files = Descs4}} = + nynja:translate_msg_room(User1, RoomName, IdMessage, <<"sv">>, false), + ct:log("User only sees translation ~p in sv other user sees it in fr", [Descs4]), + [ TRDesc4, TRDesc5 ] = [ D || #'Desc'{mime = <<"translate">>} = D <- Descs4], + Languages = [ get_desc_value(<<"LANGUAGE">>, TRDesc4), get_desc_value(<<"LANGUAGE">>, TRDesc5)], + ?assert(true, lists:member(<<"sv">>, Languages) andalso lists:member(<<"fr">>, Languages)), + + _ = nynja:ws_receive(User2, any, 10), %% Second user and Admin have received translate messages + _ = nynja:ws_receive(AdminUser, any, 10), + + %% Now try untranslate the french message with invalid client + #mqtt_packet{ payload = #'Message'{status = update, files = Descs5}} = + nynja:untranslate_msg_room(AdminUser, RoomName, IdMessage, FrenchId, false), + ct:log("AdminUser UnTranslate = ~p", [Descs5]), + ?assertEqual(3, length(Descs5)), %% welcome message and both translations + + %% Cleanup + _ = nynja:ws_receive(User1, any, 10), + _ = nynja:ws_receive(User2, any, 10), + + %% Now untranslate the french message with valid client + #mqtt_packet{ payload = #'Message'{status = update, files = Descs6}} = + nynja:untranslate_msg_room(User2, RoomName, IdMessage, FrenchId, false), + ct:log("User2 UnTranslate = ~p", [Descs6]), + [ TRDesc6 ] = [ D || #'Desc'{mime = <<"translate">>} = D <- Descs6], + ?assertEqual(<<"sv">>, get_desc_value(<<"LANGUAGE">>, TRDesc6)), + ?assertEqual(1, length(string:split(get_desc_value(<<"USERS">>, TRDesc6), ",", all))), + ok. %%% ------------- helpers --------------------- @@ -220,9 +352,9 @@ request(Id, Params, Config) -> {ok, Status, Headers, Body} = api_nynja:http_request(Request, [{timeout, 10000}], [], - [{logfun, fun(F, As) -> - ct:log(F, As) - end}]), + [{logfun, fun(F, As) -> + ct:log(F, As) + end}]), Response = api_nynja:validate_response(Id, Request, Status, Headers, Body), [ ct:log("Validated ~p\n", [Response]) || true], Response. @@ -234,3 +366,15 @@ basic_auth() -> Password = proplists:get_value(password, BasicAuth), {security, {authorization_basic, [#{username => User, password => Password}]}}. + +ack_and_msgid(Msgs) -> + case Msgs of + [#mqtt_packet{ payload = #'Message'{ id = IdM } }, #mqtt_packet{ payload = #'MessageAck'{} } ] -> IdM; + [#mqtt_packet{ payload = #'MessageAck'{} }, #mqtt_packet{ payload = #'Message'{ id = IdM } } ] -> IdM + end. + +get_desc_value(Key, Desc) -> + case roster:get_data(Key, Desc) of + [] -> undefined; + #'Feature'{value = V} -> V + end.