From 1cc10038600bed8d3200302b190561a92b7e0389 Mon Sep 17 00:00:00 2001 From: Tobias Lindahl Date: Thu, 14 May 2020 18:46:14 +0200 Subject: [PATCH 1/7] Restrict size of log entries --- sys.config | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/sys.config b/sys.config index a50aeb27c..6bd0fd3c9 100644 --- a/sys.config +++ b/sys.config @@ -12,7 +12,10 @@ , filesync_repeat_interval => 1000 }, formatter => {logger_formatter, - #{template => [time," [",level,"] ",pid," [", mfa, "] ", msg,"\n"]}} + #{template => [time," [",level,"] ",pid," [", mfa, "] ", msg,"\n"], + chars_limit => 4096, + depth => 20 + }} }}, {handler, info_handler, logger_disk_log_h, #{level => info, @@ -23,7 +26,10 @@ , filesync_repeat_interval => 1000 }, formatter => {logger_formatter, - #{template => [time," [",level,"] ",pid," [", mfa, "] ", msg,"\n"]}} + #{template => [time," [",level,"] ",pid," [", mfa, "] ", msg,"\n"], + chars_limit => 4096, + depth => 20 + }} }}, {handler, debug_handler, logger_disk_log_h, #{level => debug, @@ -35,7 +41,10 @@ }, filter_default => stop, formatter => {logger_formatter, - #{template => [time," [",level,"] ",pid," [", mfa, "] ", msg,"\n"]}} + #{template => [time," [",level,"] ",pid," [", mfa, "] ", msg,"\n"], + chars_limit => 4096, + depth => 20 + }} }} ]}]}, {review,[{host,"ns.synrc.com"}]}, -- GitLab From 062cbea31730251fe48f392881d81609ade9443e Mon Sep 17 00:00:00 2001 From: Tobias Lindahl Date: Thu, 14 May 2020 18:46:31 +0200 Subject: [PATCH 2/7] Lower number of log files in rotation --- sys.config | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/sys.config b/sys.config index 6bd0fd3c9..7c79fa832 100644 --- a/sys.config +++ b/sys.config @@ -7,7 +7,7 @@ #{level => error, config => #{ file => "log/erlang.log" , type => wrap - , max_no_files => 1000 + , max_no_files => 200 , max_no_bytes => 10000000 , filesync_repeat_interval => 1000 }, @@ -21,7 +21,7 @@ #{level => info, config => #{ file => "log/info.log" , type => wrap - , max_no_files => 1000 + , max_no_files => 200 , max_no_bytes => 10000000 , filesync_repeat_interval => 1000 }, @@ -35,7 +35,7 @@ #{level => debug, config => #{ file => "log/debug.log" , type => wrap - , max_no_files => 1000 + , max_no_files => 200 , max_no_bytes => 10000000 , filesync_repeat_interval => 1000 }, -- GitLab From 6411225b6a5e6104e4daa45365e37f4a811a1ea2 Mon Sep 17 00:00:00 2001 From: Tobias Lindahl Date: Fri, 15 May 2020 12:12:05 +0200 Subject: [PATCH 3/7] Add log backend proxy for n2o to logger --- apps/roster/src/roster_n2o_log_backend.erl | 18 ++++++++++++++++++ sys.config | 1 + 2 files changed, 19 insertions(+) create mode 100644 apps/roster/src/roster_n2o_log_backend.erl diff --git a/apps/roster/src/roster_n2o_log_backend.erl b/apps/roster/src/roster_n2o_log_backend.erl new file mode 100644 index 000000000..2d353d12a --- /dev/null +++ b/apps/roster/src/roster_n2o_log_backend.erl @@ -0,0 +1,18 @@ +%%%------------------------------------------------------------------- +%%% @doc Logging backend for n2o interfacing with logger. +%%% +%%% @end +%%%------------------------------------------------------------------- + +-module(roster_n2o_log_backend). + +-export([info/3, warning/3, error/3]). + +info(Module, FormatString, Args) -> + logger:info(FormatString, Args, #{mfa => {Module, 'undefined', 0}}). + +warning(Module, FormatString, Args) -> + logger:warning(FormatString, Args, #{mfa => {Module, 'undefined', 0}}). + +error(Module, FormatString, Args) -> + logger:error(FormatString, Args, #{mfa => {Module, 'undefined', 0}}). diff --git a/sys.config b/sys.config index a50aeb27c..3b8f63fb9 100644 --- a/sys.config +++ b/sys.config @@ -115,6 +115,7 @@ {protocols,[n2o_nitro,n2o_ftp,roster_proto]}, {log_modules,[roster, n2o_vnode]}, {log_level,roster}, + {logger, roster_n2o_log_backend}, {vnode, {roster, get_vnode}}, {validate, {roster, validate}} ]}, -- GitLab From 8c09af63fe900d90000715e8a327c065f4499d68 Mon Sep 17 00:00:00 2001 From: Tobias Lindahl Date: Fri, 15 May 2020 12:20:48 +0200 Subject: [PATCH 4/7] Make emqttc clients log with error_logger which will be captured by logger --- apps/roster/src/protocol/micro_auth.erl | 4 +++- apps/roster/src/protocol/roster_auth.erl | 2 +- apps/roster/src/protocol/roster_bpe.erl | 4 +++- apps/roster/src/protocol/roster_message.erl | 4 +++- apps/roster/src/rest/helpers/rest_main_helper.erl | 3 ++- apps/roster/src/roster_client.erl | 9 ++++++--- 6 files changed, 18 insertions(+), 8 deletions(-) diff --git a/apps/roster/src/protocol/micro_auth.erl b/apps/roster/src/protocol/micro_auth.erl index e97661e8f..aeca814a4 100644 --- a/apps/roster/src/protocol/micro_auth.erl +++ b/apps/roster/src/protocol/micro_auth.erl @@ -118,7 +118,9 @@ info(#'Auth'{phone = Phone} = Auth, Req, #cx{params = ClientId} = State) -> %% micro_auth handlers proc(init, #handler{name = micro_auth} = Async) -> - {ok, C} = emqttc:start_link([{client_id, <<"micro_auth">>}, {logger, {console, error}}, {reconnect, 5}]), + {ok, C} = emqttc:start_link([{client_id, <<"micro_auth">>}, + {logger, {error_logger, error}}, + {reconnect, 5}]), ?LOG_INFO("ASYNC AUTH PROC started", []), {ok, Async#handler{state = C, seq = 0}}; diff --git a/apps/roster/src/protocol/roster_auth.erl b/apps/roster/src/protocol/roster_auth.erl index 0c22755a7..da4a8dede 100644 --- a/apps/roster/src/protocol/roster_auth.erl +++ b/apps/roster/src/protocol/roster_auth.erl @@ -592,7 +592,7 @@ info(#'Auth'{phone = Phone} = Auth, proc(init, #handler{name = roster_auth} = Async) -> {ok, C} = emqttc:start_link([ {client_id, <<"roster_auth">>}, - {logger, {console, error}}, + {logger, {error_logger, error}}, {reconnect, 5} ]), catch locus:start_loader(city, application:get_env(locus, url, ?GEO_URL)), diff --git a/apps/roster/src/protocol/roster_bpe.erl b/apps/roster/src/protocol/roster_bpe.erl index 96b89d464..31a8e7a9e 100644 --- a/apps/roster/src/protocol/roster_bpe.erl +++ b/apps/roster/src/protocol/roster_bpe.erl @@ -196,7 +196,9 @@ info(#'Job'{feed_id = Feed} = RequestData, R, S) -> info(M, R, S) -> ?LOG_INFO("UNKNOWN:~w", [M]), {unknown, M, R, S}. proc(init, #handler{name = roster_bpe} = Async) -> - {ok, C} = emqttc:start_link([{client_id, <<"sys_bpe">>}, {logger, {console, error}}, {reconnect, 5}]), + {ok, C} = emqttc:start_link([{client_id, <<"sys_bpe">>}, + {logger, {error_logger, error}}, + {reconnect, 5}]), Table = process, Proc = case kvs:get(feed, Table) of {ok, #feed{top = Top} =_Feed} -> diff --git a/apps/roster/src/protocol/roster_message.erl b/apps/roster/src/protocol/roster_message.erl index 0469b1a96..d3b289946 100644 --- a/apps/roster/src/protocol/roster_message.erl +++ b/apps/roster/src/protocol/roster_message.erl @@ -448,7 +448,9 @@ info(#'Message'{from = From, to = To} = ReqData, Req, State) -> proc(init, #handler{name = roster_message} = Async) -> ?LOG_INFO("ASYNC:~p", [?MODULE]), - {ok, C} = emqttc:start_link([{client_id, <<"roster_message">>}, {logger, {console, error}}, {reconnect, 5}]), + {ok, C} = emqttc:start_link([{client_id, <<"roster_message">>}, + {logger, {error_logger, error}}, + {reconnect, 5}]), {ok, Async#handler{state = C, seq = 0}}; proc({send_push, From, To, #'Message'{type = TypeList} = Msg, Action}, #handler{} = H) -> diff --git a/apps/roster/src/rest/helpers/rest_main_helper.erl b/apps/roster/src/rest/helpers/rest_main_helper.erl index 33bc6dea6..2b46f3929 100644 --- a/apps/roster/src/rest/helpers/rest_main_helper.erl +++ b/apps/roster/src/rest/helpers/rest_main_helper.erl @@ -26,7 +26,8 @@ rest_pid() -> case whereis(?REST_HELPER_NAME) of undefined -> {ok, C} = emqttc:start_link([{client_id, ?SYS_REST_CLIENT}, - {logger, {console, error}}, {reconnect, 5}]), + {logger, {error_logger, error}}, + {reconnect, 5}]), register(?REST_HELPER_NAME, C), unlink(C), C; diff --git a/apps/roster/src/roster_client.erl b/apps/roster/src/roster_client.erl index ae94ab202..aec8cd2b4 100644 --- a/apps/roster/src/roster_client.erl +++ b/apps/roster/src/roster_client.erl @@ -74,7 +74,9 @@ proc(init, #handler{name = ?SYS_REST_CLIENT, state = #state{receive_pid = Self} Self ! init, {ok, Async#handler{state = State#state{mqttc = C, client_id = ?SYS_REST_CLIENT}, seq = 0}}; proc(init, #handler{name = ?SYS_BRIDGE_TEST_CLIENT, state = #state{receive_pid = Self} = State} = Async) -> - {ok, C} = emqttc:start_link([{client_id, ?SYS_BRIDGE_TEST_CLIENT}, {logger, {console, error}}, {reconnect, 5}]), + {ok, C} = emqttc:start_link([{client_id, ?SYS_BRIDGE_TEST_CLIENT}, + {logger, {error_logger, error}}, + {reconnect, 5}]), ?LOG_INFO("MICRO BRIDGE SIMULATOR PROC started", []), register(sys_bridge, C), Self ! init, @@ -82,8 +84,9 @@ proc(init, #handler{name = ?SYS_BRIDGE_TEST_CLIENT, state = #state{receive_pid = proc(init, #handler{name = ClientId, state = #state{mqtt_opts = Opts, receive_pid = Self, username = Username} = State} = Async) -> ?LOG_INFO("ClientInit:~p\r", [ClientId]), {ok, C} = emqttc:start_link([{client_id, ClientId}, - {clean_sess, false}, {logger, {console, error}}, - {username, Username}, {reconnect, 5}] ++ Opts), + {clean_sess, false}, + {logger, {error_logger, error}}, + {username, Username}, {reconnect, 5}] ++ Opts), Self ! init, {ok, Async#handler{state = State#state{mqttc = C, client_id = ClientId}, seq = 0}}; -- GitLab From 0afb391cfc2726334d3372b2f58b49ac69f9b6f2 Mon Sep 17 00:00:00 2001 From: Tobias Lindahl Date: Fri, 15 May 2020 12:43:22 +0200 Subject: [PATCH 5/7] Make sure esockd application uses error_logger as logger Which will be forwarded to logger --- sys.config | 1 + 1 file changed, 1 insertion(+) diff --git a/sys.config b/sys.config index 3b8f63fb9..b147321d4 100644 --- a/sys.config +++ b/sys.config @@ -210,6 +210,7 @@ {large_heap,67108864}, {busy_port,false}, {busy_dist_port,true}]}]}, + {esockd, [{logger, {error_logger, info}}]}, {kvs, [{dba,store_mnesia}, {schema, [kvs_user, kvs_acl, kvs_feed, kvs_subscription, roster, micro, emqttd_kvs, bpe_metainfo]} %% ,{generation, {roster_test, limit}} -- GitLab From e42415c51be5ab654fd7c6b414fec9a365cf23ed Mon Sep 17 00:00:00 2001 From: sennui Date: Fri, 15 May 2020 14:05:02 +0200 Subject: [PATCH 6/7] release notes 1.1 --- doc/release-notes/RELEASE-NOTES-1.1.md | 47 ++++++++++++++++++++++++++ 1 file changed, 47 insertions(+) create mode 100644 doc/release-notes/RELEASE-NOTES-1.1.md diff --git a/doc/release-notes/RELEASE-NOTES-1.1.md b/doc/release-notes/RELEASE-NOTES-1.1.md new file mode 100644 index 000000000..ba9261593 --- /dev/null +++ b/doc/release-notes/RELEASE-NOTES-1.1.md @@ -0,0 +1,47 @@ +# Release 1.1! + +This is the first release of Nynja with Erlang release mechanism! +We are happy to inform that it will run Erlang/OTP21. + +### Highlights + +* Upgrade to Erlang/OTP21 for performance and maintainability +* Prepare database for future distributed setup +* Initial fixes of Unread, History and History Limit +* Support of SignUp +* Number of internal fixes and server side improvements like logging + + +### List of changes from the newest + +* Multiple fixes and changes to logging mechanics + (handling large outputs) +* Further fixes to releases +* Multiple fix-ups of REST API caused by webserver upgrade +* Add redbug to the release (debugging software) +* Introduce local build scripts that use docker equivalent of production +* Replace expired push certificates +* Fix History get and double_get +* Improve Thumbnail validation +* Implement message send to groups (SignUp) +* Handle all states of "Friendship" in p2p REST API +* Use stdlib in number of places, remove some of unused code +* Fix all compilation warnings +* Fix transcribe, improve handling of amazon_api +* Fix MIME type handling of REST APIs +* Unify loggers, migrate to OTP logger, extend logging +* Extend Makefile with test targets +* Improve validation and validation module handling +* Progress with application tree cleanup +* Discontinue unsupported webserves for REST; migrate to cowboy +* Manage forks of dependencies, improve & move to our github org +* Remove potential dead loop on bad Friend structure +* Erlang/OTP21 & rebar3 +* Build OAM module for Mnesia db (upgrades, syncing etc.) +* History Limit - Message/delete support +* History Limit - Message/edit support +* Remove noisy crashes from production log +* Improve internal shared_room computation +* Support accountId in get_profile (SignUp) +* Fix Unread in case of calls of sub-group + -- GitLab From b435239e64d9360704c9537ce3ec89d1fb922d25 Mon Sep 17 00:00:00 2001 From: Thomas Arts Date: Fri, 15 May 2020 14:51:07 +0200 Subject: [PATCH 7/7] Disable the ability to use ^C in Erlang attached node. One should use bin/server remote_console, but for those that have the wrong fingers. --- vm.args | 1 + 1 file changed, 1 insertion(+) diff --git a/vm.args b/vm.args index 4bea209fa..17f9f1963 100644 --- a/vm.args +++ b/vm.args @@ -5,6 +5,7 @@ -mnesia dump_log_write_threshold 50000 +Q 524288 +A 32 ++Bi +K true -smp auto -setcookie emq_dist_cookie -- GitLab