diff --git a/doc/load_testing.md b/doc/load_testing.md new file mode 100644 index 0000000000000000000000000000000000000000..5c9bd10b84a267de0f25fdf0a80b511ab61e7bef --- /dev/null +++ b/doc/load_testing.md @@ -0,0 +1,143 @@ +# Load Testing + +This document describes how to run the load tests (very much WIP) and some results. + +## Run load test + +By default the load tests run towards localhost (127.0.0.1) but by setting the +ENV variable `NYNJA_HOST` they can easily be pointed towards any running +server. Also the load tests expect a fresh set of users, this is achieved by +prefix:ing every user in the test. The prefix is stored in Nynja, and if the +load tests have not been run before the prefix has to be initialized. Thus +running some load tests could look like: + +``` +$ ./rebar3 as test compile +$ cd eqc/ebin +$ . shell +1> make:all([load]). +... +up_to_date +2> nynja:init_prefix(). +{ok, fresh} %% or {ok, existed} +3> os:putenv("NYNJA_HOST", "127.0.0.1"). +true +4> load_test:do_test(#{setup_workers => 10, users => 150, stagger => 3000, every => 1500, for => 20000, + p2p_cfg => #{p2p => 150}, room_cfg => #{rooms => 50, max_size => 10}}). +--- SETUP DONE (in 97.48 ms) + +--- LOAD TEST REPORT + check_friends : Total = 2939.73 ms, Throughput = 19.60 ms, N = 150, Avg = 2.24 ms, Max = 11.44 ms, StdDev: 1.05 ms + connect : Total = 2995.12 ms, Throughput = 9.98 ms, N = 300, Avg = 25.80 ms, Max = 34.96 ms, StdDev: 2.92 ms + create_room : Total = 3077.24 ms, Throughput = 61.54 ms, N = 50, Avg = 8.24 ms, Max = 15.45 ms, StdDev: 4.07 ms + make_friends : Total = 2943.25 ms, Throughput = 19.75 ms, N = 149, Avg = 5.83 ms, Max = 10.95 ms, StdDev: 1.12 ms + send_p2p : Total = 22511.24 ms, Throughput = 10.72 ms, N = 2100, Avg = 2.73 ms, Max = 10.61 ms, StdDev: 0.61 ms + send_room : Total = 22865.99 ms, Throughput = 32.67 ms, N = 700, Avg = 3.11 ms, Max = 9.43 ms, StdDev: 0.70 ms +``` + +The parameters are: + - `setup_workers` - number of parallel processes used to create users - use ~10 locally and ~50 remotely. + - `users` - number of users in the tests + - `stagger` - spread the start of the parallel scenarios over this amount of time (default 1000) + - `every` - how often each actor should act + - `for` - how long the actor should run + - `p2p_cfg` - configuration for P2P scenario (can include `users`, `stagger`, `every`, and `for` if individual values for P2P should be used). + - `p2p` - how many friends pairs to use + - `room_cfg` - configuration for Room scenario (can include `users`, `stagger`, `every`, and `for` if individual values for rooms should be used). + - `rooms` - number of rooms created + - `max_size` - maximum room size (minimum is 2) + +## Load test results + +### Local tests - MacBook Pro (2,7 GHz Intel Core i7, 16 GB) + +Using 500 users, 300 P2P-pairs and 300 rooms (size 2-10 users) + +With one action every 1500ms (i.e. 400 msgs per second): +``` +8> load_test:do_test(#{setup_workers => 12, users => 500, stagger => 3000, every => 1500, for => 30000, p2p_cfg => #{p2p => 300}, room_cfg => #{rooms => 300, max_size => 10}}). +--- SETUP DONE (in 254.04 ms) + +--- LOAD TEST REPORT + check_friends : Total = 3112.09 ms, Throughput = 10.37 ms, N = 300, Avg = 17.70 ms, Max = 128.15 ms, StdDev: 25.62 ms + connect : Total = 3162.89 ms, Throughput = 5.27 ms, N = 600, Avg = 40.81 ms, Max = 146.79 ms, StdDev: 21.12 ms + create_room : Total = 3320.24 ms, Throughput = 11.07 ms, N = 300, Avg = 40.34 ms, Max = 132.56 ms, StdDev: 30.46 ms + make_friends : Total = 3197.83 ms, Throughput = 10.66 ms, N = 300, Avg = 34.96 ms, Max = 167.11 ms, StdDev: 37.71 ms + send_p2p : Total = 32991.18 ms, Throughput = 5.39 ms, N = 6116, Avg = 12.66 ms, Max = 314.85 ms, StdDev: 27.76 ms + send_room : Total = 33201.38 ms, Throughput = 5.43 ms, N = 6118, Avg = 13.06 ms, Max = 286.25 ms, StdDev: 26.96 ms +``` + +Ramping it up to one action every 1000ms we see saturation: + +``` +9> load_test:do_test(#{setup_workers => 12, users => 500, stagger => 3000, every => 1000, for => 30000, p2p_cfg => #{p2p => 300}, room_cfg => #{rooms => 300, max_size => 10}}). +--- SETUP DONE (in 272.69 ms) + +--- LOAD TEST REPORT + check_friends : Total = 3533.02 ms, Throughput = 11.78 ms, N = 300, Avg = 47.57 ms, Max = 259.80 ms, StdDev: 53.51 ms + connect : Total = 3368.27 ms, Throughput = 5.61 ms, N = 600, Avg = 105.47 ms, Max = 475.34 ms, StdDev: 88.23 ms + create_room : Total = 3920.42 ms, Throughput = 13.07 ms, N = 300, Avg = 104.43 ms, Max = 357.81 ms, StdDev: 66.37 ms + make_friends : Total = 3792.28 ms, Throughput = 12.68 ms, N = 299, Avg = 109.29 ms, Max = 489.62 ms, StdDev: 98.47 ms + send_p2p : Total = 33990.79 ms, Throughput = 5.23 ms, N = 6495, Avg = 313.13 ms, Max = 1839.77 ms, StdDev: 230.72 ms + send_room : Total = 33691.56 ms, Throughput = 4.36 ms, N = 7732, Avg = 336.03 ms, Max = 1235.90 ms, StdDev: 218.27 ms +``` + +Turning off logging improves the situation: +``` +12> load_test:do_test(#{setup_workers => 12, users => 500, stagger => 3000, every => 1000, for => 30000, p2p_cfg => #{p2p => 300}, room_cfg => #{rooms => 300, max_size => 10}}). +--- SETUP DONE (in 168.66 ms) + +--- LOAD TEST REPORT + check_friends : Total = 2970.82 ms, Throughput = 9.90 ms, N = 300, Avg = 2.15 ms, Max = 21.10 ms, StdDev: 2.52 ms + connect : Total = 3025.21 ms, Throughput = 5.04 ms, N = 600, Avg = 25.75 ms, Max = 38.51 ms, StdDev: 3.15 ms + create_room : Total = 3144.94 ms, Throughput = 10.48 ms, N = 300, Avg = 10.66 ms, Max = 35.80 ms, StdDev: 5.99 ms + make_friends : Total = 2973.85 ms, Throughput = 9.91 ms, N = 300, Avg = 5.81 ms, Max = 26.74 ms, StdDev: 3.49 ms + send_p2p : Total = 32984.35 ms, Throughput = 3.65 ms, N = 9025, Avg = 49.70 ms, Max = 1235.68 ms, StdDev: 79.81 ms + send_room : Total = 33039.19 ms, Throughput = 3.65 ms, N = 9043, Avg = 51.75 ms, Max = 583.77 ms, StdDev: 79.27 ms +``` + +### Remote tests = staging2 (34.82.252.134) + +To account for the longer TCP roundtrip we increase the number of `setup_workers` and `stagger`. + +It works (but the roundtrip makes the numbers hard to compare), it looks to be a bit saturated: +``` +14> load_test:do_test(#{setup_workers => 60, users => 500, stagger => 5000, every => 1500, for => 60000, p2p_cfg => #{p2p => 300}, room_cfg => #{rooms => 300, max_size => 10}}). +--- SETUP DONE (in 3944.74 ms) + +--- LOAD TEST REPORT + check_friends : Total = 6036.14 ms, Throughput = 20.12 ms, N = 300, Avg = 225.81 ms, Max = 488.03 ms, StdDev: 69.41 ms + connect : Total = 6276.18 ms, Throughput = 10.46 ms, N = 600, Avg = 533.31 ms, Max = 924.94 ms, StdDev: 228.85 ms + create_room : Total = 14538.03 ms, Throughput = 48.46 ms, N = 300, Avg = 499.67 ms, Max = 1897.40 ms, StdDev: 312.09 ms + make_friends : Total = 6634.62 ms, Throughput = 22.12 ms, N = 300, Avg = 685.18 ms, Max = 907.18 ms, StdDev: 62.30 ms + send_p2p : Total = 71207.10 ms, Throughput = 13.21 ms, N = 5392, Avg = 1162.62 ms, Max = 4131.50 ms, StdDev: 1058.66 ms + send_room : Total = 73278.76 ms, Throughput = 10.98 ms, N = 6672, Avg = 1302.62 ms, Max = 4168.00 ms, StdDev: 1143.75 ms +``` + +Turning off logging: +``` +15> load_test:do_test(#{setup_workers => 60, users => 500, stagger => 5000, every => 1500, for => 60000, p2p_cfg => #{p2p => 300}, room_cfg => #{rooms => 300, max_size => 10}}). +--- SETUP DONE (in 3936.78 ms) + +--- LOAD TEST REPORT + check_friends : Total = 6866.24 ms, Throughput = 22.89 ms, N = 300, Avg = 268.59 ms, Max = 628.72 ms, StdDev: 79.62 ms + connect : Total = 6103.51 ms, Throughput = 10.17 ms, N = 600, Avg = 505.26 ms, Max = 1007.03 ms, StdDev: 232.28 ms + create_room : Total = 9863.89 ms, Throughput = 32.88 ms, N = 300, Avg = 265.12 ms, Max = 466.60 ms, StdDev: 70.65 ms + make_friends : Total = 7440.43 ms, Throughput = 24.80 ms, N = 300, Avg = 693.47 ms, Max = 1122.52 ms, StdDev: 106.96 ms + send_p2p : Total = 66682.35 ms, Throughput = 8.27 ms, N = 8062, Avg = 703.26 ms, Max = 2308.39 ms, StdDev: 555.51 ms + send_room : Total = 67929.38 ms, Throughput = 6.91 ms, N = 9836, Avg = 765.28 ms, Max = 2119.36 ms, StdDev: 560.12 ms +``` + +And finally with a bit more users: +``` +2> load_test:do_test(#{setup_workers => 100, users => 1000, stagger => 15000, every => 20000, for => 120000, p2p_cfg => #{p2p => 500}, room_cfg => #{rooms => 1000, max_size => 10}}). +--- SETUP DONE (in 4859.73 ms) + +--- LOAD TEST REPORT + check_friends : Total = 19288.50 ms, Throughput = 38.58 ms, N = 500, Avg = 353.93 ms, Max = 2001.90 ms, StdDev: 265.19 ms + connect : Total = 17348.66 ms, Throughput = 17.35 ms, N = 1000, Avg = 582.46 ms, Max = 2065.91 ms, StdDev: 259.11 ms + create_room : Total = 25174.11 ms, Throughput = 25.17 ms, N = 1000, Avg = 756.28 ms, Max = 2950.40 ms, StdDev: 603.22 ms + make_friends : Total = 21898.45 ms, Throughput = 43.80 ms, N = 500, Avg = 1012.48 ms, Max = 4157.28 ms, StdDev: 646.03 ms + send_p2p : Total = 141291.54 ms, Throughput = 43.53 ms, N = 3246, Avg = 401.54 ms, Max = 3094.39 ms, StdDev: 290.48 ms + send_room : Total = 143369.10 ms, Throughput = 22.04 ms, N = 6506, Avg = 412.61 ms, Max = 2973.66 ms, StdDev: 307.97 ms +``` diff --git a/test/load_test.erl b/test/load_test.erl index 1a73e76154be63f4da4d09be900d9b95bba4b00c..8c6a157398bf594a4111d20d39a6152046246bca 100644 --- a/test/load_test.erl +++ b/test/load_test.erl @@ -20,8 +20,9 @@ %% - Connections (web socket), one or more per User %% - User %% - Scenario, one or more Connections/Users -do_test(Cfg) -> - run(Cfg, in_parallel(Cfg#{fanout => 1}, [test2(Cfg), test_room(Cfg)])). +do_test(Cfg = #{ room_cfg := RoomCfg, p2p_cfg := P2PCfg }) -> + run(Cfg, in_parallel(Cfg#{fanout => 1}, [test2(maps:merge(Cfg, P2PCfg)), + test_room(maps:merge(Cfg, RoomCfg))])). test(Cfg) -> N = maps:get(p2p, Cfg, 100), @@ -483,6 +484,7 @@ connect_user(Client, DeviceId) when is_integer(DeviceId) -> connect_user(Client, connect_user(Client, Tag) -> connect_user(Client, 1, Tag). connect_user(Client, DeviceId, Tag) -> connect(Client, DeviceId, Tag), + timer:sleep(20), send(Client, DeviceId, Tag, #'Profile'{ status = get }), #'Profile'{ phone = Phone, rosters = [#'Roster'{ id = RosterIx }] } = recv(Client, DeviceId, Tag, ?any('Profile')),