Sign Up
Log In
Log In
or
Sign Up
Places
All Projects
Status Monitor
Collapse sidebar
home:Ledest:erlang:24
erlang
4125-Restructure-benchmark-to-always-run-at-lea...
Overview
Repositories
Revisions
Requests
Users
Attributes
Meta
File 4125-Restructure-benchmark-to-always-run-at-least-reduced.patch of Package erlang
From a0a9169bee39cb7191cee86af4ef2977d6c2130f Mon Sep 17 00:00:00 2001 From: Raimo Niskanen <raimo@erlang.org> Date: Thu, 29 Sep 2022 11:49:16 +0200 Subject: [PATCH 05/27] Restructure benchmark to always run, at least reduced --- lib/ssl/test/ssl.spec | 12 +- lib/ssl/test/ssl_bench.spec | 5 +- lib/ssl/test/ssl_bench_SUITE.erl | 83 +++++++--- lib/ssl/test/ssl_dist_bench_SUITE.erl | 220 ++++++++++++++------------ 4 files changed, 184 insertions(+), 136 deletions(-) diff --git a/lib/ssl/test/ssl.spec b/lib/ssl/test/ssl.spec index 15587abecd..c6d6aa3a8b 100644 --- a/lib/ssl/test/ssl.spec +++ b/lib/ssl/test/ssl.spec @@ -1,12 +1,2 @@ % {merge_tests,false}. -{alias,dir,"../ssl_test"}. - -{suites,dir,all}. -{skip_groups,dir,ssl_bench_SUITE,setup,"Benchmarks run separately"}. -{skip_groups,dir,ssl_bench_SUITE,payload,"Benchmarks run separately"}. -{skip_groups,dir,ssl_bench_SUITE,pem_cache,"Benchmarks run separately"}. -{skip_groups,dir,ssl_dist_bench_SUITE,setup,"Benchmarks run separately"}. -{skip_groups,dir,ssl_dist_bench_SUITE,roundtrip,"Benchmarks run separately"}. -{skip_groups,dir,ssl_dist_bench_SUITE,throughput,"Benchmarks run separately"}. -{skip_groups,dir,ssl_dist_bench_SUITE,sched_utilization,"Benchmarks run separately"}. - +{suites, "../ssl_test", all}. diff --git a/lib/ssl/test/ssl_bench.spec b/lib/ssl/test/ssl_bench.spec index 217cc6fc83..38c8b35cf4 100644 --- a/lib/ssl/test/ssl_bench.spec +++ b/lib/ssl/test/ssl_bench.spec @@ -1,2 +1,3 @@ -{suites,"../ssl_test",[ssl_bench_SUITE, ssl_dist_bench_SUITE]}. -{skip_groups,"../ssl_test",ssl_bench_SUITE,basic,"Benchmarks run separately"}. +{alias, dir, "../ssl_test"}. +{groups, dir, ssl_dist_bench_SUITE, [benchmark]}. +{groups, dir, ssl_bench_SUITE, [benchmark]}. diff --git a/lib/ssl/test/ssl_bench_SUITE.erl b/lib/ssl/test/ssl_bench_SUITE.erl index 104c373d26..908aa4540a 100644 --- a/lib/ssl/test/ssl_bench_SUITE.erl +++ b/lib/ssl/test/ssl_bench_SUITE.erl @@ -60,38 +60,64 @@ test/2 ]). + +-define(COUNT, 400). +-define(REDUCE, 40). % (?COUNT rem ?REDUCE) should be 0 +%% (?COUNT div ?REDUCE) is the count used for a non-benchmark run + %%-------------------------------------------------------------------- %% Common Test interface functions ----------------------------------- %%-------------------------------------------------------------------- suite() -> [{ct_hooks,[{ts_install_cth,[{nodenames,2}]}]}]. -all() -> [{group, setup}, {group, payload}]. +all() -> + %% {repeat,_} here overrides {repeat,_} in groups() + [{group, setup, [{repeat,1}]}, + {group, payload, [{repeat,1}]}]. groups() -> - [{setup, [{repeat, 3}], [setup_sequential, setup_sequential_noreuse, setup_sequential_13, - setup_concurrent, setup_concurrent_noreuse, setup_concurrent_13]}, - {payload, [{repeat, 3}], [payload, payload_13]} + [{benchmark, [{group, G} || {group, G, _} <- all()]}, + %% + {setup, [{repeat,3}], + [setup_sequential, setup_sequential_noreuse, setup_sequential_13, + setup_concurrent, setup_concurrent_noreuse, setup_concurrent_13]}, + {payload, [{repeat,3}], + [payload, payload_13]} ]. -init_per_group(_GroupName, Config) -> - Config. - -end_per_group(_GroupName, _Config) -> - ok. +%%------- init_per_suite(Config) -> ct:timetrap({minutes, 1}), - case node() of - nonode@nohost -> - {skipped, "Node not distributed"}; - _ -> - ssl_test_lib:clean_start(), - [{server_node, ssl_bench_test_lib:setup(perf_server)}|Config] + Skipped = make_ref(), + try + node() =/= nonode@nohost + orelse throw({Skipped, "Node not distributed"}), + %% + ssl_test_lib:clean_start(), + [{server_node, ssl_bench_test_lib:setup(perf_server)}|Config] + catch + throw : {Skipped, Reason} -> + {skipped, Reason}; + Class : Reason : Stacktrace -> + {failed, {Class, Reason, Stacktrace}} end. end_per_suite(_Config) -> ok. +%%------- + +init_per_group(benchmark, Config) -> + [{effort,?REDUCE} | Config]; +init_per_group(_GroupName, Config) -> + Config. + +end_per_group(_GroupName, _Config) -> + ok. + +%%------- + init_per_testcase(_Func, Conf) -> Conf. @@ -103,8 +129,9 @@ end_per_testcase(_Func, _Conf) -> %%-------------------------------------------------------------------- --define(COUNT, 400). %%-define(TC(Cmd), tc(fun() -> Cmd end, ?MODULE, ?LINE)). +count(Config) -> + (proplists:get_value(effort, Config, 1) * ?COUNT) div ?REDUCE. -define(FPROF_CLIENT, false). -define(FPROF_SERVER, false). @@ -117,7 +144,8 @@ setup_sequential(Config) -> Server = proplists:get_value(server_node, Config), Server =/= undefined orelse error(no_server), Cfg = [{version, 'tlsv1.2'}], - {ok, Result} = do_test(ssl, {setup_connection,Cfg}, ?COUNT * 20, 1, Server), + {ok, Result} = + do_test(ssl, {setup_connection,Cfg}, 20*count(Config), 1, Server), ct_event:notify(#event{name = benchmark_data, data=[{value, Result}, {suite, "ssl"}, {name, "Sequential setup"}]}), @@ -127,7 +155,8 @@ setup_sequential_noreuse(Config) -> Server = proplists:get_value(server_node, Config), Server =/= undefined orelse error(no_server), Cfg = [{version, 'tlsv1.2'}, no_reuse], - {ok, Result} = do_test(ssl, {setup_connection,Cfg}, ?COUNT * 20, 1, Server), + {ok, Result} = + do_test(ssl, {setup_connection,Cfg}, 20*count(Config), 1, Server), ct_event:notify(#event{name = benchmark_data, data=[{value, Result}, {suite, "ssl"}, {name, "Seq setup 1.2 no session"}]}), @@ -137,7 +166,8 @@ setup_sequential_13(Config) -> Server = proplists:get_value(server_node, Config), Server =/= undefined orelse error(no_server), Cfg = [{version, 'tlsv1.3'}], - {ok, Result} = do_test(ssl, {setup_connection,Cfg}, ?COUNT * 20, 1, Server), + {ok, Result} = + do_test(ssl, {setup_connection,Cfg}, 20*count(Config), 1, Server), ct_event:notify(#event{name = benchmark_data, data=[{value, Result}, {suite, "ssl"}, {name, "Seq setup 1.3"}]}), @@ -147,7 +177,8 @@ setup_concurrent(Config) -> Server = proplists:get_value(server_node, Config), Server =/= undefined orelse error(no_server), Cfg = [{version, 'tlsv1.2'}], - {ok, Result} = do_test(ssl, {setup_connection,Cfg}, ?COUNT, 100, Server), + {ok, Result} = + do_test(ssl, {setup_connection,Cfg}, count(Config), 100, Server), ct_event:notify(#event{name = benchmark_data, data=[{value, Result}, {suite, "ssl"}, {name, "Concurrent setup"}]}), @@ -157,7 +188,8 @@ setup_concurrent_noreuse(Config) -> Server = proplists:get_value(server_node, Config), Server =/= undefined orelse error(no_server), Cfg = [{version, 'tlsv1.2'}, no_reuse], - {ok, Result} = do_test(ssl, {setup_connection,Cfg}, ?COUNT, 100, Server), + {ok, Result} = + do_test(ssl, {setup_connection,Cfg}, count(Config), 100, Server), ct_event:notify(#event{name = benchmark_data, data=[{value, Result}, {suite, "ssl"}, {name, "Conc setup 1.2 no session"}]}), @@ -167,7 +199,8 @@ setup_concurrent_13(Config) -> Server = proplists:get_value(server_node, Config), Server =/= undefined orelse error(no_server), Cfg = [{version, 'tlsv1.3'}], - {ok, Result} = do_test(ssl, {setup_connection,Cfg}, ?COUNT, 100, Server), + {ok, Result} = + do_test(ssl, {setup_connection,Cfg}, count(Config), 100, Server), ct_event:notify(#event{name = benchmark_data, data=[{value, Result}, {suite, "ssl"}, {name, "Conc setup 1.3"}]}), @@ -177,7 +210,8 @@ payload(Config) -> Server = proplists:get_value(server_node, Config), Server =/= undefined orelse error(no_server), Cfg = [{version, 'tlsv1.2'}], - {ok, Result} = do_test(ssl, {payload, Cfg}, ?COUNT*300, 10, Server), + {ok, Result} = + do_test(ssl, {payload, Cfg}, 300*count(Config), 10, Server), ct_event:notify(#event{name = benchmark_data, data=[{value, Result}, {suite, "ssl"}, {name, "Payload simple"}]}), @@ -187,7 +221,8 @@ payload_13(Config) -> Server = proplists:get_value(server_node, Config), Server =/= undefined orelse error(no_server), Cfg = [{version, 'tlsv1.3'}], - {ok, Result} = do_test(ssl, {payload, Cfg}, ?COUNT*300, 10, Server), + {ok, Result} = + do_test(ssl, {payload, Cfg}, 300*count(Config), 10, Server), ct_event:notify(#event{name = benchmark_data, data=[{value, Result}, {suite, "ssl"}, {name, "Payload 1.3"}]}), diff --git a/lib/ssl/test/ssl_dist_bench_SUITE.erl b/lib/ssl/test/ssl_dist_bench_SUITE.erl index 58d9b92e0b..164ccfd99b 100644 --- a/lib/ssl/test/ssl_dist_bench_SUITE.erl +++ b/lib/ssl/test/ssl_dist_bench_SUITE.erl @@ -57,7 +57,9 @@ all() -> {group, plain}]. groups() -> - [{ssl, all_groups()}, + [{benchmark, all()}, + %% + {ssl, all_groups()}, {crypto, all_groups()}, {plain, all_groups()}, %% @@ -92,19 +94,20 @@ init_per_suite(Config) -> prf => sha256}, %% Node = node(), + Skipped = make_ref(), try Node =/= nonode@nohost orelse - throw({skipped,"Node not distributed"}), + throw({Skipped,"Node not distributed"}), verify_node_src_addr(), {supported, SSLVersions} = lists:keyfind(supported, 1, ssl:versions()), lists:member(TLSVersion, SSLVersions) orelse throw( - {skipped, + {Skipped, "SSL does not support " ++ term_to_string(TLSVersion)}), lists:member(ECCurve, ssl:eccs(TLSVersion)) orelse throw( - {skipped, + {Skipped, "SSL does not support " ++ term_to_string(ECCurve)}), TLSCipherKeys = maps:keys(TLSCipher), lists:any( @@ -113,64 +116,66 @@ init_per_suite(Config) -> end, ssl:cipher_suites(default, TLSVersion)) orelse throw( - {skipped, - "SSL does not support " ++ term_to_string(TLSCipher)}) - of - _ -> - PrivDir = proplists:get_value(priv_dir, Config), - %% - [_, HostA] = split_node(Node), - NodeAName = ?MODULE_STRING ++ "_node_a", - NodeAString = NodeAName ++ "@" ++ HostA, - NodeAConfFile = filename:join(PrivDir, NodeAString ++ ".conf"), - NodeA = list_to_atom(NodeAString), - %% - ServerNode = ssl_bench_test_lib:setup(dist_server), - [_, HostB] = split_node(ServerNode), - NodeBName = ?MODULE_STRING ++ "_node_b", - NodeBString = NodeBName ++ "@" ++ HostB, - NodeBConfFile = filename:join(PrivDir, NodeBString ++ ".conf"), - NodeB = list_to_atom(NodeBString), - %% - CertOptions = - [{digest, Digest}, - {key, {namedCurve, ECCurve}}], - RootCert = - public_key:pkix_test_root_cert( - ?MODULE_STRING ++ " ROOT CA", CertOptions), - SSLConf = - [{verify, verify_peer}, - {versions, [TLSVersion]}, - {ciphers, [TLSCipher]}], - ServerConf = - [{fail_if_no_peer_cert, true}, - {verify_fun, - {fun inet_tls_dist:verify_client/3,[]}} - | SSLConf], - ClientConf = SSLConf, - %% - write_node_conf( - NodeAConfFile, NodeA, ServerConf, ClientConf, - CertOptions, RootCert), - write_node_conf( - NodeBConfFile, NodeB, ServerConf, ClientConf, - CertOptions, RootCert), - %% - [{node_a_name, NodeAName}, - {node_a, NodeA}, - {node_a_dist_args, - "-proto_dist inet_tls " - "-ssl_dist_optfile " ++ NodeAConfFile ++ " "}, - {node_b_name, NodeBName}, - {node_b, NodeB}, - {node_b_dist_args, - "-proto_dist inet_tls " - "-ssl_dist_optfile " ++ NodeBConfFile ++ " "}, - {server_node, ServerNode} - |Config] + {Skipped, + "SSL does not support " ++ term_to_string(TLSCipher)}), + %% + %% + %% + PrivDir = proplists:get_value(priv_dir, Config), + [_, HostA] = split_node(Node), + NodeAName = ?MODULE_STRING ++ "_node_a", + NodeAString = NodeAName ++ "@" ++ HostA, + NodeAConfFile = filename:join(PrivDir, NodeAString ++ ".conf"), + NodeA = list_to_atom(NodeAString), + %% + ServerNode = ssl_bench_test_lib:setup(dist_server), + [_, HostB] = split_node(ServerNode), + NodeBName = ?MODULE_STRING ++ "_node_b", + NodeBString = NodeBName ++ "@" ++ HostB, + NodeBConfFile = filename:join(PrivDir, NodeBString ++ ".conf"), + NodeB = list_to_atom(NodeBString), + %% + CertOptions = + [{digest, Digest}, + {key, {namedCurve, ECCurve}}], + RootCert = + public_key:pkix_test_root_cert( + ?MODULE_STRING ++ " ROOT CA", CertOptions), + SSLConf = + [{verify, verify_peer}, + {versions, [TLSVersion]}, + {ciphers, [TLSCipher]}], + ServerConf = + [{fail_if_no_peer_cert, true}, + {verify_fun, + {fun inet_tls_dist:verify_client/3,[]}} + | SSLConf], + ClientConf = SSLConf, + %% + write_node_conf( + NodeAConfFile, NodeA, ServerConf, ClientConf, + CertOptions, RootCert), + write_node_conf( + NodeBConfFile, NodeB, ServerConf, ClientConf, + CertOptions, RootCert), + %% + [{node_a_name, NodeAName}, + {node_a, NodeA}, + {node_a_dist_args, + "-proto_dist inet_tls " + "-ssl_dist_optfile " ++ NodeAConfFile ++ " "}, + {node_b_name, NodeBName}, + {node_b, NodeB}, + {node_b_dist_args, + "-proto_dist inet_tls " + "-ssl_dist_optfile " ++ NodeBConfFile ++ " "}, + {server_node, ServerNode} + |Config] catch - throw:Result -> - Result + throw : {Skipped, Reason} -> + {skipped, Reason}; + Class : Reason : Stacktrace -> + {failed, {Class, Reason, Stacktrace}} end. end_per_suite(Config) -> @@ -186,20 +191,33 @@ init_per_group(crypto, Config) -> |Config]; init_per_group(plain, Config) -> [{ssl_dist, false}, {ssl_dist_prefix, "Plain"}|Config]; +init_per_group(benchmark, Config) -> + [{effort,10}|Config]; init_per_group(_GroupName, Config) -> Config. end_per_group(_GroupName, _Config) -> ok. -init_per_testcase(_Func, Conf) -> - Conf. +init_per_testcase(Func, Conf) -> + case proplists:is_defined(effort, Conf) of + false -> + %% Not a benchmark run + case atom_to_list(Func) of + "throughput_64" -> + Conf; + "throughput_"++_ -> + {skipped, "Benchmarks run separately"}; + _ -> + Conf + end; + true -> + Conf + end. end_per_testcase(_Func, _Conf) -> ok. --define(COUNT, 400). - %%%------------------------------------------------------------------- %%% CommonTest API helpers @@ -267,10 +285,10 @@ split_node(Node) -> %% Connection setup speed setup(Config) -> - run_nodepair_test(fun setup/5, Config). + run_nodepair_test(fun setup/6, Config). -setup(A, B, Prefix, HA, HB) -> - Rounds = 50, +setup(A, B, Prefix, Effort, HA, HB) -> + Rounds = 5 * Effort, [] = ssl_apply(HA, erlang, nodes, []), [] = ssl_apply(HB, erlang, nodes, []), {SetupTime, CycleTime} = @@ -325,10 +343,10 @@ setup_wait_nodedown(A, Time) -> %% Roundtrip speed roundtrip(Config) -> - run_nodepair_test(fun roundtrip/5, Config). + run_nodepair_test(fun roundtrip/6, Config). -roundtrip(A, B, Prefix, HA, HB) -> - Rounds = 40000, +roundtrip(A, B, Prefix, Effort, HA, HB) -> + Rounds = 4000 * Effort, [] = ssl_apply(HA, erlang, nodes, []), [] = ssl_apply(HB, erlang, nodes, []), ok = ssl_apply(HA, net_kernel, allow, [[B]]), @@ -381,15 +399,17 @@ roundtrip_client(Pid, Mon, StartTime, N) -> sched_utilization(Config) -> run_nodepair_test( - fun(A, B, Prefix, HA, HB) -> - sched_utilization(A, B, Prefix, HA, HB, proplists:get_value(ssl_dist, Config)) + fun(A, B, Prefix, Effort, HA, HB) -> + sched_utilization( + A, B, Prefix, Effort, HA, HB, + proplists:get_value(ssl_dist, Config)) end, Config). -sched_utilization(A, B, Prefix, HA, HB, SSL) -> +sched_utilization(A, B, Prefix, Effort, HA, HB, SSL) -> [] = ssl_apply(HA, erlang, nodes, []), [] = ssl_apply(HB, erlang, nodes, []), {ClientMsacc, ServerMsacc, Msgs} = - ssl_apply(HA, fun () -> sched_util_runner(A, B, SSL) end), + ssl_apply(HA, fun () -> sched_util_runner(A, B, Effort, SSL) end), [B] = ssl_apply(HA, erlang, nodes, []), [A] = ssl_apply(HB, erlang, nodes, []), msacc:print(ClientMsacc), @@ -423,12 +443,13 @@ sched_utilization(A, B, Prefix, HA, HB, SSL) -> %% Runs on node A and spawns a server on node B %% We want to avoid getting busy_dist_port as it hides the true SU usage %% of the receiver and sender. -sched_util_runner(A, B, true) -> - sched_util_runner(A, B, 250); -sched_util_runner(A, B, false) -> - sched_util_runner(A, B, 250); -sched_util_runner(A, B, Senders) -> +sched_util_runner(A, B, Effort, true) -> + sched_util_runner(A, B, Effort, 250); +sched_util_runner(A, B, Effort, false) -> + sched_util_runner(A, B, Effort, 250); +sched_util_runner(A, B, Effort, Senders) -> Payload = payload(5), + Time = 1000 * Effort, [A] = rpc:call(B, erlang, nodes, []), ServerPids = [erlang:spawn_link( @@ -440,7 +461,7 @@ sched_util_runner(A, B, Senders) -> fun() -> receive {start,Pid} -> - msacc:start(10000), + msacc:start(Time), receive {done,Pid} -> Pid ! {self(),msacc:stats()} @@ -457,7 +478,7 @@ sched_util_runner(A, B, Senders) -> %% receive after 1000 -> ok end, ServerMsacc ! {start,self()}, - msacc:start(10000), + msacc:start(Time), ClientMsaccStats = msacc:stats(), receive after 1000 -> ok end, ServerMsacc ! {done,self()}, @@ -495,50 +516,50 @@ throughput_client(Pid, Payload) -> throughput_0(Config) -> run_nodepair_test( - fun (A, B, Prefix, HA, HB) -> - throughput(A, B, Prefix, HA, HB, 500000, 0) + fun (A, B, Prefix, Effort, HA, HB) -> + throughput(A, B, Prefix, HA, HB, 50000 * Effort, 0) end, Config). throughput_64(Config) -> run_nodepair_test( - fun (A, B, Prefix, HA, HB) -> - throughput(A, B, Prefix, HA, HB, 500000, 64) + fun (A, B, Prefix, Effort, HA, HB) -> + throughput(A, B, Prefix, HA, HB, 50000 * Effort, 64) end, Config). throughput_1024(Config) -> run_nodepair_test( - fun (A, B, Prefix, HA, HB) -> - throughput(A, B, Prefix, HA, HB, 100000, 1024) + fun (A, B, Prefix, Effort, HA, HB) -> + throughput(A, B, Prefix, HA, HB, 10000 * Effort, 1024) end, Config). throughput_4096(Config) -> run_nodepair_test( - fun (A, B, Prefix, HA, HB) -> - throughput(A, B, Prefix, HA, HB, 50000, 4096) + fun (A, B, Prefix, Effort, HA, HB) -> + throughput(A, B, Prefix, HA, HB, 5000 * Effort, 4096) end, Config). throughput_16384(Config) -> run_nodepair_test( - fun (A, B, Prefix, HA, HB) -> - throughput(A, B, Prefix, HA, HB, 10000, 16384) + fun (A, B, Prefix, Effort, HA, HB) -> + throughput(A, B, Prefix, HA, HB, 1000 * Effort, 16384) end, Config). throughput_65536(Config) -> run_nodepair_test( - fun (A, B, Prefix, HA, HB) -> - throughput(A, B, Prefix, HA, HB, 2000, 65536) + fun (A, B, Prefix, Effort, HA, HB) -> + throughput(A, B, Prefix, HA, HB, 200 * Effort, 65536) end, Config). throughput_262144(Config) -> run_nodepair_test( - fun (A, B, Prefix, HA, HB) -> - throughput(A, B, Prefix, HA, HB, 500, 262144) + fun (A, B, Prefix, Effort, HA, HB) -> + throughput(A, B, Prefix, HA, HB, 50 * Effort, 262144) end, Config). throughput_1048576(Config) -> run_nodepair_test( - fun (A, B, Prefix, HA, HB) -> - throughput(A, B, Prefix, HA, HB, 200, 1048576) + fun (A, B, Prefix, Effort, HA, HB) -> + throughput(A, B, Prefix, HA, HB, 20 * Effort, 1048576) end, Config). throughput(A, B, Prefix, HA, HB, Packets, Size) -> @@ -817,9 +838,10 @@ run_nodepair_test(TestFun, Config) -> A = proplists:get_value(node_a, Config), B = proplists:get_value(node_b, Config), Prefix = proplists:get_value(ssl_dist_prefix, Config), + Effort = proplists:get_value(effort, Config, 1), HA = start_ssl_node_a(Config), HB = start_ssl_node_b(Config), - try TestFun(A, B, Prefix, HA, HB) + try TestFun(A, B, Prefix, Effort, HA, HB) after stop_ssl_node_a(HA), stop_ssl_node_b(HB, Config), -- 2.35.3
Locations
Projects
Search
Status Monitor
Help
OpenBuildService.org
Documentation
API Documentation
Code of Conduct
Contact
Support
@OBShq
Terms
openSUSE Build Service is sponsored by
The Open Build Service is an
openSUSE project
.
Sign Up
Log In
Places
Places
All Projects
Status Monitor
