multi-stage profiles

[hyperthunk]

this could either by done in terms of running one profile after another, or by chaining the test executions within a given (umbrella profile) scope. The latter is probably going to work better, as experiments thus far have demonstrated that starting and stopping systest repeatedly cause all kinds of problems with cover compilation (and reporting).

[hyperthunk]

Some ideas for doing this:

diff --git a/src/systest_main.erl b/src/systest_main.erl
index 6f81062..e567904 100644
--- a/src/systest_main.erl
+++ b/src/systest_main.erl
@@ -61,49 +61,63 @@ parse_args(Args) ->
            end,
     Spec = opt_spec(),
     OptsWithVals = lists:map(fun erlang:atom_to_list/1,
-                            lists:flatten([[L, S] || {L, S, string} <- Spec])),
+                            lists:flatten([[L, S] || {L, S, T} <- Spec,
+                                                     T =:= string orelse
+                                                     T =:= {list, string}])),
     {Options, RawOpts} = niceopt:parse(Args, [{mode, Mode},
                                               {opts_with_vals, OptsWithVals}]),
     [{raw_opts, RawOpts}|validate(Options, Spec)].

 validate(Options, Spec) ->
-    [begin
-        case lists:keyfind(K, 1, Spec) of
-            false ->
-                case lists:keyfind(K, 2, Spec) of
-                    false ->
-                        RawOpt;
-                    Def ->
-                        unpack(V, Def)
-                end;
-            OptDef ->
-                unpack(V, OptDef)
-        end
-     end || {K, V}=RawOpt <- Options].
+    lists:foldl(
+      fun({K, V}=RawOpt, Acc) ->
+              case lists:keyfind(K, 1, Spec) of
+                  false ->
+                      case lists:keyfind(K, 2, Spec) of
+                          false ->
+                              [RawOpt|Acc];
+                          Def ->
+                              unpack(V, Def, Acc)
+                      end;
+                  OptDef ->
+                      unpack(V, OptDef, Acc)
+              end
+      end, [], Options).

-unpack(true, {L, _, V}) when V =:= integer orelse
-                             V =:= string ->
-                             io:format("Argument ~p requires a value!~n", [L]),
-                             help(),
-                             erlang:halt(1);
-unpack(V,    {L, _, integer}) -> {L, list_to_integer(V)};
-unpack(_,    {ignore,_,flag}) -> {error_handler,
-                                    fun(_, _) ->
-                                      io:format("[passed] all test "
+unpack(true, {L, _, V}, _Opt)
+              when V =:= integer orelse
+                   V =:= string ->
+    io:format("Argument ~p requires a value!~n", [L]),
+    help(),
+    erlang:halt(1);
+unpack(V, {L, _, integer}, Acc)  -> [{L, list_to_integer(V)}|Acc];
+unpack(_, {ignore,_,flag}, Acc)  -> [{error_handler,
+                                      fun(_, _) ->
+                                              io:format(
+                                                "[passed] all test "
                                                 "cases succeeded or "
                                                 "were explicitly ignored~n")
-                                    end};
-unpack(V,    {target, _, _})  -> case string:tokens(V, ":") of
-                                     [Suite] ->
-                                         {testsuite, [Suite]};
-                                     [Suite, Case] ->
-                                         {testcase, {Suite, Case}}
-                                 end;
-unpack(V,    {L, _, string})  -> {L, V};
-unpack(V,    {L, _, flag})    -> {L, V}.
+                                      end}|Acc];
+unpack(V, {target, _, _}, Acc)   -> [case string:tokens(V, ":") of
+                                         [Suite] ->
+                                             {testsuite, [Suite]};
+                                         [Suite, Case] ->
+                                             {testcase, {Suite, Case}}
+                                     end|Acc];
+unpack(V, {L, _, string}, Acc)   -> [{L, V}|Acc];
+unpack(V, {L, _, flag}, Acc)     -> [{L, V}|Acc];
+unpack(V, {L, _,
+           {list, string}}, Acc) -> case lists:keyfind(L, 1, Acc) of
+                                        false ->
+                                            [{L, [V]}|Acc];
+                                        {K, [_|_]=V2} ->
+                                            V3 = {K, V2 ++ [V]},
+                                            lists:keyreplace(L, 1, Acc, V3)
+                                    end.

 opt_spec() ->
     [{profile,          'P', string},
+     {multi_profile,    'K', {list, string}},
      {target,           'Z', string},
      {logging,          'L', string},
      {dryrun,           'n', flag},
diff --git a/src/systest_runner.erl b/src/systest_runner.erl
index a536e93..f5f3255 100644
--- a/src/systest_runner.erl
+++ b/src/systest_runner.erl
@@ -56,6 +56,18 @@ behaviour_info(_) ->
 %% @end
 -spec execute(systest_config:config()) -> 'ok'.
 execute(Config) ->
+    case proplists:get_all_values(multi_profile, Config) of
+        [] ->
+            run(Config);
+        [[[H|_]|_]=Profiles] when is_integer(H) ->
+            print_banner(Config),
+            application:set_env(systest, banner, ""),
+            io:format("**** NOTICE: SysTest "
+                      "is running multiple profiles....~n~n"),
+            [run([{profile, P}|Config]) || P <- Profiles]
+    end.
+
+run(Config) ->
     systest:start(),
     start_logging(Config),
     maybe_start_net_kernel(Config),