Quantification wiki page and memory usage tests

docs/quantification.md

@@ -0,0 +1,47 @@
+# Quantification #
+This page describes LINC's resources consumption.
+
+## Adding resources  ##
+To add ports and logical switches to the LINC instance you have to edit `rel/files/sys.config` file. You will there comments on how to do this.
+
+## Memory consumptions ##
+LINC has three major components that make up memory consumption:
+* ports and logical switches,
+* flow entries.
+
+### Average values ###
+On average a LINC port consumes ~ 11 kb.
+Memory consumed by logical switches is not linear. For example a logical switch with 20 ports consumes:
+* ~ 1200 kb if we have 10 such switches
+* ~ 2200 kb if we have 20 such switches
+* ~ 5100 kb if we have 30 such switches
+
+### Measuring ###
+There is a script `scripts/mem_usage_test` that helps with estimating memory requirements.
+
+To measure how much memory one port will take use it as follows:
+```bash
+./scripts/mem_usage_test ports <interval> <max_ports_number>
+```
+In the test amount of ports indicated by the `interval` will be added to the LINC instance in each iteration until reaching the `max_port_number`. The script produces output file `ports.test` that has three columns:
+* number of ports in the LINC instance,
+* memory consumed by the LINC instance,
+* memory consumed by this LINC instance minus the amount of memory consumed in the previous test.
+At the bottom of the file there's a summary line.
+
+To memory how much memory one logical switch witch fixed number of ports invoke the script as follows:
+
+```bash
+./scripts/mem_usage_test switches <ports_per_switch> <max_switches>
+```
+In this test an additional logical switch will be added to the LINC instance in each iteration until reaching the `max_switches`. The test produces an output file that has three columns:
+* number of logical switches started in a LINC instance,
+* memory consumed by the LINC instance,
+* memory consumed by this LINC instance minus the amount of memory consumed in the previous test.
+At the bottom of the file there's a summary line.
+
+## Tuning Erlang VM  ##
+Each logical switch allocates approximately 280 ETS tables. To change this value you have to edit `rel/files/vm.args` file and change the value of ERL_MAX_ETS_TABLES. You find details in the file.
+
+## TODO  ##
+Measure flow entries.

rel/files/sys.config

@@ -41,6 +41,11 @@
    %% Configuration of the logical switches.
    {logical_switches,
     [
+     %% To add a new logical switch add a new entry to this list similar to
+     %% the one below. Keep in mind that:
+     %% a. logical switches need uniquie interger id (second element
+     %%    of the tuple),
+     %% b. you have to assign ports to the logical switch manually.
      {switch, 0,
       [
        %% Configuration of switch backend implementation used by ofs_logic

scripts/config_generator.erl

@@ -0,0 +1,76 @@
+-module(config_generator).
+
+-export([generate/2]).
+
+-spec generate(PortsCnt :: non_neg_integer(),
+      LogicalSwitchesCnt :: non_neg_integer()) ->
+                      Result :: term().
+generate(PortsCnt, LogicalSwitchesCnt) ->
+    [
+     {linc,
+      [
+       {of_config, enabled},
+       {capable_switch_ports, generate_ports(PortsCnt * LogicalSwitchesCnt)},
+       {capable_switch_queues, []},
+       {logical_switches,
+        generate_logical_switches(PortsCnt, LogicalSwitchesCnt)}
+      ]},
+     {enetconf,
+      [
+       {capabilities, [{base, {1, 1}},
+                       {startup, {1, 0}},
+                       {'writable-running', {1, 0}}]},
+       {callback_module, linc_ofconfig},
+       {sshd_ip, any},
+       {sshd_port, 1830},
+       {sshd_user_passwords,
+        [
+         {"linc", "linc"}
+        ]}
+      ]},
+     {lager,
+      [
+       {handlers,
+        [
+         {lager_console_backend, info},
+         {lager_file_backend,
+          [
+           {"log/error.log", error, 10485760, "$D0", 5},
+           {"log/console.log", info, 10485760, "$D0", 5}
+          ]}
+        ]}
+      ]},
+     {sasl,
+      [
+       {sasl_error_logger, {file, "log/sasl-error.log"}},
+       {errlog_type, error},
+       {error_logger_mf_dir, "log/sasl"},      % Log directory
+       {error_logger_mf_maxbytes, 10485760},   % 10 MB max file size
+       {error_logger_mf_maxfiles, 5}           % 5 files max
+      ]},
+     {sync,
+      [
+       {excluded_modules, [procket]}
+      ]}
+    ].
+
+generate_ports(PortsCnt) ->
+    [{port, N, [{interface, "tap" ++ integer_to_list(N)}]}
+     || N <- lists:seq(1, PortsCnt)].
+
+generate_logical_switches(_, 0) ->
+    [];
+generate_logical_switches(PortsCnt, LogicalSwitchesCnt) ->
+    [
+     {switch, N,
+      [
+       {backend, linc_us4},
+       {controllers,[]},
+       {queues_status, disabled},
+       {ports, generate_logical_switch_ports(PortsCnt, (N - 1) * PortsCnt + 1)}
+      ]}
+     || N <- lists:seq(1, LogicalSwitchesCnt)].
+
+generate_logical_switch_ports(PortsCnt, StartPortNo) ->
+    [{port, N, {queues, []}} || N <- lists:seq(StartPortNo,
+                                               StartPortNo + PortsCnt - 1)].

scripts/memory_usage_test

@@ -0,0 +1,240 @@
+#!/usr/bin/env escript
+%%! -sname linc_tester@localhost
+
+-define(CONFIG_FILE, "tmp.sys.config").
+-define(COOKIE, cookie).
+-define(LINC_NODE_NAME_BASE, "linc_under_test").
+-define(MNESIA_DIR_BASE, "Mnesia." ++ ?LINC_NODE_NAME_BASE).
+-define(LOG_DIR, "log/").
+
+main([Type, Arg1, Arg2]) ->
+    process_flag(trap_exit, true),
+    compile:file("config_generator.erl"),
+    register(linc_tester_proc, self()),
+    erlang:set_cookie(node(), ?COOKIE),
+    case Type of
+        "ports" ->
+            increasing_ports(get_linc_node_name(),
+                             0,
+                             list_to_integer(Arg1),
+                             list_to_integer(Arg2),
+                             {0, []});
+        "switches" ->
+            increasing_switches(get_linc_node_name(),
+                                list_to_integer(Arg1),
+                                0,
+                                list_to_integer(Arg2), {0, 0});
+        _ ->
+            io:format("Unknown test.~n"),
+            usage()
+    end,
+    clean_up();
+main(_) ->
+    usage().
+
+usage() ->
+    io:format("~s [ports <interval> <max ports>"
+              ++ " | switches <ports per switch> <max switches>]~n",
+              [escript:script_name()]).
+
+
+
+increasing_switches(_ , Ports, Switches, MaxSwitches, {_, MemPerSwitchAcc})
+  when Switches > MaxSwitches ->
+    file:write_file(result_file(switches),
+                    io_lib:fwrite("On avarage a logical switch with ~p ports"
+                                  ++ " consumes ~p kb.~n",
+                                  [Ports, MemPerSwitchAcc/(Switches - 1)]),
+                    [append]);
+increasing_switches(LincNode, Ports, Switches, MaxSwitches, Data) ->
+    ok = generate_config(Ports, Switches),
+    LincNodePort = start_linc_node(LincNode),
+    Mem = test_memory(LincNode),
+    ok = stop_linc_node(LincNode, LincNodePort),
+    NewData = case Switches of
+                  0 ->
+                      write_result(switches, Switches, Mem, undef),
+                      {Mem, 0};
+                  _ ->
+                      {LastMem, MemPerSwitchAcc} = Data,
+                      Diff = Mem - LastMem,
+                      write_result(switches, Switches, Mem, Diff),
+                      {Mem, MemPerSwitchAcc + Diff}
+              end,
+    increasing_switches(
+      get_linc_node_name(), Ports, Switches + 1, MaxSwitches, NewData).
+
+increasing_ports(LincNode, PortsCnt, Interval, MaxPortsCnt, TestData)
+  when PortsCnt > MaxPortsCnt ->
+    increasing_ports(
+      LincNode,
+      MaxPortsCnt,
+      MaxPortsCnt rem Interval,
+      MaxPortsCnt,
+      TestData);
+increasing_ports(LincNode, PortsCnt, Interval, MaxPortsCnt, TestData) ->
+    ok  = generate_config(PortsCnt, 1),
+    LincNodePort = start_linc_node(LincNode),
+    NewTestData = case PortsCnt of
+                      0 ->
+                          Mem = test_memory(LincNode),
+                          write_result(ports, PortsCnt, Mem, undef),
+                          {Mem, []};
+                      _ ->
+                          Mem = test_memory(LincNode),
+                          {LastMem, PortAvgMem} = TestData,
+                          write_result(ports, PortsCnt, Mem,
+                                       Diff = Mem - LastMem),
+                          {Mem, [Diff div Interval | PortAvgMem]}
+                  end,
+    ok = stop_linc_node(LincNode, LincNodePort),
+    case PortsCnt == MaxPortsCnt of
+        true ->
+            {_, PortAvgMem2} = NewTestData,
+            file:write_file(result_file(ports),
+                            io_lib:fwrite("On average a port consumes ~p kb.~n",
+                                          [lists:foldl(fun(X, Acc) ->
+                                                               X + Acc end,
+                                                       0, PortAvgMem2)
+                                           / length(PortAvgMem2)]),
+                            [append]);
+        _ ->
+            increasing_ports(get_linc_node_name(),
+                             PortsCnt + Interval,
+                             Interval,
+                             MaxPortsCnt,
+                             NewTestData)
+    end.
+
+test_memory(LincNode) ->
+    Pid = self(),
+    spawn(LincNode, fun() ->
+                            Pid ! {mem_test, erlang:memory(total)}
+                    end),
+    receive
+        {mem_test, Memory} ->
+            io:format("Got memory usage: ~p kb.~n", [Memory]),
+            Memory div 1024
+    after
+        5000 ->
+            io:format("No response from ~p.~n", [LincNode]),
+            0
+    end.
+
+start_linc_node(LincNode) ->
+    Cmd = "erl -env ERL_MAX_ETS_TABLES 6000 "
+        ++ code_paths()
+        ++ " -config " ++ ?CONFIG_FILE
+        ++ " -setcookie " ++ atom_to_list(?COOKIE)
+        ++ " -sname " ++ atom_to_list(LincNode)
+        ++ " -detached"
+        ++ " -eval \""
+        ++ eval(Msg = hello)
+        ++ "\"",
+    LincNodePort = erlang:open_port({spawn, Cmd}, []),
+    io:format("Waiting for test node to connect..."),
+    receive
+        Msg ->
+            io:format("connected~n")
+    after
+        20000 ->
+            io:format("Error while starting test node~n")
+    end,
+    LincNodePort.
+
+stop_linc_node(LincNode, LincNodePort) ->
+    %% port_close(LincNodePort).
+    spawn(LincNode, fun() ->
+                            init:stop()
+                    end),
+    receive
+        {'EXIT', LincNodePort, normal} ->
+            ok
+    after
+        10000 ->
+            io:format("LINC node: not exitted~n"),
+            error
+    end.
+
+code_paths() ->
+    Paths = lists:foldl(fun(Path, Acc) ->
+                                Acc ++ filelib:wildcard(Path)
+                        end, [], ["../apps/*/ebin", "../deps/*/ebin"]),
+    lists:foldl(fun(Path, Acc) ->
+                        Acc ++ " -pa " ++ Path
+                end, [], Paths).
+
+eval(Msg) ->
+    {registered_name, ProcName} = process_info(self(), registered_name),
+    "lists:map(fun application:start/1,
+                                       [kernel, stdlib, public_key, crypto, ssl,
+                                        compiler, syntax_tools, runtime_tools,
+                                        xmerl, mnesia, lager, linc]),"
+        ++ "true = net_kernel:connect_node(" ++ atom_to_list(node()) ++ "),"
+        ++ "{" ++ atom_to_list(ProcName) ++ "," ++ atom_to_list(node()) ++ "}"
+        ++ " ! " ++ atom_to_list(Msg) ++ ".".
+
+
+ports_result_header() ->
+    "Ports | Memory | Difference\n".
+
+switches_result_header() ->
+    "Switches | Memory | Difference\n".
+
+result_line(0, Mem, _) ->
+    "0 " ++ integer_to_list(Mem) ++ " none\n";
+result_line(PortsCnt, Mem, Diff) ->
+    integer_to_list(PortsCnt)
+        ++ " "
+        ++ integer_to_list(Mem)
+        ++ " "
+        ++ integer_to_list(Diff)
+        ++ "\n".
+
+generate_config(Ports, Switches) ->
+    file:write_file(?CONFIG_FILE,
+                         io_lib:fwrite(
+                           "~p.~n",
+                           [config_generator:generate(Ports, Switches)])).
+
+write_result(Type, Cnt, Mem, Diff) ->
+    Content = case Cnt of
+                  0 ->
+                      result_header(Type) ++ result_line(0, Mem, undef);
+                  _ ->
+                      result_line(Cnt, Mem, Diff)
+              end,
+    ok = file:write_file(result_file(Type), Content, case Cnt of
+                                                    0 -> [write];
+                                                    _ -> [append]
+                                                end).
+result_file(Type) ->
+    case Type of
+        ports -> "ports.test";
+        switches -> "switches.test"
+    end.
+
+result_header(Type) ->
+    case Type of
+        ports ->
+            ports_result_header();
+        switches ->
+            switches_result_header()
+    end.
+
+get_linc_node_name() ->
+    list_to_atom(?LINC_NODE_NAME_BASE
+                 ++ integer_to_list(element(3, now()))
+                 ++ "@localhost").
+
+clean_up() ->
+    file:delete(?CONFIG_FILE),
+    lists:foreach(fun(Dir) ->
+                          ok = delete_directory(Dir)
+                  end, [ ?LOG_DIR | filelib:wildcard(?MNESIA_DIR_BASE ++ "*")]).
+
+delete_directory(Dir) ->
+    lists:foreach(fun(File) ->
+                          ok = file:delete(File)
+                  end, filelib:wildcard(Dir ++ "/*")),
+    ok = file:del_dir(Dir).