Skip to content

Commit

Permalink
feat(sync-service): Reduce memory use by hibernating idle shape proce…
Browse files Browse the repository at this point in the history
…sses (#2240)

Potential fix for #2229

Electric's memory footprint can get very large due to shape processes
holding on to memory even when idle. This PR hibernates idle shape
processes after 30 seconds of inactivity which garbage collects unused
memory.

This first graph shows 10MB changes coming in for 100 shapes (2 shapes
per second). You can see this taking up 2000MB of memory which is not
released even though the memory is no longer needed (it's been written
to disk).
<img width="679" alt="Screenshot 2025-01-22 at 12 06 41"
src="https://github.com/user-attachments/assets/ae0d5da9-731f-4a9b-9fc1-3f47fa2ffab4"
/>
Not only is the memory not released, but the memory is amplified. In
this example it's amplified by 2x since 2000MB is twice the 1000MB
needed (10MB * 100 shapes). This is due to a binary copy that happens
when CubDB persists data to file. Amplification can increase further do
to having a binary version of the change as well as an Elixir term
version of the change which is used for where clause filtering.

This second graph is with `ELECTRIC_SHAPE_HIBERNATE_AFTER` set to
`50ms`. A value this short is unlikely to be used in production but
allows the effect to be seen quickly for the purposes of this benchmark.
<img width="667" alt="Screenshot 2025-01-22 at 12 09 38"
src="https://github.com/user-attachments/assets/5c22fa32-5e83-49aa-ab2b-d233a5af9c4d"
/>
As you can see, memory use flatlines at 120MB with hibernation rather
than blowing up to 2000MB without hibernation.
  • Loading branch information
robacourt authored Jan 22, 2025
1 parent 4b379aa commit 7c72c1e
Show file tree
Hide file tree
Showing 7 changed files with 31 additions and 5 deletions.
5 changes: 5 additions & 0 deletions .changeset/three-pumas-listen.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
---
"@core/sync-service": patch
---

Reduce memory footprint by hibernating idle shapes
4 changes: 4 additions & 0 deletions packages/sync-service/config/runtime.exs
Original file line number Diff line number Diff line change
Expand Up @@ -190,6 +190,9 @@ call_home_telemetry_url =
nil
)

shape_hibernate_after =
env!("ELECTRIC_SHAPE_HIBERNATE_AFTER", &Electric.Config.parse_human_readable_time!/1, nil)

system_metrics_poll_interval =
env!(
"ELECTRIC_SYSTEM_METRICS_POLL_INTERVAL",
Expand Down Expand Up @@ -218,6 +221,7 @@ config :electric,
replication_stream_id: replication_stream_id,
replication_slot_temporary?: env!("CLEANUP_REPLICATION_SLOTS_ON_SHUTDOWN", :boolean, nil),
service_port: env!("ELECTRIC_PORT", :integer, nil),
shape_hibernate_after: shape_hibernate_after,
storage: storage,
persistent_kv: persistent_kv,
listen_on_ipv6?: env!("ELECTRIC_LISTEN_ON_IPV6", :boolean, nil)
4 changes: 3 additions & 1 deletion packages/sync-service/lib/electric/config.ex
Original file line number Diff line number Diff line change
Expand Up @@ -53,7 +53,9 @@ defmodule Electric.Config do
call_home_telemetry?: @build_env == :prod,
telemetry_statsd_host: nil,
telemetry_url: URI.new!("https://checkpoint.electric-sql.com"),
system_metrics_poll_interval: :timer.seconds(5)
system_metrics_poll_interval: :timer.seconds(5),
# Memory
shape_hibernate_after: :timer.seconds(30)
]

def default(key) do
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -80,7 +80,11 @@ defmodule Electric.ShapeCache.FileStorage do
@impl Electric.ShapeCache.Storage
def start_link(%FS{cubdb_dir: dir, db: db} = opts) do
with :ok <- initialise_filesystem(opts) do
CubDB.start_link(data_dir: dir, name: db)
CubDB.start_link(
data_dir: dir,
name: db,
hibernate_after: Electric.Config.get_env(:shape_hibernate_after)
)
end
end

Expand Down
5 changes: 4 additions & 1 deletion packages/sync-service/lib/electric/shapes/consumer.ex
Original file line number Diff line number Diff line change
Expand Up @@ -54,7 +54,10 @@ defmodule Electric.Shapes.Consumer do
end

def start_link(config) when is_map(config) do
GenStage.start_link(__MODULE__, config, name: name(config))
GenStage.start_link(__MODULE__, config,
name: name(config),
hibernate_after: Electric.Config.get_env(:shape_hibernate_after)
)
end

def init(config) do
Expand Down
9 changes: 8 additions & 1 deletion packages/sync-service/mix.exs
Original file line number Diff line number Diff line change
Expand Up @@ -76,7 +76,14 @@ defmodule Electric.MixProject do
[
{:backoff, "~> 1.1"},
{:bandit, "~> 1.5"},
{:cubdb, "~> 2.0.2"},
# Here we use a fork of CubDB that allows for hibernating all of it's processes.
#
# Electric currently uses a CubDB instance per shape, each instance has 4 processes, and those processes
# can keep references to large binaries. Hibernate these processes when not in use can significantly reduce
# the memory footprint.
#
# See: https://github.com/lucaong/cubdb/pull/78
{:electric_cubdb, "~> 2.0"},
{:dotenvy, "~> 0.8"},
{:ecto, "~> 3.11"},
{:gen_stage, "~> 1.2"},
Expand Down
3 changes: 2 additions & 1 deletion packages/sync-service/mix.lock
Original file line number Diff line number Diff line change
Expand Up @@ -5,13 +5,14 @@
"chatterbox": {:hex, :ts_chatterbox, "0.15.1", "5cac4d15dd7ad61fc3c4415ce4826fc563d4643dee897a558ec4ea0b1c835c9c", [:rebar3], [{:hpack, "~> 0.3.0", [hex: :hpack_erl, repo: "hexpm", optional: false]}], "hexpm", "4f75b91451338bc0da5f52f3480fa6ef6e3a2aeecfc33686d6b3d0a0948f31aa"},
"combine": {:hex, :combine, "0.10.0", "eff8224eeb56498a2af13011d142c5e7997a80c8f5b97c499f84c841032e429f", [:mix], [], "hexpm", "1b1dbc1790073076580d0d1d64e42eae2366583e7aecd455d1215b0d16f2451b"},
"ctx": {:hex, :ctx, "0.6.0", "8ff88b70e6400c4df90142e7f130625b82086077a45364a78d208ed3ed53c7fe", [:rebar3], [], "hexpm", "a14ed2d1b67723dbebbe423b28d7615eb0bdcba6ff28f2d1f1b0a7e1d4aa5fc2"},
"cubdb": {:hex, :cubdb, "2.0.2", "d4253885084dae37a8ff73887d232864eb38ecac962aa08543e686b0183a1d62", [:mix], [], "hexpm", "c99cc8f9e6c4deb98d16cca5ded1928edd22e48b4736b76e8a1a85367d7fe921"},
"cubdb": {:git, "https://github.com/electric-sql/cubdb", "72254274d4249d2f5fb0120929ea868143e081fe", [branch: "hibernate-all-processes"]},
"db_connection": {:hex, :db_connection, "2.7.0", "b99faa9291bb09892c7da373bb82cba59aefa9b36300f6145c5f201c7adf48ec", [:mix], [{:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "dcf08f31b2701f857dfc787fbad78223d61a32204f217f15e881dd93e4bdd3ff"},
"decimal": {:hex, :decimal, "2.1.1", "5611dca5d4b2c3dd497dec8f68751f1f1a54755e8ed2a966c2633cf885973ad6", [:mix], [], "hexpm", "53cfe5f497ed0e7771ae1a475575603d77425099ba5faef9394932b35020ffcc"},
"dialyxir": {:hex, :dialyxir, "1.4.3", "edd0124f358f0b9e95bfe53a9fcf806d615d8f838e2202a9f430d59566b6b53b", [:mix], [{:erlex, ">= 0.2.6", [hex: :erlex, repo: "hexpm", optional: false]}], "hexpm", "bf2cfb75cd5c5006bec30141b131663299c661a864ec7fbbc72dfa557487a986"},
"dotenvy": {:hex, :dotenvy, "0.8.0", "777486ad485668317c56afc53a7cbcd74f43e4e34588ba8e95a73e15a360050e", [:mix], [], "hexpm", "1f535066282388cbd109743d337ac46ff0708195780d4b5778bb83491ab1b654"},
"earmark_parser": {:hex, :earmark_parser, "1.4.41", "ab34711c9dc6212dda44fcd20ecb87ac3f3fce6f0ca2f28d4a00e4154f8cd599", [:mix], [], "hexpm", "a81a04c7e34b6617c2792e291b5a2e57ab316365c2644ddc553bb9ed863ebefa"},
"ecto": {:hex, :ecto, "3.11.2", "e1d26be989db350a633667c5cda9c3d115ae779b66da567c68c80cfb26a8c9ee", [:mix], [{:decimal, "~> 2.0", [hex: :decimal, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}, {:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "3c38bca2c6f8d8023f2145326cc8a80100c3ffe4dcbd9842ff867f7fc6156c65"},
"electric_cubdb": {:hex, :electric_cubdb, "2.0.2", "36f86e3c52dc26f4e077a49fbef813b1a38d3897421cece851f149190b34c16c", [:mix], [], "hexpm", "0c0e24b31fb76ad1b33c5de2ab35c41a4ff9da153f5c1f9b15e2de78575acaf2"},
"elixir_make": {:hex, :elixir_make, "0.8.4", "4960a03ce79081dee8fe119d80ad372c4e7badb84c493cc75983f9d3bc8bde0f", [:mix], [{:castore, "~> 0.1 or ~> 1.0", [hex: :castore, repo: "hexpm", optional: true]}, {:certifi, "~> 2.0", [hex: :certifi, repo: "hexpm", optional: true]}], "hexpm", "6e7f1d619b5f61dfabd0a20aa268e575572b542ac31723293a4c1a567d5ef040"},
"erlex": {:hex, :erlex, "0.2.7", "810e8725f96ab74d17aac676e748627a07bc87eb950d2b83acd29dc047a30595", [:mix], [], "hexpm", "3ed95f79d1a844c3f6bf0cea61e0d5612a42ce56da9c03f01df538685365efb0"},
"ex_doc": {:hex, :ex_doc, "0.34.2", "13eedf3844ccdce25cfd837b99bea9ad92c4e511233199440488d217c92571e8", [:mix], [{:earmark_parser, "~> 1.4.39", [hex: :earmark_parser, repo: "hexpm", optional: false]}, {:makeup_c, ">= 0.1.0", [hex: :makeup_c, repo: "hexpm", optional: true]}, {:makeup_elixir, "~> 0.14 or ~> 1.0", [hex: :makeup_elixir, repo: "hexpm", optional: false]}, {:makeup_erlang, "~> 0.1 or ~> 1.0", [hex: :makeup_erlang, repo: "hexpm", optional: false]}, {:makeup_html, ">= 0.1.0", [hex: :makeup_html, repo: "hexpm", optional: true]}], "hexpm", "5ce5f16b41208a50106afed3de6a2ed34f4acfd65715b82a0b84b49d995f95c1"},
Expand Down

0 comments on commit 7c72c1e

Please sign in to comment.