apply Stephan workshop optimisation (#17)

* apply stephan workshop optimisation

* lint

Gemfile

@@ -99,6 +99,7 @@ gem "avo", ">= 3.2"
 gem "ransack", "~> 4.1"
 
 gem "activerecord-enhancedsqlite3-adapter"
+gem "litestream", "~> 0.10.1"
 
 # sitepress
 gem "sitepress-rails", "~> 4.0"
@@ -111,6 +112,8 @@ gem "blazer"
 
 gem "solid_errors"
 
-gem "solid_queue", "~> 0.2.2"
+gem "solid_cache"
+
+gem "solid_queue", github: "rails/solid_queue", branch: "main"
 
 gem "mission_control-jobs", "~> 0.1.1"

Gemfile.lock

@@ -11,7 +11,7 @@ GIT
 
 GIT
   remote: https://github.com/rails/rails.git
-  revision: 462e9143189ad46d3f26a52d0c132cf69868aa4f
+  revision: ad0e3123e186c919973fe9e1cec04272c75dd252
   branch: main
   specs:
     actioncable (7.2.0.alpha)
@@ -105,6 +105,18 @@ GIT
       thor (~> 1.0, >= 1.2.2)
       zeitwerk (~> 2.6)
 
+GIT
+  remote: https://github.com/rails/solid_queue.git
+  revision: 023f6108754673b80b65c31680a5d6693eedfc21
+  branch: main
+  specs:
+    solid_queue (0.3.0)
+      activejob (>= 7.1)
+      activerecord (>= 7.1)
+      concurrent-ruby (~> 1.2.2)
+      fugit (~> 1.10.1)
+      railties (>= 7.1)
+
 GEM
   remote: https://rubygems.org/
   specs:
@@ -220,7 +232,12 @@ GEM
       rubocop
       smart_properties
     erubi (1.12.0)
+    et-orbi (1.2.11)
+      tzinfo
     ffi (1.16.3)
+    fugit (1.10.1)
+      et-orbi (~> 1, >= 1.2.7)
+      raabro (~> 1.4)
     globalid (1.2.1)
       activesupport (>= 6.1)
     heroicon (1.0.0)
@@ -287,6 +304,19 @@ GEM
     listen (3.9.0)
       rb-fsevent (~> 0.10, >= 0.10.3)
       rb-inotify (~> 0.9, >= 0.9.10)
+    litestream (0.10.1)
+      logfmt (>= 0.0.10)
+      sqlite3
+    litestream (0.10.1-arm64-darwin)
+      logfmt (>= 0.0.10)
+      sqlite3
+    litestream (0.10.1-x86_64-darwin)
+      logfmt (>= 0.0.10)
+      sqlite3
+    litestream (0.10.1-x86_64-linux)
+      logfmt (>= 0.0.10)
+      sqlite3
+    logfmt (0.0.10)
     loofah (2.22.0)
       crass (~> 1.0.2)
       nokogiri (>= 1.12.0)
@@ -368,6 +398,7 @@ GEM
     public_suffix (5.0.5)
     puma (6.4.2)
       nio4r (~> 2.0)
+    raabro (1.4.0)
     racc (1.7.3)
     rack (3.0.11)
     rack-session (2.0.0)
@@ -447,15 +478,17 @@ GEM
       sitepress-core (= 4.0.2)
       sprockets-rails (>= 2.0.0)
     smart_properties (1.17.0)
+    solid_cache (0.6.0)
+      activejob (>= 7)
+      activerecord (>= 7)
+      railties (>= 7)
     solid_errors (0.4.2)
       actionmailer (~> 7.0)
       actionpack (~> 7.0)
       actionview (~> 7.0)
       activerecord (~> 7.0)
       activesupport (~> 7.0)
       railties (~> 7.0)
-    solid_queue (0.2.2)
-      rails (~> 7.1)
     sorbet-runtime (0.5.11372)
     sprockets (4.2.1)
       concurrent-ruby (~> 1.0)
@@ -555,6 +588,7 @@ DEPENDENCIES
   jsbundling-rails
   kamal
   letter_opener
+  litestream (~> 0.10.1)
   magic_test!
   markdown-rails (~> 2.1)
   mission_control-jobs (~> 0.1.1)
@@ -568,8 +602,9 @@ DEPENDENCIES
   ruby-lsp-rails
   selenium-webdriver
   sitepress-rails (~> 4.0)
+  solid_cache
   solid_errors
-  solid_queue (~> 0.2.2)
+  solid_queue!
   sqlite3 (~> 1.4)
   standard (>= 1.35.1)
   stimulus-rails

config/application.rb

@@ -23,5 +23,18 @@ class Application < Rails::Application
     #
     # config.time_zone = "Central Time (US & Canada)"
     # config.eager_load_paths << Rails.root.join("extras")
+
+    # Use Solid Queue as the Active Job backend
+    config.active_job.queue_adapter = :solid_queue
+    config.solid_queue.connects_to = {database: {writing: :queue, reading: :queue}}
+
+    # Use Solid Cache Store as the cache store
+    config.cache_store = :solid_cache_store
+
+    # Use a separate database for error monitoring
+    config.solid_errors.connects_to = {database: {writing: :errors, reading: :errors}}
+    # config.solid_errors.send_emails = true
+    # config.solid_errors.email_from = "[email protected]"
+    # config.solid_errors.email_to = "[email protected]"
   end
 end

config/database.yml

@@ -4,22 +4,56 @@
 #   Ensure the SQLite 3 gem is defined in your Gemfile
 #   gem "sqlite3"
 #
+# DEFAULT CONNECTION CONFIGURATION
 default: &default
   adapter: sqlite3
   pool: <%= ENV.fetch("RAILS_MAX_THREADS") { 5 } %>
   timeout: 5000
 
-development:
+# DATABASE CONFIGURATIONS
+primary: &primary
+  <<: *default
+  database: storage/<%= Rails.env %>.sqlite3
+
+queue: &queue
+  <<: *default
+  migrations_paths: db/queue_migrate
+  database: storage/<%= Rails.env %>-queue.sqlite3
+
+cache: &cache
+  <<: *default
+  migrations_paths: db/cache_migrate
+  database: storage/<%= Rails.env %>-cache.sqlite3
+
+errors: &errors
   <<: *default
-  database: storage/development.sqlite3
+  migrations_paths: db/errors_migrate
+  database: storage/<%= Rails.env %>-errors.sqlite3
+
+# ENVIRONMENT CONFIGURATIONS
+development:
+  primary: *primary
+  queue: *queue
+  cache: *cache
+  errors: *errors
 
 # Warning: The database defined as "test" will be erased and
 # re-generated from your development database when you run "rake".
 # Do not set this db to the same as development or production.
 test:
-  <<: *default
-  database: storage/test.sqlite3
+  primary: *primary
+  queue: *queue
+  cache: *cache
+  errors: *errors
 
+# SQLite3 write its data on the local filesystem, as such it requires
+# persistent disks. If you are deploying to a managed service, you should
+# make sure it provides disk persistence, as many don't.
+#
+# Similarly, if you deploy your application as a Docker container, you must
+# ensure the database is located in a persisted volume.
 production:
-  <<: *default
-  database: storage/production.sqlite3
+  primary: *primary
+  queue: *queue
+  cache: *cache
+  errors: *errors

config/environments/development.rb

@@ -23,7 +23,6 @@
     config.action_controller.perform_caching = true
     config.action_controller.enable_fragment_cache_logging = true
 
-    config.cache_store = :memory_store
     config.public_file_server.headers = {"Cache-Control" => "public, max-age=#{2.days.to_i}"}
   else
     config.action_controller.perform_caching = false

config/environments/production.rb

@@ -60,11 +60,7 @@
   # want to log everything, set the level to "debug".
   config.log_level = ENV.fetch("RAILS_LOG_LEVEL", "info")
 
-  # Use a different cache store in production.
-  # config.cache_store = :mem_cache_store
-
   # Use a real queuing backend for Active Job (and separate queues per environment).
-  config.active_job.queue_adapter = :solid_queue
   # config.active_job.queue_name_prefix = "daisy_on_rails_production"
 
   config.action_mailer.perform_caching = false

config/environments/test.rb

@@ -25,6 +25,8 @@
   config.action_controller.perform_caching = false
   config.cache_store = :null_store
 
+  config.active_job.queue_adapter = :test
+
   # Render exception templates for rescuable exceptions and raise for other exceptions.
   config.action_dispatch.show_exceptions = :rescuable
 

config/initializers/litesream.rb

@@ -0,0 +1,33 @@
+# Use this hook to configure the litestream-ruby gem.
+# All configuration options will be available as environment variables, e.g.
+# config.replica_bucket becomes LITESTREAM_REPLICA_BUCKET
+# This allows you to configure Litestream using Rails encrypted credentials,
+# or some other mechanism where the values are only avaialble at runtime.
+
+Litestream.configure do |config|
+  # An example of using Rails encrypted credentials to configure Litestream.
+  litestream_credentials = Rails.application.credentials.litestream
+
+  # Replica-specific bucket location.
+  # This will be your bucket's URL without the `https://` prefix.
+  # For example, if you used DigitalOcean Spaces, your bucket URL could look like:
+  #   https://myapp.fra1.digitaloceanspaces.com
+  # And so you should set your `replica_bucket` to:
+  #   myapp.fra1.digitaloceanspaces.com
+  # Litestream supports Azure Blog Storage, Backblaze B2, DigitalOcean Spaces,
+  # Scaleway Object Storage, Google Cloud Storage, Linode Object Storage, and
+  # any SFTP server.
+  # In this example, we are using Rails encrypted credentials to store the URL to
+  # our storage provider bucket.
+  config.replica_bucket = litestream_credentials&.replica_bucket
+
+  # Replica-specific authentication key.
+  # Litestream needs authentication credentials to access your storage provider bucket.
+  # In this example, we are using Rails encrypted credentials to store the access key ID.
+  config.replica_key_id = litestream_credentials&.replica_key_id
+
+  # Replica-specific secret key.
+  # Litestream needs authentication credentials to access your storage provider bucket.
+  # In this example, we are using Rails encrypted credentials to store the secret access key.
+  config.replica_access_key = litestream_credentials&.replica_access_key
+end

config/puma.rb

@@ -2,34 +2,58 @@
 # are invoked here are part of Puma's configuration DSL. For more information
 # about methods provided by the DSL, see https://puma.io/puma/Puma/DSL.html.
 
-# Puma can serve each request in a thread from an internal thread pool.
-# The `threads` method setting takes two numbers: a minimum and maximum.
-# Any libraries that use thread pools should be configured to match
-# the maximum value specified for Puma. Default is set to 5 threads for minimum
-# and maximum; this matches the default thread size of Active Record.
-max_threads_count = ENV.fetch("RAILS_MAX_THREADS") { 5 }
-min_threads_count = ENV.fetch("RAILS_MIN_THREADS") { max_threads_count }
-threads min_threads_count, max_threads_count
-
-# Specifies that the worker count should equal the number of processors in production.
-if ENV["RAILS_ENV"] == "production"
-  require "concurrent-ruby"
-  worker_count = Integer(ENV.fetch("WEB_CONCURRENCY") { Concurrent.physical_processor_count })
-  workers worker_count if worker_count > 1
-end
+# Puma starts a configurable number of processes (workers) and each process
+# serves each request in a thread from an internal thread pool.
+#
+# The ideal number of threads per worker depends both on how much time the
+# application spends waiting for IO operations and on how much you wish to
+# to prioritize throughput over latency.
+#
+# As a rule of thumb, increasing the number of threads will increase how much
+# traffic a given process can handle (throughput), but due to CRuby's
+# Global VM Lock (GVL) it has diminishing returns and will degrade the
+# response time (latency) of the application.
+#
+# The default is set to 3 threads as it's deemed a decent compromise between
+# throughput and latency for the average Rails application.
+#
+# Any libraries that use a connection pool or another resource pool should
+# be configured to provide at least as many connections as the number of
+# threads. This includes Active Record's `pool` parameter in `database.yml`.
+threads_count = ENV.fetch("RAILS_MAX_THREADS", 3)
+threads threads_count, threads_count
 
-# Specifies the `worker_timeout` threshold that Puma will use to wait before
-# terminating a worker in development environments.
-worker_timeout 3600 if ENV.fetch("RAILS_ENV", "development") == "development"
+# Specifies the `environment` that Puma will run in.
+rails_env = ENV.fetch("RAILS_ENV", "development")
+environment rails_env
 
-# Specifies the `port` that Puma will listen on to receive requests; default is 3000.
-port ENV.fetch("PORT") { 3000 }
+case rails_env
+when "production"
+  # If you are running more than 1 thread per process, the workers count
+  # should be equal to the number of processors (CPU cores) in production.
+  #
+  # It defaults to 1 because it's impossible to reliably detect how many
+  # CPU cores are available. Make sure to set the `WEB_CONCURRENCY` environment
+  # variable to match the number of processors.
+  workers_count = Integer(ENV.fetch("WEB_CONCURRENCY", 1))
+  workers workers_count if workers_count > 1
 
-# Specifies the `environment` that Puma will run in.
-environment ENV.fetch("RAILS_ENV") { "development" }
+  preload_app!
+when "development"
+  # Specifies a very generous `worker_timeout` so that the worker
+  # isn't killed by Puma when suspended by a debugger.
+  worker_timeout 3600
+end
 
-# Specifies the `pidfile` that Puma will use.
-pidfile ENV.fetch("PIDFILE") { "tmp/pids/server.pid" }
+# Specifies the `port` that Puma will listen on to receive requests; default is 3000.
+port ENV.fetch("PORT", 3000)
 
 # Allow puma to be restarted by `bin/rails restart` command.
 plugin :tmp_restart
+
+plugin :litestream
+
+plugin :solid_queue
+
+# Only use a pidfile when requested
+pidfile ENV["PIDFILE"] if ENV["PIDFILE"]

config/solid_cache.yml

@@ -0,0 +1,15 @@
+default: &default
+  database: cache
+  store_options:
+    max_age: <%= 1.week.to_i %>
+    max_size: <%= 512.megabytes %>
+    namespace: <%= Rails.env %>
+
+development:
+  <<: *default
+
+test:
+  <<: *default
+
+production:
+  <<: *default

config/solid_queue.yml

@@ -2,6 +2,11 @@ default: &default
   dispatchers:
     - polling_interval: 1
       batch_size: 500
+      recurring_tasks:
+        # periodic_litestream_backup_verfication_job:
+        # class: Litestream::VerificationJob
+        # args: []
+        # schedule: every day at 1am EST
   workers:
     - queues: "*"
       threads: 3

db/cache_migrate/20240512200132_create_solid_cache_entries.solid_cache.rb

@@ -0,0 +1,12 @@
+# This migration comes from solid_cache (originally 20230724121448)
+class CreateSolidCacheEntries < ActiveRecord::Migration[7.0]
+  def change
+    create_table :solid_cache_entries do |t|
+      t.binary :key, null: false, limit: 1024
+      t.binary :value, null: false, limit: 512.megabytes
+      t.datetime :created_at, null: false
+
+      t.index :key, unique: true
+    end
+  end
+end