backups for production. better provisioning script

Backup/config.rb

@@ -0,0 +1,123 @@
+# encoding: utf-8
+
+##
+# Backup v4.x Configuration
+#
+# Documentation: http://backup.github.io/backup
+# Issue Tracker: https://github.com/backup/backup/issues
+
+##
+# Config Options
+#
+# The options here may be overridden on the command line, but the result
+# will depend on the use of --root-path on the command line.
+#
+# If --root-path is used on the command line, then all paths set here
+# will be overridden. If a path (like --tmp-path) is not given along with
+# --root-path, that path will use it's default location _relative to --root-path_.
+#
+# If --root-path is not used on the command line, a path option (like --tmp-path)
+# given on the command line will override the tmp_path set here, but all other
+# paths set here will be used.
+#
+# Note that relative paths given on the command line without --root-path
+# are relative to the current directory. The root_path set here only applies
+# to relative paths set here.
+#
+# ---
+#
+# Sets the root path for all relative paths, including default paths.
+# May be an absolute path, or relative to the current working directory.
+#
+# root_path 'my/root'
+#
+# Sets the path where backups are processed until they're stored.
+# This must have enough free space to hold apx. 2 backups.
+# May be an absolute path, or relative to the current directory or +root_path+.
+#
+# tmp_path  'my/tmp'
+#
+# Sets the path where backup stores persistent information.
+# When Backup's Cycler is used, small YAML files are stored here.
+# May be an absolute path, or relative to the current directory or +root_path+.
+#
+# data_path 'my/data'
+
+##
+# Utilities
+#
+# If you need to use a utility other than the one Backup detects,
+# or a utility can not be found in your $PATH.
+#
+#   Utilities.configure do
+#     tar       '/usr/bin/gnutar'
+#     redis_cli '/opt/redis/redis-cli'
+#   end
+
+##
+# Logging
+#
+# Logging options may be set on the command line, but certain settings
+# may only be configured here.
+#
+#   Logger.configure do
+#     console.quiet     = true            # Same as command line: --quiet
+#     logfile.max_bytes = 2_000_000       # Default: 500_000
+#     syslog.enabled    = true            # Same as command line: --syslog
+#     syslog.ident      = 'my_app_backup' # Default: 'backup'
+#   end
+#
+# Command line options will override those set here.
+# For example, the following would override the example settings above
+# to disable syslog and enable console output.
+#   backup perform --trigger my_backup --no-syslog --no-quiet
+
+##
+# Component Defaults
+#
+# Set default options to be applied to components in all models.
+# Options set within a model will override those set here.
+#
+#   Storage::S3.defaults do |s3|
+#     s3.access_key_id     = "my_access_key_id"
+#     s3.secret_access_key = "my_secret_access_key"
+#   end
+#
+#   Notifier::Mail.defaults do |mail|
+#     mail.from                 = '[email protected]'
+#     mail.to                   = '[email protected]'
+#     mail.address              = 'smtp.gmail.com'
+#     mail.port                 = 587
+#     mail.domain               = 'your.host.name'
+#     mail.user_name            = '[email protected]'
+#     mail.password             = 'my_password'
+#     mail.authentication       = 'plain'
+#     mail.encryption           = :starttls
+#   end
+
+##
+# Preconfigured Models
+#
+# Create custom models with preconfigured components.
+# Components added within the model definition will
+# +add to+ the preconfigured components.
+#
+#   preconfigure 'MyModel' do
+#     archive :user_pictures do |archive|
+#       archive.add '~/pictures'
+#     end
+#
+#     notify_by Mail do |mail|
+#       mail.to = '[email protected]'
+#     end
+#   end
+#
+#   MyModel.new(:john_smith, 'John Smith Backup') do
+#     archive :user_music do |archive|
+#       archive.add '~/music'
+#     end
+#
+#     notify_by Mail do |mail|
+#       mail.to = '[email protected]'
+#     end
+#   end

Backup/models/my_backup.rb

@@ -0,0 +1,117 @@
+# encoding: utf-8
+
+##
+# Backup Generated: my_backup
+# Once configured, you can run the backup with the following command:
+#
+# $ backup perform -t my_backup [-c <path_to_configuration_file>]
+#
+# For more information about Backup's components, see the documentation at:
+# http://backup.github.io/backup
+#
+require 'yaml'
+Model.new(:my_backup, 'Description for my_backup') do
+  env_file = "/var/www/logan-#{ENV['RAILS_ENV']}/shared/local_env.yml"
+  YAML.load(File.open(env_file)).each do |key, value|
+    ENV[key.to_s] = value
+  end if File.exist?(env_file)
+  ##
+  # Archive [Archive]
+  #
+  # Adding a file or directory (including sub-directories):
+  #   archive.add "/path/to/a/file.rb"
+  #   archive.add "/path/to/a/directory/"
+  #
+  # Excluding a file or directory (including sub-directories):
+  #   archive.exclude "/path/to/an/excluded_file.rb"
+  #   archive.exclude "/path/to/an/excluded_directory
+  #
+  # By default, relative paths will be relative to the directory
+  # where `backup perform` is executed, and they will be expanded
+  # to the root of the filesystem when added to the archive.
+  #
+  # If a `root` path is set, relative paths will be relative to the
+  # given `root` path and will not be expanded when added to the archive.
+  #
+  #   archive.root '/path/to/archive/root'
+  #
+  # archive :my_archive do |archive|
+  #   # Run the `tar` command using `sudo`
+  #   # archive.use_sudo
+  #   archive.add "/path/to/a/file.rb"
+  #   archive.add "/path/to/a/folder/"
+  #   archive.exclude "/path/to/a/excluded_file.rb"
+  #   archive.exclude "/path/to/a/excluded_folder"
+  # end
+
+  ##
+  # MySQL [Database]
+  #
+  database MySQL do |db|
+    # To dump all databases, set `db.name = :all` (or leave blank)
+    db.name               = ENV['RAILS_ENV']
+    db.username           = 'root'
+    db.password           = 'password'
+    db.host               = 'localhost'
+    db.port               = 3306
+    # Note: when using `skip_tables` with the `db.name = :all` option,
+    # table names should be prefixed with a database name.
+    # e.g. ["db_name.table_to_skip", ...]
+    # db.skip_tables        = ["skip", "these", "tables"]
+    # db.only_tables        = ["only", "these", "tables"]
+    db.additional_options = ['--quick', '--single-transaction']
+  end
+
+  ##
+  # Local (Copy) [Storage]
+  #
+  store_with Local do |local|
+    local.path       = '~/backups/'
+    local.keep       = 5
+    # local.keep       = Time.now - 2592000 # Remove all backups older than 1 month.
+  end
+
+  ##
+  # Amazon Simple Storage Service [Storage]
+  #
+  store_with S3 do |s3|
+    # AWS Credentials
+    s3.access_key_id     = ENV['AWS_API_TOKEN']
+    s3.secret_access_key = ENV['AWS_API_SECRET']
+    # Or, to use a IAM Profile:
+    # s3.use_iam_profile = true
+
+    s3.region            = 'us-east-1'
+    s3.bucket            = ENV['AWS_S3_BUCKET']
+    s3.path              = "/patterns_backups_#{ENV['RAILS_ENV']}"
+    s3.keep              = 15
+    # s3.keep              = Time.now - 2592000 # Remove all backups older than 1 month.
+  end
+
+  ##
+  # Gzip [Compressor]
+  #
+  compress_with Gzip
+
+  ##
+  # Mail [Notifier]
+  #
+  # The default delivery method for Mail Notifiers is 'SMTP'.
+  # See the documentation for other delivery options.
+  #
+  notify_by Mail do |mail|
+    mail.on_success           = false
+    mail.on_warning           = true
+    mail.on_failure           = true
+
+    mail.from                 = ENV['MAILER_SENDER']
+    mail.to                   = ENV['MAIL_ADMIN']
+    mail.address              = ENV['SMTP_HOST']
+    mail.port                 = ENV['SMTP_PORT']
+    mail.domain               = ENV["#{ENV['RAILS_ENV'].upcase}_SERVER"]
+    mail.user_name            = ENV['SMTP_USERNAME']
+    mail.password             = ENV['SMTP_PASSWORD']
+    mail.authentication       = 'plain'
+    mail.encryption           = :starttls
+  end
+end

app/models/concerns/searchable.rb

@@ -111,7 +111,7 @@ def complex_search(params, per_page)
             must { string "preferred_contact_method:#{params[:preferred_contact_method]}" } unless params[:preferred_contact_method].blank?
           end
         end
-        #filter :terms, tag_values: params[:tags] if params[:tags].present?
+        # filter :terms, tag_values: params[:tags] if params[:tags].present?
       end
     end
     # rubocop:enable Metrics/MethodLength, Metrics/AbcSize, Metrics/CyclomaticComplexity, Metrics/PerceivedComplexity

config/sample.local_env.yml

@@ -23,7 +23,11 @@ STAGING_BRANCH: "devlopment"
 GIT_REPOSITORY: "[email protected]:example/example.git"
 TIME_ZONE: "Central Time (US & Canada)"
 MAILER_SENDER: "[email protected]"
+MAIL_ADMIN: "[email protected]"
 SMTP_HOST: "smtp.mandrillapp.com"
 SMTP_PORT: "587"
 SMTP_USERNAME: "Blue_Ridge"
-SMTP_PASSWORD: "FFVglMV3NMxLsyzdJe63iw"
+SMTP_PASSWORD: "241234"
+AWS_API_TOKEN: ""
+AWS_API_SECRET: ""
+AWS_S3_BUCKET: ''

lib/tasks/backup.rake

@@ -0,0 +1,4 @@
+desc 'back up the db to S3'
+task :backup do
+  sh "backup perform --trigger my_backup -r /var/www/logan-production/current/"
+end

provision_new_server.sh

@@ -30,24 +30,32 @@ echo "MYSQL_PASSWORD=password" >> /etc/environment
 echo "MYSQL_HOST=localhost" >> /etc/environment
 echo "RAILS_ENV=production" >> /etc/environment
 
-apt-get update && apt-get install -y mysql-server libmysqlclient-dev redis-server openjdk-6-jre elasticsearch git git-core curl zlib1g-dev build-essential libssl-dev libreadline-dev libgmp-dev libyaml-dev libsqlite3-dev sqlite3 libxml2-dev libxslt1-dev libcurl4-openssl-dev python-software-properties libffi-dev nginx gpg ruby1.8-dev openjdk-7-jre elasticsearch
-
+apt-get update && apt-get install -y mysql-server libmysqlclient-dev redis-server openjdk-6-jre elasticsearch git git-core curl zlib1g-dev build-essential libssl-dev libreadline-dev libgmp-dev libyaml-dev libsqlite3-dev sqlite3 libxml2-dev libxslt1-dev libcurl4-openssl-dev python-software-properties libffi-dev nginx gpgv2 ruby-dev openjdk-7-jre autoconf libgdbm-dev libncurses5-dev automake libtool bison gawk g++ gcc make libreadline6-dev zlib1g-dev libssl-dev libyaml-dev libsqlite3-dev sqlite3 autoconf libgdbm-dev libncurses5-dev automake libtool bison pkg-config libffi-dev nodejs
 
+service elasticsearch start
 # we don't want the default nginx server setup.
 rm /etc/nginx/sites-enabled/default
-openssl dhparam -out /etc/nginx/dhparam.pem 4096
-service nginx stop
+openssl dhparam -dsaparam -out /etc/nginx/dhparam.pem 2048
+#service nginx stop
 cd /root
 git clone https://github.com/letsencrypt/letsencrypt
-/root/letsencrypt/letsencrypt-auto certonly --standalone --agree-tos --email [email protected] -d patterns.brl.nyc
+#/root/letsencrypt/letsencrypt-auto certonly --standalone --agree-tos --email [email protected] -d staging.patterns.brl.nyc
 
+#service nginx start
 cat >/etc/cron.weekly/letsencrypt.sh <<EOL
 PATH=/usr/local/sbin:/usr/local/bin:/sbin:/bin:/usr/sbin:/usr/bin:$PATH
 /root/letsencrypt/letsencrypt-auto certonly --webroot -w /var/www/logan-production/current/public --agree-tos --email [email protected] -d patterns.brl.nyc
 service nginx restart
 EOL
 chmod +x /etc/cron.weekly/letsencrypt.sh
 
+#passwordless sudo for logan, or else we can't install rvm
+echo 'logan ALL=(ALL) NOPASSWD: ALL' > /etc/sudoers.d/logan
+
+mkdir /var/www/logan-production
+mkdir /var/www/logan-staging
+
+
 # creating the logan user.
 getent passwd logan  > /dev/null
 if [ $? -eq 0 ]; then
@@ -56,12 +64,33 @@ else
   useradd -m -s /bin/bash logan;
   su - logan
   mkdir ~/.ssh/
-  echo "ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQCUkhUCqUdEjpm92sN5OGW7cLekAJNdT0HTDqCsUR28I3eB1lelKLWGDhIkR2L3TZmiX511+ZfaydgrFJEUqT+gotUKmWmW9CVpt5OQTZPPNJBkZ99uXYqg2sLHpAptacVIn/UGS4RRvMG6gT+pYiI1epyY0F0uqeNDVwO0HAo7pLxS7K/eK49QUZQMszjkv7TxykIDDe8wjVkkNIABbnz0vYWibaCdyYsTOqqDhrywXhX3uIoUHYqlQdN5Wk11jqnxGFrixojEhy0LEosHry8qjFBNP6H/jyfuFQeZW6+tDW8H3dY+WXYRkcN6harXmi4o/GewkAkukRVE12+nLXdX deploy@patterns" >> ~/.ssh/authorized_keys
-
+  cat >~/.ssh/authorized_keys <<EOL
+ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQCUkhUCqUdEjpm92sN5OGW7cLekAJNdT0HTDqCsUR28I3eB1lelKLWGDhIkR2L3TZmiX511+ZfaydgrFJEUqT+gotUKmWmW9CVpt5OQTZPPNJBkZ99uXYqg2sLHpAptacVIn/UGS4RRvMG6gT+pYiI1epyY0F0uqeNDVwO0HAo7pLxS7K/eK49QUZQMszjkv7TxykIDDe8wjVkkNIABbnz0vYWibaCdyYsTOqqDhrywXhX3uIoUHYqlQdN5Wk11jqnxGFrixojEhy0LEosHry8qjFBNP6H/jyfuFQeZW6+tDW8H3dY+WXYRkcN6harXmi4o/GewkAkukRVE12+nLXdX deploy@patterns
+ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQDRCFqdXUioU3N1GIRK5bowUfJ9DKswJeMp6diQDOfCU4rKN4Y6jg/Xzl8ijTXsH3e+q3hvpPAbynjNF9cK3af93tdMQ49fJajPRVlM+mZW2MXkJAnI0TkqGWqwk93KqnVAajVdaDo+jEFqdNvYzYLeqwAJUaED0OyD/GlOBlF0NV9kT2mVXGtCdcJ+ItTqFwtn6NcAuXg+/5S2ZpBJGjf1mOVyLAHdbGg00L5YY2GpU4s7L02fKqIdOzNgmU2ek74ba0F74KTcEvReRNePFjlCNZqrbqiw6dgOoo9BGjbCploNdmUzA4DJ9CQHx3lBPQXLjEiNx+kMUkxC0JxlVQbb [email protected]
+EOL
+  # so we don't have key failures for github
   ssh-keyscan -H github.com >> ~/.ssh/known_hosts
+
+  # installing ruby and rvm
+  gpg --keyserver hkp://keys.gnupg.net --recv-keys 409B6B1796C275462A1703113804BB82D39DC0E3
+  curl -sSL https://get.rvm.io | bash -s stable
+  source /home/logan/.rvm/scripts/rvm
+  rvm install 2.2.4
+  rvm use 2.2.4@staging --create
+  rvm use 2.2.4@production --create
+  rvm @global do gem install backup bundler
+  ln -s /var/www/logan-production
+  ln -s /var/www/logan-staging
   exit # back to root.
 fi
 
+# remove our logan passwordless sudo, for security
+rm /etc/sudoers.d/logan
+chown -R logan:logan /var/www/logan*
+
 #we've provisioned this server
+touch /etc/provisioned
 
-touch /etc/provisioned
+# now run:
+# cap staging deploy:setup
+# cap staging deploy:cold