store tls certs + fix dockerfile

the server is now actually production ready. we generate tls
certificates if the ones in the database are going to expire within 30
days, and then store them in the database. still a bit precarious sinc
ethis isn't really atomic, but eh

Dockerfile

@@ -3,8 +3,6 @@ FROM nixos/nix:latest as builder
 COPY flake.nix flake.lock /tmp/build/
 COPY rust-toolchain.toml /tmp/build/
 COPY Cargo.toml Cargo.lock /tmp/build/
-COPY migrations /tmp/build/migrations
-
 WORKDIR /tmp/build
 
 RUN nix --extra-experimental-features "nix-command flakes" --option filter-syscalls false build '.#build-deps'
@@ -16,12 +14,19 @@ RUN nix --extra-experimental-features "nix-command flakes" --option filter-sysca
 RUN mkdir -p /tmp/nix-store-closure
 RUN cp -R $(nix-store -qR result/) /tmp/nix-store-closure
 
+FROM rust:slim-bookworm as builder-sqlx
+WORKDIR /build
+RUN apt-get update && apt-get install -y pkg-config libssl-dev
+RUN cargo install sqlx-cli --no-default-features --features postgres --root .
+
 FROM debian:bookworm-slim
 WORKDIR /app
-RUN apt-get update && apt-get install ca-certificates -y
+RUN apt-get update && apt-get install ca-certificates nix -y
 RUN update-ca-certificates -f
 COPY --from=builder /tmp/nix-store-closure /nix/store
 COPY --from=builder /tmp/build/result /app
+
+COPY --from=builder-sqlx /build/bin/sqlx /usr/bin/sqlx
 COPY migrations /app/migrations
 
 CMD ["/app/bin/file_server"]

fly.toml

@@ -1,30 +1,38 @@
-# fly.toml app configuration file generated for encrypted-file-server-spring-surf-6898 on 2024-02-15T10:44:17-05:00
+# fly.toml app configuration file generated for encrypted-file-server on 2024-04-23T15:32:04-04:00
 #
 # See https://fly.io/docs/reference/configuration/ for information about how to use this file.
 #
 
-app = 'encrypted-file-server'
+app = 'big-file-server'
 primary_region = 'ord'
 
+[deploy]
+  release_command = 'sqlx migrate run'
+
+[env]
+  RUST_BACKTRACE = '1'
+  TOKEN_PUBLIC_KEY = 'd27fb9c11d7608f86aa9e90a00133d58688b2fe4e7903a35199a25f7e905f658'
+
+[http_service]
+  internal_port = 80
+
 [[services]]
-  protocol = "udp"
+  protocol = 'udp'
   internal_port = 9999
   auto_stop_machines = true
   auto_start_machines = true
-  min_machines_running = 1
+  min_machines_running = 0
   processes = ['app']
+
   [[services.ports]]
     port = 9999
 
-[http_service]
-  internal_port = 80
-  force_https = false
-
-[env]
-  RUST_BACKTRACE="1"
-  TOKEN_PUBLIC_KEY="d27fb9c11d7608f86aa9e90a00133d58688b2fe4e7903a35199a25f7e905f658"
+[[restart]]
+  policy = "never"
+  retries = 10
+  processes = ["app"]
 
 [[vm]]
+  memory = '1gb'
   cpu_kind = 'shared'
   cpus = 1
-  memory_mb = 1024

migrations/20240421203437_tls_certs.sql

@@ -0,0 +1,6 @@
+CREATE TABLE tls_certs(
+    id SERIAL PRIMARY KEY,
+    cert_chain_pem TEXT NOT NULL,
+    private_key_pem TEXT NOT NULL,
+    expires_at TIMESTAMP NOT NULL
+)

src/main.rs

@@ -23,7 +23,6 @@ use wtransport::Identity;
 use wtransport::ServerConfig;
 
 use crate::db::{InsertChunkError, MetaDB, PostgresMetaDB};
-use crate::tls::order_tls_certs;
 use anyhow::Result;
 use bfsp::{
     chunks_uploaded_query_resp::{ChunkUploaded, ChunksUploaded},
@@ -36,6 +35,7 @@ use bfsp::{
 };
 use bfsp::{EncryptedFileMetadata, EncryptionNonce, PrependLen};
 use log::{debug, info, trace};
+use tls::get_tls_cert;
 use tokio::{
     fs,
     io::{AsyncReadExt, AsyncWriteExt},
@@ -98,28 +98,28 @@ async fn main() -> Result<()> {
     };
 
     if !cfg!(debug_assertions) && env::var("FLY_APP_NAME").is_ok() {
-        let cert_info = order_tls_certs().await?;
+        let cert_info = get_tls_cert().await?;
 
-        fs::create_dir_all("/etc/letsencrypt/live/encrypted-file-server.fly.dev/").await?;
+        fs::create_dir_all("/etc/letsencrypt/live/big-file-server.fly.dev/").await?;
         fs::write(
-            "/etc/letsencrypt/live/encrypted-file-server.fly.dev/chain.pem",
+            "/etc/letsencrypt/live/big-file-server.fly.dev/chain.pem",
             cert_info.cert_chain_pem,
         )
         .await?;
         fs::write(
-            "/etc/letsencrypt/live/encrypted-file-server.fly.dev/privkey.pem",
+            "/etc/letsencrypt/live/big-file-server.fly.dev/privkey.pem",
             cert_info.private_key_pem,
         )
         .await?;
     }
 
     let chain_file = match cfg!(debug_assertions) {
         true => "certs/localhost.pem",
-        false => "/etc/letsencrypt/live/encrypted-file-server.fly.dev/chain.pem",
+        false => "/etc/letsencrypt/live/big-file-server.fly.dev/chain.pem",
     };
     let key_file = match cfg!(debug_assertions) {
         true => "certs/localhost-key.pem",
-        false => "/etc/letsencrypt/live/encrypted-file-server.fly.dev/privkey.pem",
+        false => "/etc/letsencrypt/live/big-file-server.fly.dev/privkey.pem",
     };
 
     let config = ServerConfig::builder()

src/tls.rs

@@ -9,6 +9,7 @@ use instant_acme::{
     Account, AccountCredentials, AuthorizationStatus, ChallengeType, Identifier, NewOrder,
     OrderState, OrderStatus,
 };
+use sqlx::Row;
 use tokio::sync::RwLock;
 
 struct AcmeChallenge {
@@ -21,7 +22,42 @@ pub(crate) struct TlsCerts {
     pub(crate) private_key_pem: String,
 }
 
-pub async fn order_tls_certs() -> anyhow::Result<TlsCerts> {
+pub async fn get_tls_cert() -> anyhow::Result<TlsCerts> {
+    let pool = sqlx::PgPool::connect(
+        &env::var("DATABASE_URL")
+            .unwrap_or_else(|_| "postgres://postgres:postgres@localhost/efs_db".to_string()),
+    )
+    .await?;
+
+    Ok(sqlx::query("
+        SELECT cert_chain_pem, private_key_pem
+        FROM tls_certs
+        WHERE expires_at > NOW()
+        ORDER BY expires_at DESC
+        LIMIT 1
+    ").fetch_optional(&pool).await.map(|row| async move {
+        match row {
+            Some(row) => TlsCerts {
+                cert_chain_pem: row.get("cert_chain_pem"),
+                private_key_pem: row.get("private_key_pem"),
+            },
+            None => {
+                let tls_certs = order_tls_certs().await.unwrap();
+
+                // they really expire in 90 days, but let's encrypt recommends rotating every 60
+                sqlx::query(
+                    "INSERT INTO tls_certs (cert_chain_pem, private_key_pem, expires_at) VALUES ($1, $2, now() + INTERVAL '60 days')",
+                )
+                    .bind(&tls_certs.cert_chain_pem)
+                    .bind(&tls_certs.private_key_pem)
+                    .execute(&pool).await.unwrap();
+                tls_certs
+            }
+        }
+    }).unwrap().await)
+}
+
+async fn order_tls_certs() -> anyhow::Result<TlsCerts> {
     let acme_challenge: Arc<RwLock<Option<AcmeChallenge>>> = Arc::new(RwLock::new(None));
 
     let acme_challenge_clone = acme_challenge.clone();
@@ -33,7 +69,7 @@ pub async fn order_tls_certs() -> anyhow::Result<TlsCerts> {
     let acc_creds = serde_json::from_str::<AccountCredentials>(acc_key.as_str()).unwrap();
     let account = Account::from_credentials(acc_creds).await.unwrap();
 
-    let identifier = Identifier::Dns("encrypted-file-server.fly.dev".to_string());
+    let identifier = Identifier::Dns("big-file-server.fly.dev".to_string());
     let mut order = account
         .new_order(&NewOrder {
             identifiers: &[identifier],
@@ -42,6 +78,10 @@ pub async fn order_tls_certs() -> anyhow::Result<TlsCerts> {
         .unwrap();
 
     let state = order.state();
+    if ![OrderStatus::Pending, OrderStatus::Ready].contains(&state.status) {
+        return Err(anyhow::anyhow!("order is not pending or ready"));
+    }
+    info!("order state: {:#?}", state);
 
     // Pick the desired challenge type and prepare the response.
     let authorizations = order.authorizations().await.unwrap();
@@ -91,11 +131,13 @@ pub async fn order_tls_certs() -> anyhow::Result<TlsCerts> {
 
         delay *= 2;
         tries += 1;
-        match tries < 5 {
+        match tries < 10 {
             true => info!("order is not ready, waiting {delay:?} {state:?} {tries}"),
             false => {
                 error!("order is not ready {state:?} {tries}");
-                return Err(anyhow::anyhow!("order is not ready"));
+                return Err(anyhow::anyhow!(
+                    "order is not ready. make sure the HTTP server is running and reachable"
+                ));
             }
         }
     };