From 553ece4ef6fb21656b43d724978cf3a9bfd0a5eb Mon Sep 17 00:00:00 2001
From: Amos Wenger <amos@bearcove.net>
Date: Mon, 26 Aug 2024 18:05:42 +0200
Subject: [PATCH 01/24] try out benchmarks

---
 crates/httpwg-loona/src/lib.rs | 114 +++++++++++++++++++++++----------
 crates/loona/src/h2/body.rs    |  58 +----------------
 crates/loona/src/h2/server.rs  |   7 +-
 crates/loona/src/types/mod.rs  |  63 ++++++++++++++++++
 k6/script.js                   |  16 +++++
 5 files changed, 162 insertions(+), 96 deletions(-)
 create mode 100644 k6/script.js

diff --git a/crates/httpwg-loona/src/lib.rs b/crates/httpwg-loona/src/lib.rs
index 1974c226..05ed5cad 100644
--- a/crates/httpwg-loona/src/lib.rs
+++ b/crates/httpwg-loona/src/lib.rs
@@ -1,12 +1,13 @@
 use b_x::{BxForResults, BX};
-use std::{cell::RefCell, rc::Rc};
+use std::{cell::RefCell, io::Write, rc::Rc};
 use tokio::{process::Command, sync::oneshot};
 
-use buffet::{IntoHalves, RollMut};
+use buffet::{IntoHalves, Piece, RollMut};
 use loona::{
+    error::NeverError,
     http::{self, StatusCode},
     Body, BodyChunk, Encoder, ExpectResponseHeaders, Responder, Response, ResponseDone,
-    ServerDriver,
+    ServerDriver, SinglePieceBody,
 };
 
 #[derive(Debug, Clone, Copy)]
@@ -173,12 +174,12 @@ where
 
     async fn handle(
         &self,
-        _req: loona::Request,
+        req: loona::Request,
         req_body: &mut impl Body,
         mut res: Responder<OurEncoder, ExpectResponseHeaders>,
     ) -> Result<Responder<OurEncoder, ResponseDone>, Self::Error> {
         // if the client sent `expect: 100-continue`, we must send a 100 status code
-        if let Some(h) = _req.headers.get(http::header::EXPECT) {
+        if let Some(h) = req.headers.get(http::header::EXPECT) {
             if &h[..] == b"100-continue" {
                 res.write_interim_response(Response {
                     status: StatusCode::CONTINUE,
@@ -188,41 +189,86 @@ where
             }
         }
 
-        // then read the full request body
-        let mut req_body_len = 0;
-        loop {
-            let chunk = req_body.next_chunk().await.bx()?;
-            match chunk {
-                BodyChunk::Done { trailers } => {
-                    // yey
-                    if let Some(trailers) = trailers {
-                        tracing::debug!(trailers_len = %trailers.len(), "received trailers");
-                    }
-                    break;
+        let res = match req.uri.path() {
+            "/echo-body" => res
+                .write_final_response_with_body(
+                    Response {
+                        status: StatusCode::OK,
+                        ..Default::default()
+                    },
+                    req_body,
+                )
+                .await
+                .bx()?,
+            "/stream-big-body" => {
+                let mut roll = RollMut::alloc().bx()?;
+                for _ in 0..256 {
+                    roll.write_all("this is a big chunk".as_bytes()).bx()?;
                 }
-                BodyChunk::Chunk(chunk) => {
-                    req_body_len += chunk.len();
+
+                struct RepeatBody {
+                    piece: Piece,
+                    n: usize,
+                    written: usize,
                 }
-            }
-        }
-        tracing::debug!(%req_body_len, "read request body");
 
-        tracing::trace!("writing final response");
-        let mut res = res
-            .write_final_response(Response {
-                status: StatusCode::OK,
-                ..Default::default()
-            })
-            .await?;
+                impl std::fmt::Debug for RepeatBody {
+                    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+                        f.debug_struct("RepeatBody")
+                            .field("piece_len", &self.piece.len())
+                            .field("n", &self.n)
+                            .field("written", &self.written)
+                            .finish()
+                    }
+                }
+
+                impl Body for RepeatBody {
+                    type Error = NeverError;
+
+                    fn content_len(&self) -> Option<u64> {
+                        Some(self.n as u64 * self.piece.len() as u64)
+                    }
+
+                    fn eof(&self) -> bool {
+                        self.written == self.n
+                    }
 
-        tracing::trace!("writing body chunk");
-        res.write_chunk("it's less dire to lose, than to lose oneself".into())
-            .await?;
+                    async fn next_chunk(&mut self) -> Result<BodyChunk, Self::Error> {
+                        if self.eof() {
+                            return Ok(BodyChunk::Done { trailers: None });
+                        }
 
-        tracing::trace!("finishing body (with no trailers)");
-        let res = res.finish_body(None).await?;
+                        let chunk = self.piece.clone();
+                        self.written += 1;
+                        Ok(BodyChunk::Chunk(chunk))
+                    }
+                }
 
-        tracing::trace!("we're done");
+                res.write_final_response_with_body(
+                    Response {
+                        status: StatusCode::OK,
+                        ..Default::default()
+                    },
+                    &mut RepeatBody {
+                        piece: roll.take_all().into(),
+                        n: 128,
+                        written: 0,
+                    },
+                )
+                .await
+                .bx()?
+            }
+            _ => res
+                .write_final_response_with_body(
+                    Response {
+                        status: StatusCode::OK,
+                        ..Default::default()
+                    },
+                    &mut SinglePieceBody::from("it's less dire to lose, than to lose oneself"),
+                )
+                .await
+                .bx()?,
+        };
         Ok(res)
     }
 }
diff --git a/crates/loona/src/h2/body.rs b/crates/loona/src/h2/body.rs
index e636cb36..5cbb7079 100644
--- a/crates/loona/src/h2/body.rs
+++ b/crates/loona/src/h2/body.rs
@@ -1,8 +1,6 @@
-use core::fmt;
-
 use tokio::sync::mpsc;
 
-use crate::{error::NeverError, Body, BodyChunk, Headers};
+use crate::{Body, BodyChunk, Headers};
 use buffet::Piece;
 
 use super::types::H2StreamError;
@@ -177,57 +175,3 @@ impl Body for H2Body {
         Ok(chunk)
     }
 }
-
-pub(crate) struct SinglePieceBody {
-    content_len: u64,
-    piece: Option<Piece>,
-}
-
-impl fmt::Debug for SinglePieceBody {
-    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
-        let mut debug_struct = f.debug_struct("SinglePieceBody");
-        debug_struct.field("content_len", &self.content_len);
-
-        if let Some(piece) = &self.piece {
-            match std::str::from_utf8(piece.as_ref()) {
-                Ok(utf8_str) => debug_struct.field("piece", &utf8_str),
-                Err(_) => debug_struct.field("piece", &"(non-utf8 string)"),
-            };
-        } else {
-            debug_struct.field("piece", &"(none)");
-        }
-
-        debug_struct.finish()
-    }
-}
-
-impl SinglePieceBody {
-    pub(crate) fn new(piece: Piece) -> Self {
-        let content_len = piece.len() as u64;
-        Self {
-            content_len,
-            piece: Some(piece),
-        }
-    }
-}
-
-impl Body for SinglePieceBody {
-    type Error = NeverError;
-
-    fn content_len(&self) -> Option<u64> {
-        Some(self.content_len)
-    }
-
-    fn eof(&self) -> bool {
-        self.piece.is_none()
-    }
-
-    async fn next_chunk(&mut self) -> Result<BodyChunk, Self::Error> {
-        tracing::trace!( has_piece = %self.piece.is_some(), "SinglePieceBody::next_chunk");
-        if let Some(piece) = self.piece.take() {
-            Ok(BodyChunk::Chunk(piece))
-        } else {
-            Ok(BodyChunk::Done { trailers: None })
-        }
-    }
-}
diff --git a/crates/loona/src/h2/server.rs b/crates/loona/src/h2/server.rs
index 3585954b..2916164c 100644
--- a/crates/loona/src/h2/server.rs
+++ b/crates/loona/src/h2/server.rs
@@ -34,13 +34,10 @@ use crate::{
         },
     },
     util::{read_and_parse, ReadAndParseError},
-    Headers, Method, Request, Responder, ResponderOrBodyError, ServeOutcome, ServerDriver,
+    Headers, Method, Request, Responder, ResponderOrBodyError, ServeOutcome, ServerDriver, SinglePieceBody,
 };
 
-use super::{
-    body::{ChunkPosition, SinglePieceBody},
-    types::H2ErrorLevel,
-};
+use super::{body::ChunkPosition, types::H2ErrorLevel};
 
 pub const MAX_WINDOW_SIZE: i64 = u32::MAX as i64;
 
diff --git a/crates/loona/src/types/mod.rs b/crates/loona/src/types/mod.rs
index d8b3588d..97d98ccb 100644
--- a/crates/loona/src/types/mod.rs
+++ b/crates/loona/src/types/mod.rs
@@ -223,3 +223,66 @@ pub enum ServeOutcome {
     /// the client
     SuccessfulHttp2GracefulShutdown,
 }
+
+pub struct SinglePieceBody {
+    content_len: u64,
+    piece: Option<Piece>,
+}
+
+impl<T> From<T> for SinglePieceBody
+where
+    T: Into<Piece>,
+{
+    fn from(piece: T) -> Self {
+        Self::new(piece.into())
+    }
+}
+
+impl fmt::Debug for SinglePieceBody {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        let mut debug_struct = f.debug_struct("SinglePieceBody");
+        debug_struct.field("content_len", &self.content_len);
+
+        if let Some(piece) = &self.piece {
+            match std::str::from_utf8(piece.as_ref()) {
+                Ok(utf8_str) => debug_struct.field("piece", &utf8_str),
+                Err(_) => debug_struct.field("piece", &"(non-utf8 string)"),
+            };
+        } else {
+            debug_struct.field("piece", &"(none)");
+        }
+
+        debug_struct.finish()
+    }
+}
+
+impl SinglePieceBody {
+    pub(crate) fn new(piece: Piece) -> Self {
+        let content_len = piece.len() as u64;
+        Self {
+            content_len,
+            piece: Some(piece),
+        }
+    }
+}
+
+impl Body for SinglePieceBody {
+    type Error = NeverError;
+
+    fn content_len(&self) -> Option<u64> {
+        Some(self.content_len)
+    }
+
+    fn eof(&self) -> bool {
+        self.piece.is_none()
+    }
+
+    async fn next_chunk(&mut self) -> Result<BodyChunk, Self::Error> {
+        tracing::trace!( has_piece = %self.piece.is_some(), "SinglePieceBody::next_chunk");
+        if let Some(piece) = self.piece.take() {
+            Ok(BodyChunk::Chunk(piece))
+        } else {
+            Ok(BodyChunk::Done { trailers: None })
+        }
+    }
+}
diff --git a/k6/script.js b/k6/script.js
new file mode 100644
index 00000000..ab7c6d86
--- /dev/null
+++ b/k6/script.js
@@ -0,0 +1,16 @@
+import http from "k6/http";
+import { sleep } from "k6";
+
+export const options = {
+  // A number specifying the number of VUs to run concurrently.
+  vus: 200,
+  // A string specifying the total duration of the test run.
+  duration: "10s",
+};
+
+export default function () {
+  http.get("http://localhost:8001/", {
+    http: {},
+  });
+  sleep(1);
+}

From 8030150aa2aa3e645a08bbccd68fcf2bb5b21893 Mon Sep 17 00:00:00 2001
From: Amos Wenger <amos@bearcove.net>
Date: Mon, 26 Aug 2024 18:35:24 +0200
Subject: [PATCH 02/24] try higher ring size

---
 crates/httpwg-hyper/src/main.rs | 2 ++
 crates/luring/src/linux.rs      | 2 +-
 2 files changed, 3 insertions(+), 1 deletion(-)

diff --git a/crates/httpwg-hyper/src/main.rs b/crates/httpwg-hyper/src/main.rs
index 243fab37..e2614763 100644
--- a/crates/httpwg-hyper/src/main.rs
+++ b/crates/httpwg-hyper/src/main.rs
@@ -110,6 +110,8 @@ async fn main() {
     println!("Using {proto:?} protocol (export TEST_PROTO=h1 or TEST_PROTO=h2 to override)");
 
     while let Ok((stream, _)) = ln.accept().await {
+        stream.set_nodelay(true).unwrap();
+
         tokio::spawn(async move {
             let mut builder = auto::Builder::new(TokioExecutor::new());
 
diff --git a/crates/luring/src/linux.rs b/crates/luring/src/linux.rs
index 2726f767..61bd997c 100644
--- a/crates/luring/src/linux.rs
+++ b/crates/luring/src/linux.rs
@@ -10,7 +10,7 @@ thread_local! {
     // for op cancellations.
     static URING: Rc<IoUringAsync> = {
         // FIXME: magic values
-        Rc::new(IoUringAsync::new(8).unwrap())
+        Rc::new(IoUringAsync::new(512).unwrap())
     };
 }
 

From a52f51e5148ca01934e07713c295bc764d31ec97 Mon Sep 17 00:00:00 2001
From: Amos Wenger <amos@bearcove.net>
Date: Wed, 28 Aug 2024 20:03:43 +0200
Subject: [PATCH 03/24] Allow listening on public addresses

---
 crates/httpwg-hyper/src/main.rs       | 3 ++-
 crates/httpwg-loona/benches/h2load.rs | 2 +-
 crates/httpwg-loona/src/lib.rs        | 4 ++--
 crates/httpwg-loona/src/main.rs       | 2 ++
 4 files changed, 7 insertions(+), 4 deletions(-)

diff --git a/crates/httpwg-hyper/src/main.rs b/crates/httpwg-hyper/src/main.rs
index e2614763..6f27d8ed 100644
--- a/crates/httpwg-hyper/src/main.rs
+++ b/crates/httpwg-hyper/src/main.rs
@@ -87,7 +87,8 @@ where
 #[tokio::main(flavor = "current_thread")]
 async fn main() {
     let port = std::env::var("PORT").unwrap_or("0".to_string());
-    let ln = tokio::net::TcpListener::bind(format!("127.0.0.1:{}", port))
+    let addr = std::env::var("ADDR").unwrap_or("127.0.0.1".to_string());
+    let ln = tokio::net::TcpListener::bind(format!("{addr}:{port}"))
         .await
         .unwrap();
     let upstream_addr = ln.local_addr().unwrap();
diff --git a/crates/httpwg-loona/benches/h2load.rs b/crates/httpwg-loona/benches/h2load.rs
index 716f0055..e37d7653 100644
--- a/crates/httpwg-loona/benches/h2load.rs
+++ b/crates/httpwg-loona/benches/h2load.rs
@@ -4,7 +4,7 @@ use httpwg_loona::{Mode, Proto};
 pub fn h2load(c: &mut Criterion) {
     c.bench_function("h2load", |b| {
         b.iter(|| {
-            httpwg_loona::do_main(0, Proto::H2, Mode::H2Load);
+            httpwg_loona::do_main("127.0.0.1".to_string(), 0, Proto::H2, Mode::H2Load);
         })
     });
 }
diff --git a/crates/httpwg-loona/src/lib.rs b/crates/httpwg-loona/src/lib.rs
index 05ed5cad..ab7b7988 100644
--- a/crates/httpwg-loona/src/lib.rs
+++ b/crates/httpwg-loona/src/lib.rs
@@ -32,7 +32,7 @@ pub enum Mode {
     H2Load,
 }
 
-pub fn do_main(port: u16, proto: Proto, mode: Mode) {
+pub fn do_main(addr: String, port: u16, proto: Proto, mode: Mode) {
     let server_start = std::time::Instant::now();
 
     let (ready_tx, cancel_rx, is_h2load) = match mode {
@@ -44,7 +44,7 @@ pub fn do_main(port: u16, proto: Proto, mode: Mode) {
     };
 
     let server_fut = async move {
-        let ln = buffet::net::TcpListener::bind(format!("127.0.0.1:{port}").parse().unwrap())
+        let ln = buffet::net::TcpListener::bind(format!("{addr}:{port}").parse().unwrap())
             .await
             .unwrap();
         let port = ln.local_addr().unwrap().port();
diff --git a/crates/httpwg-loona/src/main.rs b/crates/httpwg-loona/src/main.rs
index ed1f2a0a..cf125b7b 100644
--- a/crates/httpwg-loona/src/main.rs
+++ b/crates/httpwg-loona/src/main.rs
@@ -9,6 +9,7 @@ fn main() {
         .unwrap_or("8001".to_string())
         .parse()
         .unwrap();
+    let addr = std::env::var("ADDR").unwrap_or_else(|_| "127.0.0.1".to_string());
     let proto = match std::env::var("TEST_PROTO")
         .unwrap_or("h1".to_string())
         .as_str()
@@ -24,6 +25,7 @@ fn main() {
 
     let server_handle = std::thread::spawn(move || {
         httpwg_loona::do_main(
+            addr,
             port,
             proto,
             httpwg_loona::Mode::Server {

From 69f8b459db0ea576db5da1cc3b46b80996cf9a56 Mon Sep 17 00:00:00 2001
From: Amos Wenger <amos@bearcove.net>
Date: Wed, 28 Aug 2024 20:32:10 +0200
Subject: [PATCH 04/24] typo

---
 crates/buffet/src/lib.rs | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/crates/buffet/src/lib.rs b/crates/buffet/src/lib.rs
index dd8aeafa..3cbb97f1 100644
--- a/crates/buffet/src/lib.rs
+++ b/crates/buffet/src/lib.rs
@@ -73,7 +73,7 @@ pub fn start<F: Future>(task: F) -> F::Output {
         if (tokio::time::timeout(cancel_submit_timeout, &mut lset).await).is_err() {
             drop(cancel_tx);
 
-            // during this second poll, the async cancellations hopefuly finish
+            // during this second poll, the async cancellations hopefully finish
             let cleanup_timeout = std::time::Duration::from_millis(500);
             if (tokio::time::timeout(cleanup_timeout, lset).await).is_err() {
                 tracing::warn!(

From 22ae020a98f12bc4d3b11be34794ceb42a061369 Mon Sep 17 00:00:00 2001
From: Amos Wenger <amos@bearcove.net>
Date: Wed, 28 Aug 2024 20:43:05 +0200
Subject: [PATCH 05/24] wat

---
 crates/httpwg-loona/src/main.rs | 30 ++++++++++++++----------------
 1 file changed, 14 insertions(+), 16 deletions(-)

diff --git a/crates/httpwg-loona/src/main.rs b/crates/httpwg-loona/src/main.rs
index cf125b7b..d80ca251 100644
--- a/crates/httpwg-loona/src/main.rs
+++ b/crates/httpwg-loona/src/main.rs
@@ -1,4 +1,4 @@
-use httpwg_loona::Proto;
+use httpwg_loona::{Proto, Ready};
 use tracing::Level;
 use tracing_subscriber::{filter::Targets, layer::SubscriberExt, util::SubscriberInitExt};
 
@@ -22,24 +22,22 @@ fn main() {
 
     let (ready_tx, ready_rx) = tokio::sync::oneshot::channel();
     let (cancel_tx, cancel_rx) = tokio::sync::oneshot::channel();
+    std::mem::forget(cancel_tx);
 
-    let server_handle = std::thread::spawn(move || {
-        httpwg_loona::do_main(
-            addr,
-            port,
-            proto,
-            httpwg_loona::Mode::Server {
-                ready_tx,
-                cancel_rx,
-            },
-        );
+    std::thread::spawn(move || {
+        let ready: Ready = ready_rx.blocking_recv().unwrap();
+        eprintln!("I listen on {}", ready.port);
     });
 
-    let ready = ready_rx.blocking_recv().unwrap();
-    eprintln!("I listen on {}", ready.port);
-
-    server_handle.join().unwrap();
-    drop(cancel_tx);
+    httpwg_loona::do_main(
+        addr,
+        port,
+        proto,
+        httpwg_loona::Mode::Server {
+            ready_tx,
+            cancel_rx,
+        },
+    );
 }
 
 fn setup_tracing_and_error_reporting() {

From 18e8f5b010744cd730f119fb7a70b453dabf6e93 Mon Sep 17 00:00:00 2001
From: Amos Wenger <amos@bearcove.net>
Date: Wed, 28 Aug 2024 21:23:54 +0200
Subject: [PATCH 06/24] Make number of entries + sqpoll configurable via env
 vars

---
 crates/luring/src/linux.rs | 40 +++++++++++++++++++++++++++++++++++---
 1 file changed, 37 insertions(+), 3 deletions(-)

diff --git a/crates/luring/src/linux.rs b/crates/luring/src/linux.rs
index 61bd997c..7d95efc1 100644
--- a/crates/luring/src/linux.rs
+++ b/crates/luring/src/linux.rs
@@ -9,8 +9,7 @@ thread_local! {
     // This is a thread-local for now, but it shouldn't be. This is only the case
     // for op cancellations.
     static URING: Rc<IoUringAsync> = {
-        // FIXME: magic values
-        Rc::new(IoUringAsync::new(512).unwrap())
+        Rc::new(IoUringAsync::new_default().unwrap())
     };
 }
 
@@ -161,9 +160,44 @@ impl<S: squeue::Entry, C: cqueue::Entry> AsRawFd for IoUringAsync<S, C> {
 }
 
 impl IoUringAsync<io_uring::squeue::Entry, io_uring::cqueue::Entry> {
+    pub fn new_default() -> std::io::Result<Self> {
+        let mut entries = 512;
+        if let Ok(env_entries) = std::env::var("IO_URING_ENTRIES") {
+            entries = env_entries
+                .parse()
+                .expect("$IO_URING_ENTRIES must be a number");
+        }
+        eprintln!(
+            "==== IO_URING RING SIZE: {} (override with $IO_URING_ENTRIES)",
+            entries
+        );
+        Self::new(entries)
+    }
+
     pub fn new(entries: u32) -> std::io::Result<Self> {
+        let mut builder = io_uring::IoUring::builder();
+        let sqpoll_enabled = matches!(
+            std::env::var("IO_URING_SQPOLL").as_deref(),
+            Ok("1") | Ok("true")
+        );
+        eprintln!("==== SQPOLL: {sqpoll_enabled} (override with $IO_URING_SQPOLL=1)");
+
+        let mut sqpoll_idle_ms = 200;
+        if let Ok(env_sqpoll_idle_ms) = std::env::var("IO_URING_SQPOLL_IDLE_MS") {
+            sqpoll_idle_ms = env_sqpoll_idle_ms
+                .parse()
+                .expect("$IO_URING_SQPOLL_IDLE_MS must be a number");
+        }
+        eprintln!(
+            "==== SQPOLL_IDLE_MS: {} (override with $IO_URING_SQPOLL_IDLE_MS)",
+            sqpoll_idle_ms
+        );
+        if sqpoll_enabled {
+            builder.setup_sqpoll(sqpoll_idle_ms);
+        }
+
         Ok(Self {
-            uring: Rc::new(io_uring::IoUring::builder().build(entries)?),
+            uring: Rc::new(builder.build(entries)?),
             slab: Rc::new(RefCell::new(slab::Slab::new())),
         })
     }

From 486aacfa47febaf440f16526728ab31a5713bf8f Mon Sep 17 00:00:00 2001
From: Amos Wenger <amos@bearcove.net>
Date: Wed, 28 Aug 2024 21:26:21 +0200
Subject: [PATCH 07/24] Allow customizing number of buffers

---
 crates/buffet/src/bufpool.rs | 15 ++++++++++++++-
 1 file changed, 14 insertions(+), 1 deletion(-)

diff --git a/crates/buffet/src/bufpool.rs b/crates/buffet/src/bufpool.rs
index 7509f898..b69c5be5 100644
--- a/crates/buffet/src/bufpool.rs
+++ b/crates/buffet/src/bufpool.rs
@@ -19,7 +19,20 @@ pub fn initialize_allocator() -> Result<()> {
     #[cfg(feature = "miri")]
     let default_num_bufs = 1024;
 
-    initialize_allocator_with_num_bufs(default_num_bufs)
+    let mut num_bufs = default_num_bufs;
+
+    if let Ok(env_num_bufs) = std::env::var("BUFFET_NUM_BUFS") {
+        if let Ok(parsed_num_bufs) = env_num_bufs.parse::<usize>() {
+            num_bufs = parsed_num_bufs;
+        }
+    }
+
+    let mem_usage_in_mb: f64 = num_bufs as f64 * (BUF_SIZE as usize) as f64 / 1024.0 / 1024.0;
+    eprintln!(
+        "===== Initializing buffer pool with {} buffers, will use {:.2} MiB (override with $BUFFET_NUM_BUFS)",
+        num_bufs, mem_usage_in_mb
+    );
+    initialize_allocator_with_num_bufs(default_num_bufs as _)
 }
 
 impl BufMut {

From fa68aef05c8d4a9ba5156b259151b5576c289cad Mon Sep 17 00:00:00 2001
From: Amos Wenger <amos@bearcove.net>
Date: Wed, 28 Aug 2024 22:08:12 +0200
Subject: [PATCH 08/24] well that's super weird

---
 Justfile                           |  2 +-
 crates/buffet/src/bufpool.rs       |  2 +-
 crates/buffet/src/io.rs            | 19 +++++++++++++++++++
 crates/buffet/src/lib.rs           |  6 ++++++
 crates/buffet/src/net/net_uring.rs |  4 ++++
 crates/httpwg-hyper/src/main.rs    | 10 ++++++++--
 crates/httpwg-loona/Cargo.toml     |  2 +-
 crates/httpwg-loona/src/lib.rs     | 11 ++++++++---
 crates/httpwg/src/lib.rs           |  2 +-
 crates/loona-h2/src/lib.rs         |  1 +
 crates/loona/src/h2/server.rs      | 11 +++++++++--
 crates/luring/src/linux.rs         | 14 +++++++++++++-
 12 files changed, 72 insertions(+), 12 deletions(-)

diff --git a/Justfile b/Justfile
index bd8d37fb..196f6f93 100644
--- a/Justfile
+++ b/Justfile
@@ -66,7 +66,7 @@ httpwg-over-tcp *args='':
     export TEST_PROTO=h2
     export PORT=8001
     export RUST_LOG=${RUST_LOG:-info}
-    ./target/release/httpwg --address localhost:8001 "$@" -- ./target/release/httpwg-loona
+    ./target/release/httpwg --frame-timeout 2000 --connect-timeout 2000 --address localhost:8001 "$@" -- ./target/release/httpwg-loona
 
 instruments:
     #!/usr/bin/env -S bash -eux
diff --git a/crates/buffet/src/bufpool.rs b/crates/buffet/src/bufpool.rs
index b69c5be5..0dc09c42 100644
--- a/crates/buffet/src/bufpool.rs
+++ b/crates/buffet/src/bufpool.rs
@@ -29,7 +29,7 @@ pub fn initialize_allocator() -> Result<()> {
 
     let mem_usage_in_mb: f64 = num_bufs as f64 * (BUF_SIZE as usize) as f64 / 1024.0 / 1024.0;
     eprintln!(
-        "===== Initializing buffer pool with {} buffers, will use {:.2} MiB (override with $BUFFET_NUM_BUFS)",
+        "==== buffet will use {} buffers, for a constant {:.2} MiB usage (override with $BUFFET_NUM_BUFS)",
         num_bufs, mem_usage_in_mb
     );
     initialize_allocator_with_num_bufs(default_num_bufs as _)
diff --git a/crates/buffet/src/io.rs b/crates/buffet/src/io.rs
index f21cb6b8..b43f9072 100644
--- a/crates/buffet/src/io.rs
+++ b/crates/buffet/src/io.rs
@@ -44,7 +44,15 @@ pub trait WriteOwned {
 
         for buf in list.pieces.iter().cloned() {
             let buf_len = buf.len();
+
+            let before_write = std::time::Instant::now();
+            tracing::trace!("doing write with buf of len {}", buf_len);
             let (res, _) = self.write_owned(buf).await;
+            tracing::trace!(
+                "doing write with buf of len {}... done in {:?}",
+                buf_len,
+                before_write.elapsed()
+            );
 
             match res {
                 Ok(0) => {
@@ -72,8 +80,15 @@ pub trait WriteOwned {
     /// Write a list of buffers, re-trying the write if the kernel does a
     /// partial write.
     async fn writev_all_owned(&mut self, mut list: PieceList) -> std::io::Result<()> {
+        tracing::trace!("writev_all_owned starts...");
+        let start = std::time::Instant::now();
+
         while !list.is_empty() {
+            tracing::trace!("doing writev_owned with {} pieces", list.len());
+            let before_writev = std::time::Instant::now();
             let n = self.writev_owned(&list).await?;
+            tracing::trace!("writev_owned took {:?}", before_writev.elapsed());
+
             if n == 0 {
                 return Err(std::io::Error::new(
                     std::io::ErrorKind::WriteZero,
@@ -101,6 +116,10 @@ pub trait WriteOwned {
                 }
             }
         }
+        tracing::trace!(
+            "writev_all_owned starts... and succeeds! took {:?}",
+            start.elapsed()
+        );
 
         Ok(())
     }
diff --git a/crates/buffet/src/lib.rs b/crates/buffet/src/lib.rs
index 3cbb97f1..ae1acaa0 100644
--- a/crates/buffet/src/lib.rs
+++ b/crates/buffet/src/lib.rs
@@ -44,7 +44,13 @@ pub fn start<F: Future>(task: F) -> F::Output {
     let rt = tokio::runtime::Builder::new_current_thread()
         .enable_all()
         .on_thread_park(move || {
+            let start = std::time::Instant::now();
+            tracing::trace!("thread park, submitting...");
             u.submit().unwrap();
+            tracing::trace!(
+                "thread park, submitting... done! (took {:?})",
+                start.elapsed()
+            );
         })
         .build()
         .unwrap();
diff --git a/crates/buffet/src/net/net_uring.rs b/crates/buffet/src/net/net_uring.rs
index 2b77fa42..941f7df3 100644
--- a/crates/buffet/src/net/net_uring.rs
+++ b/crates/buffet/src/net/net_uring.rs
@@ -142,6 +142,7 @@ pub struct TcpWriteHalf(Rc<TcpStream>);
 
 impl WriteOwned for TcpWriteHalf {
     async fn write_owned(&mut self, buf: impl Into<Piece>) -> BufResult<usize, Piece> {
+        tracing::trace!("making new sqe");
         let buf = buf.into();
         let sqe = Write::new(
             io_uring::types::Fd(self.0.fd),
@@ -149,11 +150,14 @@ impl WriteOwned for TcpWriteHalf {
             buf.len().try_into().expect("usize -> u32"),
         )
         .build();
+        tracing::trace!("pushing sqe");
         let cqe = get_ring().push(sqe).await;
+        tracing::trace!("pushing sqe.. done!");
         let ret = match cqe.error_for_errno() {
             Ok(ret) => ret,
             Err(e) => return (Err(std::io::Error::from(e)), buf),
         };
+        tracing::trace!("pushing sqe.. done! (it even succeeded)");
         (Ok(ret as usize), buf)
     }
 
diff --git a/crates/httpwg-hyper/src/main.rs b/crates/httpwg-hyper/src/main.rs
index 6f27d8ed..fd6f72a7 100644
--- a/crates/httpwg-hyper/src/main.rs
+++ b/crates/httpwg-hyper/src/main.rs
@@ -36,12 +36,12 @@ where
 
     fn call(&self, req: Request<B>) -> Self::Future {
         Box::pin(async move {
-            let (parts, body) = req.into_parts();
+            let (parts, mut req_body) = req.into_parts();
 
             let path = parts.uri.path();
             match path {
                 "/echo-body" => {
-                    let body: BoxBody<E> = Box::pin(body);
+                    let body: BoxBody<E> = Box::pin(req_body);
                     let res = Response::builder().body(body).unwrap();
                     Ok(res)
                 }
@@ -64,6 +64,12 @@ where
                 }
                 _ => {
                     let parts = path.trim_start_matches('/').split('/').collect::<Vec<_>>();
+
+                    // read everything from req body
+                    while let Some(_frame) = req_body.frame().await {
+                        // got frame, nice
+                    }
+
                     let body: BoxBody<E> =
                         Box::pin(http_body_util::Empty::new().map_err(|_| unreachable!()));
 
diff --git a/crates/httpwg-loona/Cargo.toml b/crates/httpwg-loona/Cargo.toml
index 89649e4b..24120249 100644
--- a/crates/httpwg-loona/Cargo.toml
+++ b/crates/httpwg-loona/Cargo.toml
@@ -24,7 +24,7 @@ harness = false
 color-eyre = "0.6.3"
 loona = { version = "0.3.0", path = "../loona" }
 buffet = { version = "0.3.0", path = "../buffet" }
-tracing = { version = "0.1.40", features = ["release_max_level_debug"] }
+tracing = { version = "0.1.40" }
 tracing-subscriber = "0.3.18"
 tokio = { version = "1.39.2", features = ["macros", "sync", "process"] }
 eyre = { version = "0.6.12", default-features = false }
diff --git a/crates/httpwg-loona/src/lib.rs b/crates/httpwg-loona/src/lib.rs
index ab7b7988..34df518d 100644
--- a/crates/httpwg-loona/src/lib.rs
+++ b/crates/httpwg-loona/src/lib.rs
@@ -258,8 +258,12 @@ where
                 .await
                 .bx()?
             }
-            _ => res
-                .write_final_response_with_body(
+            _ => {
+                while (req_body.next_chunk().await).is_ok() {
+                    // got a chunk, nice
+                }
+
+                res.write_final_response_with_body(
                     Response {
                         status: StatusCode::OK,
                         ..Default::default()
@@ -267,7 +271,8 @@ where
                     &mut SinglePieceBody::from("it's less dire to lose, than to lose oneself"),
                 )
                 .await
-                .bx()?,
+                .bx()?
+            }
         };
         Ok(res)
     }
diff --git a/crates/httpwg/src/lib.rs b/crates/httpwg/src/lib.rs
index 88f9c1f3..ed40839f 100644
--- a/crates/httpwg/src/lib.rs
+++ b/crates/httpwg/src/lib.rs
@@ -335,7 +335,7 @@ impl<IO: IntoHalves> Conn<IO> {
                             {
                                 Ok(res) => res,
                                 Err(_) => {
-                                    debug!("timed out reading frame header");
+                                    debug!("timed out reading frame header (re-filling)");
                                     break 'read;
                                 }
                             };
diff --git a/crates/loona-h2/src/lib.rs b/crates/loona-h2/src/lib.rs
index 29c57d27..bfd3e81a 100644
--- a/crates/loona-h2/src/lib.rs
+++ b/crates/loona-h2/src/lib.rs
@@ -232,6 +232,7 @@ impl fmt::Display for StreamId {
 }
 
 /// See <https://httpwg.org/specs/rfc9113.html#FrameHeader>
+#[derive(Clone, Copy)]
 pub struct Frame {
     pub frame_type: FrameType,
     pub reserved: u8,
diff --git a/crates/loona/src/h2/server.rs b/crates/loona/src/h2/server.rs
index 2916164c..66a1a691 100644
--- a/crates/loona/src/h2/server.rs
+++ b/crates/loona/src/h2/server.rs
@@ -34,7 +34,8 @@ use crate::{
         },
     },
     util::{read_and_parse, ReadAndParseError},
-    Headers, Method, Request, Responder, ResponderOrBodyError, ServeOutcome, ServerDriver, SinglePieceBody,
+    Headers, Method, Request, Responder, ResponderOrBodyError, ServeOutcome, ServerDriver,
+    SinglePieceBody,
 };
 
 use super::{body::ChunkPosition, types::H2ErrorLevel};
@@ -1334,7 +1335,13 @@ where
 
         let frame = Frame::new(FrameType::RstStream, stream_id)
             .with_len((payload.len()).try_into().unwrap());
-        self.write_frame(frame, PieceList::single(payload)).await?;
+
+        for i in 0..15 {
+            tracing::trace!("Sending rst {i}");
+            self.write_frame(frame, PieceList::single(payload.clone()))
+                .await?;
+        }
+        tracing::trace!("All rsts sent");
 
         Ok(())
     }
diff --git a/crates/luring/src/linux.rs b/crates/luring/src/linux.rs
index 7d95efc1..37aed589 100644
--- a/crates/luring/src/linux.rs
+++ b/crates/luring/src/linux.rs
@@ -105,14 +105,19 @@ impl<C: cqueue::Entry> Future for OpInner<C> {
         let lifecycle = &mut guard[self.index];
         match lifecycle {
             Lifecycle::Submitted => {
+                tracing::trace!("polling OpInner {}... submitted!", self.index);
                 *lifecycle = Lifecycle::Waiting(cx.waker().clone());
                 std::task::Poll::Pending
             }
             Lifecycle::Waiting(_) => {
+                tracing::trace!("polling OpInner {}... waiting!", self.index);
                 *lifecycle = Lifecycle::Waiting(cx.waker().clone());
                 std::task::Poll::Pending
             }
-            Lifecycle::Completed(cqe) => std::task::Poll::Ready(cqe.clone()),
+            Lifecycle::Completed(cqe) => {
+                tracing::trace!("polling OpInner {}... completed!", self.index);
+                std::task::Poll::Ready(cqe.clone())
+            }
         }
     }
 }
@@ -207,7 +212,13 @@ impl<S: squeue::Entry, C: cqueue::Entry> IoUringAsync<S, C> {
     pub async fn listen(uring: Rc<IoUringAsync<S, C>>) {
         let async_fd = AsyncFd::new(uring).unwrap();
         loop {
+            let start = std::time::Instant::now();
+            tracing::trace!("waiting for uring fd to be ready...");
             let mut guard = async_fd.readable().await.unwrap();
+            tracing::trace!(
+                "waiting for uring fd to be ready... it is! (took {:?})",
+                start.elapsed()
+            );
             guard.get_inner().handle_cqe();
             guard.clear_ready();
         }
@@ -239,6 +250,7 @@ impl<S: squeue::Entry, C: cqueue::Entry> IoUringAsync<S, C> {
         let mut guard = self.slab.borrow_mut();
         while let Some(cqe) = unsafe { self.uring.completion_shared() }.next() {
             let index = cqe.user_data();
+            tracing::trace!("received cqe for index {}", index);
             let lifecycle = &mut guard[index.try_into().unwrap()];
             match lifecycle {
                 Lifecycle::Submitted => {

From db547148c4dc6884a4b1419c74b45f6d7091e2e3 Mon Sep 17 00:00:00 2001
From: Amos Wenger <amos@bearcove.net>
Date: Wed, 28 Aug 2024 22:21:16 +0200
Subject: [PATCH 09/24] debug things

---
 crates/buffet/src/net/net_uring.rs | 14 ++++++++------
 crates/buffet/src/roll.rs          |  2 +-
 crates/luring/src/linux.rs         | 12 ++++++------
 3 files changed, 15 insertions(+), 13 deletions(-)

diff --git a/crates/buffet/src/net/net_uring.rs b/crates/buffet/src/net/net_uring.rs
index 941f7df3..4e46d54a 100644
--- a/crates/buffet/src/net/net_uring.rs
+++ b/crates/buffet/src/net/net_uring.rs
@@ -67,8 +67,8 @@ impl TcpListener {
         socket.set_nodelay(true)?;
 
         // FIXME: don't hardcode
-        socket.set_reuse_port(true)?;
-        socket.set_reuse_address(true)?;
+        // socket.set_reuse_port(true)?;
+        // socket.set_reuse_address(true)?;
         socket.bind(&addr)?;
 
         // FIXME: magic values
@@ -129,6 +129,12 @@ impl ReadOwned for TcpReadHalf {
             buf.io_buf_mut_capacity() as u32,
         )
         .build();
+        tracing::trace!(
+            "submitting read_owned, reading from fd {} to {:p} with capacity {}",
+            self.0.fd,
+            buf.io_buf_mut_stable_mut_ptr(),
+            buf.io_buf_mut_capacity()
+        );
         let cqe = get_ring().push(sqe).await;
         let ret = match cqe.error_for_errno() {
             Ok(ret) => ret,
@@ -142,7 +148,6 @@ pub struct TcpWriteHalf(Rc<TcpStream>);
 
 impl WriteOwned for TcpWriteHalf {
     async fn write_owned(&mut self, buf: impl Into<Piece>) -> BufResult<usize, Piece> {
-        tracing::trace!("making new sqe");
         let buf = buf.into();
         let sqe = Write::new(
             io_uring::types::Fd(self.0.fd),
@@ -150,14 +155,11 @@ impl WriteOwned for TcpWriteHalf {
             buf.len().try_into().expect("usize -> u32"),
         )
         .build();
-        tracing::trace!("pushing sqe");
         let cqe = get_ring().push(sqe).await;
-        tracing::trace!("pushing sqe.. done!");
         let ret = match cqe.error_for_errno() {
             Ok(ret) => ret,
             Err(e) => return (Err(std::io::Error::from(e)), buf),
         };
-        tracing::trace!("pushing sqe.. done! (it even succeeded)");
         (Ok(ret as usize), buf)
     }
 
diff --git a/crates/buffet/src/roll.rs b/crates/buffet/src/roll.rs
index b20dc031..0b015c23 100644
--- a/crates/buffet/src/roll.rs
+++ b/crates/buffet/src/roll.rs
@@ -19,7 +19,7 @@ use nom::{
 #[macro_export]
 macro_rules! trace {
     ($($tt:tt)*) => {
-        // tracing::trace!($($tt)*)
+        tracing::trace!($($tt)*)
     };
 }
 
diff --git a/crates/luring/src/linux.rs b/crates/luring/src/linux.rs
index 37aed589..96f5049e 100644
--- a/crates/luring/src/linux.rs
+++ b/crates/luring/src/linux.rs
@@ -70,8 +70,7 @@ impl<C: cqueue::Entry> Drop for Op<C> {
                     Lifecycle::Waiting(_) => "Waiting",
                     Lifecycle::Completed(_) => "Completed",
                 };
-                tracing::debug!("dropping op {index} ({})", state_name);
-
+                tracing::debug!(%index, "dropping op in state {state_name}");
                 drop(guard);
 
                 // submit cancel op
@@ -105,17 +104,17 @@ impl<C: cqueue::Entry> Future for OpInner<C> {
         let lifecycle = &mut guard[self.index];
         match lifecycle {
             Lifecycle::Submitted => {
-                tracing::trace!("polling OpInner {}... submitted!", self.index);
+                tracing::trace!(index = %self.index, "poll: submitted!");
                 *lifecycle = Lifecycle::Waiting(cx.waker().clone());
                 std::task::Poll::Pending
             }
             Lifecycle::Waiting(_) => {
-                tracing::trace!("polling OpInner {}... waiting!", self.index);
+                tracing::trace!(index = %self.index, "poll: waiting!");
                 *lifecycle = Lifecycle::Waiting(cx.waker().clone());
                 std::task::Poll::Pending
             }
             Lifecycle::Completed(cqe) => {
-                tracing::trace!("polling OpInner {}... completed!", self.index);
+                tracing::trace!(index = %self.index, "poll: completed!");
                 std::task::Poll::Ready(cqe.clone())
             }
         }
@@ -234,6 +233,7 @@ impl<S: squeue::Entry, C: cqueue::Entry> IoUringAsync<S, C> {
     pub fn push(&self, entry: impl Into<S>) -> Op<C> {
         let mut guard = self.slab.borrow_mut();
         let index = guard.insert(Lifecycle::Submitted);
+        tracing::trace!(%index, "pushing op with index");
         let entry = entry.into().user_data(index.try_into().unwrap());
         while unsafe { self.uring.submission_shared().push(&entry).is_err() } {
             self.uring.submit().unwrap();
@@ -250,7 +250,7 @@ impl<S: squeue::Entry, C: cqueue::Entry> IoUringAsync<S, C> {
         let mut guard = self.slab.borrow_mut();
         while let Some(cqe) = unsafe { self.uring.completion_shared() }.next() {
             let index = cqe.user_data();
-            tracing::trace!("received cqe for index {}", index);
+            tracing::trace!(%index, "received cqe for index");
             let lifecycle = &mut guard[index.try_into().unwrap()];
             match lifecycle {
                 Lifecycle::Submitted => {

From e30c3c7ecee9601652c950306c7f35bcb6ece18c Mon Sep 17 00:00:00 2001
From: Amos Wenger <amos@bearcove.net>
Date: Wed, 28 Aug 2024 22:43:17 +0200
Subject: [PATCH 10/24] tests pass again

---
 crates/httpwg-loona/src/lib.rs | 38 ++++++++++++++++++++++++++++++++--
 1 file changed, 36 insertions(+), 2 deletions(-)

diff --git a/crates/httpwg-loona/src/lib.rs b/crates/httpwg-loona/src/lib.rs
index 34df518d..aa18c26f 100644
--- a/crates/httpwg-loona/src/lib.rs
+++ b/crates/httpwg-loona/src/lib.rs
@@ -201,6 +201,25 @@ where
                 .await
                 .bx()?,
             "/stream-big-body" => {
+                // then read the full request body
+                let mut req_body_len = 0;
+                loop {
+                    let chunk = req_body.next_chunk().await.bx()?;
+                    match chunk {
+                        BodyChunk::Done { trailers } => {
+                            // yey
+                            if let Some(trailers) = trailers {
+                                tracing::debug!(trailers_len = %trailers.len(), "received trailers");
+                            }
+                            break;
+                        }
+                        BodyChunk::Chunk(chunk) => {
+                            req_body_len += chunk.len();
+                        }
+                    }
+                }
+                tracing::debug!(%req_body_len, "read request body");
+
                 let mut roll = RollMut::alloc().bx()?;
                 for _ in 0..256 {
                     roll.write_all("this is a big chunk".as_bytes()).bx()?;
@@ -259,9 +278,24 @@ where
                 .bx()?
             }
             _ => {
-                while (req_body.next_chunk().await).is_ok() {
-                    // got a chunk, nice
+                // then read the full request body
+                let mut req_body_len = 0;
+                loop {
+                    let chunk = req_body.next_chunk().await.bx()?;
+                    match chunk {
+                        BodyChunk::Done { trailers } => {
+                            // yey
+                            if let Some(trailers) = trailers {
+                                tracing::debug!(trailers_len = %trailers.len(), "received trailers");
+                            }
+                            break;
+                        }
+                        BodyChunk::Chunk(chunk) => {
+                            req_body_len += chunk.len();
+                        }
+                    }
                 }
+                tracing::debug!(%req_body_len, "read request body");
 
                 res.write_final_response_with_body(
                     Response {

From 3a731ac1814e93d0fa29f2a3fa4b064be0c0bc3e Mon Sep 17 00:00:00 2001
From: Amos Wenger <amos@bearcove.net>
Date: Wed, 28 Aug 2024 22:46:30 +0200
Subject: [PATCH 11/24] ignore some failures

---
 crates/buffet/src/net/net_uring.rs |  7 ++++---
 crates/httpwg-loona/src/lib.rs     | 21 +++++++++++++++++++--
 crates/loona/src/h2/mod.rs         |  2 +-
 3 files changed, 24 insertions(+), 6 deletions(-)

diff --git a/crates/buffet/src/net/net_uring.rs b/crates/buffet/src/net/net_uring.rs
index 4e46d54a..2d4399c8 100644
--- a/crates/buffet/src/net/net_uring.rs
+++ b/crates/buffet/src/net/net_uring.rs
@@ -66,9 +66,10 @@ impl TcpListener {
 
         socket.set_nodelay(true)?;
 
-        // FIXME: don't hardcode
-        // socket.set_reuse_port(true)?;
-        // socket.set_reuse_address(true)?;
+        // FIXME: don't hardcode, but we get test failures on Linux otherwise for some
+        // reason
+        socket.set_reuse_port(true)?;
+        socket.set_reuse_address(true)?;
         socket.bind(&addr)?;
 
         // FIXME: magic values
diff --git a/crates/httpwg-loona/src/lib.rs b/crates/httpwg-loona/src/lib.rs
index aa18c26f..5b061ad7 100644
--- a/crates/httpwg-loona/src/lib.rs
+++ b/crates/httpwg-loona/src/lib.rs
@@ -4,7 +4,8 @@ use tokio::{process::Command, sync::oneshot};
 
 use buffet::{IntoHalves, Piece, RollMut};
 use loona::{
-    error::NeverError,
+    error::{NeverError, ServeError},
+    h2::types::H2ConnectionError,
     http::{self, StatusCode},
     Body, BodyChunk, Encoder, ExpectResponseHeaders, Responder, Response, ResponseDone,
     ServerDriver, SinglePieceBody,
@@ -127,7 +128,23 @@ pub fn do_main(addr: String, port: u16, proto: Proto, mode: Mode) {
                             if let Err(e) =
                                 loona::h2::serve(io, server_conf, client_buf, driver).await
                             {
-                                tracing::warn!("http/2 server error: {e:?}");
+                                let mut should_ignore = false;
+                                match &e {
+                                    ServeError::H2ConnectionError(
+                                        H2ConnectionError::WriteError(e),
+                                    ) => {
+                                        if e.kind() == std::io::ErrorKind::BrokenPipe {
+                                            should_ignore = true;
+                                        }
+                                    }
+                                    _ => {
+                                        // okay
+                                    }
+                                }
+
+                                if !should_ignore {
+                                    tracing::warn!("http/2 server error: {e:?}");
+                                }
                             }
                             tracing::debug!("http/2 server done");
                         }
diff --git a/crates/loona/src/h2/mod.rs b/crates/loona/src/h2/mod.rs
index 5d4208a2..4aad47a8 100644
--- a/crates/loona/src/h2/mod.rs
+++ b/crates/loona/src/h2/mod.rs
@@ -8,4 +8,4 @@ mod body;
 mod encode;
 pub use encode::H2EncoderError;
 
-pub(crate) mod types;
+pub mod types;

From 0d91d72d340f86ce08eb0ba2a4ca3674502e6605 Mon Sep 17 00:00:00 2001
From: Amos Wenger <amos@bearcove.net>
Date: Thu, 29 Aug 2024 18:42:43 +0200
Subject: [PATCH 12/24] hacky writev implementation

---
 crates/buffet/src/bufpool.rs             |  8 +++++-
 crates/buffet/src/bufpool/privatepool.rs |  4 +++
 crates/buffet/src/net/net_uring.rs       | 35 ++++++++++++++++++++++--
 crates/httpwg-loona/benches/h2load.rs    | 10 +++++--
 crates/httpwg-loona/src/lib.rs           |  2 +-
 5 files changed, 52 insertions(+), 7 deletions(-)

diff --git a/crates/buffet/src/bufpool.rs b/crates/buffet/src/bufpool.rs
index 0dc09c42..51444e93 100644
--- a/crates/buffet/src/bufpool.rs
+++ b/crates/buffet/src/bufpool.rs
@@ -7,11 +7,17 @@ mod privatepool;
 
 pub type BufResult<T, B> = (std::io::Result<T>, B);
 
-pub use privatepool::{initialize_allocator_with_num_bufs, num_free, Error, Result, BUF_SIZE};
+pub use privatepool::{
+    initialize_allocator_with_num_bufs, is_allocator_initialized, num_free, Error, Result, BUF_SIZE,
+};
 
 /// Initialize the allocator. Must be called before any other
 /// allocation function.
 pub fn initialize_allocator() -> Result<()> {
+    if is_allocator_initialized() {
+        return Ok(());
+    }
+
     // 64 * 1024 * 4096 bytes = 256 MiB
     #[cfg(not(feature = "miri"))]
     let default_num_bufs = 64 * 1024;
diff --git a/crates/buffet/src/bufpool/privatepool.rs b/crates/buffet/src/bufpool/privatepool.rs
index 7d1c2105..6939b3f5 100644
--- a/crates/buffet/src/bufpool/privatepool.rs
+++ b/crates/buffet/src/bufpool/privatepool.rs
@@ -73,6 +73,10 @@ fn with<T>(f: impl FnOnce(&mut Inner) -> T) -> T {
 /// The size of a buffer, in bytes (4 KiB)
 pub const BUF_SIZE: u16 = 4096;
 
+pub fn is_allocator_initialized() -> bool {
+    POOL.with(|pool| unsafe { (*pool.inner.get()).is_some() })
+}
+
 /// Initializes the allocator with the given number of buffers
 pub fn initialize_allocator_with_num_bufs(num_bufs: u32) -> Result<()> {
     POOL.with(|pool| {
diff --git a/crates/buffet/src/net/net_uring.rs b/crates/buffet/src/net/net_uring.rs
index 2d4399c8..e9b4807c 100644
--- a/crates/buffet/src/net/net_uring.rs
+++ b/crates/buffet/src/net/net_uring.rs
@@ -5,13 +5,14 @@ use std::{
     rc::Rc,
 };
 
-use io_uring::opcode::{Accept, Read, Write};
+use io_uring::opcode::{Accept, Read, Write, Writev};
+use libc::iovec;
 use nix::errno::Errno;
 
 use crate::{
     get_ring,
     io::{IntoHalves, ReadOwned, WriteOwned},
-    BufResult, IoBufMut, Piece,
+    BufResult, IoBufMut, Piece, PieceList,
 };
 
 pub struct TcpStream {
@@ -156,6 +157,7 @@ impl WriteOwned for TcpWriteHalf {
             buf.len().try_into().expect("usize -> u32"),
         )
         .build();
+
         let cqe = get_ring().push(sqe).await;
         let ret = match cqe.error_for_errno() {
             Ok(ret) => ret,
@@ -166,6 +168,35 @@ impl WriteOwned for TcpWriteHalf {
 
     // TODO: implement writev
 
+    async fn writev_owned(&mut self, list: &PieceList) -> std::io::Result<usize> {
+        let mut iovecs: Vec<iovec> = vec![];
+        for piece in list.pieces.iter() {
+            let iov = iovec {
+                iov_base: piece.as_ref().as_ptr() as *mut libc::c_void,
+                iov_len: piece.len(),
+            };
+            iovecs.push(iov);
+        }
+        let iovecs = iovecs.into_boxed_slice();
+        let iov_cnt = iovecs.len();
+        // FIXME: don't leak, duh
+        let iovecs = Box::leak(iovecs);
+
+        let sqe = Writev::new(
+            io_uring::types::Fd(self.0.fd),
+            iovecs.as_ptr() as *const _,
+            iov_cnt as u32,
+        )
+        .build();
+
+        let cqe = get_ring().push(sqe).await;
+        let ret = match cqe.error_for_errno() {
+            Ok(ret) => ret,
+            Err(e) => return Err(std::io::Error::from(e)),
+        };
+        Ok(ret as usize)
+    }
+
     async fn shutdown(&mut self) -> std::io::Result<()> {
         tracing::debug!("requesting shutdown");
         let sqe =
diff --git a/crates/httpwg-loona/benches/h2load.rs b/crates/httpwg-loona/benches/h2load.rs
index e37d7653..1997fc5b 100644
--- a/crates/httpwg-loona/benches/h2load.rs
+++ b/crates/httpwg-loona/benches/h2load.rs
@@ -3,9 +3,13 @@ use httpwg_loona::{Mode, Proto};
 
 pub fn h2load(c: &mut Criterion) {
     c.bench_function("h2load", |b| {
-        b.iter(|| {
-            httpwg_loona::do_main("127.0.0.1".to_string(), 0, Proto::H2, Mode::H2Load);
-        })
+        b.iter_with_setup(
+            || {},
+            |()| {
+                buffet::bufpool::initialize_allocator_with_num_bufs(64 * 1024).unwrap();
+                httpwg_loona::do_main("127.0.0.1".to_string(), 0, Proto::H2, Mode::H2Load);
+            },
+        )
     });
 }
 
diff --git a/crates/httpwg-loona/src/lib.rs b/crates/httpwg-loona/src/lib.rs
index 5b061ad7..b5616857 100644
--- a/crates/httpwg-loona/src/lib.rs
+++ b/crates/httpwg-loona/src/lib.rs
@@ -58,7 +58,7 @@ pub fn do_main(addr: String, port: u16, proto: Proto, mode: Mode) {
             if is_h2load {
                 let mut child = Command::new("h2load")
                     .arg("-n")
-                    .arg("100")
+                    .arg("2500")
                     .arg("-c")
                     .arg("10")
                     .arg(format!("http://127.0.0.1:{}", port))

From 65d7617ed724b894126fdce897248fb5aa046424 Mon Sep 17 00:00:00 2001
From: Amos Wenger <amos@bearcove.net>
Date: Fri, 30 Aug 2024 15:50:56 +0200
Subject: [PATCH 13/24] Add perfstat script

---
 scripts/perfstat.sh | 49 +++++++++++++++++++++++++++++++++++++++++++++
 scripts/syscalls    |  5 +++++
 2 files changed, 54 insertions(+)
 create mode 100644 scripts/perfstat.sh
 create mode 100644 scripts/syscalls

diff --git a/scripts/perfstat.sh b/scripts/perfstat.sh
new file mode 100644
index 00000000..1bbd732e
--- /dev/null
+++ b/scripts/perfstat.sh
@@ -0,0 +1,49 @@
+#!/usr/bin/env -S bash -euo pipefail
+#PERF_EVENTS="cpu-clock,context-switches,cycles,instructions,branches,branch-misses,cache-references,cache-misses,page-faults,$(paste -sd ',' syscalls)"
+PERF_EVENTS="cpu-clock,cycles,branch-misses,cache-misses,page-faults,$(paste -sd ',' syscalls)"
+
+LOONA_DIR=~/bearcove/loona
+
+# Build the servers
+cargo build --release --manifest-path="$LOONA_DIR/Cargo.toml" -F tracing/release_max_level_info
+
+# Launch hyper server
+export TEST_PROTO=h2 ADDR=0.0.0.0 PORT=8001
+"$LOONA_DIR/target/release/httpwg-hyper" &
+HYPER_PID=$!
+
+# Launch loona server
+export TEST_PROTO=h2 ADDR=0.0.0.0 PORT=8002
+"$LOONA_DIR/target/release/httpwg-loona" &
+LOONA_PID=$!
+
+HYPER_ADDR="http://localhost:8001"
+LOONA_ADDR="http://localhost:8002"
+
+ENDPOINT="${ENDPOINT:-/stream-big-body}"
+
+declare -A servers=(
+    [hyper]="$HYPER_PID $HYPER_ADDR"
+    [loona]="$LOONA_PID $LOONA_ADDR"
+)
+
+if [[ -n "${SERVER:-}" ]]; then
+    # If SERVER is set, only benchmark that one
+    if [[ -v "servers[$SERVER]" ]]; then
+        servers=([${SERVER}]="${servers[$SERVER]}")
+    else
+        echo "Error: SERVER '$SERVER' not found in the list of servers."
+        exit 1
+    fi
+fi
+
+echo -e "\033[1;34m📊 Benchmark parameters: RPS=${RPS:-2}, CONNS=${CONNS:-40}, STREAMS=${STREAMS:-8}, NUM_REQUESTS=${NUM_REQUESTS:-100}, ENDPOINT=${ENDPOINT:-/stream-big-body}\033[0m"
+for server in "${!servers[@]}"; do
+    read -r PID ADDR <<< "${servers[$server]}"
+    echo -e "\033[1;33m🚀 Benchmarking \033[1;32m$(cat /proc/$PID/cmdline | tr '\0' ' ')\033[0m"
+    perf stat -e "$PERF_EVENTS" -p "$PID" -- h2load --rps "${RPS:-2}" -c "${CONNS:-40}" -m "${STREAMS:-8}" -n "${NUM_REQUESTS:-100}" "${ADDR}${ENDPOINT}"
+    echo -e "\033[1;36mLoona Git SHA: $(cd ~/bearcove/loona && git rev-parse --short HEAD)\033[0m"
+done
+
+# Kill the servers
+kill $HYPER_PID $LOONA_PID
diff --git a/scripts/syscalls b/scripts/syscalls
new file mode 100644
index 00000000..573bb944
--- /dev/null
+++ b/scripts/syscalls
@@ -0,0 +1,5 @@
+syscalls:sys_enter_accept*
+syscalls:sys_enter_write*
+syscalls:sys_enter_brk
+syscalls:sys_enter_epoll_wait
+syscalls:sys_enter_io_uring_enter

From 495dbbe796565c84c2379b8ec17596ddb3a0b5f3 Mon Sep 17 00:00:00 2001
From: Amos Wenger <amos@bearcove.net>
Date: Fri, 30 Aug 2024 15:51:39 +0200
Subject: [PATCH 14/24] comment out writev_owned

---
 crates/buffet/src/net/net_uring.rs | 56 +++++++++++++++---------------
 1 file changed, 28 insertions(+), 28 deletions(-)

diff --git a/crates/buffet/src/net/net_uring.rs b/crates/buffet/src/net/net_uring.rs
index e9b4807c..30e2555f 100644
--- a/crates/buffet/src/net/net_uring.rs
+++ b/crates/buffet/src/net/net_uring.rs
@@ -168,34 +168,34 @@ impl WriteOwned for TcpWriteHalf {
 
     // TODO: implement writev
 
-    async fn writev_owned(&mut self, list: &PieceList) -> std::io::Result<usize> {
-        let mut iovecs: Vec<iovec> = vec![];
-        for piece in list.pieces.iter() {
-            let iov = iovec {
-                iov_base: piece.as_ref().as_ptr() as *mut libc::c_void,
-                iov_len: piece.len(),
-            };
-            iovecs.push(iov);
-        }
-        let iovecs = iovecs.into_boxed_slice();
-        let iov_cnt = iovecs.len();
-        // FIXME: don't leak, duh
-        let iovecs = Box::leak(iovecs);
-
-        let sqe = Writev::new(
-            io_uring::types::Fd(self.0.fd),
-            iovecs.as_ptr() as *const _,
-            iov_cnt as u32,
-        )
-        .build();
-
-        let cqe = get_ring().push(sqe).await;
-        let ret = match cqe.error_for_errno() {
-            Ok(ret) => ret,
-            Err(e) => return Err(std::io::Error::from(e)),
-        };
-        Ok(ret as usize)
-    }
+    // async fn writev_owned(&mut self, list: &PieceList) -> std::io::Result<usize>
+    // {     let mut iovecs: Vec<iovec> = vec![];
+    //     for piece in list.pieces.iter() {
+    //         let iov = iovec {
+    //             iov_base: piece.as_ref().as_ptr() as *mut libc::c_void,
+    //             iov_len: piece.len(),
+    //         };
+    //         iovecs.push(iov);
+    //     }
+    //     let iovecs = iovecs.into_boxed_slice();
+    //     let iov_cnt = iovecs.len();
+    //     // FIXME: don't leak, duh
+    //     let iovecs = Box::leak(iovecs);
+
+    //     let sqe = Writev::new(
+    //         io_uring::types::Fd(self.0.fd),
+    //         iovecs.as_ptr() as *const _,
+    //         iov_cnt as u32,
+    //     )
+    //     .build();
+
+    //     let cqe = get_ring().push(sqe).await;
+    //     let ret = match cqe.error_for_errno() {
+    //         Ok(ret) => ret,
+    //         Err(e) => return Err(std::io::Error::from(e)),
+    //     };
+    //     Ok(ret as usize)
+    // }
 
     async fn shutdown(&mut self) -> std::io::Result<()> {
         tracing::debug!("requesting shutdown");

From 99be74fc7f5f3b3e3e62994a8f23ecdfad7cabb5 Mon Sep 17 00:00:00 2001
From: Amos Wenger <amos@bearcove.net>
Date: Fri, 30 Aug 2024 15:54:17 +0200
Subject: [PATCH 15/24] make executable

---
 scripts/perfstat.sh | 4 ++++
 1 file changed, 4 insertions(+)
 mode change 100644 => 100755 scripts/perfstat.sh

diff --git a/scripts/perfstat.sh b/scripts/perfstat.sh
old mode 100644
new mode 100755
index 1bbd732e..d663baec
--- a/scripts/perfstat.sh
+++ b/scripts/perfstat.sh
@@ -1,4 +1,8 @@
 #!/usr/bin/env -S bash -euo pipefail
+
+# Change to the script's directory
+cd "$(dirname "$0")"
+
 #PERF_EVENTS="cpu-clock,context-switches,cycles,instructions,branches,branch-misses,cache-references,cache-misses,page-faults,$(paste -sd ',' syscalls)"
 PERF_EVENTS="cpu-clock,cycles,branch-misses,cache-misses,page-faults,$(paste -sd ',' syscalls)"
 

From d223859a1c9d2a258ae6d05ab16de321df476b54 Mon Sep 17 00:00:00 2001
From: Amos Wenger <amos@bearcove.net>
Date: Fri, 30 Aug 2024 15:55:47 +0200
Subject: [PATCH 16/24] source cargo env

---
 scripts/perfstat.sh | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/scripts/perfstat.sh b/scripts/perfstat.sh
index d663baec..fd0abde4 100755
--- a/scripts/perfstat.sh
+++ b/scripts/perfstat.sh
@@ -1,5 +1,7 @@
 #!/usr/bin/env -S bash -euo pipefail
 
+. /root/.cargo/env
+
 # Change to the script's directory
 cd "$(dirname "$0")"
 

From 12a5394ba63db46a1fd0305322412246bee6170e Mon Sep 17 00:00:00 2001
From: Amos Wenger <amos@bearcove.net>
Date: Fri, 30 Aug 2024 15:57:29 +0200
Subject: [PATCH 17/24] clean up

---
 scripts/perfstat.sh | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/scripts/perfstat.sh b/scripts/perfstat.sh
index fd0abde4..1a3f394a 100755
--- a/scripts/perfstat.sh
+++ b/scripts/perfstat.sh
@@ -43,12 +43,11 @@ if [[ -n "${SERVER:-}" ]]; then
     fi
 fi
 
-echo -e "\033[1;34m📊 Benchmark parameters: RPS=${RPS:-2}, CONNS=${CONNS:-40}, STREAMS=${STREAMS:-8}, NUM_REQUESTS=${NUM_REQUESTS:-100}, ENDPOINT=${ENDPOINT:-/stream-big-body}\033[0m"
 for server in "${!servers[@]}"; do
     read -r PID ADDR <<< "${servers[$server]}"
-    echo -e "\033[1;33m🚀 Benchmarking \033[1;32m$(cat /proc/$PID/cmdline | tr '\0' ' ')\033[0m"
+    echo -e "\033[1;33m🚀 Benchmarking \033[1;32m$(cat /proc/$PID/cmdline | tr '\0' ' ')\033[0m \033[1;36mLoona Git SHA: $(cd ~/bearcove/loona && git rev-parse --short HEAD)\033[0m"
+    echo -e "\033[1;34m📊 Benchmark parameters: RPS=${RPS:-2}, CONNS=${CONNS:-40}, STREAMS=${STREAMS:-8}, NUM_REQUESTS=${NUM_REQUESTS:-100}, ENDPOINT=${ENDPOINT:-/stream-big-body}\033[0m"
     perf stat -e "$PERF_EVENTS" -p "$PID" -- h2load --rps "${RPS:-2}" -c "${CONNS:-40}" -m "${STREAMS:-8}" -n "${NUM_REQUESTS:-100}" "${ADDR}${ENDPOINT}"
-    echo -e "\033[1;36mLoona Git SHA: $(cd ~/bearcove/loona && git rev-parse --short HEAD)\033[0m"
 done
 
 # Kill the servers

From 34e3ace9c36411b385da20a66bacc115ac0b50fa Mon Sep 17 00:00:00 2001
From: Amos Wenger <amos@bearcove.net>
Date: Fri, 30 Aug 2024 15:58:08 +0200
Subject: [PATCH 18/24] wut

---
 scripts/perfstat.sh | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/scripts/perfstat.sh b/scripts/perfstat.sh
index 1a3f394a..4eafd821 100755
--- a/scripts/perfstat.sh
+++ b/scripts/perfstat.sh
@@ -45,7 +45,8 @@ fi
 
 for server in "${!servers[@]}"; do
     read -r PID ADDR <<< "${servers[$server]}"
-    echo -e "\033[1;33m🚀 Benchmarking \033[1;32m$(cat /proc/$PID/cmdline | tr '\0' ' ')\033[0m \033[1;36mLoona Git SHA: $(cd ~/bearcove/loona && git rev-parse --short HEAD)\033[0m"
+    echo -e "\033[1;36mLoona Git SHA: $(cd ~/bearcove/loona && git rev-parse --short HEAD)\033[0m"
+    echo -e "\033[1;33m🚀 Benchmarking \033[1;32m$(cat /proc/$PID/cmdline | tr '\0' ' ')\033[0m"
     echo -e "\033[1;34m📊 Benchmark parameters: RPS=${RPS:-2}, CONNS=${CONNS:-40}, STREAMS=${STREAMS:-8}, NUM_REQUESTS=${NUM_REQUESTS:-100}, ENDPOINT=${ENDPOINT:-/stream-big-body}\033[0m"
     perf stat -e "$PERF_EVENTS" -p "$PID" -- h2load --rps "${RPS:-2}" -c "${CONNS:-40}" -m "${STREAMS:-8}" -n "${NUM_REQUESTS:-100}" "${ADDR}${ENDPOINT}"
 done

From 5ea32e74e711471b2fe97848423b9f766073bcbd Mon Sep 17 00:00:00 2001
From: Amos Wenger <amos@bearcove.net>
Date: Fri, 30 Aug 2024 15:58:32 +0200
Subject: [PATCH 19/24] print pid

---
 scripts/perfstat.sh | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/scripts/perfstat.sh b/scripts/perfstat.sh
index 4eafd821..55eb418a 100755
--- a/scripts/perfstat.sh
+++ b/scripts/perfstat.sh
@@ -17,11 +17,13 @@ cargo build --release --manifest-path="$LOONA_DIR/Cargo.toml" -F tracing/release
 export TEST_PROTO=h2 ADDR=0.0.0.0 PORT=8001
 "$LOONA_DIR/target/release/httpwg-hyper" &
 HYPER_PID=$!
+echo "Hyper PID: $HYPER_PID"
 
 # Launch loona server
 export TEST_PROTO=h2 ADDR=0.0.0.0 PORT=8002
 "$LOONA_DIR/target/release/httpwg-loona" &
 LOONA_PID=$!
+echo "Loona PID: $LOONA_PID"
 
 HYPER_ADDR="http://localhost:8001"
 LOONA_ADDR="http://localhost:8002"

From 03abe3b0367773d5c2c2ff56911c3ecfa216e447 Mon Sep 17 00:00:00 2001
From: Amos Wenger <amos@bearcove.net>
Date: Fri, 30 Aug 2024 16:26:44 +0200
Subject: [PATCH 20/24] Standardize endpoints a bit

---
 Cargo.lock                                 |  2 +
 crates/httpwg-hyper/Cargo.toml             |  2 +
 crates/httpwg-hyper/src/async_read_body.rs | 87 ++++++++++++++++++++++
 crates/httpwg-hyper/src/main.rs            | 70 ++++++++++++-----
 scripts/mkfiles.sh                         | 37 +++++++++
 scripts/perfstat.sh                        | 11 ++-
 6 files changed, 188 insertions(+), 21 deletions(-)
 create mode 100644 crates/httpwg-hyper/src/async_read_body.rs
 create mode 100755 scripts/mkfiles.sh

diff --git a/Cargo.lock b/Cargo.lock
index 5ac1ef0f..0aaa524c 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -785,8 +785,10 @@ dependencies = [
  "http-body-util",
  "hyper",
  "hyper-util",
+ "pin-project-lite",
  "tokio",
  "tokio-stream",
+ "tokio-util",
  "tracing",
 ]
 
diff --git a/crates/httpwg-hyper/Cargo.toml b/crates/httpwg-hyper/Cargo.toml
index e9558652..c76d7c22 100644
--- a/crates/httpwg-hyper/Cargo.toml
+++ b/crates/httpwg-hyper/Cargo.toml
@@ -19,6 +19,8 @@ hyper-util = { version = "0.1.7", features = [
     "http2",
     "tokio",
 ] }
+pin-project-lite = "0.2.14"
 tokio = { version = "1.39.2", features = ["full"] }
 tokio-stream = "0.1.15"
+tokio-util = { version = "0.7.11", features = ["io"] }
 tracing = "0.1.40"
diff --git a/crates/httpwg-hyper/src/async_read_body.rs b/crates/httpwg-hyper/src/async_read_body.rs
new file mode 100644
index 00000000..43a5f8e3
--- /dev/null
+++ b/crates/httpwg-hyper/src/async_read_body.rs
@@ -0,0 +1,87 @@
+// use hyper::body::{Body, Frame};
+// use pin_project_lite::pin_project;
+// use std::{
+//     pin::Pin,
+//     task::{Context, Poll},
+// };
+// use tokio::io::AsyncRead;
+// use tokio_util::io::ReaderStream;
+
+// pin_project! {
+//     /// An [`HttpBody`] created from an [`AsyncRead`].
+//     ///
+//     /// # Example
+//     ///
+//     /// `AsyncReadBody` can be used to stream the contents of a file:
+//     ///
+//     /// ```rust
+//     /// use axum::{
+//     ///     Router,
+//     ///     routing::get,
+//     ///     http::{StatusCode, header::CONTENT_TYPE},
+//     ///     response::{Response, IntoResponse},
+//     /// };
+//     /// use axum_extra::body::AsyncReadBody;
+//     /// use tokio::fs::File;
+//     ///
+//     /// async fn cargo_toml() -> Result<Response, (StatusCode, String)> {
+//     ///     let file = File::open("Cargo.toml")
+//     ///         .await
+//     ///         .map_err(|err| {
+//     ///             (StatusCode::NOT_FOUND, format!("File not found: {err}"))
+//     ///         })?;
+//     ///
+//     ///     let headers = [(CONTENT_TYPE, "text/x-toml")];
+//     ///     let body = AsyncReadBody::new(file);
+//     ///     Ok((headers, body).into_response())
+//     /// }
+//     ///
+//     /// let app = Router::new().route("/Cargo.toml", get(cargo_toml));
+//     /// # let _: Router = app;
+//     /// ```
+//     #[derive(Debug)]
+//     #[must_use]
+//     pub struct AsyncReadBody<R> {
+//         #[pin]
+//         read: R,
+//     }
+// }
+
+// impl AsyncReadBody {
+//     /// Create a new `AsyncReadBody`.
+//     pub fn new<R>(read: R) -> Self
+//     where
+//         R: AsyncRead + Send + 'static,
+//     {
+//         Self { read }
+//     }
+// }
+
+// impl Body for AsyncReadBody {
+//     type Data = Bytes;
+//     type Error = Error;
+
+//     #[inline]
+//     fn poll_frame(
+//         self: Pin<&mut Self>,
+//         cx: &mut Context<'_>,
+//     ) -> Poll<Option<Result<Frame<Self::Data>, Self::Error>>> {
+//         self.project().body.poll_frame(cx)
+//     }
+
+//     #[inline]
+//     fn is_end_stream(&self) -> bool {
+//         self.body.is_end_stream()
+//     }
+
+//     #[inline]
+//     fn size_hint(&self) -> http_body::SizeHint {
+//         self.body.size_hint()
+//     }
+// }
+
+// impl IntoResponse for AsyncReadBody {
+//     fn into_response(self) -> Response {
+//         self.body.into_response()
+//     }
+// }
diff --git a/crates/httpwg-hyper/src/main.rs b/crates/httpwg-hyper/src/main.rs
index fd6f72a7..fcbca0a7 100644
--- a/crates/httpwg-hyper/src/main.rs
+++ b/crates/httpwg-hyper/src/main.rs
@@ -1,6 +1,9 @@
 use http_body_util::{BodyExt, StreamBody};
 use hyper_util::rt::TokioExecutor;
 use hyper_util::rt::TokioIo;
+use tokio::io::AsyncReadExt;
+use tokio_stream::StreamExt;
+use tokio_util::io::ReaderStream;
 
 use hyper_util::server::conn::auto;
 use std::{convert::Infallible, fmt::Debug, pin::Pin};
@@ -18,9 +21,9 @@ use tracing::debug;
 
 pub(crate) struct TestService;
 
-pub fn big_body() -> String {
-    "this is a big chunk".repeat(256).repeat(128)
-}
+mod async_read_body;
+
+const BLOCK: &[u8] = b"K75HZ+W4P2+Z+K1eI/lPJkc+HiVr/+snBmi0fu5IAIseZ6HumAEX2bfv4ok9Rzqm8Eq1dP3ap8pscfD4IBqBHnxtMdc6+Vvf81WDDqf3yXL3yvoA0N0jxuVs9jXTllu/h+ABUf8dBymieg/xhJsn7NQDJvb/fh5+ZZpP8++ihiUgwgc+yM04rtSIP+O6Ul0RdoeHftzguVujmB9bnf+JtrUAL+AFCxIommB7IszrCLyz+0ysE2Ke1Mvv5Et88p4wvPc4TcKJC53OmyHcFp4HOI8tZXJC2eIaWC59bpTxWuzt0w0x0P8dou1uvCQTSRDHcHIo4VevzgqtCVnISEhdxjBUU6bNa4rCmXKEjSCd09fYe/Wsd45mji9J9cco1kQs4wU43se8oCSzcKnYI4cB0iyvDD3/ceIATVrYv3R8QH69J1NFWTvsILMf+TXfVQgfJmthIF/aY417hJjhvEjyoez27dZrcAMUXlvAXDozt3IsFS9D1KJvzt1SSaKENi/WjC+WMCTZr4guBNbNQdyd8NLRf/Ilum3zrIJDwcT+IecgdtIDtG3koYqVJ1ihAxFYMaZFk32R4iaNhUxyibX1DE2w8Xfz3g0HiAxGl+rWMREldUTEBlwk8Ig5ccanXwJ8fLXOn/UduZQkIKuH4ucb+T40T/iNubbi4/5SSVphTEnGJ0y1fcowKPxxseyZ5SZHVoYxHGvEYeCl+hw5XgqiZaIpHZZMiAQh38zGGd6J8mLOsPG6BSpWV8Cj00UusRnO/V2tAxiR7Vuh8EiDPV728a3XsZI5xGc4MMWbqTSmMGm2x8XybIe/vL6U7Y9ptr4c18nfQErH/Yt4OmmFGP0VTmbSo2aGGMkJ1VwX/6BAxIxOMXoqshNfZ2Nh+0py0V/Ly+SQr6OcTxX857d0I3l0P8GWsLcZxER9EpkEO6NKUMdOIqZdRoC1p1lnzMsL5UvWDFrFoIXJqAA3jHmXN+zZgJbg7+sLdWE2HR2EvsepXUdK0t31SqkBkn0YHJbklSivWe9FbLOIstB2kigkYmnFT0a49aW+uTlgU6Tc+hx9ufW6l17EHf8I37WIvInLNKsk+wOqeYzspRf8rE4mfYyFunhDDXSe/eFaVnb53otiGsYA3GRutY5FfBrYkK2ZQRIND5B+AqwGa+4V47yPkq217iCKgBSYXA5Ux0e138LUMNq2Yn9YqbdMP3XEPUBBaiT8q2GE+w/ay7dZOid1jiV72OET90aSA8FFev6jnhQhvlR6qndOYexk1GWO+mFanlUU/PEZ0+0v9tj93TlPZp/0xfWNyXpXh5ubDLRNoxX/RRQ6hMIkbpDEeCiI4zBRk1vVMpI6myc76tvMk97APMJDpKt3QGCLCQD0vb2UEqMkEKFxggR46PvlCI3zo0LQr5oigB3kaSShFzTAm8hKOzg5M9NpN/l+hQHQJv9lFhxjsuHCvdM6sNF3rxLtEKCc45IicsJRM/CyZc7cadMurqBGBUSQHpLmtndFaLNvjRQMI1gYYGcEr34/WOGG5LRQvo0I7toSjcVFc2JdfGuT/71JNJupS89l6nrSisFPCuCCgaN5O4jZAb4vnhrHHZs8r0IuFtd39pT24obpLYsheBT2+tdCf3QsEIvkGZ/VQkn/4jaMyCsGw37mm8dZNyGtn3cWcP9DYytYNNmbjc8Ks3rvkbLttMch8AyEQClqvgXwVMNPHBI/gL0OY8cPyCXxh7x4NCt0bmS9AUb+YCkEmXxDOkxrDntRFvmavacZbF6jNjMXfqG2dkMmZ9obz7M31r3eDYa1bd2MLgb5H3napVjILcRnuPrgR+EdqonE8+fIVZjGZL6Jgwi1ja0VHsoyI8d5dPDazD4U5q2EaPbkX/62RMCRz7FRJX368NBZigOwVzR3/oIJZjeuNTlsoe4cP17jGXXCkNXXY7gUmN7A2hOH9Wg5IDdPahBCf7kpL3wOcXYoyN1fciwfq+kvN8jqNtMJcGrEls2wGnWNc5OITtHTqT7xltIdE2rjkBDo4PIwfdmOZxpbnscbfVSG5HANXA6B+6caN3hor27E8Y9aEmdhPSDP0vdedzXWPzeyTQK82bbA4PB+mny+FP1IImUuVxV9jzPLPPxylx6EaR+SsxHNdUrMETboaK70mViWZpSJhSgMDQGGs1tkV22qRZFnZgIppTh4C0fBiKNK1TxkXHA7CZqndMXbA9w7C2ywBEuPCBvHZPm5qre1jLAbXC7z8TNJ/EDxdJI8yXSrKesKQiNiEZ5rEUORy3Omxi0GaPG/LfwgHmmEdTfttfzk24LbHs51XLbX5cGM+7sQ9nLVCjiaMZEsfx87At4CnbzliC3UI/ZVkYAlby0fp2TXxfMdN5VRDueDlSUdIz88tLgWJQ8lHEI90HLl4n2dNfUr08Eea4QdjI+r3INuhdS7RFm+jUWXnbPaoQpn7rev4p3tRV0YL4N3lj4eXHMsrQ4NM3ASlwvuPXfun/b+QWWTqS/k+c6vuQP1H0utoAOlv3Lmzeczq+vC35QUHJdGvi43+nrNRYNWrDP0FtFIlC1q5DN+XIL7Pq2eX8dYku/2cLEYQokY7Pq4+0frobbTIxo2AVpT41qmRhgQc2iNGLk9PDhLoopDEcS5dSql06IIo8r6Xx/tthaToqyDk+aAoQZf7wz7rvVmi0Mj158+KVRn4z2b6sCEe8yl+u9DpYmNbU4THEQSvTSsEyez0Fps23NmIDWqXpMevUYxIgZXNorbEClxPqSOHzbiL/K02E2HhjD3JA8q+XkJdvX97orDqC/BNPp0Ivp7P9TAqmjbJ6AYHMoYh/25SFq6jQQFUwuFS98wd1CJMDdewd0VzFEuzeuz69krNwv/jMNrGAUmTLeDE9jOKPMmGixOUyNLtXGpLHKleE7iVkj7LKDu2zlqYRTrDkz36JroclE+7GROXWT8+OJO4KnMep+v+ZXPFkf26/KXzKya25nqe0h200bJ/eUsFg74f9NTq+FMfEsXpRacnIVJo/yJLnObOKGL5K3VrHkrx3ccubhcPHR7MkvBmhIWcOXB3KwCnvkfsA0ttvNQQ4w5ojOz0nxaiaP6NbFz0xuehwDrkaTFSF2QLfGvXI9pY/v3PJtWAj33EEwSMs46crAX++NVBuWOKGdzgvmaCxnh5oFojrvwLrr2xdJK2nzoGQJD78HMHZ1hmYfZ8UFOigZ2PtjV/Tyt6XXZ3BhFjxjkCbvR4nsGoHbYVOxkNlmXsSKSRyhttRQ0r3WfHG7ot3YnoJpogHBy0T+O8Yu+SIPCIe6b+ac7rvewOi4kwobtygQBJFNoUN+0z3Ztqf49yc3viPfTXW4nlooWcyJhUs5Tk1FVLDOEJeDp8clCxYw/XtlMr+BLbVF7w3koa+aHU1PJmo562IeH/sDiANKw1GnGvcxqhmMsb4aPOTpvnpq16JLVtmdIl83j2oVOb1Ql1U6b0zv1pphHq8MwESFDm1tSThDbs41vkFWHplb5SpTLxAA2e9H/Ch+cb7h9OXt7HwNPsq/+0zzT9D2rlhoDatqqTnbWpyozcRDvNKOJvPlnUCvKzHJNMcp/d9q1AaTcOrNYFVDZeEOTw7+/vCAmLxihRINycQND+/x180V22WcT9I9dRbuaEPM2XpfRlkENbERqDWeGfKmuhK5r1PkF7G8QxnDgrekFVHvqudGINzi+1ELzobztD7AoyBKkIUWKzSWm/HLk5zEm9lZ2Dkh9+13faXcxjifGkOvIm6g0BF+XqpvBJSyxfKg58/x0tksvI8HOfgJmPfLFdUJbmcM+WTtebp10b9+35qN0KZJbdEwZcrRrgdLbWCIvSRvNUR2SakZbYMSy08zthER446WCeRCmzzook/Scxk+Mn3WeOyMmJsXR1zXfoD7plogXvR4nJPWpawrjl13hVZ1XCj6DszYdeIuVdonMYh3zn0TToAB/4xaNKev1IOAaU08exxD/DKWBZEM3LbZGsXuH7F1jOySuagkl5+JeffpMTx0sRpHMzEzfdX/WOFJ/w9BR5kJjGB6KtBLic1Oy9JNCez21wC4Oo4DAPqK/W4cnDgUeYev2OkiyeX47WhDRSLES4iQcsWLJ4img";
 
 type BoxBody<E> = Pin<Box<dyn Body<Data = Bytes, Error = E> + Send + Sync + 'static>>;
 
@@ -45,23 +48,6 @@ where
                     let res = Response::builder().body(body).unwrap();
                     Ok(res)
                 }
-                "/stream-big-body" => {
-                    let (tx, rx) = mpsc::channel::<Result<Frame<Bytes>, E>>(1);
-
-                    tokio::spawn(async move {
-                        let chunk = "this is a big chunk".repeat(256);
-                        let chunk = Bytes::from(chunk);
-                        for _ in 0..128 {
-                            let frame = Frame::data(chunk.clone());
-                            let _ = tx.send(Ok(frame)).await;
-                        }
-                    });
-
-                    let rx = ReceiverStream::new(rx);
-                    let body: BoxBody<E> = Box::pin(StreamBody::new(rx));
-                    let res = Response::builder().body(body).unwrap();
-                    Ok(res)
-                }
                 _ => {
                     let parts = path.trim_start_matches('/').split('/').collect::<Vec<_>>();
 
@@ -78,6 +64,50 @@ where
                         let res = Response::builder().status(code).body(body).unwrap();
                         debug!("Replying with {:?} {:?}", res.status(), res.headers());
                         Ok(res)
+                    } else if let ["repeat-4k-blocks", repeat] = parts.as_slice() {
+                        let repeat = repeat.parse::<usize>().unwrap();
+
+                        // TODO: custom impl of the Body trait to avoid channel overhead
+                        let (tx, rx) = mpsc::channel::<Result<Frame<Bytes>, E>>(1);
+
+                        tokio::spawn(async move {
+                            let block = "this is a fairly large block".repeat(4096);
+                            let block = Bytes::from(block);
+                            for _ in 0..repeat {
+                                let frame = Frame::data(block.clone());
+                                let _ = tx.send(Ok(frame)).await;
+                            }
+                        });
+
+                        let rx = ReceiverStream::new(rx);
+                        let body: BoxBody<E> = Box::pin(StreamBody::new(rx));
+                        let res = Response::builder().body(body).unwrap();
+                        Ok(res)
+                    } else if let ["stream-file", name] = parts.as_slice() {
+                        let name = name.to_string();
+
+                        // stream 64KB blocks of the file
+                        let (tx, rx) = mpsc::channel::<Result<Frame<Bytes>, E>>(1);
+                        tokio::spawn(async move {
+                            let mut file =
+                                tokio::fs::File::open(format!("/tmp/stream-file/{name}"))
+                                    .await
+                                    .unwrap();
+                            let mut buf = Vec::with_capacity(64 * 1024);
+                            while let Ok(n) = file.read(&mut buf).await {
+                                if n == 0 {
+                                    break;
+                                }
+                                let frame = Frame::data(Bytes::copy_from_slice(&buf[..n]));
+                                let _ = tx.send(Ok(frame)).await;
+                                buf.drain(..n);
+                            }
+                        });
+
+                        let rx = ReceiverStream::new(rx);
+                        let body: BoxBody<E> = Box::pin(StreamBody::new(rx));
+                        let res = Response::builder().body(body).unwrap();
+                        Ok(res)
                     } else {
                         let body = "it's less dire to lose, than to lose oneself".to_string();
                         let body: BoxBody<E> = Box::pin(body.map_err(|_| unreachable!()));
diff --git a/scripts/mkfiles.sh b/scripts/mkfiles.sh
new file mode 100755
index 00000000..f061a485
--- /dev/null
+++ b/scripts/mkfiles.sh
@@ -0,0 +1,37 @@
+#!/usr/bin/env -S bash -euo pipefail
+
+# Change to the script's directory
+cd "$(dirname "$0")"
+
+# Define an array of file names and sizes
+declare -A files=(
+    ["4M"]="4M"
+    ["16M"]="16M"
+    ["256M"]="256M"
+    ["1GB"]="1G"
+)
+
+# Print summary and ask for consent
+echo -e "\e[1;33mThis script will generate the following files in /tmp/stream-file:\e[0m"
+for file in "${!files[@]}"; do
+    echo "- $file (${files[$file]})"
+done
+echo -e "\n\e[1;33mDo you want to proceed? (y/n)\e[0m"
+read -r consent
+
+if [[ ! $consent =~ ^[Yy]$ ]]; then
+    echo "Aborting script execution."
+    exit 1
+fi
+
+# Create directory if it doesn't exist
+mkdir -p /tmp/stream-file
+
+# Loop through the array and generate files
+for file in "${!files[@]}"; do
+    echo -e "\e[1;34mGenerating $file file...\e[0m"
+    dd if=/dev/urandom of="/tmp/stream-file/$file" bs=${files[$file]} count=1 status=progress
+    echo -e "\e[1;32mCompleted generating $file file!\e[0m"
+done
+
+echo -e "\e[1;35mAll files have been generated successfully in /tmp/stream-file\e[0m"
diff --git a/scripts/perfstat.sh b/scripts/perfstat.sh
index 55eb418a..1448303d 100755
--- a/scripts/perfstat.sh
+++ b/scripts/perfstat.sh
@@ -13,6 +13,15 @@ LOONA_DIR=~/bearcove/loona
 # Build the servers
 cargo build --release --manifest-path="$LOONA_DIR/Cargo.toml" -F tracing/release_max_level_info
 
+# Create a new process group
+set -m
+
+# Set trap to kill the process group on script exit
+trap 'kill -TERM -$$' EXIT
+
+pkill -9 -f httpwg-hyper
+pkill -9 -f httpwg-loona
+
 # Launch hyper server
 export TEST_PROTO=h2 ADDR=0.0.0.0 PORT=8001
 "$LOONA_DIR/target/release/httpwg-hyper" &
@@ -28,7 +37,7 @@ echo "Loona PID: $LOONA_PID"
 HYPER_ADDR="http://localhost:8001"
 LOONA_ADDR="http://localhost:8002"
 
-ENDPOINT="${ENDPOINT:-/stream-big-body}"
+ENDPOINT="${ENDPOINT:-/repeat-4k-blocks/128}"
 
 declare -A servers=(
     [hyper]="$HYPER_PID $HYPER_ADDR"

From 3571e4a9dda2ce02c481c2e552834cda93665928 Mon Sep 17 00:00:00 2001
From: Amos Wenger <amos@bearcove.net>
Date: Fri, 30 Aug 2024 16:48:43 +0200
Subject: [PATCH 21/24] standardize testing some more

---
 Cargo.lock                      |  35 +++++-----
 Justfile                        |   4 +-
 crates/httpwg-hyper/Cargo.toml  |   2 +
 crates/httpwg-hyper/src/main.rs | 110 ++++++++++++++++++++++----------
 crates/httpwg-loona/Cargo.toml  |   3 +
 crates/loona/Cargo.toml         |   2 +-
 scripts/mkfiles.sh              |   7 ++
 7 files changed, 107 insertions(+), 56 deletions(-)

diff --git a/Cargo.lock b/Cargo.lock
index 0aaa524c..a1088aba 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -59,6 +59,7 @@ dependencies = [
  "aws-lc-sys",
  "mirai-annotations",
  "paste",
+ "untrusted 0.7.1",
  "zeroize",
 ]
 
@@ -96,12 +97,6 @@ dependencies = [
  "rustc-demangle",
 ]
 
-[[package]]
-name = "base64"
-version = "0.22.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6"
-
 [[package]]
 name = "bindgen"
 version = "0.69.4"
@@ -786,7 +781,9 @@ dependencies = [
  "hyper",
  "hyper-util",
  "pin-project-lite",
+ "rcgen",
  "tokio",
+ "tokio-rustls",
  "tokio-stream",
  "tokio-util",
  "tracing",
@@ -801,8 +798,11 @@ dependencies = [
  "codspeed-criterion-compat",
  "color-eyre",
  "eyre",
+ "ktls",
  "loona",
+ "rcgen",
  "tokio",
+ "tokio-rustls",
  "tracing",
  "tracing-subscriber",
 ]
@@ -1281,16 +1281,6 @@ version = "1.0.15"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "57c0d7b74b563b49d38dae00a0c37d4d6de9b432382b2892f0574ddcae73fd0a"
 
-[[package]]
-name = "pem"
-version = "3.0.4"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8e459365e590736a54c3fa561947c84837534b8e9af6fc5bf781307e82658fae"
-dependencies = [
- "base64",
- "serde",
-]
-
 [[package]]
 name = "pin-project-lite"
 version = "0.2.14"
@@ -1416,8 +1406,7 @@ version = "0.13.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "54077e1872c46788540de1ea3d7f4ccb1983d12f9aa909b234468676c1a36779"
 dependencies = [
- "pem",
- "ring",
+ "aws-lc-rs",
  "rustls-pki-types",
  "time",
  "yasna",
@@ -1487,7 +1476,7 @@ dependencies = [
  "getrandom",
  "libc",
  "spin",
- "untrusted",
+ "untrusted 0.9.0",
  "windows-sys 0.52.0",
 ]
 
@@ -1546,7 +1535,7 @@ dependencies = [
  "aws-lc-rs",
  "ring",
  "rustls-pki-types",
- "untrusted",
+ "untrusted 0.9.0",
 ]
 
 [[package]]
@@ -1914,6 +1903,12 @@ version = "1.0.12"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "3354b9ac3fae1ff6755cb6db53683adb661634f67557942dea4facebec0fee4b"
 
+[[package]]
+name = "untrusted"
+version = "0.7.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a156c684c91ea7d62626509bce3cb4e1d9ed5c4d978f7b4352658f96a4c26b4a"
+
 [[package]]
 name = "untrusted"
 version = "0.9.0"
diff --git a/Justfile b/Justfile
index 196f6f93..f2ee4384 100644
--- a/Justfile
+++ b/Justfile
@@ -63,7 +63,7 @@ httpwg-over-tcp *args='':
     cargo build --release \
         --package httpwg-loona \
         --package httpwg-cli
-    export TEST_PROTO=h2
+    export PROTO=h2
     export PORT=8001
     export RUST_LOG=${RUST_LOG:-info}
     ./target/release/httpwg --frame-timeout 2000 --connect-timeout 2000 --address localhost:8001 "$@" -- ./target/release/httpwg-loona
@@ -84,7 +84,7 @@ samply:
         --package httpwg-loona \
         --profile profiling \
         --features tracing/release_max_level_info
-    export TEST_PROTO=h2
+    export PROTO=h2
     export PORT=8002
     target/profiling/httpwg-loona
 
diff --git a/crates/httpwg-hyper/Cargo.toml b/crates/httpwg-hyper/Cargo.toml
index c76d7c22..ff6ddbeb 100644
--- a/crates/httpwg-hyper/Cargo.toml
+++ b/crates/httpwg-hyper/Cargo.toml
@@ -20,7 +20,9 @@ hyper-util = { version = "0.1.7", features = [
     "tokio",
 ] }
 pin-project-lite = "0.2.14"
+rcgen = { version = "0.13.1", default-features = false, features = ["aws_lc_rs"] }
 tokio = { version = "1.39.2", features = ["full"] }
+tokio-rustls = "0.26.0"
 tokio-stream = "0.1.15"
 tokio-util = { version = "0.7.11", features = ["io"] }
 tracing = "0.1.40"
diff --git a/crates/httpwg-hyper/src/main.rs b/crates/httpwg-hyper/src/main.rs
index fcbca0a7..4dd5a3ca 100644
--- a/crates/httpwg-hyper/src/main.rs
+++ b/crates/httpwg-hyper/src/main.rs
@@ -2,12 +2,13 @@ use http_body_util::{BodyExt, StreamBody};
 use hyper_util::rt::TokioExecutor;
 use hyper_util::rt::TokioIo;
 use tokio::io::AsyncReadExt;
-use tokio_stream::StreamExt;
-use tokio_util::io::ReaderStream;
 
 use hyper_util::server::conn::auto;
+use std::sync::Arc;
 use std::{convert::Infallible, fmt::Debug, pin::Pin};
 use tokio::sync::mpsc;
+use tokio_rustls::rustls::pki_types::PrivatePkcs8KeyDer;
+use tokio_rustls::rustls::ServerConfig;
 
 use bytes::Bytes;
 use futures::Future;
@@ -71,8 +72,7 @@ where
                         let (tx, rx) = mpsc::channel::<Result<Frame<Bytes>, E>>(1);
 
                         tokio::spawn(async move {
-                            let block = "this is a fairly large block".repeat(4096);
-                            let block = Bytes::from(block);
+                            let block = Bytes::copy_from_slice(BLOCK);
                             for _ in 0..repeat {
                                 let frame = Frame::data(block.clone());
                                 let _ = tx.send(Ok(frame)).await;
@@ -86,6 +86,7 @@ where
                     } else if let ["stream-file", name] = parts.as_slice() {
                         let name = name.to_string();
 
+                        // TODO: custom impl of the Body trait to avoid channel overhead
                         // stream 64KB blocks of the file
                         let (tx, rx) = mpsc::channel::<Result<Frame<Bytes>, E>>(1);
                         tokio::spawn(async move {
@@ -93,14 +94,13 @@ where
                                 tokio::fs::File::open(format!("/tmp/stream-file/{name}"))
                                     .await
                                     .unwrap();
-                            let mut buf = Vec::with_capacity(64 * 1024);
+                            let mut buf = vec![0u8; 64 * 1024];
                             while let Ok(n) = file.read(&mut buf).await {
                                 if n == 0 {
                                     break;
                                 }
                                 let frame = Frame::data(Bytes::copy_from_slice(&buf[..n]));
                                 let _ = tx.send(Ok(frame)).await;
-                                buf.drain(..n);
                             }
                         });
 
@@ -108,11 +108,17 @@ where
                         let body: BoxBody<E> = Box::pin(StreamBody::new(rx));
                         let res = Response::builder().body(body).unwrap();
                         Ok(res)
-                    } else {
+                    } else if parts.as_slice().is_empty() {
                         let body = "it's less dire to lose, than to lose oneself".to_string();
                         let body: BoxBody<E> = Box::pin(body.map_err(|_| unreachable!()));
                         let res = Response::builder().status(200).body(body).unwrap();
                         Ok(res)
+                    } else {
+                        // return a 404
+                        let body = "404 Not Found".to_string();
+                        let body: BoxBody<E> = Box::pin(body.map_err(|_| unreachable!()));
+                        let res = Response::builder().status(404).body(body).unwrap();
+                        Ok(res)
                     }
                 }
             }
@@ -131,38 +137,76 @@ async fn main() {
     println!("I listen on {upstream_addr}");
 
     #[derive(Debug, Clone, Copy)]
+    #[allow(clippy::upper_case_acronyms)]
     enum Proto {
         H1,
-        H2,
+        H2C,
+        TLS,
     }
 
-    let proto = match std::env::var("TEST_PROTO")
-        .unwrap_or("h1".to_string())
-        .as_str()
-    {
+    let proto = match std::env::var("PROTO").unwrap_or("h1".to_string()).as_str() {
+        // plaintext HTTP/1.1
         "h1" => Proto::H1,
-        "h2" => Proto::H2,
-        _ => panic!("TEST_PROTO must be either 'h1' or 'h2'"),
+        // HTTP/2 with prior knowledge
+        "h2c" => Proto::H2C,
+        // TLS with ALPN
+        "tls" => Proto::TLS,
+        _ => panic!("PROTO must be one of 'h1', 'h2c', or 'tls'"),
     };
-    println!("Using {proto:?} protocol (export TEST_PROTO=h1 or TEST_PROTO=h2 to override)");
-
-    while let Ok((stream, _)) = ln.accept().await {
-        stream.set_nodelay(true).unwrap();
-
-        tokio::spawn(async move {
-            let mut builder = auto::Builder::new(TokioExecutor::new());
-
-            match proto {
-                Proto::H1 => {
-                    builder = builder.http1_only();
-                }
-                Proto::H2 => {
-                    builder = builder.http2_only();
-                }
+    println!("Using {proto:?} (use PROTO=[h1,h2c,tls])");
+
+    match proto {
+        Proto::TLS => {
+            let certified_key =
+                rcgen::generate_simple_self_signed(vec!["localhost".to_string()]).unwrap();
+            let crt = certified_key.cert.der();
+            let key = certified_key.key_pair.serialize_der();
+
+            let server_config = ServerConfig::builder()
+                .with_no_client_auth()
+                .with_single_cert(
+                    vec![crt.clone()],
+                    PrivatePkcs8KeyDer::from(key.clone()).into(),
+                )
+                .unwrap();
+
+            let acceptor = tokio_rustls::TlsAcceptor::from(Arc::new(server_config));
+
+            while let Ok((stream, _)) = ln.accept().await {
+                stream.set_nodelay(true).unwrap();
+                let acceptor = acceptor.clone();
+                tokio::spawn(async move {
+                    let stream = acceptor.accept(stream).await.unwrap();
+                    let builder = auto::Builder::new(TokioExecutor::new());
+                    builder
+                        .serve_connection(TokioIo::new(stream), TestService)
+                        .await
+                });
+            }
+        }
+        _ => {
+            while let Ok((stream, _)) = ln.accept().await {
+                stream.set_nodelay(true).unwrap();
+
+                tokio::spawn(async move {
+                    let mut builder = auto::Builder::new(TokioExecutor::new());
+
+                    match proto {
+                        Proto::H1 => {
+                            builder = builder.http1_only();
+                        }
+                        Proto::H2C => {
+                            builder = builder.http2_only();
+                        }
+                        _ => {
+                            // nothing
+                        }
+                    }
+                    builder
+                        .serve_connection(TokioIo::new(stream), TestService)
+                        .await
+                });
             }
-            builder
-                .serve_connection(TokioIo::new(stream), TestService)
-                .await
-        });
+        }
     }
 }
diff --git a/crates/httpwg-loona/Cargo.toml b/crates/httpwg-loona/Cargo.toml
index 24120249..054583ac 100644
--- a/crates/httpwg-loona/Cargo.toml
+++ b/crates/httpwg-loona/Cargo.toml
@@ -29,6 +29,9 @@ tracing-subscriber = "0.3.18"
 tokio = { version = "1.39.2", features = ["macros", "sync", "process"] }
 eyre = { version = "0.6.12", default-features = false }
 b-x = { version = "1.0.0", path = "../b-x" }
+rcgen = { version = "0.13.1", default-features = false, features = ["aws_lc_rs"] }
+tokio-rustls = "0.26.0"
+ktls = "6.0.0"
 
 [dev-dependencies]
 codspeed-criterion-compat = "2.6.0"
diff --git a/crates/loona/Cargo.toml b/crates/loona/Cargo.toml
index 3251f3fe..c50a4d34 100644
--- a/crates/loona/Cargo.toml
+++ b/crates/loona/Cargo.toml
@@ -71,7 +71,7 @@ cargo-husky = { version = "1", features = ["user-hooks"] }
 criterion = "0.5.1"
 codspeed-criterion-compat = "2.6.0"
 itoa = "1.0.11"
-rcgen = "0.13.1"
+rcgen = { version = "0.13.1", default-features = false, features = ["aws_lc_rs"] }
 socket2 = "0.5.7"
 
 [target.'cfg(target_os = "linux")'.dev-dependencies]
diff --git a/scripts/mkfiles.sh b/scripts/mkfiles.sh
index f061a485..27d44da3 100755
--- a/scripts/mkfiles.sh
+++ b/scripts/mkfiles.sh
@@ -5,10 +5,17 @@ cd "$(dirname "$0")"
 
 # Define an array of file names and sizes
 declare -A files=(
+    ["1M"]="1M"
     ["4M"]="4M"
+    ["8M"]="8M"
     ["16M"]="16M"
+    ["32M"]="32M"
+    ["64M"]="64M"
+    ["128M"]="128M"
     ["256M"]="256M"
+    ["512M"]="512M"
     ["1GB"]="1G"
+    ["2GB"]="2G"
 )
 
 # Print summary and ask for consent

From f71c92c5d198bd8348c9cb071fe95a55cf6c7193 Mon Sep 17 00:00:00 2001
From: Amos Wenger <amos@bearcove.net>
Date: Fri, 30 Aug 2024 16:53:47 +0200
Subject: [PATCH 22/24] httpwg-hyper: configure ALPN and respect results of
 negotiation properly

---
 crates/httpwg-hyper/src/main.rs | 15 +++++++++++++--
 1 file changed, 13 insertions(+), 2 deletions(-)

diff --git a/crates/httpwg-hyper/src/main.rs b/crates/httpwg-hyper/src/main.rs
index 4dd5a3ca..6d5e5c18 100644
--- a/crates/httpwg-hyper/src/main.rs
+++ b/crates/httpwg-hyper/src/main.rs
@@ -162,13 +162,14 @@ async fn main() {
             let crt = certified_key.cert.der();
             let key = certified_key.key_pair.serialize_der();
 
-            let server_config = ServerConfig::builder()
+            let mut server_config = ServerConfig::builder()
                 .with_no_client_auth()
                 .with_single_cert(
                     vec![crt.clone()],
                     PrivatePkcs8KeyDer::from(key.clone()).into(),
                 )
                 .unwrap();
+            server_config.alpn_protocols = vec![b"h2".to_vec(), b"http/1.1".to_vec()];
 
             let acceptor = tokio_rustls::TlsAcceptor::from(Arc::new(server_config));
 
@@ -177,7 +178,17 @@ async fn main() {
                 let acceptor = acceptor.clone();
                 tokio::spawn(async move {
                     let stream = acceptor.accept(stream).await.unwrap();
-                    let builder = auto::Builder::new(TokioExecutor::new());
+
+                    let mut builder = auto::Builder::new(TokioExecutor::new());
+                    match stream.get_ref().1.alpn_protocol() {
+                        Some(b"h2") => {
+                            builder = builder.http2_only();
+                        }
+                        Some(b"http/1.1") => {
+                            builder = builder.http1_only();
+                        }
+                        _ => {}
+                    }
                     builder
                         .serve_connection(TokioIo::new(stream), TestService)
                         .await

From f8408eddccddcd6e47cd5bed1a21cdaf4d5120ce Mon Sep 17 00:00:00 2001
From: Amos Wenger <amos@bearcove.net>
Date: Fri, 30 Aug 2024 18:53:29 +0200
Subject: [PATCH 23/24] both hyper and loona httpwg clis use the harness now

---
 Cargo.lock                                 |  13 +
 crates/buffet/src/net/net_uring.rs         |  15 +-
 crates/httpwg-harness/Cargo.toml           |   9 +
 crates/httpwg-harness/src/lib.rs           |  78 +++++
 crates/httpwg-hyper/Cargo.toml             |   1 +
 crates/httpwg-hyper/src/async_read_body.rs |  87 ------
 crates/httpwg-hyper/src/main.rs            | 175 +----------
 crates/httpwg-hyper/src/service.rs         | 128 ++++++++
 crates/httpwg-loona/Cargo.toml             |  14 +-
 crates/httpwg-loona/benches/h2load.rs      |  17 --
 crates/httpwg-loona/src/driver.rs          | 158 ++++++++++
 crates/httpwg-loona/src/lib.rs             | 330 ---------------------
 crates/httpwg-loona/src/main.rs            | 186 +++++++++---
 crates/loona/Cargo.toml                    |   1 +
 crates/loona/tests/testbed.rs              |   9 +-
 15 files changed, 572 insertions(+), 649 deletions(-)
 create mode 100644 crates/httpwg-harness/Cargo.toml
 create mode 100644 crates/httpwg-harness/src/lib.rs
 delete mode 100644 crates/httpwg-hyper/src/async_read_body.rs
 create mode 100644 crates/httpwg-hyper/src/service.rs
 delete mode 100644 crates/httpwg-loona/benches/h2load.rs
 create mode 100644 crates/httpwg-loona/src/driver.rs
 delete mode 100644 crates/httpwg-loona/src/lib.rs

diff --git a/Cargo.lock b/Cargo.lock
index a1088aba..bdef83f9 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -771,6 +771,15 @@ dependencies = [
  "serde_json",
 ]
 
+[[package]]
+name = "httpwg-harness"
+version = "0.1.0"
+dependencies = [
+ "eyre",
+ "rcgen",
+ "rustls",
+]
+
 [[package]]
 name = "httpwg-hyper"
 version = "0.2.0"
@@ -778,6 +787,7 @@ dependencies = [
  "bytes",
  "futures",
  "http-body-util",
+ "httpwg-harness",
  "hyper",
  "hyper-util",
  "pin-project-lite",
@@ -798,9 +808,11 @@ dependencies = [
  "codspeed-criterion-compat",
  "color-eyre",
  "eyre",
+ "httpwg-harness",
  "ktls",
  "loona",
  "rcgen",
+ "socket2",
  "tokio",
  "tokio-rustls",
  "tracing",
@@ -1027,6 +1039,7 @@ dependencies = [
  "http",
  "httparse",
  "httpwg",
+ "httpwg-harness",
  "httpwg-macros",
  "itoa",
  "ktls",
diff --git a/crates/buffet/src/net/net_uring.rs b/crates/buffet/src/net/net_uring.rs
index 30e2555f..aa7e8acf 100644
--- a/crates/buffet/src/net/net_uring.rs
+++ b/crates/buffet/src/net/net_uring.rs
@@ -1,18 +1,17 @@
 use std::{
     mem::ManuallyDrop,
     net::SocketAddr,
-    os::fd::{AsRawFd, FromRawFd, RawFd},
+    os::fd::{AsRawFd, FromRawFd, IntoRawFd, RawFd},
     rc::Rc,
 };
 
-use io_uring::opcode::{Accept, Read, Write, Writev};
-use libc::iovec;
+use io_uring::opcode::{Accept, Read, Write};
 use nix::errno::Errno;
 
 use crate::{
     get_ring,
     io::{IntoHalves, ReadOwned, WriteOwned},
-    BufResult, IoBufMut, Piece, PieceList,
+    BufResult, IoBufMut, Piece,
 };
 
 pub struct TcpStream {
@@ -54,6 +53,14 @@ impl Drop for TcpStream {
     }
 }
 
+impl IntoRawFd for TcpStream {
+    fn into_raw_fd(self) -> RawFd {
+        let fd = self.fd;
+        std::mem::forget(self);
+        fd
+    }
+}
+
 pub struct TcpListener {
     fd: i32,
 }
diff --git a/crates/httpwg-harness/Cargo.toml b/crates/httpwg-harness/Cargo.toml
new file mode 100644
index 00000000..bd3787e1
--- /dev/null
+++ b/crates/httpwg-harness/Cargo.toml
@@ -0,0 +1,9 @@
+[package]
+name = "httpwg-harness"
+version = "0.1.0"
+edition = "2021"
+
+[dependencies]
+eyre = { version = "0.6.12", default-features = false }
+rcgen = { version = "0.13.1", default-features = false, features = ["aws_lc_rs"] }
+rustls = "0.23.12"
diff --git a/crates/httpwg-harness/src/lib.rs b/crates/httpwg-harness/src/lib.rs
new file mode 100644
index 00000000..ace82888
--- /dev/null
+++ b/crates/httpwg-harness/src/lib.rs
@@ -0,0 +1,78 @@
+use std::{net::SocketAddr, str::FromStr, sync::Arc};
+
+use rustls::{pki_types::PrivatePkcs8KeyDer, KeyLogFile, ServerConfig};
+
+#[derive(Debug, Clone, Copy)]
+#[allow(clippy::upper_case_acronyms)]
+pub enum Proto {
+    H1,
+    H2C,
+    TLS,
+}
+
+pub struct Settings {
+    pub listen_addr: SocketAddr,
+    pub proto: Proto,
+}
+
+impl Settings {
+    pub fn from_env() -> eyre::Result<Self> {
+        let port = std::env::var("PORT").unwrap_or("0".to_string());
+        let addr = std::env::var("ADDR").unwrap_or("127.0.0.1".to_string());
+        let listen_addr = SocketAddr::from_str(&format!("{}:{}", addr, port))?;
+
+        let proto = std::env::var("PROTO").unwrap_or("h2c".to_string());
+        let proto = match proto.as_str() {
+            // plaintext HTTP/1.1
+            "h1" => Proto::H1,
+            // HTTP/2 with prior knowledge
+            "h2c" => Proto::H2C,
+            // TLS with ALPN
+            "tls" => Proto::TLS,
+            _ => panic!("PROTO must be one of 'h1', 'h2c', or 'tls'"),
+        };
+        Ok(Self { listen_addr, proto })
+    }
+
+    pub const LISTEN_LINE_PREFIX: &'static str = "🌎🦊👉";
+
+    pub fn print_listen_line(&self, addr: SocketAddr) {
+        println!("🌎🦊👉 {addr} ({:?})", self.proto)
+    }
+
+    pub fn decode_listen_line(&self, line: &str) -> eyre::Result<Option<SocketAddr>> {
+        let line = match line.strip_prefix(Self::LISTEN_LINE_PREFIX) {
+            Some(l) => l,
+            None => return Ok(None),
+        };
+        let addr_token = line
+            .split_whitespace()
+            .next()
+            .ok_or_else(|| eyre::eyre!("No address token found"))?;
+        let addr = addr_token
+            .parse::<SocketAddr>()
+            .map_err(|e| eyre::eyre!("Failed to parse SocketAddr: {}", e))?;
+        Ok(Some(addr))
+    }
+
+    pub fn gen_rustls_server_config() -> eyre::Result<ServerConfig> {
+        let certified_key =
+            rcgen::generate_simple_self_signed(vec!["localhost".to_string()]).unwrap();
+        let crt = certified_key.cert.der();
+        let key = certified_key.key_pair.serialize_der();
+
+        let mut server_config = ServerConfig::builder()
+            .with_no_client_auth()
+            .with_single_cert(
+                vec![crt.clone()],
+                PrivatePkcs8KeyDer::from(key.clone()).into(),
+            )
+            .unwrap();
+        server_config.key_log = Arc::new(KeyLogFile::new());
+        server_config.alpn_protocols = vec![b"h2".to_vec(), b"http/1.1".to_vec()];
+        Ok(server_config)
+    }
+}
+
+/// A sample block of 4KiB of data.
+pub const SAMPLE_4K_BLOCK: &[u8] = b"K75HZ+W4P2+Z+K1eI/lPJkc+HiVr/+snBmi0fu5IAIseZ6HumAEX2bfv4ok9Rzqm8Eq1dP3ap8pscfD4IBqBHnxtMdc6+Vvf81WDDqf3yXL3yvoA0N0jxuVs9jXTllu/h+ABUf8dBymieg/xhJsn7NQDJvb/fh5+ZZpP8++ihiUgwgc+yM04rtSIP+O6Ul0RdoeHftzguVujmB9bnf+JtrUAL+AFCxIommB7IszrCLyz+0ysE2Ke1Mvv5Et88p4wvPc4TcKJC53OmyHcFp4HOI8tZXJC2eIaWC59bpTxWuzt0w0x0P8dou1uvCQTSRDHcHIo4VevzgqtCVnISEhdxjBUU6bNa4rCmXKEjSCd09fYe/Wsd45mji9J9cco1kQs4wU43se8oCSzcKnYI4cB0iyvDD3/ceIATVrYv3R8QH69J1NFWTvsILMf+TXfVQgfJmthIF/aY417hJjhvEjyoez27dZrcAMUXlvAXDozt3IsFS9D1KJvzt1SSaKENi/WjC+WMCTZr4guBNbNQdyd8NLRf/Ilum3zrIJDwcT+IecgdtIDtG3koYqVJ1ihAxFYMaZFk32R4iaNhUxyibX1DE2w8Xfz3g0HiAxGl+rWMREldUTEBlwk8Ig5ccanXwJ8fLXOn/UduZQkIKuH4ucb+T40T/iNubbi4/5SSVphTEnGJ0y1fcowKPxxseyZ5SZHVoYxHGvEYeCl+hw5XgqiZaIpHZZMiAQh38zGGd6J8mLOsPG6BSpWV8Cj00UusRnO/V2tAxiR7Vuh8EiDPV728a3XsZI5xGc4MMWbqTSmMGm2x8XybIe/vL6U7Y9ptr4c18nfQErH/Yt4OmmFGP0VTmbSo2aGGMkJ1VwX/6BAxIxOMXoqshNfZ2Nh+0py0V/Ly+SQr6OcTxX857d0I3l0P8GWsLcZxER9EpkEO6NKUMdOIqZdRoC1p1lnzMsL5UvWDFrFoIXJqAA3jHmXN+zZgJbg7+sLdWE2HR2EvsepXUdK0t31SqkBkn0YHJbklSivWe9FbLOIstB2kigkYmnFT0a49aW+uTlgU6Tc+hx9ufW6l17EHf8I37WIvInLNKsk+wOqeYzspRf8rE4mfYyFunhDDXSe/eFaVnb53otiGsYA3GRutY5FfBrYkK2ZQRIND5B+AqwGa+4V47yPkq217iCKgBSYXA5Ux0e138LUMNq2Yn9YqbdMP3XEPUBBaiT8q2GE+w/ay7dZOid1jiV72OET90aSA8FFev6jnhQhvlR6qndOYexk1GWO+mFanlUU/PEZ0+0v9tj93TlPZp/0xfWNyXpXh5ubDLRNoxX/RRQ6hMIkbpDEeCiI4zBRk1vVMpI6myc76tvMk97APMJDpKt3QGCLCQD0vb2UEqMkEKFxggR46PvlCI3zo0LQr5oigB3kaSShFzTAm8hKOzg5M9NpN/l+hQHQJv9lFhxjsuHCvdM6sNF3rxLtEKCc45IicsJRM/CyZc7cadMurqBGBUSQHpLmtndFaLNvjRQMI1gYYGcEr34/WOGG5LRQvo0I7toSjcVFc2JdfGuT/71JNJupS89l6nrSisFPCuCCgaN5O4jZAb4vnhrHHZs8r0IuFtd39pT24obpLYsheBT2+tdCf3QsEIvkGZ/VQkn/4jaMyCsGw37mm8dZNyGtn3cWcP9DYytYNNmbjc8Ks3rvkbLttMch8AyEQClqvgXwVMNPHBI/gL0OY8cPyCXxh7x4NCt0bmS9AUb+YCkEmXxDOkxrDntRFvmavacZbF6jNjMXfqG2dkMmZ9obz7M31r3eDYa1bd2MLgb5H3napVjILcRnuPrgR+EdqonE8+fIVZjGZL6Jgwi1ja0VHsoyI8d5dPDazD4U5q2EaPbkX/62RMCRz7FRJX368NBZigOwVzR3/oIJZjeuNTlsoe4cP17jGXXCkNXXY7gUmN7A2hOH9Wg5IDdPahBCf7kpL3wOcXYoyN1fciwfq+kvN8jqNtMJcGrEls2wGnWNc5OITtHTqT7xltIdE2rjkBDo4PIwfdmOZxpbnscbfVSG5HANXA6B+6caN3hor27E8Y9aEmdhPSDP0vdedzXWPzeyTQK82bbA4PB+mny+FP1IImUuVxV9jzPLPPxylx6EaR+SsxHNdUrMETboaK70mViWZpSJhSgMDQGGs1tkV22qRZFnZgIppTh4C0fBiKNK1TxkXHA7CZqndMXbA9w7C2ywBEuPCBvHZPm5qre1jLAbXC7z8TNJ/EDxdJI8yXSrKesKQiNiEZ5rEUORy3Omxi0GaPG/LfwgHmmEdTfttfzk24LbHs51XLbX5cGM+7sQ9nLVCjiaMZEsfx87At4CnbzliC3UI/ZVkYAlby0fp2TXxfMdN5VRDueDlSUdIz88tLgWJQ8lHEI90HLl4n2dNfUr08Eea4QdjI+r3INuhdS7RFm+jUWXnbPaoQpn7rev4p3tRV0YL4N3lj4eXHMsrQ4NM3ASlwvuPXfun/b+QWWTqS/k+c6vuQP1H0utoAOlv3Lmzeczq+vC35QUHJdGvi43+nrNRYNWrDP0FtFIlC1q5DN+XIL7Pq2eX8dYku/2cLEYQokY7Pq4+0frobbTIxo2AVpT41qmRhgQc2iNGLk9PDhLoopDEcS5dSql06IIo8r6Xx/tthaToqyDk+aAoQZf7wz7rvVmi0Mj158+KVRn4z2b6sCEe8yl+u9DpYmNbU4THEQSvTSsEyez0Fps23NmIDWqXpMevUYxIgZXNorbEClxPqSOHzbiL/K02E2HhjD3JA8q+XkJdvX97orDqC/BNPp0Ivp7P9TAqmjbJ6AYHMoYh/25SFq6jQQFUwuFS98wd1CJMDdewd0VzFEuzeuz69krNwv/jMNrGAUmTLeDE9jOKPMmGixOUyNLtXGpLHKleE7iVkj7LKDu2zlqYRTrDkz36JroclE+7GROXWT8+OJO4KnMep+v+ZXPFkf26/KXzKya25nqe0h200bJ/eUsFg74f9NTq+FMfEsXpRacnIVJo/yJLnObOKGL5K3VrHkrx3ccubhcPHR7MkvBmhIWcOXB3KwCnvkfsA0ttvNQQ4w5ojOz0nxaiaP6NbFz0xuehwDrkaTFSF2QLfGvXI9pY/v3PJtWAj33EEwSMs46crAX++NVBuWOKGdzgvmaCxnh5oFojrvwLrr2xdJK2nzoGQJD78HMHZ1hmYfZ8UFOigZ2PtjV/Tyt6XXZ3BhFjxjkCbvR4nsGoHbYVOxkNlmXsSKSRyhttRQ0r3WfHG7ot3YnoJpogHBy0T+O8Yu+SIPCIe6b+ac7rvewOi4kwobtygQBJFNoUN+0z3Ztqf49yc3viPfTXW4nlooWcyJhUs5Tk1FVLDOEJeDp8clCxYw/XtlMr+BLbVF7w3koa+aHU1PJmo562IeH/sDiANKw1GnGvcxqhmMsb4aPOTpvnpq16JLVtmdIl83j2oVOb1Ql1U6b0zv1pphHq8MwESFDm1tSThDbs41vkFWHplb5SpTLxAA2e9H/Ch+cb7h9OXt7HwNPsq/+0zzT9D2rlhoDatqqTnbWpyozcRDvNKOJvPlnUCvKzHJNMcp/d9q1AaTcOrNYFVDZeEOTw7+/vCAmLxihRINycQND+/x180V22WcT9I9dRbuaEPM2XpfRlkENbERqDWeGfKmuhK5r1PkF7G8QxnDgrekFVHvqudGINzi+1ELzobztD7AoyBKkIUWKzSWm/HLk5zEm9lZ2Dkh9+13faXcxjifGkOvIm6g0BF+XqpvBJSyxfKg58/x0tksvI8HOfgJmPfLFdUJbmcM+WTtebp10b9+35qN0KZJbdEwZcrRrgdLbWCIvSRvNUR2SakZbYMSy08zthER446WCeRCmzzook/Scxk+Mn3WeOyMmJsXR1zXfoD7plogXvR4nJPWpawrjl13hVZ1XCj6DszYdeIuVdonMYh3zn0TToAB/4xaNKev1IOAaU08exxD/DKWBZEM3LbZGsXuH7F1jOySuagkl5+JeffpMTx0sRpHMzEzfdX/WOFJ/w9BR5kJjGB6KtBLic1Oy9JNCez21wC4Oo4DAPqK/W4cnDgUeYev2OkiyeX47WhDRSLES4iQcsWLJ4img";
diff --git a/crates/httpwg-hyper/Cargo.toml b/crates/httpwg-hyper/Cargo.toml
index ff6ddbeb..f6975c82 100644
--- a/crates/httpwg-hyper/Cargo.toml
+++ b/crates/httpwg-hyper/Cargo.toml
@@ -12,6 +12,7 @@ A reference HTTP 1+2 server for httpwg, powered by hyper
 bytes = "1.7.1"
 futures = "0.3.30"
 http-body-util = "0.1.2"
+httpwg-harness = { version = "0.1.0", path = "../httpwg-harness" }
 hyper = { version = "1.4.1", features = ["client", "server", "http1", "http2"] }
 hyper-util = { version = "0.1.7", features = [
     "server",
diff --git a/crates/httpwg-hyper/src/async_read_body.rs b/crates/httpwg-hyper/src/async_read_body.rs
deleted file mode 100644
index 43a5f8e3..00000000
--- a/crates/httpwg-hyper/src/async_read_body.rs
+++ /dev/null
@@ -1,87 +0,0 @@
-// use hyper::body::{Body, Frame};
-// use pin_project_lite::pin_project;
-// use std::{
-//     pin::Pin,
-//     task::{Context, Poll},
-// };
-// use tokio::io::AsyncRead;
-// use tokio_util::io::ReaderStream;
-
-// pin_project! {
-//     /// An [`HttpBody`] created from an [`AsyncRead`].
-//     ///
-//     /// # Example
-//     ///
-//     /// `AsyncReadBody` can be used to stream the contents of a file:
-//     ///
-//     /// ```rust
-//     /// use axum::{
-//     ///     Router,
-//     ///     routing::get,
-//     ///     http::{StatusCode, header::CONTENT_TYPE},
-//     ///     response::{Response, IntoResponse},
-//     /// };
-//     /// use axum_extra::body::AsyncReadBody;
-//     /// use tokio::fs::File;
-//     ///
-//     /// async fn cargo_toml() -> Result<Response, (StatusCode, String)> {
-//     ///     let file = File::open("Cargo.toml")
-//     ///         .await
-//     ///         .map_err(|err| {
-//     ///             (StatusCode::NOT_FOUND, format!("File not found: {err}"))
-//     ///         })?;
-//     ///
-//     ///     let headers = [(CONTENT_TYPE, "text/x-toml")];
-//     ///     let body = AsyncReadBody::new(file);
-//     ///     Ok((headers, body).into_response())
-//     /// }
-//     ///
-//     /// let app = Router::new().route("/Cargo.toml", get(cargo_toml));
-//     /// # let _: Router = app;
-//     /// ```
-//     #[derive(Debug)]
-//     #[must_use]
-//     pub struct AsyncReadBody<R> {
-//         #[pin]
-//         read: R,
-//     }
-// }
-
-// impl AsyncReadBody {
-//     /// Create a new `AsyncReadBody`.
-//     pub fn new<R>(read: R) -> Self
-//     where
-//         R: AsyncRead + Send + 'static,
-//     {
-//         Self { read }
-//     }
-// }
-
-// impl Body for AsyncReadBody {
-//     type Data = Bytes;
-//     type Error = Error;
-
-//     #[inline]
-//     fn poll_frame(
-//         self: Pin<&mut Self>,
-//         cx: &mut Context<'_>,
-//     ) -> Poll<Option<Result<Frame<Self::Data>, Self::Error>>> {
-//         self.project().body.poll_frame(cx)
-//     }
-
-//     #[inline]
-//     fn is_end_stream(&self) -> bool {
-//         self.body.is_end_stream()
-//     }
-
-//     #[inline]
-//     fn size_hint(&self) -> http_body::SizeHint {
-//         self.body.size_hint()
-//     }
-// }
-
-// impl IntoResponse for AsyncReadBody {
-//     fn into_response(self) -> Response {
-//         self.body.into_response()
-//     }
-// }
diff --git a/crates/httpwg-hyper/src/main.rs b/crates/httpwg-hyper/src/main.rs
index 6d5e5c18..817339f9 100644
--- a/crates/httpwg-hyper/src/main.rs
+++ b/crates/httpwg-hyper/src/main.rs
@@ -1,176 +1,25 @@
-use http_body_util::{BodyExt, StreamBody};
+use httpwg_harness::Proto;
+use httpwg_harness::Settings;
 use hyper_util::rt::TokioExecutor;
 use hyper_util::rt::TokioIo;
-use tokio::io::AsyncReadExt;
 
 use hyper_util::server::conn::auto;
+use service::TestService;
 use std::sync::Arc;
-use std::{convert::Infallible, fmt::Debug, pin::Pin};
-use tokio::sync::mpsc;
-use tokio_rustls::rustls::pki_types::PrivatePkcs8KeyDer;
-use tokio_rustls::rustls::ServerConfig;
+use tokio::net::TcpListener;
 
-use bytes::Bytes;
-use futures::Future;
-use hyper::{
-    body::{Body, Frame},
-    service::Service,
-    Request, Response,
-};
-use tokio_stream::wrappers::ReceiverStream;
-use tracing::debug;
-
-pub(crate) struct TestService;
-
-mod async_read_body;
-
-const BLOCK: &[u8] = b"K75HZ+W4P2+Z+K1eI/lPJkc+HiVr/+snBmi0fu5IAIseZ6HumAEX2bfv4ok9Rzqm8Eq1dP3ap8pscfD4IBqBHnxtMdc6+Vvf81WDDqf3yXL3yvoA0N0jxuVs9jXTllu/h+ABUf8dBymieg/xhJsn7NQDJvb/fh5+ZZpP8++ihiUgwgc+yM04rtSIP+O6Ul0RdoeHftzguVujmB9bnf+JtrUAL+AFCxIommB7IszrCLyz+0ysE2Ke1Mvv5Et88p4wvPc4TcKJC53OmyHcFp4HOI8tZXJC2eIaWC59bpTxWuzt0w0x0P8dou1uvCQTSRDHcHIo4VevzgqtCVnISEhdxjBUU6bNa4rCmXKEjSCd09fYe/Wsd45mji9J9cco1kQs4wU43se8oCSzcKnYI4cB0iyvDD3/ceIATVrYv3R8QH69J1NFWTvsILMf+TXfVQgfJmthIF/aY417hJjhvEjyoez27dZrcAMUXlvAXDozt3IsFS9D1KJvzt1SSaKENi/WjC+WMCTZr4guBNbNQdyd8NLRf/Ilum3zrIJDwcT+IecgdtIDtG3koYqVJ1ihAxFYMaZFk32R4iaNhUxyibX1DE2w8Xfz3g0HiAxGl+rWMREldUTEBlwk8Ig5ccanXwJ8fLXOn/UduZQkIKuH4ucb+T40T/iNubbi4/5SSVphTEnGJ0y1fcowKPxxseyZ5SZHVoYxHGvEYeCl+hw5XgqiZaIpHZZMiAQh38zGGd6J8mLOsPG6BSpWV8Cj00UusRnO/V2tAxiR7Vuh8EiDPV728a3XsZI5xGc4MMWbqTSmMGm2x8XybIe/vL6U7Y9ptr4c18nfQErH/Yt4OmmFGP0VTmbSo2aGGMkJ1VwX/6BAxIxOMXoqshNfZ2Nh+0py0V/Ly+SQr6OcTxX857d0I3l0P8GWsLcZxER9EpkEO6NKUMdOIqZdRoC1p1lnzMsL5UvWDFrFoIXJqAA3jHmXN+zZgJbg7+sLdWE2HR2EvsepXUdK0t31SqkBkn0YHJbklSivWe9FbLOIstB2kigkYmnFT0a49aW+uTlgU6Tc+hx9ufW6l17EHf8I37WIvInLNKsk+wOqeYzspRf8rE4mfYyFunhDDXSe/eFaVnb53otiGsYA3GRutY5FfBrYkK2ZQRIND5B+AqwGa+4V47yPkq217iCKgBSYXA5Ux0e138LUMNq2Yn9YqbdMP3XEPUBBaiT8q2GE+w/ay7dZOid1jiV72OET90aSA8FFev6jnhQhvlR6qndOYexk1GWO+mFanlUU/PEZ0+0v9tj93TlPZp/0xfWNyXpXh5ubDLRNoxX/RRQ6hMIkbpDEeCiI4zBRk1vVMpI6myc76tvMk97APMJDpKt3QGCLCQD0vb2UEqMkEKFxggR46PvlCI3zo0LQr5oigB3kaSShFzTAm8hKOzg5M9NpN/l+hQHQJv9lFhxjsuHCvdM6sNF3rxLtEKCc45IicsJRM/CyZc7cadMurqBGBUSQHpLmtndFaLNvjRQMI1gYYGcEr34/WOGG5LRQvo0I7toSjcVFc2JdfGuT/71JNJupS89l6nrSisFPCuCCgaN5O4jZAb4vnhrHHZs8r0IuFtd39pT24obpLYsheBT2+tdCf3QsEIvkGZ/VQkn/4jaMyCsGw37mm8dZNyGtn3cWcP9DYytYNNmbjc8Ks3rvkbLttMch8AyEQClqvgXwVMNPHBI/gL0OY8cPyCXxh7x4NCt0bmS9AUb+YCkEmXxDOkxrDntRFvmavacZbF6jNjMXfqG2dkMmZ9obz7M31r3eDYa1bd2MLgb5H3napVjILcRnuPrgR+EdqonE8+fIVZjGZL6Jgwi1ja0VHsoyI8d5dPDazD4U5q2EaPbkX/62RMCRz7FRJX368NBZigOwVzR3/oIJZjeuNTlsoe4cP17jGXXCkNXXY7gUmN7A2hOH9Wg5IDdPahBCf7kpL3wOcXYoyN1fciwfq+kvN8jqNtMJcGrEls2wGnWNc5OITtHTqT7xltIdE2rjkBDo4PIwfdmOZxpbnscbfVSG5HANXA6B+6caN3hor27E8Y9aEmdhPSDP0vdedzXWPzeyTQK82bbA4PB+mny+FP1IImUuVxV9jzPLPPxylx6EaR+SsxHNdUrMETboaK70mViWZpSJhSgMDQGGs1tkV22qRZFnZgIppTh4C0fBiKNK1TxkXHA7CZqndMXbA9w7C2ywBEuPCBvHZPm5qre1jLAbXC7z8TNJ/EDxdJI8yXSrKesKQiNiEZ5rEUORy3Omxi0GaPG/LfwgHmmEdTfttfzk24LbHs51XLbX5cGM+7sQ9nLVCjiaMZEsfx87At4CnbzliC3UI/ZVkYAlby0fp2TXxfMdN5VRDueDlSUdIz88tLgWJQ8lHEI90HLl4n2dNfUr08Eea4QdjI+r3INuhdS7RFm+jUWXnbPaoQpn7rev4p3tRV0YL4N3lj4eXHMsrQ4NM3ASlwvuPXfun/b+QWWTqS/k+c6vuQP1H0utoAOlv3Lmzeczq+vC35QUHJdGvi43+nrNRYNWrDP0FtFIlC1q5DN+XIL7Pq2eX8dYku/2cLEYQokY7Pq4+0frobbTIxo2AVpT41qmRhgQc2iNGLk9PDhLoopDEcS5dSql06IIo8r6Xx/tthaToqyDk+aAoQZf7wz7rvVmi0Mj158+KVRn4z2b6sCEe8yl+u9DpYmNbU4THEQSvTSsEyez0Fps23NmIDWqXpMevUYxIgZXNorbEClxPqSOHzbiL/K02E2HhjD3JA8q+XkJdvX97orDqC/BNPp0Ivp7P9TAqmjbJ6AYHMoYh/25SFq6jQQFUwuFS98wd1CJMDdewd0VzFEuzeuz69krNwv/jMNrGAUmTLeDE9jOKPMmGixOUyNLtXGpLHKleE7iVkj7LKDu2zlqYRTrDkz36JroclE+7GROXWT8+OJO4KnMep+v+ZXPFkf26/KXzKya25nqe0h200bJ/eUsFg74f9NTq+FMfEsXpRacnIVJo/yJLnObOKGL5K3VrHkrx3ccubhcPHR7MkvBmhIWcOXB3KwCnvkfsA0ttvNQQ4w5ojOz0nxaiaP6NbFz0xuehwDrkaTFSF2QLfGvXI9pY/v3PJtWAj33EEwSMs46crAX++NVBuWOKGdzgvmaCxnh5oFojrvwLrr2xdJK2nzoGQJD78HMHZ1hmYfZ8UFOigZ2PtjV/Tyt6XXZ3BhFjxjkCbvR4nsGoHbYVOxkNlmXsSKSRyhttRQ0r3WfHG7ot3YnoJpogHBy0T+O8Yu+SIPCIe6b+ac7rvewOi4kwobtygQBJFNoUN+0z3Ztqf49yc3viPfTXW4nlooWcyJhUs5Tk1FVLDOEJeDp8clCxYw/XtlMr+BLbVF7w3koa+aHU1PJmo562IeH/sDiANKw1GnGvcxqhmMsb4aPOTpvnpq16JLVtmdIl83j2oVOb1Ql1U6b0zv1pphHq8MwESFDm1tSThDbs41vkFWHplb5SpTLxAA2e9H/Ch+cb7h9OXt7HwNPsq/+0zzT9D2rlhoDatqqTnbWpyozcRDvNKOJvPlnUCvKzHJNMcp/d9q1AaTcOrNYFVDZeEOTw7+/vCAmLxihRINycQND+/x180V22WcT9I9dRbuaEPM2XpfRlkENbERqDWeGfKmuhK5r1PkF7G8QxnDgrekFVHvqudGINzi+1ELzobztD7AoyBKkIUWKzSWm/HLk5zEm9lZ2Dkh9+13faXcxjifGkOvIm6g0BF+XqpvBJSyxfKg58/x0tksvI8HOfgJmPfLFdUJbmcM+WTtebp10b9+35qN0KZJbdEwZcrRrgdLbWCIvSRvNUR2SakZbYMSy08zthER446WCeRCmzzook/Scxk+Mn3WeOyMmJsXR1zXfoD7plogXvR4nJPWpawrjl13hVZ1XCj6DszYdeIuVdonMYh3zn0TToAB/4xaNKev1IOAaU08exxD/DKWBZEM3LbZGsXuH7F1jOySuagkl5+JeffpMTx0sRpHMzEzfdX/WOFJ/w9BR5kJjGB6KtBLic1Oy9JNCez21wC4Oo4DAPqK/W4cnDgUeYev2OkiyeX47WhDRSLES4iQcsWLJ4img";
-
-type BoxBody<E> = Pin<Box<dyn Body<Data = Bytes, Error = E> + Send + Sync + 'static>>;
-
-impl<B, E> Service<Request<B>> for TestService
-where
-    B: Body<Data = Bytes, Error = E> + Send + Sync + Unpin + 'static,
-    E: Debug + Send + Sync + 'static,
-{
-    type Response = Response<BoxBody<E>>;
-    type Error = Infallible;
-    type Future =
-        Pin<Box<dyn Future<Output = Result<Self::Response, Self::Error>> + Send + 'static>>;
-
-    fn call(&self, req: Request<B>) -> Self::Future {
-        Box::pin(async move {
-            let (parts, mut req_body) = req.into_parts();
-
-            let path = parts.uri.path();
-            match path {
-                "/echo-body" => {
-                    let body: BoxBody<E> = Box::pin(req_body);
-                    let res = Response::builder().body(body).unwrap();
-                    Ok(res)
-                }
-                _ => {
-                    let parts = path.trim_start_matches('/').split('/').collect::<Vec<_>>();
-
-                    // read everything from req body
-                    while let Some(_frame) = req_body.frame().await {
-                        // got frame, nice
-                    }
-
-                    let body: BoxBody<E> =
-                        Box::pin(http_body_util::Empty::new().map_err(|_| unreachable!()));
-
-                    if let ["status", code] = parts.as_slice() {
-                        let code = code.parse::<u16>().unwrap();
-                        let res = Response::builder().status(code).body(body).unwrap();
-                        debug!("Replying with {:?} {:?}", res.status(), res.headers());
-                        Ok(res)
-                    } else if let ["repeat-4k-blocks", repeat] = parts.as_slice() {
-                        let repeat = repeat.parse::<usize>().unwrap();
-
-                        // TODO: custom impl of the Body trait to avoid channel overhead
-                        let (tx, rx) = mpsc::channel::<Result<Frame<Bytes>, E>>(1);
-
-                        tokio::spawn(async move {
-                            let block = Bytes::copy_from_slice(BLOCK);
-                            for _ in 0..repeat {
-                                let frame = Frame::data(block.clone());
-                                let _ = tx.send(Ok(frame)).await;
-                            }
-                        });
-
-                        let rx = ReceiverStream::new(rx);
-                        let body: BoxBody<E> = Box::pin(StreamBody::new(rx));
-                        let res = Response::builder().body(body).unwrap();
-                        Ok(res)
-                    } else if let ["stream-file", name] = parts.as_slice() {
-                        let name = name.to_string();
-
-                        // TODO: custom impl of the Body trait to avoid channel overhead
-                        // stream 64KB blocks of the file
-                        let (tx, rx) = mpsc::channel::<Result<Frame<Bytes>, E>>(1);
-                        tokio::spawn(async move {
-                            let mut file =
-                                tokio::fs::File::open(format!("/tmp/stream-file/{name}"))
-                                    .await
-                                    .unwrap();
-                            let mut buf = vec![0u8; 64 * 1024];
-                            while let Ok(n) = file.read(&mut buf).await {
-                                if n == 0 {
-                                    break;
-                                }
-                                let frame = Frame::data(Bytes::copy_from_slice(&buf[..n]));
-                                let _ = tx.send(Ok(frame)).await;
-                            }
-                        });
-
-                        let rx = ReceiverStream::new(rx);
-                        let body: BoxBody<E> = Box::pin(StreamBody::new(rx));
-                        let res = Response::builder().body(body).unwrap();
-                        Ok(res)
-                    } else if parts.as_slice().is_empty() {
-                        let body = "it's less dire to lose, than to lose oneself".to_string();
-                        let body: BoxBody<E> = Box::pin(body.map_err(|_| unreachable!()));
-                        let res = Response::builder().status(200).body(body).unwrap();
-                        Ok(res)
-                    } else {
-                        // return a 404
-                        let body = "404 Not Found".to_string();
-                        let body: BoxBody<E> = Box::pin(body.map_err(|_| unreachable!()));
-                        let res = Response::builder().status(404).body(body).unwrap();
-                        Ok(res)
-                    }
-                }
-            }
-        })
-    }
-}
+mod service;
 
 #[tokio::main(flavor = "current_thread")]
 async fn main() {
-    let port = std::env::var("PORT").unwrap_or("0".to_string());
-    let addr = std::env::var("ADDR").unwrap_or("127.0.0.1".to_string());
-    let ln = tokio::net::TcpListener::bind(format!("{addr}:{port}"))
-        .await
-        .unwrap();
-    let upstream_addr = ln.local_addr().unwrap();
-    println!("I listen on {upstream_addr}");
-
-    #[derive(Debug, Clone, Copy)]
-    #[allow(clippy::upper_case_acronyms)]
-    enum Proto {
-        H1,
-        H2C,
-        TLS,
-    }
+    let settings = Settings::from_env().unwrap();
+    let ln = TcpListener::bind(settings.listen_addr).await.unwrap();
+    let listen_addr = ln.local_addr().unwrap();
+    settings.print_listen_line(listen_addr);
 
-    let proto = match std::env::var("PROTO").unwrap_or("h1".to_string()).as_str() {
-        // plaintext HTTP/1.1
-        "h1" => Proto::H1,
-        // HTTP/2 with prior knowledge
-        "h2c" => Proto::H2C,
-        // TLS with ALPN
-        "tls" => Proto::TLS,
-        _ => panic!("PROTO must be one of 'h1', 'h2c', or 'tls'"),
-    };
-    println!("Using {proto:?} (use PROTO=[h1,h2c,tls])");
-
-    match proto {
+    match settings.proto {
         Proto::TLS => {
-            let certified_key =
-                rcgen::generate_simple_self_signed(vec!["localhost".to_string()]).unwrap();
-            let crt = certified_key.cert.der();
-            let key = certified_key.key_pair.serialize_der();
-
-            let mut server_config = ServerConfig::builder()
-                .with_no_client_auth()
-                .with_single_cert(
-                    vec![crt.clone()],
-                    PrivatePkcs8KeyDer::from(key.clone()).into(),
-                )
-                .unwrap();
-            server_config.alpn_protocols = vec![b"h2".to_vec(), b"http/1.1".to_vec()];
-
+            let server_config = Settings::gen_rustls_server_config().unwrap();
             let acceptor = tokio_rustls::TlsAcceptor::from(Arc::new(server_config));
 
             while let Ok((stream, _)) = ln.accept().await {
@@ -202,7 +51,7 @@ async fn main() {
                 tokio::spawn(async move {
                     let mut builder = auto::Builder::new(TokioExecutor::new());
 
-                    match proto {
+                    match settings.proto {
                         Proto::H1 => {
                             builder = builder.http1_only();
                         }
diff --git a/crates/httpwg-hyper/src/service.rs b/crates/httpwg-hyper/src/service.rs
new file mode 100644
index 00000000..54b4aa0c
--- /dev/null
+++ b/crates/httpwg-hyper/src/service.rs
@@ -0,0 +1,128 @@
+use http_body_util::{BodyExt, StreamBody};
+use httpwg_harness::SAMPLE_4K_BLOCK;
+use tokio::io::AsyncReadExt;
+
+use std::{convert::Infallible, fmt::Debug, pin::Pin};
+use tokio::sync::mpsc;
+
+use bytes::Bytes;
+use futures::Future;
+use hyper::{
+    body::{Body, Frame},
+    service::Service,
+    Request, Response,
+};
+use tokio_stream::wrappers::ReceiverStream;
+use tracing::debug;
+
+type BoxBody<E> = Pin<Box<dyn Body<Data = Bytes, Error = E> + Send + Sync + 'static>>;
+
+pub(super) struct TestService;
+
+impl<B, E> Service<Request<B>> for TestService
+where
+    B: Body<Data = Bytes, Error = E> + Send + Sync + Unpin + 'static,
+    E: Debug + Send + Sync + 'static,
+{
+    type Response = Response<BoxBody<E>>;
+    type Error = Infallible;
+    type Future =
+        Pin<Box<dyn Future<Output = Result<Self::Response, Self::Error>> + Send + 'static>>;
+
+    fn call(&self, req: Request<B>) -> Self::Future {
+        Box::pin(async move {
+            let (parts, mut req_body) = req.into_parts();
+
+            let path = parts.uri.path();
+            match path {
+                "/echo-body" => {
+                    let body: BoxBody<E> = Box::pin(req_body);
+                    let res = Response::builder().body(body).unwrap();
+                    Ok(res)
+                }
+                _ => {
+                    let parts = path.trim_start_matches('/').split('/').collect::<Vec<_>>();
+
+                    let body: BoxBody<E> =
+                        Box::pin(http_body_util::Empty::new().map_err(|_| unreachable!()));
+
+                    if let ["status", code] = parts.as_slice() {
+                        // drain body
+                        while let Some(_frame) = req_body.frame().await {}
+
+                        let code = code.parse::<u16>().unwrap();
+                        let res = Response::builder().status(code).body(body).unwrap();
+                        debug!("Replying with {:?} {:?}", res.status(), res.headers());
+                        Ok(res)
+                    } else if let ["repeat-4k-blocks", repeat] = parts.as_slice() {
+                        // drain body
+                        while let Some(_frame) = req_body.frame().await {}
+
+                        let repeat = repeat.parse::<usize>().unwrap();
+
+                        // TODO: custom impl of the Body trait to avoid channel overhead
+                        let (tx, rx) = mpsc::channel::<Result<Frame<Bytes>, E>>(1);
+
+                        tokio::spawn(async move {
+                            let block = Bytes::copy_from_slice(SAMPLE_4K_BLOCK);
+                            for _ in 0..repeat {
+                                let frame = Frame::data(block.clone());
+                                let _ = tx.send(Ok(frame)).await;
+                            }
+                        });
+
+                        let rx = ReceiverStream::new(rx);
+                        let body: BoxBody<E> = Box::pin(StreamBody::new(rx));
+                        let res = Response::builder().body(body).unwrap();
+                        Ok(res)
+                    } else if let ["stream-file", name] = parts.as_slice() {
+                        // drain body
+                        while let Some(_frame) = req_body.frame().await {}
+
+                        let name = name.to_string();
+
+                        // TODO: custom impl of the Body trait to avoid channel overhead
+                        // stream 64KB blocks of the file
+                        let (tx, rx) = mpsc::channel::<Result<Frame<Bytes>, E>>(1);
+                        tokio::spawn(async move {
+                            let mut file =
+                                tokio::fs::File::open(format!("/tmp/stream-file/{name}"))
+                                    .await
+                                    .unwrap();
+                            let mut buf = vec![0u8; 64 * 1024];
+                            while let Ok(n) = file.read(&mut buf).await {
+                                if n == 0 {
+                                    break;
+                                }
+                                let frame = Frame::data(Bytes::copy_from_slice(&buf[..n]));
+                                let _ = tx.send(Ok(frame)).await;
+                            }
+                        });
+
+                        let rx = ReceiverStream::new(rx);
+                        let body: BoxBody<E> = Box::pin(StreamBody::new(rx));
+                        let res = Response::builder().body(body).unwrap();
+                        Ok(res)
+                    } else if parts.as_slice().is_empty() {
+                        // drain body
+                        while let Some(_frame) = req_body.frame().await {}
+
+                        let body = "it's less dire to lose, than to lose oneself".to_string();
+                        let body: BoxBody<E> = Box::pin(body.map_err(|_| unreachable!()));
+                        let res = Response::builder().status(200).body(body).unwrap();
+                        Ok(res)
+                    } else {
+                        // drain body
+                        while let Some(_frame) = req_body.frame().await {}
+
+                        // return a 404
+                        let body = "404 Not Found".to_string();
+                        let body: BoxBody<E> = Box::pin(body.map_err(|_| unreachable!()));
+                        let res = Response::builder().status(404).body(body).unwrap();
+                        Ok(res)
+                    }
+                }
+            }
+        })
+    }
+}
diff --git a/crates/httpwg-loona/Cargo.toml b/crates/httpwg-loona/Cargo.toml
index 054583ac..44d386e2 100644
--- a/crates/httpwg-loona/Cargo.toml
+++ b/crates/httpwg-loona/Cargo.toml
@@ -8,18 +8,6 @@ description = """
 A reference HTTP 1+2 server for httpwg, powered by loona
 """
 
-[lib]
-name = "httpwg_loona"
-path = "src/lib.rs"
-
-[[bin]]
-name = "httpwg-loona"
-path = "src/main.rs"
-
-[[bench]]
-name = "h2load"
-harness = false
-
 [dependencies]
 color-eyre = "0.6.3"
 loona = { version = "0.3.0", path = "../loona" }
@@ -32,6 +20,8 @@ b-x = { version = "1.0.0", path = "../b-x" }
 rcgen = { version = "0.13.1", default-features = false, features = ["aws_lc_rs"] }
 tokio-rustls = "0.26.0"
 ktls = "6.0.0"
+httpwg-harness = { version = "0.1.0", path = "../httpwg-harness" }
+socket2 = "0.5.7"
 
 [dev-dependencies]
 codspeed-criterion-compat = "2.6.0"
diff --git a/crates/httpwg-loona/benches/h2load.rs b/crates/httpwg-loona/benches/h2load.rs
deleted file mode 100644
index 1997fc5b..00000000
--- a/crates/httpwg-loona/benches/h2load.rs
+++ /dev/null
@@ -1,17 +0,0 @@
-use codspeed_criterion_compat::{criterion_group, criterion_main, Criterion};
-use httpwg_loona::{Mode, Proto};
-
-pub fn h2load(c: &mut Criterion) {
-    c.bench_function("h2load", |b| {
-        b.iter_with_setup(
-            || {},
-            |()| {
-                buffet::bufpool::initialize_allocator_with_num_bufs(64 * 1024).unwrap();
-                httpwg_loona::do_main("127.0.0.1".to_string(), 0, Proto::H2, Mode::H2Load);
-            },
-        )
-    });
-}
-
-criterion_group!(benches, h2load);
-criterion_main!(benches);
diff --git a/crates/httpwg-loona/src/driver.rs b/crates/httpwg-loona/src/driver.rs
new file mode 100644
index 00000000..9179c05f
--- /dev/null
+++ b/crates/httpwg-loona/src/driver.rs
@@ -0,0 +1,158 @@
+use b_x::{BxForResults, BX};
+use std::io::Write;
+
+use buffet::{Piece, RollMut};
+use loona::{
+    error::NeverError,
+    http::{self, StatusCode},
+    Body, BodyChunk, Encoder, ExpectResponseHeaders, Responder, Response, ResponseDone,
+    ServerDriver, SinglePieceBody,
+};
+
+pub(super) struct TestDriver;
+
+impl<OurEncoder> ServerDriver<OurEncoder> for TestDriver
+where
+    OurEncoder: Encoder,
+{
+    type Error = BX;
+
+    async fn handle(
+        &self,
+        req: loona::Request,
+        req_body: &mut impl Body,
+        mut res: Responder<OurEncoder, ExpectResponseHeaders>,
+    ) -> Result<Responder<OurEncoder, ResponseDone>, Self::Error> {
+        // if the client sent `expect: 100-continue`, we must send a 100 status code
+        if let Some(h) = req.headers.get(http::header::EXPECT) {
+            if &h[..] == b"100-continue" {
+                res.write_interim_response(Response {
+                    status: StatusCode::CONTINUE,
+                    ..Default::default()
+                })
+                .await?;
+            }
+        }
+
+        let res = match req.uri.path() {
+            "/echo-body" => res
+                .write_final_response_with_body(
+                    Response {
+                        status: StatusCode::OK,
+                        ..Default::default()
+                    },
+                    req_body,
+                )
+                .await
+                .bx()?,
+            "/stream-big-body" => {
+                // then read the full request body
+                let mut req_body_len = 0;
+                loop {
+                    let chunk = req_body.next_chunk().await.bx()?;
+                    match chunk {
+                        BodyChunk::Done { trailers } => {
+                            // yey
+                            if let Some(trailers) = trailers {
+                                tracing::debug!(trailers_len = %trailers.len(), "received trailers");
+                            }
+                            break;
+                        }
+                        BodyChunk::Chunk(chunk) => {
+                            req_body_len += chunk.len();
+                        }
+                    }
+                }
+                tracing::debug!(%req_body_len, "read request body");
+
+                let mut roll = RollMut::alloc().bx()?;
+                for _ in 0..256 {
+                    roll.write_all("this is a big chunk".as_bytes()).bx()?;
+                }
+
+                struct RepeatBody {
+                    piece: Piece,
+                    n: usize,
+                    written: usize,
+                }
+
+                impl std::fmt::Debug for RepeatBody {
+                    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+                        f.debug_struct("RepeatBody")
+                            .field("piece_len", &self.piece.len())
+                            .field("n", &self.n)
+                            .field("written", &self.written)
+                            .finish()
+                    }
+                }
+
+                impl Body for RepeatBody {
+                    type Error = NeverError;
+
+                    fn content_len(&self) -> Option<u64> {
+                        Some(self.n as u64 * self.piece.len() as u64)
+                    }
+
+                    fn eof(&self) -> bool {
+                        self.written == self.n
+                    }
+
+                    async fn next_chunk(&mut self) -> Result<BodyChunk, Self::Error> {
+                        if self.eof() {
+                            return Ok(BodyChunk::Done { trailers: None });
+                        }
+
+                        let chunk = self.piece.clone();
+                        self.written += 1;
+                        Ok(BodyChunk::Chunk(chunk))
+                    }
+                }
+
+                res.write_final_response_with_body(
+                    Response {
+                        status: StatusCode::OK,
+                        ..Default::default()
+                    },
+                    &mut RepeatBody {
+                        piece: roll.take_all().into(),
+                        n: 128,
+                        written: 0,
+                    },
+                )
+                .await
+                .bx()?
+            }
+            _ => {
+                // then read the full request body
+                let mut req_body_len = 0;
+                loop {
+                    let chunk = req_body.next_chunk().await.bx()?;
+                    match chunk {
+                        BodyChunk::Done { trailers } => {
+                            // yey
+                            if let Some(trailers) = trailers {
+                                tracing::debug!(trailers_len = %trailers.len(), "received trailers");
+                            }
+                            break;
+                        }
+                        BodyChunk::Chunk(chunk) => {
+                            req_body_len += chunk.len();
+                        }
+                    }
+                }
+                tracing::debug!(%req_body_len, "read request body");
+
+                res.write_final_response_with_body(
+                    Response {
+                        status: StatusCode::OK,
+                        ..Default::default()
+                    },
+                    &mut SinglePieceBody::from("it's less dire to lose, than to lose oneself"),
+                )
+                .await
+                .bx()?
+            }
+        };
+        Ok(res)
+    }
+}
diff --git a/crates/httpwg-loona/src/lib.rs b/crates/httpwg-loona/src/lib.rs
deleted file mode 100644
index b5616857..00000000
--- a/crates/httpwg-loona/src/lib.rs
+++ /dev/null
@@ -1,330 +0,0 @@
-use b_x::{BxForResults, BX};
-use std::{cell::RefCell, io::Write, rc::Rc};
-use tokio::{process::Command, sync::oneshot};
-
-use buffet::{IntoHalves, Piece, RollMut};
-use loona::{
-    error::{NeverError, ServeError},
-    h2::types::H2ConnectionError,
-    http::{self, StatusCode},
-    Body, BodyChunk, Encoder, ExpectResponseHeaders, Responder, Response, ResponseDone,
-    ServerDriver, SinglePieceBody,
-};
-
-#[derive(Debug, Clone, Copy)]
-pub enum Proto {
-    H1,
-    H2,
-}
-
-/// Message sent when the server is ready to accept connections.
-#[derive(Debug)]
-pub struct Ready {
-    pub port: u16,
-}
-
-pub enum Mode {
-    /// Run the server
-    Server {
-        ready_tx: oneshot::Sender<Ready>,
-        cancel_rx: oneshot::Receiver<()>,
-    },
-    /// Run the server, run h2load against it, and report the results
-    H2Load,
-}
-
-pub fn do_main(addr: String, port: u16, proto: Proto, mode: Mode) {
-    let server_start = std::time::Instant::now();
-
-    let (ready_tx, cancel_rx, is_h2load) = match mode {
-        Mode::Server {
-            ready_tx,
-            cancel_rx,
-        } => (Some(ready_tx), Some(cancel_rx), false),
-        Mode::H2Load => (None, None, true),
-    };
-
-    let server_fut = async move {
-        let ln = buffet::net::TcpListener::bind(format!("{addr}:{port}").parse().unwrap())
-            .await
-            .unwrap();
-        let port = ln.local_addr().unwrap().port();
-
-        if let Some(ready_tx) = ready_tx {
-            ready_tx.send(Ready { port }).unwrap();
-        }
-
-        let child_fut = async move {
-            if is_h2load {
-                let mut child = Command::new("h2load")
-                    .arg("-n")
-                    .arg("2500")
-                    .arg("-c")
-                    .arg("10")
-                    .arg(format!("http://127.0.0.1:{}", port))
-                    .stdout(std::process::Stdio::null())
-                    .stderr(std::process::Stdio::null())
-                    .stdin(std::process::Stdio::null())
-                    .spawn()
-                    .unwrap();
-                child.wait().await.unwrap();
-            } else {
-                // wait forever?
-                loop {
-                    tokio::time::sleep(std::time::Duration::from_secs(1)).await;
-                }
-            }
-        };
-        let loop_fut = async move {
-            let num_conns = Rc::new(RefCell::new(0));
-            loop {
-                let num_conns = num_conns.clone();
-                tracing::debug!("Accepting...");
-                let before_accept = std::time::Instant::now();
-                let (stream, addr) = ln.accept().await.unwrap();
-
-                *num_conns.borrow_mut() += 1;
-                tracing::debug!(
-                    ?addr,
-                    "Accepted connection in {:?} ({:?} since start), total conns = {}",
-                    before_accept.elapsed(),
-                    server_start.elapsed(),
-                    num_conns.borrow()
-                );
-
-                let conn_fut = async move {
-                    struct DecrementOnDrop(Rc<RefCell<usize>>);
-                    impl Drop for DecrementOnDrop {
-                        fn drop(&mut self) {
-                            let mut num_conns = self.0.borrow_mut();
-                            *num_conns -= 1;
-                        }
-                    }
-                    let _guard = DecrementOnDrop(num_conns);
-
-                    let client_buf = RollMut::alloc().unwrap();
-                    let io = stream.into_halves();
-
-                    match proto {
-                        Proto::H1 => {
-                            let driver = TestDriver;
-                            let server_conf = Rc::new(loona::h1::ServerConf {
-                                ..Default::default()
-                            });
-
-                            if let Err(e) =
-                                loona::h1::serve(io, server_conf, client_buf, driver).await
-                            {
-                                tracing::warn!("http/1 server error: {e:?}");
-                            }
-                            tracing::debug!("http/1 server done");
-                        }
-                        Proto::H2 => {
-                            let driver = Rc::new(TestDriver);
-                            let server_conf = Rc::new(loona::h2::ServerConf {
-                                ..Default::default()
-                            });
-
-                            if let Err(e) =
-                                loona::h2::serve(io, server_conf, client_buf, driver).await
-                            {
-                                let mut should_ignore = false;
-                                match &e {
-                                    ServeError::H2ConnectionError(
-                                        H2ConnectionError::WriteError(e),
-                                    ) => {
-                                        if e.kind() == std::io::ErrorKind::BrokenPipe {
-                                            should_ignore = true;
-                                        }
-                                    }
-                                    _ => {
-                                        // okay
-                                    }
-                                }
-
-                                if !should_ignore {
-                                    tracing::warn!("http/2 server error: {e:?}");
-                                }
-                            }
-                            tracing::debug!("http/2 server done");
-                        }
-                    }
-                };
-
-                let before_spawn = std::time::Instant::now();
-                buffet::spawn(conn_fut);
-                tracing::debug!("spawned connection in {:?}", before_spawn.elapsed());
-            }
-        };
-
-        tokio::select! {
-            _ = child_fut => {
-
-            },
-            _ = loop_fut => {},
-        }
-    };
-
-    if let Some(cancel_rx) = cancel_rx {
-        let cancellable_server_fut = async move {
-            tokio::select! {
-                _ = server_fut => {},
-                _ = cancel_rx => {
-                    tracing::info!("Cancelled");
-                }
-            }
-        };
-
-        buffet::start(cancellable_server_fut);
-    } else {
-        buffet::start(server_fut);
-    }
-}
-
-struct TestDriver;
-
-impl<OurEncoder> ServerDriver<OurEncoder> for TestDriver
-where
-    OurEncoder: Encoder,
-{
-    type Error = BX;
-
-    async fn handle(
-        &self,
-        req: loona::Request,
-        req_body: &mut impl Body,
-        mut res: Responder<OurEncoder, ExpectResponseHeaders>,
-    ) -> Result<Responder<OurEncoder, ResponseDone>, Self::Error> {
-        // if the client sent `expect: 100-continue`, we must send a 100 status code
-        if let Some(h) = req.headers.get(http::header::EXPECT) {
-            if &h[..] == b"100-continue" {
-                res.write_interim_response(Response {
-                    status: StatusCode::CONTINUE,
-                    ..Default::default()
-                })
-                .await?;
-            }
-        }
-
-        let res = match req.uri.path() {
-            "/echo-body" => res
-                .write_final_response_with_body(
-                    Response {
-                        status: StatusCode::OK,
-                        ..Default::default()
-                    },
-                    req_body,
-                )
-                .await
-                .bx()?,
-            "/stream-big-body" => {
-                // then read the full request body
-                let mut req_body_len = 0;
-                loop {
-                    let chunk = req_body.next_chunk().await.bx()?;
-                    match chunk {
-                        BodyChunk::Done { trailers } => {
-                            // yey
-                            if let Some(trailers) = trailers {
-                                tracing::debug!(trailers_len = %trailers.len(), "received trailers");
-                            }
-                            break;
-                        }
-                        BodyChunk::Chunk(chunk) => {
-                            req_body_len += chunk.len();
-                        }
-                    }
-                }
-                tracing::debug!(%req_body_len, "read request body");
-
-                let mut roll = RollMut::alloc().bx()?;
-                for _ in 0..256 {
-                    roll.write_all("this is a big chunk".as_bytes()).bx()?;
-                }
-
-                struct RepeatBody {
-                    piece: Piece,
-                    n: usize,
-                    written: usize,
-                }
-
-                impl std::fmt::Debug for RepeatBody {
-                    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
-                        f.debug_struct("RepeatBody")
-                            .field("piece_len", &self.piece.len())
-                            .field("n", &self.n)
-                            .field("written", &self.written)
-                            .finish()
-                    }
-                }
-
-                impl Body for RepeatBody {
-                    type Error = NeverError;
-
-                    fn content_len(&self) -> Option<u64> {
-                        Some(self.n as u64 * self.piece.len() as u64)
-                    }
-
-                    fn eof(&self) -> bool {
-                        self.written == self.n
-                    }
-
-                    async fn next_chunk(&mut self) -> Result<BodyChunk, Self::Error> {
-                        if self.eof() {
-                            return Ok(BodyChunk::Done { trailers: None });
-                        }
-
-                        let chunk = self.piece.clone();
-                        self.written += 1;
-                        Ok(BodyChunk::Chunk(chunk))
-                    }
-                }
-
-                res.write_final_response_with_body(
-                    Response {
-                        status: StatusCode::OK,
-                        ..Default::default()
-                    },
-                    &mut RepeatBody {
-                        piece: roll.take_all().into(),
-                        n: 128,
-                        written: 0,
-                    },
-                )
-                .await
-                .bx()?
-            }
-            _ => {
-                // then read the full request body
-                let mut req_body_len = 0;
-                loop {
-                    let chunk = req_body.next_chunk().await.bx()?;
-                    match chunk {
-                        BodyChunk::Done { trailers } => {
-                            // yey
-                            if let Some(trailers) = trailers {
-                                tracing::debug!(trailers_len = %trailers.len(), "received trailers");
-                            }
-                            break;
-                        }
-                        BodyChunk::Chunk(chunk) => {
-                            req_body_len += chunk.len();
-                        }
-                    }
-                }
-                tracing::debug!(%req_body_len, "read request body");
-
-                res.write_final_response_with_body(
-                    Response {
-                        status: StatusCode::OK,
-                        ..Default::default()
-                    },
-                    &mut SinglePieceBody::from("it's less dire to lose, than to lose oneself"),
-                )
-                .await
-                .bx()?
-            }
-        };
-        Ok(res)
-    }
-}
diff --git a/crates/httpwg-loona/src/main.rs b/crates/httpwg-loona/src/main.rs
index d80ca251..b8301aab 100644
--- a/crates/httpwg-loona/src/main.rs
+++ b/crates/httpwg-loona/src/main.rs
@@ -1,43 +1,143 @@
-use httpwg_loona::{Proto, Ready};
+use driver::TestDriver;
+use httpwg_harness::{Proto, Settings};
+use ktls::CorkStream;
+use std::{
+    mem::ManuallyDrop,
+    os::fd::{AsRawFd, FromRawFd, IntoRawFd},
+    rc::Rc,
+    sync::Arc,
+};
+use tokio_rustls::TlsAcceptor;
+
+use buffet::{
+    net::{TcpListener, TcpStream},
+    IntoHalves, RollMut,
+};
+use loona::{
+    error::ServeError,
+    h1,
+    h2::{self, types::H2ConnectionError},
+};
 use tracing::Level;
 use tracing_subscriber::{filter::Targets, layer::SubscriberExt, util::SubscriberInitExt};
 
+mod driver;
+
 fn main() {
     setup_tracing_and_error_reporting();
+    buffet::start(real_main());
+}
 
-    let port: u16 = std::env::var("PORT")
-        .unwrap_or("8001".to_string())
-        .parse()
-        .unwrap();
-    let addr = std::env::var("ADDR").unwrap_or_else(|_| "127.0.0.1".to_string());
-    let proto = match std::env::var("TEST_PROTO")
-        .unwrap_or("h1".to_string())
-        .as_str()
-    {
-        "h1" => Proto::H1,
-        "h2" => Proto::H2,
-        _ => panic!("TEST_PROTO must be either 'h1' or 'h2'"),
-    };
-    eprintln!("Using {proto:?} protocol (export TEST_PROTO=h1 or TEST_PROTO=h2 to override)");
-
-    let (ready_tx, ready_rx) = tokio::sync::oneshot::channel();
-    let (cancel_tx, cancel_rx) = tokio::sync::oneshot::channel();
-    std::mem::forget(cancel_tx);
-
-    std::thread::spawn(move || {
-        let ready: Ready = ready_rx.blocking_recv().unwrap();
-        eprintln!("I listen on {}", ready.port);
-    });
-
-    httpwg_loona::do_main(
-        addr,
-        port,
-        proto,
-        httpwg_loona::Mode::Server {
-            ready_tx,
-            cancel_rx,
-        },
-    );
+async fn real_main() {
+    let settings = Settings::from_env().unwrap();
+    let ln = TcpListener::bind(settings.listen_addr).await.unwrap();
+    let listen_addr = ln.local_addr().unwrap();
+    settings.print_listen_line(listen_addr);
+
+    loop {
+        tracing::debug!("Accepting...");
+        let (stream, _addr) = ln.accept().await.unwrap();
+
+        let conn_fut = async move {
+            let client_buf = RollMut::alloc().unwrap();
+
+            match settings.proto {
+                Proto::H1 => {
+                    let driver = TestDriver;
+                    let server_conf = Rc::new(h1::ServerConf {
+                        ..Default::default()
+                    });
+                    let io = stream.into_halves();
+
+                    if let Err(e) = h1::serve(io, server_conf, client_buf, driver).await {
+                        tracing::warn!("http/1 server error: {e:?}");
+                    }
+                    tracing::debug!("http/1 server done");
+                }
+                Proto::H2C => {
+                    let driver = Rc::new(TestDriver);
+                    let server_conf = Rc::new(h2::ServerConf {
+                        ..Default::default()
+                    });
+                    let io = stream.into_halves();
+
+                    if let Err(e) = h2::serve(io, server_conf, client_buf, driver).await {
+                        let mut should_ignore = false;
+                        match &e {
+                            ServeError::H2ConnectionError(H2ConnectionError::WriteError(e)) => {
+                                if e.kind() == std::io::ErrorKind::BrokenPipe {
+                                    should_ignore = true;
+                                }
+                            }
+                            _ => {
+                                // okay
+                            }
+                        }
+
+                        if !should_ignore {
+                            tracing::warn!("http/2 server error: {e:?}");
+                        }
+                    }
+                    tracing::debug!("http/2 server done");
+                }
+                #[cfg(not(target_os = "linux"))]
+                Proto::TLS => {
+                    panic!("TLS support is provided through kTLS, which we only support the Linux variant of right now");
+                }
+
+                #[cfg(target_os = "linux")]
+                Proto::TLS => {
+                    let mut server_config = Settings::gen_rustls_server_config().unwrap();
+                    server_config.enable_secret_extraction = true;
+                    let driver = TestDriver;
+                    let h1_conf = Rc::new(h1::ServerConf::default());
+                    let h2_conf = Rc::new(h2::ServerConf::default());
+
+                    // until we come up with `loona-rustls`, we need to temporarily go through a
+                    // tokio TcpStream
+                    let acceptor = TlsAcceptor::from(Arc::new(server_config));
+                    let stream = unsafe { std::net::TcpStream::from_raw_fd(stream.into_raw_fd()) };
+                    stream.set_nonblocking(true).unwrap();
+                    let stream = tokio::net::TcpStream::from_std(stream)?;
+                    let stream = CorkStream::new(stream);
+                    let stream = acceptor.accept(stream).await?;
+
+                    let is_h2 = matches!(stream.get_ref().1.alpn_protocol(), Some(b"h2"));
+                    tracing::debug!(%is_h2, "Performed TLS handshake");
+
+                    let stream = ktls::config_ktls_server(stream).await?;
+
+                    tracing::debug!("Set up kTLS");
+                    let (drained, stream) = stream.into_raw();
+                    let drained = drained.unwrap_or_default();
+                    tracing::debug!("{} bytes already decoded by rustls", drained.len());
+
+                    // and back to a buffet TcpStream
+                    let stream = stream.to_uring_tcp_stream()?;
+
+                    let mut client_buf = RollMut::alloc()?;
+                    client_buf.put(&drained[..])?;
+
+                    if is_h2 {
+                        tracing::info!("Using HTTP/2");
+                        h2::serve(stream.into_halves(), h2_conf, client_buf, Rc::new(driver))
+                            .await
+                            .map_err(|e| eyre::eyre!("h2 server error: {e:?}"))?;
+                    } else {
+                        tracing::info!("Using HTTP/1.1");
+                        h1::serve(stream.into_halves(), h1_conf, client_buf, driver)
+                            .await
+                            .map_err(|e| eyre::eyre!("h1 server error: {e:?}"))?;
+                    }
+                }
+            }
+            Ok::<_, eyre::Report>(())
+        };
+
+        let before_spawn = std::time::Instant::now();
+        buffet::spawn(conn_fut);
+        tracing::debug!("spawned connection in {:?}", before_spawn.elapsed());
+    }
 }
 
 fn setup_tracing_and_error_reporting() {
@@ -64,3 +164,21 @@ fn setup_tracing_and_error_reporting() {
         .with(fmt_layer)
         .init();
 }
+
+pub trait ToUringTcpStream {
+    fn to_uring_tcp_stream(self) -> std::io::Result<TcpStream>;
+}
+
+impl ToUringTcpStream for tokio::net::TcpStream {
+    fn to_uring_tcp_stream(self) -> std::io::Result<TcpStream> {
+        {
+            let sock = ManuallyDrop::new(unsafe { socket2::Socket::from_raw_fd(self.as_raw_fd()) });
+            // tokio needs the socket to be "non-blocking" (as in: return EAGAIN)
+            // buffet needs it to be "blocking" (as in: let io_uring do the op async)
+            sock.set_nonblocking(false)?;
+        }
+        let stream = unsafe { TcpStream::from_raw_fd(self.as_raw_fd()) };
+        std::mem::forget(self);
+        Ok(stream)
+    }
+}
diff --git a/crates/loona/Cargo.toml b/crates/loona/Cargo.toml
index c50a4d34..17c4fe15 100644
--- a/crates/loona/Cargo.toml
+++ b/crates/loona/Cargo.toml
@@ -73,6 +73,7 @@ codspeed-criterion-compat = "2.6.0"
 itoa = "1.0.11"
 rcgen = { version = "0.13.1", default-features = false, features = ["aws_lc_rs"] }
 socket2 = "0.5.7"
+httpwg-harness = { path = "../httpwg-harness" }
 
 [target.'cfg(target_os = "linux")'.dev-dependencies]
 ktls = "6.0.0"
diff --git a/crates/loona/tests/testbed.rs b/crates/loona/tests/testbed.rs
index 0e1fc431..1583eaf9 100644
--- a/crates/loona/tests/testbed.rs
+++ b/crates/loona/tests/testbed.rs
@@ -48,8 +48,13 @@ pub async fn start() -> b_x::Result<(SocketAddr, impl Any)> {
         let stdout = BufReader::new(stdout);
         let mut lines = stdout.lines();
         while let Some(line) = lines.next_line().await.unwrap() {
-            if let Some(rest) = line.strip_prefix("I listen on ") {
-                let addr = rest.parse::<SocketAddr>().unwrap();
+            if let Some(rest) = line.strip_prefix("🌎🦊👉 ") {
+                let addr = rest
+                    .split_whitespace()
+                    .next()
+                    .unwrap()
+                    .parse::<SocketAddr>()
+                    .unwrap();
                 if let Some(addr_tx) = addr_tx.take() {
                     addr_tx.send(addr).unwrap();
                 }

From 7e15f28093a0f7524ed78efa12b97a37ff431d4c Mon Sep 17 00:00:00 2001
From: Amos Wenger <amos@bearcove.net>
Date: Fri, 30 Aug 2024 19:09:07 +0200
Subject: [PATCH 24/24] Implement standard endpoints for httpwg-loona (save for
 /stream-file)

---
 crates/httpwg-hyper/src/service.rs | 174 ++++++++++++-----------
 crates/httpwg-loona/src/driver.rs  | 214 +++++++++++++++--------------
 2 files changed, 204 insertions(+), 184 deletions(-)

diff --git a/crates/httpwg-hyper/src/service.rs b/crates/httpwg-hyper/src/service.rs
index 54b4aa0c..0091bc56 100644
--- a/crates/httpwg-hyper/src/service.rs
+++ b/crates/httpwg-hyper/src/service.rs
@@ -1,3 +1,13 @@
+//! //! This service provides the following routes:
+//!
+//! - `/echo-body`: Echoes back the request body.
+//! - `/status/{code}`: Returns a response with the specified status code.
+//! - `/repeat-4k-blocks/{repeat}`: Streams the specified number of 4KB blocks.
+//! - `/stream-file/{name}`: Streams the contents of a file from
+//!   `/tmp/stream-file/`.
+//! - `/`: Returns a default message.
+//! - Any other path: Returns a 404 Not Found response.
+
 use http_body_util::{BodyExt, StreamBody};
 use httpwg_harness::SAMPLE_4K_BLOCK;
 use tokio::io::AsyncReadExt;
@@ -34,93 +44,89 @@ where
             let (parts, mut req_body) = req.into_parts();
 
             let path = parts.uri.path();
-            match path {
-                "/echo-body" => {
-                    let body: BoxBody<E> = Box::pin(req_body);
+            let parts = path.trim_start_matches('/').split('/').collect::<Vec<_>>();
+
+            if let ["echo-body"] = parts.as_slice() {
+                let body: BoxBody<E> = Box::pin(req_body);
+                let res = Response::builder().body(body).unwrap();
+                Ok(res)
+            } else {
+                let body: BoxBody<E> =
+                    Box::pin(http_body_util::Empty::new().map_err(|_| unreachable!()));
+
+                if let ["status", code] = parts.as_slice() {
+                    // drain body
+                    while let Some(_frame) = req_body.frame().await {}
+
+                    let code = code.parse::<u16>().unwrap();
+                    let res = Response::builder().status(code).body(body).unwrap();
+                    debug!("Replying with {:?} {:?}", res.status(), res.headers());
+                    Ok(res)
+                } else if let ["repeat-4k-blocks", repeat] = parts.as_slice() {
+                    // drain body
+                    while let Some(_frame) = req_body.frame().await {}
+
+                    let repeat = repeat.parse::<usize>().unwrap();
+
+                    // TODO: custom impl of the Body trait to avoid channel overhead
+                    let (tx, rx) = mpsc::channel::<Result<Frame<Bytes>, E>>(1);
+
+                    tokio::spawn(async move {
+                        let block = Bytes::copy_from_slice(SAMPLE_4K_BLOCK);
+                        for _ in 0..repeat {
+                            let frame = Frame::data(block.clone());
+                            let _ = tx.send(Ok(frame)).await;
+                        }
+                    });
+
+                    let rx = ReceiverStream::new(rx);
+                    let body: BoxBody<E> = Box::pin(StreamBody::new(rx));
                     let res = Response::builder().body(body).unwrap();
                     Ok(res)
-                }
-                _ => {
-                    let parts = path.trim_start_matches('/').split('/').collect::<Vec<_>>();
-
-                    let body: BoxBody<E> =
-                        Box::pin(http_body_util::Empty::new().map_err(|_| unreachable!()));
-
-                    if let ["status", code] = parts.as_slice() {
-                        // drain body
-                        while let Some(_frame) = req_body.frame().await {}
-
-                        let code = code.parse::<u16>().unwrap();
-                        let res = Response::builder().status(code).body(body).unwrap();
-                        debug!("Replying with {:?} {:?}", res.status(), res.headers());
-                        Ok(res)
-                    } else if let ["repeat-4k-blocks", repeat] = parts.as_slice() {
-                        // drain body
-                        while let Some(_frame) = req_body.frame().await {}
-
-                        let repeat = repeat.parse::<usize>().unwrap();
-
-                        // TODO: custom impl of the Body trait to avoid channel overhead
-                        let (tx, rx) = mpsc::channel::<Result<Frame<Bytes>, E>>(1);
-
-                        tokio::spawn(async move {
-                            let block = Bytes::copy_from_slice(SAMPLE_4K_BLOCK);
-                            for _ in 0..repeat {
-                                let frame = Frame::data(block.clone());
-                                let _ = tx.send(Ok(frame)).await;
+                } else if let ["stream-file", name] = parts.as_slice() {
+                    // drain body
+                    while let Some(_frame) = req_body.frame().await {}
+
+                    let name = name.to_string();
+
+                    // TODO: custom impl of the Body trait to avoid channel overhead
+                    // stream 64KB blocks of the file
+                    let (tx, rx) = mpsc::channel::<Result<Frame<Bytes>, E>>(1);
+                    tokio::spawn(async move {
+                        let mut file = tokio::fs::File::open(format!("/tmp/stream-file/{name}"))
+                            .await
+                            .unwrap();
+                        let mut buf = vec![0u8; 64 * 1024];
+                        while let Ok(n) = file.read(&mut buf).await {
+                            if n == 0 {
+                                break;
                             }
-                        });
-
-                        let rx = ReceiverStream::new(rx);
-                        let body: BoxBody<E> = Box::pin(StreamBody::new(rx));
-                        let res = Response::builder().body(body).unwrap();
-                        Ok(res)
-                    } else if let ["stream-file", name] = parts.as_slice() {
-                        // drain body
-                        while let Some(_frame) = req_body.frame().await {}
-
-                        let name = name.to_string();
-
-                        // TODO: custom impl of the Body trait to avoid channel overhead
-                        // stream 64KB blocks of the file
-                        let (tx, rx) = mpsc::channel::<Result<Frame<Bytes>, E>>(1);
-                        tokio::spawn(async move {
-                            let mut file =
-                                tokio::fs::File::open(format!("/tmp/stream-file/{name}"))
-                                    .await
-                                    .unwrap();
-                            let mut buf = vec![0u8; 64 * 1024];
-                            while let Ok(n) = file.read(&mut buf).await {
-                                if n == 0 {
-                                    break;
-                                }
-                                let frame = Frame::data(Bytes::copy_from_slice(&buf[..n]));
-                                let _ = tx.send(Ok(frame)).await;
-                            }
-                        });
-
-                        let rx = ReceiverStream::new(rx);
-                        let body: BoxBody<E> = Box::pin(StreamBody::new(rx));
-                        let res = Response::builder().body(body).unwrap();
-                        Ok(res)
-                    } else if parts.as_slice().is_empty() {
-                        // drain body
-                        while let Some(_frame) = req_body.frame().await {}
-
-                        let body = "it's less dire to lose, than to lose oneself".to_string();
-                        let body: BoxBody<E> = Box::pin(body.map_err(|_| unreachable!()));
-                        let res = Response::builder().status(200).body(body).unwrap();
-                        Ok(res)
-                    } else {
-                        // drain body
-                        while let Some(_frame) = req_body.frame().await {}
-
-                        // return a 404
-                        let body = "404 Not Found".to_string();
-                        let body: BoxBody<E> = Box::pin(body.map_err(|_| unreachable!()));
-                        let res = Response::builder().status(404).body(body).unwrap();
-                        Ok(res)
-                    }
+                            let frame = Frame::data(Bytes::copy_from_slice(&buf[..n]));
+                            let _ = tx.send(Ok(frame)).await;
+                        }
+                    });
+
+                    let rx = ReceiverStream::new(rx);
+                    let body: BoxBody<E> = Box::pin(StreamBody::new(rx));
+                    let res = Response::builder().body(body).unwrap();
+                    Ok(res)
+                } else if parts.as_slice().is_empty() {
+                    // drain body
+                    while let Some(_frame) = req_body.frame().await {}
+
+                    let body = "it's less dire to lose, than to lose oneself".to_string();
+                    let body: BoxBody<E> = Box::pin(body.map_err(|_| unreachable!()));
+                    let res = Response::builder().status(200).body(body).unwrap();
+                    Ok(res)
+                } else {
+                    // drain body
+                    while let Some(_frame) = req_body.frame().await {}
+
+                    // return a 404
+                    let body = "404 Not Found".to_string();
+                    let body: BoxBody<E> = Box::pin(body.map_err(|_| unreachable!()));
+                    let res = Response::builder().status(404).body(body).unwrap();
+                    Ok(res)
                 }
             }
         })
diff --git a/crates/httpwg-loona/src/driver.rs b/crates/httpwg-loona/src/driver.rs
index 9179c05f..b0a69550 100644
--- a/crates/httpwg-loona/src/driver.rs
+++ b/crates/httpwg-loona/src/driver.rs
@@ -1,12 +1,10 @@
 use b_x::{BxForResults, BX};
-use std::io::Write;
+use httpwg_harness::SAMPLE_4K_BLOCK;
 
-use buffet::{Piece, RollMut};
+use buffet::Piece;
 use loona::{
-    error::NeverError,
-    http::{self, StatusCode},
-    Body, BodyChunk, Encoder, ExpectResponseHeaders, Responder, Response, ResponseDone,
-    ServerDriver, SinglePieceBody,
+    error::NeverError, http::StatusCode, Body, BodyChunk, Encoder, ExpectResponseHeaders,
+    HeadersExt, Responder, Response, ResponseDone, ServerDriver, SinglePieceBody,
 };
 
 pub(super) struct TestDriver;
@@ -23,19 +21,23 @@ where
         req_body: &mut impl Body,
         mut res: Responder<OurEncoder, ExpectResponseHeaders>,
     ) -> Result<Responder<OurEncoder, ResponseDone>, Self::Error> {
-        // if the client sent `expect: 100-continue`, we must send a 100 status code
-        if let Some(h) = req.headers.get(http::header::EXPECT) {
-            if &h[..] == b"100-continue" {
-                res.write_interim_response(Response {
-                    status: StatusCode::CONTINUE,
-                    ..Default::default()
-                })
-                .await?;
-            }
+        if req.headers.expects_100_continue() {
+            res.write_interim_response(Response {
+                status: StatusCode::CONTINUE,
+                ..Default::default()
+            })
+            .await?;
         }
 
-        let res = match req.uri.path() {
-            "/echo-body" => res
+        let parts = req
+            .uri
+            .path()
+            .trim_start_matches('/')
+            .split('/')
+            .collect::<Vec<_>>();
+
+        let res = match parts.as_slice() {
+            ["echo-body"] => res
                 .write_final_response_with_body(
                     Response {
                         status: StatusCode::OK,
@@ -45,109 +47,62 @@ where
                 )
                 .await
                 .bx()?,
-            "/stream-big-body" => {
-                // then read the full request body
-                let mut req_body_len = 0;
-                loop {
-                    let chunk = req_body.next_chunk().await.bx()?;
-                    match chunk {
-                        BodyChunk::Done { trailers } => {
-                            // yey
-                            if let Some(trailers) = trailers {
-                                tracing::debug!(trailers_len = %trailers.len(), "received trailers");
-                            }
-                            break;
-                        }
-                        BodyChunk::Chunk(chunk) => {
-                            req_body_len += chunk.len();
-                        }
-                    }
-                }
-                tracing::debug!(%req_body_len, "read request body");
-
-                let mut roll = RollMut::alloc().bx()?;
-                for _ in 0..256 {
-                    roll.write_all("this is a big chunk".as_bytes()).bx()?;
-                }
-
-                struct RepeatBody {
-                    piece: Piece,
-                    n: usize,
-                    written: usize,
-                }
+            ["status", code] => {
+                drain_body(req_body).await?;
 
-                impl std::fmt::Debug for RepeatBody {
-                    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
-                        f.debug_struct("RepeatBody")
-                            .field("piece_len", &self.piece.len())
-                            .field("n", &self.n)
-                            .field("written", &self.written)
-                            .finish()
-                    }
-                }
-
-                impl Body for RepeatBody {
-                    type Error = NeverError;
-
-                    fn content_len(&self) -> Option<u64> {
-                        Some(self.n as u64 * self.piece.len() as u64)
-                    }
-
-                    fn eof(&self) -> bool {
-                        self.written == self.n
-                    }
-
-                    async fn next_chunk(&mut self) -> Result<BodyChunk, Self::Error> {
-                        if self.eof() {
-                            return Ok(BodyChunk::Done { trailers: None });
-                        }
-
-                        let chunk = self.piece.clone();
-                        self.written += 1;
-                        Ok(BodyChunk::Chunk(chunk))
-                    }
-                }
+                let code = code.parse::<u16>().bx()?;
+                res.write_final_response_with_body(
+                    Response {
+                        status: StatusCode::from_u16(code).bx()?,
+                        ..Default::default()
+                    },
+                    &mut (),
+                )
+                .await
+                .bx()?
+            }
+            ["repeat-4k-blocks", repeat] => {
+                drain_body(req_body).await?;
 
+                let repeat = repeat.parse::<usize>().bx()?;
                 res.write_final_response_with_body(
                     Response {
                         status: StatusCode::OK,
                         ..Default::default()
                     },
                     &mut RepeatBody {
-                        piece: roll.take_all().into(),
-                        n: 128,
+                        piece: SAMPLE_4K_BLOCK.into(),
+                        n: repeat,
                         written: 0,
                     },
                 )
                 .await
                 .bx()?
             }
+            ["stream-file", name] => {
+                drain_body(req_body).await?;
+                let _ = name;
+
+                res.write_final_response_with_body(
+                    Response {
+                        status: StatusCode::INTERNAL_SERVER_ERROR,
+                        ..Default::default()
+                    },
+                    &mut SinglePieceBody::from("/stream-file: not implemented yet"),
+                )
+                .await
+                .bx()?
+            }
             _ => {
-                // then read the full request body
-                let mut req_body_len = 0;
-                loop {
-                    let chunk = req_body.next_chunk().await.bx()?;
-                    match chunk {
-                        BodyChunk::Done { trailers } => {
-                            // yey
-                            if let Some(trailers) = trailers {
-                                tracing::debug!(trailers_len = %trailers.len(), "received trailers");
-                            }
-                            break;
-                        }
-                        BodyChunk::Chunk(chunk) => {
-                            req_body_len += chunk.len();
-                        }
-                    }
-                }
-                tracing::debug!(%req_body_len, "read request body");
+                drain_body(req_body).await?;
 
+                // return a 404
                 res.write_final_response_with_body(
                     Response {
-                        status: StatusCode::OK,
+                        status: StatusCode::NOT_FOUND,
                         ..Default::default()
                     },
-                    &mut SinglePieceBody::from("it's less dire to lose, than to lose oneself"),
+                    &mut SinglePieceBody::from("404 Not Found"),
                 )
                 .await
                 .bx()?
@@ -156,3 +111,62 @@ where
         Ok(res)
     }
 }
+
+async fn drain_body(body: &mut impl Body) -> Result<(), BX> {
+    let mut req_body_len = 0;
+    loop {
+        let chunk = body.next_chunk().await.bx()?;
+        match chunk {
+            BodyChunk::Done { trailers } => {
+                // yey
+                if let Some(trailers) = trailers {
+                    tracing::debug!(trailers_len = %trailers.len(), "received trailers");
+                }
+                break;
+            }
+            BodyChunk::Chunk(chunk) => {
+                req_body_len += chunk.len();
+            }
+        }
+    }
+    tracing::debug!(%req_body_len, "read request body");
+    Ok(())
+}
+
+struct RepeatBody {
+    piece: Piece,
+    n: usize,
+    written: usize,
+}
+
+impl std::fmt::Debug for RepeatBody {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        f.debug_struct("RepeatBody")
+            .field("piece_len", &self.piece.len())
+            .field("n", &self.n)
+            .field("written", &self.written)
+            .finish()
+    }
+}
+
+impl Body for RepeatBody {
+    type Error = NeverError;
+
+    fn content_len(&self) -> Option<u64> {
+        Some(self.n as u64 * self.piece.len() as u64)
+    }
+
+    fn eof(&self) -> bool {
+        self.written == self.n
+    }
+
+    async fn next_chunk(&mut self) -> Result<BodyChunk, Self::Error> {
+        if self.eof() {
+            return Ok(BodyChunk::Done { trailers: None });
+        }
+
+        let chunk = self.piece.clone();
+        self.written += 1;
+        Ok(BodyChunk::Chunk(chunk))
+    }
+}