diff --git a/command/src/config.rs b/command/src/config.rs index 56c4ba0d9..409d1b63b 100644 --- a/command/src/config.rs +++ b/command/src/config.rs @@ -159,6 +159,8 @@ pub const DEFAULT_MAX_COMMAND_BUFFER_SIZE: usize = 2_000_000; /// wether to avoid register cluster metrics in the local drain pub const DEFAULT_DISABLE_CLUSTER_METRICS: bool = false; +pub const MAX_LOOP_ITERATIONS: usize = 100000; + /// Number of TLS 1.3 tickets to send to a client when establishing a connection. /// The tickets allow the client to resume a session. This protects the client /// agains session tracking. Increases the number of getrandom syscalls, diff --git a/command/src/logging.rs b/command/src/logging.rs index ebaece98e..35c79a4e5 100644 --- a/command/src/logging.rs +++ b/command/src/logging.rs @@ -650,7 +650,6 @@ macro_rules! error_access { #[macro_export] macro_rules! warn { ($format:expr, $($arg:tt)*) => { - use time; log!($crate::logging::LogLevel::Warn, $format, "WARN", $($arg)*); }; ($format:expr) => { diff --git a/lib/src/protocol/kawa_h1/mod.rs b/lib/src/protocol/kawa_h1/mod.rs index a10958bb2..d1a42bf1b 100644 --- a/lib/src/protocol/kawa_h1/mod.rs +++ b/lib/src/protocol/kawa_h1/mod.rs @@ -14,7 +14,7 @@ use mio::{net::TcpStream, Interest, Token}; use rusty_ulid::Ulid; use time::{Duration, Instant}; -use sozu_command::proto::command::{Event, EventKind, ListenerType}; +use sozu_command::{proto::command::{Event, EventKind, ListenerType}, config::MAX_LOOP_ITERATIONS}; use crate::{ backends::{Backend, BackendError}, @@ -350,7 +350,7 @@ impl Http Http { - let kawa = &self.request_stream; + let kawa = &self.response_stream; parser::view( kawa.storage.used(), 16, @@ -1481,7 +1481,6 @@ impl Http SessionResult { let mut counter = 0; - let max_loop_iterations = 100000; if self.backend_connection_status.is_connecting() && !self.backend_readiness.event.is_empty() @@ -1510,7 +1509,7 @@ impl Http { - error!("Error connecting to backend: {}", connection_error) + warn!("Error connecting to backend: {}", connection_error) } } } else { @@ -1527,7 +1526,7 @@ impl Http Http { - error!("Error connecting to backend: {}", connection_error) + warn!("Error connecting to backend: {}", connection_error) } } } @@ -1626,10 +1625,10 @@ impl Http= MAX_LOOP_ITERATIONS { error!( "PROXY\thandling session {:?} went through {} iterations, there's a probable infinite loop bug, closing the connection", - self.frontend_token, max_loop_iterations + self.frontend_token, MAX_LOOP_ITERATIONS ); incr!("http.infinite_loop.error"); diff --git a/lib/src/protocol/kawa_h1/parser.rs b/lib/src/protocol/kawa_h1/parser.rs index 5a269a9d1..d168d8386 100644 --- a/lib/src/protocol/kawa_h1/parser.rs +++ b/lib/src/protocol/kawa_h1/parser.rs @@ -1,4 +1,4 @@ -use std::{fmt::{self, Write}, str::from_utf8_unchecked}; +use std::{cmp::min, fmt, str::from_utf8_unchecked}; use nom::{ bytes::{self, complete::take_while}, @@ -115,35 +115,41 @@ pub fn hostname_and_port(i: &[u8]) -> IResult<&[u8], (&[u8], Option<&[u8]>)> { } pub fn view(buf: &[u8], size: usize, points: &[usize]) -> String { - let mut view = String::new(); - let mut end = 0; - for (i, point) in points.iter().enumerate() { - let start = if end + size < *point { - view.push_str("... "); - point - size - } else { - end - }; - let stop = if i + 1 < points.len() { - points[i + 1] - } else { - buf.len() - }; - end = if point + size > stop { - stop - } else { - point + size - }; - for element in &buf[start..*point] { - let _ = view.write_fmt(format_args!("{element:02X} ")); - } - view.push_str("| "); - for element in &buf[*point..end] { - let _ = view.write_fmt(format_args!("{element:02X} ")); - } - } - if end < buf.len() { - view.push_str("...") - } - view + let len = buf.len(); + let (start, end) = match (points.first(), points.last()) { + (Some(start), Some(end)) => (min(*start, len), min(*end + size, len)), + _ => return "NO POINTS".to_string(), + }; + return format!("{:02X?}", &buf[start..end]); + // let mut view = String::new(); + // let mut end = 0; + // for (i, point) in points.iter().enumerate() { + // let start = if end + size < *point { + // view.push_str("... "); + // point - size + // } else { + // end + // }; + // let stop = if i + 1 < points.len() { + // points[i + 1] + // } else { + // buf.len() + // }; + // end = if point + size > stop { + // stop + // } else { + // point + size + // }; + // for element in &buf[start..*point] { + // let _ = view.write_fmt(format_args!("{element:02X} ")); + // } + // view.push_str("| "); + // for element in &buf[*point..end] { + // let _ = view.write_fmt(format_args!("{element:02X} ")); + // } + // } + // if end < buf.len() { + // view.push_str("...") + // } + // view } diff --git a/lib/src/protocol/pipe.rs b/lib/src/protocol/pipe.rs index 2430b918c..6399de035 100644 --- a/lib/src/protocol/pipe.rs +++ b/lib/src/protocol/pipe.rs @@ -2,6 +2,7 @@ use std::{cell::RefCell, net::SocketAddr, rc::Rc}; use mio::{net::TcpStream, Token}; use rusty_ulid::Ulid; +use sozu_command::config::MAX_LOOP_ITERATIONS; use crate::{ backends::Backend, @@ -624,14 +625,13 @@ impl SessionState for Pipe { metrics: &mut SessionMetrics, ) -> SessionResult { let mut counter = 0; - let max_loop_iterations = 100000; if self.frontend_readiness.event.is_hup() { return SessionResult::Close; } let token = self.frontend_token; - while counter < max_loop_iterations { + while counter < MAX_LOOP_ITERATIONS { let frontend_interest = self.frontend_readiness.filter_interest(); let backend_interest = self.backend_readiness.filter_interest(); @@ -709,10 +709,10 @@ impl SessionState for Pipe { counter += 1; } - if counter == max_loop_iterations { + if counter >= MAX_LOOP_ITERATIONS { error!( "PROXY\thandling session {:?} went through {} iterations, there's a probable infinite loop bug, closing the connection", - self.frontend_token, max_loop_iterations + self.frontend_token, MAX_LOOP_ITERATIONS ); incr!("http.infinite_loop.error"); diff --git a/lib/src/protocol/proxy_protocol/expect.rs b/lib/src/protocol/proxy_protocol/expect.rs index 2a16a55f1..80f9aef7a 100644 --- a/lib/src/protocol/proxy_protocol/expect.rs +++ b/lib/src/protocol/proxy_protocol/expect.rs @@ -3,6 +3,7 @@ use std::{cell::RefCell, rc::Rc}; use mio::{net::TcpStream, *}; use nom::{Err, HexDisplay}; use rusty_ulid::Ulid; +use sozu_command::config::MAX_LOOP_ITERATIONS; use crate::{ logs::LogContext, @@ -211,13 +212,12 @@ impl SessionState for ExpectProxyProtocol { metrics: &mut SessionMetrics, ) -> SessionResult { let mut counter = 0; - let max_loop_iterations = 100000; if self.frontend_readiness.event.is_hup() { return SessionResult::Close; } - while counter < max_loop_iterations { + while counter < MAX_LOOP_ITERATIONS { let frontend_interest = self.frontend_readiness.filter_interest(); trace!( @@ -251,10 +251,10 @@ impl SessionState for ExpectProxyProtocol { counter += 1; } - if counter == max_loop_iterations { + if counter >= MAX_LOOP_ITERATIONS { error!( "PROXY\thandling session {:?} went through {} iterations, there's a probable infinite loop bug, closing the connection", - self.frontend_token, max_loop_iterations + self.frontend_token, MAX_LOOP_ITERATIONS ); incr!("http.infinite_loop.error"); diff --git a/lib/src/protocol/rustls.rs b/lib/src/protocol/rustls.rs index 3e15116ba..218177835 100644 --- a/lib/src/protocol/rustls.rs +++ b/lib/src/protocol/rustls.rs @@ -3,6 +3,7 @@ use std::{cell::RefCell, io::ErrorKind, rc::Rc}; use mio::{net::TcpStream, Token}; use rustls::ServerConnection; use rusty_ulid::Ulid; +use sozu_command::config::MAX_LOOP_ITERATIONS; use crate::{ logs::LogContext, protocol::SessionState, timer::TimeoutContainer, Readiness, Ready, @@ -186,13 +187,12 @@ impl SessionState for TlsHandshake { _metrics: &mut SessionMetrics, ) -> SessionResult { let mut counter = 0; - let max_loop_iterations = 100000; if self.frontend_readiness.event.is_hup() { return SessionResult::Close; } - while counter < max_loop_iterations { + while counter < MAX_LOOP_ITERATIONS { let frontend_interest = self.frontend_readiness.filter_interest(); trace!( @@ -233,10 +233,10 @@ impl SessionState for TlsHandshake { counter += 1; } - if counter == max_loop_iterations { + if counter >= MAX_LOOP_ITERATIONS { error!( "PROXY\thandling session {:?} went through {} iterations, there's a probable infinite loop bug, closing the connection", - self.frontend_token, max_loop_iterations + self.frontend_token, MAX_LOOP_ITERATIONS ); incr!("http.infinite_loop.error"); diff --git a/lib/src/socket.rs b/lib/src/socket.rs index 8e50a2225..9440fb054 100644 --- a/lib/src/socket.rs +++ b/lib/src/socket.rs @@ -6,6 +6,7 @@ use std::{ use mio::net::{TcpListener, TcpStream}; use rustls::{ProtocolVersion, ServerConnection}; use socket2::{Domain, Protocol, Socket, Type}; +use sozu_command::config::MAX_LOOP_ITERATIONS; #[derive(thiserror::Error, Debug)] pub enum ServerBindError { @@ -60,7 +61,13 @@ pub trait SocketHandler { impl SocketHandler for TcpStream { fn socket_read(&mut self, buf: &mut [u8]) -> (usize, SocketResult) { let mut size = 0usize; + let mut counter = 0; loop { + counter += 1; + if counter > MAX_LOOP_ITERATIONS { + error!("MAX_LOOP_ITERATION reached in TcpStream::socket_read"); + incr!("socket.read.infinite_loop.error"); + } if size == buf.len() { return (size, SocketResult::Continue); } @@ -83,7 +90,13 @@ impl SocketHandler for TcpStream { fn socket_write(&mut self, buf: &[u8]) -> (usize, SocketResult) { let mut size = 0usize; + let mut counter = 0; loop { + counter += 1; + if counter > MAX_LOOP_ITERATIONS { + error!("MAX_LOOP_ITERATION reached in TcpStream::socket_write"); + incr!("socket.write.infinite_loop.error"); + } if size == buf.len() { return (size, SocketResult::Continue); } @@ -165,7 +178,14 @@ impl SocketHandler for FrontRustls { let mut is_error = false; let mut is_closed = false; + let mut counter = 0; loop { + counter += 1; + if counter > MAX_LOOP_ITERATIONS { + error!("MAX_LOOP_ITERATION reached in FrontRustls::socket_read"); + incr!("socket.read.infinite_loop.error"); + } + if size == buf.len() { break; } @@ -251,7 +271,13 @@ impl SocketHandler for FrontRustls { let mut is_error = false; let mut is_closed = false; + let mut counter = 0; loop { + counter += 1; + if counter > MAX_LOOP_ITERATIONS { + error!("MAX_LOOP_ITERATION reached in FrontRustls::socket_write"); + incr!("socket.write.infinite_loop.error"); + } if buffered_size == buf.len() { break; } @@ -297,7 +323,10 @@ impl SocketHandler for FrontRustls { } Ok(_sz) => {} Err(e) => match e.kind() { - ErrorKind::WouldBlock => can_write = false, + ErrorKind::WouldBlock => { + can_write = false; + break; + } ErrorKind::ConnectionReset | ErrorKind::ConnectionAborted | ErrorKind::BrokenPipe => { @@ -358,7 +387,13 @@ impl SocketHandler for FrontRustls { }, } + let mut counter = 0; loop { + counter += 1; + if counter > MAX_LOOP_ITERATIONS { + error!("MAX_LOOP_ITERATION reached in FrontRustls::socket_write_vectored"); + incr!("socket.write.infinite_loop.error"); + } match self.session.write_tls(&mut self.stream) { Ok(0) => { //can_write = false; @@ -366,7 +401,10 @@ impl SocketHandler for FrontRustls { } Ok(_sz) => {} Err(e) => match e.kind() { - ErrorKind::WouldBlock => can_write = false, + ErrorKind::WouldBlock => { + can_write = false; + break; + } ErrorKind::ConnectionReset | ErrorKind::ConnectionAborted | ErrorKind::BrokenPipe => { diff --git a/lib/src/tcp.rs b/lib/src/tcp.rs index 6d59ff46e..cc456a738 100644 --- a/lib/src/tcp.rs +++ b/lib/src/tcp.rs @@ -18,7 +18,7 @@ use rusty_ulid::Ulid; use slab::Slab; use time::{Duration, Instant}; -use sozu_command::{proto::command::request::RequestType, ObjectKind}; +use sozu_command::{config::MAX_LOOP_ITERATIONS, proto::command::request::RequestType, ObjectKind}; use crate::{ backends::{Backend, BackendMap}, @@ -572,7 +572,6 @@ impl TcpSession { fn ready_inner(&mut self, session: Rc>) -> StateResult { let mut counter = 0; - let max_loop_iterations = 100000; let back_connected = self.back_connected(); if back_connected.is_connecting() { @@ -634,7 +633,7 @@ impl TcpSession { } let token = self.frontend_token; - while counter < max_loop_iterations { + while counter < MAX_LOOP_ITERATIONS { let front_interest = self.front_readiness().interest & self.front_readiness().event; let back_interest = self .back_readiness() @@ -754,8 +753,11 @@ impl TcpSession { counter += 1; } - if counter == max_loop_iterations { - error!("PROXY\thandling session {:?} went through {} iterations, there's a probable infinite loop bug, closing the connection", self.frontend_token, max_loop_iterations); + if counter >= MAX_LOOP_ITERATIONS { + error!( + "PROXY\thandling session {:?} went through {} iterations, there's a probable infinite loop bug, closing the connection", + self.frontend_token, MAX_LOOP_ITERATIONS + ); incr!("tcp.infinite_loop.error"); let front_interest = self.front_readiness().interest & self.front_readiness().event;