From dbb22fbf0088ab43872381a7de80ed27fa15db01 Mon Sep 17 00:00:00 2001 From: terkwood <38859656+Terkwood@users.noreply.github.com> Date: Thu, 27 May 2021 08:11:00 -0400 Subject: [PATCH 01/54] feat: implement sorted streams --- redis_streams/Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/redis_streams/Cargo.toml b/redis_streams/Cargo.toml index df7af0ae..0cfbaafd 100644 --- a/redis_streams/Cargo.toml +++ b/redis_streams/Cargo.toml @@ -2,7 +2,7 @@ authors = ["terkwood <38859656+Terkwood@users.noreply.github.com>"] edition = "2018" name = "redis_streams" -version = "0.4.1" +version = "1.0.0" [dependencies] uuid = { version = "0.8.2", features = ["serde", "v4"] } From e6ecf37917ce82d863db51db98baa76214cf3ebf Mon Sep 17 00:00:00 2001 From: terkwood <38859656+Terkwood@users.noreply.github.com> Date: Thu, 27 May 2021 08:20:28 -0400 Subject: [PATCH 02/54] add readme and stub consumer --- redis_streams/Cargo.toml | 2 ++ redis_streams/README.md | 9 +++++++++ redis_streams/src/consumer.rs | 23 +++++++++++++++++++++++ redis_streams/src/lib.rs | 2 ++ 4 files changed, 36 insertions(+) create mode 100644 redis_streams/README.md create mode 100644 redis_streams/src/consumer.rs diff --git a/redis_streams/Cargo.toml b/redis_streams/Cargo.toml index 0cfbaafd..b10d126c 100644 --- a/redis_streams/Cargo.toml +++ b/redis_streams/Cargo.toml @@ -5,4 +5,6 @@ name = "redis_streams" version = "1.0.0" [dependencies] +anyhow = "1.0.40" +redis = "0.20.1" uuid = { version = "0.8.2", features = ["serde", "v4"] } diff --git a/redis_streams/README.md b/redis_streams/README.md new file mode 100644 index 00000000..ba8f1fbb --- /dev/null +++ b/redis_streams/README.md @@ -0,0 +1,9 @@ +# BUGOUT redis_streams + +A generic way to handle reading from redis streams using consumer groups. + +It has been customized process messages from multiple streams, in order by time. + +## Attribution + +Thanks to [klaxit/redis-stream-rs](https://github.com/klaxit/redis-stream-rs) for the bulk of this implementation. diff --git a/redis_streams/src/consumer.rs b/redis_streams/src/consumer.rs new file mode 100644 index 00000000..a19429ec --- /dev/null +++ b/redis_streams/src/consumer.rs @@ -0,0 +1,23 @@ +use anyhow::{Context, Result}; +use redis::streams::{StreamReadOptions, StreamReadReply}; +use redis::{Commands, Connection, RedisResult, Value}; +use std::collections::HashMap; + +pub type Message = HashMap; + +// A Consumer or Group Consumer handling connection to Redis and able to consume +// messages. +pub struct Consumer<'a, F> +where + F: FnMut(&str, &Message) -> Result<()>, +{ + pub count: Option, + pub group: Option<(String, String)>, + pub handled_messages: u32, + pub handler: F, + pub next_pos: String, + pub process_pending: bool, + pub redis: &'a mut Connection, + pub stream: String, + pub timeout: usize, +} diff --git a/redis_streams/src/lib.rs b/redis_streams/src/lib.rs index 14f353f4..a6d41404 100644 --- a/redis_streams/src/lib.rs +++ b/redis_streams/src/lib.rs @@ -1,3 +1,5 @@ +pub mod consumer; + #[derive(Debug, Copy, Clone, Eq, Ord, PartialEq, PartialOrd, Hash)] pub struct XReadEntryId { pub millis_time: u64, From d61f872f2186ab31109561318459da40e31311b8 Mon Sep 17 00:00:00 2001 From: terkwood <38859656+Terkwood@users.noreply.github.com> Date: Thu, 27 May 2021 08:20:48 -0400 Subject: [PATCH 03/54] make group required --- redis_streams/src/consumer.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/redis_streams/src/consumer.rs b/redis_streams/src/consumer.rs index a19429ec..b4e38537 100644 --- a/redis_streams/src/consumer.rs +++ b/redis_streams/src/consumer.rs @@ -12,7 +12,7 @@ where F: FnMut(&str, &Message) -> Result<()>, { pub count: Option, - pub group: Option<(String, String)>, + pub group: (String, String), pub handled_messages: u32, pub handler: F, pub next_pos: String, From 6994daa3c169975d559374cc08f4849e6c9744f3 Mon Sep 17 00:00:00 2001 From: terkwood <38859656+Terkwood@users.noreply.github.com> Date: Thu, 27 May 2021 08:28:27 -0400 Subject: [PATCH 04/54] stub out opts --- redis_streams/src/consumer.rs | 100 +++++++++++++++++++++++++++++++++- 1 file changed, 97 insertions(+), 3 deletions(-) diff --git a/redis_streams/src/consumer.rs b/redis_streams/src/consumer.rs index b4e38537..4ddd4fc3 100644 --- a/redis_streams/src/consumer.rs +++ b/redis_streams/src/consumer.rs @@ -1,3 +1,5 @@ +//! Defines consumer groups and streaming logic. + use anyhow::{Context, Result}; use redis::streams::{StreamReadOptions, StreamReadReply}; use redis::{Commands, Connection, RedisResult, Value}; @@ -5,9 +7,9 @@ use std::collections::HashMap; pub type Message = HashMap; -// A Consumer or Group Consumer handling connection to Redis and able to consume -// messages. -pub struct Consumer<'a, F> +/// Handles connection to Redis and consumes messages from an individual stream. +/// Uses XREADGROUP only, never XREAD. +pub struct ConsumerGroup<'a, F> where F: FnMut(&str, &Message) -> Result<()>, { @@ -21,3 +23,95 @@ where pub stream: String, pub timeout: usize, } + +impl<'a, F> ConsumerGroup<'a, F> +where + F: FnMut(&str, &Message) -> Result<()>, +{ + /// Initializes a new `stream::Consumer`. + pub fn init( + redis: &'a mut Connection, + stream: &str, + handler: F, + opts: ConsumerGroupOpts, + ) -> Result { + todo!() + } +} + +#[derive(Clone, Debug)] +pub enum StartPosition { + EndOfStream, + Other(String), + StartOfStream, +} + +#[derive(Debug)] +pub struct Group { + pub group_name: String, + pub consumer_name: String, +} + +#[derive(Debug)] +pub struct ConsumerGroupOpts { + pub count: Option, + pub create_stream_if_not_exists: bool, + pub group: Group, + pub process_pending: bool, + pub start_pos: StartPosition, + pub timeout: usize, +} + +impl ConsumerGroupOpts { + pub fn new(group: Group) -> Self { + Self { + count: None, + create_stream_if_not_exists: true, + group, + process_pending: true, + start_pos: StartPosition::EndOfStream, + timeout: 2_000, + } + } + + /// Maximum number of message to read from the stream in one batch + pub fn count(mut self, count: usize) -> Self { + self.count = Some(count); + self + } + + /// Create the stream in Redis before registering the group (default: `true`). + pub fn create_stream_if_not_exists(mut self, create_stream_if_not_exists: bool) -> Self { + self.create_stream_if_not_exists = create_stream_if_not_exists; + self + } + + /// Name of the group and consumer. Enables Redis group consumer behavior if + /// specified + pub fn group(mut self, group_name: &str, consumer_name: &str) -> Self { + self.group = Group { + group_name: group_name.to_string(), + consumer_name: consumer_name.to_string(), + }; + self + } + + /// Start by processing pending messages before switching to real time data + /// (default: `true`) + pub fn process_pending(mut self, process_pending: bool) -> Self { + self.process_pending = process_pending; + self + } + + /// Where to start reading messages in the stream. + pub fn start_pos(mut self, start_pos: StartPosition) -> Self { + self.start_pos = start_pos; + self + } + + /// Maximum ms duration to block waiting for messages. + pub fn timeout(mut self, timeout: usize) -> Self { + self.timeout = timeout; + self + } +} From cc2bf870fe3f36239965dda94efede61d9936ca1 Mon Sep 17 00:00:00 2001 From: terkwood <38859656+Terkwood@users.noreply.github.com> Date: Thu, 27 May 2021 08:28:39 -0400 Subject: [PATCH 05/54] rename --- redis_streams/src/{consumer.rs => consumer_group.rs} | 0 redis_streams/src/lib.rs | 2 +- 2 files changed, 1 insertion(+), 1 deletion(-) rename redis_streams/src/{consumer.rs => consumer_group.rs} (100%) diff --git a/redis_streams/src/consumer.rs b/redis_streams/src/consumer_group.rs similarity index 100% rename from redis_streams/src/consumer.rs rename to redis_streams/src/consumer_group.rs diff --git a/redis_streams/src/lib.rs b/redis_streams/src/lib.rs index a6d41404..be310626 100644 --- a/redis_streams/src/lib.rs +++ b/redis_streams/src/lib.rs @@ -1,4 +1,4 @@ -pub mod consumer; +pub mod consumer_group; #[derive(Debug, Copy, Clone, Eq, Ord, PartialEq, PartialOrd, Hash)] pub struct XReadEntryId { From 0e21916bba61e5ae269e58cc43fdf426411fd0e1 Mon Sep 17 00:00:00 2001 From: terkwood <38859656+Terkwood@users.noreply.github.com> Date: Thu, 27 May 2021 10:44:10 -0400 Subject: [PATCH 06/54] rename --- redis_streams/src/consumer_group.rs | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/redis_streams/src/consumer_group.rs b/redis_streams/src/consumer_group.rs index 4ddd4fc3..4f76b397 100644 --- a/redis_streams/src/consumer_group.rs +++ b/redis_streams/src/consumer_group.rs @@ -30,10 +30,10 @@ where { /// Initializes a new `stream::Consumer`. pub fn init( - redis: &'a mut Connection, - stream: &str, - handler: F, - opts: ConsumerGroupOpts, + _redis: &'a mut Connection, + _stream: &str, + _handler: F, + _opts: ConsumerGroupOpts, ) -> Result { todo!() } @@ -59,7 +59,7 @@ pub struct ConsumerGroupOpts { pub group: Group, pub process_pending: bool, pub start_pos: StartPosition, - pub timeout: usize, + pub timeout_ms: usize, } impl ConsumerGroupOpts { @@ -70,7 +70,7 @@ impl ConsumerGroupOpts { group, process_pending: true, start_pos: StartPosition::EndOfStream, - timeout: 2_000, + timeout_ms: 5_000, } } @@ -110,8 +110,8 @@ impl ConsumerGroupOpts { } /// Maximum ms duration to block waiting for messages. - pub fn timeout(mut self, timeout: usize) -> Self { - self.timeout = timeout; + pub fn timeout(mut self, timeout_ms: usize) -> Self { + self.timeout_ms = timeout_ms; self } } From 182e2b8ceb3c4bb99ba8bcc5776b6acc18f81a4e Mon Sep 17 00:00:00 2001 From: terkwood <38859656+Terkwood@users.noreply.github.com> Date: Thu, 27 May 2021 10:46:54 -0400 Subject: [PATCH 07/54] stub out struct --- redis_streams/src/lib.rs | 1 + redis_streams/src/sorted_streams.rs | 1 + 2 files changed, 2 insertions(+) create mode 100644 redis_streams/src/sorted_streams.rs diff --git a/redis_streams/src/lib.rs b/redis_streams/src/lib.rs index be310626..0700465a 100644 --- a/redis_streams/src/lib.rs +++ b/redis_streams/src/lib.rs @@ -1,4 +1,5 @@ pub mod consumer_group; +pub mod sorted_streams; #[derive(Debug, Copy, Clone, Eq, Ord, PartialEq, PartialOrd, Hash)] pub struct XReadEntryId { diff --git a/redis_streams/src/sorted_streams.rs b/redis_streams/src/sorted_streams.rs new file mode 100644 index 00000000..150fc6f2 --- /dev/null +++ b/redis_streams/src/sorted_streams.rs @@ -0,0 +1 @@ +pub struct SortedStreams {} From cef00939eee9acdf1954283832cc31b211ec9532 Mon Sep 17 00:00:00 2001 From: terkwood <38859656+Terkwood@users.noreply.github.com> Date: Thu, 27 May 2021 15:53:50 -0400 Subject: [PATCH 08/54] make a trait --- redis_streams/src/sorted_streams.rs | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/redis_streams/src/sorted_streams.rs b/redis_streams/src/sorted_streams.rs index 150fc6f2..6341c4f8 100644 --- a/redis_streams/src/sorted_streams.rs +++ b/redis_streams/src/sorted_streams.rs @@ -1 +1,12 @@ -pub struct SortedStreams {} +use crate::consumer_group::{ConsumerGroup, Message}; +use anyhow::{Context, Result}; + +trait SortedStreams { + fn consume() -> Result<()>; +} + +impl<'a, F> SortedStreams for Vec> where F: FnMut(&str, &Message) -> Result<()> { + fn consume() -> Result<()> { + todo!() + } +} From 1d177cc5708a22bb68a572461dc466d25e3c4d74 Mon Sep 17 00:00:00 2001 From: terkwood <38859656+Terkwood@users.noreply.github.com> Date: Thu, 27 May 2021 17:24:20 -0400 Subject: [PATCH 09/54] add test, and part of a method --- redis_streams/src/sorted_streams.rs | 23 ++++++++++++++++++++--- 1 file changed, 20 insertions(+), 3 deletions(-) diff --git a/redis_streams/src/sorted_streams.rs b/redis_streams/src/sorted_streams.rs index 6341c4f8..d878bffb 100644 --- a/redis_streams/src/sorted_streams.rs +++ b/redis_streams/src/sorted_streams.rs @@ -2,11 +2,28 @@ use crate::consumer_group::{ConsumerGroup, Message}; use anyhow::{Context, Result}; trait SortedStreams { - fn consume() -> Result<()>; + fn consume(&self) -> Result<()>; } -impl<'a, F> SortedStreams for Vec> where F: FnMut(&str, &Message) -> Result<()> { - fn consume() -> Result<()> { +impl<'a, F> SortedStreams for Vec> +where + F: FnMut(&str, &Message) -> Result<()>, +{ + fn consume(&self) -> Result<()> { + for _consumer_group in self { + todo!() + } + + Ok(()) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn can_consume() { todo!() } } From 5d0e94392d0cee3a6981f49b3a6e5394e762354f Mon Sep 17 00:00:00 2001 From: terkwood <38859656+Terkwood@users.noreply.github.com> Date: Thu, 27 May 2021 17:35:04 -0400 Subject: [PATCH 10/54] ack and handle --- redis_streams/src/consumer_group.rs | 24 +++++++++++++++++++++--- redis_streams/src/lib.rs | 14 +++++++------- redis_streams/src/sorted_streams.rs | 4 ++-- 3 files changed, 30 insertions(+), 12 deletions(-) diff --git a/redis_streams/src/consumer_group.rs b/redis_streams/src/consumer_group.rs index 4f76b397..83d4ce03 100644 --- a/redis_streams/src/consumer_group.rs +++ b/redis_streams/src/consumer_group.rs @@ -1,5 +1,6 @@ //! Defines consumer groups and streaming logic. +use super::XId; use anyhow::{Context, Result}; use redis::streams::{StreamReadOptions, StreamReadReply}; use redis::{Commands, Connection, RedisResult, Value}; @@ -11,10 +12,10 @@ pub type Message = HashMap; /// Uses XREADGROUP only, never XREAD. pub struct ConsumerGroup<'a, F> where - F: FnMut(&str, &Message) -> Result<()>, + F: FnMut(XId, &Message) -> Result<()>, { pub count: Option, - pub group: (String, String), + pub group: Group, pub handled_messages: u32, pub handler: F, pub next_pos: String, @@ -26,7 +27,7 @@ where impl<'a, F> ConsumerGroup<'a, F> where - F: FnMut(&str, &Message) -> Result<()>, + F: FnMut(XId, &Message) -> Result<()>, { /// Initializes a new `stream::Consumer`. pub fn init( @@ -37,6 +38,23 @@ where ) -> Result { todo!() } + + /// Process a message by calling the handler, returning the same XId + /// passed to the handler. + fn handle_message(&mut self, xid: XId, message: &Message) -> Result { + // Call handler + (self.handler)(xid, message)?; + self.handled_messages += 1; + Ok(xid) + } + + /// Acknowledge messages by ID + fn ack_messages(&mut self, xids: &[XId]) -> Result<()> { + let ids: Vec = xids.iter().map(|xid| xid.to_string()).collect(); + Ok(self + .redis + .xack(&self.stream, &self.group.group_name, &ids)?) + } } #[derive(Clone, Debug)] diff --git a/redis_streams/src/lib.rs b/redis_streams/src/lib.rs index 0700465a..cf8155c5 100644 --- a/redis_streams/src/lib.rs +++ b/redis_streams/src/lib.rs @@ -2,28 +2,28 @@ pub mod consumer_group; pub mod sorted_streams; #[derive(Debug, Copy, Clone, Eq, Ord, PartialEq, PartialOrd, Hash)] -pub struct XReadEntryId { +pub struct XId { pub millis_time: u64, pub seq_no: u64, } -impl Default for XReadEntryId { +impl Default for XId { fn default() -> Self { - XReadEntryId { + XId { millis_time: 0, seq_no: 0, } } } -impl XReadEntryId { - pub fn from_str(s: &str) -> Result { +impl XId { + pub fn from_str(s: &str) -> Result { let parts: Vec<&str> = s.split('-').collect(); if parts.len() != 2 { Err(StreamDeserError) } else { let millis_time = parts[0].parse::()?; let seq_no = parts[1].parse::()?; - Ok(XReadEntryId { + Ok(XId { millis_time, seq_no, }) @@ -53,6 +53,6 @@ mod tests { use super::*; #[test] fn xread_entry_id_default_string() { - assert_eq!(XReadEntryId::default().to_string(), "0-0".to_string()) + assert_eq!(XId::default().to_string(), "0-0".to_string()) } } diff --git a/redis_streams/src/sorted_streams.rs b/redis_streams/src/sorted_streams.rs index d878bffb..cae2114a 100644 --- a/redis_streams/src/sorted_streams.rs +++ b/redis_streams/src/sorted_streams.rs @@ -1,4 +1,4 @@ -use crate::consumer_group::{ConsumerGroup, Message}; +use crate::{XId, consumer_group::{ConsumerGroup, Message}}; use anyhow::{Context, Result}; trait SortedStreams { @@ -7,7 +7,7 @@ trait SortedStreams { impl<'a, F> SortedStreams for Vec> where - F: FnMut(&str, &Message) -> Result<()>, + F: FnMut(XId, &Message) -> Result<()>, { fn consume(&self) -> Result<()> { for _consumer_group in self { From 5bf9fc9e00107dbe16912cb1d99b8cff06dc7a97 Mon Sep 17 00:00:00 2001 From: terkwood <38859656+Terkwood@users.noreply.github.com> Date: Thu, 27 May 2021 17:41:21 -0400 Subject: [PATCH 11/54] define unacked --- redis_streams/src/consumer_group.rs | 3 +-- redis_streams/src/sorted_streams.rs | 14 +++++++++++++- 2 files changed, 14 insertions(+), 3 deletions(-) diff --git a/redis_streams/src/consumer_group.rs b/redis_streams/src/consumer_group.rs index 83d4ce03..521d8d13 100644 --- a/redis_streams/src/consumer_group.rs +++ b/redis_streams/src/consumer_group.rs @@ -42,8 +42,7 @@ where /// Process a message by calling the handler, returning the same XId /// passed to the handler. fn handle_message(&mut self, xid: XId, message: &Message) -> Result { - // Call handler - (self.handler)(xid, message)?; + (self.handler)(xid, message)?; // Call handler self.handled_messages += 1; Ok(xid) } diff --git a/redis_streams/src/sorted_streams.rs b/redis_streams/src/sorted_streams.rs index cae2114a..408ee40d 100644 --- a/redis_streams/src/sorted_streams.rs +++ b/redis_streams/src/sorted_streams.rs @@ -1,5 +1,9 @@ -use crate::{XId, consumer_group::{ConsumerGroup, Message}}; +use crate::{ + consumer_group::{ConsumerGroup, Message}, + XId, +}; use anyhow::{Context, Result}; +use std::collections::HashMap; trait SortedStreams { fn consume(&self) -> Result<()>; @@ -10,6 +14,7 @@ where F: FnMut(XId, &Message) -> Result<()>, { fn consume(&self) -> Result<()> { + let mut unacked = Unacknowledged::default(); for _consumer_group in self { todo!() } @@ -18,6 +23,13 @@ where } } +/// Track unacknowledged messages by stream name +struct Unacknowledged(pub HashMap>); +impl Default for Unacknowledged { + fn default() -> Self { + Self(HashMap::new()) + } +} #[cfg(test)] mod tests { use super::*; From 2f558187de7d73bdd033e37f54bb13d1502f84bf Mon Sep 17 00:00:00 2001 From: terkwood <38859656+Terkwood@users.noreply.github.com> Date: Thu, 27 May 2021 17:46:37 -0400 Subject: [PATCH 12/54] trim --- redis_streams/src/consumer_group.rs | 19 ------------------- 1 file changed, 19 deletions(-) diff --git a/redis_streams/src/consumer_group.rs b/redis_streams/src/consumer_group.rs index 521d8d13..06e80cb5 100644 --- a/redis_streams/src/consumer_group.rs +++ b/redis_streams/src/consumer_group.rs @@ -18,8 +18,6 @@ where pub group: Group, pub handled_messages: u32, pub handler: F, - pub next_pos: String, - pub process_pending: bool, pub redis: &'a mut Connection, pub stream: String, pub timeout: usize, @@ -74,8 +72,6 @@ pub struct ConsumerGroupOpts { pub count: Option, pub create_stream_if_not_exists: bool, pub group: Group, - pub process_pending: bool, - pub start_pos: StartPosition, pub timeout_ms: usize, } @@ -85,8 +81,6 @@ impl ConsumerGroupOpts { count: None, create_stream_if_not_exists: true, group, - process_pending: true, - start_pos: StartPosition::EndOfStream, timeout_ms: 5_000, } } @@ -113,19 +107,6 @@ impl ConsumerGroupOpts { self } - /// Start by processing pending messages before switching to real time data - /// (default: `true`) - pub fn process_pending(mut self, process_pending: bool) -> Self { - self.process_pending = process_pending; - self - } - - /// Where to start reading messages in the stream. - pub fn start_pos(mut self, start_pos: StartPosition) -> Self { - self.start_pos = start_pos; - self - } - /// Maximum ms duration to block waiting for messages. pub fn timeout(mut self, timeout_ms: usize) -> Self { self.timeout_ms = timeout_ms; From fd9b9925fb711c36fda13eac79e54728845fd822 Mon Sep 17 00:00:00 2001 From: terkwood <38859656+Terkwood@users.noreply.github.com> Date: Thu, 27 May 2021 18:14:23 -0400 Subject: [PATCH 13/54] try this then --- redis_streams/src/consumer_group.rs | 47 +++++++++++------------------ redis_streams/src/sorted_streams.rs | 37 ++++++++++++----------- 2 files changed, 38 insertions(+), 46 deletions(-) diff --git a/redis_streams/src/consumer_group.rs b/redis_streams/src/consumer_group.rs index 06e80cb5..8ae4f291 100644 --- a/redis_streams/src/consumer_group.rs +++ b/redis_streams/src/consumer_group.rs @@ -6,11 +6,9 @@ use redis::streams::{StreamReadOptions, StreamReadReply}; use redis::{Commands, Connection, RedisResult, Value}; use std::collections::HashMap; -pub type Message = HashMap; - /// Handles connection to Redis and consumes messages from an individual stream. /// Uses XREADGROUP only, never XREAD. -pub struct ConsumerGroup<'a, F> +pub struct ConsumerGroup where F: FnMut(XId, &Message) -> Result<()>, { @@ -18,42 +16,41 @@ where pub group: Group, pub handled_messages: u32, pub handler: F, - pub redis: &'a mut Connection, pub stream: String, - pub timeout: usize, } -impl<'a, F> ConsumerGroup<'a, F> +impl ConsumerGroup where F: FnMut(XId, &Message) -> Result<()>, { - /// Initializes a new `stream::Consumer`. + /// Initializes a new `stream::Consumer` and returns a ConsumerGroup struct. pub fn init( - _redis: &'a mut Connection, - _stream: &str, - _handler: F, - _opts: ConsumerGroupOpts, + stream: &str, + handler: F, + opts: ConsumerGroupOpts, + redis: &mut Connection, ) -> Result { - todo!() + redis.xgroup_create_mkstream(stream, &opts.group.group_name, "$")?; + Ok(ConsumerGroup { + count: opts.count, + group: opts.group, + handled_messages: 0, + stream: stream.to_string(), + handler, + }) } /// Process a message by calling the handler, returning the same XId /// passed to the handler. - fn handle_message(&mut self, xid: XId, message: &Message) -> Result { + fn _handle_message(&mut self, xid: XId, message: &Message) -> Result { (self.handler)(xid, message)?; // Call handler self.handled_messages += 1; Ok(xid) } - - /// Acknowledge messages by ID - fn ack_messages(&mut self, xids: &[XId]) -> Result<()> { - let ids: Vec = xids.iter().map(|xid| xid.to_string()).collect(); - Ok(self - .redis - .xack(&self.stream, &self.group.group_name, &ids)?) - } } +pub type Message = HashMap; + #[derive(Clone, Debug)] pub enum StartPosition { EndOfStream, @@ -70,7 +67,6 @@ pub struct Group { #[derive(Debug)] pub struct ConsumerGroupOpts { pub count: Option, - pub create_stream_if_not_exists: bool, pub group: Group, pub timeout_ms: usize, } @@ -79,7 +75,6 @@ impl ConsumerGroupOpts { pub fn new(group: Group) -> Self { Self { count: None, - create_stream_if_not_exists: true, group, timeout_ms: 5_000, } @@ -91,12 +86,6 @@ impl ConsumerGroupOpts { self } - /// Create the stream in Redis before registering the group (default: `true`). - pub fn create_stream_if_not_exists(mut self, create_stream_if_not_exists: bool) -> Self { - self.create_stream_if_not_exists = create_stream_if_not_exists; - self - } - /// Name of the group and consumer. Enables Redis group consumer behavior if /// specified pub fn group(mut self, group_name: &str, consumer_name: &str) -> Self { diff --git a/redis_streams/src/sorted_streams.rs b/redis_streams/src/sorted_streams.rs index 408ee40d..2c81075d 100644 --- a/redis_streams/src/sorted_streams.rs +++ b/redis_streams/src/sorted_streams.rs @@ -1,41 +1,44 @@ use crate::{ - consumer_group::{ConsumerGroup, Message}, + consumer_group::{ConsumerGroup, Group, Message}, XId, }; use anyhow::{Context, Result}; +use redis::{streams::StreamReadReply, Commands, Connection}; use std::collections::HashMap; -trait SortedStreams { - fn consume(&self) -> Result<()>; +pub struct SortedStreams<'a, F> +where + F: FnMut(XId, &Message) -> Result<()>, +{ + pub consumer_groups: Vec>, + pub timeout: usize, + pub redis: &'a mut Connection, } -impl<'a, F> SortedStreams for Vec> +impl<'a, F> SortedStreams<'a, F> where F: FnMut(XId, &Message) -> Result<()>, { - fn consume(&self) -> Result<()> { - let mut unacked = Unacknowledged::default(); - for _consumer_group in self { + /// "xreadgroup >" across streams, handle all the messages in time order, and acknowledge them all + pub fn consume(&mut self) -> Result<()> { + let unacked = Unacknowledged::default(); + for _consumer_group in &self.consumer_groups { todo!() } + for ((stream, group), xids) in unacked.0 { + let ids: Vec = xids.iter().map(|xid| xid.to_string()).collect(); + self.redis.xack(&stream, &group.group_name, &ids)? + } + Ok(()) } } /// Track unacknowledged messages by stream name -struct Unacknowledged(pub HashMap>); +struct Unacknowledged(pub HashMap<(String, Group), Vec>); impl Default for Unacknowledged { fn default() -> Self { Self(HashMap::new()) } } -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn can_consume() { - todo!() - } -} From dedcd23f2478126f0f4004bbf1d01cd938eff71e Mon Sep 17 00:00:00 2001 From: terkwood <38859656+Terkwood@users.noreply.github.com> Date: Thu, 27 May 2021 18:23:48 -0400 Subject: [PATCH 14/54] do a nice, healthy XREADGROUP --- redis_streams/src/lib.rs | 2 +- redis_streams/src/sorted_streams.rs | 22 +++++++++++++++---- .../{consumer_group.rs => stream_handler.rs} | 10 ++++----- 3 files changed, 23 insertions(+), 11 deletions(-) rename redis_streams/src/{consumer_group.rs => stream_handler.rs} (92%) diff --git a/redis_streams/src/lib.rs b/redis_streams/src/lib.rs index cf8155c5..749f1e48 100644 --- a/redis_streams/src/lib.rs +++ b/redis_streams/src/lib.rs @@ -1,4 +1,4 @@ -pub mod consumer_group; +pub mod stream_handler; pub mod sorted_streams; #[derive(Debug, Copy, Clone, Eq, Ord, PartialEq, PartialOrd, Hash)] diff --git a/redis_streams/src/sorted_streams.rs b/redis_streams/src/sorted_streams.rs index 2c81075d..f222deb0 100644 --- a/redis_streams/src/sorted_streams.rs +++ b/redis_streams/src/sorted_streams.rs @@ -1,20 +1,25 @@ use crate::{ - consumer_group::{ConsumerGroup, Group, Message}, + stream_handler::{Group, Message, StreamHandler}, XId, }; use anyhow::{Context, Result}; -use redis::{streams::StreamReadReply, Commands, Connection}; +use redis::{ + streams::{StreamReadOptions, StreamReadReply}, + Commands, Connection, +}; use std::collections::HashMap; pub struct SortedStreams<'a, F> where F: FnMut(XId, &Message) -> Result<()>, { - pub consumer_groups: Vec>, + pub handlers: Vec>, + pub group: Group, pub timeout: usize, pub redis: &'a mut Connection, } +const READ_OP: &str = ">"; impl<'a, F> SortedStreams<'a, F> where F: FnMut(XId, &Message) -> Result<()>, @@ -22,7 +27,16 @@ where /// "xreadgroup >" across streams, handle all the messages in time order, and acknowledge them all pub fn consume(&mut self) -> Result<()> { let unacked = Unacknowledged::default(); - for _consumer_group in &self.consumer_groups { + let opts = StreamReadOptions::default() + .block(self.timeout) + .group(&self.group.group_name, &self.group.consumer_name); + let stream_names: Vec = + self.handlers.iter().map(|h| h.stream.to_string()).collect(); + let read_ops: Vec = stream_names.iter().map(|_| READ_OP.to_string()).collect(); + + let _xrr: StreamReadReply = self.redis.xread_options(&stream_names, &read_ops, opts)?; + + for _consumer_group in &self.handlers { todo!() } diff --git a/redis_streams/src/consumer_group.rs b/redis_streams/src/stream_handler.rs similarity index 92% rename from redis_streams/src/consumer_group.rs rename to redis_streams/src/stream_handler.rs index 8ae4f291..da64c2b5 100644 --- a/redis_streams/src/consumer_group.rs +++ b/redis_streams/src/stream_handler.rs @@ -8,22 +8,21 @@ use std::collections::HashMap; /// Handles connection to Redis and consumes messages from an individual stream. /// Uses XREADGROUP only, never XREAD. -pub struct ConsumerGroup +pub struct StreamHandler where F: FnMut(XId, &Message) -> Result<()>, { pub count: Option, - pub group: Group, pub handled_messages: u32, pub handler: F, pub stream: String, } -impl ConsumerGroup +impl StreamHandler where F: FnMut(XId, &Message) -> Result<()>, { - /// Initializes a new `stream::Consumer` and returns a ConsumerGroup struct. + /// Calls xgroup_create_mkstream on the given stream name and returns this struct. pub fn init( stream: &str, handler: F, @@ -31,9 +30,8 @@ where redis: &mut Connection, ) -> Result { redis.xgroup_create_mkstream(stream, &opts.group.group_name, "$")?; - Ok(ConsumerGroup { + Ok(StreamHandler { count: opts.count, - group: opts.group, handled_messages: 0, stream: stream.to_string(), handler, From a15437044302070ff49bc41e4ddf387a28a8ffc1 Mon Sep 17 00:00:00 2001 From: terkwood <38859656+Terkwood@users.noreply.github.com> Date: Thu, 27 May 2021 18:25:23 -0400 Subject: [PATCH 15/54] simplify --- redis_streams/src/sorted_streams.rs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/redis_streams/src/sorted_streams.rs b/redis_streams/src/sorted_streams.rs index f222deb0..c9ff4021 100644 --- a/redis_streams/src/sorted_streams.rs +++ b/redis_streams/src/sorted_streams.rs @@ -33,16 +33,16 @@ where let stream_names: Vec = self.handlers.iter().map(|h| h.stream.to_string()).collect(); let read_ops: Vec = stream_names.iter().map(|_| READ_OP.to_string()).collect(); - + let _xrr: StreamReadReply = self.redis.xread_options(&stream_names, &read_ops, opts)?; for _consumer_group in &self.handlers { todo!() } - for ((stream, group), xids) in unacked.0 { + for (stream, xids) in unacked.0 { let ids: Vec = xids.iter().map(|xid| xid.to_string()).collect(); - self.redis.xack(&stream, &group.group_name, &ids)? + self.redis.xack(&stream, &self.group.group_name, &ids)? } Ok(()) @@ -50,7 +50,7 @@ where } /// Track unacknowledged messages by stream name -struct Unacknowledged(pub HashMap<(String, Group), Vec>); +struct Unacknowledged(pub HashMap>); impl Default for Unacknowledged { fn default() -> Self { Self(HashMap::new()) From 9450ee23cdaa8684fcec95bbc4875de3348a34c8 Mon Sep 17 00:00:00 2001 From: terkwood <38859656+Terkwood@users.noreply.github.com> Date: Fri, 28 May 2021 12:05:23 -0400 Subject: [PATCH 16/54] use hashmap --- redis_streams/src/sorted_streams.rs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/redis_streams/src/sorted_streams.rs b/redis_streams/src/sorted_streams.rs index c9ff4021..54ea3e30 100644 --- a/redis_streams/src/sorted_streams.rs +++ b/redis_streams/src/sorted_streams.rs @@ -13,7 +13,7 @@ pub struct SortedStreams<'a, F> where F: FnMut(XId, &Message) -> Result<()>, { - pub handlers: Vec>, + pub handlers: HashMap>, pub group: Group, pub timeout: usize, pub redis: &'a mut Connection, @@ -24,14 +24,14 @@ impl<'a, F> SortedStreams<'a, F> where F: FnMut(XId, &Message) -> Result<()>, { - /// "xreadgroup >" across streams, handle all the messages in time order, and acknowledge them all + /// "XREADGROUP >" across streams, handle all the messages in time order, + /// and acknowledge them all pub fn consume(&mut self) -> Result<()> { let unacked = Unacknowledged::default(); let opts = StreamReadOptions::default() .block(self.timeout) .group(&self.group.group_name, &self.group.consumer_name); - let stream_names: Vec = - self.handlers.iter().map(|h| h.stream.to_string()).collect(); + let stream_names: Vec = self.handlers.keys().cloned().collect(); let read_ops: Vec = stream_names.iter().map(|_| READ_OP.to_string()).collect(); let _xrr: StreamReadReply = self.redis.xread_options(&stream_names, &read_ops, opts)?; From f9c48e7fecf61bae67ce849ae9950331e84fac31 Mon Sep 17 00:00:00 2001 From: terkwood <38859656+Terkwood@users.noreply.github.com> Date: Fri, 28 May 2021 12:14:14 -0400 Subject: [PATCH 17/54] walk through stream read result --- redis_streams/src/sorted_streams.rs | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/redis_streams/src/sorted_streams.rs b/redis_streams/src/sorted_streams.rs index 54ea3e30..fb6d79d8 100644 --- a/redis_streams/src/sorted_streams.rs +++ b/redis_streams/src/sorted_streams.rs @@ -20,6 +20,7 @@ where } const READ_OP: &str = ">"; +struct StreamMessage(pub String, pub Message); impl<'a, F> SortedStreams<'a, F> where F: FnMut(XId, &Message) -> Result<()>, @@ -34,12 +35,21 @@ where let stream_names: Vec = self.handlers.keys().cloned().collect(); let read_ops: Vec = stream_names.iter().map(|_| READ_OP.to_string()).collect(); - let _xrr: StreamReadReply = self.redis.xread_options(&stream_names, &read_ops, opts)?; + let xrr: StreamReadReply = self.redis.xread_options(&stream_names, &read_ops, opts)?; + let mut by_time: HashMap = HashMap::new(); - for _consumer_group in &self.handlers { - todo!() + for k in xrr.keys { + let key = k.key; + for x in k.ids { + if let Ok(xid) = XId::from_str(&x.id) { + by_time.insert(xid, StreamMessage(key.clone(), x.map)); + } + todo!("ANYTHING ELSE ?!") + } } + todo!("sort etc"); + for (stream, xids) in unacked.0 { let ids: Vec = xids.iter().map(|xid| xid.to_string()).collect(); self.redis.xack(&stream, &self.group.group_name, &ids)? From 85b870f56603ede6a5eb860f6b9de58dc1b19f02 Mon Sep 17 00:00:00 2001 From: terkwood <38859656+Terkwood@users.noreply.github.com> Date: Fri, 28 May 2021 12:25:58 -0400 Subject: [PATCH 18/54] write out sort, stub handle call --- redis_streams/src/sorted_streams.rs | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/redis_streams/src/sorted_streams.rs b/redis_streams/src/sorted_streams.rs index fb6d79d8..44b15859 100644 --- a/redis_streams/src/sorted_streams.rs +++ b/redis_streams/src/sorted_streams.rs @@ -20,6 +20,7 @@ where } const READ_OP: &str = ">"; + struct StreamMessage(pub String, pub Message); impl<'a, F> SortedStreams<'a, F> where @@ -36,19 +37,22 @@ where let read_ops: Vec = stream_names.iter().map(|_| READ_OP.to_string()).collect(); let xrr: StreamReadReply = self.redis.xread_options(&stream_names, &read_ops, opts)?; - let mut by_time: HashMap = HashMap::new(); + let mut by_time: Vec<(XId, StreamMessage)> = Vec::with_capacity(50); for k in xrr.keys { let key = k.key; for x in k.ids { if let Ok(xid) = XId::from_str(&x.id) { - by_time.insert(xid, StreamMessage(key.clone(), x.map)); + by_time.push((xid, StreamMessage(key.clone(), x.map))); } - todo!("ANYTHING ELSE ?!") } } - todo!("sort etc"); + by_time.sort_by_key(|t| t.0); + + for (xid, StreamMessage(stream, message)) in by_time { + todo!("handle -- think about casing") + } for (stream, xids) in unacked.0 { let ids: Vec = xids.iter().map(|xid| xid.to_string()).collect(); From 78d3361aa789ba8ba88e4fbc97ed09d7b5955e78 Mon Sep 17 00:00:00 2001 From: terkwood <38859656+Terkwood@users.noreply.github.com> Date: Sat, 29 May 2021 08:36:14 -0400 Subject: [PATCH 19/54] write sorted stream --- redis_streams/src/sorted_streams.rs | 9 +++++++-- redis_streams/src/stream_handler.rs | 2 +- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/redis_streams/src/sorted_streams.rs b/redis_streams/src/sorted_streams.rs index 44b15859..94e2c566 100644 --- a/redis_streams/src/sorted_streams.rs +++ b/redis_streams/src/sorted_streams.rs @@ -29,7 +29,7 @@ where /// "XREADGROUP >" across streams, handle all the messages in time order, /// and acknowledge them all pub fn consume(&mut self) -> Result<()> { - let unacked = Unacknowledged::default(); + let mut unacked = Unacknowledged::default(); let opts = StreamReadOptions::default() .block(self.timeout) .group(&self.group.group_name, &self.group.consumer_name); @@ -51,7 +51,12 @@ where by_time.sort_by_key(|t| t.0); for (xid, StreamMessage(stream, message)) in by_time { - todo!("handle -- think about casing") + if let Some(handler) = self.handlers.get_mut(&stream) { + handler.handle_message(xid, &message)?; + if let Some(unacked_for_stream) = unacked.0.get_mut(&stream) { + unacked_for_stream.push(xid); + } + } } for (stream, xids) in unacked.0 { diff --git a/redis_streams/src/stream_handler.rs b/redis_streams/src/stream_handler.rs index da64c2b5..9a8e9db5 100644 --- a/redis_streams/src/stream_handler.rs +++ b/redis_streams/src/stream_handler.rs @@ -40,7 +40,7 @@ where /// Process a message by calling the handler, returning the same XId /// passed to the handler. - fn _handle_message(&mut self, xid: XId, message: &Message) -> Result { + pub fn handle_message(&mut self, xid: XId, message: &Message) -> Result { (self.handler)(xid, message)?; // Call handler self.handled_messages += 1; Ok(xid) From dea4e203b30f9272c9563d0842663c0d0ce592d9 Mon Sep 17 00:00:00 2001 From: terkwood <38859656+Terkwood@users.noreply.github.com> Date: Sat, 29 May 2021 08:36:20 -0400 Subject: [PATCH 20/54] set version --- redis_streams/Cargo.toml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/redis_streams/Cargo.toml b/redis_streams/Cargo.toml index b10d126c..8c436685 100644 --- a/redis_streams/Cargo.toml +++ b/redis_streams/Cargo.toml @@ -2,9 +2,9 @@ authors = ["terkwood <38859656+Terkwood@users.noreply.github.com>"] edition = "2018" name = "redis_streams" -version = "1.0.0" +version = "1.0.0-a" [dependencies] anyhow = "1.0.40" redis = "0.20.1" -uuid = { version = "0.8.2", features = ["serde", "v4"] } +uuid = {version = "0.8.2", features = ["serde", "v4"]} From b40031458f0ea063cba8dbeae0cb914b3221ddc1 Mon Sep 17 00:00:00 2001 From: terkwood <38859656+Terkwood@users.noreply.github.com> Date: Sat, 29 May 2021 08:36:58 -0400 Subject: [PATCH 21/54] write readme --- redis_streams/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/redis_streams/README.md b/redis_streams/README.md index ba8f1fbb..fbee36de 100644 --- a/redis_streams/README.md +++ b/redis_streams/README.md @@ -2,7 +2,7 @@ A generic way to handle reading from redis streams using consumer groups. -It has been customized process messages from multiple streams, in order by time. +It has been customized to process messages from multiple streams, in order by time. ## Attribution From bbba933d021602ab59fe1bd25c6a08b86df35816 Mon Sep 17 00:00:00 2001 From: terkwood <38859656+Terkwood@users.noreply.github.com> Date: Sat, 29 May 2021 08:40:02 -0400 Subject: [PATCH 22/54] renames --- redis_streams/src/sorted_streams.rs | 2 +- redis_streams/src/stream_handler.rs | 18 ++++++++---------- 2 files changed, 9 insertions(+), 11 deletions(-) diff --git a/redis_streams/src/sorted_streams.rs b/redis_streams/src/sorted_streams.rs index 94e2c566..ce79797e 100644 --- a/redis_streams/src/sorted_streams.rs +++ b/redis_streams/src/sorted_streams.rs @@ -2,7 +2,7 @@ use crate::{ stream_handler::{Group, Message, StreamHandler}, XId, }; -use anyhow::{Context, Result}; +use anyhow::Result; use redis::{ streams::{StreamReadOptions, StreamReadReply}, Commands, Connection, diff --git a/redis_streams/src/stream_handler.rs b/redis_streams/src/stream_handler.rs index 9a8e9db5..d23d89a9 100644 --- a/redis_streams/src/stream_handler.rs +++ b/redis_streams/src/stream_handler.rs @@ -1,9 +1,7 @@ -//! Defines consumer groups and streaming logic. - +//! Defines XREADGROUP / XACK streaming logic tied to a redis connection. use super::XId; -use anyhow::{Context, Result}; -use redis::streams::{StreamReadOptions, StreamReadReply}; -use redis::{Commands, Connection, RedisResult, Value}; +use anyhow::Result; +use redis::{Commands, Connection, Value}; use std::collections::HashMap; /// Handles connection to Redis and consumes messages from an individual stream. @@ -23,7 +21,7 @@ where F: FnMut(XId, &Message) -> Result<()>, { /// Calls xgroup_create_mkstream on the given stream name and returns this struct. - pub fn init( + pub fn init_redis_stream( stream: &str, handler: F, opts: ConsumerGroupOpts, @@ -66,7 +64,7 @@ pub struct Group { pub struct ConsumerGroupOpts { pub count: Option, pub group: Group, - pub timeout_ms: usize, + pub block_ms: usize, } impl ConsumerGroupOpts { @@ -74,7 +72,7 @@ impl ConsumerGroupOpts { Self { count: None, group, - timeout_ms: 5_000, + block_ms: 5_000, } } @@ -95,8 +93,8 @@ impl ConsumerGroupOpts { } /// Maximum ms duration to block waiting for messages. - pub fn timeout(mut self, timeout_ms: usize) -> Self { - self.timeout_ms = timeout_ms; + pub fn block_ms(mut self, timeout_ms: usize) -> Self { + self.block_ms = timeout_ms; self } } From a1f7d15ca0d7cfc37c5f76b91a22cf442011b154 Mon Sep 17 00:00:00 2001 From: terkwood <38859656+Terkwood@users.noreply.github.com> Date: Sat, 29 May 2021 08:40:59 -0400 Subject: [PATCH 23/54] trim unused --- redis_streams/src/stream_handler.rs | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/redis_streams/src/stream_handler.rs b/redis_streams/src/stream_handler.rs index d23d89a9..fbc52a7f 100644 --- a/redis_streams/src/stream_handler.rs +++ b/redis_streams/src/stream_handler.rs @@ -1,4 +1,5 @@ -//! Defines XREADGROUP / XACK streaming logic tied to a redis connection. +//! Wraps a a function which is used to process individual +//! messages from given stream, in time order. use super::XId; use anyhow::Result; use redis::{Commands, Connection, Value}; @@ -47,13 +48,6 @@ where pub type Message = HashMap; -#[derive(Clone, Debug)] -pub enum StartPosition { - EndOfStream, - Other(String), - StartOfStream, -} - #[derive(Debug)] pub struct Group { pub group_name: String, From 35d6090388ac801156c07db038bc3554ea2a7fe0 Mon Sep 17 00:00:00 2001 From: terkwood <38859656+Terkwood@users.noreply.github.com> Date: Sat, 29 May 2021 08:45:01 -0400 Subject: [PATCH 24/54] move stuff around --- redis_streams/src/consumer_group_opts.rs | 39 ++++++++++++++++++++++ redis_streams/src/lib.rs | 9 ++++-- redis_streams/src/sorted_streams.rs | 9 +++--- redis_streams/src/stream_handler.rs | 41 +----------------------- 4 files changed, 52 insertions(+), 46 deletions(-) create mode 100644 redis_streams/src/consumer_group_opts.rs diff --git a/redis_streams/src/consumer_group_opts.rs b/redis_streams/src/consumer_group_opts.rs new file mode 100644 index 00000000..80d83e96 --- /dev/null +++ b/redis_streams/src/consumer_group_opts.rs @@ -0,0 +1,39 @@ +use crate::*; +#[derive(Debug)] +pub struct ConsumerGroupOpts { + pub count: Option, + pub group: Group, + pub block_ms: usize, +} + +impl ConsumerGroupOpts { + pub fn new(group: Group) -> Self { + Self { + count: None, + group, + block_ms: 5_000, + } + } + + /// Maximum number of message to read from the stream in one batch + pub fn count(mut self, count: usize) -> Self { + self.count = Some(count); + self + } + + /// Name of the group and consumer. Enables Redis group consumer behavior if + /// specified + pub fn group(mut self, group_name: &str, consumer_name: &str) -> Self { + self.group = Group { + group_name: group_name.to_string(), + consumer_name: consumer_name.to_string(), + }; + self + } + + /// Maximum ms duration to block waiting for messages. + pub fn block_ms(mut self, timeout_ms: usize) -> Self { + self.block_ms = timeout_ms; + self + } +} diff --git a/redis_streams/src/lib.rs b/redis_streams/src/lib.rs index 749f1e48..fbd224fb 100644 --- a/redis_streams/src/lib.rs +++ b/redis_streams/src/lib.rs @@ -1,5 +1,10 @@ -pub mod stream_handler; -pub mod sorted_streams; +mod consumer_group_opts; +mod sorted_streams; +mod stream_handler; + +pub use consumer_group_opts::*; +pub use sorted_streams::*; +pub use stream_handler::*; #[derive(Debug, Copy, Clone, Eq, Ord, PartialEq, PartialOrd, Hash)] pub struct XId { diff --git a/redis_streams/src/sorted_streams.rs b/redis_streams/src/sorted_streams.rs index ce79797e..f54234d1 100644 --- a/redis_streams/src/sorted_streams.rs +++ b/redis_streams/src/sorted_streams.rs @@ -1,7 +1,4 @@ -use crate::{ - stream_handler::{Group, Message, StreamHandler}, - XId, -}; +use crate::*; use anyhow::Result; use redis::{ streams::{StreamReadOptions, StreamReadReply}, @@ -26,6 +23,10 @@ impl<'a, F> SortedStreams<'a, F> where F: FnMut(XId, &Message) -> Result<()>, { + pub fn init_redis_streams(opts: ConsumerGroupOpts, redis: &'a mut Connection) { + todo!() + } + /// "XREADGROUP >" across streams, handle all the messages in time order, /// and acknowledge them all pub fn consume(&mut self) -> Result<()> { diff --git a/redis_streams/src/stream_handler.rs b/redis_streams/src/stream_handler.rs index fbc52a7f..172505d1 100644 --- a/redis_streams/src/stream_handler.rs +++ b/redis_streams/src/stream_handler.rs @@ -1,6 +1,6 @@ //! Wraps a a function which is used to process individual //! messages from given stream, in time order. -use super::XId; +use crate::*; use anyhow::Result; use redis::{Commands, Connection, Value}; use std::collections::HashMap; @@ -53,42 +53,3 @@ pub struct Group { pub group_name: String, pub consumer_name: String, } - -#[derive(Debug)] -pub struct ConsumerGroupOpts { - pub count: Option, - pub group: Group, - pub block_ms: usize, -} - -impl ConsumerGroupOpts { - pub fn new(group: Group) -> Self { - Self { - count: None, - group, - block_ms: 5_000, - } - } - - /// Maximum number of message to read from the stream in one batch - pub fn count(mut self, count: usize) -> Self { - self.count = Some(count); - self - } - - /// Name of the group and consumer. Enables Redis group consumer behavior if - /// specified - pub fn group(mut self, group_name: &str, consumer_name: &str) -> Self { - self.group = Group { - group_name: group_name.to_string(), - consumer_name: consumer_name.to_string(), - }; - self - } - - /// Maximum ms duration to block waiting for messages. - pub fn block_ms(mut self, timeout_ms: usize) -> Self { - self.block_ms = timeout_ms; - self - } -} From c520afb45b7020d7ce985736df73b32cd3aecbf6 Mon Sep 17 00:00:00 2001 From: terkwood <38859656+Terkwood@users.noreply.github.com> Date: Sat, 29 May 2021 08:54:46 -0400 Subject: [PATCH 25/54] creation method --- redis_streams/src/consumer_group_opts.rs | 14 +++----------- redis_streams/src/sorted_streams.rs | 24 ++++++++++++++++++++---- redis_streams/src/stream_handler.rs | 17 ++++------------- 3 files changed, 27 insertions(+), 28 deletions(-) diff --git a/redis_streams/src/consumer_group_opts.rs b/redis_streams/src/consumer_group_opts.rs index 80d83e96..8453a6f3 100644 --- a/redis_streams/src/consumer_group_opts.rs +++ b/redis_streams/src/consumer_group_opts.rs @@ -1,26 +1,18 @@ use crate::*; #[derive(Debug)] -pub struct ConsumerGroupOpts { - pub count: Option, +pub struct ConsumerGroupOpts { pub group: Group, pub block_ms: usize, } impl ConsumerGroupOpts { pub fn new(group: Group) -> Self { - Self { - count: None, + Self { group, block_ms: 5_000, } } - - /// Maximum number of message to read from the stream in one batch - pub fn count(mut self, count: usize) -> Self { - self.count = Some(count); - self - } - + /// Name of the group and consumer. Enables Redis group consumer behavior if /// specified pub fn group(mut self, group_name: &str, consumer_name: &str) -> Self { diff --git a/redis_streams/src/sorted_streams.rs b/redis_streams/src/sorted_streams.rs index f54234d1..9e0dce8e 100644 --- a/redis_streams/src/sorted_streams.rs +++ b/redis_streams/src/sorted_streams.rs @@ -12,7 +12,7 @@ where { pub handlers: HashMap>, pub group: Group, - pub timeout: usize, + pub block_ms: usize, pub redis: &'a mut Connection, } @@ -23,8 +23,24 @@ impl<'a, F> SortedStreams<'a, F> where F: FnMut(XId, &Message) -> Result<()>, { - pub fn init_redis_streams(opts: ConsumerGroupOpts, redis: &'a mut Connection) { - todo!() + /// Calls xgroup_create_mkstream on the given stream name and returns this struct. + pub fn xgroup_create_mkstreams( + stream_handlers: Vec<(&str, F)>, + opts: &ConsumerGroupOpts, + redis: &'a mut Connection, + ) -> Result { + let mut handlers = HashMap::new(); + for (stream, handler) in stream_handlers { + redis.xgroup_create_mkstream(stream, &opts.group.group_name, "$")?; + handlers.insert(stream.to_string(), StreamHandler::new(stream, handler)); + } + + Ok(Self { + group: opts.group.clone(), + redis, + block_ms: opts.block_ms, + handlers, + }) } /// "XREADGROUP >" across streams, handle all the messages in time order, @@ -32,7 +48,7 @@ where pub fn consume(&mut self) -> Result<()> { let mut unacked = Unacknowledged::default(); let opts = StreamReadOptions::default() - .block(self.timeout) + .block(self.block_ms) .group(&self.group.group_name, &self.group.consumer_name); let stream_names: Vec = self.handlers.keys().cloned().collect(); let read_ops: Vec = stream_names.iter().map(|_| READ_OP.to_string()).collect(); diff --git a/redis_streams/src/stream_handler.rs b/redis_streams/src/stream_handler.rs index 172505d1..a8b287cf 100644 --- a/redis_streams/src/stream_handler.rs +++ b/redis_streams/src/stream_handler.rs @@ -11,7 +11,6 @@ pub struct StreamHandler where F: FnMut(XId, &Message) -> Result<()>, { - pub count: Option, pub handled_messages: u32, pub handler: F, pub stream: String, @@ -21,20 +20,12 @@ impl StreamHandler where F: FnMut(XId, &Message) -> Result<()>, { - /// Calls xgroup_create_mkstream on the given stream name and returns this struct. - pub fn init_redis_stream( - stream: &str, - handler: F, - opts: ConsumerGroupOpts, - redis: &mut Connection, - ) -> Result { - redis.xgroup_create_mkstream(stream, &opts.group.group_name, "$")?; - Ok(StreamHandler { - count: opts.count, + pub fn new(stream: &str, handler: F) -> Self { + StreamHandler { handled_messages: 0, stream: stream.to_string(), handler, - }) + } } /// Process a message by calling the handler, returning the same XId @@ -48,7 +39,7 @@ where pub type Message = HashMap; -#[derive(Debug)] +#[derive(Debug, Clone)] pub struct Group { pub group_name: String, pub consumer_name: String, From 30deb56a082eb637a72ccc992dbdd1c9606fd1ec Mon Sep 17 00:00:00 2001 From: terkwood <38859656+Terkwood@users.noreply.github.com> Date: Sat, 29 May 2021 08:55:22 -0400 Subject: [PATCH 26/54] clean up --- redis_streams/src/sorted_streams.rs | 7 ++++--- redis_streams/src/stream_handler.rs | 2 +- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/redis_streams/src/sorted_streams.rs b/redis_streams/src/sorted_streams.rs index 9e0dce8e..ce5f8d01 100644 --- a/redis_streams/src/sorted_streams.rs +++ b/redis_streams/src/sorted_streams.rs @@ -16,9 +16,6 @@ where pub redis: &'a mut Connection, } -const READ_OP: &str = ">"; - -struct StreamMessage(pub String, pub Message); impl<'a, F> SortedStreams<'a, F> where F: FnMut(XId, &Message) -> Result<()>, @@ -85,6 +82,10 @@ where } } +const READ_OP: &str = ">"; + +struct StreamMessage(pub String, pub Message); + /// Track unacknowledged messages by stream name struct Unacknowledged(pub HashMap>); impl Default for Unacknowledged { diff --git a/redis_streams/src/stream_handler.rs b/redis_streams/src/stream_handler.rs index a8b287cf..65c6aa6b 100644 --- a/redis_streams/src/stream_handler.rs +++ b/redis_streams/src/stream_handler.rs @@ -2,7 +2,7 @@ //! messages from given stream, in time order. use crate::*; use anyhow::Result; -use redis::{Commands, Connection, Value}; +use redis::Value; use std::collections::HashMap; /// Handles connection to Redis and consumes messages from an individual stream. From da2dc2f185092844657b5d238294231d7e35db80 Mon Sep 17 00:00:00 2001 From: terkwood <38859656+Terkwood@users.noreply.github.com> Date: Sat, 29 May 2021 08:58:51 -0400 Subject: [PATCH 27/54] address lints --- redis_streams/src/lib.rs | 12 ++++++++---- redis_streams/src/sorted_streams.rs | 1 + 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/redis_streams/src/lib.rs b/redis_streams/src/lib.rs index fbd224fb..f6a3a3f4 100644 --- a/redis_streams/src/lib.rs +++ b/redis_streams/src/lib.rs @@ -20,8 +20,10 @@ impl Default for XId { } } -impl XId { - pub fn from_str(s: &str) -> Result { +impl std::str::FromStr for XId { + type Err = StreamDeserError; + + fn from_str(s: &str) -> Result { let parts: Vec<&str> = s.split('-').collect(); if parts.len() != 2 { Err(StreamDeserError) @@ -34,9 +36,11 @@ impl XId { }) } } +} - pub fn to_string(&self) -> String { - format!("{}-{}", self.millis_time, self.seq_no) +impl std::fmt::Display for XId { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "{}-{}", self.millis_time, self.seq_no) } } diff --git a/redis_streams/src/sorted_streams.rs b/redis_streams/src/sorted_streams.rs index ce5f8d01..7b5b9f82 100644 --- a/redis_streams/src/sorted_streams.rs +++ b/redis_streams/src/sorted_streams.rs @@ -5,6 +5,7 @@ use redis::{ Commands, Connection, }; use std::collections::HashMap; +use std::str::FromStr; pub struct SortedStreams<'a, F> where From 9c5de569a751bcd2a8cfe42cb0efa2e5819a4a91 Mon Sep 17 00:00:00 2001 From: terkwood <38859656+Terkwood@users.noreply.github.com> Date: Sat, 29 May 2021 09:01:22 -0400 Subject: [PATCH 28/54] fill in doc --- redis_streams/src/sorted_streams.rs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/redis_streams/src/sorted_streams.rs b/redis_streams/src/sorted_streams.rs index 7b5b9f82..497ad4df 100644 --- a/redis_streams/src/sorted_streams.rs +++ b/redis_streams/src/sorted_streams.rs @@ -42,7 +42,9 @@ where } /// "XREADGROUP >" across streams, handle all the messages in time order, - /// and acknowledge them all + /// and acknowledge them all. The XACK calls happen once per stream, + /// acknowleding all XIds for that stream at once. The XACK calls happen + /// after _all_ individual message handlers have been invoked. pub fn consume(&mut self) -> Result<()> { let mut unacked = Unacknowledged::default(); let opts = StreamReadOptions::default() From 0685ae32adb816195db819903e4f38adfc90c605 Mon Sep 17 00:00:00 2001 From: terkwood <38859656+Terkwood@users.noreply.github.com> Date: Sat, 29 May 2021 09:09:32 -0400 Subject: [PATCH 29/54] fix: replace game lobby streaming impl --- micro-game-lobby/Cargo.lock | 2 +- micro-game-lobby/Cargo.toml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/micro-game-lobby/Cargo.lock b/micro-game-lobby/Cargo.lock index 43d5f28a..84abbff4 100644 --- a/micro-game-lobby/Cargo.lock +++ b/micro-game-lobby/Cargo.lock @@ -618,7 +618,7 @@ dependencies = [ [[package]] name = "micro-game-lobby" -version = "0.2.1" +version = "1.0.0" dependencies = [ "bincode", "core-model", diff --git a/micro-game-lobby/Cargo.toml b/micro-game-lobby/Cargo.toml index 25afd654..e8a3e3ef 100644 --- a/micro-game-lobby/Cargo.toml +++ b/micro-game-lobby/Cargo.toml @@ -2,7 +2,7 @@ authors = ["terkwood <38859656+Terkwood@users.noreply.github.com>"] edition = "2018" name = "micro-game-lobby" -version = "0.2.1" +version = "1.0.0" [dependencies] bincode = "1.2.1" From 2d455950e7d4b3db8da6328ea2f5f11cf3bd30c1 Mon Sep 17 00:00:00 2001 From: terkwood <38859656+Terkwood@users.noreply.github.com> Date: Sat, 29 May 2021 09:14:51 -0400 Subject: [PATCH 30/54] cut a lot of things --- micro-game-lobby/Cargo.lock | 871 ++++----------------------- micro-game-lobby/Cargo.toml | 4 +- micro-game-lobby/src/components.rs | 8 +- micro-game-lobby/src/stream/mod.rs | 120 +--- micro-game-lobby/src/stream/xack.rs | 111 ---- micro-game-lobby/src/stream/xread.rs | 7 - 6 files changed, 127 insertions(+), 994 deletions(-) delete mode 100644 micro-game-lobby/src/stream/xack.rs diff --git a/micro-game-lobby/Cargo.lock b/micro-game-lobby/Cargo.lock index 84abbff4..162fe479 100644 --- a/micro-game-lobby/Cargo.lock +++ b/micro-game-lobby/Cargo.lock @@ -2,130 +2,30 @@ # It is not intended for manual editing. [[package]] name = "aho-corasick" -version = "0.7.14" +version = "0.7.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b476ce7103678b0c6d3d395dbbae31d48ff910bd28be979ba5d48c6351131d0d" +checksum = "1e37cfd5e7657ada45f742d6e99ca5788580b5c529dc78faf11ece6dc702656f" dependencies = [ "memchr", ] [[package]] -name = "async-channel" -version = "1.5.1" +name = "anyhow" +version = "1.0.40" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "59740d83946db6a5af71ae25ddf9562c2b176b2ca42cf99a455f09f4a220d6b9" -dependencies = [ - "concurrent-queue", - "event-listener", - "futures-core", -] - -[[package]] -name = "async-executor" -version = "1.3.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d373d78ded7d0b3fa8039375718cde0aace493f2e34fb60f51cbf567562ca801" -dependencies = [ - "async-task", - "concurrent-queue", - "fastrand", - "futures-lite", - "once_cell", - "vec-arena", -] - -[[package]] -name = "async-global-executor" -version = "1.4.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "124ac8c265e407641c3362b8f4d39cdb4e243885b71eef087be27199790f5a3a" -dependencies = [ - "async-executor", - "async-io", - "futures-lite", - "num_cpus", - "once_cell", -] - -[[package]] -name = "async-io" -version = "1.1.10" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d54bc4c1c7292475efb2253227dbcfad8fe1ca4c02bc62c510cc2f3da5c4704e" -dependencies = [ - "concurrent-queue", - "fastrand", - "futures-lite", - "libc", - "log", - "nb-connect", - "once_cell", - "parking", - "polling", - "vec-arena", - "waker-fn", - "winapi 0.3.9", -] - -[[package]] -name = "async-mutex" -version = "1.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "479db852db25d9dbf6204e6cb6253698f175c15726470f78af0d918e99d6156e" -dependencies = [ - "event-listener", -] - -[[package]] -name = "async-std" -version = "1.6.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a9fa76751505e8df1c7a77762f60486f60c71bbd9b8557f4da6ad47d083732ed" -dependencies = [ - "async-global-executor", - "async-io", - "async-mutex", - "blocking", - "crossbeam-utils", - "futures-channel", - "futures-core", - "futures-io", - "futures-lite", - "gloo-timers", - "kv-log-macro", - "log", - "memchr", - "num_cpus", - "once_cell", - "pin-project-lite", - "pin-utils", - "slab", - "wasm-bindgen-futures", -] - -[[package]] -name = "async-task" -version = "4.0.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e91831deabf0d6d7ec49552e489aed63b7456a7a3c46cff62adad428110b0af0" +checksum = "28b2cd92db5cbd74e8e5028f7e27dd7aa3090e89e4f2a197cc7c8dfb69c7063b" [[package]] name = "async-trait" -version = "0.1.41" +version = "0.1.50" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b246867b8b3b6ae56035f1eb1ed557c1d8eae97f0d53696138a50fa0e3a3b8c0" +checksum = "0b98e84bbb4cbcdd97da190ba0c58a1bb0de2c1fdf67d159e192ed766aeca722" dependencies = [ "proc-macro2", "quote", "syn", ] -[[package]] -name = "atomic-waker" -version = "1.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "065374052e7df7ee4047b1160cca5e1467a12351a40b3da123c870ba0b8eda2a" - [[package]] name = "atty" version = "0.2.14" @@ -134,7 +34,7 @@ checksum = "d9b39be18770d11421cdb1b9947a45dd3f37e93092cbf377614828a319d5fee8" dependencies = [ "hermit-abi", "libc", - "winapi 0.3.9", + "winapi", ] [[package]] @@ -145,11 +45,10 @@ checksum = "cdb031dd78e28731d87d56cc8ffef4a8f36ca26c38fe2de700543e627f8a464a" [[package]] name = "bincode" -version = "1.3.1" +version = "1.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f30d3a39baa26f9651f17b375061f3233dde33424a8b72b0dbe93a68a0bc896d" +checksum = "b1f45e9417d87227c7a56d22e471c6206462cba514c7590c09aff4cf6d1ddcad" dependencies = [ - "byteorder", "serde", ] @@ -159,49 +58,11 @@ version = "1.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cf1de2fe8c75bc145a2f577add951f8134889b4795d47466a54a5c846d691693" -[[package]] -name = "blocking" -version = "1.0.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c5e170dbede1f740736619b776d7251cb1b9095c435c34d8ca9f57fcd2f335e9" -dependencies = [ - "async-channel", - "async-task", - "atomic-waker", - "fastrand", - "futures-lite", - "once_cell", -] - -[[package]] -name = "bumpalo" -version = "3.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2e8c087f005730276d1096a652e92a8bacee2e2472bcc9715a74d2bec38b5820" - -[[package]] -name = "byteorder" -version = "1.3.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "08c48aae112d48ed9f069b33538ea9e3e90aa263cfa3d1c24309612b1f7472de" - [[package]] name = "bytes" -version = "0.5.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0e4cec68f03f32e44924783795810fa50a7035d8c8ebe78580ad7e6c703fba38" - -[[package]] -name = "cache-padded" -version = "1.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "631ae5198c9be5e753e5cc215e1bd73c2b466a3565173db433f52bb9d3e66dba" - -[[package]] -name = "cc" -version = "1.0.61" +version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ed67cbde08356238e75fc4656be4749481eeffb09e19f320a25237d5221c985d" +checksum = "b700ce4376041dcd0a327fd0097c41095743c4c8af8887265942faf1100bd040" [[package]] name = "cfg-if" @@ -210,34 +71,19 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4785bdd1c96b2a846b2bd7cc02e86b6b3dbf14e7e53446c4f54c92a361040822" [[package]] -name = "cloudabi" -version = "0.1.0" +name = "cfg-if" +version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4344512281c643ae7638bbabc3af17a11307803ec8f0fcad9fae512a8bf36467" -dependencies = [ - "bitflags", -] +checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" [[package]] name = "combine" -version = "4.3.2" +version = "4.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2809f67365382d65fd2b6d9c22577231b954ed27400efeafbe687bda75abcc0b" +checksum = "cc4369b5e4c0cddf64ad8981c0111e7df4f7078f4d6ba98fb31f2e17c4c57b7e" dependencies = [ "bytes", - "futures-util", "memchr", - "pin-project-lite", - "tokio", -] - -[[package]] -name = "concurrent-queue" -version = "1.2.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "30ed07550be01594c6026cff2a1d7fe9c8f683caa798e12b68694ac9e88286a3" -dependencies = [ - "cache-padded", ] [[package]] @@ -256,7 +102,7 @@ version = "0.7.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "69323bff1fb41c635347b8ead484a5ca6c3f11914d784170b158d8449ab07f8e" dependencies = [ - "cfg-if", + "cfg-if 0.1.10", "crossbeam-channel", "crossbeam-deque", "crossbeam-epoch", @@ -292,7 +138,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "058ed274caafc1f60c4997b5fc07bf7dc7cca454af7c6e81edffe5f33f70dace" dependencies = [ "autocfg", - "cfg-if", + "cfg-if 0.1.10", "crossbeam-utils", "lazy_static", "maybe-uninit", @@ -306,7 +152,7 @@ version = "0.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "774ba60a54c213d409d5353bda12d49cd68d14e45036a285234c8d6f91f92570" dependencies = [ - "cfg-if", + "cfg-if 0.1.10", "crossbeam-utils", "maybe-uninit", ] @@ -318,15 +164,15 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c3c7c73a2d1e9fc0886a08b93e98eb643461230d5f1925e4036204d5f2e261a8" dependencies = [ "autocfg", - "cfg-if", + "cfg-if 0.1.10", "lazy_static", ] [[package]] name = "dtoa" -version = "0.4.6" +version = "0.4.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "134951f4028bdadb9b84baf4232681efbf277da25144b9b0ad65df75946c422b" +checksum = "56899898ce76aaf4a0f24d914c97ea6ed976d42fec6ad33fcbb0a1103e07b2b0" [[package]] name = "env_logger" @@ -342,138 +188,31 @@ dependencies = [ ] [[package]] -name = "event-listener" -version = "2.5.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f7531096570974c3a9dcf9e4b8e1cede1ec26cf5046219fb3b9d897503b9be59" - -[[package]] -name = "fastrand" -version = "1.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ca5faf057445ce5c9d4329e382b2ce7ca38550ef3b73a5348362d5f24e0c7fe3" -dependencies = [ - "instant", -] - -[[package]] -name = "fnv" -version = "1.0.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" - -[[package]] -name = "fuchsia-zircon" -version = "0.3.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2e9763c69ebaae630ba35f74888db465e49e259ba1bc0eda7d06f4a067615d82" -dependencies = [ - "bitflags", - "fuchsia-zircon-sys", -] - -[[package]] -name = "fuchsia-zircon-sys" -version = "0.3.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3dcaa9ae7725d12cdb85b3ad99a434db70b468c09ded17e012d86b5c1010f7a7" - -[[package]] -name = "futures-channel" -version = "0.3.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a7a4d35f7401e948629c9c3d6638fb9bf94e0b2121e96c3b428cc4e631f3eb74" -dependencies = [ - "futures-core", -] - -[[package]] -name = "futures-core" -version = "0.3.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d674eaa0056896d5ada519900dbf97ead2e46a7b6621e8160d79e2f2e1e2784b" - -[[package]] -name = "futures-io" -version = "0.3.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5fc94b64bb39543b4e432f1790b6bf18e3ee3b74653c5449f63310e9a74b123c" - -[[package]] -name = "futures-lite" -version = "1.11.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "381a7ad57b1bad34693f63f6f377e1abded7a9c85c9d3eb6771e11c60aaadab9" -dependencies = [ - "fastrand", - "futures-core", - "futures-io", - "memchr", - "parking", - "pin-project-lite", - "waker-fn", -] - -[[package]] -name = "futures-sink" -version = "0.3.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0d8764258ed64ebc5d9ed185cf86a95db5cac810269c5d20ececb32e0088abbd" - -[[package]] -name = "futures-task" -version = "0.3.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4dd26820a9f3637f1302da8bceba3ff33adbe53464b54ca24d4e2d4f1db30f94" -dependencies = [ - "once_cell", -] - -[[package]] -name = "futures-util" -version = "0.3.6" +name = "form_urlencoded" +version = "1.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8a894a0acddba51a2d49a6f4263b1e64b8c579ece8af50fa86503d52cd1eea34" +checksum = "5fc25a87fa4fd2094bffb06925852034d90a17f0d1e05197d4956d3555752191" dependencies = [ - "futures-core", - "futures-io", - "futures-sink", - "futures-task", - "memchr", - "pin-project", - "pin-utils", - "slab", + "matches", + "percent-encoding", ] [[package]] name = "getrandom" -version = "0.1.15" +version = "0.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fc587bc0ec293155d5bfa6b9891ec18a1e330c234f896ea47fbada4cadbe47e6" +checksum = "7fcd999463524c52659517fe2cea98493cfe485d10565e7b0fb07dbba7ad2753" dependencies = [ - "cfg-if", + "cfg-if 1.0.0", "libc", "wasi", ] -[[package]] -name = "gloo-timers" -version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "47204a46aaff920a1ea58b11d03dec6f704287d27561724a4631e450654a891f" -dependencies = [ - "futures-channel", - "futures-core", - "js-sys", - "wasm-bindgen", - "web-sys", -] - [[package]] name = "hermit-abi" -version = "0.1.17" +version = "0.1.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5aca5565f760fb5b220e499d72710ed156fdb74e631659e99377d9ebfbd13ae8" +checksum = "322f4de77956e22ed0e5032c359a0f1273f1f7f0d79bfa3b8ffbc730d7fbcc5c" dependencies = [ "libc", ] @@ -489,9 +228,9 @@ dependencies = [ [[package]] name = "idna" -version = "0.2.0" +version = "0.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "02e2673c30ee86b5b96a9cb52ad15718aa1f966f5ab9ad54a8b95d5ca33120a9" +checksum = "418a0a6fab821475f634efe3ccc45c013f742efe03d853e8d3355d5cb850ecf8" dependencies = [ "matches", "unicode-bidi", @@ -500,55 +239,18 @@ dependencies = [ [[package]] name = "instant" -version = "0.1.7" +version = "0.1.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "63312a18f7ea8760cdd0a7c5aac1a619752a246b833545e3e36d1f81f7cd9e66" +checksum = "61124eeebbd69b8190558df225adf7e4caafce0d743919e5d6b19652314ec5ec" dependencies = [ - "cfg-if", -] - -[[package]] -name = "iovec" -version = "0.1.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b2b3ea6ff95e175473f8ffe6a7eb7c00d054240321b84c57051175fe3c1e075e" -dependencies = [ - "libc", + "cfg-if 1.0.0", ] [[package]] name = "itoa" -version = "0.4.6" +version = "0.4.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dc6f3ad7b9d11a0c00842ff8de1b60ee58661048eb8049ed33c73594f359d7e6" - -[[package]] -name = "js-sys" -version = "0.3.45" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ca059e81d9486668f12d455a4ea6daa600bd408134cd17e3d3fb5a32d1f016f8" -dependencies = [ - "wasm-bindgen", -] - -[[package]] -name = "kernel32-sys" -version = "0.2.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7507624b29483431c0ba2d82aece8ca6cdba9382bff4ddd0f7490560c056098d" -dependencies = [ - "winapi 0.2.8", - "winapi-build", -] - -[[package]] -name = "kv-log-macro" -version = "1.0.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0de8b303297635ad57c9f5059fd9cee7a47f8e8daa09df0fcd07dd39fb22977f" -dependencies = [ - "log", -] +checksum = "dd25036021b0de88a0aff6b850051563c6516d0bf53f8638938edbb9de732736" [[package]] name = "lazy_static" @@ -558,9 +260,9 @@ checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" [[package]] name = "libc" -version = "0.2.79" +version = "0.2.95" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2448f6066e80e3bfc792e9c98bf705b4b0fc6e8ef5b43e5889aff0eaa9c58743" +checksum = "789da6d93f1b866ffe175afc5322a4d76c038605a1c3319bb57b06967ca98a36" [[package]] name = "lobby-model" @@ -573,20 +275,20 @@ dependencies = [ [[package]] name = "lock_api" -version = "0.4.1" +version = "0.4.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "28247cc5a5be2f05fbcd76dd0cf2c7d3b5400cb978a28042abcd4fa0b3f8261c" +checksum = "0382880606dff6d15c9476c416d18690b72742aa7b605bb6dd6ec9030fbf07eb" dependencies = [ "scopeguard", ] [[package]] name = "log" -version = "0.4.11" +version = "0.4.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4fabed175da42fed1fa0746b0ea71f412aa9d35e76e95e59b192c64b9dc2bf8b" +checksum = "51b9bbe6c47d51fc3e1a9b945965946b4c44142ab8792c50835a980d362c2710" dependencies = [ - "cfg-if", + "cfg-if 1.0.0", ] [[package]] @@ -603,9 +305,9 @@ checksum = "60302e4db3a61da70c0cb7991976248362f30319e88850c487b9b95bbf059e00" [[package]] name = "memchr" -version = "2.3.3" +version = "2.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3728d817d99e5ac407411fa471ff9800a778d88a24685968b36824eaf4bee400" +checksum = "b16bd47d9e329435e309c58469fe0791c2d0d1ba96ec0954152a5ae2b04387dc" [[package]] name = "memoffset" @@ -628,52 +330,10 @@ dependencies = [ "lobby-model", "log", "move-model", - "redis 0.17.0", + "redis", "redis_streams", ] -[[package]] -name = "mio" -version = "0.6.22" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fce347092656428bc8eaf6201042cb551b8d67855af7374542a92a0fbfcac430" -dependencies = [ - "cfg-if", - "fuchsia-zircon", - "fuchsia-zircon-sys", - "iovec", - "kernel32-sys", - "libc", - "log", - "miow", - "net2", - "slab", - "winapi 0.2.8", -] - -[[package]] -name = "mio-uds" -version = "0.6.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "afcb699eb26d4332647cc848492bbc15eafb26f08d0304550d5aa1f612e066f0" -dependencies = [ - "iovec", - "libc", - "mio", -] - -[[package]] -name = "miow" -version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8c1f2f3b1cf331de6896aabf6e9d55dca90356cc9960cca7eaaf408a355ae919" -dependencies = [ - "kernel32-sys", - "net2", - "winapi 0.2.8", - "ws2_32-sys", -] - [[package]] name = "move-model" version = "0.4.0" @@ -685,54 +345,11 @@ dependencies = [ "serde_derive", ] -[[package]] -name = "nb-connect" -version = "1.0.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8123a81538e457d44b933a02faf885d3fe8408806b23fa700e8f01c6c3a98998" -dependencies = [ - "libc", - "winapi 0.3.9", -] - -[[package]] -name = "net2" -version = "0.2.35" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3ebc3ec692ed7c9a255596c67808dee269f64655d8baf7b4f0638e51ba1d6853" -dependencies = [ - "cfg-if", - "libc", - "winapi 0.3.9", -] - -[[package]] -name = "num_cpus" -version = "1.13.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "05499f3756671c15885fee9034446956fff3f243d6077b91e5767df161f766b3" -dependencies = [ - "hermit-abi", - "libc", -] - -[[package]] -name = "once_cell" -version = "1.4.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "260e51e7efe62b592207e9e13a68e43692a7a279171d6ba57abd208bf23645ad" - -[[package]] -name = "parking" -version = "2.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "427c3892f9e783d91cc128285287e70a59e206ca452770ece88a76f7a3eddd72" - [[package]] name = "parking_lot" -version = "0.11.0" +version = "0.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a4893845fa2ca272e647da5d0e46660a314ead9c2fdd9a883aabc32e481a8733" +checksum = "6d7744ac029df22dca6284efe4e898991d28e3085c706c972bcd7da4a27a15eb" dependencies = [ "instant", "lock_api", @@ -741,17 +358,16 @@ dependencies = [ [[package]] name = "parking_lot_core" -version = "0.8.0" +version = "0.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c361aa727dd08437f2f1447be8b59a33b0edd15e0fcee698f935613d9efbca9b" +checksum = "fa7a782938e745763fe6907fc6ba86946d72f49fe7e21de074e08128a99fb018" dependencies = [ - "cfg-if", - "cloudabi", + "cfg-if 1.0.0", "instant", "libc", "redox_syscall", "smallvec", - "winapi 0.3.9", + "winapi", ] [[package]] @@ -760,62 +376,11 @@ version = "2.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d4fd5641d01c8f18a23da7b6fe29298ff4b55afcccdf78973b24cf3175fee32e" -[[package]] -name = "pin-project" -version = "0.4.27" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2ffbc8e94b38ea3d2d8ba92aea2983b503cd75d0888d75b86bb37970b5698e15" -dependencies = [ - "pin-project-internal", -] - -[[package]] -name = "pin-project-internal" -version = "0.4.27" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "65ad2ae56b6abe3a1ee25f15ee605bacadb9a764edaba9c2bf4103800d4a1895" -dependencies = [ - "proc-macro2", - "quote", - "syn", -] - -[[package]] -name = "pin-project-lite" -version = "0.1.10" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e555d9e657502182ac97b539fb3dae8b79cda19e3e4f8ffb5e8de4f18df93c95" - -[[package]] -name = "pin-utils" -version = "0.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" - -[[package]] -name = "polling" -version = "2.0.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a2a7bc6b2a29e632e45451c941832803a18cce6781db04de8a04696cdca8bde4" -dependencies = [ - "cfg-if", - "libc", - "log", - "wepoll-sys", - "winapi 0.3.9", -] - -[[package]] -name = "ppv-lite86" -version = "0.2.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c36fa947111f5c62a733b652544dd0016a43ce89619538a8ef92724a6f501a20" - [[package]] name = "proc-macro2" -version = "1.0.24" +version = "1.0.27" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1e0704ee1a7e00d7bb417d0770ea303c1bccbabf0ef1667dae92b5967f5f8a71" +checksum = "f0d8caf72986c1a598726adc988bb5984792ef84f5ee5aa50209145ee8077038" dependencies = [ "unicode-xid", ] @@ -828,9 +393,9 @@ checksum = "a1d01941d82fa2ab50be1e79e6714289dd7cde78eba4c074bc5a4374f650dfe0" [[package]] name = "quote" -version = "1.0.7" +version = "1.0.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "aa563d17ecb180e500da1cfd2b028310ac758de548efdd203e18f283af693f37" +checksum = "c3d0b9745dc2debf507c8422de05d7226cc1f0644216dfdfead988f9b1ab32a7" dependencies = [ "proc-macro2", ] @@ -846,122 +411,57 @@ dependencies = [ "scheduled-thread-pool", ] -[[package]] -name = "rand" -version = "0.7.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6a6b1679d49b24bbfe0c803429aa1874472f50d9b363131f0e89fc356b544d03" -dependencies = [ - "getrandom", - "libc", - "rand_chacha", - "rand_core", - "rand_hc", -] - -[[package]] -name = "rand_chacha" -version = "0.2.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f4c8ed856279c9737206bf725bf36935d8666ead7aa69b52be55af369d193402" -dependencies = [ - "ppv-lite86", - "rand_core", -] - -[[package]] -name = "rand_core" -version = "0.5.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "90bde5296fc891b0cef12a6d03ddccc162ce7b2aff54160af9338f8d40df6d19" -dependencies = [ - "getrandom", -] - -[[package]] -name = "rand_hc" -version = "0.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ca3129af7b92a17112d59ad498c6f81eaf463253766b90396d39ea7a39d6613c" -dependencies = [ - "rand_core", -] - -[[package]] -name = "redis" -version = "0.16.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7b94c6247d45d78d24481a5b7aca146f414ec0f5e39e175f294d1876b943eeeb" -dependencies = [ - "async-std", - "async-trait", - "bytes", - "combine", - "dtoa", - "futures-util", - "itoa", - "percent-encoding", - "pin-project-lite", - "sha1", - "tokio", - "tokio-util", - "url", -] - [[package]] name = "redis" -version = "0.17.0" +version = "0.20.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "95357caf2640abc54651b93c98a8df4fe1ccbf44b8e601ccdf43d5c1451f29ac" +checksum = "0a32cb439c4e89c1e6415e5b3b23d9d8cc6dc1bf5a8cade19adbd5418de803be" dependencies = [ - "async-std", "async-trait", - "bytes", "combine", "dtoa", - "futures-util", "itoa", "percent-encoding", - "pin-project-lite", "r2d2", "sha1", - "tokio", - "tokio-util", "url", ] [[package]] name = "redis_streams" -version = "0.3.0" -source = "git+https://github.com/Terkwood/BUGOUT?rev=92b8d91abba494737caca525ecb2a4f0d7044dbd#92b8d91abba494737caca525ecb2a4f0d7044dbd" +version = "1.0.0-a" +source = "git+https://github.com/Terkwood/BUGOUT?rev=9c5de56#9c5de569a751bcd2a8cfe42cb0efa2e5819a4a91" dependencies = [ - "redis 0.16.0", + "anyhow", + "redis", "uuid", ] [[package]] name = "redox_syscall" -version = "0.1.57" +version = "0.2.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "41cc0f7e4d5d4544e8861606a285bb08d3e70712ccc7d2b84d7c0ccfaf4b05ce" +checksum = "742739e41cd49414de871ea5e549afb7e2a3ac77b589bcbebe8c82fab37147fc" +dependencies = [ + "bitflags", +] [[package]] name = "regex" -version = "1.4.1" +version = "1.5.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8963b85b8ce3074fecffde43b4b0dded83ce2f367dc8d363afc56679f3ee820b" +checksum = "d07a8629359eb56f1e2fb1652bb04212c072a87ba68546a04065d525673ac461" dependencies = [ "aho-corasick", "memchr", "regex-syntax", - "thread_local", ] [[package]] name = "regex-syntax" -version = "0.6.20" +version = "0.6.25" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8cab7a364d15cde1e505267766a2d3c4e22a843e1a601f0fa7564c0f82ced11c" +checksum = "f497285884f3fcff424ffc933e56d7cbca511def0c9831a7f9b5f6153e3cc89b" [[package]] name = "scheduled-thread-pool" @@ -980,18 +480,15 @@ checksum = "d29ab0c6d3fc0ee92fe66e2d99f700eab17a8d57d1c1d3b748380fb20baa78cd" [[package]] name = "serde" -version = "1.0.117" +version = "1.0.126" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b88fa983de7720629c9387e9f517353ed404164b1e482c970a90c1a4aaf7dc1a" -dependencies = [ - "serde_derive", -] +checksum = "ec7505abeacaec74ae4778d9d9328fe5a5d04253220a85c4ee022239fc996d03" [[package]] name = "serde_derive" -version = "1.0.117" +version = "1.0.126" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cbd1ae72adb44aab48f325a02444a5fc079349a8d804c1fc922aed3f7454c74e" +checksum = "963a7dbc9895aeac7ac90e74f34a5d5261828f79df35cbed41e10189d3804d43" dependencies = [ "proc-macro2", "quote", @@ -1004,23 +501,17 @@ version = "0.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2579985fda508104f7587689507983eadd6a6e84dd35d6d115361f530916fa0d" -[[package]] -name = "slab" -version = "0.4.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c111b5bd5695e56cffe5129854aa230b39c93a305372fdbb2668ca2394eea9f8" - [[package]] name = "smallvec" -version = "1.4.2" +version = "1.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fbee7696b84bbf3d89a1c2eccff0850e3047ed46bfcd2e92c29a2d074d57e252" +checksum = "fe0f37c9e8f3c5a4a66ad655a93c74daac4ad00c441533bf5c6e7990bb42604e" [[package]] name = "syn" -version = "1.0.45" +version = "1.0.72" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ea9c5432ff16d6152371f808fb5a871cd67368171b09bb21b43df8e4a47a3556" +checksum = "a1e8cdbefb79a9a5a65e0db8b47b723ee907b7c7f8496c76a1770b5c310bab82" dependencies = [ "proc-macro2", "quote", @@ -1029,90 +520,59 @@ dependencies = [ [[package]] name = "termcolor" -version = "1.1.0" +version = "1.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bb6bfa289a4d7c5766392812c0a1f4c1ba45afa1ad47803c11e1f407d846d75f" +checksum = "2dfed899f0eb03f32ee8c6a0aabdb8a7949659e3466561fc0adf54e26d88c5f4" dependencies = [ "winapi-util", ] -[[package]] -name = "thread_local" -version = "1.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d40c6d1b69745a6ec6fb1ca717914848da4b44ae29d9b3080cbee91d72a69b14" -dependencies = [ - "lazy_static", -] - [[package]] name = "tinyvec" -version = "0.3.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "238ce071d267c5710f9d31451efec16c5ee22de34df17cc05e56cbc92e967117" - -[[package]] -name = "tokio" -version = "0.2.22" +version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5d34ca54d84bf2b5b4d7d31e901a8464f7b60ac145a284fba25ceb801f2ddccd" +checksum = "5b5220f05bb7de7f3f53c7c065e1199b3172696fe2db9f9c4d8ad9b4ee74c342" dependencies = [ - "bytes", - "fnv", - "futures-core", - "iovec", - "lazy_static", - "libc", - "memchr", - "mio", - "mio-uds", - "pin-project-lite", + "tinyvec_macros", ] [[package]] -name = "tokio-util" -version = "0.3.1" +name = "tinyvec_macros" +version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "be8242891f2b6cbef26a2d7e8605133c2c554cd35b3e4948ea892d6d68436499" -dependencies = [ - "bytes", - "futures-core", - "futures-sink", - "log", - "pin-project-lite", - "tokio", -] +checksum = "cda74da7e1a664f795bb1f8a87ec406fb89a02522cf6e50620d016add6dbbf5c" [[package]] name = "unicode-bidi" -version = "0.3.4" +version = "0.3.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "49f2bd0c6468a8230e1db229cff8029217cf623c767ea5d60bfbd42729ea54d5" +checksum = "eeb8be209bb1c96b7c177c7420d26e04eccacb0eeae6b980e35fcb74678107e0" dependencies = [ "matches", ] [[package]] name = "unicode-normalization" -version = "0.1.13" +version = "0.1.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6fb19cf769fa8c6a80a162df694621ebeb4dafb606470b2b2fce0be40a98a977" +checksum = "33717dca7ac877f497014e10d73f3acf948c342bee31b5ca7892faf94ccc6b49" dependencies = [ "tinyvec", ] [[package]] name = "unicode-xid" -version = "0.2.1" +version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f7fe0bb3479651439c9112f72b6c505038574c9fbb575ed1bf3b797fa39dd564" +checksum = "8ccb82d61f80a663efe1f787a51b16b5a51e3314d6ac365b08639f52387b33f3" [[package]] name = "url" -version = "2.1.1" +version = "2.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "829d4a8476c35c9bf0bbce5a3b23f4106f79728039b726d292bb93bc106787cb" +checksum = "a507c383b2d33b5fc35d1861e77e6b383d158b2da5e14fe51b83dfedf6fd578c" dependencies = [ + "form_urlencoded", "idna", "matches", "percent-encoding", @@ -1120,122 +580,19 @@ dependencies = [ [[package]] name = "uuid" -version = "0.8.1" +version = "0.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9fde2f6a4bea1d6e007c4ad38c6839fa71cbb63b6dbf5b595aa38dc9b1093c11" +checksum = "bc5cf98d8186244414c848017f0e2676b3fcb46807f6668a97dfe67359a3c4b7" dependencies = [ - "rand", + "getrandom", "serde", ] -[[package]] -name = "vec-arena" -version = "1.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eafc1b9b2dfc6f5529177b62cf806484db55b32dc7c9658a118e11bbeb33061d" - -[[package]] -name = "waker-fn" -version = "1.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9d5b2c62b4012a3e1eca5a7e077d13b3bf498c4073e33ccd58626607748ceeca" - [[package]] name = "wasi" -version = "0.9.0+wasi-snapshot-preview1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cccddf32554fecc6acb585f82a32a72e28b48f8c4c1883ddfeeeaa96f7d8e519" - -[[package]] -name = "wasm-bindgen" -version = "0.2.68" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1ac64ead5ea5f05873d7c12b545865ca2b8d28adfc50a49b84770a3a97265d42" -dependencies = [ - "cfg-if", - "wasm-bindgen-macro", -] - -[[package]] -name = "wasm-bindgen-backend" -version = "0.2.68" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f22b422e2a757c35a73774860af8e112bff612ce6cb604224e8e47641a9e4f68" -dependencies = [ - "bumpalo", - "lazy_static", - "log", - "proc-macro2", - "quote", - "syn", - "wasm-bindgen-shared", -] - -[[package]] -name = "wasm-bindgen-futures" -version = "0.4.18" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b7866cab0aa01de1edf8b5d7936938a7e397ee50ce24119aef3e1eaa3b6171da" -dependencies = [ - "cfg-if", - "js-sys", - "wasm-bindgen", - "web-sys", -] - -[[package]] -name = "wasm-bindgen-macro" -version = "0.2.68" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6b13312a745c08c469f0b292dd2fcd6411dba5f7160f593da6ef69b64e407038" -dependencies = [ - "quote", - "wasm-bindgen-macro-support", -] - -[[package]] -name = "wasm-bindgen-macro-support" -version = "0.2.68" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f249f06ef7ee334cc3b8ff031bfc11ec99d00f34d86da7498396dc1e3b1498fe" -dependencies = [ - "proc-macro2", - "quote", - "syn", - "wasm-bindgen-backend", - "wasm-bindgen-shared", -] - -[[package]] -name = "wasm-bindgen-shared" -version = "0.2.68" +version = "0.10.2+wasi-snapshot-preview1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1d649a3145108d7d3fbcde896a468d1bd636791823c9921135218ad89be08307" - -[[package]] -name = "web-sys" -version = "0.3.45" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4bf6ef87ad7ae8008e15a355ce696bed26012b7caa21605188cfd8214ab51e2d" -dependencies = [ - "js-sys", - "wasm-bindgen", -] - -[[package]] -name = "wepoll-sys" -version = "3.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "142bc2cba3fe88be1a8fcb55c727fa4cd5b0cf2d7438722792e22f26f04bc1e0" -dependencies = [ - "cc", -] - -[[package]] -name = "winapi" -version = "0.2.8" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "167dc9d6949a9b857f3451275e911c3f44255842c1f7a76f33c55103a909087a" +checksum = "fd6fbd9a79829dd1ad0cc20627bf1ed606756a7f77edff7b66b7064f9cb327c6" [[package]] name = "winapi" @@ -1247,12 +604,6 @@ dependencies = [ "winapi-x86_64-pc-windows-gnu", ] -[[package]] -name = "winapi-build" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2d315eee3b34aca4797b2da6b13ed88266e6d612562a0c46390af8299fc699bc" - [[package]] name = "winapi-i686-pc-windows-gnu" version = "0.4.0" @@ -1265,7 +616,7 @@ version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "70ec6ce85bb158151cae5e5c87f95a8e97d2c0c4b001223f33a334e3ce5de178" dependencies = [ - "winapi 0.3.9", + "winapi", ] [[package]] @@ -1273,13 +624,3 @@ name = "winapi-x86_64-pc-windows-gnu" version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" - -[[package]] -name = "ws2_32-sys" -version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d59cefebd0c892fa2dd6de581e937301d8552cb44489cdff035c6187cb63fa5e" -dependencies = [ - "winapi 0.2.8", - "winapi-build", -] diff --git a/micro-game-lobby/Cargo.toml b/micro-game-lobby/Cargo.toml index e8a3e3ef..d6548079 100644 --- a/micro-game-lobby/Cargo.toml +++ b/micro-game-lobby/Cargo.toml @@ -13,5 +13,5 @@ env_logger = "0.7.1" lobby-model = {path = "lobby-model"} log = "0.4.8" move-model = {git = "https://github.com/Terkwood/BUGOUT", rev = "20e6620"} -redis = {version = "0.17.0", features = ["r2d2"]} -redis_streams = {git = "https://github.com/Terkwood/BUGOUT", version = "0.3.0", rev = "92b8d91abba494737caca525ecb2a4f0d7044dbd"} +redis = {version = "0.20.0", features = ["r2d2"]} +redis_streams = {git = "https://github.com/Terkwood/BUGOUT", rev = "9c5de56"} diff --git a/micro-game-lobby/src/components.rs b/micro-game-lobby/src/components.rs index 212032bc..4a758bab 100644 --- a/micro-game-lobby/src/components.rs +++ b/micro-game-lobby/src/components.rs @@ -1,13 +1,11 @@ use crate::repo::GameLobbyRepo; -use crate::stream::{XAck, XAdd, XRead}; +use crate::stream::XAdd; use std::rc::Rc; pub struct Components { pub game_lobby_repo: Box, - pub xread: Box, pub xadd: Box, - pub xack: Box, } const REDIS_URL: &str = "redis://redis/"; @@ -20,9 +18,7 @@ impl Components { pub fn new(client: Rc) -> Self { Components { game_lobby_repo: Box::new(client.clone()), - xread: Box::new(client.clone()), - xadd: Box::new(client.clone()), - xack: Box::new(client), + xadd: Box::new(client), } } } diff --git a/micro-game-lobby/src/stream/mod.rs b/micro-game-lobby/src/stream/mod.rs index 8c2f5708..d090e933 100644 --- a/micro-game-lobby/src/stream/mod.rs +++ b/micro-game-lobby/src/stream/mod.rs @@ -1,12 +1,8 @@ mod init; -mod xack; mod xadd; -mod xread; pub use init::*; -pub use xack::*; pub use xadd::*; -pub use xread::*; use crate::components::Components; use crate::game_lobby::GameLobbyOps; @@ -16,10 +12,18 @@ use lobby_model::api::*; use lobby_model::*; use log::{error, info, trace}; use move_model::GameState; -use redis_streams::XReadEntryId; +use redis_streams::XId; pub const GROUP_NAME: &str = "micro-game-lobby"; +#[derive(Clone, Debug)] +pub enum StreamInput { + FPG(FindPublicGame), + CG(CreateGame), + JPG(JoinPrivateGame), + SD(SessionDisconnected), +} + #[derive(Debug, Clone, PartialEq)] pub enum StreamOutput { WFO(WaitForOpponent), @@ -29,24 +33,22 @@ pub enum StreamOutput { } pub fn process(reg: &Components) { - let mut unacked = Unacknowledged::default(); loop { - match reg.xread.xread_sorted() { + todo!("generic consumer") + /* + match todo!("was xread sorted") { Ok(xrr) => { for (xid, data) in xrr { info!("🧮 Processing {:?}", &data); consume(xid, &data, ®); - unacked.push(xid, data); } } Err(e) => error!("Stream err {:?}", e), - } - - unacked.ack_all(®) + }*/ } } -fn consume(_eid: XReadEntryId, event: &StreamInput, reg: &Components) { +fn consume(_eid: XId, event: &StreamInput, reg: &Components) { match event { StreamInput::FPG(fpg) => consume_fpg(fpg, reg), StreamInput::CG(cg) => consume_cg(cg, reg), @@ -203,7 +205,7 @@ mod test { use crate::components::Components; use crate::repo::*; use crossbeam_channel::{select, unbounded, Sender}; - use redis_streams::XReadEntryId; + use redis_streams::XId; use std::sync::{Arc, Mutex}; use std::time::Duration; @@ -224,29 +226,6 @@ mod test { } } - struct FakeXRead { - sorted_data: Arc>>, - } - impl XRead for FakeXRead { - /// Be careful, this implementation assumes - /// that the underlying data is pre-sorted - fn xread_sorted( - &self, - ) -> Result, XReadErr> { - { - let mut data = self.sorted_data.lock().expect("lock"); - if data.is_empty() { - // stop the test thread from spinning like crazy - std::thread::sleep(Duration::from_millis(1)); - Ok(vec![]) - } else { - let result = data.clone(); - *data = vec![]; - Ok(result) - } - } - } - } struct FakeXAdd(Sender); impl XAdd for FakeXAdd { fn xadd(&self, data: StreamOutput) -> Result<(), XAddErr> { @@ -255,60 +234,9 @@ mod test { } } - enum AckType { - FPG, - CG, - JPG, - SD, - } - struct ItWasAcked { - ack_type: AckType, - xids: Vec, - } - struct FakeXAck(Sender); - impl XAck for FakeXAck { - fn ack_find_public_game(&self, xids: &[XReadEntryId]) -> Result<(), StreamAckErr> { - if let Err(_) = self.0.send(ItWasAcked { - ack_type: AckType::FPG, - xids: xids.to_vec(), - }) {} - Ok(()) - } - - fn ack_join_priv_game(&self, xids: &[XReadEntryId]) -> Result<(), StreamAckErr> { - Ok(self - .0 - .send(ItWasAcked { - ack_type: AckType::JPG, - xids: xids.to_vec(), - }) - .expect("send")) - } - - fn ack_create_game(&self, xids: &[XReadEntryId]) -> Result<(), StreamAckErr> { - Ok(self - .0 - .send(ItWasAcked { - ack_type: AckType::CG, - xids: xids.to_vec(), - }) - .expect("send")) - } - - fn ack_session_disconnected(&self, xids: &[XReadEntryId]) -> Result<(), StreamAckErr> { - Ok(self - .0 - .send(ItWasAcked { - ack_type: AckType::SD, - xids: xids.to_vec(), - }) - .expect("send")) - } - } #[test] fn test_process() { let (xadd_in, xadd_out) = unbounded(); - let (xack_in, xack_out) = unbounded(); let sorted_fake_stream = Arc::new(Mutex::new(vec![])); @@ -323,11 +251,7 @@ mod test { thread::spawn(move || { let components = Components { game_lobby_repo: Box::new(FakeGameLobbyRepo { contents: fgl }), - xread: Box::new(FakeXRead { - sorted_data: sfs.clone(), - }), xadd: Box::new(FakeXAdd(xadd_in)), - xack: Box::new(FakeXAck(xack_in)), }; process(&components); }); @@ -353,16 +277,6 @@ mod test { )); thread::sleep(timeout); - // We should have seen a single XACK for a find public game record - select! { - recv(xack_out) -> msg => match msg { - Ok(ItWasAcked { ack_type: AckType::FPG , xids }) => { - assert_eq!(xids.len(), 1); - assert_eq!(xids[0], xid0) - } - _ => panic!("fpg ack") - } - } // The game lobby repo should now contain one game assert_eq!( @@ -406,8 +320,8 @@ mod test { } } - fn quick_eid(ms: u64) -> XReadEntryId { - XReadEntryId { + fn quick_eid(ms: u64) -> XId { + XId { millis_time: ms, seq_no: 0, } diff --git a/micro-game-lobby/src/stream/xack.rs b/micro-game-lobby/src/stream/xack.rs deleted file mode 100644 index 884c2c90..00000000 --- a/micro-game-lobby/src/stream/xack.rs +++ /dev/null @@ -1,111 +0,0 @@ -use crate::stream::StreamInput; -use crate::topics; -use crate::Components; -use log::error; -use redis::{Client, Commands}; -use redis_streams::XReadEntryId; - -pub trait XAck { - fn ack_find_public_game(&self, xids: &[XReadEntryId]) -> Result<(), StreamAckErr>; - fn ack_join_priv_game(&self, xids: &[XReadEntryId]) -> Result<(), StreamAckErr>; - fn ack_create_game(&self, xids: &[XReadEntryId]) -> Result<(), StreamAckErr>; - fn ack_session_disconnected(&self, xids: &[XReadEntryId]) -> Result<(), StreamAckErr>; -} - -impl XAck for std::rc::Rc { - fn ack_find_public_game(&self, xids: &[XReadEntryId]) -> Result<(), StreamAckErr> { - ack(self, topics::FIND_PUBLIC_GAME, xids) - } - - fn ack_join_priv_game(&self, xids: &[XReadEntryId]) -> Result<(), StreamAckErr> { - ack(self, topics::JOIN_PRIVATE_GAME, xids) - } - - fn ack_create_game(&self, xids: &[XReadEntryId]) -> Result<(), StreamAckErr> { - ack(self, topics::CREATE_GAME, xids) - } - - fn ack_session_disconnected(&self, xids: &[XReadEntryId]) -> Result<(), StreamAckErr> { - ack(self, topics::SESSION_DISCONNECTED, xids) - } -} - -#[derive(Debug)] -pub struct StreamAckErr; - -pub struct Unacknowledged { - fpg: Vec, - jpg: Vec, - cg: Vec, - sd: Vec, -} - -impl Unacknowledged { - pub fn ack_all(&mut self, reg: &Components) { - if !self.fpg.is_empty() { - if let Err(_e) = reg.xack.ack_find_public_game(&self.fpg) { - error!("ack for fpg failed") - } else { - self.fpg.clear(); - } - } - - if !self.jpg.is_empty() { - if let Err(_e) = reg.xack.ack_join_priv_game(&self.jpg) { - error!("ack for jpg failed") - } else { - self.jpg.clear(); - } - } - if !self.cg.is_empty() { - if let Err(_e) = reg.xack.ack_create_game(&self.cg) { - error!("ack for create game failed") - } else { - self.cg.clear(); - } - } - if !self.sd.is_empty() { - if let Err(_e) = reg.xack.ack_session_disconnected(&self.sd) { - error!("ack for session disconn failed") - } else { - self.sd.clear(); - } - } - } - pub fn push(&mut self, xid: XReadEntryId, event: StreamInput) { - match event { - StreamInput::FPG(_) => self.fpg.push(xid), - StreamInput::JPG(_) => self.jpg.push(xid), - StreamInput::CG(_) => self.cg.push(xid), - StreamInput::SD(_) => self.sd.push(xid), - } - } -} - -const INIT_ACK_CAPACITY: usize = 50; -impl Default for Unacknowledged { - fn default() -> Self { - Self { - fpg: Vec::with_capacity(INIT_ACK_CAPACITY), - jpg: Vec::with_capacity(INIT_ACK_CAPACITY), - cg: Vec::with_capacity(INIT_ACK_CAPACITY), - sd: Vec::with_capacity(INIT_ACK_CAPACITY), - } - } -} - -fn ack(client: &Client, key: &str, ids: &[XReadEntryId]) -> Result<(), StreamAckErr> { - if let Ok(mut conn) = client.get_connection() { - let idstrs: Vec = ids.iter().map(|id| id.to_string()).collect(); - let _: usize = conn.xack(key, super::GROUP_NAME, &idstrs)?; - Ok(()) - } else { - Err(StreamAckErr) - } -} - -impl From for StreamAckErr { - fn from(_: redis::RedisError) -> Self { - Self - } -} diff --git a/micro-game-lobby/src/stream/xread.rs b/micro-game-lobby/src/stream/xread.rs index 2114adf9..5f9cd16e 100644 --- a/micro-game-lobby/src/stream/xread.rs +++ b/micro-game-lobby/src/stream/xread.rs @@ -116,10 +116,3 @@ pub enum XReadDeserErr { DataDeser(String), } -#[derive(Clone, Debug)] -pub enum StreamInput { - FPG(FindPublicGame), - CG(CreateGame), - JPG(JoinPrivateGame), - SD(SessionDisconnected), -} From a2ae1b85a4aca76b922a325bce08fe6594ef1cc8 Mon Sep 17 00:00:00 2001 From: terkwood <38859656+Terkwood@users.noreply.github.com> Date: Sat, 29 May 2021 09:22:53 -0400 Subject: [PATCH 31/54] provide trait --- redis_streams/src/sorted_streams.rs | 23 ++++++++++++++++------- 1 file changed, 16 insertions(+), 7 deletions(-) diff --git a/redis_streams/src/sorted_streams.rs b/redis_streams/src/sorted_streams.rs index 497ad4df..ab2cad2d 100644 --- a/redis_streams/src/sorted_streams.rs +++ b/redis_streams/src/sorted_streams.rs @@ -7,7 +7,15 @@ use redis::{ use std::collections::HashMap; use std::str::FromStr; -pub struct SortedStreams<'a, F> +trait SortedStreams { + /// "XREADGROUP >" across streams, handle all the messages in time order, + /// and acknowledge them all. The XACK calls happen once per stream, + /// acknowleding all XIds for that stream at once. The XACK calls happen + /// after _all_ individual message handlers have been invoked. + fn consume(&mut self) -> Result<()>; +} + +pub struct RedisSortedStreams<'a, F> where F: FnMut(XId, &Message) -> Result<()>, { @@ -17,7 +25,7 @@ where pub redis: &'a mut Connection, } -impl<'a, F> SortedStreams<'a, F> +impl<'a, F> RedisSortedStreams<'a, F> where F: FnMut(XId, &Message) -> Result<()>, { @@ -40,12 +48,13 @@ where handlers, }) } +} - /// "XREADGROUP >" across streams, handle all the messages in time order, - /// and acknowledge them all. The XACK calls happen once per stream, - /// acknowleding all XIds for that stream at once. The XACK calls happen - /// after _all_ individual message handlers have been invoked. - pub fn consume(&mut self) -> Result<()> { +impl<'a, F> SortedStreams for RedisSortedStreams<'a, F> +where + F: FnMut(XId, &Message) -> Result<()>, +{ + fn consume(&mut self) -> Result<()> { let mut unacked = Unacknowledged::default(); let opts = StreamReadOptions::default() .block(self.block_ms) From 7d265268a47b8b34171b6198f1ae508bac8c07bd Mon Sep 17 00:00:00 2001 From: terkwood <38859656+Terkwood@users.noreply.github.com> Date: Sat, 29 May 2021 09:27:29 -0400 Subject: [PATCH 32/54] make trait public --- redis_streams/src/sorted_streams.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/redis_streams/src/sorted_streams.rs b/redis_streams/src/sorted_streams.rs index ab2cad2d..0b33c3ea 100644 --- a/redis_streams/src/sorted_streams.rs +++ b/redis_streams/src/sorted_streams.rs @@ -7,7 +7,7 @@ use redis::{ use std::collections::HashMap; use std::str::FromStr; -trait SortedStreams { +pub trait SortedStreams { /// "XREADGROUP >" across streams, handle all the messages in time order, /// and acknowledge them all. The XACK calls happen once per stream, /// acknowleding all XIds for that stream at once. The XACK calls happen From 8e58b2f0f705e4815ec946fc790a0209582d347c Mon Sep 17 00:00:00 2001 From: terkwood <38859656+Terkwood@users.noreply.github.com> Date: Sat, 29 May 2021 09:31:28 -0400 Subject: [PATCH 33/54] hack --- micro-game-lobby/Cargo.lock | 2 +- micro-game-lobby/Cargo.toml | 2 +- micro-game-lobby/src/stream/mod.rs | 12 ++++++++---- 3 files changed, 10 insertions(+), 6 deletions(-) diff --git a/micro-game-lobby/Cargo.lock b/micro-game-lobby/Cargo.lock index 162fe479..1663d509 100644 --- a/micro-game-lobby/Cargo.lock +++ b/micro-game-lobby/Cargo.lock @@ -430,7 +430,7 @@ dependencies = [ [[package]] name = "redis_streams" version = "1.0.0-a" -source = "git+https://github.com/Terkwood/BUGOUT?rev=9c5de56#9c5de569a751bcd2a8cfe42cb0efa2e5819a4a91" +source = "git+https://github.com/Terkwood/BUGOUT?rev=7d26526#7d265268a47b8b34171b6198f1ae508bac8c07bd" dependencies = [ "anyhow", "redis", diff --git a/micro-game-lobby/Cargo.toml b/micro-game-lobby/Cargo.toml index d6548079..61dd9d01 100644 --- a/micro-game-lobby/Cargo.toml +++ b/micro-game-lobby/Cargo.toml @@ -14,4 +14,4 @@ lobby-model = {path = "lobby-model"} log = "0.4.8" move-model = {git = "https://github.com/Terkwood/BUGOUT", rev = "20e6620"} redis = {version = "0.20.0", features = ["r2d2"]} -redis_streams = {git = "https://github.com/Terkwood/BUGOUT", rev = "9c5de56"} +redis_streams = {git = "https://github.com/Terkwood/BUGOUT", rev = "7d26526"} diff --git a/micro-game-lobby/src/stream/mod.rs b/micro-game-lobby/src/stream/mod.rs index d090e933..f438f66e 100644 --- a/micro-game-lobby/src/stream/mod.rs +++ b/micro-game-lobby/src/stream/mod.rs @@ -234,6 +234,13 @@ mod test { } } + struct FakeSortedStreams; + impl redis_streams::SortedStreams for FakeSortedStreams { + fn consume(&mut self) -> anyhow::Result<()> { + todo!() + } + } + #[test] fn test_process() { let (xadd_in, xadd_out) = unbounded(); @@ -245,7 +252,6 @@ mod test { // set up a loop to process game lobby requests let fake_game_lobby_contents = Arc::new(Mutex::new(GameLobby::default())); - let sfs = sorted_fake_stream.clone(); let fgl = fake_game_lobby_contents.clone(); thread::spawn(move || { @@ -256,9 +262,7 @@ mod test { process(&components); }); - // emit some events in a time-ordered fashion - // (we need to use time-ordered push since the - // FakeXRead impl won't sort its underlying data ) + // emit some events let mut fake_time_ms = 100; let incr_ms = 100; From d95de671fc11908109e853335420c45177474d20 Mon Sep 17 00:00:00 2001 From: terkwood <38859656+Terkwood@users.noreply.github.com> Date: Sat, 29 May 2021 09:32:00 -0400 Subject: [PATCH 34/54] expose anyhow --- redis_streams/src/lib.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/redis_streams/src/lib.rs b/redis_streams/src/lib.rs index f6a3a3f4..ec287e4e 100644 --- a/redis_streams/src/lib.rs +++ b/redis_streams/src/lib.rs @@ -2,6 +2,7 @@ mod consumer_group_opts; mod sorted_streams; mod stream_handler; +pub use anyhow; pub use consumer_group_opts::*; pub use sorted_streams::*; pub use stream_handler::*; From 14493f0d9ae0da8b27f3663b1d7901e765268f12 Mon Sep 17 00:00:00 2001 From: terkwood <38859656+Terkwood@users.noreply.github.com> Date: Sat, 29 May 2021 09:34:44 -0400 Subject: [PATCH 35/54] hack --- micro-game-lobby/Cargo.lock | 2 +- micro-game-lobby/Cargo.toml | 2 +- micro-game-lobby/src/components.rs | 4 +++- micro-game-lobby/src/stream/mod.rs | 3 ++- 4 files changed, 7 insertions(+), 4 deletions(-) diff --git a/micro-game-lobby/Cargo.lock b/micro-game-lobby/Cargo.lock index 1663d509..9caf1d74 100644 --- a/micro-game-lobby/Cargo.lock +++ b/micro-game-lobby/Cargo.lock @@ -430,7 +430,7 @@ dependencies = [ [[package]] name = "redis_streams" version = "1.0.0-a" -source = "git+https://github.com/Terkwood/BUGOUT?rev=7d26526#7d265268a47b8b34171b6198f1ae508bac8c07bd" +source = "git+https://github.com/Terkwood/BUGOUT?rev=d95de67#d95de671fc11908109e853335420c45177474d20" dependencies = [ "anyhow", "redis", diff --git a/micro-game-lobby/Cargo.toml b/micro-game-lobby/Cargo.toml index 61dd9d01..7350e39c 100644 --- a/micro-game-lobby/Cargo.toml +++ b/micro-game-lobby/Cargo.toml @@ -14,4 +14,4 @@ lobby-model = {path = "lobby-model"} log = "0.4.8" move-model = {git = "https://github.com/Terkwood/BUGOUT", rev = "20e6620"} redis = {version = "0.20.0", features = ["r2d2"]} -redis_streams = {git = "https://github.com/Terkwood/BUGOUT", rev = "7d26526"} +redis_streams = {git = "https://github.com/Terkwood/BUGOUT", rev = "d95de67"} diff --git a/micro-game-lobby/src/components.rs b/micro-game-lobby/src/components.rs index 4a758bab..01eda4ef 100644 --- a/micro-game-lobby/src/components.rs +++ b/micro-game-lobby/src/components.rs @@ -1,11 +1,12 @@ use crate::repo::GameLobbyRepo; use crate::stream::XAdd; - +use redis_streams::{RedisSortedStreams, SortedStreams}; use std::rc::Rc; pub struct Components { pub game_lobby_repo: Box, pub xadd: Box, + pub sorted_streams: Box, } const REDIS_URL: &str = "redis://redis/"; @@ -19,6 +20,7 @@ impl Components { Components { game_lobby_repo: Box::new(client.clone()), xadd: Box::new(client), + sorted_streams: todo!(), } } } diff --git a/micro-game-lobby/src/stream/mod.rs b/micro-game-lobby/src/stream/mod.rs index f438f66e..4d9ac361 100644 --- a/micro-game-lobby/src/stream/mod.rs +++ b/micro-game-lobby/src/stream/mod.rs @@ -236,7 +236,7 @@ mod test { struct FakeSortedStreams; impl redis_streams::SortedStreams for FakeSortedStreams { - fn consume(&mut self) -> anyhow::Result<()> { + fn consume(&mut self) -> redis_streams::anyhow::Result<()> { todo!() } } @@ -258,6 +258,7 @@ mod test { let components = Components { game_lobby_repo: Box::new(FakeGameLobbyRepo { contents: fgl }), xadd: Box::new(FakeXAdd(xadd_in)), + sorted_streams: Box::new(FakeSortedStreams) }; process(&components); }); From 2420bdd686b65a0961082660e8ee3442e07c69e9 Mon Sep 17 00:00:00 2001 From: terkwood <38859656+Terkwood@users.noreply.github.com> Date: Sat, 29 May 2021 09:48:56 -0400 Subject: [PATCH 36/54] stub out components --- micro-game-lobby/src/components.rs | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/micro-game-lobby/src/components.rs b/micro-game-lobby/src/components.rs index 01eda4ef..64be25b0 100644 --- a/micro-game-lobby/src/components.rs +++ b/micro-game-lobby/src/components.rs @@ -17,10 +17,27 @@ pub fn redis_client() -> Rc { impl Components { pub fn new(client: Rc) -> Self { + let mut conn = client.get_connection().expect("redis conn"); + let stream_handlers: Vec<( + &str, + Box< + FnMut( + redis_streams::XId, + &redis_streams::Message, + ) -> Result<(), redis_streams::anyhow::Error>, + >, + )> = vec![ + ("some-stream", todo!()), + ("another-stream", todo!()), + ("fix-the-names", todo!()), + ]; + let sorted_streams = + RedisSortedStreams::xgroup_create_mkstreams(stream_handlers, todo!("opts"), &mut conn) + .expect("stream creation"); Components { game_lobby_repo: Box::new(client.clone()), xadd: Box::new(client), - sorted_streams: todo!(), + sorted_streams: Box::new(sorted_streams), } } } From 6d4cfd3a753c7b88e5a886df7f4efa81f540df4a Mon Sep 17 00:00:00 2001 From: terkwood <38859656+Terkwood@users.noreply.github.com> Date: Sat, 29 May 2021 12:30:38 -0400 Subject: [PATCH 37/54] halp --- micro-game-lobby/src/components.rs | 23 +----- micro-game-lobby/src/main.rs | 3 +- micro-game-lobby/src/stream/mod.rs | 124 +++++++++++++++++------------ 3 files changed, 78 insertions(+), 72 deletions(-) diff --git a/micro-game-lobby/src/components.rs b/micro-game-lobby/src/components.rs index 64be25b0..f67deda2 100644 --- a/micro-game-lobby/src/components.rs +++ b/micro-game-lobby/src/components.rs @@ -6,7 +6,6 @@ use std::rc::Rc; pub struct Components { pub game_lobby_repo: Box, pub xadd: Box, - pub sorted_streams: Box, } const REDIS_URL: &str = "redis://redis/"; @@ -15,29 +14,13 @@ pub fn redis_client() -> Rc { Rc::new(redis::Client::open(REDIS_URL).expect("redis client")) } + impl Components { pub fn new(client: Rc) -> Self { - let mut conn = client.get_connection().expect("redis conn"); - let stream_handlers: Vec<( - &str, - Box< - FnMut( - redis_streams::XId, - &redis_streams::Message, - ) -> Result<(), redis_streams::anyhow::Error>, - >, - )> = vec![ - ("some-stream", todo!()), - ("another-stream", todo!()), - ("fix-the-names", todo!()), - ]; - let sorted_streams = - RedisSortedStreams::xgroup_create_mkstreams(stream_handlers, todo!("opts"), &mut conn) - .expect("stream creation"); + Components { game_lobby_repo: Box::new(client.clone()), - xadd: Box::new(client), - sorted_streams: Box::new(sorted_streams), + xadd: Box::new(client), } } } diff --git a/micro-game-lobby/src/main.rs b/micro-game-lobby/src/main.rs index a3a11021..554f5168 100644 --- a/micro-game-lobby/src/main.rs +++ b/micro-game-lobby/src/main.rs @@ -9,5 +9,6 @@ fn main() { let client = redis_client(); let components = Components::new(client.clone()); stream::create_consumer_group(&client); - stream::process(&components) + let mut lobby_streams = stream::LobbyStreams::new(components, client); + lobby_streams.process() } diff --git a/micro-game-lobby/src/stream/mod.rs b/micro-game-lobby/src/stream/mod.rs index 4d9ac361..847ce053 100644 --- a/micro-game-lobby/src/stream/mod.rs +++ b/micro-game-lobby/src/stream/mod.rs @@ -2,6 +2,7 @@ mod init; mod xadd; pub use init::*; +use redis_streams::SortedStreams; pub use xadd::*; use crate::components::Components; @@ -32,66 +33,86 @@ pub enum StreamOutput { LOG(GameState), } -pub fn process(reg: &Components) { - loop { - todo!("generic consumer") - /* - match todo!("was xread sorted") { - Ok(xrr) => { - for (xid, data) in xrr { - info!("🧮 Processing {:?}", &data); - consume(xid, &data, ®); - } +pub struct LobbyStreams { + pub reg: Components, + pub sorted_streams: Box, +} + +use redis_streams::{anyhow, Message, RedisSortedStreams}; +use std::rc::Rc; +impl LobbyStreams { + pub fn new(reg: Components, client: Rc) -> Self { + let mut conn = client.get_connection().expect("redis conn"); + let stream_handlers: Vec<(&str, Box anyhow::Result<()>>)> = vec![ + ("some-stream", todo!()), + ("another-stream", todo!()), + ("fix-the-names", todo!()), + ]; + let sorted_streams = Box::new( + RedisSortedStreams::xgroup_create_mkstreams(stream_handlers, todo!("opts"), &mut conn) + .expect("stream creation"), + ); + + Self { + reg, + sorted_streams, + } + } + + pub fn process(&mut self) { + loop { + if let Err(e) = self.sorted_streams.consume() { + error!("Stream err {:?}", e) } - Err(e) => error!("Stream err {:?}", e), - }*/ + } } -} -fn consume(_eid: XId, event: &StreamInput, reg: &Components) { - match event { - StreamInput::FPG(fpg) => consume_fpg(fpg, reg), - StreamInput::CG(cg) => consume_cg(cg, reg), - StreamInput::JPG(jpg) => consume_jpg(jpg, reg), - StreamInput::SD(sd) => consume_sd(sd, reg), + fn consume(&mut self, _eid: XId, event: &StreamInput, reg: &Components) { + match event { + StreamInput::FPG(fpg) => self.consume_fpg(fpg), + StreamInput::CG(cg) => todo!(), //consume_cg(cg), + StreamInput::JPG(jpg) => todo!(), //consume_jpg(jpg), + StreamInput::SD(sd) => todo!(), //consume_sd(sd), + } } -} -fn consume_fpg(fpg: &FindPublicGame, reg: &Components) { - let visibility = Visibility::Public; - let session_id = &fpg.session_id; - if let Ok(lobby) = reg.game_lobby_repo.get() { - if let Some(queued) = lobby - .games - .iter() - .find(|g| g.visibility == Visibility::Public) - { - ready_game(session_id, &lobby, queued, reg) - } else { - let game_id = GameId::new(); - let updated: GameLobby = lobby.open(Game { - board_size: PUBLIC_GAME_BOARD_SIZE, - creator: session_id.clone(), - visibility, - game_id: game_id.clone(), - }); - if let Err(_) = reg.game_lobby_repo.put(&updated) { - error!("game lobby write F2"); + fn consume_fpg(&mut self, fpg: &FindPublicGame) { + let reg = &self.reg; + let visibility = Visibility::Public; + let session_id = &fpg.session_id; + if let Ok(lobby) = reg.game_lobby_repo.get() { + if let Some(queued) = lobby + .games + .iter() + .find(|g| g.visibility == Visibility::Public) + { + ready_game(session_id, &lobby, queued, ®) } else { - if let Err(_) = reg.xadd.xadd(StreamOutput::WFO(WaitForOpponent { - event_id: EventId::new(), - game_id, - session_id: session_id.clone(), + let game_id = GameId::new(); + let updated: GameLobby = lobby.open(Game { + board_size: PUBLIC_GAME_BOARD_SIZE, + creator: session_id.clone(), visibility, - })) { - error!("XADD: Wait for oppo") + game_id: game_id.clone(), + }); + if let Err(_) = reg.game_lobby_repo.put(&updated) { + error!("game lobby write F2"); } else { - trace!("Public game open. Lobby: {:?}", &updated) + if let Err(_) = reg.xadd.xadd(StreamOutput::WFO(WaitForOpponent { + event_id: EventId::new(), + game_id, + session_id: session_id.clone(), + visibility, + })) { + error!("XADD: Wait for oppo") + } else { + trace!("Public game open. Lobby: {:?}", &updated) + } } } + } else { + error!("Failed to fetch game lobby: FPG") } - } else { - error!("Failed to fetch game lobby: FPG") } } @@ -258,9 +279,10 @@ mod test { let components = Components { game_lobby_repo: Box::new(FakeGameLobbyRepo { contents: fgl }), xadd: Box::new(FakeXAdd(xadd_in)), - sorted_streams: Box::new(FakeSortedStreams) }; - process(&components); + let fake_sorted_streams = FakeSortedStreams; + let lobby_streams = LobbyStreams::new(components, todo!()); + lobby_streams.process(); }); // emit some events From 9f801e6f10ebe355132e06f533d99476a7a0a0ed Mon Sep 17 00:00:00 2001 From: terkwood <38859656+Terkwood@users.noreply.github.com> Date: Sat, 29 May 2021 12:46:57 -0400 Subject: [PATCH 38/54] work it out --- micro-game-lobby/src/main.rs | 23 +++++++++++++++-- micro-game-lobby/src/stream/mod.rs | 41 ++++++++---------------------- 2 files changed, 31 insertions(+), 33 deletions(-) diff --git a/micro-game-lobby/src/main.rs b/micro-game-lobby/src/main.rs index 554f5168..831603d4 100644 --- a/micro-game-lobby/src/main.rs +++ b/micro-game-lobby/src/main.rs @@ -1,14 +1,33 @@ +use std::borrow::BorrowMut; + use log::info; use micro_game_lobby::*; const VERSION: &str = env!("CARGO_PKG_VERSION"); +use redis_streams::{anyhow, Message, RedisSortedStreams, XId}; + fn main() { env_logger::init(); info!("🔢 {}", VERSION); let client = redis_client(); let components = Components::new(client.clone()); stream::create_consumer_group(&client); - let mut lobby_streams = stream::LobbyStreams::new(components, client); - lobby_streams.process() + + let mut lobby_streams = stream::LobbyStreams::new(components); + + let mut conn = client.get_connection().expect("redis conn"); + let stream_handlers: Vec<(&str, Box anyhow::Result<()>>)> = vec![ + ( + "some-stream", + Box::new(|xid, msg| Ok(lobby_streams.consume_fpg(msg))), + ), + ("another-stream", todo!()), + ("fix-the-names", todo!()), + ]; + let mut sorted_streams = + RedisSortedStreams::xgroup_create_mkstreams(stream_handlers, todo!("opts"), &mut conn) + .expect("stream creation"); + + lobby_streams.process(&mut sorted_streams) } diff --git a/micro-game-lobby/src/stream/mod.rs b/micro-game-lobby/src/stream/mod.rs index 847ce053..d458adf3 100644 --- a/micro-game-lobby/src/stream/mod.rs +++ b/micro-game-lobby/src/stream/mod.rs @@ -13,7 +13,7 @@ use lobby_model::api::*; use lobby_model::*; use log::{error, info, trace}; use move_model::GameState; -use redis_streams::XId; +use redis_streams::{Message, XId}; pub const GROUP_NAME: &str = "micro-game-lobby"; @@ -35,48 +35,27 @@ pub enum StreamOutput { pub struct LobbyStreams { pub reg: Components, - pub sorted_streams: Box, } -use redis_streams::{anyhow, Message, RedisSortedStreams}; +use redis_streams::{anyhow, RedisSortedStreams}; +use std::borrow::BorrowMut; use std::rc::Rc; impl LobbyStreams { - pub fn new(reg: Components, client: Rc) -> Self { - let mut conn = client.get_connection().expect("redis conn"); - let stream_handlers: Vec<(&str, Box anyhow::Result<()>>)> = vec![ - ("some-stream", todo!()), - ("another-stream", todo!()), - ("fix-the-names", todo!()), - ]; - let sorted_streams = Box::new( - RedisSortedStreams::xgroup_create_mkstreams(stream_handlers, todo!("opts"), &mut conn) - .expect("stream creation"), - ); - - Self { - reg, - sorted_streams, - } + pub fn new(reg: Components) -> Self { + Self { reg } } - pub fn process(&mut self) { + pub fn process(&mut self, streams: &mut dyn SortedStreams) { loop { - if let Err(e) = self.sorted_streams.consume() { + if let Err(e) = streams.consume() { error!("Stream err {:?}", e) } } } - fn consume(&mut self, _eid: XId, event: &StreamInput, reg: &Components) { - match event { - StreamInput::FPG(fpg) => self.consume_fpg(fpg), - StreamInput::CG(cg) => todo!(), //consume_cg(cg), - StreamInput::JPG(jpg) => todo!(), //consume_jpg(jpg), - StreamInput::SD(sd) => todo!(), //consume_sd(sd), - } - } - - fn consume_fpg(&mut self, fpg: &FindPublicGame) { + pub fn consume_fpg(&mut self, msg: &Message) { + todo!("deser"); + let fpg: FindPublicGame = todo!(); let reg = &self.reg; let visibility = Visibility::Public; let session_id = &fpg.session_id; From 6658c76ec00c213ea3898b1666e6d331a7130ea4 Mon Sep 17 00:00:00 2001 From: terkwood <38859656+Terkwood@users.noreply.github.com> Date: Sat, 29 May 2021 12:49:04 -0400 Subject: [PATCH 39/54] typecheck test --- micro-game-lobby/src/main.rs | 2 +- micro-game-lobby/src/stream/mod.rs | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/micro-game-lobby/src/main.rs b/micro-game-lobby/src/main.rs index 831603d4..62b45ff7 100644 --- a/micro-game-lobby/src/main.rs +++ b/micro-game-lobby/src/main.rs @@ -19,7 +19,7 @@ fn main() { let mut conn = client.get_connection().expect("redis conn"); let stream_handlers: Vec<(&str, Box anyhow::Result<()>>)> = vec![ ( - "some-stream", + topics::FIND_PUBLIC_GAME, Box::new(|xid, msg| Ok(lobby_streams.consume_fpg(msg))), ), ("another-stream", todo!()), diff --git a/micro-game-lobby/src/stream/mod.rs b/micro-game-lobby/src/stream/mod.rs index d458adf3..4d9c7f08 100644 --- a/micro-game-lobby/src/stream/mod.rs +++ b/micro-game-lobby/src/stream/mod.rs @@ -259,9 +259,9 @@ mod test { game_lobby_repo: Box::new(FakeGameLobbyRepo { contents: fgl }), xadd: Box::new(FakeXAdd(xadd_in)), }; - let fake_sorted_streams = FakeSortedStreams; - let lobby_streams = LobbyStreams::new(components, todo!()); - lobby_streams.process(); + let mut fake_sorted_streams = FakeSortedStreams; + let mut lobby_streams = LobbyStreams::new(components); + lobby_streams.process(&mut fake_sorted_streams); }); // emit some events From b29bd360a1e8cfe64fdf639bf905b58bd1082952 Mon Sep 17 00:00:00 2001 From: terkwood <38859656+Terkwood@users.noreply.github.com> Date: Sat, 29 May 2021 12:53:00 -0400 Subject: [PATCH 40/54] move method --- micro-game-lobby/src/main.rs | 9 ++++-- micro-game-lobby/src/stream/mod.rs | 48 ++++++++++++++++-------------- 2 files changed, 31 insertions(+), 26 deletions(-) diff --git a/micro-game-lobby/src/main.rs b/micro-game-lobby/src/main.rs index 62b45ff7..b2e03ed8 100644 --- a/micro-game-lobby/src/main.rs +++ b/micro-game-lobby/src/main.rs @@ -20,10 +20,13 @@ fn main() { let stream_handlers: Vec<(&str, Box anyhow::Result<()>>)> = vec![ ( topics::FIND_PUBLIC_GAME, - Box::new(|xid, msg| Ok(lobby_streams.consume_fpg(msg))), + Box::new(|_xid, msg| Ok(lobby_streams.consume_fpg(msg))), + ), + (topics::JOIN_PRIVATE_GAME, todo!()), + ( + topics::CREATE_GAME, + Box::new(|_xid, msg| Ok(lobby_streams.consume_cg(msg))), ), - ("another-stream", todo!()), - ("fix-the-names", todo!()), ]; let mut sorted_streams = RedisSortedStreams::xgroup_create_mkstreams(stream_handlers, todo!("opts"), &mut conn) diff --git a/micro-game-lobby/src/stream/mod.rs b/micro-game-lobby/src/stream/mod.rs index 4d9c7f08..60231e62 100644 --- a/micro-game-lobby/src/stream/mod.rs +++ b/micro-game-lobby/src/stream/mod.rs @@ -93,34 +93,36 @@ impl LobbyStreams { error!("Failed to fetch game lobby: FPG") } } -} -fn consume_cg(cg: &CreateGame, reg: &Components) { - let session_id = &cg.session_id; - let game_id = cg.game_id.clone().unwrap_or(GameId::new()); - if let Ok(lobby) = reg.game_lobby_repo.get() { - let updated: GameLobby = lobby.open(Game { - game_id: game_id.clone(), - board_size: cg.board_size, - creator: session_id.clone(), - visibility: cg.visibility, - }); - if let Err(_) = reg.game_lobby_repo.put(&updated) { - error!("game lobby write F1"); - } else { - if let Err(_) = reg.xadd.xadd(StreamOutput::WFO(WaitForOpponent { + pub fn consume_cg(&mut self, msg: &Message) { + todo!("deser"); + let mut cg: CreateGame = todo!(); + let session_id = &cg.session_id; + let game_id = cg.game_id.clone().unwrap_or(GameId::new()); + if let Ok(lobby) = self.reg.game_lobby_repo.get() { + let updated: GameLobby = lobby.open(Game { game_id: game_id.clone(), - session_id: session_id.clone(), - event_id: EventId::new(), + board_size: cg.board_size, + creator: session_id.clone(), visibility: cg.visibility, - })) { - error!("XADD Game ready") + }); + if let Err(_) = self.reg.game_lobby_repo.put(&updated) { + error!("game lobby write F1"); } else { - trace!("Game created. Lobby: {:?}", &updated); + if let Err(_) = self.reg.xadd.xadd(StreamOutput::WFO(WaitForOpponent { + game_id: game_id.clone(), + session_id: session_id.clone(), + event_id: EventId::new(), + visibility: cg.visibility, + })) { + error!("XADD Game ready") + } else { + trace!("Game created. Lobby: {:?}", &updated); + } } + } else { + error!("CG GAME REPO GET") } - } else { - error!("CG GAME REPO GET") } } /// Consumes the command to join a private game. @@ -237,7 +239,7 @@ mod test { struct FakeSortedStreams; impl redis_streams::SortedStreams for FakeSortedStreams { fn consume(&mut self) -> redis_streams::anyhow::Result<()> { - todo!() + todo!("write some sort of test, i guess") } } From 1703569af96a72d07d9f4fca67c4907e9349f6d8 Mon Sep 17 00:00:00 2001 From: terkwood <38859656+Terkwood@users.noreply.github.com> Date: Sat, 29 May 2021 12:54:05 -0400 Subject: [PATCH 41/54] move method --- micro-game-lobby/src/stream/mod.rs | 54 ++++++++++++++++-------------- 1 file changed, 29 insertions(+), 25 deletions(-) diff --git a/micro-game-lobby/src/stream/mod.rs b/micro-game-lobby/src/stream/mod.rs index 60231e62..c39f840b 100644 --- a/micro-game-lobby/src/stream/mod.rs +++ b/micro-game-lobby/src/stream/mod.rs @@ -124,35 +124,39 @@ impl LobbyStreams { error!("CG GAME REPO GET") } } -} -/// Consumes the command to join a private game. -/// In the event that the game is invalid, -/// we will simply log a warning. -/// Consider implementing logic related to handling -/// private game rejection: https://github.com/Terkwood/BUGOUT/issues/304 -fn consume_jpg(jpg: &JoinPrivateGame, reg: &Components) { - if let Ok(lobby) = reg.game_lobby_repo.get() { - if let Some(queued) = lobby - .games - .iter() - .find(|g| g.visibility == Visibility::Private && g.game_id == jpg.game_id) - { - ready_game(&jpg.session_id, &lobby, queued, reg) - } else { - if let Err(e) = reg.xadd.xadd(StreamOutput::PGR(PrivateGameRejected { - client_id: jpg.client_id.clone(), - event_id: EventId::new(), - game_id: jpg.game_id.clone(), - session_id: jpg.session_id.clone(), - })) { - error!("Error writing private game rejection to stream {:?}", e) + + /// Consumes the command to join a private game. + /// In the event that the game is invalid, + /// we will simply log a warning. + /// Consider implementing logic related to handling + /// private game rejection: https://github.com/Terkwood/BUGOUT/issues/304 + pub fn consume_jpg(&mut self) { + let jpg: JoinPrivateGame = todo!(); + todo!("deser"); + + let reg = &self.reg; + if let Ok(lobby) = reg.game_lobby_repo.get() { + if let Some(queued) = lobby + .games + .iter() + .find(|g| g.visibility == Visibility::Private && g.game_id == jpg.game_id) + { + ready_game(&jpg.session_id, &lobby, queued, reg) + } else { + if let Err(e) = reg.xadd.xadd(StreamOutput::PGR(PrivateGameRejected { + client_id: jpg.client_id.clone(), + event_id: EventId::new(), + game_id: jpg.game_id.clone(), + session_id: jpg.session_id.clone(), + })) { + error!("Error writing private game rejection to stream {:?}", e) + } } + } else { + error!("game lobby JPG get") } - } else { - error!("game lobby JPG get") } } - fn consume_sd(sd: &SessionDisconnected, reg: &Components) { if let Ok(game_lobby) = reg.game_lobby_repo.get() { let updated: GameLobby = game_lobby.abandon(&sd.session_id); From 1c1293f615dae9fc66ce3d32b5e72820cc5cc7a8 Mon Sep 17 00:00:00 2001 From: terkwood <38859656+Terkwood@users.noreply.github.com> Date: Sat, 29 May 2021 12:55:45 -0400 Subject: [PATCH 42/54] make some handlers --- micro-game-lobby/src/main.rs | 5 ++++- micro-game-lobby/src/stream/mod.rs | 6 +++--- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/micro-game-lobby/src/main.rs b/micro-game-lobby/src/main.rs index b2e03ed8..376c6f9b 100644 --- a/micro-game-lobby/src/main.rs +++ b/micro-game-lobby/src/main.rs @@ -22,7 +22,10 @@ fn main() { topics::FIND_PUBLIC_GAME, Box::new(|_xid, msg| Ok(lobby_streams.consume_fpg(msg))), ), - (topics::JOIN_PRIVATE_GAME, todo!()), + ( + topics::JOIN_PRIVATE_GAME, + Box::new(|_xid, msg| Ok(lobby_streams.consume_jpg(msg))), + ), ( topics::CREATE_GAME, Box::new(|_xid, msg| Ok(lobby_streams.consume_cg(msg))), diff --git a/micro-game-lobby/src/stream/mod.rs b/micro-game-lobby/src/stream/mod.rs index c39f840b..b79a164c 100644 --- a/micro-game-lobby/src/stream/mod.rs +++ b/micro-game-lobby/src/stream/mod.rs @@ -53,7 +53,7 @@ impl LobbyStreams { } } - pub fn consume_fpg(&mut self, msg: &Message) { + pub fn consume_fpg(&self, msg: &Message) { todo!("deser"); let fpg: FindPublicGame = todo!(); let reg = &self.reg; @@ -94,7 +94,7 @@ impl LobbyStreams { } } - pub fn consume_cg(&mut self, msg: &Message) { + pub fn consume_cg(&self, msg: &Message) { todo!("deser"); let mut cg: CreateGame = todo!(); let session_id = &cg.session_id; @@ -130,7 +130,7 @@ impl LobbyStreams { /// we will simply log a warning. /// Consider implementing logic related to handling /// private game rejection: https://github.com/Terkwood/BUGOUT/issues/304 - pub fn consume_jpg(&mut self) { + pub fn consume_jpg(&self, msg: &Message) { let jpg: JoinPrivateGame = todo!(); todo!("deser"); From 192801f1c55703dd5b6bb65b8df641da54fcf7c4 Mon Sep 17 00:00:00 2001 From: terkwood <38859656+Terkwood@users.noreply.github.com> Date: Sat, 29 May 2021 12:57:30 -0400 Subject: [PATCH 43/54] move method --- micro-game-lobby/src/main.rs | 4 ++++ micro-game-lobby/src/stream/mod.rs | 25 +++++++++++++------------ 2 files changed, 17 insertions(+), 12 deletions(-) diff --git a/micro-game-lobby/src/main.rs b/micro-game-lobby/src/main.rs index 376c6f9b..8901d502 100644 --- a/micro-game-lobby/src/main.rs +++ b/micro-game-lobby/src/main.rs @@ -30,6 +30,10 @@ fn main() { topics::CREATE_GAME, Box::new(|_xid, msg| Ok(lobby_streams.consume_cg(msg))), ), + ( + topics::SESSION_DISCONNECTED, + Box::new(|_xid, msg| Ok(lobby_streams.consume_sd(msg))), + ), ]; let mut sorted_streams = RedisSortedStreams::xgroup_create_mkstreams(stream_handlers, todo!("opts"), &mut conn) diff --git a/micro-game-lobby/src/stream/mod.rs b/micro-game-lobby/src/stream/mod.rs index b79a164c..8fd2ec68 100644 --- a/micro-game-lobby/src/stream/mod.rs +++ b/micro-game-lobby/src/stream/mod.rs @@ -131,8 +131,7 @@ impl LobbyStreams { /// Consider implementing logic related to handling /// private game rejection: https://github.com/Terkwood/BUGOUT/issues/304 pub fn consume_jpg(&self, msg: &Message) { - let jpg: JoinPrivateGame = todo!(); - todo!("deser"); + let jpg: JoinPrivateGame = todo!("deser"); let reg = &self.reg; if let Ok(lobby) = reg.game_lobby_repo.get() { @@ -156,20 +155,22 @@ impl LobbyStreams { error!("game lobby JPG get") } } -} -fn consume_sd(sd: &SessionDisconnected, reg: &Components) { - if let Ok(game_lobby) = reg.game_lobby_repo.get() { - let updated: GameLobby = game_lobby.abandon(&sd.session_id); - if let Err(_) = reg.game_lobby_repo.put(&updated) { - error!("game lobby write F1"); + + pub fn consume_sd(&self, msg: &Message) { + let sd: SessionDisconnected = todo!("deser"); + let reg = &self.reg; + if let Ok(game_lobby) = reg.game_lobby_repo.get() { + let updated: GameLobby = game_lobby.abandon(&sd.session_id); + if let Err(_) = reg.game_lobby_repo.put(&updated) { + error!("game lobby write F1"); + } else { + trace!("session {} abandoned: {:?}", sd.session_id.0, &updated); + } } else { - trace!("session {} abandoned: {:?}", sd.session_id.0, &updated); + error!("SD GAME REPO GET") } - } else { - error!("SD GAME REPO GET") } } - fn ready_game(session_id: &SessionId, lobby: &GameLobby, queued: &Game, reg: &Components) { let updated: GameLobby = lobby.ready(queued); if let Err(_) = reg.game_lobby_repo.put(&updated) { From 8a20f221f543597714117a3dff997fcb838cb679 Mon Sep 17 00:00:00 2001 From: terkwood <38859656+Terkwood@users.noreply.github.com> Date: Sat, 29 May 2021 13:01:39 -0400 Subject: [PATCH 44/54] work things out --- micro-game-lobby/src/main.rs | 6 ++---- micro-game-lobby/src/stream/mod.rs | 22 ++++++++++++++++------ micro-game-lobby/src/stream/xread.rs | 1 - 3 files changed, 18 insertions(+), 11 deletions(-) diff --git a/micro-game-lobby/src/main.rs b/micro-game-lobby/src/main.rs index 8901d502..baee36cd 100644 --- a/micro-game-lobby/src/main.rs +++ b/micro-game-lobby/src/main.rs @@ -1,5 +1,3 @@ -use std::borrow::BorrowMut; - use log::info; use micro_game_lobby::*; @@ -14,7 +12,7 @@ fn main() { let components = Components::new(client.clone()); stream::create_consumer_group(&client); - let mut lobby_streams = stream::LobbyStreams::new(components); + let lobby_streams = stream::LobbyStreams::new(components); let mut conn = client.get_connection().expect("redis conn"); let stream_handlers: Vec<(&str, Box anyhow::Result<()>>)> = vec![ @@ -36,7 +34,7 @@ fn main() { ), ]; let mut sorted_streams = - RedisSortedStreams::xgroup_create_mkstreams(stream_handlers, todo!("opts"), &mut conn) + RedisSortedStreams::xgroup_create_mkstreams(stream_handlers, &stream::opts(), &mut conn) .expect("stream creation"); lobby_streams.process(&mut sorted_streams) diff --git a/micro-game-lobby/src/stream/mod.rs b/micro-game-lobby/src/stream/mod.rs index 8fd2ec68..6bedc487 100644 --- a/micro-game-lobby/src/stream/mod.rs +++ b/micro-game-lobby/src/stream/mod.rs @@ -2,6 +2,7 @@ mod init; mod xadd; pub use init::*; +use redis_streams::ConsumerGroupOpts; use redis_streams::SortedStreams; pub use xadd::*; @@ -11,9 +12,9 @@ use crate::PUBLIC_GAME_BOARD_SIZE; use core_model::*; use lobby_model::api::*; use lobby_model::*; -use log::{error, info, trace}; +use log::{error, trace}; use move_model::GameState; -use redis_streams::{Message, XId}; +use redis_streams::Message; pub const GROUP_NAME: &str = "micro-game-lobby"; @@ -37,15 +38,24 @@ pub struct LobbyStreams { pub reg: Components, } -use redis_streams::{anyhow, RedisSortedStreams}; -use std::borrow::BorrowMut; -use std::rc::Rc; +use redis_streams::Group; +const BLOCK_MS: usize = 5000; +pub fn opts() -> ConsumerGroupOpts { + ConsumerGroupOpts { + block_ms: BLOCK_MS, + group: Group { + group_name: GROUP_NAME.to_string(), + consumer_name: "singleton".to_string(), + }, + } +} + impl LobbyStreams { pub fn new(reg: Components) -> Self { Self { reg } } - pub fn process(&mut self, streams: &mut dyn SortedStreams) { + pub fn process(&self, streams: &mut dyn SortedStreams) { loop { if let Err(e) = streams.consume() { error!("Stream err {:?}", e) diff --git a/micro-game-lobby/src/stream/xread.rs b/micro-game-lobby/src/stream/xread.rs index 5f9cd16e..7e013e29 100644 --- a/micro-game-lobby/src/stream/xread.rs +++ b/micro-game-lobby/src/stream/xread.rs @@ -8,7 +8,6 @@ use redis_streams::XReadEntryId; use std::collections::HashMap; use std::rc::Rc; -const BLOCK_MS: usize = 5000; /// xread_sorted performs a redis xread then sorts the results /// From b0990d05408eaa4e77aa49bdae2ee8359ada9cef Mon Sep 17 00:00:00 2001 From: terkwood <38859656+Terkwood@users.noreply.github.com> Date: Sat, 29 May 2021 13:02:35 -0400 Subject: [PATCH 45/54] rename --- micro-game-lobby/src/main.rs | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/micro-game-lobby/src/main.rs b/micro-game-lobby/src/main.rs index baee36cd..c5bfadc1 100644 --- a/micro-game-lobby/src/main.rs +++ b/micro-game-lobby/src/main.rs @@ -12,30 +12,30 @@ fn main() { let components = Components::new(client.clone()); stream::create_consumer_group(&client); - let lobby_streams = stream::LobbyStreams::new(components); + let lobby = stream::LobbyStreams::new(components); let mut conn = client.get_connection().expect("redis conn"); let stream_handlers: Vec<(&str, Box anyhow::Result<()>>)> = vec![ ( topics::FIND_PUBLIC_GAME, - Box::new(|_xid, msg| Ok(lobby_streams.consume_fpg(msg))), + Box::new(|_xid, msg| Ok(lobby.consume_fpg(msg))), ), ( topics::JOIN_PRIVATE_GAME, - Box::new(|_xid, msg| Ok(lobby_streams.consume_jpg(msg))), + Box::new(|_xid, msg| Ok(lobby.consume_jpg(msg))), ), ( topics::CREATE_GAME, - Box::new(|_xid, msg| Ok(lobby_streams.consume_cg(msg))), + Box::new(|_xid, msg| Ok(lobby.consume_cg(msg))), ), ( topics::SESSION_DISCONNECTED, - Box::new(|_xid, msg| Ok(lobby_streams.consume_sd(msg))), + Box::new(|_xid, msg| Ok(lobby.consume_sd(msg))), ), ]; - let mut sorted_streams = + let mut streams = RedisSortedStreams::xgroup_create_mkstreams(stream_handlers, &stream::opts(), &mut conn) .expect("stream creation"); - lobby_streams.process(&mut sorted_streams) + lobby.process(&mut streams) } From d7d42f7d6c3477c08c3152c9f60b95925b287d05 Mon Sep 17 00:00:00 2001 From: terkwood <38859656+Terkwood@users.noreply.github.com> Date: Sat, 29 May 2021 13:08:20 -0400 Subject: [PATCH 46/54] =?UTF-8?q?DELETE=20THE=20TEST=20=F0=9F=98=83?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- micro-game-lobby/src/main.rs | 1 - micro-game-lobby/src/stream/mod.rs | 135 ----------------------------- 2 files changed, 136 deletions(-) diff --git a/micro-game-lobby/src/main.rs b/micro-game-lobby/src/main.rs index c5bfadc1..4048ff2e 100644 --- a/micro-game-lobby/src/main.rs +++ b/micro-game-lobby/src/main.rs @@ -13,7 +13,6 @@ fn main() { stream::create_consumer_group(&client); let lobby = stream::LobbyStreams::new(components); - let mut conn = client.get_connection().expect("redis conn"); let stream_handlers: Vec<(&str, Box anyhow::Result<()>>)> = vec![ ( diff --git a/micro-game-lobby/src/stream/mod.rs b/micro-game-lobby/src/stream/mod.rs index 6bedc487..dc5eb626 100644 --- a/micro-game-lobby/src/stream/mod.rs +++ b/micro-game-lobby/src/stream/mod.rs @@ -215,138 +215,3 @@ fn init_changelog(game_id: &GameId, board_size: u16, reg: &Components) { error!("could not write game state changelog {:?}", chgerr) } } - -#[cfg(test)] -mod test { - use super::*; - use crate::components::Components; - use crate::repo::*; - use crossbeam_channel::{select, unbounded, Sender}; - use redis_streams::XId; - use std::sync::{Arc, Mutex}; - use std::time::Duration; - - use std::thread; - - struct FakeGameLobbyRepo { - pub contents: Arc>, - } - - impl GameLobbyRepo for FakeGameLobbyRepo { - fn get(&self) -> Result { - Ok(self.contents.lock().expect("mutex lock").clone()) - } - fn put(&self, game_lobby: &GameLobby) -> Result<(), WriteErr> { - let mut data = self.contents.lock().expect("lock"); - *data = game_lobby.clone(); - Ok(()) - } - } - - struct FakeXAdd(Sender); - impl XAdd for FakeXAdd { - fn xadd(&self, data: StreamOutput) -> Result<(), XAddErr> { - if let Err(_) = self.0.send(data) {} - Ok(()) - } - } - - struct FakeSortedStreams; - impl redis_streams::SortedStreams for FakeSortedStreams { - fn consume(&mut self) -> redis_streams::anyhow::Result<()> { - todo!("write some sort of test, i guess") - } - } - - #[test] - fn test_process() { - let (xadd_in, xadd_out) = unbounded(); - - let sorted_fake_stream = Arc::new(Mutex::new(vec![])); - - let timeout = Duration::from_millis(160); - - // set up a loop to process game lobby requests - let fake_game_lobby_contents = Arc::new(Mutex::new(GameLobby::default())); - - let fgl = fake_game_lobby_contents.clone(); - - thread::spawn(move || { - let components = Components { - game_lobby_repo: Box::new(FakeGameLobbyRepo { contents: fgl }), - xadd: Box::new(FakeXAdd(xadd_in)), - }; - let mut fake_sorted_streams = FakeSortedStreams; - let mut lobby_streams = LobbyStreams::new(components); - lobby_streams.process(&mut fake_sorted_streams); - }); - - // emit some events - - let mut fake_time_ms = 100; - let incr_ms = 100; - - let session_b = SessionId::new(); - let session_w = SessionId::new(); - let client_b = ClientId::new(); - let client_w = ClientId::new(); - let xid0 = quick_eid(fake_time_ms); - sorted_fake_stream.lock().expect("lock").push(( - xid0, - StreamInput::FPG(FindPublicGame { - client_id: client_w.clone(), - session_id: session_w.clone(), - }), - )); - - thread::sleep(timeout); - - // The game lobby repo should now contain one game - assert_eq!( - fake_game_lobby_contents - .clone() - .lock() - .expect("gl") - .games - .iter() - .collect::>() - .len(), - 1 - ); - - // There should be an XADD triggered for a wait-for-opponent - // message - select! { - recv(xadd_out) -> msg => match msg { - Ok(StreamOutput::WFO(_)) => assert!(true), - _ => panic!("wrong output") - }, - default(timeout) => panic!("WAIT timeout") - } - - fake_time_ms += incr_ms; - sorted_fake_stream.lock().expect("lock").push(( - quick_eid(fake_time_ms), - StreamInput::FPG(FindPublicGame { - client_id: client_b, - session_id: session_b, - }), - )); - - // There should now be GameReady in stream - select! { - recv(xadd_out) -> msg => match msg { - Ok(StreamOutput::GR(_)) => assert!(true), - _ => assert!(false) - }, - default(timeout) => panic!("GR time out") - } - } - - fn quick_eid(ms: u64) -> XId { - XId { - millis_time: ms, - seq_no: 0, - } - } -} From 8d5a89d9031d25e861e470296104c27ae9c81cea Mon Sep 17 00:00:00 2001 From: terkwood <38859656+Terkwood@users.noreply.github.com> Date: Sat, 29 May 2021 13:15:51 -0400 Subject: [PATCH 47/54] hack --- micro-game-lobby/src/main.rs | 2 +- micro-game-lobby/src/stream/mod.rs | 83 +++++++++++++++------------- micro-game-lobby/src/stream/xread.rs | 4 +- 3 files changed, 47 insertions(+), 42 deletions(-) diff --git a/micro-game-lobby/src/main.rs b/micro-game-lobby/src/main.rs index 4048ff2e..7cc36634 100644 --- a/micro-game-lobby/src/main.rs +++ b/micro-game-lobby/src/main.rs @@ -17,7 +17,7 @@ fn main() { let stream_handlers: Vec<(&str, Box anyhow::Result<()>>)> = vec![ ( topics::FIND_PUBLIC_GAME, - Box::new(|_xid, msg| Ok(lobby.consume_fpg(msg))), + Box::new(|_xid, msg| lobby.consume_fpg(msg)), ), ( topics::JOIN_PRIVATE_GAME, diff --git a/micro-game-lobby/src/stream/mod.rs b/micro-game-lobby/src/stream/mod.rs index dc5eb626..750dd0ae 100644 --- a/micro-game-lobby/src/stream/mod.rs +++ b/micro-game-lobby/src/stream/mod.rs @@ -50,6 +50,7 @@ pub fn opts() -> ConsumerGroupOpts { } } +use redis_streams::anyhow; impl LobbyStreams { pub fn new(reg: Components) -> Self { Self { reg } @@ -63,45 +64,49 @@ impl LobbyStreams { } } - pub fn consume_fpg(&self, msg: &Message) { - todo!("deser"); - let fpg: FindPublicGame = todo!(); - let reg = &self.reg; - let visibility = Visibility::Public; - let session_id = &fpg.session_id; - if let Ok(lobby) = reg.game_lobby_repo.get() { - if let Some(queued) = lobby - .games - .iter() - .find(|g| g.visibility == Visibility::Public) - { - ready_game(session_id, &lobby, queued, ®) - } else { - let game_id = GameId::new(); - let updated: GameLobby = lobby.open(Game { - board_size: PUBLIC_GAME_BOARD_SIZE, - creator: session_id.clone(), - visibility, - game_id: game_id.clone(), - }); - if let Err(_) = reg.game_lobby_repo.put(&updated) { - error!("game lobby write F2"); - } else { - if let Err(_) = reg.xadd.xadd(StreamOutput::WFO(WaitForOpponent { - event_id: EventId::new(), - game_id, - session_id: session_id.clone(), - visibility, - })) { - error!("XADD: Wait for oppo") - } else { - trace!("Public game open. Lobby: {:?}", &updated) - } - } - } - } else { - error!("Failed to fetch game lobby: FPG") - } + pub fn consume_fpg(&self, msg: &Message) -> anyhow::Result<()> { + let maybe_value = msg.get("data"); + Ok(if let Some(redis::Value::Data(data)) = maybe_value { + /* let fpg: FindPublicGame = bincode::deserialize(&data)?; + let reg = &self.reg; + let visibility = Visibility::Public; + let session_id = &fpg.session_id; + if let Ok(lobby) = reg.game_lobby_repo.get() { + if let Some(queued) = lobby + .games + .iter() + .find(|g| g.visibility == Visibility::Public) + { + ready_game(session_id, &lobby, queued, ®) + } else { + let game_id = GameId::new(); + let updated: GameLobby = lobby.open(Game { + board_size: PUBLIC_GAME_BOARD_SIZE, + creator: session_id.clone(), + visibility, + game_id: game_id.clone(), + }); + if let Err(_) = reg.game_lobby_repo.put(&updated) { + error!("game lobby write F2"); + } else { + if let Err(_) = reg.xadd.xadd(StreamOutput::WFO(WaitForOpponent { + event_id: EventId::new(), + game_id, + session_id: session_id.clone(), + visibility, + })) { + error!("XADD: Wait for oppo") + } else { + trace!("Public game open. Lobby: {:?}", &updated) + } + } + } + } else { + error!("Failed to fetch game lobby: FPG") + }} + + */ + }) } pub fn consume_cg(&self, msg: &Message) { diff --git a/micro-game-lobby/src/stream/xread.rs b/micro-game-lobby/src/stream/xread.rs index 7e013e29..5465ddfc 100644 --- a/micro-game-lobby/src/stream/xread.rs +++ b/micro-game-lobby/src/stream/xread.rs @@ -72,10 +72,10 @@ fn deser(srr: StreamReadReply) -> Result, XRe let key = k.key; for e in k.ids { if let Ok(eid) = XReadEntryId::from_str(&e.id) { - let maybe_data: Option> = e.get("data"); + if let Some(data) = maybe_data { let sd: Option = if key == FIND_PUBLIC_GAME { - bincode::deserialize(&data) + .map(|fpg| StreamInput::FPG(fpg)) .ok() } else if key == CREATE_GAME { From 5c06940bc654bc82c8c803fcb91bae7b99b920d5 Mon Sep 17 00:00:00 2001 From: terkwood <38859656+Terkwood@users.noreply.github.com> Date: Sat, 29 May 2021 13:16:46 -0400 Subject: [PATCH 48/54] deser --- micro-game-lobby/src/stream/mod.rs | 74 +++++++++++++++--------------- 1 file changed, 36 insertions(+), 38 deletions(-) diff --git a/micro-game-lobby/src/stream/mod.rs b/micro-game-lobby/src/stream/mod.rs index 750dd0ae..7e68c2fb 100644 --- a/micro-game-lobby/src/stream/mod.rs +++ b/micro-game-lobby/src/stream/mod.rs @@ -67,45 +67,43 @@ impl LobbyStreams { pub fn consume_fpg(&self, msg: &Message) -> anyhow::Result<()> { let maybe_value = msg.get("data"); Ok(if let Some(redis::Value::Data(data)) = maybe_value { - /* let fpg: FindPublicGame = bincode::deserialize(&data)?; - let reg = &self.reg; - let visibility = Visibility::Public; - let session_id = &fpg.session_id; - if let Ok(lobby) = reg.game_lobby_repo.get() { - if let Some(queued) = lobby - .games - .iter() - .find(|g| g.visibility == Visibility::Public) - { - ready_game(session_id, &lobby, queued, ®) - } else { - let game_id = GameId::new(); - let updated: GameLobby = lobby.open(Game { - board_size: PUBLIC_GAME_BOARD_SIZE, - creator: session_id.clone(), - visibility, - game_id: game_id.clone(), - }); - if let Err(_) = reg.game_lobby_repo.put(&updated) { - error!("game lobby write F2"); - } else { - if let Err(_) = reg.xadd.xadd(StreamOutput::WFO(WaitForOpponent { - event_id: EventId::new(), - game_id, - session_id: session_id.clone(), - visibility, - })) { - error!("XADD: Wait for oppo") - } else { - trace!("Public game open. Lobby: {:?}", &updated) - } - } - } + let fpg: FindPublicGame = bincode::deserialize(&data)?; + let reg = &self.reg; + let visibility = Visibility::Public; + let session_id = &fpg.session_id; + if let Ok(lobby) = reg.game_lobby_repo.get() { + if let Some(queued) = lobby + .games + .iter() + .find(|g| g.visibility == Visibility::Public) + { + ready_game(session_id, &lobby, queued, ®) + } else { + let game_id = GameId::new(); + let updated: GameLobby = lobby.open(Game { + board_size: PUBLIC_GAME_BOARD_SIZE, + creator: session_id.clone(), + visibility, + game_id: game_id.clone(), + }); + if let Err(_) = reg.game_lobby_repo.put(&updated) { + error!("game lobby write F2"); + } else { + if let Err(_) = reg.xadd.xadd(StreamOutput::WFO(WaitForOpponent { + event_id: EventId::new(), + game_id, + session_id: session_id.clone(), + visibility, + })) { + error!("XADD: Wait for oppo") } else { - error!("Failed to fetch game lobby: FPG") - }} - - */ + trace!("Public game open. Lobby: {:?}", &updated) + } + } + } + } else { + error!("Failed to fetch game lobby: FPG") + } }) } From 3669369ad71173d5a9289b630988d7fc510f2230 Mon Sep 17 00:00:00 2001 From: terkwood <38859656+Terkwood@users.noreply.github.com> Date: Sat, 29 May 2021 13:21:07 -0400 Subject: [PATCH 49/54] impl topics --- micro-game-lobby/src/main.rs | 4 +- micro-game-lobby/src/stream/mod.rs | 98 +++++++++++++++++------------- 2 files changed, 57 insertions(+), 45 deletions(-) diff --git a/micro-game-lobby/src/main.rs b/micro-game-lobby/src/main.rs index 7cc36634..7ec46a3f 100644 --- a/micro-game-lobby/src/main.rs +++ b/micro-game-lobby/src/main.rs @@ -21,11 +21,11 @@ fn main() { ), ( topics::JOIN_PRIVATE_GAME, - Box::new(|_xid, msg| Ok(lobby.consume_jpg(msg))), + Box::new(|_xid, msg| lobby.consume_jpg(msg)), ), ( topics::CREATE_GAME, - Box::new(|_xid, msg| Ok(lobby.consume_cg(msg))), + Box::new(|_xid, msg| lobby.consume_cg(msg)), ), ( topics::SESSION_DISCONNECTED, diff --git a/micro-game-lobby/src/stream/mod.rs b/micro-game-lobby/src/stream/mod.rs index 7e68c2fb..9e086bdb 100644 --- a/micro-game-lobby/src/stream/mod.rs +++ b/micro-game-lobby/src/stream/mod.rs @@ -104,38 +104,45 @@ impl LobbyStreams { } else { error!("Failed to fetch game lobby: FPG") } + } else { + error!("could not deserialize FPG data field") }) } - pub fn consume_cg(&self, msg: &Message) { - todo!("deser"); - let mut cg: CreateGame = todo!(); - let session_id = &cg.session_id; - let game_id = cg.game_id.clone().unwrap_or(GameId::new()); - if let Ok(lobby) = self.reg.game_lobby_repo.get() { - let updated: GameLobby = lobby.open(Game { - game_id: game_id.clone(), - board_size: cg.board_size, - creator: session_id.clone(), - visibility: cg.visibility, - }); - if let Err(_) = self.reg.game_lobby_repo.put(&updated) { - error!("game lobby write F1"); - } else { - if let Err(_) = self.reg.xadd.xadd(StreamOutput::WFO(WaitForOpponent { + pub fn consume_cg(&self, msg: &Message) -> anyhow::Result<()> { + let maybe_value = msg.get("data"); + Ok(if let Some(redis::Value::Data(data)) = maybe_value { + let cg: CreateGame = bincode::deserialize(&data)?; + + let session_id = &cg.session_id; + let game_id = cg.game_id.clone().unwrap_or(GameId::new()); + if let Ok(lobby) = self.reg.game_lobby_repo.get() { + let updated: GameLobby = lobby.open(Game { game_id: game_id.clone(), - session_id: session_id.clone(), - event_id: EventId::new(), + board_size: cg.board_size, + creator: session_id.clone(), visibility: cg.visibility, - })) { - error!("XADD Game ready") + }); + if let Err(_) = self.reg.game_lobby_repo.put(&updated) { + error!("game lobby write F1"); } else { - trace!("Game created. Lobby: {:?}", &updated); + if let Err(_) = self.reg.xadd.xadd(StreamOutput::WFO(WaitForOpponent { + game_id: game_id.clone(), + session_id: session_id.clone(), + event_id: EventId::new(), + visibility: cg.visibility, + })) { + error!("XADD Game ready") + } else { + trace!("Game created. Lobby: {:?}", &updated); + } } + } else { + error!("CG GAME REPO GET") } } else { - error!("CG GAME REPO GET") - } + error!("could not deser create game data field") + }) } /// Consumes the command to join a private game. @@ -143,30 +150,35 @@ impl LobbyStreams { /// we will simply log a warning. /// Consider implementing logic related to handling /// private game rejection: https://github.com/Terkwood/BUGOUT/issues/304 - pub fn consume_jpg(&self, msg: &Message) { - let jpg: JoinPrivateGame = todo!("deser"); + pub fn consume_jpg(&self, msg: &Message) -> anyhow::Result<()> { + let maybe_value = msg.get("data"); + Ok(if let Some(redis::Value::Data(data)) = maybe_value { + let jpg: JoinPrivateGame = bincode::deserialize(&data)?; - let reg = &self.reg; - if let Ok(lobby) = reg.game_lobby_repo.get() { - if let Some(queued) = lobby - .games - .iter() - .find(|g| g.visibility == Visibility::Private && g.game_id == jpg.game_id) - { - ready_game(&jpg.session_id, &lobby, queued, reg) - } else { - if let Err(e) = reg.xadd.xadd(StreamOutput::PGR(PrivateGameRejected { - client_id: jpg.client_id.clone(), - event_id: EventId::new(), - game_id: jpg.game_id.clone(), - session_id: jpg.session_id.clone(), - })) { - error!("Error writing private game rejection to stream {:?}", e) + let reg = &self.reg; + if let Ok(lobby) = reg.game_lobby_repo.get() { + if let Some(queued) = lobby + .games + .iter() + .find(|g| g.visibility == Visibility::Private && g.game_id == jpg.game_id) + { + ready_game(&jpg.session_id, &lobby, queued, reg) + } else { + if let Err(e) = reg.xadd.xadd(StreamOutput::PGR(PrivateGameRejected { + client_id: jpg.client_id.clone(), + event_id: EventId::new(), + game_id: jpg.game_id.clone(), + session_id: jpg.session_id.clone(), + })) { + error!("Error writing private game rejection to stream {:?}", e) + } } + } else { + error!("game lobby JPG get") } } else { - error!("game lobby JPG get") - } + error!("could not consume JoinPrivateGame data field") + }) } pub fn consume_sd(&self, msg: &Message) { From 6910a80563365eab00acb8ad7ae38b6e616ffd62 Mon Sep 17 00:00:00 2001 From: terkwood <38859656+Terkwood@users.noreply.github.com> Date: Sat, 29 May 2021 13:28:16 -0400 Subject: [PATCH 50/54] fiddle --- micro-game-lobby/src/main.rs | 3 +- micro-game-lobby/src/stream/mod.rs | 25 +++--- micro-game-lobby/src/stream/xread.rs | 117 --------------------------- 3 files changed, 17 insertions(+), 128 deletions(-) delete mode 100644 micro-game-lobby/src/stream/xread.rs diff --git a/micro-game-lobby/src/main.rs b/micro-game-lobby/src/main.rs index 7ec46a3f..040e4aac 100644 --- a/micro-game-lobby/src/main.rs +++ b/micro-game-lobby/src/main.rs @@ -13,6 +13,7 @@ fn main() { stream::create_consumer_group(&client); let lobby = stream::LobbyStreams::new(components); + let mut conn = client.get_connection().expect("redis conn"); let stream_handlers: Vec<(&str, Box anyhow::Result<()>>)> = vec![ ( @@ -29,7 +30,7 @@ fn main() { ), ( topics::SESSION_DISCONNECTED, - Box::new(|_xid, msg| Ok(lobby.consume_sd(msg))), + Box::new(|_xid, msg| lobby.consume_sd(msg)), ), ]; let mut streams = diff --git a/micro-game-lobby/src/stream/mod.rs b/micro-game-lobby/src/stream/mod.rs index 9e086bdb..da3015ba 100644 --- a/micro-game-lobby/src/stream/mod.rs +++ b/micro-game-lobby/src/stream/mod.rs @@ -181,19 +181,24 @@ impl LobbyStreams { }) } - pub fn consume_sd(&self, msg: &Message) { - let sd: SessionDisconnected = todo!("deser"); - let reg = &self.reg; - if let Ok(game_lobby) = reg.game_lobby_repo.get() { - let updated: GameLobby = game_lobby.abandon(&sd.session_id); - if let Err(_) = reg.game_lobby_repo.put(&updated) { - error!("game lobby write F1"); + pub fn consume_sd(&self, msg: &Message) -> anyhow::Result<()> { + let maybe_value = msg.get("data"); + Ok(if let Some(redis::Value::Data(data)) = maybe_value { + let sd: SessionDisconnected = bincode::deserialize(&data)?; + let reg = &self.reg; + if let Ok(game_lobby) = reg.game_lobby_repo.get() { + let updated: GameLobby = game_lobby.abandon(&sd.session_id); + if let Err(_) = reg.game_lobby_repo.put(&updated) { + error!("game lobby write F1"); + } else { + trace!("session {} abandoned: {:?}", sd.session_id.0, &updated); + } } else { - trace!("session {} abandoned: {:?}", sd.session_id.0, &updated); + error!("SD GAME REPO GET") } } else { - error!("SD GAME REPO GET") - } + error!("could not deser session disconn data field") + }) } } fn ready_game(session_id: &SessionId, lobby: &GameLobby, queued: &Game, reg: &Components) { diff --git a/micro-game-lobby/src/stream/xread.rs b/micro-game-lobby/src/stream/xread.rs deleted file mode 100644 index 5465ddfc..00000000 --- a/micro-game-lobby/src/stream/xread.rs +++ /dev/null @@ -1,117 +0,0 @@ -use super::GROUP_NAME; -use crate::topics::*; -use lobby_model::api::*; -use log::{error, warn}; -use redis::streams::{StreamReadOptions, StreamReadReply}; -use redis::{Client, Commands}; -use redis_streams::XReadEntryId; -use std::collections::HashMap; -use std::rc::Rc; - - -/// xread_sorted performs a redis xread then sorts the results -/// -/// entry_ids: the minimum entry ids from which to read -pub trait XRead { - fn xread_sorted(&self) -> Result, XReadErr>; -} - -#[derive(Debug)] -pub enum XReadErr { - Deser(XReadDeserErr), - Other, -} -const READ_OP: &str = ">"; - -impl XRead for Rc { - fn xread_sorted(&self) -> Result, XReadErr> { - if let Ok(mut conn) = self.get_connection() { - let opts = StreamReadOptions::default() - .block(BLOCK_MS) - .group(GROUP_NAME, "singleton"); - let xrr = conn.xread_options( - &[ - FIND_PUBLIC_GAME, - CREATE_GAME, - JOIN_PRIVATE_GAME, - SESSION_DISCONNECTED, - ], - &[READ_OP, READ_OP, READ_OP, READ_OP], - opts, - ); - - if let Ok(x) = xrr { - match deser(x) { - Ok(unsorted) => { - let mut sorted_keys: Vec = - unsorted.keys().map(|k| *k).collect(); - sorted_keys.sort(); - - let mut answer = vec![]; - for sk in sorted_keys { - if let Some(data) = unsorted.get(&sk) { - answer.push((sk, data.clone())) - } - } - Ok(answer) - } - Err(e) => Err(XReadErr::Deser(e)), - } - } else { - Err(XReadErr::Other) - } - } else { - Err(XReadErr::Other) - } - } -} - -fn deser(srr: StreamReadReply) -> Result, XReadDeserErr> { - let mut out = HashMap::new(); - for k in srr.keys { - let key = k.key; - for e in k.ids { - if let Ok(eid) = XReadEntryId::from_str(&e.id) { - - if let Some(data) = maybe_data { - let sd: Option = if key == FIND_PUBLIC_GAME { - - .map(|fpg| StreamInput::FPG(fpg)) - .ok() - } else if key == CREATE_GAME { - bincode::deserialize(&data) - .map(|cg| StreamInput::CG(cg)) - .ok() - } else if key == JOIN_PRIVATE_GAME { - bincode::deserialize(&data) - .map(|jpg| StreamInput::JPG(jpg)) - .ok() - } else if key == SESSION_DISCONNECTED { - bincode::deserialize(&data) - .map(|sd| StreamInput::SD(sd)) - .ok() - } else { - warn!("Unknown key {}", key); - None - }; - if let Some(s) = sd { - out.insert(eid, s); - } else { - return Err(XReadDeserErr::DataDeser(key)); - } - } - } else { - error!("eid-ish"); - return Err(XReadDeserErr::EIDFormat); - } - } - } - Ok(out) -} - -#[derive(Debug)] -pub enum XReadDeserErr { - EIDFormat, - DataDeser(String), -} - From 63a69013dd736fe478961d285c30c435f39ff5fd Mon Sep 17 00:00:00 2001 From: terkwood <38859656+Terkwood@users.noreply.github.com> Date: Sat, 29 May 2021 13:28:28 -0400 Subject: [PATCH 51/54] trim --- micro-game-lobby/src/components.rs | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/micro-game-lobby/src/components.rs b/micro-game-lobby/src/components.rs index f67deda2..8633942f 100644 --- a/micro-game-lobby/src/components.rs +++ b/micro-game-lobby/src/components.rs @@ -1,6 +1,5 @@ use crate::repo::GameLobbyRepo; use crate::stream::XAdd; -use redis_streams::{RedisSortedStreams, SortedStreams}; use std::rc::Rc; pub struct Components { @@ -14,13 +13,11 @@ pub fn redis_client() -> Rc { Rc::new(redis::Client::open(REDIS_URL).expect("redis client")) } - impl Components { pub fn new(client: Rc) -> Self { - Components { game_lobby_repo: Box::new(client.clone()), - xadd: Box::new(client), + xadd: Box::new(client), } } } From 0f8bb0192ccd6b0f48e4dbfb85dfdf6d777be2df Mon Sep 17 00:00:00 2001 From: terkwood <38859656+Terkwood@users.noreply.github.com> Date: Sat, 29 May 2021 13:35:15 -0400 Subject: [PATCH 52/54] ignore busygroup stream creation --- redis_streams/src/sorted_streams.rs | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/redis_streams/src/sorted_streams.rs b/redis_streams/src/sorted_streams.rs index 0b33c3ea..2422bb91 100644 --- a/redis_streams/src/sorted_streams.rs +++ b/redis_streams/src/sorted_streams.rs @@ -37,7 +37,19 @@ where ) -> Result { let mut handlers = HashMap::new(); for (stream, handler) in stream_handlers { - redis.xgroup_create_mkstream(stream, &opts.group.group_name, "$")?; + let _: Result<(), redis::RedisError> = match redis + .xgroup_create_mkstream(stream, &opts.group.group_name, "$") + .map(|_: redis::Value| ()) + { + Err(err) => { + if err.to_string() == "BUSYGROUP: Consumer Group name already exists" { + Ok(()) // ignore busygroup + } else { + Err(err) + } + } + _ => Ok(()), + }; handlers.insert(stream.to_string(), StreamHandler::new(stream, handler)); } From 19e65b2a5c834a3edd10e6703d154bb51b079ad4 Mon Sep 17 00:00:00 2001 From: terkwood <38859656+Terkwood@users.noreply.github.com> Date: Sat, 29 May 2021 13:35:46 -0400 Subject: [PATCH 53/54] set dep ver --- micro-game-lobby/Cargo.lock | 2 +- micro-game-lobby/Cargo.toml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/micro-game-lobby/Cargo.lock b/micro-game-lobby/Cargo.lock index 9caf1d74..ee33f3be 100644 --- a/micro-game-lobby/Cargo.lock +++ b/micro-game-lobby/Cargo.lock @@ -430,7 +430,7 @@ dependencies = [ [[package]] name = "redis_streams" version = "1.0.0-a" -source = "git+https://github.com/Terkwood/BUGOUT?rev=d95de67#d95de671fc11908109e853335420c45177474d20" +source = "git+https://github.com/Terkwood/BUGOUT?rev=0f8bb01#0f8bb0192ccd6b0f48e4dbfb85dfdf6d777be2df" dependencies = [ "anyhow", "redis", diff --git a/micro-game-lobby/Cargo.toml b/micro-game-lobby/Cargo.toml index 7350e39c..72c886b8 100644 --- a/micro-game-lobby/Cargo.toml +++ b/micro-game-lobby/Cargo.toml @@ -14,4 +14,4 @@ lobby-model = {path = "lobby-model"} log = "0.4.8" move-model = {git = "https://github.com/Terkwood/BUGOUT", rev = "20e6620"} redis = {version = "0.20.0", features = ["r2d2"]} -redis_streams = {git = "https://github.com/Terkwood/BUGOUT", rev = "d95de67"} +redis_streams = {git = "https://github.com/Terkwood/BUGOUT", rev = "0f8bb01"} From 510dbd6cc0e8e1142f69a18a978404fec2b75ca7 Mon Sep 17 00:00:00 2001 From: terkwood <38859656+Terkwood@users.noreply.github.com> Date: Sat, 29 May 2021 13:43:51 -0400 Subject: [PATCH 54/54] set dependency version --- micro-game-lobby/Cargo.lock | 4 ++-- micro-game-lobby/Cargo.toml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/micro-game-lobby/Cargo.lock b/micro-game-lobby/Cargo.lock index ee33f3be..4d29ff68 100644 --- a/micro-game-lobby/Cargo.lock +++ b/micro-game-lobby/Cargo.lock @@ -429,8 +429,8 @@ dependencies = [ [[package]] name = "redis_streams" -version = "1.0.0-a" -source = "git+https://github.com/Terkwood/BUGOUT?rev=0f8bb01#0f8bb0192ccd6b0f48e4dbfb85dfdf6d777be2df" +version = "1.0.0" +source = "git+https://github.com/Terkwood/BUGOUT?rev=a394d46#a394d4650faba1c834770d9213323617208c53ab" dependencies = [ "anyhow", "redis", diff --git a/micro-game-lobby/Cargo.toml b/micro-game-lobby/Cargo.toml index 72c886b8..9f1118c9 100644 --- a/micro-game-lobby/Cargo.toml +++ b/micro-game-lobby/Cargo.toml @@ -14,4 +14,4 @@ lobby-model = {path = "lobby-model"} log = "0.4.8" move-model = {git = "https://github.com/Terkwood/BUGOUT", rev = "20e6620"} redis = {version = "0.20.0", features = ["r2d2"]} -redis_streams = {git = "https://github.com/Terkwood/BUGOUT", rev = "0f8bb01"} +redis_streams = {git = "https://github.com/Terkwood/BUGOUT", rev = "a394d46"}