From 64c6da29a86d21c520f1eec176a69d44ddc8d9dd Mon Sep 17 00:00:00 2001 From: John Behm Date: Mon, 26 Feb 2024 23:43:38 +0100 Subject: [PATCH 01/76] [WIP] the context update - add context to every sesion methos, every start method - remove queue and replace with buffered channel - add context to TopoloyFunc and to HandlerFunc, - add special errors for rejecting messages - rework flow control handling (WIP) - improve pool closing - and more ... - remove publish/await timeouts and replace with the single context that's passed to publish --- amqpx.go | 33 ++--- amqpx_options.go | 24 ---- amqpx_test.go | 171 ++++++++++++++---------- docker-compose.yaml | 32 ++++- docker/toxiproxy.json | 6 + go.mod | 1 - go.sum | 27 ---- helpers_test.go | 69 +++++----- pool/connection.go | 140 ++++++++++++------- pool/connection_options.go | 10 -- pool/connection_pool.go | 123 ++++++++--------- pool/connection_pool_options.go | 10 -- pool/connection_pool_test.go | 15 ++- pool/connection_test.go | 14 +- pool/errors.go | 17 +++ pool/pool.go | 20 +-- pool/pool_options.go | 8 -- pool/pool_test.go | 15 ++- pool/publisher.go | 84 ++++++------ pool/publisher_option.go | 26 +--- pool/publisher_test.go | 128 ++++++++++++++++-- pool/session.go | 157 ++++++++++------------ pool/session_pool.go | 73 ++++------ pool/session_pool_test.go | 8 +- pool/session_test.go | 42 +++--- pool/subscriber.go | 168 +++++++++++++---------- pool/subscriber_batch_handler.go | 2 + pool/subscriber_handler.go | 2 + pool/subscriber_handler_options_test.go | 3 +- pool/subscriber_test.go | 105 +++++++++------ pool/topologer.go | 116 ++++++++-------- 31 files changed, 892 insertions(+), 757 deletions(-) diff --git a/amqpx.go b/amqpx.go index 4398bf8..abcc15d 100644 --- a/amqpx.go +++ b/amqpx.go @@ -17,7 +17,7 @@ var ( ) type ( - TopologyFunc func(*pool.Topologer) error + TopologyFunc func(context.Context, *pool.Topologer) error ) type AMQPX struct { @@ -152,7 +152,7 @@ func (a *AMQPX) RegisterBatchHandler(queue string, handlerFunc pool.BatchHandler // settings like publish confirmations or a custom context which can signal an application shutdown. // This customcontext does not replace the Close() call. Always defer a Close() call. // Start is a non-blocking operation. -func (a *AMQPX) Start(connectUrl string, options ...Option) (err error) { +func (a *AMQPX) Start(ctx context.Context, connectUrl string, options ...Option) (err error) { a.mu.Lock() defer a.mu.Unlock() @@ -178,6 +178,7 @@ func (a *AMQPX) Start(connectUrl string, options ...Option) (err error) { // publisher and subscriber need to have different tcp connections (tcp pushback prevention) a.pubPool, err = pool.New( + ctx, connectUrl, option.PublisherConnections, option.PublisherSessions, @@ -196,7 +197,7 @@ func (a *AMQPX) Start(connectUrl string, options ...Option) (err error) { topologer := pool.NewTopologer(a.pubPool) for _, t := range a.topologies { - err = t(topologer) + err = t(ctx, topologer) if err != nil { return } @@ -225,6 +226,7 @@ func (a *AMQPX) Start(connectUrl string, options ...Option) (err error) { // with each other var subPool *pool.Pool subPool, err = pool.New( + ctx, connectUrl, connections, sessions, @@ -244,7 +246,7 @@ func (a *AMQPX) Start(connectUrl string, options ...Option) (err error) { for _, bh := range a.batchHandlers { a.sub.RegisterBatchHandler(bh) } - err = a.sub.Start() + err = a.sub.Start(ctx) if err != nil { return } @@ -280,7 +282,7 @@ func (a *AMQPX) close() (err error) { pool.TopologerWithTransientSessions(true), ) for _, f := range a.topologyDeleters { - err = errors.Join(err, f(topologer)) + err = errors.Join(err, f(ctx, topologer)) } } @@ -295,18 +297,18 @@ func (a *AMQPX) close() (err error) { // Publish a message to a specific exchange with a given routingKey. // You may set exchange to "" and routingKey to your queue name in order to publish directly to a queue. -func (a *AMQPX) Publish(exchange string, routingKey string, msg pool.Publishing) error { +func (a *AMQPX) Publish(ctx context.Context, exchange string, routingKey string, msg pool.Publishing) error { a.mu.RLock() defer a.mu.RUnlock() if a.pub == nil { panic("amqpx package was not started") } - return a.pub.Publish(exchange, routingKey, msg) + return a.pub.Publish(ctx, exchange, routingKey, msg) } // Get is only supposed to be used for testing, do not use get for polling any broker queues. -func (a *AMQPX) Get(queue string, autoAck bool) (msg pool.Delivery, ok bool, err error) { +func (a *AMQPX) Get(ctx context.Context, queue string, autoAck bool) (msg pool.Delivery, ok bool, err error) { a.mu.RLock() defer a.mu.RUnlock() if a.pub == nil { @@ -314,7 +316,7 @@ func (a *AMQPX) Get(queue string, autoAck bool) (msg pool.Delivery, ok bool, err } // publisher is used because this is a testing method for the publisher - return a.pub.Get(queue, autoAck) + return a.pub.Get(ctx, queue, autoAck) } // RegisterTopology registers a topology creating function that is called upon @@ -351,8 +353,9 @@ func RegisterBatchHandler(queue string, handlerFunc pool.BatchHandlerFunc, optio // settings like publish confirmations or a custom context which can signal an application shutdown. // This customcontext does not replace the Close() call. Always defer a Close() call. // Start is a non-blocking operation. -func Start(connectUrl string, options ...Option) (err error) { - return amqpx.Start(connectUrl, options...) +// The startup context may differ from the cancelation context provided via the options. +func Start(ctx context.Context, connectUrl string, options ...Option) (err error) { + return amqpx.Start(ctx, connectUrl, options...) } func Close() error { @@ -361,13 +364,13 @@ func Close() error { // Publish a message to a specific exchange with a given routingKey. // You may set exchange to "" and routingKey to your queue name in order to publish directly to a queue. -func Publish(exchange string, routingKey string, msg pool.Publishing) error { - return amqpx.Publish(exchange, routingKey, msg) +func Publish(ctx context.Context, exchange string, routingKey string, msg pool.Publishing) error { + return amqpx.Publish(ctx, exchange, routingKey, msg) } // Get is only supposed to be used for testing, do not use get for polling any broker queues. -func Get(queue string, autoAck bool) (msg pool.Delivery, ok bool, err error) { - return amqpx.Get(queue, autoAck) +func Get(ctx context.Context, queue string, autoAck bool) (msg pool.Delivery, ok bool, err error) { + return amqpx.Get(ctx, queue, autoAck) } // Reset closes the current package and resets its state before it was initialized and started. diff --git a/amqpx_options.go b/amqpx_options.go index 0cdf1cf..9b14431 100644 --- a/amqpx_options.go +++ b/amqpx_options.go @@ -1,7 +1,6 @@ package amqpx import ( - "context" "crypto/tls" "time" @@ -51,13 +50,6 @@ func WithConnectionTimeout(timeout time.Duration) Option { } } -// WithContext allows to set a custom connection timeout, that MUST be >= 1 * time.Second -func WithContext(ctx context.Context) Option { - return func(o *option) { - o.PoolOptions = append(o.PoolOptions, pool.WithContext(ctx)) - } -} - // WithTLS allows to configure tls connectivity. func WithTLS(config *tls.Config) Option { return func(o *option) { @@ -81,22 +73,6 @@ func WithConfirms(requirePublishConfirms bool) Option { } } -// WithConfirmTimeout is the timout before another publishing attempt is tried. -// As the broker did not send any confirmation that our published message arrived. -func WithConfirmTimeout(timeout time.Duration) Option { - return func(o *option) { - o.PublisherOptions = append(o.PublisherOptions, pool.PublisherWithConfirmTimeout(timeout)) - } -} - -// WithPublishTimeout is the timout that we attempt to try sending our message to the broker -// before aborting. This does not affect the overall time of publishing, as publishing is retried indefinitely. -func WithPublishTimeout(timeout time.Duration) Option { - return func(o *option) { - o.PublisherOptions = append(o.PublisherOptions, pool.PublisherWithPublishTimeout(timeout)) - } -} - // WithPublisherConnections defines the number of tcp connections of the publisher. func WithPublisherConnections(connections int) Option { if connections < 1 { diff --git a/amqpx_test.go b/amqpx_test.go index c8e541f..53cac2c 100644 --- a/amqpx_test.go +++ b/amqpx_test.go @@ -32,19 +32,21 @@ func TestMain(m *testing.M) { } func TestExchangeDeclarePassive(t *testing.T) { + ctx := context.TODO() defer amqpx.Reset() eName := "exchange-01" var err error - amqpx.RegisterTopologyCreator(func(t *pool.Topologer) error { - return createExchange(eName, t) + amqpx.RegisterTopologyCreator(func(ctx context.Context, t *pool.Topologer) error { + return createExchange(ctx, eName, t) }) - amqpx.RegisterTopologyDeleter(func(t *pool.Topologer) error { - return deleteExchange(eName, t) + amqpx.RegisterTopologyDeleter(func(ctx context.Context, t *pool.Topologer) error { + return deleteExchange(ctx, eName, t) }) err = amqpx.Start( + ctx, connectURL, amqpx.WithLogger(logging.NewTestLogger(t)), amqpx.WithPublisherConnections(1), @@ -54,19 +56,21 @@ func TestExchangeDeclarePassive(t *testing.T) { } func TestQueueDeclarePassive(t *testing.T) { + ctx := context.TODO() defer amqpx.Reset() qName := "queue-01" var err error - amqpx.RegisterTopologyCreator(func(t *pool.Topologer) error { - return createQueue(qName, t) + amqpx.RegisterTopologyCreator(func(ctx context.Context, t *pool.Topologer) error { + return createQueue(ctx, qName, t) }) - amqpx.RegisterTopologyDeleter(func(t *pool.Topologer) error { - return deleteQueue(qName, t) + amqpx.RegisterTopologyDeleter(func(ctx context.Context, t *pool.Topologer) error { + return deleteQueue(ctx, qName, t) }) err = amqpx.Start( + ctx, connectURL, amqpx.WithLogger(logging.NewTestLogger(t)), amqpx.WithPublisherConnections(1), @@ -76,12 +80,14 @@ func TestQueueDeclarePassive(t *testing.T) { } func TestAMQPXPub(t *testing.T) { + ctx := context.TODO() defer amqpx.Reset() amqpx.RegisterTopologyCreator(createTopology) amqpx.RegisterTopologyDeleter(deleteTopology) err := amqpx.Start( + ctx, connectURL, amqpx.WithLogger(logging.NewNoOpLogger()), amqpx.WithPublisherConnections(1), @@ -100,7 +106,7 @@ func TestAMQPXPub(t *testing.T) { event := "TestAMQPXPub - event content" // publish event to first queue - err = amqpx.Publish("exchange-01", "event-01", pool.Publishing{ + err = amqpx.Publish(ctx, "exchange-01", "event-01", pool.Publishing{ ContentType: "application/json", Body: []byte(event), }) @@ -114,7 +120,7 @@ func TestAMQPXPub(t *testing.T) { ok bool ) for i := 0; i < 20; i++ { - msg, ok, err = amqpx.Get("queue-01", false) + msg, ok, err = amqpx.Get(ctx, "queue-01", false) if err != nil { assert.NoError(t, err) return @@ -135,24 +141,26 @@ func TestAMQPXPub(t *testing.T) { } func TestAMQPXSubAndPub(t *testing.T) { + ctx := context.TODO() log := logging.NewTestLogger(t) defer amqpx.Reset() amqpx.RegisterTopologyCreator(createTopology) amqpx.RegisterTopologyDeleter(deleteTopology) - ctx, cancel := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGINT) + ctx, cancel := signal.NotifyContext(ctx, syscall.SIGINT, syscall.SIGINT) defer cancel() eventContent := "TestAMQPXSubAndPub - event content" - amqpx.RegisterHandler("queue-01", func(msg pool.Delivery) error { + amqpx.RegisterHandler("queue-01", func(ctx context.Context, msg pool.Delivery) error { log.Info("subscriber of queue-01") cancel() return nil }) err := amqpx.Start( + ctx, connectURL, amqpx.WithLogger(logging.NewNoOpLogger()), amqpx.WithPublisherConnections(1), @@ -165,7 +173,7 @@ func TestAMQPXSubAndPub(t *testing.T) { // publish event to first queue - err = amqpx.Publish("exchange-01", "event-01", pool.Publishing{ + err = amqpx.Publish(ctx, "exchange-01", "event-01", pool.Publishing{ ContentType: "application/json", Body: []byte(eventContent), }) @@ -182,22 +190,23 @@ func TestAMQPXSubAndPub(t *testing.T) { } func TestAMQPXSubAndPubMulti(t *testing.T) { + ctx := context.TODO() log := logging.NewTestLogger(t) defer amqpx.Reset() amqpx.RegisterTopologyCreator(createTopology) amqpx.RegisterTopologyDeleter(deleteTopology) - ctx, cancel := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGINT) + ctx, cancel := signal.NotifyContext(ctx, syscall.SIGINT, syscall.SIGINT) defer cancel() eventContent := "TestAMQPXSubAndPub - event content" // publish -> queue-01 -> subscriber-01 -> queue-02 -> subscriber-02 -> queue-03 -> subscriber-03 -> cancel context - amqpx.RegisterHandler("queue-01", func(msg pool.Delivery) error { + amqpx.RegisterHandler("queue-01", func(ctx context.Context, msg pool.Delivery) error { log.Info("handler of subscriber-01") - err := amqpx.Publish("exchange-02", "event-02", pool.Publishing{ + err := amqpx.Publish(ctx, "exchange-02", "event-02", pool.Publishing{ ContentType: msg.ContentType, Body: msg.Body, }) @@ -212,10 +221,10 @@ func TestAMQPXSubAndPubMulti(t *testing.T) { pool.ConsumeOptions{ConsumerTag: "subscriber-01"}, ) - amqpx.RegisterHandler("queue-02", func(msg pool.Delivery) error { + amqpx.RegisterHandler("queue-02", func(ctx context.Context, msg pool.Delivery) error { log.Info("handler of subscriber-02") - err := amqpx.Publish("exchange-03", "event-03", pool.Publishing{ + err := amqpx.Publish(ctx, "exchange-03", "event-03", pool.Publishing{ ContentType: msg.ContentType, Body: msg.Body, }) @@ -227,13 +236,14 @@ func TestAMQPXSubAndPubMulti(t *testing.T) { return nil }, pool.ConsumeOptions{ConsumerTag: "subscriber-02"}) - amqpx.RegisterHandler("queue-03", func(msg pool.Delivery) error { + amqpx.RegisterHandler("queue-03", func(ctx context.Context, msg pool.Delivery) error { log.Info("handler of subscriber-03: canceling context!") cancel() return nil }, pool.ConsumeOptions{ConsumerTag: "subscriber-03"}) err := amqpx.Start( + ctx, connectURL, amqpx.WithLogger(log), amqpx.WithPublisherConnections(1), @@ -246,7 +256,7 @@ func TestAMQPXSubAndPubMulti(t *testing.T) { // publish event to first queue - err = amqpx.Publish("exchange-01", "event-01", pool.Publishing{ + err = amqpx.Publish(ctx, "exchange-01", "event-01", pool.Publishing{ ContentType: "application/json", Body: []byte(eventContent), }) @@ -263,24 +273,26 @@ func TestAMQPXSubAndPubMulti(t *testing.T) { } func TestAMQPXSubHandler(t *testing.T) { + ctx := context.TODO() log := logging.NewTestLogger(t) defer amqpx.Reset() amqpx.RegisterTopologyCreator(createTopology) amqpx.RegisterTopologyDeleter(deleteTopology) - ctx, cancel := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGINT) + ctx, cancel := signal.NotifyContext(ctx, syscall.SIGINT, syscall.SIGINT) defer cancel() eventContent := "TestAMQPXSubAndPub - event content" - amqpx.RegisterHandler("queue-01", func(msg pool.Delivery) error { + amqpx.RegisterHandler("queue-01", func(ctx context.Context, msg pool.Delivery) error { log.Info("subscriber of queue-01") cancel() return nil }) err := amqpx.Start( + ctx, connectURL, amqpx.WithLogger(logging.NewNoOpLogger()), amqpx.WithPublisherConnections(1), @@ -293,7 +305,7 @@ func TestAMQPXSubHandler(t *testing.T) { // publish event to first queue - err = amqpx.Publish("exchange-01", "event-01", pool.Publishing{ + err = amqpx.Publish(ctx, "exchange-01", "event-01", pool.Publishing{ ContentType: "application/json", Body: []byte(eventContent), }) @@ -310,6 +322,7 @@ func TestAMQPXSubHandler(t *testing.T) { } func TestCreateDeleteTopology(t *testing.T) { + ctx := context.TODO() log := logging.NewTestLogger(t) defer amqpx.Reset() @@ -317,6 +330,7 @@ func TestCreateDeleteTopology(t *testing.T) { amqpx.RegisterTopologyDeleter(deleteTopology) err := amqpx.Start( + ctx, connectURL, amqpx.WithLogger(log), amqpx.WithPublisherConnections(1), @@ -334,32 +348,31 @@ func TestPauseResumeHandlerNoProcessing(t *testing.T) { log := logging.NewTestLogger(t) amqp := amqpx.New() - amqp.RegisterTopologyCreator(func(t *pool.Topologer) error { - _, err := t.QueueDeclare(queueName) + amqp.RegisterTopologyCreator(func(ctx context.Context, t *pool.Topologer) error { + _, err := t.QueueDeclare(ctx, queueName) if err != nil { return err } return nil }) - amqp.RegisterTopologyDeleter(func(t *pool.Topologer) error { - _, err := t.QueueDelete(queueName) + amqp.RegisterTopologyDeleter(func(ctx context.Context, t *pool.Topologer) error { + _, err := t.QueueDelete(ctx, queueName) if err != nil { return err } return nil }) - handler := amqp.RegisterHandler(queueName, func(d pool.Delivery) error { + handler := amqp.RegisterHandler(queueName, func(ctx context.Context, d pool.Delivery) error { log.Info("received message") return nil }) - err = amqp.Start(connectURL, - amqpx.WithLogger( - logging.NewNoOpLogger(), - ), - amqpx.WithContext(ctx), + err = amqp.Start( + ctx, + connectURL, + amqpx.WithLogger(logging.NewNoOpLogger()), ) if err != nil { assert.NoError(t, err) @@ -427,12 +440,16 @@ func testHandlerPauseAndResume(t *testing.T) { // step 1 - fill queue with messages amqpx.RegisterTopologyDeleter(deleteTopology) - err = amqpxPublish.Start(connectURL, options...) + err = amqpxPublish.Start( + ctx, + connectURL, + options..., + ) require.NoError(t, err) // fill queue with messages for i := 0; i < publish; i++ { - err := amqpxPublish.Publish("exchange-01", "event-01", pool.Publishing{ + err := amqpxPublish.Publish(ctx, "exchange-01", "event-01", pool.Publishing{ ContentType: "application/json", Body: []byte(fmt.Sprintf("%s: message number %d", eventContent, i)), }) @@ -443,10 +460,10 @@ func testHandlerPauseAndResume(t *testing.T) { } // step 2 - process messages, pause, wait, resume, process rest, cancel context - handler01 := amqpx.RegisterHandler("queue-01", func(msg pool.Delivery) (err error) { + handler01 := amqpx.RegisterHandler("queue-01", func(ctx context.Context, msg pool.Delivery) (err error) { cnt++ if cnt == publish/3 || cnt == publish/3*2 { - err = amqpx.Publish("exchange-02", "event-02", pool.Publishing{ + err = amqpx.Publish(ctx, "exchange-02", "event-02", pool.Publishing{ ContentType: "application/json", Body: []byte(fmt.Sprintf("%s: hit %d messages, toggling processing", eventContent, cnt)), }) @@ -457,7 +474,7 @@ func testHandlerPauseAndResume(t *testing.T) { }) running := true - amqpx.RegisterHandler("queue-02", func(msg pool.Delivery) (err error) { + amqpx.RegisterHandler("queue-02", func(ctx context.Context, msg pool.Delivery) (err error) { log.Infof("received toggle request: %s", string(msg.Body)) queue := handler01.Queue() @@ -482,7 +499,7 @@ func testHandlerPauseAndResume(t *testing.T) { assertActive(t, handler01, true) // trigger cancelation - err = amqpxPublish.Publish("exchange-03", "event-03", pool.Publishing{ + err = amqpxPublish.Publish(ctx, "exchange-03", "event-03", pool.Publishing{ ContentType: "application/json", Body: []byte(fmt.Sprintf("%s: delayed toggle back", eventContent)), }) @@ -492,7 +509,7 @@ func testHandlerPauseAndResume(t *testing.T) { }) var once sync.Once - amqpx.RegisterHandler("queue-03", func(msg pool.Delivery) (err error) { + amqpx.RegisterHandler("queue-03", func(ctx context.Context, msg pool.Delivery) (err error) { once.Do(func() { log.Info("pausing handler") @@ -516,6 +533,7 @@ func testHandlerPauseAndResume(t *testing.T) { assertActive(t, handler01, false) err = amqpx.Start( + ctx, connectURL, amqpx.WithLogger(logging.NewNoOpLogger()), amqpx.WithPublisherConnections(1), @@ -557,7 +575,7 @@ func testBatchHandlerPauseAndResume(t *testing.T) { amqpxPublish := amqpx.New() amqpxPublish.RegisterTopologyCreator(createTopology) - err = amqpxPublish.Start(connectURL, options...) + err = amqpxPublish.Start(ctx, connectURL, options...) require.NoError(t, err) eventContent := "TestBatchHandlerPauseAndResume - event content" @@ -572,7 +590,7 @@ func testBatchHandlerPauseAndResume(t *testing.T) { // fill queue with messages for i := 0; i < publish; i++ { - err := amqpxPublish.Publish("exchange-01", "event-01", pool.Publishing{ + err := amqpxPublish.Publish(ctx, "exchange-01", "event-01", pool.Publishing{ ContentType: "application/json", Body: []byte(fmt.Sprintf("%s: message number %d", eventContent, i)), }) @@ -583,11 +601,11 @@ func testBatchHandlerPauseAndResume(t *testing.T) { } // step 2 - process messages, pause, wait, resume, process rest, cancel context - handler01 := amqpx.RegisterBatchHandler("queue-01", func(msgs []pool.Delivery) (err error) { + handler01 := amqpx.RegisterBatchHandler("queue-01", func(ctx context.Context, msgs []pool.Delivery) (err error) { for _, msg := range msgs { cnt++ if cnt == publish/3 || cnt == publish/3*2 { - err = amqpx.Publish("exchange-02", "event-02", pool.Publishing{ + err = amqpx.Publish(ctx, "exchange-02", "event-02", pool.Publishing{ ContentType: "application/json", Body: []byte(fmt.Sprintf("%s: hit %d messages, toggling processing: %s", eventContent, cnt, string(msg.Body))), }) @@ -598,7 +616,7 @@ func testBatchHandlerPauseAndResume(t *testing.T) { }) running := true - amqpx.RegisterBatchHandler("queue-02", func(msgs []pool.Delivery) (err error) { + amqpx.RegisterBatchHandler("queue-02", func(ctx context.Context, msgs []pool.Delivery) (err error) { queue := handler01.Queue() for _, msg := range msgs { @@ -624,7 +642,7 @@ func testBatchHandlerPauseAndResume(t *testing.T) { assertActive(t, handler01, true) // trigger cancelation - err = amqpxPublish.Publish("exchange-03", "event-03", pool.Publishing{ + err = amqpxPublish.Publish(ctx, "exchange-03", "event-03", pool.Publishing{ ContentType: "application/json", Body: []byte(fmt.Sprintf("%s: delayed toggle back", eventContent)), }) @@ -635,7 +653,7 @@ func testBatchHandlerPauseAndResume(t *testing.T) { }) var once sync.Once - amqpx.RegisterBatchHandler("queue-03", func(msgs []pool.Delivery) (err error) { + amqpx.RegisterBatchHandler("queue-03", func(ctx context.Context, msgs []pool.Delivery) (err error) { _ = msgs[0] once.Do(func() { @@ -660,6 +678,7 @@ func testBatchHandlerPauseAndResume(t *testing.T) { assertActive(t, handler01, false) err = amqpx.Start( + ctx, connectURL, amqpx.WithLogger(logging.NewNoOpLogger()), amqpx.WithPublisherConnections(1), @@ -688,32 +707,33 @@ func TestQueueDeletedConsumerReconnect(t *testing.T) { assert.NoError(t, amqpx.Reset()) }() - ts, closer := newTransientSession(t, connectURL) + ts, closer := newTransientSession(t, ctx, connectURL) defer closer() // step 1 - fill queue with messages - amqpx.RegisterTopologyCreator(func(t *pool.Topologer) error { - _, err := t.QueueDeclare(queueName) + amqpx.RegisterTopologyCreator(func(ctx context.Context, t *pool.Topologer) error { + _, err := t.QueueDeclare(ctx, queueName) if err != nil { return err } return nil }) - amqpx.RegisterTopologyDeleter(func(t *pool.Topologer) error { - _, err := t.QueueDelete(queueName) + amqpx.RegisterTopologyDeleter(func(ctx context.Context, t *pool.Topologer) error { + _, err := t.QueueDelete(ctx, queueName) if err != nil { return err } return nil }) - h := amqpx.RegisterHandler(queueName, func(msg pool.Delivery) (err error) { + h := amqpx.RegisterHandler(queueName, func(ctx context.Context, msg pool.Delivery) (err error) { return nil }) assertActive(t, h, false) err = amqpx.Start( + ctx, connectURL, amqpx.WithLogger(log), ) @@ -722,10 +742,10 @@ func TestQueueDeletedConsumerReconnect(t *testing.T) { return } - assert.NoError(t, h.Pause(context.Background())) + assert.NoError(t, h.Pause(ctx)) assertActive(t, h, false) - _, err = ts.QueueDelete(queueName) + _, err = ts.QueueDelete(ctx, queueName) assert.NoError(t, err) tctx, cancel := context.WithTimeout(ctx, 5*time.Second) @@ -734,7 +754,7 @@ func TestQueueDeletedConsumerReconnect(t *testing.T) { assert.Error(t, err) assertActive(t, h, false) - _, err = ts.QueueDeclare(queueName) + _, err = ts.QueueDeclare(ctx, queueName) assert.NoError(t, err) tctx, cancel = context.WithTimeout(ctx, 5*time.Second) @@ -755,32 +775,33 @@ func TestQueueDeletedBatchConsumerReconnect(t *testing.T) { assert.NoError(t, amqpx.Reset()) }() - ts, closer := newTransientSession(t, connectURL) + ts, closer := newTransientSession(t, ctx, connectURL) defer closer() // step 1 - fill queue with messages - amqpx.RegisterTopologyCreator(func(t *pool.Topologer) error { - _, err := t.QueueDeclare(queueName) + amqpx.RegisterTopologyCreator(func(ctx context.Context, t *pool.Topologer) error { + _, err := t.QueueDeclare(ctx, queueName) if err != nil { return err } return nil }) - amqpx.RegisterTopologyDeleter(func(t *pool.Topologer) error { - _, err := t.QueueDelete(queueName) + amqpx.RegisterTopologyDeleter(func(ctx context.Context, t *pool.Topologer) error { + _, err := t.QueueDelete(ctx, queueName) if err != nil { return err } return nil }) - h := amqpx.RegisterBatchHandler(queueName, func(msg []pool.Delivery) (err error) { + h := amqpx.RegisterBatchHandler(queueName, func(ctx context.Context, msg []pool.Delivery) (err error) { return nil }) assertActive(t, h, false) err = amqpx.Start( + ctx, connectURL, amqpx.WithLogger(log), ) @@ -789,10 +810,10 @@ func TestQueueDeletedBatchConsumerReconnect(t *testing.T) { return } - assert.NoError(t, h.Pause(context.Background())) + assert.NoError(t, h.Pause(ctx)) assertActive(t, h, false) - _, err = ts.QueueDelete(queueName) + _, err = ts.QueueDelete(ctx, queueName) assert.NoError(t, err) tctx, cancel := context.WithTimeout(ctx, 5*time.Second) @@ -801,7 +822,7 @@ func TestQueueDeletedBatchConsumerReconnect(t *testing.T) { assert.Error(t, err) assertActive(t, h, false) - _, err = ts.QueueDeclare(queueName) + _, err = ts.QueueDeclare(ctx, queueName) assert.NoError(t, err) tctx, cancel = context.WithTimeout(ctx, 5*time.Second) @@ -811,11 +832,11 @@ func TestQueueDeletedBatchConsumerReconnect(t *testing.T) { assertActive(t, h, true) } -func newTransientSession(t *testing.T, connectUrl string) (session *pool.Session, closer func()) { - p, err := pool.New(connectUrl, 1, 1, pool.WithLogger(logging.NewTestLogger(t))) +func newTransientSession(t *testing.T, ctx context.Context, connectUrl string) (session *pool.Session, closer func()) { + p, err := pool.New(ctx, connectUrl, 1, 1, pool.WithLogger(logging.NewTestLogger(t))) require.NoError(t, err) - s, err := p.GetSession() + s, err := p.GetSession(ctx) require.NoError(t, err) return s, func() { @@ -856,7 +877,7 @@ func testHandlerReset(t *testing.T) { amqpxPublish := amqpx.New() amqpxPublish.RegisterTopologyCreator(createTopology) - err = amqpxPublish.Start(connectURL, options...) + err = amqpxPublish.Start(ctx, connectURL, options...) require.NoError(t, err) eventContent := "TestBatchHandlerReset - event content" @@ -871,7 +892,7 @@ func testHandlerReset(t *testing.T) { // fill queue with messages for i := 0; i < publish; i++ { - err := amqpxPublish.Publish("exchange-01", "event-01", pool.Publishing{ + err := amqpxPublish.Publish(ctx, "exchange-01", "event-01", pool.Publishing{ ContentType: "application/json", Body: []byte(fmt.Sprintf("%s: message number %d", eventContent, i)), }) @@ -883,7 +904,7 @@ func testHandlerReset(t *testing.T) { done := make(chan struct{}) // step 2 - process messages, pause, wait, resume, process rest, cancel context - handler01 := amqpx.RegisterHandler("queue-01", func(msgs pool.Delivery) (err error) { + handler01 := amqpx.RegisterHandler("queue-01", func(ctx context.Context, msgs pool.Delivery) (err error) { cnt++ if cnt == publish { close(done) @@ -894,6 +915,7 @@ func testHandlerReset(t *testing.T) { assertActive(t, handler01, false) err = amqpx.Start( + ctx, connectURL, amqpx.WithLogger(logging.NewNoOpLogger()), amqpx.WithPublisherConnections(1), @@ -943,7 +965,7 @@ func testBatchHandlerReset(t *testing.T) { amqpxPublish := amqpx.New() amqpxPublish.RegisterTopologyCreator(createTopology) - err = amqpxPublish.Start(connectURL, options...) + err = amqpxPublish.Start(ctx, connectURL, options...) require.NoError(t, err) eventContent := "TestBatchHandlerReset - event content" @@ -958,7 +980,7 @@ func testBatchHandlerReset(t *testing.T) { // fill queue with messages for i := 0; i < publish; i++ { - err := amqpxPublish.Publish("exchange-01", "event-01", pool.Publishing{ + err := amqpxPublish.Publish(ctx, "exchange-01", "event-01", pool.Publishing{ ContentType: "application/json", Body: []byte(fmt.Sprintf("%s: message number %d", eventContent, i)), }) @@ -970,7 +992,7 @@ func testBatchHandlerReset(t *testing.T) { done := make(chan struct{}) // step 2 - process messages, pause, wait, resume, process rest, cancel context - handler01 := amqpx.RegisterBatchHandler("queue-01", func(msgs []pool.Delivery) (err error) { + handler01 := amqpx.RegisterBatchHandler("queue-01", func(ctx context.Context, msgs []pool.Delivery) (err error) { cnt += len(msgs) if cnt == publish { @@ -982,6 +1004,7 @@ func testBatchHandlerReset(t *testing.T) { assertActive(t, handler01, false) err = amqpx.Start( + ctx, connectURL, amqpx.WithLogger(logging.NewNoOpLogger()), amqpx.WithPublisherConnections(1), diff --git a/docker-compose.yaml b/docker-compose.yaml index bfa36bd..b6f35ac 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -4,11 +4,11 @@ services: rabbitmq: image: docker.io/bitnami/rabbitmq:latest ports: - - '4369:4369' - - '5551:5551' - - '5552:5552' + #- '4369:4369' + #- '5551:5551' + #- '5552:5552' #- '5672:5672' # proxied through toxiproxy - - '25672:25672' + #- '25672:25672' # user interface - '15672:15672' environment: @@ -22,6 +22,27 @@ services: networks: - rabbitnet + rabbitmq-broken: + image: docker.io/bitnami/rabbitmq:latest + ports: + #- '14369:4369' + #- '15551:5551' + #- '15552:5552' + #- '5672:5672' # proxied through toxiproxy + #- '35672:25672' + # user interface + - '25672:15672' + environment: + RABBITMQ_NODE_TYPE: stats + RABBITMQ_USERNAME: admin + RABBITMQ_PASSWORD: password + #RABBITMQ_DISK_FREE_RELATIVE_LIMIT: 100 + RABBITMQ_VM_MEMORY_HIGH_WATERMARK: 1000 + # See: https://www.rabbitmq.com/access-control.html#loopback-users + RABBITMQ_SERVER_ADDITIONAL_ERL_ARGS: "-rabbit loopback_users []" + networks: + - rabbitnet + toxiproxy: image: ghcr.io/shopify/toxiproxy:2.5.0 command: @@ -29,7 +50,8 @@ services: - -config=/toxiproxy.json ports: - 8474:8474 - - 5672:5672 + - 5672:5672 # normal rabbitmq + - 5673:5673 # broken rabbitmq networks: - rabbitnet volumes: diff --git a/docker/toxiproxy.json b/docker/toxiproxy.json index bde8c22..bfd5437 100644 --- a/docker/toxiproxy.json +++ b/docker/toxiproxy.json @@ -4,5 +4,11 @@ "listen": "[::]:5672", "upstream": "rabbitmq:5672", "enabled": true + }, + { + "name": "rabbitmq-broken", + "listen": "[::]:5673", + "upstream": "rabbitmq-broken:5672", + "enabled": true } ] \ No newline at end of file diff --git a/go.mod b/go.mod index f2a873d..318b70a 100644 --- a/go.mod +++ b/go.mod @@ -4,7 +4,6 @@ go 1.20 require ( github.com/Shopify/toxiproxy/v2 v2.7.0 - github.com/Workiva/go-datastructures v1.1.1 github.com/rabbitmq/amqp091-go v1.9.0 github.com/stretchr/testify v1.8.4 go.uber.org/goleak v1.3.0 diff --git a/go.sum b/go.sum index b0b84af..7448880 100644 --- a/go.sum +++ b/go.sum @@ -1,7 +1,5 @@ github.com/Shopify/toxiproxy/v2 v2.7.0 h1:Zz2jdyqtYw1SpihfMWzLFGpOO92p9effjAkURG57ifc= github.com/Shopify/toxiproxy/v2 v2.7.0/go.mod h1:k0V84e/dLQmVNuI6S0G7TpXCl611OSRYdptoxm0XTzA= -github.com/Workiva/go-datastructures v1.1.1 h1:9G5u1UqKt6ABseAffHGNfbNQd7omRlWE5QaxNruzhE0= -github.com/Workiva/go-datastructures v1.1.1/go.mod h1:1yZL+zfsztete+ePzZz/Zb1/t5BnDuE2Ya2MMGhzP6A= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= @@ -14,7 +12,6 @@ github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= -github.com/philhofer/fwd v1.1.1/go.mod h1:gk3iGcWd9+svBvR0sR+KPcfE+RNWozjowpeBVG3ZVNU= github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= @@ -25,37 +22,13 @@ github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjR github.com/rogpeppe/go-internal v1.10.0/go.mod h1:UQnix2H7Ngw/k4C5ijL5+65zddjncjaFoBhdsK/akog= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= -github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= -github.com/tinylib/msgp v1.1.5/go.mod h1:eQsjooMTnV42mHu917E26IogZ2930nFyBQdofk10Udg= -github.com/ttacon/chalk v0.0.0-20160626202418-22c06c80ed31/go.mod h1:onvgF043R+lC5RZ8IT9rBXDaEDnpnw/Cl+HFiw+v/7Q= -github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= go.uber.org/goleak v1.2.1/go.mod h1:qlT2yGI9QafXHhZZLxlSuNsMw3FFLxBr+tBRlmO1xH4= go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= -golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= -golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= -golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= -golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= -golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= -golang.org/x/tools v0.0.0-20201022035929-9cf592e881e9/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= -golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= diff --git a/helpers_test.go b/helpers_test.go index 40eb281..84d345a 100644 --- a/helpers_test.go +++ b/helpers_test.go @@ -1,89 +1,90 @@ package amqpx_test import ( + "context" "errors" "fmt" "github.com/jxsl13/amqpx/pool" ) -func createTopology(t *pool.Topologer) (err error) { +func createTopology(ctx context.Context, t *pool.Topologer) (err error) { // documentation: https://www.cloudamqp.com/blog/part4-rabbitmq-for-beginners-exchanges-routing-keys-bindings.html#:~:text=The%20routing%20key%20is%20a%20message%20attribute%20added%20to%20the,routing%20key%20of%20the%20message. - err = createExchange("exchange-01", t) + err = createExchange(ctx, "exchange-01", t) if err != nil { return err } - err = createQueue("queue-01", t) + err = createQueue(ctx, "queue-01", t) if err != nil { return err } - err = t.QueueBind("queue-01", "event-01", "exchange-01") + err = t.QueueBind(ctx, "queue-01", "event-01", "exchange-01") if err != nil { return err } - err = createExchange("exchange-02", t) + err = createExchange(ctx, "exchange-02", t) if err != nil { return err } - err = createQueue("queue-02", t) + err = createQueue(ctx, "queue-02", t) if err != nil { return err } - err = t.QueueBind("queue-02", "event-02", "exchange-02") + err = t.QueueBind(ctx, "queue-02", "event-02", "exchange-02") if err != nil { return err } - err = createExchange("exchange-03", t) + err = createExchange(ctx, "exchange-03", t) if err != nil { return err } - err = createQueue("queue-03", t) + err = createQueue(ctx, "queue-03", t) if err != nil { return err } - err = t.QueueBind("queue-03", "event-03", "exchange-03") + err = t.QueueBind(ctx, "queue-03", "event-03", "exchange-03") if err != nil { return err } return nil -} -func deleteTopology(t *pool.Topologer) (err error) { +} - err = deleteQueue("queue-01", t) +func deleteTopology(ctx context.Context, t *pool.Topologer) (err error) { + err = deleteQueue(ctx, "queue-01", t) if err != nil { return err } - err = deleteQueue("queue-02", t) + err = deleteQueue(ctx, "queue-02", t) if err != nil { return err } - err = deleteQueue("queue-03", t) + err = deleteQueue(ctx, "queue-03", t) if err != nil { return err } - err = deleteExchange("exchange-01", t) + err = deleteExchange(ctx, "exchange-01", t) if err != nil { return err } - err = deleteExchange("exchange-02", t) + err = deleteExchange(ctx, "exchange-02", t) if err != nil { return err } - err = deleteExchange("exchange-03", t) + err = deleteExchange(ctx, "exchange-03", t) if err != nil { return err } @@ -91,8 +92,8 @@ func deleteTopology(t *pool.Topologer) (err error) { return nil } -func createQueue(name string, t *pool.Topologer) (err error) { - _, err = t.QueueDeclarePassive(name) +func createQueue(ctx context.Context, name string, t *pool.Topologer) (err error) { + _, err = t.QueueDeclarePassive(ctx, name) if !errors.Is(err, pool.ErrNotFound) { if err != nil { return fmt.Errorf("queue %s was found even tho it should not exist: %w", name, err) @@ -100,38 +101,38 @@ func createQueue(name string, t *pool.Topologer) (err error) { return fmt.Errorf("queue %s was found even tho it should not exist", name) } - _, err = t.QueueDeclare(name) + _, err = t.QueueDeclare(ctx, name) if err != nil { return err } - _, err = t.QueueDeclarePassive(name) + _, err = t.QueueDeclarePassive(ctx, name) if err != nil { return fmt.Errorf("queue %s was not found even tho it should exist: %w", name, err) } return nil } -func deleteQueue(name string, t *pool.Topologer) (err error) { - _, err = t.QueueDeclarePassive(name) +func deleteQueue(ctx context.Context, name string, t *pool.Topologer) (err error) { + _, err = t.QueueDeclarePassive(ctx, name) if err != nil { return fmt.Errorf("%q does not exist but is supposed to be deleted", name) } - _, err = t.QueueDelete(name) + _, err = t.QueueDelete(ctx, name) if err != nil { return err } - _, err = t.QueueDeclarePassive(name) + _, err = t.QueueDeclarePassive(ctx, name) if err == nil { return fmt.Errorf("%q still exists after deletion", name) } return nil } -func createExchange(name string, t *pool.Topologer) (err error) { - err = t.ExchangeDeclarePassive(name, pool.ExchangeKindTopic) +func createExchange(ctx context.Context, name string, t *pool.Topologer) (err error) { + err = t.ExchangeDeclarePassive(ctx, name, pool.ExchangeKindTopic) if !errors.Is(err, pool.ErrNotFound) { if err != nil { return fmt.Errorf("exchange %s was found even tho it should not exist: %w", name, err) @@ -139,30 +140,30 @@ func createExchange(name string, t *pool.Topologer) (err error) { return fmt.Errorf("exchange %s was found even tho it should not exist", name) } - err = t.ExchangeDeclare(name, pool.ExchangeKindTopic) + err = t.ExchangeDeclare(ctx, name, pool.ExchangeKindTopic) if err != nil { return err } - err = t.ExchangeDeclarePassive(name, pool.ExchangeKindTopic) + err = t.ExchangeDeclarePassive(ctx, name, pool.ExchangeKindTopic) if err != nil { return fmt.Errorf("exchange %s was not found even tho it should exist: %w", name, err) } return nil } -func deleteExchange(name string, t *pool.Topologer) (err error) { - err = t.ExchangeDeclarePassive(name, pool.ExchangeKindTopic) +func deleteExchange(ctx context.Context, name string, t *pool.Topologer) (err error) { + err = t.ExchangeDeclarePassive(ctx, name, pool.ExchangeKindTopic) if err != nil { return fmt.Errorf("exchange %s was not found even tho it should exist: %w", name, err) } - err = t.ExchangeDelete(name) + err = t.ExchangeDelete(ctx, name) if err != nil { return err } - err = t.ExchangeDeclarePassive(name, pool.ExchangeKindTopic) + err = t.ExchangeDeclarePassive(ctx, name, pool.ExchangeKindTopic) if !errors.Is(err, pool.ErrNotFound) { return fmt.Errorf("exchange %s was found even tho it should not exist: %w", name, err) } diff --git a/pool/connection.go b/pool/connection.go index 0ee5d51..f512fb0 100644 --- a/pool/connection.go +++ b/pool/connection.go @@ -3,6 +3,7 @@ package pool import ( "context" "crypto/tls" + "errors" "fmt" "sync" "time" @@ -15,24 +16,34 @@ import ( // Connection is an internal representation of amqp.Connection. type Connection struct { + // connection url (user ,password, host, port, vhost, etc) url string name string - cached bool - flagged bool // whether an error occurred on this connection or not, indicating the connectionmust be recovered + // indicates that the connection is part of a connection pool. + cached bool + // if set to true, the connection is marked as broken, indicating the connection must be recovered + flagged bool tls *tls.Config + // underlying amqp connection conn *amqp.Connection lastConnLoss time.Time + // backoff policy errorBackoff BackoffFunc - heartbeat time.Duration + heartbeat time.Duration + + // connection timeout is only used for the inital connection + // recovering connections are recovered as long as the calling context + // is not canceled connTimeout time.Duration - errors chan *amqp.Error - blockers chan amqp.Blocking + errors chan *amqp.Error + // flow control messages from rabbitmq + blocking chan amqp.Blocking mu sync.Mutex ctx context.Context @@ -45,7 +56,7 @@ type Connection struct { // NewConnection creates a connection wrapper. // name: unique connection name -func NewConnection(connectUrl, name string, options ...ConnectionOption) (*Connection, error) { +func NewConnection(ctx context.Context, connectUrl, name string, options ...ConnectionOption) (*Connection, error) { // use sane defaults option := connectionOption{ Logger: logging.NewNoOpLogger(), @@ -53,7 +64,7 @@ func NewConnection(connectUrl, name string, options ...ConnectionOption) (*Conne HeartbeatInterval: 15 * time.Second, ConnectionTimeout: 30 * time.Second, BackoffPolicy: newDefaultBackoffPolicy(time.Second, 15*time.Second), - Ctx: context.Background(), + Ctx: ctx, RecoverCallback: nil, } @@ -62,7 +73,7 @@ func NewConnection(connectUrl, name string, options ...ConnectionOption) (*Conne o(&option) } - u, err := url.Parse(connectUrl) + u, err := url.ParseRequestURI(connectUrl) if err != nil { return nil, fmt.Errorf("%w: %v", ErrInvalidConnectURL, err) } @@ -73,7 +84,7 @@ func NewConnection(connectUrl, name string, options ...ConnectionOption) (*Conne // we derive a new context from the parent one in order to // be able to close it without affecting the parent - ctx, cancel := context.WithCancel(option.Ctx) + cCtx, cancel := context.WithCancel(option.Ctx) conn := &Connection{ url: u.String(), @@ -89,9 +100,9 @@ func NewConnection(connectUrl, name string, options ...ConnectionOption) (*Conne errorBackoff: option.BackoffPolicy, errors: make(chan *amqp.Error, 10), - blockers: make(chan amqp.Blocking, 10), + blocking: make(chan amqp.Blocking, 10), - ctx: ctx, + ctx: cCtx, cancel: cancel, log: option.Logger, @@ -100,7 +111,7 @@ func NewConnection(connectUrl, name string, options ...ConnectionOption) (*Conne recoverCB: option.RecoverCallback, } - err = conn.Connect() + err = conn.Connect(ctx) if err == nil { return conn, nil } @@ -109,7 +120,7 @@ func NewConnection(connectUrl, name string, options ...ConnectionOption) (*Conne return nil, err } - err = conn.Recover() + err = conn.Recover(ctx) if err != nil { return nil, err } @@ -151,16 +162,22 @@ func (ch *Connection) Flag(flagged bool) { } } +func (ch *Connection) IsFlagged() bool { + ch.mu.Lock() + defer ch.mu.Unlock() + return ch.flagged +} + // Connect tries to connect (or reconnect) // Does not block indefinitely, but returns an error // upon connection failure. -func (ch *Connection) Connect() error { +func (ch *Connection) Connect(ctx context.Context) error { ch.mu.Lock() defer ch.mu.Unlock() - return ch.connect() + return ch.connect(ctx) } -func (ch *Connection) connect() error { +func (ch *Connection) connect(ctx context.Context) error { // not closed, close before reconnecting if !ch.isClosed() { @@ -172,7 +189,7 @@ func (ch *Connection) connect() error { amqpConn, err := amqp.DialConfig(ch.url, amqp.Config{ Heartbeat: ch.heartbeat, - Dial: defaultDial(ch.ctx, ch.connTimeout), + Dial: defaultDial(ctx, ch.connTimeout), TLSClientConfig: ch.tls.Clone(), Properties: amqp.Table{ "connection_name": ch.name, @@ -183,31 +200,37 @@ func (ch *Connection) connect() error { return fmt.Errorf("%v: %w", ErrConnectionFailed, err) } - ch.info("connected") // override upon reconnect ch.conn = amqpConn ch.errors = make(chan *amqp.Error, 10) - ch.blockers = make(chan amqp.Blocking, 10) + ch.blocking = make(chan amqp.Blocking, 10) // ch.Errors is closed by streadway/amqp in some scenarios :( ch.conn.NotifyClose(ch.errors) - ch.conn.NotifyBlocked(ch.blockers) + ch.conn.NotifyBlocked(ch.blocking) + ch.info("connected") return nil } // PauseOnFlowControl allows you to wait and sleep while receiving flow control messages. // Sleeps for one second, repeatedly until the blocking has stopped. // Such messages will most likely be received when the broker hits its memory or disk limits. -func (ch *Connection) PauseOnFlowControl() { +// Returns an error in case that flow control was detected and is resolved or in case that the connection is closed. +func (ch *Connection) PauseOnFlowControl(ctx context.Context) error { ch.mu.Lock() defer ch.mu.Unlock() - ch.pauseOnFlowControl() + return ch.pauseOnFlowControl(ctx) } // not threadsafe -func (ch *Connection) pauseOnFlowControl() { +func (ch *Connection) pauseOnFlowControl(ctx context.Context) error { + + if ch.isClosed() { + return ErrClosed + } + var ( duration = time.Second timer = time.NewTimer(duration) @@ -215,32 +238,53 @@ func (ch *Connection) pauseOnFlowControl() { ) defer closeTimer(timer, &drained) - for !ch.conn.IsClosed() && !ch.isShutdown() { + var ( + flowControl = false + start = time.Now() + reason = "" + err error = nil + ) + for !ch.isClosed() { select { - case blocker, ok := <-ch.blockers: // Check for flow control issues. + case <-ch.catchShutdown(): + return ErrClosed + case <-ctx.Done(): + return fmt.Errorf("pause on flow control failed: %w", ctx.Err()) + case blocker, ok := <-ch.blocking: // Check for flow control issues. if !ok { - return + return errFlowControlClosed } if !blocker.Active { - return + return nil + } + + if !flowControl || reason != blocker.Reason { + reason = blocker.Reason + flowControl = true + err = fmt.Errorf("%w: reason: %s", errFlowControl, reason) + ch.warn(err) } - ch.info("pausing on flow control") resetTimer(timer, duration, &drained) select { case <-ch.catchShutdown(): - return + return ErrClosed case <-timer.C: drained = true continue } default: - return + if flowControl { + ch.warnf("flow control with last reason: %s: ended after %s", reason, time.Since(start)) + } + return err } } + + return nil } func (ch *Connection) IsClosed() bool { @@ -286,12 +330,7 @@ func (ch *Connection) error() error { return fmt.Errorf("connection and errors channel %w", ErrClosed) } // only overwrite with the first error - if err == nil { - err = e - } else { - // flush all other errors after the first one - continue - } + err = errors.Join(err, e) default: // return err after flushing errors channel return err @@ -301,18 +340,17 @@ func (ch *Connection) error() error { // Recover tries to recover the connection until // a shutdown occurs via context cancelation. -func (ch *Connection) Recover() error { +func (ch *Connection) Recover(ctx context.Context) error { ch.mu.Lock() defer ch.mu.Unlock() - return ch.recover() + return ch.recover(ctx) } -func (ch *Connection) recover() error { - healthy := ch.error() == nil +func (ch *Connection) recover(ctx context.Context) error { + healthy := !ch.flagged && ch.error() == nil if healthy && !ch.isClosed() { - ch.pauseOnFlowControl() - return nil + return ch.pauseOnFlowControl(ctx) } var ( @@ -324,7 +362,7 @@ func (ch *Connection) recover() error { ch.info("recovering") for try := 0; ; try++ { ch.lastConnLoss = time.Now() - err := ch.connect() + err := ch.connect(ctx) if err == nil { // connection established successfully break @@ -347,6 +385,9 @@ func (ch *Connection) recover() error { case <-ch.catchShutdown(): // catch shutdown signal return fmt.Errorf("connection recovery failed: connection %w", ErrClosed) + case <-ctx.Done(): + // catch context cancelation + return fmt.Errorf("connection recovery failed: %w", ctx.Err()) case <-timer.C: drained = true // retry after sleep @@ -382,15 +423,6 @@ func (ch *Connection) catchShutdown() <-chan struct{} { return ch.ctx.Done() } -func (ch *Connection) isShutdown() bool { - select { - case <-ch.ctx.Done(): - return true - default: - return false - } -} - func (ch *Connection) info(a ...any) { ch.log.WithField("connection", ch.Name()).Info(a...) } @@ -399,6 +431,10 @@ func (ch *Connection) warn(err error, a ...any) { ch.log.WithField("connection", ch.Name()).WithField("error", err.Error()).Warn(a...) } +func (ch *Connection) warnf(format string, a ...any) { + ch.log.WithField("connection", ch.Name()).Warnf(format, a...) +} + func (ch *Connection) debug(a ...any) { ch.log.WithField("connection", ch.Name()).Debug(a...) } diff --git a/pool/connection_options.go b/pool/connection_options.go index b7b6d26..9492073 100644 --- a/pool/connection_options.go +++ b/pool/connection_options.go @@ -64,16 +64,6 @@ func ConnectionWithBackoffPolicy(policy BackoffFunc) ConnectionOption { } } -// ConnectionWithContext allows to set a custom connection timeout, that MUST be >= 1 * time.Second -func ConnectionWithContext(ctx context.Context) ConnectionOption { - if ctx == nil { - panic("nil context passed") - } - return func(co *connectionOption) { - co.Ctx = ctx - } -} - // ConnectionWithTLS allows to configure tls connectivity. func ConnectionWithTLS(config *tls.Config) ConnectionOption { return func(co *connectionOption) { diff --git a/pool/connection_pool.go b/pool/connection_pool.go index 2414d78..7bb4184 100644 --- a/pool/connection_pool.go +++ b/pool/connection_pool.go @@ -9,14 +9,16 @@ import ( "sync/atomic" "time" - "github.com/Workiva/go-datastructures/queue" "github.com/jxsl13/amqpx/logging" ) // ConnectionPool houses the pool of RabbitMQ connections. type ConnectionPool struct { + // connection pool name will be added to all of its connections name string - url string + + // connection url to connect to the RabbitMQ server (user, password, url, port, vhost, etc) + url string heartbeat time.Duration connTimeout time.Duration @@ -24,7 +26,7 @@ type ConnectionPool struct { size int tls *tls.Config - connections *queue.Queue + connections chan *Connection transientID int64 @@ -38,7 +40,7 @@ type ConnectionPool struct { // NewConnectionPool creates a new connection pool which has a maximum size it // can become and an idle size of connections that are always open. -func NewConnectionPool(connectUrl string, numConns int, options ...ConnectionPoolOption) (*ConnectionPool, error) { +func NewConnectionPool(ctx context.Context, connectUrl string, numConns int, options ...ConnectionPoolOption) (*ConnectionPool, error) { if numConns < 1 { return nil, fmt.Errorf("%w: %d", errInvalidPoolSize, numConns) } @@ -48,7 +50,7 @@ func NewConnectionPool(connectUrl string, numConns int, options ...ConnectionPoo Name: defaultAppName(), Size: numConns, - Ctx: context.Background(), + Ctx: ctx, ConnHeartbeatInterval: 15 * time.Second, // https://www.rabbitmq.com/heartbeats.html#false-positives ConnTimeout: 30 * time.Second, @@ -68,7 +70,7 @@ func NewConnectionPool(connectUrl string, numConns int, options ...ConnectionPoo } func newConnectionPoolFromOption(connectUrl string, option connectionPoolOption) (_ *ConnectionPool, err error) { - u, err := url.Parse(connectUrl) + u, err := url.ParseRequestURI(connectUrl) if err != nil { return nil, fmt.Errorf("%w: %v", ErrInvalidConnectURL, err) } @@ -89,7 +91,7 @@ func newConnectionPoolFromOption(connectUrl string, option connectionPoolOption) size: option.Size, tls: option.TLSConfig, - connections: queue.New(int64(option.Size)), + connections: make(chan *Connection, option.Size), ctx: ctx, cancel: cancel, @@ -123,9 +125,15 @@ func (cp *ConnectionPool) initCachedConns() error { return fmt.Errorf("%w: %v", ErrPoolInitializationFailed, err) } - if err = cp.connections.Put(conn); err != nil { - return fmt.Errorf("%w: %v", ErrPoolInitializationFailed, err) + select { + case cp.connections <- conn: + case <-cp.ctx.Done(): + return fmt.Errorf("%w: %v", ErrPoolInitializationFailed, cp.ctx.Err()) + default: + // should not happen + return fmt.Errorf("%w: pool channel buffer full", ErrPoolInitializationFailed) } + } return nil } @@ -137,8 +145,7 @@ func (cp *ConnectionPool) deriveConnection(ctx context.Context, id int, cached b } else { name = fmt.Sprintf("%s-transient-connection-%d", cp.name, id) } - return NewConnection(cp.url, name, - ConnectionWithContext(ctx), + return NewConnection(ctx, cp.url, name, ConnectionWithTimeout(cp.connTimeout), ConnectionWithHeartbeatInterval(cp.heartbeat), ConnectionWithTLS(cp.tls), @@ -149,18 +156,23 @@ func (cp *ConnectionPool) deriveConnection(ctx context.Context, id int, cached b } // GetConnection only returns an error upon shutdown -func (cp *ConnectionPool) GetConnection() (*Connection, error) { - conn, err := cp.getConnectionFromPool() - if err != nil { - return nil, err - } +func (cp *ConnectionPool) GetConnection(ctx context.Context) (*Connection, error) { + select { + case <-cp.catchShutdown(): + return nil, fmt.Errorf("connection pool %w", ErrClosed) + case <-ctx.Done(): + return nil, ctx.Err() + case conn, ok := <-cp.connections: + if !ok { + return nil, fmt.Errorf("connection pool %w", ErrClosed) + } + err := conn.Recover(ctx) + if err != nil { + return nil, fmt.Errorf("failed to get connection: %w", err) + } - err = conn.Recover() - if err != nil { - return nil, fmt.Errorf("failed to get connection: %w", err) + return conn, nil } - - return conn, nil } // GetTransientConnection may return an error when the context was cancelled before the connection could be obtained. @@ -174,7 +186,7 @@ func (cp *ConnectionPool) GetTransientConnection(ctx context.Context) (*Connecti } // recover until context is closed - err = conn.Recover() + err = conn.Recover(ctx) if err != nil { return nil, fmt.Errorf("failed to get transient connection: %w", err) } @@ -182,41 +194,23 @@ func (cp *ConnectionPool) GetTransientConnection(ctx context.Context) (*Connecti return conn, nil } -func (cp *ConnectionPool) getConnectionFromPool() (*Connection, error) { - - // Pull from the queue. - // Pauses here indefinitely if the queue is empty. - objects, err := cp.connections.Get(1) - if err != nil { - return nil, fmt.Errorf("connection pool %w: %v", ErrClosed, err) - } - - conn, ok := objects[0].(*Connection) - if !ok { - panic("invalid type in queue") - } - return conn, nil -} - // ReturnConnection puts the connection back in the queue and flag it for error. // This helps maintain a Round Robin on Connections and their resources. -func (cp *ConnectionPool) ReturnConnection(conn *Connection, flag bool) { - if flag { - conn.Flag(flag) - } - +func (cp *ConnectionPool) ReturnConnection(ctx context.Context, conn *Connection, flag bool) { // close transient connections if !conn.IsCached() { _ = conn.Close() + return } + conn.Flag(flag) - err := cp.connections.Put(conn) - if err != nil { - // queue was disposed of, - // indicating pool shutdown - // -> close connection upon pool shutdown - _ = conn.Close() + if conn.IsFlagged() { + // try to recover until context is canceled + // if recovery fails, we put the broken connection into the pool + _ = conn.Recover(ctx) } + + cp.connections <- conn } // Close closes the connection pool. @@ -229,27 +223,24 @@ func (cp *ConnectionPool) Close() { defer cp.info("closed") wg := &sync.WaitGroup{} - - // close all connections - for !cp.connections.Empty() { - items := cp.connections.Dispose() - - wg.Add(len(items)) - for _, item := range items { - conn, ok := item.(*Connection) - if !ok { - panic("item in connection queue is not a connection") - } - - go func(c *Connection) { - defer wg.Done() - _ = c.Close() - }(conn) - } + wg.Add(cp.size) + cp.cancel() + + for i := 0; i < cp.size; i++ { + go func() { + defer wg.Done() + conn := <-cp.connections + _ = conn.Close() + }() } + wg.Wait() } +func (cp *ConnectionPool) catchShutdown() <-chan struct{} { + return cp.ctx.Done() +} + func (cp *ConnectionPool) Name() string { return cp.name } diff --git a/pool/connection_pool_options.go b/pool/connection_pool_options.go index 24fd87f..15b9b42 100644 --- a/pool/connection_pool_options.go +++ b/pool/connection_pool_options.go @@ -98,16 +98,6 @@ func ConnectionPoolWithConnectionTimeout(timeout time.Duration) ConnectionPoolOp } } -// ConnectionPoolWithContext allows to set a custom connection timeout, that MUST be >= 1 * time.Second -func ConnectionPoolWithContext(ctx context.Context) ConnectionPoolOption { - if ctx == nil { - panic("nil context passed") - } - return func(po *connectionPoolOption) { - po.Ctx = ctx - } -} - // ConnectionPoolWithTLS allows to configure tls connectivity. func ConnectionPoolWithTLS(config *tls.Config) ConnectionPoolOption { return func(po *connectionPoolOption) { diff --git a/pool/connection_pool_test.go b/pool/connection_pool_test.go index c480cb4..7f7bd0d 100644 --- a/pool/connection_pool_test.go +++ b/pool/connection_pool_test.go @@ -1,6 +1,7 @@ package pool_test import ( + "context" "sync" "testing" "time" @@ -11,8 +12,9 @@ import ( ) func TestNewConnectionPool(t *testing.T) { + ctx := context.TODO() connections := 5 - p, err := pool.NewConnectionPool(connectURL, connections, + p, err := pool.NewConnectionPool(ctx, connectURL, connections, pool.ConnectionPoolWithName("TestNewConnectionPool"), pool.ConnectionPoolWithLogger(logging.NewTestLogger(t)), ) @@ -27,13 +29,13 @@ func TestNewConnectionPool(t *testing.T) { wg.Add(1) go func() { defer wg.Done() - c, err := p.GetConnection() + c, err := p.GetConnection(ctx) if err != nil { assert.NoError(t, err) return } time.Sleep(5 * time.Second) - p.ReturnConnection(c, false) + p.ReturnConnection(ctx, c, false) }() } @@ -41,8 +43,9 @@ func TestNewConnectionPool(t *testing.T) { } func TestNewConnectionPoolDisconnect(t *testing.T) { + ctx := context.TODO() connections := 100 - p, err := pool.NewConnectionPool(connectURL, connections, + p, err := pool.NewConnectionPool(ctx, connectURL, connections, pool.ConnectionPoolWithName("TestNewConnectionPoolDisconnect"), pool.ConnectionPoolWithLogger(logging.NewTestLogger(t)), ) @@ -64,14 +67,14 @@ func TestNewConnectionPoolDisconnect(t *testing.T) { awaitStarted() // wait for connection loss // no connection, this should retry until there is a connection - c, err := p.GetConnection() + c, err := p.GetConnection(ctx) if err != nil { assert.NoError(t, err) return } time.Sleep(1 * time.Second) - p.ReturnConnection(c, false) + p.ReturnConnection(ctx, c, false) }(i) } diff --git a/pool/connection_test.go b/pool/connection_test.go index 12d95de..3b53ad8 100644 --- a/pool/connection_test.go +++ b/pool/connection_test.go @@ -1,6 +1,7 @@ package pool_test import ( + "context" "fmt" "sync" "testing" @@ -13,7 +14,9 @@ import ( ) func TestNewSingleConnection(t *testing.T) { + ctx := context.TODO() c, err := pool.NewConnection( + ctx, connectURL, "TestNewSingleConnection", pool.ConnectionWithLogger(logging.NewTestLogger(t)), @@ -30,12 +33,14 @@ func TestNewSingleConnection(t *testing.T) { } func TestNewSingleConnectionWithDisconnect(t *testing.T) { + ctx := context.TODO() started, stopped := DisconnectWithStartedStopped(t, 0, 0, 15*time.Second) started() defer stopped() c, err := pool.NewConnection( + ctx, connectURL, - "TestNewSingleConnection", + "TestNewSingleConnectionWithDisconnect", pool.ConnectionWithLogger(logging.NewTestLogger(t)), ) @@ -50,6 +55,7 @@ func TestNewSingleConnectionWithDisconnect(t *testing.T) { } func TestNewConnection(t *testing.T) { + ctx := context.TODO() var wg sync.WaitGroup connections := 5 @@ -59,6 +65,7 @@ func TestNewConnection(t *testing.T) { defer wg.Done() c, err := pool.NewConnection( + ctx, connectURL, fmt.Sprintf("TestNewConnection-%d", id), pool.ConnectionWithLogger(logging.NewTestLogger(t)), @@ -80,7 +87,7 @@ func TestNewConnection(t *testing.T) { } func TestNewConnectionDisconnect(t *testing.T) { - + ctx := context.TODO() var wg sync.WaitGroup connections := 100 @@ -95,6 +102,7 @@ func TestNewConnectionDisconnect(t *testing.T) { defer wg.Done() c, err := pool.NewConnection( + ctx, connectURL, fmt.Sprintf("TestNewConnectionDisconnect-%d", id), //pool.ConnectionWithLogger(logging.NewTestLogger(t)), @@ -110,7 +118,7 @@ func TestNewConnectionDisconnect(t *testing.T) { wait() // wait for connection to work again. - assert.NoError(t, c.Recover()) + assert.NoError(t, c.Recover(ctx)) assert.NoError(t, c.Error()) }(int64(i)) } diff --git a/pool/errors.go b/pool/errors.go index dfb331c..642d0b5 100644 --- a/pool/errors.go +++ b/pool/errors.go @@ -21,6 +21,23 @@ var ( // ErrNotFound is returned by ExchangeDeclarePassive or QueueDeclarePassive in the case that // the queue was not found. ErrNotFound = errors.New("not found") + + // errFlowControl is returned when the server is under flow control + // TODO: make public api after a while + errFlowControl = errors.New("flow control") + + // errFlowControlClosed is returned when the flow control channel is closed + // Specifically interesting when awaiting publish confirms + // TODO: make public api after a while + errFlowControlClosed = errors.New("flow control channel closed") + + // ErrReject can be used to reject a specific message + // This is a special error that negatively acknowledges messages and does not reuque them. + ErrReject = errors.New("message rejected") + + // ErrRejectSingle can be used to reject a specific message + // This is a special error that negatively acknowledges messages and does not reuque them + ErrRejectSingle = errors.New("single message rejected") ) var ( diff --git a/pool/pool.go b/pool/pool.go index bff25c9..1759e15 100644 --- a/pool/pool.go +++ b/pool/pool.go @@ -19,7 +19,7 @@ type Pool struct { sp *SessionPool } -func New(connectUrl string, numConns, numSessions int, options ...Option) (*Pool, error) { +func New(ctx context.Context, connectUrl string, numConns, numSessions int, options ...Option) (*Pool, error) { if numConns < 1 { return nil, fmt.Errorf("%w: %d", errInvalidPoolSize, numConns) } @@ -28,8 +28,6 @@ func New(connectUrl string, numConns, numSessions int, options ...Option) (*Pool numSessions = numConns } - ctx := context.Background() - logger := logging.NewNoOpLogger() // use sane defaults @@ -81,16 +79,8 @@ func (p *Pool) Close() { } // GetSession returns a new session from the pool, only returns an error upon shutdown. -func (p *Pool) GetSession() (*Session, error) { - return p.sp.GetSession() -} - -// GetSessionCtx returns a new session from the pool, only returns an error upon shutdown or when the passed context was canceled. -func (p *Pool) GetSessionCtx(ctx context.Context) (*Session, error) { - if p.sp.ctx == ctx { - return p.sp.GetSession() - } - return p.sp.GetSessionCtx(ctx) +func (p *Pool) GetSession(ctx context.Context) (*Session, error) { + return p.sp.GetSession(ctx) } // GetTransientSession returns a new session which is decoupled from anyshutdown mechanism, thus @@ -103,8 +93,8 @@ func (p *Pool) GetTransientSession(ctx context.Context) (*Session, error) { // ReturnSession returns a Session back to the pool. // If the session was returned due to an error, erred should be set to true, otherwise // erred should be set to false. -func (p *Pool) ReturnSession(session *Session, erred bool) { - p.sp.ReturnSession(session, erred) +func (p *Pool) ReturnSession(ctx context.Context, session *Session, erred bool) { + p.sp.ReturnSession(ctx, session, erred) } func (p *Pool) Context() context.Context { diff --git a/pool/pool_options.go b/pool/pool_options.go index 723ee5f..f596abe 100644 --- a/pool/pool_options.go +++ b/pool/pool_options.go @@ -1,7 +1,6 @@ package pool import ( - "context" "crypto/tls" "time" @@ -61,13 +60,6 @@ func WithConnectionTimeout(timeout time.Duration) Option { } } -// WithContext allows to set a custom connection timeout, that MUST be >= 1 * time.Second -func WithContext(ctx context.Context) Option { - return func(po *poolOption) { - ConnectionPoolWithContext(ctx)(&po.cpo) - } -} - // WithTLS allows to configure tls connectivity. func WithTLS(config *tls.Config) Option { return func(po *poolOption) { diff --git a/pool/pool_test.go b/pool/pool_test.go index d0ff8b6..94d4459 100644 --- a/pool/pool_test.go +++ b/pool/pool_test.go @@ -1,6 +1,7 @@ package pool_test import ( + "context" "sync" "testing" "time" @@ -13,7 +14,8 @@ import ( ) var ( - connectURL = amqpx.NewURL("localhost", 5672, "admin", "password") + connectURL = amqpx.NewURL("localhost", 5672, "admin", "password") + brokenConnectURL = amqpx.NewURL("localhost", 5673, "admin", "password") ) func TestMain(m *testing.M) { @@ -26,10 +28,15 @@ func TestMain(m *testing.M) { } func TestNew(t *testing.T) { + ctx := context.TODO() connections := 2 sessions := 10 - p, err := pool.New(connectURL, connections, sessions, + p, err := pool.New( + ctx, + connectURL, + connections, + sessions, pool.WithName("TestNew"), pool.WithLogger(logging.NewTestLogger(t)), ) @@ -46,14 +53,14 @@ func TestNew(t *testing.T) { go func() { defer wg.Done() - session, err := p.GetSession() + session, err := p.GetSession(ctx) if err != nil { assert.NoError(t, err) return } time.Sleep(1 * time.Second) - p.ReturnSession(session, false) + p.ReturnSession(ctx, session, false) }() } diff --git a/pool/publisher.go b/pool/publisher.go index 4a0a87f..87c0b80 100644 --- a/pool/publisher.go +++ b/pool/publisher.go @@ -3,20 +3,20 @@ package pool import ( "context" "errors" - "time" + "sync" "github.com/jxsl13/amqpx/logging" ) type Publisher struct { - pool *Pool - autoClosePool bool - publishTimeout time.Duration - confirmTimeout time.Duration + pool *Pool + autoClosePool bool ctx context.Context cancel context.CancelFunc + mu sync.Mutex + log logging.Logger } @@ -38,11 +38,10 @@ func NewPublisher(p *Pool, options ...PublisherOption) *Publisher { // sane defaults, prefer fault tolerance over performance option := publisherOption{ - Ctx: p.Context(), - PublishTimeout: 15 * time.Second, - ConfirmTimeout: 15 * time.Second, - AutoClosePool: false, - Logger: p.sp.log, // derive logger from session pool + Ctx: p.Context(), + + AutoClosePool: false, + Logger: p.sp.log, // derive logger from session pool } for _, o := range options { @@ -52,13 +51,10 @@ func NewPublisher(p *Pool, options ...PublisherOption) *Publisher { ctx, cancel := context.WithCancel(option.Ctx) pub := &Publisher{ - pool: p, - autoClosePool: option.AutoClosePool, - confirmTimeout: option.ConfirmTimeout, - publishTimeout: option.PublishTimeout, - - ctx: ctx, - cancel: cancel, + pool: p, + autoClosePool: option.AutoClosePool, + ctx: ctx, + cancel: cancel, log: option.Logger, } @@ -69,22 +65,30 @@ func NewPublisher(p *Pool, options ...PublisherOption) *Publisher { // Publish a message to a specific exchange with a given routingKey. // You may set exchange to "" and routingKey to your queue name in order to publish directly to a queue. -func (p *Publisher) Publish(exchange string, routingKey string, msg Publishing) error { +func (p *Publisher) Publish(ctx context.Context, exchange string, routingKey string, msg Publishing) error { for { - err := p.publish(exchange, routingKey, msg) - if err == nil { + err := p.publish(ctx, exchange, routingKey, msg) + switch { + case err == nil: return nil - } else if errors.Is(err, ErrClosed) { + case errors.Is(err, context.Canceled): return err - } else { + case errors.Is(err, context.DeadlineExceeded): + return err + case errors.Is(err, ErrClosed): + return err + case errors.Is(err, ErrNack): + return err + case errors.Is(err, ErrDeliveryTagMismatch): + return err + default: p.warn(exchange, routingKey, err, "publish failed, retrying") } - // continue in any other error case } } -func (p *Publisher) publish(exchange string, routingKey string, msg Publishing) (err error) { +func (p *Publisher) publish(ctx context.Context, exchange string, routingKey string, msg Publishing) (err error) { defer func() { if err != nil { p.warn(exchange, routingKey, err) @@ -93,28 +97,25 @@ func (p *Publisher) publish(exchange string, routingKey string, msg Publishing) } }() - s, err := p.pool.GetSession() + s, err := p.pool.GetSession(ctx) if err != nil && errors.Is(err, ErrClosed) { return err } defer func() { // return session if err == nil { - p.pool.ReturnSession(s, false) + p.pool.ReturnSession(ctx, s, false) } else if errors.Is(err, ErrClosed) { // TODO: potential message loss upon shutdown // might try a transient session for this one - p.pool.ReturnSession(s, false) + p.pool.ReturnSession(ctx, s, false) } else { - p.pool.ReturnSession(s, true) + p.pool.ReturnSession(ctx, s, true) } }() - pubCtx, pubCancel := context.WithTimeout(p.ctx, p.publishTimeout) - defer pubCancel() - - tag, err := s.Publish(pubCtx, exchange, routingKey, msg) - if err != nil && errors.Is(err, ErrClosed) { + tag, err := s.Publish(ctx, exchange, routingKey, msg) + if err != nil { return err } @@ -122,30 +123,27 @@ func (p *Publisher) publish(exchange string, routingKey string, msg Publishing) return nil } - confirmCtx, confirmCancel := context.WithTimeout(p.ctx, p.confirmTimeout) - defer confirmCancel() - - return s.AwaitConfirm(confirmCtx, tag) + return s.AwaitConfirm(ctx, tag) } // Get is only supposed to be used for testing, do not use get for polling any broker queues. -func (p *Publisher) Get(queue string, autoAck bool) (msg Delivery, ok bool, err error) { - s, err := p.pool.GetSession() +func (p *Publisher) Get(ctx context.Context, queue string, autoAck bool) (msg Delivery, ok bool, err error) { + s, err := p.pool.GetSession(ctx) if err != nil && errors.Is(err, ErrClosed) { return Delivery{}, false, err } defer func() { // return session if err == nil { - p.pool.ReturnSession(s, false) + p.pool.ReturnSession(ctx, s, false) } else if errors.Is(err, ErrClosed) { - p.pool.ReturnSession(s, false) + p.pool.ReturnSession(ctx, s, false) } else { - p.pool.ReturnSession(s, true) + p.pool.ReturnSession(ctx, s, true) } }() - return s.Get(queue, autoAck) + return s.Get(ctx, queue, autoAck) } func (p *Publisher) info(exchange, routingKey string, a ...any) { diff --git a/pool/publisher_option.go b/pool/publisher_option.go index 9e8e162..0417bdd 100644 --- a/pool/publisher_option.go +++ b/pool/publisher_option.go @@ -2,16 +2,14 @@ package pool import ( "context" - "time" "github.com/jxsl13/amqpx/logging" ) type publisherOption struct { - Ctx context.Context - PublishTimeout time.Duration - ConfirmTimeout time.Duration - AutoClosePool bool + Ctx context.Context + + AutoClosePool bool Logger logging.Logger } @@ -35,21 +33,3 @@ func PublisherWithAutoClosePool(autoClose bool) PublisherOption { po.AutoClosePool = autoClose } } - -func PublisherWithConfirmTimeout(timeout time.Duration) PublisherOption { - if timeout < time.Second { - timeout = time.Second - } - return func(po *publisherOption) { - po.ConfirmTimeout = timeout - } -} - -func PublisherWithPublishTimeout(timeout time.Duration) PublisherOption { - if timeout < time.Second { - timeout = time.Second - } - return func(po *publisherOption) { - po.PublishTimeout = timeout - } -} diff --git a/pool/publisher_test.go b/pool/publisher_test.go index 36d881c..2b220e8 100644 --- a/pool/publisher_test.go +++ b/pool/publisher_test.go @@ -1,7 +1,10 @@ package pool_test import ( + "context" "fmt" + "os" + "os/signal" "sync" "testing" "time" @@ -12,9 +15,12 @@ import ( ) func TestPublisher(t *testing.T) { + ctx := context.TODO() connections := 1 sessions := 10 // publisher sessions + consumer sessions - p, err := pool.New(connectURL, + p, err := pool.New( + ctx, + connectURL, connections, sessions, pool.WithName("TestPublisher"), @@ -35,42 +41,45 @@ func TestPublisher(t *testing.T) { go func(id int64) { defer wg.Done() - s, err := p.GetSession() + s, err := p.GetSession(ctx) if err != nil { assert.NoError(t, err) return } + defer func() { + p.ReturnSession(ctx, s, false) + }() queueName := fmt.Sprintf("TestPublisher-Queue-%d", id) - _, err = s.QueueDeclare(queueName) + _, err = s.QueueDeclare(ctx, queueName) if err != nil { assert.NoError(t, err) return } defer func() { - i, err := s.QueueDelete(queueName) + i, err := s.QueueDelete(ctx, queueName) assert.NoError(t, err) assert.Equal(t, 0, i) }() exchangeName := fmt.Sprintf("TestPublisher-Exchange-%d", id) - err = s.ExchangeDeclare(exchangeName, "topic") + err = s.ExchangeDeclare(ctx, exchangeName, "topic") if err != nil { assert.NoError(t, err) return } defer func() { - err := s.ExchangeDelete(exchangeName) + err := s.ExchangeDelete(ctx, exchangeName) assert.NoError(t, err) }() - err = s.QueueBind(queueName, "#", exchangeName) + err = s.QueueBind(ctx, queueName, "#", exchangeName) if err != nil { assert.NoError(t, err) return } defer func() { - err := s.QueueUnbind(queueName, "#", exchangeName, nil) + err := s.QueueUnbind(ctx, queueName, "#", exchangeName, nil) assert.NoError(t, err) }() @@ -104,7 +113,7 @@ func TestPublisher(t *testing.T) { pub := pool.NewPublisher(p) defer pub.Close() - pub.Publish(exchangeName, "", pool.Publishing{ + pub.Publish(ctx, exchangeName, "", pool.Publishing{ Mandatory: true, ContentType: "application/json", Body: []byte(message), @@ -117,3 +126,104 @@ func TestPublisher(t *testing.T) { wg.Wait() } + +func TestPauseOnFlowControl(t *testing.T) { + ctx, cancel := signal.NotifyContext(context.TODO(), os.Interrupt) + defer cancel() + + connections := 1 + sessions := 2 // publisher sessions + consumer sessions + p, err := pool.New( + ctx, + brokenConnectURL, // + connections, + sessions, + pool.WithName("TestPauseOnFlowControl"), + pool.WithConfirms(true), + pool.WithLogger(logging.NewTestLogger(t)), + ) + if err != nil { + assert.NoError(t, err) + return + } + defer p.Close() + + s, err := p.GetSession(ctx) + if err != nil { + assert.NoError(t, err) + return + } + + var ( + exchangeName = "TestPauseOnFlowControl-Exchange" + ) + + cleanup := initQueueExchange(t, s, ctx, "TestPauseOnFlowControl-Queue", exchangeName) + defer cleanup() + + pub := pool.NewPublisher(p) + defer pub.Close() + + pubGen := PublishingGenerator("TestPauseOnFlowControl") + + err = pub.Publish(ctx, exchangeName, "", pubGen()) + assert.NoError(t, err) + +} + +func initQueueExchange(t *testing.T, s *pool.Session, ctx context.Context, queueName, exchangeName string) (cleanup func()) { + _, err := s.QueueDeclare(ctx, queueName) + if err != nil { + assert.NoError(t, err) + return + } + cleanupList := []func(){} + + cleanupQueue := func() { + i, err := s.QueueDelete(ctx, queueName) + assert.NoError(t, err) + assert.Equal(t, 0, i) + } + cleanupList = append(cleanupList, cleanupQueue) + + err = s.ExchangeDeclare(ctx, exchangeName, "topic") + if err != nil { + assert.NoError(t, err) + return + } + cleanupExchange := func() { + err := s.ExchangeDelete(ctx, exchangeName) + assert.NoError(t, err) + } + cleanupList = append(cleanupList, cleanupExchange) + + err = s.QueueBind(ctx, queueName, "#", exchangeName) + if err != nil { + assert.NoError(t, err) + return + } + cleanupBind := func() { + err := s.QueueUnbind(ctx, queueName, "#", exchangeName, nil) + assert.NoError(t, err) + } + cleanupList = append(cleanupList, cleanupBind) + + return func() { + for i := len(cleanupList) - 1; i >= 0; i-- { + cleanupList[i]() + } + } +} + +func PublishingGenerator(MessagePrefix string) func() pool.Publishing { + i := 0 + return func() pool.Publishing { + defer func() { + i++ + }() + return pool.Publishing{ + ContentType: "application/json", + Body: []byte(fmt.Sprintf("%s-%d", MessagePrefix, i)), + } + } +} diff --git a/pool/session.go b/pool/session.go index 6fd7e9b..6f7de0b 100644 --- a/pool/session.go +++ b/pool/session.go @@ -218,6 +218,12 @@ func (s *Session) connect() (err error) { if err != nil { return fmt.Errorf("%v: %w", ErrConnectionFailed, err) } + defer func() { + if err != nil { + // close channel upon error + _ = channel.Close() + } + }() if s.confirmable { s.confirms = make(chan amqp091.Confirmation, s.bufferSize) @@ -240,28 +246,33 @@ func (s *Session) connect() (err error) { } -func (s *Session) Recover() error { +func (s *Session) Recover(ctx context.Context) error { s.mu.Lock() defer s.mu.Unlock() - return s.recover() + return s.recover(ctx) } -func (s *Session) tryRecover(err error) error { +func (s *Session) tryRecover(ctx context.Context, err error) error { if err == nil { return nil } + if !recoverable(err) { return err } - return s.recover() + return s.recover(ctx) } -func (s *Session) recover() error { +func (s *Session) recover(ctx context.Context) error { // tries to recover session forever for try := 0; ; try++ { + // try closing the channel before recovering + // in case of a bug in this library we do not want to flood the rabbitmq with + // open channels (which already happened) + _ = s.channel.Close() - err := s.conn.Recover() // recovers connection with a backoff mechanism + err := s.conn.Recover(ctx) // recovers connection with a backoff mechanism if err != nil { // upon shutdown this will fail return fmt.Errorf("failed to recover session: %w", err) @@ -306,11 +317,14 @@ func (s *Session) AwaitConfirm(ctx context.Context, expectedTag uint64) error { } if !confirm.Ack { - // in case the server did not accept the message, it might be due to - // resource problems. + // in case the server did not accept the message, it might be due to resource problems. // TODO: do we want to pause here upon flow control messages - s.conn.PauseOnFlowControl() - return fmt.Errorf("await confirm failed: %w", ErrNack) + err := fmt.Errorf("await confirm failed: %w", ErrNack) + flowErr := s.conn.PauseOnFlowControl(ctx) + if flowErr != nil { + err = errors.Join(err, flowErr) + } + return err } if confirm.DeliveryTag != expectedTag { return fmt.Errorf("await confirm failed: %w: expected %d, got %d", ErrDeliveryTagMismatch, expectedTag, confirm.DeliveryTag) @@ -368,7 +382,7 @@ type Publishing struct { // The way to ensure that all publishings reach the server is to add a listener to Channel.NotifyPublish and put the channel in confirm mode with Channel.Confirm. // Publishing delivery tags and their corresponding confirmations start at 1. Exit when all publishings are confirmed. // When Publish does not return an error and the channel is in confirm mode, the internal counter for DeliveryTags with the first confirmation starts at 1. -func (s *Session) Publish(ctx context.Context, exchange string, routingKey string, msg Publishing) (tag uint64, err error) { +func (s *Session) Publish(ctx context.Context, exchange string, routingKey string, msg Publishing) (deliveryTag uint64, err error) { s.mu.Lock() defer s.mu.Unlock() @@ -383,10 +397,10 @@ func (s *Session) Publish(ctx context.Context, exchange string, routingKey strin amqpDeliverMode = 2 // persistent (persisted to disk upon arrival in queue) } - return s.retryPublish(s.publishRetryCB, func() (uint64, error) { - tag = 0 + err = s.retry(ctx, s.publishRetryCB, func() error { + deliveryTag = 0 if s.confirmable { - tag = s.channel.GetNextPublishSeqNo() + deliveryTag = s.channel.GetNextPublishSeqNo() } err = s.channel.PublishWithContext( @@ -413,36 +427,22 @@ func (s *Session) Publish(ctx context.Context, exchange string, routingKey strin }, ) if err != nil { - return 0, err + return err } - return tag, err + return nil }) -} - -func (s *Session) retryPublish(cb sessionRetryCallback, f func() (uint64, error)) (tag uint64, err error) { - for try := 0; ; try++ { - tag, err := f() - if err == nil { - return tag, nil - } - - if cb != nil { - cb(s.conn.Name(), s.name, try, err) - } - - err = s.tryRecover(err) - if err != nil { - return 0, err - } + if err != nil { + return 0, err } + return deliveryTag, nil } // Get is only supposed to be used for testing purposes, do not us eit to poll the queue periodically. -func (s *Session) Get(queue string, autoAck bool) (msg Delivery, ok bool, err error) { +func (s *Session) Get(ctx context.Context, queue string, autoAck bool) (msg Delivery, ok bool, err error) { s.mu.Lock() defer s.mu.Unlock() - err = s.retry(s.getRetryCB, func() error { + err = s.retry(ctx, s.getRetryCB, func() error { msg, ok, err = s.channel.Get(queue, autoAck) if err != nil { return err @@ -537,7 +537,7 @@ func (s *Session) Consume(queue string, option ...ConsumeOptions) (<-chan Delive err error ) // retries to connect and attempts to start a consumer - err = s.retry(s.consumeRetryCB, func() error { + err = s.retry(s.ctx, s.consumeRetryCB, func() error { c, err = s.channel.Consume( queue, o.ConsumerTag, @@ -602,7 +602,7 @@ func (s *Session) ConsumeWithContext(ctx context.Context, queue string, option . err error ) // retries to connect and attempts to start a consumer - err = s.retry(s.consumeContextRetryCB, func() error { + err = s.retry(ctx, s.consumeContextRetryCB, func() error { c, err = s.channel.ConsumeWithContext( ctx, queue, @@ -626,7 +626,7 @@ func (s *Session) ConsumeWithContext(ctx context.Context, queue string, option . return c, nil } -func (s *Session) retry(cb sessionRetryCallback, f func() error) error { +func (s *Session) retry(ctx context.Context, cb sessionRetryCallback, f func() error) error { for try := 0; ; try++ { err := f() @@ -637,7 +637,7 @@ func (s *Session) retry(cb sessionRetryCallback, f func() error) error { if cb != nil { cb(s.conn.Name(), s.name, try, err) } - err = s.tryRecover(err) + err = s.tryRecover(ctx, err) if err != nil { return err } @@ -715,7 +715,7 @@ type ExchangeDeclareOptions struct { // how messages are routed through it. Once an exchange is declared, its type // cannot be changed. The common types are "direct", "fanout", "topic" and // "headers". -func (s *Session) ExchangeDeclare(name string, kind ExchangeKind, option ...ExchangeDeclareOptions) error { +func (s *Session) ExchangeDeclare(ctx context.Context, name string, kind ExchangeKind, option ...ExchangeDeclareOptions) error { s.mu.Lock() defer s.mu.Unlock() @@ -731,7 +731,7 @@ func (s *Session) ExchangeDeclare(name string, kind ExchangeKind, option ...Exch o = option[0] } - return s.retry(s.exchangeDeclareRetryCB, func() error { + return s.retry(ctx, s.exchangeDeclareRetryCB, func() error { return s.channel.ExchangeDeclare( name, string(kind), @@ -749,7 +749,7 @@ func (s *Session) ExchangeDeclare(name string, kind ExchangeKind, option ...Exch // exchange is assumed by RabbitMQ to already exist, and attempting to connect to a // non-existent exchange will cause RabbitMQ to throw an exception. This function // can be used to detect the existence of an exchange. -func (s *Session) ExchangeDeclarePassive(name string, kind ExchangeKind, option ...ExchangeDeclareOptions) error { +func (s *Session) ExchangeDeclarePassive(ctx context.Context, name string, kind ExchangeKind, option ...ExchangeDeclareOptions) error { s.mu.Lock() defer s.mu.Unlock() @@ -765,7 +765,7 @@ func (s *Session) ExchangeDeclarePassive(name string, kind ExchangeKind, option o = option[0] } - err := s.retry(s.exchangeDeclarePassiveRetryCB, func() error { + err := s.retry(ctx, s.exchangeDeclarePassiveRetryCB, func() error { return s.channel.ExchangeDeclarePassive( name, string(kind), @@ -802,7 +802,7 @@ type ExchangeDeleteOptions struct { // ExchangeDelete removes the named exchange from the server. When an exchange is // deleted all queue bindings on the exchange are also deleted. If this exchange // does not exist, the channel will be closed with an error. -func (s *Session) ExchangeDelete(name string, option ...ExchangeDeleteOptions) error { +func (s *Session) ExchangeDelete(ctx context.Context, name string, option ...ExchangeDeleteOptions) error { s.mu.Lock() defer s.mu.Unlock() @@ -814,7 +814,7 @@ func (s *Session) ExchangeDelete(name string, option ...ExchangeDeleteOptions) e o = option[0] } - return s.retry(s.exchangeDeleteRetryCB, func() error { + return s.retry(ctx, s.exchangeDeleteRetryCB, func() error { return s.channel.ExchangeDelete(name, o.IfUnused, o.NoWait) }) } @@ -886,7 +886,7 @@ type QueueDeclareOptions struct { // // When the error return value is not nil, you can assume the queue could not be // declared with these parameters, and the channel will be closed. -func (s *Session) QueueDeclare(name string, option ...QueueDeclareOptions) (Queue, error) { +func (s *Session) QueueDeclare(ctx context.Context, name string, option ...QueueDeclareOptions) (Queue, error) { s.mu.Lock() defer s.mu.Unlock() @@ -904,7 +904,7 @@ func (s *Session) QueueDeclare(name string, option ...QueueDeclareOptions) (Queu err error queue amqp091.Queue ) - err = s.retry(s.queueDeclareRetryCB, func() error { + err = s.retry(ctx, s.queueDeclareRetryCB, func() error { queue, err = s.channel.QueueDeclare( name, o.Durable, @@ -925,7 +925,7 @@ func (s *Session) QueueDeclare(name string, option ...QueueDeclareOptions) (Queu // QueueDeclarePassive is functionally and parametrically equivalent to QueueDeclare, except that it sets the "passive" attribute to true. // A passive queue is assumed by RabbitMQ to already exist, and attempting to connect to a non-existent queue will cause RabbitMQ to throw an exception. // This function can be used to test for the existence of a queue. -func (s *Session) QueueDeclarePassive(name string, option ...QueueDeclareOptions) (Queue, error) { +func (s *Session) QueueDeclarePassive(ctx context.Context, name string, option ...QueueDeclareOptions) (Queue, error) { s.mu.Lock() defer s.mu.Unlock() @@ -944,7 +944,7 @@ func (s *Session) QueueDeclarePassive(name string, option ...QueueDeclareOptions err error queue amqp091.Queue ) - err = s.retry(s.queueDeclarePassiveRetryCB, func() error { + err = s.retry(ctx, s.queueDeclarePassiveRetryCB, func() error { queue, err = s.channel.QueueDeclarePassive( name, o.Durable, @@ -987,7 +987,7 @@ type QueueDeleteOptions struct { // QueueDelete removes the queue from the server including all bindings then // purges the messages based on server configuration, returning the number of // messages purged. -func (s *Session) QueueDelete(name string, option ...QueueDeleteOptions) (int, error) { +func (s *Session) QueueDelete(ctx context.Context, name string, option ...QueueDeleteOptions) (purgedMsgs int, err error) { s.mu.Lock() defer s.mu.Unlock() @@ -1000,32 +1000,19 @@ func (s *Session) QueueDelete(name string, option ...QueueDeleteOptions) (int, e o = option[0] } - return s.retryQueueDelete(func() (int, error) { - return s.channel.QueueDelete( + err = s.retry(ctx, s.queueDeleteRetryCB, func() error { + purgedMsgs, err = s.channel.QueueDelete( name, o.IfUnused, o.IfEmpty, o.NoWait, ) + return err }) -} - -func (s *Session) retryQueueDelete(f func() (int, error)) (int, error) { - for try := 0; ; try++ { - i, err := f() - if err == nil { - return i, nil - } - - if s.queueDeleteRetryCB != nil { - s.queueDeleteRetryCB(s.conn.Name(), s.name, try, err) - } - - err = s.tryRecover(err) - if err != nil { - return 0, err - } + if err != nil { + return 0, err } + return purgedMsgs, nil } type QueueBindOptions struct { @@ -1074,7 +1061,7 @@ type QueueBindOptions struct { // // If the binding could not complete, an error will be returned and the channel // will be closed. -func (s *Session) QueueBind(queueName string, routingKey string, exchange string, option ...QueueBindOptions) error { +func (s *Session) QueueBind(ctx context.Context, queueName string, routingKey string, exchange string, option ...QueueBindOptions) error { s.mu.Lock() defer s.mu.Unlock() @@ -1088,7 +1075,7 @@ func (s *Session) QueueBind(queueName string, routingKey string, exchange string o = option[0] } - return s.retry(s.queueBindRetryCB, func() error { + return s.retry(ctx, s.queueBindRetryCB, func() error { return s.channel.QueueBind( queueName, routingKey, @@ -1103,7 +1090,7 @@ func (s *Session) QueueBind(queueName string, routingKey string, exchange string // arguments. // It is possible to send and empty string for the exchange name which means to // unbind the queue from the default exchange. -func (s *Session) QueueUnbind(name string, routingKey string, exchange string, arg ...Table) error { +func (s *Session) QueueUnbind(ctx context.Context, name string, routingKey string, exchange string, arg ...Table) error { s.mu.Lock() defer s.mu.Unlock() @@ -1113,7 +1100,7 @@ func (s *Session) QueueUnbind(name string, routingKey string, exchange string, a option = arg[0] } - return s.retry(s.queueUnbindRetryCB, func() error { + return s.retry(ctx, s.queueUnbindRetryCB, func() error { return s.channel.QueueUnbind(name, routingKey, exchange, option) }) } @@ -1126,7 +1113,7 @@ type QueuePurgeOptions struct { // QueuePurge removes all messages from the named queue which are not waiting to be acknowledged. // Messages that have been delivered but have not yet been acknowledged will not be removed. // When successful, returns the number of messages purged. -func (s *Session) QueuePurge(name string, options ...QueuePurgeOptions) (int, error) { +func (s *Session) QueuePurge(ctx context.Context, name string, options ...QueuePurgeOptions) (int, error) { opt := QueuePurgeOptions{ NoWait: false, } @@ -1139,7 +1126,7 @@ func (s *Session) QueuePurge(name string, options ...QueuePurgeOptions) (int, er err error ) - err = s.retry(s.queuePurgeRetryCB, func() error { + err = s.retry(ctx, s.queuePurgeRetryCB, func() error { numPurgedMessages, err = s.channel.QueuePurge(name, opt.NoWait) if err != nil { return err @@ -1186,7 +1173,7 @@ type ExchangeBindOptions struct { // ----------------------------------------------- // key: AAPL --> trade ----> MSFT sell // \---> AAPL --> buy -func (s *Session) ExchangeBind(destination string, routingKey string, source string, option ...ExchangeBindOptions) error { +func (s *Session) ExchangeBind(ctx context.Context, destination string, routingKey string, source string, option ...ExchangeBindOptions) error { s.mu.Lock() defer s.mu.Unlock() @@ -1199,7 +1186,7 @@ func (s *Session) ExchangeBind(destination string, routingKey string, source str o = option[0] } - return s.retry(s.exchangeBindRetryCB, func() error { + return s.retry(ctx, s.exchangeBindRetryCB, func() error { return s.channel.ExchangeBind( destination, routingKey, @@ -1227,7 +1214,7 @@ type ExchangeUnbindOptions struct { // server by removing the routing key between them. This is the inverse of // ExchangeBind. If the binding does not currently exist, an error will be // returned. -func (s *Session) ExchangeUnbind(destination string, routingKey string, source string, option ...ExchangeUnbindOptions) error { +func (s *Session) ExchangeUnbind(ctx context.Context, destination string, routingKey string, source string, option ...ExchangeUnbindOptions) error { s.mu.Lock() defer s.mu.Unlock() @@ -1239,7 +1226,7 @@ func (s *Session) ExchangeUnbind(destination string, routingKey string, source s o = option[0] } - return s.retry(s.exchangeUnbindRetryCB, func() error { + return s.retry(ctx, s.exchangeUnbindRetryCB, func() error { return s.channel.ExchangeUnbind( destination, routingKey, @@ -1276,11 +1263,11 @@ greater as described by benchmarks on RabbitMQ. http://www.rabbitmq.com/blog/2012/04/25/rabbitmq-performance-measurements-part-2/ */ -func (s *Session) Qos(prefetchCount int, prefetchSize int) error { +func (s *Session) Qos(ctx context.Context, prefetchCount int, prefetchSize int) error { s.mu.Lock() defer s.mu.Unlock() - return s.retry(s.qosRetryCB, func() error { + return s.retry(ctx, s.qosRetryCB, func() error { // session quos should not affect new sessions of the same connection return s.channel.Qos(prefetchCount, prefetchSize, false) }) @@ -1307,18 +1294,18 @@ func (s *Session) Qos(prefetchCount int, prefetchSize int) error { // Note: RabbitMQ prefers to use TCP push back to control flow for all channels on // a connection, so under high volume scenarios, it's wise to open separate // Connections for publishings and deliveries. -func (s *Session) Flow(active bool) error { +func (s *Session) Flow(ctx context.Context, active bool) error { s.mu.Lock() defer s.mu.Unlock() - return s.retry(s.flowRetryCB, func() error { + return s.retry(ctx, s.flowRetryCB, func() error { return s.channel.Flow(active) }) } -// flushConfirms removes all previous confirmations pending processing. +// FlushConfirms removes all previous confirmations pending processing. // You can use the returned value -func (s *Session) flushConfirms() []amqp091.Confirmation { +func (s *Session) FlushConfirms() []amqp091.Confirmation { s.mu.Lock() defer s.mu.Unlock() diff --git a/pool/session_pool.go b/pool/session_pool.go index 9ed080d..b97da4e 100644 --- a/pool/session_pool.go +++ b/pool/session_pool.go @@ -129,22 +129,7 @@ func (sp *SessionPool) Size() int { // GetSession gets a pooled session. // blocks until a session is acquired from the pool. -func (sp *SessionPool) GetSession() (*Session, error) { - select { - case <-sp.catchShutdown(): - return nil, ErrClosed - case session, ok := <-sp.sessions: - if !ok { - return nil, fmt.Errorf("failed to get session: %w", ErrClosed) - } - - return session, nil - } -} - -// GetSessionCtx gets a pooled session. -// blocks until a session is acquired from the pool, the session pool was closed or the passed context was canceled. -func (sp *SessionPool) GetSessionCtx(ctx context.Context) (*Session, error) { +func (sp *SessionPool) GetSession(ctx context.Context) (*Session, error) { select { case <-sp.catchShutdown(): return nil, ErrClosed @@ -214,7 +199,7 @@ func (sp *SessionPool) deriveSession(ctx context.Context, conn *Connection, id i // ReturnSession returns a Session. // If Session is not a cached channel, it is simply closed here. // If Cache Session, we check if erred, new Session is created instead and then returned to the cache. -func (sp *SessionPool) ReturnSession(session *Session, erred bool) { +func (sp *SessionPool) ReturnSession(ctx context.Context, session *Session, erred bool) { // don't ass non-managed sessions back to the channel if !session.IsCached() { @@ -223,25 +208,19 @@ func (sp *SessionPool) ReturnSession(session *Session, erred bool) { } if erred { - err := session.Recover() - if err != nil { - // error is only returned on shutdown, - // don't recover upon shutdown - return - } + // try recovering until context closed or shutdown + _ = session.Recover(ctx) } else { // healthy sessions may contain pending confirmation messages // cleanup confirmations from previous session usage - _ = session.flushConfirms() + _ = session.FlushConfirms() // flush errors _ = session.Error() } - select { - case <-sp.catchShutdown(): - _ = session.Close() - case sp.sessions <- session: - } + // always put the session back into the pool + // even if it is still broken + sp.sessions <- session } func (sp *SessionPool) catchShutdown() <-chan struct{} { @@ -254,26 +233,20 @@ func (sp *SessionPool) Close() { sp.info("closing session pool...") defer sp.info("closed") + // trigger session cancelation + // consumers, publishers, etc. + sp.cancel() + wg := &sync.WaitGroup{} - // close all sessions -SessionClose: - for { - select { - // flush sessions channel - case session, ok := <-sp.sessions: - if !ok { - break SessionClose - } - wg.Add(1) - go func(*Session) { - defer wg.Done() - _ = session.Close() - }(session) - - default: - break SessionClose - } + // close all sessions: + for i := 0; i < sp.size; i++ { + session := <-sp.sessions + wg.Add(1) + go func(s *Session) { + defer wg.Done() + _ = s.Close() + }(session) } wg.Wait() @@ -299,7 +272,7 @@ func (sp *SessionPool) initCachedSession(id int) (*Session, error) { // retry until we get a channel // or until shutdown for { - conn, err := sp.pool.GetConnection() + conn, err := sp.pool.GetConnection(sp.ctx) if err != nil { // error is only returned upon shutdown return nil, err @@ -307,11 +280,11 @@ func (sp *SessionPool) initCachedSession(id int) (*Session, error) { session, err := sp.deriveSession(sp.ctx, conn, id) if err != nil { - sp.pool.ReturnConnection(conn, true) + sp.pool.ReturnConnection(sp.ctx, conn, true) continue } - sp.pool.ReturnConnection(conn, false) + sp.pool.ReturnConnection(sp.ctx, conn, false) return session, nil } } diff --git a/pool/session_pool_test.go b/pool/session_pool_test.go index ec9a219..d1034dd 100644 --- a/pool/session_pool_test.go +++ b/pool/session_pool_test.go @@ -1,6 +1,7 @@ package pool_test import ( + "context" "sync" "testing" "time" @@ -11,9 +12,10 @@ import ( ) func TestNewSessionPool(t *testing.T) { + ctx := context.TODO() connections := 1 sessions := 10 - p, err := pool.NewConnectionPool(connectURL, connections, + p, err := pool.NewConnectionPool(ctx, connectURL, connections, pool.ConnectionPoolWithName("TestNewConnectionPool"), pool.ConnectionPoolWithLogger(logging.NewTestLogger(t)), ) @@ -35,13 +37,13 @@ func TestNewSessionPool(t *testing.T) { wg.Add(1) go func() { defer wg.Done() - s, err := sp.GetSession() + s, err := sp.GetSession(ctx) if err != nil { assert.NoError(t, err) return } time.Sleep(3 * time.Second) - sp.ReturnSession(s, false) + sp.ReturnSession(ctx, s, false) }() } diff --git a/pool/session_test.go b/pool/session_test.go index 0d481de..0040354 100644 --- a/pool/session_test.go +++ b/pool/session_test.go @@ -13,8 +13,9 @@ import ( ) func TestNewSession(t *testing.T) { - + ctx := context.TODO() c, err := pool.NewConnection( + ctx, connectURL, "TestNewSession", pool.ConnectionWithLogger(logging.NewTestLogger(t)), @@ -42,35 +43,35 @@ func TestNewSession(t *testing.T) { }() queueName := fmt.Sprintf("TestNewSession-Queue-%d", id) - _, err = s.QueueDeclare(queueName) + _, err = s.QueueDeclare(ctx, queueName) if err != nil { assert.NoError(t, err) return } defer func() { - i, err := s.QueueDelete(queueName) + i, err := s.QueueDelete(ctx, queueName) assert.NoError(t, err) assert.Equal(t, 0, i) }() exchangeName := fmt.Sprintf("TestNewSession-Exchange-%d", id) - err = s.ExchangeDeclare(exchangeName, "topic") + err = s.ExchangeDeclare(ctx, exchangeName, "topic") if err != nil { assert.NoError(t, err) return } defer func() { - err := s.ExchangeDelete(exchangeName) + err := s.ExchangeDelete(ctx, exchangeName) assert.NoError(t, err) }() - err = s.QueueBind(queueName, "#", exchangeName) + err = s.QueueBind(ctx, queueName, "#", exchangeName) if err != nil { assert.NoError(t, err) return } defer func() { - err := s.QueueUnbind(queueName, "#", exchangeName, nil) + err := s.QueueUnbind(ctx, queueName, "#", exchangeName, nil) assert.NoError(t, err) }() @@ -132,8 +133,9 @@ func TestNewSession(t *testing.T) { } func TestNewSessionDisconnect(t *testing.T) { - + ctx := context.TODO() c, err := pool.NewConnection( + ctx, connectURL, "TestNewSessionDisconnect", pool.ConnectionWithLogger(logging.NewTestLogger(t)), @@ -184,7 +186,7 @@ func TestNewSessionDisconnect(t *testing.T) { started() exchangeName := fmt.Sprintf("TestNewSession-Exchange-%d", id) - err = s.ExchangeDeclare(exchangeName, "topic") + err = s.ExchangeDeclare(ctx, exchangeName, "topic") if err != nil { assert.NoError(t, err) return @@ -196,7 +198,7 @@ func TestNewSessionDisconnect(t *testing.T) { start9() started9() - err := s.ExchangeDelete(exchangeName) + err := s.ExchangeDelete(ctx, exchangeName) assert.NoError(t, err) stopped9() @@ -206,7 +208,7 @@ func TestNewSessionDisconnect(t *testing.T) { started2() queueName := fmt.Sprintf("TestNewSession-Queue-%d", id) - _, err = s.QueueDeclare(queueName) + _, err = s.QueueDeclare(ctx, queueName) if err != nil { assert.NoError(t, err) return @@ -218,14 +220,14 @@ func TestNewSessionDisconnect(t *testing.T) { started8() stopped8() - _, err := s.QueueDelete(queueName) + _, err := s.QueueDelete(ctx, queueName) assert.NoError(t, err) }() start3() started3() - err = s.QueueBind(queueName, "#", exchangeName) + err = s.QueueBind(ctx, queueName, "#", exchangeName) if err != nil { assert.NoError(t, err) return @@ -236,7 +238,7 @@ func TestNewSessionDisconnect(t *testing.T) { start7() started7() - err := s.QueueUnbind(queueName, "#", exchangeName, nil) + err := s.QueueUnbind(ctx, queueName, "#", exchangeName, nil) assert.NoError(t, err) stopped7() @@ -282,7 +284,7 @@ func TestNewSessionDisconnect(t *testing.T) { var once sync.Once for { - tag, err := s.Publish(context.Background(), exchangeName, "", pool.Publishing{ + tag, err := s.Publish(ctx, exchangeName, "", pool.Publishing{ Mandatory: true, ContentType: "application/json", Body: []byte(message), @@ -300,7 +302,7 @@ func TestNewSessionDisconnect(t *testing.T) { stopped6() }) - err = s.AwaitConfirm(context.Background(), tag) + err = s.AwaitConfirm(ctx, tag) if err != nil { // retry because the first attempt at confirmation failed continue @@ -316,6 +318,7 @@ func TestNewSessionDisconnect(t *testing.T) { } func TestNewSessionQueueDeclarePassive(t *testing.T) { + ctx := context.TODO() var wg sync.WaitGroup defer func() { @@ -324,6 +327,7 @@ func TestNewSessionQueueDeclarePassive(t *testing.T) { }() c, err := pool.NewConnection( + ctx, connectURL, "TestNewSessionQueueDeclarePassive", pool.ConnectionWithLogger(logging.NewTestLogger(t)), @@ -347,7 +351,7 @@ func TestNewSessionQueueDeclarePassive(t *testing.T) { for i := 0; i < 100; i++ { qname := fmt.Sprintf("TestNewSessionQueueDeclarePassive-queue-%d", i) - q, err := session.QueueDeclare(qname) + q, err := session.QueueDeclare(ctx, qname) if err != nil { assert.NoError(t, err) return @@ -356,11 +360,11 @@ func TestNewSessionQueueDeclarePassive(t *testing.T) { // executed upon return defer func() { - _, err := session.QueueDelete(qname) + _, err := session.QueueDelete(ctx, qname) assert.NoErrorf(t, err, "failed to delete queue: %s", qname) }() - q, err = session.QueueDeclarePassive(qname) + q, err = session.QueueDeclarePassive(ctx, qname) if err != nil { assert.NoErrorf(t, err, "QueueDeclarePassive failed for queue: %s", qname) return diff --git a/pool/subscriber.go b/pool/subscriber.go index c7619a2..3874fe7 100644 --- a/pool/subscriber.go +++ b/pool/subscriber.go @@ -79,10 +79,10 @@ func NewSubscriber(p *Pool, options ...SubscriberOption) *Subscriber { } // HandlerFunc is basically a handler for incoming messages/events. -type HandlerFunc func(Delivery) error +type HandlerFunc func(context.Context, Delivery) error // BatchHandlerFunc is a handler for incoming batches of messages/events -type BatchHandlerFunc func([]Delivery) error +type BatchHandlerFunc func(context.Context, []Delivery) error // RegisterHandlerFunc registers a consumer function that starts a consumer upon subscriber startup. // The consumer is identified by a string that is unique and scoped for all consumers on this channel. @@ -174,7 +174,7 @@ func (s *Subscriber) RegisterBatchHandler(handler *BatchHandler) { // Start starts the consumers for all registered handler functions // This method is not blocking. Use Wait() to wait for all routines to shut down // via context cancelation (e.g. via a signal) -func (s *Subscriber) Start() (err error) { +func (s *Subscriber) Start(ctx context.Context) (err error) { s.mu.Lock() defer s.mu.Unlock() if s.started { @@ -196,7 +196,7 @@ func (s *Subscriber) Start() (err error) { for _, h := range s.handlers { s.wg.Add(1) go s.consumer(h, &s.wg) - err = h.awaitResumed(s.ctx) + err = h.awaitResumed(ctx) if err != nil { return fmt.Errorf("failed to start consumer for queue %s: %w", h.Queue(), err) } @@ -207,7 +207,7 @@ func (s *Subscriber) Start() (err error) { s.wg.Add(1) go s.batchConsumer(bh, &s.wg) - err = bh.awaitResumed(s.ctx) + err = bh.awaitResumed(ctx) if err != nil { return fmt.Errorf("failed to start batch consumer for queue %s: %w", bh.Queue(), err) } @@ -297,7 +297,7 @@ func (s *Subscriber) consume(h *Handler) (err error) { s.debugConsumer(opts.ConsumerTag, "starting consumer...") - session, err := s.pool.GetSession() + session, err := s.pool.GetSession(s.ctx) if err != nil { return err } @@ -328,7 +328,7 @@ func (s *Subscriber) consume(h *Handler) (err error) { } s.infoHandler(opts.ConsumerTag, msg.Exchange, msg.RoutingKey, opts.Queue, "received message") - err = opts.HandlerFunc(msg) + err = opts.HandlerFunc(h.pausing(), msg) if opts.AutoAck { if err != nil { // we cannot really do anything to recover from a processing error in this case @@ -349,35 +349,38 @@ func (s *Subscriber) consume(h *Handler) (err error) { // (n)ack delivery and signal that message was processed by the service func (s *Subscriber) ackPostHandle(opts HandlerConfig, deliveryTag uint64, exchange, routingKey string, session *Session, handlerErr error) (err error) { var ackErr error - if handlerErr != nil { + if handlerErr == nil { + ackErr = session.Ack(deliveryTag, false) + } else if errors.Is(handlerErr, ErrReject) || errors.Is(handlerErr, ErrRejectSingle) { + ackErr = session.Nack(deliveryTag, false, false) + } else { // requeue message if possible ackErr = session.Nack(deliveryTag, false, true) - } else { - ackErr = session.Ack(deliveryTag, false) } - // if (n)ack fails, we know that the connection died - // potentially before processing already. - if ackErr != nil { - s.warnHandler(opts.ConsumerTag, exchange, routingKey, opts.Queue, ackErr, "(n)ack failed") - poolErr := session.Recover() - if poolErr != nil { - // only returns an error upon shutdown - return poolErr + if ackErr == nil { + // (n)acked or rejected successfully + if handlerErr == nil { + s.infoHandler(opts.ConsumerTag, exchange, routingKey, opts.Queue, "acked message") + } else if errors.Is(handlerErr, ErrReject) { + s.infoHandler(opts.ConsumerTag, exchange, routingKey, opts.Queue, "rejected message") + } else { + s.infoHandler(opts.ConsumerTag, exchange, routingKey, opts.Queue, "nacked message") } - - // do not retry, because the broker will requeue the un(n)acked message - // after a timeout of (by default) 30 minutes. return nil } - // (n)acked successfully - if handlerErr != nil { - s.infoHandler(opts.ConsumerTag, exchange, routingKey, opts.Queue, "nacked message") - } else { - s.infoHandler(opts.ConsumerTag, exchange, routingKey, opts.Queue, "acked message") + // if (n)ack fails, we know that the connection died + // potentially before processing already. + s.warnHandler(opts.ConsumerTag, exchange, routingKey, opts.Queue, ackErr, "(n)ack failed") + poolErr := session.Recover(s.ctx) + if poolErr != nil { + // only returns an error upon shutdown + return poolErr } - // successfully handled message + + // do not retry, because the broker will requeue the un(n)acked message + // after a timeout of (by default) 30 minutes. return nil } @@ -413,7 +416,7 @@ func (s *Subscriber) batchConsume(h *BatchHandler) (err error) { s.debugConsumer(opts.ConsumerTag, "starting batch consumer...") - session, err := s.pool.GetSession() + session, err := s.pool.GetSession(s.ctx) if err != nil { return err } @@ -522,7 +525,7 @@ func (s *Subscriber) batchConsume(h *BatchHandler) (err error) { ) s.infoBatchHandler(opts.ConsumerTag, opts.Queue, batchSize, batchBytes, "received batch") - err = opts.HandlerFunc(batch) + err = opts.HandlerFunc(h.pausing(), batch) // no acks required if opts.AutoAck { if err != nil { @@ -554,55 +557,76 @@ func (s *Subscriber) batchConsume(h *BatchHandler) (err error) { func (s *Subscriber) ackBatchPostHandle(opts BatchHandlerConfig, lastDeliveryTag uint64, currentBatchSize, currentBatchBytes int, session *Session, handlerErr error) (err error) { var ackErr error // processing failed - if handlerErr != nil { - // requeue message if possible & nack all previous messages - ackErr = session.Nack(lastDeliveryTag, true, true) - } else { + if handlerErr == nil { // ack last and all previous messages ackErr = session.Ack(lastDeliveryTag, true) + } else if errors.Is(handlerErr, ErrReject) { + // reject multiple + ackErr = session.Nack(lastDeliveryTag, true, false) + } else if errors.Is(handlerErr, ErrRejectSingle) { + // reject single + ackErr = session.Nack(lastDeliveryTag, false, false) + } else { + // requeue message if possible & nack all previous messages + ackErr = session.Nack(lastDeliveryTag, true, true) } - // if (n)ack fails, we know that the connection died - // potentially before processing already. - if ackErr != nil { - s.warnBatchHandler( - opts.ConsumerTag, - opts.Queue, - currentBatchSize, - currentBatchBytes, - ackErr, - "batch (n)ack failed", - ) - poolErr := session.Recover() - if poolErr != nil { - // only returns an error upon shutdown - return poolErr + if ackErr == nil { + if handlerErr == nil { + s.infoBatchHandler( + opts.ConsumerTag, + opts.Queue, + currentBatchSize, + currentBatchBytes, + "acked batch", + ) + } else if errors.Is(handlerErr, ErrReject) { + s.infoBatchHandler( + opts.ConsumerTag, + opts.Queue, + currentBatchSize, + currentBatchBytes, + "rejected batch", + ) + } else if errors.Is(handlerErr, ErrRejectSingle) { + s.infoBatchHandler( + opts.ConsumerTag, + opts.Queue, + currentBatchSize, + currentBatchBytes, + "rejected single message in batch", + ) + } else { + s.infoBatchHandler( + opts.ConsumerTag, + opts.Queue, + currentBatchSize, + currentBatchBytes, + "nacked batch", + ) } - - // do not retry, because the broker will requeue the un(n)acked message - // after a timeout of by default 30 minutes. + // successfully handled message return nil } - // (n)acked successfully - if handlerErr != nil { - s.infoBatchHandler( - opts.ConsumerTag, - opts.Queue, - currentBatchSize, - currentBatchBytes, - "nacked batch", - ) - } else { - s.infoBatchHandler( - opts.ConsumerTag, - opts.Queue, - currentBatchSize, - currentBatchBytes, - "acked batch", - ) + // if (n)ack fails, we know that the connection died + // potentially before processing already. + s.warnBatchHandler( + opts.ConsumerTag, + opts.Queue, + currentBatchSize, + currentBatchBytes, + ackErr, + "batch (n)ack failed", + ) + poolErr := session.Recover(s.ctx) + if poolErr != nil { + // only returns an error upon shutdown + return poolErr } - // successfully handled message + + // do not retry, because the broker will requeue the un(n)acked message + // after a timeout of by default 30 minutes. return nil } @@ -616,7 +640,7 @@ func (s *Subscriber) returnSession(h handler, session *Session, err error) { if errors.Is(err, ErrClosed) { // graceful shutdown - s.pool.ReturnSession(session, false) + s.pool.ReturnSession(s.ctx, session, false) s.infoConsumer(opts.ConsumerTag, "closed") return } @@ -626,11 +650,11 @@ func (s *Subscriber) returnSession(h handler, session *Session, err error) { // expected closing due to context cancelation // cancel errors the underlying channel // A canceled session is an erred session. - s.pool.ReturnSession(session, true) + s.pool.ReturnSession(s.ctx, session, true) s.infoConsumer(opts.ConsumerTag, "paused") default: // actual error - s.pool.ReturnSession(session, true) + s.pool.ReturnSession(s.ctx, session, true) s.warnConsumer(opts.ConsumerTag, err, "closed unexpectedly") } } diff --git a/pool/subscriber_batch_handler.go b/pool/subscriber_batch_handler.go index f5ea9db..09002e9 100644 --- a/pool/subscriber_batch_handler.go +++ b/pool/subscriber_batch_handler.go @@ -163,9 +163,11 @@ func (h *BatchHandler) awaitResumed(ctx context.Context) error { return h.sc.AwaitResumed(ctx) } +/* func (h *BatchHandler) awaitPaused(ctx context.Context) error { return h.sc.AwaitPaused(ctx) } +*/ func (h *BatchHandler) Queue() string { h.mu.RLock() diff --git a/pool/subscriber_handler.go b/pool/subscriber_handler.go index 0062afa..62ad702 100644 --- a/pool/subscriber_handler.go +++ b/pool/subscriber_handler.go @@ -139,9 +139,11 @@ func (h *Handler) awaitResumed(ctx context.Context) error { return h.sc.AwaitResumed(ctx) } +/* func (h *Handler) awaitPaused(ctx context.Context) error { return h.sc.AwaitPaused(ctx) } +*/ func (h *Handler) Queue() string { h.mu.Lock() diff --git a/pool/subscriber_handler_options_test.go b/pool/subscriber_handler_options_test.go index 5c2c79d..507eabb 100644 --- a/pool/subscriber_handler_options_test.go +++ b/pool/subscriber_handler_options_test.go @@ -1,13 +1,14 @@ package pool import ( + "context" "testing" "github.com/stretchr/testify/assert" ) func TestWithMaxBatchSize(t *testing.T) { - dummyHandler := func([]Delivery) error { return nil } + dummyHandler := func(context.Context, []Delivery) error { return nil } bh := NewBatchHandler("test", dummyHandler, WithMaxBatchSize(0), WithMaxBatchBytes(0)) assert.Equal(t, defaultMaxBatchSize, bh.MaxBatchSize()) diff --git a/pool/subscriber_test.go b/pool/subscriber_test.go index 5d28607..968c5b9 100644 --- a/pool/subscriber_test.go +++ b/pool/subscriber_test.go @@ -13,9 +13,16 @@ import ( ) func TestSubscriber(t *testing.T) { - + ctx := context.TODO() sessions := 2 // publisher sessions + consumer sessions - p, err := pool.New(connectURL, 1, sessions, pool.WithConfirms(true), pool.WithLogger(logging.NewTestLogger(t))) + p, err := pool.New( + ctx, + connectURL, + 1, + sessions, + pool.WithConfirms(true), + pool.WithLogger(logging.NewTestLogger(t)), + ) if err != nil { assert.NoError(t, err) return @@ -35,50 +42,50 @@ func TestSubscriber(t *testing.T) { assert.NoError(t, err) return } - defer p.ReturnSession(ts, false) + defer p.ReturnSession(ctx, ts, false) queueName := fmt.Sprintf("TestSubscriber-Queue-%d", id) - _, err = ts.QueueDeclare(queueName) + _, err = ts.QueueDeclare(ctx, queueName) if err != nil { assert.NoError(t, err) return } defer func() { - i, err := ts.QueueDelete(queueName) + i, err := ts.QueueDelete(ctx, queueName) assert.NoError(t, err) assert.Equal(t, 0, i) }() exchangeName := fmt.Sprintf("TestSubscriber-Exchange-%d", id) - err = ts.ExchangeDeclare(exchangeName, "topic") + err = ts.ExchangeDeclare(ctx, exchangeName, "topic") if err != nil { assert.NoError(t, err) return } defer func() { - err := ts.ExchangeDelete(exchangeName) + err := ts.ExchangeDelete(ctx, exchangeName) assert.NoError(t, err) }() - err = ts.QueueBind(queueName, "#", exchangeName) + err = ts.QueueBind(ctx, queueName, "#", exchangeName) if err != nil { assert.NoError(t, err) return } defer func() { - err := ts.QueueUnbind(queueName, "#", exchangeName, nil) + err := ts.QueueUnbind(ctx, queueName, "#", exchangeName, nil) assert.NoError(t, err) }() message := fmt.Sprintf("Message-%s", queueName) - ctx, cancel := context.WithCancel(p.Context()) + cctx, cancel := context.WithCancel(p.Context()) - sub := pool.NewSubscriber(p, pool.SubscriberWithContext(ctx)) + sub := pool.NewSubscriber(p, pool.SubscriberWithContext(cctx)) defer sub.Close() sub.RegisterHandlerFunc(queueName, - func(msg pool.Delivery) error { + func(ctx context.Context, msg pool.Delivery) error { // handler func receivedMsg := string(msg.Body) @@ -94,13 +101,17 @@ func TestSubscriber(t *testing.T) { Exclusive: true, }, ) - sub.Start() + err = sub.Start(ctx) + if err != nil { + assert.NoError(t, err) + return + } time.Sleep(5 * time.Second) pub := pool.NewPublisher(p) defer pub.Close() - pub.Publish(exchangeName, "", pool.Publishing{ + pub.Publish(ctx, exchangeName, "", pool.Publishing{ Mandatory: true, ContentType: "application/json", Body: []byte(message), @@ -117,13 +128,20 @@ func TestSubscriber(t *testing.T) { } func TestBatchSubscriber(t *testing.T) { - var ( + ctx = context.TODO() sessions = 2 // publisher sessions + consumer sessions numMessages = 50 batchTimeout = 10 * time.Second // keep this at a higher number for slow machines ) - p, err := pool.New(connectURL, 1, sessions, pool.WithConfirms(true), pool.WithLogger(logging.NewTestLogger(t))) + p, err := pool.New( + ctx, + connectURL, + 1, + sessions, + pool.WithConfirms(true), + pool.WithLogger(logging.NewTestLogger(t)), + ) if err != nil { assert.NoError(t, err) return @@ -143,38 +161,38 @@ func TestBatchSubscriber(t *testing.T) { assert.NoError(t, err) return } - defer p.ReturnSession(ts, false) + defer p.ReturnSession(ctx, ts, false) queueName := fmt.Sprintf("TestBatchSubscriber-Queue-%d", id) - _, err = ts.QueueDeclare(queueName) + _, err = ts.QueueDeclare(ctx, queueName) if err != nil { assert.NoError(t, err) return } defer func() { - i, err := ts.QueueDelete(queueName) + i, err := ts.QueueDelete(ctx, queueName) assert.NoError(t, err) assert.Equal(t, 0, i) }() exchangeName := fmt.Sprintf("TestBatchSubscriber-Exchange-%d", id) - err = ts.ExchangeDeclare(exchangeName, "topic") + err = ts.ExchangeDeclare(ctx, exchangeName, "topic") if err != nil { assert.NoError(t, err) return } defer func() { - err := ts.ExchangeDelete(exchangeName) + err := ts.ExchangeDelete(ctx, exchangeName) assert.NoError(t, err) }() - err = ts.QueueBind(queueName, "#", exchangeName) + err = ts.QueueBind(ctx, queueName, "#", exchangeName) if err != nil { assert.NoError(t, err) return } defer func() { - err := ts.QueueUnbind(queueName, "#", exchangeName, nil) + err := ts.QueueUnbind(ctx, queueName, "#", exchangeName, nil) assert.NoError(t, err) }() @@ -185,7 +203,7 @@ func TestBatchSubscriber(t *testing.T) { for i := 0; i < numMessages; i++ { message := fmt.Sprintf("Message-%s-%d", queueName, i) - pub.Publish(exchangeName, "", pool.Publishing{ + pub.Publish(ctx, exchangeName, "", pool.Publishing{ Mandatory: true, ContentType: "application/json", Body: []byte(message), @@ -202,7 +220,7 @@ func TestBatchSubscriber(t *testing.T) { batchCount := 0 messageCount := 0 sub.RegisterBatchHandlerFunc(queueName, - func(msgs []pool.Delivery) error { + func(ctx context.Context, msgs []pool.Delivery) error { log := logging.NewTestLogger(t) assert.Equal(t, batchSize, len(msgs)) @@ -227,7 +245,7 @@ func TestBatchSubscriber(t *testing.T) { Exclusive: true, }), ) - sub.Start() + sub.Start(ctx) // this should be canceled upon context cancelation from within the // subscriber handler. @@ -243,7 +261,6 @@ func TestBatchSubscriber(t *testing.T) { } func TestBatchSubscriberMaxBytes(t *testing.T) { - for i := 1; i <= 2048; i = i*2 + 1 { testBatchSubscriberMaxBytes(t, i) } @@ -253,11 +270,19 @@ func testBatchSubscriberMaxBytes(t *testing.T, maxBatchBytes int) { t.Helper() var ( + ctx = context.TODO() sessions = 2 // publisher sessions + consumer sessions numMessages = 50 batchTimeout = 5 * time.Second // keep this at a higher number for slow machines ) - p, err := pool.New(connectURL, 1, sessions, pool.WithConfirms(true), pool.WithLogger(logging.NewTestLogger(t))) + p, err := pool.New( + ctx, + connectURL, + 1, + sessions, + pool.WithConfirms(true), + pool.WithLogger(logging.NewTestLogger(t)), + ) if err != nil { assert.NoError(t, err) return @@ -277,38 +302,38 @@ func testBatchSubscriberMaxBytes(t *testing.T, maxBatchBytes int) { assert.NoError(t, err) return } - defer p.ReturnSession(ts, false) + defer p.ReturnSession(ctx, ts, false) queueName := fmt.Sprintf("TestBatchSubscriberMaxBytes-Queue-%d", id) - _, err = ts.QueueDeclare(queueName) + _, err = ts.QueueDeclare(ctx, queueName) if err != nil { assert.NoError(t, err) return } defer func() { - i, err := ts.QueueDelete(queueName) + i, err := ts.QueueDelete(ctx, queueName) assert.NoError(t, err) assert.Equal(t, 0, i) }() exchangeName := fmt.Sprintf("TestBatchSubscriberMaxBytes-Exchange-%d", id) - err = ts.ExchangeDeclare(exchangeName, "topic") + err = ts.ExchangeDeclare(ctx, exchangeName, "topic") if err != nil { assert.NoError(t, err) return } defer func() { - err := ts.ExchangeDelete(exchangeName) + err := ts.ExchangeDelete(ctx, exchangeName) assert.NoError(t, err) }() - err = ts.QueueBind(queueName, "#", exchangeName) + err = ts.QueueBind(ctx, queueName, "#", exchangeName) if err != nil { assert.NoError(t, err) return } defer func() { - err := ts.QueueUnbind(queueName, "#", exchangeName, nil) + err := ts.QueueUnbind(ctx, queueName, "#", exchangeName, nil) assert.NoError(t, err) }() @@ -326,7 +351,7 @@ func testBatchSubscriberMaxBytes(t *testing.T, maxBatchBytes int) { maxMsgLen = mlen } - pub.Publish(exchangeName, "", pool.Publishing{ + pub.Publish(ctx, exchangeName, "", pool.Publishing{ Mandatory: true, ContentType: "application/json", Body: []byte(message), @@ -345,15 +370,15 @@ func testBatchSubscriberMaxBytes(t *testing.T, maxBatchBytes int) { } log.Debugf("expected batches: %d", expectedBatches) - ctx, cancel := context.WithCancel(p.Context()) + cctx, cancel := context.WithCancel(p.Context()) - sub := pool.NewSubscriber(p, pool.SubscriberWithContext(ctx)) + sub := pool.NewSubscriber(p, pool.SubscriberWithContext(cctx)) defer sub.Close() batchCount := 0 messageCount := 0 sub.RegisterBatchHandlerFunc(queueName, - func(msgs []pool.Delivery) error { + func(ctx context.Context, msgs []pool.Delivery) error { for idx, msg := range msgs { assert.Truef(t, len(msg.Body) > 0, "msg body is empty: message index: %d", idx) @@ -383,7 +408,7 @@ func testBatchSubscriberMaxBytes(t *testing.T, maxBatchBytes int) { Exclusive: true, }), ) - sub.Start() + sub.Start(ctx) // this should be canceled upon context cancelation from within the // subscriber handler. diff --git a/pool/topologer.go b/pool/topologer.go index f900092..73b0cbc 100644 --- a/pool/topologer.go +++ b/pool/topologer.go @@ -38,12 +38,12 @@ func NewTopologer(p *Pool, options ...TopologerOption) *Topologer { // TODO: it should be possible to pass a custom context in here so that we can define // timeouts, especially for a topology deleter which operates on a closed context and needs a new one. -func (t *Topologer) getSession() (*Session, error) { +func (t *Topologer) getSession(ctx context.Context) (*Session, error) { if t.transientOnly || t.pool.SessionPoolSize() == 0 { - return t.pool.GetTransientSession(t.ctx) + return t.pool.GetTransientSession(ctx) } - return t.pool.GetSessionCtx(t.ctx) + return t.pool.GetSession(ctx) } // ExchangeDeclare declares an exchange on the server. If the exchange does not @@ -62,19 +62,19 @@ func (t *Topologer) getSession() (*Session, error) { // how messages are routed through it. Once an exchange is declared, its type // cannot be changed. The common types are "direct", "fanout", "topic" and // "headers". -func (t *Topologer) ExchangeDeclare(name string, kind ExchangeKind, option ...ExchangeDeclareOptions) (err error) { - s, err := t.getSession() +func (t *Topologer) ExchangeDeclare(ctx context.Context, name string, kind ExchangeKind, option ...ExchangeDeclareOptions) (err error) { + s, err := t.getSession(ctx) if err != nil { return err } defer func() { if err != nil { - t.pool.ReturnSession(s, true) + t.pool.ReturnSession(ctx, s, true) } else { - t.pool.ReturnSession(s, false) + t.pool.ReturnSession(ctx, s, false) } }() - return s.ExchangeDeclare(name, kind, option...) + return s.ExchangeDeclare(ctx, name, kind, option...) } // ExchangeDeclarePassive is functionally and parametrically equivalent to @@ -82,38 +82,38 @@ func (t *Topologer) ExchangeDeclare(name string, kind ExchangeKind, option ...Ex // exchange is assumed by RabbitMQ to already exist, and attempting to connect to a // non-existent exchange will cause RabbitMQ to throw an exception. This function // can be used to detect the existence of an exchange. -func (t *Topologer) ExchangeDeclarePassive(name string, kind ExchangeKind, option ...ExchangeDeclareOptions) (err error) { - s, err := t.getSession() +func (t *Topologer) ExchangeDeclarePassive(ctx context.Context, name string, kind ExchangeKind, option ...ExchangeDeclareOptions) (err error) { + s, err := t.getSession(ctx) if err != nil { return err } defer func() { if err != nil { - t.pool.ReturnSession(s, true) + t.pool.ReturnSession(ctx, s, true) } else { - t.pool.ReturnSession(s, false) + t.pool.ReturnSession(ctx, s, false) } }() - return s.ExchangeDeclarePassive(name, kind, option...) + return s.ExchangeDeclarePassive(ctx, name, kind, option...) } // ExchangeDelete removes the named exchange from the server. When an exchange is // deleted all queue bindings on the exchange are also deleted. If this exchange // does not exist, the channel will be closed with an error. -func (t *Topologer) ExchangeDelete(name string, option ...ExchangeDeleteOptions) (err error) { - s, err := t.getSession() +func (t *Topologer) ExchangeDelete(ctx context.Context, name string, option ...ExchangeDeleteOptions) (err error) { + s, err := t.getSession(ctx) if err != nil { return err } defer func() { if err != nil { - t.pool.ReturnSession(s, true) + t.pool.ReturnSession(ctx, s, true) } else { - t.pool.ReturnSession(s, false) + t.pool.ReturnSession(ctx, s, false) } }() - return s.ExchangeDelete(name, option...) + return s.ExchangeDelete(ctx, name, option...) } // QueueDeclare declares a queue to hold messages and deliver to consumers. @@ -134,73 +134,73 @@ func (t *Topologer) ExchangeDelete(name string, option ...ExchangeDeleteOptions) // // The queue name may be empty, in which case the server will generate a unique name // which will be returned in the Name field of Queue struct. -func (t *Topologer) QueueDeclare(name string, option ...QueueDeclareOptions) (queue Queue, err error) { - s, err := t.getSession() +func (t *Topologer) QueueDeclare(ctx context.Context, name string, option ...QueueDeclareOptions) (queue Queue, err error) { + s, err := t.getSession(ctx) if err != nil { return Queue{}, err } defer func() { if err != nil { - t.pool.ReturnSession(s, true) + t.pool.ReturnSession(ctx, s, true) } else { - t.pool.ReturnSession(s, false) + t.pool.ReturnSession(ctx, s, false) } }() - return s.QueueDeclare(name, option...) + return s.QueueDeclare(ctx, name, option...) } // QueueDeclarePassive is functionally and parametrically equivalent to QueueDeclare, except that it sets the "passive" attribute to true. // A passive queue is assumed by RabbitMQ to already exist, and attempting to connect to a non-existent queue will cause RabbitMQ to throw an exception. // This function can be used to test for the existence of a queue. -func (t *Topologer) QueueDeclarePassive(name string, option ...QueueDeclareOptions) (queue Queue, err error) { - s, err := t.getSession() +func (t *Topologer) QueueDeclarePassive(ctx context.Context, name string, option ...QueueDeclareOptions) (queue Queue, err error) { + s, err := t.getSession(ctx) if err != nil { return Queue{}, err } defer func() { if err != nil { - t.pool.ReturnSession(s, true) + t.pool.ReturnSession(ctx, s, true) } else { - t.pool.ReturnSession(s, false) + t.pool.ReturnSession(ctx, s, false) } }() - return s.QueueDeclarePassive(name, option...) + return s.QueueDeclarePassive(ctx, name, option...) } // QueuePurge removes all messages from the named queue which are not waiting to be acknowledged. // Messages that have been delivered but have not yet been acknowledged will not be removed. // When successful, returns the number of messages purged. -func (t *Topologer) QueuePurge(name string, options ...QueuePurgeOptions) (int, error) { - s, err := t.getSession() +func (t *Topologer) QueuePurge(ctx context.Context, name string, options ...QueuePurgeOptions) (int, error) { + s, err := t.getSession(ctx) if err != nil { return 0, err } defer func() { if err != nil { - t.pool.ReturnSession(s, true) + t.pool.ReturnSession(ctx, s, true) } else { - t.pool.ReturnSession(s, false) + t.pool.ReturnSession(ctx, s, false) } }() - return s.QueuePurge(name, options...) + return s.QueuePurge(ctx, name, options...) } // QueueDelete removes the queue from the server including all bindings then // purges the messages based on server configuration, returning the number of // messages purged. -func (t *Topologer) QueueDelete(name string, option ...QueueDeleteOptions) (purged int, err error) { - s, err := t.getSession() +func (t *Topologer) QueueDelete(ctx context.Context, name string, option ...QueueDeleteOptions) (purged int, err error) { + s, err := t.getSession(ctx) if err != nil { return 0, err } defer func() { if err != nil { - t.pool.ReturnSession(s, true) + t.pool.ReturnSession(ctx, s, true) } else { - t.pool.ReturnSession(s, false) + t.pool.ReturnSession(ctx, s, false) } }() - return s.QueueDelete(name, option...) + return s.QueueDelete(ctx, name, option...) } // QueueBind binds an exchange to a queue so that publishings to the exchange will @@ -234,19 +234,19 @@ func (t *Topologer) QueueDelete(name string, option ...QueueDeleteOptions) (purg // key: info ---> amq.topic ----> # ------> emails // \---> info ---/ // key: debug --> amq.topic ----> # ------> emails -func (t *Topologer) QueueBind(name string, routingKey string, exchange string, option ...QueueBindOptions) (err error) { - s, err := t.getSession() +func (t *Topologer) QueueBind(ctx context.Context, name string, routingKey string, exchange string, option ...QueueBindOptions) (err error) { + s, err := t.getSession(ctx) if err != nil { return err } defer func() { if err != nil { - t.pool.ReturnSession(s, true) + t.pool.ReturnSession(ctx, s, true) } else { - t.pool.ReturnSession(s, false) + t.pool.ReturnSession(ctx, s, false) } }() - return s.QueueBind(name, routingKey, exchange, option...) + return s.QueueBind(ctx, name, routingKey, exchange, option...) } // QueueUnbind removes a binding between an exchange and queue matching the key and @@ -254,19 +254,19 @@ func (t *Topologer) QueueBind(name string, routingKey string, exchange string, o // It is possible to send and empty string for the exchange name which means to // unbind the queue from the default exchange. -func (t *Topologer) QueueUnbind(name string, routingKey string, exchange string, args ...Table) (err error) { - s, err := t.getSession() +func (t *Topologer) QueueUnbind(ctx context.Context, name string, routingKey string, exchange string, args ...Table) (err error) { + s, err := t.getSession(ctx) if err != nil { return err } defer func() { if err != nil { - t.pool.ReturnSession(s, true) + t.pool.ReturnSession(ctx, s, true) } else { - t.pool.ReturnSession(s, false) + t.pool.ReturnSession(ctx, s, false) } }() - return s.QueueUnbind(name, routingKey, exchange, args...) + return s.QueueUnbind(ctx, name, routingKey, exchange, args...) } // ExchangeBind binds an exchange to another exchange to create inter-exchange @@ -292,37 +292,37 @@ func (t *Topologer) QueueUnbind(name string, routingKey string, exchange string, // ----------------------------------------------- // key: AAPL --> trade ----> MSFT sell // \---> AAPL --> buy -func (t *Topologer) ExchangeBind(destination string, routingKey string, source string, option ...ExchangeBindOptions) (err error) { - s, err := t.getSession() +func (t *Topologer) ExchangeBind(ctx context.Context, destination string, routingKey string, source string, option ...ExchangeBindOptions) (err error) { + s, err := t.getSession(ctx) if err != nil { return err } defer func() { if err != nil { - t.pool.ReturnSession(s, true) + t.pool.ReturnSession(ctx, s, true) } else { - t.pool.ReturnSession(s, false) + t.pool.ReturnSession(ctx, s, false) } }() - return s.ExchangeBind(destination, routingKey, source, option...) + return s.ExchangeBind(ctx, destination, routingKey, source, option...) } // ExchangeUnbind unbinds the destination exchange from the source exchange on the // server by removing the routing key between them. This is the inverse of // ExchangeBind. If the binding does not currently exist, an error will be // returned. -func (t *Topologer) ExchangeUnbind(destination string, routingKey string, source string, option ...ExchangeUnbindOptions) (err error) { - s, err := t.getSession() +func (t *Topologer) ExchangeUnbind(ctx context.Context, destination string, routingKey string, source string, option ...ExchangeUnbindOptions) (err error) { + s, err := t.getSession(ctx) if err != nil { return err } defer func() { if err != nil { - t.pool.ReturnSession(s, true) + t.pool.ReturnSession(ctx, s, true) } else { - t.pool.ReturnSession(s, false) + t.pool.ReturnSession(ctx, s, false) } }() - return s.ExchangeUnbind(destination, routingKey, source, option...) + return s.ExchangeUnbind(ctx, destination, routingKey, source, option...) } From 716e69048a4bfc566cc1104bd65e7fc3f31a12f9 Mon Sep 17 00:00:00 2001 From: John Behm Date: Wed, 28 Feb 2024 23:56:01 +0100 Subject: [PATCH 02/76] improve flow control handling, reduce redundant code, immediately return sessions and connections to the pool. --- .gitignore | 2 + DEBUG.md | 453 ----------------------------------- amqpx.go | 1 - amqpx_test.go | 2 +- helpers_test.go | 2 +- pool/connection.go | 98 ++------ pool/connection_pool.go | 109 ++++++--- pool/connection_pool_test.go | 4 +- pool/errors.go | 6 +- pool/flag.go | 7 + pool/helpers_context.go | 21 +- pool/pool.go | 4 +- pool/pool_test.go | 2 +- pool/publisher.go | 34 +-- pool/publisher_test.go | 15 +- pool/session.go | 52 +++- pool/session_pool.go | 105 ++++---- pool/session_pool_test.go | 12 +- pool/session_test.go | 10 +- pool/subscriber.go | 23 +- pool/subscriber_test.go | 6 +- pool/topologer.go | 66 +---- pool/utils.go | 11 +- 23 files changed, 309 insertions(+), 736 deletions(-) delete mode 100644 DEBUG.md create mode 100644 pool/flag.go diff --git a/.gitignore b/.gitignore index 1f84f06..ba0327e 100644 --- a/.gitignore +++ b/.gitignore @@ -2,3 +2,5 @@ *.test *trace* coverage.txt +DEBUG.md +debug.md diff --git a/DEBUG.md b/DEBUG.md deleted file mode 100644 index 89d8e25..0000000 --- a/DEBUG.md +++ /dev/null @@ -1,453 +0,0 @@ -=== RUN TestHandlerPauseAndResume - amqpx_test.go:430: - Error Trace: /home/behm015/Development/amqpx/amqpx_test.go:430 - /home/behm015/Development/amqpx/pool/subscriber.go:287 - /home/behm015/Development/amqpx/pool/subscriber.go:222 - /usr/local/go/src/runtime/asm_amd64.s:1650 - Error: Not equal: - expected: false - actual : true - Test: TestHandlerPauseAndResume - Messages: expected active to be false - - - -panic: test timed out after 10m0s -running tests: - TestBatchHandlerPauseAndResume (8m49s) - -goroutine 316 [running]: -testing.(*M).startAlarm.func1() - /usr/local/go/src/testing/testing.go:2259 +0x1fc -created by time.goFunc - /usr/local/go/src/time/sleep.go:176 +0x45 - -goroutine 1 [chan receive, 8 minutes]: -testing.(*T).Run(0xc000082ea0, {0x89c94f, 0x1e}, 0x8c4010) - /usr/local/go/src/testing/testing.go:1649 +0x856 -testing.runTests.func1(0x0?) - /usr/local/go/src/testing/testing.go:2054 +0x85 -testing.tRunner(0xc000082ea0, 0xc0000f9908) - /usr/local/go/src/testing/testing.go:1595 +0x239 -testing.runTests(0xc00009abe0?, {0xb33520, 0x9, 0x9}, {0x4a8459?, 0x4a9c31?, 0xb397c0?}) - /usr/local/go/src/testing/testing.go:2052 +0x897 -testing.(*M).Run(0xc00009abe0) - /usr/local/go/src/testing/testing.go:1925 +0xb58 -go.uber.org/goleak.VerifyTestMain({0x929a20, 0xc00009abe0}, {0xc0000f9e18, 0x3, 0x3}) - /home/behm015/go/pkg/mod/go.uber.org/goleak@v1.3.0/testmain.go:53 +0x65 -github.com/jxsl13/amqpx_test.TestMain(0xfdfb0802185865db?) - /home/behm015/Development/amqpx/amqpx_test.go:24 +0x2e9 -main.main() - _testmain.go:67 +0x308 - -goroutine 170 [semacquire, 7 minutes]: -sync.runtime_Semacquire(0xc0001287e8?) - /usr/local/go/src/runtime/sema.go:62 +0x25 -sync.(*WaitGroup).Wait(0xc0001287e0) - /usr/local/go/src/sync/waitgroup.go:116 +0xa5 -github.com/jxsl13/amqpx/pool.(*Subscriber).Close(0xc000128780) - /home/behm015/Development/amqpx/pool/subscriber.go:35 +0x12a -github.com/jxsl13/amqpx.(*AMQPX).Close.(*AMQPX).close.func1() - /home/behm015/Development/amqpx/amqpx.go:236 +0x96 -sync.(*Once).doSlow(0xb398d4, 0xc0000ad888) - /usr/local/go/src/sync/once.go:74 +0xf1 -sync.(*Once).Do(0xb398d4, 0xc0000ad878?) - /usr/local/go/src/sync/once.go:65 +0x45 -github.com/jxsl13/amqpx.(*AMQPX).close(...) - /home/behm015/Development/amqpx/amqpx.go:233 -github.com/jxsl13/amqpx.(*AMQPX).Close(0xb39840) - /home/behm015/Development/amqpx/amqpx.go:229 +0xf2 -github.com/jxsl13/amqpx.Close(...) - /home/behm015/Development/amqpx/amqpx.go:351 -github.com/jxsl13/amqpx_test.testBatchHandlerPauseAndResume(0xc00029a4e0) - /home/behm015/Development/amqpx/amqpx_test.go:751 +0x1331 -github.com/jxsl13/amqpx_test.TestBatchHandlerPauseAndResume(0x0?) - /home/behm015/Development/amqpx/amqpx_test.go:539 +0x31 -testing.tRunner(0xc00029a4e0, 0x8c4010) - /usr/local/go/src/testing/testing.go:1595 +0x239 -created by testing.(*T).Run in goroutine 1 - /usr/local/go/src/testing/testing.go:1648 +0x82b - -goroutine 34 [syscall, 9 minutes]: -os/signal.signal_recv() - /usr/local/go/src/runtime/sigqueue.go:152 +0x29 -os/signal.loop() - /usr/local/go/src/os/signal/signal_unix.go:23 +0x1d -created by os/signal.Notify.func1.1 in goroutine 19 - /usr/local/go/src/os/signal/signal.go:151 +0x47 - -goroutine 33 [select]: -github.com/rabbitmq/amqp091-go.(*Connection).heartbeater(0xc00017e6e0, 0x1bf08eb00, 0xc0002d82a0) - /home/behm015/go/pkg/mod/github.com/rabbitmq/amqp091-go@v1.9.0/connection.go:761 +0x26d -created by github.com/rabbitmq/amqp091-go.(*Connection).openTune in goroutine 19 - /home/behm015/go/pkg/mod/github.com/rabbitmq/amqp091-go@v1.9.0/connection.go:1016 +0xb8c - -goroutine 291 [IO wait]: -internal/poll.runtime_pollWait(0x7f8b9ce04e68, 0x72) - /usr/local/go/src/runtime/netpoll.go:343 +0x85 -internal/poll.(*pollDesc).wait(0xc00014a4a0, 0xc000495000?, 0x0) - /usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0xb1 -internal/poll.(*pollDesc).waitRead(...) - /usr/local/go/src/internal/poll/fd_poll_runtime.go:89 -internal/poll.(*FD).Read(0xc00014a480, {0xc000495000, 0x1000, 0x1000}) - /usr/local/go/src/internal/poll/fd_unix.go:164 +0x3e5 -net.(*netFD).Read(0xc00014a480, {0xc000495000, 0x1000, 0x1000}) - /usr/local/go/src/net/fd_posix.go:55 +0x4b -net.(*conn).Read(0xc0002da170, {0xc000495000, 0x1000, 0x1000}) - /usr/local/go/src/net/net.go:179 +0xad -bufio.(*Reader).Read(0xc0002d95c0, {0xc00023a159, 0x7, 0x7}) - /usr/local/go/src/bufio/bufio.go:244 +0x4be -io.ReadAtLeast({0x929c40, 0xc0002d95c0}, {0xc00023a159, 0x7, 0x7}, 0x7) - /usr/local/go/src/io/io.go:335 +0xd0 -io.ReadFull(...) - /usr/local/go/src/io/io.go:354 -github.com/rabbitmq/amqp091-go.(*reader).ReadFrame(0xc000165f18) - /home/behm015/go/pkg/mod/github.com/rabbitmq/amqp091-go@v1.9.0/read.go:49 +0x98 -github.com/rabbitmq/amqp091-go.(*Connection).reader(0xc0000566e0, {0x929fa0?, 0xc0002da170}) - /home/behm015/go/pkg/mod/github.com/rabbitmq/amqp091-go@v1.9.0/connection.go:726 +0x2ab -created by github.com/rabbitmq/amqp091-go.Open in goroutine 170 - /home/behm015/go/pkg/mod/github.com/rabbitmq/amqp091-go@v1.9.0/connection.go:271 +0x67a - -goroutine 41 [IO wait]: -internal/poll.runtime_pollWait(0x7f8b9ce04d70, 0x72) - /usr/local/go/src/runtime/netpoll.go:343 +0x85 -internal/poll.(*pollDesc).wait(0xc0001283a0, 0xc00029f000?, 0x0) - /usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0xb1 -internal/poll.(*pollDesc).waitRead(...) - /usr/local/go/src/internal/poll/fd_poll_runtime.go:89 -internal/poll.(*FD).Read(0xc000128380, {0xc00029f000, 0x1000, 0x1000}) - /usr/local/go/src/internal/poll/fd_unix.go:164 +0x3e5 -net.(*netFD).Read(0xc000128380, {0xc00029f000, 0x1000, 0x1000}) - /usr/local/go/src/net/fd_posix.go:55 +0x4b -net.(*conn).Read(0xc000158090, {0xc00029f000, 0x1000, 0x1000}) - /usr/local/go/src/net/net.go:179 +0xad -bufio.(*Reader).Read(0xc00014c780, {0xc00013c259, 0x7, 0x7}) - /usr/local/go/src/bufio/bufio.go:244 +0x4be -io.ReadAtLeast({0x929c40, 0xc00014c780}, {0xc00013c259, 0x7, 0x7}, 0x7) - /usr/local/go/src/io/io.go:335 +0xd0 -io.ReadFull(...) - /usr/local/go/src/io/io.go:354 -github.com/rabbitmq/amqp091-go.(*reader).ReadFrame(0xc0000a9f18) - /home/behm015/go/pkg/mod/github.com/rabbitmq/amqp091-go@v1.9.0/read.go:49 +0x98 -github.com/rabbitmq/amqp091-go.(*Connection).reader(0xc00017e2c0, {0x929fa0?, 0xc000158090}) - /home/behm015/go/pkg/mod/github.com/rabbitmq/amqp091-go@v1.9.0/connection.go:726 +0x2ab -created by github.com/rabbitmq/amqp091-go.Open in goroutine 19 - /home/behm015/go/pkg/mod/github.com/rabbitmq/amqp091-go@v1.9.0/connection.go:271 +0x67a - -goroutine 21 [select]: -github.com/rabbitmq/amqp091-go.(*Connection).heartbeater(0xc00017e2c0, 0x1bf08eb00, 0xc00008e660) - /home/behm015/go/pkg/mod/github.com/rabbitmq/amqp091-go@v1.9.0/connection.go:761 +0x26d -created by github.com/rabbitmq/amqp091-go.(*Connection).openTune in goroutine 19 - /home/behm015/go/pkg/mod/github.com/rabbitmq/amqp091-go@v1.9.0/connection.go:1016 +0xb8c - -goroutine 58 [IO wait]: -internal/poll.runtime_pollWait(0x7f8b9ce04c78, 0x72) - /usr/local/go/src/runtime/netpoll.go:343 +0x85 -internal/poll.(*pollDesc).wait(0xc00034c120, 0xc000365000?, 0x0) - /usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0xb1 -internal/poll.(*pollDesc).waitRead(...) - /usr/local/go/src/internal/poll/fd_poll_runtime.go:89 -internal/poll.(*FD).Read(0xc00034c100, {0xc000365000, 0x1000, 0x1000}) - /usr/local/go/src/internal/poll/fd_unix.go:164 +0x3e5 -net.(*netFD).Read(0xc00034c100, {0xc000365000, 0x1000, 0x1000}) - /usr/local/go/src/net/fd_posix.go:55 +0x4b -net.(*conn).Read(0xc000346030, {0xc000365000, 0x1000, 0x1000}) - /usr/local/go/src/net/net.go:179 +0xad -bufio.(*Reader).Read(0xc000344240, {0xc0003cc6d9, 0x7, 0x7}) - /usr/local/go/src/bufio/bufio.go:244 +0x4be -io.ReadAtLeast({0x929c40, 0xc000344240}, {0xc0003cc6d9, 0x7, 0x7}, 0x7) - /usr/local/go/src/io/io.go:335 +0xd0 -io.ReadFull(...) - /usr/local/go/src/io/io.go:354 -github.com/rabbitmq/amqp091-go.(*reader).ReadFrame(0xc00016bf18) - /home/behm015/go/pkg/mod/github.com/rabbitmq/amqp091-go@v1.9.0/read.go:49 +0x98 -github.com/rabbitmq/amqp091-go.(*Connection).reader(0xc00017e160, {0x929fa0?, 0xc000346030}) - /home/behm015/go/pkg/mod/github.com/rabbitmq/amqp091-go@v1.9.0/connection.go:726 +0x2ab -created by github.com/rabbitmq/amqp091-go.Open in goroutine 19 - /home/behm015/go/pkg/mod/github.com/rabbitmq/amqp091-go@v1.9.0/connection.go:271 +0x67a - -goroutine 137 [select]: -github.com/rabbitmq/amqp091-go.(*Connection).heartbeater(0xc000056000, 0x1bf08eb00, 0xc00008e420) - /home/behm015/go/pkg/mod/github.com/rabbitmq/amqp091-go@v1.9.0/connection.go:761 +0x26d -created by github.com/rabbitmq/amqp091-go.(*Connection).openTune in goroutine 19 - /home/behm015/go/pkg/mod/github.com/rabbitmq/amqp091-go@v1.9.0/connection.go:1016 +0xb8c - -goroutine 238 [select]: -github.com/rabbitmq/amqp091-go.(*Connection).heartbeater(0xc000056840, 0x1bf08eb00, 0xc00008f3e0) - /home/behm015/go/pkg/mod/github.com/rabbitmq/amqp091-go@v1.9.0/connection.go:761 +0x26d -created by github.com/rabbitmq/amqp091-go.(*Connection).openTune in goroutine 170 - /home/behm015/go/pkg/mod/github.com/rabbitmq/amqp091-go@v1.9.0/connection.go:1016 +0xb8c - -goroutine 198 [select]: -github.com/rabbitmq/amqp091-go.(*Connection).heartbeater(0xc000056160, 0x1bf08eb00, 0xc000183b60) - /home/behm015/go/pkg/mod/github.com/rabbitmq/amqp091-go@v1.9.0/connection.go:761 +0x26d -created by github.com/rabbitmq/amqp091-go.(*Connection).openTune in goroutine 19 - /home/behm015/go/pkg/mod/github.com/rabbitmq/amqp091-go@v1.9.0/connection.go:1016 +0xb8c - -goroutine 162 [IO wait]: -internal/poll.runtime_pollWait(0x7f8b9ce04a88, 0x72) - /usr/local/go/src/runtime/netpoll.go:343 +0x85 -internal/poll.(*pollDesc).wait(0xc000002120, 0xc0000d8000?, 0x0) - /usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0xb1 -internal/poll.(*pollDesc).waitRead(...) - /usr/local/go/src/internal/poll/fd_poll_runtime.go:89 -internal/poll.(*FD).Read(0xc000002100, {0xc0000d8000, 0x1000, 0x1000}) - /usr/local/go/src/internal/poll/fd_unix.go:164 +0x3e5 -net.(*netFD).Read(0xc000002100, {0xc0000d8000, 0x1000, 0x1000}) - /usr/local/go/src/net/fd_posix.go:55 +0x4b -net.(*conn).Read(0xc000346048, {0xc0000d8000, 0x1000, 0x1000}) - /usr/local/go/src/net/net.go:179 +0xad -bufio.(*Reader).Read(0xc00008e240, {0xc0004db449, 0x7, 0x7}) - /usr/local/go/src/bufio/bufio.go:244 +0x4be -io.ReadAtLeast({0x929c40, 0xc00008e240}, {0xc0004db449, 0x7, 0x7}, 0x7) - /usr/local/go/src/io/io.go:335 +0xd0 -io.ReadFull(...) - /usr/local/go/src/io/io.go:354 -github.com/rabbitmq/amqp091-go.(*reader).ReadFrame(0xc0003ddf18) - /home/behm015/go/pkg/mod/github.com/rabbitmq/amqp091-go@v1.9.0/read.go:49 +0x98 -github.com/rabbitmq/amqp091-go.(*Connection).reader(0xc000056000, {0x929fa0?, 0xc000346048}) - /home/behm015/go/pkg/mod/github.com/rabbitmq/amqp091-go@v1.9.0/connection.go:726 +0x2ab -created by github.com/rabbitmq/amqp091-go.Open in goroutine 19 - /home/behm015/go/pkg/mod/github.com/rabbitmq/amqp091-go@v1.9.0/connection.go:271 +0x67a - -goroutine 294 [IO wait]: -internal/poll.runtime_pollWait(0x7f8b9ce04898, 0x72) - /usr/local/go/src/runtime/netpoll.go:343 +0x85 -internal/poll.(*pollDesc).wait(0xc00014a620, 0xc0001c5000?, 0x0) - /usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0xb1 -internal/poll.(*pollDesc).waitRead(...) - /usr/local/go/src/internal/poll/fd_poll_runtime.go:89 -internal/poll.(*FD).Read(0xc00014a600, {0xc0001c5000, 0x1000, 0x1000}) - /usr/local/go/src/internal/poll/fd_unix.go:164 +0x3e5 -net.(*netFD).Read(0xc00014a600, {0xc0001c5000, 0x1000, 0x1000}) - /usr/local/go/src/net/fd_posix.go:55 +0x4b -net.(*conn).Read(0xc0002da1d0, {0xc0001c5000, 0x1000, 0x1000}) - /usr/local/go/src/net/net.go:179 +0xad -bufio.(*Reader).Read(0xc0002d98c0, {0xc0003cc459, 0x7, 0x7}) - /usr/local/go/src/bufio/bufio.go:244 +0x4be -io.ReadAtLeast({0x929c40, 0xc0002d98c0}, {0xc0003cc459, 0x7, 0x7}, 0x7) - /usr/local/go/src/io/io.go:335 +0xd0 -io.ReadFull(...) - /usr/local/go/src/io/io.go:354 -github.com/rabbitmq/amqp091-go.(*reader).ReadFrame(0xc000169f18) - /home/behm015/go/pkg/mod/github.com/rabbitmq/amqp091-go@v1.9.0/read.go:49 +0x98 -github.com/rabbitmq/amqp091-go.(*Connection).reader(0xc000056840, {0x929fa0?, 0xc0002da1d0}) - /home/behm015/go/pkg/mod/github.com/rabbitmq/amqp091-go@v1.9.0/connection.go:726 +0x2ab -created by github.com/rabbitmq/amqp091-go.Open in goroutine 170 - /home/behm015/go/pkg/mod/github.com/rabbitmq/amqp091-go@v1.9.0/connection.go:271 +0x67a - -goroutine 77 [select]: -github.com/rabbitmq/amqp091-go.(*Connection).heartbeater(0xc00017e160, 0x1bf08eb00, 0xc000183200) - /home/behm015/go/pkg/mod/github.com/rabbitmq/amqp091-go@v1.9.0/connection.go:761 +0x26d -created by github.com/rabbitmq/amqp091-go.(*Connection).openTune in goroutine 19 - /home/behm015/go/pkg/mod/github.com/rabbitmq/amqp091-go@v1.9.0/connection.go:1016 +0xb8c - -goroutine 123 [IO wait]: -internal/poll.runtime_pollWait(0x7f8b9ce04b80, 0x72) - /usr/local/go/src/runtime/netpoll.go:343 +0x85 -internal/poll.(*pollDesc).wait(0xc000128520, 0xc00033d000?, 0x0) - /usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0xb1 -internal/poll.(*pollDesc).waitRead(...) - /usr/local/go/src/internal/poll/fd_poll_runtime.go:89 -internal/poll.(*FD).Read(0xc000128500, {0xc00033d000, 0x1000, 0x1000}) - /usr/local/go/src/internal/poll/fd_unix.go:164 +0x3e5 -net.(*netFD).Read(0xc000128500, {0xc00033d000, 0x1000, 0x1000}) - /usr/local/go/src/net/fd_posix.go:55 +0x4b -net.(*conn).Read(0xc00017c110, {0xc00033d000, 0x1000, 0x1000}) - /usr/local/go/src/net/net.go:179 +0xad -bufio.(*Reader).Read(0xc000182e40, {0xc00052b5d9, 0x7, 0x7}) - /usr/local/go/src/bufio/bufio.go:244 +0x4be -io.ReadAtLeast({0x929c40, 0xc000182e40}, {0xc00052b5d9, 0x7, 0x7}, 0x7) - /usr/local/go/src/io/io.go:335 +0xd0 -io.ReadFull(...) - /usr/local/go/src/io/io.go:354 -github.com/rabbitmq/amqp091-go.(*reader).ReadFrame(0xc0003d9f18) - /home/behm015/go/pkg/mod/github.com/rabbitmq/amqp091-go@v1.9.0/read.go:49 +0x98 -github.com/rabbitmq/amqp091-go.(*Connection).reader(0xc00017e6e0, {0x929fa0?, 0xc00017c110}) - /home/behm015/go/pkg/mod/github.com/rabbitmq/amqp091-go@v1.9.0/connection.go:726 +0x2ab -created by github.com/rabbitmq/amqp091-go.Open in goroutine 19 - /home/behm015/go/pkg/mod/github.com/rabbitmq/amqp091-go@v1.9.0/connection.go:271 +0x67a - -goroutine 212 [IO wait]: -internal/poll.runtime_pollWait(0x7f8b9ce04990, 0x72) - /usr/local/go/src/runtime/netpoll.go:343 +0x85 -internal/poll.(*pollDesc).wait(0xc000128620, 0xc000241000?, 0x0) - /usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0xb1 -internal/poll.(*pollDesc).waitRead(...) - /usr/local/go/src/internal/poll/fd_poll_runtime.go:89 -internal/poll.(*FD).Read(0xc000128600, {0xc000241000, 0x1000, 0x1000}) - /usr/local/go/src/internal/poll/fd_unix.go:164 +0x3e5 -net.(*netFD).Read(0xc000128600, {0xc000241000, 0x1000, 0x1000}) - /usr/local/go/src/net/fd_posix.go:55 +0x4b -net.(*conn).Read(0xc0003461b8, {0xc000241000, 0x1000, 0x1000}) - /usr/local/go/src/net/net.go:179 +0xad -bufio.(*Reader).Read(0xc00014cd80, {0xc000431469, 0x7, 0x7}) - /usr/local/go/src/bufio/bufio.go:244 +0x4be -io.ReadAtLeast({0x929c40, 0xc00014cd80}, {0xc000431469, 0x7, 0x7}, 0x7) - /usr/local/go/src/io/io.go:335 +0xd0 -io.ReadFull(...) - /usr/local/go/src/io/io.go:354 -github.com/rabbitmq/amqp091-go.(*reader).ReadFrame(0xc0003e3f18) - /home/behm015/go/pkg/mod/github.com/rabbitmq/amqp091-go@v1.9.0/read.go:49 +0x98 -github.com/rabbitmq/amqp091-go.(*Connection).reader(0xc000056160, {0x929fa0?, 0xc0003461b8}) - /home/behm015/go/pkg/mod/github.com/rabbitmq/amqp091-go@v1.9.0/connection.go:726 +0x2ab -created by github.com/rabbitmq/amqp091-go.Open in goroutine 19 - /home/behm015/go/pkg/mod/github.com/rabbitmq/amqp091-go@v1.9.0/connection.go:271 +0x67a - -goroutine 308 [sync.Mutex.Lock]: -sync.runtime_SemacquireMutex(0x929960?, 0x0?, 0xc0004252c0?) - /usr/local/go/src/runtime/sema.go:77 +0x25 -sync.(*Mutex).lockSlow(0xc0003c81d8) - /usr/local/go/src/sync/mutex.go:171 +0x213 -sync.(*Mutex).Lock(0xc0003c81d8) - /usr/local/go/src/sync/mutex.go:90 +0x55 -github.com/jxsl13/amqpx/pool.(*Connection).Recover(0xc0003c8160) - /home/behm015/Development/amqpx/pool/connection.go:300 +0x48 -github.com/jxsl13/amqpx/pool.(*Session).recover(0xc00014a100) - /home/behm015/Development/amqpx/pool/session.go:211 +0x46 -github.com/jxsl13/amqpx/pool.(*Session).Recover(0xc00014a100) - /home/behm015/Development/amqpx/pool/session.go:193 +0x89 -github.com/jxsl13/amqpx/pool.(*SessionPool).ReturnSession(0xc00014d260, 0xc00014a100, 0x1) - /home/behm015/Development/amqpx/pool/session_pool.go:155 +0x77 -github.com/jxsl13/amqpx/pool.(*Pool).ReturnSession(...) - /home/behm015/Development/amqpx/pool/pool.go:107 -github.com/jxsl13/amqpx/pool.(*Subscriber).batchConsume.func1() - /home/behm015/Development/amqpx/pool/subscriber.go:393 +0x22c -github.com/jxsl13/amqpx/pool.(*Subscriber).batchConsume(0xc000128780, 0xc00014c240) - /home/behm015/Development/amqpx/pool/subscriber.go:409 +0x6dc -github.com/jxsl13/amqpx/pool.(*Subscriber).batchConsumer(0xc000128780, 0xc00014c240, 0xc0001287e0) - /home/behm015/Development/amqpx/pool/subscriber.go:359 +0x3e5 -created by github.com/jxsl13/amqpx/pool.(*Subscriber).Start in goroutine 170 - /home/behm015/Development/amqpx/pool/subscriber.go:200 +0x5cf - -goroutine 223 [select]: -github.com/rabbitmq/amqp091-go.(*Connection).heartbeater(0xc0000562c0, 0x1bf08eb00, 0xc0002d8480) - /home/behm015/go/pkg/mod/github.com/rabbitmq/amqp091-go@v1.9.0/connection.go:761 +0x26d -created by github.com/rabbitmq/amqp091-go.(*Connection).openTune in goroutine 170 - /home/behm015/go/pkg/mod/github.com/rabbitmq/amqp091-go@v1.9.0/connection.go:1016 +0xb8c - -goroutine 278 [select]: -github.com/rabbitmq/amqp091-go.(*Connection).heartbeater(0xc0000566e0, 0x1bf08eb00, 0xc0001820c0) - /home/behm015/go/pkg/mod/github.com/rabbitmq/amqp091-go@v1.9.0/connection.go:761 +0x26d -created by github.com/rabbitmq/amqp091-go.(*Connection).openTune in goroutine 170 - /home/behm015/go/pkg/mod/github.com/rabbitmq/amqp091-go@v1.9.0/connection.go:1016 +0xb8c - -goroutine 306 [IO wait]: -internal/poll.runtime_pollWait(0x7f8b9ce046a8, 0x72) - /usr/local/go/src/runtime/netpoll.go:343 +0x85 -internal/poll.(*pollDesc).wait(0xc0001285a0, 0xc000262000?, 0x0) - /usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0xb1 -internal/poll.(*pollDesc).waitRead(...) - /usr/local/go/src/internal/poll/fd_poll_runtime.go:89 -internal/poll.(*FD).Read(0xc000128580, {0xc000262000, 0x1000, 0x1000}) - /usr/local/go/src/internal/poll/fd_unix.go:164 +0x3e5 -net.(*netFD).Read(0xc000128580, {0xc000262000, 0x1000, 0x1000}) - /usr/local/go/src/net/fd_posix.go:55 +0x4b -net.(*conn).Read(0xc0002da0d0, {0xc000262000, 0x1000, 0x1000}) - /usr/local/go/src/net/net.go:179 +0xad -bufio.(*Reader).Read(0xc00014cf00, {0xc0003cc819, 0x7, 0x7}) - /usr/local/go/src/bufio/bufio.go:244 +0x4be -io.ReadAtLeast({0x929c40, 0xc00014cf00}, {0xc0003cc819, 0x7, 0x7}, 0x7) - /usr/local/go/src/io/io.go:335 +0xd0 -io.ReadFull(...) - /usr/local/go/src/io/io.go:354 -github.com/rabbitmq/amqp091-go.(*reader).ReadFrame(0xc0003dff18) - /home/behm015/go/pkg/mod/github.com/rabbitmq/amqp091-go@v1.9.0/read.go:49 +0x98 -github.com/rabbitmq/amqp091-go.(*Connection).reader(0xc000056580, {0x929fa0?, 0xc0002da0d0}) - /home/behm015/go/pkg/mod/github.com/rabbitmq/amqp091-go@v1.9.0/connection.go:726 +0x2ab -created by github.com/rabbitmq/amqp091-go.Open in goroutine 170 - /home/behm015/go/pkg/mod/github.com/rabbitmq/amqp091-go@v1.9.0/connection.go:271 +0x67a - -goroutine 255 [IO wait]: -internal/poll.runtime_pollWait(0x7f8b9ce047a0, 0x72) - /usr/local/go/src/runtime/netpoll.go:343 +0x85 -internal/poll.(*pollDesc).wait(0xc000128120, 0xc0003ea000?, 0x0) - /usr/local/go/src/internal/poll/fd_poll_runtime.go:84 +0xb1 -internal/poll.(*pollDesc).waitRead(...) - /usr/local/go/src/internal/poll/fd_poll_runtime.go:89 -internal/poll.(*FD).Read(0xc000128100, {0xc0003ea000, 0x1000, 0x1000}) - /usr/local/go/src/internal/poll/fd_unix.go:164 +0x3e5 -net.(*netFD).Read(0xc000128100, {0xc0003ea000, 0x1000, 0x1000}) - /usr/local/go/src/net/fd_posix.go:55 +0x4b -net.(*conn).Read(0xc0002da030, {0xc0003ea000, 0x1000, 0x1000}) - /usr/local/go/src/net/net.go:179 +0xad -bufio.(*Reader).Read(0xc0002d8420, {0xc00046d2a9, 0x7, 0x7}) - /usr/local/go/src/bufio/bufio.go:244 +0x4be -io.ReadAtLeast({0x929c40, 0xc0002d8420}, {0xc00046d2a9, 0x7, 0x7}, 0x7) - /usr/local/go/src/io/io.go:335 +0xd0 -io.ReadFull(...) - /usr/local/go/src/io/io.go:354 -github.com/rabbitmq/amqp091-go.(*reader).ReadFrame(0xc0000abf18) - /home/behm015/go/pkg/mod/github.com/rabbitmq/amqp091-go@v1.9.0/read.go:49 +0x98 -github.com/rabbitmq/amqp091-go.(*Connection).reader(0xc0000562c0, {0x929fa0?, 0xc0002da030}) - /home/behm015/go/pkg/mod/github.com/rabbitmq/amqp091-go@v1.9.0/connection.go:726 +0x2ab -created by github.com/rabbitmq/amqp091-go.Open in goroutine 170 - /home/behm015/go/pkg/mod/github.com/rabbitmq/amqp091-go@v1.9.0/connection.go:271 +0x67a - -goroutine 307 [select]: -github.com/rabbitmq/amqp091-go.(*Connection).heartbeater(0xc000056580, 0x1bf08eb00, 0xc000345da0) - /home/behm015/go/pkg/mod/github.com/rabbitmq/amqp091-go@v1.9.0/connection.go:761 +0x26d -created by github.com/rabbitmq/amqp091-go.(*Connection).openTune in goroutine 170 - /home/behm015/go/pkg/mod/github.com/rabbitmq/amqp091-go@v1.9.0/connection.go:1016 +0xb8c - -goroutine 310 [sync.Mutex.Lock]: -sync.runtime_SemacquireMutex(0x4b3cce?, 0xd8?, 0x4a9c69?) - /usr/local/go/src/runtime/sema.go:77 +0x25 -sync.(*Mutex).lockSlow(0xc0003c81d8) - /usr/local/go/src/sync/mutex.go:171 +0x213 -sync.(*Mutex).Lock(0xc0003c81d8) - /usr/local/go/src/sync/mutex.go:90 +0x55 -github.com/jxsl13/amqpx/pool.(*Connection).channel(0xc0003c8160) - /home/behm015/Development/amqpx/pool/connection.go:356 +0x59 -github.com/jxsl13/amqpx/pool.(*Session).connect(0xc000128700) - /home/behm015/Development/amqpx/pool/session.go:164 +0x156 -github.com/jxsl13/amqpx/pool.(*Session).recover(0xc000128700) - /home/behm015/Development/amqpx/pool/session.go:219 +0x55 -github.com/jxsl13/amqpx/pool.(*Session).Recover(0xc000128700) - /home/behm015/Development/amqpx/pool/session.go:193 +0x89 -github.com/jxsl13/amqpx/pool.(*SessionPool).ReturnSession(0xc00014d260, 0xc000128700, 0x1) - /home/behm015/Development/amqpx/pool/session_pool.go:155 +0x77 -github.com/jxsl13/amqpx/pool.(*Pool).ReturnSession(...) - /home/behm015/Development/amqpx/pool/pool.go:107 -github.com/jxsl13/amqpx/pool.(*Subscriber).batchConsume.func1() - /home/behm015/Development/amqpx/pool/subscriber.go:393 +0x22c -github.com/jxsl13/amqpx/pool.(*Subscriber).batchConsume(0xc000128780, 0xc00014c300) - /home/behm015/Development/amqpx/pool/subscriber.go:409 +0x6dc -github.com/jxsl13/amqpx/pool.(*Subscriber).batchConsumer(0xc000128780, 0xc00014c300, 0xc0001287e0) - /home/behm015/Development/amqpx/pool/subscriber.go:359 +0x3e5 -created by github.com/jxsl13/amqpx/pool.(*Subscriber).Start in goroutine 170 - /home/behm015/Development/amqpx/pool/subscriber.go:200 +0x5cf - -goroutine 309 [runnable]: -github.com/rabbitmq/amqp091-go.(*allocator).reserve(0xc000505fa0, 0x6d0) - /home/behm015/go/pkg/mod/github.com/rabbitmq/amqp091-go@v1.9.0/allocator.go:103 +0x105 -github.com/rabbitmq/amqp091-go.(*allocator).next(0xc000505fa0) - /home/behm015/go/pkg/mod/github.com/rabbitmq/amqp091-go@v1.9.0/allocator.go:84 +0x176 -github.com/rabbitmq/amqp091-go.(*Connection).allocateChannel(0xc000056580) - /home/behm015/go/pkg/mod/github.com/rabbitmq/amqp091-go@v1.9.0/connection.go:819 +0xf6 -github.com/rabbitmq/amqp091-go.(*Connection).openChannel(0xc0003c81d8?) - /home/behm015/go/pkg/mod/github.com/rabbitmq/amqp091-go@v1.9.0/connection.go:847 +0x33 -github.com/rabbitmq/amqp091-go.(*Connection).Channel(...) - /home/behm015/go/pkg/mod/github.com/rabbitmq/amqp091-go@v1.9.0/connection.go:873 -github.com/jxsl13/amqpx/pool.(*Connection).channel(0xc0003c8160) - /home/behm015/Development/amqpx/pool/connection.go:358 +0xb6 -github.com/jxsl13/amqpx/pool.(*Session).connect(0xc00014a180) - /home/behm015/Development/amqpx/pool/session.go:164 +0x156 -github.com/jxsl13/amqpx/pool.(*Session).recover(0xc00014a180) - /home/behm015/Development/amqpx/pool/session.go:219 +0x55 -github.com/jxsl13/amqpx/pool.(*Session).Recover(0xc00014a180) - /home/behm015/Development/amqpx/pool/session.go:193 +0x89 -github.com/jxsl13/amqpx/pool.(*SessionPool).ReturnSession(0xc00014d260, 0xc00014a180, 0x1) - /home/behm015/Development/amqpx/pool/session_pool.go:155 +0x77 -github.com/jxsl13/amqpx/pool.(*Pool).ReturnSession(...) - /home/behm015/Development/amqpx/pool/pool.go:107 -github.com/jxsl13/amqpx/pool.(*Subscriber).batchConsume.func1() - /home/behm015/Development/amqpx/pool/subscriber.go:393 +0x22c -github.com/jxsl13/amqpx/pool.(*Subscriber).batchConsume(0xc000128780, 0xc00014c2a0) - /home/behm015/Development/amqpx/pool/subscriber.go:409 +0x6dc -github.com/jxsl13/amqpx/pool.(*Subscriber).batchConsumer(0xc000128780, 0xc00014c2a0, 0xc0001287e0) - /home/behm015/Development/amqpx/pool/subscriber.go:359 +0x3e5 -created by github.com/jxsl13/amqpx/pool.(*Subscriber).Start in goroutine 170 - /home/behm015/Development/amqpx/pool/subscriber.go:200 +0x5cf -FAIL github.com/jxsl13/amqpx 600.024s -FAIL \ No newline at end of file diff --git a/amqpx.go b/amqpx.go index abcc15d..037ed15 100644 --- a/amqpx.go +++ b/amqpx.go @@ -202,7 +202,6 @@ func (a *AMQPX) Start(ctx context.Context, connectUrl string, options ...Option) return } } - } // publisher must before subscribers, as subscriber handlers might be using the publisher. diff --git a/amqpx_test.go b/amqpx_test.go index 53cac2c..1b6d95b 100644 --- a/amqpx_test.go +++ b/amqpx_test.go @@ -21,7 +21,7 @@ var ( connectURL = amqpx.NewURL("localhost", 5672, "admin", "password") ) -// WARNING: Do not assert consumer counts, as those values are too flaky and break tests all over th eplace +// WARNING: Do not assert consumer counts, as those values are too flaky and break tests all over the place func TestMain(m *testing.M) { goleak.VerifyTestMain( m, diff --git a/helpers_test.go b/helpers_test.go index 84d345a..6f491fe 100644 --- a/helpers_test.go +++ b/helpers_test.go @@ -116,7 +116,7 @@ func createQueue(ctx context.Context, name string, t *pool.Topologer) (err error func deleteQueue(ctx context.Context, name string, t *pool.Topologer) (err error) { _, err = t.QueueDeclarePassive(ctx, name) if err != nil { - return fmt.Errorf("%q does not exist but is supposed to be deleted", name) + return fmt.Errorf("%q does not exist but is supposed to be deleted: %w", name, err) } _, err = t.QueueDelete(ctx, name) diff --git a/pool/connection.go b/pool/connection.go index f512fb0..b72e6df 100644 --- a/pool/connection.go +++ b/pool/connection.go @@ -84,7 +84,8 @@ func NewConnection(ctx context.Context, connectUrl, name string, options ...Conn // we derive a new context from the parent one in order to // be able to close it without affecting the parent - cCtx, cancel := context.WithCancel(option.Ctx) + cCtx, cc := context.WithCancelCause(option.Ctx) + cancel := toCancelFunc(fmt.Errorf("connection %w", ErrClosed), cc) conn := &Connection{ url: u.String(), @@ -213,78 +214,11 @@ func (ch *Connection) connect(ctx context.Context) error { return nil } -// PauseOnFlowControl allows you to wait and sleep while receiving flow control messages. -// Sleeps for one second, repeatedly until the blocking has stopped. -// Such messages will most likely be received when the broker hits its memory or disk limits. -// Returns an error in case that flow control was detected and is resolved or in case that the connection is closed. -func (ch *Connection) PauseOnFlowControl(ctx context.Context) error { +func (ch *Connection) FlowControl() <-chan amqp.Blocking { ch.mu.Lock() defer ch.mu.Unlock() - return ch.pauseOnFlowControl(ctx) -} - -// not threadsafe -func (ch *Connection) pauseOnFlowControl(ctx context.Context) error { - - if ch.isClosed() { - return ErrClosed - } - - var ( - duration = time.Second - timer = time.NewTimer(duration) - drained = false - ) - defer closeTimer(timer, &drained) - - var ( - flowControl = false - start = time.Now() - reason = "" - err error = nil - ) - - for !ch.isClosed() { - select { - case <-ch.catchShutdown(): - return ErrClosed - case <-ctx.Done(): - return fmt.Errorf("pause on flow control failed: %w", ctx.Err()) - case blocker, ok := <-ch.blocking: // Check for flow control issues. - if !ok { - return errFlowControlClosed - } - if !blocker.Active { - return nil - } - - if !flowControl || reason != blocker.Reason { - reason = blocker.Reason - flowControl = true - err = fmt.Errorf("%w: reason: %s", errFlowControl, reason) - ch.warn(err) - } - - resetTimer(timer, duration, &drained) - - select { - case <-ch.catchShutdown(): - return ErrClosed - case <-timer.C: - drained = true - continue - } - - default: - if flowControl { - ch.warnf("flow control with last reason: %s: ended after %s", reason, time.Since(start)) - } - return err - } - } - - return nil + return ch.blocking } func (ch *Connection) IsClosed() bool { @@ -321,7 +255,7 @@ func (ch *Connection) error() error { for { select { case <-ch.catchShutdown(): - return fmt.Errorf("connection %w", ErrClosed) + return ch.shutdownErr() case e, ok := <-ch.errors: if !ok { // because the amqp library might close this @@ -347,10 +281,20 @@ func (ch *Connection) Recover(ctx context.Context) error { } func (ch *Connection) recover(ctx context.Context) error { - healthy := !ch.flagged && ch.error() == nil - if healthy && !ch.isClosed() { - return ch.pauseOnFlowControl(ctx) + select { + case <-ctx.Done(): + return fmt.Errorf("connection recovery failed: %w", ctx.Err()) + case <-ch.catchShutdown(): + return fmt.Errorf("connection recovery failed: %w", ch.shutdownErr()) + default: + // try recovering after checking if contexts are still valid + } + + healthy := !ch.flagged && ch.error() == nil && !ch.isClosed() + + if healthy { + return nil } var ( @@ -384,7 +328,7 @@ func (ch *Connection) recover(ctx context.Context) error { select { case <-ch.catchShutdown(): // catch shutdown signal - return fmt.Errorf("connection recovery failed: connection %w", ErrClosed) + return fmt.Errorf("connection recovery failed: %w", ch.shutdownErr()) case <-ctx.Done(): // catch context cancelation return fmt.Errorf("connection recovery failed: %w", ctx.Err()) @@ -423,6 +367,10 @@ func (ch *Connection) catchShutdown() <-chan struct{} { return ch.ctx.Done() } +func (ch *Connection) shutdownErr() error { + return ch.ctx.Err() +} + func (ch *Connection) info(a ...any) { ch.log.WithField("connection", ch.Name()).Info(a...) } diff --git a/pool/connection_pool.go b/pool/connection_pool.go index 7bb4184..d3eafcf 100644 --- a/pool/connection_pool.go +++ b/pool/connection_pool.go @@ -6,7 +6,6 @@ import ( "fmt" "net/url" "sync" - "sync/atomic" "time" "github.com/jxsl13/amqpx/logging" @@ -25,10 +24,7 @@ type ConnectionPool struct { size int - tls *tls.Config - connections chan *Connection - - transientID int64 + tls *tls.Config ctx context.Context cancel context.CancelFunc @@ -36,6 +32,12 @@ type ConnectionPool struct { log logging.Logger recoverCB ConnectionRecoverCallback + + connections chan *Connection + + mu sync.Mutex + transientID int64 + concurrentTransient int } // NewConnectionPool creates a new connection pool which has a maximum size it @@ -80,7 +82,8 @@ func newConnectionPoolFromOption(connectUrl string, option connectionPoolOption) } // decouple from parent context, in case we want to close this context ourselves. - ctx, cancel := context.WithCancel(option.Ctx) + ctx, cc := context.WithCancelCause(option.Ctx) + cancel := toCancelFunc(fmt.Errorf("connection pool %w", ErrClosed), cc) cp := &ConnectionPool{ name: option.Name, @@ -119,7 +122,7 @@ func newConnectionPoolFromOption(connectUrl string, option connectionPoolOption) } func (cp *ConnectionPool) initCachedConns() error { - for id := 0; id < cp.size; id++ { + for id := int64(0); id < int64(cp.size); id++ { conn, err := cp.deriveConnection(cp.ctx, id, true) if err != nil { return fmt.Errorf("%w: %v", ErrPoolInitializationFailed, err) @@ -138,7 +141,7 @@ func (cp *ConnectionPool) initCachedConns() error { return nil } -func (cp *ConnectionPool) deriveConnection(ctx context.Context, id int, cached bool) (*Connection, error) { +func (cp *ConnectionPool) deriveConnection(ctx context.Context, id int64, cached bool) (*Connection, error) { var name string if cached { name = fmt.Sprintf("%s-cached-connection-%d", cp.name, id) @@ -158,29 +161,55 @@ func (cp *ConnectionPool) deriveConnection(ctx context.Context, id int, cached b // GetConnection only returns an error upon shutdown func (cp *ConnectionPool) GetConnection(ctx context.Context) (*Connection, error) { select { - case <-cp.catchShutdown(): - return nil, fmt.Errorf("connection pool %w", ErrClosed) - case <-ctx.Done(): - return nil, ctx.Err() case conn, ok := <-cp.connections: if !ok { return nil, fmt.Errorf("connection pool %w", ErrClosed) } - err := conn.Recover(ctx) - if err != nil { - return nil, fmt.Errorf("failed to get connection: %w", err) + if conn.IsFlagged() { + err := conn.Recover(ctx) + if err != nil { + return nil, fmt.Errorf("failed to get connection: %w", err) + } } return conn, nil + case <-ctx.Done(): + return nil, ctx.Err() + case <-cp.catchShutdown(): + return nil, fmt.Errorf("connection pool %w", ErrClosed) } } +func (cp *ConnectionPool) nextTransientID() int64 { + cp.mu.Lock() + defer cp.mu.Unlock() + id := cp.transientID + cp.transientID++ + return id +} + +func (cp *ConnectionPool) incTransient() { + cp.mu.Lock() + cp.concurrentTransient++ + cp.mu.Unlock() +} + +func (cp *ConnectionPool) decTransient() { + cp.mu.Lock() + cp.concurrentTransient-- + cp.mu.Unlock() +} + // GetTransientConnection may return an error when the context was cancelled before the connection could be obtained. // Transient connections may be returned to the pool. The are closed properly upon returning. -func (cp *ConnectionPool) GetTransientConnection(ctx context.Context) (*Connection, error) { - tID := atomic.AddInt64(&cp.transientID, 1) +func (cp *ConnectionPool) GetTransientConnection(ctx context.Context) (_ *Connection, err error) { + defer func() { + if err == nil { + cp.incTransient() + } + }() - conn, err := cp.deriveConnection(ctx, int(tID), false) + conn, err := cp.deriveConnection(ctx, cp.nextTransientID(), false) if err == nil { return conn, nil } @@ -196,21 +225,23 @@ func (cp *ConnectionPool) GetTransientConnection(ctx context.Context) (*Connecti // ReturnConnection puts the connection back in the queue and flag it for error. // This helps maintain a Round Robin on Connections and their resources. -func (cp *ConnectionPool) ReturnConnection(ctx context.Context, conn *Connection, flag bool) { +// If the connection is flagged, it will be recovered and returned to the pool. +// If the context is canceled, the connection will be immediately returned to the pool +// without any recovery attempt. +func (cp *ConnectionPool) ReturnConnection(ctx context.Context, conn *Connection, err error) { // close transient connections if !conn.IsCached() { + cp.decTransient() // decrease transient cinnections _ = conn.Close() return } - conn.Flag(flag) + conn.Flag(flaggable(err)) - if conn.IsFlagged() { - // try to recover until context is canceled - // if recovery fails, we put the broken connection into the pool - _ = conn.Recover(ctx) + select { + case cp.connections <- conn: + default: + panic("connection pool connections buffer full: not supposed to happen") } - - cp.connections <- conn } // Close closes the connection pool. @@ -237,6 +268,28 @@ func (cp *ConnectionPool) Close() { wg.Wait() } +// StatTransientActive returns the number of active transient connections. +func (cp *ConnectionPool) StatTransientActive() int { + cp.mu.Lock() + defer cp.mu.Unlock() + return cp.concurrentTransient +} + +// StatCachedIdle returns the number of idle cached connections. +func (cp *ConnectionPool) StatCachedIdle() int { + return len(cp.connections) +} + +// StatCachedActive returns the number of active cached connections. +func (cp *ConnectionPool) StatCachedActive() int { + return cp.size - len(cp.connections) +} + +// Size is the total size of the cached connection pool without any transient connections. +func (cp *ConnectionPool) Size() int { + return cp.size +} + func (cp *ConnectionPool) catchShutdown() <-chan struct{} { return cp.ctx.Done() } @@ -256,7 +309,3 @@ func (cp *ConnectionPool) error(err error, a ...any) { func (cp *ConnectionPool) debug(a ...any) { cp.log.WithField("connectionPool", cp.name).Debug(a...) } - -func (cp *ConnectionPool) Size() int { - return cp.size -} diff --git a/pool/connection_pool_test.go b/pool/connection_pool_test.go index 7f7bd0d..cf75480 100644 --- a/pool/connection_pool_test.go +++ b/pool/connection_pool_test.go @@ -35,7 +35,7 @@ func TestNewConnectionPool(t *testing.T) { return } time.Sleep(5 * time.Second) - p.ReturnConnection(ctx, c, false) + p.ReturnConnection(ctx, c, nil) }() } @@ -74,7 +74,7 @@ func TestNewConnectionPoolDisconnect(t *testing.T) { } time.Sleep(1 * time.Second) - p.ReturnConnection(ctx, c, false) + p.ReturnConnection(ctx, c, nil) }(i) } diff --git a/pool/errors.go b/pool/errors.go index 642d0b5..a0b0466 100644 --- a/pool/errors.go +++ b/pool/errors.go @@ -22,9 +22,9 @@ var ( // the queue was not found. ErrNotFound = errors.New("not found") - // errFlowControl is returned when the server is under flow control - // TODO: make public api after a while - errFlowControl = errors.New("flow control") + // ErrFlowControl is returned when the server is under flow control + // Your HTTP api may return 503 Service Unavailable or 429 Too Many Requests with a Retry-After header (https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Retry-After) + ErrFlowControl = errors.New("flow control") // errFlowControlClosed is returned when the flow control channel is closed // Specifically interesting when awaiting publish confirms diff --git a/pool/flag.go b/pool/flag.go new file mode 100644 index 0000000..5c31944 --- /dev/null +++ b/pool/flag.go @@ -0,0 +1,7 @@ +package pool + +import "errors" + +func flaggable(err error) bool { + return err != nil && !errors.Is(err, ErrFlowControl) && !errors.Is(err, ErrClosed) +} diff --git a/pool/helpers_context.go b/pool/helpers_context.go index 2ee7746..578d6ca 100644 --- a/pool/helpers_context.go +++ b/pool/helpers_context.go @@ -2,12 +2,21 @@ package pool import ( "context" + "errors" "fmt" "sync" ) -func newCancelContext(parentCtx context.Context) *cancelContext { - ctx, cancel := context.WithCancel(parentCtx) +var ( + errPausingCancel = errors.New("pausing") + errPausedCancel = errors.New("paused") + errResumingCancel = errors.New("resuming") + errResumedCancel = errors.New("resumed") +) + +func newCancelContext(parentCtx context.Context, cancelCause error) *cancelContext { + ctx, cc := context.WithCancelCause(parentCtx) + cancel := toCancelFunc(cancelCause, cc) return &cancelContext{ ctx: ctx, cancel: cancel, @@ -141,10 +150,10 @@ type stateContext struct { func newStateContext(ctx context.Context) *stateContext { sc := &stateContext{ parentCtx: ctx, - pausing: newCancelContext(ctx), - resuming: newCancelContext(ctx), - paused: newCancelContext(ctx), - resumed: newCancelContext(ctx), + pausing: newCancelContext(ctx, errPausingCancel), + resuming: newCancelContext(ctx, errResumingCancel), + paused: newCancelContext(ctx, errPausedCancel), + resumed: newCancelContext(ctx, errResumedCancel), } sc.pausing.Cancel() diff --git a/pool/pool.go b/pool/pool.go index 1759e15..268ee5f 100644 --- a/pool/pool.go +++ b/pool/pool.go @@ -93,8 +93,8 @@ func (p *Pool) GetTransientSession(ctx context.Context) (*Session, error) { // ReturnSession returns a Session back to the pool. // If the session was returned due to an error, erred should be set to true, otherwise // erred should be set to false. -func (p *Pool) ReturnSession(ctx context.Context, session *Session, erred bool) { - p.sp.ReturnSession(ctx, session, erred) +func (p *Pool) ReturnSession(session *Session, err error) { + p.sp.ReturnSession(session, err) } func (p *Pool) Context() context.Context { diff --git a/pool/pool_test.go b/pool/pool_test.go index 94d4459..07084dd 100644 --- a/pool/pool_test.go +++ b/pool/pool_test.go @@ -60,7 +60,7 @@ func TestNew(t *testing.T) { } time.Sleep(1 * time.Second) - p.ReturnSession(ctx, session, false) + p.ReturnSession(session, nil) }() } diff --git a/pool/publisher.go b/pool/publisher.go index 87c0b80..9020878 100644 --- a/pool/publisher.go +++ b/pool/publisher.go @@ -3,7 +3,7 @@ package pool import ( "context" "errors" - "sync" + "fmt" "github.com/jxsl13/amqpx/logging" ) @@ -15,8 +15,6 @@ type Publisher struct { ctx context.Context cancel context.CancelFunc - mu sync.Mutex - log logging.Logger } @@ -48,7 +46,8 @@ func NewPublisher(p *Pool, options ...PublisherOption) *Publisher { o(&option) } - ctx, cancel := context.WithCancel(option.Ctx) + ctx, cc := context.WithCancelCause(option.Ctx) + cancel := toCancelFunc(fmt.Errorf("publisher %w", ErrClosed), cc) pub := &Publisher{ pool: p, @@ -82,6 +81,9 @@ func (p *Publisher) Publish(ctx context.Context, exchange string, routingKey str return err case errors.Is(err, ErrDeliveryTagMismatch): return err + case errors.Is(err, ErrFlowControl): + p.warn(exchange, routingKey, err, "publish failed, retrying") + return err default: p.warn(exchange, routingKey, err, "publish failed, retrying") } @@ -98,20 +100,11 @@ func (p *Publisher) publish(ctx context.Context, exchange string, routingKey str }() s, err := p.pool.GetSession(ctx) - if err != nil && errors.Is(err, ErrClosed) { + if err != nil { return err } defer func() { - // return session - if err == nil { - p.pool.ReturnSession(ctx, s, false) - } else if errors.Is(err, ErrClosed) { - // TODO: potential message loss upon shutdown - // might try a transient session for this one - p.pool.ReturnSession(ctx, s, false) - } else { - p.pool.ReturnSession(ctx, s, true) - } + p.pool.ReturnSession(s, err) }() tag, err := s.Publish(ctx, exchange, routingKey, msg) @@ -129,18 +122,11 @@ func (p *Publisher) publish(ctx context.Context, exchange string, routingKey str // Get is only supposed to be used for testing, do not use get for polling any broker queues. func (p *Publisher) Get(ctx context.Context, queue string, autoAck bool) (msg Delivery, ok bool, err error) { s, err := p.pool.GetSession(ctx) - if err != nil && errors.Is(err, ErrClosed) { + if err != nil { return Delivery{}, false, err } defer func() { - // return session - if err == nil { - p.pool.ReturnSession(ctx, s, false) - } else if errors.Is(err, ErrClosed) { - p.pool.ReturnSession(ctx, s, false) - } else { - p.pool.ReturnSession(ctx, s, true) - } + p.pool.ReturnSession(s, err) }() return s.Get(ctx, queue, autoAck) diff --git a/pool/publisher_test.go b/pool/publisher_test.go index 2b220e8..47ff32a 100644 --- a/pool/publisher_test.go +++ b/pool/publisher_test.go @@ -47,7 +47,7 @@ func TestPublisher(t *testing.T) { return } defer func() { - p.ReturnSession(ctx, s, false) + p.ReturnSession(s, nil) }() queueName := fmt.Sprintf("TestPublisher-Queue-%d", id) @@ -127,7 +127,7 @@ func TestPublisher(t *testing.T) { wg.Wait() } -func TestPauseOnFlowControl(t *testing.T) { +func TestPublishAwaitFlowControl(t *testing.T) { ctx, cancel := signal.NotifyContext(context.TODO(), os.Interrupt) defer cancel() @@ -138,7 +138,7 @@ func TestPauseOnFlowControl(t *testing.T) { brokenConnectURL, // connections, sessions, - pool.WithName("TestPauseOnFlowControl"), + pool.WithName("TestPublishAwaitFlowControl"), pool.WithConfirms(true), pool.WithLogger(logging.NewTestLogger(t)), ) @@ -153,21 +153,22 @@ func TestPauseOnFlowControl(t *testing.T) { assert.NoError(t, err) return } + defer p.ReturnSession(s, nil) var ( - exchangeName = "TestPauseOnFlowControl-Exchange" + exchangeName = "TestPublishAwaitFlowControl-Exchange" ) - cleanup := initQueueExchange(t, s, ctx, "TestPauseOnFlowControl-Queue", exchangeName) + cleanup := initQueueExchange(t, s, ctx, "TestPublishAwaitFlowControl-Queue", exchangeName) defer cleanup() pub := pool.NewPublisher(p) defer pub.Close() - pubGen := PublishingGenerator("TestPauseOnFlowControl") + pubGen := PublishingGenerator("TestPublishAwaitFlowControl") err = pub.Publish(ctx, exchangeName, "", pubGen()) - assert.NoError(t, err) + assert.ErrorIs(t, err, pool.ErrFlowControl) } diff --git a/pool/session.go b/pool/session.go index 6f7de0b..18fc436 100644 --- a/pool/session.go +++ b/pool/session.go @@ -19,6 +19,7 @@ const ( type Session struct { name string cached bool + flagged bool confirmable bool bufferSize int @@ -96,7 +97,8 @@ func NewSession(conn *Connection, name string, options ...SessionOption) (*Sessi o(&option) } - ctx, cancel := context.WithCancel(option.Ctx) + ctx, cc := context.WithCancelCause(option.Ctx) + cancel := toCancelFunc(fmt.Errorf("session %w", ErrClosed), cc) session := &Session{ name: name, @@ -144,6 +146,25 @@ func NewSession(conn *Connection, name string, options ...SessionOption) (*Sessi return session, nil } +// Flag marks the session as flagged. +// This is useful in case of a connection pool, where the session is returned to the pool +// and should be recovered by the next user. +func (s *Session) Flag(flagged bool) { + s.mu.Lock() + defer s.mu.Unlock() + + if !s.flagged && flagged { + s.flagged = flagged + } +} + +// IsFlagged returns whether the session is flagged. +func (s *Session) IsFlagged() bool { + s.mu.Lock() + defer s.mu.Unlock() + return s.flagged +} + // Close closes the session completely. // Do not use this method in case you have acquired the session // from a connection pool. @@ -265,6 +286,15 @@ func (s *Session) tryRecover(ctx context.Context, err error) error { func (s *Session) recover(ctx context.Context) error { + select { + case <-ctx.Done(): + return ctx.Err() + case <-s.catchShutdown(): + return fmt.Errorf("failed to recover session: %w", s.shutdownErr()) + default: + // check if context was closed before starting a recovery. + } + // tries to recover session forever for try := 0; ; try++ { // try closing the channel before recovering @@ -282,6 +312,8 @@ func (s *Session) recover(ctx context.Context) error { // with a backoff. Sessions should be instantly created on a healthy connection err = s.connect() // Creates a new channel and flushes internal buffers automatically. if err == nil { + // successfully recovered + s.flagged = false return nil } @@ -316,20 +348,24 @@ func (s *Session) AwaitConfirm(ctx context.Context, expectedTag uint64) error { return fmt.Errorf("await confirm failed: confirms channel %w", ErrClosed) } if !confirm.Ack { - // in case the server did not accept the message, it might be due to resource problems. // TODO: do we want to pause here upon flow control messages err := fmt.Errorf("await confirm failed: %w", ErrNack) - flowErr := s.conn.PauseOnFlowControl(ctx) - if flowErr != nil { - err = errors.Join(err, flowErr) - } return err } if confirm.DeliveryTag != expectedTag { return fmt.Errorf("await confirm failed: %w: expected %d, got %d", ErrDeliveryTagMismatch, expectedTag, confirm.DeliveryTag) } return nil + case blocking, ok := <-s.conn.FlowControl(): + if !ok { + err := s.error() + if err != nil { + return fmt.Errorf("await confirm failed: %w", err) + } + return fmt.Errorf("await confirm failed: %w", errFlowControlClosed) + } + return fmt.Errorf("await confirm failed: %w: %s", ErrFlowControl, blocking.Reason) case <-ctx.Done(): err := ctx.Err() return fmt.Errorf("await confirm: failed context %w: %w", ErrClosed, err) @@ -1375,6 +1411,10 @@ func (s *Session) catchShutdown() <-chan struct{} { return s.ctx.Done() } +func (s *Session) shutdownErr() error { + return s.ctx.Err() +} + func (s *Session) info(a ...any) { s.log.WithField("connection", s.conn.Name()).WithField("session", s.Name()).Info(a...) } diff --git a/pool/session_pool.go b/pool/session_pool.go index b97da4e..512150d 100644 --- a/pool/session_pool.go +++ b/pool/session_pool.go @@ -69,7 +69,8 @@ func NewSessionPool(pool *ConnectionPool, numSessions int, options ...SessionPoo func newSessionPoolFromOption(pool *ConnectionPool, ctx context.Context, option sessionPoolOption) (sp *SessionPool, err error) { // decouple from parent context, in case we want to close this context ourselves. - ctx, cancel := context.WithCancel(ctx) + ctx, cc := context.WithCancelCause(ctx) + cancel := toCancelFunc(fmt.Errorf("session pool %w", ErrClosed), cc) sp = &SessionPool{ pool: pool, @@ -122,6 +123,40 @@ func newSessionPoolFromOption(pool *ConnectionPool, ctx context.Context, option return sp, nil } +func (sp *SessionPool) initCachedSessions() error { + for i := 0; i < sp.size; i++ { + session, err := sp.initCachedSession(i) + if err != nil { + return err + } + sp.sessions <- session + } + return nil +} + +// initCachedSession allows you create a pooled Session. +func (sp *SessionPool) initCachedSession(id int) (*Session, error) { + + // retry until we get a channel + // or until shutdown + for { + conn, err := sp.pool.GetConnection(sp.ctx) + if err != nil { + // error is only returned upon shutdown + return nil, err + } + + session, err := sp.deriveSession(sp.ctx, conn, id) + if err != nil { + sp.pool.ReturnConnection(sp.ctx, conn, err) + continue + } + + sp.pool.ReturnConnection(sp.ctx, conn, nil) + return session, nil + } +} + // Size returns the size of the session pool which indicate sthe number of available cached sessions. func (sp *SessionPool) Size() int { return sp.size @@ -132,7 +167,7 @@ func (sp *SessionPool) Size() int { func (sp *SessionPool) GetSession(ctx context.Context) (*Session, error) { select { case <-sp.catchShutdown(): - return nil, ErrClosed + return nil, sp.shutdownErr() case <-ctx.Done(): return nil, ctx.Err() case session, ok := <-sp.sessions: @@ -196,10 +231,9 @@ func (sp *SessionPool) deriveSession(ctx context.Context, conn *Connection, id i ) } -// ReturnSession returns a Session. +// ReturnSession returns a Session to the pool. // If Session is not a cached channel, it is simply closed here. -// If Cache Session, we check if erred, new Session is created instead and then returned to the cache. -func (sp *SessionPool) ReturnSession(ctx context.Context, session *Session, erred bool) { +func (sp *SessionPool) ReturnSession(session *Session, err error) { // don't ass non-managed sessions back to the channel if !session.IsCached() { @@ -207,26 +241,31 @@ func (sp *SessionPool) ReturnSession(ctx context.Context, session *Session, erre return } - if erred { - // try recovering until context closed or shutdown - _ = session.Recover(ctx) - } else { - // healthy sessions may contain pending confirmation messages - // cleanup confirmations from previous session usage - _ = session.FlushConfirms() - // flush errors - _ = session.Error() - } + // try recovering until context closed or shutdown + session.Flag(flaggable(err)) + // healthy sessions may contain pending confirmation messages + // cleanup confirmations from previous session usage + _ = session.FlushConfirms() + // flush errors + _ = session.Error() // always put the session back into the pool // even if it is still broken - sp.sessions <- session + select { + case sp.sessions <- session: + default: + panic("session buffer full: not supposed to happen") + } } func (sp *SessionPool) catchShutdown() <-chan struct{} { return sp.ctx.Done() } +func (sp *SessionPool) shutdownErr() error { + return sp.ctx.Err() +} + // Closes the session pool with all of its sessions func (sp *SessionPool) Close() { @@ -255,40 +294,6 @@ func (sp *SessionPool) Close() { } } -func (sp *SessionPool) initCachedSessions() error { - for i := 0; i < sp.size; i++ { - session, err := sp.initCachedSession(i) - if err != nil { - return err - } - sp.sessions <- session - } - return nil -} - -// initCachedSession allows you create a pooled Session. -func (sp *SessionPool) initCachedSession(id int) (*Session, error) { - - // retry until we get a channel - // or until shutdown - for { - conn, err := sp.pool.GetConnection(sp.ctx) - if err != nil { - // error is only returned upon shutdown - return nil, err - } - - session, err := sp.deriveSession(sp.ctx, conn, id) - if err != nil { - sp.pool.ReturnConnection(sp.ctx, conn, true) - continue - } - - sp.pool.ReturnConnection(sp.ctx, conn, false) - return session, nil - } -} - func (sp *SessionPool) info(a ...any) { sp.log.WithField("sessionPool", sp.pool.name).Info(a...) } diff --git a/pool/session_pool_test.go b/pool/session_pool_test.go index d1034dd..0eba311 100644 --- a/pool/session_pool_test.go +++ b/pool/session_pool_test.go @@ -15,7 +15,9 @@ func TestNewSessionPool(t *testing.T) { ctx := context.TODO() connections := 1 sessions := 10 - p, err := pool.NewConnectionPool(ctx, connectURL, connections, + p, err := pool.NewConnectionPool(ctx, + connectURL, + connections, pool.ConnectionPoolWithName("TestNewConnectionPool"), pool.ConnectionPoolWithLogger(logging.NewTestLogger(t)), ) @@ -24,7 +26,11 @@ func TestNewSessionPool(t *testing.T) { return } - sp, err := pool.NewSessionPool(p, sessions, pool.SessionPoolWithAutoCloseConnectionPool(true)) + sp, err := pool.NewSessionPool( + p, + sessions, + pool.SessionPoolWithAutoCloseConnectionPool(true), + ) if err != nil { assert.NoError(t, err) return @@ -43,7 +49,7 @@ func TestNewSessionPool(t *testing.T) { return } time.Sleep(3 * time.Second) - sp.ReturnSession(ctx, s, false) + sp.ReturnSession(s, nil) }() } diff --git a/pool/session_test.go b/pool/session_test.go index 0040354..440cd2d 100644 --- a/pool/session_test.go +++ b/pool/session_test.go @@ -326,7 +326,7 @@ func TestNewSessionQueueDeclarePassive(t *testing.T) { time.Sleep(10 * time.Second) // await dangling io goroutines to timeout }() - c, err := pool.NewConnection( + conn, err := pool.NewConnection( ctx, connectURL, "TestNewSessionQueueDeclarePassive", @@ -337,10 +337,14 @@ func TestNewSessionQueueDeclarePassive(t *testing.T) { return } defer func() { - c.Close() // can be nil or error + conn.Close() // can be nil or error }() - session, err := pool.NewSession(c, fmt.Sprintf("TestNewSessionQueueDeclarePassive-%d", 1), pool.SessionWithConfirms(true)) + session, err := pool.NewSession( + conn, + fmt.Sprintf("TestNewSessionQueueDeclarePassive-%d", 1), + pool.SessionWithConfirms(true), + ) if err != nil { assert.NoError(t, err) return diff --git a/pool/subscriber.go b/pool/subscriber.go index 3874fe7..9e04a86 100644 --- a/pool/subscriber.go +++ b/pool/subscriber.go @@ -64,7 +64,8 @@ func NewSubscriber(p *Pool, options ...SubscriberOption) *Subscriber { } // decouple from parent in order to individually close the context - ctx, cancel := context.WithCancel(option.Ctx) + ctx, cc := context.WithCancelCause(option.Ctx) + cancel := toCancelFunc(fmt.Errorf("subscriber %w", ErrClosed), cc) sub := &Subscriber{ pool: p, @@ -234,7 +235,7 @@ func (s *Subscriber) retry(consumerType string, h handler, f func() (err error)) return } // consume was canceled due to context being closed (paused) - if errors.Is(err, context.Canceled) { + if errors.Is(err, errPausingCancel) { s.infoConsumer(opts.ConsumerTag, fmt.Sprintf("%s paused: pausing %v", consumerType, err)) continue } @@ -281,7 +282,7 @@ func (s *Subscriber) consumer(h *Handler, wg *sync.WaitGroup) { s.retry("consumer", h, func() error { select { case <-s.catchShutdown(): - return ErrClosed + return s.shutdownErr() case <-h.resuming().Done(): return s.consume(h) } @@ -321,7 +322,7 @@ func (s *Subscriber) consume(h *Handler) (err error) { for { select { case <-s.catchShutdown(): - return ErrClosed + return s.shutdownErr() case msg, ok := <-delivery: if !ok { return ErrDeliveryClosed @@ -400,7 +401,7 @@ func (s *Subscriber) batchConsumer(h *BatchHandler, wg *sync.WaitGroup) { s.retry("batch consumer", h, func() error { select { case <-s.catchShutdown(): - return ErrClosed + return s.shutdownErr() case <-h.resuming().Done(): return s.batchConsume(h) } @@ -482,7 +483,7 @@ func (s *Subscriber) batchConsume(h *BatchHandler) (err error) { select { case <-s.catchShutdown(): - return ErrClosed + return s.shutdownErr() case msg, ok := <-delivery: if !ok { return ErrDeliveryClosed @@ -640,7 +641,7 @@ func (s *Subscriber) returnSession(h handler, session *Session, err error) { if errors.Is(err, ErrClosed) { // graceful shutdown - s.pool.ReturnSession(s.ctx, session, false) + s.pool.ReturnSession(session, err) s.infoConsumer(opts.ConsumerTag, "closed") return } @@ -650,11 +651,11 @@ func (s *Subscriber) returnSession(h handler, session *Session, err error) { // expected closing due to context cancelation // cancel errors the underlying channel // A canceled session is an erred session. - s.pool.ReturnSession(s.ctx, session, true) + s.pool.ReturnSession(session, h.pausing().Err()) s.infoConsumer(opts.ConsumerTag, "paused") default: // actual error - s.pool.ReturnSession(s.ctx, session, true) + s.pool.ReturnSession(session, err) s.warnConsumer(opts.ConsumerTag, err, "closed unexpectedly") } } @@ -663,6 +664,10 @@ func (s *Subscriber) catchShutdown() <-chan struct{} { return s.ctx.Done() } +func (s *Subscriber) shutdownErr() error { + return s.ctx.Err() +} + func (s *Subscriber) infoBatchHandler(consumer, queue string, batchSize, batchBytes int, a ...any) { s.log.WithFields(withConsumerIfSet(consumer, map[string]any{ diff --git a/pool/subscriber_test.go b/pool/subscriber_test.go index 968c5b9..1229730 100644 --- a/pool/subscriber_test.go +++ b/pool/subscriber_test.go @@ -42,7 +42,7 @@ func TestSubscriber(t *testing.T) { assert.NoError(t, err) return } - defer p.ReturnSession(ctx, ts, false) + defer p.ReturnSession(ts, nil) queueName := fmt.Sprintf("TestSubscriber-Queue-%d", id) _, err = ts.QueueDeclare(ctx, queueName) @@ -161,7 +161,7 @@ func TestBatchSubscriber(t *testing.T) { assert.NoError(t, err) return } - defer p.ReturnSession(ctx, ts, false) + defer p.ReturnSession(ts, nil) queueName := fmt.Sprintf("TestBatchSubscriber-Queue-%d", id) _, err = ts.QueueDeclare(ctx, queueName) @@ -302,7 +302,7 @@ func testBatchSubscriberMaxBytes(t *testing.T, maxBatchBytes int) { assert.NoError(t, err) return } - defer p.ReturnSession(ctx, ts, false) + defer p.ReturnSession(ts, nil) queueName := fmt.Sprintf("TestBatchSubscriberMaxBytes-Queue-%d", id) _, err = ts.QueueDeclare(ctx, queueName) diff --git a/pool/topologer.go b/pool/topologer.go index 73b0cbc..58e608e 100644 --- a/pool/topologer.go +++ b/pool/topologer.go @@ -68,11 +68,7 @@ func (t *Topologer) ExchangeDeclare(ctx context.Context, name string, kind Excha return err } defer func() { - if err != nil { - t.pool.ReturnSession(ctx, s, true) - } else { - t.pool.ReturnSession(ctx, s, false) - } + t.pool.ReturnSession(s, err) }() return s.ExchangeDeclare(ctx, name, kind, option...) } @@ -88,11 +84,7 @@ func (t *Topologer) ExchangeDeclarePassive(ctx context.Context, name string, kin return err } defer func() { - if err != nil { - t.pool.ReturnSession(ctx, s, true) - } else { - t.pool.ReturnSession(ctx, s, false) - } + t.pool.ReturnSession(s, err) }() return s.ExchangeDeclarePassive(ctx, name, kind, option...) } @@ -106,11 +98,7 @@ func (t *Topologer) ExchangeDelete(ctx context.Context, name string, option ...E return err } defer func() { - if err != nil { - t.pool.ReturnSession(ctx, s, true) - } else { - t.pool.ReturnSession(ctx, s, false) - } + t.pool.ReturnSession(s, err) }() return s.ExchangeDelete(ctx, name, option...) @@ -140,11 +128,7 @@ func (t *Topologer) QueueDeclare(ctx context.Context, name string, option ...Que return Queue{}, err } defer func() { - if err != nil { - t.pool.ReturnSession(ctx, s, true) - } else { - t.pool.ReturnSession(ctx, s, false) - } + t.pool.ReturnSession(s, err) }() return s.QueueDeclare(ctx, name, option...) } @@ -158,11 +142,7 @@ func (t *Topologer) QueueDeclarePassive(ctx context.Context, name string, option return Queue{}, err } defer func() { - if err != nil { - t.pool.ReturnSession(ctx, s, true) - } else { - t.pool.ReturnSession(ctx, s, false) - } + t.pool.ReturnSession(s, err) }() return s.QueueDeclarePassive(ctx, name, option...) } @@ -176,11 +156,7 @@ func (t *Topologer) QueuePurge(ctx context.Context, name string, options ...Queu return 0, err } defer func() { - if err != nil { - t.pool.ReturnSession(ctx, s, true) - } else { - t.pool.ReturnSession(ctx, s, false) - } + t.pool.ReturnSession(s, err) }() return s.QueuePurge(ctx, name, options...) } @@ -194,11 +170,7 @@ func (t *Topologer) QueueDelete(ctx context.Context, name string, option ...Queu return 0, err } defer func() { - if err != nil { - t.pool.ReturnSession(ctx, s, true) - } else { - t.pool.ReturnSession(ctx, s, false) - } + t.pool.ReturnSession(s, err) }() return s.QueueDelete(ctx, name, option...) } @@ -240,11 +212,7 @@ func (t *Topologer) QueueBind(ctx context.Context, name string, routingKey strin return err } defer func() { - if err != nil { - t.pool.ReturnSession(ctx, s, true) - } else { - t.pool.ReturnSession(ctx, s, false) - } + t.pool.ReturnSession(s, err) }() return s.QueueBind(ctx, name, routingKey, exchange, option...) } @@ -260,11 +228,7 @@ func (t *Topologer) QueueUnbind(ctx context.Context, name string, routingKey str return err } defer func() { - if err != nil { - t.pool.ReturnSession(ctx, s, true) - } else { - t.pool.ReturnSession(ctx, s, false) - } + t.pool.ReturnSession(s, err) }() return s.QueueUnbind(ctx, name, routingKey, exchange, args...) } @@ -298,11 +262,7 @@ func (t *Topologer) ExchangeBind(ctx context.Context, destination string, routin return err } defer func() { - if err != nil { - t.pool.ReturnSession(ctx, s, true) - } else { - t.pool.ReturnSession(ctx, s, false) - } + t.pool.ReturnSession(s, err) }() return s.ExchangeBind(ctx, destination, routingKey, source, option...) } @@ -317,11 +277,7 @@ func (t *Topologer) ExchangeUnbind(ctx context.Context, destination string, rout return err } defer func() { - if err != nil { - t.pool.ReturnSession(ctx, s, true) - } else { - t.pool.ReturnSession(ctx, s, false) - } + t.pool.ReturnSession(s, err) }() return s.ExchangeUnbind(ctx, destination, routingKey, source, option...) diff --git a/pool/utils.go b/pool/utils.go index c2e286b..82b4b3b 100644 --- a/pool/utils.go +++ b/pool/utils.go @@ -1,6 +1,9 @@ package pool -import "time" +import ( + "context" + "time" +) // closeTimer should be used as a deferred function // in order to cleanly shut down a timer @@ -30,3 +33,9 @@ func resetTimer(timer *time.Timer, duration time.Duration, drained *bool) { timer.Reset(duration) *drained = false } + +func toCancelFunc(err error, ccf context.CancelCauseFunc) context.CancelFunc { + return func() { + ccf(err) + } +} From 59cd9a139a9704f5dc44c5dc6891890935b354a5 Mon Sep 17 00:00:00 2001 From: John Behm Date: Mon, 4 Mar 2024 20:05:23 +0100 Subject: [PATCH 03/76] update basic tests --- internal/testutils/generator.go | 185 ++++++++++++++++++++++++ internal/testutils/rand.go | 46 ++++++ internal/testutils/testutils.go | 40 +++++ pool/connection_test.go | 69 +++++---- pool/session_test.go | 249 +++++++++++++++++--------------- pool/utils_test.go | 163 +++++++++++++++++++++ 6 files changed, 607 insertions(+), 145 deletions(-) create mode 100644 internal/testutils/generator.go create mode 100644 internal/testutils/rand.go create mode 100644 internal/testutils/testutils.go create mode 100644 pool/utils_test.go diff --git a/internal/testutils/generator.go b/internal/testutils/generator.go new file mode 100644 index 0000000..ef292d5 --- /dev/null +++ b/internal/testutils/generator.go @@ -0,0 +1,185 @@ +package testutils + +import ( + "fmt" + "strings" + "sync" +) + +type generatorOptions struct { + prefix string + up int + suffix string + randomSuffix bool + suffixSize int +} + +func (o *generatorOptions) ToSuffix() string { + var suffix string + suffix += o.suffix + if o.randomSuffix { + suffix += "-" + RandString(o.suffixSize) + } + return suffix +} + +type GeneratorOption func(*generatorOptions) + +func WithRandomSuffix(addRandomSuffix bool) GeneratorOption { + return func(o *generatorOptions) { + o.randomSuffix = addRandomSuffix + } +} + +func WithPrefix(prefix string) GeneratorOption { + return func(o *generatorOptions) { + o.prefix = prefix + } +} + +func WithSuffix(suffix string) GeneratorOption { + return func(o *generatorOptions) { + o.suffix = suffix + } +} + +func ExchangeNameGenerator(sessionName string, options ...GeneratorOption) (nextExchangeName func() string) { + opts := generatorOptions{ + prefix: "", + randomSuffix: false, + + up: 2, + } + + for _, opt := range options { + opt(&opts) + } + + var mu sync.Mutex + var counter int64 + return func() string { + mu.Lock() + cnt := counter + counter++ + mu.Unlock() + return fmt.Sprintf("%s-%sexchange-%d%s", sessionName, opts.prefix, cnt, opts.ToSuffix()) + } +} + +func ConsumerNameGenerator(queueName string, options ...GeneratorOption) (nextConsumerName func() string) { + opts := generatorOptions{ + prefix: "", + randomSuffix: false, + + up: 2, + } + + for _, opt := range options { + opt(&opts) + } + + var mu sync.Mutex + var counter int64 + return func() string { + mu.Lock() + cnt := counter + counter++ + mu.Unlock() + return fmt.Sprintf("%s-%sconsumer-%d%s", queueName, opts.prefix, cnt, opts.ToSuffix()) + } +} + +func QueueNameGenerator(sessionName string, options ...GeneratorOption) (nextQueueName func() string) { + opts := generatorOptions{ + prefix: "", + randomSuffix: false, + + up: 2, + } + + for _, opt := range options { + opt(&opts) + } + + var mu sync.Mutex + var counter int64 + return func() string { + mu.Lock() + cnt := counter + counter++ + mu.Unlock() + return fmt.Sprintf("%s-%squeue-%d%s", sessionName, opts.prefix, cnt, opts.ToSuffix()) + } +} + +func SessionNameGenerator(connectionName string, options ...GeneratorOption) (nextSessionName func() string) { + opts := generatorOptions{ + prefix: "", + randomSuffix: false, + + up: 2, + } + + for _, opt := range options { + opt(&opts) + } + + var mu sync.Mutex + var counter int64 + return func() string { + mu.Lock() + cnt := counter + counter++ + mu.Unlock() + return fmt.Sprintf("%s-%ssession-%d%s", connectionName, opts.prefix, cnt, opts.ToSuffix()) + } +} + +func ConnectionNameGenerator(options ...GeneratorOption) (nextConnName func() string) { + opts := generatorOptions{ + prefix: "", + randomSuffix: false, + + up: 2, + } + + for _, opt := range options { + opt(&opts) + } + + var mu sync.Mutex + funcName := FuncName(opts.up) + parts := strings.Split(funcName, ".") + funcName = parts[len(parts)-1] + + var counter int64 + return func() string { + mu.Lock() + cnt := counter + counter++ + mu.Unlock() + return fmt.Sprintf("%s%s-%d%s", opts.prefix, funcName, cnt, opts.ToSuffix()) + } +} + +func MessageGenerator(queueOrExchangeName string, options ...GeneratorOption) (nextMessage func() string) { + opts := generatorOptions{ + prefix: "", + randomSuffix: false, + up: 2, + } + + for _, opt := range options { + opt(&opts) + } + + var mu sync.Mutex + var counter int64 + return func() string { + mu.Lock() + cnt := counter + counter++ + mu.Unlock() + return fmt.Sprintf("%s-message-%d-%s", queueOrExchangeName, cnt, opts.ToSuffix()) + } +} diff --git a/internal/testutils/rand.go b/internal/testutils/rand.go new file mode 100644 index 0000000..c24b78a --- /dev/null +++ b/internal/testutils/rand.go @@ -0,0 +1,46 @@ +package testutils + +import ( + "fmt" + "math/rand" +) + +const ( + // CharSetAlphaNum is the alphanumeric character set for use with + // RandStringFromCharSet + CharSetAlphaNum = "abcdefghijklmnopqrstuvwxyz012346789" + + // CharSetAlpha is the alphabetical character set for use with + // RandStringFromCharSet + CharSetAlpha = "abcdefghijklmnopqrstuvwxyz" +) + +// RandInt generates a random integer +func RandInt() int { + return rand.Int() +} + +// RandIntWithPrefix is used to generate a unique name with a prefix +func RandIntWithPrefix(name string) string { + return fmt.Sprintf("%s-%d", name, RandInt()) +} + +// RandIntRange returns a random integer between min (inclusive) and max (exclusive) +func RandIntRange(min int, max int) int { + return min + rand.Intn(max-min) +} + +// RandString generates a random alphanumeric string of the length specified +func RandString(strlen int) string { + return RandStringFromCharSet(strlen, CharSetAlphaNum) +} + +// RandStringFromCharSet generates a random string by selecting characters from +// the charset provided +func RandStringFromCharSet(strlen int, charSet string) string { + result := make([]byte, strlen) + for i := 0; i < strlen; i++ { + result[i] = charSet[RandIntRange(0, len(charSet))] + } + return string(result) +} diff --git a/internal/testutils/testutils.go b/internal/testutils/testutils.go new file mode 100644 index 0000000..bc80077 --- /dev/null +++ b/internal/testutils/testutils.go @@ -0,0 +1,40 @@ +package testutils + +import ( + "fmt" + "path/filepath" + "runtime" +) + +func FilePath(relative string, up ...int) string { + offset := 1 + if len(up) > 0 && up[0] > 0 { + offset = up[0] + } + _, file, _, ok := runtime.Caller(offset) + if !ok { + panic("failed to get caller") + } + if filepath.IsAbs(relative) { + panic(fmt.Sprintf("%s is an absolute file path, must be relative to the current go source file", relative)) + } + abs := filepath.Join(filepath.Dir(file), relative) + return abs +} + +func FuncName(up ...int) string { + offset := 1 + if len(up) > 0 && up[0] > 0 { + offset = up[0] + } + pc, _, _, ok := runtime.Caller(offset) + if !ok { + panic("failed to get caller") + } + + f := runtime.FuncForPC(pc) + if f == nil { + panic("failed to get function name") + } + return f.Name() +} diff --git a/pool/connection_test.go b/pool/connection_test.go index 3b53ad8..51b4f47 100644 --- a/pool/connection_test.go +++ b/pool/connection_test.go @@ -2,11 +2,11 @@ package pool_test import ( "context" - "fmt" "sync" "testing" "time" + "github.com/jxsl13/amqpx/internal/testutils" "github.com/jxsl13/amqpx/logging" "github.com/jxsl13/amqpx/pool" "github.com/stretchr/testify/assert" @@ -14,11 +14,15 @@ import ( ) func TestNewSingleConnection(t *testing.T) { - ctx := context.TODO() + var ( + ctx = context.TODO() + nextName = testutils.ConnectionNameGenerator() + ) + c, err := pool.NewConnection( ctx, connectURL, - "TestNewSingleConnection", + nextName(), pool.ConnectionWithLogger(logging.NewTestLogger(t)), ) @@ -33,14 +37,19 @@ func TestNewSingleConnection(t *testing.T) { } func TestNewSingleConnectionWithDisconnect(t *testing.T) { - ctx := context.TODO() - started, stopped := DisconnectWithStartedStopped(t, 0, 0, 15*time.Second) + var ( + ctx = context.TODO() + nextName = testutils.ConnectionNameGenerator() + ) + + started, stopped := DisconnectWithStartedStopped(t, 0, 0, 10*time.Second) started() defer stopped() + c, err := pool.NewConnection( ctx, connectURL, - "TestNewSingleConnectionWithDisconnect", + nextName(), pool.ConnectionWithLogger(logging.NewTestLogger(t)), ) @@ -49,25 +58,27 @@ func TestNewSingleConnectionWithDisconnect(t *testing.T) { return } defer func() { - err := c.Close() - require.NoError(t, err) + require.NoError(t, c.Close()) }() } -func TestNewConnection(t *testing.T) { - ctx := context.TODO() - var wg sync.WaitGroup +func TestManyNewConnection(t *testing.T) { + var ( + ctx = context.TODO() + wg sync.WaitGroup + connections = 5 + nextName = testutils.ConnectionNameGenerator() + ) - connections := 5 wg.Add(connections) for i := 0; i < connections; i++ { - go func(id int64) { + go func() { defer wg.Done() c, err := pool.NewConnection( ctx, connectURL, - fmt.Sprintf("TestNewConnection-%d", id), + nextName(), pool.ConnectionWithLogger(logging.NewTestLogger(t)), ) if err != nil { @@ -75,36 +86,37 @@ func TestNewConnection(t *testing.T) { return } defer func() { + // error closed assert.Error(t, c.Error()) }() defer c.Close() time.Sleep(2 * time.Second) assert.NoError(t, c.Error()) - }(int64(i)) + }() } wg.Wait() } -func TestNewConnectionDisconnect(t *testing.T) { - ctx := context.TODO() - var wg sync.WaitGroup - - connections := 100 - wg.Add(connections) - - // disconnect directly for 10 seconds +func TestManyNewConnectionWithDisconnect(t *testing.T) { + var ( + ctx = context.TODO() + wg sync.WaitGroup + connections = 100 + nextName = testutils.ConnectionNameGenerator() + ) wait := DisconnectWithStopped(t, 0, 0, time.Second) defer wait() // wait for goroutine to properly close & unblock the proxy + wg.Add(connections) for i := 0; i < connections; i++ { - go func(id int64) { + go func() { defer wg.Done() c, err := pool.NewConnection( ctx, connectURL, - fmt.Sprintf("TestNewConnectionDisconnect-%d", id), + nextName(), //pool.ConnectionWithLogger(logging.NewTestLogger(t)), ) if err != nil { @@ -112,15 +124,18 @@ func TestNewConnectionDisconnect(t *testing.T) { return } defer func() { + // err closed assert.Error(t, c.Error()) }() defer c.Close() wait() // wait for connection to work again. - assert.NoError(t, c.Recover(ctx)) + tctx, cancel := context.WithTimeout(ctx, 5*time.Second) + defer cancel() + assert.NoError(t, c.Recover(tctx)) assert.NoError(t, c.Error()) - }(int64(i)) + }() } wg.Wait() diff --git a/pool/session_test.go b/pool/session_test.go index 440cd2d..164ba29 100644 --- a/pool/session_test.go +++ b/pool/session_test.go @@ -2,142 +2,142 @@ package pool_test import ( "context" - "fmt" "sync" "testing" "time" + "github.com/jxsl13/amqpx/internal/testutils" "github.com/jxsl13/amqpx/logging" "github.com/jxsl13/amqpx/pool" "github.com/stretchr/testify/assert" ) -func TestNewSession(t *testing.T) { - ctx := context.TODO() +func TestNewSingleSession(t *testing.T) { + var ( + ctx = context.TODO() + wg sync.WaitGroup + nextConnName = testutils.ConnectionNameGenerator() + connName = nextConnName() + nextSessionName = testutils.SessionNameGenerator(connName) + sessionName = nextSessionName() + nextQueueName = testutils.QueueNameGenerator(sessionName) + queueName = nextQueueName() + nextExchangeName = testutils.ExchangeNameGenerator(sessionName) + exchangeName = nextExchangeName() + nextConsumerName = testutils.ConsumerNameGenerator(queueName) + consumerName = nextConsumerName() + consumeMessageGenerator = testutils.MessageGenerator(queueName) + publishMessageGenerator = testutils.MessageGenerator(queueName) + ) + c, err := pool.NewConnection( ctx, connectURL, - "TestNewSession", + connName, pool.ConnectionWithLogger(logging.NewTestLogger(t)), ) if err != nil { assert.NoError(t, err) return } - defer c.Close() - - var wg sync.WaitGroup - - sessions := 5 - wg.Add(sessions) - for id := 0; id < sessions; id++ { - go func(id int64) { - defer wg.Done() - s, err := pool.NewSession(c, fmt.Sprintf("TestNewSession-%d", id), pool.SessionWithConfirms(true)) - if err != nil { - assert.NoError(t, err) - return - } - defer func() { - assert.NoError(t, s.Close()) - }() - - queueName := fmt.Sprintf("TestNewSession-Queue-%d", id) - _, err = s.QueueDeclare(ctx, queueName) - if err != nil { - assert.NoError(t, err) - return - } - defer func() { - i, err := s.QueueDelete(ctx, queueName) - assert.NoError(t, err) - assert.Equal(t, 0, i) - }() - - exchangeName := fmt.Sprintf("TestNewSession-Exchange-%d", id) - err = s.ExchangeDeclare(ctx, exchangeName, "topic") - if err != nil { - assert.NoError(t, err) - return - } - defer func() { - err := s.ExchangeDelete(ctx, exchangeName) - assert.NoError(t, err) - }() - - err = s.QueueBind(ctx, queueName, "#", exchangeName) - if err != nil { - assert.NoError(t, err) - return - } - defer func() { - err := s.QueueUnbind(ctx, queueName, "#", exchangeName, nil) - assert.NoError(t, err) - }() - - delivery, err := s.Consume( - queueName, - pool.ConsumeOptions{ - ConsumerTag: fmt.Sprintf("Consumer-%s", queueName), - Exclusive: true, - }, - ) - if err != nil { - assert.NoError(t, err) - return - } + defer func() { + assert.NoError(t, c.Close()) + }() - message := fmt.Sprintf("Message-%s", queueName) + s, err := pool.NewSession( + c, + sessionName, + pool.SessionWithConfirms(true), + ) + if err != nil { + assert.NoError(t, err) + return + } + defer func() { + assert.NoError(t, s.Close()) + }() - wg.Add(1) - go func(msg string) { - defer wg.Done() + cleanup := DeclareExchangeQueue(t, ctx, s, exchangeName, queueName) + defer cleanup() - msgsReceived := 0 - for val := range delivery { - receivedMsg := string(val.Body) - assert.Equal(t, message, receivedMsg) - msgsReceived++ - } - assert.Equal(t, 1, msgsReceived) - // this routine must be closed upon session closure - }(message) + ConsumeAsyncN(t, ctx, &wg, s, queueName, consumerName, consumeMessageGenerator, 1) + PublishAsyncN(t, ctx, &wg, s, exchangeName, publishMessageGenerator, 1) - time.Sleep(5 * time.Second) + wg.Wait() +} - ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) - defer cancel() +func TestManyNewSessions(t *testing.T) { + var ( + ctx = context.TODO() + wg sync.WaitGroup + nextConnName = testutils.ConnectionNameGenerator() + connName = nextConnName() + nextSessionName = testutils.SessionNameGenerator(connName) + sessions = 5 + ) - tag, err := s.Publish(ctx, exchangeName, "", pool.Publishing{ - Mandatory: true, - ContentType: "application/json", - Body: []byte(message), - }) - if err != nil { - assert.NoError(t, err) - return - } + c, err := pool.NewConnection( + ctx, + connectURL, + connName, + pool.ConnectionWithLogger(logging.NewTestLogger(t)), + ) + if err != nil { + assert.NoError(t, err) + return + } + defer func() { + assert.NoError(t, c.Close()) + }() - err = s.AwaitConfirm(ctx, tag) - if err != nil { - assert.NoError(t, err) - return - } + for id := 0; id < sessions; id++ { + var ( + sessionName = nextSessionName() + nextQueueName = testutils.QueueNameGenerator(sessionName) + queueName = nextQueueName() + nextExchangeName = testutils.ExchangeNameGenerator(sessionName) + exchangeName = nextExchangeName() + nextConsumerName = testutils.ConsumerNameGenerator(queueName) + consumerName = nextConsumerName() + // generate equal consume & publish messages for comparison + consumeNextMessage = testutils.MessageGenerator(queueName) + publishNextMessage = testutils.MessageGenerator(queueName) + ) + + s, err := pool.NewSession( + c, + sessionName, + pool.SessionWithConfirms(true), + ) + if err != nil { + assert.NoError(t, err) + return + } + defer func() { + assert.NoError(t, s.Close()) + }() - time.Sleep(5 * time.Second) + cleanup := DeclareExchangeQueue(t, ctx, s, exchangeName, queueName) + defer cleanup() - }(int64(id)) + ConsumeAsyncN(t, ctx, &wg, s, queueName, consumerName, consumeNextMessage, 1) + PublishAsyncN(t, ctx, &wg, s, exchangeName, publishNextMessage, 1) } wg.Wait() } func TestNewSessionDisconnect(t *testing.T) { - ctx := context.TODO() + var ( + ctx = context.TODO() + nextConnName = testutils.ConnectionNameGenerator() + connName = nextConnName() + nextSessionName = testutils.SessionNameGenerator(connName) + ) c, err := pool.NewConnection( ctx, connectURL, - "TestNewSessionDisconnect", + connName, pool.ConnectionWithLogger(logging.NewTestLogger(t)), ) if err != nil { @@ -167,9 +167,20 @@ func TestNewSessionDisconnect(t *testing.T) { start10, started10, stopped10 := DisconnectWithStartStartedStopped(t, time.Second) for id := 0; id < sessions; id++ { - go func(id int64) { + go func() { defer wg.Done() - s, err := pool.NewSession(c, fmt.Sprintf("TestNewSession-%d", id), pool.SessionWithConfirms(true)) + + var ( + sessionName = nextSessionName() + nextQueueName = testutils.QueueNameGenerator(sessionName) + queueName = nextQueueName() + nextExchangeName = testutils.ExchangeNameGenerator(sessionName) + exchangeName = nextExchangeName() + nextConsumerName = testutils.ConsumerNameGenerator(queueName) + consumerName = nextConsumerName() + nextMessage = testutils.MessageGenerator(queueName) + ) + s, err := pool.NewSession(c, sessionName, pool.SessionWithConfirms(true)) if err != nil { assert.NoError(t, err) return @@ -185,7 +196,6 @@ func TestNewSessionDisconnect(t *testing.T) { start() // await connection loss start started() - exchangeName := fmt.Sprintf("TestNewSession-Exchange-%d", id) err = s.ExchangeDeclare(ctx, exchangeName, "topic") if err != nil { assert.NoError(t, err) @@ -207,7 +217,6 @@ func TestNewSessionDisconnect(t *testing.T) { start2() started2() - queueName := fmt.Sprintf("TestNewSession-Queue-%d", id) _, err = s.QueueDeclare(ctx, queueName) if err != nil { assert.NoError(t, err) @@ -250,7 +259,7 @@ func TestNewSessionDisconnect(t *testing.T) { delivery, err := s.Consume( queueName, pool.ConsumeOptions{ - ConsumerTag: fmt.Sprintf("Consumer-%s", queueName), + ConsumerTag: consumerName, Exclusive: true, }, ) @@ -260,7 +269,7 @@ func TestNewSessionDisconnect(t *testing.T) { } stopped4() - message := fmt.Sprintf("Message-%s", queueName) + message := nextMessage() wg.Add(1) go func(msg string) { @@ -310,26 +319,29 @@ func TestNewSessionDisconnect(t *testing.T) { break } - }(int64(id)) + }() } wg.Wait() - time.Sleep(10 * time.Second) // await dangling io goroutines to timeout } func TestNewSessionQueueDeclarePassive(t *testing.T) { - ctx := context.TODO() - var wg sync.WaitGroup - - defer func() { - wg.Wait() - time.Sleep(10 * time.Second) // await dangling io goroutines to timeout - }() + t.Parallel() + + var ( + ctx = context.TODO() + wg sync.WaitGroup + nextConnName = testutils.ConnectionNameGenerator() + connName = nextConnName() + nextSessionName = testutils.SessionNameGenerator(connName) + sessionName = nextSessionName() + nextQueueName = testutils.QueueNameGenerator(sessionName) + ) conn, err := pool.NewConnection( ctx, connectURL, - "TestNewSessionQueueDeclarePassive", + connName, pool.ConnectionWithLogger(logging.NewTestLogger(t)), ) if err != nil { @@ -342,7 +354,7 @@ func TestNewSessionQueueDeclarePassive(t *testing.T) { session, err := pool.NewSession( conn, - fmt.Sprintf("TestNewSessionQueueDeclarePassive-%d", 1), + sessionName, pool.SessionWithConfirms(true), ) if err != nil { @@ -354,7 +366,7 @@ func TestNewSessionQueueDeclarePassive(t *testing.T) { }() for i := 0; i < 100; i++ { - qname := fmt.Sprintf("TestNewSessionQueueDeclarePassive-queue-%d", i) + qname := nextQueueName() q, err := session.QueueDeclare(ctx, qname) if err != nil { assert.NoError(t, err) @@ -377,4 +389,5 @@ func TestNewSessionQueueDeclarePassive(t *testing.T) { assert.Equalf(t, 0, q.Consumers, "queue should not have any consumers: %s", qname) } + wg.Wait() } diff --git a/pool/utils_test.go b/pool/utils_test.go new file mode 100644 index 0000000..c79c8bd --- /dev/null +++ b/pool/utils_test.go @@ -0,0 +1,163 @@ +package pool_test + +import ( + "context" + "sync" + "testing" + "time" + + "github.com/jxsl13/amqpx/pool" + "github.com/stretchr/testify/assert" +) + +func ConsumeAsyncN( + t *testing.T, + ctx context.Context, + wg *sync.WaitGroup, + s *pool.Session, + queueName string, + consumerName string, + messageGenerator func() string, + n int, +) { + delivery, err := s.Consume( + queueName, + pool.ConsumeOptions{ + ConsumerTag: consumerName, + Exclusive: true, + }, + ) + if err != nil { + assert.NoError(t, err) + return + } + + wg.Add(1) + go func(wg *sync.WaitGroup, messageGenerator func() string, n int) { + defer wg.Done() + cctx, ccancel := context.WithCancel(ctx) + defer ccancel() + + msgsReceived := 0 + defer func() { + assert.Equal(t, n, msgsReceived) + }() + for { + select { + case <-cctx.Done(): + return + case val, ok := <-delivery: + if !ok { + return + } + err := val.Ack(false) + if err != nil { + assert.NoError(t, err) + return + } + + receivedMsg := string(val.Body) + assert.Equal(t, messageGenerator(), receivedMsg) + msgsReceived++ + if msgsReceived == n { + ccancel() + } + } + } + }(wg, messageGenerator, n) +} + +func PublishAsyncN( + t *testing.T, + ctx context.Context, + wg *sync.WaitGroup, + s *pool.Session, + exchangeName string, + publishMessageGenerator func() string, + n int, +) { + wg.Add(1) + go func(wg *sync.WaitGroup, publishMessageGenerator func() string, n int) { + defer wg.Done() + + for i := 0; i < n; i++ { + func() { + tctx, cancel := context.WithTimeout(ctx, 5*time.Second) + defer cancel() + + tag, err := s.Publish( + tctx, + exchangeName, "", + pool.Publishing{ + Mandatory: true, + ContentType: "text/plain", + Body: []byte(publishMessageGenerator()), + }) + if err != nil { + assert.NoError(t, err) + return + } + if s.IsConfirmable() { + err = s.AwaitConfirm(tctx, tag) + if err != nil { + assert.NoError(t, err) + return + } + } + }() + } + }(wg, publishMessageGenerator, n) +} + +func DeclareExchangeQueue( + t *testing.T, + ctx context.Context, + s *pool.Session, + exchangeName string, + queueName string, +) (cleanup func()) { + cleanup = func() {} + var err error + + err = s.ExchangeDeclare(ctx, exchangeName, pool.ExchangeKindTopic) + if err != nil { + assert.NoError(t, err) + return + } + defer func() { + if err != nil { + assert.NoError(t, s.ExchangeDelete(ctx, exchangeName)) + } + }() + + _, err = s.QueueDeclare(ctx, queueName) + if err != nil { + assert.NoError(t, err) + return + } + defer func() { + if err != nil { + _, e := s.QueueDelete(ctx, queueName) + assert.NoError(t, e) + } + }() + + err = s.QueueBind(ctx, queueName, "#", exchangeName) + if err != nil { + assert.NoError(t, err) + return + } + defer func() { + if err != nil { + assert.NoError(t, s.QueueUnbind(ctx, queueName, "#", exchangeName, nil)) + } + }() + + return func() { + assert.NoError(t, s.QueueUnbind(ctx, queueName, "#", exchangeName, nil)) + + _, e := s.QueueDelete(ctx, queueName) + assert.NoError(t, e) + assert.NoError(t, s.ExchangeDelete(ctx, exchangeName)) + } +} From 47dc36787ab2e2f08bc2076c1cf6ceb959e9e1fc Mon Sep 17 00:00:00 2001 From: John Behm Date: Wed, 6 Mar 2024 00:56:28 +0100 Subject: [PATCH 04/76] add Jitter test utility --- internal/testutils/jitter.go | 10 ++++++++++ 1 file changed, 10 insertions(+) create mode 100644 internal/testutils/jitter.go diff --git a/internal/testutils/jitter.go b/internal/testutils/jitter.go new file mode 100644 index 0000000..ab3e51e --- /dev/null +++ b/internal/testutils/jitter.go @@ -0,0 +1,10 @@ +package testutils + +import ( + "math/rand" + "time" +) + +func Jitter(min, max time.Duration) time.Duration { + return min + time.Duration(rand.Int63n(int64(max-min))) +} From da853656558836a548febffc45b1e3630610ea14 Mon Sep 17 00:00:00 2001 From: John Behm Date: Wed, 6 Mar 2024 00:56:45 +0100 Subject: [PATCH 05/76] rename test utility parameter --- pool/toxiproxy_client_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pool/toxiproxy_client_test.go b/pool/toxiproxy_client_test.go index 4422292..dd434c9 100644 --- a/pool/toxiproxy_client_test.go +++ b/pool/toxiproxy_client_test.go @@ -138,8 +138,8 @@ func DisconnectWithStopped(t *testing.T, block, timeout, duration time.Duration) // block current thread // timeout: how long the asynchronous goroutine needs to wait until it disables the connection // duration: how long the asynchronous goroutine wait suntil it reenables the connection -func DisconnectWithStartedStopped(t *testing.T, block, timeout, duration time.Duration) (awaitStarted, awaitStopped func()) { - start := time.Now().Add(timeout) +func DisconnectWithStartedStopped(t *testing.T, block, startIn, duration time.Duration) (awaitStarted, awaitStopped func()) { + start := time.Now().Add(startIn) var ( wgStart sync.WaitGroup wgStop sync.WaitGroup From c29f64070f92850b4b47ac424a689b2773bbdebc Mon Sep 17 00:00:00 2001 From: John Behm Date: Wed, 6 Mar 2024 00:57:07 +0100 Subject: [PATCH 06/76] add logging to test helpers --- pool/utils_test.go | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/pool/utils_test.go b/pool/utils_test.go index c79c8bd..978d97d 100644 --- a/pool/utils_test.go +++ b/pool/utils_test.go @@ -6,6 +6,7 @@ import ( "testing" "time" + "github.com/jxsl13/amqpx/logging" "github.com/jxsl13/amqpx/pool" "github.com/stretchr/testify/assert" ) @@ -60,6 +61,7 @@ func ConsumeAsyncN( assert.Equal(t, messageGenerator(), receivedMsg) msgsReceived++ if msgsReceived == n { + logging.NewTestLogger(t).Infof("consumed %d messages, closing consumer", n) ccancel() } } @@ -82,7 +84,7 @@ func PublishAsyncN( for i := 0; i < n; i++ { func() { - tctx, cancel := context.WithTimeout(ctx, 5*time.Second) + tctx, cancel := context.WithTimeout(ctx, 60*time.Second) defer cancel() tag, err := s.Publish( @@ -106,6 +108,7 @@ func PublishAsyncN( } }() } + logging.NewTestLogger(t).Infof("published %d messages, closing publisher", n) }(wg, publishMessageGenerator, n) } From e0eeaf9384cbbadbe1c6287117b97d5e32da1871 Mon Sep 17 00:00:00 2001 From: John Behm Date: Wed, 6 Mar 2024 00:58:49 +0100 Subject: [PATCH 07/76] fix callback bug & refactor session --- pool/session.go | 137 +++++++++++++++++++++++++----------------------- 1 file changed, 71 insertions(+), 66 deletions(-) diff --git a/pool/session.go b/pool/session.go index 18fc436..373fa7b 100644 --- a/pool/session.go +++ b/pool/session.go @@ -107,9 +107,9 @@ func NewSession(conn *Connection, name string, options ...SessionOption) (*Sessi bufferSize: option.BufferSize, consumers: map[string]bool{}, - channel: nil, // will be created below - confirms: nil, // will be created below - errors: nil, // will be created below + channel: nil, // will be created on connect + errors: nil, // will be created on connect + confirms: nil, // will be created on connect conn: conn, autoCloseConn: option.AutoCloseConn, @@ -182,7 +182,25 @@ func (s *Session) Close() (err error) { } }() + if s.autoCloseConn { + defer func() { + err = errors.Join(err, s.conn.Close()) + }() + } + s.cancel() + return s.close() +} + +func (s *Session) close() (err error) { + defer func() { + flush(s.errors) + flush(s.confirms) + if s.channel != nil { + s.channel = nil + } + }() + if s.channel == nil || s.channel.IsClosed() { return nil } @@ -190,12 +208,8 @@ func (s *Session) Close() (err error) { for consumer := range s.consumers { // ignore error, as at this point we cannot do anything about the error // tell server to cancel consumer deliveries. - _ = s.channel.Cancel(consumer, false) - } - - if s.autoCloseConn { - _ = s.channel.Close() - return s.conn.Close() + cerr := s.channel.Cancel(consumer, false) + err = errors.Join(err, cerr) } return s.channel.Close() @@ -213,52 +227,41 @@ func (s *Session) Connect() (err error) { } func (s *Session) connect() (err error) { + s.debug("opening session...") defer func() { // reset state in case of an error if err != nil { - s.channel = nil - s.errors = make(chan *amqp091.Error) - s.confirms = make(chan amqp091.Confirmation) - - close(s.errors) - close(s.confirms) - + s.close() s.warn(err, "failed to open session") } else { s.info("opened session") } }() - s.debug("opening session...") if s.conn.IsClosed() { // do not reconnect connection explicitly return ErrClosed } + s.close() // close any open rabbitmq channel & cleanup Go channels + channel, err := s.conn.channel() if err != nil { return fmt.Errorf("%v: %w", ErrConnectionFailed, err) } - defer func() { - if err != nil { - // close channel upon error - _ = channel.Close() - } - }() + + s.errors = make(chan *amqp091.Error, s.bufferSize) + channel.NotifyClose(s.errors) if s.confirmable { s.confirms = make(chan amqp091.Confirmation, s.bufferSize) channel.NotifyPublish(s.confirms) - err = channel.Confirm(false) if err != nil { return err } } - s.errors = make(chan *amqp091.Error, s.bufferSize) - channel.NotifyClose(s.errors) - // reset consumer tracking upon reconnect s.consumers = map[string]bool{} s.channel = channel @@ -295,14 +298,28 @@ func (s *Session) recover(ctx context.Context) error { // check if context was closed before starting a recovery. } + // check if session/channel needs to be recovered + err := s.error() + if err == nil { + return nil + } + s.warnf(err, "recovering session due to error: %v", err) + + // necessary for cleanup and to cleanup potentially dangling open sessions + // already ran into a bug, where recovery spawned infinitely many channels. + _ = s.close() + // tries to recover session forever for try := 0; ; try++ { - // try closing the channel before recovering - // in case of a bug in this library we do not want to flood the rabbitmq with - // open channels (which already happened) - _ = s.channel.Close() - err := s.conn.Recover(ctx) // recovers connection with a backoff mechanism + if s.recoverCB != nil { + // allow a user to hook into the recovery process of a session + // this is expected to not be called often, as the connection should be the main cause + // of errors and not necessarily the session. + s.recoverCB(s.conn.Name(), s.name, try, err) + } + + err = s.conn.Recover(ctx) // recovers connection with a backoff mechanism if err != nil { // upon shutdown this will fail return fmt.Errorf("failed to recover session: %w", err) @@ -316,13 +333,6 @@ func (s *Session) recover(ctx context.Context) error { s.flagged = false return nil } - - if s.conn.recoverCB != nil { - // allow a user to hook into the recovery process of a session - // this is expected to not be called often, as the connection should be the main cause - // of errors and not necessarily the session. - s.recoverCB(s.conn.Name(), s.name, try, err) - } } } @@ -1339,33 +1349,6 @@ func (s *Session) Flow(ctx context.Context, active bool) error { }) } -// FlushConfirms removes all previous confirmations pending processing. -// You can use the returned value -func (s *Session) FlushConfirms() []amqp091.Confirmation { - s.mu.Lock() - defer s.mu.Unlock() - - confirms := make([]amqp091.Confirmation, 0, len(s.confirms)) -flush: - for { - // Some weird use case where the Channel is being flooded with confirms after connection disruption - // It lead to an infinite loop when this method was called. - select { - case c, ok := <-s.confirms: - if !ok { - break flush - } - // flush confirmations in channel - confirms = append(confirms, c) - case <-s.catchShutdown(): - break flush - default: - break flush - } - } - return confirms -} - // Error returns all errors from the errors channel // and flushes all other pending errors from the channel // In case that there are no errors, nil is returned. @@ -1423,6 +1406,28 @@ func (s *Session) warn(err error, a ...any) { s.log.WithField("connection", s.conn.Name()).WithField("session", s.Name()).WithField("error", err.Error()).Warn(a...) } +func (s *Session) warnf(err error, format string, a ...any) { + s.log.WithField("connection", s.conn.Name()).WithField("session", s.Name()).WithField("error", err.Error()).Warnf(format, a...) +} + func (s *Session) debug(a ...any) { s.log.WithField("connection", s.conn.Name()).WithField("session", s.Name()).Debug(a...) } + +// flush is a helper function to flush a channel +func flush[T any](c <-chan T) []T { + var ( + slice []T + ) + for { + select { + case e, ok := <-c: + if !ok { + return slice + } + slice = append(slice, e) + default: + return slice + } + } +} From 0801433968dc6b8fc4c71934e4ae932916e01688 Mon Sep 17 00:00:00 2001 From: John Behm Date: Wed, 6 Mar 2024 00:59:50 +0100 Subject: [PATCH 08/76] make connection tests that do not use toxiproxy run in parallel --- pool/connection.go | 2 +- pool/connection_test.go | 6 +++++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/pool/connection.go b/pool/connection.go index b72e6df..bb8259c 100644 --- a/pool/connection.go +++ b/pool/connection.go @@ -273,7 +273,7 @@ func (ch *Connection) error() error { } // Recover tries to recover the connection until -// a shutdown occurs via context cancelation. +// a shutdown occurs via context cancelation or until the passed context is closed. func (ch *Connection) Recover(ctx context.Context) error { ch.mu.Lock() defer ch.mu.Unlock() diff --git a/pool/connection_test.go b/pool/connection_test.go index 51b4f47..927847a 100644 --- a/pool/connection_test.go +++ b/pool/connection_test.go @@ -14,6 +14,8 @@ import ( ) func TestNewSingleConnection(t *testing.T) { + t.Parallel() // can be run in parallel because the connection to the rabbitmq is never broken + var ( ctx = context.TODO() nextName = testutils.ConnectionNameGenerator() @@ -63,6 +65,8 @@ func TestNewSingleConnectionWithDisconnect(t *testing.T) { } func TestManyNewConnection(t *testing.T) { + t.Parallel() // can be run in parallel because the connection to the rabbitmq is never broken + var ( ctx = context.TODO() wg sync.WaitGroup @@ -90,7 +94,7 @@ func TestManyNewConnection(t *testing.T) { assert.Error(t, c.Error()) }() defer c.Close() - time.Sleep(2 * time.Second) + time.Sleep(testutils.Jitter(time.Second, 3*time.Second)) assert.NoError(t, c.Error()) }() } From bee775f544786d054aa753364d009cfcfbae4c8f Mon Sep 17 00:00:00 2001 From: John Behm Date: Wed, 6 Mar 2024 01:10:58 +0100 Subject: [PATCH 09/76] remove context parameter from ReturnConnection and recover (if necessary) the session when fetching it from the session pool --- pool/connection_pool.go | 2 +- pool/session.go | 11 +++++++++++ pool/session_pool.go | 20 +++++++++++--------- 3 files changed, 23 insertions(+), 10 deletions(-) diff --git a/pool/connection_pool.go b/pool/connection_pool.go index d3eafcf..1b9dcf3 100644 --- a/pool/connection_pool.go +++ b/pool/connection_pool.go @@ -228,7 +228,7 @@ func (cp *ConnectionPool) GetTransientConnection(ctx context.Context) (_ *Connec // If the connection is flagged, it will be recovered and returned to the pool. // If the context is canceled, the connection will be immediately returned to the pool // without any recovery attempt. -func (cp *ConnectionPool) ReturnConnection(ctx context.Context, conn *Connection, err error) { +func (cp *ConnectionPool) ReturnConnection(conn *Connection, err error) { // close transient connections if !conn.IsCached() { cp.decTransient() // decrease transient cinnections diff --git a/pool/session.go b/pool/session.go index 373fa7b..d2928d9 100644 --- a/pool/session.go +++ b/pool/session.go @@ -1414,6 +1414,17 @@ func (s *Session) debug(a ...any) { s.log.WithField("connection", s.conn.Name()).WithField("session", s.Name()).Debug(a...) } +// flush should be called when you return the session back to the pool +// TODO: improve the name of this function and decide whether it should be exported or not +func (s *Session) flush() { + s.mu.Lock() + defer s.mu.Unlock() + // do not flush the errors channel + // as it i sneeded for checking whether a session recovery is needed + + flush(s.confirms) +} + // flush is a helper function to flush a channel func flush[T any](c <-chan T) []T { var ( diff --git a/pool/session_pool.go b/pool/session_pool.go index 512150d..89d1708 100644 --- a/pool/session_pool.go +++ b/pool/session_pool.go @@ -148,11 +148,11 @@ func (sp *SessionPool) initCachedSession(id int) (*Session, error) { session, err := sp.deriveSession(sp.ctx, conn, id) if err != nil { - sp.pool.ReturnConnection(sp.ctx, conn, err) + sp.pool.ReturnConnection(conn, err) continue } - sp.pool.ReturnConnection(sp.ctx, conn, nil) + sp.pool.ReturnConnection(conn, nil) return session, nil } } @@ -175,6 +175,10 @@ func (sp *SessionPool) GetSession(ctx context.Context) (*Session, error) { return nil, fmt.Errorf("failed to get session: %w", ErrClosed) } + err := session.Recover(ctx) + if err != nil { + return nil, fmt.Errorf("failed to get session: %w", err) + } return session, nil } } @@ -235,7 +239,7 @@ func (sp *SessionPool) deriveSession(ctx context.Context, conn *Connection, id i // If Session is not a cached channel, it is simply closed here. func (sp *SessionPool) ReturnSession(session *Session, err error) { - // don't ass non-managed sessions back to the channel + // don't put non-managed sessions back into the channel if !session.IsCached() { _ = session.Close() return @@ -243,14 +247,12 @@ func (sp *SessionPool) ReturnSession(session *Session, err error) { // try recovering until context closed or shutdown session.Flag(flaggable(err)) - // healthy sessions may contain pending confirmation messages - // cleanup confirmations from previous session usage - _ = session.FlushConfirms() - // flush errors - _ = session.Error() + + // flush confirms channel + session.flush() // always put the session back into the pool - // even if it is still broken + // even if the session is still broken select { case sp.sessions <- session: default: From 49d58595754a3f68fcbb23add38f4d942ffbccc0 Mon Sep 17 00:00:00 2001 From: John Behm Date: Wed, 6 Mar 2024 01:11:43 +0100 Subject: [PATCH 10/76] derive connection names for pool tests from test function name --- pool/pool_test.go | 21 +++++++++++++++------ 1 file changed, 15 insertions(+), 6 deletions(-) diff --git a/pool/pool_test.go b/pool/pool_test.go index 07084dd..3e4d5b0 100644 --- a/pool/pool_test.go +++ b/pool/pool_test.go @@ -7,6 +7,7 @@ import ( "time" "github.com/jxsl13/amqpx" + "github.com/jxsl13/amqpx/internal/testutils" "github.com/jxsl13/amqpx/logging" "github.com/jxsl13/amqpx/pool" "github.com/stretchr/testify/assert" @@ -27,17 +28,22 @@ func TestMain(m *testing.M) { ) } -func TestNew(t *testing.T) { - ctx := context.TODO() - connections := 2 - sessions := 10 +func TestNewPool(t *testing.T) { + t.Parallel() // can be run in parallel because the connection to the rabbitmq is never broken + + var ( + ctx = context.TODO() + poolName = testutils.FuncName() + connections = 2 + sessions = 10 + ) p, err := pool.New( ctx, connectURL, connections, sessions, - pool.WithName("TestNew"), + pool.WithName(poolName), pool.WithLogger(logging.NewTestLogger(t)), ) if err != nil { @@ -58,7 +64,10 @@ func TestNew(t *testing.T) { assert.NoError(t, err) return } - time.Sleep(1 * time.Second) + time.Sleep(testutils.Jitter(1*time.Second, 3*time.Second)) + + // recovering should not be neccessary + assert.NoError(t, session.Recover(ctx)) p.ReturnSession(session, nil) }() From 91c0e8b23946802a2981529f4760f09ec611e4ad Mon Sep 17 00:00:00 2001 From: John Behm Date: Wed, 6 Mar 2024 01:12:50 +0100 Subject: [PATCH 11/76] update connection pool tests --- pool/connection_pool_test.go | 83 ++++++++++++++++++++++++++++++------ 1 file changed, 71 insertions(+), 12 deletions(-) diff --git a/pool/connection_pool_test.go b/pool/connection_pool_test.go index cf75480..fd88c5f 100644 --- a/pool/connection_pool_test.go +++ b/pool/connection_pool_test.go @@ -6,16 +6,57 @@ import ( "testing" "time" + "github.com/jxsl13/amqpx/internal/testutils" "github.com/jxsl13/amqpx/logging" "github.com/jxsl13/amqpx/pool" "github.com/stretchr/testify/assert" ) +func TestNewSingleConnectionPool(t *testing.T) { + t.Parallel() // can be run in parallel because the connection to the rabbitmq is never broken + + poolName := testutils.FuncName() + + ctx := context.TODO() + connections := 1 + p, err := pool.NewConnectionPool( + ctx, + connectURL, + connections, + pool.ConnectionPoolWithName(poolName), + pool.ConnectionPoolWithLogger(logging.NewTestLogger(t)), + ) + if err != nil { + assert.NoError(t, err) + return + } + defer p.Close() + + cctx, cancel := context.WithTimeout(ctx, 5*time.Second) + defer cancel() + + c, err := p.GetConnection(cctx) + if err != nil { + assert.NoError(t, err) + return + } + + assert.NoError(t, c.Recover(ctx)) // should not need to recover + + time.Sleep(testutils.Jitter(1*time.Second, 5*time.Second)) + p.ReturnConnection(c, nil) + +} + func TestNewConnectionPool(t *testing.T) { + t.Parallel() // can be run in parallel because the connection to the rabbitmq is never broken + + poolName := testutils.FuncName() + ctx := context.TODO() connections := 5 p, err := pool.NewConnectionPool(ctx, connectURL, connections, - pool.ConnectionPoolWithName("TestNewConnectionPool"), + pool.ConnectionPoolWithName(poolName), pool.ConnectionPoolWithLogger(logging.NewTestLogger(t)), ) if err != nil { @@ -25,28 +66,38 @@ func TestNewConnectionPool(t *testing.T) { defer p.Close() var wg sync.WaitGroup + wg.Add(connections) for i := 0; i < connections; i++ { - wg.Add(1) - go func() { + go func(i int) { defer wg.Done() - c, err := p.GetConnection(ctx) + + cctx, cancel := context.WithTimeout(ctx, 5*time.Second) + defer cancel() + + c, err := p.GetConnection(cctx) if err != nil { assert.NoError(t, err) return } - time.Sleep(5 * time.Second) - p.ReturnConnection(ctx, c, nil) - }() + + // should not need to recover + assert.NoError(t, c.Recover(ctx)) + + time.Sleep(testutils.Jitter(1*time.Second, 5*time.Second)) + p.ReturnConnection(c, nil) + }(i) } wg.Wait() } -func TestNewConnectionPoolDisconnect(t *testing.T) { +func TestNewConnectionPoolWithDisconnect(t *testing.T) { + poolName := testutils.FuncName() + ctx := context.TODO() connections := 100 p, err := pool.NewConnectionPool(ctx, connectURL, connections, - pool.ConnectionPoolWithName("TestNewConnectionPoolDisconnect"), + pool.ConnectionPoolWithName(poolName), pool.ConnectionPoolWithLogger(logging.NewTestLogger(t)), ) if err != nil { @@ -56,7 +107,9 @@ func TestNewConnectionPoolDisconnect(t *testing.T) { defer p.Close() var wg sync.WaitGroup - awaitStarted, awaitStopped := DisconnectWithStartedStopped(t, 0, 10*time.Second, 2*time.Second) + disconnectDuration := 5 * time.Second + + awaitStarted, awaitStopped := DisconnectWithStartedStopped(t, 0, 5*time.Second, disconnectDuration) defer awaitStopped() for i := 0; i < connections; i++ { @@ -72,9 +125,15 @@ func TestNewConnectionPoolDisconnect(t *testing.T) { assert.NoError(t, err) return } + maxSleep := disconnectDuration - time.Second + sleep := testutils.Jitter(maxSleep/2, maxSleep) + + time.Sleep(sleep) + cctx, cancel := context.WithTimeout(ctx, disconnectDuration) + defer cancel() + assert.NoError(t, c.Recover(cctx)) - time.Sleep(1 * time.Second) - p.ReturnConnection(ctx, c, nil) + p.ReturnConnection(c, nil) }(i) } From 8bb9559e50ed0bcf9c86a03ce03693230e4d060c Mon Sep 17 00:00:00 2001 From: John Behm Date: Wed, 6 Mar 2024 01:13:33 +0100 Subject: [PATCH 12/76] update session pool tests --- pool/session_pool_test.go | 59 +++++++++++++++++++++++++++++++++++---- 1 file changed, 54 insertions(+), 5 deletions(-) diff --git a/pool/session_pool_test.go b/pool/session_pool_test.go index 0eba311..3508da5 100644 --- a/pool/session_pool_test.go +++ b/pool/session_pool_test.go @@ -6,19 +6,68 @@ import ( "testing" "time" + "github.com/jxsl13/amqpx/internal/testutils" "github.com/jxsl13/amqpx/logging" "github.com/jxsl13/amqpx/pool" "github.com/stretchr/testify/assert" ) +func TestSingleSessionPool(t *testing.T) { + t.Parallel() // can be run in parallel because the connection to the rabbitmq is never + + var ( + poolName = testutils.FuncName() + ctx = context.TODO() + connections = 1 + sessions = 1 + ) + p, err := pool.NewConnectionPool(ctx, + connectURL, + connections, + pool.ConnectionPoolWithName(poolName), + pool.ConnectionPoolWithLogger(logging.NewTestLogger(t)), + ) + if err != nil { + assert.NoError(t, err) + return + } + + sp, err := pool.NewSessionPool( + p, + sessions, + pool.SessionPoolWithAutoCloseConnectionPool(true), + ) + if err != nil { + assert.NoError(t, err) + return + } + defer sp.Close() + + s, err := sp.GetSession(ctx) + if err != nil { + assert.NoError(t, err) + return + } + time.Sleep(testutils.Jitter(1*time.Second, 3*time.Second)) + + assert.NoError(t, s.Recover(ctx)) + + sp.ReturnSession(s, nil) +} + func TestNewSessionPool(t *testing.T) { - ctx := context.TODO() - connections := 1 - sessions := 10 + t.Parallel() // can be run in parallel because the connection to the rabbitmq is never + + var ( + poolName = testutils.FuncName() + ctx = context.TODO() + connections = 1 + sessions = 10 + ) p, err := pool.NewConnectionPool(ctx, connectURL, connections, - pool.ConnectionPoolWithName("TestNewConnectionPool"), + pool.ConnectionPoolWithName(poolName), pool.ConnectionPoolWithLogger(logging.NewTestLogger(t)), ) if err != nil { @@ -48,7 +97,7 @@ func TestNewSessionPool(t *testing.T) { assert.NoError(t, err) return } - time.Sleep(3 * time.Second) + time.Sleep(testutils.Jitter(1*time.Second, 5*time.Second)) sp.ReturnSession(s, nil) }() } From 298684fd23f7108f5ea0c78fd9bf4493a5d5220c Mon Sep 17 00:00:00 2001 From: John Behm Date: Wed, 6 Mar 2024 01:14:33 +0100 Subject: [PATCH 13/76] [WIP] split session test into smaller and easier to understand tests --- pool/session_test.go | 435 ++++++++++++++++++++++++++++++++++++------- 1 file changed, 366 insertions(+), 69 deletions(-) diff --git a/pool/session_test.go b/pool/session_test.go index 164ba29..30bc45a 100644 --- a/pool/session_test.go +++ b/pool/session_test.go @@ -3,6 +3,7 @@ package pool_test import ( "context" "sync" + "sync/atomic" "testing" "time" @@ -13,6 +14,8 @@ import ( ) func TestNewSingleSession(t *testing.T) { + t.Parallel() // can be run in parallel because the connection to the rabbitmq is never broken + var ( ctx = context.TODO() wg sync.WaitGroup @@ -35,6 +38,9 @@ func TestNewSingleSession(t *testing.T) { connectURL, connName, pool.ConnectionWithLogger(logging.NewTestLogger(t)), + pool.ConnectionWithRecoverCallback(func(name string, retry int, err error) { + assert.NoErrorf(t, err, "name=%s retry=%d", name, retry) + }), ) if err != nil { assert.NoError(t, err) @@ -48,6 +54,11 @@ func TestNewSingleSession(t *testing.T) { c, sessionName, pool.SessionWithConfirms(true), + pool.SessionWithRetryCallback( + func(operation, connName, sessionName string, retry int, err error) { + assert.NoErrorf(t, err, "operation=%s connName=%s sessionName=%s retry=%d", operation, connName, sessionName, retry) + }, + ), ) if err != nil { assert.NoError(t, err) @@ -67,6 +78,8 @@ func TestNewSingleSession(t *testing.T) { } func TestManyNewSessions(t *testing.T) { + t.Parallel() // can be run in parallel because the connection to the rabbitmq is never broken + var ( ctx = context.TODO() wg sync.WaitGroup @@ -81,6 +94,9 @@ func TestManyNewSessions(t *testing.T) { connectURL, connName, pool.ConnectionWithLogger(logging.NewTestLogger(t)), + pool.ConnectionWithRecoverCallback(func(name string, retry int, err error) { + assert.NoErrorf(t, err, "unextected connection recovery: name=%s retry=%d", name, retry) + }), ) if err != nil { assert.NoError(t, err) @@ -127,18 +143,365 @@ func TestManyNewSessions(t *testing.T) { wg.Wait() } -func TestNewSessionDisconnect(t *testing.T) { +func TestNewSessionQueueDeclarePassive(t *testing.T) { + t.Parallel() // can be run in parallel because the connection to the rabbitmq is never broken + + var ( + ctx = context.TODO() + wg sync.WaitGroup + nextConnName = testutils.ConnectionNameGenerator() + connName = nextConnName() + nextSessionName = testutils.SessionNameGenerator(connName) + sessionName = nextSessionName() + nextQueueName = testutils.QueueNameGenerator(sessionName) + ) + + conn, err := pool.NewConnection( + ctx, + connectURL, + connName, + pool.ConnectionWithLogger(logging.NewTestLogger(t)), + pool.ConnectionWithRecoverCallback(func(name string, retry int, err error) { + assert.NoErrorf(t, err, "unextected connection recovery: name=%s retry=%d", name, retry) + }), + ) + if err != nil { + assert.NoError(t, err) + return + } + defer func() { + assert.NoError(t, conn.Close()) // can be nil or error + }() + + session, err := pool.NewSession( + conn, + sessionName, + pool.SessionWithConfirms(true), + pool.SessionWithRetryCallback( + func(operation, connName, sessionName string, retry int, err error) { + assert.NoErrorf(t, err, "unexpected session recovery: operation=%s connName=%s sessionName=%s retry=%d", operation, connName, sessionName, retry) + }, + ), + ) + if err != nil { + assert.NoError(t, err) + return + } + defer func() { + assert.NoError(t, session.Close()) + }() + + for i := 0; i < 100; i++ { + func() { + qname := nextQueueName() + q, err := session.QueueDeclare(ctx, qname) + if err != nil { + assert.NoError(t, err) + return + } + assert.Equalf(t, 0, q.Consumers, "expected 0 consumers when declaring a queue: %s", qname) + + // executed upon return + defer func() { + _, err := session.QueueDelete(ctx, qname) + assert.NoErrorf(t, err, "failed to delete queue: %s", qname) + }() + + q, err = session.QueueDeclarePassive(ctx, qname) + if err != nil { + assert.NoErrorf(t, err, "QueueDeclarePassive failed for queue: %s", qname) + return + } + + assert.Equalf(t, 0, q.Consumers, "queue should not have any consumers: %s", qname) + }() + } + + wg.Wait() +} + +func TestNewSessionExchangeDeclareWithDisconnect(t *testing.T) { + var ( + ctx = context.TODO() + nextConnName = testutils.ConnectionNameGenerator() + connName = nextConnName() + nextSessionName = testutils.SessionNameGenerator(connName) + ) + + var ( + reconnectCounter int64 = 0 + expectedReconnects int64 = 1 + ) + + defer func() { + assert.Equal(t, expectedReconnects, reconnectCounter, "number of reconnection attempts") + }() + c, err := pool.NewConnection( + ctx, + connectURL, + connName, + pool.ConnectionWithLogger(logging.NewTestLogger(t)), + pool.ConnectionWithRecoverCallback(func(name string, retry int, err error) { + if retry == 0 { + atomic.AddInt64(&reconnectCounter, 1) + } + }), + ) + if err != nil { + assert.NoError(t, err) + return + } + defer func() { + c.Close() // can be nil or error + }() + + var ( + sessionName = nextSessionName() + nextExchangeName = testutils.ExchangeNameGenerator(sessionName) + exchangeName = nextExchangeName() + + start, started, stopped = DisconnectWithStartStartedStopped(t, time.Second) + ) + s, err := pool.NewSession(c, sessionName) + if err != nil { + assert.NoError(t, err) + return + } + defer s.Close() // can be nil or error + + start() // await connection loss start + started() + + err = s.ExchangeDeclare(ctx, exchangeName, pool.ExchangeKindTopic) + if err != nil { + assert.NoError(t, err) + return + } + + stopped() + + defer func() { + err := s.ExchangeDelete(ctx, exchangeName) + assert.NoError(t, err) + }() +} + +func TestNewSessionExchangeDeleteWithDisconnect(t *testing.T) { var ( ctx = context.TODO() nextConnName = testutils.ConnectionNameGenerator() connName = nextConnName() nextSessionName = testutils.SessionNameGenerator(connName) ) + + var ( + reconnectCounter int64 = 0 + expectedReconnects int64 = 1 + ) + + defer func() { + assert.Equal(t, expectedReconnects, reconnectCounter, "number of reconnection attempts") + }() c, err := pool.NewConnection( ctx, connectURL, connName, pool.ConnectionWithLogger(logging.NewTestLogger(t)), + pool.ConnectionWithRecoverCallback(func(name string, retry int, err error) { + if retry == 0 { + atomic.AddInt64(&reconnectCounter, 1) + } + }), + ) + if err != nil { + assert.NoError(t, err) + return + } + defer func() { + c.Close() // can be nil or error + }() + + var ( + sessionName = nextSessionName() + nextExchangeName = testutils.ExchangeNameGenerator(sessionName) + exchangeName = nextExchangeName() + + start, started, stopped = DisconnectWithStartStartedStopped(t, time.Second) + ) + s, err := pool.NewSession(c, sessionName) + if err != nil { + assert.NoError(t, err) + return + } + defer s.Close() // can be nil or error + + err = s.ExchangeDeclare(ctx, exchangeName, pool.ExchangeKindTopic) + if err != nil { + assert.NoError(t, err) + return + } + + start() // await connection loss start + started() + + err = s.ExchangeDelete(ctx, exchangeName) + assert.NoError(t, err) + stopped() +} + +func TestNewSessionQueueDeclareWithDisconnect(t *testing.T) { + var ( + ctx = context.TODO() + nextConnName = testutils.ConnectionNameGenerator() + connName = nextConnName() + nextSessionName = testutils.SessionNameGenerator(connName) + ) + + var ( + reconnectCounter int64 = 0 + expectedReconnects int64 = 1 + ) + + defer func() { + assert.Equal(t, expectedReconnects, reconnectCounter, "number of reconnection attempts") + }() + c, err := pool.NewConnection( + ctx, + connectURL, + connName, + pool.ConnectionWithLogger(logging.NewTestLogger(t)), + pool.ConnectionWithRecoverCallback(func(name string, retry int, err error) { + if retry == 0 { + atomic.AddInt64(&reconnectCounter, 1) + } + }), + ) + if err != nil { + assert.NoError(t, err) + return + } + defer func() { + c.Close() // can be nil or error + }() + + var ( + sessionName = nextSessionName() + nextQueueName = testutils.QueueNameGenerator(sessionName) + queueName = nextQueueName() + + start, started, stopped = DisconnectWithStartStartedStopped(t, time.Second) + ) + s, err := pool.NewSession(c, sessionName) + if err != nil { + assert.NoError(t, err) + return + } + defer s.Close() // can be nil or error + + start() // await connection loss start + started() + + _, err = s.QueueDeclare(ctx, queueName) + if err != nil { + assert.NoError(t, err) + return + } + + stopped() + + defer func() { + delMsgs, err := s.QueueDelete(ctx, queueName) + assert.NoError(t, err) + assert.Equal(t, 0, delMsgs, "expected 0 messages to be deleted") + }() +} + +func TestNewSessionQueueDeleteWithDisconnect(t *testing.T) { + var ( + ctx = context.TODO() + nextConnName = testutils.ConnectionNameGenerator() + connName = nextConnName() + nextSessionName = testutils.SessionNameGenerator(connName) + ) + + var ( + reconnectCounter int64 = 0 + expectedReconnects int64 = 1 + ) + + defer func() { + assert.Equal(t, expectedReconnects, reconnectCounter, "number of reconnection attempts") + }() + c, err := pool.NewConnection( + ctx, + connectURL, + connName, + pool.ConnectionWithLogger(logging.NewTestLogger(t)), + pool.ConnectionWithRecoverCallback(func(name string, retry int, err error) { + if retry == 0 { + atomic.AddInt64(&reconnectCounter, 1) + } + }), + ) + if err != nil { + assert.NoError(t, err) + return + } + defer func() { + c.Close() // can be nil or error + }() + + var ( + sessionName = nextSessionName() + nextQueueName = testutils.QueueNameGenerator(sessionName) + queueName = nextQueueName() + + start, started, stopped = DisconnectWithStartStartedStopped(t, time.Second) + ) + s, err := pool.NewSession(c, sessionName) + if err != nil { + assert.NoError(t, err) + return + } + defer s.Close() // can be nil or error + + _, err = s.QueueDeclare(ctx, queueName) + if err != nil { + assert.NoError(t, err) + return + } + + start() // await connection loss start + started() + + delMsgs, err := s.QueueDelete(ctx, queueName) + assert.NoError(t, err) + assert.Equal(t, 0, delMsgs, "expected 0 messages to be deleted") + stopped() +} + +func TestNewSessionWithDisconnect(t *testing.T) { + var ( + ctx = context.TODO() + nextConnName = testutils.ConnectionNameGenerator() + connName = nextConnName() + nextSessionName = testutils.SessionNameGenerator(connName) + ) + + var reconnectCounter int64 = 0 + defer func() { + assert.Equal(t, 10, reconnectCounter-1) + }() + c, err := pool.NewConnection( + ctx, + connectURL, + connName, + pool.ConnectionWithLogger(logging.NewTestLogger(t)), + pool.ConnectionWithRecoverCallback(func(name string, retry int, err error) { + if retry == 0 { + atomic.AddInt64(&reconnectCounter, 1) + } + }), ) if err != nil { assert.NoError(t, err) @@ -186,6 +549,7 @@ func TestNewSessionDisconnect(t *testing.T) { return } defer func() { + // INFO: does not lead to a recovery start10() started10() @@ -196,7 +560,7 @@ func TestNewSessionDisconnect(t *testing.T) { start() // await connection loss start started() - err = s.ExchangeDeclare(ctx, exchangeName, "topic") + err = s.ExchangeDeclare(ctx, exchangeName, pool.ExchangeKindTopic) if err != nil { assert.NoError(t, err) return @@ -324,70 +688,3 @@ func TestNewSessionDisconnect(t *testing.T) { wg.Wait() } - -func TestNewSessionQueueDeclarePassive(t *testing.T) { - t.Parallel() - - var ( - ctx = context.TODO() - wg sync.WaitGroup - nextConnName = testutils.ConnectionNameGenerator() - connName = nextConnName() - nextSessionName = testutils.SessionNameGenerator(connName) - sessionName = nextSessionName() - nextQueueName = testutils.QueueNameGenerator(sessionName) - ) - - conn, err := pool.NewConnection( - ctx, - connectURL, - connName, - pool.ConnectionWithLogger(logging.NewTestLogger(t)), - ) - if err != nil { - assert.NoError(t, err) - return - } - defer func() { - conn.Close() // can be nil or error - }() - - session, err := pool.NewSession( - conn, - sessionName, - pool.SessionWithConfirms(true), - ) - if err != nil { - assert.NoError(t, err) - return - } - defer func() { - assert.NoError(t, session.Close()) - }() - - for i := 0; i < 100; i++ { - qname := nextQueueName() - q, err := session.QueueDeclare(ctx, qname) - if err != nil { - assert.NoError(t, err) - return - } - assert.Equalf(t, 0, q.Consumers, "expected 0 consumers when declaring a queue: %s", qname) - - // executed upon return - defer func() { - _, err := session.QueueDelete(ctx, qname) - assert.NoErrorf(t, err, "failed to delete queue: %s", qname) - }() - - q, err = session.QueueDeclarePassive(ctx, qname) - if err != nil { - assert.NoErrorf(t, err, "QueueDeclarePassive failed for queue: %s", qname) - return - } - - assert.Equalf(t, 0, q.Consumers, "queue should not have any consumers: %s", qname) - } - - wg.Wait() -} From af569a097b3fdf42c791544a10e23efdf1f60df8 Mon Sep 17 00:00:00 2001 From: John Behm Date: Thu, 7 Mar 2024 21:14:11 +0100 Subject: [PATCH 14/76] add healthy rabbitmq connection --- docker-compose.yaml | 3 ++- docker/toxiproxy.json | 6 ++++++ 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/docker-compose.yaml b/docker-compose.yaml index b6f35ac..40b404f 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -50,7 +50,8 @@ services: - -config=/toxiproxy.json ports: - 8474:8474 - - 5672:5672 # normal rabbitmq + - 5671:5671 # healthy rabbitmq + - 5672:5672 # normal toggleable rabbitmq - 5673:5673 # broken rabbitmq networks: - rabbitnet diff --git a/docker/toxiproxy.json b/docker/toxiproxy.json index bfd5437..ed65fbf 100644 --- a/docker/toxiproxy.json +++ b/docker/toxiproxy.json @@ -1,4 +1,10 @@ [ + { + "name": "rabbitmq-healthy", + "listen": "[::]:5671", + "upstream": "rabbitmq:5672", + "enabled": true + }, { "name": "rabbitmq", "listen": "[::]:5672", From 48c1bf14862f5ecc25e52b8ffe77c2741a54969f Mon Sep 17 00:00:00 2001 From: John Behm Date: Thu, 7 Mar 2024 21:14:22 +0100 Subject: [PATCH 15/76] fix message generator --- internal/testutils/generator.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/testutils/generator.go b/internal/testutils/generator.go index ef292d5..9ca95f0 100644 --- a/internal/testutils/generator.go +++ b/internal/testutils/generator.go @@ -180,6 +180,6 @@ func MessageGenerator(queueOrExchangeName string, options ...GeneratorOption) (n cnt := counter counter++ mu.Unlock() - return fmt.Sprintf("%s-message-%d-%s", queueOrExchangeName, cnt, opts.ToSuffix()) + return fmt.Sprintf("%s-message-%d%s", queueOrExchangeName, cnt, opts.ToSuffix()) } } From c464b33742db8b9ecfd3ab7abf8237fe0f02fd38 Mon Sep 17 00:00:00 2001 From: John Behm Date: Thu, 7 Mar 2024 21:15:40 +0100 Subject: [PATCH 16/76] improve ConsumeAsyncN test utility & add utility for creating test sessions --- pool/utils_test.go | 120 ++++++++++++++++++++++++++++++++++----------- 1 file changed, 92 insertions(+), 28 deletions(-) diff --git a/pool/utils_test.go b/pool/utils_test.go index 978d97d..37967ae 100644 --- a/pool/utils_test.go +++ b/pool/utils_test.go @@ -6,9 +6,11 @@ import ( "testing" "time" + "github.com/jxsl13/amqpx/internal/testutils" "github.com/jxsl13/amqpx/logging" "github.com/jxsl13/amqpx/pool" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" ) func ConsumeAsyncN( @@ -21,48 +23,54 @@ func ConsumeAsyncN( messageGenerator func() string, n int, ) { - delivery, err := s.Consume( - queueName, - pool.ConsumeOptions{ - ConsumerTag: consumerName, - Exclusive: true, - }, - ) - if err != nil { - assert.NoError(t, err) - return - } wg.Add(1) go func(wg *sync.WaitGroup, messageGenerator func() string, n int) { defer wg.Done() cctx, ccancel := context.WithCancel(ctx) defer ccancel() + log := logging.NewTestLogger(t) msgsReceived := 0 defer func() { assert.Equal(t, n, msgsReceived) }() + outer: for { - select { - case <-cctx.Done(): + delivery, err := s.Consume( + queueName, + pool.ConsumeOptions{ + ConsumerTag: consumerName, + Exclusive: true, + }, + ) + if err != nil { + assert.NoError(t, err) return - case val, ok := <-delivery: - if !ok { - return - } - err := val.Ack(false) - if err != nil { - assert.NoError(t, err) + } + + for { + select { + case <-cctx.Done(): return - } + case val, ok := <-delivery: + if !ok { + continue outer + } + err := val.Ack(false) + if err != nil { + assert.NoError(t, err) + return + } - receivedMsg := string(val.Body) - assert.Equal(t, messageGenerator(), receivedMsg) - msgsReceived++ - if msgsReceived == n { - logging.NewTestLogger(t).Infof("consumed %d messages, closing consumer", n) - ccancel() + receivedMsg := string(val.Body) + assert.Equal(t, messageGenerator(), receivedMsg) + log.Infof("consumed message: %s", receivedMsg) + msgsReceived++ + if msgsReceived == n { + logging.NewTestLogger(t).Infof("consumed %d messages, closing consumer", n) + ccancel() + } } } } @@ -140,8 +148,9 @@ func DeclareExchangeQueue( } defer func() { if err != nil { - _, e := s.QueueDelete(ctx, queueName) + deleted, e := s.QueueDelete(ctx, queueName) assert.NoError(t, e) + assert.Equalf(t, 0, deleted, "expected 0 deleted messages, got %d for queue %s", deleted, queueName) } }() @@ -164,3 +173,58 @@ func DeclareExchangeQueue( assert.NoError(t, s.ExchangeDelete(ctx, exchangeName)) } } + +func NewSession(t *testing.T, ctx context.Context, connectURL, connectionName string, options ...pool.ConnectionOption) (_ *pool.Session, cleanup func()) { + cleanup = func() {} + log := logging.NewTestLogger(t) + c, err := pool.NewConnection( + ctx, + connectURL, + connectionName, + append([]pool.ConnectionOption{ + pool.ConnectionWithLogger(log), + }, options...)...) + if err != nil { + require.NoError(t, err) + return nil, cleanup + } + nextSessionName := testutils.SessionNameGenerator(connectionName) + s, err := pool.NewSession( + c, + nextSessionName(), + pool.SessionWithConfirms(true), + pool.SessionWithLogger(log), + pool.SessionWithRetryCallback(func(operation, connName, sessionName string, retry int, err error) { + log.Infof("retrying %s on connection %s, session %s, attempt %d, error: %s", operation, connName, sessionName, retry, err) + }), + ) + if err != nil { + require.NoError(t, err) + return nil, cleanup + } + return s, func() { + assert.NoError(t, s.Close()) + assert.NoError(t, c.Close()) + } +} + +func AssertConnectionReconnectAttempts(t *testing.T, n int) (callback pool.ConnectionRecoverCallback, deferredAssert func()) { + var ( + i int + mu sync.Mutex + log = logging.NewTestLogger(t) + ) + return func(name string, retry int, err error) { + if retry == 0 { + log.Infof("connection %s retry %d, error: %v", name, retry, err) + mu.Lock() + i++ + mu.Unlock() + } + }, + func() { + mu.Lock() + defer mu.Unlock() + assert.Equal(t, n, i) + } +} From f547dee283b84445732b40704ab80da89b067030 Mon Sep 17 00:00:00 2001 From: John Behm Date: Thu, 7 Mar 2024 21:17:13 +0100 Subject: [PATCH 17/76] add Disconnect utility --- pool/toxiproxy_client_test.go | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/pool/toxiproxy_client_test.go b/pool/toxiproxy_client_test.go index dd434c9..becf856 100644 --- a/pool/toxiproxy_client_test.go +++ b/pool/toxiproxy_client_test.go @@ -183,6 +183,23 @@ func DisconnectWithStartedStopped(t *testing.T, block, startIn, duration time.Du } } +func Disconnect(t *testing.T, duration time.Duration) (started, stopped func()) { + var ( + disconnectOnce sync.Once + reconnectOnce sync.Once + ) + + disconnect, awaitStarted, awaitStopped := DisconnectWithStartStartedStopped(t, duration) + return func() { + disconnectOnce.Do(func() { + disconnect() + awaitStarted() + }) + }, func() { + reconnectOnce.Do(awaitStopped) + } +} + // block current thread // timeout: how long the asynchronous goroutine needs to wait until it disables the connection // duration: how long the asynchronous goroutine wait suntil it reenables the connection From d3b8abd5dbb9765cc32d8c051ab001aeec6ac2ef Mon Sep 17 00:00:00 2001 From: John Behm Date: Thu, 7 Mar 2024 21:17:36 +0100 Subject: [PATCH 18/76] add healthy connect url to tests --- pool/pool_test.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/pool/pool_test.go b/pool/pool_test.go index 3e4d5b0..4e520a0 100644 --- a/pool/pool_test.go +++ b/pool/pool_test.go @@ -15,8 +15,9 @@ import ( ) var ( - connectURL = amqpx.NewURL("localhost", 5672, "admin", "password") - brokenConnectURL = amqpx.NewURL("localhost", 5673, "admin", "password") + healthyConnectURL = amqpx.NewURL("localhost", 5671, "admin", "password") + connectURL = amqpx.NewURL("localhost", 5672, "admin", "password") + brokenConnectURL = amqpx.NewURL("localhost", 5673, "admin", "password") ) func TestMain(m *testing.M) { From 88a8e37917b1e440538ad5418072cfc1dbae5ce4 Mon Sep 17 00:00:00 2001 From: John Behm Date: Thu, 7 Mar 2024 23:04:15 +0100 Subject: [PATCH 19/76] improve tests & allow running all tests in parallel. --- Makefile | 6 + docker-compose.yaml | 110 +++++- docker/toxiproxy.json | 596 +++++++++++++++++++++++++++++- internal/testutils/connect_url.go | 96 +++++ pool/connection_pool_test.go | 28 +- pool/connection_test.go | 86 +++-- pool/pool_test.go | 9 +- pool/publisher_test.go | 5 +- pool/session_pool_test.go | 4 +- pool/session_test.go | 444 +++++++++++++++------- pool/subscriber_test.go | 8 +- pool/toxiproxy_client_test.go | 24 +- pool/utils_test.go | 149 ++++---- 13 files changed, 1298 insertions(+), 267 deletions(-) create mode 100644 internal/testutils/connect_url.go diff --git a/Makefile b/Makefile index fd6dfa1..7b6465b 100644 --- a/Makefile +++ b/Makefile @@ -9,3 +9,9 @@ down: test: go test -timeout 900s -v -race -count=1 ./... + +count-tests: + grep -REn 'func Test.+\(.+testing\.T.*\)' . | wc -l + +count-disconnect-tests: + grep -REn 'func Test.+WithDisconnect.*\(.+testing\.T.*\)' . | wc -l \ No newline at end of file diff --git a/docker-compose.yaml b/docker-compose.yaml index 40b404f..273d630 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -7,7 +7,7 @@ services: #- '4369:4369' #- '5551:5551' #- '5552:5552' - #- '5672:5672' # proxied through toxiproxy + - 5671:5672 # healthy rabbitmq #- '25672:25672' # user interface - '15672:15672' @@ -28,7 +28,7 @@ services: #- '14369:4369' #- '15551:5551' #- '15552:5552' - #- '5672:5672' # proxied through toxiproxy + - 5670:5672 # broken rabbitmq #- '35672:25672' # user interface - '25672:15672' @@ -49,10 +49,108 @@ services: - -host=0.0.0.0 - -config=/toxiproxy.json ports: - - 8474:8474 - - 5671:5671 # healthy rabbitmq - - 5672:5672 # normal toggleable rabbitmq - - 5673:5673 # broken rabbitmq + - 8474:8474 # toxyproxy port + - 5672:5672 # rabbitmq-5672 + - 5673:5673 # rabbitmq-5673 + - 5674:5674 # rabbitmq-5674 + - 5675:5675 # rabbitmq-5675 + - 5676:5676 # rabbitmq-5676 + - 5677:5677 # rabbitmq-5677 + - 5678:5678 # rabbitmq-5678 + - 5679:5679 # rabbitmq-5679 + - 5680:5680 # rabbitmq-5680 + - 5681:5681 # rabbitmq-5681 + - 5682:5682 # rabbitmq-5682 + - 5683:5683 # rabbitmq-5683 + - 5684:5684 # rabbitmq-5684 + - 5685:5685 # rabbitmq-5685 + - 5686:5686 # rabbitmq-5686 + - 5687:5687 # rabbitmq-5687 + - 5688:5688 # rabbitmq-5688 + - 5689:5689 # rabbitmq-5689 + - 5690:5690 # rabbitmq-5690 + - 5691:5691 # rabbitmq-5691 + - 5692:5692 # rabbitmq-5692 + - 5693:5693 # rabbitmq-5693 + - 5694:5694 # rabbitmq-5694 + - 5695:5695 # rabbitmq-5695 + - 5696:5696 # rabbitmq-5696 + - 5697:5697 # rabbitmq-5697 + - 5698:5698 # rabbitmq-5698 + - 5699:5699 # rabbitmq-5699 + - 5700:5700 # rabbitmq-5700 + - 5701:5701 # rabbitmq-5701 + - 5702:5702 # rabbitmq-5702 + - 5703:5703 # rabbitmq-5703 + - 5704:5704 # rabbitmq-5704 + - 5705:5705 # rabbitmq-5705 + - 5706:5706 # rabbitmq-5706 + - 5707:5707 # rabbitmq-5707 + - 5708:5708 # rabbitmq-5708 + - 5709:5709 # rabbitmq-5709 + - 5710:5710 # rabbitmq-5710 + - 5711:5711 # rabbitmq-5711 + - 5712:5712 # rabbitmq-5712 + - 5713:5713 # rabbitmq-5713 + - 5714:5714 # rabbitmq-5714 + - 5715:5715 # rabbitmq-5715 + - 5716:5716 # rabbitmq-5716 + - 5717:5717 # rabbitmq-5717 + - 5718:5718 # rabbitmq-5718 + - 5719:5719 # rabbitmq-5719 + - 5720:5720 # rabbitmq-5720 + - 5721:5721 # rabbitmq-5721 + - 5722:5722 # rabbitmq-5722 + - 5723:5723 # rabbitmq-5723 + - 5724:5724 # rabbitmq-5724 + - 5725:5725 # rabbitmq-5725 + - 5726:5726 # rabbitmq-5726 + - 5727:5727 # rabbitmq-5727 + - 5728:5728 # rabbitmq-5728 + - 5729:5729 # rabbitmq-5729 + - 5730:5730 # rabbitmq-5730 + - 5731:5731 # rabbitmq-5731 + - 5732:5732 # rabbitmq-5732 + - 5733:5733 # rabbitmq-5733 + - 5734:5734 # rabbitmq-5734 + - 5735:5735 # rabbitmq-5735 + - 5736:5736 # rabbitmq-5736 + - 5737:5737 # rabbitmq-5737 + - 5738:5738 # rabbitmq-5738 + - 5739:5739 # rabbitmq-5739 + - 5740:5740 # rabbitmq-5740 + - 5741:5741 # rabbitmq-5741 + - 5742:5742 # rabbitmq-5742 + - 5743:5743 # rabbitmq-5743 + - 5744:5744 # rabbitmq-5744 + - 5745:5745 # rabbitmq-5745 + - 5746:5746 # rabbitmq-5746 + - 5747:5747 # rabbitmq-5747 + - 5748:5748 # rabbitmq-5748 + - 5749:5749 # rabbitmq-5749 + - 5750:5750 # rabbitmq-5750 + - 5751:5751 # rabbitmq-5751 + - 5752:5752 # rabbitmq-5752 + - 5753:5753 # rabbitmq-5753 + - 5754:5754 # rabbitmq-5754 + - 5755:5755 # rabbitmq-5755 + - 5756:5756 # rabbitmq-5756 + - 5757:5757 # rabbitmq-5757 + - 5758:5758 # rabbitmq-5758 + - 5759:5759 # rabbitmq-5759 + - 5760:5760 # rabbitmq-5760 + - 5761:5761 # rabbitmq-5761 + - 5762:5762 # rabbitmq-5762 + - 5763:5763 # rabbitmq-5763 + - 5764:5764 # rabbitmq-5764 + - 5765:5765 # rabbitmq-5765 + - 5766:5766 # rabbitmq-5766 + - 5767:5767 # rabbitmq-5767 + - 5768:5768 # rabbitmq-5768 + - 5769:5769 # rabbitmq-5769 + - 5770:5770 # rabbitmq-5770 + - 5771:5771 # rabbitmq-5771 + networks: - rabbitnet volumes: diff --git a/docker/toxiproxy.json b/docker/toxiproxy.json index ed65fbf..cee8368 100644 --- a/docker/toxiproxy.json +++ b/docker/toxiproxy.json @@ -1,20 +1,602 @@ [ { - "name": "rabbitmq-healthy", - "listen": "[::]:5671", + "name": "rabbitmq-5672", + "listen": "[::]:5672", "upstream": "rabbitmq:5672", "enabled": true }, { - "name": "rabbitmq", - "listen": "[::]:5672", + "name": "rabbitmq-5673", + "listen": "[::]:5673", "upstream": "rabbitmq:5672", "enabled": true }, { - "name": "rabbitmq-broken", - "listen": "[::]:5673", - "upstream": "rabbitmq-broken:5672", + "name": "rabbitmq-5674", + "listen": "[::]:5674", + "upstream": "rabbitmq:5672", + "enabled": true + }, + { + "name": "rabbitmq-5675", + "listen": "[::]:5675", + "upstream": "rabbitmq:5672", + "enabled": true + }, + { + "name": "rabbitmq-5676", + "listen": "[::]:5676", + "upstream": "rabbitmq:5672", + "enabled": true + }, + { + "name": "rabbitmq-5677", + "listen": "[::]:5677", + "upstream": "rabbitmq:5672", + "enabled": true + }, + { + "name": "rabbitmq-5678", + "listen": "[::]:5678", + "upstream": "rabbitmq:5672", + "enabled": true + }, + { + "name": "rabbitmq-5679", + "listen": "[::]:5679", + "upstream": "rabbitmq:5672", + "enabled": true + }, + { + "name": "rabbitmq-5680", + "listen": "[::]:5680", + "upstream": "rabbitmq:5672", + "enabled": true + }, + { + "name": "rabbitmq-5681", + "listen": "[::]:5681", + "upstream": "rabbitmq:5672", + "enabled": true + }, + { + "name": "rabbitmq-5682", + "listen": "[::]:5682", + "upstream": "rabbitmq:5672", + "enabled": true + }, + { + "name": "rabbitmq-5683", + "listen": "[::]:5683", + "upstream": "rabbitmq:5672", + "enabled": true + }, + { + "name": "rabbitmq-5684", + "listen": "[::]:5684", + "upstream": "rabbitmq:5672", + "enabled": true + }, + { + "name": "rabbitmq-5685", + "listen": "[::]:5685", + "upstream": "rabbitmq:5672", + "enabled": true + }, + { + "name": "rabbitmq-5686", + "listen": "[::]:5686", + "upstream": "rabbitmq:5672", + "enabled": true + }, + { + "name": "rabbitmq-5687", + "listen": "[::]:5687", + "upstream": "rabbitmq:5672", + "enabled": true + }, + { + "name": "rabbitmq-5688", + "listen": "[::]:5688", + "upstream": "rabbitmq:5672", + "enabled": true + }, + { + "name": "rabbitmq-5689", + "listen": "[::]:5689", + "upstream": "rabbitmq:5672", + "enabled": true + }, + { + "name": "rabbitmq-5690", + "listen": "[::]:5690", + "upstream": "rabbitmq:5672", + "enabled": true + }, + { + "name": "rabbitmq-5691", + "listen": "[::]:5691", + "upstream": "rabbitmq:5672", + "enabled": true + }, + { + "name": "rabbitmq-5692", + "listen": "[::]:5692", + "upstream": "rabbitmq:5672", + "enabled": true + }, + { + "name": "rabbitmq-5693", + "listen": "[::]:5693", + "upstream": "rabbitmq:5672", + "enabled": true + }, + { + "name": "rabbitmq-5694", + "listen": "[::]:5694", + "upstream": "rabbitmq:5672", + "enabled": true + }, + { + "name": "rabbitmq-5695", + "listen": "[::]:5695", + "upstream": "rabbitmq:5672", + "enabled": true + }, + { + "name": "rabbitmq-5696", + "listen": "[::]:5696", + "upstream": "rabbitmq:5672", + "enabled": true + }, + { + "name": "rabbitmq-5697", + "listen": "[::]:5697", + "upstream": "rabbitmq:5672", + "enabled": true + }, + { + "name": "rabbitmq-5698", + "listen": "[::]:5698", + "upstream": "rabbitmq:5672", + "enabled": true + }, + { + "name": "rabbitmq-5699", + "listen": "[::]:5699", + "upstream": "rabbitmq:5672", + "enabled": true + }, + { + "name": "rabbitmq-5700", + "listen": "[::]:5700", + "upstream": "rabbitmq:5672", + "enabled": true + }, + { + "name": "rabbitmq-5701", + "listen": "[::]:5701", + "upstream": "rabbitmq:5672", + "enabled": true + }, + { + "name": "rabbitmq-5702", + "listen": "[::]:5702", + "upstream": "rabbitmq:5672", + "enabled": true + }, + { + "name": "rabbitmq-5703", + "listen": "[::]:5703", + "upstream": "rabbitmq:5672", + "enabled": true + }, + { + "name": "rabbitmq-5704", + "listen": "[::]:5704", + "upstream": "rabbitmq:5672", + "enabled": true + }, + { + "name": "rabbitmq-5705", + "listen": "[::]:5705", + "upstream": "rabbitmq:5672", + "enabled": true + }, + { + "name": "rabbitmq-5706", + "listen": "[::]:5706", + "upstream": "rabbitmq:5672", + "enabled": true + }, + { + "name": "rabbitmq-5707", + "listen": "[::]:5707", + "upstream": "rabbitmq:5672", + "enabled": true + }, + { + "name": "rabbitmq-5708", + "listen": "[::]:5708", + "upstream": "rabbitmq:5672", + "enabled": true + }, + { + "name": "rabbitmq-5709", + "listen": "[::]:5709", + "upstream": "rabbitmq:5672", + "enabled": true + }, + { + "name": "rabbitmq-5710", + "listen": "[::]:5710", + "upstream": "rabbitmq:5672", + "enabled": true + }, + { + "name": "rabbitmq-5711", + "listen": "[::]:5711", + "upstream": "rabbitmq:5672", + "enabled": true + }, + { + "name": "rabbitmq-5712", + "listen": "[::]:5712", + "upstream": "rabbitmq:5672", + "enabled": true + }, + { + "name": "rabbitmq-5713", + "listen": "[::]:5713", + "upstream": "rabbitmq:5672", + "enabled": true + }, + { + "name": "rabbitmq-5714", + "listen": "[::]:5714", + "upstream": "rabbitmq:5672", + "enabled": true + }, + { + "name": "rabbitmq-5715", + "listen": "[::]:5715", + "upstream": "rabbitmq:5672", + "enabled": true + }, + { + "name": "rabbitmq-5716", + "listen": "[::]:5716", + "upstream": "rabbitmq:5672", + "enabled": true + }, + { + "name": "rabbitmq-5717", + "listen": "[::]:5717", + "upstream": "rabbitmq:5672", + "enabled": true + }, + { + "name": "rabbitmq-5718", + "listen": "[::]:5718", + "upstream": "rabbitmq:5672", + "enabled": true + }, + { + "name": "rabbitmq-5719", + "listen": "[::]:5719", + "upstream": "rabbitmq:5672", + "enabled": true + }, + { + "name": "rabbitmq-5720", + "listen": "[::]:5720", + "upstream": "rabbitmq:5672", + "enabled": true + }, + { + "name": "rabbitmq-5721", + "listen": "[::]:5721", + "upstream": "rabbitmq:5672", + "enabled": true + }, + { + "name": "rabbitmq-5722", + "listen": "[::]:5722", + "upstream": "rabbitmq:5672", + "enabled": true + }, + { + "name": "rabbitmq-5723", + "listen": "[::]:5723", + "upstream": "rabbitmq:5672", + "enabled": true + }, + { + "name": "rabbitmq-5724", + "listen": "[::]:5724", + "upstream": "rabbitmq:5672", + "enabled": true + }, + { + "name": "rabbitmq-5725", + "listen": "[::]:5725", + "upstream": "rabbitmq:5672", + "enabled": true + }, + { + "name": "rabbitmq-5726", + "listen": "[::]:5726", + "upstream": "rabbitmq:5672", + "enabled": true + }, + { + "name": "rabbitmq-5727", + "listen": "[::]:5727", + "upstream": "rabbitmq:5672", + "enabled": true + }, + { + "name": "rabbitmq-5728", + "listen": "[::]:5728", + "upstream": "rabbitmq:5672", + "enabled": true + }, + { + "name": "rabbitmq-5729", + "listen": "[::]:5729", + "upstream": "rabbitmq:5672", + "enabled": true + }, + { + "name": "rabbitmq-5730", + "listen": "[::]:5730", + "upstream": "rabbitmq:5672", + "enabled": true + }, + { + "name": "rabbitmq-5731", + "listen": "[::]:5731", + "upstream": "rabbitmq:5672", + "enabled": true + }, + { + "name": "rabbitmq-5732", + "listen": "[::]:5732", + "upstream": "rabbitmq:5672", + "enabled": true + }, + { + "name": "rabbitmq-5733", + "listen": "[::]:5733", + "upstream": "rabbitmq:5672", + "enabled": true + }, + { + "name": "rabbitmq-5734", + "listen": "[::]:5734", + "upstream": "rabbitmq:5672", + "enabled": true + }, + { + "name": "rabbitmq-5735", + "listen": "[::]:5735", + "upstream": "rabbitmq:5672", + "enabled": true + }, + { + "name": "rabbitmq-5736", + "listen": "[::]:5736", + "upstream": "rabbitmq:5672", + "enabled": true + }, + { + "name": "rabbitmq-5737", + "listen": "[::]:5737", + "upstream": "rabbitmq:5672", + "enabled": true + }, + { + "name": "rabbitmq-5738", + "listen": "[::]:5738", + "upstream": "rabbitmq:5672", + "enabled": true + }, + { + "name": "rabbitmq-5739", + "listen": "[::]:5739", + "upstream": "rabbitmq:5672", + "enabled": true + }, + { + "name": "rabbitmq-5740", + "listen": "[::]:5740", + "upstream": "rabbitmq:5672", + "enabled": true + }, + { + "name": "rabbitmq-5741", + "listen": "[::]:5741", + "upstream": "rabbitmq:5672", + "enabled": true + }, + { + "name": "rabbitmq-5742", + "listen": "[::]:5742", + "upstream": "rabbitmq:5672", + "enabled": true + }, + { + "name": "rabbitmq-5743", + "listen": "[::]:5743", + "upstream": "rabbitmq:5672", + "enabled": true + }, + { + "name": "rabbitmq-5744", + "listen": "[::]:5744", + "upstream": "rabbitmq:5672", + "enabled": true + }, + { + "name": "rabbitmq-5745", + "listen": "[::]:5745", + "upstream": "rabbitmq:5672", + "enabled": true + }, + { + "name": "rabbitmq-5746", + "listen": "[::]:5746", + "upstream": "rabbitmq:5672", + "enabled": true + }, + { + "name": "rabbitmq-5747", + "listen": "[::]:5747", + "upstream": "rabbitmq:5672", + "enabled": true + }, + { + "name": "rabbitmq-5748", + "listen": "[::]:5748", + "upstream": "rabbitmq:5672", + "enabled": true + }, + { + "name": "rabbitmq-5749", + "listen": "[::]:5749", + "upstream": "rabbitmq:5672", + "enabled": true + }, + { + "name": "rabbitmq-5750", + "listen": "[::]:5750", + "upstream": "rabbitmq:5672", + "enabled": true + }, + { + "name": "rabbitmq-5751", + "listen": "[::]:5751", + "upstream": "rabbitmq:5672", + "enabled": true + }, + { + "name": "rabbitmq-5752", + "listen": "[::]:5752", + "upstream": "rabbitmq:5672", + "enabled": true + }, + { + "name": "rabbitmq-5753", + "listen": "[::]:5753", + "upstream": "rabbitmq:5672", + "enabled": true + }, + { + "name": "rabbitmq-5754", + "listen": "[::]:5754", + "upstream": "rabbitmq:5672", + "enabled": true + }, + { + "name": "rabbitmq-5755", + "listen": "[::]:5755", + "upstream": "rabbitmq:5672", + "enabled": true + }, + { + "name": "rabbitmq-5756", + "listen": "[::]:5756", + "upstream": "rabbitmq:5672", + "enabled": true + }, + { + "name": "rabbitmq-5757", + "listen": "[::]:5757", + "upstream": "rabbitmq:5672", + "enabled": true + }, + { + "name": "rabbitmq-5758", + "listen": "[::]:5758", + "upstream": "rabbitmq:5672", + "enabled": true + }, + { + "name": "rabbitmq-5759", + "listen": "[::]:5759", + "upstream": "rabbitmq:5672", + "enabled": true + }, + { + "name": "rabbitmq-5760", + "listen": "[::]:5760", + "upstream": "rabbitmq:5672", + "enabled": true + }, + { + "name": "rabbitmq-5761", + "listen": "[::]:5761", + "upstream": "rabbitmq:5672", + "enabled": true + }, + { + "name": "rabbitmq-5762", + "listen": "[::]:5762", + "upstream": "rabbitmq:5672", + "enabled": true + }, + { + "name": "rabbitmq-5763", + "listen": "[::]:5763", + "upstream": "rabbitmq:5672", + "enabled": true + }, + { + "name": "rabbitmq-5764", + "listen": "[::]:5764", + "upstream": "rabbitmq:5672", + "enabled": true + }, + { + "name": "rabbitmq-5765", + "listen": "[::]:5765", + "upstream": "rabbitmq:5672", + "enabled": true + }, + { + "name": "rabbitmq-5766", + "listen": "[::]:5766", + "upstream": "rabbitmq:5672", + "enabled": true + }, + { + "name": "rabbitmq-5767", + "listen": "[::]:5767", + "upstream": "rabbitmq:5672", + "enabled": true + }, + { + "name": "rabbitmq-5768", + "listen": "[::]:5768", + "upstream": "rabbitmq:5672", + "enabled": true + }, + { + "name": "rabbitmq-5769", + "listen": "[::]:5769", + "upstream": "rabbitmq:5672", + "enabled": true + }, + { + "name": "rabbitmq-5770", + "listen": "[::]:5770", + "upstream": "rabbitmq:5672", + "enabled": true + }, + { + "name": "rabbitmq-5771", + "listen": "[::]:5771", + "upstream": "rabbitmq:5672", "enabled": true } ] \ No newline at end of file diff --git a/internal/testutils/connect_url.go b/internal/testutils/connect_url.go new file mode 100644 index 0000000..27703c5 --- /dev/null +++ b/internal/testutils/connect_url.go @@ -0,0 +1,96 @@ +package testutils + +import ( + "encoding/json" + "fmt" + "sync" + "testing" +) + +var ( + NumTests = 100 + Upstream = "rabbitmq:5672" + Username = "admin" + Password = "password" + Hostname = "localhost" + BrokenConnectURL = fmt.Sprintf("amqp://%s:%s@%s:%d/", Username, Password, Hostname, 5670) + HealthyConnectURL = fmt.Sprintf("amqp://%s:%s@%s:%d/", Username, Password, Hostname, 5671) +) + +var ( + port int = 5672 + mu sync.Mutex +) + +func NextConnectURL() (proxyName, connectURL string, port int) { + proxyPort := NextPort() + return fmt.Sprintf("rabbitmq-%d", proxyPort), + fmt.Sprintf("amqp://%s:%s@%s:%d/", Username, Password, Hostname, proxyPort), + proxyPort +} + +func NextPort() int { + mu.Lock() + defer mu.Unlock() + defer func() { + loop: + for { + port++ + switch port { + case 5670, 5671, 8474: + default: + break loop + } + + } + }() + return port +} + +/* +[ + + { + "name": "rabbitmq", + "listen": "[::]:5672", + "upstream": "rabbitmq:5672", + "enabled": true + } + +] +*/ +func TestGenerateProxyConfig(_ *testing.T) { + list := []struct { + Name string `json:"name"` + Listen string `json:"listen"` + Upstream string `json:"upstream"` + Enabled bool `json:"enabled"` + }{} + + for i := 0; i < NumTests; i++ { + _, proxyName, proxyPort := NextConnectURL() + list = append(list, struct { + Name string `json:"name"` + Listen string `json:"listen"` + Upstream string `json:"upstream"` + Enabled bool `json:"enabled"` + }{ + Name: proxyName, + Listen: fmt.Sprintf("[::]:%d", proxyPort), + Upstream: Upstream, + Enabled: true, + }) + } + + data, _ := json.MarshalIndent(list, "", " ") + fmt.Println(string(data)) +} + +func TestGenerateDockerPortForwards(_ *testing.T) { + str := "" + for i := 0; i < NumTests; i++ { + proxyName, _, proxyPort := NextConnectURL() + str += fmt.Sprintf(" - %[1]d:%[1]d # %s\n", proxyPort, proxyName) + } + fmt.Println(str) +} diff --git a/pool/connection_pool_test.go b/pool/connection_pool_test.go index fd88c5f..bdd674b 100644 --- a/pool/connection_pool_test.go +++ b/pool/connection_pool_test.go @@ -16,12 +16,12 @@ func TestNewSingleConnectionPool(t *testing.T) { t.Parallel() // can be run in parallel because the connection to the rabbitmq is never broken poolName := testutils.FuncName() - ctx := context.TODO() + connections := 1 p, err := pool.NewConnectionPool( ctx, - connectURL, + testutils.HealthyConnectURL, connections, pool.ConnectionPoolWithName(poolName), pool.ConnectionPoolWithLogger(logging.NewTestLogger(t)), @@ -55,7 +55,9 @@ func TestNewConnectionPool(t *testing.T) { ctx := context.TODO() connections := 5 - p, err := pool.NewConnectionPool(ctx, connectURL, connections, + p, err := pool.NewConnectionPool(ctx, + testutils.HealthyConnectURL, + connections, pool.ConnectionPoolWithName(poolName), pool.ConnectionPoolWithLogger(logging.NewTestLogger(t)), ) @@ -92,11 +94,17 @@ func TestNewConnectionPool(t *testing.T) { } func TestNewConnectionPoolWithDisconnect(t *testing.T) { - poolName := testutils.FuncName() + var ( + ctx = context.TODO() + poolName = testutils.FuncName() + proxyName, connectURL, _ = testutils.NextConnectURL() + ) - ctx := context.TODO() connections := 100 - p, err := pool.NewConnectionPool(ctx, connectURL, connections, + p, err := pool.NewConnectionPool( + ctx, + connectURL, + connections, pool.ConnectionPoolWithName(poolName), pool.ConnectionPoolWithLogger(logging.NewTestLogger(t)), ) @@ -109,7 +117,13 @@ func TestNewConnectionPoolWithDisconnect(t *testing.T) { disconnectDuration := 5 * time.Second - awaitStarted, awaitStopped := DisconnectWithStartedStopped(t, 0, 5*time.Second, disconnectDuration) + awaitStarted, awaitStopped := DisconnectWithStartedStopped( + t, + proxyName, + 0, + 5*time.Second, + disconnectDuration, + ) defer awaitStopped() for i := 0; i < connections; i++ { diff --git a/pool/connection_test.go b/pool/connection_test.go index 927847a..65ea351 100644 --- a/pool/connection_test.go +++ b/pool/connection_test.go @@ -10,57 +10,32 @@ import ( "github.com/jxsl13/amqpx/logging" "github.com/jxsl13/amqpx/pool" "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" ) func TestNewSingleConnection(t *testing.T) { t.Parallel() // can be run in parallel because the connection to the rabbitmq is never broken - var ( ctx = context.TODO() nextName = testutils.ConnectionNameGenerator() ) - c, err := pool.NewConnection( - ctx, - connectURL, - nextName(), - pool.ConnectionWithLogger(logging.NewTestLogger(t)), - ) - - if err != nil { - require.NoError(t, err) - return - } - defer func() { - err := c.Close() - require.NoError(t, err) - }() -} - -func TestNewSingleConnectionWithDisconnect(t *testing.T) { - var ( - ctx = context.TODO() - nextName = testutils.ConnectionNameGenerator() - ) - - started, stopped := DisconnectWithStartedStopped(t, 0, 0, 10*time.Second) - started() - defer stopped() + reconnectCB, deferredAssert := AssertConnectionReconnectAttempts(t, 0) + defer deferredAssert() c, err := pool.NewConnection( ctx, - connectURL, + testutils.HealthyConnectURL, nextName(), pool.ConnectionWithLogger(logging.NewTestLogger(t)), + pool.ConnectionWithRecoverCallback(reconnectCB), ) if err != nil { - require.NoError(t, err) + assert.NoError(t, err) return } defer func() { - require.NoError(t, c.Close()) + assert.NoError(t, c.Close()) }() } @@ -79,11 +54,14 @@ func TestManyNewConnection(t *testing.T) { go func() { defer wg.Done() + reconnectCB, deferredAssert := AssertConnectionReconnectAttempts(t, 0) + defer deferredAssert() c, err := pool.NewConnection( ctx, - connectURL, + testutils.HealthyConnectURL, nextName(), pool.ConnectionWithLogger(logging.NewTestLogger(t)), + pool.ConnectionWithRecoverCallback(reconnectCB), ) if err != nil { assert.NoError(t, err) @@ -102,14 +80,50 @@ func TestManyNewConnection(t *testing.T) { wg.Wait() } +func TestNewSingleConnectionWithDisconnect(t *testing.T) { + t.Parallel() + var ( + ctx = context.TODO() + proxyName, connectURL, _ = testutils.NextConnectURL() + nextName = testutils.ConnectionNameGenerator() + ) + + reconnectCB, deferredAssert := AssertConnectionReconnectAttempts(t, 1) + defer deferredAssert() + + started, stopped := DisconnectWithStartedStopped(t, proxyName, 0, 0, 10*time.Second) + started() + defer stopped() + + c, err := pool.NewConnection( + ctx, + connectURL, + nextName(), + pool.ConnectionWithLogger(logging.NewTestLogger(t)), + pool.ConnectionWithRecoverCallback(reconnectCB), + ) + if err != nil { + assert.NoError(t, err) + return + } + defer func() { + assert.NoError(t, c.Close()) + }() +} + func TestManyNewConnectionWithDisconnect(t *testing.T) { + t.Parallel() + + var ( + ctx = context.TODO() + proxyName, connectURL, _ = testutils.NextConnectURL() + ) var ( - ctx = context.TODO() wg sync.WaitGroup connections = 100 nextName = testutils.ConnectionNameGenerator() ) - wait := DisconnectWithStopped(t, 0, 0, time.Second) + wait := DisconnectWithStopped(t, proxyName, 0, 0, time.Second) defer wait() // wait for goroutine to properly close & unblock the proxy wg.Add(connections) @@ -117,11 +131,13 @@ func TestManyNewConnectionWithDisconnect(t *testing.T) { go func() { defer wg.Done() + reconnectCB, deferredAssert := AssertConnectionReconnectAttempts(t, 1) + defer deferredAssert() c, err := pool.NewConnection( ctx, connectURL, nextName(), - //pool.ConnectionWithLogger(logging.NewTestLogger(t)), + pool.ConnectionWithRecoverCallback(reconnectCB), ) if err != nil { assert.NoError(t, err) diff --git a/pool/pool_test.go b/pool/pool_test.go index 4e520a0..1c2fcfc 100644 --- a/pool/pool_test.go +++ b/pool/pool_test.go @@ -6,7 +6,6 @@ import ( "testing" "time" - "github.com/jxsl13/amqpx" "github.com/jxsl13/amqpx/internal/testutils" "github.com/jxsl13/amqpx/logging" "github.com/jxsl13/amqpx/pool" @@ -14,12 +13,6 @@ import ( "go.uber.org/goleak" ) -var ( - healthyConnectURL = amqpx.NewURL("localhost", 5671, "admin", "password") - connectURL = amqpx.NewURL("localhost", 5672, "admin", "password") - brokenConnectURL = amqpx.NewURL("localhost", 5673, "admin", "password") -) - func TestMain(m *testing.M) { goleak.VerifyTestMain( m, @@ -41,7 +34,7 @@ func TestNewPool(t *testing.T) { p, err := pool.New( ctx, - connectURL, + testutils.HealthyConnectURL, connections, sessions, pool.WithName(poolName), diff --git a/pool/publisher_test.go b/pool/publisher_test.go index 47ff32a..4a2f58d 100644 --- a/pool/publisher_test.go +++ b/pool/publisher_test.go @@ -9,6 +9,7 @@ import ( "testing" "time" + "github.com/jxsl13/amqpx/internal/testutils" "github.com/jxsl13/amqpx/logging" "github.com/jxsl13/amqpx/pool" "github.com/stretchr/testify/assert" @@ -20,7 +21,7 @@ func TestPublisher(t *testing.T) { sessions := 10 // publisher sessions + consumer sessions p, err := pool.New( ctx, - connectURL, + testutils.HealthyConnectURL, connections, sessions, pool.WithName("TestPublisher"), @@ -135,7 +136,7 @@ func TestPublishAwaitFlowControl(t *testing.T) { sessions := 2 // publisher sessions + consumer sessions p, err := pool.New( ctx, - brokenConnectURL, // + testutils.BrokenConnectURL, // memory limit or disk limit reached connections, sessions, pool.WithName("TestPublishAwaitFlowControl"), diff --git a/pool/session_pool_test.go b/pool/session_pool_test.go index 3508da5..a4cb7f9 100644 --- a/pool/session_pool_test.go +++ b/pool/session_pool_test.go @@ -22,7 +22,7 @@ func TestSingleSessionPool(t *testing.T) { sessions = 1 ) p, err := pool.NewConnectionPool(ctx, - connectURL, + testutils.HealthyConnectURL, connections, pool.ConnectionPoolWithName(poolName), pool.ConnectionPoolWithLogger(logging.NewTestLogger(t)), @@ -65,7 +65,7 @@ func TestNewSessionPool(t *testing.T) { sessions = 10 ) p, err := pool.NewConnectionPool(ctx, - connectURL, + testutils.HealthyConnectURL, connections, pool.ConnectionPoolWithName(poolName), pool.ConnectionPoolWithLogger(logging.NewTestLogger(t)), diff --git a/pool/session_test.go b/pool/session_test.go index 30bc45a..f5ca1ea 100644 --- a/pool/session_test.go +++ b/pool/session_test.go @@ -3,7 +3,6 @@ package pool_test import ( "context" "sync" - "sync/atomic" "testing" "time" @@ -13,7 +12,7 @@ import ( "github.com/stretchr/testify/assert" ) -func TestNewSingleSession(t *testing.T) { +func TestNewSingleSessionPublishAndConsume(t *testing.T) { t.Parallel() // can be run in parallel because the connection to the rabbitmq is never broken var ( @@ -33,14 +32,14 @@ func TestNewSingleSession(t *testing.T) { publishMessageGenerator = testutils.MessageGenerator(queueName) ) + reconnectCB, deferredAssert := AssertConnectionReconnectAttempts(t, 0) + defer deferredAssert() c, err := pool.NewConnection( ctx, - connectURL, + testutils.HealthyConnectURL, connName, pool.ConnectionWithLogger(logging.NewTestLogger(t)), - pool.ConnectionWithRecoverCallback(func(name string, retry int, err error) { - assert.NoErrorf(t, err, "name=%s retry=%d", name, retry) - }), + pool.ConnectionWithRecoverCallback(reconnectCB), ) if err != nil { assert.NoError(t, err) @@ -71,13 +70,13 @@ func TestNewSingleSession(t *testing.T) { cleanup := DeclareExchangeQueue(t, ctx, s, exchangeName, queueName) defer cleanup() - ConsumeAsyncN(t, ctx, &wg, s, queueName, consumerName, consumeMessageGenerator, 1) - PublishAsyncN(t, ctx, &wg, s, exchangeName, publishMessageGenerator, 1) + ConsumeAsyncN(t, ctx, &wg, s, queueName, consumerName, consumeMessageGenerator, 20) + PublishAsyncN(t, ctx, &wg, s, exchangeName, publishMessageGenerator, 20) wg.Wait() } -func TestManyNewSessions(t *testing.T) { +func TestManyNewSessionsPublishAndConsume(t *testing.T) { t.Parallel() // can be run in parallel because the connection to the rabbitmq is never broken var ( @@ -89,14 +88,14 @@ func TestManyNewSessions(t *testing.T) { sessions = 5 ) + reconnectCB, deferredAssert := AssertConnectionReconnectAttempts(t, 0) + defer deferredAssert() c, err := pool.NewConnection( ctx, - connectURL, + testutils.HealthyConnectURL, connName, pool.ConnectionWithLogger(logging.NewTestLogger(t)), - pool.ConnectionWithRecoverCallback(func(name string, retry int, err error) { - assert.NoErrorf(t, err, "unextected connection recovery: name=%s retry=%d", name, retry) - }), + pool.ConnectionWithRecoverCallback(reconnectCB), ) if err != nil { assert.NoError(t, err) @@ -136,8 +135,8 @@ func TestManyNewSessions(t *testing.T) { cleanup := DeclareExchangeQueue(t, ctx, s, exchangeName, queueName) defer cleanup() - ConsumeAsyncN(t, ctx, &wg, s, queueName, consumerName, consumeNextMessage, 1) - PublishAsyncN(t, ctx, &wg, s, exchangeName, publishNextMessage, 1) + ConsumeAsyncN(t, ctx, &wg, s, queueName, consumerName, consumeNextMessage, 20) + PublishAsyncN(t, ctx, &wg, s, exchangeName, publishNextMessage, 20) } wg.Wait() @@ -156,14 +155,14 @@ func TestNewSessionQueueDeclarePassive(t *testing.T) { nextQueueName = testutils.QueueNameGenerator(sessionName) ) + reconnectCB, deferredAssert := AssertConnectionReconnectAttempts(t, 0) + defer deferredAssert() conn, err := pool.NewConnection( ctx, - connectURL, + testutils.HealthyConnectURL, connName, pool.ConnectionWithLogger(logging.NewTestLogger(t)), - pool.ConnectionWithRecoverCallback(func(name string, retry int, err error) { - assert.NoErrorf(t, err, "unextected connection recovery: name=%s retry=%d", name, retry) - }), + pool.ConnectionWithRecoverCallback(reconnectCB), ) if err != nil { assert.NoError(t, err) @@ -221,38 +220,31 @@ func TestNewSessionQueueDeclarePassive(t *testing.T) { } func TestNewSessionExchangeDeclareWithDisconnect(t *testing.T) { - var ( - ctx = context.TODO() - nextConnName = testutils.ConnectionNameGenerator() - connName = nextConnName() - nextSessionName = testutils.SessionNameGenerator(connName) - ) + t.Parallel() var ( - reconnectCounter int64 = 0 - expectedReconnects int64 = 1 + ctx = context.TODO() + proxyName, connectURL, _ = testutils.NextConnectURL() + nextConnName = testutils.ConnectionNameGenerator() + connName = nextConnName() + nextSessionName = testutils.SessionNameGenerator(connName) ) - defer func() { - assert.Equal(t, expectedReconnects, reconnectCounter, "number of reconnection attempts") - }() + reconnectCB, deferredAssert := AssertConnectionReconnectAttempts(t, 1) + defer deferredAssert() c, err := pool.NewConnection( ctx, connectURL, connName, pool.ConnectionWithLogger(logging.NewTestLogger(t)), - pool.ConnectionWithRecoverCallback(func(name string, retry int, err error) { - if retry == 0 { - atomic.AddInt64(&reconnectCounter, 1) - } - }), + pool.ConnectionWithRecoverCallback(reconnectCB), ) if err != nil { assert.NoError(t, err) return } defer func() { - c.Close() // can be nil or error + assert.NoError(t, c.Close()) }() var ( @@ -260,17 +252,19 @@ func TestNewSessionExchangeDeclareWithDisconnect(t *testing.T) { nextExchangeName = testutils.ExchangeNameGenerator(sessionName) exchangeName = nextExchangeName() - start, started, stopped = DisconnectWithStartStartedStopped(t, time.Second) + disconnected, reconnected = Disconnect(t, proxyName, 5*time.Second) ) s, err := pool.NewSession(c, sessionName) if err != nil { assert.NoError(t, err) return } - defer s.Close() // can be nil or error + defer func() { + assert.NoError(t, s.Close()) + }() - start() // await connection loss start - started() + disconnected() + defer reconnected() err = s.ExchangeDeclare(ctx, exchangeName, pool.ExchangeKindTopic) if err != nil { @@ -278,8 +272,6 @@ func TestNewSessionExchangeDeclareWithDisconnect(t *testing.T) { return } - stopped() - defer func() { err := s.ExchangeDelete(ctx, exchangeName) assert.NoError(t, err) @@ -287,38 +279,31 @@ func TestNewSessionExchangeDeclareWithDisconnect(t *testing.T) { } func TestNewSessionExchangeDeleteWithDisconnect(t *testing.T) { - var ( - ctx = context.TODO() - nextConnName = testutils.ConnectionNameGenerator() - connName = nextConnName() - nextSessionName = testutils.SessionNameGenerator(connName) - ) + t.Parallel() var ( - reconnectCounter int64 = 0 - expectedReconnects int64 = 1 + ctx = context.TODO() + proxyName, connectURL, _ = testutils.NextConnectURL() + nextConnName = testutils.ConnectionNameGenerator() + connName = nextConnName() + nextSessionName = testutils.SessionNameGenerator(connName) ) - defer func() { - assert.Equal(t, expectedReconnects, reconnectCounter, "number of reconnection attempts") - }() + reconnectCB, deferredAssert := AssertConnectionReconnectAttempts(t, 1) + defer deferredAssert() c, err := pool.NewConnection( ctx, connectURL, connName, pool.ConnectionWithLogger(logging.NewTestLogger(t)), - pool.ConnectionWithRecoverCallback(func(name string, retry int, err error) { - if retry == 0 { - atomic.AddInt64(&reconnectCounter, 1) - } - }), + pool.ConnectionWithRecoverCallback(reconnectCB), ) if err != nil { assert.NoError(t, err) return } defer func() { - c.Close() // can be nil or error + assert.NoError(t, c.Close()) }() var ( @@ -326,14 +311,16 @@ func TestNewSessionExchangeDeleteWithDisconnect(t *testing.T) { nextExchangeName = testutils.ExchangeNameGenerator(sessionName) exchangeName = nextExchangeName() - start, started, stopped = DisconnectWithStartStartedStopped(t, time.Second) + disconnected, reconnected = Disconnect(t, proxyName, 5*time.Second) ) s, err := pool.NewSession(c, sessionName) if err != nil { assert.NoError(t, err) return } - defer s.Close() // can be nil or error + defer func() { + assert.NoError(t, s.Close()) + }() err = s.ExchangeDeclare(ctx, exchangeName, pool.ExchangeKindTopic) if err != nil { @@ -341,47 +328,100 @@ func TestNewSessionExchangeDeleteWithDisconnect(t *testing.T) { return } - start() // await connection loss start - started() + disconnected() + defer reconnected() err = s.ExchangeDelete(ctx, exchangeName) assert.NoError(t, err) - stopped() } func TestNewSessionQueueDeclareWithDisconnect(t *testing.T) { + t.Parallel() + var ( - ctx = context.TODO() - nextConnName = testutils.ConnectionNameGenerator() - connName = nextConnName() - nextSessionName = testutils.SessionNameGenerator(connName) + ctx = context.TODO() + proxyName, connectURL, _ = testutils.NextConnectURL() + nextConnName = testutils.ConnectionNameGenerator() + connName = nextConnName() + nextSessionName = testutils.SessionNameGenerator(connName) + ) + + reconnectCB, deferredAssert := AssertConnectionReconnectAttempts(t, 1) + defer deferredAssert() + c, err := pool.NewConnection( + ctx, + connectURL, + connName, + pool.ConnectionWithLogger(logging.NewTestLogger(t)), + pool.ConnectionWithRecoverCallback(reconnectCB), ) + if err != nil { + assert.NoError(t, err, "expected no error when creating new connection") + return + } + defer func() { + assert.NoError(t, c.Close(), "expected no error when closing connection") + }() var ( - reconnectCounter int64 = 0 - expectedReconnects int64 = 1 + sessionName = nextSessionName() + nextQueueName = testutils.QueueNameGenerator(sessionName) + queueName = nextQueueName() + + disconnected, reconnected = Disconnect(t, proxyName, 5*time.Second) ) + s, err := pool.NewSession(c, sessionName) + if err != nil { + assert.NoError(t, err) + return + } + defer func() { + assert.NoError(t, s.Close()) + }() + + disconnected() + defer reconnected() + + _, err = s.QueueDeclare(ctx, queueName) + if err != nil { + assert.NoError(t, err) + return + } defer func() { - assert.Equal(t, expectedReconnects, reconnectCounter, "number of reconnection attempts") + _, err := s.QueueDelete(ctx, queueName) + assert.NoError(t, err, "expected no error when deleting queue") + // TODO: asserting the number of deleted messages seems to be pretty flaky, so we do not assert it here + // assert.Equal(t, 0, delMsgs, "expected 0 messages to be deleted") }() +} + +func TestNewSessionQueueDeleteWithDisconnect(t *testing.T) { + t.Parallel() + + var ( + ctx = context.TODO() + proxyName, connectURL, _ = testutils.NextConnectURL() + nextConnName = testutils.ConnectionNameGenerator() + connName = nextConnName() + nextSessionName = testutils.SessionNameGenerator(connName) + ) + + reconnectCB, deferredAssert := AssertConnectionReconnectAttempts(t, 1) + defer deferredAssert() c, err := pool.NewConnection( ctx, connectURL, connName, pool.ConnectionWithLogger(logging.NewTestLogger(t)), - pool.ConnectionWithRecoverCallback(func(name string, retry int, err error) { - if retry == 0 { - atomic.AddInt64(&reconnectCounter, 1) - } - }), + pool.ConnectionWithRecoverCallback(reconnectCB), ) if err != nil { assert.NoError(t, err) return } defer func() { - c.Close() // can be nil or error + assert.NoError(t, c.Close()) }() var ( @@ -389,17 +429,16 @@ func TestNewSessionQueueDeclareWithDisconnect(t *testing.T) { nextQueueName = testutils.QueueNameGenerator(sessionName) queueName = nextQueueName() - start, started, stopped = DisconnectWithStartStartedStopped(t, time.Second) + disconnected, reconnected = Disconnect(t, proxyName, 5*time.Second) ) s, err := pool.NewSession(c, sessionName) if err != nil { assert.NoError(t, err) return } - defer s.Close() // can be nil or error - - start() // await connection loss start - started() + defer func() { + assert.NoError(t, s.Close()) + }() _, err = s.QueueDeclare(ctx, queueName) if err != nil { @@ -407,85 +446,238 @@ func TestNewSessionQueueDeclareWithDisconnect(t *testing.T) { return } - stopped() + disconnected() + defer reconnected() + + delMsgs, err := s.QueueDelete(ctx, queueName) + assert.NoError(t, err) + assert.Equal(t, 0, delMsgs, "expected 0 messages to be deleted") +} + +func TestNewSessionQueueBindWithDisconnect(t *testing.T) { + t.Parallel() + + var ( + ctx = context.TODO() + proxyName, connectURL, _ = testutils.NextConnectURL() + nextConnName = testutils.ConnectionNameGenerator() + connName = nextConnName() + nextSessionName = testutils.SessionNameGenerator(connName) + ) + + reconnectCB, deferredAssert := AssertConnectionReconnectAttempts(t, 1) + defer deferredAssert() + c, err := pool.NewConnection( + ctx, + connectURL, + connName, + pool.ConnectionWithLogger(logging.NewTestLogger(t)), + pool.ConnectionWithRecoverCallback(reconnectCB), + ) + if err != nil { + assert.NoError(t, err) + return + } + defer func() { + assert.NoError(t, c.Close()) + }() + + var ( + sessionName = nextSessionName() + nextExchangeName = testutils.ExchangeNameGenerator(sessionName) + exchangeName = nextExchangeName() + nextQueueName = testutils.QueueNameGenerator(sessionName) + queueName = nextQueueName() + + disconnected, reconnected = Disconnect(t, proxyName, 5*time.Second) + ) + s, err := pool.NewSession(c, sessionName) + if err != nil { + assert.NoError(t, err) + return + } + defer func() { + assert.NoError(t, s.Close()) + }() + err = s.ExchangeDeclare(ctx, exchangeName, pool.ExchangeKindTopic) + if err != nil { + assert.NoError(t, err) + return + } + defer func() { + err = s.ExchangeDelete(ctx, exchangeName) + assert.NoError(t, err) + }() + + _, err = s.QueueDeclare(ctx, queueName) + if err != nil { + assert.NoError(t, err) + return + } defer func() { delMsgs, err := s.QueueDelete(ctx, queueName) assert.NoError(t, err) assert.Equal(t, 0, delMsgs, "expected 0 messages to be deleted") }() + + disconnected() + defer reconnected() + + err = s.QueueBind(ctx, queueName, "#", exchangeName) + if err != nil { + assert.NoError(t, err) + return + } } -func TestNewSessionQueueDeleteWithDisconnect(t *testing.T) { - var ( - ctx = context.TODO() - nextConnName = testutils.ConnectionNameGenerator() - connName = nextConnName() - nextSessionName = testutils.SessionNameGenerator(connName) - ) +func TestNewSessionQueueUnbindWithDisconnect(t *testing.T) { + t.Parallel() var ( - reconnectCounter int64 = 0 - expectedReconnects int64 = 1 + ctx = context.TODO() + proxyName, connectURL, _ = testutils.NextConnectURL() + nextConnName = testutils.ConnectionNameGenerator() + connName = nextConnName() + nextSessionName = testutils.SessionNameGenerator(connName) ) - defer func() { - assert.Equal(t, expectedReconnects, reconnectCounter, "number of reconnection attempts") - }() + reconnectCB, deferredAssert := AssertConnectionReconnectAttempts(t, 1) + defer deferredAssert() c, err := pool.NewConnection( ctx, connectURL, connName, pool.ConnectionWithLogger(logging.NewTestLogger(t)), - pool.ConnectionWithRecoverCallback(func(name string, retry int, err error) { - if retry == 0 { - atomic.AddInt64(&reconnectCounter, 1) - } - }), + pool.ConnectionWithRecoverCallback(reconnectCB), ) if err != nil { assert.NoError(t, err) return } defer func() { - c.Close() // can be nil or error + assert.NoError(t, c.Close()) }() var ( - sessionName = nextSessionName() - nextQueueName = testutils.QueueNameGenerator(sessionName) - queueName = nextQueueName() + sessionName = nextSessionName() + nextExchangeName = testutils.ExchangeNameGenerator(sessionName) + exchangeName = nextExchangeName() + nextQueueName = testutils.QueueNameGenerator(sessionName) + queueName = nextQueueName() - start, started, stopped = DisconnectWithStartStartedStopped(t, time.Second) + disconnected, reconnected = Disconnect(t, proxyName, 5*time.Second) ) s, err := pool.NewSession(c, sessionName) if err != nil { assert.NoError(t, err) return } - defer s.Close() // can be nil or error + defer func() { + assert.NoError(t, s.Close()) + }() + + err = s.ExchangeDeclare(ctx, exchangeName, pool.ExchangeKindTopic) + if err != nil { + assert.NoError(t, err) + return + } + defer func() { + err = s.ExchangeDelete(ctx, exchangeName) + assert.NoError(t, err) + }() _, err = s.QueueDeclare(ctx, queueName) if err != nil { assert.NoError(t, err) return } + defer func() { + delMsgs, err := s.QueueDelete(ctx, queueName) + assert.NoError(t, err) + assert.Equal(t, 0, delMsgs, "expected 0 messages to be deleted") + }() - start() // await connection loss start - started() + err = s.QueueBind(ctx, queueName, "#", exchangeName) + if err != nil { + assert.NoError(t, err) + return + } - delMsgs, err := s.QueueDelete(ctx, queueName) + disconnected() + defer reconnected() + + err = s.QueueUnbind(ctx, queueName, "#", exchangeName, nil) assert.NoError(t, err) - assert.Equal(t, 0, delMsgs, "expected 0 messages to be deleted") - stopped() } +func TestNewSessionPublishWithDisconnect(t *testing.T) { + t.Parallel() + + var ( + proxyName, connectURL, _ = testutils.NextConnectURL() + ctx = context.TODO() + nextConnName = testutils.ConnectionNameGenerator() + ) + + healthyConnCB, hcbAssert := AssertConnectionReconnectAttempts(t, 0) + defer hcbAssert() + hs, hsclose := NewSession( + t, + ctx, + testutils.HealthyConnectURL, + nextConnName(), + pool.ConnectionWithRecoverCallback(healthyConnCB), + ) + defer hsclose() + + brokenReconnCB, scbAssert := AssertConnectionReconnectAttempts(t, 1) + defer scbAssert() + s, sclose := NewSession( + t, + ctx, + connectURL, + nextConnName(), + pool.ConnectionWithRecoverCallback(brokenReconnCB), + ) + defer sclose() + + var ( + nextExchangeName = testutils.ExchangeNameGenerator(hs.Name()) + nextQueueName = testutils.QueueNameGenerator(hs.Name()) + + exchangeName = nextExchangeName() + queueName = nextQueueName() + nextConsumerName = testutils.ConsumerNameGenerator(queueName) + ) + + cleanup := DeclareExchangeQueue(t, ctx, hs, exchangeName, queueName) + defer cleanup() + + var ( + msgGen = func() string { return "test message content" } + wg sync.WaitGroup + ) + + ConsumeAsyncN(t, ctx, &wg, hs, queueName, nextConsumerName(), msgGen, 1) + + disconnected, reconnected := Disconnect(t, proxyName, 5*time.Second) + disconnected() + PublishAsyncN(t, ctx, &wg, s, exchangeName, msgGen, 1) + reconnected() + + wg.Wait() +} + +/* func TestNewSessionWithDisconnect(t *testing.T) { + var ( - ctx = context.TODO() - nextConnName = testutils.ConnectionNameGenerator() - connName = nextConnName() - nextSessionName = testutils.SessionNameGenerator(connName) + ctx = context.TODO() + proxyName, connectURL, _ = testutils.NextConnectURL() + nextConnName = testutils.ConnectionNameGenerator() + connName = nextConnName() + nextSessionName = testutils.SessionNameGenerator(connName) ) var reconnectCounter int64 = 0 @@ -516,18 +708,18 @@ func TestNewSessionWithDisconnect(t *testing.T) { sessions := 1 wg.Add(sessions) - start, started, stopped := DisconnectWithStartStartedStopped(t, time.Second) - start2, started2, stopped2 := DisconnectWithStartStartedStopped(t, time.Second) - start3, started3, stopped3 := DisconnectWithStartStartedStopped(t, time.Second) - start4, started4, stopped4 := DisconnectWithStartStartedStopped(t, time.Second) - start5, started5, stopped5 := DisconnectWithStartStartedStopped(t, time.Second) - start6, started6, stopped6 := DisconnectWithStartStartedStopped(t, time.Second) + start, started, stopped := DisconnectWithStartStartedStopped(t, proxyName, time.Second) + start2, started2, stopped2 := DisconnectWithStartStartedStopped(t, proxyName, time.Second) + start3, started3, stopped3 := DisconnectWithStartStartedStopped(t, proxyName, time.Second) + start4, started4, stopped4 := DisconnectWithStartStartedStopped(t, proxyName, time.Second) + start5, started5, stopped5 := DisconnectWithStartStartedStopped(t, proxyName, time.Second) + start6, started6, stopped6 := DisconnectWithStartStartedStopped(t, proxyName, time.Second) // deferred - start7, started7, stopped7 := DisconnectWithStartStartedStopped(t, time.Second) - start8, started8, stopped8 := DisconnectWithStartStartedStopped(t, time.Second) - start9, started9, stopped9 := DisconnectWithStartStartedStopped(t, time.Second) - start10, started10, stopped10 := DisconnectWithStartStartedStopped(t, time.Second) + start7, started7, stopped7 := DisconnectWithStartStartedStopped(t, proxyName, time.Second) + start8, started8, stopped8 := DisconnectWithStartStartedStopped(t, proxyName, time.Second) + start9, started9, stopped9 := DisconnectWithStartStartedStopped(t, proxyName, time.Second) + start10, started10, stopped10 := DisconnectWithStartStartedStopped(t, proxyName, time.Second) for id := 0; id < sessions; id++ { go func() { @@ -688,3 +880,5 @@ func TestNewSessionWithDisconnect(t *testing.T) { wg.Wait() } + +*/ diff --git a/pool/subscriber_test.go b/pool/subscriber_test.go index 1229730..f1f809c 100644 --- a/pool/subscriber_test.go +++ b/pool/subscriber_test.go @@ -7,17 +7,19 @@ import ( "testing" "time" + "github.com/jxsl13/amqpx/internal/testutils" "github.com/jxsl13/amqpx/logging" "github.com/jxsl13/amqpx/pool" "github.com/stretchr/testify/assert" ) func TestSubscriber(t *testing.T) { + ctx := context.TODO() sessions := 2 // publisher sessions + consumer sessions p, err := pool.New( ctx, - connectURL, + testutils.HealthyConnectURL, 1, sessions, pool.WithConfirms(true), @@ -136,7 +138,7 @@ func TestBatchSubscriber(t *testing.T) { ) p, err := pool.New( ctx, - connectURL, + testutils.HealthyConnectURL, 1, sessions, pool.WithConfirms(true), @@ -277,7 +279,7 @@ func testBatchSubscriberMaxBytes(t *testing.T, maxBatchBytes int) { ) p, err := pool.New( ctx, - connectURL, + testutils.HealthyConnectURL, 1, sessions, pool.WithConfirms(true), diff --git a/pool/toxiproxy_client_test.go b/pool/toxiproxy_client_test.go index becf856..b4eaae6 100644 --- a/pool/toxiproxy_client_test.go +++ b/pool/toxiproxy_client_test.go @@ -17,7 +17,7 @@ type Proxy struct { mu sync.Mutex } -func NewProxy(t *testing.T) *Proxy { +func NewProxy(t *testing.T, proxyName string) *Proxy { log := logging.NewTestLogger(t) toxi := toxiproxy.NewClient("localhost:8474") @@ -27,13 +27,17 @@ func NewProxy(t *testing.T) *Proxy { } var proxy *toxiproxy.Proxy + proxy, found := m[proxyName] + if !found { + log.Fatalf("no proxy with name %s found", proxyName) + } for k, p := range m { if k == "rabbitmq" { proxy = p } } if proxy == nil { - log.Fatal("no rabbitmq proxy found") + log.Fatalf("proxy with name %s is nil", proxyName) } return &Proxy{ @@ -94,12 +98,12 @@ func (p *Proxy) Close() error { // block current thread // timeout: how long the asynchronous goroutine needs to wait until it disables the connection // duration: how long the asynchronous goroutine wait suntil it reenables the connection -func DisconnectWithStopped(t *testing.T, block, timeout, duration time.Duration) (wait func()) { +func DisconnectWithStopped(t *testing.T, proxyName string, block, timeout, duration time.Duration) (wait func()) { start := time.Now().Add(timeout) var ( wg sync.WaitGroup - proxy = NewProxy(t) + proxy = NewProxy(t, proxyName) ) wg.Add(1) go func(start time.Time) { @@ -138,12 +142,12 @@ func DisconnectWithStopped(t *testing.T, block, timeout, duration time.Duration) // block current thread // timeout: how long the asynchronous goroutine needs to wait until it disables the connection // duration: how long the asynchronous goroutine wait suntil it reenables the connection -func DisconnectWithStartedStopped(t *testing.T, block, startIn, duration time.Duration) (awaitStarted, awaitStopped func()) { +func DisconnectWithStartedStopped(t *testing.T, proxyName string, block, startIn, duration time.Duration) (awaitStarted, awaitStopped func()) { start := time.Now().Add(startIn) var ( wgStart sync.WaitGroup wgStop sync.WaitGroup - proxy = NewProxy(t) + proxy = NewProxy(t, proxyName) ) wgStart.Add(1) @@ -183,13 +187,13 @@ func DisconnectWithStartedStopped(t *testing.T, block, startIn, duration time.Du } } -func Disconnect(t *testing.T, duration time.Duration) (started, stopped func()) { +func Disconnect(t *testing.T, proxyName string, duration time.Duration) (started, stopped func()) { var ( disconnectOnce sync.Once reconnectOnce sync.Once ) - disconnect, awaitStarted, awaitStopped := DisconnectWithStartStartedStopped(t, duration) + disconnect, awaitStarted, awaitStopped := DisconnectWithStartStartedStopped(t, proxyName, duration) return func() { disconnectOnce.Do(func() { disconnect() @@ -203,11 +207,11 @@ func Disconnect(t *testing.T, duration time.Duration) (started, stopped func()) // block current thread // timeout: how long the asynchronous goroutine needs to wait until it disables the connection // duration: how long the asynchronous goroutine wait suntil it reenables the connection -func DisconnectWithStartStartedStopped(t *testing.T, duration time.Duration) (disconnect, awaitStarted, awaitStopped func()) { +func DisconnectWithStartStartedStopped(t *testing.T, proxyName string, duration time.Duration) (disconnect, awaitStarted, awaitStopped func()) { var ( wgStart sync.WaitGroup wgStop sync.WaitGroup - proxy = NewProxy(t) + proxy = NewProxy(t, proxyName) ) disconnect = func() { wgStart.Add(1) diff --git a/pool/utils_test.go b/pool/utils_test.go index 37967ae..21cdd00 100644 --- a/pool/utils_test.go +++ b/pool/utils_test.go @@ -9,79 +9,103 @@ import ( "github.com/jxsl13/amqpx/internal/testutils" "github.com/jxsl13/amqpx/logging" "github.com/jxsl13/amqpx/pool" + "github.com/rabbitmq/amqp091-go" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) -func ConsumeAsyncN( +type Consumer interface { + Consume(queue string, option ...pool.ConsumeOptions) (<-chan amqp091.Delivery, error) +} + +func ConsumeN( t *testing.T, ctx context.Context, wg *sync.WaitGroup, - s *pool.Session, + c Consumer, queueName string, consumerName string, messageGenerator func() string, n int, ) { + cctx, ccancel := context.WithCancel(ctx) + defer ccancel() + log := logging.NewTestLogger(t) + + msgsReceived := 0 + defer func() { + assert.Equal(t, n, msgsReceived, "expected to consume %d messages, got %d", n, msgsReceived) + }() +outer: + for { + delivery, err := c.Consume( + queueName, + pool.ConsumeOptions{ + ConsumerTag: consumerName, + Exclusive: true, + }, + ) + if err != nil { + assert.NoError(t, err) + return + } - wg.Add(1) - go func(wg *sync.WaitGroup, messageGenerator func() string, n int) { - defer wg.Done() - cctx, ccancel := context.WithCancel(ctx) - defer ccancel() - log := logging.NewTestLogger(t) - - msgsReceived := 0 - defer func() { - assert.Equal(t, n, msgsReceived) - }() - outer: for { - delivery, err := s.Consume( - queueName, - pool.ConsumeOptions{ - ConsumerTag: consumerName, - Exclusive: true, - }, - ) - if err != nil { - assert.NoError(t, err) + select { + case <-cctx.Done(): return - } - - for { - select { - case <-cctx.Done(): + case val, ok := <-delivery: + if !ok { + continue outer + } + err := val.Ack(false) + if err != nil { + assert.NoError(t, err) return - case val, ok := <-delivery: - if !ok { - continue outer - } - err := val.Ack(false) - if err != nil { - assert.NoError(t, err) - return - } + } - receivedMsg := string(val.Body) - assert.Equal(t, messageGenerator(), receivedMsg) - log.Infof("consumed message: %s", receivedMsg) - msgsReceived++ - if msgsReceived == n { - logging.NewTestLogger(t).Infof("consumed %d messages, closing consumer", n) - ccancel() - } + receivedMsg := string(val.Body) + assert.Equal(t, messageGenerator(), receivedMsg) + log.Infof("consumed message: %s", receivedMsg) + msgsReceived++ + if msgsReceived == n { + logging.NewTestLogger(t).Infof("consumed %d messages, closing consumer", n) + ccancel() } } } - }(wg, messageGenerator, n) + } +} + +func ConsumeAsyncN( + t *testing.T, + ctx context.Context, + wg *sync.WaitGroup, + c Consumer, + queueName string, + consumerName string, + messageGenerator func() string, + n int, +) { + + wg.Add(1) + go func() { + defer wg.Done() + ConsumeN(t, ctx, wg, c, queueName, consumerName, messageGenerator, n) + }() +} + +type Producer interface { + Publish(ctx context.Context, exchange string, routingKey string, msg pool.Publishing) (deliveryTag uint64, err error) + IsConfirmable() bool + AwaitConfirm(ctx context.Context, expectedTag uint64) error } func PublishAsyncN( t *testing.T, ctx context.Context, wg *sync.WaitGroup, - s *pool.Session, + p Producer, exchangeName string, publishMessageGenerator func() string, n int, @@ -95,7 +119,7 @@ func PublishAsyncN( tctx, cancel := context.WithTimeout(ctx, 60*time.Second) defer cancel() - tag, err := s.Publish( + tag, err := p.Publish( tctx, exchangeName, "", pool.Publishing{ @@ -104,13 +128,13 @@ func PublishAsyncN( Body: []byte(publishMessageGenerator()), }) if err != nil { - assert.NoError(t, err) + assert.NoError(t, err, "expected no error when publishing message") return } - if s.IsConfirmable() { - err = s.AwaitConfirm(tctx, tag) + if p.IsConfirmable() { + err = p.AwaitConfirm(tctx, tag) if err != nil { - assert.NoError(t, err) + assert.NoError(t, err, "expected no error when awaiting confirmation") return } } @@ -132,12 +156,12 @@ func DeclareExchangeQueue( err = s.ExchangeDeclare(ctx, exchangeName, pool.ExchangeKindTopic) if err != nil { - assert.NoError(t, err) + assert.NoError(t, err, "expected no error when declaring exchange") return } defer func() { if err != nil { - assert.NoError(t, s.ExchangeDelete(ctx, exchangeName)) + assert.NoError(t, s.ExchangeDelete(ctx, exchangeName), "expected no error when deleting exchange") } }() @@ -148,9 +172,10 @@ func DeclareExchangeQueue( } defer func() { if err != nil { - deleted, e := s.QueueDelete(ctx, queueName) - assert.NoError(t, e) - assert.Equalf(t, 0, deleted, "expected 0 deleted messages, got %d for queue %s", deleted, queueName) + _, e := s.QueueDelete(ctx, queueName) + assert.NoError(t, e, "expected no error when deleting queue") + // TODO: asserting the number of purged messages seems to be flaky, so we do not do that for now. + //assert.Equalf(t, 0, deleted, "expected 0 deleted messages, got %d for queue %s", deleted, queueName) } }() @@ -185,7 +210,7 @@ func NewSession(t *testing.T, ctx context.Context, connectURL, connectionName st pool.ConnectionWithLogger(log), }, options...)...) if err != nil { - require.NoError(t, err) + require.NoError(t, err, "expected no error when creating new connection") return nil, cleanup } nextSessionName := testutils.SessionNameGenerator(connectionName) @@ -199,12 +224,12 @@ func NewSession(t *testing.T, ctx context.Context, connectURL, connectionName st }), ) if err != nil { - require.NoError(t, err) + assert.NoError(t, err, "expected no error when creating new session") return nil, cleanup } return s, func() { - assert.NoError(t, s.Close()) - assert.NoError(t, c.Close()) + assert.NoError(t, s.Close(), "expected no error when closing session") + assert.NoError(t, c.Close(), "expected no error when closing connection") } } @@ -225,6 +250,6 @@ func AssertConnectionReconnectAttempts(t *testing.T, n int) (callback pool.Conne func() { mu.Lock() defer mu.Unlock() - assert.Equal(t, n, i) + assert.Equal(t, n, i, "expected %d reconnect attempts, got %d", n, i) } } From 26e54cf80d8f9f0e1e106f71b1d29825d755d780 Mon Sep 17 00:00:00 2001 From: John Behm Date: Fri, 8 Mar 2024 16:05:30 +0100 Subject: [PATCH 20/76] update testutils port generator --- internal/testutils/connect_url.go | 102 ++++++++++--------------- internal/testutils/connect_url_test.go | 59 ++++++++++++++ 2 files changed, 98 insertions(+), 63 deletions(-) create mode 100644 internal/testutils/connect_url_test.go diff --git a/internal/testutils/connect_url.go b/internal/testutils/connect_url.go index 27703c5..a678519 100644 --- a/internal/testutils/connect_url.go +++ b/internal/testutils/connect_url.go @@ -1,10 +1,8 @@ package testutils import ( - "encoding/json" "fmt" "sync" - "testing" ) var ( @@ -13,13 +11,14 @@ var ( Username = "admin" Password = "password" Hostname = "localhost" - BrokenConnectURL = fmt.Sprintf("amqp://%s:%s@%s:%d/", Username, Password, Hostname, 5670) - HealthyConnectURL = fmt.Sprintf("amqp://%s:%s@%s:%d/", Username, Password, Hostname, 5671) -) + ToxiProxyPort = 8474 + BrokenPort = 5670 + HealthyPort = 5671 + ExcludedPorts = []int{BrokenPort, HealthyPort, ToxiProxyPort} + BrokenConnectURL = fmt.Sprintf("amqp://%s:%s@%s:%d/", Username, Password, Hostname, BrokenPort) + HealthyConnectURL = fmt.Sprintf("amqp://%s:%s@%s:%d/", Username, Password, Hostname, HealthyPort) -var ( - port int = 5672 - mu sync.Mutex + nextPort = NewPortGenerator(ExcludedPorts...) ) func NextConnectURL() (proxyName, connectURL string, port int) { @@ -30,67 +29,44 @@ func NextConnectURL() (proxyName, connectURL string, port int) { } func NextPort() int { - mu.Lock() - defer mu.Unlock() - defer func() { - loop: - for { - port++ - switch port { - case 5670, 5671, 8474: - default: - break loop - } - - } - }() - return port + return nextPort() } -/* -[ - - { - "name": "rabbitmq", - "listen": "[::]:5672", - "upstream": "rabbitmq:5672", - "enabled": true +func NewConnectURLGenerator(excludePorts ...int) func() (proxyName, connectURL string, port int) { + portGen := NewPortGenerator(excludePorts...) + return func() (proxyName, connectURL string, port int) { + proxyPort := portGen() + return fmt.Sprintf("rabbitmq-%d", proxyPort), + fmt.Sprintf("amqp://%s:%s@%s:%d/", Username, Password, Hostname, proxyPort), + proxyPort } +} -] -*/ -func TestGenerateProxyConfig(_ *testing.T) { - list := []struct { - Name string `json:"name"` - Listen string `json:"listen"` - Upstream string `json:"upstream"` - Enabled bool `json:"enabled"` - }{} - - for i := 0; i < NumTests; i++ { - _, proxyName, proxyPort := NextConnectURL() - list = append(list, struct { - Name string `json:"name"` - Listen string `json:"listen"` - Upstream string `json:"upstream"` - Enabled bool `json:"enabled"` - }{ - Name: proxyName, - Listen: fmt.Sprintf("[::]:%d", proxyPort), - Upstream: Upstream, - Enabled: true, - }) +func NewPortGenerator(excludePorts ...int) func() int { + var ( + port int = 5672 + mu sync.Mutex + ) + excludeMap := make(map[int]struct{}, len(excludePorts)) + for _, p := range excludePorts { + excludeMap[p] = struct{}{} } + return func() int { + mu.Lock() + defer mu.Unlock() + defer func() { + loop: + for { + port++ - data, _ := json.MarshalIndent(list, "", " ") - fmt.Println(string(data)) -} + if _, ok := excludeMap[port]; ok { + continue loop + } + break loop -func TestGenerateDockerPortForwards(_ *testing.T) { - str := "" - for i := 0; i < NumTests; i++ { - proxyName, _, proxyPort := NextConnectURL() - str += fmt.Sprintf(" - %[1]d:%[1]d # %s\n", proxyPort, proxyName) + } + }() + return port } - fmt.Println(str) + } diff --git a/internal/testutils/connect_url_test.go b/internal/testutils/connect_url_test.go new file mode 100644 index 0000000..c06a0cb --- /dev/null +++ b/internal/testutils/connect_url_test.go @@ -0,0 +1,59 @@ +package testutils + +import ( + "encoding/json" + "fmt" + "testing" +) + +/* +[ + + { + "name": "rabbitmq-5673", + "listen": "[::]:5673", + "upstream": "rabbitmq:5672", + "enabled": true + } + +] +*/ +func TestGenerateProxyConfig(_ *testing.T) { + list := []struct { + Name string `json:"name"` + Listen string `json:"listen"` + Upstream string `json:"upstream"` + Enabled bool `json:"enabled"` + }{} + + nextConnectURL := NewConnectURLGenerator(ExcludedPorts...) + + for i := 0; i < NumTests; i++ { + proxyName, _, proxyPort := nextConnectURL() + list = append(list, struct { + Name string `json:"name"` + Listen string `json:"listen"` + Upstream string `json:"upstream"` + Enabled bool `json:"enabled"` + }{ + Name: proxyName, + Listen: fmt.Sprintf("[::]:%d", proxyPort), + Upstream: Upstream, + Enabled: true, + }) + } + + data, _ := json.MarshalIndent(list, "", " ") + fmt.Println(string(data)) +} + +func TestGenerateDockerPortForwards(_ *testing.T) { + nextConnectURL := NewConnectURLGenerator(ExcludedPorts...) + + str := "" + for i := 0; i < NumTests; i++ { + proxyName, _, proxyPort := nextConnectURL() + str += fmt.Sprintf(" - %[1]d:%[1]d # %s\n", proxyPort, proxyName) + } + fmt.Println(str) +} From 0aca705c2b6e78a15136889e96ff5b3cd2da26d9 Mon Sep 17 00:00:00 2001 From: John Behm Date: Fri, 8 Mar 2024 16:06:31 +0100 Subject: [PATCH 21/76] add close error checks --- pool/session.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pool/session.go b/pool/session.go index d2928d9..750351f 100644 --- a/pool/session.go +++ b/pool/session.go @@ -231,7 +231,7 @@ func (s *Session) connect() (err error) { defer func() { // reset state in case of an error if err != nil { - s.close() + err = errors.Join(err, s.close()) s.warn(err, "failed to open session") } else { s.info("opened session") @@ -243,7 +243,7 @@ func (s *Session) connect() (err error) { return ErrClosed } - s.close() // close any open rabbitmq channel & cleanup Go channels + _ = s.close() // close any open rabbitmq channel & cleanup Go channels channel, err := s.conn.channel() if err != nil { From b8bdb85d1ed63be010a8857ea982e8e1f4ff964f Mon Sep 17 00:00:00 2001 From: John Behm Date: Fri, 8 Mar 2024 16:06:48 +0100 Subject: [PATCH 22/76] finalize session tests --- pool/session_test.go | 246 +++++++++---------------------------------- pool/utils_test.go | 82 +++++++++------ 2 files changed, 97 insertions(+), 231 deletions(-) diff --git a/pool/session_test.go b/pool/session_test.go index f5ca1ea..5716dc7 100644 --- a/pool/session_test.go +++ b/pool/session_test.go @@ -655,230 +655,78 @@ func TestNewSessionPublishWithDisconnect(t *testing.T) { defer cleanup() var ( - msgGen = func() string { return "test message content" } - wg sync.WaitGroup + consumeMsgGen = testutils.MessageGenerator(queueName) + publishMsgGen = testutils.MessageGenerator(queueName) + numMsgs = 20 + wg sync.WaitGroup ) - ConsumeAsyncN(t, ctx, &wg, hs, queueName, nextConsumerName(), msgGen, 1) + ConsumeAsyncN(t, ctx, &wg, hs, queueName, nextConsumerName(), consumeMsgGen, numMsgs) disconnected, reconnected := Disconnect(t, proxyName, 5*time.Second) disconnected() - PublishAsyncN(t, ctx, &wg, s, exchangeName, msgGen, 1) + PublishN(t, ctx, s, exchangeName, publishMsgGen, numMsgs) reconnected() wg.Wait() } -/* -func TestNewSessionWithDisconnect(t *testing.T) { +func TestNewSessionConsumeWithDisconnect(t *testing.T) { + t.Parallel() var ( - ctx = context.TODO() proxyName, connectURL, _ = testutils.NextConnectURL() + ctx = context.TODO() nextConnName = testutils.ConnectionNameGenerator() - connName = nextConnName() - nextSessionName = testutils.SessionNameGenerator(connName) ) - var reconnectCounter int64 = 0 - defer func() { - assert.Equal(t, 10, reconnectCounter-1) - }() - c, err := pool.NewConnection( + healthyConnCB, hcbAssert := AssertConnectionReconnectAttempts(t, 0) + defer hcbAssert() + hs, hsclose := NewSession( + t, ctx, - connectURL, - connName, - pool.ConnectionWithLogger(logging.NewTestLogger(t)), - pool.ConnectionWithRecoverCallback(func(name string, retry int, err error) { - if retry == 0 { - atomic.AddInt64(&reconnectCounter, 1) - } - }), + testutils.HealthyConnectURL, + nextConnName(), + pool.ConnectionWithRecoverCallback(healthyConnCB), ) - if err != nil { - assert.NoError(t, err) - return - } - defer func() { - c.Close() // can be nil or error - }() - - var wg sync.WaitGroup - - sessions := 1 - wg.Add(sessions) - - start, started, stopped := DisconnectWithStartStartedStopped(t, proxyName, time.Second) - start2, started2, stopped2 := DisconnectWithStartStartedStopped(t, proxyName, time.Second) - start3, started3, stopped3 := DisconnectWithStartStartedStopped(t, proxyName, time.Second) - start4, started4, stopped4 := DisconnectWithStartStartedStopped(t, proxyName, time.Second) - start5, started5, stopped5 := DisconnectWithStartStartedStopped(t, proxyName, time.Second) - start6, started6, stopped6 := DisconnectWithStartStartedStopped(t, proxyName, time.Second) - - // deferred - start7, started7, stopped7 := DisconnectWithStartStartedStopped(t, proxyName, time.Second) - start8, started8, stopped8 := DisconnectWithStartStartedStopped(t, proxyName, time.Second) - start9, started9, stopped9 := DisconnectWithStartStartedStopped(t, proxyName, time.Second) - start10, started10, stopped10 := DisconnectWithStartStartedStopped(t, proxyName, time.Second) - - for id := 0; id < sessions; id++ { - go func() { - defer wg.Done() - - var ( - sessionName = nextSessionName() - nextQueueName = testutils.QueueNameGenerator(sessionName) - queueName = nextQueueName() - nextExchangeName = testutils.ExchangeNameGenerator(sessionName) - exchangeName = nextExchangeName() - nextConsumerName = testutils.ConsumerNameGenerator(queueName) - consumerName = nextConsumerName() - nextMessage = testutils.MessageGenerator(queueName) - ) - s, err := pool.NewSession(c, sessionName, pool.SessionWithConfirms(true)) - if err != nil { - assert.NoError(t, err) - return - } - defer func() { - // INFO: does not lead to a recovery - start10() - started10() - - s.Close() // can be nil or error - stopped10() - }() - - start() // await connection loss start - started() - - err = s.ExchangeDeclare(ctx, exchangeName, pool.ExchangeKindTopic) - if err != nil { - assert.NoError(t, err) - return - } - - stopped() - - defer func() { - start9() - started9() - - err := s.ExchangeDelete(ctx, exchangeName) - assert.NoError(t, err) - - stopped9() - }() - - start2() - started2() - - _, err = s.QueueDeclare(ctx, queueName) - if err != nil { - assert.NoError(t, err) - return - } - stopped2() - - defer func() { - start8() - started8() - stopped8() - - _, err := s.QueueDelete(ctx, queueName) - assert.NoError(t, err) - }() - - start3() - started3() + defer hsclose() - err = s.QueueBind(ctx, queueName, "#", exchangeName) - if err != nil { - assert.NoError(t, err) - return - } - stopped3() + brokenReconnCB, scbAssert := AssertConnectionReconnectAttempts(t, 1) + defer scbAssert() + s, sclose := NewSession( + t, + ctx, + connectURL, + nextConnName(), + pool.ConnectionWithRecoverCallback(brokenReconnCB), + ) + defer sclose() - defer func() { - start7() - started7() + var ( + nextExchangeName = testutils.ExchangeNameGenerator(hs.Name()) + nextQueueName = testutils.QueueNameGenerator(hs.Name()) - err := s.QueueUnbind(ctx, queueName, "#", exchangeName, nil) - assert.NoError(t, err) + exchangeName = nextExchangeName() + queueName = nextQueueName() + nextConsumerName = testutils.ConsumerNameGenerator(queueName) + ) - stopped7() - }() + cleanup := DeclareExchangeQueue(t, ctx, hs, exchangeName, queueName) + defer cleanup() - start4() - started4() + var ( + publisherMsgGen = testutils.MessageGenerator(queueName) + consumerMsgGen = testutils.MessageGenerator(queueName) + numMsgs = 20 + wg sync.WaitGroup + ) - delivery, err := s.Consume( - queueName, - pool.ConsumeOptions{ - ConsumerTag: consumerName, - Exclusive: true, - }, - ) - if err != nil { - assert.NoError(t, err) - return - } - stopped4() - - message := nextMessage() - - wg.Add(1) - go func(msg string) { - defer wg.Done() - - msgsReceived := 0 - for val := range delivery { - receivedMsg := string(val.Body) - assert.Equal(t, message, receivedMsg) - msgsReceived++ - } - // consumption fails because the connection will be closed - assert.Equal(t, 0, msgsReceived) - // this routine must be closed upon session closure - }(message) - - time.Sleep(2 * time.Second) - - start5() - started5() - var once sync.Once - - for { - tag, err := s.Publish(ctx, exchangeName, "", pool.Publishing{ - Mandatory: true, - ContentType: "application/json", - Body: []byte(message), - }) - if err != nil { - assert.NoError(t, err) - return - } - - stopped5() - - once.Do(func() { - start6() - started6() - stopped6() - }) - - err = s.AwaitConfirm(ctx, tag) - if err != nil { - // retry because the first attempt at confirmation failed - continue - } - break - } + PublishAsyncN(t, ctx, &wg, hs, exchangeName, publisherMsgGen, numMsgs) - }() - } + disconnected, reconnected := Disconnect(t, proxyName, 5*time.Second) + disconnected() + ConsumeN(t, ctx, s, queueName, nextConsumerName(), consumerMsgGen, numMsgs) + reconnected() wg.Wait() } - -*/ diff --git a/pool/utils_test.go b/pool/utils_test.go index 21cdd00..fdec588 100644 --- a/pool/utils_test.go +++ b/pool/utils_test.go @@ -2,9 +2,9 @@ package pool_test import ( "context" + "fmt" "sync" "testing" - "time" "github.com/jxsl13/amqpx/internal/testutils" "github.com/jxsl13/amqpx/logging" @@ -21,7 +21,6 @@ type Consumer interface { func ConsumeN( t *testing.T, ctx context.Context, - wg *sync.WaitGroup, c Consumer, queueName string, consumerName string, @@ -91,7 +90,7 @@ func ConsumeAsyncN( wg.Add(1) go func() { defer wg.Done() - ConsumeN(t, ctx, wg, c, queueName, consumerName, messageGenerator, n) + ConsumeN(t, ctx, c, queueName, consumerName, messageGenerator, n) }() } @@ -101,6 +100,43 @@ type Producer interface { AwaitConfirm(ctx context.Context, expectedTag uint64) error } +func PublishN( + t *testing.T, + ctx context.Context, + p Producer, + exchangeName string, + publishMessageGenerator func() string, + n int, +) { + for i := 0; i < n; i++ { + message := publishMessageGenerator() + err := publish(t, ctx, p, exchangeName, message) + assert.NoError(t, err) + } + logging.NewTestLogger(t).Infof("published %d messages, closing publisher", n) +} + +func publish(t *testing.T, ctx context.Context, p Producer, exchangeName string, message string) error { + tag, err := p.Publish( + ctx, + exchangeName, "", + pool.Publishing{ + Mandatory: true, + ContentType: "text/plain", + Body: []byte(message), + }) + if err != nil { + return fmt.Errorf("expected no error when publishing message: %w", err) + } + if p.IsConfirmable() { + err = p.AwaitConfirm(ctx, tag) + if err != nil { + return fmt.Errorf("expected no error when awaiting confirmation: %w", err) + } + } + return nil +} + func PublishAsyncN( t *testing.T, ctx context.Context, @@ -113,41 +149,23 @@ func PublishAsyncN( wg.Add(1) go func(wg *sync.WaitGroup, publishMessageGenerator func() string, n int) { defer wg.Done() - - for i := 0; i < n; i++ { - func() { - tctx, cancel := context.WithTimeout(ctx, 60*time.Second) - defer cancel() - - tag, err := p.Publish( - tctx, - exchangeName, "", - pool.Publishing{ - Mandatory: true, - ContentType: "text/plain", - Body: []byte(publishMessageGenerator()), - }) - if err != nil { - assert.NoError(t, err, "expected no error when publishing message") - return - } - if p.IsConfirmable() { - err = p.AwaitConfirm(tctx, tag) - if err != nil { - assert.NoError(t, err, "expected no error when awaiting confirmation") - return - } - } - }() - } - logging.NewTestLogger(t).Infof("published %d messages, closing publisher", n) + PublishN(t, ctx, p, exchangeName, publishMessageGenerator, n) }(wg, publishMessageGenerator, n) } +type Topologer interface { + ExchangeDeclare(ctx context.Context, name string, kind pool.ExchangeKind, option ...pool.ExchangeDeclareOptions) error + ExchangeDelete(ctx context.Context, name string, option ...pool.ExchangeDeleteOptions) error + QueueDeclare(ctx context.Context, name string, option ...pool.QueueDeclareOptions) (pool.Queue, error) + QueueDelete(ctx context.Context, name string, option ...pool.QueueDeleteOptions) (purgedMsgs int, err error) + QueueBind(ctx context.Context, queueName string, routingKey string, exchange string, option ...pool.QueueBindOptions) error + QueueUnbind(ctx context.Context, name string, routingKey string, exchange string, arg ...amqp091.Table) error +} + func DeclareExchangeQueue( t *testing.T, ctx context.Context, - s *pool.Session, + s Topologer, exchangeName string, queueName string, ) (cleanup func()) { From f225b8b701fcc01f2b465f7556bca2146f6091d1 Mon Sep 17 00:00:00 2001 From: John Behm Date: Fri, 8 Mar 2024 17:03:26 +0100 Subject: [PATCH 23/76] cleanup & improve test utilities --- ...private_test.go => backoff_policy_test.go} | 2 + pool/helpers_context_test.go | 102 +++++++++--------- pool/pool_test.go | 1 - pool/session_test.go | 20 ++-- pool/utils_test.go | 24 ++++- 5 files changed, 85 insertions(+), 64 deletions(-) rename pool/{private_test.go => backoff_policy_test.go} (98%) diff --git a/pool/private_test.go b/pool/backoff_policy_test.go similarity index 98% rename from pool/private_test.go rename to pool/backoff_policy_test.go index b56c702..58294fd 100644 --- a/pool/private_test.go +++ b/pool/backoff_policy_test.go @@ -8,6 +8,8 @@ import ( ) func TestBackoffPolicy(t *testing.T) { + t.Parallel() + backoffMultiTest(t, 15, 3) backoffMultiTest(t, 32, 4) backoffMultiTest(t, 64, 5) diff --git a/pool/helpers_context_test.go b/pool/helpers_context_test.go index 2010d18..d8f0602 100644 --- a/pool/helpers_context_test.go +++ b/pool/helpers_context_test.go @@ -13,56 +13,9 @@ import ( "github.com/stretchr/testify/assert" ) -func worker(t *testing.T, ctx context.Context, wg *sync.WaitGroup, sc *stateContext) { - defer wg.Done() - - log := logging.NewTestLogger(t) - defer func() { - log.Debug("worker pausing (closing)") - sc.Paused() - log.Debug("worker paused (closing)") - log.Debug("worker closed") - }() - log.Debug("worker started") - - for { - select { - case <-ctx.Done(): - //log.Debug("worker done") - return - case <-sc.Resuming().Done(): - //log.Debug("worker resuming") - sc.Resumed() - go func() { - select { - case <-ctx.Done(): - return - case <-time.After(time.Second): - sc.Pause(ctx) // always have at least one goroutine that triggers the switch back to the other state after a specific time - } - }() - //log.Debug("worker resumed") - } - select { - case <-ctx.Done(): - return - case <-sc.Pausing().Done(): - //log.Debug("worker pausing") - sc.Paused() - //log.Debug("worker paused") - go func() { - select { - case <-ctx.Done(): - return - case <-time.After(time.Second): - sc.Resume(ctx) // always have at least one goroutine that triggers the switch back to the other state after a specific time - } - }() - } - } -} - func TestStateContextSimpleSynchronized(t *testing.T) { + t.Parallel() + ctx, cancel := signal.NotifyContext(context.Background(), os.Interrupt, os.Kill) defer cancel() @@ -87,6 +40,8 @@ func TestStateContextSimpleSynchronized(t *testing.T) { } func TestStateContextConcurrentTransitions(t *testing.T) { + t.Parallel() + log := logging.NewTestLogger(t) ctx, cancel := signal.NotifyContext(context.Background(), os.Interrupt, os.Kill) defer cancel() @@ -182,3 +137,52 @@ func TestStateContextConcurrentTransitions(t *testing.T) { log.Debugf("awaitPaused: %d", awaitPaused.Load()) log.Debugf("awaitResumed: %d", awaitResumed.Load()) } + +func worker(t *testing.T, ctx context.Context, wg *sync.WaitGroup, sc *stateContext) { + defer wg.Done() + + log := logging.NewTestLogger(t) + defer func() { + log.Debug("worker pausing (closing)") + sc.Paused() + log.Debug("worker paused (closing)") + log.Debug("worker closed") + }() + log.Debug("worker started") + + for { + select { + case <-ctx.Done(): + //log.Debug("worker done") + return + case <-sc.Resuming().Done(): + //log.Debug("worker resuming") + sc.Resumed() + go func() { + select { + case <-ctx.Done(): + return + case <-time.After(time.Second): + sc.Pause(ctx) // always have at least one goroutine that triggers the switch back to the other state after a specific time + } + }() + //log.Debug("worker resumed") + } + select { + case <-ctx.Done(): + return + case <-sc.Pausing().Done(): + //log.Debug("worker pausing") + sc.Paused() + //log.Debug("worker paused") + go func() { + select { + case <-ctx.Done(): + return + case <-time.After(time.Second): + sc.Resume(ctx) // always have at least one goroutine that triggers the switch back to the other state after a specific time + } + }() + } + } +} diff --git a/pool/pool_test.go b/pool/pool_test.go index 1c2fcfc..01b4708 100644 --- a/pool/pool_test.go +++ b/pool/pool_test.go @@ -68,5 +68,4 @@ func TestNewPool(t *testing.T) { } wg.Wait() - } diff --git a/pool/session_test.go b/pool/session_test.go index 5716dc7..ecd33d9 100644 --- a/pool/session_test.go +++ b/pool/session_test.go @@ -655,15 +655,15 @@ func TestNewSessionPublishWithDisconnect(t *testing.T) { defer cleanup() var ( - consumeMsgGen = testutils.MessageGenerator(queueName) - publishMsgGen = testutils.MessageGenerator(queueName) - numMsgs = 20 - wg sync.WaitGroup + wg sync.WaitGroup + consumeMsgGen = testutils.MessageGenerator(queueName) + publishMsgGen = testutils.MessageGenerator(queueName) + numMsgs = 20 + disconnected, reconnected = Disconnect(t, proxyName, 5*time.Second) ) ConsumeAsyncN(t, ctx, &wg, hs, queueName, nextConsumerName(), consumeMsgGen, numMsgs) - disconnected, reconnected := Disconnect(t, proxyName, 5*time.Second) disconnected() PublishN(t, ctx, s, exchangeName, publishMsgGen, numMsgs) reconnected() @@ -715,15 +715,15 @@ func TestNewSessionConsumeWithDisconnect(t *testing.T) { defer cleanup() var ( - publisherMsgGen = testutils.MessageGenerator(queueName) - consumerMsgGen = testutils.MessageGenerator(queueName) - numMsgs = 20 - wg sync.WaitGroup + publisherMsgGen = testutils.MessageGenerator(queueName) + consumerMsgGen = testutils.MessageGenerator(queueName) + numMsgs = 20 + wg sync.WaitGroup + disconnected, reconnected = Disconnect(t, proxyName, 5*time.Second) ) PublishAsyncN(t, ctx, &wg, hs, exchangeName, publisherMsgGen, numMsgs) - disconnected, reconnected := Disconnect(t, proxyName, 5*time.Second) disconnected() ConsumeN(t, ctx, s, queueName, nextConsumerName(), consumerMsgGen, numMsgs) reconnected() diff --git a/pool/utils_test.go b/pool/utils_test.go index fdec588..48c62e8 100644 --- a/pool/utils_test.go +++ b/pool/utils_test.go @@ -49,6 +49,8 @@ outer: return } + var previouslyReceivedMsg string + for { select { case <-cctx.Done(): @@ -63,14 +65,28 @@ outer: return } - receivedMsg := string(val.Body) - assert.Equal(t, messageGenerator(), receivedMsg) + var ( + expectedMsg = messageGenerator() + receivedMsg = string(val.Body) + ) + assert.Equalf( + t, + expectedMsg, + receivedMsg, + "expected message %s, got %s, previously received message: %s", + expectedMsg, + receivedMsg, + previouslyReceivedMsg, + ) + log.Infof("consumed message: %s", receivedMsg) msgsReceived++ if msgsReceived == n { logging.NewTestLogger(t).Infof("consumed %d messages, closing consumer", n) ccancel() } + // update last received message + previouslyReceivedMsg = receivedMsg } } } @@ -110,13 +126,13 @@ func PublishN( ) { for i := 0; i < n; i++ { message := publishMessageGenerator() - err := publish(t, ctx, p, exchangeName, message) + err := publish(ctx, p, exchangeName, message) assert.NoError(t, err) } logging.NewTestLogger(t).Infof("published %d messages, closing publisher", n) } -func publish(t *testing.T, ctx context.Context, p Producer, exchangeName string, message string) error { +func publish(ctx context.Context, p Producer, exchangeName string, message string) error { tag, err := p.Publish( ctx, exchangeName, "", From 5f3102ab743e9e8998a9d53b9c940cf14e24fdfe Mon Sep 17 00:00:00 2001 From: John Behm Date: Fri, 8 Mar 2024 22:21:35 +0100 Subject: [PATCH 24/76] improve error messages --- pool/session.go | 76 ++++++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 72 insertions(+), 4 deletions(-) diff --git a/pool/session.go b/pool/session.go index 750351f..37f2507 100644 --- a/pool/session.go +++ b/pool/session.go @@ -184,16 +184,18 @@ func (s *Session) Close() (err error) { if s.autoCloseConn { defer func() { + s.debug("closing session connection...") err = errors.Join(err, s.conn.Close()) }() } - + s.debug("closing session context...") s.cancel() return s.close() } func (s *Session) close() (err error) { defer func() { + s.debug("flushing channels...") flush(s.errors) flush(s.confirms) if s.channel != nil { @@ -205,6 +207,7 @@ func (s *Session) close() (err error) { return nil } + s.debug("canceling consumers...") for consumer := range s.consumers { // ignore error, as at this point we cannot do anything about the error // tell server to cancel consumer deliveries. @@ -212,6 +215,13 @@ func (s *Session) close() (err error) { err = errors.Join(err, cerr) } + if s.error() != nil { + // cannot close erred channel + s.debug("not closing erred amqp channel") + return nil + } + + s.debug("closing amqp channel...") return s.channel.Close() } @@ -338,7 +348,7 @@ func (s *Session) recover(ctx context.Context) error { // AwaitConfirm tries to await a confirmation from the broker for a published message // You may check for ErrNack in order to see whether the broker rejected the message temporatily. -// AwaitConfirm cannot be retried in case the channel dies. +// WARNING: AwaitConfirm cannot be retried in case the channel dies or errors. // You must resend your message and attempt to await it again. func (s *Session) AwaitConfirm(ctx context.Context, expectedTag uint64) error { s.mu.Lock() @@ -353,7 +363,7 @@ func (s *Session) AwaitConfirm(ctx context.Context, expectedTag uint64) error { if !ok { err := s.error() if err != nil { - return fmt.Errorf("await confirm failed: %w", err) + return fmt.Errorf("await confirm failed: confirms channel closed: %w", err) } return fmt.Errorf("await confirm failed: confirms channel %w", ErrClosed) } @@ -371,7 +381,7 @@ func (s *Session) AwaitConfirm(ctx context.Context, expectedTag uint64) error { if !ok { err := s.error() if err != nil { - return fmt.Errorf("await confirm failed: %w", err) + return fmt.Errorf("await confirm failed: blocking channel closed: %w", err) } return fmt.Errorf("await confirm failed: %w", errFlowControlClosed) } @@ -1349,6 +1359,64 @@ func (s *Session) Flow(ctx context.Context, active bool) error { }) } +// Tx puts the channel into transaction mode on the server. All publishings and acknowledgments following this method +// will be atomically committed or rolled back for a single queue. Call either Channel.TxCommit or Channel.TxRollback +// to leave this transaction and immediately start a new transaction. +// +// The atomicity across multiple queues is not defined as queue declarations and bindings are not included in the transaction. +// +// The behavior of publishings that are delivered as mandatory or immediate while the channel is in a transaction is not defined. +// +// Once a channel has been put into transaction mode, it cannot be taken out of transaction mode. +// Use a different channel for non-transactional semantics. +func (s *Session) Tx() error { + s.mu.Lock() + defer s.mu.Unlock() + + return s.channel.Tx() +} + +// TxCommit atomically commits all publishings and acknowledgments for a single queue and immediately start a new transaction. +// +// Calling this method without having called Channel.Tx is an error. +func (s *Session) TxCommit() error { + s.mu.Lock() + defer s.mu.Unlock() + + return s.channel.TxCommit() +} + +// TxRollback atomically rolls back all publishings and acknowledgments for a single queue and immediately start a new transaction. +// +// Calling this method without having called Channel.Tx is an error. +func (s *Session) TxRollback() error { + s.mu.Lock() + defer s.mu.Unlock() + + return s.channel.TxRollback() +} + +func (s *Session) Do(ctx context.Context, f func() error) error { + s.mu.Lock() + defer s.mu.Unlock() + + return s.retry(ctx, nil, func() (err error) { + err = s.channel.Tx() + if err != nil { + return err + } + defer func() { + if err != nil { + err = errors.Join(err, s.channel.TxRollback()) + } else { + err = s.channel.TxCommit() + } + }() + + return f() + }) +} + // Error returns all errors from the errors channel // and flushes all other pending errors from the channel // In case that there are no errors, nil is returned. From f2218c86aa1712d82c3fe28b82df03d0189d6fe1 Mon Sep 17 00:00:00 2001 From: John Behm Date: Fri, 8 Mar 2024 22:21:44 +0100 Subject: [PATCH 25/76] update .gitignore --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index ba0327e..c0df5a7 100644 --- a/.gitignore +++ b/.gitignore @@ -4,3 +4,4 @@ coverage.txt DEBUG.md debug.md +__debug_bin* From 09a2ff3801b445f7cbe4d39cbcb7823636521237 Mon Sep 17 00:00:00 2001 From: John Behm Date: Fri, 8 Mar 2024 22:27:36 +0100 Subject: [PATCH 26/76] update publisher & test (utlities) --- pool/publisher.go | 8 +- pool/publisher_test.go | 277 ++++++++++++++++------------------------- pool/session_test.go | 14 ++- pool/utils_test.go | 22 ++-- 4 files changed, 134 insertions(+), 187 deletions(-) diff --git a/pool/publisher.go b/pool/publisher.go index 9020878..42d0edd 100644 --- a/pool/publisher.go +++ b/pool/publisher.go @@ -82,10 +82,14 @@ func (p *Publisher) Publish(ctx context.Context, exchange string, routingKey str case errors.Is(err, ErrDeliveryTagMismatch): return err case errors.Is(err, ErrFlowControl): - p.warn(exchange, routingKey, err, "publish failed, retrying") return err default: - p.warn(exchange, routingKey, err, "publish failed, retrying") + if recoverable(err) { + p.warn(exchange, routingKey, err, "publish failed due to recoverable error, retrying") + // retry + } else { + return err + } } } } diff --git a/pool/publisher_test.go b/pool/publisher_test.go index 4a2f58d..07b38f3 100644 --- a/pool/publisher_test.go +++ b/pool/publisher_test.go @@ -3,8 +3,6 @@ package pool_test import ( "context" "fmt" - "os" - "os/signal" "sync" "testing" "time" @@ -15,18 +13,34 @@ import ( "github.com/stretchr/testify/assert" ) -func TestPublisher(t *testing.T) { - ctx := context.TODO() - connections := 1 - sessions := 10 // publisher sessions + consumer sessions - p, err := pool.New( +func TestSinglePublisher(t *testing.T) { + t.Parallel() + + var ( + proxyName, connectURL, _ = testutils.NextConnectURL() + ctx = context.TODO() + nextConnName = testutils.ConnectionNameGenerator() + numMsgs = 20 + ) + + healthyConnCB, hcbAssert := AssertConnectionReconnectAttempts(t, 0) + defer hcbAssert() + hs, hsclose := NewSession( + t, ctx, testutils.HealthyConnectURL, - connections, - sessions, - pool.WithName("TestPublisher"), - pool.WithConfirms(true), + nextConnName(), + pool.ConnectionWithRecoverCallback(healthyConnCB), + ) + defer hsclose() + + p, err := pool.New( + ctx, + connectURL, + 1, + 1, pool.WithLogger(logging.NewTestLogger(t)), + pool.WithConfirms(true), ) if err != nil { assert.NoError(t, err) @@ -34,198 +48,117 @@ func TestPublisher(t *testing.T) { } defer p.Close() - var wg sync.WaitGroup - - channels := sessions / 2 // one sessions for consumer and one for publisher - wg.Add(channels) - for id := 0; id < channels; id++ { - go func(id int64) { - defer wg.Done() - - s, err := p.GetSession(ctx) - if err != nil { - assert.NoError(t, err) - return - } - defer func() { - p.ReturnSession(s, nil) - }() + var ( + nextExchangeName = testutils.ExchangeNameGenerator(hs.Name()) + nextQueueName = testutils.QueueNameGenerator(hs.Name()) + exchangeName = nextExchangeName() + queueName = nextQueueName() + ) + cleanup := DeclareExchangeQueue(t, ctx, hs, exchangeName, queueName) + defer cleanup() - queueName := fmt.Sprintf("TestPublisher-Queue-%d", id) - _, err = s.QueueDeclare(ctx, queueName) - if err != nil { - assert.NoError(t, err) - return - } - defer func() { - i, err := s.QueueDelete(ctx, queueName) - assert.NoError(t, err) - assert.Equal(t, 0, i) - }() + var ( + nextConsumerName = testutils.ConsumerNameGenerator(queueName) + publisherMsgGen = testutils.MessageGenerator(queueName) + consumerMsgGen = testutils.MessageGenerator(queueName) + wg sync.WaitGroup + ) - exchangeName := fmt.Sprintf("TestPublisher-Exchange-%d", id) - err = s.ExchangeDeclare(ctx, exchangeName, "topic") - if err != nil { - assert.NoError(t, err) - return - } - defer func() { - err := s.ExchangeDelete(ctx, exchangeName) - assert.NoError(t, err) - }() + pub := pool.NewPublisher(p) + defer pub.Close() - err = s.QueueBind(ctx, queueName, "#", exchangeName) - if err != nil { - assert.NoError(t, err) - return - } + // TODO: currently this test allows duplication of messages + ConsumeAsyncN(t, ctx, &wg, hs, queueName, nextConsumerName(), consumerMsgGen, numMsgs, true) + + for i := 0; i < numMsgs; i++ { + msg := publisherMsgGen() + err = func(msg string) error { + disconnected, reconnected := DisconnectWithStartedStopped( + t, + proxyName, + 0, + testutils.Jitter(time.Millisecond, 20*time.Millisecond), + testutils.Jitter(100*time.Millisecond, 150*time.Millisecond), + ) defer func() { - err := s.QueueUnbind(ctx, queueName, "#", exchangeName, nil) - assert.NoError(t, err) + disconnected() + reconnected() }() - delivery, err := s.Consume( - queueName, - pool.ConsumeOptions{ - ConsumerTag: fmt.Sprintf("Consumer-%s", queueName), - Exclusive: true, - }, - ) - if err != nil { - assert.NoError(t, err) - return - } - - message := fmt.Sprintf("Message-%s", queueName) - - wg.Add(1) - go func(msg string) { - defer wg.Done() - - for val := range delivery { - receivedMsg := string(val.Body) - assert.Equal(t, message, receivedMsg) - } - // this routine must be closed upon session closure - }(message) - - time.Sleep(5 * time.Second) - - pub := pool.NewPublisher(p) - defer pub.Close() - - pub.Publish(ctx, exchangeName, "", pool.Publishing{ + return pub.Publish(ctx, exchangeName, "", pool.Publishing{ Mandatory: true, - ContentType: "application/json", - Body: []byte(message), + ContentType: "text/plain", + Body: []byte(msg), }) - - time.Sleep(5 * time.Second) - - }(int64(id)) + }(msg) + if err != nil { + assert.NoError(t, err, fmt.Sprintf("when publishing message %s", msg)) + return + } } - wg.Wait() } func TestPublishAwaitFlowControl(t *testing.T) { - ctx, cancel := signal.NotifyContext(context.TODO(), os.Interrupt) - defer cancel() + t.Parallel() + + var ( + ctx = context.TODO() + nextConnName = testutils.ConnectionNameGenerator() + ) + + healthyConnCB, hcbAssert := AssertConnectionReconnectAttempts(t, 0) + defer hcbAssert() + hs, hsclose := NewSession( + t, + ctx, + testutils.HealthyConnectURL, + nextConnName(), + pool.ConnectionWithRecoverCallback(healthyConnCB), + ) + defer hsclose() - connections := 1 - sessions := 2 // publisher sessions + consumer sessions p, err := pool.New( ctx, - testutils.BrokenConnectURL, // memory limit or disk limit reached - connections, - sessions, - pool.WithName("TestPublishAwaitFlowControl"), - pool.WithConfirms(true), + testutils.BrokenConnectURL, + 1, + 1, pool.WithLogger(logging.NewTestLogger(t)), + pool.WithConfirms(true), ) if err != nil { assert.NoError(t, err) return } - defer p.Close() - - s, err := p.GetSession(ctx) + defer func() { + p.Close() + }() + var ( + nextExchangeName = testutils.ExchangeNameGenerator(hs.Name()) + nextQueueName = testutils.QueueNameGenerator(hs.Name()) + exchangeName = nextExchangeName() + queueName = nextQueueName() + ) + ts, err := p.GetTransientSession(ctx) if err != nil { assert.NoError(t, err) return } - defer p.ReturnSession(s, nil) + defer func() { + p.ReturnSession(ts, nil) + }() + cleanup := DeclareExchangeQueue(t, ctx, ts, exchangeName, queueName) + defer cleanup() var ( - exchangeName = "TestPublishAwaitFlowControl-Exchange" + publisherMsgGen = testutils.MessageGenerator(queueName) ) - - cleanup := initQueueExchange(t, s, ctx, "TestPublishAwaitFlowControl-Queue", exchangeName) - defer cleanup() - pub := pool.NewPublisher(p) defer pub.Close() - pubGen := PublishingGenerator("TestPublishAwaitFlowControl") - - err = pub.Publish(ctx, exchangeName, "", pubGen()) + err = pub.Publish(ctx, exchangeName, "", pool.Publishing{ + ContentType: "text/plain", + Body: []byte(publisherMsgGen()), + }) assert.ErrorIs(t, err, pool.ErrFlowControl) - -} - -func initQueueExchange(t *testing.T, s *pool.Session, ctx context.Context, queueName, exchangeName string) (cleanup func()) { - _, err := s.QueueDeclare(ctx, queueName) - if err != nil { - assert.NoError(t, err) - return - } - cleanupList := []func(){} - - cleanupQueue := func() { - i, err := s.QueueDelete(ctx, queueName) - assert.NoError(t, err) - assert.Equal(t, 0, i) - } - cleanupList = append(cleanupList, cleanupQueue) - - err = s.ExchangeDeclare(ctx, exchangeName, "topic") - if err != nil { - assert.NoError(t, err) - return - } - cleanupExchange := func() { - err := s.ExchangeDelete(ctx, exchangeName) - assert.NoError(t, err) - } - cleanupList = append(cleanupList, cleanupExchange) - - err = s.QueueBind(ctx, queueName, "#", exchangeName) - if err != nil { - assert.NoError(t, err) - return - } - cleanupBind := func() { - err := s.QueueUnbind(ctx, queueName, "#", exchangeName, nil) - assert.NoError(t, err) - } - cleanupList = append(cleanupList, cleanupBind) - - return func() { - for i := len(cleanupList) - 1; i >= 0; i-- { - cleanupList[i]() - } - } -} - -func PublishingGenerator(MessagePrefix string) func() pool.Publishing { - i := 0 - return func() pool.Publishing { - defer func() { - i++ - }() - return pool.Publishing{ - ContentType: "application/json", - Body: []byte(fmt.Sprintf("%s-%d", MessagePrefix, i)), - } - } } diff --git a/pool/session_test.go b/pool/session_test.go index ecd33d9..526e18b 100644 --- a/pool/session_test.go +++ b/pool/session_test.go @@ -30,6 +30,7 @@ func TestNewSingleSessionPublishAndConsume(t *testing.T) { consumerName = nextConsumerName() consumeMessageGenerator = testutils.MessageGenerator(queueName) publishMessageGenerator = testutils.MessageGenerator(queueName) + numMsgs = 20 ) reconnectCB, deferredAssert := AssertConnectionReconnectAttempts(t, 0) @@ -70,8 +71,8 @@ func TestNewSingleSessionPublishAndConsume(t *testing.T) { cleanup := DeclareExchangeQueue(t, ctx, s, exchangeName, queueName) defer cleanup() - ConsumeAsyncN(t, ctx, &wg, s, queueName, consumerName, consumeMessageGenerator, 20) - PublishAsyncN(t, ctx, &wg, s, exchangeName, publishMessageGenerator, 20) + ConsumeAsyncN(t, ctx, &wg, s, queueName, consumerName, consumeMessageGenerator, numMsgs, false) + PublishAsyncN(t, ctx, &wg, s, exchangeName, publishMessageGenerator, numMsgs) wg.Wait() } @@ -86,6 +87,7 @@ func TestManyNewSessionsPublishAndConsume(t *testing.T) { connName = nextConnName() nextSessionName = testutils.SessionNameGenerator(connName) sessions = 5 + numMsgs = 20 ) reconnectCB, deferredAssert := AssertConnectionReconnectAttempts(t, 0) @@ -135,8 +137,8 @@ func TestManyNewSessionsPublishAndConsume(t *testing.T) { cleanup := DeclareExchangeQueue(t, ctx, s, exchangeName, queueName) defer cleanup() - ConsumeAsyncN(t, ctx, &wg, s, queueName, consumerName, consumeNextMessage, 20) - PublishAsyncN(t, ctx, &wg, s, exchangeName, publishNextMessage, 20) + ConsumeAsyncN(t, ctx, &wg, s, queueName, consumerName, consumeNextMessage, numMsgs, false) + PublishAsyncN(t, ctx, &wg, s, exchangeName, publishNextMessage, numMsgs) } wg.Wait() @@ -662,7 +664,7 @@ func TestNewSessionPublishWithDisconnect(t *testing.T) { disconnected, reconnected = Disconnect(t, proxyName, 5*time.Second) ) - ConsumeAsyncN(t, ctx, &wg, hs, queueName, nextConsumerName(), consumeMsgGen, numMsgs) + ConsumeAsyncN(t, ctx, &wg, hs, queueName, nextConsumerName(), consumeMsgGen, numMsgs, false) disconnected() PublishN(t, ctx, s, exchangeName, publishMsgGen, numMsgs) @@ -725,7 +727,7 @@ func TestNewSessionConsumeWithDisconnect(t *testing.T) { PublishAsyncN(t, ctx, &wg, hs, exchangeName, publisherMsgGen, numMsgs) disconnected() - ConsumeN(t, ctx, s, queueName, nextConsumerName(), consumerMsgGen, numMsgs) + ConsumeN(t, ctx, s, queueName, nextConsumerName(), consumerMsgGen, numMsgs, false) reconnected() wg.Wait() diff --git a/pool/utils_test.go b/pool/utils_test.go index 48c62e8..6332b8a 100644 --- a/pool/utils_test.go +++ b/pool/utils_test.go @@ -26,6 +26,7 @@ func ConsumeN( consumerName string, messageGenerator func() string, n int, + allowDuplicates bool, ) { cctx, ccancel := context.WithCancel(ctx) defer ccancel() @@ -35,7 +36,7 @@ func ConsumeN( defer func() { assert.Equal(t, n, msgsReceived, "expected to consume %d messages, got %d", n, msgsReceived) }() -outer: + for { delivery, err := c.Consume( queueName, @@ -56,8 +57,9 @@ outer: case <-cctx.Done(): return case val, ok := <-delivery: + require.True(t, ok, "expected delivery channel to be open of consumer %s in ConsumeN", consumerName) if !ok { - continue outer + return } err := val.Ack(false) if err != nil { @@ -65,10 +67,15 @@ outer: return } - var ( - expectedMsg = messageGenerator() - receivedMsg = string(val.Body) - ) + var receivedMsg = string(val.Body) + if allowDuplicates && receivedMsg == previouslyReceivedMsg { + // TODO: it is possible that messages are duplicated, but this is not a problem + // due to network issues. We should not fail the test in this case. + log.Warnf("received duplicate message: %s", receivedMsg) + continue + } + + var expectedMsg = messageGenerator() assert.Equalf( t, expectedMsg, @@ -101,12 +108,13 @@ func ConsumeAsyncN( consumerName string, messageGenerator func() string, n int, + alllowDuplicates bool, ) { wg.Add(1) go func() { defer wg.Done() - ConsumeN(t, ctx, c, queueName, consumerName, messageGenerator, n) + ConsumeN(t, ctx, c, queueName, consumerName, messageGenerator, n, alllowDuplicates) }() } From 2e907dac5145380b1f425ffb7583e5f34f83b413 Mon Sep 17 00:00:00 2001 From: John Behm Date: Fri, 8 Mar 2024 22:36:06 +0100 Subject: [PATCH 27/76] update dockerfile to only port forward to localhost --- docker-compose.yaml | 210 ++++++++++++++++++++++---------------------- 1 file changed, 105 insertions(+), 105 deletions(-) diff --git a/docker-compose.yaml b/docker-compose.yaml index 273d630..e4cde20 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -7,10 +7,10 @@ services: #- '4369:4369' #- '5551:5551' #- '5552:5552' - - 5671:5672 # healthy rabbitmq + - 127.0.0.1:5671:5672 # healthy rabbitmq #- '25672:25672' # user interface - - '15672:15672' + - '127.0.0.1:15672:15672' environment: RABBITMQ_NODE_TYPE: stats RABBITMQ_USERNAME: admin @@ -28,10 +28,10 @@ services: #- '14369:4369' #- '15551:5551' #- '15552:5552' - - 5670:5672 # broken rabbitmq + - 127.0.0.1:5670:5672 # broken rabbitmq #- '35672:25672' # user interface - - '25672:15672' + - '127.0.0.1:25672:15672' environment: RABBITMQ_NODE_TYPE: stats RABBITMQ_USERNAME: admin @@ -49,107 +49,107 @@ services: - -host=0.0.0.0 - -config=/toxiproxy.json ports: - - 8474:8474 # toxyproxy port - - 5672:5672 # rabbitmq-5672 - - 5673:5673 # rabbitmq-5673 - - 5674:5674 # rabbitmq-5674 - - 5675:5675 # rabbitmq-5675 - - 5676:5676 # rabbitmq-5676 - - 5677:5677 # rabbitmq-5677 - - 5678:5678 # rabbitmq-5678 - - 5679:5679 # rabbitmq-5679 - - 5680:5680 # rabbitmq-5680 - - 5681:5681 # rabbitmq-5681 - - 5682:5682 # rabbitmq-5682 - - 5683:5683 # rabbitmq-5683 - - 5684:5684 # rabbitmq-5684 - - 5685:5685 # rabbitmq-5685 - - 5686:5686 # rabbitmq-5686 - - 5687:5687 # rabbitmq-5687 - - 5688:5688 # rabbitmq-5688 - - 5689:5689 # rabbitmq-5689 - - 5690:5690 # rabbitmq-5690 - - 5691:5691 # rabbitmq-5691 - - 5692:5692 # rabbitmq-5692 - - 5693:5693 # rabbitmq-5693 - - 5694:5694 # rabbitmq-5694 - - 5695:5695 # rabbitmq-5695 - - 5696:5696 # rabbitmq-5696 - - 5697:5697 # rabbitmq-5697 - - 5698:5698 # rabbitmq-5698 - - 5699:5699 # rabbitmq-5699 - - 5700:5700 # rabbitmq-5700 - - 5701:5701 # rabbitmq-5701 - - 5702:5702 # rabbitmq-5702 - - 5703:5703 # rabbitmq-5703 - - 5704:5704 # rabbitmq-5704 - - 5705:5705 # rabbitmq-5705 - - 5706:5706 # rabbitmq-5706 - - 5707:5707 # rabbitmq-5707 - - 5708:5708 # rabbitmq-5708 - - 5709:5709 # rabbitmq-5709 - - 5710:5710 # rabbitmq-5710 - - 5711:5711 # rabbitmq-5711 - - 5712:5712 # rabbitmq-5712 - - 5713:5713 # rabbitmq-5713 - - 5714:5714 # rabbitmq-5714 - - 5715:5715 # rabbitmq-5715 - - 5716:5716 # rabbitmq-5716 - - 5717:5717 # rabbitmq-5717 - - 5718:5718 # rabbitmq-5718 - - 5719:5719 # rabbitmq-5719 - - 5720:5720 # rabbitmq-5720 - - 5721:5721 # rabbitmq-5721 - - 5722:5722 # rabbitmq-5722 - - 5723:5723 # rabbitmq-5723 - - 5724:5724 # rabbitmq-5724 - - 5725:5725 # rabbitmq-5725 - - 5726:5726 # rabbitmq-5726 - - 5727:5727 # rabbitmq-5727 - - 5728:5728 # rabbitmq-5728 - - 5729:5729 # rabbitmq-5729 - - 5730:5730 # rabbitmq-5730 - - 5731:5731 # rabbitmq-5731 - - 5732:5732 # rabbitmq-5732 - - 5733:5733 # rabbitmq-5733 - - 5734:5734 # rabbitmq-5734 - - 5735:5735 # rabbitmq-5735 - - 5736:5736 # rabbitmq-5736 - - 5737:5737 # rabbitmq-5737 - - 5738:5738 # rabbitmq-5738 - - 5739:5739 # rabbitmq-5739 - - 5740:5740 # rabbitmq-5740 - - 5741:5741 # rabbitmq-5741 - - 5742:5742 # rabbitmq-5742 - - 5743:5743 # rabbitmq-5743 - - 5744:5744 # rabbitmq-5744 - - 5745:5745 # rabbitmq-5745 - - 5746:5746 # rabbitmq-5746 - - 5747:5747 # rabbitmq-5747 - - 5748:5748 # rabbitmq-5748 - - 5749:5749 # rabbitmq-5749 - - 5750:5750 # rabbitmq-5750 - - 5751:5751 # rabbitmq-5751 - - 5752:5752 # rabbitmq-5752 - - 5753:5753 # rabbitmq-5753 - - 5754:5754 # rabbitmq-5754 - - 5755:5755 # rabbitmq-5755 - - 5756:5756 # rabbitmq-5756 - - 5757:5757 # rabbitmq-5757 - - 5758:5758 # rabbitmq-5758 - - 5759:5759 # rabbitmq-5759 - - 5760:5760 # rabbitmq-5760 - - 5761:5761 # rabbitmq-5761 - - 5762:5762 # rabbitmq-5762 - - 5763:5763 # rabbitmq-5763 - - 5764:5764 # rabbitmq-5764 - - 5765:5765 # rabbitmq-5765 - - 5766:5766 # rabbitmq-5766 - - 5767:5767 # rabbitmq-5767 - - 5768:5768 # rabbitmq-5768 - - 5769:5769 # rabbitmq-5769 - - 5770:5770 # rabbitmq-5770 - - 5771:5771 # rabbitmq-5771 + - 127.0.0.1:8474:8474 # toxyproxy port + - 127.0.0.1:5672:5672 # rabbitmq-5672 + - 127.0.0.1:5673:5673 # rabbitmq-5673 + - 127.0.0.1:5674:5674 # rabbitmq-5674 + - 127.0.0.1:5675:5675 # rabbitmq-5675 + - 127.0.0.1:5676:5676 # rabbitmq-5676 + - 127.0.0.1:5677:5677 # rabbitmq-5677 + - 127.0.0.1:5678:5678 # rabbitmq-5678 + - 127.0.0.1:5679:5679 # rabbitmq-5679 + - 127.0.0.1:5680:5680 # rabbitmq-5680 + - 127.0.0.1:5681:5681 # rabbitmq-5681 + - 127.0.0.1:5682:5682 # rabbitmq-5682 + - 127.0.0.1:5683:5683 # rabbitmq-5683 + - 127.0.0.1:5684:5684 # rabbitmq-5684 + - 127.0.0.1:5685:5685 # rabbitmq-5685 + - 127.0.0.1:5686:5686 # rabbitmq-5686 + - 127.0.0.1:5687:5687 # rabbitmq-5687 + - 127.0.0.1:5688:5688 # rabbitmq-5688 + - 127.0.0.1:5689:5689 # rabbitmq-5689 + - 127.0.0.1:5690:5690 # rabbitmq-5690 + - 127.0.0.1:5691:5691 # rabbitmq-5691 + - 127.0.0.1:5692:5692 # rabbitmq-5692 + - 127.0.0.1:5693:5693 # rabbitmq-5693 + - 127.0.0.1:5694:5694 # rabbitmq-5694 + - 127.0.0.1:5695:5695 # rabbitmq-5695 + - 127.0.0.1:5696:5696 # rabbitmq-5696 + - 127.0.0.1:5697:5697 # rabbitmq-5697 + - 127.0.0.1:5698:5698 # rabbitmq-5698 + - 127.0.0.1:5699:5699 # rabbitmq-5699 + - 127.0.0.1:5700:5700 # rabbitmq-5700 + - 127.0.0.1:5701:5701 # rabbitmq-5701 + - 127.0.0.1:5702:5702 # rabbitmq-5702 + - 127.0.0.1:5703:5703 # rabbitmq-5703 + - 127.0.0.1:5704:5704 # rabbitmq-5704 + - 127.0.0.1:5705:5705 # rabbitmq-5705 + - 127.0.0.1:5706:5706 # rabbitmq-5706 + - 127.0.0.1:5707:5707 # rabbitmq-5707 + - 127.0.0.1:5708:5708 # rabbitmq-5708 + - 127.0.0.1:5709:5709 # rabbitmq-5709 + - 127.0.0.1:5710:5710 # rabbitmq-5710 + - 127.0.0.1:5711:5711 # rabbitmq-5711 + - 127.0.0.1:5712:5712 # rabbitmq-5712 + - 127.0.0.1:5713:5713 # rabbitmq-5713 + - 127.0.0.1:5714:5714 # rabbitmq-5714 + - 127.0.0.1:5715:5715 # rabbitmq-5715 + - 127.0.0.1:5716:5716 # rabbitmq-5716 + - 127.0.0.1:5717:5717 # rabbitmq-5717 + - 127.0.0.1:5718:5718 # rabbitmq-5718 + - 127.0.0.1:5719:5719 # rabbitmq-5719 + - 127.0.0.1:5720:5720 # rabbitmq-5720 + - 127.0.0.1:5721:5721 # rabbitmq-5721 + - 127.0.0.1:5722:5722 # rabbitmq-5722 + - 127.0.0.1:5723:5723 # rabbitmq-5723 + - 127.0.0.1:5724:5724 # rabbitmq-5724 + - 127.0.0.1:5725:5725 # rabbitmq-5725 + - 127.0.0.1:5726:5726 # rabbitmq-5726 + - 127.0.0.1:5727:5727 # rabbitmq-5727 + - 127.0.0.1:5728:5728 # rabbitmq-5728 + - 127.0.0.1:5729:5729 # rabbitmq-5729 + - 127.0.0.1:5730:5730 # rabbitmq-5730 + - 127.0.0.1:5731:5731 # rabbitmq-5731 + - 127.0.0.1:5732:5732 # rabbitmq-5732 + - 127.0.0.1:5733:5733 # rabbitmq-5733 + - 127.0.0.1:5734:5734 # rabbitmq-5734 + - 127.0.0.1:5735:5735 # rabbitmq-5735 + - 127.0.0.1:5736:5736 # rabbitmq-5736 + - 127.0.0.1:5737:5737 # rabbitmq-5737 + - 127.0.0.1:5738:5738 # rabbitmq-5738 + - 127.0.0.1:5739:5739 # rabbitmq-5739 + - 127.0.0.1:5740:5740 # rabbitmq-5740 + - 127.0.0.1:5741:5741 # rabbitmq-5741 + - 127.0.0.1:5742:5742 # rabbitmq-5742 + - 127.0.0.1:5743:5743 # rabbitmq-5743 + - 127.0.0.1:5744:5744 # rabbitmq-5744 + - 127.0.0.1:5745:5745 # rabbitmq-5745 + - 127.0.0.1:5746:5746 # rabbitmq-5746 + - 127.0.0.1:5747:5747 # rabbitmq-5747 + - 127.0.0.1:5748:5748 # rabbitmq-5748 + - 127.0.0.1:5749:5749 # rabbitmq-5749 + - 127.0.0.1:5750:5750 # rabbitmq-5750 + - 127.0.0.1:5751:5751 # rabbitmq-5751 + - 127.0.0.1:5752:5752 # rabbitmq-5752 + - 127.0.0.1:5753:5753 # rabbitmq-5753 + - 127.0.0.1:5754:5754 # rabbitmq-5754 + - 127.0.0.1:5755:5755 # rabbitmq-5755 + - 127.0.0.1:5756:5756 # rabbitmq-5756 + - 127.0.0.1:5757:5757 # rabbitmq-5757 + - 127.0.0.1:5758:5758 # rabbitmq-5758 + - 127.0.0.1:5759:5759 # rabbitmq-5759 + - 127.0.0.1:5760:5760 # rabbitmq-5760 + - 127.0.0.1:5761:5761 # rabbitmq-5761 + - 127.0.0.1:5762:5762 # rabbitmq-5762 + - 127.0.0.1:5763:5763 # rabbitmq-5763 + - 127.0.0.1:5764:5764 # rabbitmq-5764 + - 127.0.0.1:5765:5765 # rabbitmq-5765 + - 127.0.0.1:5766:5766 # rabbitmq-5766 + - 127.0.0.1:5767:5767 # rabbitmq-5767 + - 127.0.0.1:5768:5768 # rabbitmq-5768 + - 127.0.0.1:5769:5769 # rabbitmq-5769 + - 127.0.0.1:5770:5770 # rabbitmq-5770 + - 127.0.0.1:5771:5771 # rabbitmq-5771 networks: - rabbitnet From cd930e7b58f34ac4b5a32603c08a767cada1fe27 Mon Sep 17 00:00:00 2001 From: John Behm Date: Fri, 8 Mar 2024 22:46:51 +0100 Subject: [PATCH 28/76] update readme --- README.md | 37 +++++++++++++++++++++++++++++++------ 1 file changed, 31 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index 67768a3..b8af61a 100644 --- a/README.md +++ b/README.md @@ -194,18 +194,43 @@ A `Subscriber` must be `Start()`ed in order for it to create consumer goroutines Test flags you might want to add: ```shell --v -race -count=1 +go test -v -race -count=1 ./... ``` - see test logs - detect data races - do not cache test results -Starting the test environment: +Starting the tests: ```shell -docker-compose up -d +go test -v -race -count=1 ./... ``` -Starting the tests: +### Test environment + +- Requires docker (and docker compose subcommand) + +Starting the test environment: ```shell -go test -v -race -count=1 ./... -``` \ No newline at end of file +make environment +#or +docker compose up -d +``` + +The test environment looks like this: + +Web interfaces: + - [rabbitmq management interface: http://127.0.0.1:15672 -> rabbitmq:15672](http://127.0.0.1:15672) + - [out of memory rabbitmq management interface: http://127.0.0.1:25672 -> rabbitmq-broken:15672](http://127.0.0.1:25672) + +``` +127.0.0.1:5670 -> rabbitmq-broken:5672 # out of memory rabbitmq +127.0.0.1:5671 -> rabbitmq:5672 # healthy rabbitmq connection which is never disconnected + + +127.0.0.1:5672 -> toxiproxy:5672 -> rabbitmq:5672 # connection which is disconnected by toxiproxy +127.0.0.1:5673 -> toxiproxy:5673 -> rabbitmq:5672 # connection which is disconnected by toxiproxy +127.0.0.1:5674 -> toxiproxy:5674 -> rabbitmq:5672 # connection which is disconnected by toxiproxy +... +127.0.0.1:5771 -> toxiproxy:5771 -> rabbitmq:5672 # connection which is disconnected by toxiproxy + +``` From 85bd49f98ebde73669f73826ae56f7771bfa4972 Mon Sep 17 00:00:00 2001 From: John Behm Date: Sat, 9 Mar 2024 00:05:55 +0100 Subject: [PATCH 29/76] add address log field to connection logs --- pool/connection.go | 20 +++++++++++++------- 1 file changed, 13 insertions(+), 7 deletions(-) diff --git a/pool/connection.go b/pool/connection.go index bb8259c..1544322 100644 --- a/pool/connection.go +++ b/pool/connection.go @@ -18,6 +18,8 @@ import ( type Connection struct { // connection url (user ,password, host, port, vhost, etc) url string + addr string + name string // indicates that the connection is part of a connection pool. @@ -89,6 +91,7 @@ func NewConnection(ctx context.Context, connectUrl, name string, options ...Conn conn := &Connection{ url: u.String(), + addr: u.Host, name: name, cached: option.Cached, flagged: false, @@ -371,18 +374,21 @@ func (ch *Connection) shutdownErr() error { return ch.ctx.Err() } -func (ch *Connection) info(a ...any) { - ch.log.WithField("connection", ch.Name()).Info(a...) +func (ch *Connection) cLog() logging.Logger { + return ch.log.WithFields(map[string]any{ + "connection": ch.name, + "address": ch.addr, + }) } -func (ch *Connection) warn(err error, a ...any) { - ch.log.WithField("connection", ch.Name()).WithField("error", err.Error()).Warn(a...) +func (ch *Connection) info(a ...any) { + ch.cLog().Info(a...) } -func (ch *Connection) warnf(format string, a ...any) { - ch.log.WithField("connection", ch.Name()).Warnf(format, a...) +func (ch *Connection) warn(err error, a ...any) { + ch.cLog().WithField("error", err.Error()).Warn(a...) } func (ch *Connection) debug(a ...any) { - ch.log.WithField("connection", ch.Name()).Debug(a...) + ch.cLog().Debug(a...) } From d3d8a08230209cd0b075c79ed3e22d25f0647f53 Mon Sep 17 00:00:00 2001 From: John Behm Date: Sat, 9 Mar 2024 00:17:40 +0100 Subject: [PATCH 30/76] add returned channel to session & to AwaitConfirm --- pool/errors.go | 6 ++++++ pool/session.go | 16 ++++++++++++++++ 2 files changed, 22 insertions(+) diff --git a/pool/errors.go b/pool/errors.go index a0b0466..9196241 100644 --- a/pool/errors.go +++ b/pool/errors.go @@ -31,6 +31,12 @@ var ( // TODO: make public api after a while errFlowControlClosed = errors.New("flow control channel closed") + // ErrReturned is returned when a message is returned by the server when publishing + ErrReturned = errors.New("returned") + + // errReturnedClosed + errReturnedClosed = errors.New("returned channel closed") + // ErrReject can be used to reject a specific message // This is a special error that negatively acknowledges messages and does not reuque them. ErrReject = errors.New("message rejected") diff --git a/pool/session.go b/pool/session.go index 37f2507..cfdb2a5 100644 --- a/pool/session.go +++ b/pool/session.go @@ -24,6 +24,7 @@ type Session struct { bufferSize int channel *amqp091.Channel + returned chan amqp091.Return confirms chan amqp091.Confirmation errors chan *amqp091.Error @@ -110,6 +111,7 @@ func NewSession(conn *Connection, name string, options ...SessionOption) (*Sessi channel: nil, // will be created on connect errors: nil, // will be created on connect confirms: nil, // will be created on connect + returned: nil, // will be created on connect conn: conn, autoCloseConn: option.AutoCloseConn, @@ -198,6 +200,8 @@ func (s *Session) close() (err error) { s.debug("flushing channels...") flush(s.errors) flush(s.confirms) + flush(s.returned) + if s.channel != nil { s.channel = nil } @@ -270,6 +274,9 @@ func (s *Session) connect() (err error) { if err != nil { return err } + + s.returned = make(chan amqp091.Return, s.bufferSize) + channel.NotifyReturn(s.returned) } // reset consumer tracking upon reconnect @@ -377,6 +384,15 @@ func (s *Session) AwaitConfirm(ctx context.Context, expectedTag uint64) error { return fmt.Errorf("await confirm failed: %w: expected %d, got %d", ErrDeliveryTagMismatch, expectedTag, confirm.DeliveryTag) } return nil + case returned, ok := <-s.returned: + if !ok { + err := s.error() + if err != nil { + return fmt.Errorf("await confirm failed: returned channel closed: %w", err) + } + return fmt.Errorf("await confirm failed: %w", errReturnedClosed) + } + return fmt.Errorf("await confirm failed: %w: %s", ErrReturned, returned.ReplyText) case blocking, ok := <-s.conn.FlowControl(): if !ok { err := s.error() From 55a7e5bc92810c13c7246f826f5c0334f4a166da Mon Sep 17 00:00:00 2001 From: John Behm Date: Sat, 9 Mar 2024 00:18:15 +0100 Subject: [PATCH 31/76] update proxy utility --- pool/toxiproxy_client_test.go | 34 ++++++++++++++++------------------ 1 file changed, 16 insertions(+), 18 deletions(-) diff --git a/pool/toxiproxy_client_test.go b/pool/toxiproxy_client_test.go index b4eaae6..d569459 100644 --- a/pool/toxiproxy_client_test.go +++ b/pool/toxiproxy_client_test.go @@ -7,6 +7,7 @@ import ( toxiproxy "github.com/Shopify/toxiproxy/v2/client" "github.com/jxsl13/amqpx/logging" + "github.com/stretchr/testify/require" ) type Proxy struct { @@ -23,22 +24,13 @@ func NewProxy(t *testing.T, proxyName string) *Proxy { m, err := toxi.Proxies() if err != nil { - log.Fatal(err) + require.NoError(t, err) } var proxy *toxiproxy.Proxy proxy, found := m[proxyName] - if !found { - log.Fatalf("no proxy with name %s found", proxyName) - } - for k, p := range m { - if k == "rabbitmq" { - proxy = p - } - } - if proxy == nil { - log.Fatalf("proxy with name %s is nil", proxyName) - } + require.Truef(t, found, "no proxy with name %s found", proxyName) + require.NotNil(t, proxy, "proxy with name %s is nil", proxyName) return &Proxy{ proxy: proxy, @@ -86,6 +78,7 @@ func (p *Proxy) Close() error { return nil } + // TODO: this can be removed waitTime := time.Until(p.lastHttpRequest.Add(10 * time.Second)) if waitTime > 0 { time.Sleep(waitTime) @@ -105,6 +98,8 @@ func DisconnectWithStopped(t *testing.T, proxyName string, block, timeout, durat wg sync.WaitGroup proxy = NewProxy(t, proxyName) ) + require.NoError(t, proxy.Enable()) + wg.Add(1) go func(start time.Time) { defer wg.Done() @@ -116,7 +111,7 @@ func DisconnectWithStopped(t *testing.T, proxyName string, block, timeout, durat log.Debug("disabled rabbitmq connection") err := proxy.Disable() if err != nil { - log.Fatal(err) + require.NoError(t, err) } if duration > 0 { @@ -125,7 +120,7 @@ func DisconnectWithStopped(t *testing.T, proxyName string, block, timeout, durat log.Debug("enabled rabbitmq connection") err = proxy.Enable() if err != nil { - log.Fatal(err) + require.NoError(t, err) } }(start) if block > 0 { @@ -149,6 +144,7 @@ func DisconnectWithStartedStopped(t *testing.T, proxyName string, block, startIn wgStop sync.WaitGroup proxy = NewProxy(t, proxyName) ) + require.NoError(t, proxy.Enable()) wgStart.Add(1) wgStop.Add(1) @@ -162,7 +158,7 @@ func DisconnectWithStartedStopped(t *testing.T, proxyName string, block, startIn log.Debug("disabled rabbitmq connection") err := proxy.Disable() if err != nil { - log.Fatal(err) + require.NoError(t, err) } wgStart.Done() @@ -172,7 +168,7 @@ func DisconnectWithStartedStopped(t *testing.T, proxyName string, block, startIn log.Debug("enabled rabbitmq connection") err = proxy.Enable() if err != nil { - log.Fatal(err) + require.NoError(t, err) } }(start) if block > 0 { @@ -213,6 +209,8 @@ func DisconnectWithStartStartedStopped(t *testing.T, proxyName string, duration wgStop sync.WaitGroup proxy = NewProxy(t, proxyName) ) + require.NoError(t, proxy.Enable()) + disconnect = func() { wgStart.Add(1) wgStop.Add(1) @@ -223,7 +221,7 @@ func DisconnectWithStartStartedStopped(t *testing.T, proxyName string, duration log.Debug("disabled rabbitmq connection") err := proxy.Disable() if err != nil { - log.Fatal(err) + require.NoError(t, err) } wgStart.Done() @@ -233,7 +231,7 @@ func DisconnectWithStartStartedStopped(t *testing.T, proxyName string, duration log.Debug("enabled rabbitmq connection") err = proxy.Enable() if err != nil { - log.Fatal(err) + require.NoError(t, err) } }() } From 7fdb5fe75a516b73de51d983067c55ba73a36a14 Mon Sep 17 00:00:00 2001 From: John Behm Date: Sat, 9 Mar 2024 00:18:27 +0100 Subject: [PATCH 32/76] update publisher test --- pool/publisher_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pool/publisher_test.go b/pool/publisher_test.go index 07b38f3..a3f35ba 100644 --- a/pool/publisher_test.go +++ b/pool/publisher_test.go @@ -20,7 +20,7 @@ func TestSinglePublisher(t *testing.T) { proxyName, connectURL, _ = testutils.NextConnectURL() ctx = context.TODO() nextConnName = testutils.ConnectionNameGenerator() - numMsgs = 20 + numMsgs = 10 ) healthyConnCB, hcbAssert := AssertConnectionReconnectAttempts(t, 0) From 33f5df3bdc177859164903b86f1c4ac5829cf87f Mon Sep 17 00:00:00 2001 From: John Behm Date: Sat, 9 Mar 2024 00:19:38 +0100 Subject: [PATCH 33/76] do not defer reconnected function --- pool/session_test.go | 53 +++++++++++++++++++++++++++++--------------- 1 file changed, 35 insertions(+), 18 deletions(-) diff --git a/pool/session_test.go b/pool/session_test.go index 526e18b..e8e6369 100644 --- a/pool/session_test.go +++ b/pool/session_test.go @@ -9,6 +9,7 @@ import ( "github.com/jxsl13/amqpx/internal/testutils" "github.com/jxsl13/amqpx/logging" "github.com/jxsl13/amqpx/pool" + amqp "github.com/rabbitmq/amqp091-go" "github.com/stretchr/testify/assert" ) @@ -71,7 +72,7 @@ func TestNewSingleSessionPublishAndConsume(t *testing.T) { cleanup := DeclareExchangeQueue(t, ctx, s, exchangeName, queueName) defer cleanup() - ConsumeAsyncN(t, ctx, &wg, s, queueName, consumerName, consumeMessageGenerator, numMsgs, false) + ConsumeAsyncN(t, ctx, &wg, s, queueName, consumerName, consumeMessageGenerator, numMsgs, true) PublishAsyncN(t, ctx, &wg, s, exchangeName, publishMessageGenerator, numMsgs) wg.Wait() @@ -137,7 +138,7 @@ func TestManyNewSessionsPublishAndConsume(t *testing.T) { cleanup := DeclareExchangeQueue(t, ctx, s, exchangeName, queueName) defer cleanup() - ConsumeAsyncN(t, ctx, &wg, s, queueName, consumerName, consumeNextMessage, numMsgs, false) + ConsumeAsyncN(t, ctx, &wg, s, queueName, consumerName, consumeNextMessage, numMsgs, true) PublishAsyncN(t, ctx, &wg, s, exchangeName, publishNextMessage, numMsgs) } @@ -266,18 +267,15 @@ func TestNewSessionExchangeDeclareWithDisconnect(t *testing.T) { }() disconnected() - defer reconnected() - err = s.ExchangeDeclare(ctx, exchangeName, pool.ExchangeKindTopic) if err != nil { assert.NoError(t, err) return } + reconnected() - defer func() { - err := s.ExchangeDelete(ctx, exchangeName) - assert.NoError(t, err) - }() + err = s.ExchangeDelete(ctx, exchangeName) + assert.NoError(t, err) } func TestNewSessionExchangeDeleteWithDisconnect(t *testing.T) { @@ -382,20 +380,18 @@ func TestNewSessionQueueDeclareWithDisconnect(t *testing.T) { }() disconnected() - defer reconnected() _, err = s.QueueDeclare(ctx, queueName) if err != nil { assert.NoError(t, err) return } + reconnected() - defer func() { - _, err := s.QueueDelete(ctx, queueName) - assert.NoError(t, err, "expected no error when deleting queue") - // TODO: asserting the number of deleted messages seems to be pretty flaky, so we do not assert it here - // assert.Equal(t, 0, delMsgs, "expected 0 messages to be deleted") - }() + _, err = s.QueueDelete(ctx, queueName) + assert.NoError(t, err, "expected no error when deleting queue") + // TODO: asserting the number of deleted messages seems to be pretty flaky, so we do not assert it here + // assert.Equal(t, 0, delMsgs, "expected 0 messages to be deleted") } func TestNewSessionQueueDeleteWithDisconnect(t *testing.T) { @@ -449,11 +445,10 @@ func TestNewSessionQueueDeleteWithDisconnect(t *testing.T) { } disconnected() - defer reconnected() - delMsgs, err := s.QueueDelete(ctx, queueName) assert.NoError(t, err) assert.Equal(t, 0, delMsgs, "expected 0 messages to be deleted") + reconnected() } func TestNewSessionQueueBindWithDisconnect(t *testing.T) { @@ -664,7 +659,7 @@ func TestNewSessionPublishWithDisconnect(t *testing.T) { disconnected, reconnected = Disconnect(t, proxyName, 5*time.Second) ) - ConsumeAsyncN(t, ctx, &wg, hs, queueName, nextConsumerName(), consumeMsgGen, numMsgs, false) + ConsumeAsyncN(t, ctx, &wg, hs, queueName, nextConsumerName(), consumeMsgGen, numMsgs, true) disconnected() PublishN(t, ctx, s, exchangeName, publishMsgGen, numMsgs) @@ -732,3 +727,25 @@ func TestNewSessionConsumeWithDisconnect(t *testing.T) { wg.Wait() } + +func TestChannelCloseOnOutOfMemoryRabbitMQ(t *testing.T) { + t.Parallel() + + amqpConn, err := amqp.Dial(testutils.BrokenConnectURL) + if err != nil { + assert.NoError(t, err) + return + } + defer func() { + assert.NoError(t, amqpConn.Close(), "expected no error when closing connection") + }() + + amqpChan, err := amqpConn.Channel() + if err != nil { + assert.NoError(t, err, "expected no error when creating channel") + return + } + + err = amqpChan.Close() + assert.NoError(t, err, "expected no error when closing channel") +} From 460785b1075203bb1ade23ff5181451de684a61c Mon Sep 17 00:00:00 2001 From: John Behm Date: Sat, 9 Mar 2024 00:33:11 +0100 Subject: [PATCH 34/76] cleanup logging --- pool/connection.go | 8 ++++---- pool/session.go | 29 +++++++++++++++++++---------- 2 files changed, 23 insertions(+), 14 deletions(-) diff --git a/pool/connection.go b/pool/connection.go index 1544322..589d8d6 100644 --- a/pool/connection.go +++ b/pool/connection.go @@ -374,7 +374,7 @@ func (ch *Connection) shutdownErr() error { return ch.ctx.Err() } -func (ch *Connection) cLog() logging.Logger { +func (ch *Connection) clog() logging.Logger { return ch.log.WithFields(map[string]any{ "connection": ch.name, "address": ch.addr, @@ -382,13 +382,13 @@ func (ch *Connection) cLog() logging.Logger { } func (ch *Connection) info(a ...any) { - ch.cLog().Info(a...) + ch.clog().Info(a...) } func (ch *Connection) warn(err error, a ...any) { - ch.cLog().WithField("error", err.Error()).Warn(a...) + ch.clog().WithError(err).Warn(a...) } func (ch *Connection) debug(a ...any) { - ch.cLog().Debug(a...) + ch.clog().Debug(a...) } diff --git a/pool/session.go b/pool/session.go index cfdb2a5..9e011a5 100644 --- a/pool/session.go +++ b/pool/session.go @@ -211,12 +211,14 @@ func (s *Session) close() (err error) { return nil } - s.debug("canceling consumers...") - for consumer := range s.consumers { - // ignore error, as at this point we cannot do anything about the error - // tell server to cancel consumer deliveries. - cerr := s.channel.Cancel(consumer, false) - err = errors.Join(err, cerr) + if len(s.consumers) > 0 { + s.debug("canceling consumers...") + for consumer := range s.consumers { + // ignore error, as at this point we cannot do anything about the error + // tell server to cancel consumer deliveries. + cerr := s.channel.Cancel(consumer, false) + err = errors.Join(err, cerr) + } } if s.error() != nil { @@ -1482,20 +1484,27 @@ func (s *Session) shutdownErr() error { return s.ctx.Err() } +func (s *Session) slog() logging.Logger { + return s.log.WithFields(logging.Fields{ + "connection": s.conn.Name(), + "session": s.name, + }) +} + func (s *Session) info(a ...any) { - s.log.WithField("connection", s.conn.Name()).WithField("session", s.Name()).Info(a...) + s.slog().Info(a...) } func (s *Session) warn(err error, a ...any) { - s.log.WithField("connection", s.conn.Name()).WithField("session", s.Name()).WithField("error", err.Error()).Warn(a...) + s.slog().WithError(err).Warn(a...) } func (s *Session) warnf(err error, format string, a ...any) { - s.log.WithField("connection", s.conn.Name()).WithField("session", s.Name()).WithField("error", err.Error()).Warnf(format, a...) + s.slog().WithError(err).Warnf(format, a...) } func (s *Session) debug(a ...any) { - s.log.WithField("connection", s.conn.Name()).WithField("session", s.Name()).Debug(a...) + s.slog().Debug(a...) } // flush should be called when you return the session back to the pool From bed77d4c23f579613fa640f92692d4f6368c87ab Mon Sep 17 00:00:00 2001 From: John Behm Date: Sat, 9 Mar 2024 01:00:13 +0100 Subject: [PATCH 35/76] refactor and simplify first subscriber tests to run in parallel --- pool/publisher_test.go | 1 + pool/subscriber_handler_options_test.go | 2 + pool/subscriber_test.go | 246 ++++++++++++++++-------- 3 files changed, 164 insertions(+), 85 deletions(-) diff --git a/pool/publisher_test.go b/pool/publisher_test.go index a3f35ba..a3f72c6 100644 --- a/pool/publisher_test.go +++ b/pool/publisher_test.go @@ -161,4 +161,5 @@ func TestPublishAwaitFlowControl(t *testing.T) { Body: []byte(publisherMsgGen()), }) assert.ErrorIs(t, err, pool.ErrFlowControl) + // FIXME: this test gets stuck when the sessions in the session pool are closed.: } diff --git a/pool/subscriber_handler_options_test.go b/pool/subscriber_handler_options_test.go index 507eabb..f2c151c 100644 --- a/pool/subscriber_handler_options_test.go +++ b/pool/subscriber_handler_options_test.go @@ -8,6 +8,8 @@ import ( ) func TestWithMaxBatchSize(t *testing.T) { + t.Parallel() + dummyHandler := func(context.Context, []Delivery) error { return nil } bh := NewBatchHandler("test", dummyHandler, WithMaxBatchSize(0), WithMaxBatchBytes(0)) diff --git a/pool/subscriber_test.go b/pool/subscriber_test.go index f1f809c..e2f4d99 100644 --- a/pool/subscriber_test.go +++ b/pool/subscriber_test.go @@ -13,120 +13,196 @@ import ( "github.com/stretchr/testify/assert" ) -func TestSubscriber(t *testing.T) { +func TestNewSingleSubscriber(t *testing.T) { + t.Parallel() - ctx := context.TODO() - sessions := 2 // publisher sessions + consumer sessions - p, err := pool.New( + var ( + ctx = context.TODO() + poolName = testutils.FuncName() + ) + + hp, err := pool.New( ctx, testutils.HealthyConnectURL, 1, - sessions, + 2, pool.WithConfirms(true), pool.WithLogger(logging.NewTestLogger(t)), + pool.WithName(poolName), ) if err != nil { assert.NoError(t, err) return } - defer p.Close() + defer hp.Close() - var wg sync.WaitGroup + var ( + nextExchangeName = testutils.ExchangeNameGenerator(poolName) + nextQueueName = testutils.QueueNameGenerator(poolName) + exchangeName = nextExchangeName() + queueName = nextQueueName() + nextConsumerName = testutils.ConsumerNameGenerator(queueName) + consumerName = nextConsumerName() + publisherMsgGen = testutils.MessageGenerator(queueName) + subscriberMsgGen = testutils.MessageGenerator(queueName) + ) - channels := sessions / 2 // one sessions for consumer and one for publisher - wg.Add(channels) - for id := 0; id < channels; id++ { - go func(id int64) { - defer wg.Done() + ts, err := hp.GetTransientSession(ctx) + if err != nil { + assert.NoError(t, err) + return + } + defer hp.ReturnSession(ts, nil) - ts, err := p.GetTransientSession(p.Context()) - if err != nil { - assert.NoError(t, err) - return - } - defer p.ReturnSession(ts, nil) + cleanup := DeclareExchangeQueue(t, ctx, ts, exchangeName, queueName) + defer cleanup() - queueName := fmt.Sprintf("TestSubscriber-Queue-%d", id) - _, err = ts.QueueDeclare(ctx, queueName) - if err != nil { - assert.NoError(t, err) - return - } - defer func() { - i, err := ts.QueueDelete(ctx, queueName) - assert.NoError(t, err) - assert.Equal(t, 0, i) - }() + cctx, cancel := context.WithCancel(ctx) - exchangeName := fmt.Sprintf("TestSubscriber-Exchange-%d", id) - err = ts.ExchangeDeclare(ctx, exchangeName, "topic") - if err != nil { - assert.NoError(t, err) - return - } - defer func() { - err := ts.ExchangeDelete(ctx, exchangeName) - assert.NoError(t, err) - }() + sub := pool.NewSubscriber(hp, pool.SubscriberWithContext(cctx)) + defer sub.Close() - err = ts.QueueBind(ctx, queueName, "#", exchangeName) - if err != nil { - assert.NoError(t, err) - return - } - defer func() { - err := ts.QueueUnbind(ctx, queueName, "#", exchangeName, nil) - assert.NoError(t, err) - }() + sub.RegisterHandlerFunc(queueName, + func(ctx context.Context, msg pool.Delivery) error { - message := fmt.Sprintf("Message-%s", queueName) + // handler func + receivedMsg := string(msg.Body) + // assert equel to message that is to be sent + assert.Equal(t, subscriberMsgGen(), receivedMsg) - cctx, cancel := context.WithCancel(p.Context()) + // close subscriber from within handler + cancel() + return nil + }, + pool.ConsumeOptions{ + ConsumerTag: consumerName, + Exclusive: true, + }, + ) + err = sub.Start(ctx) + if err != nil { + assert.NoError(t, err) + return + } + time.Sleep(5 * time.Second) - sub := pool.NewSubscriber(p, pool.SubscriberWithContext(cctx)) - defer sub.Close() + pub := pool.NewPublisher(hp) + defer pub.Close() - sub.RegisterHandlerFunc(queueName, - func(ctx context.Context, msg pool.Delivery) error { + err = pub.Publish(ctx, exchangeName, "", pool.Publishing{ + Mandatory: true, + ContentType: "application/json", + Body: []byte(publisherMsgGen()), + }) + assert.NoError(t, err) - // handler func - receivedMsg := string(msg.Body) - // assert equel to message that is to be sent - assert.Equal(t, message, receivedMsg) + // this should be canceled upon context cancelation from within the + // subscriber handler. + sub.Wait() +} - // close subscriber from within handler - cancel() - return nil - }, - pool.ConsumeOptions{ - ConsumerTag: fmt.Sprintf("Consumer-%s", queueName), - Exclusive: true, - }, - ) - err = sub.Start(ctx) - if err != nil { - assert.NoError(t, err) - return - } - time.Sleep(5 * time.Second) +func TestNewSubscriberWithDisconnect(t *testing.T) { + t.Parallel() - pub := pool.NewPublisher(p) - defer pub.Close() + var ( + ctx = context.TODO() + poolName = testutils.FuncName() + proxyName, connectURL, _ = testutils.NextConnectURL() + disconnected, reconnected = Disconnect(t, proxyName, 5*time.Second) + ) - pub.Publish(ctx, exchangeName, "", pool.Publishing{ - Mandatory: true, - ContentType: "application/json", - Body: []byte(message), - }) + hp, err := pool.New( + ctx, + testutils.HealthyConnectURL, + 1, + 1, + pool.WithConfirms(true), + pool.WithLogger(logging.NewTestLogger(t)), + pool.WithName(poolName), + ) + if err != nil { + assert.NoError(t, err) + return + } + defer hp.Close() - // this should be canceled upon context cancelation from within the - // subscriber handler. - sub.Wait() + bp, err := pool.New( + ctx, + connectURL, + 1, + 1, + pool.WithConfirms(true), + pool.WithLogger(logging.NewTestLogger(t)), + pool.WithName(poolName+"-broken"), + ) + if err != nil { + assert.NoError(t, err) + return + } + defer bp.Close() - }(int64(id)) + var ( + nextExchangeName = testutils.ExchangeNameGenerator(poolName) + nextQueueName = testutils.QueueNameGenerator(poolName) + exchangeName = nextExchangeName() + queueName = nextQueueName() + nextConsumerName = testutils.ConsumerNameGenerator(queueName) + consumerName = nextConsumerName() + publisherMsgGen = testutils.MessageGenerator(queueName) + subscriberMsgGen = testutils.MessageGenerator(queueName) + ) + + ts, err := hp.GetTransientSession(ctx) + if err != nil { + assert.NoError(t, err) + return } + defer hp.ReturnSession(ts, nil) - wg.Wait() + cleanup := DeclareExchangeQueue(t, ctx, ts, exchangeName, queueName) + defer cleanup() + + cctx, cancel := context.WithCancel(ctx) + + sub := pool.NewSubscriber(hp, pool.SubscriberWithContext(cctx)) + defer sub.Close() + + sub.RegisterHandlerFunc(queueName, + func(ctx context.Context, msg pool.Delivery) error { + + // handler func + receivedMsg := string(msg.Body) + // assert equel to message that is to be sent + assert.Equal(t, subscriberMsgGen(), receivedMsg) + + // close subscriber from within handler + cancel() + return nil + }, + pool.ConsumeOptions{ + ConsumerTag: consumerName, + Exclusive: true, + }, + ) + pub := pool.NewPublisher(hp) + defer pub.Close() + + err = pub.Publish(ctx, exchangeName, "", pool.Publishing{ + Mandatory: true, + ContentType: "application/json", + Body: []byte(publisherMsgGen()), + }) + assert.NoError(t, err) + + disconnected() + err = sub.Start(ctx) + if err != nil { + assert.NoError(t, err) + return + } + + sub.Wait() + reconnected() } func TestBatchSubscriber(t *testing.T) { From 6b44962c9b5fbd8eed27c3c827e9eef8578f93a7 Mon Sep 17 00:00:00 2001 From: John Behm Date: Sat, 9 Mar 2024 01:06:07 +0100 Subject: [PATCH 36/76] update toxiproxy image to 2.7.0 --- docker-compose.yaml | 2 +- go.mod | 2 +- go.sum | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/docker-compose.yaml b/docker-compose.yaml index e4cde20..1664ce0 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -44,7 +44,7 @@ services: - rabbitnet toxiproxy: - image: ghcr.io/shopify/toxiproxy:2.5.0 + image: ghcr.io/shopify/toxiproxy:2.7.0 command: - -host=0.0.0.0 - -config=/toxiproxy.json diff --git a/go.mod b/go.mod index 318b70a..2121669 100644 --- a/go.mod +++ b/go.mod @@ -5,7 +5,7 @@ go 1.20 require ( github.com/Shopify/toxiproxy/v2 v2.7.0 github.com/rabbitmq/amqp091-go v1.9.0 - github.com/stretchr/testify v1.8.4 + github.com/stretchr/testify v1.9.0 go.uber.org/goleak v1.3.0 ) diff --git a/go.sum b/go.sum index 7448880..b437b0c 100644 --- a/go.sum +++ b/go.sum @@ -24,8 +24,8 @@ github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+ github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= -github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= -github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= +github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= +github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= go.uber.org/goleak v1.2.1/go.mod h1:qlT2yGI9QafXHhZZLxlSuNsMw3FFLxBr+tBRlmO1xH4= go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= From 3d076ee2a0dcff1213dcea9bc9ca4f5db3fd6cd7 Mon Sep 17 00:00:00 2001 From: John Behm Date: Tue, 12 Mar 2024 15:55:07 +0100 Subject: [PATCH 37/76] improve toxiproxy log messages --- pool/toxiproxy_client_test.go | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/pool/toxiproxy_client_test.go b/pool/toxiproxy_client_test.go index d569459..21f1831 100644 --- a/pool/toxiproxy_client_test.go +++ b/pool/toxiproxy_client_test.go @@ -117,7 +117,7 @@ func DisconnectWithStopped(t *testing.T, proxyName string, block, timeout, durat if duration > 0 { time.Sleep(duration) } - log.Debug("enabled rabbitmq connection") + log.Debugf("enabled rabbitmq connection proxy: %s", proxyName) err = proxy.Enable() if err != nil { require.NoError(t, err) @@ -155,7 +155,7 @@ func DisconnectWithStartedStopped(t *testing.T, proxyName string, block, startIn if wait := time.Until(start); wait > 0 { time.Sleep(wait) } - log.Debug("disabled rabbitmq connection") + log.Debugf("disabled rabbitmq connection proxy: %s", proxyName) err := proxy.Disable() if err != nil { require.NoError(t, err) @@ -165,7 +165,7 @@ func DisconnectWithStartedStopped(t *testing.T, proxyName string, block, startIn if duration > 0 { time.Sleep(duration) } - log.Debug("enabled rabbitmq connection") + log.Debugf("enabled rabbitmq connection proxy: %s", proxyName) err = proxy.Enable() if err != nil { require.NoError(t, err) @@ -218,7 +218,7 @@ func DisconnectWithStartStartedStopped(t *testing.T, proxyName string, duration defer wgStop.Done() log := logging.NewTestLogger(t) - log.Debug("disabled rabbitmq connection") + log.Debugf("disabled rabbitmq connection proxy: %s", proxyName) err := proxy.Disable() if err != nil { require.NoError(t, err) @@ -228,7 +228,7 @@ func DisconnectWithStartStartedStopped(t *testing.T, proxyName string, duration if duration > 0 { time.Sleep(duration) } - log.Debug("enabled rabbitmq connection") + log.Debugf("enabled rabbitmq connection proxy: %s", proxyName) err = proxy.Enable() if err != nil { require.NoError(t, err) From dcf12106caf6a960db8cb5adbb0cb8862ac9fc87 Mon Sep 17 00:00:00 2001 From: John Behm Date: Tue, 12 Mar 2024 15:56:09 +0100 Subject: [PATCH 38/76] export Flush method & flag connections using errors instead of bools --- pool/connection.go | 4 +++- pool/session.go | 10 ++++++---- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/pool/connection.go b/pool/connection.go index 589d8d6..e26a29d 100644 --- a/pool/connection.go +++ b/pool/connection.go @@ -157,10 +157,12 @@ func (ch *Connection) Close() (err error) { // Flag flags the connection as broken which must be recovered. // A flagged connection implies a closed connection. // Flagging of a connectioncan only be undone by Recover-ing the connection. -func (ch *Connection) Flag(flagged bool) { +func (ch *Connection) Flag(err error) { ch.mu.Lock() defer ch.mu.Unlock() + flagged := err != nil && recoverable(err) + if !ch.flagged && flagged { ch.flagged = flagged } diff --git a/pool/session.go b/pool/session.go index 9e011a5..e17eda0 100644 --- a/pool/session.go +++ b/pool/session.go @@ -151,10 +151,11 @@ func NewSession(conn *Connection, name string, options ...SessionOption) (*Sessi // Flag marks the session as flagged. // This is useful in case of a connection pool, where the session is returned to the pool // and should be recovered by the next user. -func (s *Session) Flag(flagged bool) { +func (s *Session) Flag(err error) { s.mu.Lock() defer s.mu.Unlock() + flagged := err != nil && recoverable(err) if !s.flagged && flagged { s.flagged = flagged } @@ -1507,15 +1508,16 @@ func (s *Session) debug(a ...any) { s.slog().Debug(a...) } -// flush should be called when you return the session back to the pool -// TODO: improve the name of this function and decide whether it should be exported or not -func (s *Session) flush() { +// Flush internal channels. +func (s *Session) Flush() { s.mu.Lock() defer s.mu.Unlock() // do not flush the errors channel // as it i sneeded for checking whether a session recovery is needed + flush(s.errors) flush(s.confirms) + flush(s.returned) } // flush is a helper function to flush a channel From 0399ae86ea6d33ff74874c10b6c60c637b8d859a Mon Sep 17 00:00:00 2001 From: John Behm Date: Tue, 12 Mar 2024 15:56:26 +0100 Subject: [PATCH 39/76] fix transient id counter --- pool/connection_pool.go | 28 ++++------------------------ 1 file changed, 4 insertions(+), 24 deletions(-) diff --git a/pool/connection_pool.go b/pool/connection_pool.go index 1b9dcf3..7954d08 100644 --- a/pool/connection_pool.go +++ b/pool/connection_pool.go @@ -104,12 +104,12 @@ func newConnectionPoolFromOption(connectUrl string, option connectionPoolOption) recoverCB: option.ConnectionRecoverCallback, } - cp.debug("initializing pool connections") + cp.debug("initializing pool connections...") defer func() { if err != nil { cp.error(err, "failed to initialize pool connections") } else { - cp.info("initialized") + cp.info("initialized pool connections") } }() @@ -183,32 +183,13 @@ func (cp *ConnectionPool) GetConnection(ctx context.Context) (*Connection, error func (cp *ConnectionPool) nextTransientID() int64 { cp.mu.Lock() defer cp.mu.Unlock() - id := cp.transientID cp.transientID++ - return id -} - -func (cp *ConnectionPool) incTransient() { - cp.mu.Lock() - cp.concurrentTransient++ - cp.mu.Unlock() -} - -func (cp *ConnectionPool) decTransient() { - cp.mu.Lock() - cp.concurrentTransient-- - cp.mu.Unlock() + return cp.transientID } // GetTransientConnection may return an error when the context was cancelled before the connection could be obtained. // Transient connections may be returned to the pool. The are closed properly upon returning. func (cp *ConnectionPool) GetTransientConnection(ctx context.Context) (_ *Connection, err error) { - defer func() { - if err == nil { - cp.incTransient() - } - }() - conn, err := cp.deriveConnection(ctx, cp.nextTransientID(), false) if err == nil { return conn, nil @@ -231,11 +212,10 @@ func (cp *ConnectionPool) GetTransientConnection(ctx context.Context) (_ *Connec func (cp *ConnectionPool) ReturnConnection(conn *Connection, err error) { // close transient connections if !conn.IsCached() { - cp.decTransient() // decrease transient cinnections _ = conn.Close() return } - conn.Flag(flaggable(err)) + conn.Flag(err) select { case cp.connections <- conn: From c01a4fb7cc2a57c2af869bf8f5c149f49fccac67 Mon Sep 17 00:00:00 2001 From: John Behm Date: Tue, 12 Mar 2024 15:57:07 +0100 Subject: [PATCH 40/76] remove flaggable helper function --- pool/errors.go | 23 +++++++++++++++++------ pool/flag.go | 7 ------- 2 files changed, 17 insertions(+), 13 deletions(-) delete mode 100644 pool/flag.go diff --git a/pool/errors.go b/pool/errors.go index 9196241..2e8700f 100644 --- a/pool/errors.go +++ b/pool/errors.go @@ -65,7 +65,24 @@ func recoverable(err error) bool { panic("checking nil error for recoverability") } + if errors.Is(err, context.Canceled) { + return false + } + + if errors.Is(err, context.DeadlineExceeded) { + return false + } + + if errors.Is(err, ErrClosed) { + return false + } + + if errors.Is(err, ErrFlowControl) { + return false + } + // invalid usage of the amqp protocol is not recoverable + // INFO: this should be checked last. ae := &amqp091.Error{} if errors.As(err, &ae) { switch ae.Code { @@ -84,12 +101,6 @@ func recoverable(err error) bool { } } - if errors.Is(err, context.Canceled) { - return false - } - - // TODO: errors.Is(err, context.DeadlineExceeded) also needed? - // every other unknown error is recoverable return true } diff --git a/pool/flag.go b/pool/flag.go deleted file mode 100644 index 5c31944..0000000 --- a/pool/flag.go +++ /dev/null @@ -1,7 +0,0 @@ -package pool - -import "errors" - -func flaggable(err error) bool { - return err != nil && !errors.Is(err, ErrFlowControl) && !errors.Is(err, ErrClosed) -} From 7bf300b8fe70154f005e781e52bbfca2fb1aaa14 Mon Sep 17 00:00:00 2001 From: John Behm Date: Tue, 12 Mar 2024 15:57:29 +0100 Subject: [PATCH 41/76] improve log message --- pool/publisher.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pool/publisher.go b/pool/publisher.go index 42d0edd..0320acb 100644 --- a/pool/publisher.go +++ b/pool/publisher.go @@ -97,7 +97,7 @@ func (p *Publisher) Publish(ctx context.Context, exchange string, routingKey str func (p *Publisher) publish(ctx context.Context, exchange string, routingKey string, msg Publishing) (err error) { defer func() { if err != nil { - p.warn(exchange, routingKey, err) + p.warn(exchange, routingKey, err, "failed to publish message") } else { p.info(exchange, routingKey, "published a message") } From 91f7ae3c552c07e5b8e8305de5bb71e5649cdf91 Mon Sep 17 00:00:00 2001 From: John Behm Date: Tue, 12 Mar 2024 15:57:50 +0100 Subject: [PATCH 42/76] use new Flag & exported Flush methods in session pool --- pool/session_pool.go | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/pool/session_pool.go b/pool/session_pool.go index 89d1708..570e12c 100644 --- a/pool/session_pool.go +++ b/pool/session_pool.go @@ -245,11 +245,10 @@ func (sp *SessionPool) ReturnSession(session *Session, err error) { return } - // try recovering until context closed or shutdown - session.Flag(flaggable(err)) + session.Flag(err) // flush confirms channel - session.flush() + session.Flush() // always put the session back into the pool // even if the session is still broken From 9f46f3a611c8c03d7f625e0f437aaf40d6e0a593 Mon Sep 17 00:00:00 2001 From: John Behm Date: Tue, 12 Mar 2024 17:04:59 +0100 Subject: [PATCH 43/76] add more logs to toxiproxy actions --- pool/utils_test.go | 28 ++++++++++++++++++++++++---- 1 file changed, 24 insertions(+), 4 deletions(-) diff --git a/pool/utils_test.go b/pool/utils_test.go index 6332b8a..ddc82f7 100644 --- a/pool/utils_test.go +++ b/pool/utils_test.go @@ -186,6 +186,9 @@ type Topologer interface { QueueUnbind(ctx context.Context, name string, routingKey string, exchange string, arg ...amqp091.Table) error } +// DeclareExchangeQueue declares an exchange and a queue and binds them together. +// It returns a cleanup function that can be used to delete the exchange and queue. +// The cleanup function is idempotent and can be called multiple times, but it will only delete the exchange and queue once. func DeclareExchangeQueue( t *testing.T, ctx context.Context, @@ -196,6 +199,9 @@ func DeclareExchangeQueue( cleanup = func() {} var err error + log := logging.NewTestLogger(t) + + log.Infof("declaring exchange %s", exchangeName) err = s.ExchangeDeclare(ctx, exchangeName, pool.ExchangeKindTopic) if err != nil { assert.NoError(t, err, "expected no error when declaring exchange") @@ -203,10 +209,12 @@ func DeclareExchangeQueue( } defer func() { if err != nil { + log.Infof("deleting exchange %s", exchangeName) assert.NoError(t, s.ExchangeDelete(ctx, exchangeName), "expected no error when deleting exchange") } }() + log.Infof("declaring queue %s", queueName) _, err = s.QueueDeclare(ctx, queueName) if err != nil { assert.NoError(t, err) @@ -214,6 +222,7 @@ func DeclareExchangeQueue( } defer func() { if err != nil { + log.Infof("deleting queue %s", queueName) _, e := s.QueueDelete(ctx, queueName) assert.NoError(t, e, "expected no error when deleting queue") // TODO: asserting the number of purged messages seems to be flaky, so we do not do that for now. @@ -221,6 +230,7 @@ func DeclareExchangeQueue( } }() + log.Infof("binding queue %s to exchange %s", queueName, exchangeName) err = s.QueueBind(ctx, queueName, "#", exchangeName) if err != nil { assert.NoError(t, err) @@ -228,16 +238,24 @@ func DeclareExchangeQueue( } defer func() { if err != nil { + log.Infof("unbinding queue %s from exchange %s", queueName, exchangeName) assert.NoError(t, s.QueueUnbind(ctx, queueName, "#", exchangeName, nil)) } }() + once := sync.Once{} return func() { - assert.NoError(t, s.QueueUnbind(ctx, queueName, "#", exchangeName, nil)) + once.Do(func() { + log.Infof("unbinding queue %s from exchange %s", queueName, exchangeName) + assert.NoError(t, s.QueueUnbind(ctx, queueName, "#", exchangeName, nil)) - _, e := s.QueueDelete(ctx, queueName) - assert.NoError(t, e) - assert.NoError(t, s.ExchangeDelete(ctx, exchangeName)) + log.Infof("deleting queue %s", queueName) + _, e := s.QueueDelete(ctx, queueName) + assert.NoError(t, e) + + log.Infof("deleting exchange %s", exchangeName) + assert.NoError(t, s.ExchangeDelete(ctx, exchangeName)) + }) } } @@ -270,7 +288,9 @@ func NewSession(t *testing.T, ctx context.Context, connectURL, connectionName st return nil, cleanup } return s, func() { + log.Infof("closing session %s", s.Name()) assert.NoError(t, s.Close(), "expected no error when closing session") + log.Infof("closing connection %s", c.Name()) assert.NoError(t, c.Close(), "expected no error when closing connection") } } From 4427e3c8f11e7860a707ccc44bb257e44f4b77d1 Mon Sep 17 00:00:00 2001 From: John Behm Date: Tue, 12 Mar 2024 17:05:45 +0100 Subject: [PATCH 44/76] remove unneded errors channel check when closing session --- pool/session.go | 6 ------ 1 file changed, 6 deletions(-) diff --git a/pool/session.go b/pool/session.go index e17eda0..b0f2e39 100644 --- a/pool/session.go +++ b/pool/session.go @@ -222,12 +222,6 @@ func (s *Session) close() (err error) { } } - if s.error() != nil { - // cannot close erred channel - s.debug("not closing erred amqp channel") - return nil - } - s.debug("closing amqp channel...") return s.channel.Close() } From 35edbd8b1bba22c1667e04b3cd469637c7546e2c Mon Sep 17 00:00:00 2001 From: John Behm Date: Tue, 12 Mar 2024 21:11:16 +0100 Subject: [PATCH 45/76] add low level tests to confirm deadlock assumptions --- pool/session_test.go | 370 ++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 367 insertions(+), 3 deletions(-) diff --git a/pool/session_test.go b/pool/session_test.go index e8e6369..9e0455e 100644 --- a/pool/session_test.go +++ b/pool/session_test.go @@ -9,6 +9,7 @@ import ( "github.com/jxsl13/amqpx/internal/testutils" "github.com/jxsl13/amqpx/logging" "github.com/jxsl13/amqpx/pool" + "github.com/rabbitmq/amqp091-go" amqp "github.com/rabbitmq/amqp091-go" "github.com/stretchr/testify/assert" ) @@ -728,16 +729,176 @@ func TestNewSessionConsumeWithDisconnect(t *testing.T) { wg.Wait() } -func TestChannelCloseOnOutOfMemoryRabbitMQ(t *testing.T) { +func TestChannelFullChainOnOutOfMemoryRabbitMQ(t *testing.T) { t.Parallel() - amqpConn, err := amqp.Dial(testutils.BrokenConnectURL) + var ( + ctx = context.TODO() + log = logging.NewTestLogger(t) + nextConnName = testutils.ConnectionNameGenerator() + nextSessionName = testutils.SessionNameGenerator(nextConnName()) + sessionName = nextSessionName() + nextExchangeName = testutils.ExchangeNameGenerator(sessionName) + nextQueueName = testutils.QueueNameGenerator(sessionName) + exchangeName = nextExchangeName() + queueName = nextQueueName() + bufferSize = 1 + ) + + log.Info("creating connection") + conn, err := amqp.Dial( + testutils.BrokenConnectURL, // out of memory rabbitmq + ) + if err != nil { + assert.NoError(t, err) + return + } + defer func() { + assert.NoError(t, conn.Close(), "expected no error when closing connection") + }() + + log.Info("registering flow control notification channel") + blocked := make(chan amqp.Blocking, bufferSize) + conn.NotifyBlocked(blocked) + + log.Info("creating channel") + c, err := conn.Channel() + if err != nil { + assert.NoError(t, err, "expected no error when creating channel") + return + } + defer func() { + log.Info("closing channel") + err = c.Close() + assert.NoError(t, err, "expected no error when closing channel") + }() + + log.Info("registering error notification channel") + errors := make(chan *amqp091.Error, bufferSize) + c.NotifyClose(errors) + + log.Info("registering confirms notification channel") + confirms := make(chan amqp.Confirmation, bufferSize) + c.NotifyPublish(confirms) + err = c.Confirm(false) + if err != nil { + assert.NoError(t, err, "expected no error when enabling confirms") + return + } + + log.Info("registering flow control notification channel") + flow := make(chan bool, bufferSize) + c.NotifyFlow(flow) + + log.Info("registering returned message notification channel") + returned := make(chan amqp091.Return, bufferSize) + c.NotifyReturn(returned) + + log.Info("declaring exchange") + err = c.ExchangeDeclare(exchangeName, "topic", true, false, false, false, nil) + if err != nil { + assert.NoError(t, err, "expected no error when declaring exchange") + return + } + defer func() { + log.Info("deleting exchange") + err = c.ExchangeDelete(exchangeName, false, false) + assert.NoError(t, err, "expected no error when deleting exchange") + }() + + log.Info("declaring queue") + _, err = c.QueueDeclare(queueName, true, false, false, false, nil) + if err != nil { + assert.NoError(t, err, "expected no error when declaring queue") + return + } + defer func() { + log.Info("deleting queue") + _, err = c.QueueDelete(queueName, false, false, false) + assert.NoError(t, err, "expected no error when deleting queue") + }() + + log.Info("binding queue") + err = c.QueueBind(queueName, "#", exchangeName, false, nil) + if err != nil { + assert.NoError(t, err, "expected no error when binding queue") + return + } + defer func() { + log.Info("unbinding queue") + err = c.QueueUnbind(queueName, "#", exchangeName, nil) + if err != nil { + assert.NoError(t, err, "expected no error when unbinding queue") + return + } + }() + + log.Info("publishing message") + msg := "hello world" + err = c.PublishWithContext(ctx, exchangeName, "", false, false, amqp.Publishing{ + ContentType: "text/plain", + Body: []byte(msg), + }) + if err != nil { + assert.NoError(t, err, "expected no error when publishing message") + return + } + + tctx, cancel := context.WithTimeout(ctx, 20*time.Second) + defer cancel() + + select { + case b, ok := <-blocked: + if !ok { + assert.Fail(t, "expected blocked channel to be open") + return + } + assert.True(t, b.Active, "expected blocked notification to be active") + assert.NotEmpty(t, b.Reason, "expected blocked notification to have a reason") + case f, ok := <-flow: + if !ok { + assert.Fail(t, "expected flow channel to be open") + return + } + assert.Fail(t, "expected no flow message when publishing message", "got=%v", f) + case confirm, ok := <-confirms: + if !ok { + assert.Fail(t, "expected confirms channel to be open") + return + } + assert.Fail(t, "expected no confirmation when publishing message", "got=%v", confirm) + case e, ok := <-errors: + if !ok { + assert.Fail(t, "expected errors channel to be open") + return + } + assert.Fail(t, "expected no error when publishing message", "got=%v", e) + case r, ok := <-returned: + if !ok { + assert.Fail(t, "expected returned channel to be open") + return + } + assert.Fail(t, "expected no returned message when publishing message", "got=%v", r) + case <-tctx.Done(): + assert.NoError(t, tctx.Err(), "expected no timeout when waiting for flow channel") + } +} + +func TestChannelCloseWithDisconnect(t *testing.T) { + t.Parallel() + + var ( + proxyName, connectURL, _ = testutils.NextConnectURL() + disconnected, reconnected = Disconnect(t, proxyName, 5*time.Second) + ) + + amqpConn, err := amqp.Dial(connectURL) if err != nil { assert.NoError(t, err) return } defer func() { - assert.NoError(t, amqpConn.Close(), "expected no error when closing connection") + assert.Error(t, amqpConn.Close(), "expected no error when closing connection") }() amqpChan, err := amqpConn.Channel() @@ -746,6 +907,209 @@ func TestChannelCloseOnOutOfMemoryRabbitMQ(t *testing.T) { return } + disconnected() + defer reconnected() err = amqpChan.Close() assert.NoError(t, err, "expected no error when closing channel") } + +func TestNewSingleSessionCloseWithDisconnect(t *testing.T) { + t.Parallel() // can be run in parallel because the connection to the rabbitmq is never broken + + var ( + ctx = context.TODO() + nextConnName = testutils.ConnectionNameGenerator() + connName = nextConnName() + nextSessionName = testutils.SessionNameGenerator(connName) + sessionName = nextSessionName() + proxyName, connectURL, _ = testutils.NextConnectURL() + disconnected, reconnected = Disconnect(t, proxyName, 5*time.Second) + ) + + reconnectCB, deferredAssert := AssertConnectionReconnectAttempts(t, 0) + defer deferredAssert() + c, err := pool.NewConnection( + ctx, + connectURL, + connName, + pool.ConnectionWithLogger(logging.NewTestLogger(t)), + pool.ConnectionWithRecoverCallback(reconnectCB), + ) + if err != nil { + assert.NoError(t, err) + return + } + defer func() { + //TODO: we do not want to assert anything here + assert.NoError(t, c.Close()) + }() + + s, err := pool.NewSession( + c, + sessionName, + pool.SessionWithConfirms(true), + pool.SessionWithRetryCallback( + func(operation, connName, sessionName string, retry int, err error) { + assert.NoErrorf(t, err, "operation=%s connName=%s sessionName=%s retry=%d", operation, connName, sessionName, retry) + }, + ), + ) + if err != nil { + assert.NoError(t, err) + return + } + + disconnected() + defer reconnected() + assert.NoError(t, s.Close()) +} + +func TestNewSingleSessionCloseWithOutOfMemoryRabbitMQ(t *testing.T) { + t.Parallel() // can be run in parallel because the connection to the rabbitmq is never broken + + var ( + ctx = context.TODO() + log = logging.NewTestLogger(t) + nextConnName = testutils.ConnectionNameGenerator() + connName = nextConnName() + nextSessionName = testutils.SessionNameGenerator(connName) + sessionName = nextSessionName() + nextExchangeName = testutils.ExchangeNameGenerator(sessionName) + nextQueueName = testutils.QueueNameGenerator(sessionName) + exchangeName = nextExchangeName() + queueName = nextQueueName() + ) + + reconnectCB, deferredAssert := AssertConnectionReconnectAttempts(t, 0) + defer deferredAssert() + c, err := pool.NewConnection( + ctx, + testutils.BrokenConnectURL, // out of memory rabbitmq + connName, + pool.ConnectionWithLogger(log), + pool.ConnectionWithRecoverCallback(reconnectCB), + ) + if err != nil { + assert.NoError(t, err) + return + } + defer func() { + //TODO: we do not want to assert anything here + assert.NoError(t, c.Close()) + }() + + s, err := pool.NewSession( + c, + sessionName, + pool.SessionWithConfirms(true), + pool.SessionWithRetryCallback( + func(operation, connName, sessionName string, retry int, err error) { + assert.NoErrorf(t, err, "operation=%s connName=%s sessionName=%s retry=%d", operation, connName, sessionName, retry) + }, + ), + ) + if err != nil { + assert.NoError(t, err) + return + } + + cleanup := DeclareExchangeQueue(t, ctx, s, exchangeName, queueName) + defer cleanup() + + log.Infof("publishing message to exchange %s", exchangeName) + tag, err := s.Publish(ctx, exchangeName, "", + pool.Publishing{ + ContentType: "text/plain", + Body: []byte("hello world"), + }, + ) + if err != nil { + assert.NoError(t, err) + return + } + + log.Infof("awaiting confirm for tag %d", tag) + err = s.AwaitConfirm(ctx, tag) + assert.Error(t, err, "expected a flow control error") + cleanup() + + log.Infof("closing session %s", s.Name()) + err = s.Close() + assert.NoError(t, err) +} + +func TestNewSingleSessionCloseWithHealthyRabbitMQ(t *testing.T) { + t.Parallel() // can be run in parallel because the connection to the rabbitmq is never broken + + var ( + ctx = context.TODO() + log = logging.NewTestLogger(t) + nextConnName = testutils.ConnectionNameGenerator() + connName = nextConnName() + nextSessionName = testutils.SessionNameGenerator(connName) + sessionName = nextSessionName() + nextExchangeName = testutils.ExchangeNameGenerator(sessionName) + nextQueueName = testutils.QueueNameGenerator(sessionName) + exchangeName = nextExchangeName() + queueName = nextQueueName() + ) + + reconnectCB, deferredAssert := AssertConnectionReconnectAttempts(t, 0) + defer deferredAssert() + c, err := pool.NewConnection( + ctx, + testutils.HealthyConnectURL, // healthy rabbitmq + connName, + pool.ConnectionWithLogger(log), + pool.ConnectionWithRecoverCallback(reconnectCB), + ) + if err != nil { + assert.NoError(t, err) + return + } + defer func() { + //TODO: we do not want to assert anything here + assert.NoError(t, c.Close()) + }() + + s, err := pool.NewSession( + c, + sessionName, + pool.SessionWithConfirms(true), + pool.SessionWithRetryCallback( + func(operation, connName, sessionName string, retry int, err error) { + assert.NoErrorf(t, err, "operation=%s connName=%s sessionName=%s retry=%d", operation, connName, sessionName, retry) + }, + ), + ) + if err != nil { + assert.NoError(t, err) + return + } + + cleanup := DeclareExchangeQueue(t, ctx, s, exchangeName, queueName) + defer cleanup() + + log.Infof("publishing message to exchange %s", exchangeName) + tag, err := s.Publish(ctx, exchangeName, "", + pool.Publishing{ + ContentType: "text/plain", + Body: []byte("hello world"), + }, + ) + if err != nil { + assert.NoError(t, err) + return + } + + log.Infof("awaiting confirm for tag %d", tag) + err = s.AwaitConfirm(ctx, tag) + log.Infof("await confirm failed(as expected): %v", err) + assert.NoError(t, err) + + cleanup() + + log.Infof("closing session %s", s.Name()) + err = s.Close() + assert.NoError(t, err) +} From b988ebcbdb1d837ec9f6377b8d6fc3d1698cb455 Mon Sep 17 00:00:00 2001 From: John Behm Date: Tue, 12 Mar 2024 21:13:24 +0100 Subject: [PATCH 46/76] make happy path have the lowest probability to be hit (golang select implementation details) --- pool/session_test.go | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/pool/session_test.go b/pool/session_test.go index 9e0455e..86fdb8a 100644 --- a/pool/session_test.go +++ b/pool/session_test.go @@ -848,13 +848,6 @@ func TestChannelFullChainOnOutOfMemoryRabbitMQ(t *testing.T) { defer cancel() select { - case b, ok := <-blocked: - if !ok { - assert.Fail(t, "expected blocked channel to be open") - return - } - assert.True(t, b.Active, "expected blocked notification to be active") - assert.NotEmpty(t, b.Reason, "expected blocked notification to have a reason") case f, ok := <-flow: if !ok { assert.Fail(t, "expected flow channel to be open") @@ -881,6 +874,13 @@ func TestChannelFullChainOnOutOfMemoryRabbitMQ(t *testing.T) { assert.Fail(t, "expected no returned message when publishing message", "got=%v", r) case <-tctx.Done(): assert.NoError(t, tctx.Err(), "expected no timeout when waiting for flow channel") + case b, ok := <-blocked: + if !ok { + assert.Fail(t, "expected blocked channel to be open") + return + } + assert.True(t, b.Active, "expected blocked notification to be active") + assert.NotEmpty(t, b.Reason, "expected blocked notification to have a reason") } } From eb1cf701127cb79aaaf3bd038405b51f3db73554 Mon Sep 17 00:00:00 2001 From: John Behm Date: Wed, 13 Mar 2024 18:21:17 +0100 Subject: [PATCH 47/76] update subscriber & publisher tests --- pool/connection.go | 2 +- pool/connection_pool_test.go | 2 + pool/errors.go | 10 +- pool/pool_test.go | 8 +- pool/publisher.go | 8 - pool/publisher_test.go | 3 + pool/session.go | 6 +- pool/session_test.go | 7 +- pool/subscriber_test.go | 412 +++++++++-------------------------- pool/utils_test.go | 252 +++++++++++++++++++++ 10 files changed, 382 insertions(+), 328 deletions(-) diff --git a/pool/connection.go b/pool/connection.go index e26a29d..0e5a39f 100644 --- a/pool/connection.go +++ b/pool/connection.go @@ -219,7 +219,7 @@ func (ch *Connection) connect(ctx context.Context) error { return nil } -func (ch *Connection) FlowControl() <-chan amqp.Blocking { +func (ch *Connection) BlockingFlowControl() <-chan amqp.Blocking { ch.mu.Lock() defer ch.mu.Unlock() diff --git a/pool/connection_pool_test.go b/pool/connection_pool_test.go index bdd674b..d7275c1 100644 --- a/pool/connection_pool_test.go +++ b/pool/connection_pool_test.go @@ -94,6 +94,8 @@ func TestNewConnectionPool(t *testing.T) { } func TestNewConnectionPoolWithDisconnect(t *testing.T) { + t.Parallel() + var ( ctx = context.TODO() poolName = testutils.FuncName() diff --git a/pool/errors.go b/pool/errors.go index 2e8700f..2dfa413 100644 --- a/pool/errors.go +++ b/pool/errors.go @@ -22,14 +22,14 @@ var ( // the queue was not found. ErrNotFound = errors.New("not found") - // ErrFlowControl is returned when the server is under flow control + // ErrBlockingFlowControl is returned when the server is under flow control // Your HTTP api may return 503 Service Unavailable or 429 Too Many Requests with a Retry-After header (https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Retry-After) - ErrFlowControl = errors.New("flow control") + ErrBlockingFlowControl = errors.New("blocking flow control") - // errFlowControlClosed is returned when the flow control channel is closed + // errBlockingFlowControlClosed is returned when the flow control channel is closed // Specifically interesting when awaiting publish confirms // TODO: make public api after a while - errFlowControlClosed = errors.New("flow control channel closed") + errBlockingFlowControlClosed = errors.New("blocking flow control channel closed") // ErrReturned is returned when a message is returned by the server when publishing ErrReturned = errors.New("returned") @@ -77,7 +77,7 @@ func recoverable(err error) bool { return false } - if errors.Is(err, ErrFlowControl) { + if errors.Is(err, ErrBlockingFlowControl) { return false } diff --git a/pool/pool_test.go b/pool/pool_test.go index 01b4708..e95a111 100644 --- a/pool/pool_test.go +++ b/pool/pool_test.go @@ -10,16 +10,10 @@ import ( "github.com/jxsl13/amqpx/logging" "github.com/jxsl13/amqpx/pool" "github.com/stretchr/testify/assert" - "go.uber.org/goleak" ) func TestMain(m *testing.M) { - goleak.VerifyTestMain( - m, - goleak.IgnoreTopFunction("internal/poll.runtime_pollWait"), - goleak.IgnoreTopFunction("github.com/rabbitmq/amqp091-go.(*Connection).heartbeater"), - goleak.IgnoreTopFunction("net/http.(*persistConn).writeLoop"), - ) + testutils.VerifyLeak(m) } func TestNewPool(t *testing.T) { diff --git a/pool/publisher.go b/pool/publisher.go index 0320acb..2961488 100644 --- a/pool/publisher.go +++ b/pool/publisher.go @@ -71,18 +71,10 @@ func (p *Publisher) Publish(ctx context.Context, exchange string, routingKey str switch { case err == nil: return nil - case errors.Is(err, context.Canceled): - return err - case errors.Is(err, context.DeadlineExceeded): - return err - case errors.Is(err, ErrClosed): - return err case errors.Is(err, ErrNack): return err case errors.Is(err, ErrDeliveryTagMismatch): return err - case errors.Is(err, ErrFlowControl): - return err default: if recoverable(err) { p.warn(exchange, routingKey, err, "publish failed due to recoverable error, retrying") diff --git a/pool/publisher_test.go b/pool/publisher_test.go index a3f72c6..b07f399 100644 --- a/pool/publisher_test.go +++ b/pool/publisher_test.go @@ -99,6 +99,8 @@ func TestSinglePublisher(t *testing.T) { wg.Wait() } +/* +// TODO: out of memory rabbitmq tests are disabled until https://github.com/rabbitmq/amqp091-go/issues/253 is resolved func TestPublishAwaitFlowControl(t *testing.T) { t.Parallel() @@ -163,3 +165,4 @@ func TestPublishAwaitFlowControl(t *testing.T) { assert.ErrorIs(t, err, pool.ErrFlowControl) // FIXME: this test gets stuck when the sessions in the session pool are closed.: } +*/ diff --git a/pool/session.go b/pool/session.go index b0f2e39..8272c07 100644 --- a/pool/session.go +++ b/pool/session.go @@ -390,15 +390,15 @@ func (s *Session) AwaitConfirm(ctx context.Context, expectedTag uint64) error { return fmt.Errorf("await confirm failed: %w", errReturnedClosed) } return fmt.Errorf("await confirm failed: %w: %s", ErrReturned, returned.ReplyText) - case blocking, ok := <-s.conn.FlowControl(): + case blocking, ok := <-s.conn.BlockingFlowControl(): if !ok { err := s.error() if err != nil { return fmt.Errorf("await confirm failed: blocking channel closed: %w", err) } - return fmt.Errorf("await confirm failed: %w", errFlowControlClosed) + return fmt.Errorf("await confirm failed: %w", errBlockingFlowControlClosed) } - return fmt.Errorf("await confirm failed: %w: %s", ErrFlowControl, blocking.Reason) + return fmt.Errorf("await confirm failed: %w: %s", ErrBlockingFlowControl, blocking.Reason) case <-ctx.Done(): err := ctx.Err() return fmt.Errorf("await confirm: failed context %w: %w", ErrClosed, err) diff --git a/pool/session_test.go b/pool/session_test.go index 86fdb8a..21d928d 100644 --- a/pool/session_test.go +++ b/pool/session_test.go @@ -9,7 +9,6 @@ import ( "github.com/jxsl13/amqpx/internal/testutils" "github.com/jxsl13/amqpx/logging" "github.com/jxsl13/amqpx/pool" - "github.com/rabbitmq/amqp091-go" amqp "github.com/rabbitmq/amqp091-go" "github.com/stretchr/testify/assert" ) @@ -729,6 +728,8 @@ func TestNewSessionConsumeWithDisconnect(t *testing.T) { wg.Wait() } +/* +// FIXME: ou of memory tests are disabled until https://github.com/rabbitmq/amqp091-go/issues/253 is resolved func TestChannelFullChainOnOutOfMemoryRabbitMQ(t *testing.T) { t.Parallel() @@ -883,6 +884,7 @@ func TestChannelFullChainOnOutOfMemoryRabbitMQ(t *testing.T) { assert.NotEmpty(t, b.Reason, "expected blocked notification to have a reason") } } +*/ func TestChannelCloseWithDisconnect(t *testing.T) { t.Parallel() @@ -964,6 +966,8 @@ func TestNewSingleSessionCloseWithDisconnect(t *testing.T) { assert.NoError(t, s.Close()) } +/* +// FIXME: ou of memory tests are disabled until https://github.com/rabbitmq/amqp091-go/issues/253 is resolved func TestNewSingleSessionCloseWithOutOfMemoryRabbitMQ(t *testing.T) { t.Parallel() // can be run in parallel because the connection to the rabbitmq is never broken @@ -1037,6 +1041,7 @@ func TestNewSingleSessionCloseWithOutOfMemoryRabbitMQ(t *testing.T) { err = s.Close() assert.NoError(t, err) } +*/ func TestNewSingleSessionCloseWithHealthyRabbitMQ(t *testing.T) { t.Parallel() // can be run in parallel because the connection to the rabbitmq is never broken diff --git a/pool/subscriber_test.go b/pool/subscriber_test.go index e2f4d99..7653513 100644 --- a/pool/subscriber_test.go +++ b/pool/subscriber_test.go @@ -17,23 +17,11 @@ func TestNewSingleSubscriber(t *testing.T) { t.Parallel() var ( - ctx = context.TODO() - poolName = testutils.FuncName() - ) - - hp, err := pool.New( - ctx, - testutils.HealthyConnectURL, - 1, - 2, - pool.WithConfirms(true), - pool.WithLogger(logging.NewTestLogger(t)), - pool.WithName(poolName), + ctx = context.TODO() + nextPoolName = testutils.PoolNameGenerator() + poolName = nextPoolName() + hp = NewPool(t, ctx, testutils.HealthyConnectURL, poolName, 1, 2) ) - if err != nil { - assert.NoError(t, err) - return - } defer hp.Close() var ( @@ -43,8 +31,10 @@ func TestNewSingleSubscriber(t *testing.T) { queueName = nextQueueName() nextConsumerName = testutils.ConsumerNameGenerator(queueName) consumerName = nextConsumerName() + publisherMsgGen = testutils.MessageGenerator(queueName) subscriberMsgGen = testutils.MessageGenerator(queueName) + numMsgs = 20 ) ts, err := hp.GetTransientSession(ctx) @@ -56,91 +46,27 @@ func TestNewSingleSubscriber(t *testing.T) { cleanup := DeclareExchangeQueue(t, ctx, ts, exchangeName, queueName) defer cleanup() + var wg sync.WaitGroup + defer wg.Wait() - cctx, cancel := context.WithCancel(ctx) - - sub := pool.NewSubscriber(hp, pool.SubscriberWithContext(cctx)) - defer sub.Close() - - sub.RegisterHandlerFunc(queueName, - func(ctx context.Context, msg pool.Delivery) error { - - // handler func - receivedMsg := string(msg.Body) - // assert equel to message that is to be sent - assert.Equal(t, subscriberMsgGen(), receivedMsg) - - // close subscriber from within handler - cancel() - return nil - }, - pool.ConsumeOptions{ - ConsumerTag: consumerName, - Exclusive: true, - }, - ) - err = sub.Start(ctx) - if err != nil { - assert.NoError(t, err) - return - } - time.Sleep(5 * time.Second) - - pub := pool.NewPublisher(hp) - defer pub.Close() - - err = pub.Publish(ctx, exchangeName, "", pool.Publishing{ - Mandatory: true, - ContentType: "application/json", - Body: []byte(publisherMsgGen()), - }) - assert.NoError(t, err) - - // this should be canceled upon context cancelation from within the - // subscriber handler. - sub.Wait() + SubscriberConsumeAsyncN(t, ctx, &wg, hp, queueName, consumerName, subscriberMsgGen, numMsgs, false) + PublisherPublishAsyncN(t, ctx, &wg, hp, exchangeName, publisherMsgGen, numMsgs) } -func TestNewSubscriberWithDisconnect(t *testing.T) { +func TestNewSingleSubscriberWithDisconnect(t *testing.T) { t.Parallel() var ( ctx = context.TODO() - poolName = testutils.FuncName() + nextPoolName = testutils.PoolNameGenerator() + poolName = nextPoolName() + hp = NewPool(t, ctx, testutils.HealthyConnectURL, poolName, 1, 2) proxyName, connectURL, _ = testutils.NextConnectURL() + pp = NewPool(t, ctx, connectURL, nextPoolName()+proxyName, 1, 1) disconnected, reconnected = Disconnect(t, proxyName, 5*time.Second) ) - - hp, err := pool.New( - ctx, - testutils.HealthyConnectURL, - 1, - 1, - pool.WithConfirms(true), - pool.WithLogger(logging.NewTestLogger(t)), - pool.WithName(poolName), - ) - if err != nil { - assert.NoError(t, err) - return - } defer hp.Close() - bp, err := pool.New( - ctx, - connectURL, - 1, - 1, - pool.WithConfirms(true), - pool.WithLogger(logging.NewTestLogger(t)), - pool.WithName(poolName+"-broken"), - ) - if err != nil { - assert.NoError(t, err) - return - } - defer bp.Close() - var ( nextExchangeName = testutils.ExchangeNameGenerator(poolName) nextQueueName = testutils.QueueNameGenerator(poolName) @@ -148,8 +74,10 @@ func TestNewSubscriberWithDisconnect(t *testing.T) { queueName = nextQueueName() nextConsumerName = testutils.ConsumerNameGenerator(queueName) consumerName = nextConsumerName() + publisherMsgGen = testutils.MessageGenerator(queueName) subscriberMsgGen = testutils.MessageGenerator(queueName) + numMsgs = 20 ) ts, err := hp.GetTransientSession(ctx) @@ -161,228 +89,128 @@ func TestNewSubscriberWithDisconnect(t *testing.T) { cleanup := DeclareExchangeQueue(t, ctx, ts, exchangeName, queueName) defer cleanup() - - cctx, cancel := context.WithCancel(ctx) - - sub := pool.NewSubscriber(hp, pool.SubscriberWithContext(cctx)) - defer sub.Close() - - sub.RegisterHandlerFunc(queueName, - func(ctx context.Context, msg pool.Delivery) error { - - // handler func - receivedMsg := string(msg.Body) - // assert equel to message that is to be sent - assert.Equal(t, subscriberMsgGen(), receivedMsg) - - // close subscriber from within handler - cancel() - return nil - }, - pool.ConsumeOptions{ - ConsumerTag: consumerName, - Exclusive: true, - }, - ) - pub := pool.NewPublisher(hp) - defer pub.Close() - - err = pub.Publish(ctx, exchangeName, "", pool.Publishing{ - Mandatory: true, - ContentType: "application/json", - Body: []byte(publisherMsgGen()), - }) - assert.NoError(t, err) + var wg sync.WaitGroup + defer wg.Wait() + PublisherPublishAsyncN(t, ctx, &wg, hp, exchangeName, publisherMsgGen, numMsgs) disconnected() - err = sub.Start(ctx) - if err != nil { - assert.NoError(t, err) - return - } - - sub.Wait() - reconnected() + defer reconnected() + SubscriberConsumeN(t, ctx, pp, queueName, consumerName, subscriberMsgGen, numMsgs, true) } -func TestBatchSubscriber(t *testing.T) { +func TestNewSingleBatchSubscriber(t *testing.T) { + t.Parallel() + var ( ctx = context.TODO() - sessions = 2 // publisher sessions + consumer sessions - numMessages = 50 - batchTimeout = 10 * time.Second // keep this at a higher number for slow machines + nextPoolName = testutils.PoolNameGenerator() + poolName = nextPoolName() + hp = NewPool(t, ctx, testutils.HealthyConnectURL, poolName, 1, 2) ) - p, err := pool.New( - ctx, - testutils.HealthyConnectURL, - 1, - sessions, - pool.WithConfirms(true), - pool.WithLogger(logging.NewTestLogger(t)), + defer hp.Close() + + var ( + nextExchangeName = testutils.ExchangeNameGenerator(poolName) + nextQueueName = testutils.QueueNameGenerator(poolName) + exchangeName = nextExchangeName() + queueName = nextQueueName() + nextConsumerName = testutils.ConsumerNameGenerator(queueName) + consumerName = nextConsumerName() + + publisherMsgGen = testutils.MessageGenerator(queueName) + subscriberMsgGen = testutils.MessageGenerator(queueName) + numMsgs = 20 + batchSize = numMsgs / 4 ) + + ts, err := hp.GetTransientSession(ctx) if err != nil { assert.NoError(t, err) return } - defer p.Close() + defer hp.ReturnSession(ts, nil) + cleanup := DeclareExchangeQueue(t, ctx, ts, exchangeName, queueName) + defer cleanup() var wg sync.WaitGroup + defer wg.Wait() - channels := sessions / 2 // one sessions for consumer and one for publisher - wg.Add(channels) - for id := 0; id < channels; id++ { - go func(id int64) { - defer wg.Done() - - ts, err := p.GetTransientSession(p.Context()) - if err != nil { - assert.NoError(t, err) - return - } - defer p.ReturnSession(ts, nil) - - queueName := fmt.Sprintf("TestBatchSubscriber-Queue-%d", id) - _, err = ts.QueueDeclare(ctx, queueName) - if err != nil { - assert.NoError(t, err) - return - } - defer func() { - i, err := ts.QueueDelete(ctx, queueName) - assert.NoError(t, err) - assert.Equal(t, 0, i) - }() - - exchangeName := fmt.Sprintf("TestBatchSubscriber-Exchange-%d", id) - err = ts.ExchangeDeclare(ctx, exchangeName, "topic") - if err != nil { - assert.NoError(t, err) - return - } - defer func() { - err := ts.ExchangeDelete(ctx, exchangeName) - assert.NoError(t, err) - }() - - err = ts.QueueBind(ctx, queueName, "#", exchangeName) - if err != nil { - assert.NoError(t, err) - return - } - defer func() { - err := ts.QueueUnbind(ctx, queueName, "#", exchangeName, nil) - assert.NoError(t, err) - }() - - // publish all messages - pub := pool.NewPublisher(p) - defer pub.Close() - - for i := 0; i < numMessages; i++ { - message := fmt.Sprintf("Message-%s-%d", queueName, i) - - pub.Publish(ctx, exchangeName, "", pool.Publishing{ - Mandatory: true, - ContentType: "application/json", - Body: []byte(message), - }) - } - - ctx, cancel := context.WithCancel(p.Context()) - - sub := pool.NewSubscriber(p, pool.SubscriberWithContext(ctx)) - defer sub.Close() - - batchSize := numMessages / 2 - - batchCount := 0 - messageCount := 0 - sub.RegisterBatchHandlerFunc(queueName, - func(ctx context.Context, msgs []pool.Delivery) error { - log := logging.NewTestLogger(t) - assert.Equal(t, batchSize, len(msgs)) - - for idx, msg := range msgs { - assert.Truef(t, len(msg.Body) > 0, "msg body is empty: message index: %d", idx) - log.Debugf("batch: %d message: %d: body: %q", batchCount, idx, string(msg.Body)) - } - - messageCount += len(msgs) - batchCount += 1 - - if messageCount == numMessages { - // close subscriber from within handler - cancel() - } - return nil - }, - pool.WithMaxBatchSize(batchSize), - pool.WithBatchFlushTimeout(batchTimeout), - pool.WithBatchConsumeOptions(pool.ConsumeOptions{ - ConsumerTag: fmt.Sprintf("Consumer-%s", queueName), - Exclusive: true, - }), - ) - sub.Start(ctx) - - // this should be canceled upon context cancelation from within the - // subscriber handler. - sub.Wait() - - assert.Equalf(t, numMessages, messageCount, "expected messages counter to have the same number as publishes messages") - assert.Equalf(t, 2, batchCount, "required to have two batches received") - - }(int64(id)) - } - - wg.Wait() + SubscriberBatchConsumeAsyncN( + t, + ctx, + &wg, + hp, + queueName, + consumerName, + subscriberMsgGen, + numMsgs, + batchSize, + numMsgs*1024, // should not be hit + false, + ) + PublisherPublishAsyncN(t, ctx, &wg, hp, exchangeName, publisherMsgGen, numMsgs) } func TestBatchSubscriberMaxBytes(t *testing.T) { + t.Parallel() + + var wg sync.WaitGroup for i := 1; i <= 2048; i = i*2 + 1 { - testBatchSubscriberMaxBytes(t, i) + wg.Add(1) + go testBatchSubscriberMaxBytes(t, i, &wg) } + + wg.Wait() } -func testBatchSubscriberMaxBytes(t *testing.T, maxBatchBytes int) { +func testBatchSubscriberMaxBytes(t *testing.T, maxBatchBytes int, w *sync.WaitGroup) { t.Helper() + defer w.Done() var ( ctx = context.TODO() - sessions = 2 // publisher sessions + consumer sessions - numMessages = 50 + nextPoolName = testutils.PoolNameGenerator( + testutils.WithUp(3), + testutils.WithSuffix(fmt.Sprintf("-max-batch-bytes-%d", maxBatchBytes)), + ) + poolName = nextPoolName() + + nextExchangeName = testutils.ExchangeNameGenerator(poolName) + nextQueueName = testutils.QueueNameGenerator(poolName) + + numSessions = 2 + hp = NewPool(t, ctx, testutils.HealthyConnectURL, poolName, 1, numSessions) // // publisher sessions + consumer sessions + + numMsgs = 20 batchTimeout = 5 * time.Second // keep this at a higher number for slow machines ) - p, err := pool.New( - ctx, - testutils.HealthyConnectURL, - 1, - sessions, - pool.WithConfirms(true), - pool.WithLogger(logging.NewTestLogger(t)), - ) - if err != nil { - assert.NoError(t, err) - return - } - defer p.Close() + defer hp.Close() var wg sync.WaitGroup - channels := sessions / 2 // one sessions for consumer and one for publisher + channels := numSessions / 2 // one sessions for consumer and one for publisher wg.Add(channels) for id := 0; id < channels; id++ { go func(id int64) { defer wg.Done() - ts, err := p.GetTransientSession(p.Context()) + var ( + log = logging.NewTestLogger(t) + exchangeName = nextExchangeName() + queueName = nextQueueName() + nextConsumerName = testutils.ConsumerNameGenerator(queueName) + ) + + ts, err := hp.GetTransientSession(ctx) if err != nil { assert.NoError(t, err) return } - defer p.ReturnSession(ts, nil) + defer hp.ReturnSession(ts, nil) + + cleanup := DeclareExchangeQueue(t, ctx, ts, exchangeName, queueName) + defer cleanup() - queueName := fmt.Sprintf("TestBatchSubscriberMaxBytes-Queue-%d", id) _, err = ts.QueueDeclare(ctx, queueName) if err != nil { assert.NoError(t, err) @@ -394,35 +222,12 @@ func testBatchSubscriberMaxBytes(t *testing.T, maxBatchBytes int) { assert.Equal(t, 0, i) }() - exchangeName := fmt.Sprintf("TestBatchSubscriberMaxBytes-Exchange-%d", id) - err = ts.ExchangeDeclare(ctx, exchangeName, "topic") - if err != nil { - assert.NoError(t, err) - return - } - defer func() { - err := ts.ExchangeDelete(ctx, exchangeName) - assert.NoError(t, err) - }() - - err = ts.QueueBind(ctx, queueName, "#", exchangeName) - if err != nil { - assert.NoError(t, err) - return - } - defer func() { - err := ts.QueueUnbind(ctx, queueName, "#", exchangeName, nil) - assert.NoError(t, err) - }() - // publish all messages - pub := pool.NewPublisher(p) + pub := pool.NewPublisher(hp) defer pub.Close() - log := logging.NewTestLogger(t) - maxMsgLen := 0 - for i := 0; i < numMessages; i++ { + for i := 0; i < numMsgs; i++ { message := fmt.Sprintf("Message-%s-%06d", queueName, i) // max 6 digits mlen := len(message) if mlen > maxMsgLen { @@ -431,7 +236,7 @@ func testBatchSubscriberMaxBytes(t *testing.T, maxBatchBytes int) { pub.Publish(ctx, exchangeName, "", pool.Publishing{ Mandatory: true, - ContentType: "application/json", + ContentType: "text/plain", Body: []byte(message), }) } @@ -442,15 +247,16 @@ func testBatchSubscriberMaxBytes(t *testing.T, maxBatchBytes int) { expectedMessagesPerBatch += 1 } log.Debugf("expected messages per batch: %d", expectedMessagesPerBatch) - expectedBatches := numMessages / expectedMessagesPerBatch - if numMessages%expectedMessagesPerBatch > 0 { + expectedBatches := numMsgs / expectedMessagesPerBatch + if numMsgs%expectedMessagesPerBatch > 0 { expectedBatches += 1 } log.Debugf("expected batches: %d", expectedBatches) - cctx, cancel := context.WithCancel(p.Context()) + cctx, cancel := context.WithCancel(ctx) + defer cancel() - sub := pool.NewSubscriber(p, pool.SubscriberWithContext(cctx)) + sub := pool.NewSubscriber(hp, pool.SubscriberWithContext(cctx)) defer sub.Close() batchCount := 0 @@ -472,7 +278,7 @@ func testBatchSubscriberMaxBytes(t *testing.T, maxBatchBytes int) { } assert.Equal(t, expectedMessages, len(msgs)) - if messageCount == numMessages { + if messageCount == numMsgs { // close subscriber from within handler cancel() } @@ -482,7 +288,7 @@ func testBatchSubscriberMaxBytes(t *testing.T, maxBatchBytes int) { pool.WithMaxBatchSize(0), // disable this check pool.WithBatchFlushTimeout(batchTimeout), pool.WithBatchConsumeOptions(pool.ConsumeOptions{ - ConsumerTag: fmt.Sprintf("Consumer-%s", queueName), + ConsumerTag: nextConsumerName(), Exclusive: true, }), ) @@ -492,7 +298,7 @@ func testBatchSubscriberMaxBytes(t *testing.T, maxBatchBytes int) { // subscriber handler. sub.Wait() - assert.Equalf(t, numMessages, messageCount, "expected messages counter to have the same number as publishes messages") + assert.Equalf(t, numMsgs, messageCount, "expected messages counter to have the same number as publishes messages") assert.Equalf(t, expectedBatches, batchCount, "required to have %d batches received", expectedBatches) }(int64(id)) diff --git a/pool/utils_test.go b/pool/utils_test.go index ddc82f7..c103217 100644 --- a/pool/utils_test.go +++ b/pool/utils_test.go @@ -5,6 +5,7 @@ import ( "fmt" "sync" "testing" + "time" "github.com/jxsl13/amqpx/internal/testutils" "github.com/jxsl13/amqpx/logging" @@ -315,3 +316,254 @@ func AssertConnectionReconnectAttempts(t *testing.T, n int) (callback pool.Conne assert.Equal(t, n, i, "expected %d reconnect attempts, got %d", n, i) } } + +func PublisherPublishN(t *testing.T, ctx context.Context, p *pool.Pool, exchangeName string, publishMessageGenerator func() string, n int) { + pub := pool.NewPublisher(p) + defer pub.Close() + + for i := 0; i < n; i++ { + message := publishMessageGenerator() + err := pub.Publish(ctx, exchangeName, "", pool.Publishing{ + ContentType: "text/plain", + Body: []byte(message), + }) + assert.NoError(t, err) + } +} + +func PublisherPublishAsyncN( + t *testing.T, + ctx context.Context, + wg *sync.WaitGroup, + p *pool.Pool, + exchangeName string, + publishMessageGenerator func() string, + n int, +) { + wg.Add(1) + go func() { + defer wg.Done() + PublisherPublishN(t, ctx, p, exchangeName, publishMessageGenerator, n) + }() +} + +func SubscriberConsumeN( + t *testing.T, + ctx context.Context, + p *pool.Pool, + queueName string, + consumerName string, + messageGenerator func() string, + n int, + allowDuplicates bool, +) { + var log = logging.NewTestLogger(t) + + processingTime := 30 * time.Millisecond + cctx, ccancel := context.WithTimeout(ctx, 30*time.Second+time.Duration((2+1)*n)*processingTime) + defer ccancel() + sub := pool.NewSubscriber( + p, + pool.SubscriberWithContext(cctx), + pool.SubscriberWithLogger(log), + ) + defer sub.Close() + + msgsReceived := 0 + defer func() { + assert.Equal(t, n, msgsReceived, "expected to consume %d messages, got %d", n, msgsReceived) + }() + + var previouslyReceivedMsg string + sub.RegisterHandlerFunc(queueName, func(ctx context.Context, d pool.Delivery) error { + var receivedMsg = string(d.Body) + + select { + case <-time.After(testutils.Jitter(0, processingTime)): + case <-ctx.Done(): + } + + if allowDuplicates && receivedMsg == previouslyReceivedMsg { + // TODO: it is possible that messages are duplicated, but this is not a problem + // due to network issues. We should not fail the test in this case. + log.Warnf("received duplicate message: %s", receivedMsg) + return nil + } + + var expectedMsg = messageGenerator() + assert.Equalf( + t, + expectedMsg, + receivedMsg, + "expected message %s, got %s, previously received message: %s", + expectedMsg, + receivedMsg, + previouslyReceivedMsg, + ) + + log.Infof("consumed message: %s", receivedMsg) + msgsReceived++ + if msgsReceived == n { + logging.NewTestLogger(t).Infof("consumed %d messages, closing consumer", n) + ccancel() + } + // update last received message + previouslyReceivedMsg = receivedMsg + return nil + }, pool.ConsumeOptions{ + ConsumerTag: consumerName, + }) + + err := sub.Start(cctx) + if err != nil { + assert.NoError(t, err) + ccancel() + } + sub.Wait() +} + +func SubscriberConsumeAsyncN( + t *testing.T, + ctx context.Context, + wg *sync.WaitGroup, + p *pool.Pool, + queueName string, + consumerName string, + messageGenerator func() string, + n int, + allowDuplicates bool, +) { + wg.Add(1) + go func() { + defer wg.Done() + SubscriberConsumeN(t, ctx, p, queueName, consumerName, messageGenerator, n, allowDuplicates) + }() +} + +func SubscriberBatchConsumeN( + t *testing.T, + ctx context.Context, + p *pool.Pool, + queueName string, + consumerName string, + messageGenerator func() string, + n int, + batchSize int, + maxBytes int, + allowDuplicates bool, +) { + var log = logging.NewTestLogger(t) + processingTime := 30 * time.Millisecond + cctx, ccancel := context.WithTimeout(ctx, 30*time.Second+time.Duration((2+1)*n)*processingTime) + defer ccancel() + sub := pool.NewSubscriber( + p, + pool.SubscriberWithContext(cctx), + pool.SubscriberWithLogger(log), + ) + defer sub.Close() + + msgsReceived := 0 + defer func() { + assert.Equal(t, n, msgsReceived, "expected to consume %d messages, got %d", n, msgsReceived) + }() + + var previouslyReceivedMsg string + sub.RegisterBatchHandlerFunc(queueName, func(ctx context.Context, ds []pool.Delivery) error { + for _, d := range ds { + var receivedMsg = string(d.Body) + + select { + case <-time.After(testutils.Jitter(0, processingTime)): + case <-ctx.Done(): + } + + if allowDuplicates && receivedMsg == previouslyReceivedMsg { + // TODO: it is possible that messages are duplicated, but this is not a problem + // due to network issues. We should not fail the test in this case. + log.Warnf("received duplicate message: %s", receivedMsg) + continue + } + + var expectedMsg = messageGenerator() + assert.Equalf( + t, + expectedMsg, + receivedMsg, + "expected message %s, got %s, previously received message: %s", + expectedMsg, + receivedMsg, + previouslyReceivedMsg, + ) + + log.Infof("consumed message: %s", receivedMsg) + msgsReceived++ + if msgsReceived == n { + logging.NewTestLogger(t).Infof("consumed %d messages, closing consumer", n) + ccancel() + } + // update last received message + previouslyReceivedMsg = receivedMsg + } + return nil + }, pool.WithBatchConsumeOptions(pool.ConsumeOptions{ + ConsumerTag: consumerName, + }), pool.WithMaxBatchSize(batchSize), + pool.WithMaxBatchBytes(maxBytes), + ) + + err := sub.Start(cctx) + if err != nil { + assert.NoError(t, err) + ccancel() + } + sub.Wait() +} + +func SubscriberBatchConsumeAsyncN( + t *testing.T, + ctx context.Context, + wg *sync.WaitGroup, + p *pool.Pool, + queueName string, + consumerName string, + messageGenerator func() string, + n int, + batchSize int, + maxBytes int, + allowDuplicates bool, +) { + wg.Add(1) + go func() { + defer wg.Done() + SubscriberBatchConsumeN( + t, + ctx, + p, + queueName, + consumerName, + messageGenerator, + n, + batchSize, + maxBytes, + allowDuplicates, + ) + }() +} + +func NewPool(t *testing.T, ctx context.Context, connectURL, poolName string, numConns, numSessions int) *pool.Pool { + p, err := pool.New( + ctx, + connectURL, + numConns, + numSessions, + pool.WithLogger(logging.NewTestLogger(t)), + pool.WithConfirms(true), + pool.WithName(poolName), + ) + if err != nil { + assert.NoError(t, err) + return nil + } + return p +} From 5370c486fe19dd61382c98bec77d1a8fc407786d Mon Sep 17 00:00:00 2001 From: John Behm Date: Wed, 13 Mar 2024 18:21:34 +0100 Subject: [PATCH 48/76] update testutils --- internal/testutils/generator.go | 44 ++++++++++++++++++++++++++++++--- internal/testutils/leak.go | 16 ++++++++++++ internal/testutils/testutils.go | 17 +++++++++++++ 3 files changed, 74 insertions(+), 3 deletions(-) create mode 100644 internal/testutils/leak.go diff --git a/internal/testutils/generator.go b/internal/testutils/generator.go index 9ca95f0..f74980d 100644 --- a/internal/testutils/generator.go +++ b/internal/testutils/generator.go @@ -25,6 +25,13 @@ func (o *generatorOptions) ToSuffix() string { type GeneratorOption func(*generatorOptions) +// WithUp sets the number of stack frames to skip when generating the name +func WithUp(up int) GeneratorOption { + return func(o *generatorOptions) { + o.up = up + } +} + func WithRandomSuffix(addRandomSuffix bool) GeneratorOption { return func(o *generatorOptions) { o.randomSuffix = addRandomSuffix @@ -101,8 +108,10 @@ func QueueNameGenerator(sessionName string, options ...GeneratorOption) (nextQue opt(&opts) } - var mu sync.Mutex - var counter int64 + var ( + mu sync.Mutex + counter int64 + ) return func() string { mu.Lock() cnt := counter @@ -124,14 +133,43 @@ func SessionNameGenerator(connectionName string, options ...GeneratorOption) (ne opt(&opts) } + var ( + mu sync.Mutex + counter int64 + ) + return func() string { + mu.Lock() + cnt := counter + counter++ + mu.Unlock() + return fmt.Sprintf("%s-%ssession-%d%s", connectionName, opts.prefix, cnt, opts.ToSuffix()) + } +} + +func PoolNameGenerator(options ...GeneratorOption) (nextConnName func() string) { + opts := generatorOptions{ + prefix: "", + randomSuffix: false, + + up: 2, + } + + for _, opt := range options { + opt(&opts) + } + var mu sync.Mutex + funcName := FuncName(opts.up) + parts := strings.Split(funcName, ".") + funcName = parts[len(parts)-1] + var counter int64 return func() string { mu.Lock() cnt := counter counter++ mu.Unlock() - return fmt.Sprintf("%s-%ssession-%d%s", connectionName, opts.prefix, cnt, opts.ToSuffix()) + return fmt.Sprintf("%s%s-%d%s", opts.prefix, funcName, cnt, opts.ToSuffix()) } } diff --git a/internal/testutils/leak.go b/internal/testutils/leak.go new file mode 100644 index 0000000..f55f7b7 --- /dev/null +++ b/internal/testutils/leak.go @@ -0,0 +1,16 @@ +package testutils + +import ( + "testing" + + "go.uber.org/goleak" +) + +func VerifyLeak(m *testing.M) { + goleak.VerifyTestMain( + m, + goleak.IgnoreTopFunction("internal/poll.runtime_pollWait"), + goleak.IgnoreTopFunction("github.com/rabbitmq/amqp091-go.(*Connection).heartbeater"), + goleak.IgnoreTopFunction("net/http.(*persistConn).writeLoop"), + ) +} diff --git a/internal/testutils/testutils.go b/internal/testutils/testutils.go index bc80077..7eb4f3c 100644 --- a/internal/testutils/testutils.go +++ b/internal/testutils/testutils.go @@ -38,3 +38,20 @@ func FuncName(up ...int) string { } return f.Name() } + +func CallerFuncName(up ...int) string { + offset := 2 + if len(up) > 0 && up[0] > 0 { + offset = up[0] + } + pc, _, _, ok := runtime.Caller(offset) + if !ok { + panic("failed to get caller") + } + + f := runtime.FuncForPC(pc) + if f == nil { + panic("failed to get function name") + } + return f.Name() +} From 96dd887034d219c99bf9f18de7ae0f1997c46c44 Mon Sep 17 00:00:00 2001 From: John Behm Date: Wed, 13 Mar 2024 18:21:48 +0100 Subject: [PATCH 49/76] update amqpx tests --- amqpx.go | 4 + amqpx_test.go | 569 +++++++++++++++++++++++++++--------------------- helpers_test.go | 134 ++++++------ 3 files changed, 394 insertions(+), 313 deletions(-) diff --git a/amqpx.go b/amqpx.go index 037ed15..9662677 100644 --- a/amqpx.go +++ b/amqpx.go @@ -11,10 +11,12 @@ import ( "github.com/jxsl13/amqpx/pool" ) +/* var ( // global variable, as we usually only need a single connection amqpx = New() ) +*/ type ( TopologyFunc func(context.Context, *pool.Topologer) error @@ -318,6 +320,7 @@ func (a *AMQPX) Get(ctx context.Context, queue string, autoAck bool) (msg pool.D return a.pub.Get(ctx, queue, autoAck) } +/* // RegisterTopology registers a topology creating function that is called upon // Start. The creation of topologie sis the first step before any publisher or subscriber is started. func RegisterTopologyCreator(topology TopologyFunc) { @@ -376,3 +379,4 @@ func Get(ctx context.Context, queue string, autoAck bool) (msg pool.Delivery, ok func Reset() error { return amqpx.Reset() } +*/ diff --git a/amqpx_test.go b/amqpx_test.go index 1b6d95b..8fe3b96 100644 --- a/amqpx_test.go +++ b/amqpx_test.go @@ -10,44 +10,44 @@ import ( "time" "github.com/jxsl13/amqpx" + "github.com/jxsl13/amqpx/internal/testutils" "github.com/jxsl13/amqpx/logging" "github.com/jxsl13/amqpx/pool" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" - "go.uber.org/goleak" -) - -var ( - connectURL = amqpx.NewURL("localhost", 5672, "admin", "password") ) // WARNING: Do not assert consumer counts, as those values are too flaky and break tests all over the place func TestMain(m *testing.M) { - goleak.VerifyTestMain( - m, - goleak.IgnoreTopFunction("internal/poll.runtime_pollWait"), - goleak.IgnoreTopFunction("github.com/rabbitmq/amqp091-go.(*Connection).heartbeater"), - goleak.IgnoreTopFunction("net/http.(*persistConn).writeLoop"), - ) + testutils.VerifyLeak(m) } func TestExchangeDeclarePassive(t *testing.T) { - ctx := context.TODO() - defer amqpx.Reset() + t.Parallel() + + var ( + amqp = amqpx.New() + ctx = context.TODO() + funcName = testutils.FuncName() + nextExchangeName = testutils.ExchangeNameGenerator(funcName) + exchangeName = nextExchangeName() + ) + defer func() { + assert.NoError(t, amqp.Close()) + }() - eName := "exchange-01" var err error - amqpx.RegisterTopologyCreator(func(ctx context.Context, t *pool.Topologer) error { - return createExchange(ctx, eName, t) + amqp.RegisterTopologyCreator(func(ctx context.Context, t *pool.Topologer) error { + return createExchange(ctx, exchangeName, t) }) - amqpx.RegisterTopologyDeleter(func(ctx context.Context, t *pool.Topologer) error { - return deleteExchange(ctx, eName, t) + amqp.RegisterTopologyDeleter(func(ctx context.Context, t *pool.Topologer) error { + return deleteExchange(ctx, exchangeName, t) }) - err = amqpx.Start( + err = amqp.Start( ctx, - connectURL, + testutils.HealthyConnectURL, amqpx.WithLogger(logging.NewTestLogger(t)), amqpx.WithPublisherConnections(1), amqpx.WithPublisherSessions(2), @@ -56,22 +56,31 @@ func TestExchangeDeclarePassive(t *testing.T) { } func TestQueueDeclarePassive(t *testing.T) { - ctx := context.TODO() - defer amqpx.Reset() + t.Parallel() + + var ( + amqp = amqpx.New() + ctx = context.TODO() + funcName = testutils.FuncName() + nextQueueName = testutils.QueueNameGenerator(funcName) + queueName = nextQueueName() + ) + defer func() { + assert.NoError(t, amqp.Close()) + }() - qName := "queue-01" var err error - amqpx.RegisterTopologyCreator(func(ctx context.Context, t *pool.Topologer) error { - return createQueue(ctx, qName, t) + amqp.RegisterTopologyCreator(func(ctx context.Context, t *pool.Topologer) error { + return createQueue(ctx, queueName, t) }) - amqpx.RegisterTopologyDeleter(func(ctx context.Context, t *pool.Topologer) error { - return deleteQueue(ctx, qName, t) + amqp.RegisterTopologyDeleter(func(ctx context.Context, t *pool.Topologer) error { + return deleteQueue(ctx, queueName, t) }) - err = amqpx.Start( + err = amqp.Start( ctx, - connectURL, + testutils.HealthyConnectURL, amqpx.WithLogger(logging.NewTestLogger(t)), amqpx.WithPublisherConnections(1), amqpx.WithPublisherSessions(2), @@ -80,15 +89,23 @@ func TestQueueDeclarePassive(t *testing.T) { } func TestAMQPXPub(t *testing.T) { - ctx := context.TODO() - defer amqpx.Reset() + t.Parallel() - amqpx.RegisterTopologyCreator(createTopology) - amqpx.RegisterTopologyDeleter(deleteTopology) + var ( + amqp = amqpx.New() + ctx = context.TODO() + funcName = testutils.FuncName() + ) + defer func() { + assert.NoError(t, amqp.Close()) + }() + + amqp.RegisterTopologyCreator(createTopology(funcName)) + amqp.RegisterTopologyDeleter(deleteTopology(funcName)) - err := amqpx.Start( + err := amqp.Start( ctx, - connectURL, + testutils.HealthyConnectURL, amqpx.WithLogger(logging.NewNoOpLogger()), amqpx.WithPublisherConnections(1), amqpx.WithPublisherSessions(2), @@ -99,15 +116,15 @@ func TestAMQPXPub(t *testing.T) { } defer func() { // will be canceled when the event has reache dthe third handler - err = amqpx.Close() + err = amqp.Close() assert.NoError(t, err) }() - event := "TestAMQPXPub - event content" + event := funcName + " - event content" // publish event to first queue - err = amqpx.Publish(ctx, "exchange-01", "event-01", pool.Publishing{ - ContentType: "application/json", + err = amqp.Publish(ctx, funcName+"exchange-01", "event-01", pool.Publishing{ + ContentType: "text/plain", Body: []byte(event), }) if err != nil { @@ -120,7 +137,7 @@ func TestAMQPXPub(t *testing.T) { ok bool ) for i := 0; i < 20; i++ { - msg, ok, err = amqpx.Get(ctx, "queue-01", false) + msg, ok, err = amqp.Get(ctx, funcName+"queue-01", false) if err != nil { assert.NoError(t, err) return @@ -141,27 +158,32 @@ func TestAMQPXPub(t *testing.T) { } func TestAMQPXSubAndPub(t *testing.T) { - ctx := context.TODO() - log := logging.NewTestLogger(t) - defer amqpx.Reset() - - amqpx.RegisterTopologyCreator(createTopology) - amqpx.RegisterTopologyDeleter(deleteTopology) - - ctx, cancel := signal.NotifyContext(ctx, syscall.SIGINT, syscall.SIGINT) + var ( + amqp = amqpx.New() + log = logging.NewTestLogger(t) + cctx, cancel = signal.NotifyContext(context.TODO(), syscall.SIGINT, syscall.SIGINT) + funcName = testutils.FuncName() + ) defer cancel() + defer func() { + log.Info("closing amqp") + assert.NoError(t, amqp.Close()) + }() - eventContent := "TestAMQPXSubAndPub - event content" + amqp.RegisterTopologyCreator(createTopology(funcName)) + amqp.RegisterTopologyDeleter(deleteTopology(funcName)) - amqpx.RegisterHandler("queue-01", func(ctx context.Context, msg pool.Delivery) error { + eventContent := funcName + " - event content" + + amqp.RegisterHandler(funcName+"queue-01", func(ctx context.Context, msg pool.Delivery) error { log.Info("subscriber of queue-01") cancel() return nil }) - err := amqpx.Start( - ctx, - connectURL, + err := amqp.Start( + cctx, + testutils.HealthyConnectURL, amqpx.WithLogger(logging.NewNoOpLogger()), amqpx.WithPublisherConnections(1), amqpx.WithPublisherSessions(5), @@ -173,8 +195,8 @@ func TestAMQPXSubAndPub(t *testing.T) { // publish event to first queue - err = amqpx.Publish(ctx, "exchange-01", "event-01", pool.Publishing{ - ContentType: "application/json", + err = amqp.Publish(cctx, funcName+"exchange-01", "event-01", pool.Publishing{ + ContentType: "text/plain", Body: []byte(eventContent), }) if err != nil { @@ -183,30 +205,36 @@ func TestAMQPXSubAndPub(t *testing.T) { } // will be canceled when the event has reache dthe third handler - <-ctx.Done() + <-cctx.Done() log.Info("context canceled, closing test.") - err = amqpx.Close() - assert.NoError(t, err) } func TestAMQPXSubAndPubMulti(t *testing.T) { - ctx := context.TODO() - log := logging.NewTestLogger(t) - defer amqpx.Reset() + t.Parallel() - amqpx.RegisterTopologyCreator(createTopology) - amqpx.RegisterTopologyDeleter(deleteTopology) + var ( + amqp = amqpx.New() + ctx = context.TODO() + funcName = testutils.FuncName() + log = logging.NewTestLogger(t) + ) + defer func() { + assert.NoError(t, amqp.Close()) + }() + + amqp.RegisterTopologyCreator(createTopology(funcName)) + amqp.RegisterTopologyDeleter(deleteTopology(funcName)) ctx, cancel := signal.NotifyContext(ctx, syscall.SIGINT, syscall.SIGINT) defer cancel() - eventContent := "TestAMQPXSubAndPub - event content" + eventContent := funcName + " - event content" // publish -> queue-01 -> subscriber-01 -> queue-02 -> subscriber-02 -> queue-03 -> subscriber-03 -> cancel context - amqpx.RegisterHandler("queue-01", func(ctx context.Context, msg pool.Delivery) error { + amqp.RegisterHandler(funcName+"queue-01", func(ctx context.Context, msg pool.Delivery) error { log.Info("handler of subscriber-01") - err := amqpx.Publish(ctx, "exchange-02", "event-02", pool.Publishing{ + err := amqp.Publish(ctx, funcName+"exchange-02", "event-02", pool.Publishing{ ContentType: msg.ContentType, Body: msg.Body, }) @@ -218,13 +246,13 @@ func TestAMQPXSubAndPubMulti(t *testing.T) { return nil }, - pool.ConsumeOptions{ConsumerTag: "subscriber-01"}, + pool.ConsumeOptions{ConsumerTag: funcName + "subscriber-01"}, ) - amqpx.RegisterHandler("queue-02", func(ctx context.Context, msg pool.Delivery) error { + amqp.RegisterHandler(funcName+"queue-02", func(ctx context.Context, msg pool.Delivery) error { log.Info("handler of subscriber-02") - err := amqpx.Publish(ctx, "exchange-03", "event-03", pool.Publishing{ + err := amqp.Publish(ctx, funcName+"exchange-03", "event-03", pool.Publishing{ ContentType: msg.ContentType, Body: msg.Body, }) @@ -234,17 +262,17 @@ func TestAMQPXSubAndPubMulti(t *testing.T) { } return nil - }, pool.ConsumeOptions{ConsumerTag: "subscriber-02"}) + }, pool.ConsumeOptions{ConsumerTag: funcName + "subscriber-02"}) - amqpx.RegisterHandler("queue-03", func(ctx context.Context, msg pool.Delivery) error { + amqp.RegisterHandler(funcName+"queue-03", func(ctx context.Context, msg pool.Delivery) error { log.Info("handler of subscriber-03: canceling context!") cancel() return nil - }, pool.ConsumeOptions{ConsumerTag: "subscriber-03"}) + }, pool.ConsumeOptions{ConsumerTag: funcName + "subscriber-03"}) - err := amqpx.Start( + err := amqp.Start( ctx, - connectURL, + testutils.HealthyConnectURL, amqpx.WithLogger(log), amqpx.WithPublisherConnections(1), amqpx.WithPublisherSessions(5), @@ -256,8 +284,8 @@ func TestAMQPXSubAndPubMulti(t *testing.T) { // publish event to first queue - err = amqpx.Publish(ctx, "exchange-01", "event-01", pool.Publishing{ - ContentType: "application/json", + err = amqp.Publish(ctx, funcName+"exchange-01", "event-01", pool.Publishing{ + ContentType: "text/plain", Body: []byte(eventContent), }) if err != nil { @@ -268,32 +296,39 @@ func TestAMQPXSubAndPubMulti(t *testing.T) { // will be canceled when the event has reache dthe third handler <-ctx.Done() log.Info("context canceled, closing test.") - err = amqpx.Close() - assert.NoError(t, err) } func TestAMQPXSubHandler(t *testing.T) { - ctx := context.TODO() - log := logging.NewTestLogger(t) - defer amqpx.Reset() + t.Parallel() - amqpx.RegisterTopologyCreator(createTopology) - amqpx.RegisterTopologyDeleter(deleteTopology) + var ( + amqp = amqpx.New() + ctx = context.TODO() + funcName = testutils.FuncName() + log = logging.NewTestLogger(t) + ) + defer func() { + assert.NoError(t, amqp.Close()) + }() + + amqp.RegisterTopologyCreator(createTopology(funcName)) + amqp.RegisterTopologyDeleter(deleteTopology(funcName)) ctx, cancel := signal.NotifyContext(ctx, syscall.SIGINT, syscall.SIGINT) defer cancel() - eventContent := "TestAMQPXSubAndPub - event content" + eventContent := funcName + " - event content" - amqpx.RegisterHandler("queue-01", func(ctx context.Context, msg pool.Delivery) error { + amqp.RegisterHandler(funcName+"queue-01", func(ctx context.Context, msg pool.Delivery) error { log.Info("subscriber of queue-01") + assert.Equal(t, eventContent, string(msg.Body)) cancel() return nil }) - err := amqpx.Start( + err := amqp.Start( ctx, - connectURL, + testutils.HealthyConnectURL, amqpx.WithLogger(logging.NewNoOpLogger()), amqpx.WithPublisherConnections(1), amqpx.WithPublisherSessions(5), @@ -305,8 +340,8 @@ func TestAMQPXSubHandler(t *testing.T) { // publish event to first queue - err = amqpx.Publish(ctx, "exchange-01", "event-01", pool.Publishing{ - ContentType: "application/json", + err = amqp.Publish(ctx, funcName+"exchange-01", "event-01", pool.Publishing{ + ContentType: "text/plain", Body: []byte(eventContent), }) if err != nil { @@ -317,21 +352,27 @@ func TestAMQPXSubHandler(t *testing.T) { // will be canceled when the event has reache dthe third handler <-ctx.Done() log.Info("context canceled, closing test.") - err = amqpx.Close() - assert.NoError(t, err) } func TestCreateDeleteTopology(t *testing.T) { - ctx := context.TODO() - log := logging.NewTestLogger(t) - defer amqpx.Reset() + t.Parallel() - amqpx.RegisterTopologyCreator(createTopology) - amqpx.RegisterTopologyDeleter(deleteTopology) + var ( + amqp = amqpx.New() + ctx = context.TODO() + funcName = testutils.FuncName() + log = logging.NewTestLogger(t) + ) + defer func() { + assert.NoError(t, amqp.Close()) + }() + + amqp.RegisterTopologyCreator(createTopology(funcName)) + amqp.RegisterTopologyDeleter(deleteTopology(funcName)) - err := amqpx.Start( + err := amqp.Start( ctx, - connectURL, + testutils.HealthyConnectURL, amqpx.WithLogger(log), amqpx.WithPublisherConnections(1), amqpx.WithPublisherSessions(2), @@ -340,14 +381,23 @@ func TestCreateDeleteTopology(t *testing.T) { } func TestPauseResumeHandlerNoProcessing(t *testing.T) { - var err error - queueName := "testPauseResumeHandler-01" - ctx, cancel := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGINT) - defer cancel() + t.Parallel() + + var ( + err error + amqp = amqpx.New() + cctx, cancel = signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGINT) + funcName = testutils.FuncName() + log = logging.NewTestLogger(t) + nextQueueName = testutils.QueueNameGenerator(funcName) + queueName = nextQueueName() + ) + defer func() { + assert.NoError(t, amqp.Close()) + }() - log := logging.NewTestLogger(t) + defer cancel() - amqp := amqpx.New() amqp.RegisterTopologyCreator(func(ctx context.Context, t *pool.Topologer) error { _, err := t.QueueDeclare(ctx, queueName) if err != nil { @@ -370,8 +420,8 @@ func TestPauseResumeHandlerNoProcessing(t *testing.T) { }) err = amqp.Start( - ctx, - connectURL, + cctx, + testutils.HealthyConnectURL, amqpx.WithLogger(logging.NewNoOpLogger()), ) if err != nil { @@ -387,7 +437,7 @@ func TestPauseResumeHandlerNoProcessing(t *testing.T) { t.Logf("iteration %d", i) assertActive(t, handler, true) - err = handler.Pause(context.Background()) + err = handler.Pause(cctx) if err != nil { assert.NoError(t, err) return @@ -395,7 +445,7 @@ func TestPauseResumeHandlerNoProcessing(t *testing.T) { assertActive(t, handler, false) - err = handler.Resume(context.Background()) + err = handler.Resume(cctx) if err != nil { assert.NoError(t, err) return @@ -407,20 +457,21 @@ func TestPauseResumeHandlerNoProcessing(t *testing.T) { func TestHandlerPauseAndResume(t *testing.T) { for i := 0; i < 10; i++ { - t.Logf("iteration %d", i) - testHandlerPauseAndResume(t) + testHandlerPauseAndResume(t, i) } } -func testHandlerPauseAndResume(t *testing.T) { - var err error - ctx, cancel := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGINT) - defer cancel() +func testHandlerPauseAndResume(t *testing.T, i int) { + t.Logf("iteration %d", i) - log := logging.NewTestLogger(t) - defer func() { - assert.NoError(t, amqpx.Reset()) - }() + var ( + err error + amqp = amqpx.New() + cctx, cancel = signal.NotifyContext(context.TODO(), syscall.SIGINT, syscall.SIGINT) + funcName = testutils.FuncName(3) + fmt.Sprintf("-i-%d", i) + log = logging.NewTestLogger(t) + ) + defer cancel() options := []amqpx.Option{ amqpx.WithLogger(logging.NewNoOpLogger()), @@ -429,7 +480,7 @@ func testHandlerPauseAndResume(t *testing.T) { } amqpxPublish := amqpx.New() - amqpxPublish.RegisterTopologyCreator(createTopology) + amqpxPublish.RegisterTopologyCreator(createTopology(funcName)) eventContent := "TestHandlerPauseAndResume - event content" @@ -439,18 +490,18 @@ func testHandlerPauseAndResume(t *testing.T) { ) // step 1 - fill queue with messages - amqpx.RegisterTopologyDeleter(deleteTopology) + amqp.RegisterTopologyDeleter(deleteTopology(funcName)) err = amqpxPublish.Start( - ctx, - connectURL, + cctx, + funcName, options..., ) require.NoError(t, err) // fill queue with messages for i := 0; i < publish; i++ { - err := amqpxPublish.Publish(ctx, "exchange-01", "event-01", pool.Publishing{ - ContentType: "application/json", + err := amqpxPublish.Publish(cctx, funcName+"exchange-01", "event-01", pool.Publishing{ + ContentType: "text/plain", Body: []byte(fmt.Sprintf("%s: message number %d", eventContent, i)), }) if err != nil { @@ -460,11 +511,11 @@ func testHandlerPauseAndResume(t *testing.T) { } // step 2 - process messages, pause, wait, resume, process rest, cancel context - handler01 := amqpx.RegisterHandler("queue-01", func(ctx context.Context, msg pool.Delivery) (err error) { + handler01 := amqp.RegisterHandler(funcName+"queue-01", func(ctx context.Context, msg pool.Delivery) (err error) { cnt++ if cnt == publish/3 || cnt == publish/3*2 { - err = amqpx.Publish(ctx, "exchange-02", "event-02", pool.Publishing{ - ContentType: "application/json", + err = amqp.Publish(ctx, funcName+"exchange-02", "event-02", pool.Publishing{ + ContentType: "text/plain", Body: []byte(fmt.Sprintf("%s: hit %d messages, toggling processing", eventContent, cnt)), }) assert.NoError(t, err) @@ -474,7 +525,7 @@ func testHandlerPauseAndResume(t *testing.T) { }) running := true - amqpx.RegisterHandler("queue-02", func(ctx context.Context, msg pool.Delivery) (err error) { + amqp.RegisterHandler(funcName+"queue-02", func(ctx context.Context, msg pool.Delivery) (err error) { log.Infof("received toggle request: %s", string(msg.Body)) queue := handler01.Queue() @@ -499,8 +550,8 @@ func testHandlerPauseAndResume(t *testing.T) { assertActive(t, handler01, true) // trigger cancelation - err = amqpxPublish.Publish(ctx, "exchange-03", "event-03", pool.Publishing{ - ContentType: "application/json", + err = amqpxPublish.Publish(ctx, funcName+"exchange-03", "event-03", pool.Publishing{ + ContentType: "text/plain", Body: []byte(fmt.Sprintf("%s: delayed toggle back", eventContent)), }) assert.NoError(t, err) @@ -509,12 +560,12 @@ func testHandlerPauseAndResume(t *testing.T) { }) var once sync.Once - amqpx.RegisterHandler("queue-03", func(ctx context.Context, msg pool.Delivery) (err error) { + amqp.RegisterHandler(funcName+"queue-03", func(ctx context.Context, msg pool.Delivery) (err error) { once.Do(func() { log.Info("pausing handler") assertActive(t, handler01, true) - err = handler01.Pause(context.Background()) + err = handler01.Pause(ctx) if err != nil { assert.NoError(t, err) return @@ -532,9 +583,9 @@ func testHandlerPauseAndResume(t *testing.T) { assertActive(t, handler01, false) - err = amqpx.Start( - ctx, - connectURL, + err = amqp.Start( + cctx, + testutils.HealthyConnectURL, amqpx.WithLogger(logging.NewNoOpLogger()), amqpx.WithPublisherConnections(1), amqpx.WithPublisherSessions(5), @@ -545,27 +596,29 @@ func testHandlerPauseAndResume(t *testing.T) { } // will be canceled when the event has reache dthe third handler - <-ctx.Done() + <-cctx.Done() log.Info("context canceled, closing test.") - assert.NoError(t, amqpx.Close()) + assert.NoError(t, amqp.Close()) assertActive(t, handler01, false) } func TestBatchHandlerPauseAndResume(t *testing.T) { for i := 0; i < 10; i++ { - testBatchHandlerPauseAndResume(t) + testBatchHandlerPauseAndResume(t, i) } } -func testBatchHandlerPauseAndResume(t *testing.T) { - var err error - ctx, cancel := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGINT) - defer cancel() +func testBatchHandlerPauseAndResume(t *testing.T, i int) { + t.Logf("iteration %d", i) - log := logging.NewTestLogger(t) - defer func() { - assert.NoError(t, amqpx.Reset()) - }() + var ( + err error + amqp = amqpx.New() + cctx, cancel = signal.NotifyContext(context.TODO(), syscall.SIGINT, syscall.SIGINT) + funcName = testutils.FuncName(3) + fmt.Sprintf("-i-%d", i) + log = logging.NewTestLogger(t) + ) + defer cancel() options := []amqpx.Option{ amqpx.WithLogger(logging.NewNoOpLogger()), @@ -574,11 +627,15 @@ func testBatchHandlerPauseAndResume(t *testing.T) { } amqpxPublish := amqpx.New() - amqpxPublish.RegisterTopologyCreator(createTopology) - err = amqpxPublish.Start(ctx, connectURL, options...) + amqpxPublish.RegisterTopologyCreator(createTopology(funcName)) + err = amqpxPublish.Start( + cctx, + testutils.HealthyConnectURL, + options..., + ) require.NoError(t, err) - eventContent := "TestBatchHandlerPauseAndResume - event content" + eventContent := funcName + " - event content" var ( publish = 500 @@ -586,12 +643,12 @@ func testBatchHandlerPauseAndResume(t *testing.T) { ) // step 1 - fill queue with messages - amqpx.RegisterTopologyDeleter(deleteTopology) + amqp.RegisterTopologyDeleter(deleteTopology(funcName)) // fill queue with messages for i := 0; i < publish; i++ { - err := amqpxPublish.Publish(ctx, "exchange-01", "event-01", pool.Publishing{ - ContentType: "application/json", + err := amqpxPublish.Publish(cctx, funcName+"exchange-01", "event-01", pool.Publishing{ + ContentType: "text/plain", Body: []byte(fmt.Sprintf("%s: message number %d", eventContent, i)), }) if err != nil { @@ -601,12 +658,12 @@ func testBatchHandlerPauseAndResume(t *testing.T) { } // step 2 - process messages, pause, wait, resume, process rest, cancel context - handler01 := amqpx.RegisterBatchHandler("queue-01", func(ctx context.Context, msgs []pool.Delivery) (err error) { + handler01 := amqp.RegisterBatchHandler(funcName+"queue-01", func(ctx context.Context, msgs []pool.Delivery) (err error) { for _, msg := range msgs { cnt++ if cnt == publish/3 || cnt == publish/3*2 { - err = amqpx.Publish(ctx, "exchange-02", "event-02", pool.Publishing{ - ContentType: "application/json", + err = amqp.Publish(ctx, funcName+"exchange-02", "event-02", pool.Publishing{ + ContentType: "text/plain", Body: []byte(fmt.Sprintf("%s: hit %d messages, toggling processing: %s", eventContent, cnt, string(msg.Body))), }) assert.NoError(t, err) @@ -616,7 +673,7 @@ func testBatchHandlerPauseAndResume(t *testing.T) { }) running := true - amqpx.RegisterBatchHandler("queue-02", func(ctx context.Context, msgs []pool.Delivery) (err error) { + amqp.RegisterBatchHandler(funcName+"queue-02", func(ctx context.Context, msgs []pool.Delivery) (err error) { queue := handler01.Queue() for _, msg := range msgs { @@ -626,7 +683,7 @@ func testBatchHandlerPauseAndResume(t *testing.T) { assertActive(t, handler01, true) - err = handler01.Pause(context.Background()) + err = handler01.Pause(ctx) assert.NoError(t, err) assertActive(t, handler01, false) @@ -635,15 +692,15 @@ func testBatchHandlerPauseAndResume(t *testing.T) { assertActive(t, handler01, false) - err = handler01.Resume(context.Background()) + err = handler01.Resume(ctx) assert.NoError(t, err) log.Infof("resumed processing of %s", queue) assertActive(t, handler01, true) // trigger cancelation - err = amqpxPublish.Publish(ctx, "exchange-03", "event-03", pool.Publishing{ - ContentType: "application/json", + err = amqpxPublish.Publish(ctx, funcName+"exchange-03", "event-03", pool.Publishing{ + ContentType: "text/plain", Body: []byte(fmt.Sprintf("%s: delayed toggle back", eventContent)), }) assert.NoError(t, err) @@ -653,12 +710,12 @@ func testBatchHandlerPauseAndResume(t *testing.T) { }) var once sync.Once - amqpx.RegisterBatchHandler("queue-03", func(ctx context.Context, msgs []pool.Delivery) (err error) { + amqp.RegisterBatchHandler(funcName+"queue-03", func(ctx context.Context, msgs []pool.Delivery) (err error) { _ = msgs[0] once.Do(func() { assertActive(t, handler01, true) - err = handler01.Pause(context.Background()) + err = handler01.Pause(ctx) if err != nil { assert.NoError(t, err) return @@ -677,9 +734,9 @@ func testBatchHandlerPauseAndResume(t *testing.T) { assertActive(t, handler01, false) - err = amqpx.Start( - ctx, - connectURL, + err = amqp.Start( + cctx, + testutils.HealthyConnectURL, amqpx.WithLogger(logging.NewNoOpLogger()), amqpx.WithPublisherConnections(1), amqpx.WithPublisherSessions(5), @@ -690,35 +747,39 @@ func testBatchHandlerPauseAndResume(t *testing.T) { } // will be canceled when the event has reache dthe third handler - <-ctx.Done() + <-cctx.Done() log.Info("context canceled, closing test.") - assert.NoError(t, amqpx.Close()) + assert.NoError(t, amqp.Close()) assertActive(t, handler01, false) } func TestQueueDeletedConsumerReconnect(t *testing.T) { - queueName := "TestQueueDeletedConsumerReconnect-01" - var err error - ctx, cancel := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGINT) + var ( + err error + amqp = amqpx.New() + log = logging.NewTestLogger(t) + cctx, cancel = signal.NotifyContext(context.TODO(), syscall.SIGINT, syscall.SIGINT) + funcName = testutils.FuncName() + nextQueueName = testutils.QueueNameGenerator(funcName) + queueName = nextQueueName() + ) defer cancel() - - log := logging.NewTestLogger(t) defer func() { - assert.NoError(t, amqpx.Reset()) + assert.NoError(t, amqp.Close()) }() - ts, closer := newTransientSession(t, ctx, connectURL) + ts, closer := newTransientSession(t, cctx, testutils.HealthyConnectURL) defer closer() // step 1 - fill queue with messages - amqpx.RegisterTopologyCreator(func(ctx context.Context, t *pool.Topologer) error { + amqp.RegisterTopologyCreator(func(ctx context.Context, t *pool.Topologer) error { _, err := t.QueueDeclare(ctx, queueName) if err != nil { return err } return nil }) - amqpx.RegisterTopologyDeleter(func(ctx context.Context, t *pool.Topologer) error { + amqp.RegisterTopologyDeleter(func(ctx context.Context, t *pool.Topologer) error { _, err := t.QueueDelete(ctx, queueName) if err != nil { return err @@ -726,15 +787,15 @@ func TestQueueDeletedConsumerReconnect(t *testing.T) { return nil }) - h := amqpx.RegisterHandler(queueName, func(ctx context.Context, msg pool.Delivery) (err error) { + h := amqp.RegisterHandler(queueName, func(ctx context.Context, msg pool.Delivery) (err error) { return nil }) assertActive(t, h, false) - err = amqpx.Start( - ctx, - connectURL, + err = amqp.Start( + cctx, + testutils.HealthyConnectURL, amqpx.WithLogger(log), ) if err != nil { @@ -742,51 +803,55 @@ func TestQueueDeletedConsumerReconnect(t *testing.T) { return } - assert.NoError(t, h.Pause(ctx)) + assert.NoError(t, h.Pause(cctx)) assertActive(t, h, false) - _, err = ts.QueueDelete(ctx, queueName) + _, err = ts.QueueDelete(cctx, queueName) assert.NoError(t, err) - tctx, cancel := context.WithTimeout(ctx, 5*time.Second) + tctx, tcancel := context.WithTimeout(cctx, 5*time.Second) err = h.Resume(tctx) - cancel() + tcancel() assert.Error(t, err) assertActive(t, h, false) - _, err = ts.QueueDeclare(ctx, queueName) + _, err = ts.QueueDeclare(cctx, queueName) assert.NoError(t, err) - tctx, cancel = context.WithTimeout(ctx, 5*time.Second) + tctx, tcancel = context.WithTimeout(cctx, 5*time.Second) err = h.Resume(tctx) - cancel() + tcancel() assert.NoError(t, err) assertActive(t, h, true) } func TestQueueDeletedBatchConsumerReconnect(t *testing.T) { - queueName := "TestQueueDeletedBatchConsumerReconnect-01" - var err error - ctx, cancel := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGINT) + var ( + err error + amqp = amqpx.New() + log = logging.NewTestLogger(t) + cctx, cancel = signal.NotifyContext(context.TODO(), syscall.SIGINT, syscall.SIGINT) + funcName = testutils.FuncName() + nextQueueName = testutils.QueueNameGenerator(funcName) + queueName = nextQueueName() + ) defer cancel() - - log := logging.NewTestLogger(t) defer func() { - assert.NoError(t, amqpx.Reset()) + assert.NoError(t, amqp.Close()) }() - ts, closer := newTransientSession(t, ctx, connectURL) + ts, closer := newTransientSession(t, cctx, testutils.HealthyConnectURL) defer closer() // step 1 - fill queue with messages - amqpx.RegisterTopologyCreator(func(ctx context.Context, t *pool.Topologer) error { + amqp.RegisterTopologyCreator(func(ctx context.Context, t *pool.Topologer) error { _, err := t.QueueDeclare(ctx, queueName) if err != nil { return err } return nil }) - amqpx.RegisterTopologyDeleter(func(ctx context.Context, t *pool.Topologer) error { + amqp.RegisterTopologyDeleter(func(ctx context.Context, t *pool.Topologer) error { _, err := t.QueueDelete(ctx, queueName) if err != nil { return err @@ -794,15 +859,15 @@ func TestQueueDeletedBatchConsumerReconnect(t *testing.T) { return nil }) - h := amqpx.RegisterBatchHandler(queueName, func(ctx context.Context, msg []pool.Delivery) (err error) { + h := amqp.RegisterBatchHandler(queueName, func(ctx context.Context, msg []pool.Delivery) (err error) { return nil }) assertActive(t, h, false) - err = amqpx.Start( - ctx, - connectURL, + err = amqp.Start( + cctx, + testutils.HealthyConnectURL, amqpx.WithLogger(log), ) if err != nil { @@ -810,24 +875,24 @@ func TestQueueDeletedBatchConsumerReconnect(t *testing.T) { return } - assert.NoError(t, h.Pause(ctx)) + assert.NoError(t, h.Pause(cctx)) assertActive(t, h, false) - _, err = ts.QueueDelete(ctx, queueName) + _, err = ts.QueueDelete(cctx, queueName) assert.NoError(t, err) - tctx, cancel := context.WithTimeout(ctx, 5*time.Second) + tctx, tcancel := context.WithTimeout(cctx, 5*time.Second) err = h.Resume(tctx) - cancel() + tcancel() assert.Error(t, err) assertActive(t, h, false) - _, err = ts.QueueDeclare(ctx, queueName) + _, err = ts.QueueDeclare(cctx, queueName) assert.NoError(t, err) - tctx, cancel = context.WithTimeout(ctx, 5*time.Second) + tctx, tcancel = context.WithTimeout(cctx, 5*time.Second) err = h.Resume(tctx) - cancel() + tcancel() assert.NoError(t, err) assertActive(t, h, true) } @@ -854,21 +919,21 @@ type handlerStats interface { func TestHandlerReset(t *testing.T) { for i := 0; i < 5; i++ { - testHandlerReset(t) + testHandlerReset(t, i) } t.Log("done") } -func testHandlerReset(t *testing.T) { - var err error - ctx, cancel := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGINT) +func testHandlerReset(t *testing.T, i int) { + var ( + err error + amqp = amqpx.New() + log = logging.NewTestLogger(t) + cctx, cancel = signal.NotifyContext(context.TODO(), syscall.SIGINT, syscall.SIGINT) + funcName = testutils.FuncName(3) + fmt.Sprintf("-i-%d", i) + ) defer cancel() - log := logging.NewTestLogger(t) - defer func() { - assert.NoError(t, amqpx.Reset()) - }() - options := []amqpx.Option{ amqpx.WithLogger(logging.NewNoOpLogger()), amqpx.WithPublisherConnections(1), @@ -876,11 +941,15 @@ func testHandlerReset(t *testing.T) { } amqpxPublish := amqpx.New() - amqpxPublish.RegisterTopologyCreator(createTopology) - err = amqpxPublish.Start(ctx, connectURL, options...) + amqpxPublish.RegisterTopologyCreator(createTopology(funcName)) + err = amqpxPublish.Start( + cctx, + testutils.HealthyConnectURL, + options..., + ) require.NoError(t, err) - eventContent := "TestBatchHandlerReset - event content" + eventContent := funcName + " - event content" var ( publish = 50 @@ -888,12 +957,12 @@ func testHandlerReset(t *testing.T) { ) // step 1 - fill queue with messages - amqpx.RegisterTopologyDeleter(deleteTopology) + amqp.RegisterTopologyDeleter(deleteTopology(funcName)) // fill queue with messages for i := 0; i < publish; i++ { - err := amqpxPublish.Publish(ctx, "exchange-01", "event-01", pool.Publishing{ - ContentType: "application/json", + err := amqpxPublish.Publish(cctx, funcName+"exchange-01", "event-01", pool.Publishing{ + ContentType: "text/plain", Body: []byte(fmt.Sprintf("%s: message number %d", eventContent, i)), }) if err != nil { @@ -904,7 +973,7 @@ func testHandlerReset(t *testing.T) { done := make(chan struct{}) // step 2 - process messages, pause, wait, resume, process rest, cancel context - handler01 := amqpx.RegisterHandler("queue-01", func(ctx context.Context, msgs pool.Delivery) (err error) { + handler01 := amqp.RegisterHandler(funcName+"queue-01", func(ctx context.Context, msgs pool.Delivery) (err error) { cnt++ if cnt == publish { close(done) @@ -914,9 +983,9 @@ func testHandlerReset(t *testing.T) { assertActive(t, handler01, false) - err = amqpx.Start( - ctx, - connectURL, + err = amqp.Start( + cctx, + testutils.HealthyConnectURL, amqpx.WithLogger(logging.NewNoOpLogger()), amqpx.WithPublisherConnections(1), amqpx.WithPublisherSessions(5), @@ -932,9 +1001,9 @@ func testHandlerReset(t *testing.T) { <-done cancel() - <-ctx.Done() + <-cctx.Done() log.Info("context canceled, closing test.") - assert.NoError(t, amqpx.Close()) + assert.NoError(t, amqp.Close()) // after close assertActive(t, handler01, false) @@ -942,21 +1011,21 @@ func testHandlerReset(t *testing.T) { func TestBatchHandlerReset(t *testing.T) { for i := 0; i < 5; i++ { - testBatchHandlerReset(t) + testBatchHandlerReset(t, i) } t.Log("done") } -func testBatchHandlerReset(t *testing.T) { - var err error - ctx, cancel := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGINT) +func testBatchHandlerReset(t *testing.T, i int) { + var ( + err error + amqp = amqpx.New() + log = logging.NewTestLogger(t) + cctx, cancel = signal.NotifyContext(context.TODO(), syscall.SIGINT, syscall.SIGINT) + funcName = testutils.FuncName(3) + fmt.Sprintf("-i-%d", i) + ) defer cancel() - log := logging.NewTestLogger(t) - defer func() { - assert.NoError(t, amqpx.Reset()) - }() - options := []amqpx.Option{ amqpx.WithLogger(logging.NewNoOpLogger()), amqpx.WithPublisherConnections(1), @@ -964,8 +1033,12 @@ func testBatchHandlerReset(t *testing.T) { } amqpxPublish := amqpx.New() - amqpxPublish.RegisterTopologyCreator(createTopology) - err = amqpxPublish.Start(ctx, connectURL, options...) + amqpxPublish.RegisterTopologyCreator(createTopology(funcName)) + err = amqpxPublish.Start( + cctx, + testutils.HealthyConnectURL, + options..., + ) require.NoError(t, err) eventContent := "TestBatchHandlerReset - event content" @@ -976,12 +1049,12 @@ func testBatchHandlerReset(t *testing.T) { ) // step 1 - fill queue with messages - amqpx.RegisterTopologyDeleter(deleteTopology) + amqp.RegisterTopologyDeleter(deleteTopology(funcName)) // fill queue with messages for i := 0; i < publish; i++ { - err := amqpxPublish.Publish(ctx, "exchange-01", "event-01", pool.Publishing{ - ContentType: "application/json", + err := amqpxPublish.Publish(cctx, funcName+"exchange-01", "event-01", pool.Publishing{ + ContentType: "text/plain", Body: []byte(fmt.Sprintf("%s: message number %d", eventContent, i)), }) if err != nil { @@ -992,7 +1065,7 @@ func testBatchHandlerReset(t *testing.T) { done := make(chan struct{}) // step 2 - process messages, pause, wait, resume, process rest, cancel context - handler01 := amqpx.RegisterBatchHandler("queue-01", func(ctx context.Context, msgs []pool.Delivery) (err error) { + handler01 := amqp.RegisterBatchHandler(funcName+"queue-01", func(ctx context.Context, msgs []pool.Delivery) (err error) { cnt += len(msgs) if cnt == publish { @@ -1003,9 +1076,9 @@ func testBatchHandlerReset(t *testing.T) { assertActive(t, handler01, false) - err = amqpx.Start( - ctx, - connectURL, + err = amqp.Start( + cctx, + testutils.HealthyConnectURL, amqpx.WithLogger(logging.NewNoOpLogger()), amqpx.WithPublisherConnections(1), amqpx.WithPublisherSessions(5), @@ -1021,9 +1094,9 @@ func testBatchHandlerReset(t *testing.T) { <-done cancel() - <-ctx.Done() + <-cctx.Done() log.Info("context canceled, closing test.") - assert.NoError(t, amqpx.Close()) + assert.NoError(t, amqp.Close()) // after close assertActive(t, handler01, false) diff --git a/helpers_test.go b/helpers_test.go index 6f491fe..b6d618e 100644 --- a/helpers_test.go +++ b/helpers_test.go @@ -5,91 +5,95 @@ import ( "errors" "fmt" + "github.com/jxsl13/amqpx" "github.com/jxsl13/amqpx/pool" ) -func createTopology(ctx context.Context, t *pool.Topologer) (err error) { - // documentation: https://www.cloudamqp.com/blog/part4-rabbitmq-for-beginners-exchanges-routing-keys-bindings.html#:~:text=The%20routing%20key%20is%20a%20message%20attribute%20added%20to%20the,routing%20key%20of%20the%20message. +func createTopology(prefix string) amqpx.TopologyFunc { + return func(ctx context.Context, t *pool.Topologer) (err error) { + // documentation: https://www.cloudamqp.com/blog/part4-rabbitmq-for-beginners-exchanges-routing-keys-bindings.html#:~:text=The%20routing%20key%20is%20a%20message%20attribute%20added%20to%20the,routing%20key%20of%20the%20message. - err = createExchange(ctx, "exchange-01", t) - if err != nil { - return err - } + err = createExchange(ctx, prefix+"exchange-01", t) + if err != nil { + return err + } - err = createQueue(ctx, "queue-01", t) - if err != nil { - return err - } + err = createQueue(ctx, prefix+"queue-01", t) + if err != nil { + return err + } - err = t.QueueBind(ctx, "queue-01", "event-01", "exchange-01") - if err != nil { - return err - } + err = t.QueueBind(ctx, prefix+"queue-01", "event-01", prefix+"exchange-01") + if err != nil { + return err + } - err = createExchange(ctx, "exchange-02", t) - if err != nil { - return err - } + err = createExchange(ctx, prefix+"exchange-02", t) + if err != nil { + return err + } - err = createQueue(ctx, "queue-02", t) - if err != nil { - return err - } + err = createQueue(ctx, prefix+"queue-02", t) + if err != nil { + return err + } - err = t.QueueBind(ctx, "queue-02", "event-02", "exchange-02") - if err != nil { - return err - } + err = t.QueueBind(ctx, prefix+"queue-02", "event-02", prefix+"exchange-02") + if err != nil { + return err + } - err = createExchange(ctx, "exchange-03", t) - if err != nil { - return err - } + err = createExchange(ctx, prefix+"exchange-03", t) + if err != nil { + return err + } - err = createQueue(ctx, "queue-03", t) - if err != nil { - return err - } - err = t.QueueBind(ctx, "queue-03", "event-03", "exchange-03") - if err != nil { - return err + err = createQueue(ctx, prefix+"queue-03", t) + if err != nil { + return err + } + err = t.QueueBind(ctx, prefix+"queue-03", "event-03", prefix+"exchange-03") + if err != nil { + return err + } + return nil } - return nil - } -func deleteTopology(ctx context.Context, t *pool.Topologer) (err error) { - err = deleteQueue(ctx, "queue-01", t) - if err != nil { - return err - } +func deleteTopology(prefix string) amqpx.TopologyFunc { + return func(ctx context.Context, t *pool.Topologer) (err error) { + err = deleteQueue(ctx, prefix+"queue-01", t) + if err != nil { + return err + } - err = deleteQueue(ctx, "queue-02", t) - if err != nil { - return err - } + err = deleteQueue(ctx, prefix+"queue-02", t) + if err != nil { + return err + } - err = deleteQueue(ctx, "queue-03", t) - if err != nil { - return err - } + err = deleteQueue(ctx, prefix+"queue-03", t) + if err != nil { + return err + } - err = deleteExchange(ctx, "exchange-01", t) - if err != nil { - return err - } + err = deleteExchange(ctx, prefix+"exchange-01", t) + if err != nil { + return err + } - err = deleteExchange(ctx, "exchange-02", t) - if err != nil { - return err - } + err = deleteExchange(ctx, prefix+"exchange-02", t) + if err != nil { + return err + } - err = deleteExchange(ctx, "exchange-03", t) - if err != nil { - return err - } + err = deleteExchange(ctx, prefix+"exchange-03", t) + if err != nil { + return err + } - return nil + return nil + } } func createQueue(ctx context.Context, name string, t *pool.Topologer) (err error) { From 94ff5b3001dbe0785d9d968e8296bf77724da173 Mon Sep 17 00:00:00 2001 From: John Behm Date: Thu, 14 Mar 2024 15:52:34 +0100 Subject: [PATCH 50/76] fix amqpx, amqpx tests & test utilities --- amqpx.go | 36 +-- amqpx_test.go | 428 ++++++++++++++++++-------------- helpers_test.go | 148 +++++------ internal/testutils/generator.go | 56 +++++ 4 files changed, 402 insertions(+), 266 deletions(-) diff --git a/amqpx.go b/amqpx.go index 9662677..68e0502 100644 --- a/amqpx.go +++ b/amqpx.go @@ -11,21 +11,23 @@ import ( "github.com/jxsl13/amqpx/pool" ) -/* var ( // global variable, as we usually only need a single connection amqpx = New() ) -*/ type ( TopologyFunc func(context.Context, *pool.Topologer) error ) +func noopCancel() {} + type AMQPX struct { - pubPool *pool.Pool - pub *pool.Publisher - sub *pool.Subscriber + pubCtx context.Context + pubCancel context.CancelFunc + pubPool *pool.Pool + pub *pool.Publisher + sub *pool.Subscriber mu sync.RWMutex handlers []*pool.Handler @@ -42,9 +44,11 @@ type AMQPX struct { func New() *AMQPX { return &AMQPX{ - pubPool: nil, - pub: nil, - sub: nil, + pubCtx: context.Background(), + pubCancel: noopCancel, + pubPool: nil, + pub: nil, + sub: nil, handlers: make([]*pool.Handler, 0), batchHandlers: make([]*pool.BatchHandler, 0), @@ -59,6 +63,8 @@ func (a *AMQPX) Reset() error { defer a.mu.Unlock() err := a.close() + a.pubCtx = context.Background() + a.pubCancel = noopCancel a.pubPool = nil a.pub = nil a.sub = nil @@ -166,7 +172,7 @@ func (a *AMQPX) Start(ctx context.Context, connectUrl string, options ...Option) PublisherOptions: make([]pool.PublisherOption, 0), PublisherConnections: 1, PublisherSessions: 10, - SubscriberConnections: 1, + SubscriberConnections: len(a.handlers) + len(a.batchHandlers), CloseTimeout: 15 * time.Second, } @@ -179,8 +185,11 @@ func (a *AMQPX) Start(ctx context.Context, connectUrl string, options ...Option) a.closeTimeout = option.CloseTimeout // publisher and subscriber need to have different tcp connections (tcp pushback prevention) + // pub pool is only closed when .Close() is called. + // This is needed so that we can correctly call the topology deleters. + a.pubCtx, a.pubCancel = context.WithCancel(context.Background()) a.pubPool, err = pool.New( - ctx, + a.pubCtx, connectUrl, option.PublisherConnections, option.PublisherSessions, @@ -218,7 +227,7 @@ func (a *AMQPX) Start(ctx context.Context, connectUrl string, options ...Option) connections = requiredHandlers ) - if option.SubscriberConnections > connections { + if option.SubscriberConnections >= connections { connections = option.SubscriberConnections } @@ -264,6 +273,7 @@ func (a *AMQPX) Close() error { func (a *AMQPX) close() (err error) { a.closeOnce.Do(func() { + defer a.pubCancel() if a.sub != nil { a.sub.Close() @@ -274,7 +284,7 @@ func (a *AMQPX) close() (err error) { } if a.pubPool != nil && len(a.topologyDeleters) > 0 { - ctx, cancel := context.WithTimeout(context.Background(), a.closeTimeout) + ctx, cancel := context.WithTimeout(a.pubCtx, a.closeTimeout) defer cancel() topologer := pool.NewTopologer( @@ -320,7 +330,6 @@ func (a *AMQPX) Get(ctx context.Context, queue string, autoAck bool) (msg pool.D return a.pub.Get(ctx, queue, autoAck) } -/* // RegisterTopology registers a topology creating function that is called upon // Start. The creation of topologie sis the first step before any publisher or subscriber is started. func RegisterTopologyCreator(topology TopologyFunc) { @@ -379,4 +388,3 @@ func Get(ctx context.Context, queue string, autoAck bool) (msg pool.Delivery, ok func Reset() error { return amqpx.Reset() } -*/ diff --git a/amqpx_test.go b/amqpx_test.go index 8fe3b96..d745a32 100644 --- a/amqpx_test.go +++ b/amqpx_test.go @@ -28,6 +28,7 @@ func TestExchangeDeclarePassive(t *testing.T) { var ( amqp = amqpx.New() ctx = context.TODO() + log = logging.NewTestLogger(t) funcName = testutils.FuncName() nextExchangeName = testutils.ExchangeNameGenerator(funcName) exchangeName = nextExchangeName() @@ -38,16 +39,17 @@ func TestExchangeDeclarePassive(t *testing.T) { var err error amqp.RegisterTopologyCreator(func(ctx context.Context, t *pool.Topologer) error { - return createExchange(ctx, exchangeName, t) + return createExchange(ctx, exchangeName, t, log) }) amqp.RegisterTopologyDeleter(func(ctx context.Context, t *pool.Topologer) error { - return deleteExchange(ctx, exchangeName, t) + return deleteExchange(ctx, exchangeName, t, log) }) err = amqp.Start( ctx, testutils.HealthyConnectURL, + amqpx.WithName(funcName), amqpx.WithLogger(logging.NewTestLogger(t)), amqpx.WithPublisherConnections(1), amqpx.WithPublisherSessions(2), @@ -61,6 +63,7 @@ func TestQueueDeclarePassive(t *testing.T) { var ( amqp = amqpx.New() ctx = context.TODO() + log = logging.NewTestLogger(t) funcName = testutils.FuncName() nextQueueName = testutils.QueueNameGenerator(funcName) queueName = nextQueueName() @@ -71,16 +74,17 @@ func TestQueueDeclarePassive(t *testing.T) { var err error amqp.RegisterTopologyCreator(func(ctx context.Context, t *pool.Topologer) error { - return createQueue(ctx, queueName, t) + return createQueue(ctx, queueName, t, log) }) amqp.RegisterTopologyDeleter(func(ctx context.Context, t *pool.Topologer) error { - return deleteQueue(ctx, queueName, t) + return deleteQueue(ctx, queueName, t, log) }) err = amqp.Start( ctx, testutils.HealthyConnectURL, + amqpx.WithName(funcName), amqpx.WithLogger(logging.NewTestLogger(t)), amqpx.WithPublisherConnections(1), amqpx.WithPublisherSessions(2), @@ -94,18 +98,23 @@ func TestAMQPXPub(t *testing.T) { var ( amqp = amqpx.New() ctx = context.TODO() + log = logging.NewTestLogger(t) funcName = testutils.FuncName() + + nextExchangeQueue = testutils.NewExchangeQueueGenerator(funcName) + eq1 = nextExchangeQueue() ) defer func() { assert.NoError(t, amqp.Close()) }() - amqp.RegisterTopologyCreator(createTopology(funcName)) - amqp.RegisterTopologyDeleter(deleteTopology(funcName)) + amqp.RegisterTopologyCreator(createTopology(log, eq1)) + amqp.RegisterTopologyDeleter(deleteTopology(log, eq1)) err := amqp.Start( ctx, testutils.HealthyConnectURL, + amqpx.WithName(funcName), amqpx.WithLogger(logging.NewNoOpLogger()), amqpx.WithPublisherConnections(1), amqpx.WithPublisherSessions(2), @@ -120,12 +129,10 @@ func TestAMQPXPub(t *testing.T) { assert.NoError(t, err) }() - event := funcName + " - event content" - // publish event to first queue - err = amqp.Publish(ctx, funcName+"exchange-01", "event-01", pool.Publishing{ + err = amqp.Publish(ctx, eq1.Exchange, eq1.RoutingKey, pool.Publishing{ ContentType: "text/plain", - Body: []byte(event), + Body: []byte(eq1.NextPubMsg()), }) if err != nil { assert.NoError(t, err) @@ -136,8 +143,9 @@ func TestAMQPXPub(t *testing.T) { msg pool.Delivery ok bool ) + for i := 0; i < 20; i++ { - msg, ok, err = amqp.Get(ctx, funcName+"queue-01", false) + msg, ok, err = amqp.Get(ctx, eq1.Queue, false) if err != nil { assert.NoError(t, err) return @@ -154,15 +162,19 @@ func TestAMQPXPub(t *testing.T) { return } - assert.Equal(t, event, string(msg.Body)) + assert.Equal(t, eq1.NextSubMsg(), string(msg.Body)) } func TestAMQPXSubAndPub(t *testing.T) { + t.Parallel() + var ( - amqp = amqpx.New() - log = logging.NewTestLogger(t) - cctx, cancel = signal.NotifyContext(context.TODO(), syscall.SIGINT, syscall.SIGINT) - funcName = testutils.FuncName() + amqp = amqpx.New() + log = logging.NewTestLogger(t) + cctx, cancel = signal.NotifyContext(context.TODO(), syscall.SIGINT, syscall.SIGINT) + funcName = testutils.FuncName() + nextExchangeQueue = testutils.NewExchangeQueueGenerator(funcName) + eq1 = nextExchangeQueue() ) defer cancel() defer func() { @@ -170,13 +182,12 @@ func TestAMQPXSubAndPub(t *testing.T) { assert.NoError(t, amqp.Close()) }() - amqp.RegisterTopologyCreator(createTopology(funcName)) - amqp.RegisterTopologyDeleter(deleteTopology(funcName)) - - eventContent := funcName + " - event content" + amqp.RegisterTopologyCreator(createTopology(log, eq1)) + amqp.RegisterTopologyDeleter(deleteTopology(log, eq1)) - amqp.RegisterHandler(funcName+"queue-01", func(ctx context.Context, msg pool.Delivery) error { - log.Info("subscriber of queue-01") + amqp.RegisterHandler(eq1.Queue, func(ctx context.Context, msg pool.Delivery) error { + log.Infof("subscriber of %s", eq1.Queue) + assert.Equal(t, eq1.NextSubMsg(), string(msg.Body)) cancel() return nil }) @@ -184,6 +195,7 @@ func TestAMQPXSubAndPub(t *testing.T) { err := amqp.Start( cctx, testutils.HealthyConnectURL, + amqpx.WithName(funcName), amqpx.WithLogger(logging.NewNoOpLogger()), amqpx.WithPublisherConnections(1), amqpx.WithPublisherSessions(5), @@ -195,9 +207,9 @@ func TestAMQPXSubAndPub(t *testing.T) { // publish event to first queue - err = amqp.Publish(cctx, funcName+"exchange-01", "event-01", pool.Publishing{ + err = amqp.Publish(cctx, eq1.Exchange, eq1.RoutingKey, pool.Publishing{ ContentType: "text/plain", - Body: []byte(eventContent), + Body: []byte(eq1.NextPubMsg()), }) if err != nil { assert.NoError(t, err) @@ -213,66 +225,73 @@ func TestAMQPXSubAndPubMulti(t *testing.T) { t.Parallel() var ( - amqp = amqpx.New() - ctx = context.TODO() - funcName = testutils.FuncName() - log = logging.NewTestLogger(t) + amqp = amqpx.New() + log = logging.NewTestLogger(t) + cctx, cancel = signal.NotifyContext(context.TODO(), syscall.SIGINT, syscall.SIGINT) + funcName = testutils.FuncName() + nextExchangeQueue = testutils.NewExchangeQueueGenerator(funcName) + + eq1 = nextExchangeQueue() + eq2 = nextExchangeQueue() + eq3 = nextExchangeQueue() ) + defer cancel() defer func() { + log.Info("closing amqp") assert.NoError(t, amqp.Close()) }() - amqp.RegisterTopologyCreator(createTopology(funcName)) - amqp.RegisterTopologyDeleter(deleteTopology(funcName)) - - ctx, cancel := signal.NotifyContext(ctx, syscall.SIGINT, syscall.SIGINT) - defer cancel() - - eventContent := funcName + " - event content" + amqp.RegisterTopologyCreator(createTopology(log, eq1, eq2, eq3)) + amqp.RegisterTopologyDeleter(deleteTopology(log, eq1, eq2, eq3)) // publish -> queue-01 -> subscriber-01 -> queue-02 -> subscriber-02 -> queue-03 -> subscriber-03 -> cancel context - amqp.RegisterHandler(funcName+"queue-01", func(ctx context.Context, msg pool.Delivery) error { - log.Info("handler of subscriber-01") + amqp.RegisterHandler(eq1.Queue, func(ctx context.Context, msg pool.Delivery) error { + log.Infof("handler of %s", eq1.Queue) - err := amqp.Publish(ctx, funcName+"exchange-02", "event-02", pool.Publishing{ + assert.Equal(t, eq1.NextSubMsg(), string(msg.Body)) + + err := amqp.Publish(ctx, eq2.Exchange, eq2.RoutingKey, pool.Publishing{ ContentType: msg.ContentType, Body: msg.Body, }) if err != nil { - log.Error("subscriber-01:", err) + log.Errorf("%s: %v", eq1.Queue, err) assert.NoError(t, err) } return nil }, - pool.ConsumeOptions{ConsumerTag: funcName + "subscriber-01"}, + pool.ConsumeOptions{ + ConsumerTag: eq1.ConsumerTag, + }, ) - amqp.RegisterHandler(funcName+"queue-02", func(ctx context.Context, msg pool.Delivery) error { - log.Info("handler of subscriber-02") + amqp.RegisterHandler(eq2.Queue, func(ctx context.Context, msg pool.Delivery) error { + log.Infof("handler of %s", eq2.Queue) - err := amqp.Publish(ctx, funcName+"exchange-03", "event-03", pool.Publishing{ + err := amqp.Publish(ctx, eq3.Exchange, eq3.RoutingKey, pool.Publishing{ ContentType: msg.ContentType, Body: msg.Body, }) if err != nil { - log.Error("subscriber-02:", err) + log.Errorf("%s: %v", eq2.Queue, err) } return nil - }, pool.ConsumeOptions{ConsumerTag: funcName + "subscriber-02"}) + }, pool.ConsumeOptions{ConsumerTag: eq2.ConsumerTag}) - amqp.RegisterHandler(funcName+"queue-03", func(ctx context.Context, msg pool.Delivery) error { - log.Info("handler of subscriber-03: canceling context!") + amqp.RegisterHandler(eq3.Queue, func(ctx context.Context, msg pool.Delivery) error { + log.Infof("handler of %s: canceling context!", eq3.Queue) cancel() return nil - }, pool.ConsumeOptions{ConsumerTag: funcName + "subscriber-03"}) + }, pool.ConsumeOptions{ConsumerTag: eq3.ConsumerTag}) err := amqp.Start( - ctx, + cctx, testutils.HealthyConnectURL, + amqpx.WithName(funcName), amqpx.WithLogger(log), amqpx.WithPublisherConnections(1), amqpx.WithPublisherSessions(5), @@ -284,9 +303,9 @@ func TestAMQPXSubAndPubMulti(t *testing.T) { // publish event to first queue - err = amqp.Publish(ctx, funcName+"exchange-01", "event-01", pool.Publishing{ + err = amqp.Publish(cctx, eq1.Exchange, eq1.RoutingKey, pool.Publishing{ ContentType: "text/plain", - Body: []byte(eventContent), + Body: []byte(eq1.NextPubMsg()), }) if err != nil { assert.NoError(t, err) @@ -294,7 +313,7 @@ func TestAMQPXSubAndPubMulti(t *testing.T) { } // will be canceled when the event has reache dthe third handler - <-ctx.Done() + <-cctx.Done() log.Info("context canceled, closing test.") } @@ -302,33 +321,33 @@ func TestAMQPXSubHandler(t *testing.T) { t.Parallel() var ( - amqp = amqpx.New() - ctx = context.TODO() - funcName = testutils.FuncName() - log = logging.NewTestLogger(t) + amqp = amqpx.New() + log = logging.NewTestLogger(t) + cctx, cancel = signal.NotifyContext(context.TODO(), syscall.SIGINT, syscall.SIGINT) + funcName = testutils.FuncName() + nextExchangeQueue = testutils.NewExchangeQueueGenerator(funcName) + eq1 = nextExchangeQueue() ) + defer cancel() defer func() { + log.Info("closing amqp") assert.NoError(t, amqp.Close()) }() - amqp.RegisterTopologyCreator(createTopology(funcName)) - amqp.RegisterTopologyDeleter(deleteTopology(funcName)) - - ctx, cancel := signal.NotifyContext(ctx, syscall.SIGINT, syscall.SIGINT) - defer cancel() + amqp.RegisterTopologyCreator(createTopology(log, eq1)) + amqp.RegisterTopologyDeleter(deleteTopology(log, eq1)) - eventContent := funcName + " - event content" - - amqp.RegisterHandler(funcName+"queue-01", func(ctx context.Context, msg pool.Delivery) error { - log.Info("subscriber of queue-01") - assert.Equal(t, eventContent, string(msg.Body)) + amqp.RegisterHandler(eq1.Queue, func(ctx context.Context, msg pool.Delivery) error { + log.Infof("subscriber of %s", eq1.Queue) + assert.Equal(t, eq1.NextSubMsg(), string(msg.Body)) cancel() return nil }) err := amqp.Start( - ctx, + cctx, testutils.HealthyConnectURL, + amqpx.WithName(funcName), amqpx.WithLogger(logging.NewNoOpLogger()), amqpx.WithPublisherConnections(1), amqpx.WithPublisherSessions(5), @@ -340,9 +359,9 @@ func TestAMQPXSubHandler(t *testing.T) { // publish event to first queue - err = amqp.Publish(ctx, funcName+"exchange-01", "event-01", pool.Publishing{ + err = amqp.Publish(cctx, eq1.Exchange, eq1.RoutingKey, pool.Publishing{ ContentType: "text/plain", - Body: []byte(eventContent), + Body: []byte(eq1.NextPubMsg()), }) if err != nil { assert.NoError(t, err) @@ -350,7 +369,7 @@ func TestAMQPXSubHandler(t *testing.T) { } // will be canceled when the event has reache dthe third handler - <-ctx.Done() + <-cctx.Done() log.Info("context canceled, closing test.") } @@ -358,21 +377,26 @@ func TestCreateDeleteTopology(t *testing.T) { t.Parallel() var ( - amqp = amqpx.New() - ctx = context.TODO() - funcName = testutils.FuncName() - log = logging.NewTestLogger(t) + amqp = amqpx.New() + log = logging.NewTestLogger(t) + cctx, cancel = signal.NotifyContext(context.TODO(), syscall.SIGINT, syscall.SIGINT) + funcName = testutils.FuncName() + nextExchangeQueue = testutils.NewExchangeQueueGenerator(funcName) + eq1 = nextExchangeQueue() ) + defer cancel() defer func() { + log.Info("closing amqp") assert.NoError(t, amqp.Close()) }() - amqp.RegisterTopologyCreator(createTopology(funcName)) - amqp.RegisterTopologyDeleter(deleteTopology(funcName)) + amqp.RegisterTopologyCreator(createTopology(log, eq1)) + amqp.RegisterTopologyDeleter(deleteTopology(log, eq1)) err := amqp.Start( - ctx, + cctx, testutils.HealthyConnectURL, + amqpx.WithName(funcName), amqpx.WithLogger(log), amqpx.WithPublisherConnections(1), amqpx.WithPublisherSessions(2), @@ -384,20 +408,19 @@ func TestPauseResumeHandlerNoProcessing(t *testing.T) { t.Parallel() var ( - err error amqp = amqpx.New() - cctx, cancel = signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGINT) - funcName = testutils.FuncName() log = logging.NewTestLogger(t) + cctx, cancel = signal.NotifyContext(context.TODO(), syscall.SIGINT, syscall.SIGINT) + funcName = testutils.FuncName() nextQueueName = testutils.QueueNameGenerator(funcName) queueName = nextQueueName() ) + defer cancel() defer func() { + log.Info("closing amqp") assert.NoError(t, amqp.Close()) }() - defer cancel() - amqp.RegisterTopologyCreator(func(ctx context.Context, t *pool.Topologer) error { _, err := t.QueueDeclare(ctx, queueName) if err != nil { @@ -419,19 +442,16 @@ func TestPauseResumeHandlerNoProcessing(t *testing.T) { return nil }) - err = amqp.Start( + err := amqp.Start( cctx, testutils.HealthyConnectURL, + amqpx.WithName(funcName), amqpx.WithLogger(logging.NewNoOpLogger()), ) if err != nil { assert.NoError(t, err) return } - defer func() { - err = amqp.Close() - assert.NoError(t, err) - }() for i := 0; i < 5; i++ { t.Logf("iteration %d", i) @@ -456,8 +476,17 @@ func TestPauseResumeHandlerNoProcessing(t *testing.T) { } func TestHandlerPauseAndResume(t *testing.T) { + t.Parallel() + + var wg sync.WaitGroup + defer wg.Wait() + for i := 0; i < 10; i++ { - testHandlerPauseAndResume(t, i) + wg.Add(1) + go func(i int) { + defer wg.Done() + testHandlerPauseAndResume(t, i) + }(i) } } @@ -465,13 +494,20 @@ func testHandlerPauseAndResume(t *testing.T, i int) { t.Logf("iteration %d", i) var ( - err error - amqp = amqpx.New() - cctx, cancel = signal.NotifyContext(context.TODO(), syscall.SIGINT, syscall.SIGINT) - funcName = testutils.FuncName(3) + fmt.Sprintf("-i-%d", i) - log = logging.NewTestLogger(t) + amqp = amqpx.New() + log = logging.NewTestLogger(t) + cctx, cancel = signal.NotifyContext(context.TODO(), syscall.SIGINT, syscall.SIGINT) + funcName = fmt.Sprintf("%s-%d", testutils.CallerFuncName(), i) + nextExchangeQueue = testutils.NewExchangeQueueGenerator(funcName) + eq1 = nextExchangeQueue() + eq2 = nextExchangeQueue() + eq3 = nextExchangeQueue() ) defer cancel() + defer func() { + log.Info("closing amqp") + assert.NoError(t, amqp.Close()) + }() options := []amqpx.Option{ amqpx.WithLogger(logging.NewNoOpLogger()), @@ -479,30 +515,30 @@ func testHandlerPauseAndResume(t *testing.T, i int) { amqpx.WithPublisherSessions(5), } - amqpxPublish := amqpx.New() - amqpxPublish.RegisterTopologyCreator(createTopology(funcName)) + amqpPub := amqpx.New() + amqpPub.RegisterTopologyCreator(createTopology(log, eq1, eq2, eq3)) + amqp.RegisterTopologyDeleter(deleteTopology(log, eq1, eq2, eq3)) + defer func() { + assert.NoError(t, amqpPub.Close()) + }() - eventContent := "TestHandlerPauseAndResume - event content" + err := amqpPub.Start( + cctx, + testutils.HealthyConnectURL, + append(options, amqpx.WithName(funcName+"-pub"))..., + ) + require.NoError(t, err) var ( publish = 500 cnt = 0 ) - // step 1 - fill queue with messages - amqp.RegisterTopologyDeleter(deleteTopology(funcName)) - err = amqpxPublish.Start( - cctx, - funcName, - options..., - ) - require.NoError(t, err) - // fill queue with messages for i := 0; i < publish; i++ { - err := amqpxPublish.Publish(cctx, funcName+"exchange-01", "event-01", pool.Publishing{ + err := amqpPub.Publish(cctx, eq1.Exchange, eq1.RoutingKey, pool.Publishing{ ContentType: "text/plain", - Body: []byte(fmt.Sprintf("%s: message number %d", eventContent, i)), + Body: []byte(eq1.NextPubMsg()), }) if err != nil { assert.NoError(t, err) @@ -511,12 +547,14 @@ func testHandlerPauseAndResume(t *testing.T, i int) { } // step 2 - process messages, pause, wait, resume, process rest, cancel context - handler01 := amqp.RegisterHandler(funcName+"queue-01", func(ctx context.Context, msg pool.Delivery) (err error) { + handler01 := amqp.RegisterHandler(eq1.Queue, func(_ context.Context, msg pool.Delivery) (err error) { + assert.Equal(t, eq1.NextSubMsg(), string(msg.Body)) + cnt++ if cnt == publish/3 || cnt == publish/3*2 { - err = amqp.Publish(ctx, funcName+"exchange-02", "event-02", pool.Publishing{ + err = amqp.Publish(cctx, eq2.Exchange, eq2.RoutingKey, pool.Publishing{ ContentType: "text/plain", - Body: []byte(fmt.Sprintf("%s: hit %d messages, toggling processing", eventContent, cnt)), + Body: []byte(eq2.NextPubMsg()), }) assert.NoError(t, err) } @@ -525,7 +563,8 @@ func testHandlerPauseAndResume(t *testing.T, i int) { }) running := true - amqp.RegisterHandler(funcName+"queue-02", func(ctx context.Context, msg pool.Delivery) (err error) { + amqp.RegisterHandler(eq2.Queue, func(_ context.Context, msg pool.Delivery) (err error) { + assert.Equal(t, eq2.NextSubMsg(), string(msg.Body)) log.Infof("received toggle request: %s", string(msg.Body)) queue := handler01.Queue() @@ -534,7 +573,7 @@ func testHandlerPauseAndResume(t *testing.T, i int) { assertActive(t, handler01, true) - err = handler01.Pause(context.Background()) + err = handler01.Pause(cctx) assert.NoError(t, err) assertActive(t, handler01, false) @@ -543,16 +582,16 @@ func testHandlerPauseAndResume(t *testing.T, i int) { assertActive(t, handler01, false) - err = handler01.Resume(context.Background()) + err = handler01.Resume(cctx) assert.NoError(t, err) log.Infof("resumed processing of %s", queue) assertActive(t, handler01, true) // trigger cancelation - err = amqpxPublish.Publish(ctx, funcName+"exchange-03", "event-03", pool.Publishing{ + err = amqpPub.Publish(cctx, eq3.Exchange, eq3.RoutingKey, pool.Publishing{ ContentType: "text/plain", - Body: []byte(fmt.Sprintf("%s: delayed toggle back", eventContent)), + Body: []byte(eq3.NextPubMsg()), }) assert.NoError(t, err) } @@ -560,12 +599,12 @@ func testHandlerPauseAndResume(t *testing.T, i int) { }) var once sync.Once - amqp.RegisterHandler(funcName+"queue-03", func(ctx context.Context, msg pool.Delivery) (err error) { + amqp.RegisterHandler(eq3.Queue, func(_ context.Context, msg pool.Delivery) (err error) { once.Do(func() { log.Info("pausing handler") assertActive(t, handler01, true) - err = handler01.Pause(ctx) + err = handler01.Pause(cctx) if err != nil { assert.NoError(t, err) return @@ -586,9 +625,7 @@ func testHandlerPauseAndResume(t *testing.T, i int) { err = amqp.Start( cctx, testutils.HealthyConnectURL, - amqpx.WithLogger(logging.NewNoOpLogger()), - amqpx.WithPublisherConnections(1), - amqpx.WithPublisherSessions(5), + append(options, amqpx.WithName(funcName))..., ) if err != nil { assert.NoError(t, err) @@ -603,8 +640,16 @@ func testHandlerPauseAndResume(t *testing.T, i int) { } func TestBatchHandlerPauseAndResume(t *testing.T) { + t.Parallel() + + var wg sync.WaitGroup + defer wg.Wait() for i := 0; i < 10; i++ { - testBatchHandlerPauseAndResume(t, i) + wg.Add(1) + go func(i int) { + defer wg.Done() + testBatchHandlerPauseAndResume(t, i) + }(i) } } @@ -612,12 +657,18 @@ func testBatchHandlerPauseAndResume(t *testing.T, i int) { t.Logf("iteration %d", i) var ( - err error - amqp = amqpx.New() - cctx, cancel = signal.NotifyContext(context.TODO(), syscall.SIGINT, syscall.SIGINT) - funcName = testutils.FuncName(3) + fmt.Sprintf("-i-%d", i) - log = logging.NewTestLogger(t) + amqp = amqpx.New() + log = logging.NewTestLogger(t) + cctx, cancel = signal.NotifyContext(context.TODO(), syscall.SIGINT, syscall.SIGINT) + funcName = fmt.Sprintf("%s-%d", testutils.CallerFuncName(), i) + nextExchangeQueue = testutils.NewExchangeQueueGenerator(funcName) + eq1 = nextExchangeQueue() + eq2 = nextExchangeQueue() + eq3 = nextExchangeQueue() ) + defer func() { + assert.NoError(t, amqp.Close()) + }() defer cancel() options := []amqpx.Option{ @@ -627,29 +678,27 @@ func testBatchHandlerPauseAndResume(t *testing.T, i int) { } amqpxPublish := amqpx.New() - amqpxPublish.RegisterTopologyCreator(createTopology(funcName)) - err = amqpxPublish.Start( + amqpxPublish.RegisterTopologyCreator(createTopology(log, eq1, eq2, eq3)) + err := amqpxPublish.Start( cctx, testutils.HealthyConnectURL, - options..., + append(options, amqpx.WithName(funcName+"-pub"))..., ) require.NoError(t, err) - eventContent := funcName + " - event content" - var ( publish = 500 cnt = 0 ) // step 1 - fill queue with messages - amqp.RegisterTopologyDeleter(deleteTopology(funcName)) + amqp.RegisterTopologyDeleter(deleteTopology(log, eq1, eq2, eq3)) // fill queue with messages for i := 0; i < publish; i++ { - err := amqpxPublish.Publish(cctx, funcName+"exchange-01", "event-01", pool.Publishing{ + err := amqpxPublish.Publish(cctx, eq1.Exchange, eq1.RoutingKey, pool.Publishing{ ContentType: "text/plain", - Body: []byte(fmt.Sprintf("%s: message number %d", eventContent, i)), + Body: []byte(eq1.NextPubMsg()), }) if err != nil { assert.NoError(t, err) @@ -658,13 +707,14 @@ func testBatchHandlerPauseAndResume(t *testing.T, i int) { } // step 2 - process messages, pause, wait, resume, process rest, cancel context - handler01 := amqp.RegisterBatchHandler(funcName+"queue-01", func(ctx context.Context, msgs []pool.Delivery) (err error) { + handler01 := amqp.RegisterBatchHandler(eq1.Queue, func(_ context.Context, msgs []pool.Delivery) (err error) { for _, msg := range msgs { + assert.Equal(t, eq1.NextSubMsg(), string(msg.Body)) cnt++ if cnt == publish/3 || cnt == publish/3*2 { - err = amqp.Publish(ctx, funcName+"exchange-02", "event-02", pool.Publishing{ + err = amqp.Publish(cctx, eq2.Exchange, eq2.RoutingKey, pool.Publishing{ ContentType: "text/plain", - Body: []byte(fmt.Sprintf("%s: hit %d messages, toggling processing: %s", eventContent, cnt, string(msg.Body))), + Body: []byte(eq2.NextPubMsg()), }) assert.NoError(t, err) } @@ -673,7 +723,7 @@ func testBatchHandlerPauseAndResume(t *testing.T, i int) { }) running := true - amqp.RegisterBatchHandler(funcName+"queue-02", func(ctx context.Context, msgs []pool.Delivery) (err error) { + amqp.RegisterBatchHandler(eq2.Queue, func(_ context.Context, msgs []pool.Delivery) (err error) { queue := handler01.Queue() for _, msg := range msgs { @@ -683,7 +733,7 @@ func testBatchHandlerPauseAndResume(t *testing.T, i int) { assertActive(t, handler01, true) - err = handler01.Pause(ctx) + err = handler01.Pause(cctx) assert.NoError(t, err) assertActive(t, handler01, false) @@ -692,16 +742,16 @@ func testBatchHandlerPauseAndResume(t *testing.T, i int) { assertActive(t, handler01, false) - err = handler01.Resume(ctx) + err = handler01.Resume(cctx) assert.NoError(t, err) log.Infof("resumed processing of %s", queue) assertActive(t, handler01, true) // trigger cancelation - err = amqpxPublish.Publish(ctx, funcName+"exchange-03", "event-03", pool.Publishing{ + err = amqpxPublish.Publish(cctx, eq3.Exchange, eq3.RoutingKey, pool.Publishing{ ContentType: "text/plain", - Body: []byte(fmt.Sprintf("%s: delayed toggle back", eventContent)), + Body: []byte(eq3.NextPubMsg()), }) assert.NoError(t, err) } @@ -710,12 +760,12 @@ func testBatchHandlerPauseAndResume(t *testing.T, i int) { }) var once sync.Once - amqp.RegisterBatchHandler(funcName+"queue-03", func(ctx context.Context, msgs []pool.Delivery) (err error) { + amqp.RegisterBatchHandler(eq3.Queue, func(_ context.Context, msgs []pool.Delivery) (err error) { _ = msgs[0] once.Do(func() { assertActive(t, handler01, true) - err = handler01.Pause(ctx) + err = handler01.Pause(cctx) if err != nil { assert.NoError(t, err) return @@ -737,9 +787,7 @@ func testBatchHandlerPauseAndResume(t *testing.T, i int) { err = amqp.Start( cctx, testutils.HealthyConnectURL, - amqpx.WithLogger(logging.NewNoOpLogger()), - amqpx.WithPublisherConnections(1), - amqpx.WithPublisherSessions(5), + append(options, amqpx.WithName(funcName))..., ) if err != nil { assert.NoError(t, err) @@ -754,6 +802,8 @@ func testBatchHandlerPauseAndResume(t *testing.T, i int) { } func TestQueueDeletedConsumerReconnect(t *testing.T) { + t.Parallel() + var ( err error amqp = amqpx.New() @@ -796,6 +846,7 @@ func TestQueueDeletedConsumerReconnect(t *testing.T) { err = amqp.Start( cctx, testutils.HealthyConnectURL, + amqpx.WithName(funcName), amqpx.WithLogger(log), ) if err != nil { @@ -826,6 +877,8 @@ func TestQueueDeletedConsumerReconnect(t *testing.T) { } func TestQueueDeletedBatchConsumerReconnect(t *testing.T) { + t.Parallel() + var ( err error amqp = amqpx.New() @@ -868,6 +921,7 @@ func TestQueueDeletedBatchConsumerReconnect(t *testing.T) { err = amqp.Start( cctx, testutils.HealthyConnectURL, + amqpx.WithName(funcName), amqpx.WithLogger(log), ) if err != nil { @@ -905,6 +959,7 @@ func newTransientSession(t *testing.T, ctx context.Context, connectUrl string) ( require.NoError(t, err) return s, func() { + p.ReturnSession(s, nil) err = s.Close() assert.NoError(t, err) p.Close() @@ -918,6 +973,8 @@ type handlerStats interface { } func TestHandlerReset(t *testing.T) { + t.Parallel() + for i := 0; i < 5; i++ { testHandlerReset(t, i) } @@ -926,12 +983,19 @@ func TestHandlerReset(t *testing.T) { func testHandlerReset(t *testing.T, i int) { var ( - err error - amqp = amqpx.New() - log = logging.NewTestLogger(t) - cctx, cancel = signal.NotifyContext(context.TODO(), syscall.SIGINT, syscall.SIGINT) - funcName = testutils.FuncName(3) + fmt.Sprintf("-i-%d", i) + err error + amqp = amqpx.New() + log = logging.NewTestLogger(t) + cctx, cancel = signal.NotifyContext(context.TODO(), syscall.SIGINT, syscall.SIGINT) + funcName = fmt.Sprintf("%s-i-%d", testutils.CallerFuncName(), i) + nextExchangeQueue = testutils.NewExchangeQueueGenerator(funcName) + eq1 = nextExchangeQueue() + eq2 = nextExchangeQueue() + eq3 = nextExchangeQueue() ) + defer func() { + assert.NoError(t, amqp.Close()) + }() defer cancel() options := []amqpx.Option{ @@ -941,29 +1005,27 @@ func testHandlerReset(t *testing.T, i int) { } amqpxPublish := amqpx.New() - amqpxPublish.RegisterTopologyCreator(createTopology(funcName)) + amqpxPublish.RegisterTopologyCreator(createTopology(log, eq1, eq2, eq3)) err = amqpxPublish.Start( cctx, testutils.HealthyConnectURL, - options..., + append(options, amqpx.WithName(funcName+"-pub"))..., ) require.NoError(t, err) - eventContent := funcName + " - event content" - var ( publish = 50 cnt = 0 ) // step 1 - fill queue with messages - amqp.RegisterTopologyDeleter(deleteTopology(funcName)) + amqp.RegisterTopologyDeleter(deleteTopology(log, eq1, eq2, eq3)) // fill queue with messages for i := 0; i < publish; i++ { - err := amqpxPublish.Publish(cctx, funcName+"exchange-01", "event-01", pool.Publishing{ + err := amqpxPublish.Publish(cctx, eq1.Exchange, eq1.RoutingKey, pool.Publishing{ ContentType: "text/plain", - Body: []byte(fmt.Sprintf("%s: message number %d", eventContent, i)), + Body: []byte(eq1.NextPubMsg()), }) if err != nil { assert.NoError(t, err) @@ -973,7 +1035,7 @@ func testHandlerReset(t *testing.T, i int) { done := make(chan struct{}) // step 2 - process messages, pause, wait, resume, process rest, cancel context - handler01 := amqp.RegisterHandler(funcName+"queue-01", func(ctx context.Context, msgs pool.Delivery) (err error) { + handler01 := amqp.RegisterHandler(eq1.Queue, func(_ context.Context, msgs pool.Delivery) (err error) { cnt++ if cnt == publish { close(done) @@ -986,9 +1048,7 @@ func testHandlerReset(t *testing.T, i int) { err = amqp.Start( cctx, testutils.HealthyConnectURL, - amqpx.WithLogger(logging.NewNoOpLogger()), - amqpx.WithPublisherConnections(1), - amqpx.WithPublisherSessions(5), + append(options, amqpx.WithName(funcName))..., ) if err != nil { assert.NoError(t, err) @@ -1010,6 +1070,8 @@ func testHandlerReset(t *testing.T, i int) { } func TestBatchHandlerReset(t *testing.T) { + t.Parallel() + for i := 0; i < 5; i++ { testBatchHandlerReset(t, i) } @@ -1018,12 +1080,19 @@ func TestBatchHandlerReset(t *testing.T) { func testBatchHandlerReset(t *testing.T, i int) { var ( - err error - amqp = amqpx.New() - log = logging.NewTestLogger(t) - cctx, cancel = signal.NotifyContext(context.TODO(), syscall.SIGINT, syscall.SIGINT) - funcName = testutils.FuncName(3) + fmt.Sprintf("-i-%d", i) + err error + amqp = amqpx.New() + log = logging.NewTestLogger(t) + cctx, cancel = signal.NotifyContext(context.TODO(), syscall.SIGINT, syscall.SIGINT) + funcName = fmt.Sprintf("%s-i-%d", testutils.CallerFuncName(), i) + nextExchangeQueue = testutils.NewExchangeQueueGenerator(funcName) + eq1 = nextExchangeQueue() + eq2 = nextExchangeQueue() + eq3 = nextExchangeQueue() ) + defer func() { + assert.NoError(t, amqp.Close()) + }() defer cancel() options := []amqpx.Option{ @@ -1033,29 +1102,27 @@ func testBatchHandlerReset(t *testing.T, i int) { } amqpxPublish := amqpx.New() - amqpxPublish.RegisterTopologyCreator(createTopology(funcName)) + amqpxPublish.RegisterTopologyCreator(createTopology(log, eq1, eq2, eq3)) err = amqpxPublish.Start( cctx, testutils.HealthyConnectURL, - options..., + append(options, amqpx.WithName(funcName+"-pub"))..., ) require.NoError(t, err) - eventContent := "TestBatchHandlerReset - event content" - var ( publish = 50 cnt = 0 ) // step 1 - fill queue with messages - amqp.RegisterTopologyDeleter(deleteTopology(funcName)) + amqp.RegisterTopologyDeleter(deleteTopology(log, eq1, eq2, eq3)) // fill queue with messages for i := 0; i < publish; i++ { - err := amqpxPublish.Publish(cctx, funcName+"exchange-01", "event-01", pool.Publishing{ + err := amqpxPublish.Publish(cctx, eq1.Exchange, eq1.RoutingKey, pool.Publishing{ ContentType: "text/plain", - Body: []byte(fmt.Sprintf("%s: message number %d", eventContent, i)), + Body: []byte(eq1.NextPubMsg()), }) if err != nil { assert.NoError(t, err) @@ -1065,7 +1132,7 @@ func testBatchHandlerReset(t *testing.T, i int) { done := make(chan struct{}) // step 2 - process messages, pause, wait, resume, process rest, cancel context - handler01 := amqp.RegisterBatchHandler(funcName+"queue-01", func(ctx context.Context, msgs []pool.Delivery) (err error) { + handler01 := amqp.RegisterBatchHandler(eq1.Queue, func(_ context.Context, msgs []pool.Delivery) (err error) { cnt += len(msgs) if cnt == publish { @@ -1079,9 +1146,7 @@ func testBatchHandlerReset(t *testing.T, i int) { err = amqp.Start( cctx, testutils.HealthyConnectURL, - amqpx.WithLogger(logging.NewNoOpLogger()), - amqpx.WithPublisherConnections(1), - amqpx.WithPublisherSessions(5), + append(options, amqpx.WithName(funcName))..., ) if err != nil { assert.NoError(t, err) @@ -1103,7 +1168,10 @@ func testBatchHandlerReset(t *testing.T, i int) { } func assertActive(t *testing.T, handler handlerStats, expected bool) { - active, err := handler.IsActive(context.Background()) + cctx, cancel := context.WithTimeout(context.Background(), 60*time.Second) + defer cancel() + + active, err := handler.IsActive(cctx) if err != nil { assert.NoError(t, err) return diff --git a/helpers_test.go b/helpers_test.go index b6d618e..e62989f 100644 --- a/helpers_test.go +++ b/helpers_test.go @@ -6,97 +6,77 @@ import ( "fmt" "github.com/jxsl13/amqpx" + "github.com/jxsl13/amqpx/internal/testutils" + "github.com/jxsl13/amqpx/logging" "github.com/jxsl13/amqpx/pool" ) -func createTopology(prefix string) amqpx.TopologyFunc { - return func(ctx context.Context, t *pool.Topologer) (err error) { - // documentation: https://www.cloudamqp.com/blog/part4-rabbitmq-for-beginners-exchanges-routing-keys-bindings.html#:~:text=The%20routing%20key%20is%20a%20message%20attribute%20added%20to%20the,routing%20key%20of%20the%20message. - - err = createExchange(ctx, prefix+"exchange-01", t) - if err != nil { - return err - } +func declareExchangeQueue(ctx context.Context, eq testutils.ExchangeQueue, t *pool.Topologer, log logging.Logger) (err error) { + err = createExchange(ctx, eq.Exchange, t, log) + if err != nil { + return err + } - err = createQueue(ctx, prefix+"queue-01", t) - if err != nil { - return err - } + err = createQueue(ctx, eq.Queue, t, log) + if err != nil { + return err + } - err = t.QueueBind(ctx, prefix+"queue-01", "event-01", prefix+"exchange-01") - if err != nil { - return err - } + err = bindQueue(ctx, eq.Queue, eq.Exchange, eq.RoutingKey, t, log) + if err != nil { + return err + } + return nil +} - err = createExchange(ctx, prefix+"exchange-02", t) - if err != nil { - return err - } +func deleteExchangeQueue(ctx context.Context, eq testutils.ExchangeQueue, t *pool.Topologer, log logging.Logger) (err error) { + err = unbindQueue(ctx, eq.Queue, eq.Exchange, eq.RoutingKey, t, log) + if err != nil { + return err + } - err = createQueue(ctx, prefix+"queue-02", t) - if err != nil { - return err - } + err = deleteQueue(ctx, eq.Queue, t, log) + if err != nil { + return err + } - err = t.QueueBind(ctx, prefix+"queue-02", "event-02", prefix+"exchange-02") - if err != nil { - return err - } + err = deleteExchange(ctx, eq.Exchange, t, log) + if err != nil { + return err + } + return nil +} - err = createExchange(ctx, prefix+"exchange-03", t) - if err != nil { - return err - } +func createTopology(log logging.Logger, eqs ...testutils.ExchangeQueue) amqpx.TopologyFunc { + return func(ctx context.Context, t *pool.Topologer) (err error) { + // documentation: https://www.cloudamqp.com/blog/part4-rabbitmq-for-beginners-exchanges-routing-keys-bindings.html#:~:text=The%20routing%20key%20is%20a%20message%20attribute%20added%20to%20the,routing%20key%20of%20the%20message. - err = createQueue(ctx, prefix+"queue-03", t) - if err != nil { - return err - } - err = t.QueueBind(ctx, prefix+"queue-03", "event-03", prefix+"exchange-03") - if err != nil { - return err + for _, eq := range eqs { + err = declareExchangeQueue(ctx, eq, t, log) + if err != nil { + return err + } } return nil } } -func deleteTopology(prefix string) amqpx.TopologyFunc { +func deleteTopology(log logging.Logger, eqs ...testutils.ExchangeQueue) amqpx.TopologyFunc { return func(ctx context.Context, t *pool.Topologer) (err error) { - err = deleteQueue(ctx, prefix+"queue-01", t) - if err != nil { - return err - } - - err = deleteQueue(ctx, prefix+"queue-02", t) - if err != nil { - return err - } - - err = deleteQueue(ctx, prefix+"queue-03", t) - if err != nil { - return err - } - - err = deleteExchange(ctx, prefix+"exchange-01", t) - if err != nil { - return err - } - - err = deleteExchange(ctx, prefix+"exchange-02", t) - if err != nil { - return err - } - err = deleteExchange(ctx, prefix+"exchange-03", t) - if err != nil { - return err + for _, eq := range eqs { + err = deleteExchangeQueue(ctx, eq, t, log) + if err != nil { + return err + } } - return nil } } -func createQueue(ctx context.Context, name string, t *pool.Topologer) (err error) { +func createQueue(ctx context.Context, name string, t *pool.Topologer, log logging.Logger) (err error) { + log.Infof("topology: creating queue: %s", name) + _, err = t.QueueDeclarePassive(ctx, name) if !errors.Is(err, pool.ErrNotFound) { if err != nil { @@ -117,7 +97,9 @@ func createQueue(ctx context.Context, name string, t *pool.Topologer) (err error return nil } -func deleteQueue(ctx context.Context, name string, t *pool.Topologer) (err error) { +func deleteQueue(ctx context.Context, name string, t *pool.Topologer, log logging.Logger) (err error) { + log.Infof("topology: deleting queue: %s", name) + _, err = t.QueueDeclarePassive(ctx, name) if err != nil { return fmt.Errorf("%q does not exist but is supposed to be deleted: %w", name, err) @@ -135,7 +117,9 @@ func deleteQueue(ctx context.Context, name string, t *pool.Topologer) (err error return nil } -func createExchange(ctx context.Context, name string, t *pool.Topologer) (err error) { +func createExchange(ctx context.Context, name string, t *pool.Topologer, log logging.Logger) (err error) { + log.Infof("topology: creating exchange: %s", name) + err = t.ExchangeDeclarePassive(ctx, name, pool.ExchangeKindTopic) if !errors.Is(err, pool.ErrNotFound) { if err != nil { @@ -156,7 +140,9 @@ func createExchange(ctx context.Context, name string, t *pool.Topologer) (err er return nil } -func deleteExchange(ctx context.Context, name string, t *pool.Topologer) (err error) { +func deleteExchange(ctx context.Context, name string, t *pool.Topologer, log logging.Logger) (err error) { + log.Infof("topology: deleting exchange: %s", name) + err = t.ExchangeDeclarePassive(ctx, name, pool.ExchangeKindTopic) if err != nil { return fmt.Errorf("exchange %s was not found even tho it should exist: %w", name, err) @@ -173,3 +159,21 @@ func deleteExchange(ctx context.Context, name string, t *pool.Topologer) (err er } return nil } + +func bindQueue(ctx context.Context, queue, exchange, routingKey string, t *pool.Topologer, log logging.Logger) (err error) { + log.Infof("topology: binding queue %s to exchange %s with routing key: %s", queue, exchange, routingKey) + err = t.QueueBind(ctx, queue, routingKey, exchange) + if err != nil { + return err + } + return nil +} + +func unbindQueue(ctx context.Context, queue, exchange, routingKey string, t *pool.Topologer, log logging.Logger) (err error) { + log.Infof("topology: unbinding queue %s from exchange %s with routing key: %s", queue, exchange, routingKey) + err = t.QueueUnbind(ctx, queue, routingKey, exchange) + if err != nil { + return err + } + return nil +} diff --git a/internal/testutils/generator.go b/internal/testutils/generator.go index f74980d..eee3d34 100644 --- a/internal/testutils/generator.go +++ b/internal/testutils/generator.go @@ -50,6 +50,29 @@ func WithSuffix(suffix string) GeneratorOption { } } +func RoutingKeyGenerator(sessionName string, options ...GeneratorOption) (nextRoutingKey func() string) { + opts := generatorOptions{ + prefix: "", + randomSuffix: false, + + up: 2, + } + + for _, opt := range options { + opt(&opts) + } + + var mu sync.Mutex + var counter int64 + return func() string { + mu.Lock() + cnt := counter + counter++ + mu.Unlock() + return fmt.Sprintf("%s-%srouting-key-%d%s", sessionName, opts.prefix, cnt, opts.ToSuffix()) + } +} + func ExchangeNameGenerator(sessionName string, options ...GeneratorOption) (nextExchangeName func() string) { opts := generatorOptions{ prefix: "", @@ -221,3 +244,36 @@ func MessageGenerator(queueOrExchangeName string, options ...GeneratorOption) (n return fmt.Sprintf("%s-message-%d%s", queueOrExchangeName, cnt, opts.ToSuffix()) } } + +func NewExchangeQueueGenerator(funcName string) func() ExchangeQueue { + var ( + nextExchangeName = ExchangeNameGenerator(funcName) + nextQueueName = QueueNameGenerator(funcName) + nextRoutingKey = RoutingKeyGenerator(funcName) + ) + return func() ExchangeQueue { + return NewExchangeQueue(nextExchangeName(), nextQueueName(), nextRoutingKey()) + } +} + +func NewExchangeQueue(exchange, queue, routingKey string) ExchangeQueue { + return ExchangeQueue{ + Exchange: exchange, + Queue: queue, + RoutingKey: routingKey, + ConsumerTag: ConsumerNameGenerator(queue)(), // generate one consumer name + NextPubMsg: MessageGenerator(exchange), + NextSubMsg: MessageGenerator(exchange), + } +} + +type ExchangeQueue struct { + Exchange string + Queue string + RoutingKey string + + ConsumerTag string + + NextPubMsg func() string + NextSubMsg func() string +} From eb0745afb2ecf74debd5ba25a8eb4a412cba16de Mon Sep 17 00:00:00 2001 From: John Behm Date: Thu, 14 Mar 2024 15:53:52 +0100 Subject: [PATCH 51/76] rename *Size methods to *Capacity and make Size methods represent the current number of idle connections or sessions on the pools --- amqpx_options.go | 6 ++--- pool/connection_pool.go | 33 +++++++++++++------------- pool/connection_pool_options.go | 2 +- pool/pool.go | 26 +++++++++++++++------ pool/pool_options.go | 6 ++--- pool/session.go | 32 ++++++++++++------------- pool/session_options.go | 16 ++++++------- pool/session_pool.go | 41 ++++++++++++++++++--------------- pool/session_pool_options.go | 16 ++++++------- pool/topologer.go | 2 -- 10 files changed, 98 insertions(+), 82 deletions(-) diff --git a/amqpx_options.go b/amqpx_options.go index 9b14431..ab1ba61 100644 --- a/amqpx_options.go +++ b/amqpx_options.go @@ -57,11 +57,11 @@ func WithTLS(config *tls.Config) Option { } } -// WithBufferSize allows to configurethe size of +// WithBufferCapacity allows to configurethe size of // the confirmation, error & blocker buffers of all sessions -func WithBufferSize(size int) Option { +func WithBufferCapacity(capacity int) Option { return func(o *option) { - o.PoolOptions = append(o.PoolOptions, pool.WithBufferSize(size)) + o.PoolOptions = append(o.PoolOptions, pool.WithBufferCapacity(capacity)) } } diff --git a/pool/connection_pool.go b/pool/connection_pool.go index 7954d08..8f910c1 100644 --- a/pool/connection_pool.go +++ b/pool/connection_pool.go @@ -22,7 +22,7 @@ type ConnectionPool struct { heartbeat time.Duration connTimeout time.Duration - size int + capacity int tls *tls.Config @@ -49,8 +49,8 @@ func NewConnectionPool(ctx context.Context, connectUrl string, numConns int, opt // use sane defaults option := connectionPoolOption{ - Name: defaultAppName(), - Size: numConns, + Name: defaultAppName(), + Capacity: numConns, Ctx: ctx, @@ -92,9 +92,9 @@ func newConnectionPoolFromOption(connectUrl string, option connectionPoolOption) heartbeat: option.ConnHeartbeatInterval, connTimeout: option.ConnTimeout, - size: option.Size, + capacity: option.Capacity, tls: option.TLSConfig, - connections: make(chan *Connection, option.Size), + connections: make(chan *Connection, option.Capacity), ctx: ctx, cancel: cancel, @@ -122,7 +122,7 @@ func newConnectionPoolFromOption(connectUrl string, option connectionPoolOption) } func (cp *ConnectionPool) initCachedConns() error { - for id := int64(0); id < int64(cp.size); id++ { + for id := int64(0); id < int64(cp.capacity); id++ { conn, err := cp.deriveConnection(cp.ctx, id, true) if err != nil { return fmt.Errorf("%w: %v", ErrPoolInitializationFailed, err) @@ -234,10 +234,10 @@ func (cp *ConnectionPool) Close() { defer cp.info("closed") wg := &sync.WaitGroup{} - wg.Add(cp.size) + wg.Add(cp.capacity) cp.cancel() - for i := 0; i < cp.size; i++ { + for i := 0; i < cp.capacity; i++ { go func() { defer wg.Done() conn := <-cp.connections @@ -255,19 +255,20 @@ func (cp *ConnectionPool) StatTransientActive() int { return cp.concurrentTransient } -// StatCachedIdle returns the number of idle cached connections. -func (cp *ConnectionPool) StatCachedIdle() int { - return len(cp.connections) -} - // StatCachedActive returns the number of active cached connections. func (cp *ConnectionPool) StatCachedActive() int { - return cp.size - len(cp.connections) + return cp.capacity - len(cp.connections) } -// Size is the total size of the cached connection pool without any transient connections. +// Size returns the number of idle cached connections. func (cp *ConnectionPool) Size() int { - return cp.size + return len(cp.connections) +} + +// Capacity is the capacity of the cached connection pool without any transient connections. +// It is the initial number of connections that were created for this connection pool. +func (cp *ConnectionPool) Capacity() int { + return cp.capacity } func (cp *ConnectionPool) catchShutdown() <-chan struct{} { diff --git a/pool/connection_pool_options.go b/pool/connection_pool_options.go index 15b9b42..086bb70 100644 --- a/pool/connection_pool_options.go +++ b/pool/connection_pool_options.go @@ -17,7 +17,7 @@ type connectionPoolOption struct { Name string Ctx context.Context - Size int + Capacity int ConnHeartbeatInterval time.Duration ConnTimeout time.Duration diff --git a/pool/pool.go b/pool/pool.go index 268ee5f..b95dd36 100644 --- a/pool/pool.go +++ b/pool/pool.go @@ -33,9 +33,9 @@ func New(ctx context.Context, connectUrl string, numConns, numSessions int, opti // use sane defaults option := poolOption{ cpo: connectionPoolOption{ - Name: defaultAppName(), - Ctx: ctx, - Size: numConns, // at least one connection + Name: defaultAppName(), + Ctx: ctx, + Capacity: numConns, // at least one connection ConnHeartbeatInterval: 15 * time.Second, ConnTimeout: 30 * time.Second, @@ -44,9 +44,9 @@ func New(ctx context.Context, connectUrl string, numConns, numSessions int, opti Logger: logger, }, spo: sessionPoolOption{ - Size: numSessions, - Confirmable: false, // require publish confirmations - BufferSize: 1, // fault tolerance over throughput + Capacity: numSessions, + Confirmable: true, // require publish confirmations + BufferCapacity: 10, Logger: logger, }, @@ -106,10 +106,22 @@ func (p *Pool) Name() string { return p.cp.Name() } +// ConnectionPoolCapacity returns the capacity of the connection pool. +func (p *Pool) ConnectionPoolCapacity() int { + return p.cp.Capacity() +} + +// ConnectionPoolSize returns the number of connections in the pool that are idling. func (p *Pool) ConnectionPoolSize() int { - return p.cp.Size() + return p.cp.Capacity() +} + +// SessionPoolCapacity returns the capacity of the session pool. +func (p *Pool) SessionPoolCapacity() int { + return p.sp.Capacity() } +// SessionPoolSize returns the number of sessions in the pool that are idling. func (p *Pool) SessionPoolSize() int { return p.sp.Size() } diff --git a/pool/pool_options.go b/pool/pool_options.go index f596abe..eb30230 100644 --- a/pool/pool_options.go +++ b/pool/pool_options.go @@ -67,11 +67,11 @@ func WithTLS(config *tls.Config) Option { } } -// WithBufferSize allows to configurethe size of +// WithBufferCapacity allows to configurethe size of // the confirmation, error & blocker buffers of all sessions -func WithBufferSize(size int) Option { +func WithBufferCapacity(size int) Option { return func(po *poolOption) { - SessionPoolWithBufferSize(size)(&po.spo) + SessionPoolWithBufferCapacity(size)(&po.spo) } } diff --git a/pool/session.go b/pool/session.go index 8272c07..d4e2f9b 100644 --- a/pool/session.go +++ b/pool/session.go @@ -17,11 +17,11 @@ const ( // Session is type Session struct { - name string - cached bool - flagged bool - confirmable bool - bufferSize int + name string + cached bool + flagged bool + confirmable bool + bufferCapacity int channel *amqp091.Channel returned chan amqp091.Return @@ -83,10 +83,10 @@ func NewSession(conn *Connection, name string, options ...SessionOption) (*Sessi // default values option := sessionOption{ - Logger: conn.log, // derive logger form connection - Cached: false, - Confirmable: false, - BufferSize: 100, + Logger: conn.log, // derive logger form connection + Cached: false, + Confirmable: false, + BufferCapacity: 10, // derive context from connection, as we are derived from the connection // so in case the connection is closed, we are closed as well. Ctx: conn.ctx, @@ -102,10 +102,10 @@ func NewSession(conn *Connection, name string, options ...SessionOption) (*Sessi cancel := toCancelFunc(fmt.Errorf("session %w", ErrClosed), cc) session := &Session{ - name: name, - cached: option.Cached, - confirmable: option.Confirmable, - bufferSize: option.BufferSize, + name: name, + cached: option.Cached, + confirmable: option.Confirmable, + bufferCapacity: option.BufferCapacity, consumers: map[string]bool{}, channel: nil, // will be created on connect @@ -261,18 +261,18 @@ func (s *Session) connect() (err error) { return fmt.Errorf("%v: %w", ErrConnectionFailed, err) } - s.errors = make(chan *amqp091.Error, s.bufferSize) + s.errors = make(chan *amqp091.Error, s.bufferCapacity) channel.NotifyClose(s.errors) if s.confirmable { - s.confirms = make(chan amqp091.Confirmation, s.bufferSize) + s.confirms = make(chan amqp091.Confirmation, s.bufferCapacity) channel.NotifyPublish(s.confirms) err = channel.Confirm(false) if err != nil { return err } - s.returned = make(chan amqp091.Return, s.bufferSize) + s.returned = make(chan amqp091.Return, s.bufferCapacity) channel.NotifyReturn(s.returned) } diff --git a/pool/session_options.go b/pool/session_options.go index 82032ac..b663d78 100644 --- a/pool/session_options.go +++ b/pool/session_options.go @@ -7,12 +7,12 @@ import ( ) type sessionOption struct { - Logger logging.Logger - Cached bool - Confirmable bool - BufferSize int - Ctx context.Context - AutoCloseConn bool + Logger logging.Logger + Cached bool + Confirmable bool + BufferCapacity int + Ctx context.Context + AutoCloseConn bool RecoverCallback SessionRetryCallback PublishRetryCallback SessionRetryCallback @@ -72,9 +72,9 @@ func SessionWithConfirms(requiresPublishConfirms bool) SessionOption { // SessionWithBufferSize allows to customize the size of th einternal channel buffers. // all buffers/channels are initialized with this size. (e.g. error or confirm channels) -func SessionWithBufferSize(size int) SessionOption { +func SessionWithBufferCapacity(capacity int) SessionOption { return func(so *sessionOption) { - so.BufferSize = size + so.BufferCapacity = capacity } } diff --git a/pool/session_pool.go b/pool/session_pool.go index 570e12c..00beecd 100644 --- a/pool/session_pool.go +++ b/pool/session_pool.go @@ -15,10 +15,10 @@ type SessionPool struct { transientID int64 - size int - bufferSize int - confirmable bool - sessions chan *Session + capacity int + bufferCapacity int + confirmable bool + sessions chan *Session ctx context.Context cancel context.CancelFunc @@ -52,11 +52,11 @@ func NewSessionPool(pool *ConnectionPool, numSessions int, options ...SessionPoo // use sane defaults option := sessionPoolOption{ - AutoClosePool: false, // caller owns the connection pool by default - Size: numSessions, - Confirmable: false, - BufferSize: 1, // fault tolerance over throughput - Logger: pool.log, // derive logger from connection pool + AutoClosePool: false, // caller owns the connection pool by default + Capacity: numSessions, + Confirmable: false, + BufferCapacity: 10, // fault tolerance over throughput + Logger: pool.log, // derive logger from connection pool } for _, o := range options { @@ -76,10 +76,10 @@ func newSessionPoolFromOption(pool *ConnectionPool, ctx context.Context, option pool: pool, autoCloseConnPool: option.AutoClosePool, - size: option.Size, - bufferSize: option.BufferSize, - confirmable: option.Confirmable, - sessions: make(chan *Session, option.Size), + capacity: option.Capacity, + bufferCapacity: option.BufferCapacity, + confirmable: option.Confirmable, + sessions: make(chan *Session, option.Capacity), ctx: ctx, cancel: cancel, @@ -124,7 +124,7 @@ func newSessionPoolFromOption(pool *ConnectionPool, ctx context.Context, option } func (sp *SessionPool) initCachedSessions() error { - for i := 0; i < sp.size; i++ { + for i := 0; i < sp.capacity; i++ { session, err := sp.initCachedSession(i) if err != nil { return err @@ -157,9 +157,14 @@ func (sp *SessionPool) initCachedSession(id int) (*Session, error) { } } -// Size returns the size of the session pool which indicate sthe number of available cached sessions. +// Size returns the number of available idle sessions in the pool. func (sp *SessionPool) Size() int { - return sp.size + return len(sp.sessions) +} + +// Capacity returns the size of the session pool which indicate t he number of available cached sessions. +func (sp *SessionPool) Capacity() int { + return sp.capacity } // GetSession gets a pooled session. @@ -209,7 +214,7 @@ func (sp *SessionPool) deriveSession(ctx context.Context, conn *Connection, id i return NewSession(conn, name, SessionWithContext(ctx), - SessionWithBufferSize(sp.size), + SessionWithBufferCapacity(sp.bufferCapacity), SessionWithCached(cached), SessionWithConfirms(sp.confirmable), SessionWithAutoCloseConnection(!cached), // only close transient connections @@ -280,7 +285,7 @@ func (sp *SessionPool) Close() { wg := &sync.WaitGroup{} // close all sessions: - for i := 0; i < sp.size; i++ { + for i := 0; i < sp.capacity; i++ { session := <-sp.sessions wg.Add(1) go func(s *Session) { diff --git a/pool/session_pool_options.go b/pool/session_pool_options.go index c974c51..02b9be7 100644 --- a/pool/session_pool_options.go +++ b/pool/session_pool_options.go @@ -5,9 +5,9 @@ import ( ) type sessionPoolOption struct { - Size int - Confirmable bool // whether published messages require confirmation awaiting - BufferSize int // size of the sessio internal confirmation and error buffers. + Capacity int + Confirmable bool // whether published messages require awaiting confirmations. + BufferCapacity int // size of the session internal confirmation and error buffers. AutoClosePool bool // whether to close the internal connection pool automatically Logger logging.Logger @@ -41,14 +41,14 @@ func SessionPoolWithLogger(logger logging.Logger) SessionPoolOption { } } -// SessionPoolWithBufferSize allows to configurethe size of +// SessionPoolWithBufferCapacity allows to configure the size of // the confirmation, error & blocker buffers of all sessions -func SessionPoolWithBufferSize(size int) SessionPoolOption { - if size < 0 { - size = 0 +func SessionPoolWithBufferCapacity(capacity int) SessionPoolOption { + if capacity < 1 { + capacity = 1 // should be at least 1 in order not to create weiird deadlocks. } return func(po *sessionPoolOption) { - po.BufferSize = size + po.BufferCapacity = capacity } } diff --git a/pool/topologer.go b/pool/topologer.go index 58e608e..df4c3d2 100644 --- a/pool/topologer.go +++ b/pool/topologer.go @@ -36,8 +36,6 @@ func NewTopologer(p *Pool, options ...TopologerOption) *Topologer { return top } -// TODO: it should be possible to pass a custom context in here so that we can define -// timeouts, especially for a topology deleter which operates on a closed context and needs a new one. func (t *Topologer) getSession(ctx context.Context) (*Session, error) { if t.transientOnly || t.pool.SessionPoolSize() == 0 { From 9d0f9e91744ced25e6948847deaeefe7f7a4d1da Mon Sep 17 00:00:00 2001 From: John Behm Date: Thu, 14 Mar 2024 16:11:37 +0100 Subject: [PATCH 52/76] update readme examples --- README.md | 44 +++++++++++++++++++++++--------------------- 1 file changed, 23 insertions(+), 21 deletions(-) diff --git a/README.md b/README.md index b8af61a..91ff850 100644 --- a/README.md +++ b/README.md @@ -46,21 +46,21 @@ func main() { ctx, cancel := signal.NotifyContext(context.Background()) defer cancel() - amqpx.RegisterTopologyCreator(func(t *pool.Topologer) error { + amqpx.RegisterTopologyCreator(func(ctx context.Context, t *pool.Topologer) error { // error handling omitted for brevity - t.ExchangeDeclare("example-exchange", "topic") // durable exchange by default - t.QueueDeclare("example-queue") // durable quorum queue by default - t.QueueBind("example-queue", "route.name.v1.event", "example-exchange") + _ = t.ExchangeDeclare(ctx, "example-exchange", "topic") // durable exchange by default + _, _ = t.QueueDeclare(ctx, "example-queue") // durable quorum queue by default + _ = t.QueueBind(ctx, "example-queue", "route.name.v1.event", "example-exchange") return nil }) - amqpx.RegisterTopologyDeleter(func(t *pool.Topologer) error { + amqpx.RegisterTopologyDeleter(func(ctx context.Context, t *pool.Topologer) error { // error handling omitted for brevity - t.QueueDelete("example-queue") - t.ExchangeDelete("example-exchange") + _, _ = t.QueueDelete(ctx, "example-queue") + _ = t.ExchangeDelete(ctx, "example-exchange") return nil }) - amqpx.RegisterHandler("example-queue", func(msg pool.Delivery) error { + amqpx.RegisterHandler("example-queue", func(ctx context.Context, msg pool.Delivery) error { fmt.Println("received message:", string(msg.Body)) fmt.Println("canceling context") cancel() @@ -70,12 +70,13 @@ func main() { }) amqpx.Start( + ctx, amqpx.NewURL("localhost", 5672, "admin", "password"), // or amqp://username@password:localhost:5672 amqpx.WithLogger(logging.NewNoOpLogger()), // provide a logger that implements the logging.Logger interface ) defer amqpx.Close() - amqpx.Publish("example-exchange", "route.name.v1.event", pool.Publishing{ + _ = amqpx.Publish(ctx, "example-exchange", "route.name.v1.event", pool.Publishing{ ContentType: "application/json", Body: []byte("my test event"), }) @@ -101,8 +102,8 @@ import ( "github.com/jxsl13/amqpx/logging" ) -func ExampleConsumer(cancel func()) amqpx.HandlerFunc { - return func(msg amqpx.Delivery) error { +func SomeConsumer(cancel func()) pool.HandlerFunc { + return func(ctx context.Context, msg pool.Delivery) error { fmt.Println("received message:", string(msg.Body)) fmt.Println("canceling context") cancel() @@ -116,45 +117,46 @@ func main() { ctx, cancel := signal.NotifyContext(context.Background()) defer cancel() - amqpx.RegisterTopologyCreator(func(t *pool.Topologer) error { + amqpx.RegisterTopologyCreator(func(ctx context.Context, t *pool.Topologer) error { // error handling omitted for brevity - t.ExchangeDeclare("example-exchange", "topic", + _ = t.ExchangeDeclare(ctx, "example-exchange", "topic", pool.ExchangeDeclareOptions{ Durable: true, }, ) - t.QueueDeclare("example-queue", + _, _ = t.QueueDeclare(ctx, "example-queue", pool.QueueDeclareOptions{ Durable: true, Args: pool.QuorumQueue, }, ) - t.QueueBind("example-queue", "route.name.v1.event", "example-exchange") + t.QueueBind(ctx, "example-queue", "route.name.v1.event", "example-exchange") return nil }) - amqpx.RegisterTopologyDeleter(func(t *amqpx.Topologer) error { + amqpx.RegisterTopologyDeleter(func(ctx context.Context, t *pool.Topologer) error { // error handling omitted for brevity - t.QueueDelete("example-queue") - t.ExchangeDelete("example-exchange") + _, _ = t.QueueDelete(ctx, "example-queue") + _ = t.ExchangeDelete(ctx, "example-exchange") return nil }) amqpx.RegisterHandler("example-queue", - ExampleConsumer(cancel), + SomeConsumer(cancel), pool.ConsumeOptions{ ConsumerTag: "example-queue-cunsumer", Exclusive: true, }, ) - amqpx.Start( + _ = amqpx.Start( + ctx, amqpx.NewURL("localhost", 5672, "admin", "password"), // or amqp://username@password:localhost:5672 amqpx.WithLogger(logging.NewNoOpLogger()), // provide a logger that implements the logging.Logger interface (logrus adapter is provided) ) defer amqpx.Close() - amqpx.Publish("example-exchange", "route.name.v1.event", amqpx.Publishing{ + _ = amqpx.Publish(ctx, "example-exchange", "route.name.v1.event", pool.Publishing{ ContentType: "application/json", Body: []byte("my test event"), }) From f40c7471053024709111324b7e42030d9dd265f0 Mon Sep 17 00:00:00 2001 From: John Behm Date: Thu, 14 Mar 2024 16:12:34 +0100 Subject: [PATCH 53/76] remove reconnect attempt assertions in tests --- pool/connection_test.go | 14 ------------ pool/publisher_test.go | 6 ----- pool/session_test.go | 50 +---------------------------------------- pool/utils_test.go | 21 ----------------- 4 files changed, 1 insertion(+), 90 deletions(-) diff --git a/pool/connection_test.go b/pool/connection_test.go index 65ea351..54f8a6d 100644 --- a/pool/connection_test.go +++ b/pool/connection_test.go @@ -19,15 +19,11 @@ func TestNewSingleConnection(t *testing.T) { nextName = testutils.ConnectionNameGenerator() ) - reconnectCB, deferredAssert := AssertConnectionReconnectAttempts(t, 0) - defer deferredAssert() - c, err := pool.NewConnection( ctx, testutils.HealthyConnectURL, nextName(), pool.ConnectionWithLogger(logging.NewTestLogger(t)), - pool.ConnectionWithRecoverCallback(reconnectCB), ) if err != nil { @@ -54,14 +50,11 @@ func TestManyNewConnection(t *testing.T) { go func() { defer wg.Done() - reconnectCB, deferredAssert := AssertConnectionReconnectAttempts(t, 0) - defer deferredAssert() c, err := pool.NewConnection( ctx, testutils.HealthyConnectURL, nextName(), pool.ConnectionWithLogger(logging.NewTestLogger(t)), - pool.ConnectionWithRecoverCallback(reconnectCB), ) if err != nil { assert.NoError(t, err) @@ -88,9 +81,6 @@ func TestNewSingleConnectionWithDisconnect(t *testing.T) { nextName = testutils.ConnectionNameGenerator() ) - reconnectCB, deferredAssert := AssertConnectionReconnectAttempts(t, 1) - defer deferredAssert() - started, stopped := DisconnectWithStartedStopped(t, proxyName, 0, 0, 10*time.Second) started() defer stopped() @@ -100,7 +90,6 @@ func TestNewSingleConnectionWithDisconnect(t *testing.T) { connectURL, nextName(), pool.ConnectionWithLogger(logging.NewTestLogger(t)), - pool.ConnectionWithRecoverCallback(reconnectCB), ) if err != nil { assert.NoError(t, err) @@ -131,13 +120,10 @@ func TestManyNewConnectionWithDisconnect(t *testing.T) { go func() { defer wg.Done() - reconnectCB, deferredAssert := AssertConnectionReconnectAttempts(t, 1) - defer deferredAssert() c, err := pool.NewConnection( ctx, connectURL, nextName(), - pool.ConnectionWithRecoverCallback(reconnectCB), ) if err != nil { assert.NoError(t, err) diff --git a/pool/publisher_test.go b/pool/publisher_test.go index b07f399..37d8ce9 100644 --- a/pool/publisher_test.go +++ b/pool/publisher_test.go @@ -23,14 +23,11 @@ func TestSinglePublisher(t *testing.T) { numMsgs = 10 ) - healthyConnCB, hcbAssert := AssertConnectionReconnectAttempts(t, 0) - defer hcbAssert() hs, hsclose := NewSession( t, ctx, testutils.HealthyConnectURL, nextConnName(), - pool.ConnectionWithRecoverCallback(healthyConnCB), ) defer hsclose() @@ -109,14 +106,11 @@ func TestPublishAwaitFlowControl(t *testing.T) { nextConnName = testutils.ConnectionNameGenerator() ) - healthyConnCB, hcbAssert := AssertConnectionReconnectAttempts(t, 0) - defer hcbAssert() hs, hsclose := NewSession( t, ctx, testutils.HealthyConnectURL, nextConnName(), - pool.ConnectionWithRecoverCallback(healthyConnCB), ) defer hsclose() diff --git a/pool/session_test.go b/pool/session_test.go index 21d928d..3515695 100644 --- a/pool/session_test.go +++ b/pool/session_test.go @@ -34,14 +34,11 @@ func TestNewSingleSessionPublishAndConsume(t *testing.T) { numMsgs = 20 ) - reconnectCB, deferredAssert := AssertConnectionReconnectAttempts(t, 0) - defer deferredAssert() c, err := pool.NewConnection( ctx, testutils.HealthyConnectURL, connName, pool.ConnectionWithLogger(logging.NewTestLogger(t)), - pool.ConnectionWithRecoverCallback(reconnectCB), ) if err != nil { assert.NoError(t, err) @@ -91,14 +88,11 @@ func TestManyNewSessionsPublishAndConsume(t *testing.T) { numMsgs = 20 ) - reconnectCB, deferredAssert := AssertConnectionReconnectAttempts(t, 0) - defer deferredAssert() c, err := pool.NewConnection( ctx, testutils.HealthyConnectURL, connName, pool.ConnectionWithLogger(logging.NewTestLogger(t)), - pool.ConnectionWithRecoverCallback(reconnectCB), ) if err != nil { assert.NoError(t, err) @@ -158,14 +152,11 @@ func TestNewSessionQueueDeclarePassive(t *testing.T) { nextQueueName = testutils.QueueNameGenerator(sessionName) ) - reconnectCB, deferredAssert := AssertConnectionReconnectAttempts(t, 0) - defer deferredAssert() conn, err := pool.NewConnection( ctx, testutils.HealthyConnectURL, connName, pool.ConnectionWithLogger(logging.NewTestLogger(t)), - pool.ConnectionWithRecoverCallback(reconnectCB), ) if err != nil { assert.NoError(t, err) @@ -233,14 +224,11 @@ func TestNewSessionExchangeDeclareWithDisconnect(t *testing.T) { nextSessionName = testutils.SessionNameGenerator(connName) ) - reconnectCB, deferredAssert := AssertConnectionReconnectAttempts(t, 1) - defer deferredAssert() c, err := pool.NewConnection( ctx, connectURL, connName, pool.ConnectionWithLogger(logging.NewTestLogger(t)), - pool.ConnectionWithRecoverCallback(reconnectCB), ) if err != nil { assert.NoError(t, err) @@ -289,14 +277,11 @@ func TestNewSessionExchangeDeleteWithDisconnect(t *testing.T) { nextSessionName = testutils.SessionNameGenerator(connName) ) - reconnectCB, deferredAssert := AssertConnectionReconnectAttempts(t, 1) - defer deferredAssert() c, err := pool.NewConnection( ctx, connectURL, connName, pool.ConnectionWithLogger(logging.NewTestLogger(t)), - pool.ConnectionWithRecoverCallback(reconnectCB), ) if err != nil { assert.NoError(t, err) @@ -346,14 +331,11 @@ func TestNewSessionQueueDeclareWithDisconnect(t *testing.T) { nextSessionName = testutils.SessionNameGenerator(connName) ) - reconnectCB, deferredAssert := AssertConnectionReconnectAttempts(t, 1) - defer deferredAssert() c, err := pool.NewConnection( ctx, connectURL, connName, pool.ConnectionWithLogger(logging.NewTestLogger(t)), - pool.ConnectionWithRecoverCallback(reconnectCB), ) if err != nil { assert.NoError(t, err, "expected no error when creating new connection") @@ -405,14 +387,11 @@ func TestNewSessionQueueDeleteWithDisconnect(t *testing.T) { nextSessionName = testutils.SessionNameGenerator(connName) ) - reconnectCB, deferredAssert := AssertConnectionReconnectAttempts(t, 1) - defer deferredAssert() c, err := pool.NewConnection( ctx, connectURL, connName, pool.ConnectionWithLogger(logging.NewTestLogger(t)), - pool.ConnectionWithRecoverCallback(reconnectCB), ) if err != nil { assert.NoError(t, err) @@ -462,14 +441,11 @@ func TestNewSessionQueueBindWithDisconnect(t *testing.T) { nextSessionName = testutils.SessionNameGenerator(connName) ) - reconnectCB, deferredAssert := AssertConnectionReconnectAttempts(t, 1) - defer deferredAssert() c, err := pool.NewConnection( ctx, connectURL, connName, pool.ConnectionWithLogger(logging.NewTestLogger(t)), - pool.ConnectionWithRecoverCallback(reconnectCB), ) if err != nil { assert.NoError(t, err) @@ -539,14 +515,11 @@ func TestNewSessionQueueUnbindWithDisconnect(t *testing.T) { nextSessionName = testutils.SessionNameGenerator(connName) ) - reconnectCB, deferredAssert := AssertConnectionReconnectAttempts(t, 1) - defer deferredAssert() c, err := pool.NewConnection( ctx, connectURL, connName, pool.ConnectionWithLogger(logging.NewTestLogger(t)), - pool.ConnectionWithRecoverCallback(reconnectCB), ) if err != nil { assert.NoError(t, err) @@ -617,25 +590,19 @@ func TestNewSessionPublishWithDisconnect(t *testing.T) { nextConnName = testutils.ConnectionNameGenerator() ) - healthyConnCB, hcbAssert := AssertConnectionReconnectAttempts(t, 0) - defer hcbAssert() hs, hsclose := NewSession( t, ctx, testutils.HealthyConnectURL, nextConnName(), - pool.ConnectionWithRecoverCallback(healthyConnCB), ) defer hsclose() - brokenReconnCB, scbAssert := AssertConnectionReconnectAttempts(t, 1) - defer scbAssert() s, sclose := NewSession( t, ctx, connectURL, nextConnName(), - pool.ConnectionWithRecoverCallback(brokenReconnCB), ) defer sclose() @@ -677,25 +644,19 @@ func TestNewSessionConsumeWithDisconnect(t *testing.T) { nextConnName = testutils.ConnectionNameGenerator() ) - healthyConnCB, hcbAssert := AssertConnectionReconnectAttempts(t, 0) - defer hcbAssert() hs, hsclose := NewSession( t, ctx, testutils.HealthyConnectURL, nextConnName(), - pool.ConnectionWithRecoverCallback(healthyConnCB), ) defer hsclose() - brokenReconnCB, scbAssert := AssertConnectionReconnectAttempts(t, 1) - defer scbAssert() s, sclose := NewSession( t, ctx, connectURL, nextConnName(), - pool.ConnectionWithRecoverCallback(brokenReconnCB), ) defer sclose() @@ -928,14 +889,11 @@ func TestNewSingleSessionCloseWithDisconnect(t *testing.T) { disconnected, reconnected = Disconnect(t, proxyName, 5*time.Second) ) - reconnectCB, deferredAssert := AssertConnectionReconnectAttempts(t, 0) - defer deferredAssert() c, err := pool.NewConnection( ctx, connectURL, connName, pool.ConnectionWithLogger(logging.NewTestLogger(t)), - pool.ConnectionWithRecoverCallback(reconnectCB), ) if err != nil { assert.NoError(t, err) @@ -967,7 +925,7 @@ func TestNewSingleSessionCloseWithDisconnect(t *testing.T) { } /* -// FIXME: ou of memory tests are disabled until https://github.com/rabbitmq/amqp091-go/issues/253 is resolved +// FIXME: out of memory tests are disabled until https://github.com/rabbitmq/amqp091-go/issues/253 is resolved func TestNewSingleSessionCloseWithOutOfMemoryRabbitMQ(t *testing.T) { t.Parallel() // can be run in parallel because the connection to the rabbitmq is never broken @@ -984,14 +942,11 @@ func TestNewSingleSessionCloseWithOutOfMemoryRabbitMQ(t *testing.T) { queueName = nextQueueName() ) - reconnectCB, deferredAssert := AssertConnectionReconnectAttempts(t, 0) - defer deferredAssert() c, err := pool.NewConnection( ctx, testutils.BrokenConnectURL, // out of memory rabbitmq connName, pool.ConnectionWithLogger(log), - pool.ConnectionWithRecoverCallback(reconnectCB), ) if err != nil { assert.NoError(t, err) @@ -1059,14 +1014,11 @@ func TestNewSingleSessionCloseWithHealthyRabbitMQ(t *testing.T) { queueName = nextQueueName() ) - reconnectCB, deferredAssert := AssertConnectionReconnectAttempts(t, 0) - defer deferredAssert() c, err := pool.NewConnection( ctx, testutils.HealthyConnectURL, // healthy rabbitmq connName, pool.ConnectionWithLogger(log), - pool.ConnectionWithRecoverCallback(reconnectCB), ) if err != nil { assert.NoError(t, err) diff --git a/pool/utils_test.go b/pool/utils_test.go index c103217..f777739 100644 --- a/pool/utils_test.go +++ b/pool/utils_test.go @@ -296,27 +296,6 @@ func NewSession(t *testing.T, ctx context.Context, connectURL, connectionName st } } -func AssertConnectionReconnectAttempts(t *testing.T, n int) (callback pool.ConnectionRecoverCallback, deferredAssert func()) { - var ( - i int - mu sync.Mutex - log = logging.NewTestLogger(t) - ) - return func(name string, retry int, err error) { - if retry == 0 { - log.Infof("connection %s retry %d, error: %v", name, retry, err) - mu.Lock() - i++ - mu.Unlock() - } - }, - func() { - mu.Lock() - defer mu.Unlock() - assert.Equal(t, n, i, "expected %d reconnect attempts, got %d", n, i) - } -} - func PublisherPublishN(t *testing.T, ctx context.Context, p *pool.Pool, exchangeName string, publishMessageGenerator func() string, n int) { pub := pool.NewPublisher(p) defer pub.Close() From 616ee399b96bd69cb6ebfa67d9bffd9eea07caaf Mon Sep 17 00:00:00 2001 From: John Behm Date: Thu, 14 Mar 2024 16:12:47 +0100 Subject: [PATCH 54/76] update makefile test timeout --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 7b6465b..d893f82 100644 --- a/Makefile +++ b/Makefile @@ -8,7 +8,7 @@ down: docker-compose down test: - go test -timeout 900s -v -race -count=1 ./... + go test -timeout 300s -v -race -count=1 ./... count-tests: grep -REn 'func Test.+\(.+testing\.T.*\)' . | wc -l From e2e046e2361a4f3cfa7732568f9273c02a55f9b4 Mon Sep 17 00:00:00 2001 From: John Behm Date: Fri, 15 Mar 2024 14:41:31 +0100 Subject: [PATCH 55/76] update test utils --- internal/testutils/testutils.go | 40 +++++++++++++++++++++++++++++++-- 1 file changed, 38 insertions(+), 2 deletions(-) diff --git a/internal/testutils/testutils.go b/internal/testutils/testutils.go index 7eb4f3c..bb61f3a 100644 --- a/internal/testutils/testutils.go +++ b/internal/testutils/testutils.go @@ -25,7 +25,7 @@ func FilePath(relative string, up ...int) string { func FuncName(up ...int) string { offset := 1 if len(up) > 0 && up[0] > 0 { - offset = up[0] + offset = +up[0] } pc, _, _, ok := runtime.Caller(offset) if !ok { @@ -39,10 +39,28 @@ func FuncName(up ...int) string { return f.Name() } +func FileLine(up ...int) string { + offset := 1 + if len(up) > 0 && up[0] > 0 { + offset += up[0] + } + pc, _, _, ok := runtime.Caller(offset) + if !ok { + panic("failed to get caller") + } + + f := runtime.FuncForPC(pc) + if f == nil { + panic("failed to get function name") + } + file, line := f.FileLine(pc) + return fmt.Sprintf("%s:%d", file, line) +} + func CallerFuncName(up ...int) string { offset := 2 if len(up) > 0 && up[0] > 0 { - offset = up[0] + offset += up[0] } pc, _, _, ok := runtime.Caller(offset) if !ok { @@ -55,3 +73,21 @@ func CallerFuncName(up ...int) string { } return f.Name() } + +func CallerFileLine(up ...int) string { + offset := 2 + if len(up) > 0 && up[0] > 0 { + offset += up[0] + } + pc, _, _, ok := runtime.Caller(offset) + if !ok { + panic("failed to get caller") + } + + f := runtime.FuncForPC(pc) + if f == nil { + panic("failed to get function name") + } + file, line := f.FileLine(pc) + return fmt.Sprintf("%s:%d", file, line) +} From 12263ba886de2ccb6df9d2ec55192290e4d211f4 Mon Sep 17 00:00:00 2001 From: John Behm Date: Fri, 15 Mar 2024 14:41:53 +0100 Subject: [PATCH 56/76] add test case for debugging a close deadlock --- Makefile | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/Makefile b/Makefile index d893f82..0da8be8 100644 --- a/Makefile +++ b/Makefile @@ -14,4 +14,9 @@ count-tests: grep -REn 'func Test.+\(.+testing\.T.*\)' . | wc -l count-disconnect-tests: - grep -REn 'func Test.+WithDisconnect.*\(.+testing\.T.*\)' . | wc -l \ No newline at end of file + grep -REn 'func Test.+WithDisconnect.*\(.+testing\.T.*\)' . | wc -l + + +pool.TestBatchSubscriberMaxBytes: + go test -timeout 0m30s github.com/jxsl13/amqpx/pool -run ^TestBatchSubscriberMaxBytes$ -v -count=1 -race 2>&1 > test.log + cat test.log | grep 'INFO: session' | sort | uniq -c \ No newline at end of file From 47b5648312cd13bd0a609e17eeb5c5a11b569ed1 Mon Sep 17 00:00:00 2001 From: John Behm Date: Fri, 15 Mar 2024 14:42:19 +0100 Subject: [PATCH 57/76] update generators --- internal/testutils/generator.go | 13 ++++--------- 1 file changed, 4 insertions(+), 9 deletions(-) diff --git a/internal/testutils/generator.go b/internal/testutils/generator.go index eee3d34..5859626 100644 --- a/internal/testutils/generator.go +++ b/internal/testutils/generator.go @@ -25,13 +25,6 @@ func (o *generatorOptions) ToSuffix() string { type GeneratorOption func(*generatorOptions) -// WithUp sets the number of stack frames to skip when generating the name -func WithUp(up int) GeneratorOption { - return func(o *generatorOptions) { - o.up = up - } -} - func WithRandomSuffix(addRandomSuffix bool) GeneratorOption { return func(o *generatorOptions) { o.randomSuffix = addRandomSuffix @@ -169,7 +162,7 @@ func SessionNameGenerator(connectionName string, options ...GeneratorOption) (ne } } -func PoolNameGenerator(options ...GeneratorOption) (nextConnName func() string) { +func PoolNameGenerator(funcName string, options ...GeneratorOption) (nextConnName func() string) { opts := generatorOptions{ prefix: "", randomSuffix: false, @@ -182,7 +175,6 @@ func PoolNameGenerator(options ...GeneratorOption) (nextConnName func() string) } var mu sync.Mutex - funcName := FuncName(opts.up) parts := strings.Split(funcName, ".") funcName = parts[len(parts)-1] @@ -247,11 +239,14 @@ func MessageGenerator(queueOrExchangeName string, options ...GeneratorOption) (n func NewExchangeQueueGenerator(funcName string) func() ExchangeQueue { var ( + mu sync.Mutex nextExchangeName = ExchangeNameGenerator(funcName) nextQueueName = QueueNameGenerator(funcName) nextRoutingKey = RoutingKeyGenerator(funcName) ) return func() ExchangeQueue { + mu.Lock() + defer mu.Unlock() return NewExchangeQueue(nextExchangeName(), nextQueueName(), nextRoutingKey()) } } From 39bd86d14da9f7206fb6a36bdb877d25dfb88512 Mon Sep 17 00:00:00 2001 From: John Behm Date: Fri, 15 Mar 2024 14:43:50 +0100 Subject: [PATCH 58/76] update subscriber tests --- pool/subscriber_test.go | 68 ++++++++++++++++++----------------------- 1 file changed, 29 insertions(+), 39 deletions(-) diff --git a/pool/subscriber_test.go b/pool/subscriber_test.go index 7653513..c86044b 100644 --- a/pool/subscriber_test.go +++ b/pool/subscriber_test.go @@ -18,7 +18,7 @@ func TestNewSingleSubscriber(t *testing.T) { var ( ctx = context.TODO() - nextPoolName = testutils.PoolNameGenerator() + nextPoolName = testutils.PoolNameGenerator(testutils.FuncName()) poolName = nextPoolName() hp = NewPool(t, ctx, testutils.HealthyConnectURL, poolName, 1, 2) ) @@ -58,7 +58,7 @@ func TestNewSingleSubscriberWithDisconnect(t *testing.T) { var ( ctx = context.TODO() - nextPoolName = testutils.PoolNameGenerator() + nextPoolName = testutils.PoolNameGenerator(testutils.FuncName()) poolName = nextPoolName() hp = NewPool(t, ctx, testutils.HealthyConnectURL, poolName, 1, 2) proxyName, connectURL, _ = testutils.NextConnectURL() @@ -103,7 +103,7 @@ func TestNewSingleBatchSubscriber(t *testing.T) { var ( ctx = context.TODO() - nextPoolName = testutils.PoolNameGenerator() + nextPoolName = testutils.PoolNameGenerator(testutils.FuncName()) poolName = nextPoolName() hp = NewPool(t, ctx, testutils.HealthyConnectURL, poolName, 1, 2) ) @@ -153,30 +153,31 @@ func TestNewSingleBatchSubscriber(t *testing.T) { func TestBatchSubscriberMaxBytes(t *testing.T) { t.Parallel() - var wg sync.WaitGroup - for i := 1; i <= 2048; i = i*2 + 1 { - wg.Add(1) - go testBatchSubscriberMaxBytes(t, i, &wg) - } + funcName := testutils.FuncName() + const ( + maxBatchBytes = 2048 + iterations = 100 + ) + for i := 0; i < iterations; i++ { + for j := 1; j <= maxBatchBytes; j = j*2 + 1 { + wg.Add(1) + go testBatchSubscriberMaxBytes(t, fmt.Sprintf("%s-%d-max-batch-bytes-%d", funcName, i, j), j, &wg) + } + } wg.Wait() } -func testBatchSubscriberMaxBytes(t *testing.T, maxBatchBytes int, w *sync.WaitGroup) { +func testBatchSubscriberMaxBytes(t *testing.T, funcName string, maxBatchBytes int, w *sync.WaitGroup) { t.Helper() defer w.Done() var ( - ctx = context.TODO() - nextPoolName = testutils.PoolNameGenerator( - testutils.WithUp(3), - testutils.WithSuffix(fmt.Sprintf("-max-batch-bytes-%d", maxBatchBytes)), - ) - poolName = nextPoolName() - - nextExchangeName = testutils.ExchangeNameGenerator(poolName) - nextQueueName = testutils.QueueNameGenerator(poolName) + ctx = context.TODO() + nextPoolName = testutils.PoolNameGenerator(funcName) + poolName = nextPoolName() + nextExchangeQueue = testutils.NewExchangeQueueGenerator(poolName) numSessions = 2 hp = NewPool(t, ctx, testutils.HealthyConnectURL, poolName, 1, numSessions) // // publisher sessions + consumer sessions @@ -195,10 +196,8 @@ func testBatchSubscriberMaxBytes(t *testing.T, maxBatchBytes int, w *sync.WaitGr defer wg.Done() var ( - log = logging.NewTestLogger(t) - exchangeName = nextExchangeName() - queueName = nextQueueName() - nextConsumerName = testutils.ConsumerNameGenerator(queueName) + log = logging.NewTestLogger(t) + eq = nextExchangeQueue() ) ts, err := hp.GetTransientSession(ctx) @@ -208,37 +207,27 @@ func testBatchSubscriberMaxBytes(t *testing.T, maxBatchBytes int, w *sync.WaitGr } defer hp.ReturnSession(ts, nil) - cleanup := DeclareExchangeQueue(t, ctx, ts, exchangeName, queueName) + cleanup := DeclareExchangeQueue(t, ctx, ts, eq.Exchange, eq.Queue) defer cleanup() - _, err = ts.QueueDeclare(ctx, queueName) - if err != nil { - assert.NoError(t, err) - return - } - defer func() { - i, err := ts.QueueDelete(ctx, queueName) - assert.NoError(t, err) - assert.Equal(t, 0, i) - }() - // publish all messages pub := pool.NewPublisher(hp) defer pub.Close() maxMsgLen := 0 for i := 0; i < numMsgs; i++ { - message := fmt.Sprintf("Message-%s-%06d", queueName, i) // max 6 digits + message := fmt.Sprintf("Message-%s-%06d", eq.Queue, i) // max 6 digits mlen := len(message) if mlen > maxMsgLen { maxMsgLen = mlen } - pub.Publish(ctx, exchangeName, "", pool.Publishing{ + err = pub.Publish(ctx, eq.Exchange, "", pool.Publishing{ Mandatory: true, ContentType: "text/plain", Body: []byte(message), }) + assert.NoError(t, err) } log.Debugf("max message length: %d", maxMsgLen) log.Debugf("max batch bytes: %d", maxBatchBytes) @@ -261,7 +250,7 @@ func testBatchSubscriberMaxBytes(t *testing.T, maxBatchBytes int, w *sync.WaitGr batchCount := 0 messageCount := 0 - sub.RegisterBatchHandlerFunc(queueName, + sub.RegisterBatchHandlerFunc(eq.Queue, func(ctx context.Context, msgs []pool.Delivery) error { for idx, msg := range msgs { @@ -288,11 +277,12 @@ func testBatchSubscriberMaxBytes(t *testing.T, maxBatchBytes int, w *sync.WaitGr pool.WithMaxBatchSize(0), // disable this check pool.WithBatchFlushTimeout(batchTimeout), pool.WithBatchConsumeOptions(pool.ConsumeOptions{ - ConsumerTag: nextConsumerName(), + ConsumerTag: eq.ConsumerTag, Exclusive: true, }), ) - sub.Start(ctx) + err = sub.Start(ctx) + assert.NoError(t, err) // this should be canceled upon context cancelation from within the // subscriber handler. From 5b8a69ccf522a4099bb609acf1e045bf495a2993 Mon Sep 17 00:00:00 2001 From: John Behm Date: Fri, 15 Mar 2024 14:44:19 +0100 Subject: [PATCH 59/76] update session pool tests --- pool/session_pool_test.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pool/session_pool_test.go b/pool/session_pool_test.go index a4cb7f9..e30a518 100644 --- a/pool/session_pool_test.go +++ b/pool/session_pool_test.go @@ -62,7 +62,7 @@ func TestNewSessionPool(t *testing.T) { poolName = testutils.FuncName() ctx = context.TODO() connections = 1 - sessions = 10 + sessions = 100 ) p, err := pool.NewConnectionPool(ctx, testutils.HealthyConnectURL, @@ -74,11 +74,11 @@ func TestNewSessionPool(t *testing.T) { assert.NoError(t, err) return } + defer p.Close() sp, err := pool.NewSessionPool( p, sessions, - pool.SessionPoolWithAutoCloseConnectionPool(true), ) if err != nil { assert.NoError(t, err) @@ -97,7 +97,7 @@ func TestNewSessionPool(t *testing.T) { assert.NoError(t, err) return } - time.Sleep(testutils.Jitter(1*time.Second, 5*time.Second)) + time.Sleep(testutils.Jitter(10*time.Millisecond, 30*time.Millisecond)) sp.ReturnSession(s, nil) }() } From b773bf31f7ce32c3a0ce6aad9fa8a83553cfbce1 Mon Sep 17 00:00:00 2001 From: John Behm Date: Fri, 15 Mar 2024 14:45:00 +0100 Subject: [PATCH 60/76] update session comments --- pool/session.go | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/pool/session.go b/pool/session.go index d4e2f9b..6aebc86 100644 --- a/pool/session.go +++ b/pool/session.go @@ -15,7 +15,8 @@ const ( notImplemented = 540 ) -// Session is +// Session is a wrapper for an amqp channel. +// It MUST not be used in a multithreaded context, but only in a single goroutine. type Session struct { name string cached bool @@ -363,6 +364,9 @@ func (s *Session) AwaitConfirm(ctx context.Context, expectedTag uint64) error { } select { + // TODO: this might lead to problems when a single session is used + // in a multithreaded context. That way we might received out of order confirmations + // which could lead to unexpected behavior. case confirm, ok := <-s.confirms: if !ok { err := s.error() From 736cc7276f65169862721f4925b4114af90b7f29 Mon Sep 17 00:00:00 2001 From: John Behm Date: Fri, 15 Mar 2024 14:51:30 +0100 Subject: [PATCH 61/76] fix session & connection no being returned back to the pool in case that they cannot be recovered upon acquiring from pool. --- pool/connection_pool.go | 25 +++++++++++++++++++------ pool/session_pool.go | 24 ++++++++++++++++++++---- 2 files changed, 39 insertions(+), 10 deletions(-) diff --git a/pool/connection_pool.go b/pool/connection_pool.go index 8f910c1..3c29ddf 100644 --- a/pool/connection_pool.go +++ b/pool/connection_pool.go @@ -159,17 +159,25 @@ func (cp *ConnectionPool) deriveConnection(ctx context.Context, id int64, cached } // GetConnection only returns an error upon shutdown -func (cp *ConnectionPool) GetConnection(ctx context.Context) (*Connection, error) { +func (cp *ConnectionPool) GetConnection(ctx context.Context) (conn *Connection, err error) { select { case conn, ok := <-cp.connections: if !ok { return nil, fmt.Errorf("connection pool %w", ErrClosed) } - if conn.IsFlagged() { - err := conn.Recover(ctx) + + // recovery may fail, that's why we MUST check for errors + // and return the connection back to the pool in case that the recovery failed + // due to e.g. the pool being closed, the context being canceled, etc. + defer func() { if err != nil { - return nil, fmt.Errorf("failed to get connection: %w", err) + cp.ReturnConnection(conn, err) } + }() + + err = conn.Recover(ctx) + if err != nil { + return nil, fmt.Errorf("failed to get connection: %w", err) } return conn, nil @@ -189,11 +197,16 @@ func (cp *ConnectionPool) nextTransientID() int64 { // GetTransientConnection may return an error when the context was cancelled before the connection could be obtained. // Transient connections may be returned to the pool. The are closed properly upon returning. -func (cp *ConnectionPool) GetTransientConnection(ctx context.Context) (_ *Connection, err error) { - conn, err := cp.deriveConnection(ctx, cp.nextTransientID(), false) +func (cp *ConnectionPool) GetTransientConnection(ctx context.Context) (conn *Connection, err error) { + conn, err = cp.deriveConnection(ctx, cp.nextTransientID(), false) if err == nil { return conn, nil } + defer func() { + if err != nil { + _ = conn.Close() + } + }() // recover until context is closed err = conn.Recover(ctx) diff --git a/pool/session_pool.go b/pool/session_pool.go index 00beecd..fcdfddb 100644 --- a/pool/session_pool.go +++ b/pool/session_pool.go @@ -76,9 +76,9 @@ func newSessionPoolFromOption(pool *ConnectionPool, ctx context.Context, option pool: pool, autoCloseConnPool: option.AutoClosePool, - capacity: option.Capacity, bufferCapacity: option.BufferCapacity, confirmable: option.Confirmable, + capacity: option.Capacity, sessions: make(chan *Session, option.Capacity), ctx: ctx, @@ -169,7 +169,7 @@ func (sp *SessionPool) Capacity() int { // GetSession gets a pooled session. // blocks until a session is acquired from the pool. -func (sp *SessionPool) GetSession(ctx context.Context) (*Session, error) { +func (sp *SessionPool) GetSession(ctx context.Context) (s *Session, err error) { select { case <-sp.catchShutdown(): return nil, sp.shutdownErr() @@ -179,6 +179,13 @@ func (sp *SessionPool) GetSession(ctx context.Context) (*Session, error) { if !ok { return nil, fmt.Errorf("failed to get session: %w", ErrClosed) } + defer func() { + // it's possible for the recovery to fail + // in that case we MUST return the session back to the pool + if err != nil { + sp.ReturnSession(session, err) + } + }() err := session.Recover(ctx) if err != nil { @@ -191,14 +198,23 @@ func (sp *SessionPool) GetSession(ctx context.Context) (*Session, error) { // GetTransientSession returns a transient session. // This method may return an error when the context ha sbeen closed before a session could be obtained. // A transient session creates a transient connection under the hood. -func (sp *SessionPool) GetTransientSession(ctx context.Context) (*Session, error) { +func (sp *SessionPool) GetTransientSession(ctx context.Context) (s *Session, err error) { conn, err := sp.pool.GetTransientConnection(ctx) if err != nil { return nil, err } + defer func() { + if err != nil { + sp.pool.ReturnConnection(conn, err) + } + }() transientID := atomic.AddInt64(&sp.transientID, 1) - return sp.deriveSession(ctx, conn, int(transientID)) + s, err = sp.deriveSession(ctx, conn, int(transientID)) + if err != nil { + return nil, err + } + return s, nil } func (sp *SessionPool) deriveSession(ctx context.Context, conn *Connection, id int) (*Session, error) { From 161387789aff86d68633e574fec9cbb66a3ae20c Mon Sep 17 00:00:00 2001 From: John Behm Date: Fri, 15 Mar 2024 15:16:16 +0100 Subject: [PATCH 62/76] only check if close session close does not deadlock, ignore returned error --- pool/session_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pool/session_test.go b/pool/session_test.go index 3515695..6088579 100644 --- a/pool/session_test.go +++ b/pool/session_test.go @@ -921,7 +921,7 @@ func TestNewSingleSessionCloseWithDisconnect(t *testing.T) { disconnected() defer reconnected() - assert.NoError(t, s.Close()) + _ = s.Close() } /* From 8927bc55eae0a00b74dee9cf1cad9dca89792347 Mon Sep 17 00:00:00 2001 From: John Behm Date: Fri, 15 Mar 2024 15:37:22 +0100 Subject: [PATCH 63/76] flush connection channels after recovery --- pool/connection.go | 10 +++++++++- pool/connection_test.go | 2 +- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/pool/connection.go b/pool/connection.go index 0e5a39f..35f7b1f 100644 --- a/pool/connection.go +++ b/pool/connection.go @@ -154,6 +154,12 @@ func (ch *Connection) Close() (err error) { return nil } +// flush all internal channels +func (ch *Connection) flush() { + flush(ch.errors) + flush(ch.blocking) +} + // Flag flags the connection as broken which must be recovered. // A flagged connection implies a closed connection. // Flagging of a connectioncan only be undone by Recover-ing the connection. @@ -285,7 +291,7 @@ func (ch *Connection) Recover(ctx context.Context) error { return ch.recover(ctx) } -func (ch *Connection) recover(ctx context.Context) error { +func (ch *Connection) recover(ctx context.Context) (err error) { select { case <-ctx.Done(): @@ -301,6 +307,8 @@ func (ch *Connection) recover(ctx context.Context) error { if healthy { return nil } + // flush all channels after recovery + defer ch.flush() var ( timer = time.NewTimer(0) diff --git a/pool/connection_test.go b/pool/connection_test.go index 54f8a6d..2dbf46d 100644 --- a/pool/connection_test.go +++ b/pool/connection_test.go @@ -137,7 +137,7 @@ func TestManyNewConnectionWithDisconnect(t *testing.T) { wait() // wait for connection to work again. - tctx, cancel := context.WithTimeout(ctx, 5*time.Second) + tctx, cancel := context.WithTimeout(ctx, 20*time.Second) defer cancel() assert.NoError(t, c.Recover(tctx)) assert.NoError(t, c.Error()) From 991323c38485b7ff353d1ed77ccc31a4f8379681 Mon Sep 17 00:00:00 2001 From: John Behm Date: Fri, 15 Mar 2024 16:01:52 +0100 Subject: [PATCH 64/76] remove connection channel flush --- pool/connection.go | 9 --------- 1 file changed, 9 deletions(-) diff --git a/pool/connection.go b/pool/connection.go index 35f7b1f..42ed131 100644 --- a/pool/connection.go +++ b/pool/connection.go @@ -154,12 +154,6 @@ func (ch *Connection) Close() (err error) { return nil } -// flush all internal channels -func (ch *Connection) flush() { - flush(ch.errors) - flush(ch.blocking) -} - // Flag flags the connection as broken which must be recovered. // A flagged connection implies a closed connection. // Flagging of a connectioncan only be undone by Recover-ing the connection. @@ -228,7 +222,6 @@ func (ch *Connection) connect(ctx context.Context) error { func (ch *Connection) BlockingFlowControl() <-chan amqp.Blocking { ch.mu.Lock() defer ch.mu.Unlock() - return ch.blocking } @@ -307,8 +300,6 @@ func (ch *Connection) recover(ctx context.Context) (err error) { if healthy { return nil } - // flush all channels after recovery - defer ch.flush() var ( timer = time.NewTimer(0) From 6abb8fc1ad35cd3b7c835c3391632953c1075bde Mon Sep 17 00:00:00 2001 From: John Behm Date: Fri, 15 Mar 2024 16:02:23 +0100 Subject: [PATCH 65/76] improve ConsumeN test utility --- pool/utils_test.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/pool/utils_test.go b/pool/utils_test.go index f777739..1eb6eaa 100644 --- a/pool/utils_test.go +++ b/pool/utils_test.go @@ -38,6 +38,9 @@ func ConsumeN( assert.Equal(t, n, msgsReceived, "expected to consume %d messages, got %d", n, msgsReceived) }() + var previouslyReceivedMsg string + +outer: for { delivery, err := c.Consume( queueName, @@ -51,16 +54,13 @@ func ConsumeN( return } - var previouslyReceivedMsg string - for { select { case <-cctx.Done(): return case val, ok := <-delivery: - require.True(t, ok, "expected delivery channel to be open of consumer %s in ConsumeN", consumerName) if !ok { - return + continue outer } err := val.Ack(false) if err != nil { From e757aef601552c1d8d329e8a767a1dbc1271085b Mon Sep 17 00:00:00 2001 From: John Behm Date: Fri, 15 Mar 2024 16:03:14 +0100 Subject: [PATCH 66/76] fix nil pointer dereference --- pool/session_pool.go | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/pool/session_pool.go b/pool/session_pool.go index fcdfddb..7fee65f 100644 --- a/pool/session_pool.go +++ b/pool/session_pool.go @@ -72,7 +72,8 @@ func newSessionPoolFromOption(pool *ConnectionPool, ctx context.Context, option ctx, cc := context.WithCancelCause(ctx) cancel := toCancelFunc(fmt.Errorf("session pool %w", ErrClosed), cc) - sp = &SessionPool{ + // DO NOT rename this variable to sp + sessionPool := &SessionPool{ pool: pool, autoCloseConnPool: option.AutoClosePool, @@ -106,21 +107,21 @@ func newSessionPoolFromOption(pool *ConnectionPool, ctx context.Context, option FlowRetryCallback: option.FlowRetryCallback, } - sp.debug("initializing pool sessions...") + sessionPool.debug("initializing pool sessions...") defer func() { if err != nil { - sp.error(err, "failed to initialize pool sessions") + sessionPool.error(err, "failed to initialize pool sessions") } else { - sp.info("initialized") + sessionPool.info("initialized") } }() - err = sp.initCachedSessions() + err = sessionPool.initCachedSessions() if err != nil { return nil, err } - return sp, nil + return sessionPool, nil } func (sp *SessionPool) initCachedSessions() error { From c675cd0e5a4efa0afd6af43ac51d919d45c65d89 Mon Sep 17 00:00:00 2001 From: John Behm Date: Fri, 15 Mar 2024 16:03:40 +0100 Subject: [PATCH 67/76] remove signal handling from tests --- amqpx_test.go | 24 +++++++++++------------- 1 file changed, 11 insertions(+), 13 deletions(-) diff --git a/amqpx_test.go b/amqpx_test.go index d745a32..7aec1ac 100644 --- a/amqpx_test.go +++ b/amqpx_test.go @@ -3,9 +3,7 @@ package amqpx_test import ( "context" "fmt" - "os/signal" "sync" - "syscall" "testing" "time" @@ -171,7 +169,7 @@ func TestAMQPXSubAndPub(t *testing.T) { var ( amqp = amqpx.New() log = logging.NewTestLogger(t) - cctx, cancel = signal.NotifyContext(context.TODO(), syscall.SIGINT, syscall.SIGINT) + cctx, cancel = context.WithCancel(context.TODO()) funcName = testutils.FuncName() nextExchangeQueue = testutils.NewExchangeQueueGenerator(funcName) eq1 = nextExchangeQueue() @@ -227,7 +225,7 @@ func TestAMQPXSubAndPubMulti(t *testing.T) { var ( amqp = amqpx.New() log = logging.NewTestLogger(t) - cctx, cancel = signal.NotifyContext(context.TODO(), syscall.SIGINT, syscall.SIGINT) + cctx, cancel = context.WithCancel(context.TODO()) funcName = testutils.FuncName() nextExchangeQueue = testutils.NewExchangeQueueGenerator(funcName) @@ -323,7 +321,7 @@ func TestAMQPXSubHandler(t *testing.T) { var ( amqp = amqpx.New() log = logging.NewTestLogger(t) - cctx, cancel = signal.NotifyContext(context.TODO(), syscall.SIGINT, syscall.SIGINT) + cctx, cancel = context.WithCancel(context.TODO()) funcName = testutils.FuncName() nextExchangeQueue = testutils.NewExchangeQueueGenerator(funcName) eq1 = nextExchangeQueue() @@ -379,7 +377,7 @@ func TestCreateDeleteTopology(t *testing.T) { var ( amqp = amqpx.New() log = logging.NewTestLogger(t) - cctx, cancel = signal.NotifyContext(context.TODO(), syscall.SIGINT, syscall.SIGINT) + cctx, cancel = context.WithCancel(context.TODO()) funcName = testutils.FuncName() nextExchangeQueue = testutils.NewExchangeQueueGenerator(funcName) eq1 = nextExchangeQueue() @@ -410,7 +408,7 @@ func TestPauseResumeHandlerNoProcessing(t *testing.T) { var ( amqp = amqpx.New() log = logging.NewTestLogger(t) - cctx, cancel = signal.NotifyContext(context.TODO(), syscall.SIGINT, syscall.SIGINT) + cctx, cancel = context.WithCancel(context.TODO()) funcName = testutils.FuncName() nextQueueName = testutils.QueueNameGenerator(funcName) queueName = nextQueueName() @@ -496,7 +494,7 @@ func testHandlerPauseAndResume(t *testing.T, i int) { var ( amqp = amqpx.New() log = logging.NewTestLogger(t) - cctx, cancel = signal.NotifyContext(context.TODO(), syscall.SIGINT, syscall.SIGINT) + cctx, cancel = context.WithCancel(context.TODO()) funcName = fmt.Sprintf("%s-%d", testutils.CallerFuncName(), i) nextExchangeQueue = testutils.NewExchangeQueueGenerator(funcName) eq1 = nextExchangeQueue() @@ -659,7 +657,7 @@ func testBatchHandlerPauseAndResume(t *testing.T, i int) { var ( amqp = amqpx.New() log = logging.NewTestLogger(t) - cctx, cancel = signal.NotifyContext(context.TODO(), syscall.SIGINT, syscall.SIGINT) + cctx, cancel = context.WithCancel(context.TODO()) funcName = fmt.Sprintf("%s-%d", testutils.CallerFuncName(), i) nextExchangeQueue = testutils.NewExchangeQueueGenerator(funcName) eq1 = nextExchangeQueue() @@ -808,7 +806,7 @@ func TestQueueDeletedConsumerReconnect(t *testing.T) { err error amqp = amqpx.New() log = logging.NewTestLogger(t) - cctx, cancel = signal.NotifyContext(context.TODO(), syscall.SIGINT, syscall.SIGINT) + cctx, cancel = context.WithCancel(context.TODO()) funcName = testutils.FuncName() nextQueueName = testutils.QueueNameGenerator(funcName) queueName = nextQueueName() @@ -883,7 +881,7 @@ func TestQueueDeletedBatchConsumerReconnect(t *testing.T) { err error amqp = amqpx.New() log = logging.NewTestLogger(t) - cctx, cancel = signal.NotifyContext(context.TODO(), syscall.SIGINT, syscall.SIGINT) + cctx, cancel = context.WithCancel(context.TODO()) funcName = testutils.FuncName() nextQueueName = testutils.QueueNameGenerator(funcName) queueName = nextQueueName() @@ -986,7 +984,7 @@ func testHandlerReset(t *testing.T, i int) { err error amqp = amqpx.New() log = logging.NewTestLogger(t) - cctx, cancel = signal.NotifyContext(context.TODO(), syscall.SIGINT, syscall.SIGINT) + cctx, cancel = context.WithCancel(context.TODO()) funcName = fmt.Sprintf("%s-i-%d", testutils.CallerFuncName(), i) nextExchangeQueue = testutils.NewExchangeQueueGenerator(funcName) eq1 = nextExchangeQueue() @@ -1083,7 +1081,7 @@ func testBatchHandlerReset(t *testing.T, i int) { err error amqp = amqpx.New() log = logging.NewTestLogger(t) - cctx, cancel = signal.NotifyContext(context.TODO(), syscall.SIGINT, syscall.SIGINT) + cctx, cancel = context.WithCancel(context.TODO()) funcName = fmt.Sprintf("%s-i-%d", testutils.CallerFuncName(), i) nextExchangeQueue = testutils.NewExchangeQueueGenerator(funcName) eq1 = nextExchangeQueue() From df88d09e472ff3a3155b1c405bf1225ee6891e25 Mon Sep 17 00:00:00 2001 From: John Behm Date: Fri, 15 Mar 2024 16:12:41 +0100 Subject: [PATCH 68/76] update low level channel close with disconnect tests --- pool/session_test.go | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/pool/session_test.go b/pool/session_test.go index 6088579..ff5f4b8 100644 --- a/pool/session_test.go +++ b/pool/session_test.go @@ -860,9 +860,8 @@ func TestChannelCloseWithDisconnect(t *testing.T) { assert.NoError(t, err) return } - defer func() { - assert.Error(t, amqpConn.Close(), "expected no error when closing connection") - }() + // only check that there is no deadlock + defer amqpConn.Close() amqpChan, err := amqpConn.Channel() if err != nil { @@ -872,8 +871,9 @@ func TestChannelCloseWithDisconnect(t *testing.T) { disconnected() defer reconnected() - err = amqpChan.Close() - assert.NoError(t, err, "expected no error when closing channel") + + // only check that there is no deadlock + _ = amqpChan.Close() } func TestNewSingleSessionCloseWithDisconnect(t *testing.T) { From a5a8c646577f805acd999d55b066dd8562e7e4ce Mon Sep 17 00:00:00 2001 From: John Behm Date: Fri, 15 Mar 2024 16:36:02 +0100 Subject: [PATCH 69/76] make two amwpx sub/sub tests more robust to connection loss --- amqpx_test.go | 44 +++++++++++++++++++++++++++++++------------- 1 file changed, 31 insertions(+), 13 deletions(-) diff --git a/amqpx_test.go b/amqpx_test.go index 7aec1ac..0ec121d 100644 --- a/amqpx_test.go +++ b/amqpx_test.go @@ -183,9 +183,13 @@ func TestAMQPXSubAndPub(t *testing.T) { amqp.RegisterTopologyCreator(createTopology(log, eq1)) amqp.RegisterTopologyDeleter(deleteTopology(log, eq1)) - amqp.RegisterHandler(eq1.Queue, func(ctx context.Context, msg pool.Delivery) error { - log.Infof("subscriber of %s", eq1.Queue) - assert.Equal(t, eq1.NextSubMsg(), string(msg.Body)) + // we only expect a single message to arrive or + // a duplicate message due to network issues. + expectedMsg := eq1.NextSubMsg() + amqp.RegisterHandler(eq1.Queue, func(ctx context.Context, d pool.Delivery) error { + msg := string(d.Body) + log.Infof("subscriber of %s received message: %s", eq1.Queue, msg) + assert.Equal(t, expectedMsg, msg) cancel() return nil }) @@ -204,8 +208,12 @@ func TestAMQPXSubAndPub(t *testing.T) { } // publish event to first queue - - err = amqp.Publish(cctx, eq1.Exchange, eq1.RoutingKey, pool.Publishing{ + // due to bad network it is possible that the message is received multiple times. + // that is why we pass context.TODO() to the publish method in order to avoid + // aborting a secondary retry which would return an error here. + tctx, tcancel := context.WithTimeout(context.TODO(), 10*time.Second) + defer tcancel() + err = amqp.Publish(tctx, eq1.Exchange, eq1.RoutingKey, pool.Publishing{ ContentType: "text/plain", Body: []byte(eq1.NextPubMsg()), }) @@ -326,8 +334,10 @@ func TestAMQPXSubHandler(t *testing.T) { nextExchangeQueue = testutils.NewExchangeQueueGenerator(funcName) eq1 = nextExchangeQueue() ) - defer cancel() defer func() { + log.Info("canceling context") + cancel() + log.Info("closing amqp") assert.NoError(t, amqp.Close()) }() @@ -335,9 +345,13 @@ func TestAMQPXSubHandler(t *testing.T) { amqp.RegisterTopologyCreator(createTopology(log, eq1)) amqp.RegisterTopologyDeleter(deleteTopology(log, eq1)) - amqp.RegisterHandler(eq1.Queue, func(ctx context.Context, msg pool.Delivery) error { - log.Infof("subscriber of %s", eq1.Queue) - assert.Equal(t, eq1.NextSubMsg(), string(msg.Body)) + // we want to only receive one message or one duplicate message + expectedMsg := eq1.NextSubMsg() + amqp.RegisterHandler(eq1.Queue, func(ctx context.Context, d pool.Delivery) error { + msg := string(d.Body) + log.Infof("subscriber of %s: received message: %s", eq1.Queue, msg) + assert.Equal(t, expectedMsg, msg) + log.Info("canceling context from within handler") cancel() return nil }) @@ -356,8 +370,12 @@ func TestAMQPXSubHandler(t *testing.T) { } // publish event to first queue - - err = amqp.Publish(cctx, eq1.Exchange, eq1.RoutingKey, pool.Publishing{ + // due to bad network it is possible that the message is received multiple times. + // that is why we pass context.TODO() to the publish method in order to avoid + // aborting a secondary retry which would return an error here. + tctx, tcancel := context.WithTimeout(context.TODO(), 10*time.Second) + defer tcancel() + err = amqp.Publish(tctx, eq1.Exchange, eq1.RoutingKey, pool.Publishing{ ContentType: "text/plain", Body: []byte(eq1.NextPubMsg()), }) @@ -858,7 +876,7 @@ func TestQueueDeletedConsumerReconnect(t *testing.T) { _, err = ts.QueueDelete(cctx, queueName) assert.NoError(t, err) - tctx, tcancel := context.WithTimeout(cctx, 5*time.Second) + tctx, tcancel := context.WithTimeout(cctx, 10*time.Second) err = h.Resume(tctx) tcancel() assert.Error(t, err) @@ -867,7 +885,7 @@ func TestQueueDeletedConsumerReconnect(t *testing.T) { _, err = ts.QueueDeclare(cctx, queueName) assert.NoError(t, err) - tctx, tcancel = context.WithTimeout(cctx, 5*time.Second) + tctx, tcancel = context.WithTimeout(cctx, 10*time.Second) err = h.Resume(tctx) tcancel() assert.NoError(t, err) From e8a7a25447d0212ea163a76ae4897d7c400879d7 Mon Sep 17 00:00:00 2001 From: John Behm Date: Fri, 15 Mar 2024 19:41:24 +0100 Subject: [PATCH 70/76] update toxiproxy test container --- docker-compose.yaml | 2 +- pool/connection_pool_test.go | 4 ++++ pool/publisher_test.go | 6 +++++- 3 files changed, 10 insertions(+), 2 deletions(-) diff --git a/docker-compose.yaml b/docker-compose.yaml index 1664ce0..d9a3ab1 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -44,7 +44,7 @@ services: - rabbitnet toxiproxy: - image: ghcr.io/shopify/toxiproxy:2.7.0 + image: ghcr.io/shopify/toxiproxy:2.9.0 command: - -host=0.0.0.0 - -config=/toxiproxy.json diff --git a/pool/connection_pool_test.go b/pool/connection_pool_test.go index d7275c1..7ecff45 100644 --- a/pool/connection_pool_test.go +++ b/pool/connection_pool_test.go @@ -98,6 +98,7 @@ func TestNewConnectionPoolWithDisconnect(t *testing.T) { var ( ctx = context.TODO() + log = logging.NewTestLogger(t) poolName = testutils.FuncName() proxyName, connectURL, _ = testutils.NextConnectURL() ) @@ -109,6 +110,9 @@ func TestNewConnectionPoolWithDisconnect(t *testing.T) { connections, pool.ConnectionPoolWithName(poolName), pool.ConnectionPoolWithLogger(logging.NewTestLogger(t)), + pool.ConnectionPoolWithRecoverCallback(func(name string, retry int, err error) { + log.Warnf("recovering connection %s, attempt %d, error: %v", name, retry, err) + }), ) if err != nil { assert.NoError(t, err) diff --git a/pool/publisher_test.go b/pool/publisher_test.go index 37d8ce9..9e196e5 100644 --- a/pool/publisher_test.go +++ b/pool/publisher_test.go @@ -19,8 +19,9 @@ func TestSinglePublisher(t *testing.T) { var ( proxyName, connectURL, _ = testutils.NextConnectURL() ctx = context.TODO() + log = logging.NewTestLogger(t) nextConnName = testutils.ConnectionNameGenerator() - numMsgs = 10 + numMsgs = 5 ) hs, hsclose := NewSession( @@ -38,6 +39,9 @@ func TestSinglePublisher(t *testing.T) { 1, pool.WithLogger(logging.NewTestLogger(t)), pool.WithConfirms(true), + pool.WithConnectionRecoverCallback(func(name string, retry int, err error) { + log.Warnf("connection %s is broken, retry %d, error: %s", name, retry, err) + }), ) if err != nil { assert.NoError(t, err) From 80179549776b4a38d811f2e1c48856c1d6c0b0c4 Mon Sep 17 00:00:00 2001 From: John Behm Date: Fri, 15 Mar 2024 19:54:41 +0100 Subject: [PATCH 71/76] increase close timeout --- amqpx_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/amqpx_test.go b/amqpx_test.go index 0ec121d..202c6f1 100644 --- a/amqpx_test.go +++ b/amqpx_test.go @@ -1091,7 +1091,6 @@ func TestBatchHandlerReset(t *testing.T) { for i := 0; i < 5; i++ { testBatchHandlerReset(t, i) } - t.Log("done") } func testBatchHandlerReset(t *testing.T, i int) { @@ -1112,9 +1111,10 @@ func testBatchHandlerReset(t *testing.T, i int) { defer cancel() options := []amqpx.Option{ - amqpx.WithLogger(logging.NewNoOpLogger()), + amqpx.WithLogger(log), amqpx.WithPublisherConnections(1), amqpx.WithPublisherSessions(5), + amqpx.WithCloseTimeout(60 * time.Second), } amqpxPublish := amqpx.New() From 3d39586612f04c1bd54485b7c48897aacca4f9aa Mon Sep 17 00:00:00 2001 From: John Behm Date: Fri, 15 Mar 2024 20:01:45 +0100 Subject: [PATCH 72/76] run less tests in parallel --- amqpx_test.go | 14 -------------- pool/session_test.go | 18 ++---------------- 2 files changed, 2 insertions(+), 30 deletions(-) diff --git a/amqpx_test.go b/amqpx_test.go index 202c6f1..fbdde41 100644 --- a/amqpx_test.go +++ b/amqpx_test.go @@ -21,7 +21,6 @@ func TestMain(m *testing.M) { } func TestExchangeDeclarePassive(t *testing.T) { - t.Parallel() var ( amqp = amqpx.New() @@ -56,7 +55,6 @@ func TestExchangeDeclarePassive(t *testing.T) { } func TestQueueDeclarePassive(t *testing.T) { - t.Parallel() var ( amqp = amqpx.New() @@ -91,7 +89,6 @@ func TestQueueDeclarePassive(t *testing.T) { } func TestAMQPXPub(t *testing.T) { - t.Parallel() var ( amqp = amqpx.New() @@ -164,7 +161,6 @@ func TestAMQPXPub(t *testing.T) { } func TestAMQPXSubAndPub(t *testing.T) { - t.Parallel() var ( amqp = amqpx.New() @@ -228,7 +224,6 @@ func TestAMQPXSubAndPub(t *testing.T) { } func TestAMQPXSubAndPubMulti(t *testing.T) { - t.Parallel() var ( amqp = amqpx.New() @@ -324,7 +319,6 @@ func TestAMQPXSubAndPubMulti(t *testing.T) { } func TestAMQPXSubHandler(t *testing.T) { - t.Parallel() var ( amqp = amqpx.New() @@ -390,7 +384,6 @@ func TestAMQPXSubHandler(t *testing.T) { } func TestCreateDeleteTopology(t *testing.T) { - t.Parallel() var ( amqp = amqpx.New() @@ -421,7 +414,6 @@ func TestCreateDeleteTopology(t *testing.T) { } func TestPauseResumeHandlerNoProcessing(t *testing.T) { - t.Parallel() var ( amqp = amqpx.New() @@ -492,7 +484,6 @@ func TestPauseResumeHandlerNoProcessing(t *testing.T) { } func TestHandlerPauseAndResume(t *testing.T) { - t.Parallel() var wg sync.WaitGroup defer wg.Wait() @@ -656,7 +647,6 @@ func testHandlerPauseAndResume(t *testing.T, i int) { } func TestBatchHandlerPauseAndResume(t *testing.T) { - t.Parallel() var wg sync.WaitGroup defer wg.Wait() @@ -818,7 +808,6 @@ func testBatchHandlerPauseAndResume(t *testing.T, i int) { } func TestQueueDeletedConsumerReconnect(t *testing.T) { - t.Parallel() var ( err error @@ -893,7 +882,6 @@ func TestQueueDeletedConsumerReconnect(t *testing.T) { } func TestQueueDeletedBatchConsumerReconnect(t *testing.T) { - t.Parallel() var ( err error @@ -989,7 +977,6 @@ type handlerStats interface { } func TestHandlerReset(t *testing.T) { - t.Parallel() for i := 0; i < 5; i++ { testHandlerReset(t, i) @@ -1086,7 +1073,6 @@ func testHandlerReset(t *testing.T, i int) { } func TestBatchHandlerReset(t *testing.T) { - t.Parallel() for i := 0; i < 5; i++ { testBatchHandlerReset(t, i) diff --git a/pool/session_test.go b/pool/session_test.go index ff5f4b8..be0c465 100644 --- a/pool/session_test.go +++ b/pool/session_test.go @@ -14,7 +14,6 @@ import ( ) func TestNewSingleSessionPublishAndConsume(t *testing.T) { - t.Parallel() // can be run in parallel because the connection to the rabbitmq is never broken var ( ctx = context.TODO() @@ -76,7 +75,6 @@ func TestNewSingleSessionPublishAndConsume(t *testing.T) { } func TestManyNewSessionsPublishAndConsume(t *testing.T) { - t.Parallel() // can be run in parallel because the connection to the rabbitmq is never broken var ( ctx = context.TODO() @@ -140,7 +138,6 @@ func TestManyNewSessionsPublishAndConsume(t *testing.T) { } func TestNewSessionQueueDeclarePassive(t *testing.T) { - t.Parallel() // can be run in parallel because the connection to the rabbitmq is never broken var ( ctx = context.TODO() @@ -214,7 +211,6 @@ func TestNewSessionQueueDeclarePassive(t *testing.T) { } func TestNewSessionExchangeDeclareWithDisconnect(t *testing.T) { - t.Parallel() var ( ctx = context.TODO() @@ -267,7 +263,6 @@ func TestNewSessionExchangeDeclareWithDisconnect(t *testing.T) { } func TestNewSessionExchangeDeleteWithDisconnect(t *testing.T) { - t.Parallel() var ( ctx = context.TODO() @@ -321,7 +316,6 @@ func TestNewSessionExchangeDeleteWithDisconnect(t *testing.T) { } func TestNewSessionQueueDeclareWithDisconnect(t *testing.T) { - t.Parallel() var ( ctx = context.TODO() @@ -377,7 +371,6 @@ func TestNewSessionQueueDeclareWithDisconnect(t *testing.T) { } func TestNewSessionQueueDeleteWithDisconnect(t *testing.T) { - t.Parallel() var ( ctx = context.TODO() @@ -431,7 +424,6 @@ func TestNewSessionQueueDeleteWithDisconnect(t *testing.T) { } func TestNewSessionQueueBindWithDisconnect(t *testing.T) { - t.Parallel() var ( ctx = context.TODO() @@ -505,7 +497,6 @@ func TestNewSessionQueueBindWithDisconnect(t *testing.T) { } func TestNewSessionQueueUnbindWithDisconnect(t *testing.T) { - t.Parallel() var ( ctx = context.TODO() @@ -582,7 +573,6 @@ func TestNewSessionQueueUnbindWithDisconnect(t *testing.T) { } func TestNewSessionPublishWithDisconnect(t *testing.T) { - t.Parallel() var ( proxyName, connectURL, _ = testutils.NextConnectURL() @@ -636,7 +626,6 @@ func TestNewSessionPublishWithDisconnect(t *testing.T) { } func TestNewSessionConsumeWithDisconnect(t *testing.T) { - t.Parallel() var ( proxyName, connectURL, _ = testutils.NextConnectURL() @@ -692,7 +681,7 @@ func TestNewSessionConsumeWithDisconnect(t *testing.T) { /* // FIXME: ou of memory tests are disabled until https://github.com/rabbitmq/amqp091-go/issues/253 is resolved func TestChannelFullChainOnOutOfMemoryRabbitMQ(t *testing.T) { - t.Parallel() + var ( ctx = context.TODO() @@ -848,7 +837,6 @@ func TestChannelFullChainOnOutOfMemoryRabbitMQ(t *testing.T) { */ func TestChannelCloseWithDisconnect(t *testing.T) { - t.Parallel() var ( proxyName, connectURL, _ = testutils.NextConnectURL() @@ -877,7 +865,6 @@ func TestChannelCloseWithDisconnect(t *testing.T) { } func TestNewSingleSessionCloseWithDisconnect(t *testing.T) { - t.Parallel() // can be run in parallel because the connection to the rabbitmq is never broken var ( ctx = context.TODO() @@ -927,7 +914,7 @@ func TestNewSingleSessionCloseWithDisconnect(t *testing.T) { /* // FIXME: out of memory tests are disabled until https://github.com/rabbitmq/amqp091-go/issues/253 is resolved func TestNewSingleSessionCloseWithOutOfMemoryRabbitMQ(t *testing.T) { - t.Parallel() // can be run in parallel because the connection to the rabbitmq is never broken + var ( ctx = context.TODO() @@ -999,7 +986,6 @@ func TestNewSingleSessionCloseWithOutOfMemoryRabbitMQ(t *testing.T) { */ func TestNewSingleSessionCloseWithHealthyRabbitMQ(t *testing.T) { - t.Parallel() // can be run in parallel because the connection to the rabbitmq is never broken var ( ctx = context.TODO() From acda79f3dae0029b87092fbcb03f6cfd8005c40c Mon Sep 17 00:00:00 2001 From: John Behm Date: Fri, 15 Mar 2024 20:06:31 +0100 Subject: [PATCH 73/76] decrease toxiproxy ports --- docker-compose.yaml | 53 ------- docker/toxiproxy.json | 318 ------------------------------------------ 2 files changed, 371 deletions(-) diff --git a/docker-compose.yaml b/docker-compose.yaml index d9a3ab1..0e41485 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -97,59 +97,6 @@ services: - 127.0.0.1:5716:5716 # rabbitmq-5716 - 127.0.0.1:5717:5717 # rabbitmq-5717 - 127.0.0.1:5718:5718 # rabbitmq-5718 - - 127.0.0.1:5719:5719 # rabbitmq-5719 - - 127.0.0.1:5720:5720 # rabbitmq-5720 - - 127.0.0.1:5721:5721 # rabbitmq-5721 - - 127.0.0.1:5722:5722 # rabbitmq-5722 - - 127.0.0.1:5723:5723 # rabbitmq-5723 - - 127.0.0.1:5724:5724 # rabbitmq-5724 - - 127.0.0.1:5725:5725 # rabbitmq-5725 - - 127.0.0.1:5726:5726 # rabbitmq-5726 - - 127.0.0.1:5727:5727 # rabbitmq-5727 - - 127.0.0.1:5728:5728 # rabbitmq-5728 - - 127.0.0.1:5729:5729 # rabbitmq-5729 - - 127.0.0.1:5730:5730 # rabbitmq-5730 - - 127.0.0.1:5731:5731 # rabbitmq-5731 - - 127.0.0.1:5732:5732 # rabbitmq-5732 - - 127.0.0.1:5733:5733 # rabbitmq-5733 - - 127.0.0.1:5734:5734 # rabbitmq-5734 - - 127.0.0.1:5735:5735 # rabbitmq-5735 - - 127.0.0.1:5736:5736 # rabbitmq-5736 - - 127.0.0.1:5737:5737 # rabbitmq-5737 - - 127.0.0.1:5738:5738 # rabbitmq-5738 - - 127.0.0.1:5739:5739 # rabbitmq-5739 - - 127.0.0.1:5740:5740 # rabbitmq-5740 - - 127.0.0.1:5741:5741 # rabbitmq-5741 - - 127.0.0.1:5742:5742 # rabbitmq-5742 - - 127.0.0.1:5743:5743 # rabbitmq-5743 - - 127.0.0.1:5744:5744 # rabbitmq-5744 - - 127.0.0.1:5745:5745 # rabbitmq-5745 - - 127.0.0.1:5746:5746 # rabbitmq-5746 - - 127.0.0.1:5747:5747 # rabbitmq-5747 - - 127.0.0.1:5748:5748 # rabbitmq-5748 - - 127.0.0.1:5749:5749 # rabbitmq-5749 - - 127.0.0.1:5750:5750 # rabbitmq-5750 - - 127.0.0.1:5751:5751 # rabbitmq-5751 - - 127.0.0.1:5752:5752 # rabbitmq-5752 - - 127.0.0.1:5753:5753 # rabbitmq-5753 - - 127.0.0.1:5754:5754 # rabbitmq-5754 - - 127.0.0.1:5755:5755 # rabbitmq-5755 - - 127.0.0.1:5756:5756 # rabbitmq-5756 - - 127.0.0.1:5757:5757 # rabbitmq-5757 - - 127.0.0.1:5758:5758 # rabbitmq-5758 - - 127.0.0.1:5759:5759 # rabbitmq-5759 - - 127.0.0.1:5760:5760 # rabbitmq-5760 - - 127.0.0.1:5761:5761 # rabbitmq-5761 - - 127.0.0.1:5762:5762 # rabbitmq-5762 - - 127.0.0.1:5763:5763 # rabbitmq-5763 - - 127.0.0.1:5764:5764 # rabbitmq-5764 - - 127.0.0.1:5765:5765 # rabbitmq-5765 - - 127.0.0.1:5766:5766 # rabbitmq-5766 - - 127.0.0.1:5767:5767 # rabbitmq-5767 - - 127.0.0.1:5768:5768 # rabbitmq-5768 - - 127.0.0.1:5769:5769 # rabbitmq-5769 - - 127.0.0.1:5770:5770 # rabbitmq-5770 - - 127.0.0.1:5771:5771 # rabbitmq-5771 networks: - rabbitnet diff --git a/docker/toxiproxy.json b/docker/toxiproxy.json index cee8368..c87a07a 100644 --- a/docker/toxiproxy.json +++ b/docker/toxiproxy.json @@ -280,323 +280,5 @@ "listen": "[::]:5718", "upstream": "rabbitmq:5672", "enabled": true - }, - { - "name": "rabbitmq-5719", - "listen": "[::]:5719", - "upstream": "rabbitmq:5672", - "enabled": true - }, - { - "name": "rabbitmq-5720", - "listen": "[::]:5720", - "upstream": "rabbitmq:5672", - "enabled": true - }, - { - "name": "rabbitmq-5721", - "listen": "[::]:5721", - "upstream": "rabbitmq:5672", - "enabled": true - }, - { - "name": "rabbitmq-5722", - "listen": "[::]:5722", - "upstream": "rabbitmq:5672", - "enabled": true - }, - { - "name": "rabbitmq-5723", - "listen": "[::]:5723", - "upstream": "rabbitmq:5672", - "enabled": true - }, - { - "name": "rabbitmq-5724", - "listen": "[::]:5724", - "upstream": "rabbitmq:5672", - "enabled": true - }, - { - "name": "rabbitmq-5725", - "listen": "[::]:5725", - "upstream": "rabbitmq:5672", - "enabled": true - }, - { - "name": "rabbitmq-5726", - "listen": "[::]:5726", - "upstream": "rabbitmq:5672", - "enabled": true - }, - { - "name": "rabbitmq-5727", - "listen": "[::]:5727", - "upstream": "rabbitmq:5672", - "enabled": true - }, - { - "name": "rabbitmq-5728", - "listen": "[::]:5728", - "upstream": "rabbitmq:5672", - "enabled": true - }, - { - "name": "rabbitmq-5729", - "listen": "[::]:5729", - "upstream": "rabbitmq:5672", - "enabled": true - }, - { - "name": "rabbitmq-5730", - "listen": "[::]:5730", - "upstream": "rabbitmq:5672", - "enabled": true - }, - { - "name": "rabbitmq-5731", - "listen": "[::]:5731", - "upstream": "rabbitmq:5672", - "enabled": true - }, - { - "name": "rabbitmq-5732", - "listen": "[::]:5732", - "upstream": "rabbitmq:5672", - "enabled": true - }, - { - "name": "rabbitmq-5733", - "listen": "[::]:5733", - "upstream": "rabbitmq:5672", - "enabled": true - }, - { - "name": "rabbitmq-5734", - "listen": "[::]:5734", - "upstream": "rabbitmq:5672", - "enabled": true - }, - { - "name": "rabbitmq-5735", - "listen": "[::]:5735", - "upstream": "rabbitmq:5672", - "enabled": true - }, - { - "name": "rabbitmq-5736", - "listen": "[::]:5736", - "upstream": "rabbitmq:5672", - "enabled": true - }, - { - "name": "rabbitmq-5737", - "listen": "[::]:5737", - "upstream": "rabbitmq:5672", - "enabled": true - }, - { - "name": "rabbitmq-5738", - "listen": "[::]:5738", - "upstream": "rabbitmq:5672", - "enabled": true - }, - { - "name": "rabbitmq-5739", - "listen": "[::]:5739", - "upstream": "rabbitmq:5672", - "enabled": true - }, - { - "name": "rabbitmq-5740", - "listen": "[::]:5740", - "upstream": "rabbitmq:5672", - "enabled": true - }, - { - "name": "rabbitmq-5741", - "listen": "[::]:5741", - "upstream": "rabbitmq:5672", - "enabled": true - }, - { - "name": "rabbitmq-5742", - "listen": "[::]:5742", - "upstream": "rabbitmq:5672", - "enabled": true - }, - { - "name": "rabbitmq-5743", - "listen": "[::]:5743", - "upstream": "rabbitmq:5672", - "enabled": true - }, - { - "name": "rabbitmq-5744", - "listen": "[::]:5744", - "upstream": "rabbitmq:5672", - "enabled": true - }, - { - "name": "rabbitmq-5745", - "listen": "[::]:5745", - "upstream": "rabbitmq:5672", - "enabled": true - }, - { - "name": "rabbitmq-5746", - "listen": "[::]:5746", - "upstream": "rabbitmq:5672", - "enabled": true - }, - { - "name": "rabbitmq-5747", - "listen": "[::]:5747", - "upstream": "rabbitmq:5672", - "enabled": true - }, - { - "name": "rabbitmq-5748", - "listen": "[::]:5748", - "upstream": "rabbitmq:5672", - "enabled": true - }, - { - "name": "rabbitmq-5749", - "listen": "[::]:5749", - "upstream": "rabbitmq:5672", - "enabled": true - }, - { - "name": "rabbitmq-5750", - "listen": "[::]:5750", - "upstream": "rabbitmq:5672", - "enabled": true - }, - { - "name": "rabbitmq-5751", - "listen": "[::]:5751", - "upstream": "rabbitmq:5672", - "enabled": true - }, - { - "name": "rabbitmq-5752", - "listen": "[::]:5752", - "upstream": "rabbitmq:5672", - "enabled": true - }, - { - "name": "rabbitmq-5753", - "listen": "[::]:5753", - "upstream": "rabbitmq:5672", - "enabled": true - }, - { - "name": "rabbitmq-5754", - "listen": "[::]:5754", - "upstream": "rabbitmq:5672", - "enabled": true - }, - { - "name": "rabbitmq-5755", - "listen": "[::]:5755", - "upstream": "rabbitmq:5672", - "enabled": true - }, - { - "name": "rabbitmq-5756", - "listen": "[::]:5756", - "upstream": "rabbitmq:5672", - "enabled": true - }, - { - "name": "rabbitmq-5757", - "listen": "[::]:5757", - "upstream": "rabbitmq:5672", - "enabled": true - }, - { - "name": "rabbitmq-5758", - "listen": "[::]:5758", - "upstream": "rabbitmq:5672", - "enabled": true - }, - { - "name": "rabbitmq-5759", - "listen": "[::]:5759", - "upstream": "rabbitmq:5672", - "enabled": true - }, - { - "name": "rabbitmq-5760", - "listen": "[::]:5760", - "upstream": "rabbitmq:5672", - "enabled": true - }, - { - "name": "rabbitmq-5761", - "listen": "[::]:5761", - "upstream": "rabbitmq:5672", - "enabled": true - }, - { - "name": "rabbitmq-5762", - "listen": "[::]:5762", - "upstream": "rabbitmq:5672", - "enabled": true - }, - { - "name": "rabbitmq-5763", - "listen": "[::]:5763", - "upstream": "rabbitmq:5672", - "enabled": true - }, - { - "name": "rabbitmq-5764", - "listen": "[::]:5764", - "upstream": "rabbitmq:5672", - "enabled": true - }, - { - "name": "rabbitmq-5765", - "listen": "[::]:5765", - "upstream": "rabbitmq:5672", - "enabled": true - }, - { - "name": "rabbitmq-5766", - "listen": "[::]:5766", - "upstream": "rabbitmq:5672", - "enabled": true - }, - { - "name": "rabbitmq-5767", - "listen": "[::]:5767", - "upstream": "rabbitmq:5672", - "enabled": true - }, - { - "name": "rabbitmq-5768", - "listen": "[::]:5768", - "upstream": "rabbitmq:5672", - "enabled": true - }, - { - "name": "rabbitmq-5769", - "listen": "[::]:5769", - "upstream": "rabbitmq:5672", - "enabled": true - }, - { - "name": "rabbitmq-5770", - "listen": "[::]:5770", - "upstream": "rabbitmq:5672", - "enabled": true - }, - { - "name": "rabbitmq-5771", - "listen": "[::]:5771", - "upstream": "rabbitmq:5672", - "enabled": true } ] \ No newline at end of file From f5cfb80fa056ef453f42c725f0aa631e00d41c39 Mon Sep 17 00:00:00 2001 From: John Behm Date: Fri, 15 Mar 2024 20:10:42 +0100 Subject: [PATCH 74/76] remove parallel tests --- pool/connection_test.go | 5 +---- pool/pool_test.go | 1 - pool/session_pool_test.go | 2 -- pool/subscriber_test.go | 4 ---- 4 files changed, 1 insertion(+), 11 deletions(-) diff --git a/pool/connection_test.go b/pool/connection_test.go index 2dbf46d..34cf2f0 100644 --- a/pool/connection_test.go +++ b/pool/connection_test.go @@ -13,7 +13,7 @@ import ( ) func TestNewSingleConnection(t *testing.T) { - t.Parallel() // can be run in parallel because the connection to the rabbitmq is never broken + var ( ctx = context.TODO() nextName = testutils.ConnectionNameGenerator() @@ -36,7 +36,6 @@ func TestNewSingleConnection(t *testing.T) { } func TestManyNewConnection(t *testing.T) { - t.Parallel() // can be run in parallel because the connection to the rabbitmq is never broken var ( ctx = context.TODO() @@ -74,7 +73,6 @@ func TestManyNewConnection(t *testing.T) { } func TestNewSingleConnectionWithDisconnect(t *testing.T) { - t.Parallel() var ( ctx = context.TODO() proxyName, connectURL, _ = testutils.NextConnectURL() @@ -101,7 +99,6 @@ func TestNewSingleConnectionWithDisconnect(t *testing.T) { } func TestManyNewConnectionWithDisconnect(t *testing.T) { - t.Parallel() var ( ctx = context.TODO() diff --git a/pool/pool_test.go b/pool/pool_test.go index e95a111..420dfe0 100644 --- a/pool/pool_test.go +++ b/pool/pool_test.go @@ -17,7 +17,6 @@ func TestMain(m *testing.M) { } func TestNewPool(t *testing.T) { - t.Parallel() // can be run in parallel because the connection to the rabbitmq is never broken var ( ctx = context.TODO() diff --git a/pool/session_pool_test.go b/pool/session_pool_test.go index e30a518..77b4a1e 100644 --- a/pool/session_pool_test.go +++ b/pool/session_pool_test.go @@ -13,7 +13,6 @@ import ( ) func TestSingleSessionPool(t *testing.T) { - t.Parallel() // can be run in parallel because the connection to the rabbitmq is never var ( poolName = testutils.FuncName() @@ -56,7 +55,6 @@ func TestSingleSessionPool(t *testing.T) { } func TestNewSessionPool(t *testing.T) { - t.Parallel() // can be run in parallel because the connection to the rabbitmq is never var ( poolName = testutils.FuncName() diff --git a/pool/subscriber_test.go b/pool/subscriber_test.go index c86044b..4fb5344 100644 --- a/pool/subscriber_test.go +++ b/pool/subscriber_test.go @@ -14,7 +14,6 @@ import ( ) func TestNewSingleSubscriber(t *testing.T) { - t.Parallel() var ( ctx = context.TODO() @@ -54,7 +53,6 @@ func TestNewSingleSubscriber(t *testing.T) { } func TestNewSingleSubscriberWithDisconnect(t *testing.T) { - t.Parallel() var ( ctx = context.TODO() @@ -99,7 +97,6 @@ func TestNewSingleSubscriberWithDisconnect(t *testing.T) { } func TestNewSingleBatchSubscriber(t *testing.T) { - t.Parallel() var ( ctx = context.TODO() @@ -152,7 +149,6 @@ func TestNewSingleBatchSubscriber(t *testing.T) { } func TestBatchSubscriberMaxBytes(t *testing.T) { - t.Parallel() var wg sync.WaitGroup funcName := testutils.FuncName() From 626c0cdc9382832a3f2714f8f66a35bd1105253c Mon Sep 17 00:00:00 2001 From: John Behm Date: Fri, 15 Mar 2024 20:43:55 +0100 Subject: [PATCH 75/76] allow parallel tests by default but run them sequentially in the CI pipeline --- .github/workflows/test.yaml | 2 +- .gitignore | 1 + Makefile | 7 ++++-- README.md | 3 +++ amqpx_test.go | 28 ++++++++++----------- internal/testutils/connect_url_test.go | 8 ++++-- pool/connection_pool_test.go | 4 +-- pool/connection_test.go | 5 ++-- pool/pool_test.go | 2 +- pool/session_pool_test.go | 4 +-- pool/session_test.go | 35 +++++++++++++------------- pool/subscriber_test.go | 6 ++--- 12 files changed, 58 insertions(+), 47 deletions(-) diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 7b584e1..372fe79 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -71,7 +71,7 @@ jobs: run: docker-compose up -d - name: Code Coverage - run: go test -timeout 900s -race -count=1 -covermode=atomic -coverprofile=coverage.txt ./... + run: go test -timeout 900s -race -count=1 -parallel 1 -covermode=atomic -coverprofile=coverage.txt ./... - name: Upload coverage to Codecov uses: codecov/codecov-action@v4 diff --git a/.gitignore b/.gitignore index c0df5a7..576e75d 100644 --- a/.gitignore +++ b/.gitignore @@ -5,3 +5,4 @@ coverage.txt DEBUG.md debug.md __debug_bin* +*.log diff --git a/Makefile b/Makefile index 0da8be8..2a3c0e5 100644 --- a/Makefile +++ b/Makefile @@ -8,7 +8,10 @@ down: docker-compose down test: - go test -timeout 300s -v -race -count=1 ./... + go test -timeout 600ss -v -race -count=1 ./... > parallel.test.log + +test-sequentially: + go test -timeout 900s -v -race -parallel 1 -count=1 ./... > sequential.test.log count-tests: grep -REn 'func Test.+\(.+testing\.T.*\)' . | wc -l @@ -18,5 +21,5 @@ count-disconnect-tests: pool.TestBatchSubscriberMaxBytes: - go test -timeout 0m30s github.com/jxsl13/amqpx/pool -run ^TestBatchSubscriberMaxBytes$ -v -count=1 -race 2>&1 > test.log + go test -timeout 0m30s github.com/jxsl13/amqpx/pool -run ^TestBatchSubscriberMaxBytes$ -v -count=1 -race 2>&1 > debug.test.log cat test.log | grep 'INFO: session' | sort | uniq -c \ No newline at end of file diff --git a/README.md b/README.md index 91ff850..7c842e7 100644 --- a/README.md +++ b/README.md @@ -194,6 +194,9 @@ A `Subscriber` must be `Start()`ed in order for it to create consumer goroutines ## Development +Tests can all be run in parallel but the parallel testing is disabled for now because of the GitHub runners starting to behave weirdly when under such a load. +That is why those tests were disabled for the CI pipeline. + Test flags you might want to add: ```shell go test -v -race -count=1 ./... diff --git a/amqpx_test.go b/amqpx_test.go index fbdde41..9a7eede 100644 --- a/amqpx_test.go +++ b/amqpx_test.go @@ -21,7 +21,7 @@ func TestMain(m *testing.M) { } func TestExchangeDeclarePassive(t *testing.T) { - + t.Parallel() var ( amqp = amqpx.New() ctx = context.TODO() @@ -55,7 +55,7 @@ func TestExchangeDeclarePassive(t *testing.T) { } func TestQueueDeclarePassive(t *testing.T) { - + t.Parallel() var ( amqp = amqpx.New() ctx = context.TODO() @@ -89,7 +89,7 @@ func TestQueueDeclarePassive(t *testing.T) { } func TestAMQPXPub(t *testing.T) { - + t.Parallel() var ( amqp = amqpx.New() ctx = context.TODO() @@ -161,7 +161,7 @@ func TestAMQPXPub(t *testing.T) { } func TestAMQPXSubAndPub(t *testing.T) { - + t.Parallel() var ( amqp = amqpx.New() log = logging.NewTestLogger(t) @@ -224,7 +224,7 @@ func TestAMQPXSubAndPub(t *testing.T) { } func TestAMQPXSubAndPubMulti(t *testing.T) { - + t.Parallel() var ( amqp = amqpx.New() log = logging.NewTestLogger(t) @@ -319,7 +319,7 @@ func TestAMQPXSubAndPubMulti(t *testing.T) { } func TestAMQPXSubHandler(t *testing.T) { - + t.Parallel() var ( amqp = amqpx.New() log = logging.NewTestLogger(t) @@ -384,7 +384,7 @@ func TestAMQPXSubHandler(t *testing.T) { } func TestCreateDeleteTopology(t *testing.T) { - + t.Parallel() var ( amqp = amqpx.New() log = logging.NewTestLogger(t) @@ -414,7 +414,7 @@ func TestCreateDeleteTopology(t *testing.T) { } func TestPauseResumeHandlerNoProcessing(t *testing.T) { - + t.Parallel() var ( amqp = amqpx.New() log = logging.NewTestLogger(t) @@ -484,7 +484,7 @@ func TestPauseResumeHandlerNoProcessing(t *testing.T) { } func TestHandlerPauseAndResume(t *testing.T) { - + t.Parallel() var wg sync.WaitGroup defer wg.Wait() @@ -647,7 +647,7 @@ func testHandlerPauseAndResume(t *testing.T, i int) { } func TestBatchHandlerPauseAndResume(t *testing.T) { - + t.Parallel() var wg sync.WaitGroup defer wg.Wait() for i := 0; i < 10; i++ { @@ -808,7 +808,7 @@ func testBatchHandlerPauseAndResume(t *testing.T, i int) { } func TestQueueDeletedConsumerReconnect(t *testing.T) { - + t.Parallel() var ( err error amqp = amqpx.New() @@ -882,7 +882,7 @@ func TestQueueDeletedConsumerReconnect(t *testing.T) { } func TestQueueDeletedBatchConsumerReconnect(t *testing.T) { - + t.Parallel() var ( err error amqp = amqpx.New() @@ -977,7 +977,7 @@ type handlerStats interface { } func TestHandlerReset(t *testing.T) { - + t.Parallel() for i := 0; i < 5; i++ { testHandlerReset(t, i) } @@ -1073,7 +1073,7 @@ func testHandlerReset(t *testing.T, i int) { } func TestBatchHandlerReset(t *testing.T) { - + t.Parallel() for i := 0; i < 5; i++ { testBatchHandlerReset(t, i) } diff --git a/internal/testutils/connect_url_test.go b/internal/testutils/connect_url_test.go index c06a0cb..146e13d 100644 --- a/internal/testutils/connect_url_test.go +++ b/internal/testutils/connect_url_test.go @@ -18,7 +18,9 @@ import ( ] */ -func TestGenerateProxyConfig(_ *testing.T) { +func TestGenerateProxyConfig(t *testing.T) { + t.Parallel() + list := []struct { Name string `json:"name"` Listen string `json:"listen"` @@ -47,7 +49,9 @@ func TestGenerateProxyConfig(_ *testing.T) { fmt.Println(string(data)) } -func TestGenerateDockerPortForwards(_ *testing.T) { +func TestGenerateDockerPortForwards(t *testing.T) { + t.Parallel() + nextConnectURL := NewConnectURLGenerator(ExcludedPorts...) str := "" diff --git a/pool/connection_pool_test.go b/pool/connection_pool_test.go index 7ecff45..c58bc46 100644 --- a/pool/connection_pool_test.go +++ b/pool/connection_pool_test.go @@ -13,7 +13,7 @@ import ( ) func TestNewSingleConnectionPool(t *testing.T) { - t.Parallel() // can be run in parallel because the connection to the rabbitmq is never broken + t.Parallel() poolName := testutils.FuncName() ctx := context.TODO() @@ -49,7 +49,7 @@ func TestNewSingleConnectionPool(t *testing.T) { } func TestNewConnectionPool(t *testing.T) { - t.Parallel() // can be run in parallel because the connection to the rabbitmq is never broken + t.Parallel() poolName := testutils.FuncName() diff --git a/pool/connection_test.go b/pool/connection_test.go index 34cf2f0..05f523a 100644 --- a/pool/connection_test.go +++ b/pool/connection_test.go @@ -13,7 +13,7 @@ import ( ) func TestNewSingleConnection(t *testing.T) { - + t.Parallel() var ( ctx = context.TODO() nextName = testutils.ConnectionNameGenerator() @@ -36,7 +36,7 @@ func TestNewSingleConnection(t *testing.T) { } func TestManyNewConnection(t *testing.T) { - + t.Parallel() var ( ctx = context.TODO() wg sync.WaitGroup @@ -73,6 +73,7 @@ func TestManyNewConnection(t *testing.T) { } func TestNewSingleConnectionWithDisconnect(t *testing.T) { + t.Parallel() var ( ctx = context.TODO() proxyName, connectURL, _ = testutils.NextConnectURL() diff --git a/pool/pool_test.go b/pool/pool_test.go index 420dfe0..6864eed 100644 --- a/pool/pool_test.go +++ b/pool/pool_test.go @@ -17,7 +17,7 @@ func TestMain(m *testing.M) { } func TestNewPool(t *testing.T) { - + t.Parallel() var ( ctx = context.TODO() poolName = testutils.FuncName() diff --git a/pool/session_pool_test.go b/pool/session_pool_test.go index 77b4a1e..966894d 100644 --- a/pool/session_pool_test.go +++ b/pool/session_pool_test.go @@ -13,7 +13,7 @@ import ( ) func TestSingleSessionPool(t *testing.T) { - + t.Parallel() var ( poolName = testutils.FuncName() ctx = context.TODO() @@ -55,7 +55,7 @@ func TestSingleSessionPool(t *testing.T) { } func TestNewSessionPool(t *testing.T) { - + t.Parallel() var ( poolName = testutils.FuncName() ctx = context.TODO() diff --git a/pool/session_test.go b/pool/session_test.go index be0c465..7e69c69 100644 --- a/pool/session_test.go +++ b/pool/session_test.go @@ -14,7 +14,7 @@ import ( ) func TestNewSingleSessionPublishAndConsume(t *testing.T) { - + t.Parallel() var ( ctx = context.TODO() wg sync.WaitGroup @@ -75,7 +75,7 @@ func TestNewSingleSessionPublishAndConsume(t *testing.T) { } func TestManyNewSessionsPublishAndConsume(t *testing.T) { - + t.Parallel() var ( ctx = context.TODO() wg sync.WaitGroup @@ -138,7 +138,7 @@ func TestManyNewSessionsPublishAndConsume(t *testing.T) { } func TestNewSessionQueueDeclarePassive(t *testing.T) { - + t.Parallel() var ( ctx = context.TODO() wg sync.WaitGroup @@ -211,7 +211,7 @@ func TestNewSessionQueueDeclarePassive(t *testing.T) { } func TestNewSessionExchangeDeclareWithDisconnect(t *testing.T) { - + t.Parallel() var ( ctx = context.TODO() proxyName, connectURL, _ = testutils.NextConnectURL() @@ -263,7 +263,7 @@ func TestNewSessionExchangeDeclareWithDisconnect(t *testing.T) { } func TestNewSessionExchangeDeleteWithDisconnect(t *testing.T) { - + t.Parallel() var ( ctx = context.TODO() proxyName, connectURL, _ = testutils.NextConnectURL() @@ -316,7 +316,7 @@ func TestNewSessionExchangeDeleteWithDisconnect(t *testing.T) { } func TestNewSessionQueueDeclareWithDisconnect(t *testing.T) { - + t.Parallel() var ( ctx = context.TODO() proxyName, connectURL, _ = testutils.NextConnectURL() @@ -371,7 +371,7 @@ func TestNewSessionQueueDeclareWithDisconnect(t *testing.T) { } func TestNewSessionQueueDeleteWithDisconnect(t *testing.T) { - + t.Parallel() var ( ctx = context.TODO() proxyName, connectURL, _ = testutils.NextConnectURL() @@ -424,7 +424,7 @@ func TestNewSessionQueueDeleteWithDisconnect(t *testing.T) { } func TestNewSessionQueueBindWithDisconnect(t *testing.T) { - + t.Parallel() var ( ctx = context.TODO() proxyName, connectURL, _ = testutils.NextConnectURL() @@ -497,7 +497,7 @@ func TestNewSessionQueueBindWithDisconnect(t *testing.T) { } func TestNewSessionQueueUnbindWithDisconnect(t *testing.T) { - + t.Parallel() var ( ctx = context.TODO() proxyName, connectURL, _ = testutils.NextConnectURL() @@ -573,7 +573,7 @@ func TestNewSessionQueueUnbindWithDisconnect(t *testing.T) { } func TestNewSessionPublishWithDisconnect(t *testing.T) { - + t.Parallel() var ( proxyName, connectURL, _ = testutils.NextConnectURL() ctx = context.TODO() @@ -626,7 +626,7 @@ func TestNewSessionPublishWithDisconnect(t *testing.T) { } func TestNewSessionConsumeWithDisconnect(t *testing.T) { - + t.Parallel() var ( proxyName, connectURL, _ = testutils.NextConnectURL() ctx = context.TODO() @@ -679,9 +679,9 @@ func TestNewSessionConsumeWithDisconnect(t *testing.T) { } /* -// FIXME: ou of memory tests are disabled until https://github.com/rabbitmq/amqp091-go/issues/253 is resolved +// FIXME: out of memory tests are disabled until https://github.com/rabbitmq/amqp091-go/issues/253 is resolved func TestChannelFullChainOnOutOfMemoryRabbitMQ(t *testing.T) { - + t.Parallel() var ( ctx = context.TODO() @@ -837,7 +837,7 @@ func TestChannelFullChainOnOutOfMemoryRabbitMQ(t *testing.T) { */ func TestChannelCloseWithDisconnect(t *testing.T) { - + t.Parallel() var ( proxyName, connectURL, _ = testutils.NextConnectURL() disconnected, reconnected = Disconnect(t, proxyName, 5*time.Second) @@ -865,7 +865,7 @@ func TestChannelCloseWithDisconnect(t *testing.T) { } func TestNewSingleSessionCloseWithDisconnect(t *testing.T) { - + t.Parallel() var ( ctx = context.TODO() nextConnName = testutils.ConnectionNameGenerator() @@ -914,8 +914,7 @@ func TestNewSingleSessionCloseWithDisconnect(t *testing.T) { /* // FIXME: out of memory tests are disabled until https://github.com/rabbitmq/amqp091-go/issues/253 is resolved func TestNewSingleSessionCloseWithOutOfMemoryRabbitMQ(t *testing.T) { - - + t.Parallel() var ( ctx = context.TODO() log = logging.NewTestLogger(t) @@ -986,7 +985,7 @@ func TestNewSingleSessionCloseWithOutOfMemoryRabbitMQ(t *testing.T) { */ func TestNewSingleSessionCloseWithHealthyRabbitMQ(t *testing.T) { - + t.Parallel() var ( ctx = context.TODO() log = logging.NewTestLogger(t) diff --git a/pool/subscriber_test.go b/pool/subscriber_test.go index 4fb5344..43275aa 100644 --- a/pool/subscriber_test.go +++ b/pool/subscriber_test.go @@ -14,7 +14,7 @@ import ( ) func TestNewSingleSubscriber(t *testing.T) { - + t.Parallel() var ( ctx = context.TODO() nextPoolName = testutils.PoolNameGenerator(testutils.FuncName()) @@ -53,7 +53,7 @@ func TestNewSingleSubscriber(t *testing.T) { } func TestNewSingleSubscriberWithDisconnect(t *testing.T) { - + t.Parallel() var ( ctx = context.TODO() nextPoolName = testutils.PoolNameGenerator(testutils.FuncName()) @@ -97,7 +97,7 @@ func TestNewSingleSubscriberWithDisconnect(t *testing.T) { } func TestNewSingleBatchSubscriber(t *testing.T) { - + t.Parallel() var ( ctx = context.TODO() nextPoolName = testutils.PoolNameGenerator(testutils.FuncName()) From 554e0a1e4f64df024b04424d23660230d34ade72 Mon Sep 17 00:00:00 2001 From: John Behm Date: Fri, 15 Mar 2024 20:53:54 +0100 Subject: [PATCH 76/76] test increased parallelism --- .github/workflows/test.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 372fe79..590bf9d 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -71,7 +71,7 @@ jobs: run: docker-compose up -d - name: Code Coverage - run: go test -timeout 900s -race -count=1 -parallel 1 -covermode=atomic -coverprofile=coverage.txt ./... + run: go test -timeout 900s -race -count=1 -parallel 2 -covermode=atomic -coverprofile=coverage.txt ./... - name: Upload coverage to Codecov uses: codecov/codecov-action@v4