From 28433ea85db99366dfe697ffdfb37c17f03b9a35 Mon Sep 17 00:00:00 2001 From: Alessio Caiazza Date: Thu, 16 Nov 2017 18:55:21 +0100 Subject: [PATCH] Wait for eventual consistency on AWS spot instances Also requesting a spot instance is subject to eventual consistency, this patch implements a retry method around spot instance waiter. Signed-off-by: Alessio Caiazza --- drivers/amazonec2/amazonec2.go | 23 +++++++++++++++++------ 1 file changed, 17 insertions(+), 6 deletions(-) diff --git a/drivers/amazonec2/amazonec2.go b/drivers/amazonec2/amazonec2.go index f313043adb..c512c9163c 100644 --- a/drivers/amazonec2/amazonec2.go +++ b/drivers/amazonec2/amazonec2.go @@ -45,7 +45,8 @@ const ( ) const ( - keypairNotFoundCode = "InvalidKeyPair.NotFound" + keypairNotFoundCode = "InvalidKeyPair.NotFound" + spotInstanceRequestNotFoundCode = "InvalidSpotInstanceRequestID.NotFound" ) var ( @@ -640,11 +641,21 @@ func (d *Driver) Create() error { } log.Info("Waiting for spot instance...") - err = d.getClient().WaitUntilSpotInstanceRequestFulfilled(&ec2.DescribeSpotInstanceRequestsInput{ - SpotInstanceRequestIds: []*string{spotInstanceRequest.SpotInstanceRequests[0].SpotInstanceRequestId}, - }) - if err != nil { - return fmt.Errorf("Error fulfilling spot request: %v", err) + for i := 0; i < 3; i++ { + // AWS eventual consistency means we could not have SpotInstanceRequest ready yet + err = d.getClient().WaitUntilSpotInstanceRequestFulfilled(&ec2.DescribeSpotInstanceRequestsInput{ + SpotInstanceRequestIds: []*string{spotInstanceRequest.SpotInstanceRequests[0].SpotInstanceRequestId}, + }) + if err != nil { + if awsErr, ok := err.(awserr.Error); ok { + if awsErr.Code() == spotInstanceRequestNotFoundCode { + time.Sleep(5 * time.Second) + continue + } + } + return fmt.Errorf("Error fulfilling spot request: %v", err) + } + break } log.Info("Created spot instance request %v", *spotInstanceRequest.SpotInstanceRequests[0].SpotInstanceRequestId) // resolve instance id