Wait for eventual consistency on AWS spot instances

Also requesting a spot instance is subject to eventual consistency, this patch implements a retry method around spot instance waiter. Signed-off-by: Alessio Caiazza <[email protected]>
afbjorklund · Nov 16, 2017 · 28433ea · 28433ea
1 parent 6f12672
commit 28433ea
Showing 1 changed file with 17 additions and 6 deletions.
diff --git a/drivers/amazonec2/amazonec2.go b/drivers/amazonec2/amazonec2.go
@@ -45,7 +45,8 @@ const (
 )
 
 const (
-	keypairNotFoundCode = "InvalidKeyPair.NotFound"
+	keypairNotFoundCode             = "InvalidKeyPair.NotFound"
+	spotInstanceRequestNotFoundCode = "InvalidSpotInstanceRequestID.NotFound"
 )
 
 var (
@@ -640,11 +641,21 @@ func (d *Driver) Create() error {
 		}
 
 		log.Info("Waiting for spot instance...")
-		err = d.getClient().WaitUntilSpotInstanceRequestFulfilled(&ec2.DescribeSpotInstanceRequestsInput{
-			SpotInstanceRequestIds: []*string{spotInstanceRequest.SpotInstanceRequests[0].SpotInstanceRequestId},
-		})
-		if err != nil {
-			return fmt.Errorf("Error fulfilling spot request: %v", err)
+		for i := 0; i < 3; i++ {
+			// AWS eventual consistency means we could not have SpotInstanceRequest ready yet
+			err = d.getClient().WaitUntilSpotInstanceRequestFulfilled(&ec2.DescribeSpotInstanceRequestsInput{
+				SpotInstanceRequestIds: []*string{spotInstanceRequest.SpotInstanceRequests[0].SpotInstanceRequestId},
+			})
+			if err != nil {
+				if awsErr, ok := err.(awserr.Error); ok {
+					if awsErr.Code() == spotInstanceRequestNotFoundCode {
+						time.Sleep(5 * time.Second)
+						continue
+					}
+				}
+				return fmt.Errorf("Error fulfilling spot request: %v", err)
+			}
+			break
 		}
 		log.Info("Created spot instance request %v", *spotInstanceRequest.SpotInstanceRequests[0].SpotInstanceRequestId)
 		// resolve instance id