Merge pull request #524 from gnought/feat/retry_insufficient_spot_capacity

feat: add retry logic for insufficient spot capacity errors

Author: kp2099

builder/common/step_run_source_instance.go

@@ -285,10 +285,7 @@ func (s *StepRunSourceInstance) Run(ctx context.Context, state multistep.StateBa
 	err = retry.Config{
 		Tries: 11,
 		ShouldRetry: func(err error) bool {
-			if awserrors.Matches(err, "InvalidParameterValue", "iamInstanceProfile") {
-				return true
-			}
-			return false
+			return awserrors.Matches(err, "InvalidParameterValue", "iamInstanceProfile")
 		},
 		RetryDelay: (&retry.Backoff{InitialBackoff: 200 * time.Millisecond, MaxBackoff: 30 * time.Second, Multiplier: 2}).Linear,
 	}.Run(ctx, func(ctx context.Context) error {
@@ -326,10 +323,7 @@ func (s *StepRunSourceInstance) Run(ctx context.Context, state multistep.StateBa
 
 	var r *ec2.DescribeInstancesOutput
 	err = retry.Config{Tries: 11, ShouldRetry: func(err error) bool {
-		if awserrors.Matches(err, "InvalidInstanceID.NotFound", "") {
-			return true
-		}
-		return false
+		return awserrors.Matches(err, "InvalidInstanceID.NotFound", "")
 	},
 		RetryDelay: (&retry.Backoff{InitialBackoff: 200 * time.Millisecond, MaxBackoff: 30 * time.Second, Multiplier: 2}).Linear,
 	}.Run(ctx, func(ctx context.Context) error {

builder/common/step_run_spot_instance.go

@@ -413,6 +413,9 @@ func (s *StepRunSpotInstance) Run(ctx context.Context, state multistep.StateBag)
 				// we can wait on those operations, this can be removed.
 				return true
 			}
+			if err.Error() == "InsufficientInstanceCapacity" {
+				return true
+			}
 			return false
 		},
 		RetryDelay: (&retry.Backoff{InitialBackoff: 500 * time.Millisecond, MaxBackoff: 30 * time.Second, Multiplier: 2}).Linear,
@@ -433,6 +436,12 @@ func (s *StepRunSpotInstance) Run(ctx context.Context, state multistep.StateBag)
 		if err != nil {
 			log.Printf("create request failed %v", err)
 		}
+		// We can end up unavailable Spot capacity, we keep retrying
+		for _, err := range createOutput.Errors {
+			if err.ErrorCode != nil && *err.ErrorCode == "InsufficientInstanceCapacity" {
+				return fmt.Errorf(*err.ErrorCode)
+			}
+		}
 		return err
 	})
 
@@ -513,10 +522,7 @@ func (s *StepRunSpotInstance) Run(ctx context.Context, state multistep.StateBag)
 
 	// Retry creating tags for about 2.5 minutes
 	err = retry.Config{Tries: 11, ShouldRetry: func(err error) bool {
-		if awserrors.Matches(err, "InvalidInstanceID.NotFound", "") {
-			return true
-		}
-		return false
+		return awserrors.Matches(err, "InvalidInstanceID.NotFound", "")
 	},
 		RetryDelay: (&retry.Backoff{InitialBackoff: 200 * time.Millisecond, MaxBackoff: 30 * time.Second, Multiplier: 2}).Linear,
 	}.Run(ctx, func(ctx context.Context) error {