File fix-for-ec2-rate-limit-failures.patch of Package py26-compat-salt

Overview Repositories Revisions Requests Users Attributes Meta

File fix-for-ec2-rate-limit-failures.patch of Package py26-compat-salt

From eb912004c8329c470b4e5ce0064158ff7079cbf8 Mon Sep 17 00:00:00 2001
From: ed lane <ed.lane.0@gmail.com>
Date: Wed, 9 May 2018 01:08:26 -0600
Subject: [PATCH] Fix for EC2 Rate Limit Failures

Fix for ec2 rate limit failures described here: https://bugzilla.suse.com/show_bug.cgi?id=1088888
---
 salt/utils/aws.py | 22 ++++++++++++++++------
 1 file changed, 16 insertions(+), 6 deletions(-)

diff --git a/salt/utils/aws.py b/salt/utils/aws.py
index 8b9c5a9aed..aacda6b157 100644
--- a/salt/utils/aws.py
+++ b/salt/utils/aws.py
@@ -20,6 +20,7 @@ import hmac
 import logging
 import salt.config
 import re
+import random
 
 # Import Salt libs
 import salt.utils.xmlutil as xml
@@ -449,8 +450,9 @@ def query(params=None, setname=None, requesturl=None, location=None,
         )
         headers = {}
 
-    attempts = 5
-    while attempts > 0:
+    MAX_RETRIES = 6
+    attempts = 0
+    while attempts < MAX_RETRIES:
         LOG.debug('AWS Request: {0}'.format(requesturl))
         LOG.trace('AWS Request Parameters: {0}'.format(params_with_headers))
         try:
@@ -475,16 +477,24 @@ def query(params=None, setname=None, requesturl=None, location=None,
 
             # check to see if we should retry the query
             err_code = data.get('Errors', {}).get('Error', {}).get('Code', '')
-            if attempts > 0 and err_code and err_code in AWS_RETRY_CODES:
-                attempts -= 1
+            if err_code in AWS_RETRY_CODES:
+                attempts += 1
                 LOG.error(
                     'AWS Response Status Code and Error: [{0} {1}] {2}; '
                     'Attempts remaining: {3}'.format(
                         exc.response.status_code, exc, data, attempts
                     )
                 )
-                # Wait a bit before continuing to prevent throttling
-                time.sleep(2)
+                # backoff an exponential amount of time to throttle requests
+                # during "API Rate Exceeded" failures as suggested by the AWS documentation here:
+                # https://docs.aws.amazon.com/AWSEC2/latest/APIReference/query-api-troubleshooting.html
+                # and also here:
+                # https://docs.aws.amazon.com/general/latest/gr/api-retries.html
+                # Failure to implement this approach results in a failure rate of >30% when using salt-cloud with
+                # "--parallel" when creating 50 or more instances with a fixed delay of 2 seconds.
+                # A failure rate of >10% is observed when using the salt-api with an asyncronous client
+                # specified (runner_async).
+                time.sleep(random.uniform(1, 2**attempts))
                 continue
 
             LOG.error(
-- 
2.13.6

Places

File fix-for-ec2-rate-limit-failures.patch of Package py26-compat-salt

Places