File calculate-fqdns-in-parallel-to-avoid-blockings-... of Package venv-salt-minion

Overview Repositories Revisions Requests Users Attributes Meta

File calculate-fqdns-in-parallel-to-avoid-blockings-bsc-1.patch of Package venv-salt-minion (Revision 67d3b5d84b16efe8436cdde2fb1b70c5)

Currently displaying revision 67d3b5d84b16efe8436cdde2fb1b70c5 , Show latest

From d9362f10681a2dfdb057939eee1ffae3a35d4a8d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Pablo=20Su=C3=A1rez=20Hern=C3=A1ndez?=
 <psuarezhernandez@suse.com>
Date: Fri, 12 Apr 2019 16:47:03 +0100
Subject: [PATCH] Calculate FQDNs in parallel to avoid blockings
 (bsc#1129079)

Fix pylint issue
---
 salt/grains/core.py | 55 +++++++++++++++++++++++++++++++++------------
 1 file changed, 41 insertions(+), 14 deletions(-)

diff --git a/salt/grains/core.py b/salt/grains/core.py
index 006878f806..883e3ebe09 100644
--- a/salt/grains/core.py
+++ b/salt/grains/core.py
@@ -20,8 +20,10 @@ import socket
 import sys
 import time
 import uuid
+import warnings
 import zlib
 from errno import EACCES, EPERM
+from multiprocessing.dummy import Pool as ThreadPool
 
 import distro
 import salt.exceptions
@@ -44,6 +46,14 @@ import salt.utils.versions
 from salt.ext.six.moves import range
 from salt.utils.network import _get_interfaces
 
+# pylint: disable=import-error
+try:
+    import dateutil.tz
+
+    _DATEUTIL_TZ = True
+except ImportError:
+    _DATEUTIL_TZ = False
+
 
 # rewrite distro.linux_distribution to allow best=True kwarg in version(), needed to get the minor version numbers in CentOS
 def _linux_distribution():
@@ -2402,22 +2412,12 @@ def fqdns():
     grains = {}
     fqdns = set()
 
-    addresses = salt.utils.network.ip_addrs(
-        include_loopback=False, interface_data=_get_interfaces()
-    )
-    addresses.extend(
-        salt.utils.network.ip_addrs6(
-            include_loopback=False, interface_data=_get_interfaces()
-        )
-    )
-    err_message = "Exception during resolving address: %s"
-    for ip in addresses:
+    def _lookup_fqdn(ip):
         try:
             name, aliaslist, addresslist = socket.gethostbyaddr(ip)
-            fqdns.update(
-                [socket.getfqdn(name)]
-                + [als for als in aliaslist if salt.utils.network.is_fqdn(als)]
-            )
+            return [socket.getfqdn(name)] + [
+                als for als in aliaslist if salt.utils.network.is_fqdn(als)
+            ]
         except socket.herror as err:
             if err.errno in (0, HOST_NOT_FOUND, NO_DATA):
                 # No FQDN for this IP address, so we don't need to know this all the time.
@@ -2427,6 +2427,33 @@ def fqdns():
         except (OSError, socket.gaierror, socket.timeout) as err:
             log.error(err_message, ip, err)
 
+    start = time.time()
+
+    addresses = salt.utils.network.ip_addrs(
+        include_loopback=False, interface_data=_get_interfaces()
+    )
+    addresses.extend(
+        salt.utils.network.ip_addrs6(
+            include_loopback=False, interface_data=_get_interfaces()
+        )
+    )
+    err_message = "Exception during resolving address: %s"
+
+    # Create a ThreadPool to process the underlying calls to 'socket.gethostbyaddr' in parallel.
+    # This avoid blocking the execution when the "fqdn" is not defined for certains IP addresses, which was causing
+    # that "socket.timeout" was reached multiple times secuencially, blocking execution for several seconds.
+    pool = ThreadPool(8)
+    results = pool.map(_lookup_fqdn, addresses)
+    pool.close()
+    pool.join()
+
+    for item in results:
+        if item:
+            fqdns.update(item)
+
+    elapsed = time.time() - start
+    log.debug("Elapsed time getting FQDNs: {} seconds".format(elapsed))
+
     return {"fqdns": sorted(list(fqdns))}
 
 
-- 
2.29.2

Places

File calculate-fqdns-in-parallel-to-avoid-blockings-bsc-1.patch of Package venv-salt-minion (Revision 67d3b5d84b16efe8436cdde2fb1b70c5)

Places