File gcc9-add-flto=auto.patch of Package gcc9

Overview Repositories Revisions Requests Users Attributes Meta

File gcc9-add-flto=auto.patch of Package gcc9 (Revision 37ffa7c3709fb39be7a6c2d25a3b67fc)

Currently displaying revision 37ffa7c3709fb39be7a6c2d25a3b67fc , Show latest

# The patch is master backport of the following patches:
# - f12fbeb535f192f742025cc4f9b69a48136730f1
# - e63ca5570db076ec7b7bdfa55d51ef6f654d9412
# - d25b1154d110c5403525b66fa54c5aefddd50de7
# - 907e3499443d0e441fcb3b7575d6432598413bff
# - 567ef43c98f6783dde4290467476f8de389c3c10
# - 267389902a985871dd172ab5c5b651f0cd082290
# - 5aa3a1348175aff8d670cb9d0fb5f28444e84aa5

diff --git a/gcc/doc/invoke.texi b/gcc/doc/invoke.texi
index 6ef36ce02aa..83d384f1933 100644
--- a/gcc/doc/invoke.texi
+++ b/gcc/doc/invoke.texi
@@ -10327,7 +10327,7 @@ If you specify the optional @var{n}, the optimization and code
 generation done at link time is executed in parallel using @var{n}
 parallel jobs by utilizing an installed @command{make} program.  The
 environment variable @env{MAKE} may be used to override the program
-used.  The default value for @var{n} is 1.
+used.
 
 You can also specify @option{-flto=jobserver} to use GNU make's
 job server mode to determine the number of parallel jobs. This
@@ -10336,6 +10336,10 @@ You must prepend a @samp{+} to the command recipe in the parent Makefile
 for this to work.  This option likely only works if @env{MAKE} is
 GNU make.
 
+Use @option{-flto=auto} to use GNU make's job server, if available,
+or otherwise fall back to autodetection of the number of CPU threads
+present in your system.
+
 @item -flto-partition=@var{alg}
 @opindex flto-partition
 Specify the partitioning algorithm used by the link-time optimizer.
diff --git a/gcc/gcc.c b/gcc/gcc.c
index 4f57765b012..69fab920938 100644
--- a/gcc/gcc.c
+++ b/gcc/gcc.c
@@ -8261,6 +8261,8 @@ driver::maybe_run_linker (const char *argv0) const
     {
       int tmp = execution_count;
 
+      detect_jobserver ();
+
       if (! have_c)
 	{
 #if HAVE_LTO_PLUGIN > 0
@@ -8350,6 +8352,46 @@ driver::final_actions () const
     }
 }
 
+/* Detect whether jobserver is active and working.  If not drop
+   --jobserver-auth from MAKEFLAGS.  */
+
+void
+driver::detect_jobserver () const
+{
+  /* Detect jobserver and drop it if it's not working.  */
+  const char *makeflags = env.get ("MAKEFLAGS");
+  if (makeflags != NULL)
+    {
+      const char *needle = "--jobserver-auth=";
+      const char *n = strstr (makeflags, needle);
+      if (n != NULL)
+	{
+	  int rfd = -1;
+	  int wfd = -1;
+
+	  bool jobserver
+	    = (sscanf (n + strlen (needle), "%d,%d", &rfd, &wfd) == 2
+	       && rfd > 0
+	       && wfd > 0
+	       && is_valid_fd (rfd)
+	       && is_valid_fd (wfd));
+
+	  /* Drop the jobserver if it's not working now.  */
+	  if (!jobserver)
+	    {
+	      unsigned offset = n - makeflags;
+	      char *dup = xstrdup (makeflags);
+	      dup[offset] = '\0';
+
+	      const char *space = strchr (makeflags + offset, ' ');
+	      if (space != NULL)
+		strcpy (dup + offset, space);
+	      xputenv (concat ("MAKEFLAGS=", dup, NULL));
+	    }
+	}
+    }
+}
+
 /* Determine what the exit code of the driver should be.  */
 
 int
diff --git a/gcc/gcc.h b/gcc/gcc.h
index a0a1d94c6e6..dc77dba67fb 100644
--- a/gcc/gcc.h
+++ b/gcc/gcc.h
@@ -51,6 +51,7 @@ class driver
   void do_spec_on_infiles () const;
   void maybe_run_linker (const char *argv0) const;
   void final_actions () const;
+  void detect_jobserver () const;
   int get_exit_code () const;
 
  private:
diff --git a/gcc/lto-wrapper.c b/gcc/lto-wrapper.c
index ac971494054..7116fdd1873 100644
--- a/gcc/lto-wrapper.c
+++ b/gcc/lto-wrapper.c
@@ -1109,6 +1109,136 @@ cmp_priority (const void *a, const void *b)
   return *((const int *)b)-*((const int *)a);
 }
 
+/* Number of CPUs that can be used for parallel LTRANS phase.  */
+
+static unsigned long nthreads_var = 0;
+
+#ifdef HAVE_PTHREAD_AFFINITY_NP
+unsigned long cpuset_size;
+static unsigned long get_cpuset_size;
+cpu_set_t *cpusetp;
+
+unsigned long
+static cpuset_popcount (unsigned long cpusetsize, cpu_set_t *cpusetp)
+{
+#ifdef CPU_COUNT_S
+  /* glibc 2.7 and above provide a macro for this.  */
+  return CPU_COUNT_S (cpusetsize, cpusetp);
+#else
+#ifdef CPU_COUNT
+  if (cpusetsize == sizeof (cpu_set_t))
+    /* glibc 2.6 and above provide a macro for this.  */
+    return CPU_COUNT (cpusetp);
+#endif
+  size_t i;
+  unsigned long ret = 0;
+  STATIC_ASSERT (sizeof (cpusetp->__bits[0]) == sizeof (unsigned long int));
+  for (i = 0; i < cpusetsize / sizeof (cpusetp->__bits[0]); i++)
+    {
+      unsigned long int mask = cpusetp->__bits[i];
+      if (mask == 0)
+	continue;
+      ret += __builtin_popcountl (mask);
+    }
+  return ret;
+#endif
+}
+#endif
+
+/* At startup, determine the default number of threads.  It would seem
+   this should be related to the number of cpus online.  */
+
+static void
+init_num_threads (void)
+{
+#ifdef HAVE_PTHREAD_AFFINITY_NP
+#if defined (_SC_NPROCESSORS_CONF) && defined (CPU_ALLOC_SIZE)
+  cpuset_size = sysconf (_SC_NPROCESSORS_CONF);
+  cpuset_size = CPU_ALLOC_SIZE (cpuset_size);
+#else
+  cpuset_size = sizeof (cpu_set_t);
+#endif
+
+  cpusetp = (cpu_set_t *) xmalloc (gomp_cpuset_size);
+  do
+    {
+      int ret = pthread_getaffinity_np (pthread_self (), gomp_cpuset_size,
+					cpusetp);
+      if (ret == 0)
+	{
+	  /* Count only the CPUs this process can use.  */
+	  nthreads_var = cpuset_popcount (cpuset_size, cpusetp);
+	  if (nthreads_var == 0)
+	    break;
+	  get_cpuset_size = cpuset_size;
+#ifdef CPU_ALLOC_SIZE
+	  unsigned long i;
+	  for (i = cpuset_size * 8; i; i--)
+	    if (CPU_ISSET_S (i - 1, cpuset_size, cpusetp))
+	      break;
+	  cpuset_size = CPU_ALLOC_SIZE (i);
+#endif
+	  return;
+	}
+      if (ret != EINVAL)
+	break;
+#ifdef CPU_ALLOC_SIZE
+      if (cpuset_size < sizeof (cpu_set_t))
+	cpuset_size = sizeof (cpu_set_t);
+      else
+	cpuset_size = cpuset_size * 2;
+      if (cpuset_size < 8 * sizeof (cpu_set_t))
+	cpusetp
+	  = (cpu_set_t *) realloc (cpusetp, cpuset_size);
+      else
+	{
+	  /* Avoid fatal if too large memory allocation would be
+	     requested, e.g. kernel returning EINVAL all the time.  */
+	  void *p = realloc (cpusetp, cpuset_size);
+	  if (p == NULL)
+	    break;
+	  cpusetp = (cpu_set_t *) p;
+	}
+#else
+      break;
+#endif
+    }
+  while (1);
+  cpuset_size = 0;
+  nthreads_var = 1;
+  free (cpusetp);
+  cpusetp = NULL;
+#endif
+#ifdef _SC_NPROCESSORS_ONLN
+  nthreads_var = sysconf (_SC_NPROCESSORS_ONLN);
+#endif
+}
+
+/* FIXME: once using -std=c11, we can use std::thread::hardware_concurrency.  */
+
+/* Return true when a jobserver is running and can accept a job.  */
+
+static bool
+jobserver_active_p (void)
+{
+  const char *makeflags = getenv ("MAKEFLAGS");
+  if (makeflags == NULL)
+    return false;
+
+  const char *needle = "--jobserver-auth=";
+  const char *n = strstr (makeflags, needle);
+  if (n == NULL)
+    return false;
+
+  int rfd = -1;
+  int wfd = -1;
+
+  return (sscanf (n + strlen (needle), "%d,%d", &rfd, &wfd) == 2
+	  && rfd > 0
+	  && wfd > 0
+	  && is_valid_fd (rfd)
+	  && is_valid_fd (wfd));
+}
 
 /* Execute gcc. ARGC is the number of arguments. ARGV contains the arguments. */
 
@@ -1123,6 +1253,7 @@ run_gcc (unsigned argc, char *argv[])
   const char *collect_gcc, *collect_gcc_options;
   int parallel = 0;
   int jobserver = 0;
+  int auto_parallel = 0;
   bool no_partition = false;
   struct cl_decoded_option *fdecoded_options = NULL;
   struct cl_decoded_option *offload_fdecoded_options = NULL;
@@ -1247,8 +1378,13 @@ run_gcc (unsigned argc, char *argv[])
 	case OPT_flto_:
 	  if (strcmp (option->arg, "jobserver") == 0)
 	    {
+	      parallel = 1;
 	      jobserver = 1;
+	    }
+	  else if (strcmp (option->arg, "auto") == 0)
+	    {
 	      parallel = 1;
+	      auto_parallel = 1;
 	    }
 	  else
 	    {
@@ -1290,8 +1426,14 @@ run_gcc (unsigned argc, char *argv[])
     {
       lto_mode = LTO_MODE_LTO;
       jobserver = 0;
+      auto_parallel = 0;
       parallel = 0;
     }
+  else if (!jobserver && auto_parallel && jobserver_active_p ())
+    {
+      parallel = 1;
+      jobserver = 1;
+    }
 
   if (linker_output)
     {
@@ -1483,7 +1625,21 @@ cont1:
       strcpy (tmp, ltrans_output_file);
 
       if (jobserver)
-	obstack_ptr_grow (&argv_obstack, xstrdup ("-fwpa=jobserver"));
+	{
+	  if (verbose)
+	    fprintf (stderr, "Using make jobserver\n");
+	  obstack_ptr_grow (&argv_obstack, xstrdup ("-fwpa=jobserver"));
+	}
+      else if (auto_parallel)
+	{
+	  char buf[256];
+	  init_num_threads ();
+	  if (verbose)
+	    fprintf (stderr, "LTO parallelism level set to %ld\n",
+		     nthreads_var);
+	  sprintf (buf, "-fwpa=%ld", nthreads_var);
+	  obstack_ptr_grow (&argv_obstack, xstrdup (buf));
+	}
       else if (parallel > 1)
 	{
 	  char buf[256];
@@ -1691,7 +1847,8 @@ cont:
 	  i = 3;
 	  if (!jobserver)
 	    {
-	      snprintf (jobs, 31, "-j%d", parallel);
+	      snprintf (jobs, 31, "-j%ld",
+			auto_parallel ? nthreads_var : parallel);
 	      new_argv[i++] = jobs;
 	    }
 	  new_argv[i++] = "all";
diff --git a/gcc/opts.c b/gcc/opts.c
index 6164d49fa08..80dc04cedaf 100644
--- a/gcc/opts.c
+++ b/gcc/opts.c
@@ -2738,6 +2738,15 @@ common_handle_option (struct gcc_options *opts,
       opts->x_flag_lto = value ? "" : NULL;
       break;
 
+    case OPT_flto_:
+      if (strcmp (arg, "none") != 0
+	  && strcmp (arg, "jobserver") != 0
+	  && strcmp (arg, "auto") != 0
+	  && atoi (arg) == 0)
+	error_at (loc,
+		  "unrecognized argument to %<-flto=%> option: %qs", arg);
+      break;
+
     case OPT_w:
       dc->dc_inhibit_warnings = true;
       break;
diff --git a/gcc/testsuite/g++.dg/lto/devirt-19_0.C b/gcc/testsuite/g++.dg/lto/devirt-19_0.C
index 696d8c0fc83..b43527e324e 100644
--- a/gcc/testsuite/g++.dg/lto/devirt-19_0.C
+++ b/gcc/testsuite/g++.dg/lto/devirt-19_0.C
@@ -1,5 +1,5 @@
 /* { dg-lto-do link } */
 /* { dg-lto-options { "-O2 -fdump-ipa-cp -Wno-return-type -flto -r -nostdlib" } } */
-/* { dg-extra-ld-options "-flinker-output=nolto-rel" } */
+/* { dg-extra-ld-options "-flinker-output=nolto-rel -flto=auto" } */
 #include "../ipa/devirt-19.C"
 /* { dg-final { scan-wpa-ipa-dump-times "Discovered a virtual call to a known target" 1 "cp"  } } */
diff --git a/gcc/testsuite/gcc.dg/spellcheck-options-21.c b/gcc/testsuite/gcc.dg/spellcheck-options-21.c
new file mode 100644
index 00000000000..3e0e8a8ebaf
--- /dev/null
+++ b/gcc/testsuite/gcc.dg/spellcheck-options-21.c
@@ -0,0 +1,3 @@
+/* { dg-do compile } */
+/* { dg-options "-flto=sparta" } */
+/* { dg-error "unrecognized argument to '-flto=' option: 'sparta'" "" { target *-*-* } 0 } */
diff --git a/include/libiberty.h b/include/libiberty.h
index 57476135026..70e8c9e1076 100644
--- a/include/libiberty.h
+++ b/include/libiberty.h
@@ -137,6 +137,10 @@ extern const char *unix_lbasename (const char *) ATTRIBUTE_RETURNS_NONNULL ATTRI
 
 extern char *lrealpath (const char *);
 
+/* Return true when FD file descriptor exists.  */
+
+extern int is_valid_fd (int fd);
+
 /* Concatenate an arbitrary number of strings.  You must pass NULL as
    the last argument of this function, to terminate the list of
    strings.  Allocates memory using xmalloc.  */
diff --git a/libiberty/Makefile.in b/libiberty/Makefile.in
index 0be45b4ae8e..f1628d4ee0d 100644
--- a/libiberty/Makefile.in
+++ b/libiberty/Makefile.in
@@ -127,7 +127,7 @@ CFILES = alloca.c argv.c asprintf.c atexit.c				\
 	calloc.c choose-temp.c clock.c concat.c cp-demangle.c		\
 	 cp-demint.c cplus-dem.c crc32.c				\
 	d-demangle.c dwarfnames.c dyn-string.c				\
-	fdmatch.c ffs.c fibheap.c filename_cmp.c floatformat.c		\
+	fdmatch.c ffs.c fibheap.c filedescriptor.c filename_cmp.c floatformat.c		\
 	fnmatch.c fopen_unlocked.c					\
 	getcwd.c getopt.c getopt1.c getpagesize.c getpwd.c getruntime.c	\
          gettimeofday.c                                                 \
@@ -171,6 +171,7 @@ REQUIRED_OFILES =							\
 	./cp-demint.$(objext) ./crc32.$(objext) ./d-demangle.$(objext)	\
 	./dwarfnames.$(objext) ./dyn-string.$(objext)			\
 	./fdmatch.$(objext) ./fibheap.$(objext)				\
+	./filedescriptor.$(objext)	\
 	./filename_cmp.$(objext) ./floatformat.$(objext)		\
 	./fnmatch.$(objext) ./fopen_unlocked.$(objext)			\
 	./getopt.$(objext) ./getopt1.$(objext) ./getpwd.$(objext)	\
@@ -756,6 +757,17 @@ $(CONFIGURED_OFILES): stamp-picdir stamp-noasandir
 	else true; fi
 	$(COMPILE.c) $(srcdir)/fibheap.c $(OUTPUT_OPTION)
 
+./filedescriptor.$(objext): $(srcdir)/filedescriptor.c config.h $(INCDIR)/ansidecl.h \
+	$(INCDIR)/libiberty.h
+	if [ x"$(PICFLAG)" != x ]; then \
+	  $(COMPILE.c) $(PICFLAG) $(srcdir)/filedescriptor.c -o pic/$@; \
+	else true; fi
+	if [ x"$(NOASANFLAG)" != x ]; then \
+	  $(COMPILE.c) $(PICFLAG) $(NOASANFLAG) $(srcdir)/filedescriptor.c -o noasan/$@; \
+	else true; fi
+	$(COMPILE.c) $(srcdir)/filedescriptor.c $(OUTPUT_OPTION)
+
+
 ./filename_cmp.$(objext): $(srcdir)/filename_cmp.c config.h $(INCDIR)/ansidecl.h \
 	$(INCDIR)/filenames.h $(INCDIR)/hashtab.h \
 	$(INCDIR)/safe-ctype.h
diff --git a/libiberty/filedescriptor.c b/libiberty/filedescriptor.c
new file mode 100644
index 00000000000..3a1a68d1eef
--- /dev/null
+++ b/libiberty/filedescriptor.c
@@ -0,0 +1,47 @@
+/* File descriptor related functions.
+
+   Copyright (C) 2019 Free Software Foundation, Inc.
+
+   This file is part of the libiberty library.
+
+   This program is free software; you can redistribute it and/or modify
+   it under the terms of the GNU General Public License as published by
+   the Free Software Foundation; either version 2 of the License, or
+   (at your option) any later version.
+
+   This program is distributed in the hope that it will be useful,
+   but WITHOUT ANY WARRANTY; without even the implied warranty of
+   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+   GNU General Public License for more details.
+
+   You should have received a copy of the GNU General Public License
+   along with this program; if not, write to the Free Software
+   Foundation, Inc., 51 Franklin Street - Fifth Floor,
+   Boston, MA 02110-1301, USA.  */
+
+#include "config.h"
+#include "ansidecl.h"
+#include "libiberty.h"
+
+#ifdef HAVE_FCNTL_H
+#include <fcntl.h>
+#endif
+
+#if defined (_WIN32)
+#define WIN32_LEAN_AND_MEAN
+#include <windows.h> /* for GetFullPathName */
+#endif
+/* Return true when FD file descriptor exists.  */
+
+int
+is_valid_fd (int fd)
+{
+#if defined(_WIN32)
+  HANDLE h = (HANDLE) _get_osfhandle (fd);
+  return h != (HANDLE) -1;
+#elif defined(F_GETFD)
+  return fcntl (fd, F_GETFD) >= 0;
+#else
+  return dup2 (fd, fd) < 0;
+#endif
+}

Places

File gcc9-add-flto=auto.patch of Package gcc9 (Revision 37ffa7c3709fb39be7a6c2d25a3b67fc)

Places