File 2901-erts-Optimize-make-depend-on-windows.patch of Package erlang

Overview Repositories Revisions Requests Users Attributes Meta

File 2901-erts-Optimize-make-depend-on-windows.patch of Package erlang (Revision b3a22b16b9b462356a1a0d804f54fae1)

Currently displaying revision b3a22b16b9b462356a1a0d804f54fae1 , Show latest

From 7613213a5e1f6c21492d9ba82b9448205531a827 Mon Sep 17 00:00:00 2001
From: Lukas Larsson <lukas@erlang.org>
Date: Wed, 9 Feb 2022 15:30:16 +0100
Subject: [PATCH] erts: Optimize make depend on windows

Change the different depend targets to be dependencies which allow
them to be run in parallel. Also redo cc.sh -MM to use /showIncludes
to list all includes instead of searching through the pre processed
files.
---
 erts/emulator/Makefile.in         |  60 +++++---
 erts/etc/win32/wsl_tools/vc/cc.sh | 219 ++++++++++++++++++++----------
 2 files changed, 186 insertions(+), 93 deletions(-)

diff --git a/erts/emulator/Makefile.in b/erts/emulator/Makefile.in
index ea52d14c46..fa331e003a 100644
--- a/erts/emulator/Makefile.in
+++ b/erts/emulator/Makefile.in
@@ -1330,9 +1330,12 @@ ifeq ($(TARGET),win32)
 
 #DEP_CC=$(EMU_CC)
 DEP_CC=$(CC)
-DEP_FLAGS=-MM  $(subst -O2,,$(CFLAGS)) $(INCLUDES) -I../etc/win32 \
+DEP_CXX=$(CXX)
+DEP_INCLUDES=$(INCLUDES) -I../etc/win32 \
 	-Idrivers/common -Idrivers/$(ERLANG_OSTYPE) \
 	-Inifs/common -Inifs/$(ERLANG_OSTYPE)
+DEP_FLAGS=-MM  $(subst -O2,,$(CFLAGS)) $(INCLUDES) 
+DEP_CXXFLAGS=-MM $(subst -O2,,$(CXXFLAGS)) $(DEP_INCLUDES) $(ASMJIT_FLAGS)
 
 # ifeq ($(USING_VC),yes)
 # VC++ used for compiling. If __GNUC__ is defined we will include
@@ -1372,36 +1375,57 @@ $(TARGET)/gen_git_version.mk:
 # rebuild.
 	$(V_at)if utils/gen_git_version $@; then touch beam/erl_bif_info.c; fi
 
-.PHONY: depend
-ifdef VOID_EMULATOR
-depend:
-	@echo $(VOID_EMULATOR)' - omitted target depend'
-else
-depend: $(TTF_DIR)/depend.mk
-$(TTF_DIR)/depend.mk: $(TTF_DIR)/GENERATED $(PRELOAD_SRC)
+DEPEND_DEPS=jit src drv nif sys target zlib ryu
+
+$(TTF_DIR)/src.depend.mk: $(TTF_DIR)/GENERATED $(PRELOAD_SRC)
 	$(gen_verbose)
 	$(V_at)$(DEP_CC) $(DEP_FLAGS) $(BEAM_SRC) \
-		| $(SED_DEPEND) > $(TTF_DIR)/depend.mk
+		| $(SED_DEPEND) > $@
+$(TTF_DIR)/drv.depend.mk: $(TTF_DIR)/GENERATED $(PRELOAD_SRC)
+	$(gen_verbose)
 	$(V_at)$(DEP_CC) $(DEP_FLAGS) -DLIBSCTP=$(LIBSCTP) $(DRV_COMMON_SRC) \
-		| $(SED_DEPEND) >> $(TTF_DIR)/depend.mk
+		| $(SED_DEPEND) > $@
 	$(V_at)$(DEP_CC) $(DEP_FLAGS) -I../etc/$(ERLANG_OSTYPE) $(DRV_OSTYPE_SRC) \
-		| $(SED_DEPEND) >> $(TTF_DIR)/depend.mk
+		| $(SED_DEPEND) >> $@
+$(TTF_DIR)/nif.depend.mk: $(TTF_DIR)/GENERATED $(PRELOAD_SRC)
+	$(gen_verbose)
 	$(V_at)$(DEP_CC) $(DEP_FLAGS) $(NIF_COMMON_SRC) \
-		| $(SED_DEPEND) >> $(TTF_DIR)/depend.mk
+		| $(SED_DEPEND) > $@
 	$(V_at)$(DEP_CC) $(DEP_FLAGS) -I../etc/$(ERLANG_OSTYPE) $(NIF_OSTYPE_SRC) \
-		| $(SED_DEPEND) >> $(TTF_DIR)/depend.mk
+		| $(SED_DEPEND) >> $@
+$(TTF_DIR)/sys.depend.mk: $(TTF_DIR)/GENERATED $(PRELOAD_SRC)
+	$(gen_verbose)
 	$(V_at)$(DEP_CC) $(DEP_FLAGS) $(SYS_SRC) \
-		| $(SED_DEPEND) >> $(TTF_DIR)/depend.mk
+		| $(SED_DEPEND) > $@
+$(TTF_DIR)/target.depend.mk: $(TTF_DIR)/GENERATED $(PRELOAD_SRC)
+	$(gen_verbose)
 	$(V_at)$(DEP_CC) $(DEP_FLAGS) $(TARGET_SRC) \
-		| $(SED_DEPEND) >> $(TTF_DIR)/depend.mk
+		| $(SED_DEPEND) > $@
+$(TTF_DIR)/zlib.depend.mk: $(TTF_DIR)/GENERATED $(PRELOAD_SRC)
+	$(gen_verbose)
 	$(V_at)$(DEP_CC) $(DEP_FLAGS) $(ZLIB_SRC) \
-		| $(SED_DEPEND_ZLIB) >> $(TTF_DIR)/depend.mk
+		| $(SED_DEPEND_ZLIB) > $@
+$(TTF_DIR)/ryu.depend.mk: $(TTF_DIR)/GENERATED $(PRELOAD_SRC)
+	$(gen_verbose)
 	$(V_at)$(DEP_CC) $(DEP_FLAGS) $(RYU_SRC) \
-		| $(SED_DEPEND_ZLIB) >> $(TTF_DIR)/depend.mk
+		| $(SED_DEPEND_ZLIB) > $@
+$(TTF_DIR)/jit.depend.mk: $(TTF_DIR)/GENERATED $(PRELOAD_SRC)
+	$(gen_verbose)
+	@touch $@
 ifeq ($(JIT_ENABLED),yes)
 	$(V_at)$(DEP_CXX) $(DEP_CXXFLAGS) $(BEAM_CPP_SRC) \
-		| $(SED_DEPEND) >> $(TTF_DIR)/depend.mk
+		| $(SED_DEPEND) > $@
 endif
+
+.PHONY: depend
+ifdef VOID_EMULATOR
+depend:
+	@echo $(VOID_EMULATOR)' - omitted target depend'
+else
+depend: $(TTF_DIR)/depend.mk
+$(TTF_DIR)/depend.mk: $(foreach dep, $(DEPEND_DEPS), $(TTF_DIR)/$(dep).depend.mk)
+	-rm $@
+	for dep in $^; do cat $$dep >> $@; done
 	$(V_at)cd $(ERTS_LIB_DIR) && $(MAKE) depend
 endif
 
diff --git a/erts/etc/win32/wsl_tools/vc/cc.sh b/erts/etc/win32/wsl_tools/vc/cc.sh
index db8dab3e14..f275259fe8 100755
--- a/erts/etc/win32/wsl_tools/vc/cc.sh
+++ b/erts/etc/win32/wsl_tools/vc/cc.sh
@@ -41,6 +41,8 @@ MD_FORCED=false
 PREPROCESSING=false
 # If we're generating dependencies (implies preprocessing)
 DEPENDENCIES=false
+# If we're generating fast dependencies
+FAST_DEPENDENCIES=false
 # If this is supposed to be a debug build
 DEBUG_BUILD=false
 # If this is supposed to be an optimized build (there can only be one...)
@@ -91,6 +93,9 @@ while test -n "$1" ; do
 	    LINKING=false;;
 	    #CMD="$CMD -c";;
 	-MM)
+	    LINKING=false;
+	    FAST_DEPENDENCIES=true;;
+	-MS)
 	    PREPROCESSING=true;
 	    LINKING=false;
 	    DEPENDENCIES=true;;
@@ -241,75 +246,139 @@ while true ; do
     esac
 done
 
-# Compile
-for x in $SOURCES; do
-    # Compile each source
-    if [ $LINKING = false ]; then
-	# We should have an output defined, which is a directory
-	# or an object file
-	case $OUTFILE in
-	    /*.o)
-		# Simple output, SOURCES should be one single
-		n=`echo $SOURCES | wc -w`;
-		if [ $n -gt 1 ]; then
-		    echo "cc.sh:Error, multiple sources, one object output.";
-		    exit 1;
-		else
-		    output_filename=`echo $OUTFILE`;
-		fi;;
-	    *.o)
-		# Relative path needs no translation
-		n=`echo $SOURCES | wc -w`
-		if [ $n -gt 1 ]; then
-		    echo "cc.sh:Error, multiple sources, one object output."
-		    exit 1
-		else
-		    output_filename=$OUTFILE
-		fi;;
-	    /*)
-		# Absolute directory
-		o=`echo $x | sed 's,.*/,,' | sed 's,\.c$,.o,'`
-		output_filename=`echo $OUTFILE`
-		output_filename="$output_filename/${o}";;
-	    *)
-		# Relative_directory or empty string (.//x.o is valid)
-		o=`echo $x | sed 's,.*/,,' | sed 's,\.cp*$,.o,'`
-		output_filename="./${OUTFILE}/${o}";;
-	esac
-    else
-	# We are linking, which means we build objects in a temporary
-	# directory and link from there. We should retain the basename
-	# of each source to make examining the exe easier...
-	o=`echo $x | sed 's,.*/,,' | sed 's,\.c$,.o,'`
-	output_filename=$TMPOBJDIR/$o
-	ACCUM_OBJECTS="$ACCUM_OBJECTS $output_filename"
-    fi
-    # Now we know enough, lets try a compilation...
-    if [ $USEABSPATH = true ]; then
-        MPATH=`w32_path.sh -a -d $x`
-    else
-        MPATH=`w32_path.sh -d $x`
-    fi
-    if [ $PREPROCESSING = true ]; then
-	output_flag="-E"
-    else
-	output_flag="/FS -c -Fo`w32_path.sh -a -d ${output_filename}`"
-    fi
+## This is a faster dependency calculation method that
+## only invokes cl.exe once for all files.
+if [ $FAST_DEPENDENCIES = true ]; then
+    MPATH=
+    for x in $SOURCES; do
+	if [ $USEABSPATH = true ]; then
+	    MPATH="$MPATH $(w32_path.sh -a -d $x)"
+	else
+	    MPATH="$MPATH $(w32_path.sh -d $x)"
+	fi
+    done
     params="$COMMON_CFLAGS $MD $DEBUG_FLAGS $OPTIMIZE_FLAGS \
-	    $CMD ${output_flag} $MPATH"
+	    $CMD /showIncludes '-Fo`w32_path.sh -a -d ${TMPOBJDIR}`\\' $MPATH"
     if [ "X$CC_SH_DEBUG_LOG" != "X" ]; then
 	echo cc.sh "$SAVE" >>$CC_SH_DEBUG_LOG
 	echo cl.exe $params >>$CC_SH_DEBUG_LOG
     fi
-    eval cl.exe $params >$MSG_FILE 2>$ERR_FILE
+    eval cl.exe $params > $MSG_FILE 2> $ERR_FILE
     RES=$?
-    if test $PREPROCESSING = false; then
-	cat $ERR_FILE >&2
-	tail -n +2 $MSG_FILE
-    else
-	tail -n +2 $ERR_FILE >&2
-	if test $DEPENDENCIES = true; then
-	    perl -e '
+    tail -n +2 $ERR_FILE | grep -v LNK1561 >&2
+    if grep "LNK1561" $ERR_FILE; then
+	rm -f $ERR_FILE $MSG_FILE
+	rm -rf $TMPOBJDIR
+	exit $RES
+    fi
+    perl -e '
+my $deps = "";
+my @sources = split(" ",$ARGV[1]);
+
+open(FH, "<", $ARGV[0]) or die $!;
+
+sub flushDeps() {
+    if ($deps ne "") {
+        ## To start a new shell can be very slow so we run all
+        ## the wslpath translations in a single run
+        my @paths = split("\n",`for x in $deps; do wslpath -u "\$x"; done`);
+        foreach (@paths) {
+            print "\\\n $_";
+        }
+        print "\n\n";
+    }
+}
+
+while (<FH>) {
+    $_ =~ tr/\r\n/\n/d;
+    if (/^(.*)\.(c|cpp)$/) {
+        my $target = "$1";
+        flushDeps();
+        $deps = shift(@sources) . " ";
+        print $target.".o:";
+    }
+    if (/^Note: including file:\s*(.*)$/) {
+        ## Ignore any deps that comes from "Program Files"
+        next if /Program Files/;
+        my $file = $1;
+        $file =~ s/\\/\//g;
+        $deps .= "'"'"'$file'"'"' ";
+    }
+}
+flushDeps();
+' $MSG_FILE "$MPATH"
+
+    rm -f $ERR_FILE $MSG_FILE
+else
+    # Compile
+    for x in $SOURCES; do
+        # Compile each source
+        if [ $LINKING = false ]; then
+	    # We should have an output defined, which is a directory
+	    # or an object file
+	    case $OUTFILE in
+	        /*.o)
+		    # Simple output, SOURCES should be one single
+		    n=`echo $SOURCES | wc -w`;
+		    if [ $n -gt 1 ]; then
+		        echo "cc.sh:Error, multiple sources, one object output.";
+		        exit 1;
+		    else
+		        output_filename=`echo $OUTFILE`;
+		    fi;;
+	        *.o)
+		    # Relative path needs no translation
+		    n=`echo $SOURCES | wc -w`
+		    if [ $n -gt 1 ]; then
+		        echo "cc.sh:Error, multiple sources, one object output."
+		        exit 1
+		    else
+		        output_filename=$OUTFILE
+		    fi;;
+	        /*)
+		    # Absolute directory
+		    o=`echo $x | sed 's,.*/,,' | sed 's,\.c$,.o,'`
+		    output_filename=`echo $OUTFILE`
+		    output_filename="$output_filename/${o}";;
+	        *)
+		    # Relative_directory or empty string (.//x.o is valid)
+		    o=`echo $x | sed 's,.*/,,' | sed 's,\.cp*$,.o,'`
+		    output_filename="./${OUTFILE}/${o}";;
+	    esac
+        else
+	    # We are linking, which means we build objects in a temporary
+	    # directory and link from there. We should retain the basename
+	    # of each source to make examining the exe easier...
+	    o=`echo $x | sed 's,.*/,,' | sed 's,\.c$,.o,'`
+	    output_filename=$TMPOBJDIR/$o
+	    ACCUM_OBJECTS="$ACCUM_OBJECTS $output_filename"
+        fi
+        # Now we know enough, lets try a compilation...
+        if [ $USEABSPATH = true ]; then
+            MPATH=`w32_path.sh -a -d $x`
+        else
+            MPATH=`w32_path.sh -d $x`
+        fi
+        if [ $PREPROCESSING = true ]; then
+	    output_flag="-E"
+        else
+	    output_flag="/FS -c -Fo`w32_path.sh -a -d ${output_filename}`"
+        fi
+        params="$COMMON_CFLAGS $MD $DEBUG_FLAGS $OPTIMIZE_FLAGS \
+	    $CMD ${output_flag} $MPATH"
+        if [ "X$CC_SH_DEBUG_LOG" != "X" ]; then
+	    echo cc.sh "$SAVE" >>$CC_SH_DEBUG_LOG
+	    echo cl.exe $params >>$CC_SH_DEBUG_LOG
+        fi
+        eval cl.exe $params >$MSG_FILE 2>$ERR_FILE
+        RES=$?
+        if test $PREPROCESSING = false; then
+	    cat $ERR_FILE >&2
+	    tail -n +2 $MSG_FILE
+        else
+	    tail -n +2 $ERR_FILE >&2
+	    if test $DEPENDENCIES = true; then
+	        perl -e '
 my $file = "'$x'";
 while (<>) {
       next unless /^#line/;
@@ -331,18 +400,18 @@ if (@f) {
      print "\n\n";
      print STDERR "Made dependencies for $file\n";
 }' $MSG_FILE
-	else
-	    cat $MSG_FILE
-	fi
-    fi
-    rm -f $ERR_FILE $MSG_FILE
-    if [ $RES != 0 ]; then
-	echo Failed: cl.exe $params
-	rm -rf $TMPOBJDIR
-	exit $RES
-    fi
-done
-
+	    else
+	        cat $MSG_FILE
+	    fi
+        fi
+        rm -f $ERR_FILE $MSG_FILE
+        if [ $RES != 0 ]; then
+	    echo Failed: cl.exe $params
+	    rm -rf $TMPOBJDIR
+	    exit $RES
+        fi
+    done
+fi
 # If we got here, we succeeded in compiling (if there were anything to compile)
 # The output filename should name an executable if we're linking
 if [ $LINKING = true ]; then
-- 
2.34.1

Places

File 2901-erts-Optimize-make-depend-on-windows.patch of Package erlang (Revision b3a22b16b9b462356a1a0d804f54fae1)

Places