File 6032-gh-Only-upload-erlang-artifacts-in-order-t... of Package erlang

Overview Repositories Revisions Requests Users Attributes Meta

File 6032-gh-Only-upload-erlang-artifacts-in-order-to-optimize.patch of Package erlang (Revision da0f8958cbfdaccc567b3effd8dde26d)

Currently displaying revision da0f8958cbfdaccc567b3effd8dde26d , Show latest

From 4fdc61f9dbe38a6d362faa9f16179e19dfe84b57 Mon Sep 17 00:00:00 2001
From: Lukas Larsson <lukas@erlang.org>
Date: Sun, 3 Jul 2022 19:49:42 +0200
Subject: [PATCH 2/5] gh: Only upload erlang artifacts in order to optimize
 building

---
 .github/dockerfiles/Dockerfile.64-bit |   7 ++
 .github/dockerfiles/init.sh           |   4 +
 .github/scripts/base-tag              |  20 ----
 .github/scripts/build-base-image.sh   |  55 +++++++++++
 .github/scripts/restore-otp-image.sh  |  12 +++
 .github/workflows/main.yaml           | 132 ++++++++++++++++----------
 .github/workflows/update-base.yaml    |  10 +-
 7 files changed, 161 insertions(+), 79 deletions(-)
 delete mode 100755 .github/scripts/base-tag
 create mode 100755 .github/scripts/build-base-image.sh
 create mode 100755 .github/scripts/restore-otp-image.sh

diff --git a/.github/dockerfiles/Dockerfile.64-bit b/.github/dockerfiles/Dockerfile.64-bit
index 785d6e8391..25aadc5e84 100644
--- a/.github/dockerfiles/Dockerfile.64-bit
+++ b/.github/dockerfiles/Dockerfile.64-bit
@@ -39,3 +39,10 @@ RUN latest () { \
 ENV ERL_LIBS=/buildroot/proper:/buildroot/jsx
 
 WORKDIR /buildroot/otp/
+
+## Update init.sh with correct env vars
+RUN echo "export MAKEFLAGS=$MAKEFLAGS" > /buildroot/env.sh && \
+    echo "export ERLC_USE_SERVER=$ERLC_USE_SERVER" >> /buildroot/env.sh && \
+    echo "export ERL_TOP=$ERL_TOP" >> /buildroot/env.sh && \
+    echo "export PATH=$PATH" >> /buildroot/env.sh && \
+    echo "export ERL_LIBS=$ERL_LIBS" >> /buildroot/env.sh
diff --git a/.github/dockerfiles/init.sh b/.github/dockerfiles/init.sh
index bac54338c1..56801f782a 100755
--- a/.github/dockerfiles/init.sh
+++ b/.github/dockerfiles/init.sh
@@ -1,5 +1,9 @@
 #!/bin/sh
 
+if [ -f "/buildroot/env.sh" ]; then
+    . "/buildroot/env.sh"
+fi
+
 sudo mkdir -p -m0755 /var/run/sshd
 
 sudo /usr/sbin/sshd
diff --git a/.github/scripts/base-tag b/.github/scripts/base-tag
deleted file mode 100755
index b2a8017bfc..0000000000
--- a/.github/scripts/base-tag
+++ /dev/null
@@ -1,20 +0,0 @@
-#!/bin/bash
-
-set -x
-
-case "$1" in
-    *i386-debian-base)
-        BASE="i386/debian:bullseye"
-        BASE_TYPE=debian-base
-        ;;
-    *debian-base)
-        BASE="debian:bullseye"
-        BASE_TYPE=debian-base
-        ;;
-    *ubuntu-base)
-        BASE="ubuntu:20.04"
-        BASE_TYPE=ubuntu-base
-        ;;
-esac
-echo "::set-output name=BASE::${BASE}"
-echo "::set-output name=BASE_TYPE::${BASE_TYPE}"
diff --git a/.github/scripts/build-base-image.sh b/.github/scripts/build-base-image.sh
new file mode 100755
index 0000000000..a064813e73
--- /dev/null
+++ b/.github/scripts/build-base-image.sh
@@ -0,0 +1,55 @@
+#!/bin/bash
+
+if [ -z "${BASE_TAG}" ]; then
+    BASE_TAG=$(grep "ARG BASE=" ".github/dockerfiles/Dockerfile.${1}" | head -1 | tr '=' ' ' | awk '{print $3}')
+    BASE_CACHE="--cache-from ${BASE_TAG}"
+fi
+
+case "${BASE_TAG}" in
+    *i386-debian-base)
+        BASE="i386/debian:bullseye"
+        BASE_TYPE=debian-base
+        ;;
+    *debian-base)
+        BASE="debian:bullseye"
+        BASE_TYPE=debian-base
+        ;;
+    *ubuntu-base)
+        BASE="ubuntu:20.04"
+        BASE_TYPE=ubuntu-base
+        ;;
+esac
+
+echo "::set-output name=BASE::${BASE}"
+echo "::set-output name=BASE_TAG::${BASE_TAG}"
+echo "::set-output name=BASE_TYPE::${BASE_TYPE}"
+
+if [ -f "otp_docker_base.tar" ]; then
+    docker load -i "otp_docker_base.tar"
+    echo "::set-output name=BASE_BUILD::loaded"
+elif [ -f "otp_docker_base/otp_docker_base.tar" ]; then
+    docker load -i "otp_docker_base/otp_docker_base.tar"
+    echo "::set-output name=BASE_BUILD::loaded"
+else
+    if [ -n "${BASE_CACHE}" ]; then
+        docker pull "${BASE_TAG}"
+    fi
+
+    BASE_IMAGE_ID=$(docker images -q "${BASE_TAG}")
+
+    docker build --pull --tag "${BASE_TAG}" \
+       ${BASE_CACHE} \
+       --file ".github/dockerfiles/Dockerfile.${BASE_TYPE}" \
+       --build-arg MAKEFLAGS=-j$(($(nproc) + 2)) \
+       --build-arg USER=otptest --build-arg GROUP=uucp \
+       --build-arg uid="$(id -u)" \
+       --build-arg BASE="${BASE}" .github/
+
+    NEW_BASE_IMAGE_ID=$(docker images -q "${BASE_TAG}")
+    if [ "${BASE_IMAGE_ID}" = "${NEW_BASE_IMAGE_ID}" ]; then
+        echo "::set-output name=BASE_BUILD::cached"
+    else
+        echo "::set-output name=BASE_BUILD::re-built"
+        docker save "${BASE_TAG}" > "otp_docker_base.tar"
+    fi
+fi
diff --git a/.github/scripts/restore-otp-image.sh b/.github/scripts/restore-otp-image.sh
new file mode 100755
index 0000000000..421e888758
--- /dev/null
+++ b/.github/scripts/restore-otp-image.sh
@@ -0,0 +1,12 @@
+#!/bin/bash
+
+BASE_TAG=$(grep "ARG BASE=" ".github/dockerfiles/Dockerfile.64-bit" | head -1 | tr '=' ' ' | awk '{print $3}')
+
+BASE_TAG="$BASE_TAG" .github/scripts/build-base-image.sh
+
+cat > Dockerfile <<EOF
+FROM ${BASE_TAG}
+ADD otp-ubuntu-20.04.tar.gz /
+WORKDIR /buildroot/otp/
+EOF
+docker build -t otp .
diff --git a/.github/workflows/main.yaml b/.github/workflows/main.yaml
index fec5ee78ee..c85c224f12 100644
--- a/.github/workflows/main.yaml
+++ b/.github/workflows/main.yaml
@@ -25,6 +25,8 @@ jobs:
   pack:
     name: Build Erlang/OTP (64-bit)
     runs-on: ubuntu-latest
+    outputs:
+        BASE_BUILD: ${{ steps.base-build.outputs.BASE_BUILD }}
     steps:
       - uses: actions/checkout@v2
       - name: Create initial pre-release tar
@@ -40,40 +42,35 @@ jobs:
           registry: docker.pkg.github.com
           username: ${{ github.repository_owner }}
           password: ${{ secrets.GITHUB_TOKEN }}
-      - name: Calculate BASE image
-        id: base
-        run: |
-          BASE_TAG=$(grep "ARG BASE=" .github/dockerfiles/Dockerfile.64-bit | head -1 | tr '=' ' ' | awk '{print $3}')
-          echo "::set-output name=BASE_TAG::${BASE_TAG}"
-          .github/scripts/base-tag "${BASE_TAG}"
-      - name: Pull BASE image
-        run: docker pull ${{ steps.base.outputs.BASE_TAG }}
       - name: Build BASE image
-        run: |
-          docker build --pull --tag ${{ steps.base.outputs.BASE_TAG }} \
-            --cache-from ${{ steps.base.outputs.BASE_TAG }} \
-            --file .github/dockerfiles/Dockerfile.${{ steps.base.outputs.BASE_TYPE }} \
-            --build-arg USER=otptest --build-arg GROUP=uucp \
-            --build-arg uid=$(id -u) \
-            --build-arg BASE=${{ steps.base.outputs.BASE }} .github/
-      - name: Build 64-bit image
+        id: base-build
+        run: .github/scripts/build-base-image.sh 64-bit
+      - name: Save BASE image
+        if: steps.base-build.outputs.BASE_BUILD == 're-built'
+        uses: actions/upload-artifact@v2
+        with:
+            name: otp_docker_base
+            path: otp_docker_base.tar
+      - name: Build image
         run: |
           mv otp_src.tar.gz .github/otp.tar.gz
           docker build --tag otp \
             --build-arg MAKEFLAGS=-j$(($(nproc) + 2)) \
-            --file .github/dockerfiles/Dockerfile.64-bit \
+            --file ".github/dockerfiles/Dockerfile.64-bit" \
             .github/
-      - name: Save docker image
-        run: docker save otp > otp_docker.tar
-      - name: Upload docker image
+      - name: Save Erlang/OTP image
+        run: |
+            docker run -v $PWD:/github --entrypoint "" otp \
+              tar czf /github/otp-ubuntu-20.04.tar.gz /buildroot/ /otp/
+      - name: Upload otp ubuntu image
         uses: actions/upload-artifact@v2
         with:
-          name: otp_docker
-          path: otp_docker.tar
+          name: otp-ubuntu-20.04
+          path: otp-ubuntu-20.04.tar.gz
       - name: Build pre-built tar archives
         run: |
-          docker run -v $PWD:/github otp \
-            "scripts/build-otp-tar -o /github/otp_clean_src.tar.gz /github/otp_src.tar.gz -b /buildroot/otp/ /buildroot/otp.tar.gz"
+          docker run -v $PWD:/github --entrypoint "" otp \
+            scripts/build-otp-tar -o /github/otp_clean_src.tar.gz /github/otp_src.tar.gz -b /buildroot/otp/ /buildroot/otp.tar.gz
       - name: Upload pre-built tar archive
         uses: actions/upload-artifact@v2
         with:
@@ -293,47 +290,45 @@ jobs:
         uses: actions/download-artifact@v2
         with:
           name: otp_prebuilt_no_chunks
-      - run: echo "${{ needs.pack.outputs.changes }}"
       - name: Docker login
         uses: docker/login-action@v1
         with:
           registry: docker.pkg.github.com
           username: ${{ github.repository_owner }}
           password: ${{ secrets.GITHUB_TOKEN }}
-      - name: Calculate BASE image
-        id: base
-        run: |
-          BASE_TAG=$(grep "ARG BASE=" .github/dockerfiles/Dockerfile.${{ matrix.type }} | head -1 | tr '=' ' ' | awk '{print $3}')
-          echo "::set-output name=BASE_TAG::${BASE_TAG}"
-          .github/scripts/base-tag "${BASE_TAG}"
-      - name: Pull BASE image
-        run: docker pull ${{ steps.base.outputs.BASE_TAG }}
-      - name: Build BASE image
-        run: |
-          docker build --pull --tag ${{ steps.base.outputs.BASE_TAG }} \
-            --cache-from ${{ steps.base.outputs.BASE_TAG }} \
-            --file .github/dockerfiles/Dockerfile.${{ steps.base.outputs.BASE_TYPE }} \
-            --build-arg USER=otptest --build-arg GROUP=uucp \
-            --build-arg uid=$(id -u) \
-            --build-arg BASE=${{ steps.base.outputs.BASE }} .github/
+      - name: Build base image
+        run: .github/scripts/build-base-image.sh ${{ matrix.type }}
       - name: Build ${{ matrix.type }} image
         run: |
           mv otp_src.tar.gz .github/otp.tar.gz
           docker build --tag otp --file .github/dockerfiles/Dockerfile.${{ matrix.type }} \
-            .github/
+            .github
 
   documentation:
     name: Build and check documentation
     runs-on: ubuntu-latest
     needs: pack
     steps:
+      - uses: actions/checkout@v2
       ## Download docker images
-      - name: Download source archive
+      - name: Docker login
+        uses: docker/login-action@v1
+        with:
+          registry: docker.pkg.github.com
+          username: ${{ github.repository_owner }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+      - name: Download base build
+        if: needs.pack.outputs.BASE_BUILD == 're-built'
         uses: actions/download-artifact@v2
         with:
-          name: otp_docker
+          name: otp_docker_base
+      - name: Download otp build
+        uses: actions/download-artifact@v2
+        with:
+          name: otp-ubuntu-20.04
       - name: Restore docker image
-        run: docker load < otp_docker.tar
+        run: .github/scripts/restore-otp-image.sh
+
       ## Build pre-built tar with chunks
       - name: Build doc chunks
         run: |
@@ -387,13 +382,25 @@ jobs:
     runs-on: ubuntu-latest
     needs: pack
     steps:
+      - uses: actions/checkout@v2
       ## Download docker images
-      - name: Download source archive
+      - name: Docker login
+        uses: docker/login-action@v1
+        with:
+          registry: docker.pkg.github.com
+          username: ${{ github.repository_owner }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+      - name: Download base build
+        if: needs.pack.outputs.BASE_BUILD == 're-built'
+        uses: actions/download-artifact@v2
+        with:
+          name: otp_docker_base
+      - name: Download otp build
         uses: actions/download-artifact@v2
         with:
-          name: otp_docker
+          name: otp-ubuntu-20.04
       - name: Restore docker image
-        run: docker load < otp_docker.tar
+        run: .github/scripts/restore-otp-image.sh
       - name: Install clang-format
         run: |
           docker build -t otp - <<EOF
@@ -419,13 +426,25 @@ jobs:
         # type: ["os_mon","sasl"]
       fail-fast: false
     steps:
+      - uses: actions/checkout@v2
       ## Download docker images
-      - name: Download source archive
+      - name: Download base build
+        if: needs.pack.outputs.BASE_BUILD == 're-built'
+        uses: actions/download-artifact@v2
+        with:
+          name: otp_docker_base
+      - name: Download otp build
         uses: actions/download-artifact@v2
         with:
-          name: otp_docker
+          name: otp-ubuntu-20.04
+      - name: Docker login
+        uses: docker/login-action@v1
+        with:
+          registry: docker.pkg.github.com
+          username: ${{ github.repository_owner }}
+          password: ${{ secrets.GITHUB_TOKEN }}
       - name: Restore docker image
-        run: docker load < otp_docker.tar
+        run: .github/scripts/restore-otp-image.sh
       - name: Run tests
         id: run-tests
         run: |
@@ -469,8 +488,18 @@ jobs:
     if: always() # Run even if the need has failed
     needs: test
     steps:
+      - uses: actions/checkout@v2
+      - name: Docker login
+        uses: docker/login-action@v1
+        with:
+          registry: docker.pkg.github.com
+          username: ${{ github.repository_owner }}
+          password: ${{ secrets.GITHUB_TOKEN }}
       - name: Download test results
         uses: actions/download-artifact@v2
+      - run: mv otp-ubuntu-20.04/otp-ubuntu-20.04.tar.gz .
+      - name: Restore docker image
+        run: .github/scripts/restore-otp-image.sh
       - name: Merge test results
         run: |
           shopt -s nullglob
@@ -478,7 +507,6 @@ jobs:
           for file in *_test_results/*.tar.gz; do
             tar xzf $file
           done
-          docker load < otp_docker/otp_docker.tar
           docker run -v $PWD/make_test_dir:/buildroot/otp/erts/make_test_dir otp \
             "ct_run -refresh_logs /buildroot/otp/erts/make_test_dir"
       - name: Run system tests
diff --git a/.github/workflows/update-base.yaml b/.github/workflows/update-base.yaml
index 0cd0be8102..fdc8a168d6 100644
--- a/.github/workflows/update-base.yaml
+++ b/.github/workflows/update-base.yaml
@@ -33,12 +33,8 @@ jobs:
           echo "::set-output name=BASE_TAG::docker.pkg.github.com/erlang/otp/${{ matrix.type }}"
           .github/scripts/base-tag "${{ matrix.type }}"
       - name: Build base image
-        run: |
-          docker build --pull --tag ${{ steps.base.outputs.BASE_TAG }} \
-            --cache-from ${{ steps.base.outputs.BASE_TAG }} \
-            --file .github/dockerfiles/Dockerfile.${{ steps.base.outputs.BASE_TYPE }} \
-            --build-arg USER=otptest --build-arg uid=$(id -u) \
-            --build-arg GROUP=uucp --build-arg gid=$(id -g uucp) \
-            --build-arg BASE=${{ steps.base.outputs.BASE }} .github
+        run: >-
+            BASE_TAG=docker.pkg.github.com/erlang/otp/${{ matrix.type }}
+            .github/scripts/build-base-image.sh
       - name: Push base image
         run: docker push ${{ steps.base.outputs.BASE_TAG }}
-- 
2.35.3

Places

File 6032-gh-Only-upload-erlang-artifacts-in-order-to-optimize.patch of Package erlang (Revision da0f8958cbfdaccc567b3effd8dde26d)

Places