From 9c88a780328d257a9af3e304095d92f81ae5f40d Mon Sep 17 00:00:00 2001 From: Hamza Remmal Date: Tue, 22 Apr 2025 07:21:22 +0100 Subject: [PATCH 01/10] chore: add build step for non-bootstrapped compiler --- .github/workflows/ci.yaml | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index db2a66b8b234..6ba35e942e70 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -53,7 +53,35 @@ env: # text on stderr and so can break tests which check the output of a program). jobs: + build-non-bootstrapped: + name: Build Non-Bootstrapped Compiler + runs-on: [self-hosted, Linux] + if: github.event_name == 'pull_request' + container: + image: lampepfl/dotty:2024-10-18 + options: --cpu-shares 4096 + volumes: + - /nfs-cache/.develocity/build-cache/${{ github.event.pull_request.number }}:/root/.sbt/1.0/.develocity/build-cache + - ${{ github.workspace }}/../../cache/sbt:/root/.sbt + - ${{ github.workspace }}/../../cache/ivy:/root/.ivy2/cache + - ${{ github.workspace }}/../../cache/general:/root/.cache + steps: + - name: Set JDK 17 as default + run: echo "/usr/lib/jvm/java-17-openjdk-amd64/bin" >> $GITHUB_PATH + - name: Checkout cleanup script + uses: actions/checkout@v4 + - name: Cleanup + run: .github/workflows/cleanup.sh + - name: Git Checkout + uses: actions/checkout@v4 + - name: Add SBT proxy repositories + run: cp -vf .github/workflows/repositories /root/.sbt/ ; true + - name: Compile Non-Bootstrapped Dotty + run: ./project/scripts/sbt "; compile" + + test_non_bootstrapped: + needs: build-non-bootstrapped runs-on: [self-hosted, Linux] container: image: lampepfl/dotty:2024-10-18 From 2e77400df2a3dfd9adac5e1a9fe37364fc472375 Mon Sep 17 00:00:00 2001 From: Hamza Remmal Date: Tue, 22 Apr 2025 07:28:00 +0100 Subject: [PATCH 02/10] chore: mount the caching folder as ro in test --- .github/workflows/ci.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 6ba35e942e70..940b332b1701 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -87,6 +87,7 @@ jobs: image: lampepfl/dotty:2024-10-18 options: --cpu-shares 4096 volumes: + - /nfs-cache/.develocity/build-cache/${{ github.event.pull_request.number }}:/root/.sbt/1.0/.develocity/build-cache:ro - ${{ github.workspace }}/../../cache/sbt:/root/.sbt - ${{ github.workspace }}/../../cache/ivy:/root/.ivy2/cache - ${{ github.workspace }}/../../cache/general:/root/.cache From fdc16417caa74a1761c5a9cc85faaaff3ddd9bc2 Mon Sep 17 00:00:00 2001 From: Hamza Remmal Date: Tue, 22 Apr 2025 07:39:55 +0100 Subject: [PATCH 03/10] chore: populate the build-cache from the NFS --- .github/workflows/ci.yaml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 940b332b1701..fd7445afff42 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -87,7 +87,7 @@ jobs: image: lampepfl/dotty:2024-10-18 options: --cpu-shares 4096 volumes: - - /nfs-cache/.develocity/build-cache/${{ github.event.pull_request.number }}:/root/.sbt/1.0/.develocity/build-cache:ro + - /nfs-cache/.develocity/build-cache/${{ github.event.pull_request.number }}:/root/build-cache-source:ro - ${{ github.workspace }}/../../cache/sbt:/root/.sbt - ${{ github.workspace }}/../../cache/ivy:/root/.ivy2/cache - ${{ github.workspace }}/../../cache/general:/root/.cache @@ -103,6 +103,8 @@ jobs: && github.repository == 'scala/scala3' )" steps: + - name: Populate the Build Cache + run: cp -r /root/build-cache-source/. /root/.sbt/1.0/.develocity/build-cache - name: Set JDK 17 as default run: echo "/usr/lib/jvm/java-17-openjdk-amd64/bin" >> $GITHUB_PATH From 991273b86500f381b642032a8f8dff9982e33099 Mon Sep 17 00:00:00 2001 From: Hamza Remmal Date: Tue, 22 Apr 2025 07:46:43 +0100 Subject: [PATCH 04/10] chore: add build bootstrapped compiler --- .github/workflows/ci.yaml | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index fd7445afff42..d77568679eda 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -79,6 +79,36 @@ jobs: - name: Compile Non-Bootstrapped Dotty run: ./project/scripts/sbt "; compile" + build-bootstrapped: + name: Build Bootstrapped Compiler + needs: build-non-bootstrapped + runs-on: [self-hosted, Linux] + container: + image: lampepfl/dotty:2024-10-18 + options: --cpu-shares 4096 + volumes: + - /nfs-cache/.develocity/build-cache/${{ github.event.pull_request.number }}:/root/build-cache-source + - ${{ github.workspace }}/../../cache/sbt:/root/.sbt + - ${{ github.workspace }}/../../cache/ivy:/root/.ivy2/cache + - ${{ github.workspace }}/../../cache/general:/root/.cache + steps: + - name: Populate the Local Build Cache + run: cp -r /root/build-cache-source/. /root/.sbt/1.0/.develocity/build-cache + - name: Set JDK 17 as default + run: echo "/usr/lib/jvm/java-17-openjdk-amd64/bin" >> $GITHUB_PATH + - name: Checkout cleanup script + uses: actions/checkout@v4 + - name: Cleanup + run: .github/workflows/cleanup.sh + - name: Git Checkout + uses: actions/checkout@v4 + - name: Add SBT proxy repositories + run: cp -vf .github/workflows/repositories /root/.sbt/ ; true + - name: Compile Bootstrapped Dotty + run: ./project/scripts/sbt "; scala3-bootstrapped/compile" + - name: Populate the Remote Build Cache + run: cp -r /root/.sbt/1.0/.develocity/build-cache/. /root/build-cache-source + test_non_bootstrapped: needs: build-non-bootstrapped From 3a1d82c10bb7cdb493720592e6fce1ba2fc60c7a Mon Sep 17 00:00:00 2001 From: Hamza Remmal Date: Tue, 22 Apr 2025 08:07:34 +0100 Subject: [PATCH 05/10] chore: new folder --- .github/workflows/ci.yaml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index d77568679eda..43285ca15dae 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -61,7 +61,7 @@ jobs: image: lampepfl/dotty:2024-10-18 options: --cpu-shares 4096 volumes: - - /nfs-cache/.develocity/build-cache/${{ github.event.pull_request.number }}:/root/.sbt/1.0/.develocity/build-cache + - /nfs-cache/.develocity/build-cache-2/${{ github.event.pull_request.number }}:/root/.sbt/1.0/.develocity/build-cache - ${{ github.workspace }}/../../cache/sbt:/root/.sbt - ${{ github.workspace }}/../../cache/ivy:/root/.ivy2/cache - ${{ github.workspace }}/../../cache/general:/root/.cache @@ -87,7 +87,7 @@ jobs: image: lampepfl/dotty:2024-10-18 options: --cpu-shares 4096 volumes: - - /nfs-cache/.develocity/build-cache/${{ github.event.pull_request.number }}:/root/build-cache-source + - /nfs-cache/.develocity/build-cache-2/${{ github.event.pull_request.number }}:/root/build-cache-source - ${{ github.workspace }}/../../cache/sbt:/root/.sbt - ${{ github.workspace }}/../../cache/ivy:/root/.ivy2/cache - ${{ github.workspace }}/../../cache/general:/root/.cache @@ -107,7 +107,7 @@ jobs: - name: Compile Bootstrapped Dotty run: ./project/scripts/sbt "; scala3-bootstrapped/compile" - name: Populate the Remote Build Cache - run: cp -r /root/.sbt/1.0/.develocity/build-cache/. /root/build-cache-source + run: cp -r /root/.sbt/1.0/.develocity/build-cache/. /root/build-cache-source/ test_non_bootstrapped: @@ -117,7 +117,7 @@ jobs: image: lampepfl/dotty:2024-10-18 options: --cpu-shares 4096 volumes: - - /nfs-cache/.develocity/build-cache/${{ github.event.pull_request.number }}:/root/build-cache-source:ro + - /nfs-cache/.develocity/build-cache-2/${{ github.event.pull_request.number }}:/root/build-cache-source:ro - ${{ github.workspace }}/../../cache/sbt:/root/.sbt - ${{ github.workspace }}/../../cache/ivy:/root/.ivy2/cache - ${{ github.workspace }}/../../cache/general:/root/.cache From 94aa93dd9f0fec624c965f73193b4318fda88db8 Mon Sep 17 00:00:00 2001 From: Hamza Remmal Date: Tue, 22 Apr 2025 08:18:30 +0100 Subject: [PATCH 06/10] debug: look at the remote cache --- .github/workflows/ci.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 43285ca15dae..75e975fe6df0 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -106,6 +106,7 @@ jobs: run: cp -vf .github/workflows/repositories /root/.sbt/ ; true - name: Compile Bootstrapped Dotty run: ./project/scripts/sbt "; scala3-bootstrapped/compile" + - run: ls -alF /root/.sbt/1.0/.develocity/build-cache/ - name: Populate the Remote Build Cache run: cp -r /root/.sbt/1.0/.develocity/build-cache/. /root/build-cache-source/ From a01de13c2f6518bf0e070e81f607d1f456fee4aa Mon Sep 17 00:00:00 2001 From: Hamza Remmal Date: Tue, 22 Apr 2025 08:20:55 +0100 Subject: [PATCH 07/10] chore: new build cache folder --- .github/workflows/ci.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 75e975fe6df0..ba3b4450edf9 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -61,7 +61,7 @@ jobs: image: lampepfl/dotty:2024-10-18 options: --cpu-shares 4096 volumes: - - /nfs-cache/.develocity/build-cache-2/${{ github.event.pull_request.number }}:/root/.sbt/1.0/.develocity/build-cache + - /nfs-cache/.develocity/build-cache-3/${{ github.event.pull_request.number }}:/root/.sbt/1.0/.develocity/build-cache - ${{ github.workspace }}/../../cache/sbt:/root/.sbt - ${{ github.workspace }}/../../cache/ivy:/root/.ivy2/cache - ${{ github.workspace }}/../../cache/general:/root/.cache @@ -87,7 +87,7 @@ jobs: image: lampepfl/dotty:2024-10-18 options: --cpu-shares 4096 volumes: - - /nfs-cache/.develocity/build-cache-2/${{ github.event.pull_request.number }}:/root/build-cache-source + - /nfs-cache/.develocity/build-cache-3/${{ github.event.pull_request.number }}:/root/build-cache-source - ${{ github.workspace }}/../../cache/sbt:/root/.sbt - ${{ github.workspace }}/../../cache/ivy:/root/.ivy2/cache - ${{ github.workspace }}/../../cache/general:/root/.cache @@ -118,7 +118,7 @@ jobs: image: lampepfl/dotty:2024-10-18 options: --cpu-shares 4096 volumes: - - /nfs-cache/.develocity/build-cache-2/${{ github.event.pull_request.number }}:/root/build-cache-source:ro + - /nfs-cache/.develocity/build-cache-3/${{ github.event.pull_request.number }}:/root/build-cache-source:ro - ${{ github.workspace }}/../../cache/sbt:/root/.sbt - ${{ github.workspace }}/../../cache/ivy:/root/.ivy2/cache - ${{ github.workspace }}/../../cache/general:/root/.cache From 9b4ba347ab3f3e54393ee7bb21240f3b6845af35 Mon Sep 17 00:00:00 2001 From: Hamza Remmal Date: Tue, 22 Apr 2025 08:37:07 +0100 Subject: [PATCH 08/10] fix: unmount the .sbt folder --- .github/workflows/ci.yaml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index ba3b4450edf9..4a53a4cf23c4 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -61,8 +61,8 @@ jobs: image: lampepfl/dotty:2024-10-18 options: --cpu-shares 4096 volumes: - - /nfs-cache/.develocity/build-cache-3/${{ github.event.pull_request.number }}:/root/.sbt/1.0/.develocity/build-cache - - ${{ github.workspace }}/../../cache/sbt:/root/.sbt + - /nfs-cache/.develocity/build-cache-4/${{ github.event.pull_request.number }}:/root/.sbt/1.0/.develocity/build-cache + #- ${{ github.workspace }}/../../cache/sbt:/root/.sbt - ${{ github.workspace }}/../../cache/ivy:/root/.ivy2/cache - ${{ github.workspace }}/../../cache/general:/root/.cache steps: @@ -87,8 +87,8 @@ jobs: image: lampepfl/dotty:2024-10-18 options: --cpu-shares 4096 volumes: - - /nfs-cache/.develocity/build-cache-3/${{ github.event.pull_request.number }}:/root/build-cache-source - - ${{ github.workspace }}/../../cache/sbt:/root/.sbt + - /nfs-cache/.develocity/build-cache-4/${{ github.event.pull_request.number }}:/root/build-cache-source + #- ${{ github.workspace }}/../../cache/sbt:/root/.sbt - ${{ github.workspace }}/../../cache/ivy:/root/.ivy2/cache - ${{ github.workspace }}/../../cache/general:/root/.cache steps: @@ -118,8 +118,8 @@ jobs: image: lampepfl/dotty:2024-10-18 options: --cpu-shares 4096 volumes: - - /nfs-cache/.develocity/build-cache-3/${{ github.event.pull_request.number }}:/root/build-cache-source:ro - - ${{ github.workspace }}/../../cache/sbt:/root/.sbt + - /nfs-cache/.develocity/build-cache-4/${{ github.event.pull_request.number }}:/root/build-cache-source:ro + #- ${{ github.workspace }}/../../cache/sbt:/root/.sbt - ${{ github.workspace }}/../../cache/ivy:/root/.ivy2/cache - ${{ github.workspace }}/../../cache/general:/root/.cache if: "github.event_name == 'schedule' && github.repository == 'scala/scala3' From ae723d7585e3b4a416120c1cceb84f87ee47be50 Mon Sep 17 00:00:00 2001 From: Hamza Remmal Date: Tue, 22 Apr 2025 08:41:10 +0100 Subject: [PATCH 09/10] chore: create the cache folder --- .github/workflows/ci.yaml | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 4a53a4cf23c4..c2745dd7cd08 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -93,7 +93,9 @@ jobs: - ${{ github.workspace }}/../../cache/general:/root/.cache steps: - name: Populate the Local Build Cache - run: cp -r /root/build-cache-source/. /root/.sbt/1.0/.develocity/build-cache + run: | + mkdir -p /root/.sbt/1.0/.develocity/build-cache + cp -r /root/build-cache-source/. /root/.sbt/1.0/.develocity/build-cache - name: Set JDK 17 as default run: echo "/usr/lib/jvm/java-17-openjdk-amd64/bin" >> $GITHUB_PATH - name: Checkout cleanup script @@ -135,7 +137,9 @@ jobs: )" steps: - name: Populate the Build Cache - run: cp -r /root/build-cache-source/. /root/.sbt/1.0/.develocity/build-cache + run: | + mkdir -p /root/.sbt/1.0/.develocity/build-cache + cp -r /root/build-cache-source/. /root/.sbt/1.0/.develocity/build-cache - name: Set JDK 17 as default run: echo "/usr/lib/jvm/java-17-openjdk-amd64/bin" >> $GITHUB_PATH From caf74dc054fa06de29c08084c2ac8a3df3847153 Mon Sep 17 00:00:00 2001 From: Hamza Remmal Date: Tue, 22 Apr 2025 08:48:36 +0100 Subject: [PATCH 10/10] chore: add cache in MiMa --- .github/workflows/ci.yaml | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index c2745dd7cd08..03e9ea4912b7 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -359,12 +359,14 @@ jobs: mima: name: MiMa + needs: build-bootstrapped runs-on: [self-hosted, Linux] container: image: lampepfl/dotty:2024-10-18 options: --cpu-shares 4096 volumes: - - ${{ github.workspace }}/../../cache/sbt:/root/.sbt + - /nfs-cache/.develocity/build-cache-4/${{ github.event.pull_request.number }}:/root/build-cache-source + #- ${{ github.workspace }}/../../cache/sbt:/root/.sbt - ${{ github.workspace }}/../../cache/ivy:/root/.ivy2/cache - ${{ github.workspace }}/../../cache/general:/root/.cache if: "github.event_name == 'schedule' && github.repository == 'scala/scala3' @@ -380,6 +382,10 @@ jobs: && github.repository == 'scala/scala3' )" steps: + - name: Populate the Local Build Cache + run: | + mkdir -p /root/.sbt/1.0/.develocity/build-cache + cp -r /root/build-cache-source/. /root/.sbt/1.0/.develocity/build-cache - name: Set JDK 17 as default run: echo "/usr/lib/jvm/java-17-openjdk-amd64/bin" >> $GITHUB_PATH @@ -408,6 +414,8 @@ jobs: run: | # This script cleans the compiler and recompiles it from scratch (keep as last run) ./project/scripts/scala2-library-tasty-mima.sh + - name: Populate the Remote Build Cache + run: cp -r /root/.sbt/1.0/.develocity/build-cache/. /root/build-cache-source/ community_build_a: runs-on: [self-hosted, Linux]