Skip to content

Enable caching in the CI #23024

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
wants to merge 10 commits into
base: main
Choose a base branch
from
+76 −2
Draft

Enable caching in the CI #23024

Changes from all commits
Commits
Show all changes
10 commits
Select commit Hold shift + click to select a range
9c88a78
chore: add build step for non-bootstrapped compiler
hamzaremmal Apr 22, 2025
2e77400
chore: mount the caching folder as ro in test
hamzaremmal Apr 22, 2025
fdc1641
chore: populate the build-cache from the NFS
hamzaremmal Apr 22, 2025
991273b
chore: add build bootstrapped compiler
hamzaremmal Apr 22, 2025
3a1d82c
chore: new folder
hamzaremmal Apr 22, 2025
94aa93d
debug: look at the remote cache
hamzaremmal Apr 22, 2025
a01de13
chore: new build cache folder
hamzaremmal Apr 22, 2025
9b4ba34
fix: unmount the .sbt folder
hamzaremmal Apr 22, 2025
ae723d7
chore: create the cache folder
hamzaremmal Apr 22, 2025
caf74dc
chore: add cache in MiMa
hamzaremmal Apr 22, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
78 changes: 76 additions & 2 deletions .github/workflows/ci.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -53,13 +53,75 @@ env:
# text on stderr and so can break tests which check the output of a program).

jobs:
build-non-bootstrapped:
name: Build Non-Bootstrapped Compiler
runs-on: [self-hosted, Linux]
if: github.event_name == 'pull_request'
container:
image: lampepfl/dotty:2024-10-18
options: --cpu-shares 4096
volumes:
- /nfs-cache/.develocity/build-cache-4/${{ github.event.pull_request.number }}:/root/.sbt/1.0/.develocity/build-cache
#- ${{ github.workspace }}/../../cache/sbt:/root/.sbt
- ${{ github.workspace }}/../../cache/ivy:/root/.ivy2/cache
- ${{ github.workspace }}/../../cache/general:/root/.cache
steps:
- name: Set JDK 17 as default
run: echo "/usr/lib/jvm/java-17-openjdk-amd64/bin" >> $GITHUB_PATH
- name: Checkout cleanup script
uses: actions/checkout@v4
- name: Cleanup
run: .github/workflows/cleanup.sh
- name: Git Checkout
uses: actions/checkout@v4
- name: Add SBT proxy repositories
run: cp -vf .github/workflows/repositories /root/.sbt/ ; true
- name: Compile Non-Bootstrapped Dotty
run: ./project/scripts/sbt "; compile"

build-bootstrapped:
name: Build Bootstrapped Compiler
needs: build-non-bootstrapped
runs-on: [self-hosted, Linux]
container:
image: lampepfl/dotty:2024-10-18
options: --cpu-shares 4096
volumes:
- /nfs-cache/.develocity/build-cache-4/${{ github.event.pull_request.number }}:/root/build-cache-source
#- ${{ github.workspace }}/../../cache/sbt:/root/.sbt
- ${{ github.workspace }}/../../cache/ivy:/root/.ivy2/cache
- ${{ github.workspace }}/../../cache/general:/root/.cache
steps:
- name: Populate the Local Build Cache
run: |
mkdir -p /root/.sbt/1.0/.develocity/build-cache
cp -r /root/build-cache-source/. /root/.sbt/1.0/.develocity/build-cache
- name: Set JDK 17 as default
run: echo "/usr/lib/jvm/java-17-openjdk-amd64/bin" >> $GITHUB_PATH
- name: Checkout cleanup script
uses: actions/checkout@v4
- name: Cleanup
run: .github/workflows/cleanup.sh
- name: Git Checkout
uses: actions/checkout@v4
- name: Add SBT proxy repositories
run: cp -vf .github/workflows/repositories /root/.sbt/ ; true
- name: Compile Bootstrapped Dotty
run: ./project/scripts/sbt "; scala3-bootstrapped/compile"
- run: ls -alF /root/.sbt/1.0/.develocity/build-cache/
- name: Populate the Remote Build Cache
run: cp -r /root/.sbt/1.0/.develocity/build-cache/. /root/build-cache-source/


test_non_bootstrapped:
needs: build-non-bootstrapped
runs-on: [self-hosted, Linux]
container:
image: lampepfl/dotty:2024-10-18
options: --cpu-shares 4096
volumes:
- ${{ github.workspace }}/../../cache/sbt:/root/.sbt
- /nfs-cache/.develocity/build-cache-4/${{ github.event.pull_request.number }}:/root/build-cache-source:ro
#- ${{ github.workspace }}/../../cache/sbt:/root/.sbt
- ${{ github.workspace }}/../../cache/ivy:/root/.ivy2/cache
- ${{ github.workspace }}/../../cache/general:/root/.cache
if: "github.event_name == 'schedule' && github.repository == 'scala/scala3'
Expand All @@ -74,6 +136,10 @@ jobs:
&& github.repository == 'scala/scala3'
)"
steps:
- name: Populate the Build Cache
run: |
mkdir -p /root/.sbt/1.0/.develocity/build-cache
cp -r /root/build-cache-source/. /root/.sbt/1.0/.develocity/build-cache
- name: Set JDK 17 as default
run: echo "/usr/lib/jvm/java-17-openjdk-amd64/bin" >> $GITHUB_PATH

Expand Down Expand Up @@ -293,12 +359,14 @@ jobs:

mima:
name: MiMa
needs: build-bootstrapped
runs-on: [self-hosted, Linux]
container:
image: lampepfl/dotty:2024-10-18
options: --cpu-shares 4096
volumes:
- ${{ github.workspace }}/../../cache/sbt:/root/.sbt
- /nfs-cache/.develocity/build-cache-4/${{ github.event.pull_request.number }}:/root/build-cache-source
#- ${{ github.workspace }}/../../cache/sbt:/root/.sbt
- ${{ github.workspace }}/../../cache/ivy:/root/.ivy2/cache
- ${{ github.workspace }}/../../cache/general:/root/.cache
if: "github.event_name == 'schedule' && github.repository == 'scala/scala3'
Expand All @@ -314,6 +382,10 @@ jobs:
&& github.repository == 'scala/scala3'
)"
steps:
- name: Populate the Local Build Cache
run: |
mkdir -p /root/.sbt/1.0/.develocity/build-cache
cp -r /root/build-cache-source/. /root/.sbt/1.0/.develocity/build-cache
- name: Set JDK 17 as default
run: echo "/usr/lib/jvm/java-17-openjdk-amd64/bin" >> $GITHUB_PATH

Expand Down Expand Up @@ -342,6 +414,8 @@ jobs:
run: |
# This script cleans the compiler and recompiles it from scratch (keep as last run)
./project/scripts/scala2-library-tasty-mima.sh
- name: Populate the Remote Build Cache
run: cp -r /root/.sbt/1.0/.develocity/build-cache/. /root/build-cache-source/

community_build_a:
runs-on: [self-hosted, Linux]
Expand Down