diff --git a/.github/scripts/archive-node-modules.sh b/.github/scripts/archive-node-modules.sh new file mode 100755 index 0000000000..5b989ca123 --- /dev/null +++ b/.github/scripts/archive-node-modules.sh @@ -0,0 +1,42 @@ +#!/usr/bin/env bash +# +# Packs the installed dependency tree into ONE compressed file for the other jobs to unpack. +# +# Paired with restore-node-modules.sh. A single ~2.9 GB file beats handing actions/cache ~900k loose +# files in both directions, and it is small enough to sit in the 10 GB per-repo cache budget — which +# the exploded ~9 GB tree never was. +set -o pipefail + +ARCHIVE="${NODE_MODULES_ARCHIVE:?NODE_MODULES_ARCHIVE must be set}" + +PATHS=() +for p in node_modules apps/*/node_modules packages/*/node_modules \ + packages/plugins/*/node_modules tools/node_modules; do + [ -d "$p" ] && PATHS+=("$p") +done +if [ ${#PATHS[@]} -eq 0 ]; then + echo "::error::install produced no node_modules directories" + exit 1 +fi +echo "archiving ${#PATHS[@]} directories" + +if command -v zstd >/dev/null 2>&1; then + COMPRESS="zstd -3 -T0" +else + echo "::warning::zstd unavailable — falling back to gzip (larger archive, slower)" + COMPRESS="gzip -3" +fi + +# `--warning=no-file-changed` because yarn's daemon can touch files while we read them; `|| [ $? -eq 1 ]` +# accepts tar's "file changed as we read it" (exit 1) while still failing on a real error (exit 2). +tar --warning=no-file-changed --use-compress-program="$COMPRESS" \ + -cf "$ARCHIVE" "${PATHS[@]}" || [ $? -eq 1 ] + +# Tolerating exit 1 means a TRUNCATED archive could be published, and every consumer would then fall +# back to its own multi-hour install. A cheap size floor catches the gross case. +size=$(stat -c%s "$ARCHIVE") +ls -lh "$ARCHIVE" +if [ "$size" -lt 104857600 ]; then + echo "::error::archive is only $size bytes — the dependency tree cannot be that small" + exit 1 +fi diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index c6300701db..caaf84c558 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -45,6 +45,9 @@ env: # Nx Cloud is disabled for this org; without this `nx run-many` hard-fails with # "Nx Cloud: Workspace is unable to be authorized. Exiting run." NX_NO_CLOUD: true + # The dependency tree travels between jobs as this one compressed file. Workspace-relative so the + # same string works for `tar` in a run step and for actions/cache. + NODE_MODULES_ARCHIVE: node-modules.tar.zst # Least-privilege scope for the automatic GITHUB_TOKEN. # This workflow only builds/tests/deploys from a checkout — read access is sufficient. @@ -67,20 +70,42 @@ jobs: - uses: actions/setup-node@v4 with: node-version: 24 - cache: yarn + + - name: Restore node_modules archive + id: cache + uses: actions/cache/restore@v4 + with: + path: ${{ env.NODE_MODULES_ARCHIVE }} + key: ${{ runner.os }}-${{ runner.arch }}-node-modules-${{ hashFiles('yarn.lock', 'patches/**', '.scripts/postinstall.js') }} - name: Install dependencies + if: steps.cache.outputs.cache-hit != 'true' run: yarn install --network-timeout 1000000 --frozen-lockfile --ignore-scripts - name: Run postinstall manually + if: steps.cache.outputs.cache-hit != 'true' run: yarn postinstall.manual + - name: Archive node_modules + if: steps.cache.outputs.cache-hit != 'true' + shell: bash + run: .github/scripts/archive-node-modules.sh + + - name: Save node_modules archive + if: steps.cache.outputs.cache-hit != 'true' + uses: actions/cache/save@v4 + # Explicit save so a FAILED install can never publish a half-built tree. + with: + path: ${{ env.NODE_MODULES_ARCHIVE }} + key: ${{ runner.os }}-${{ runner.arch }}-node-modules-${{ hashFiles('yarn.lock', 'patches/**', '.scripts/postinstall.js') }} + # The strict library verdict, as its own early-reporting check: this is the task set whose # per-library `strictTemplates` tsconfigs catch template type errors (the class that broke the # demo webapp image). It has no `needs` edge and no app build behind it, so it reports as soon # as install + the 71 library builds finish. build-libs: name: build-libs + needs: build-monorepo-root runs-on: ${{ vars.RUNNER_LINUX_X64_8 || 'ubuntu-latest' }} timeout-minutes: 180 steps: @@ -89,19 +114,25 @@ jobs: - uses: actions/setup-node@v4 with: node-version: 24 - cache: yarn - - name: Install dependencies - run: yarn install --network-timeout 1000000 --frozen-lockfile --ignore-scripts + - name: Restore node_modules archive + uses: actions/cache/restore@v4 + with: + path: ${{ env.NODE_MODULES_ARCHIVE }} + key: ${{ runner.os }}-${{ runner.arch }}-node-modules-${{ hashFiles('yarn.lock', 'patches/**', '.scripts/postinstall.js') }} - - name: Run postinstall manually - run: yarn postinstall.manual + - name: Restore node_modules + shell: bash + # Unpacks the tree build-monorepo-root published, or installs from scratch if the cache is + # unavailable. One step, so there is no `if:` on a previous step's outcome to get wrong. + run: .github/scripts/restore-node-modules.sh - name: Build packages run: yarn build:package:all build-api: name: build-api + needs: build-monorepo-root runs-on: ${{ vars.RUNNER_LINUX_X64_8 || 'ubuntu-latest' }} timeout-minutes: 180 steps: @@ -110,19 +141,25 @@ jobs: - uses: actions/setup-node@v4 with: node-version: 24 - cache: yarn - - name: Install dependencies - run: yarn install --network-timeout 1000000 --frozen-lockfile --ignore-scripts + - name: Restore node_modules archive + uses: actions/cache/restore@v4 + with: + path: ${{ env.NODE_MODULES_ARCHIVE }} + key: ${{ runner.os }}-${{ runner.arch }}-node-modules-${{ hashFiles('yarn.lock', 'patches/**', '.scripts/postinstall.js') }} - - name: Run postinstall manually - run: yarn postinstall.manual + - name: Restore node_modules + shell: bash + # Unpacks the tree build-monorepo-root published, or installs from scratch if the cache is + # unavailable. One step, so there is no `if:` on a previous step's outcome to get wrong. + run: .github/scripts/restore-node-modules.sh - name: Build API run: yarn build:api:prod:ci build-web: name: build-web + needs: build-monorepo-root runs-on: ${{ vars.RUNNER_LINUX_X64_8 || 'ubuntu-latest' }} timeout-minutes: 180 steps: @@ -131,19 +168,25 @@ jobs: - uses: actions/setup-node@v4 with: node-version: 24 - cache: yarn - - name: Install dependencies - run: yarn install --network-timeout 1000000 --frozen-lockfile --ignore-scripts + - name: Restore node_modules archive + uses: actions/cache/restore@v4 + with: + path: ${{ env.NODE_MODULES_ARCHIVE }} + key: ${{ runner.os }}-${{ runner.arch }}-node-modules-${{ hashFiles('yarn.lock', 'patches/**', '.scripts/postinstall.js') }} - - name: Run postinstall manually - run: yarn postinstall.manual + - name: Restore node_modules + shell: bash + # Unpacks the tree build-monorepo-root published, or installs from scratch if the cache is + # unavailable. One step, so there is no `if:` on a previous step's outcome to get wrong. + run: .github/scripts/restore-node-modules.sh - name: Build web run: yarn build:gauzy:prod:ci build-desktop: name: build-desktop + needs: build-monorepo-root # Matches the CircleCI branch filter: desktop was never built on ordinary PR branches. if: github.ref == 'refs/heads/develop' || github.ref == 'refs/heads/stage' || github.ref == 'refs/heads/master' runs-on: ${{ vars.RUNNER_LINUX_X64_8 || 'ubuntu-latest' }} @@ -154,7 +197,6 @@ jobs: - uses: actions/setup-node@v4 with: node-version: 24 - cache: yarn - name: Install system dependencies for Electron run: | @@ -162,11 +204,17 @@ jobs: sudo apt-get install -y --no-install-recommends \ build-essential icnsutils graphicsmagick binutils libappindicator3-1 || true - - name: Install dependencies - run: yarn install --network-timeout 1000000 --frozen-lockfile --ignore-scripts + - name: Restore node_modules archive + uses: actions/cache/restore@v4 + with: + path: ${{ env.NODE_MODULES_ARCHIVE }} + key: ${{ runner.os }}-${{ runner.arch }}-node-modules-${{ hashFiles('yarn.lock', 'patches/**', '.scripts/postinstall.js') }} - - name: Run postinstall manually - run: yarn postinstall.manual + - name: Restore node_modules + shell: bash + # Unpacks the tree build-monorepo-root published, or installs from scratch if the cache is + # unavailable. One step, so there is no `if:` on a previous step's outcome to get wrong. + run: .github/scripts/restore-node-modules.sh - name: Build desktop run: yarn build:desktop diff --git a/.github/workflows/test_playwright.yml b/.github/workflows/test_playwright.yml index 044f5fbd93..fd52dc4738 100644 --- a/.github/workflows/test_playwright.yml +++ b/.github/workflows/test_playwright.yml @@ -96,46 +96,8 @@ jobs: # was killed at the 120-minute job timeout 1 h 58 m in. All four shards died that way and # the 12 h 34 m run produced ZERO test results. # - # And this job no longer writes a cache AT ALL. After pruning duplicates the budget holds - # 9.33 GB of caches that other workflows genuinely need — `build.yml`'s 4.66 GB yarn cache - # (the PR gate, five jobs, every pull request) and the 4.64 GB macOS desktop-build cache. - # Storing even a ~3 GB archive would evict one of them, making the PR gate slower to speed up - # a suite that runs only on stage pushes. That is a bad trade, so deps pays the bootstrap and - # the artifact makes sure nothing downstream pays it again. - # - # zstd on one file also beats the cache action's per-file handling of a huge tree in both - # directions. `--warning=no-file-changed` because yarn's own daemon can touch files while we - # read them; `|| [ $? -eq 1 ]` accepts tar's "file changed as we read it" (exit 1) but still - # fails on a real error (exit 2). - run: | - set -o pipefail - PATHS=() - for p in node_modules apps/*/node_modules packages/*/node_modules \ - packages/plugins/*/node_modules tools/node_modules; do - [ -d "$p" ] && PATHS+=("$p") - done - if [ ${#PATHS[@]} -eq 0 ]; then - echo "::error::bootstrap produced no node_modules directories"; exit 1 - fi - echo "archiving ${#PATHS[@]} directories" - if command -v zstd >/dev/null 2>&1; then - COMPRESS="zstd -3 -T0" - else - echo "::warning::zstd unavailable — falling back to gzip (larger archive, slower)" - COMPRESS="gzip -3" - fi - tar --warning=no-file-changed --use-compress-program="$COMPRESS" \ - -cf "$NODE_MODULES_ARCHIVE" "${PATHS[@]}" || [ $? -eq 1 ] - ls -lh "$NODE_MODULES_ARCHIVE" - # Tolerating tar's exit 1 means a TRUNCATED archive could be published, and every - # downstream job would then fall back to its own multi-hour bootstrap. A cheap size floor - # catches the gross case; a full `tar -t` would re-read every byte for little extra. - size=$(stat -c%s "$NODE_MODULES_ARCHIVE") - echo "archive bytes: $size" - if [ "$size" -lt 104857600 ]; then - echo "::error::archive is only $size bytes — the dependency tree cannot be that small" - exit 1 - fi + # Shares its implementation with build.yml — see the script. + run: .github/scripts/archive-node-modules.sh - name: Upload node_modules archive uses: actions/upload-artifact@v7