mirror of
https://github.com/NVIDIA/Model-Optimizer.git
synced 2026-11-08 09:03:20 +08:00
Keval Morabia
ae69d5d9b9
## What does this PR do? [Short term]: Megatron based tests take a long time often resulting in CICD timeout. Splitting megatron tests into a dedicated CICD job for faster overall CI/CD run [Mid/Long term]: Run all megatron gpu tests using `torchrun` instead of `pytest` so all dist processes are already created and all individual tests no longer need to setup and destroy their processes which adds a lot of overhead per test ## Testing <!-- Mention how have you tested your change if applicable. --> - [x] 1-GPU CI/CD passing (on this PR) - [x] 2-GPU CI/CD passing (on nightly run - manually triggered): https://github.com/NVIDIA/Model-Optimizer/actions/runs/22000517688 --------- Signed-off-by: Keval Morabia <28916987+kevalmorabia97@users.noreply.github.com>
109 lines
4.0 KiB
YAML
109 lines
4.0 KiB
YAML
# TODO: Optimize gpu tests runtime!
|
|
name: GPU tests
|
|
|
|
on:
|
|
push:
|
|
branches: ["pull-request/[0-9]+"]
|
|
# NOTE: paths cannot be used since push happens to copied PR and only latest commit to PR is used
|
|
schedule:
|
|
- cron: "0 0 * * *" # Nightly
|
|
workflow_dispatch: # On-demand
|
|
|
|
# Cancel previous runs if new commit is pushed to the same PR
|
|
concurrency:
|
|
group: ${{ github.workflow }}-${{ startsWith(github.ref, 'refs/heads/pull-request/') && github.ref || github.sha }}
|
|
cancel-in-progress: true
|
|
|
|
jobs:
|
|
check-file-changes:
|
|
if: startsWith(github.ref, 'refs/heads/pull-request/')
|
|
runs-on: ubuntu-latest
|
|
outputs:
|
|
any_changed: ${{ steps.changed-tests.outputs.any_changed }}
|
|
steps:
|
|
- uses: actions/checkout@v6
|
|
with:
|
|
fetch-depth: 0
|
|
- id: get-pr-info
|
|
uses: nv-gha-runners/get-pr-info@main
|
|
# Get commit from main branch that is present in the PR to use as base for changed files
|
|
- id: calculate-merge-base
|
|
env:
|
|
PR_SHA: ${{ fromJSON(steps.get-pr-info.outputs.pr-info).head.sha }}
|
|
BASE_SHA: ${{ fromJSON(steps.get-pr-info.outputs.pr-info).base.sha }}
|
|
run: |
|
|
(echo -n "merge-base="; git merge-base "$BASE_SHA" "$PR_SHA") | tee --append "${GITHUB_OUTPUT}"
|
|
- name: Check for changes in test-relevant directories
|
|
id: changed-tests
|
|
uses: step-security/changed-files@v46.0.5
|
|
with:
|
|
base_sha: ${{ steps.calculate-merge-base.outputs.merge-base }}
|
|
sha: ${{ fromJSON(steps.get-pr-info.outputs.pr-info).head.sha }}
|
|
files: |
|
|
.github/workflows/gpu_tests.yml
|
|
modelopt/**
|
|
tests/gpu/**
|
|
tox.ini
|
|
setup.py
|
|
fail_on_initial_diff_error: true
|
|
wait-checks:
|
|
needs: [check-file-changes]
|
|
if: needs.check-file-changes.outputs.any_changed == 'true'
|
|
uses: ./.github/workflows/_wait_for_checks.yml
|
|
permissions:
|
|
checks: read
|
|
secrets: inherit
|
|
with:
|
|
match_pattern: "^DCO$|^linux$" # Wait for DCO and Unit tests / linux to pass
|
|
delay: 300s
|
|
gpu-tests-pr:
|
|
needs: [check-file-changes, wait-checks]
|
|
if: needs.check-file-changes.outputs.any_changed == 'true'
|
|
strategy:
|
|
fail-fast: false
|
|
matrix:
|
|
include:
|
|
- example: py312-cuda12-gpu
|
|
timeout: 90
|
|
- example: py312-cuda12-gpu-megatron
|
|
timeout: 120
|
|
runs-on: linux-amd64-gpu-l4-latest-1
|
|
timeout-minutes: ${{ matrix.timeout }}
|
|
container: &gpu_container
|
|
image: nvcr.io/nvidia/pytorch:25.06-py3
|
|
env:
|
|
GIT_DEPTH: 1000 # For correct version for tests/gpu/torch/quantization/plugins/test_megatron.py
|
|
PIP_CONSTRAINT: "" # Disable pip constraint for upgrading packages
|
|
HF_TOKEN: ${{ secrets.HF_TOKEN }}
|
|
steps: &gpu_steps
|
|
- uses: actions/checkout@v6
|
|
- uses: nv-gha-runners/setup-proxy-cache@main
|
|
- name: Setup environment variables
|
|
run: |
|
|
echo "LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:/usr/include:/usr/lib/x86_64-linux-gnu" >> $GITHUB_ENV
|
|
- name: Run gpu tests
|
|
run: pip install tox-current-env && tox -e ${{ matrix.example }} --current-env
|
|
gpu-tests-non-pr:
|
|
if: ${{ !startsWith(github.ref, 'refs/heads/pull-request/') }}
|
|
strategy:
|
|
fail-fast: false
|
|
matrix:
|
|
include:
|
|
- example: py312-cuda12-gpu
|
|
timeout: 90
|
|
- example: py312-cuda12-gpu-megatron
|
|
timeout: 120
|
|
runs-on: linux-amd64-gpu-h100-latest-2
|
|
timeout-minutes: ${{ matrix.timeout }}
|
|
container: *gpu_container
|
|
steps: *gpu_steps
|
|
gpu-pr-required-check:
|
|
# Run even if gpu-tests-pr is skipped
|
|
if: ${{ startsWith(github.ref, 'refs/heads/pull-request/') && always() }}
|
|
needs: [check-file-changes, gpu-tests-pr]
|
|
runs-on: ubuntu-latest
|
|
steps:
|
|
- name: Required GPU tests did not succeed
|
|
if: ${{ needs.check-file-changes.result != 'success' || (needs.check-file-changes.outputs.any_changed == 'true' && needs.gpu-tests-pr.result != 'success') }}
|
|
run: exit 1
|