diff --git a/.github/workflows/demo_inference_test.yml b/.github/workflows/demo_inference_test.yml new file mode 100644 index 0000000000..32775c4363 --- /dev/null +++ b/.github/workflows/demo_inference_test.yml @@ -0,0 +1,180 @@ +name: Demo Inference Test - Qwen3 + +# Demo workflow to test the test-lyz-infer runner with real Qwen3 inference +# Uses FlagScale inference tests from tests/functional_tests/inference/qwen3 + +on: + pull_request: + branches: ["main"] + workflow_dispatch: + inputs: + test_case: + description: 'Test case to run' + required: false + default: '4b_tp2_ascend' + type: choice + options: + - 4b_tp2 + - 4b_tp2_ascend + +jobs: + demo_inference: + name: Qwen3 Inference Demo + runs-on: test-lyz-infer + env: + PROJECT_ROOT: ${{ github.workspace }} + + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Print test info + run: | + echo "==========================================" + echo "Qwen3 Inference Demo Test" + echo "==========================================" + echo "Runner: test-lyz-infer" + echo "Test Case: ${{ inputs.test_case || '4b_tp2_ascend' }}" + echo "Workflow: ${{ github.workflow }}" + echo "Run ID: ${{ github.run_id }}" + echo "Project Root: $PROJECT_ROOT" + echo "==========================================" + + - name: Check system info + run: | + echo "=== System Information ===" + echo "Hostname: $(hostname)" + echo "OS: $(uname -s)" + echo "Kernel: $(uname -r)" + echo "Architecture: $(uname -m)" + echo "" + echo "=== CPU Information ===" + lscpu | grep -E "Model name|CPU\(s\)|Thread|Core" || echo "lscpu not available" + echo "" + echo "=== Memory Information ===" + free -h || echo "free command not available" + echo "" + echo "=== Disk Space ===" + df -h / || echo "df command not available" + + - name: Check GPU availability + run: | + echo "=== GPU Information ===" + if command -v nvidia-smi &> /dev/null; then + echo "NVIDIA GPU detected:" + nvidia-smi + else + echo "nvidia-smi not found - no NVIDIA GPU or driver not installed" + fi + + if command -v npu-smi &> /dev/null; then + echo "Ascend NPU detected:" + npu-smi info + else + echo "npu-smi not found - no Ascend NPU" + fi + + - name: Check Python environment + run: | + echo "=== Python Environment ===" + if command -v python &> /dev/null; then + echo "Python version: $(python --version)" + echo "Python location: $(which python)" + else + echo "Python not found in PATH" + fi + + - name: Install FlagScale + run: | + echo "=== Installing FlagScale ===" + cd $PROJECT_ROOT + pip install . --no-build-isolation || { echo "❌ FlagScale install failed"; exit 1; } + + # Install vllm-plugin-FL + pip install vllm-plugin-fl==0.1.0+vllm0.13.0 \ + --extra-index-url https://resource.flagos.net/repository/flagos-pypi-hosted/simple \ + || { echo "❌ vllm-plugin-FL install failed"; exit 1; } + echo "✅ vllm-plugin-FL installed successfully" + + # Verify installation + command -v flagscale || { echo "❌ FlagScale CLI not found in PATH"; exit 1; } + echo "✅ FlagScale CLI installed: $(flagscale --version 2>/dev/null || echo 'version unknown')" + + - name: Run Qwen3 inference test + id: inference_test + run: | + set -euo pipefail + cd $PROJECT_ROOT + + TEST_CASE="${{ inputs.test_case || '4b_tp2_ascend' }}" + TEST_DIR="tests/functional_tests/inference/qwen3" + CONFIG_FILE="$TEST_DIR/conf/${TEST_CASE}.yaml" + RESULTS_DIR="$TEST_DIR/test_results/${TEST_CASE}" + GOLD_FILE="$TEST_DIR/results_gold/${TEST_CASE}" + + echo "=== Running Qwen3 Inference Test ===" + echo "Test case: $TEST_CASE" + echo "Config: $CONFIG_FILE" + echo "Results dir: $RESULTS_DIR" + echo "" + + # Check if config exists + if [ ! -f "$CONFIG_FILE" ]; then + echo "❌ Config file not found: $CONFIG_FILE" + exit 1 + fi + + # Create results directory + mkdir -p "$RESULTS_DIR" + + # Run inference using flagscale + echo "Starting inference..." + flagscale inference qwen3 --config "$TEST_DIR/conf/${TEST_CASE}.yaml" --test || { + echo "❌ Inference failed" + exit 1 + } + + echo "✅ Inference completed" + + # Check results + if [ -f "$RESULTS_DIR/output.txt" ]; then + echo "" + echo "=== Inference Output ===" + cat "$RESULTS_DIR/output.txt" + echo "" + + # Compare with gold results if available + if [ -f "$GOLD_FILE" ]; then + echo "=== Comparing with gold results ===" + if diff -u "$GOLD_FILE" "$RESULTS_DIR/output.txt"; then + echo "✅ Results match gold standard" + else + echo "⚠️ Results differ from gold standard (this may be expected)" + fi + fi + else + echo "⚠️ Output file not found at $RESULTS_DIR/output.txt" + fi + + echo "" + echo "==========================================" + echo "✅ Qwen3 inference test completed!" + echo "==========================================" + + - name: Upload test results + if: always() + uses: actions/upload-artifact@v4 + with: + name: qwen3-inference-results-${{ inputs.test_case || '4b_tp2_ascend' }}-${{ github.run_id }} + path: tests/functional_tests/inference/qwen3/test_results/${{ inputs.test_case || '4b_tp2_ascend' }} + retention-days: 7 + if-no-files-found: warn + + - name: Test summary + if: always() + run: | + echo "=== Test Summary ===" + echo "Status: ${{ job.status }}" + echo "Runner: test-lyz-infer" + echo "Test Case: ${{ inputs.test_case || '4b_tp2_ascend' }}" + echo "Timestamp: $(date -u '+%Y-%m-%d %H:%M:%S UTC')" diff --git a/.github/workflows/demo_train_test.yml b/.github/workflows/demo_train_test.yml new file mode 100644 index 0000000000..f72da983c7 --- /dev/null +++ b/.github/workflows/demo_train_test.yml @@ -0,0 +1,204 @@ +name: Demo Train Test - Qwen3 + +# Demo workflow to test the flagscale-metax-c550 runner with real Qwen3 training +# Uses FlagScale training tests from tests/functional_tests/train/qwen3 + +on: + pull_request: + branches: ["main"] + workflow_dispatch: + inputs: + test_case: + description: 'Test case to run' + required: false + default: '0_6b_metax' + type: choice + options: + - 0_6b_metax + +jobs: + demo_train: + name: Qwen3 Train Demo + runs-on: test-lyz-train-metax + env: + PROJECT_ROOT: ${{ github.workspace }} + + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Print test info + run: | + echo "==========================================" + echo "Qwen3 Train Demo Test" + echo "==========================================" + echo "Runner: ${{ runner.name }}" + echo "Test Case: ${{ inputs.test_case || '0_6b_metax' }}" + echo "Workflow: ${{ github.workflow }}" + echo "Run ID: ${{ github.run_id }}" + echo "Project Root: $PROJECT_ROOT" + echo "Current User: $(whoami)" + echo "=== Environment Variables ===" + env | sort + echo "==========================================" + + - name: Check system info + run: | + echo "=== System Information ===" + echo "Hostname: $(hostname)" + echo "OS: $(uname -s)" + echo "Kernel: $(uname -r)" + echo "Architecture: $(uname -m)" + echo "" + echo "=== CPU Information ===" + lscpu | grep -E "Model name|CPU\(s\)|Thread|Core" || echo "lscpu not available" + echo "" + echo "=== Memory Information ===" + free -h || echo "free command not available" + echo "" + echo "=== Disk Space ===" + df -h / || echo "df command not available" + + - name: Check GPU availability + run: | + echo "=== GPU Information ===" + if command -v mx-smi &> /dev/null; then + echo "MetaX GPU detected:" + mx-smi + else + echo "mx-smi not found - no MetaX GPU or driver not installed" + fi + + if command -v nvidia-smi &> /dev/null; then + echo "NVIDIA GPU detected:" + nvidia-smi + else + echo "nvidia-smi not found" + fi + + - name: Check Python environment + run: | + echo "=== Python Environment ===" + if command -v python &> /dev/null; then + echo "Python version: $(python --version)" + echo "Python location: $(which python)" + else + echo "Python not found in PATH" + fi + + - name: Prepare data + run: | + mkdir -p /opt/data && cd /opt/data + wget https://baai-flagscale.ks3-cn-beijing.ksyuncs.com/datasets/enron_emails_demo_text_document_qwen/enron_emails_demo_text_document_qwen.idx + wget https://baai-flagscale.ks3-cn-beijing.ksyuncs.com/datasets/enron_emails_demo_text_document_qwen/enron_emails_demo_text_document_qwen.bin + mkdir -p /opt/qwentokenizer && cd /opt/qwentokenizer + wget "https://baai-flagscale.ks3-cn-beijing.ksyuncs.com/tokenizers/qwentokenizer/tokenizer_config.json" -O tokenizer_config.json + wget "https://baai-flagscale.ks3-cn-beijing.ksyuncs.com/tokenizers/qwentokenizer/qwen.tiktoken" -O qwen.tiktoken + wget "https://baai-flagscale.ks3-cn-beijing.ksyuncs.com/tokenizers/qwentokenizer/qwen_generation_utils.py" -O qwen_generation_utils.py + wget "https://baai-flagscale.ks3-cn-beijing.ksyuncs.com/tokenizers/qwentokenizer/tokenization_qwen.py" -O tokenization_qwen.py + + - name: Install FlagScale + run: | + source /etc/profile.d/conda.sh + conda activate base + + echo "=== Installing FlagScale ===" + cd $PROJECT_ROOT + + # Install Megatron-LM-FL + git clone \ + "https://github.com/flagos-ai/Megatron-LM-FL.git" /tmp/Megatron-LM-FL + git -C /tmp/Megatron-LM-FL checkout d092f8df49f7c0b5b4cae42d036b7e4a26b8fc81 + + echo "Installing Megatron-LM-FL via pip..." + pip install /tmp/Megatron-LM-FL --no-build-isolation --root-user-action=ignore \ + || { echo "Megatron-LM-FL install failed"; exit 1; } + echo "✅ Megatron-LM-FL installed successfully" + + # Install TransformerEngine-FL and dependencies + git clone --depth 1 https://github.com/flagos-ai/TransformerEngine-FL.git /workspace/TransformerEngine-FL \ + || { echo "❌ TransformerEngine-FL clone failed"; exit 1; } + TE_FL_SKIP_CUDA=1 pip install /workspace/TransformerEngine-FL --no-build-isolation \ + || { echo "❌ TransformerEngine-FL install failed"; exit 1; } + echo "✅ TransformerEngine-FL installed successfully" + + # Install FlagScale + pip install . --no-build-isolation || { echo "❌ FlagScale install failed"; exit 1; } + + # Verify installation + command -v flagscale || { echo "❌ FlagScale CLI not found in PATH"; exit 1; } + echo "✅ FlagScale CLI installed: $(flagscale --version 2>/dev/null || echo 'version unknown')" + + - name: Run Qwen3 train test + id: train_test + run: | + source /etc/profile.d/conda.sh + conda activate base + + set -euo pipefail + cd $PROJECT_ROOT + + TEST_CASE="${{ inputs.test_case || '0_6b_metax' }}" + TEST_DIR="tests/functional_tests/train/qwen3" + CONFIG_FILE="$TEST_DIR/conf/${TEST_CASE}.yaml" + RESULTS_DIR="$TEST_DIR/test_results/${TEST_CASE}" + + echo "=== Running Qwen3 Train Test ===" + echo "Test case: $TEST_CASE" + echo "Config: $CONFIG_FILE" + echo "Results dir: $RESULTS_DIR" + echo "" + + # Check if config exists + if [ ! -f "$CONFIG_FILE" ]; then + echo "❌ Config file not found: $CONFIG_FILE" + exit 1 + fi + + echo "Starting training..." + flagscale train qwen3 --config "$CONFIG_FILE" --test || { + echo "❌ Training failed" + exit 1 + } + + echo "✅ Training completed" + + # Check results + if [ -d "$RESULTS_DIR" ]; then + echo "" + echo "=== Training Results ===" + ls -la "$RESULTS_DIR" + echo "" + + # Check for loss log if available + if [ -f "$RESULTS_DIR/hydra/logs/flagscale/flagscale.log" ]; then + echo "=== Training Log (last 50 lines) ===" + tail -50 "$RESULTS_DIR/hydra/logs/flagscale/flagscale.log" + echo "" + fi + else + echo "⚠️ Results directory not found at $RESULTS_DIR" + fi + + echo "" + echo "==========================================" + echo "✅ Qwen3 train test completed!" + echo "==========================================" + + - name: Upload test results + if: always() + uses: actions/upload-artifact@v4 + with: + name: qwen3-train-results-${{ inputs.test_case || '0_6b_metax' }}-${{ github.run_id }} + path: tests/functional_tests/train/qwen3/test_results/${{ inputs.test_case || '0_6b_metax' }} + retention-days: 7 + if-no-files-found: warn + + - name: Test summary + if: always() + run: | + echo "=== Test Summary ===" + echo "Status: ${{ job.status }}" + echo "Runner: ${{ runner.name }}" + echo "Test Case: ${{ inputs.test_case || '0_6b_metax' }}" + echo "Timestamp: $(date -u '+%Y-%m-%d %H:%M:%S UTC')" diff --git a/.github/workflows/push_image_harbor.yml b/.github/workflows/push_image_harbor.yml index 118cb4d033..e2b2043656 100644 --- a/.github/workflows/push_image_harbor.yml +++ b/.github/workflows/push_image_harbor.yml @@ -10,6 +10,7 @@ on: - 'tools/install/**' - 'requirements/**' - '.github/workflows/build_image_cuda.yml' + workflow_dispatch: permissions: contents: read @@ -25,6 +26,13 @@ jobs: prepare: name: Detect tar files for ${{ github.sha }} runs-on: [self-hosted, Linux, X64, nvidia-0, gpus-8] + timeout-minutes: 15 + container: + image: harbor.baai.ac.cn/flagscale/flagscale-train:dev-cu128-py3.12-20260228210721 + volumes: + - /data/image_tar:/data/image_tar + options: >- + --user root outputs: needs_promotion: ${{ steps.detect.outputs.needs_promotion }} train_tar: ${{ steps.detect.outputs.train_tar }} @@ -42,7 +50,14 @@ jobs: id: config run: | TAR_DIR=$(grep '^tar_dir:' .github/configs/cuda.yml | awk '{print $2}') - echo "tar_dir=${TAR_DIR}" >> $GITHUB_OUTPUT + echo "tar_dir=${TAR_DIR}" >> "$GITHUB_OUTPUT" + + - name: Install gh CLI + run: | + if ! command -v gh &>/dev/null; then + apt-get update && apt-get install -y gh + fi + gh --version - name: Resolve PR number from merge commit id: resolve_pr @@ -61,10 +76,10 @@ jobs: if [ -n "$PR_NUMBER" ]; then echo "Found PR #${PR_NUMBER} for merge commit ${MERGE_SHA}" - echo "suffix=pr${PR_NUMBER}" >> $GITHUB_OUTPUT + echo "suffix=pr${PR_NUMBER}" >> "$GITHUB_OUTPUT" else echo "::warning::Could not find PR for merge commit ${MERGE_SHA}, falling back to short SHA" - echo "suffix=${MERGE_SHA:0:7}" >> $GITHUB_OUTPUT + echo "suffix=${MERGE_SHA:0:7}" >> "$GITHUB_OUTPUT" fi - name: Find tar files by PR number @@ -78,29 +93,30 @@ jobs: INFERENCE_TAR=$(find "$TAR_DIR" -maxdepth 1 -name "*flagscale-inference*-${SUFFIX}.tar" 2>/dev/null | sort -r | head -1) ALL_TAR=$(find "$TAR_DIR" -maxdepth 1 -name "*flagscale-all*-${SUFFIX}.tar" 2>/dev/null | sort -r | head -1) - echo "train_tar=${TRAIN_TAR}" >> $GITHUB_OUTPUT - echo "inference_tar=${INFERENCE_TAR}" >> $GITHUB_OUTPUT - echo "all_tar=${ALL_TAR}" >> $GITHUB_OUTPUT + echo "train_tar=${TRAIN_TAR}" >> "$GITHUB_OUTPUT" + echo "inference_tar=${INFERENCE_TAR}" >> "$GITHUB_OUTPUT" + echo "all_tar=${ALL_TAR}" >> "$GITHUB_OUTPUT" if [ -n "$TRAIN_TAR" ] || [ -n "$INFERENCE_TAR" ] || [ -n "$ALL_TAR" ]; then - echo "needs_promotion=true" >> $GITHUB_OUTPUT + echo "needs_promotion=true" >> "$GITHUB_OUTPUT" echo "Detected tars for ${SUFFIX}:" if [ -n "$TRAIN_TAR" ]; then echo " train: $TRAIN_TAR"; fi if [ -n "$INFERENCE_TAR" ]; then echo " inference: $INFERENCE_TAR"; fi if [ -n "$ALL_TAR" ]; then echo " all: $ALL_TAR"; fi else - echo "needs_promotion=false" >> $GITHUB_OUTPUT + echo "needs_promotion=false" >> "$GITHUB_OUTPUT" echo "::warning::No tar files found for ${SUFFIX} in $TAR_DIR, skipping promotion" fi # --------------------------------------------------------------------------- - # Promote: load tar → retag → push to Harbor → delete tar + # Promote: load tar → retag → push to Harbor # --------------------------------------------------------------------------- promote: name: Push validated images to Harbor needs: prepare if: needs.prepare.outputs.needs_promotion == 'true' runs-on: [self-hosted, Linux, X64, nvidia-0, gpus-8] + timeout-minutes: 60 outputs: remote_train_tag: ${{ steps.promote_train.outputs.remote_tag }} remote_inference_tag: ${{ steps.promote_inference.outputs.remote_tag }} @@ -121,16 +137,19 @@ jobs: TAR_PATH="${{ needs.prepare.outputs.train_tar }}" echo "Loading tar: $TAR_PATH" - LOCAL_TAG=$(sudo docker load -i "$TAR_PATH" | grep 'Loaded image:' | awk '{print $NF}') + LOCAL_TAG=$(sudo docker load -i "$TAR_PATH" | grep -oP '(?<=Loaded image: ).+' | tail -1) + if [ -z "$LOCAL_TAG" ]; then + echo "::error::Failed to parse image tag from docker load output" + exit 1 + fi echo "Loaded image tag: $LOCAL_TAG" - # Strip local registry prefix (e.g. localhost:5000/) to get image:tag IMAGE_AND_TAG="${LOCAL_TAG#*/}" REMOTE_TAG="${{ env.REMOTE_REGISTRY }}/${{ env.REMOTE_IMAGE_PREFIX }}/${IMAGE_AND_TAG}" sudo docker tag "${LOCAL_TAG}" "${REMOTE_TAG}" sudo docker push "${REMOTE_TAG}" - echo "remote_tag=${REMOTE_TAG}" >> $GITHUB_OUTPUT + echo "remote_tag=${REMOTE_TAG}" >> "$GITHUB_OUTPUT" echo "Pushed: ${REMOTE_TAG}" - name: Promote inference image to Harbor @@ -141,7 +160,11 @@ jobs: TAR_PATH="${{ needs.prepare.outputs.inference_tar }}" echo "Loading tar: $TAR_PATH" - LOCAL_TAG=$(sudo docker load -i "$TAR_PATH" | grep 'Loaded image:' | awk '{print $NF}') + LOCAL_TAG=$(sudo docker load -i "$TAR_PATH" | grep -oP '(?<=Loaded image: ).+' | tail -1) + if [ -z "$LOCAL_TAG" ]; then + echo "::error::Failed to parse image tag from docker load output" + exit 1 + fi echo "Loaded image tag: $LOCAL_TAG" IMAGE_AND_TAG="${LOCAL_TAG#*/}" @@ -149,7 +172,7 @@ jobs: sudo docker tag "${LOCAL_TAG}" "${REMOTE_TAG}" sudo docker push "${REMOTE_TAG}" - echo "remote_tag=${REMOTE_TAG}" >> $GITHUB_OUTPUT + echo "remote_tag=${REMOTE_TAG}" >> "$GITHUB_OUTPUT" echo "Pushed: ${REMOTE_TAG}" - name: Promote all image to Harbor @@ -160,7 +183,11 @@ jobs: TAR_PATH="${{ needs.prepare.outputs.all_tar }}" echo "Loading tar: $TAR_PATH" - LOCAL_TAG=$(sudo docker load -i "$TAR_PATH" | grep 'Loaded image:' | awk '{print $NF}') + LOCAL_TAG=$(sudo docker load -i "$TAR_PATH" | grep -oP '(?<=Loaded image: ).+' | tail -1) + if [ -z "$LOCAL_TAG" ]; then + echo "::error::Failed to parse image tag from docker load output" + exit 1 + fi echo "Loaded image tag: $LOCAL_TAG" IMAGE_AND_TAG="${LOCAL_TAG#*/}" @@ -168,7 +195,7 @@ jobs: sudo docker tag "${LOCAL_TAG}" "${REMOTE_TAG}" sudo docker push "${REMOTE_TAG}" - echo "remote_tag=${REMOTE_TAG}" >> $GITHUB_OUTPUT + echo "remote_tag=${REMOTE_TAG}" >> "$GITHUB_OUTPUT" echo "Pushed: ${REMOTE_TAG}" # --------------------------------------------------------------------------- @@ -178,6 +205,7 @@ jobs: name: Update cuda.yml with Harbor image tags needs: promote runs-on: ubuntu-latest + timeout-minutes: 10 permissions: contents: write steps: @@ -185,6 +213,7 @@ jobs: uses: actions/checkout@v4 with: token: ${{ secrets.GITHUB_TOKEN }} + ref: main - name: Update image tags in cuda.yml run: | @@ -229,7 +258,8 @@ jobs: name: Clean up build cache needs: ['prepare', 'promote'] runs-on: [self-hosted, Linux, X64, nvidia-0, gpus-8] - if: always() + timeout-minutes: 10 + if: always() && needs.prepare.result == 'success' steps: - name: Remove stale tar files for this PR run: | @@ -241,20 +271,20 @@ jobs: fi - name: Remove dangling images - run: docker image prune -f 2>/dev/null || true + run: sudo docker image prune -f 2>/dev/null || true - name: Remove build cache older than 7 days - run: docker builder prune -f --filter "until=168h" 2>/dev/null || true + run: sudo docker builder prune -f --filter "until=168h" 2>/dev/null || true - name: Remove old localhost registry images run: | - docker images --format '{{.Repository}}:{{.Tag}} {{.CreatedSince}}' \ + sudo docker images --format '{{.Repository}}:{{.Tag}} {{.CreatedSince}}' \ | grep 'localhost:5000' \ | grep -E '(weeks|months)' \ | awk '{print $1}' \ - | xargs -r docker rmi 2>/dev/null || true + | xargs -r sudo docker rmi 2>/dev/null || true - name: Report disk usage run: | echo "Docker disk usage:" - docker system df + sudo docker system df diff --git a/.github/workflows/test_demo_musa.yml b/.github/workflows/test_demo_musa.yml new file mode 100644 index 0000000000..ec4830aea5 --- /dev/null +++ b/.github/workflows/test_demo_musa.yml @@ -0,0 +1,188 @@ +name: Demo Train Test Musa - Qwen3 + +# Demo workflow to test the flagscale-metax-c550 runner with real Qwen3 training +# Uses FlagScale training tests from tests/functional_tests/train/qwen3 + +on: + pull_request: + branches: ["main"] + workflow_dispatch: + inputs: + test_case: + description: 'Test case to run' + required: false + default: '0_6b_metax' + type: choice + options: + - 0_6b_metax + +jobs: + demo_train: + name: Qwen3 Train Demo + runs-on: test-lyz-musa + env: + PROJECT_ROOT: ${{ github.workspace }} + + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Print test info + run: | + echo "==========================================" + echo "Qwen3 Train Demo Test" + echo "==========================================" + echo "Runner: ${{ runner.name }}" + echo "Test Case: ${{ inputs.test_case || '0_6b_metax' }}" + echo "Workflow: ${{ github.workflow }}" + echo "Run ID: ${{ github.run_id }}" + echo "Project Root: $PROJECT_ROOT" + echo "Current User: $(whoami)" + echo "=== Environment Variables ===" + env | sort + echo "==========================================" + + - name: Check system info + run: | + echo "=== System Information ===" + echo "Hostname: $(hostname)" + echo "OS: $(uname -s)" + echo "Kernel: $(uname -r)" + echo "Architecture: $(uname -m)" + echo "" + echo "=== CPU Information ===" + lscpu | grep -E "Model name|CPU\(s\)|Thread|Core" || echo "lscpu not available" + echo "" + echo "=== Memory Information ===" + free -h || echo "free command not available" + echo "" + echo "=== Disk Space ===" + df -h / || echo "df command not available" + + - name: Check GPU availability + run: | + echo "=== GPU Information ===" + mthreads-gmi + + - name: Check Python environment + run: | + echo "=== Python Environment ===" + if command -v python &> /dev/null; then + echo "Python version: $(python --version)" + echo "Python location: $(which python)" + else + echo "Python not found in PATH" + fi + + - name: Prepare data + run: | + mkdir -p /opt/data && cd /opt/data + wget https://baai-flagscale.ks3-cn-beijing.ksyuncs.com/datasets/enron_emails_demo_text_document_qwen/enron_emails_demo_text_document_qwen.idx + wget https://baai-flagscale.ks3-cn-beijing.ksyuncs.com/datasets/enron_emails_demo_text_document_qwen/enron_emails_demo_text_document_qwen.bin + mkdir -p /opt/qwentokenizer && cd /opt/qwentokenizer + wget "https://baai-flagscale.ks3-cn-beijing.ksyuncs.com/tokenizers/qwentokenizer/tokenizer_config.json" -O tokenizer_config.json + wget "https://baai-flagscale.ks3-cn-beijing.ksyuncs.com/tokenizers/qwentokenizer/qwen.tiktoken" -O qwen.tiktoken + wget "https://baai-flagscale.ks3-cn-beijing.ksyuncs.com/tokenizers/qwentokenizer/qwen_generation_utils.py" -O qwen_generation_utils.py + wget "https://baai-flagscale.ks3-cn-beijing.ksyuncs.com/tokenizers/qwentokenizer/tokenization_qwen.py" -O tokenization_qwen.py + + - name: Install FlagScale + run: | + + echo "=== Installing FlagScale ===" + cd $PROJECT_ROOT + + # Install Megatron-LM-FL + git clone \ + "https://github.com/flagos-ai/Megatron-LM-FL.git" /tmp/Megatron-LM-FL + git -C /tmp/Megatron-LM-FL checkout d092f8df49f7c0b5b4cae42d036b7e4a26b8fc81 + + echo "Installing Megatron-LM-FL via pip..." + pip install /tmp/Megatron-LM-FL --no-build-isolation --root-user-action=ignore \ + || { echo "Megatron-LM-FL install failed"; exit 1; } + echo "✅ Megatron-LM-FL installed successfully" + + # Install TransformerEngine-FL and dependencies + git clone --depth 1 https://github.com/flagos-ai/TransformerEngine-FL.git /workspace/TransformerEngine-FL \ + || { echo "❌ TransformerEngine-FL clone failed"; exit 1; } + TE_FL_SKIP_CUDA=1 pip install /workspace/TransformerEngine-FL --no-build-isolation \ + || { echo "❌ TransformerEngine-FL install failed"; exit 1; } + echo "✅ TransformerEngine-FL installed successfully" + + # Install FlagScale + pip install . --no-build-isolation || { echo "❌ FlagScale install failed"; exit 1; } + + # Verify installation + command -v flagscale || { echo "❌ FlagScale CLI not found in PATH"; exit 1; } + echo "✅ FlagScale CLI installed: $(flagscale --version 2>/dev/null || echo 'version unknown')" + + - name: Run Qwen3 train test + id: train_test + run: | + + set -euo pipefail + cd $PROJECT_ROOT + + TEST_CASE="${{ inputs.test_case || '0_6b_metax' }}" + TEST_DIR="tests/functional_tests/train/qwen3" + CONFIG_FILE="$TEST_DIR/conf/${TEST_CASE}.yaml" + RESULTS_DIR="$TEST_DIR/test_results/${TEST_CASE}" + + echo "=== Running Qwen3 Train Test ===" + echo "Test case: $TEST_CASE" + echo "Config: $CONFIG_FILE" + echo "Results dir: $RESULTS_DIR" + echo "" + + # Check if config exists + if [ ! -f "$CONFIG_FILE" ]; then + echo "❌ Config file not found: $CONFIG_FILE" + exit 1 + fi + + echo "Starting training..." + flagscale train qwen3 --config "$CONFIG_FILE" --test || { + echo "❌ Training failed" + exit 1 + } + + echo "✅ Training completed" + + # Check results + if [ -d "$RESULTS_DIR" ]; then + echo "" + echo "=== Training Results ===" + ls -la "$RESULTS_DIR" + echo "" + + # Check for loss log if available + if [ -f "$RESULTS_DIR/hydra/logs/flagscale/flagscale.log" ]; then + echo "=== Training Log (last 50 lines) ===" + tail -50 "$RESULTS_DIR/hydra/logs/flagscale/flagscale.log" + echo "" + fi + else + echo "⚠️ Results directory not found at $RESULTS_DIR" + fi + + echo "" + echo "==========================================" + echo "✅ Qwen3 train test completed!" + echo "==========================================" + + - name: Upload test results + if: always() + uses: actions/upload-artifact@v4 + with: + name: qwen3-train-results-${{ inputs.test_case || '0_6b_metax' }}-${{ github.run_id }} + path: tests/functional_tests/train/qwen3/test_results/${{ inputs.test_case || '0_6b_metax' }} + retention-days: 7 + if-no-files-found: warn + + - name: Test summary + if: always() + run: | + echo "=== Test Summary ===" + echo "Status: ${{ job.status }}" + echo "Runner: ${{ runner.name }}" + echo "Test Case: ${{ inputs.test_case || '0_6b_metax' }}" + echo "Timestamp: $(date -u '+%Y-%m-%d %H:%M:%S UTC')" diff --git a/tests/functional_tests/inference/qwen3/conf/inference/4b_tp2_ascend.yaml b/tests/functional_tests/inference/qwen3/conf/inference/4b_tp2_ascend.yaml index 366e720810..31b6d36fce 100644 --- a/tests/functional_tests/inference/qwen3/conf/inference/4b_tp2_ascend.yaml +++ b/tests/functional_tests/inference/qwen3/conf/inference/4b_tp2_ascend.yaml @@ -1,6 +1,6 @@ llm: - model: /home/gitlab-runner/data/Qwen3-4B - tokenizer: /home/gitlab-runner/data/Qwen3-4B + model: /flagcicd/model/0b0ec7d3-1439-4e5d-9847-f5026942e397/latest + tokenizer: /flagcicd/model/0b0ec7d3-1439-4e5d-9847-f5026942e397/latest trust_remote_code: true tensor_parallel_size: 2 pipeline_parallel_size: 1 diff --git a/tests/functional_tests/train/qwen3/conf/train/data.yaml b/tests/functional_tests/train/qwen3/conf/train/data.yaml index 8d9d4bd4ca..bc9db18d8d 100644 --- a/tests/functional_tests/train/qwen3/conf/train/data.yaml +++ b/tests/functional_tests/train/qwen3/conf/train/data.yaml @@ -1,9 +1,9 @@ data: - data_path: /home/gitlab-runner/data/pile_wikipedia_demo/pile_wikipedia_demo + data_path: /opt/data/pile_wikipedia_demo split: 1 no_mmap_bin_files: true tokenizer: tokenizer_type: QwenTokenizerFS - tokenizer_path: /home/gitlab-runner/tokenizers/qwentokenizer + tokenizer_path: /opt/qwentokenizer vocab_size: 151936 make_vocab_size_divisible_by: 64