From 3ca1e223c8ef3e6c87190fc1072a1dec48e243e4 Mon Sep 17 00:00:00 2001 From: Dhiraj Kumar Sah Date: Wed, 10 Dec 2025 11:05:57 +0000 Subject: [PATCH 1/3] HOTFIX: Testing the Finetune base CI failure by installing pytorch2.9.0 before run Signed-off-by: Dhiraj Kumar Sah --- scripts/Jenkinsfile | 161 +++++++++++++++++++++----------------------- 1 file changed, 78 insertions(+), 83 deletions(-) diff --git a/scripts/Jenkinsfile b/scripts/Jenkinsfile index 8f95c1d98..ecbe6f755 100644 --- a/scripts/Jenkinsfile +++ b/scripts/Jenkinsfile @@ -22,7 +22,6 @@ pipeline { . preflight_qeff/bin/activate && pip install --upgrade pip setuptools && pip install .[test] && - pip install .[diffusers] && pip install junitparser pytest-xdist && pip install librosa==0.10.2 soundfile==0.13.1 && #packages needed to load example for whisper testing pip install --extra-index-url https://download.pytorch.org/whl/cpu timm==1.0.14 torchvision==0.22.0+cpu einops==0.8.1 && #packages to load VLMs @@ -30,80 +29,80 @@ pipeline { ''' } } - stage('Non CLI Tests') { - parallel { - stage('Run Non-CLI Non-QAIC Tests') { - steps { - timeout(time: 40, unit: 'MINUTES') { - sh ''' - sudo docker exec ${BUILD_TAG} bash -c " - cd /efficient-transformers && - . preflight_qeff/bin/activate && - mkdir -p $PWD/Non_cli_qaic && - export TOKENIZERS_PARALLELISM=false && - export QEFF_HOME=$PWD/Non_cli_qaic && - pytest tests -m '(not cli) and (not on_qaic) and (not finetune)' --ignore tests/vllm -n 4 --junitxml=tests/tests_log1.xml && - junitparser merge tests/tests_log1.xml tests/tests_log.xml && - deactivate" - ''' - } - } - } - stage('Run Non-CLI QAIC Tests') { - steps { - timeout(time: 200, unit: 'MINUTES') { - sh ''' - sudo docker exec ${BUILD_TAG} bash -c " - cd /efficient-transformers && - . preflight_qeff/bin/activate && - mkdir -p $PWD/Non_qaic && - export TOKENIZERS_PARALLELISM=false && - export QEFF_HOME=$PWD/Non_qaic && - pytest tests -m '(not cli) and (on_qaic) and (not nightly) and (not multimodal) and (not qnn) and (not finetune)' --ignore tests/vllm --junitxml=tests/tests_log2.xml && - junitparser merge tests/tests_log2.xml tests/tests_log.xml && - deactivate" - ''' - } - } - } - } - } - stage('QAIC MultiModal Tests') { - steps { - timeout(time: 120, unit: 'MINUTES') { - sh ''' - sudo docker exec ${BUILD_TAG} bash -c " - cd /efficient-transformers && - . preflight_qeff/bin/activate && - mkdir -p $PWD/Non_cli_qaic_multimodal && - export TOKENIZERS_PARALLELISM=false && - export QEFF_HOME=$PWD/Non_cli_qaic_multimodal && - pytest tests -m '(not cli) and (on_qaic) and (multimodal) and (not qnn) and (not finetune)' --ignore tests/vllm --junitxml=tests/tests_log6.xml && - junitparser merge tests/tests_log6.xml tests/tests_log.xml && - deactivate" - ''' - } - } - } - stage('Inference Tests') { - steps { - timeout(time: 120, unit: 'MINUTES') { - sh ''' - sudo docker exec ${BUILD_TAG} bash -c " - #source /qnn_sdk/bin/envsetup.sh && - #source /qnn_sdk/bin/envcheck -c && - cd /efficient-transformers && - . preflight_qeff/bin/activate && - mkdir -p $PWD/cli && - export TOKENIZERS_PARALLELISM=false && - export QEFF_HOME=$PWD/cli && - pytest tests -m '(cli and not qnn) and (not finetune)' --ignore tests/vllm --junitxml=tests/tests_log3.xml && - junitparser merge tests/tests_log3.xml tests/tests_log.xml && - deactivate" - ''' - } - } - } + // stage('Non CLI Tests') { + // parallel { + // stage('Run Non-CLI Non-QAIC Tests') { + // steps { + // timeout(time: 25, unit: 'MINUTES') { + // sh ''' + // sudo docker exec ${BUILD_TAG} bash -c " + // cd /efficient-transformers && + // . preflight_qeff/bin/activate && + // mkdir -p $PWD/Non_cli_qaic && + // export TOKENIZERS_PARALLELISM=false && + // export QEFF_HOME=$PWD/Non_cli_qaic && + // pytest tests -m '(not cli) and (not on_qaic) and (not finetune)' --ignore tests/vllm --junitxml=tests/tests_log1.xml && + // junitparser merge tests/tests_log1.xml tests/tests_log.xml && + // deactivate" + // ''' + // } + // } + // } + // stage('Run Non-CLI QAIC Tests') { + // steps { + // timeout(time: 200, unit: 'MINUTES') { + // sh ''' + // sudo docker exec ${BUILD_TAG} bash -c " + // cd /efficient-transformers && + // . preflight_qeff/bin/activate && + // mkdir -p $PWD/Non_qaic && + // export TOKENIZERS_PARALLELISM=false && + // export QEFF_HOME=$PWD/Non_qaic && + // pytest tests -m '(not cli) and (on_qaic) and (not nightly) and (not multimodal) and (not qnn) and (not finetune)' --ignore tests/vllm --junitxml=tests/tests_log2.xml && + // junitparser merge tests/tests_log2.xml tests/tests_log.xml && + // deactivate" + // ''' + // } + // } + // } + // } + // } + // stage('QAIC MultiModal Tests') { + // steps { + // timeout(time: 60, unit: 'MINUTES') { + // sh ''' + // sudo docker exec ${BUILD_TAG} bash -c " + // cd /efficient-transformers && + // . preflight_qeff/bin/activate && + // mkdir -p $PWD/Non_cli_qaic_multimodal && + // export TOKENIZERS_PARALLELISM=false && + // export QEFF_HOME=$PWD/Non_cli_qaic_multimodal && + // pytest tests -m '(not cli) and (on_qaic) and (multimodal) and (not qnn) and (not finetune)' --ignore tests/vllm --junitxml=tests/tests_log6.xml && + // junitparser merge tests/tests_log6.xml tests/tests_log.xml && + // deactivate" + // ''' + // } + // } + // } + // stage('Inference Tests') { + // steps { + // timeout(time: 60, unit: 'MINUTES') { + // sh ''' + // sudo docker exec ${BUILD_TAG} bash -c " + // #source /qnn_sdk/bin/envsetup.sh && + // #source /qnn_sdk/bin/envcheck -c && + // cd /efficient-transformers && + // . preflight_qeff/bin/activate && + // mkdir -p $PWD/cli && + // export TOKENIZERS_PARALLELISM=false && + // export QEFF_HOME=$PWD/cli && + // pytest tests -m '(cli and not qnn) and (not finetune)' --ignore tests/vllm --junitxml=tests/tests_log3.xml && + // junitparser merge tests/tests_log3.xml tests/tests_log.xml && + // deactivate" + // ''' + // } + // } + // } // stage('QNN CLI Tests') { // steps { // timeout(time: 30, unit: 'MINUTES') { @@ -163,12 +162,13 @@ pipeline { // } stage('Finetune CLI Tests') { steps { - timeout(time: 20, unit: 'MINUTES') { + timeout(time: 5, unit: 'MINUTES') { sh ''' sudo docker exec ${BUILD_TAG} bash -c " cd /efficient-transformers && . preflight_qeff/bin/activate && pip install /opt/qti-aic/integrations/torch_qaic/py310/torch_qaic-0.1.0-cp310-cp310-linux_x86_64.whl && + pip install torch==2.9.0 torchvision==0.24.0 torchaudio==2.9.0 --index-url https://download.pytorch.org/whl/cpu && mkdir -p $PWD/cli_qaic_finetuning && export TOKENIZERS_PARALLELISM=false && export QEFF_HOME=$PWD/cli_qaic_finetuning && @@ -201,13 +201,8 @@ pipeline { echo "Failed to change ownership: ${error}" } } - script { - try { - junit testResults: 'tests/tests_log.xml', allowEmptyResults: true - } catch (error) { - echo "No test results file found or parsing failed: ${error}" - } - } + junit testResults: 'tests/tests_log.xml' + script { try { sh ''' From 844c9fd7ef9c7fce3baf2091d93893aa78d08234 Mon Sep 17 00:00:00 2001 From: Dhiraj Kumar Sah Date: Wed, 10 Dec 2025 11:31:32 +0000 Subject: [PATCH 2/3] Updated the Jenkinsfile for normal run. Can be merged now Signed-off-by: Dhiraj Kumar Sah --- scripts/Jenkinsfile | 160 +++++++++++++++++++++++--------------------- 1 file changed, 83 insertions(+), 77 deletions(-) diff --git a/scripts/Jenkinsfile b/scripts/Jenkinsfile index ecbe6f755..9a8dfb376 100644 --- a/scripts/Jenkinsfile +++ b/scripts/Jenkinsfile @@ -22,6 +22,7 @@ pipeline { . preflight_qeff/bin/activate && pip install --upgrade pip setuptools && pip install .[test] && + pip install .[diffusers] && pip install junitparser pytest-xdist && pip install librosa==0.10.2 soundfile==0.13.1 && #packages needed to load example for whisper testing pip install --extra-index-url https://download.pytorch.org/whl/cpu timm==1.0.14 torchvision==0.22.0+cpu einops==0.8.1 && #packages to load VLMs @@ -29,80 +30,80 @@ pipeline { ''' } } - // stage('Non CLI Tests') { - // parallel { - // stage('Run Non-CLI Non-QAIC Tests') { - // steps { - // timeout(time: 25, unit: 'MINUTES') { - // sh ''' - // sudo docker exec ${BUILD_TAG} bash -c " - // cd /efficient-transformers && - // . preflight_qeff/bin/activate && - // mkdir -p $PWD/Non_cli_qaic && - // export TOKENIZERS_PARALLELISM=false && - // export QEFF_HOME=$PWD/Non_cli_qaic && - // pytest tests -m '(not cli) and (not on_qaic) and (not finetune)' --ignore tests/vllm --junitxml=tests/tests_log1.xml && - // junitparser merge tests/tests_log1.xml tests/tests_log.xml && - // deactivate" - // ''' - // } - // } - // } - // stage('Run Non-CLI QAIC Tests') { - // steps { - // timeout(time: 200, unit: 'MINUTES') { - // sh ''' - // sudo docker exec ${BUILD_TAG} bash -c " - // cd /efficient-transformers && - // . preflight_qeff/bin/activate && - // mkdir -p $PWD/Non_qaic && - // export TOKENIZERS_PARALLELISM=false && - // export QEFF_HOME=$PWD/Non_qaic && - // pytest tests -m '(not cli) and (on_qaic) and (not nightly) and (not multimodal) and (not qnn) and (not finetune)' --ignore tests/vllm --junitxml=tests/tests_log2.xml && - // junitparser merge tests/tests_log2.xml tests/tests_log.xml && - // deactivate" - // ''' - // } - // } - // } - // } - // } - // stage('QAIC MultiModal Tests') { - // steps { - // timeout(time: 60, unit: 'MINUTES') { - // sh ''' - // sudo docker exec ${BUILD_TAG} bash -c " - // cd /efficient-transformers && - // . preflight_qeff/bin/activate && - // mkdir -p $PWD/Non_cli_qaic_multimodal && - // export TOKENIZERS_PARALLELISM=false && - // export QEFF_HOME=$PWD/Non_cli_qaic_multimodal && - // pytest tests -m '(not cli) and (on_qaic) and (multimodal) and (not qnn) and (not finetune)' --ignore tests/vllm --junitxml=tests/tests_log6.xml && - // junitparser merge tests/tests_log6.xml tests/tests_log.xml && - // deactivate" - // ''' - // } - // } - // } - // stage('Inference Tests') { - // steps { - // timeout(time: 60, unit: 'MINUTES') { - // sh ''' - // sudo docker exec ${BUILD_TAG} bash -c " - // #source /qnn_sdk/bin/envsetup.sh && - // #source /qnn_sdk/bin/envcheck -c && - // cd /efficient-transformers && - // . preflight_qeff/bin/activate && - // mkdir -p $PWD/cli && - // export TOKENIZERS_PARALLELISM=false && - // export QEFF_HOME=$PWD/cli && - // pytest tests -m '(cli and not qnn) and (not finetune)' --ignore tests/vllm --junitxml=tests/tests_log3.xml && - // junitparser merge tests/tests_log3.xml tests/tests_log.xml && - // deactivate" - // ''' - // } - // } - // } + stage('Non CLI Tests') { + parallel { + stage('Run Non-CLI Non-QAIC Tests') { + steps { + timeout(time: 40, unit: 'MINUTES') { + sh ''' + sudo docker exec ${BUILD_TAG} bash -c " + cd /efficient-transformers && + . preflight_qeff/bin/activate && + mkdir -p $PWD/Non_cli_qaic && + export TOKENIZERS_PARALLELISM=false && + export QEFF_HOME=$PWD/Non_cli_qaic && + pytest tests -m '(not cli) and (not on_qaic) and (not finetune)' --ignore tests/vllm --junitxml=tests/tests_log1.xml && + junitparser merge tests/tests_log1.xml tests/tests_log.xml && + deactivate" + ''' + } + } + } + stage('Run Non-CLI QAIC Tests') { + steps { + timeout(time: 200, unit: 'MINUTES') { + sh ''' + sudo docker exec ${BUILD_TAG} bash -c " + cd /efficient-transformers && + . preflight_qeff/bin/activate && + mkdir -p $PWD/Non_qaic && + export TOKENIZERS_PARALLELISM=false && + export QEFF_HOME=$PWD/Non_qaic && + pytest tests -m '(not cli) and (on_qaic) and (not nightly) and (not multimodal) and (not qnn) and (not finetune)' --ignore tests/vllm --junitxml=tests/tests_log2.xml && + junitparser merge tests/tests_log2.xml tests/tests_log.xml && + deactivate" + ''' + } + } + } + } + } + stage('QAIC MultiModal Tests') { + steps { + timeout(time: 120, unit: 'MINUTES') { + sh ''' + sudo docker exec ${BUILD_TAG} bash -c " + cd /efficient-transformers && + . preflight_qeff/bin/activate && + mkdir -p $PWD/Non_cli_qaic_multimodal && + export TOKENIZERS_PARALLELISM=false && + export QEFF_HOME=$PWD/Non_cli_qaic_multimodal && + pytest tests -m '(not cli) and (on_qaic) and (multimodal) and (not qnn) and (not finetune)' --ignore tests/vllm --junitxml=tests/tests_log6.xml && + junitparser merge tests/tests_log6.xml tests/tests_log.xml && + deactivate" + ''' + } + } + } + stage('Inference Tests') { + steps { + timeout(time: 120, unit: 'MINUTES') { + sh ''' + sudo docker exec ${BUILD_TAG} bash -c " + #source /qnn_sdk/bin/envsetup.sh && + #source /qnn_sdk/bin/envcheck -c && + cd /efficient-transformers && + . preflight_qeff/bin/activate && + mkdir -p $PWD/cli && + export TOKENIZERS_PARALLELISM=false && + export QEFF_HOME=$PWD/cli && + pytest tests -m '(cli and not qnn) and (not finetune)' --ignore tests/vllm --junitxml=tests/tests_log3.xml && + junitparser merge tests/tests_log3.xml tests/tests_log.xml && + deactivate" + ''' + } + } + } // stage('QNN CLI Tests') { // steps { // timeout(time: 30, unit: 'MINUTES') { @@ -162,7 +163,7 @@ pipeline { // } stage('Finetune CLI Tests') { steps { - timeout(time: 5, unit: 'MINUTES') { + timeout(time: 20, unit: 'MINUTES') { sh ''' sudo docker exec ${BUILD_TAG} bash -c " cd /efficient-transformers && @@ -201,8 +202,13 @@ pipeline { echo "Failed to change ownership: ${error}" } } - junit testResults: 'tests/tests_log.xml' - + script { + try { + junit testResults: 'tests/tests_log.xml', allowEmptyResults: true + } catch (error) { + echo "No test results file found or parsing failed: ${error}" + } + } script { try { sh ''' From aecac705b744b993567a92ad7bf217e0555a9a32 Mon Sep 17 00:00:00 2001 From: Dhiraj Kumar Sah Date: Thu, 18 Dec 2025 06:17:03 +0000 Subject: [PATCH 3/3] Minor patchup due to issue with rebase. Signed-off-by: Dhiraj Kumar Sah --- scripts/Jenkinsfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/Jenkinsfile b/scripts/Jenkinsfile index 9a8dfb376..e9925dee2 100644 --- a/scripts/Jenkinsfile +++ b/scripts/Jenkinsfile @@ -42,7 +42,7 @@ pipeline { mkdir -p $PWD/Non_cli_qaic && export TOKENIZERS_PARALLELISM=false && export QEFF_HOME=$PWD/Non_cli_qaic && - pytest tests -m '(not cli) and (not on_qaic) and (not finetune)' --ignore tests/vllm --junitxml=tests/tests_log1.xml && + pytest tests -m '(not cli) and (not on_qaic) and (not finetune)' --ignore tests/vllm -n 4 --junitxml=tests/tests_log1.xml && junitparser merge tests/tests_log1.xml tests/tests_log.xml && deactivate" '''