diff --git a/.github/workflows/openvmm-ci.yaml b/.github/workflows/openvmm-ci.yaml index 74b9d1b04c..87ef6bbda1 100644 --- a/.github/workflows/openvmm-ci.yaml +++ b/.github/workflows/openvmm-ci.yaml @@ -342,11 +342,11 @@ jobs: run: flowey e 1 flowey_lib_common::cache 3 shell: bash job10: - name: clippy [x64-windows], unit tests [x64-windows] + name: build openhcl [x64-linux] runs-on: - self-hosted - - 1ES.Pool=OpenVMM-GitHub-Win-Pool-WestUS3 - - 1ES.ImageOverride=HvLite-CI-Win-Ge-Image-256GB + - 1ES.Pool=OpenVMM-GitHub-Linux-Pool-WestUS3 + - 1ES.ImageOverride=MMSUbuntu22.04-256GB permissions: contents: read id-token: write @@ -385,11 +385,11 @@ jobs: - name: Build flowey run: | set -x - CARGO_INCREMENTAL=0 cargo build -p flowey_hvlite --target x86_64-pc-windows-msvc --profile flowey-ci + CARGO_INCREMENTAL=0 cargo build -p flowey_hvlite --target x86_64-unknown-linux-gnu --profile flowey-ci OutDirNormal=$(echo "${{ runner.temp }}/bootstrapped-flowey" | sed -e 's|\\|\/|g' -e 's|^\([A-Za-z]\)\:/\(.*\)|/\L\1\E/\2|') mkdir -p "$OutDirNormal" mv ./.github/workflows/openvmm-ci.yaml "$OutDirNormal/pipeline.yaml" - mv target/x86_64-pc-windows-msvc/flowey-ci/flowey_hvlite.exe "$OutDirNormal/flowey.exe" + mv target/x86_64-unknown-linux-gnu/flowey-ci/flowey_hvlite "$OutDirNormal/flowey" working-directory: flowey_bootstrap shell: bash - run: echo "${{ runner.temp }}/bootstrapped-flowey" >> $GITHUB_PATH @@ -402,7 +402,7 @@ jobs: ${{ runner.temp }} EOF ) - flowey.exe pipeline github --runtime $ESCAPED_AGENT_TEMPDIR\\bootstrapped-flowey\\pipeline.yaml --out .github/workflows/openvmm-ci.yaml ci checkin-gates --config=ci + flowey pipeline github --runtime $ESCAPED_AGENT_TEMPDIR/bootstrapped-flowey/pipeline.yaml --out .github/workflows/openvmm-ci.yaml ci checkin-gates --config=ci shell: bash - name: πŸŒΌπŸ›« Initialize job run: | @@ -410,179 +410,392 @@ jobs: AgentTempDirNormal=$(echo "$AgentTempDirNormal" | sed -e 's|\\|\/|g' -e 's|^\([A-Za-z]\)\:/\(.*\)|/\L\1\E/\2|') echo "AgentTempDirNormal=$AgentTempDirNormal" >> $GITHUB_ENV - chmod +x $AgentTempDirNormal/bootstrapped-flowey/flowey.exe + chmod +x $AgentTempDirNormal/bootstrapped-flowey/flowey - echo '"debug"' | flowey.exe v 10 'FLOWEY_LOG' update - echo "${{ runner.temp }}/work" | flowey.exe v 10 '_internal_WORKING_DIR' --is-raw-string update + echo '"debug"' | flowey v 10 'FLOWEY_LOG' update + echo "${{ runner.temp }}/work" | flowey v 10 '_internal_WORKING_DIR' --is-raw-string update - cat <<'EOF' | flowey.exe v 10 'verbose' update + cat <<'EOF' | flowey v 10 'verbose' update ${{ inputs.verbose != '' && inputs.verbose || 'false' }} EOF + mkdir -p "$AgentTempDirNormal/publish_artifacts/x64-linux-musl-pipette" + echo "$AgentTempDirNormal/publish_artifacts/x64-linux-musl-pipette" | flowey v 10 'artifact_publish_from_x64-linux-musl-pipette' --is-raw-string update + mkdir -p "$AgentTempDirNormal/publish_artifacts/x64-linux-musl-tmk_vmm" + echo "$AgentTempDirNormal/publish_artifacts/x64-linux-musl-tmk_vmm" | flowey v 10 'artifact_publish_from_x64-linux-musl-tmk_vmm' --is-raw-string update + mkdir -p "$AgentTempDirNormal/publish_artifacts/x64-openhcl-baseline" + echo "$AgentTempDirNormal/publish_artifacts/x64-openhcl-baseline" | flowey v 10 'artifact_publish_from_x64-openhcl-baseline' --is-raw-string update + mkdir -p "$AgentTempDirNormal/publish_artifacts/x64-openhcl-igvm" + echo "$AgentTempDirNormal/publish_artifacts/x64-openhcl-igvm" | flowey v 10 'artifact_publish_from_x64-openhcl-igvm' --is-raw-string update + mkdir -p "$AgentTempDirNormal/publish_artifacts/x64-openhcl-igvm-extras" + echo "$AgentTempDirNormal/publish_artifacts/x64-openhcl-igvm-extras" | flowey v 10 'artifact_publish_from_x64-openhcl-igvm-extras' --is-raw-string update + shell: bash + - name: checking if packages need to be installed + run: flowey e 10 flowey_lib_common::install_dist_pkg 0 + shell: bash + - name: installing packages + run: flowey e 10 flowey_lib_common::install_dist_pkg 1 + shell: bash + - name: create gh-release-download cache dir + run: flowey e 10 flowey_lib_common::download_gh_release 0 + shell: bash + - name: Pre-processing cache vars + run: |- + flowey e 10 flowey_lib_common::cache 0 + flowey v 10 'flowey_lib_common::cache:2:flowey_lib_common/src/cache.rs:407:72' --is-raw-string write-to-env github floweyvar1 + flowey v 10 'flowey_lib_common::cache:1:flowey_lib_common/src/cache.rs:406:72' --is-raw-string write-to-env github floweyvar2 + shell: bash + - id: flowey_lib_common__cache__1 + uses: actions/cache@v4 + with: + key: ${{ env.floweyvar1 }} + path: ${{ env.floweyvar2 }} + name: 'Restore cache: gh-release-download' + - name: download artifacts from github releases + run: |- + flowey v 10 'flowey_lib_common::cache:4:flowey_lib_common/src/cache.rs:462:70' --is-raw-string update --env-source steps.flowey_lib_common__cache__1.outputs.cache-hit <> $GITHUB_PATH @@ -918,7 +1211,7 @@ jobs: ${{ runner.temp }} EOF ) - flowey pipeline github --runtime $ESCAPED_AGENT_TEMPDIR/bootstrapped-flowey/pipeline.yaml --out .github/workflows/openvmm-ci.yaml ci checkin-gates --config=ci + flowey.exe pipeline github --runtime $ESCAPED_AGENT_TEMPDIR\\bootstrapped-flowey\\pipeline.yaml --out .github/workflows/openvmm-ci.yaml ci checkin-gates --config=ci shell: bash - name: πŸŒΌπŸ›« Initialize job run: | @@ -926,31 +1219,31 @@ jobs: AgentTempDirNormal=$(echo "$AgentTempDirNormal" | sed -e 's|\\|\/|g' -e 's|^\([A-Za-z]\)\:/\(.*\)|/\L\1\E/\2|') echo "AgentTempDirNormal=$AgentTempDirNormal" >> $GITHUB_ENV - chmod +x $AgentTempDirNormal/bootstrapped-flowey/flowey + chmod +x $AgentTempDirNormal/bootstrapped-flowey/flowey.exe - echo '"debug"' | flowey v 12 'FLOWEY_LOG' update - echo "${{ runner.temp }}/work" | flowey v 12 '_internal_WORKING_DIR' --is-raw-string update + echo '"debug"' | flowey.exe v 12 'FLOWEY_LOG' update + echo "${{ runner.temp }}/work" | flowey.exe v 12 '_internal_WORKING_DIR' --is-raw-string update - cat <<'EOF' | flowey v 12 'verbose' update + cat <<'EOF' | flowey.exe v 12 'verbose' update ${{ inputs.verbose != '' && inputs.verbose || 'false' }} EOF shell: bash - name: add default cargo home to path - run: flowey e 12 flowey_lib_common::install_rust 0 + run: flowey.exe e 12 flowey_lib_common::install_rust 0 shell: bash - name: install Rust - run: flowey e 12 flowey_lib_common::install_rust 1 + run: flowey.exe e 12 flowey_lib_common::install_rust 1 shell: bash - name: detect active toolchain run: |- - flowey e 12 flowey_lib_common::install_rust 2 - flowey e 12 flowey_lib_common::cfg_cargo_common_flags 0 + flowey.exe e 12 flowey_lib_common::install_rust 2 + flowey.exe e 12 flowey_lib_common::cfg_cargo_common_flags 0 shell: bash - name: check if openvmm needs to be cloned run: |- - flowey e 12 flowey_lib_common::git_checkout 0 - flowey v 12 'flowey_lib_common::git_checkout:0:flowey_lib_common/src/git_checkout.rs:469:80' --is-raw-string --condvar flowey_lib_common::git_checkout:1:flowey_lib_common/src/git_checkout.rs:470:46 write-to-env github floweyvar6 - flowey v 12 'flowey_lib_common::git_checkout:1:flowey_lib_common/src/git_checkout.rs:470:46' write-to-env github FLOWEY_CONDITION + flowey.exe e 12 flowey_lib_common::git_checkout 0 + flowey.exe v 12 'flowey_lib_common::git_checkout:0:flowey_lib_common/src/git_checkout.rs:469:80' --is-raw-string --condvar flowey_lib_common::git_checkout:1:flowey_lib_common/src/git_checkout.rs:470:46 write-to-env github floweyvar6 + flowey.exe v 12 'flowey_lib_common::git_checkout:1:flowey_lib_common/src/git_checkout.rs:470:46' write-to-env github FLOWEY_CONDITION shell: bash - id: flowey_lib_common__git_checkout__1 uses: actions/checkout@v4 @@ -962,27 +1255,23 @@ jobs: if: ${{ fromJSON(env.FLOWEY_CONDITION) }} - name: report cloned repo directories run: |- - flowey v 12 'flowey_lib_common::git_checkout:4:flowey_core/src/node/github_context.rs:55:41' --is-raw-string update --env-source github.workspace <> $GITHUB_PATH @@ -1185,7 +1451,7 @@ jobs: ${{ runner.temp }} EOF ) - flowey.exe pipeline github --runtime $ESCAPED_AGENT_TEMPDIR\\bootstrapped-flowey\\pipeline.yaml --out .github/workflows/openvmm-ci.yaml ci checkin-gates --config=ci + flowey pipeline github --runtime $ESCAPED_AGENT_TEMPDIR/bootstrapped-flowey/pipeline.yaml --out .github/workflows/openvmm-ci.yaml ci checkin-gates --config=ci shell: bash - name: πŸŒΌπŸ›« Initialize job run: | @@ -1193,31 +1459,31 @@ jobs: AgentTempDirNormal=$(echo "$AgentTempDirNormal" | sed -e 's|\\|\/|g' -e 's|^\([A-Za-z]\)\:/\(.*\)|/\L\1\E/\2|') echo "AgentTempDirNormal=$AgentTempDirNormal" >> $GITHUB_ENV - chmod +x $AgentTempDirNormal/bootstrapped-flowey/flowey.exe + chmod +x $AgentTempDirNormal/bootstrapped-flowey/flowey - echo '"debug"' | flowey.exe v 13 'FLOWEY_LOG' update - echo "${{ runner.temp }}/work" | flowey.exe v 13 '_internal_WORKING_DIR' --is-raw-string update + echo '"debug"' | flowey v 13 'FLOWEY_LOG' update + echo "${{ runner.temp }}/work" | flowey v 13 '_internal_WORKING_DIR' --is-raw-string update - cat <<'EOF' | flowey.exe v 13 'verbose' update + cat <<'EOF' | flowey v 13 'verbose' update ${{ inputs.verbose != '' && inputs.verbose || 'false' }} EOF shell: bash - name: add default cargo home to path - run: flowey.exe e 13 flowey_lib_common::install_rust 0 + run: flowey e 13 flowey_lib_common::install_rust 0 shell: bash - name: install Rust - run: flowey.exe e 13 flowey_lib_common::install_rust 1 + run: flowey e 13 flowey_lib_common::install_rust 1 shell: bash - name: detect active toolchain run: |- - flowey.exe e 13 flowey_lib_common::install_rust 2 - flowey.exe e 13 flowey_lib_common::cfg_cargo_common_flags 0 + flowey e 13 flowey_lib_common::install_rust 2 + flowey e 13 flowey_lib_common::cfg_cargo_common_flags 0 shell: bash - name: check if openvmm needs to be cloned run: |- - flowey.exe e 13 flowey_lib_common::git_checkout 0 - flowey.exe v 13 'flowey_lib_common::git_checkout:0:flowey_lib_common/src/git_checkout.rs:469:80' --is-raw-string --condvar flowey_lib_common::git_checkout:1:flowey_lib_common/src/git_checkout.rs:470:46 write-to-env github floweyvar6 - flowey.exe v 13 'flowey_lib_common::git_checkout:1:flowey_lib_common/src/git_checkout.rs:470:46' write-to-env github FLOWEY_CONDITION + flowey e 13 flowey_lib_common::git_checkout 0 + flowey v 13 'flowey_lib_common::git_checkout:0:flowey_lib_common/src/git_checkout.rs:469:80' --is-raw-string --condvar flowey_lib_common::git_checkout:1:flowey_lib_common/src/git_checkout.rs:470:46 write-to-env github floweyvar6 + flowey v 13 'flowey_lib_common::git_checkout:1:flowey_lib_common/src/git_checkout.rs:470:46' write-to-env github FLOWEY_CONDITION shell: bash - id: flowey_lib_common__git_checkout__1 uses: actions/checkout@v4 @@ -1229,23 +1495,23 @@ jobs: if: ${{ fromJSON(env.FLOWEY_CONDITION) }} - name: report cloned repo directories run: |- - flowey.exe v 13 'flowey_lib_common::git_checkout:4:flowey_core/src/node/github_context.rs:55:41' --is-raw-string update --env-source github.workspace <> $GITHUB_PATH @@ -1453,12 +1755,6 @@ jobs: flowey e 14 flowey_lib_common::install_rust 2 flowey e 14 flowey_lib_common::cfg_cargo_common_flags 0 shell: bash - - name: checking if packages need to be installed - run: flowey e 14 flowey_lib_common::install_dist_pkg 0 - shell: bash - - name: installing packages - run: flowey e 14 flowey_lib_common::install_dist_pkg 1 - shell: bash - name: check if openvmm needs to be cloned run: |- flowey e 14 flowey_lib_common::git_checkout 0 @@ -1480,9 +1776,13 @@ jobs: EOF flowey e 14 flowey_lib_common::git_checkout 3 flowey e 14 flowey_lib_hvlite::git_checkout_openvmm_repo 0 + flowey e 14 flowey_lib_hvlite::cfg_openvmm_magicpath 0 shell: bash - - name: set '-Dwarnings' in .cargo/config.toml - run: flowey e 14 flowey_lib_hvlite::init_openvmm_cargo_config_deny_warnings 0 + - name: checking if packages need to be installed + run: flowey e 14 flowey_lib_common::install_dist_pkg 0 + shell: bash + - name: installing packages + run: flowey e 14 flowey_lib_common::install_dist_pkg 1 shell: bash - name: create gh-release-download cache dir run: flowey e 14 flowey_lib_common::download_gh_release 0 @@ -1507,16 +1807,23 @@ jobs: flowey e 14 flowey_lib_common::cache 6 flowey e 14 flowey_lib_common::download_gh_release 1 shell: bash + - name: unpack openvmm-deps archive + run: flowey e 14 flowey_lib_hvlite::resolve_openvmm_deps 0 + shell: bash + - name: extract X64 sysroot.tar.gz + run: flowey e 14 flowey_lib_hvlite::init_openvmm_magicpath_openhcl_sysroot 0 + shell: bash + - name: set '-Dwarnings' in .cargo/config.toml + run: flowey e 14 flowey_lib_hvlite::init_openvmm_cargo_config_deny_warnings 0 + shell: bash - name: unpack protoc - run: |- - flowey e 14 flowey_lib_common::download_protoc 0 - flowey e 14 flowey_lib_hvlite::cfg_openvmm_magicpath 0 + run: flowey e 14 flowey_lib_common::download_protoc 0 shell: bash - name: symlink protoc run: |- flowey e 14 flowey_lib_hvlite::init_openvmm_magicpath_protoc 0 - flowey e 14 flowey_lib_hvlite::init_cross_build 0 flowey e 14 flowey_lib_hvlite::init_cross_build 2 + flowey e 14 flowey_lib_hvlite::init_cross_build 0 shell: bash - name: cargo build xtask run: |- @@ -1535,6 +1842,12 @@ jobs: - name: cargo clippy run: flowey e 14 flowey_lib_common::run_cargo_clippy 0 shell: bash + - name: cargo clippy + run: flowey e 14 flowey_lib_common::run_cargo_clippy 2 + shell: bash + - name: cargo clippy + run: flowey e 14 flowey_lib_common::run_cargo_clippy 1 + shell: bash - name: create cargo-nextest cache dir run: |- flowey e 14 flowey_lib_common::download_cargo_nextest 0 @@ -1568,7 +1881,7 @@ jobs: - name: installing cargo-nextest run: |- flowey e 14 flowey_lib_common::install_cargo_nextest 0 - flowey e 14 flowey_lib_hvlite::init_cross_build 3 + flowey e 14 flowey_lib_hvlite::init_cross_build 1 shell: bash - name: cargo build xtask run: |- @@ -1584,7 +1897,7 @@ jobs: - name: determine unit test exclusions run: |- flowey e 14 flowey_lib_hvlite::build_nextest_unit_tests 0 - flowey e 14 flowey_lib_hvlite::init_cross_build 1 + flowey e 14 flowey_lib_hvlite::init_cross_build 3 flowey e 14 flowey_lib_hvlite::run_cargo_nextest_run 0 shell: bash - name: generate nextest command @@ -1604,14 +1917,14 @@ jobs: - id: flowey_lib_common__publish_test_results__3 uses: actions/upload-artifact@v4 with: - name: aarch64-linux-unit-tests-junit-xml + name: x64-linux-musl-unit-tests-junit-xml path: ${{ env.floweyvar1 }} - name: 'publish test results: aarch64-linux-unit-tests (JUnit XML)' + name: 'publish test results: x64-linux-musl-unit-tests (JUnit XML)' if: ${{ fromJSON(env.FLOWEY_CONDITION) }} - name: report test results to overall pipeline status run: flowey e 14 flowey_lib_hvlite::_jobs::build_and_run_nextest_unit_tests 1 shell: bash - - name: run doctests for aarch64-unknown-linux-gnu + - name: run doctests for x86_64-unknown-linux-musl run: flowey e 14 flowey_lib_hvlite::_jobs::build_and_run_doc_tests 0 shell: bash - name: 'validate cache entry: cargo-nextest' @@ -1621,11 +1934,11 @@ jobs: run: flowey e 14 flowey_lib_common::cache 7 shell: bash job15: - name: clippy [aarch64-linux-musl, misc nostd], unit tests [aarch64-linux-musl] + name: clippy [aarch64-windows], unit tests [aarch64-windows] runs-on: - self-hosted - 1ES.Pool=OpenVMM-GitHub-ARM64-Pool-WestUS2 - - 1ES.ImageOverride=OpenVMM-CI-Ubuntu-ARM64 + - 1ES.ImageOverride=OpenVMM-CI-Windows-ARM64 permissions: contents: read id-token: write @@ -1664,11 +1977,11 @@ jobs: - name: Build flowey run: | set -x - CARGO_INCREMENTAL=0 cargo build -p flowey_hvlite --target aarch64-unknown-linux-gnu --profile flowey-ci + CARGO_INCREMENTAL=0 cargo build -p flowey_hvlite --target aarch64-pc-windows-msvc --profile flowey-ci OutDirNormal=$(echo "${{ runner.temp }}/bootstrapped-flowey" | sed -e 's|\\|\/|g' -e 's|^\([A-Za-z]\)\:/\(.*\)|/\L\1\E/\2|') mkdir -p "$OutDirNormal" mv ./.github/workflows/openvmm-ci.yaml "$OutDirNormal/pipeline.yaml" - mv target/aarch64-unknown-linux-gnu/flowey-ci/flowey_hvlite "$OutDirNormal/flowey" + mv target/aarch64-pc-windows-msvc/flowey-ci/flowey_hvlite.exe "$OutDirNormal/flowey.exe" working-directory: flowey_bootstrap shell: bash - run: echo "${{ runner.temp }}/bootstrapped-flowey" >> $GITHUB_PATH @@ -1681,7 +1994,7 @@ jobs: ${{ runner.temp }} EOF ) - flowey pipeline github --runtime $ESCAPED_AGENT_TEMPDIR/bootstrapped-flowey/pipeline.yaml --out .github/workflows/openvmm-ci.yaml ci checkin-gates --config=ci + flowey.exe pipeline github --runtime $ESCAPED_AGENT_TEMPDIR\\bootstrapped-flowey\\pipeline.yaml --out .github/workflows/openvmm-ci.yaml ci checkin-gates --config=ci shell: bash - name: πŸŒΌπŸ›« Initialize job run: | @@ -1689,31 +2002,31 @@ jobs: AgentTempDirNormal=$(echo "$AgentTempDirNormal" | sed -e 's|\\|\/|g' -e 's|^\([A-Za-z]\)\:/\(.*\)|/\L\1\E/\2|') echo "AgentTempDirNormal=$AgentTempDirNormal" >> $GITHUB_ENV - chmod +x $AgentTempDirNormal/bootstrapped-flowey/flowey + chmod +x $AgentTempDirNormal/bootstrapped-flowey/flowey.exe - echo '"debug"' | flowey v 15 'FLOWEY_LOG' update - echo "${{ runner.temp }}/work" | flowey v 15 '_internal_WORKING_DIR' --is-raw-string update + echo '"debug"' | flowey.exe v 15 'FLOWEY_LOG' update + echo "${{ runner.temp }}/work" | flowey.exe v 15 '_internal_WORKING_DIR' --is-raw-string update - cat <<'EOF' | flowey v 15 'verbose' update + cat <<'EOF' | flowey.exe v 15 'verbose' update ${{ inputs.verbose != '' && inputs.verbose || 'false' }} EOF shell: bash - name: add default cargo home to path - run: flowey e 15 flowey_lib_common::install_rust 0 + run: flowey.exe e 15 flowey_lib_common::install_rust 0 shell: bash - name: install Rust - run: flowey e 15 flowey_lib_common::install_rust 1 + run: flowey.exe e 15 flowey_lib_common::install_rust 1 shell: bash - name: detect active toolchain run: |- - flowey e 15 flowey_lib_common::install_rust 2 - flowey e 15 flowey_lib_common::cfg_cargo_common_flags 0 + flowey.exe e 15 flowey_lib_common::install_rust 2 + flowey.exe e 15 flowey_lib_common::cfg_cargo_common_flags 0 shell: bash - name: check if openvmm needs to be cloned run: |- - flowey e 15 flowey_lib_common::git_checkout 0 - flowey v 15 'flowey_lib_common::git_checkout:0:flowey_lib_common/src/git_checkout.rs:469:80' --is-raw-string --condvar flowey_lib_common::git_checkout:1:flowey_lib_common/src/git_checkout.rs:470:46 write-to-env github floweyvar6 - flowey v 15 'flowey_lib_common::git_checkout:1:flowey_lib_common/src/git_checkout.rs:470:46' write-to-env github FLOWEY_CONDITION + flowey.exe e 15 flowey_lib_common::git_checkout 0 + flowey.exe v 15 'flowey_lib_common::git_checkout:0:flowey_lib_common/src/git_checkout.rs:469:80' --is-raw-string --condvar flowey_lib_common::git_checkout:1:flowey_lib_common/src/git_checkout.rs:470:46 write-to-env github floweyvar6 + flowey.exe v 15 'flowey_lib_common::git_checkout:1:flowey_lib_common/src/git_checkout.rs:470:46' write-to-env github FLOWEY_CONDITION shell: bash - id: flowey_lib_common__git_checkout__1 uses: actions/checkout@v4 @@ -1725,27 +2038,23 @@ jobs: if: ${{ fromJSON(env.FLOWEY_CONDITION) }} - name: report cloned repo directories run: |- - flowey v 15 'flowey_lib_common::git_checkout:4:flowey_core/src/node/github_context.rs:55:41' --is-raw-string update --env-source github.workspace <> "$GITHUB_PATH" + rustup show + if: runner.os == 'Linux' + name: rustup (Linux) + shell: bash + - run: | + set -x + curl --fail -sSfLo rustup-init.exe https://win.rustup.rs/x86_64 --output rustup-init + ./rustup-init.exe -y --default-toolchain=1.91.1 + echo "$USERPROFILE\\.cargo\\bin" >> $GITHUB_PATH + if: runner.os == 'Windows' && runner.arch == 'X64' + name: rustup (Windows X64) + shell: bash + - run: | + set -x + curl --fail -sSfLo rustup-init.exe https://win.rustup.rs/aarch64 --output rustup-init + ./rustup-init.exe -y --default-toolchain=1.91.1 + echo "$USERPROFILE\\.cargo\\bin" >> $GITHUB_PATH + if: runner.os == 'Windows' && runner.arch == 'ARM64' + name: rustup (Windows ARM64) + shell: bash + - uses: actions/checkout@v4 with: - pattern: '{_internal-flowey-bootstrap-x86_64-windows-uid-12,x64-guest_test_uefi,x64-linux-musl-pipette,x64-linux-musl-tmk_vmm,x64-linux-tpm_guest_tests,x64-openhcl-igvm,x64-tmks,x64-windows-openvmm,x64-windows-pipette,x64-windows-prep_steps,x64-windows-tmk_vmm,x64-windows-tpm_guest_tests,x64-windows-vmgstool,x64-windows-vmm-tests-archive}' - path: ${{ runner.temp }}/used_artifacts/ - - run: echo "${{ runner.temp }}/used_artifacts/_internal-flowey-bootstrap-x86_64-windows-uid-12" >> $GITHUB_PATH + path: flowey_bootstrap + - name: Build flowey + run: | + set -x + CARGO_INCREMENTAL=0 cargo build -p flowey_hvlite --target aarch64-unknown-linux-gnu --profile flowey-ci + OutDirNormal=$(echo "${{ runner.temp }}/bootstrapped-flowey" | sed -e 's|\\|\/|g' -e 's|^\([A-Za-z]\)\:/\(.*\)|/\L\1\E/\2|') + mkdir -p "$OutDirNormal" + mv ./.github/workflows/openvmm-ci.yaml "$OutDirNormal/pipeline.yaml" + mv target/aarch64-unknown-linux-gnu/flowey-ci/flowey_hvlite "$OutDirNormal/flowey" + working-directory: flowey_bootstrap + shell: bash + - run: echo "${{ runner.temp }}/bootstrapped-flowey" >> $GITHUB_PATH shell: bash name: πŸŒΌπŸ“¦ Add flowey to PATH + - name: πŸŒΌπŸ”Ž Self-check YAML + run: |- + ESCAPED_AGENT_TEMPDIR=$( + cat <<'EOF' | sed 's/\\/\\\\/g' + ${{ runner.temp }} + EOF + ) + flowey pipeline github --runtime $ESCAPED_AGENT_TEMPDIR/bootstrapped-flowey/pipeline.yaml --out .github/workflows/openvmm-ci.yaml ci checkin-gates --config=ci + shell: bash - name: πŸŒΌπŸ›« Initialize job run: | AgentTempDirNormal="${{ runner.temp }}" AgentTempDirNormal=$(echo "$AgentTempDirNormal" | sed -e 's|\\|\/|g' -e 's|^\([A-Za-z]\)\:/\(.*\)|/\L\1\E/\2|') echo "AgentTempDirNormal=$AgentTempDirNormal" >> $GITHUB_ENV - chmod +x $AgentTempDirNormal/used_artifacts/_internal-flowey-bootstrap-x86_64-windows-uid-12/flowey.exe + chmod +x $AgentTempDirNormal/bootstrapped-flowey/flowey - echo '"debug"' | flowey.exe v 16 'FLOWEY_LOG' update - echo "${{ runner.temp }}/work" | flowey.exe v 16 '_internal_WORKING_DIR' --is-raw-string update + echo '"debug"' | flowey v 16 'FLOWEY_LOG' update + echo "${{ runner.temp }}/work" | flowey v 16 '_internal_WORKING_DIR' --is-raw-string update - cat <<'EOF' | flowey.exe v 16 'verbose' update + cat <<'EOF' | flowey v 16 'verbose' update ${{ inputs.verbose != '' && inputs.verbose || 'false' }} EOF - echo "${{ runner.temp }}\\used_artifacts\\x64-guest_test_uefi" | flowey.exe v 16 'artifact_use_from_x64-guest_test_uefi' --is-raw-string update - echo "${{ runner.temp }}\\used_artifacts\\x64-linux-musl-pipette" | flowey.exe v 16 'artifact_use_from_x64-linux-musl-pipette' --is-raw-string update - echo "${{ runner.temp }}\\used_artifacts\\x64-linux-musl-tmk_vmm" | flowey.exe v 16 'artifact_use_from_x64-linux-musl-tmk_vmm' --is-raw-string update - echo "${{ runner.temp }}\\used_artifacts\\x64-linux-tpm_guest_tests" | flowey.exe v 16 'artifact_use_from_x64-linux-tpm_guest_tests' --is-raw-string update - echo "${{ runner.temp }}\\used_artifacts\\x64-openhcl-igvm" | flowey.exe v 16 'artifact_use_from_x64-openhcl-igvm' --is-raw-string update - echo "${{ runner.temp }}\\used_artifacts\\x64-tmks" | flowey.exe v 16 'artifact_use_from_x64-tmks' --is-raw-string update - echo "${{ runner.temp }}\\used_artifacts\\x64-windows-openvmm" | flowey.exe v 16 'artifact_use_from_x64-windows-openvmm' --is-raw-string update - echo "${{ runner.temp }}\\used_artifacts\\x64-windows-pipette" | flowey.exe v 16 'artifact_use_from_x64-windows-pipette' --is-raw-string update - echo "${{ runner.temp }}\\used_artifacts\\x64-windows-prep_steps" | flowey.exe v 16 'artifact_use_from_x64-windows-prep_steps' --is-raw-string update - echo "${{ runner.temp }}\\used_artifacts\\x64-windows-tmk_vmm" | flowey.exe v 16 'artifact_use_from_x64-windows-tmk_vmm' --is-raw-string update - echo "${{ runner.temp }}\\used_artifacts\\x64-windows-tpm_guest_tests" | flowey.exe v 16 'artifact_use_from_x64-windows-tpm_guest_tests' --is-raw-string update - echo "${{ runner.temp }}\\used_artifacts\\x64-windows-vmgstool" | flowey.exe v 16 'artifact_use_from_x64-windows-vmgstool' --is-raw-string update - echo "${{ runner.temp }}\\used_artifacts\\x64-windows-vmm-tests-archive" | flowey.exe v 16 'artifact_use_from_x64-windows-vmm-tests-archive' --is-raw-string update - shell: bash - - name: creating new test content dir - run: |- - flowey.exe e 16 flowey_core::pipeline::artifact::resolve 5 - flowey.exe e 16 flowey_core::pipeline::artifact::resolve 6 - flowey.exe e 16 flowey_core::pipeline::artifact::resolve 1 - flowey.exe e 16 flowey_core::pipeline::artifact::resolve 0 - flowey.exe e 16 flowey_core::pipeline::artifact::resolve 8 - flowey.exe e 16 flowey_core::pipeline::artifact::resolve 2 - flowey.exe e 16 flowey_core::pipeline::artifact::resolve 4 - flowey.exe e 16 flowey_core::pipeline::artifact::resolve 10 - flowey.exe e 16 flowey_core::pipeline::artifact::resolve 9 - flowey.exe e 16 flowey_core::pipeline::artifact::resolve 3 - flowey.exe e 16 flowey_lib_hvlite::_jobs::consume_and_test_nextest_vmm_tests_archive 0 - shell: bash - - name: resolve OpenHCL igvm artifact - run: flowey.exe e 16 flowey_lib_hvlite::artifact_openhcl_igvm_from_recipe::resolve 0 shell: bash - - name: create azcopy cache dir - run: flowey.exe e 16 flowey_lib_common::download_azcopy 0 + - name: add default cargo home to path + run: flowey e 16 flowey_lib_common::install_rust 0 shell: bash - - name: Pre-processing cache vars - run: |- - flowey.exe e 16 flowey_lib_common::cache 0 - flowey.exe v 16 'flowey_lib_common::cache:2:flowey_lib_common/src/cache.rs:407:72' --is-raw-string write-to-env github floweyvar4 - flowey.exe v 16 'flowey_lib_common::cache:1:flowey_lib_common/src/cache.rs:406:72' --is-raw-string write-to-env github floweyvar5 + - name: install Rust + run: flowey e 16 flowey_lib_common::install_rust 1 shell: bash - - id: flowey_lib_common__cache__1 - uses: actions/cache@v4 - with: - key: ${{ env.floweyvar4 }} - path: ${{ env.floweyvar5 }} - name: 'Restore cache: azcopy' - - name: installing azcopy + - name: detect active toolchain run: |- - flowey.exe v 16 'flowey_lib_common::cache:4:flowey_lib_common/src/cache.rs:462:70' --is-raw-string update --env-source steps.flowey_lib_common__cache__1.outputs.cache-hit <> $GITHUB_PATH + - run: | + set -x + i=0; while [ $i -lt 5 ] && ! sudo apt-get update; do let "i=i+1"; sleep 1; done; + sudo apt-get -o DPkg::Lock::Timeout=60 install gcc -y + curl --fail --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- --default-toolchain=1.91.1 -y + . "$HOME/.cargo/env" + echo "$HOME/.cargo/bin" >> "$GITHUB_PATH" + rustup show + if: runner.os == 'Linux' + name: rustup (Linux) shell: bash - name: πŸŒΌπŸ“¦ Add flowey to PATH - - name: πŸŒΌπŸ›« Initialize job - run: | - AgentTempDirNormal="${{ runner.temp }}" - AgentTempDirNormal=$(echo "$AgentTempDirNormal" | sed -e 's|\\|\/|g' -e 's|^\([A-Za-z]\)\:/\(.*\)|/\L\1\E/\2|') - echo "AgentTempDirNormal=$AgentTempDirNormal" >> $GITHUB_ENV - - chmod +x $AgentTempDirNormal/used_artifacts/_internal-flowey-bootstrap-x86_64-windows-uid-12/flowey.exe - - echo '"debug"' | flowey.exe v 17 'FLOWEY_LOG' update - echo "${{ runner.temp }}/work" | flowey.exe v 17 '_internal_WORKING_DIR' --is-raw-string update - - cat <<'EOF' | flowey.exe v 17 'verbose' update - ${{ inputs.verbose != '' && inputs.verbose || 'false' }} - EOF - echo "${{ runner.temp }}\\used_artifacts\\x64-guest_test_uefi" | flowey.exe v 17 'artifact_use_from_x64-guest_test_uefi' --is-raw-string update - echo "${{ runner.temp }}\\used_artifacts\\x64-linux-musl-pipette" | flowey.exe v 17 'artifact_use_from_x64-linux-musl-pipette' --is-raw-string update - echo "${{ runner.temp }}\\used_artifacts\\x64-linux-musl-tmk_vmm" | flowey.exe v 17 'artifact_use_from_x64-linux-musl-tmk_vmm' --is-raw-string update - echo "${{ runner.temp }}\\used_artifacts\\x64-linux-tpm_guest_tests" | flowey.exe v 17 'artifact_use_from_x64-linux-tpm_guest_tests' --is-raw-string update - echo "${{ runner.temp }}\\used_artifacts\\x64-openhcl-igvm" | flowey.exe v 17 'artifact_use_from_x64-openhcl-igvm' --is-raw-string update - echo "${{ runner.temp }}\\used_artifacts\\x64-tmks" | flowey.exe v 17 'artifact_use_from_x64-tmks' --is-raw-string update - echo "${{ runner.temp }}\\used_artifacts\\x64-windows-openvmm" | flowey.exe v 17 'artifact_use_from_x64-windows-openvmm' --is-raw-string update - echo "${{ runner.temp }}\\used_artifacts\\x64-windows-pipette" | flowey.exe v 17 'artifact_use_from_x64-windows-pipette' --is-raw-string update - echo "${{ runner.temp }}\\used_artifacts\\x64-windows-prep_steps" | flowey.exe v 17 'artifact_use_from_x64-windows-prep_steps' --is-raw-string update - echo "${{ runner.temp }}\\used_artifacts\\x64-windows-tmk_vmm" | flowey.exe v 17 'artifact_use_from_x64-windows-tmk_vmm' --is-raw-string update - echo "${{ runner.temp }}\\used_artifacts\\x64-windows-tpm_guest_tests" | flowey.exe v 17 'artifact_use_from_x64-windows-tpm_guest_tests' --is-raw-string update - echo "${{ runner.temp }}\\used_artifacts\\x64-windows-vmgstool" | flowey.exe v 17 'artifact_use_from_x64-windows-vmgstool' --is-raw-string update - echo "${{ runner.temp }}\\used_artifacts\\x64-windows-vmm-tests-archive" | flowey.exe v 17 'artifact_use_from_x64-windows-vmm-tests-archive' --is-raw-string update + - run: | + set -x + curl --fail -sSfLo rustup-init.exe https://win.rustup.rs/x86_64 --output rustup-init + ./rustup-init.exe -y --default-toolchain=1.91.1 + echo "$USERPROFILE\\.cargo\\bin" >> $GITHUB_PATH + if: runner.os == 'Windows' && runner.arch == 'X64' + name: rustup (Windows X64) shell: bash - - name: creating new test content dir - run: |- - flowey.exe e 17 flowey_core::pipeline::artifact::resolve 5 - flowey.exe e 17 flowey_core::pipeline::artifact::resolve 6 - flowey.exe e 17 flowey_core::pipeline::artifact::resolve 1 - flowey.exe e 17 flowey_core::pipeline::artifact::resolve 0 - flowey.exe e 17 flowey_core::pipeline::artifact::resolve 8 - flowey.exe e 17 flowey_core::pipeline::artifact::resolve 2 - flowey.exe e 17 flowey_core::pipeline::artifact::resolve 4 - flowey.exe e 17 flowey_core::pipeline::artifact::resolve 10 - flowey.exe e 17 flowey_core::pipeline::artifact::resolve 9 - flowey.exe e 17 flowey_core::pipeline::artifact::resolve 3 - flowey.exe e 17 flowey_lib_hvlite::_jobs::consume_and_test_nextest_vmm_tests_archive 0 + - run: | + set -x + curl --fail -sSfLo rustup-init.exe https://win.rustup.rs/aarch64 --output rustup-init + ./rustup-init.exe -y --default-toolchain=1.91.1 + echo "$USERPROFILE\\.cargo\\bin" >> $GITHUB_PATH + if: runner.os == 'Windows' && runner.arch == 'ARM64' + name: rustup (Windows ARM64) shell: bash - - name: resolve OpenHCL igvm artifact - run: flowey.exe e 17 flowey_lib_hvlite::artifact_openhcl_igvm_from_recipe::resolve 0 + - uses: actions/checkout@v4 + with: + path: flowey_bootstrap + - name: Build flowey + run: | + set -x + CARGO_INCREMENTAL=0 cargo build -p flowey_hvlite --target aarch64-unknown-linux-gnu --profile flowey-ci + OutDirNormal=$(echo "${{ runner.temp }}/bootstrapped-flowey" | sed -e 's|\\|\/|g' -e 's|^\([A-Za-z]\)\:/\(.*\)|/\L\1\E/\2|') + mkdir -p "$OutDirNormal" + mv ./.github/workflows/openvmm-ci.yaml "$OutDirNormal/pipeline.yaml" + mv target/aarch64-unknown-linux-gnu/flowey-ci/flowey_hvlite "$OutDirNormal/flowey" + working-directory: flowey_bootstrap shell: bash - - name: create azcopy cache dir - run: flowey.exe e 17 flowey_lib_common::download_azcopy 0 + - run: echo "${{ runner.temp }}/bootstrapped-flowey" >> $GITHUB_PATH shell: bash - - name: Pre-processing cache vars + name: πŸŒΌπŸ“¦ Add flowey to PATH + - name: πŸŒΌπŸ”Ž Self-check YAML run: |- - flowey.exe e 17 flowey_lib_common::cache 0 - flowey.exe v 17 'flowey_lib_common::cache:2:flowey_lib_common/src/cache.rs:407:72' --is-raw-string write-to-env github floweyvar4 - flowey.exe v 17 'flowey_lib_common::cache:1:flowey_lib_common/src/cache.rs:406:72' --is-raw-string write-to-env github floweyvar5 + ESCAPED_AGENT_TEMPDIR=$( + cat <<'EOF' | sed 's/\\/\\\\/g' + ${{ runner.temp }} + EOF + ) + flowey pipeline github --runtime $ESCAPED_AGENT_TEMPDIR/bootstrapped-flowey/pipeline.yaml --out .github/workflows/openvmm-ci.yaml ci checkin-gates --config=ci shell: bash - - id: flowey_lib_common__cache__1 - uses: actions/cache@v4 - with: - key: ${{ env.floweyvar4 }} - path: ${{ env.floweyvar5 }} - name: 'Restore cache: azcopy' - - name: installing azcopy - run: |- - flowey.exe v 17 'flowey_lib_common::cache:4:flowey_lib_common/src/cache.rs:462:70' --is-raw-string update --env-source steps.flowey_lib_common__cache__1.outputs.cache-hit <> $GITHUB_ENV + + chmod +x $AgentTempDirNormal/bootstrapped-flowey/flowey + + echo '"debug"' | flowey v 17 'FLOWEY_LOG' update + echo "${{ runner.temp }}/work" | flowey v 17 '_internal_WORKING_DIR' --is-raw-string update + + cat <<'EOF' | flowey v 17 'verbose' update + ${{ inputs.verbose != '' && inputs.verbose || 'false' }} EOF - flowey.exe e 17 flowey_lib_common::cache 2 - flowey.exe e 17 flowey_lib_common::download_azcopy 1 shell: bash - - name: calculating required VMM tests disk images - run: flowey.exe e 17 flowey_lib_hvlite::download_openvmm_vmm_tests_artifacts 0 + - name: add default cargo home to path + run: flowey e 17 flowey_lib_common::install_rust 0 shell: bash - - name: downloading VMM test disk images - run: |- - flowey.exe e 17 flowey_lib_hvlite::download_openvmm_vmm_tests_artifacts 1 - flowey.exe e 17 flowey_lib_hvlite::download_openvmm_vmm_tests_artifacts 2 + - name: install Rust + run: flowey e 17 flowey_lib_common::install_rust 1 shell: bash - - name: create gh cache dir - run: flowey.exe e 17 flowey_lib_common::download_gh_cli 0 + - name: detect active toolchain + run: |- + flowey e 17 flowey_lib_common::install_rust 2 + flowey e 17 flowey_lib_common::cfg_cargo_common_flags 0 shell: bash - - name: Pre-processing cache vars + - name: check if openvmm needs to be cloned run: |- - flowey.exe e 17 flowey_lib_common::cache 8 - flowey.exe v 17 'flowey_lib_common::cache:18:flowey_lib_common/src/cache.rs:407:72' --is-raw-string write-to-env github floweyvar8 - flowey.exe v 17 'flowey_lib_common::cache:17:flowey_lib_common/src/cache.rs:406:72' --is-raw-string write-to-env github floweyvar9 + flowey e 17 flowey_lib_common::git_checkout 0 + flowey v 17 'flowey_lib_common::git_checkout:0:flowey_lib_common/src/git_checkout.rs:469:80' --is-raw-string --condvar flowey_lib_common::git_checkout:1:flowey_lib_common/src/git_checkout.rs:470:46 write-to-env github floweyvar6 + flowey v 17 'flowey_lib_common::git_checkout:1:flowey_lib_common/src/git_checkout.rs:470:46' write-to-env github FLOWEY_CONDITION shell: bash - - id: flowey_lib_common__cache__9 - uses: actions/cache@v4 + - id: flowey_lib_common__git_checkout__1 + uses: actions/checkout@v4 with: - key: ${{ env.floweyvar8 }} - path: ${{ env.floweyvar9 }} - name: 'Restore cache: gh-cli' - - name: installing gh + fetch-depth: '1' + path: repo0 + persist-credentials: ${{ env.floweyvar6 }} + name: checkout repo openvmm + if: ${{ fromJSON(env.FLOWEY_CONDITION) }} + - name: report cloned repo directories run: |- - flowey.exe v 17 'flowey_lib_common::cache:20:flowey_lib_common/src/cache.rs:462:70' --is-raw-string update --env-source steps.flowey_lib_common__cache__9.outputs.cache-hit <> $GITHUB_PATH + - run: echo "${{ runner.temp }}/used_artifacts/_internal-flowey-bootstrap-x86_64-windows-uid-14" >> $GITHUB_PATH shell: bash name: πŸŒΌπŸ“¦ Add flowey to PATH - name: πŸŒΌπŸ›« Initialize job @@ -2475,7 +2725,7 @@ jobs: AgentTempDirNormal=$(echo "$AgentTempDirNormal" | sed -e 's|\\|\/|g' -e 's|^\([A-Za-z]\)\:/\(.*\)|/\L\1\E/\2|') echo "AgentTempDirNormal=$AgentTempDirNormal" >> $GITHUB_ENV - chmod +x $AgentTempDirNormal/used_artifacts/_internal-flowey-bootstrap-x86_64-windows-uid-12/flowey.exe + chmod +x $AgentTempDirNormal/used_artifacts/_internal-flowey-bootstrap-x86_64-windows-uid-14/flowey.exe echo '"debug"' | flowey.exe v 18 'FLOWEY_LOG' update echo "${{ runner.temp }}/work" | flowey.exe v 18 '_internal_WORKING_DIR' --is-raw-string update @@ -2607,7 +2857,7 @@ jobs: flowey.exe e 18 flowey_lib_common::download_gh_release 1 shell: bash - name: unpack openvmm-deps archive - run: flowey.exe e 18 flowey_lib_hvlite::download_openvmm_deps 0 + run: flowey.exe e 18 flowey_lib_hvlite::resolve_openvmm_deps 0 shell: bash - name: unpack mu_msvm package (x64) run: flowey.exe e 18 flowey_lib_hvlite::download_uefi_mu_msvm 0 @@ -2687,9 +2937,9 @@ jobs: - id: flowey_lib_common__publish_test_results__6 uses: actions/upload-artifact@v4 with: - name: x64-windows-amd-vmm-tests-logs + name: x64-windows-intel-vmm-tests-logs path: ${{ env.floweyvar2 }} - name: 'publish test results: x64-windows-amd-vmm-tests (logs)' + name: 'publish test results: x64-windows-intel-vmm-tests (logs)' if: ${{ fromJSON(env.FLOWEY_CONDITION) }} - name: πŸ¦€ flowey rust steps run: |- @@ -2703,9 +2953,9 @@ jobs: - id: flowey_lib_common__publish_test_results__3 uses: actions/upload-artifact@v4 with: - name: x64-windows-amd-vmm-tests-junit-xml + name: x64-windows-intel-vmm-tests-junit-xml path: ${{ env.floweyvar1 }} - name: 'publish test results: x64-windows-amd-vmm-tests (JUnit XML)' + name: 'publish test results: x64-windows-intel-vmm-tests (JUnit XML)' if: ${{ fromJSON(env.FLOWEY_CONDITION) }} - name: report test results to overall pipeline status run: flowey.exe e 18 flowey_lib_hvlite::_jobs::consume_and_test_nextest_vmm_tests_archive 3 @@ -2723,28 +2973,28 @@ jobs: run: flowey.exe e 18 flowey_lib_common::cache 15 shell: bash job19: - name: run vmm-tests [x64-windows-amd-snp] + name: run vmm-tests [x64-windows-intel-tdx] runs-on: - self-hosted - Windows - X64 - - SNP + - TDX - Baremetal permissions: contents: read id-token: write needs: + - job10 - job5 - job7 - - job9 if: github.event.pull_request.draft == false steps: - name: πŸŒΌπŸ“¦ Download artifacts uses: actions/download-artifact@v4 with: - pattern: '{_internal-flowey-bootstrap-x86_64-windows-uid-12,x64-guest_test_uefi,x64-linux-musl-pipette,x64-linux-musl-tmk_vmm,x64-linux-tpm_guest_tests,x64-openhcl-igvm,x64-tmks,x64-windows-openvmm,x64-windows-pipette,x64-windows-prep_steps,x64-windows-tmk_vmm,x64-windows-tpm_guest_tests,x64-windows-vmgstool,x64-windows-vmm-tests-archive}' + pattern: '{_internal-flowey-bootstrap-x86_64-windows-uid-14,x64-guest_test_uefi,x64-linux-musl-pipette,x64-linux-musl-tmk_vmm,x64-linux-tpm_guest_tests,x64-openhcl-igvm,x64-tmks,x64-windows-openvmm,x64-windows-pipette,x64-windows-prep_steps,x64-windows-tmk_vmm,x64-windows-tpm_guest_tests,x64-windows-vmgstool,x64-windows-vmm-tests-archive}' path: ${{ runner.temp }}/used_artifacts/ - - run: echo "${{ runner.temp }}/used_artifacts/_internal-flowey-bootstrap-x86_64-windows-uid-12" >> $GITHUB_PATH + - run: echo "${{ runner.temp }}/used_artifacts/_internal-flowey-bootstrap-x86_64-windows-uid-14" >> $GITHUB_PATH shell: bash name: πŸŒΌπŸ“¦ Add flowey to PATH - name: πŸŒΌπŸ›« Initialize job @@ -2753,7 +3003,7 @@ jobs: AgentTempDirNormal=$(echo "$AgentTempDirNormal" | sed -e 's|\\|\/|g' -e 's|^\([A-Za-z]\)\:/\(.*\)|/\L\1\E/\2|') echo "AgentTempDirNormal=$AgentTempDirNormal" >> $GITHUB_ENV - chmod +x $AgentTempDirNormal/used_artifacts/_internal-flowey-bootstrap-x86_64-windows-uid-12/flowey.exe + chmod +x $AgentTempDirNormal/used_artifacts/_internal-flowey-bootstrap-x86_64-windows-uid-14/flowey.exe echo '"debug"' | flowey.exe v 19 'FLOWEY_LOG' update echo "${{ runner.temp }}/work" | flowey.exe v 19 '_internal_WORKING_DIR' --is-raw-string update @@ -2790,16 +3040,511 @@ jobs: flowey.exe e 19 flowey_lib_hvlite::_jobs::consume_and_test_nextest_vmm_tests_archive 0 shell: bash - name: resolve OpenHCL igvm artifact - run: flowey.exe e 19 flowey_lib_hvlite::artifact_openhcl_igvm_from_recipe::resolve 0 + run: flowey.exe e 19 flowey_lib_hvlite::artifact_openhcl_igvm_from_recipe::resolve 0 + shell: bash + - name: create azcopy cache dir + run: flowey.exe e 19 flowey_lib_common::download_azcopy 0 + shell: bash + - name: Pre-processing cache vars + run: |- + flowey.exe e 19 flowey_lib_common::cache 0 + flowey.exe v 19 'flowey_lib_common::cache:2:flowey_lib_common/src/cache.rs:407:72' --is-raw-string write-to-env github floweyvar4 + flowey.exe v 19 'flowey_lib_common::cache:1:flowey_lib_common/src/cache.rs:406:72' --is-raw-string write-to-env github floweyvar5 + shell: bash + - id: flowey_lib_common__cache__1 + uses: actions/cache@v4 + with: + key: ${{ env.floweyvar4 }} + path: ${{ env.floweyvar5 }} + name: 'Restore cache: azcopy' + - name: installing azcopy + run: |- + flowey.exe v 19 'flowey_lib_common::cache:4:flowey_lib_common/src/cache.rs:462:70' --is-raw-string update --env-source steps.flowey_lib_common__cache__1.outputs.cache-hit <> "$GITHUB_PATH" + rustup show + if: runner.os == 'Linux' + name: rustup (Linux) + shell: bash + - run: | + set -x + curl --fail -sSfLo rustup-init.exe https://win.rustup.rs/x86_64 --output rustup-init + ./rustup-init.exe -y --default-toolchain=1.91.1 + echo "$USERPROFILE\\.cargo\\bin" >> $GITHUB_PATH + if: runner.os == 'Windows' && runner.arch == 'X64' + name: rustup (Windows X64) + shell: bash + - run: | + set -x + curl --fail -sSfLo rustup-init.exe https://win.rustup.rs/aarch64 --output rustup-init + ./rustup-init.exe -y --default-toolchain=1.91.1 + echo "$USERPROFILE\\.cargo\\bin" >> $GITHUB_PATH + if: runner.os == 'Windows' && runner.arch == 'ARM64' + name: rustup (Windows ARM64) + shell: bash + - uses: actions/checkout@v4 + with: + path: flowey_bootstrap + - name: Build flowey + run: | + set -x + CARGO_INCREMENTAL=0 cargo build -p flowey_hvlite --target x86_64-pc-windows-msvc --profile flowey-ci + OutDirNormal=$(echo "${{ runner.temp }}/bootstrapped-flowey" | sed -e 's|\\|\/|g' -e 's|^\([A-Za-z]\)\:/\(.*\)|/\L\1\E/\2|') + mkdir -p "$OutDirNormal" + mv ./.github/workflows/openvmm-ci.yaml "$OutDirNormal/pipeline.yaml" + mv target/x86_64-pc-windows-msvc/flowey-ci/flowey_hvlite.exe "$OutDirNormal/flowey.exe" + working-directory: flowey_bootstrap + shell: bash + - run: echo "${{ runner.temp }}/bootstrapped-flowey" >> $GITHUB_PATH + shell: bash + name: πŸŒΌπŸ“¦ Add flowey to PATH + - name: πŸŒΌπŸ”Ž Self-check YAML + run: |- + ESCAPED_AGENT_TEMPDIR=$( + cat <<'EOF' | sed 's/\\/\\\\/g' + ${{ runner.temp }} + EOF + ) + flowey.exe pipeline github --runtime $ESCAPED_AGENT_TEMPDIR\\bootstrapped-flowey\\pipeline.yaml --out .github/workflows/openvmm-ci.yaml ci checkin-gates --config=ci + shell: bash + - name: πŸŒΌπŸ›« Initialize job + run: | + AgentTempDirNormal="${{ runner.temp }}" + AgentTempDirNormal=$(echo "$AgentTempDirNormal" | sed -e 's|\\|\/|g' -e 's|^\([A-Za-z]\)\:/\(.*\)|/\L\1\E/\2|') + echo "AgentTempDirNormal=$AgentTempDirNormal" >> $GITHUB_ENV + + chmod +x $AgentTempDirNormal/bootstrapped-flowey/flowey.exe + + echo '"debug"' | flowey.exe v 2 'FLOWEY_LOG' update + echo "${{ runner.temp }}/work" | flowey.exe v 2 '_internal_WORKING_DIR' --is-raw-string update + + cat <<'EOF' | flowey.exe v 2 'verbose' update + ${{ inputs.verbose != '' && inputs.verbose || 'false' }} + EOF + mkdir -p "$AgentTempDirNormal/publish_artifacts/aarch64-windows-hypestv" + echo "${{ runner.temp }}\\publish_artifacts\\aarch64-windows-hypestv" | flowey.exe v 2 'artifact_publish_from_aarch64-windows-hypestv' --is-raw-string update + mkdir -p "$AgentTempDirNormal/publish_artifacts/aarch64-windows-igvmfilegen" + echo "${{ runner.temp }}\\publish_artifacts\\aarch64-windows-igvmfilegen" | flowey.exe v 2 'artifact_publish_from_aarch64-windows-igvmfilegen' --is-raw-string update + mkdir -p "$AgentTempDirNormal/publish_artifacts/aarch64-windows-ohcldiag-dev" + echo "${{ runner.temp }}\\publish_artifacts\\aarch64-windows-ohcldiag-dev" | flowey.exe v 2 'artifact_publish_from_aarch64-windows-ohcldiag-dev' --is-raw-string update + mkdir -p "$AgentTempDirNormal/publish_artifacts/aarch64-windows-vmgs_lib" + echo "${{ runner.temp }}\\publish_artifacts\\aarch64-windows-vmgs_lib" | flowey.exe v 2 'artifact_publish_from_aarch64-windows-vmgs_lib' --is-raw-string update + shell: bash + - name: add default cargo home to path + run: flowey.exe e 2 flowey_lib_common::install_rust 0 + shell: bash + - name: install Rust + run: flowey.exe e 2 flowey_lib_common::install_rust 1 + shell: bash + - name: detect active toolchain + run: |- + flowey.exe e 2 flowey_lib_common::install_rust 2 + flowey.exe e 2 flowey_lib_common::cfg_cargo_common_flags 0 + shell: bash + - name: check if openvmm needs to be cloned + run: |- + flowey.exe e 2 flowey_lib_common::git_checkout 0 + flowey.exe v 2 'flowey_lib_common::git_checkout:0:flowey_lib_common/src/git_checkout.rs:469:80' --is-raw-string --condvar flowey_lib_common::git_checkout:1:flowey_lib_common/src/git_checkout.rs:470:46 write-to-env github floweyvar1 + flowey.exe v 2 'flowey_lib_common::git_checkout:1:flowey_lib_common/src/git_checkout.rs:470:46' write-to-env github FLOWEY_CONDITION + shell: bash + - id: flowey_lib_common__git_checkout__1 + uses: actions/checkout@v4 + with: + fetch-depth: '1' + path: repo0 + persist-credentials: ${{ env.floweyvar1 }} + name: checkout repo openvmm + if: ${{ fromJSON(env.FLOWEY_CONDITION) }} + - name: report cloned repo directories + run: |- + flowey.exe v 2 'flowey_lib_common::git_checkout:4:flowey_core/src/node/github_context.rs:55:41' --is-raw-string update --env-source github.workspace <> $GITHUB_PATH + shell: bash + name: πŸŒΌπŸ“¦ Add flowey to PATH + - name: πŸŒΌπŸ›« Initialize job + run: | + AgentTempDirNormal="${{ runner.temp }}" + AgentTempDirNormal=$(echo "$AgentTempDirNormal" | sed -e 's|\\|\/|g' -e 's|^\([A-Za-z]\)\:/\(.*\)|/\L\1\E/\2|') + echo "AgentTempDirNormal=$AgentTempDirNormal" >> $GITHUB_ENV + + chmod +x $AgentTempDirNormal/used_artifacts/_internal-flowey-bootstrap-x86_64-windows-uid-14/flowey.exe + + echo '"debug"' | flowey.exe v 20 'FLOWEY_LOG' update + echo "${{ runner.temp }}/work" | flowey.exe v 20 '_internal_WORKING_DIR' --is-raw-string update + + cat <<'EOF' | flowey.exe v 20 'verbose' update + ${{ inputs.verbose != '' && inputs.verbose || 'false' }} + EOF + echo "${{ runner.temp }}\\used_artifacts\\x64-guest_test_uefi" | flowey.exe v 20 'artifact_use_from_x64-guest_test_uefi' --is-raw-string update + echo "${{ runner.temp }}\\used_artifacts\\x64-linux-musl-pipette" | flowey.exe v 20 'artifact_use_from_x64-linux-musl-pipette' --is-raw-string update + echo "${{ runner.temp }}\\used_artifacts\\x64-linux-musl-tmk_vmm" | flowey.exe v 20 'artifact_use_from_x64-linux-musl-tmk_vmm' --is-raw-string update + echo "${{ runner.temp }}\\used_artifacts\\x64-linux-tpm_guest_tests" | flowey.exe v 20 'artifact_use_from_x64-linux-tpm_guest_tests' --is-raw-string update + echo "${{ runner.temp }}\\used_artifacts\\x64-openhcl-igvm" | flowey.exe v 20 'artifact_use_from_x64-openhcl-igvm' --is-raw-string update + echo "${{ runner.temp }}\\used_artifacts\\x64-tmks" | flowey.exe v 20 'artifact_use_from_x64-tmks' --is-raw-string update + echo "${{ runner.temp }}\\used_artifacts\\x64-windows-openvmm" | flowey.exe v 20 'artifact_use_from_x64-windows-openvmm' --is-raw-string update + echo "${{ runner.temp }}\\used_artifacts\\x64-windows-pipette" | flowey.exe v 20 'artifact_use_from_x64-windows-pipette' --is-raw-string update + echo "${{ runner.temp }}\\used_artifacts\\x64-windows-prep_steps" | flowey.exe v 20 'artifact_use_from_x64-windows-prep_steps' --is-raw-string update + echo "${{ runner.temp }}\\used_artifacts\\x64-windows-tmk_vmm" | flowey.exe v 20 'artifact_use_from_x64-windows-tmk_vmm' --is-raw-string update + echo "${{ runner.temp }}\\used_artifacts\\x64-windows-tpm_guest_tests" | flowey.exe v 20 'artifact_use_from_x64-windows-tpm_guest_tests' --is-raw-string update + echo "${{ runner.temp }}\\used_artifacts\\x64-windows-vmgstool" | flowey.exe v 20 'artifact_use_from_x64-windows-vmgstool' --is-raw-string update + echo "${{ runner.temp }}\\used_artifacts\\x64-windows-vmm-tests-archive" | flowey.exe v 20 'artifact_use_from_x64-windows-vmm-tests-archive' --is-raw-string update + shell: bash + - name: creating new test content dir + run: |- + flowey.exe e 20 flowey_core::pipeline::artifact::resolve 5 + flowey.exe e 20 flowey_core::pipeline::artifact::resolve 6 + flowey.exe e 20 flowey_core::pipeline::artifact::resolve 1 + flowey.exe e 20 flowey_core::pipeline::artifact::resolve 0 + flowey.exe e 20 flowey_core::pipeline::artifact::resolve 8 + flowey.exe e 20 flowey_core::pipeline::artifact::resolve 2 + flowey.exe e 20 flowey_core::pipeline::artifact::resolve 4 + flowey.exe e 20 flowey_core::pipeline::artifact::resolve 10 + flowey.exe e 20 flowey_core::pipeline::artifact::resolve 9 + flowey.exe e 20 flowey_core::pipeline::artifact::resolve 3 + flowey.exe e 20 flowey_lib_hvlite::_jobs::consume_and_test_nextest_vmm_tests_archive 0 + shell: bash + - name: resolve OpenHCL igvm artifact + run: flowey.exe e 20 flowey_lib_hvlite::artifact_openhcl_igvm_from_recipe::resolve 0 shell: bash - name: create azcopy cache dir - run: flowey.exe e 19 flowey_lib_common::download_azcopy 0 + run: flowey.exe e 20 flowey_lib_common::download_azcopy 0 shell: bash - name: Pre-processing cache vars run: |- - flowey.exe e 19 flowey_lib_common::cache 0 - flowey.exe v 19 'flowey_lib_common::cache:2:flowey_lib_common/src/cache.rs:407:72' --is-raw-string write-to-env github floweyvar4 - flowey.exe v 19 'flowey_lib_common::cache:1:flowey_lib_common/src/cache.rs:406:72' --is-raw-string write-to-env github floweyvar5 + flowey.exe e 20 flowey_lib_common::cache 0 + flowey.exe v 20 'flowey_lib_common::cache:2:flowey_lib_common/src/cache.rs:407:72' --is-raw-string write-to-env github floweyvar4 + flowey.exe v 20 'flowey_lib_common::cache:1:flowey_lib_common/src/cache.rs:406:72' --is-raw-string write-to-env github floweyvar5 shell: bash - id: flowey_lib_common__cache__1 uses: actions/cache@v4 @@ -2809,28 +3554,28 @@ jobs: name: 'Restore cache: azcopy' - name: installing azcopy run: |- - flowey.exe v 19 'flowey_lib_common::cache:4:flowey_lib_common/src/cache.rs:462:70' --is-raw-string update --env-source steps.flowey_lib_common__cache__1.outputs.cache-hit <> "$GITHUB_PATH" - rustup show - if: runner.os == 'Linux' - name: rustup (Linux) + - name: πŸŒΌπŸ“¦ Download artifacts + uses: actions/download-artifact@v4 + with: + pattern: '{_internal-flowey-bootstrap-x86_64-windows-uid-14,x64-guest_test_uefi,x64-linux-musl-pipette,x64-linux-musl-tmk_vmm,x64-linux-tpm_guest_tests,x64-openhcl-igvm,x64-tmks,x64-windows-openvmm,x64-windows-pipette,x64-windows-prep_steps,x64-windows-tmk_vmm,x64-windows-tpm_guest_tests,x64-windows-vmgstool,x64-windows-vmm-tests-archive}' + path: ${{ runner.temp }}/used_artifacts/ + - run: echo "${{ runner.temp }}/used_artifacts/_internal-flowey-bootstrap-x86_64-windows-uid-14" >> $GITHUB_PATH shell: bash - - run: | - set -x - curl --fail -sSfLo rustup-init.exe https://win.rustup.rs/x86_64 --output rustup-init - ./rustup-init.exe -y --default-toolchain=1.91.1 - echo "$USERPROFILE\\.cargo\\bin" >> $GITHUB_PATH - if: runner.os == 'Windows' && runner.arch == 'X64' - name: rustup (Windows X64) + name: πŸŒΌπŸ“¦ Add flowey to PATH + - name: πŸŒΌπŸ›« Initialize job + run: | + AgentTempDirNormal="${{ runner.temp }}" + AgentTempDirNormal=$(echo "$AgentTempDirNormal" | sed -e 's|\\|\/|g' -e 's|^\([A-Za-z]\)\:/\(.*\)|/\L\1\E/\2|') + echo "AgentTempDirNormal=$AgentTempDirNormal" >> $GITHUB_ENV + + chmod +x $AgentTempDirNormal/used_artifacts/_internal-flowey-bootstrap-x86_64-windows-uid-14/flowey.exe + + echo '"debug"' | flowey.exe v 21 'FLOWEY_LOG' update + echo "${{ runner.temp }}/work" | flowey.exe v 21 '_internal_WORKING_DIR' --is-raw-string update + + cat <<'EOF' | flowey.exe v 21 'verbose' update + ${{ inputs.verbose != '' && inputs.verbose || 'false' }} + EOF + echo "${{ runner.temp }}\\used_artifacts\\x64-guest_test_uefi" | flowey.exe v 21 'artifact_use_from_x64-guest_test_uefi' --is-raw-string update + echo "${{ runner.temp }}\\used_artifacts\\x64-linux-musl-pipette" | flowey.exe v 21 'artifact_use_from_x64-linux-musl-pipette' --is-raw-string update + echo "${{ runner.temp }}\\used_artifacts\\x64-linux-musl-tmk_vmm" | flowey.exe v 21 'artifact_use_from_x64-linux-musl-tmk_vmm' --is-raw-string update + echo "${{ runner.temp }}\\used_artifacts\\x64-linux-tpm_guest_tests" | flowey.exe v 21 'artifact_use_from_x64-linux-tpm_guest_tests' --is-raw-string update + echo "${{ runner.temp }}\\used_artifacts\\x64-openhcl-igvm" | flowey.exe v 21 'artifact_use_from_x64-openhcl-igvm' --is-raw-string update + echo "${{ runner.temp }}\\used_artifacts\\x64-tmks" | flowey.exe v 21 'artifact_use_from_x64-tmks' --is-raw-string update + echo "${{ runner.temp }}\\used_artifacts\\x64-windows-openvmm" | flowey.exe v 21 'artifact_use_from_x64-windows-openvmm' --is-raw-string update + echo "${{ runner.temp }}\\used_artifacts\\x64-windows-pipette" | flowey.exe v 21 'artifact_use_from_x64-windows-pipette' --is-raw-string update + echo "${{ runner.temp }}\\used_artifacts\\x64-windows-prep_steps" | flowey.exe v 21 'artifact_use_from_x64-windows-prep_steps' --is-raw-string update + echo "${{ runner.temp }}\\used_artifacts\\x64-windows-tmk_vmm" | flowey.exe v 21 'artifact_use_from_x64-windows-tmk_vmm' --is-raw-string update + echo "${{ runner.temp }}\\used_artifacts\\x64-windows-tpm_guest_tests" | flowey.exe v 21 'artifact_use_from_x64-windows-tpm_guest_tests' --is-raw-string update + echo "${{ runner.temp }}\\used_artifacts\\x64-windows-vmgstool" | flowey.exe v 21 'artifact_use_from_x64-windows-vmgstool' --is-raw-string update + echo "${{ runner.temp }}\\used_artifacts\\x64-windows-vmm-tests-archive" | flowey.exe v 21 'artifact_use_from_x64-windows-vmm-tests-archive' --is-raw-string update shell: bash - - run: | - set -x - curl --fail -sSfLo rustup-init.exe https://win.rustup.rs/aarch64 --output rustup-init - ./rustup-init.exe -y --default-toolchain=1.91.1 - echo "$USERPROFILE\\.cargo\\bin" >> $GITHUB_PATH - if: runner.os == 'Windows' && runner.arch == 'ARM64' - name: rustup (Windows ARM64) + - name: creating new test content dir + run: |- + flowey.exe e 21 flowey_core::pipeline::artifact::resolve 5 + flowey.exe e 21 flowey_core::pipeline::artifact::resolve 6 + flowey.exe e 21 flowey_core::pipeline::artifact::resolve 1 + flowey.exe e 21 flowey_core::pipeline::artifact::resolve 0 + flowey.exe e 21 flowey_core::pipeline::artifact::resolve 8 + flowey.exe e 21 flowey_core::pipeline::artifact::resolve 2 + flowey.exe e 21 flowey_core::pipeline::artifact::resolve 4 + flowey.exe e 21 flowey_core::pipeline::artifact::resolve 10 + flowey.exe e 21 flowey_core::pipeline::artifact::resolve 9 + flowey.exe e 21 flowey_core::pipeline::artifact::resolve 3 + flowey.exe e 21 flowey_lib_hvlite::_jobs::consume_and_test_nextest_vmm_tests_archive 0 shell: bash - - uses: actions/checkout@v4 + - name: resolve OpenHCL igvm artifact + run: flowey.exe e 21 flowey_lib_hvlite::artifact_openhcl_igvm_from_recipe::resolve 0 + shell: bash + - name: create azcopy cache dir + run: flowey.exe e 21 flowey_lib_common::download_azcopy 0 + shell: bash + - name: Pre-processing cache vars + run: |- + flowey.exe e 21 flowey_lib_common::cache 0 + flowey.exe v 21 'flowey_lib_common::cache:2:flowey_lib_common/src/cache.rs:407:72' --is-raw-string write-to-env github floweyvar4 + flowey.exe v 21 'flowey_lib_common::cache:1:flowey_lib_common/src/cache.rs:406:72' --is-raw-string write-to-env github floweyvar5 + shell: bash + - id: flowey_lib_common__cache__1 + uses: actions/cache@v4 with: - path: flowey_bootstrap - - name: Build flowey - run: | - set -x - CARGO_INCREMENTAL=0 cargo build -p flowey_hvlite --target x86_64-pc-windows-msvc --profile flowey-ci - OutDirNormal=$(echo "${{ runner.temp }}/bootstrapped-flowey" | sed -e 's|\\|\/|g' -e 's|^\([A-Za-z]\)\:/\(.*\)|/\L\1\E/\2|') - mkdir -p "$OutDirNormal" - mv ./.github/workflows/openvmm-ci.yaml "$OutDirNormal/pipeline.yaml" - mv target/x86_64-pc-windows-msvc/flowey-ci/flowey_hvlite.exe "$OutDirNormal/flowey.exe" - working-directory: flowey_bootstrap + key: ${{ env.floweyvar4 }} + path: ${{ env.floweyvar5 }} + name: 'Restore cache: azcopy' + - name: installing azcopy + run: |- + flowey.exe v 21 'flowey_lib_common::cache:4:flowey_lib_common/src/cache.rs:462:70' --is-raw-string update --env-source steps.flowey_lib_common__cache__1.outputs.cache-hit <> $GITHUB_PATH + - name: calculating required VMM tests disk images + run: flowey.exe e 21 flowey_lib_hvlite::download_openvmm_vmm_tests_artifacts 0 shell: bash - name: πŸŒΌπŸ“¦ Add flowey to PATH - - name: πŸŒΌπŸ”Ž Self-check YAML + - name: downloading VMM test disk images run: |- - ESCAPED_AGENT_TEMPDIR=$( - cat <<'EOF' | sed 's/\\/\\\\/g' - ${{ runner.temp }} + flowey.exe e 21 flowey_lib_hvlite::download_openvmm_vmm_tests_artifacts 1 + flowey.exe e 21 flowey_lib_hvlite::download_openvmm_vmm_tests_artifacts 2 + shell: bash + - name: create gh cache dir + run: flowey.exe e 21 flowey_lib_common::download_gh_cli 0 + shell: bash + - name: Pre-processing cache vars + run: |- + flowey.exe e 21 flowey_lib_common::cache 8 + flowey.exe v 21 'flowey_lib_common::cache:18:flowey_lib_common/src/cache.rs:407:72' --is-raw-string write-to-env github floweyvar8 + flowey.exe v 21 'flowey_lib_common::cache:17:flowey_lib_common/src/cache.rs:406:72' --is-raw-string write-to-env github floweyvar9 + shell: bash + - id: flowey_lib_common__cache__9 + uses: actions/cache@v4 + with: + key: ${{ env.floweyvar8 }} + path: ${{ env.floweyvar9 }} + name: 'Restore cache: gh-cli' + - name: installing gh + run: |- + flowey.exe v 21 'flowey_lib_common::cache:20:flowey_lib_common/src/cache.rs:462:70' --is-raw-string update --env-source steps.flowey_lib_common__cache__9.outputs.cache-hit <> $GITHUB_ENV - - chmod +x $AgentTempDirNormal/bootstrapped-flowey/flowey.exe - - echo '"debug"' | flowey.exe v 2 'FLOWEY_LOG' update - echo "${{ runner.temp }}/work" | flowey.exe v 2 '_internal_WORKING_DIR' --is-raw-string update - - cat <<'EOF' | flowey.exe v 2 'verbose' update - ${{ inputs.verbose != '' && inputs.verbose || 'false' }} - EOF - mkdir -p "$AgentTempDirNormal/publish_artifacts/aarch64-windows-hypestv" - echo "${{ runner.temp }}\\publish_artifacts\\aarch64-windows-hypestv" | flowey.exe v 2 'artifact_publish_from_aarch64-windows-hypestv' --is-raw-string update - mkdir -p "$AgentTempDirNormal/publish_artifacts/aarch64-windows-igvmfilegen" - echo "${{ runner.temp }}\\publish_artifacts\\aarch64-windows-igvmfilegen" | flowey.exe v 2 'artifact_publish_from_aarch64-windows-igvmfilegen' --is-raw-string update - mkdir -p "$AgentTempDirNormal/publish_artifacts/aarch64-windows-ohcldiag-dev" - echo "${{ runner.temp }}\\publish_artifacts\\aarch64-windows-ohcldiag-dev" | flowey.exe v 2 'artifact_publish_from_aarch64-windows-ohcldiag-dev' --is-raw-string update - mkdir -p "$AgentTempDirNormal/publish_artifacts/aarch64-windows-vmgs_lib" - echo "${{ runner.temp }}\\publish_artifacts\\aarch64-windows-vmgs_lib" | flowey.exe v 2 'artifact_publish_from_aarch64-windows-vmgs_lib' --is-raw-string update + - name: unpack openvmm-deps archive + run: flowey.exe e 21 flowey_lib_hvlite::resolve_openvmm_deps 0 shell: bash - - name: add default cargo home to path - run: flowey.exe e 2 flowey_lib_common::install_rust 0 + - name: unpack mu_msvm package (x64) + run: flowey.exe e 21 flowey_lib_hvlite::download_uefi_mu_msvm 0 shell: bash - - name: install Rust - run: flowey.exe e 2 flowey_lib_common::install_rust 1 + - name: setting up vmm_tests env + run: flowey.exe e 21 flowey_lib_hvlite::init_vmm_tests_env 0 shell: bash - - name: detect active toolchain + - name: create cargo-nextest cache dir run: |- - flowey.exe e 2 flowey_lib_common::install_rust 2 - flowey.exe e 2 flowey_lib_common::cfg_cargo_common_flags 0 + flowey.exe e 21 flowey_lib_common::download_cargo_nextest 0 + flowey.exe e 21 flowey_lib_common::download_cargo_nextest 1 + flowey.exe e 21 flowey_lib_common::download_cargo_nextest 2 + flowey.exe e 21 flowey_lib_common::download_cargo_nextest 3 + shell: bash + - name: Pre-processing cache vars + run: |- + flowey.exe e 21 flowey_lib_common::cache 4 + flowey.exe v 21 'flowey_lib_common::cache:10:flowey_lib_common/src/cache.rs:407:72' --is-raw-string write-to-env github floweyvar6 + flowey.exe v 21 'flowey_lib_common::cache:9:flowey_lib_common/src/cache.rs:406:72' --is-raw-string write-to-env github floweyvar7 + shell: bash + - id: flowey_lib_common__cache__5 + uses: actions/cache@v4 + with: + key: ${{ env.floweyvar6 }} + path: ${{ env.floweyvar7 }} + name: 'Restore cache: cargo-nextest' + - name: downloading cargo-nextest + run: |- + flowey.exe v 21 'flowey_lib_common::cache:12:flowey_lib_common/src/cache.rs:462:70' --is-raw-string update --env-source steps.flowey_lib_common__cache__5.outputs.cache-hit <> $GITHUB_PATH + - run: echo "${{ runner.temp }}/used_artifacts/_internal-flowey-bootstrap-x86_64-linux-uid-9" >> $GITHUB_PATH shell: bash name: πŸŒΌπŸ“¦ Add flowey to PATH - name: πŸŒΌπŸ›« Initialize job @@ -3248,40 +4057,40 @@ jobs: AgentTempDirNormal=$(echo "$AgentTempDirNormal" | sed -e 's|\\|\/|g' -e 's|^\([A-Za-z]\)\:/\(.*\)|/\L\1\E/\2|') echo "AgentTempDirNormal=$AgentTempDirNormal" >> $GITHUB_ENV - chmod +x $AgentTempDirNormal/used_artifacts/_internal-flowey-bootstrap-x86_64-linux-uid-8/flowey + chmod +x $AgentTempDirNormal/used_artifacts/_internal-flowey-bootstrap-x86_64-linux-uid-9/flowey - echo '"debug"' | flowey v 20 'FLOWEY_LOG' update - echo "${{ runner.temp }}/work" | flowey v 20 '_internal_WORKING_DIR' --is-raw-string update + echo '"debug"' | flowey v 22 'FLOWEY_LOG' update + echo "${{ runner.temp }}/work" | flowey v 22 '_internal_WORKING_DIR' --is-raw-string update - cat <<'EOF' | flowey v 20 'verbose' update + cat <<'EOF' | flowey v 22 'verbose' update ${{ inputs.verbose != '' && inputs.verbose || 'false' }} EOF - echo "$AgentTempDirNormal/used_artifacts/x64-guest_test_uefi" | flowey v 20 'artifact_use_from_x64-guest_test_uefi' --is-raw-string update - echo "$AgentTempDirNormal/used_artifacts/x64-linux-musl-pipette" | flowey v 20 'artifact_use_from_x64-linux-musl-pipette' --is-raw-string update - echo "$AgentTempDirNormal/used_artifacts/x64-linux-musl-tmk_vmm" | flowey v 20 'artifact_use_from_x64-linux-musl-tmk_vmm' --is-raw-string update - echo "$AgentTempDirNormal/used_artifacts/x64-linux-openvmm" | flowey v 20 'artifact_use_from_x64-linux-openvmm' --is-raw-string update - echo "$AgentTempDirNormal/used_artifacts/x64-linux-vmm-tests-archive" | flowey v 20 'artifact_use_from_x64-linux-vmm-tests-archive' --is-raw-string update - echo "$AgentTempDirNormal/used_artifacts/x64-tmks" | flowey v 20 'artifact_use_from_x64-tmks' --is-raw-string update - echo "$AgentTempDirNormal/used_artifacts/x64-windows-pipette" | flowey v 20 'artifact_use_from_x64-windows-pipette' --is-raw-string update + echo "$AgentTempDirNormal/used_artifacts/x64-guest_test_uefi" | flowey v 22 'artifact_use_from_x64-guest_test_uefi' --is-raw-string update + echo "$AgentTempDirNormal/used_artifacts/x64-linux-musl-pipette" | flowey v 22 'artifact_use_from_x64-linux-musl-pipette' --is-raw-string update + echo "$AgentTempDirNormal/used_artifacts/x64-linux-musl-tmk_vmm" | flowey v 22 'artifact_use_from_x64-linux-musl-tmk_vmm' --is-raw-string update + echo "$AgentTempDirNormal/used_artifacts/x64-linux-openvmm" | flowey v 22 'artifact_use_from_x64-linux-openvmm' --is-raw-string update + echo "$AgentTempDirNormal/used_artifacts/x64-linux-vmm-tests-archive" | flowey v 22 'artifact_use_from_x64-linux-vmm-tests-archive' --is-raw-string update + echo "$AgentTempDirNormal/used_artifacts/x64-tmks" | flowey v 22 'artifact_use_from_x64-tmks' --is-raw-string update + echo "$AgentTempDirNormal/used_artifacts/x64-windows-pipette" | flowey v 22 'artifact_use_from_x64-windows-pipette' --is-raw-string update shell: bash - name: creating new test content dir run: |- - flowey e 20 flowey_core::pipeline::artifact::resolve 3 - flowey e 20 flowey_core::pipeline::artifact::resolve 6 - flowey e 20 flowey_core::pipeline::artifact::resolve 1 - flowey e 20 flowey_core::pipeline::artifact::resolve 0 - flowey e 20 flowey_core::pipeline::artifact::resolve 2 - flowey e 20 flowey_core::pipeline::artifact::resolve 5 - flowey e 20 flowey_lib_hvlite::_jobs::consume_and_test_nextest_vmm_tests_archive 0 + flowey e 22 flowey_core::pipeline::artifact::resolve 3 + flowey e 22 flowey_core::pipeline::artifact::resolve 6 + flowey e 22 flowey_core::pipeline::artifact::resolve 1 + flowey e 22 flowey_core::pipeline::artifact::resolve 0 + flowey e 22 flowey_core::pipeline::artifact::resolve 2 + flowey e 22 flowey_core::pipeline::artifact::resolve 5 + flowey e 22 flowey_lib_hvlite::_jobs::consume_and_test_nextest_vmm_tests_archive 0 shell: bash - name: create azcopy cache dir - run: flowey e 20 flowey_lib_common::download_azcopy 0 + run: flowey e 22 flowey_lib_common::download_azcopy 0 shell: bash - name: Pre-processing cache vars run: |- - flowey e 20 flowey_lib_common::cache 0 - flowey v 20 'flowey_lib_common::cache:2:flowey_lib_common/src/cache.rs:407:72' --is-raw-string write-to-env github floweyvar4 - flowey v 20 'flowey_lib_common::cache:1:flowey_lib_common/src/cache.rs:406:72' --is-raw-string write-to-env github floweyvar5 + flowey e 22 flowey_lib_common::cache 0 + flowey v 22 'flowey_lib_common::cache:2:flowey_lib_common/src/cache.rs:407:72' --is-raw-string write-to-env github floweyvar4 + flowey v 22 'flowey_lib_common::cache:1:flowey_lib_common/src/cache.rs:406:72' --is-raw-string write-to-env github floweyvar5 shell: bash - id: flowey_lib_common__cache__1 uses: actions/cache@v4 @@ -3291,34 +4100,34 @@ jobs: name: 'Restore cache: azcopy' - name: checking if packages need to be installed run: |- - flowey v 20 'flowey_lib_common::cache:4:flowey_lib_common/src/cache.rs:462:70' --is-raw-string update --env-source steps.flowey_lib_common__cache__1.outputs.cache-hit <> $GITHUB_PATH @@ -411,7 +411,7 @@ jobs: ${{ runner.temp }} EOF ) - flowey.exe pipeline github --runtime $ESCAPED_AGENT_TEMPDIR\\bootstrapped-flowey\\pipeline.yaml --out .github/workflows/openvmm-pr-release.yaml ci checkin-gates --config=pr-release + flowey pipeline github --runtime $ESCAPED_AGENT_TEMPDIR/bootstrapped-flowey/pipeline.yaml --out .github/workflows/openvmm-pr-release.yaml ci checkin-gates --config=pr-release shell: bash - name: πŸŒΌπŸ›« Initialize job run: | @@ -419,179 +419,368 @@ jobs: AgentTempDirNormal=$(echo "$AgentTempDirNormal" | sed -e 's|\\|\/|g' -e 's|^\([A-Za-z]\)\:/\(.*\)|/\L\1\E/\2|') echo "AgentTempDirNormal=$AgentTempDirNormal" >> $GITHUB_ENV - chmod +x $AgentTempDirNormal/bootstrapped-flowey/flowey.exe + chmod +x $AgentTempDirNormal/bootstrapped-flowey/flowey - echo '"debug"' | flowey.exe v 10 'FLOWEY_LOG' update - echo "${{ runner.temp }}/work" | flowey.exe v 10 '_internal_WORKING_DIR' --is-raw-string update + echo '"debug"' | flowey v 10 'FLOWEY_LOG' update + echo "${{ runner.temp }}/work" | flowey v 10 '_internal_WORKING_DIR' --is-raw-string update - cat <<'EOF' | flowey.exe v 10 'verbose' update + cat <<'EOF' | flowey v 10 'verbose' update ${{ inputs.verbose != '' && inputs.verbose || 'false' }} EOF + mkdir -p "$AgentTempDirNormal/publish_artifacts/x64-linux-musl-pipette" + echo "$AgentTempDirNormal/publish_artifacts/x64-linux-musl-pipette" | flowey v 10 'artifact_publish_from_x64-linux-musl-pipette' --is-raw-string update + mkdir -p "$AgentTempDirNormal/publish_artifacts/x64-linux-musl-tmk_vmm" + echo "$AgentTempDirNormal/publish_artifacts/x64-linux-musl-tmk_vmm" | flowey v 10 'artifact_publish_from_x64-linux-musl-tmk_vmm' --is-raw-string update + mkdir -p "$AgentTempDirNormal/publish_artifacts/x64-openhcl-igvm" + echo "$AgentTempDirNormal/publish_artifacts/x64-openhcl-igvm" | flowey v 10 'artifact_publish_from_x64-openhcl-igvm' --is-raw-string update + mkdir -p "$AgentTempDirNormal/publish_artifacts/x64-openhcl-igvm-extras" + echo "$AgentTempDirNormal/publish_artifacts/x64-openhcl-igvm-extras" | flowey v 10 'artifact_publish_from_x64-openhcl-igvm-extras' --is-raw-string update + shell: bash + - name: checking if packages need to be installed + run: flowey e 10 flowey_lib_common::install_dist_pkg 0 + shell: bash + - name: installing packages + run: flowey e 10 flowey_lib_common::install_dist_pkg 1 + shell: bash + - name: create gh-release-download cache dir + run: flowey e 10 flowey_lib_common::download_gh_release 0 + shell: bash + - name: Pre-processing cache vars + run: |- + flowey e 10 flowey_lib_common::cache 0 + flowey v 10 'flowey_lib_common::cache:2:flowey_lib_common/src/cache.rs:407:72' --is-raw-string write-to-env github floweyvar1 + flowey v 10 'flowey_lib_common::cache:1:flowey_lib_common/src/cache.rs:406:72' --is-raw-string write-to-env github floweyvar2 + shell: bash + - id: flowey_lib_common__cache__1 + uses: actions/cache@v4 + with: + key: ${{ env.floweyvar1 }} + path: ${{ env.floweyvar2 }} + name: 'Restore cache: gh-release-download' + - name: download artifacts from github releases + run: |- + flowey v 10 'flowey_lib_common::cache:4:flowey_lib_common/src/cache.rs:462:70' --is-raw-string update --env-source steps.flowey_lib_common__cache__1.outputs.cache-hit <> $GITHUB_PATH @@ -927,7 +1196,7 @@ jobs: ${{ runner.temp }} EOF ) - flowey pipeline github --runtime $ESCAPED_AGENT_TEMPDIR/bootstrapped-flowey/pipeline.yaml --out .github/workflows/openvmm-pr-release.yaml ci checkin-gates --config=pr-release + flowey.exe pipeline github --runtime $ESCAPED_AGENT_TEMPDIR\\bootstrapped-flowey\\pipeline.yaml --out .github/workflows/openvmm-pr-release.yaml ci checkin-gates --config=pr-release shell: bash - name: πŸŒΌπŸ›« Initialize job run: | @@ -935,31 +1204,31 @@ jobs: AgentTempDirNormal=$(echo "$AgentTempDirNormal" | sed -e 's|\\|\/|g' -e 's|^\([A-Za-z]\)\:/\(.*\)|/\L\1\E/\2|') echo "AgentTempDirNormal=$AgentTempDirNormal" >> $GITHUB_ENV - chmod +x $AgentTempDirNormal/bootstrapped-flowey/flowey + chmod +x $AgentTempDirNormal/bootstrapped-flowey/flowey.exe - echo '"debug"' | flowey v 12 'FLOWEY_LOG' update - echo "${{ runner.temp }}/work" | flowey v 12 '_internal_WORKING_DIR' --is-raw-string update + echo '"debug"' | flowey.exe v 12 'FLOWEY_LOG' update + echo "${{ runner.temp }}/work" | flowey.exe v 12 '_internal_WORKING_DIR' --is-raw-string update - cat <<'EOF' | flowey v 12 'verbose' update + cat <<'EOF' | flowey.exe v 12 'verbose' update ${{ inputs.verbose != '' && inputs.verbose || 'false' }} EOF shell: bash - name: add default cargo home to path - run: flowey e 12 flowey_lib_common::install_rust 0 + run: flowey.exe e 12 flowey_lib_common::install_rust 0 shell: bash - name: install Rust - run: flowey e 12 flowey_lib_common::install_rust 1 + run: flowey.exe e 12 flowey_lib_common::install_rust 1 shell: bash - name: detect active toolchain run: |- - flowey e 12 flowey_lib_common::install_rust 2 - flowey e 12 flowey_lib_common::cfg_cargo_common_flags 0 + flowey.exe e 12 flowey_lib_common::install_rust 2 + flowey.exe e 12 flowey_lib_common::cfg_cargo_common_flags 0 shell: bash - name: check if openvmm needs to be cloned run: |- - flowey e 12 flowey_lib_common::git_checkout 0 - flowey v 12 'flowey_lib_common::git_checkout:0:flowey_lib_common/src/git_checkout.rs:469:80' --is-raw-string --condvar flowey_lib_common::git_checkout:1:flowey_lib_common/src/git_checkout.rs:470:46 write-to-env github floweyvar6 - flowey v 12 'flowey_lib_common::git_checkout:1:flowey_lib_common/src/git_checkout.rs:470:46' write-to-env github FLOWEY_CONDITION + flowey.exe e 12 flowey_lib_common::git_checkout 0 + flowey.exe v 12 'flowey_lib_common::git_checkout:0:flowey_lib_common/src/git_checkout.rs:469:80' --is-raw-string --condvar flowey_lib_common::git_checkout:1:flowey_lib_common/src/git_checkout.rs:470:46 write-to-env github floweyvar6 + flowey.exe v 12 'flowey_lib_common::git_checkout:1:flowey_lib_common/src/git_checkout.rs:470:46' write-to-env github FLOWEY_CONDITION shell: bash - id: flowey_lib_common__git_checkout__1 uses: actions/checkout@v4 @@ -971,27 +1240,23 @@ jobs: if: ${{ fromJSON(env.FLOWEY_CONDITION) }} - name: report cloned repo directories run: |- - flowey v 12 'flowey_lib_common::git_checkout:4:flowey_core/src/node/github_context.rs:55:41' --is-raw-string update --env-source github.workspace <> $GITHUB_PATH @@ -1194,7 +1436,7 @@ jobs: ${{ runner.temp }} EOF ) - flowey.exe pipeline github --runtime $ESCAPED_AGENT_TEMPDIR\\bootstrapped-flowey\\pipeline.yaml --out .github/workflows/openvmm-pr-release.yaml ci checkin-gates --config=pr-release + flowey pipeline github --runtime $ESCAPED_AGENT_TEMPDIR/bootstrapped-flowey/pipeline.yaml --out .github/workflows/openvmm-pr-release.yaml ci checkin-gates --config=pr-release shell: bash - name: πŸŒΌπŸ›« Initialize job run: | @@ -1202,31 +1444,31 @@ jobs: AgentTempDirNormal=$(echo "$AgentTempDirNormal" | sed -e 's|\\|\/|g' -e 's|^\([A-Za-z]\)\:/\(.*\)|/\L\1\E/\2|') echo "AgentTempDirNormal=$AgentTempDirNormal" >> $GITHUB_ENV - chmod +x $AgentTempDirNormal/bootstrapped-flowey/flowey.exe + chmod +x $AgentTempDirNormal/bootstrapped-flowey/flowey - echo '"debug"' | flowey.exe v 13 'FLOWEY_LOG' update - echo "${{ runner.temp }}/work" | flowey.exe v 13 '_internal_WORKING_DIR' --is-raw-string update + echo '"debug"' | flowey v 13 'FLOWEY_LOG' update + echo "${{ runner.temp }}/work" | flowey v 13 '_internal_WORKING_DIR' --is-raw-string update - cat <<'EOF' | flowey.exe v 13 'verbose' update + cat <<'EOF' | flowey v 13 'verbose' update ${{ inputs.verbose != '' && inputs.verbose || 'false' }} EOF shell: bash - name: add default cargo home to path - run: flowey.exe e 13 flowey_lib_common::install_rust 0 + run: flowey e 13 flowey_lib_common::install_rust 0 shell: bash - name: install Rust - run: flowey.exe e 13 flowey_lib_common::install_rust 1 + run: flowey e 13 flowey_lib_common::install_rust 1 shell: bash - name: detect active toolchain run: |- - flowey.exe e 13 flowey_lib_common::install_rust 2 - flowey.exe e 13 flowey_lib_common::cfg_cargo_common_flags 0 + flowey e 13 flowey_lib_common::install_rust 2 + flowey e 13 flowey_lib_common::cfg_cargo_common_flags 0 shell: bash - name: check if openvmm needs to be cloned run: |- - flowey.exe e 13 flowey_lib_common::git_checkout 0 - flowey.exe v 13 'flowey_lib_common::git_checkout:0:flowey_lib_common/src/git_checkout.rs:469:80' --is-raw-string --condvar flowey_lib_common::git_checkout:1:flowey_lib_common/src/git_checkout.rs:470:46 write-to-env github floweyvar6 - flowey.exe v 13 'flowey_lib_common::git_checkout:1:flowey_lib_common/src/git_checkout.rs:470:46' write-to-env github FLOWEY_CONDITION + flowey e 13 flowey_lib_common::git_checkout 0 + flowey v 13 'flowey_lib_common::git_checkout:0:flowey_lib_common/src/git_checkout.rs:469:80' --is-raw-string --condvar flowey_lib_common::git_checkout:1:flowey_lib_common/src/git_checkout.rs:470:46 write-to-env github floweyvar6 + flowey v 13 'flowey_lib_common::git_checkout:1:flowey_lib_common/src/git_checkout.rs:470:46' write-to-env github FLOWEY_CONDITION shell: bash - id: flowey_lib_common__git_checkout__1 uses: actions/checkout@v4 @@ -1238,23 +1480,23 @@ jobs: if: ${{ fromJSON(env.FLOWEY_CONDITION) }} - name: report cloned repo directories run: |- - flowey.exe v 13 'flowey_lib_common::git_checkout:4:flowey_core/src/node/github_context.rs:55:41' --is-raw-string update --env-source github.workspace <> $GITHUB_PATH @@ -1462,12 +1740,6 @@ jobs: flowey e 14 flowey_lib_common::install_rust 2 flowey e 14 flowey_lib_common::cfg_cargo_common_flags 0 shell: bash - - name: checking if packages need to be installed - run: flowey e 14 flowey_lib_common::install_dist_pkg 0 - shell: bash - - name: installing packages - run: flowey e 14 flowey_lib_common::install_dist_pkg 1 - shell: bash - name: check if openvmm needs to be cloned run: |- flowey e 14 flowey_lib_common::git_checkout 0 @@ -1489,9 +1761,13 @@ jobs: EOF flowey e 14 flowey_lib_common::git_checkout 3 flowey e 14 flowey_lib_hvlite::git_checkout_openvmm_repo 0 + flowey e 14 flowey_lib_hvlite::cfg_openvmm_magicpath 0 shell: bash - - name: set '-Dwarnings' in .cargo/config.toml - run: flowey e 14 flowey_lib_hvlite::init_openvmm_cargo_config_deny_warnings 0 + - name: checking if packages need to be installed + run: flowey e 14 flowey_lib_common::install_dist_pkg 0 + shell: bash + - name: installing packages + run: flowey e 14 flowey_lib_common::install_dist_pkg 1 shell: bash - name: create gh-release-download cache dir run: flowey e 14 flowey_lib_common::download_gh_release 0 @@ -1516,16 +1792,23 @@ jobs: flowey e 14 flowey_lib_common::cache 6 flowey e 14 flowey_lib_common::download_gh_release 1 shell: bash + - name: unpack openvmm-deps archive + run: flowey e 14 flowey_lib_hvlite::resolve_openvmm_deps 0 + shell: bash + - name: extract X64 sysroot.tar.gz + run: flowey e 14 flowey_lib_hvlite::init_openvmm_magicpath_openhcl_sysroot 0 + shell: bash + - name: set '-Dwarnings' in .cargo/config.toml + run: flowey e 14 flowey_lib_hvlite::init_openvmm_cargo_config_deny_warnings 0 + shell: bash - name: unpack protoc - run: |- - flowey e 14 flowey_lib_common::download_protoc 0 - flowey e 14 flowey_lib_hvlite::cfg_openvmm_magicpath 0 + run: flowey e 14 flowey_lib_common::download_protoc 0 shell: bash - name: symlink protoc run: |- flowey e 14 flowey_lib_hvlite::init_openvmm_magicpath_protoc 0 - flowey e 14 flowey_lib_hvlite::init_cross_build 0 flowey e 14 flowey_lib_hvlite::init_cross_build 2 + flowey e 14 flowey_lib_hvlite::init_cross_build 0 shell: bash - name: cargo build xtask run: |- @@ -1544,6 +1827,12 @@ jobs: - name: cargo clippy run: flowey e 14 flowey_lib_common::run_cargo_clippy 0 shell: bash + - name: cargo clippy + run: flowey e 14 flowey_lib_common::run_cargo_clippy 2 + shell: bash + - name: cargo clippy + run: flowey e 14 flowey_lib_common::run_cargo_clippy 1 + shell: bash - name: create cargo-nextest cache dir run: |- flowey e 14 flowey_lib_common::download_cargo_nextest 0 @@ -1577,7 +1866,7 @@ jobs: - name: installing cargo-nextest run: |- flowey e 14 flowey_lib_common::install_cargo_nextest 0 - flowey e 14 flowey_lib_hvlite::init_cross_build 3 + flowey e 14 flowey_lib_hvlite::init_cross_build 1 shell: bash - name: cargo build xtask run: |- @@ -1593,7 +1882,7 @@ jobs: - name: determine unit test exclusions run: |- flowey e 14 flowey_lib_hvlite::build_nextest_unit_tests 0 - flowey e 14 flowey_lib_hvlite::init_cross_build 1 + flowey e 14 flowey_lib_hvlite::init_cross_build 3 flowey e 14 flowey_lib_hvlite::run_cargo_nextest_run 0 shell: bash - name: generate nextest command @@ -1613,14 +1902,14 @@ jobs: - id: flowey_lib_common__publish_test_results__3 uses: actions/upload-artifact@v4 with: - name: aarch64-linux-unit-tests-junit-xml + name: x64-linux-musl-unit-tests-junit-xml path: ${{ env.floweyvar1 }} - name: 'publish test results: aarch64-linux-unit-tests (JUnit XML)' + name: 'publish test results: x64-linux-musl-unit-tests (JUnit XML)' if: ${{ fromJSON(env.FLOWEY_CONDITION) }} - name: report test results to overall pipeline status run: flowey e 14 flowey_lib_hvlite::_jobs::build_and_run_nextest_unit_tests 1 shell: bash - - name: run doctests for aarch64-unknown-linux-gnu + - name: run doctests for x86_64-unknown-linux-musl run: flowey e 14 flowey_lib_hvlite::_jobs::build_and_run_doc_tests 0 shell: bash - name: 'validate cache entry: cargo-nextest' @@ -1630,11 +1919,11 @@ jobs: run: flowey e 14 flowey_lib_common::cache 7 shell: bash job15: - name: clippy [aarch64-linux-musl, misc nostd], unit tests [aarch64-linux-musl] + name: clippy [aarch64-windows], unit tests [aarch64-windows] runs-on: - self-hosted - 1ES.Pool=OpenVMM-GitHub-ARM64-Pool-WestUS2 - - 1ES.ImageOverride=OpenVMM-CI-Ubuntu-ARM64 + - 1ES.ImageOverride=OpenVMM-CI-Windows-ARM64 permissions: contents: read id-token: write @@ -1673,11 +1962,11 @@ jobs: - name: Build flowey run: | set -x - CARGO_INCREMENTAL=0 cargo build -p flowey_hvlite --target aarch64-unknown-linux-gnu --profile flowey-ci + CARGO_INCREMENTAL=0 cargo build -p flowey_hvlite --target aarch64-pc-windows-msvc --profile flowey-ci OutDirNormal=$(echo "${{ runner.temp }}/bootstrapped-flowey" | sed -e 's|\\|\/|g' -e 's|^\([A-Za-z]\)\:/\(.*\)|/\L\1\E/\2|') mkdir -p "$OutDirNormal" mv ./.github/workflows/openvmm-pr-release.yaml "$OutDirNormal/pipeline.yaml" - mv target/aarch64-unknown-linux-gnu/flowey-ci/flowey_hvlite "$OutDirNormal/flowey" + mv target/aarch64-pc-windows-msvc/flowey-ci/flowey_hvlite.exe "$OutDirNormal/flowey.exe" working-directory: flowey_bootstrap shell: bash - run: echo "${{ runner.temp }}/bootstrapped-flowey" >> $GITHUB_PATH @@ -1690,7 +1979,7 @@ jobs: ${{ runner.temp }} EOF ) - flowey pipeline github --runtime $ESCAPED_AGENT_TEMPDIR/bootstrapped-flowey/pipeline.yaml --out .github/workflows/openvmm-pr-release.yaml ci checkin-gates --config=pr-release + flowey.exe pipeline github --runtime $ESCAPED_AGENT_TEMPDIR\\bootstrapped-flowey\\pipeline.yaml --out .github/workflows/openvmm-pr-release.yaml ci checkin-gates --config=pr-release shell: bash - name: πŸŒΌπŸ›« Initialize job run: | @@ -1698,31 +1987,31 @@ jobs: AgentTempDirNormal=$(echo "$AgentTempDirNormal" | sed -e 's|\\|\/|g' -e 's|^\([A-Za-z]\)\:/\(.*\)|/\L\1\E/\2|') echo "AgentTempDirNormal=$AgentTempDirNormal" >> $GITHUB_ENV - chmod +x $AgentTempDirNormal/bootstrapped-flowey/flowey + chmod +x $AgentTempDirNormal/bootstrapped-flowey/flowey.exe - echo '"debug"' | flowey v 15 'FLOWEY_LOG' update - echo "${{ runner.temp }}/work" | flowey v 15 '_internal_WORKING_DIR' --is-raw-string update + echo '"debug"' | flowey.exe v 15 'FLOWEY_LOG' update + echo "${{ runner.temp }}/work" | flowey.exe v 15 '_internal_WORKING_DIR' --is-raw-string update - cat <<'EOF' | flowey v 15 'verbose' update + cat <<'EOF' | flowey.exe v 15 'verbose' update ${{ inputs.verbose != '' && inputs.verbose || 'false' }} EOF shell: bash - name: add default cargo home to path - run: flowey e 15 flowey_lib_common::install_rust 0 + run: flowey.exe e 15 flowey_lib_common::install_rust 0 shell: bash - name: install Rust - run: flowey e 15 flowey_lib_common::install_rust 1 + run: flowey.exe e 15 flowey_lib_common::install_rust 1 shell: bash - name: detect active toolchain run: |- - flowey e 15 flowey_lib_common::install_rust 2 - flowey e 15 flowey_lib_common::cfg_cargo_common_flags 0 + flowey.exe e 15 flowey_lib_common::install_rust 2 + flowey.exe e 15 flowey_lib_common::cfg_cargo_common_flags 0 shell: bash - name: check if openvmm needs to be cloned run: |- - flowey e 15 flowey_lib_common::git_checkout 0 - flowey v 15 'flowey_lib_common::git_checkout:0:flowey_lib_common/src/git_checkout.rs:469:80' --is-raw-string --condvar flowey_lib_common::git_checkout:1:flowey_lib_common/src/git_checkout.rs:470:46 write-to-env github floweyvar6 - flowey v 15 'flowey_lib_common::git_checkout:1:flowey_lib_common/src/git_checkout.rs:470:46' write-to-env github FLOWEY_CONDITION + flowey.exe e 15 flowey_lib_common::git_checkout 0 + flowey.exe v 15 'flowey_lib_common::git_checkout:0:flowey_lib_common/src/git_checkout.rs:469:80' --is-raw-string --condvar flowey_lib_common::git_checkout:1:flowey_lib_common/src/git_checkout.rs:470:46 write-to-env github floweyvar6 + flowey.exe v 15 'flowey_lib_common::git_checkout:1:flowey_lib_common/src/git_checkout.rs:470:46' write-to-env github FLOWEY_CONDITION shell: bash - id: flowey_lib_common__git_checkout__1 uses: actions/checkout@v4 @@ -1734,27 +2023,23 @@ jobs: if: ${{ fromJSON(env.FLOWEY_CONDITION) }} - name: report cloned repo directories run: |- - flowey v 15 'flowey_lib_common::git_checkout:4:flowey_core/src/node/github_context.rs:55:41' --is-raw-string update --env-source github.workspace <> "$GITHUB_PATH" + rustup show + if: runner.os == 'Linux' + name: rustup (Linux) + shell: bash + - run: | + set -x + curl --fail -sSfLo rustup-init.exe https://win.rustup.rs/x86_64 --output rustup-init + ./rustup-init.exe -y --default-toolchain=1.91.1 + echo "$USERPROFILE\\.cargo\\bin" >> $GITHUB_PATH + if: runner.os == 'Windows' && runner.arch == 'X64' + name: rustup (Windows X64) + shell: bash + - run: | + set -x + curl --fail -sSfLo rustup-init.exe https://win.rustup.rs/aarch64 --output rustup-init + ./rustup-init.exe -y --default-toolchain=1.91.1 + echo "$USERPROFILE\\.cargo\\bin" >> $GITHUB_PATH + if: runner.os == 'Windows' && runner.arch == 'ARM64' + name: rustup (Windows ARM64) + shell: bash + - uses: actions/checkout@v4 with: - pattern: '{_internal-flowey-bootstrap-x86_64-windows-uid-12,x64-guest_test_uefi,x64-linux-musl-pipette,x64-linux-musl-tmk_vmm,x64-linux-tpm_guest_tests,x64-openhcl-igvm,x64-tmks,x64-windows-openvmm,x64-windows-pipette,x64-windows-prep_steps,x64-windows-tmk_vmm,x64-windows-tpm_guest_tests,x64-windows-vmgstool,x64-windows-vmm-tests-archive}' - path: ${{ runner.temp }}/used_artifacts/ - - run: echo "${{ runner.temp }}/used_artifacts/_internal-flowey-bootstrap-x86_64-windows-uid-12" >> $GITHUB_PATH + path: flowey_bootstrap + - name: Build flowey + run: | + set -x + CARGO_INCREMENTAL=0 cargo build -p flowey_hvlite --target aarch64-unknown-linux-gnu --profile flowey-ci + OutDirNormal=$(echo "${{ runner.temp }}/bootstrapped-flowey" | sed -e 's|\\|\/|g' -e 's|^\([A-Za-z]\)\:/\(.*\)|/\L\1\E/\2|') + mkdir -p "$OutDirNormal" + mv ./.github/workflows/openvmm-pr-release.yaml "$OutDirNormal/pipeline.yaml" + mv target/aarch64-unknown-linux-gnu/flowey-ci/flowey_hvlite "$OutDirNormal/flowey" + working-directory: flowey_bootstrap + shell: bash + - run: echo "${{ runner.temp }}/bootstrapped-flowey" >> $GITHUB_PATH shell: bash name: πŸŒΌπŸ“¦ Add flowey to PATH + - name: πŸŒΌπŸ”Ž Self-check YAML + run: |- + ESCAPED_AGENT_TEMPDIR=$( + cat <<'EOF' | sed 's/\\/\\\\/g' + ${{ runner.temp }} + EOF + ) + flowey pipeline github --runtime $ESCAPED_AGENT_TEMPDIR/bootstrapped-flowey/pipeline.yaml --out .github/workflows/openvmm-pr-release.yaml ci checkin-gates --config=pr-release + shell: bash - name: πŸŒΌπŸ›« Initialize job run: | AgentTempDirNormal="${{ runner.temp }}" AgentTempDirNormal=$(echo "$AgentTempDirNormal" | sed -e 's|\\|\/|g' -e 's|^\([A-Za-z]\)\:/\(.*\)|/\L\1\E/\2|') echo "AgentTempDirNormal=$AgentTempDirNormal" >> $GITHUB_ENV - chmod +x $AgentTempDirNormal/used_artifacts/_internal-flowey-bootstrap-x86_64-windows-uid-12/flowey.exe + chmod +x $AgentTempDirNormal/bootstrapped-flowey/flowey - echo '"debug"' | flowey.exe v 16 'FLOWEY_LOG' update - echo "${{ runner.temp }}/work" | flowey.exe v 16 '_internal_WORKING_DIR' --is-raw-string update + echo '"debug"' | flowey v 16 'FLOWEY_LOG' update + echo "${{ runner.temp }}/work" | flowey v 16 '_internal_WORKING_DIR' --is-raw-string update - cat <<'EOF' | flowey.exe v 16 'verbose' update + cat <<'EOF' | flowey v 16 'verbose' update ${{ inputs.verbose != '' && inputs.verbose || 'false' }} EOF - echo "${{ runner.temp }}\\used_artifacts\\x64-guest_test_uefi" | flowey.exe v 16 'artifact_use_from_x64-guest_test_uefi' --is-raw-string update - echo "${{ runner.temp }}\\used_artifacts\\x64-linux-musl-pipette" | flowey.exe v 16 'artifact_use_from_x64-linux-musl-pipette' --is-raw-string update - echo "${{ runner.temp }}\\used_artifacts\\x64-linux-musl-tmk_vmm" | flowey.exe v 16 'artifact_use_from_x64-linux-musl-tmk_vmm' --is-raw-string update - echo "${{ runner.temp }}\\used_artifacts\\x64-linux-tpm_guest_tests" | flowey.exe v 16 'artifact_use_from_x64-linux-tpm_guest_tests' --is-raw-string update - echo "${{ runner.temp }}\\used_artifacts\\x64-openhcl-igvm" | flowey.exe v 16 'artifact_use_from_x64-openhcl-igvm' --is-raw-string update - echo "${{ runner.temp }}\\used_artifacts\\x64-tmks" | flowey.exe v 16 'artifact_use_from_x64-tmks' --is-raw-string update - echo "${{ runner.temp }}\\used_artifacts\\x64-windows-openvmm" | flowey.exe v 16 'artifact_use_from_x64-windows-openvmm' --is-raw-string update - echo "${{ runner.temp }}\\used_artifacts\\x64-windows-pipette" | flowey.exe v 16 'artifact_use_from_x64-windows-pipette' --is-raw-string update - echo "${{ runner.temp }}\\used_artifacts\\x64-windows-prep_steps" | flowey.exe v 16 'artifact_use_from_x64-windows-prep_steps' --is-raw-string update - echo "${{ runner.temp }}\\used_artifacts\\x64-windows-tmk_vmm" | flowey.exe v 16 'artifact_use_from_x64-windows-tmk_vmm' --is-raw-string update - echo "${{ runner.temp }}\\used_artifacts\\x64-windows-tpm_guest_tests" | flowey.exe v 16 'artifact_use_from_x64-windows-tpm_guest_tests' --is-raw-string update - echo "${{ runner.temp }}\\used_artifacts\\x64-windows-vmgstool" | flowey.exe v 16 'artifact_use_from_x64-windows-vmgstool' --is-raw-string update - echo "${{ runner.temp }}\\used_artifacts\\x64-windows-vmm-tests-archive" | flowey.exe v 16 'artifact_use_from_x64-windows-vmm-tests-archive' --is-raw-string update - shell: bash - - name: creating new test content dir - run: |- - flowey.exe e 16 flowey_core::pipeline::artifact::resolve 5 - flowey.exe e 16 flowey_core::pipeline::artifact::resolve 6 - flowey.exe e 16 flowey_core::pipeline::artifact::resolve 1 - flowey.exe e 16 flowey_core::pipeline::artifact::resolve 0 - flowey.exe e 16 flowey_core::pipeline::artifact::resolve 8 - flowey.exe e 16 flowey_core::pipeline::artifact::resolve 2 - flowey.exe e 16 flowey_core::pipeline::artifact::resolve 4 - flowey.exe e 16 flowey_core::pipeline::artifact::resolve 10 - flowey.exe e 16 flowey_core::pipeline::artifact::resolve 9 - flowey.exe e 16 flowey_core::pipeline::artifact::resolve 3 - flowey.exe e 16 flowey_lib_hvlite::_jobs::consume_and_test_nextest_vmm_tests_archive 0 - shell: bash - - name: resolve OpenHCL igvm artifact - run: flowey.exe e 16 flowey_lib_hvlite::artifact_openhcl_igvm_from_recipe::resolve 0 shell: bash - - name: create azcopy cache dir - run: flowey.exe e 16 flowey_lib_common::download_azcopy 0 + - name: add default cargo home to path + run: flowey e 16 flowey_lib_common::install_rust 0 shell: bash - - name: Pre-processing cache vars - run: |- - flowey.exe e 16 flowey_lib_common::cache 0 - flowey.exe v 16 'flowey_lib_common::cache:2:flowey_lib_common/src/cache.rs:407:72' --is-raw-string write-to-env github floweyvar4 - flowey.exe v 16 'flowey_lib_common::cache:1:flowey_lib_common/src/cache.rs:406:72' --is-raw-string write-to-env github floweyvar5 + - name: install Rust + run: flowey e 16 flowey_lib_common::install_rust 1 shell: bash - - id: flowey_lib_common__cache__1 - uses: actions/cache@v4 - with: - key: ${{ env.floweyvar4 }} - path: ${{ env.floweyvar5 }} - name: 'Restore cache: azcopy' - - name: installing azcopy + - name: detect active toolchain run: |- - flowey.exe v 16 'flowey_lib_common::cache:4:flowey_lib_common/src/cache.rs:462:70' --is-raw-string update --env-source steps.flowey_lib_common__cache__1.outputs.cache-hit <> "$GITHUB_PATH" + rustup show + if: runner.os == 'Linux' + name: rustup (Linux) + shell: bash + - run: | + set -x + curl --fail -sSfLo rustup-init.exe https://win.rustup.rs/x86_64 --output rustup-init + ./rustup-init.exe -y --default-toolchain=1.91.1 + echo "$USERPROFILE\\.cargo\\bin" >> $GITHUB_PATH + if: runner.os == 'Windows' && runner.arch == 'X64' + name: rustup (Windows X64) + shell: bash + - run: | + set -x + curl --fail -sSfLo rustup-init.exe https://win.rustup.rs/aarch64 --output rustup-init + ./rustup-init.exe -y --default-toolchain=1.91.1 + echo "$USERPROFILE\\.cargo\\bin" >> $GITHUB_PATH + if: runner.os == 'Windows' && runner.arch == 'ARM64' + name: rustup (Windows ARM64) + shell: bash + - uses: actions/checkout@v4 with: - pattern: '{_internal-flowey-bootstrap-x86_64-windows-uid-12,x64-guest_test_uefi,x64-linux-musl-pipette,x64-linux-musl-tmk_vmm,x64-linux-tpm_guest_tests,x64-openhcl-igvm,x64-tmks,x64-windows-openvmm,x64-windows-pipette,x64-windows-prep_steps,x64-windows-tmk_vmm,x64-windows-tpm_guest_tests,x64-windows-vmgstool,x64-windows-vmm-tests-archive}' - path: ${{ runner.temp }}/used_artifacts/ - - run: echo "${{ runner.temp }}/used_artifacts/_internal-flowey-bootstrap-x86_64-windows-uid-12" >> $GITHUB_PATH + path: flowey_bootstrap + - name: Build flowey + run: | + set -x + CARGO_INCREMENTAL=0 cargo build -p flowey_hvlite --target aarch64-unknown-linux-gnu --profile flowey-ci + OutDirNormal=$(echo "${{ runner.temp }}/bootstrapped-flowey" | sed -e 's|\\|\/|g' -e 's|^\([A-Za-z]\)\:/\(.*\)|/\L\1\E/\2|') + mkdir -p "$OutDirNormal" + mv ./.github/workflows/openvmm-pr-release.yaml "$OutDirNormal/pipeline.yaml" + mv target/aarch64-unknown-linux-gnu/flowey-ci/flowey_hvlite "$OutDirNormal/flowey" + working-directory: flowey_bootstrap + shell: bash + - run: echo "${{ runner.temp }}/bootstrapped-flowey" >> $GITHUB_PATH shell: bash name: πŸŒΌπŸ“¦ Add flowey to PATH + - name: πŸŒΌπŸ”Ž Self-check YAML + run: |- + ESCAPED_AGENT_TEMPDIR=$( + cat <<'EOF' | sed 's/\\/\\\\/g' + ${{ runner.temp }} + EOF + ) + flowey pipeline github --runtime $ESCAPED_AGENT_TEMPDIR/bootstrapped-flowey/pipeline.yaml --out .github/workflows/openvmm-pr-release.yaml ci checkin-gates --config=pr-release + shell: bash - name: πŸŒΌπŸ›« Initialize job run: | AgentTempDirNormal="${{ runner.temp }}" AgentTempDirNormal=$(echo "$AgentTempDirNormal" | sed -e 's|\\|\/|g' -e 's|^\([A-Za-z]\)\:/\(.*\)|/\L\1\E/\2|') echo "AgentTempDirNormal=$AgentTempDirNormal" >> $GITHUB_ENV - chmod +x $AgentTempDirNormal/used_artifacts/_internal-flowey-bootstrap-x86_64-windows-uid-12/flowey.exe + chmod +x $AgentTempDirNormal/bootstrapped-flowey/flowey - echo '"debug"' | flowey.exe v 17 'FLOWEY_LOG' update - echo "${{ runner.temp }}/work" | flowey.exe v 17 '_internal_WORKING_DIR' --is-raw-string update + echo '"debug"' | flowey v 17 'FLOWEY_LOG' update + echo "${{ runner.temp }}/work" | flowey v 17 '_internal_WORKING_DIR' --is-raw-string update - cat <<'EOF' | flowey.exe v 17 'verbose' update + cat <<'EOF' | flowey v 17 'verbose' update ${{ inputs.verbose != '' && inputs.verbose || 'false' }} EOF - echo "${{ runner.temp }}\\used_artifacts\\x64-guest_test_uefi" | flowey.exe v 17 'artifact_use_from_x64-guest_test_uefi' --is-raw-string update - echo "${{ runner.temp }}\\used_artifacts\\x64-linux-musl-pipette" | flowey.exe v 17 'artifact_use_from_x64-linux-musl-pipette' --is-raw-string update - echo "${{ runner.temp }}\\used_artifacts\\x64-linux-musl-tmk_vmm" | flowey.exe v 17 'artifact_use_from_x64-linux-musl-tmk_vmm' --is-raw-string update - echo "${{ runner.temp }}\\used_artifacts\\x64-linux-tpm_guest_tests" | flowey.exe v 17 'artifact_use_from_x64-linux-tpm_guest_tests' --is-raw-string update - echo "${{ runner.temp }}\\used_artifacts\\x64-openhcl-igvm" | flowey.exe v 17 'artifact_use_from_x64-openhcl-igvm' --is-raw-string update - echo "${{ runner.temp }}\\used_artifacts\\x64-tmks" | flowey.exe v 17 'artifact_use_from_x64-tmks' --is-raw-string update - echo "${{ runner.temp }}\\used_artifacts\\x64-windows-openvmm" | flowey.exe v 17 'artifact_use_from_x64-windows-openvmm' --is-raw-string update - echo "${{ runner.temp }}\\used_artifacts\\x64-windows-pipette" | flowey.exe v 17 'artifact_use_from_x64-windows-pipette' --is-raw-string update - echo "${{ runner.temp }}\\used_artifacts\\x64-windows-prep_steps" | flowey.exe v 17 'artifact_use_from_x64-windows-prep_steps' --is-raw-string update - echo "${{ runner.temp }}\\used_artifacts\\x64-windows-tmk_vmm" | flowey.exe v 17 'artifact_use_from_x64-windows-tmk_vmm' --is-raw-string update - echo "${{ runner.temp }}\\used_artifacts\\x64-windows-tpm_guest_tests" | flowey.exe v 17 'artifact_use_from_x64-windows-tpm_guest_tests' --is-raw-string update - echo "${{ runner.temp }}\\used_artifacts\\x64-windows-vmgstool" | flowey.exe v 17 'artifact_use_from_x64-windows-vmgstool' --is-raw-string update - echo "${{ runner.temp }}\\used_artifacts\\x64-windows-vmm-tests-archive" | flowey.exe v 17 'artifact_use_from_x64-windows-vmm-tests-archive' --is-raw-string update shell: bash - - name: creating new test content dir - run: |- - flowey.exe e 17 flowey_core::pipeline::artifact::resolve 5 - flowey.exe e 17 flowey_core::pipeline::artifact::resolve 6 - flowey.exe e 17 flowey_core::pipeline::artifact::resolve 1 - flowey.exe e 17 flowey_core::pipeline::artifact::resolve 0 - flowey.exe e 17 flowey_core::pipeline::artifact::resolve 8 - flowey.exe e 17 flowey_core::pipeline::artifact::resolve 2 - flowey.exe e 17 flowey_core::pipeline::artifact::resolve 4 - flowey.exe e 17 flowey_core::pipeline::artifact::resolve 10 - flowey.exe e 17 flowey_core::pipeline::artifact::resolve 9 - flowey.exe e 17 flowey_core::pipeline::artifact::resolve 3 - flowey.exe e 17 flowey_lib_hvlite::_jobs::consume_and_test_nextest_vmm_tests_archive 0 + - name: add default cargo home to path + run: flowey e 17 flowey_lib_common::install_rust 0 shell: bash - - name: resolve OpenHCL igvm artifact - run: flowey.exe e 17 flowey_lib_hvlite::artifact_openhcl_igvm_from_recipe::resolve 0 + - name: install Rust + run: flowey e 17 flowey_lib_common::install_rust 1 shell: bash - - name: create azcopy cache dir - run: flowey.exe e 17 flowey_lib_common::download_azcopy 0 + - name: detect active toolchain + run: |- + flowey e 17 flowey_lib_common::install_rust 2 + flowey e 17 flowey_lib_common::cfg_cargo_common_flags 0 shell: bash - - name: Pre-processing cache vars + - name: check if openvmm needs to be cloned run: |- - flowey.exe e 17 flowey_lib_common::cache 0 - flowey.exe v 17 'flowey_lib_common::cache:2:flowey_lib_common/src/cache.rs:407:72' --is-raw-string write-to-env github floweyvar4 - flowey.exe v 17 'flowey_lib_common::cache:1:flowey_lib_common/src/cache.rs:406:72' --is-raw-string write-to-env github floweyvar5 + flowey e 17 flowey_lib_common::git_checkout 0 + flowey v 17 'flowey_lib_common::git_checkout:0:flowey_lib_common/src/git_checkout.rs:469:80' --is-raw-string --condvar flowey_lib_common::git_checkout:1:flowey_lib_common/src/git_checkout.rs:470:46 write-to-env github floweyvar6 + flowey v 17 'flowey_lib_common::git_checkout:1:flowey_lib_common/src/git_checkout.rs:470:46' write-to-env github FLOWEY_CONDITION shell: bash - - id: flowey_lib_common__cache__1 - uses: actions/cache@v4 + - id: flowey_lib_common__git_checkout__1 + uses: actions/checkout@v4 with: - key: ${{ env.floweyvar4 }} - path: ${{ env.floweyvar5 }} - name: 'Restore cache: azcopy' - - name: installing azcopy + fetch-depth: '1' + path: repo0 + persist-credentials: ${{ env.floweyvar6 }} + name: checkout repo openvmm + if: ${{ fromJSON(env.FLOWEY_CONDITION) }} + - name: report cloned repo directories run: |- - flowey.exe v 17 'flowey_lib_common::cache:4:flowey_lib_common/src/cache.rs:462:70' --is-raw-string update --env-source steps.flowey_lib_common__cache__1.outputs.cache-hit <> $GITHUB_PATH + - run: echo "${{ runner.temp }}/used_artifacts/_internal-flowey-bootstrap-x86_64-windows-uid-14" >> $GITHUB_PATH shell: bash name: πŸŒΌπŸ“¦ Add flowey to PATH - name: πŸŒΌπŸ›« Initialize job @@ -2484,7 +2710,7 @@ jobs: AgentTempDirNormal=$(echo "$AgentTempDirNormal" | sed -e 's|\\|\/|g' -e 's|^\([A-Za-z]\)\:/\(.*\)|/\L\1\E/\2|') echo "AgentTempDirNormal=$AgentTempDirNormal" >> $GITHUB_ENV - chmod +x $AgentTempDirNormal/used_artifacts/_internal-flowey-bootstrap-x86_64-windows-uid-12/flowey.exe + chmod +x $AgentTempDirNormal/used_artifacts/_internal-flowey-bootstrap-x86_64-windows-uid-14/flowey.exe echo '"debug"' | flowey.exe v 18 'FLOWEY_LOG' update echo "${{ runner.temp }}/work" | flowey.exe v 18 '_internal_WORKING_DIR' --is-raw-string update @@ -2616,7 +2842,7 @@ jobs: flowey.exe e 18 flowey_lib_common::download_gh_release 1 shell: bash - name: unpack openvmm-deps archive - run: flowey.exe e 18 flowey_lib_hvlite::download_openvmm_deps 0 + run: flowey.exe e 18 flowey_lib_hvlite::resolve_openvmm_deps 0 shell: bash - name: unpack mu_msvm package (x64) run: flowey.exe e 18 flowey_lib_hvlite::download_uefi_mu_msvm 0 @@ -2696,9 +2922,9 @@ jobs: - id: flowey_lib_common__publish_test_results__6 uses: actions/upload-artifact@v4 with: - name: x64-windows-amd-vmm-tests-logs + name: x64-windows-intel-vmm-tests-logs path: ${{ env.floweyvar2 }} - name: 'publish test results: x64-windows-amd-vmm-tests (logs)' + name: 'publish test results: x64-windows-intel-vmm-tests (logs)' if: ${{ fromJSON(env.FLOWEY_CONDITION) }} - name: πŸ¦€ flowey rust steps run: |- @@ -2712,9 +2938,9 @@ jobs: - id: flowey_lib_common__publish_test_results__3 uses: actions/upload-artifact@v4 with: - name: x64-windows-amd-vmm-tests-junit-xml + name: x64-windows-intel-vmm-tests-junit-xml path: ${{ env.floweyvar1 }} - name: 'publish test results: x64-windows-amd-vmm-tests (JUnit XML)' + name: 'publish test results: x64-windows-intel-vmm-tests (JUnit XML)' if: ${{ fromJSON(env.FLOWEY_CONDITION) }} - name: report test results to overall pipeline status run: flowey.exe e 18 flowey_lib_hvlite::_jobs::consume_and_test_nextest_vmm_tests_archive 3 @@ -2732,28 +2958,28 @@ jobs: run: flowey.exe e 18 flowey_lib_common::cache 15 shell: bash job19: - name: run vmm-tests [x64-windows-amd-snp] + name: run vmm-tests [x64-windows-intel-tdx] runs-on: - self-hosted - Windows - X64 - - SNP + - TDX - Baremetal permissions: contents: read id-token: write needs: + - job10 - job5 - job7 - - job9 if: contains(github.event.pull_request.labels.*.name, 'release-ci-required') && github.event.pull_request.draft == false steps: - name: πŸŒΌπŸ“¦ Download artifacts uses: actions/download-artifact@v4 with: - pattern: '{_internal-flowey-bootstrap-x86_64-windows-uid-12,x64-guest_test_uefi,x64-linux-musl-pipette,x64-linux-musl-tmk_vmm,x64-linux-tpm_guest_tests,x64-openhcl-igvm,x64-tmks,x64-windows-openvmm,x64-windows-pipette,x64-windows-prep_steps,x64-windows-tmk_vmm,x64-windows-tpm_guest_tests,x64-windows-vmgstool,x64-windows-vmm-tests-archive}' + pattern: '{_internal-flowey-bootstrap-x86_64-windows-uid-14,x64-guest_test_uefi,x64-linux-musl-pipette,x64-linux-musl-tmk_vmm,x64-linux-tpm_guest_tests,x64-openhcl-igvm,x64-tmks,x64-windows-openvmm,x64-windows-pipette,x64-windows-prep_steps,x64-windows-tmk_vmm,x64-windows-tpm_guest_tests,x64-windows-vmgstool,x64-windows-vmm-tests-archive}' path: ${{ runner.temp }}/used_artifacts/ - - run: echo "${{ runner.temp }}/used_artifacts/_internal-flowey-bootstrap-x86_64-windows-uid-12" >> $GITHUB_PATH + - run: echo "${{ runner.temp }}/used_artifacts/_internal-flowey-bootstrap-x86_64-windows-uid-14" >> $GITHUB_PATH shell: bash name: πŸŒΌπŸ“¦ Add flowey to PATH - name: πŸŒΌπŸ›« Initialize job @@ -2762,53 +2988,548 @@ jobs: AgentTempDirNormal=$(echo "$AgentTempDirNormal" | sed -e 's|\\|\/|g' -e 's|^\([A-Za-z]\)\:/\(.*\)|/\L\1\E/\2|') echo "AgentTempDirNormal=$AgentTempDirNormal" >> $GITHUB_ENV - chmod +x $AgentTempDirNormal/used_artifacts/_internal-flowey-bootstrap-x86_64-windows-uid-12/flowey.exe + chmod +x $AgentTempDirNormal/used_artifacts/_internal-flowey-bootstrap-x86_64-windows-uid-14/flowey.exe echo '"debug"' | flowey.exe v 19 'FLOWEY_LOG' update echo "${{ runner.temp }}/work" | flowey.exe v 19 '_internal_WORKING_DIR' --is-raw-string update - cat <<'EOF' | flowey.exe v 19 'verbose' update + cat <<'EOF' | flowey.exe v 19 'verbose' update + ${{ inputs.verbose != '' && inputs.verbose || 'false' }} + EOF + echo "${{ runner.temp }}\\used_artifacts\\x64-guest_test_uefi" | flowey.exe v 19 'artifact_use_from_x64-guest_test_uefi' --is-raw-string update + echo "${{ runner.temp }}\\used_artifacts\\x64-linux-musl-pipette" | flowey.exe v 19 'artifact_use_from_x64-linux-musl-pipette' --is-raw-string update + echo "${{ runner.temp }}\\used_artifacts\\x64-linux-musl-tmk_vmm" | flowey.exe v 19 'artifact_use_from_x64-linux-musl-tmk_vmm' --is-raw-string update + echo "${{ runner.temp }}\\used_artifacts\\x64-linux-tpm_guest_tests" | flowey.exe v 19 'artifact_use_from_x64-linux-tpm_guest_tests' --is-raw-string update + echo "${{ runner.temp }}\\used_artifacts\\x64-openhcl-igvm" | flowey.exe v 19 'artifact_use_from_x64-openhcl-igvm' --is-raw-string update + echo "${{ runner.temp }}\\used_artifacts\\x64-tmks" | flowey.exe v 19 'artifact_use_from_x64-tmks' --is-raw-string update + echo "${{ runner.temp }}\\used_artifacts\\x64-windows-openvmm" | flowey.exe v 19 'artifact_use_from_x64-windows-openvmm' --is-raw-string update + echo "${{ runner.temp }}\\used_artifacts\\x64-windows-pipette" | flowey.exe v 19 'artifact_use_from_x64-windows-pipette' --is-raw-string update + echo "${{ runner.temp }}\\used_artifacts\\x64-windows-prep_steps" | flowey.exe v 19 'artifact_use_from_x64-windows-prep_steps' --is-raw-string update + echo "${{ runner.temp }}\\used_artifacts\\x64-windows-tmk_vmm" | flowey.exe v 19 'artifact_use_from_x64-windows-tmk_vmm' --is-raw-string update + echo "${{ runner.temp }}\\used_artifacts\\x64-windows-tpm_guest_tests" | flowey.exe v 19 'artifact_use_from_x64-windows-tpm_guest_tests' --is-raw-string update + echo "${{ runner.temp }}\\used_artifacts\\x64-windows-vmgstool" | flowey.exe v 19 'artifact_use_from_x64-windows-vmgstool' --is-raw-string update + echo "${{ runner.temp }}\\used_artifacts\\x64-windows-vmm-tests-archive" | flowey.exe v 19 'artifact_use_from_x64-windows-vmm-tests-archive' --is-raw-string update + shell: bash + - name: creating new test content dir + run: |- + flowey.exe e 19 flowey_core::pipeline::artifact::resolve 5 + flowey.exe e 19 flowey_core::pipeline::artifact::resolve 6 + flowey.exe e 19 flowey_core::pipeline::artifact::resolve 1 + flowey.exe e 19 flowey_core::pipeline::artifact::resolve 0 + flowey.exe e 19 flowey_core::pipeline::artifact::resolve 8 + flowey.exe e 19 flowey_core::pipeline::artifact::resolve 2 + flowey.exe e 19 flowey_core::pipeline::artifact::resolve 4 + flowey.exe e 19 flowey_core::pipeline::artifact::resolve 10 + flowey.exe e 19 flowey_core::pipeline::artifact::resolve 9 + flowey.exe e 19 flowey_core::pipeline::artifact::resolve 3 + flowey.exe e 19 flowey_lib_hvlite::_jobs::consume_and_test_nextest_vmm_tests_archive 0 + shell: bash + - name: resolve OpenHCL igvm artifact + run: flowey.exe e 19 flowey_lib_hvlite::artifact_openhcl_igvm_from_recipe::resolve 0 + shell: bash + - name: create azcopy cache dir + run: flowey.exe e 19 flowey_lib_common::download_azcopy 0 + shell: bash + - name: Pre-processing cache vars + run: |- + flowey.exe e 19 flowey_lib_common::cache 0 + flowey.exe v 19 'flowey_lib_common::cache:2:flowey_lib_common/src/cache.rs:407:72' --is-raw-string write-to-env github floweyvar4 + flowey.exe v 19 'flowey_lib_common::cache:1:flowey_lib_common/src/cache.rs:406:72' --is-raw-string write-to-env github floweyvar5 + shell: bash + - id: flowey_lib_common__cache__1 + uses: actions/cache@v4 + with: + key: ${{ env.floweyvar4 }} + path: ${{ env.floweyvar5 }} + name: 'Restore cache: azcopy' + - name: installing azcopy + run: |- + flowey.exe v 19 'flowey_lib_common::cache:4:flowey_lib_common/src/cache.rs:462:70' --is-raw-string update --env-source steps.flowey_lib_common__cache__1.outputs.cache-hit <> "$GITHUB_PATH" + rustup show + if: runner.os == 'Linux' + name: rustup (Linux) + shell: bash + - run: | + set -x + curl --fail -sSfLo rustup-init.exe https://win.rustup.rs/x86_64 --output rustup-init + ./rustup-init.exe -y --default-toolchain=1.91.1 + echo "$USERPROFILE\\.cargo\\bin" >> $GITHUB_PATH + if: runner.os == 'Windows' && runner.arch == 'X64' + name: rustup (Windows X64) + shell: bash + - run: | + set -x + curl --fail -sSfLo rustup-init.exe https://win.rustup.rs/aarch64 --output rustup-init + ./rustup-init.exe -y --default-toolchain=1.91.1 + echo "$USERPROFILE\\.cargo\\bin" >> $GITHUB_PATH + if: runner.os == 'Windows' && runner.arch == 'ARM64' + name: rustup (Windows ARM64) + shell: bash + - uses: actions/checkout@v4 + with: + path: flowey_bootstrap + - name: Build flowey + run: | + set -x + CARGO_INCREMENTAL=0 cargo build -p flowey_hvlite --target x86_64-pc-windows-msvc --profile flowey-ci + OutDirNormal=$(echo "${{ runner.temp }}/bootstrapped-flowey" | sed -e 's|\\|\/|g' -e 's|^\([A-Za-z]\)\:/\(.*\)|/\L\1\E/\2|') + mkdir -p "$OutDirNormal" + mv ./.github/workflows/openvmm-pr-release.yaml "$OutDirNormal/pipeline.yaml" + mv target/x86_64-pc-windows-msvc/flowey-ci/flowey_hvlite.exe "$OutDirNormal/flowey.exe" + working-directory: flowey_bootstrap + shell: bash + - run: echo "${{ runner.temp }}/bootstrapped-flowey" >> $GITHUB_PATH + shell: bash + name: πŸŒΌπŸ“¦ Add flowey to PATH + - name: πŸŒΌπŸ”Ž Self-check YAML + run: |- + ESCAPED_AGENT_TEMPDIR=$( + cat <<'EOF' | sed 's/\\/\\\\/g' + ${{ runner.temp }} + EOF + ) + flowey.exe pipeline github --runtime $ESCAPED_AGENT_TEMPDIR\\bootstrapped-flowey\\pipeline.yaml --out .github/workflows/openvmm-pr-release.yaml ci checkin-gates --config=pr-release + shell: bash + - name: πŸŒΌπŸ›« Initialize job + run: | + AgentTempDirNormal="${{ runner.temp }}" + AgentTempDirNormal=$(echo "$AgentTempDirNormal" | sed -e 's|\\|\/|g' -e 's|^\([A-Za-z]\)\:/\(.*\)|/\L\1\E/\2|') + echo "AgentTempDirNormal=$AgentTempDirNormal" >> $GITHUB_ENV + + chmod +x $AgentTempDirNormal/bootstrapped-flowey/flowey.exe + + echo '"debug"' | flowey.exe v 2 'FLOWEY_LOG' update + echo "${{ runner.temp }}/work" | flowey.exe v 2 '_internal_WORKING_DIR' --is-raw-string update + + cat <<'EOF' | flowey.exe v 2 'verbose' update + ${{ inputs.verbose != '' && inputs.verbose || 'false' }} + EOF + mkdir -p "$AgentTempDirNormal/publish_artifacts/aarch64-windows-hypestv" + echo "${{ runner.temp }}\\publish_artifacts\\aarch64-windows-hypestv" | flowey.exe v 2 'artifact_publish_from_aarch64-windows-hypestv' --is-raw-string update + mkdir -p "$AgentTempDirNormal/publish_artifacts/aarch64-windows-igvmfilegen" + echo "${{ runner.temp }}\\publish_artifacts\\aarch64-windows-igvmfilegen" | flowey.exe v 2 'artifact_publish_from_aarch64-windows-igvmfilegen' --is-raw-string update + mkdir -p "$AgentTempDirNormal/publish_artifacts/aarch64-windows-ohcldiag-dev" + echo "${{ runner.temp }}\\publish_artifacts\\aarch64-windows-ohcldiag-dev" | flowey.exe v 2 'artifact_publish_from_aarch64-windows-ohcldiag-dev' --is-raw-string update + mkdir -p "$AgentTempDirNormal/publish_artifacts/aarch64-windows-vmgs_lib" + echo "${{ runner.temp }}\\publish_artifacts\\aarch64-windows-vmgs_lib" | flowey.exe v 2 'artifact_publish_from_aarch64-windows-vmgs_lib' --is-raw-string update + shell: bash + - name: add default cargo home to path + run: flowey.exe e 2 flowey_lib_common::install_rust 0 + shell: bash + - name: install Rust + run: flowey.exe e 2 flowey_lib_common::install_rust 1 + shell: bash + - name: detect active toolchain + run: |- + flowey.exe e 2 flowey_lib_common::install_rust 2 + flowey.exe e 2 flowey_lib_common::cfg_cargo_common_flags 0 + shell: bash + - name: check if openvmm needs to be cloned + run: |- + flowey.exe e 2 flowey_lib_common::git_checkout 0 + flowey.exe v 2 'flowey_lib_common::git_checkout:0:flowey_lib_common/src/git_checkout.rs:469:80' --is-raw-string --condvar flowey_lib_common::git_checkout:1:flowey_lib_common/src/git_checkout.rs:470:46 write-to-env github floweyvar1 + flowey.exe v 2 'flowey_lib_common::git_checkout:1:flowey_lib_common/src/git_checkout.rs:470:46' write-to-env github FLOWEY_CONDITION + shell: bash + - id: flowey_lib_common__git_checkout__1 + uses: actions/checkout@v4 + with: + fetch-depth: '1' + path: repo0 + persist-credentials: ${{ env.floweyvar1 }} + name: checkout repo openvmm + if: ${{ fromJSON(env.FLOWEY_CONDITION) }} + - name: report cloned repo directories + run: |- + flowey.exe v 2 'flowey_lib_common::git_checkout:4:flowey_core/src/node/github_context.rs:55:41' --is-raw-string update --env-source github.workspace <> $GITHUB_PATH + shell: bash + name: πŸŒΌπŸ“¦ Add flowey to PATH + - name: πŸŒΌπŸ›« Initialize job + run: | + AgentTempDirNormal="${{ runner.temp }}" + AgentTempDirNormal=$(echo "$AgentTempDirNormal" | sed -e 's|\\|\/|g' -e 's|^\([A-Za-z]\)\:/\(.*\)|/\L\1\E/\2|') + echo "AgentTempDirNormal=$AgentTempDirNormal" >> $GITHUB_ENV + + chmod +x $AgentTempDirNormal/used_artifacts/_internal-flowey-bootstrap-x86_64-windows-uid-14/flowey.exe + + echo '"debug"' | flowey.exe v 20 'FLOWEY_LOG' update + echo "${{ runner.temp }}/work" | flowey.exe v 20 '_internal_WORKING_DIR' --is-raw-string update + + cat <<'EOF' | flowey.exe v 20 'verbose' update ${{ inputs.verbose != '' && inputs.verbose || 'false' }} EOF - echo "${{ runner.temp }}\\used_artifacts\\x64-guest_test_uefi" | flowey.exe v 19 'artifact_use_from_x64-guest_test_uefi' --is-raw-string update - echo "${{ runner.temp }}\\used_artifacts\\x64-linux-musl-pipette" | flowey.exe v 19 'artifact_use_from_x64-linux-musl-pipette' --is-raw-string update - echo "${{ runner.temp }}\\used_artifacts\\x64-linux-musl-tmk_vmm" | flowey.exe v 19 'artifact_use_from_x64-linux-musl-tmk_vmm' --is-raw-string update - echo "${{ runner.temp }}\\used_artifacts\\x64-linux-tpm_guest_tests" | flowey.exe v 19 'artifact_use_from_x64-linux-tpm_guest_tests' --is-raw-string update - echo "${{ runner.temp }}\\used_artifacts\\x64-openhcl-igvm" | flowey.exe v 19 'artifact_use_from_x64-openhcl-igvm' --is-raw-string update - echo "${{ runner.temp }}\\used_artifacts\\x64-tmks" | flowey.exe v 19 'artifact_use_from_x64-tmks' --is-raw-string update - echo "${{ runner.temp }}\\used_artifacts\\x64-windows-openvmm" | flowey.exe v 19 'artifact_use_from_x64-windows-openvmm' --is-raw-string update - echo "${{ runner.temp }}\\used_artifacts\\x64-windows-pipette" | flowey.exe v 19 'artifact_use_from_x64-windows-pipette' --is-raw-string update - echo "${{ runner.temp }}\\used_artifacts\\x64-windows-prep_steps" | flowey.exe v 19 'artifact_use_from_x64-windows-prep_steps' --is-raw-string update - echo "${{ runner.temp }}\\used_artifacts\\x64-windows-tmk_vmm" | flowey.exe v 19 'artifact_use_from_x64-windows-tmk_vmm' --is-raw-string update - echo "${{ runner.temp }}\\used_artifacts\\x64-windows-tpm_guest_tests" | flowey.exe v 19 'artifact_use_from_x64-windows-tpm_guest_tests' --is-raw-string update - echo "${{ runner.temp }}\\used_artifacts\\x64-windows-vmgstool" | flowey.exe v 19 'artifact_use_from_x64-windows-vmgstool' --is-raw-string update - echo "${{ runner.temp }}\\used_artifacts\\x64-windows-vmm-tests-archive" | flowey.exe v 19 'artifact_use_from_x64-windows-vmm-tests-archive' --is-raw-string update + echo "${{ runner.temp }}\\used_artifacts\\x64-guest_test_uefi" | flowey.exe v 20 'artifact_use_from_x64-guest_test_uefi' --is-raw-string update + echo "${{ runner.temp }}\\used_artifacts\\x64-linux-musl-pipette" | flowey.exe v 20 'artifact_use_from_x64-linux-musl-pipette' --is-raw-string update + echo "${{ runner.temp }}\\used_artifacts\\x64-linux-musl-tmk_vmm" | flowey.exe v 20 'artifact_use_from_x64-linux-musl-tmk_vmm' --is-raw-string update + echo "${{ runner.temp }}\\used_artifacts\\x64-linux-tpm_guest_tests" | flowey.exe v 20 'artifact_use_from_x64-linux-tpm_guest_tests' --is-raw-string update + echo "${{ runner.temp }}\\used_artifacts\\x64-openhcl-igvm" | flowey.exe v 20 'artifact_use_from_x64-openhcl-igvm' --is-raw-string update + echo "${{ runner.temp }}\\used_artifacts\\x64-tmks" | flowey.exe v 20 'artifact_use_from_x64-tmks' --is-raw-string update + echo "${{ runner.temp }}\\used_artifacts\\x64-windows-openvmm" | flowey.exe v 20 'artifact_use_from_x64-windows-openvmm' --is-raw-string update + echo "${{ runner.temp }}\\used_artifacts\\x64-windows-pipette" | flowey.exe v 20 'artifact_use_from_x64-windows-pipette' --is-raw-string update + echo "${{ runner.temp }}\\used_artifacts\\x64-windows-prep_steps" | flowey.exe v 20 'artifact_use_from_x64-windows-prep_steps' --is-raw-string update + echo "${{ runner.temp }}\\used_artifacts\\x64-windows-tmk_vmm" | flowey.exe v 20 'artifact_use_from_x64-windows-tmk_vmm' --is-raw-string update + echo "${{ runner.temp }}\\used_artifacts\\x64-windows-tpm_guest_tests" | flowey.exe v 20 'artifact_use_from_x64-windows-tpm_guest_tests' --is-raw-string update + echo "${{ runner.temp }}\\used_artifacts\\x64-windows-vmgstool" | flowey.exe v 20 'artifact_use_from_x64-windows-vmgstool' --is-raw-string update + echo "${{ runner.temp }}\\used_artifacts\\x64-windows-vmm-tests-archive" | flowey.exe v 20 'artifact_use_from_x64-windows-vmm-tests-archive' --is-raw-string update shell: bash - name: creating new test content dir run: |- - flowey.exe e 19 flowey_core::pipeline::artifact::resolve 5 - flowey.exe e 19 flowey_core::pipeline::artifact::resolve 6 - flowey.exe e 19 flowey_core::pipeline::artifact::resolve 1 - flowey.exe e 19 flowey_core::pipeline::artifact::resolve 0 - flowey.exe e 19 flowey_core::pipeline::artifact::resolve 8 - flowey.exe e 19 flowey_core::pipeline::artifact::resolve 2 - flowey.exe e 19 flowey_core::pipeline::artifact::resolve 4 - flowey.exe e 19 flowey_core::pipeline::artifact::resolve 10 - flowey.exe e 19 flowey_core::pipeline::artifact::resolve 9 - flowey.exe e 19 flowey_core::pipeline::artifact::resolve 3 - flowey.exe e 19 flowey_lib_hvlite::_jobs::consume_and_test_nextest_vmm_tests_archive 0 + flowey.exe e 20 flowey_core::pipeline::artifact::resolve 5 + flowey.exe e 20 flowey_core::pipeline::artifact::resolve 6 + flowey.exe e 20 flowey_core::pipeline::artifact::resolve 1 + flowey.exe e 20 flowey_core::pipeline::artifact::resolve 0 + flowey.exe e 20 flowey_core::pipeline::artifact::resolve 8 + flowey.exe e 20 flowey_core::pipeline::artifact::resolve 2 + flowey.exe e 20 flowey_core::pipeline::artifact::resolve 4 + flowey.exe e 20 flowey_core::pipeline::artifact::resolve 10 + flowey.exe e 20 flowey_core::pipeline::artifact::resolve 9 + flowey.exe e 20 flowey_core::pipeline::artifact::resolve 3 + flowey.exe e 20 flowey_lib_hvlite::_jobs::consume_and_test_nextest_vmm_tests_archive 0 shell: bash - name: resolve OpenHCL igvm artifact - run: flowey.exe e 19 flowey_lib_hvlite::artifact_openhcl_igvm_from_recipe::resolve 0 + run: flowey.exe e 20 flowey_lib_hvlite::artifact_openhcl_igvm_from_recipe::resolve 0 shell: bash - name: create azcopy cache dir - run: flowey.exe e 19 flowey_lib_common::download_azcopy 0 + run: flowey.exe e 20 flowey_lib_common::download_azcopy 0 shell: bash - name: Pre-processing cache vars run: |- - flowey.exe e 19 flowey_lib_common::cache 0 - flowey.exe v 19 'flowey_lib_common::cache:2:flowey_lib_common/src/cache.rs:407:72' --is-raw-string write-to-env github floweyvar4 - flowey.exe v 19 'flowey_lib_common::cache:1:flowey_lib_common/src/cache.rs:406:72' --is-raw-string write-to-env github floweyvar5 + flowey.exe e 20 flowey_lib_common::cache 0 + flowey.exe v 20 'flowey_lib_common::cache:2:flowey_lib_common/src/cache.rs:407:72' --is-raw-string write-to-env github floweyvar4 + flowey.exe v 20 'flowey_lib_common::cache:1:flowey_lib_common/src/cache.rs:406:72' --is-raw-string write-to-env github floweyvar5 shell: bash - id: flowey_lib_common__cache__1 uses: actions/cache@v4 @@ -2818,28 +3539,28 @@ jobs: name: 'Restore cache: azcopy' - name: installing azcopy run: |- - flowey.exe v 19 'flowey_lib_common::cache:4:flowey_lib_common/src/cache.rs:462:70' --is-raw-string update --env-source steps.flowey_lib_common__cache__1.outputs.cache-hit <> "$GITHUB_PATH" - rustup show - if: runner.os == 'Linux' - name: rustup (Linux) + - name: πŸŒΌπŸ“¦ Download artifacts + uses: actions/download-artifact@v4 + with: + pattern: '{_internal-flowey-bootstrap-x86_64-windows-uid-14,x64-guest_test_uefi,x64-linux-musl-pipette,x64-linux-musl-tmk_vmm,x64-linux-tpm_guest_tests,x64-openhcl-igvm,x64-tmks,x64-windows-openvmm,x64-windows-pipette,x64-windows-prep_steps,x64-windows-tmk_vmm,x64-windows-tpm_guest_tests,x64-windows-vmgstool,x64-windows-vmm-tests-archive}' + path: ${{ runner.temp }}/used_artifacts/ + - run: echo "${{ runner.temp }}/used_artifacts/_internal-flowey-bootstrap-x86_64-windows-uid-14" >> $GITHUB_PATH shell: bash - - run: | - set -x - curl --fail -sSfLo rustup-init.exe https://win.rustup.rs/x86_64 --output rustup-init - ./rustup-init.exe -y --default-toolchain=1.91.1 - echo "$USERPROFILE\\.cargo\\bin" >> $GITHUB_PATH - if: runner.os == 'Windows' && runner.arch == 'X64' - name: rustup (Windows X64) + name: πŸŒΌπŸ“¦ Add flowey to PATH + - name: πŸŒΌπŸ›« Initialize job + run: | + AgentTempDirNormal="${{ runner.temp }}" + AgentTempDirNormal=$(echo "$AgentTempDirNormal" | sed -e 's|\\|\/|g' -e 's|^\([A-Za-z]\)\:/\(.*\)|/\L\1\E/\2|') + echo "AgentTempDirNormal=$AgentTempDirNormal" >> $GITHUB_ENV + + chmod +x $AgentTempDirNormal/used_artifacts/_internal-flowey-bootstrap-x86_64-windows-uid-14/flowey.exe + + echo '"debug"' | flowey.exe v 21 'FLOWEY_LOG' update + echo "${{ runner.temp }}/work" | flowey.exe v 21 '_internal_WORKING_DIR' --is-raw-string update + + cat <<'EOF' | flowey.exe v 21 'verbose' update + ${{ inputs.verbose != '' && inputs.verbose || 'false' }} + EOF + echo "${{ runner.temp }}\\used_artifacts\\x64-guest_test_uefi" | flowey.exe v 21 'artifact_use_from_x64-guest_test_uefi' --is-raw-string update + echo "${{ runner.temp }}\\used_artifacts\\x64-linux-musl-pipette" | flowey.exe v 21 'artifact_use_from_x64-linux-musl-pipette' --is-raw-string update + echo "${{ runner.temp }}\\used_artifacts\\x64-linux-musl-tmk_vmm" | flowey.exe v 21 'artifact_use_from_x64-linux-musl-tmk_vmm' --is-raw-string update + echo "${{ runner.temp }}\\used_artifacts\\x64-linux-tpm_guest_tests" | flowey.exe v 21 'artifact_use_from_x64-linux-tpm_guest_tests' --is-raw-string update + echo "${{ runner.temp }}\\used_artifacts\\x64-openhcl-igvm" | flowey.exe v 21 'artifact_use_from_x64-openhcl-igvm' --is-raw-string update + echo "${{ runner.temp }}\\used_artifacts\\x64-tmks" | flowey.exe v 21 'artifact_use_from_x64-tmks' --is-raw-string update + echo "${{ runner.temp }}\\used_artifacts\\x64-windows-openvmm" | flowey.exe v 21 'artifact_use_from_x64-windows-openvmm' --is-raw-string update + echo "${{ runner.temp }}\\used_artifacts\\x64-windows-pipette" | flowey.exe v 21 'artifact_use_from_x64-windows-pipette' --is-raw-string update + echo "${{ runner.temp }}\\used_artifacts\\x64-windows-prep_steps" | flowey.exe v 21 'artifact_use_from_x64-windows-prep_steps' --is-raw-string update + echo "${{ runner.temp }}\\used_artifacts\\x64-windows-tmk_vmm" | flowey.exe v 21 'artifact_use_from_x64-windows-tmk_vmm' --is-raw-string update + echo "${{ runner.temp }}\\used_artifacts\\x64-windows-tpm_guest_tests" | flowey.exe v 21 'artifact_use_from_x64-windows-tpm_guest_tests' --is-raw-string update + echo "${{ runner.temp }}\\used_artifacts\\x64-windows-vmgstool" | flowey.exe v 21 'artifact_use_from_x64-windows-vmgstool' --is-raw-string update + echo "${{ runner.temp }}\\used_artifacts\\x64-windows-vmm-tests-archive" | flowey.exe v 21 'artifact_use_from_x64-windows-vmm-tests-archive' --is-raw-string update shell: bash - - run: | - set -x - curl --fail -sSfLo rustup-init.exe https://win.rustup.rs/aarch64 --output rustup-init - ./rustup-init.exe -y --default-toolchain=1.91.1 - echo "$USERPROFILE\\.cargo\\bin" >> $GITHUB_PATH - if: runner.os == 'Windows' && runner.arch == 'ARM64' - name: rustup (Windows ARM64) + - name: creating new test content dir + run: |- + flowey.exe e 21 flowey_core::pipeline::artifact::resolve 5 + flowey.exe e 21 flowey_core::pipeline::artifact::resolve 6 + flowey.exe e 21 flowey_core::pipeline::artifact::resolve 1 + flowey.exe e 21 flowey_core::pipeline::artifact::resolve 0 + flowey.exe e 21 flowey_core::pipeline::artifact::resolve 8 + flowey.exe e 21 flowey_core::pipeline::artifact::resolve 2 + flowey.exe e 21 flowey_core::pipeline::artifact::resolve 4 + flowey.exe e 21 flowey_core::pipeline::artifact::resolve 10 + flowey.exe e 21 flowey_core::pipeline::artifact::resolve 9 + flowey.exe e 21 flowey_core::pipeline::artifact::resolve 3 + flowey.exe e 21 flowey_lib_hvlite::_jobs::consume_and_test_nextest_vmm_tests_archive 0 shell: bash - - uses: actions/checkout@v4 + - name: resolve OpenHCL igvm artifact + run: flowey.exe e 21 flowey_lib_hvlite::artifact_openhcl_igvm_from_recipe::resolve 0 + shell: bash + - name: create azcopy cache dir + run: flowey.exe e 21 flowey_lib_common::download_azcopy 0 + shell: bash + - name: Pre-processing cache vars + run: |- + flowey.exe e 21 flowey_lib_common::cache 0 + flowey.exe v 21 'flowey_lib_common::cache:2:flowey_lib_common/src/cache.rs:407:72' --is-raw-string write-to-env github floweyvar4 + flowey.exe v 21 'flowey_lib_common::cache:1:flowey_lib_common/src/cache.rs:406:72' --is-raw-string write-to-env github floweyvar5 + shell: bash + - id: flowey_lib_common__cache__1 + uses: actions/cache@v4 with: - path: flowey_bootstrap - - name: Build flowey - run: | - set -x - CARGO_INCREMENTAL=0 cargo build -p flowey_hvlite --target x86_64-pc-windows-msvc --profile flowey-ci - OutDirNormal=$(echo "${{ runner.temp }}/bootstrapped-flowey" | sed -e 's|\\|\/|g' -e 's|^\([A-Za-z]\)\:/\(.*\)|/\L\1\E/\2|') - mkdir -p "$OutDirNormal" - mv ./.github/workflows/openvmm-pr-release.yaml "$OutDirNormal/pipeline.yaml" - mv target/x86_64-pc-windows-msvc/flowey-ci/flowey_hvlite.exe "$OutDirNormal/flowey.exe" - working-directory: flowey_bootstrap + key: ${{ env.floweyvar4 }} + path: ${{ env.floweyvar5 }} + name: 'Restore cache: azcopy' + - name: installing azcopy + run: |- + flowey.exe v 21 'flowey_lib_common::cache:4:flowey_lib_common/src/cache.rs:462:70' --is-raw-string update --env-source steps.flowey_lib_common__cache__1.outputs.cache-hit <> $GITHUB_PATH + - name: create gh-release-download cache dir + run: flowey.exe e 21 flowey_lib_common::download_gh_release 0 shell: bash - name: πŸŒΌπŸ“¦ Add flowey to PATH - - name: πŸŒΌπŸ”Ž Self-check YAML + - name: Pre-processing cache vars run: |- - ESCAPED_AGENT_TEMPDIR=$( - cat <<'EOF' | sed 's/\\/\\\\/g' - ${{ runner.temp }} - EOF - ) - flowey.exe pipeline github --runtime $ESCAPED_AGENT_TEMPDIR\\bootstrapped-flowey\\pipeline.yaml --out .github/workflows/openvmm-pr-release.yaml ci checkin-gates --config=pr-release + flowey.exe e 21 flowey_lib_common::cache 12 + flowey.exe v 21 'flowey_lib_common::cache:26:flowey_lib_common/src/cache.rs:407:72' --is-raw-string write-to-env github floweyvar10 + flowey.exe v 21 'flowey_lib_common::cache:25:flowey_lib_common/src/cache.rs:406:72' --is-raw-string write-to-env github floweyvar11 shell: bash - - name: πŸŒΌπŸ›« Initialize job - run: | - AgentTempDirNormal="${{ runner.temp }}" - AgentTempDirNormal=$(echo "$AgentTempDirNormal" | sed -e 's|\\|\/|g' -e 's|^\([A-Za-z]\)\:/\(.*\)|/\L\1\E/\2|') - echo "AgentTempDirNormal=$AgentTempDirNormal" >> $GITHUB_ENV - - chmod +x $AgentTempDirNormal/bootstrapped-flowey/flowey.exe - - echo '"debug"' | flowey.exe v 2 'FLOWEY_LOG' update - echo "${{ runner.temp }}/work" | flowey.exe v 2 '_internal_WORKING_DIR' --is-raw-string update - - cat <<'EOF' | flowey.exe v 2 'verbose' update - ${{ inputs.verbose != '' && inputs.verbose || 'false' }} + - id: flowey_lib_common__cache__13 + uses: actions/cache@v4 + with: + key: ${{ env.floweyvar10 }} + path: ${{ env.floweyvar11 }} + name: 'Restore cache: gh-release-download' + - name: download artifacts from github releases + run: |- + flowey.exe v 21 'flowey_lib_common::cache:28:flowey_lib_common/src/cache.rs:462:70' --is-raw-string update --env-source steps.flowey_lib_common__cache__13.outputs.cache-hit <> $GITHUB_PATH + - run: echo "${{ runner.temp }}/used_artifacts/_internal-flowey-bootstrap-x86_64-linux-uid-9" >> $GITHUB_PATH shell: bash name: πŸŒΌπŸ“¦ Add flowey to PATH - name: πŸŒΌπŸ›« Initialize job @@ -3257,40 +4042,40 @@ jobs: AgentTempDirNormal=$(echo "$AgentTempDirNormal" | sed -e 's|\\|\/|g' -e 's|^\([A-Za-z]\)\:/\(.*\)|/\L\1\E/\2|') echo "AgentTempDirNormal=$AgentTempDirNormal" >> $GITHUB_ENV - chmod +x $AgentTempDirNormal/used_artifacts/_internal-flowey-bootstrap-x86_64-linux-uid-8/flowey + chmod +x $AgentTempDirNormal/used_artifacts/_internal-flowey-bootstrap-x86_64-linux-uid-9/flowey - echo '"debug"' | flowey v 20 'FLOWEY_LOG' update - echo "${{ runner.temp }}/work" | flowey v 20 '_internal_WORKING_DIR' --is-raw-string update + echo '"debug"' | flowey v 22 'FLOWEY_LOG' update + echo "${{ runner.temp }}/work" | flowey v 22 '_internal_WORKING_DIR' --is-raw-string update - cat <<'EOF' | flowey v 20 'verbose' update + cat <<'EOF' | flowey v 22 'verbose' update ${{ inputs.verbose != '' && inputs.verbose || 'false' }} EOF - echo "$AgentTempDirNormal/used_artifacts/x64-guest_test_uefi" | flowey v 20 'artifact_use_from_x64-guest_test_uefi' --is-raw-string update - echo "$AgentTempDirNormal/used_artifacts/x64-linux-musl-pipette" | flowey v 20 'artifact_use_from_x64-linux-musl-pipette' --is-raw-string update - echo "$AgentTempDirNormal/used_artifacts/x64-linux-musl-tmk_vmm" | flowey v 20 'artifact_use_from_x64-linux-musl-tmk_vmm' --is-raw-string update - echo "$AgentTempDirNormal/used_artifacts/x64-linux-openvmm" | flowey v 20 'artifact_use_from_x64-linux-openvmm' --is-raw-string update - echo "$AgentTempDirNormal/used_artifacts/x64-linux-vmm-tests-archive" | flowey v 20 'artifact_use_from_x64-linux-vmm-tests-archive' --is-raw-string update - echo "$AgentTempDirNormal/used_artifacts/x64-tmks" | flowey v 20 'artifact_use_from_x64-tmks' --is-raw-string update - echo "$AgentTempDirNormal/used_artifacts/x64-windows-pipette" | flowey v 20 'artifact_use_from_x64-windows-pipette' --is-raw-string update + echo "$AgentTempDirNormal/used_artifacts/x64-guest_test_uefi" | flowey v 22 'artifact_use_from_x64-guest_test_uefi' --is-raw-string update + echo "$AgentTempDirNormal/used_artifacts/x64-linux-musl-pipette" | flowey v 22 'artifact_use_from_x64-linux-musl-pipette' --is-raw-string update + echo "$AgentTempDirNormal/used_artifacts/x64-linux-musl-tmk_vmm" | flowey v 22 'artifact_use_from_x64-linux-musl-tmk_vmm' --is-raw-string update + echo "$AgentTempDirNormal/used_artifacts/x64-linux-openvmm" | flowey v 22 'artifact_use_from_x64-linux-openvmm' --is-raw-string update + echo "$AgentTempDirNormal/used_artifacts/x64-linux-vmm-tests-archive" | flowey v 22 'artifact_use_from_x64-linux-vmm-tests-archive' --is-raw-string update + echo "$AgentTempDirNormal/used_artifacts/x64-tmks" | flowey v 22 'artifact_use_from_x64-tmks' --is-raw-string update + echo "$AgentTempDirNormal/used_artifacts/x64-windows-pipette" | flowey v 22 'artifact_use_from_x64-windows-pipette' --is-raw-string update shell: bash - name: creating new test content dir run: |- - flowey e 20 flowey_core::pipeline::artifact::resolve 3 - flowey e 20 flowey_core::pipeline::artifact::resolve 6 - flowey e 20 flowey_core::pipeline::artifact::resolve 1 - flowey e 20 flowey_core::pipeline::artifact::resolve 0 - flowey e 20 flowey_core::pipeline::artifact::resolve 2 - flowey e 20 flowey_core::pipeline::artifact::resolve 5 - flowey e 20 flowey_lib_hvlite::_jobs::consume_and_test_nextest_vmm_tests_archive 0 + flowey e 22 flowey_core::pipeline::artifact::resolve 3 + flowey e 22 flowey_core::pipeline::artifact::resolve 6 + flowey e 22 flowey_core::pipeline::artifact::resolve 1 + flowey e 22 flowey_core::pipeline::artifact::resolve 0 + flowey e 22 flowey_core::pipeline::artifact::resolve 2 + flowey e 22 flowey_core::pipeline::artifact::resolve 5 + flowey e 22 flowey_lib_hvlite::_jobs::consume_and_test_nextest_vmm_tests_archive 0 shell: bash - name: create azcopy cache dir - run: flowey e 20 flowey_lib_common::download_azcopy 0 + run: flowey e 22 flowey_lib_common::download_azcopy 0 shell: bash - name: Pre-processing cache vars run: |- - flowey e 20 flowey_lib_common::cache 0 - flowey v 20 'flowey_lib_common::cache:2:flowey_lib_common/src/cache.rs:407:72' --is-raw-string write-to-env github floweyvar4 - flowey v 20 'flowey_lib_common::cache:1:flowey_lib_common/src/cache.rs:406:72' --is-raw-string write-to-env github floweyvar5 + flowey e 22 flowey_lib_common::cache 0 + flowey v 22 'flowey_lib_common::cache:2:flowey_lib_common/src/cache.rs:407:72' --is-raw-string write-to-env github floweyvar4 + flowey v 22 'flowey_lib_common::cache:1:flowey_lib_common/src/cache.rs:406:72' --is-raw-string write-to-env github floweyvar5 shell: bash - id: flowey_lib_common__cache__1 uses: actions/cache@v4 @@ -3300,34 +4085,34 @@ jobs: name: 'Restore cache: azcopy' - name: checking if packages need to be installed run: |- - flowey v 20 'flowey_lib_common::cache:4:flowey_lib_common/src/cache.rs:462:70' --is-raw-string update --env-source steps.flowey_lib_common__cache__1.outputs.cache-hit <> $GITHUB_PATH @@ -1074,7 +1079,7 @@ jobs: ${{ runner.temp }} EOF ) - flowey.exe pipeline github --runtime $ESCAPED_AGENT_TEMPDIR\\bootstrapped-flowey\\pipeline.yaml --out .github/workflows/openvmm-pr.yaml ci checkin-gates --config=pr + flowey pipeline github --runtime $ESCAPED_AGENT_TEMPDIR/bootstrapped-flowey/pipeline.yaml --out .github/workflows/openvmm-pr.yaml ci checkin-gates --config=pr shell: bash - name: πŸŒΌπŸ›« Initialize job run: | @@ -1082,183 +1087,294 @@ jobs: AgentTempDirNormal=$(echo "$AgentTempDirNormal" | sed -e 's|\\|\/|g' -e 's|^\([A-Za-z]\)\:/\(.*\)|/\L\1\E/\2|') echo "AgentTempDirNormal=$AgentTempDirNormal" >> $GITHUB_ENV - chmod +x $AgentTempDirNormal/bootstrapped-flowey/flowey.exe + chmod +x $AgentTempDirNormal/bootstrapped-flowey/flowey - echo '"debug"' | flowey.exe v 12 'FLOWEY_LOG' update - echo "${{ runner.temp }}/work" | flowey.exe v 12 '_internal_WORKING_DIR' --is-raw-string update + echo '"debug"' | flowey v 12 'FLOWEY_LOG' update + echo "${{ runner.temp }}/work" | flowey v 12 '_internal_WORKING_DIR' --is-raw-string update - cat <<'EOF' | flowey.exe v 12 'verbose' update + cat <<'EOF' | flowey v 12 'verbose' update ${{ inputs.verbose != '' && inputs.verbose || 'false' }} EOF + mkdir -p "$AgentTempDirNormal/publish_artifacts/x64-openhcl-igvm-extras-nix" + echo "$AgentTempDirNormal/publish_artifacts/x64-openhcl-igvm-extras-nix" | flowey v 12 'artifact_publish_from_x64-openhcl-igvm-extras-nix' --is-raw-string update + mkdir -p "$AgentTempDirNormal/publish_artifacts/x64-openhcl-igvm-nix" + echo "$AgentTempDirNormal/publish_artifacts/x64-openhcl-igvm-nix" | flowey v 12 'artifact_publish_from_x64-openhcl-igvm-nix' --is-raw-string update shell: bash - - name: add default cargo home to path - run: flowey.exe e 12 flowey_lib_common::install_rust 0 - shell: bash - - name: install Rust - run: flowey.exe e 12 flowey_lib_common::install_rust 1 + - name: add nix profile to path + run: flowey e 12 flowey_lib_common::install_nix 0 shell: bash - - name: detect active toolchain - run: |- - flowey.exe e 12 flowey_lib_common::install_rust 2 - flowey.exe e 12 flowey_lib_common::cfg_cargo_common_flags 0 + - name: install Nix + run: flowey e 12 flowey_lib_common::install_nix 1 shell: bash - name: check if openvmm needs to be cloned run: |- - flowey.exe e 12 flowey_lib_common::git_checkout 0 - flowey.exe v 12 'flowey_lib_common::git_checkout:0:flowey_lib_common/src/git_checkout.rs:469:80' --is-raw-string --condvar flowey_lib_common::git_checkout:1:flowey_lib_common/src/git_checkout.rs:470:46 write-to-env github floweyvar6 - flowey.exe v 12 'flowey_lib_common::git_checkout:1:flowey_lib_common/src/git_checkout.rs:470:46' write-to-env github FLOWEY_CONDITION + flowey e 12 flowey_lib_common::git_checkout 0 + flowey v 12 'flowey_lib_common::git_checkout:0:flowey_lib_common/src/git_checkout.rs:469:80' --is-raw-string --condvar flowey_lib_common::git_checkout:1:flowey_lib_common/src/git_checkout.rs:470:46 write-to-env github floweyvar1 + flowey v 12 'flowey_lib_common::git_checkout:1:flowey_lib_common/src/git_checkout.rs:470:46' write-to-env github FLOWEY_CONDITION shell: bash - id: flowey_lib_common__git_checkout__1 uses: actions/checkout@v4 with: fetch-depth: '1' path: repo0 - persist-credentials: ${{ env.floweyvar6 }} + persist-credentials: ${{ env.floweyvar1 }} name: checkout repo openvmm if: ${{ fromJSON(env.FLOWEY_CONDITION) }} - name: report cloned repo directories run: |- - flowey.exe v 12 'flowey_lib_common::git_checkout:4:flowey_core/src/node/github_context.rs:55:41' --is-raw-string update --env-source github.workspace <> $GITHUB_PATH @@ -1590,7 +1668,7 @@ jobs: ${{ runner.temp }} EOF ) - flowey pipeline github --runtime $ESCAPED_AGENT_TEMPDIR/bootstrapped-flowey/pipeline.yaml --out .github/workflows/openvmm-pr.yaml ci checkin-gates --config=pr + flowey.exe pipeline github --runtime $ESCAPED_AGENT_TEMPDIR\\bootstrapped-flowey\\pipeline.yaml --out .github/workflows/openvmm-pr.yaml ci checkin-gates --config=pr shell: bash - name: πŸŒΌπŸ›« Initialize job run: | @@ -1598,31 +1676,31 @@ jobs: AgentTempDirNormal=$(echo "$AgentTempDirNormal" | sed -e 's|\\|\/|g' -e 's|^\([A-Za-z]\)\:/\(.*\)|/\L\1\E/\2|') echo "AgentTempDirNormal=$AgentTempDirNormal" >> $GITHUB_ENV - chmod +x $AgentTempDirNormal/bootstrapped-flowey/flowey + chmod +x $AgentTempDirNormal/bootstrapped-flowey/flowey.exe - echo '"debug"' | flowey v 14 'FLOWEY_LOG' update - echo "${{ runner.temp }}/work" | flowey v 14 '_internal_WORKING_DIR' --is-raw-string update + echo '"debug"' | flowey.exe v 14 'FLOWEY_LOG' update + echo "${{ runner.temp }}/work" | flowey.exe v 14 '_internal_WORKING_DIR' --is-raw-string update - cat <<'EOF' | flowey v 14 'verbose' update + cat <<'EOF' | flowey.exe v 14 'verbose' update ${{ inputs.verbose != '' && inputs.verbose || 'false' }} EOF shell: bash - name: add default cargo home to path - run: flowey e 14 flowey_lib_common::install_rust 0 + run: flowey.exe e 14 flowey_lib_common::install_rust 0 shell: bash - name: install Rust - run: flowey e 14 flowey_lib_common::install_rust 1 + run: flowey.exe e 14 flowey_lib_common::install_rust 1 shell: bash - name: detect active toolchain run: |- - flowey e 14 flowey_lib_common::install_rust 2 - flowey e 14 flowey_lib_common::cfg_cargo_common_flags 0 + flowey.exe e 14 flowey_lib_common::install_rust 2 + flowey.exe e 14 flowey_lib_common::cfg_cargo_common_flags 0 shell: bash - name: check if openvmm needs to be cloned run: |- - flowey e 14 flowey_lib_common::git_checkout 0 - flowey v 14 'flowey_lib_common::git_checkout:0:flowey_lib_common/src/git_checkout.rs:469:80' --is-raw-string --condvar flowey_lib_common::git_checkout:1:flowey_lib_common/src/git_checkout.rs:470:46 write-to-env github floweyvar6 - flowey v 14 'flowey_lib_common::git_checkout:1:flowey_lib_common/src/git_checkout.rs:470:46' write-to-env github FLOWEY_CONDITION + flowey.exe e 14 flowey_lib_common::git_checkout 0 + flowey.exe v 14 'flowey_lib_common::git_checkout:0:flowey_lib_common/src/git_checkout.rs:469:80' --is-raw-string --condvar flowey_lib_common::git_checkout:1:flowey_lib_common/src/git_checkout.rs:470:46 write-to-env github floweyvar6 + flowey.exe v 14 'flowey_lib_common::git_checkout:1:flowey_lib_common/src/git_checkout.rs:470:46' write-to-env github FLOWEY_CONDITION shell: bash - id: flowey_lib_common__git_checkout__1 uses: actions/checkout@v4 @@ -1634,27 +1712,23 @@ jobs: if: ${{ fromJSON(env.FLOWEY_CONDITION) }} - name: report cloned repo directories run: |- - flowey v 14 'flowey_lib_common::git_checkout:4:flowey_core/src/node/github_context.rs:55:41' --is-raw-string update --env-source github.workspace <> $GITHUB_PATH @@ -1854,7 +1908,7 @@ jobs: ${{ runner.temp }} EOF ) - flowey.exe pipeline github --runtime $ESCAPED_AGENT_TEMPDIR\\bootstrapped-flowey\\pipeline.yaml --out .github/workflows/openvmm-pr.yaml ci checkin-gates --config=pr + flowey pipeline github --runtime $ESCAPED_AGENT_TEMPDIR/bootstrapped-flowey/pipeline.yaml --out .github/workflows/openvmm-pr.yaml ci checkin-gates --config=pr shell: bash - name: πŸŒΌπŸ›« Initialize job run: | @@ -1862,31 +1916,31 @@ jobs: AgentTempDirNormal=$(echo "$AgentTempDirNormal" | sed -e 's|\\|\/|g' -e 's|^\([A-Za-z]\)\:/\(.*\)|/\L\1\E/\2|') echo "AgentTempDirNormal=$AgentTempDirNormal" >> $GITHUB_ENV - chmod +x $AgentTempDirNormal/bootstrapped-flowey/flowey.exe + chmod +x $AgentTempDirNormal/bootstrapped-flowey/flowey - echo '"debug"' | flowey.exe v 15 'FLOWEY_LOG' update - echo "${{ runner.temp }}/work" | flowey.exe v 15 '_internal_WORKING_DIR' --is-raw-string update + echo '"debug"' | flowey v 15 'FLOWEY_LOG' update + echo "${{ runner.temp }}/work" | flowey v 15 '_internal_WORKING_DIR' --is-raw-string update - cat <<'EOF' | flowey.exe v 15 'verbose' update + cat <<'EOF' | flowey v 15 'verbose' update ${{ inputs.verbose != '' && inputs.verbose || 'false' }} EOF shell: bash - name: add default cargo home to path - run: flowey.exe e 15 flowey_lib_common::install_rust 0 + run: flowey e 15 flowey_lib_common::install_rust 0 shell: bash - name: install Rust - run: flowey.exe e 15 flowey_lib_common::install_rust 1 + run: flowey e 15 flowey_lib_common::install_rust 1 shell: bash - name: detect active toolchain run: |- - flowey.exe e 15 flowey_lib_common::install_rust 2 - flowey.exe e 15 flowey_lib_common::cfg_cargo_common_flags 0 + flowey e 15 flowey_lib_common::install_rust 2 + flowey e 15 flowey_lib_common::cfg_cargo_common_flags 0 shell: bash - name: check if openvmm needs to be cloned run: |- - flowey.exe e 15 flowey_lib_common::git_checkout 0 - flowey.exe v 15 'flowey_lib_common::git_checkout:0:flowey_lib_common/src/git_checkout.rs:469:80' --is-raw-string --condvar flowey_lib_common::git_checkout:1:flowey_lib_common/src/git_checkout.rs:470:46 write-to-env github floweyvar6 - flowey.exe v 15 'flowey_lib_common::git_checkout:1:flowey_lib_common/src/git_checkout.rs:470:46' write-to-env github FLOWEY_CONDITION + flowey e 15 flowey_lib_common::git_checkout 0 + flowey v 15 'flowey_lib_common::git_checkout:0:flowey_lib_common/src/git_checkout.rs:469:80' --is-raw-string --condvar flowey_lib_common::git_checkout:1:flowey_lib_common/src/git_checkout.rs:470:46 write-to-env github floweyvar6 + flowey v 15 'flowey_lib_common::git_checkout:1:flowey_lib_common/src/git_checkout.rs:470:46' write-to-env github FLOWEY_CONDITION shell: bash - id: flowey_lib_common__git_checkout__1 uses: actions/checkout@v4 @@ -1898,23 +1952,23 @@ jobs: if: ${{ fromJSON(env.FLOWEY_CONDITION) }} - name: report cloned repo directories run: |- - flowey.exe v 15 'flowey_lib_common::git_checkout:4:flowey_core/src/node/github_context.rs:55:41' --is-raw-string update --env-source github.workspace <> $GITHUB_PATH @@ -2119,12 +2212,6 @@ jobs: flowey e 16 flowey_lib_common::install_rust 2 flowey e 16 flowey_lib_common::cfg_cargo_common_flags 0 shell: bash - - name: checking if packages need to be installed - run: flowey e 16 flowey_lib_common::install_dist_pkg 0 - shell: bash - - name: installing packages - run: flowey e 16 flowey_lib_common::install_dist_pkg 1 - shell: bash - name: check if openvmm needs to be cloned run: |- flowey e 16 flowey_lib_common::git_checkout 0 @@ -2146,9 +2233,13 @@ jobs: EOF flowey e 16 flowey_lib_common::git_checkout 3 flowey e 16 flowey_lib_hvlite::git_checkout_openvmm_repo 0 + flowey e 16 flowey_lib_hvlite::cfg_openvmm_magicpath 0 shell: bash - - name: set '-Dwarnings' in .cargo/config.toml - run: flowey e 16 flowey_lib_hvlite::init_openvmm_cargo_config_deny_warnings 0 + - name: checking if packages need to be installed + run: flowey e 16 flowey_lib_common::install_dist_pkg 0 + shell: bash + - name: installing packages + run: flowey e 16 flowey_lib_common::install_dist_pkg 1 shell: bash - name: create gh-release-download cache dir run: flowey e 16 flowey_lib_common::download_gh_release 0 @@ -2173,16 +2264,23 @@ jobs: flowey e 16 flowey_lib_common::cache 6 flowey e 16 flowey_lib_common::download_gh_release 1 shell: bash + - name: unpack openvmm-deps archive + run: flowey e 16 flowey_lib_hvlite::resolve_openvmm_deps 0 + shell: bash + - name: extract X64 sysroot.tar.gz + run: flowey e 16 flowey_lib_hvlite::init_openvmm_magicpath_openhcl_sysroot 0 + shell: bash + - name: set '-Dwarnings' in .cargo/config.toml + run: flowey e 16 flowey_lib_hvlite::init_openvmm_cargo_config_deny_warnings 0 + shell: bash - name: unpack protoc - run: |- - flowey e 16 flowey_lib_common::download_protoc 0 - flowey e 16 flowey_lib_hvlite::cfg_openvmm_magicpath 0 + run: flowey e 16 flowey_lib_common::download_protoc 0 shell: bash - name: symlink protoc run: |- flowey e 16 flowey_lib_hvlite::init_openvmm_magicpath_protoc 0 - flowey e 16 flowey_lib_hvlite::init_cross_build 0 flowey e 16 flowey_lib_hvlite::init_cross_build 2 + flowey e 16 flowey_lib_hvlite::init_cross_build 0 shell: bash - name: cargo build xtask run: |- @@ -2201,6 +2299,12 @@ jobs: - name: cargo clippy run: flowey e 16 flowey_lib_common::run_cargo_clippy 0 shell: bash + - name: cargo clippy + run: flowey e 16 flowey_lib_common::run_cargo_clippy 2 + shell: bash + - name: cargo clippy + run: flowey e 16 flowey_lib_common::run_cargo_clippy 1 + shell: bash - name: create cargo-nextest cache dir run: |- flowey e 16 flowey_lib_common::download_cargo_nextest 0 @@ -2234,7 +2338,7 @@ jobs: - name: installing cargo-nextest run: |- flowey e 16 flowey_lib_common::install_cargo_nextest 0 - flowey e 16 flowey_lib_hvlite::init_cross_build 3 + flowey e 16 flowey_lib_hvlite::init_cross_build 1 shell: bash - name: cargo build xtask run: |- @@ -2250,7 +2354,7 @@ jobs: - name: determine unit test exclusions run: |- flowey e 16 flowey_lib_hvlite::build_nextest_unit_tests 0 - flowey e 16 flowey_lib_hvlite::init_cross_build 1 + flowey e 16 flowey_lib_hvlite::init_cross_build 3 flowey e 16 flowey_lib_hvlite::run_cargo_nextest_run 0 shell: bash - name: generate nextest command @@ -2270,14 +2374,14 @@ jobs: - id: flowey_lib_common__publish_test_results__3 uses: actions/upload-artifact@v4 with: - name: aarch64-linux-unit-tests-junit-xml + name: x64-linux-musl-unit-tests-junit-xml path: ${{ env.floweyvar1 }} - name: 'publish test results: aarch64-linux-unit-tests (JUnit XML)' + name: 'publish test results: x64-linux-musl-unit-tests (JUnit XML)' if: ${{ fromJSON(env.FLOWEY_CONDITION) }} - name: report test results to overall pipeline status run: flowey e 16 flowey_lib_hvlite::_jobs::build_and_run_nextest_unit_tests 1 shell: bash - - name: run doctests for aarch64-unknown-linux-gnu + - name: run doctests for x86_64-unknown-linux-musl run: flowey e 16 flowey_lib_hvlite::_jobs::build_and_run_doc_tests 0 shell: bash - name: 'validate cache entry: cargo-nextest' @@ -2287,8 +2391,8 @@ jobs: run: flowey e 16 flowey_lib_common::cache 7 shell: bash job17: - name: clippy [aarch64-linux-musl, misc nostd], unit tests [aarch64-linux-musl] - runs-on: ubuntu-24.04-arm + name: clippy [aarch64-windows], unit tests [aarch64-windows] + runs-on: windows-11-arm permissions: contents: read id-token: write @@ -2327,11 +2431,11 @@ jobs: - name: Build flowey run: | set -x - CARGO_INCREMENTAL=0 cargo build -p flowey_hvlite --target aarch64-unknown-linux-gnu --profile flowey-ci + CARGO_INCREMENTAL=0 cargo build -p flowey_hvlite --target aarch64-pc-windows-msvc --profile flowey-ci OutDirNormal=$(echo "${{ runner.temp }}/bootstrapped-flowey" | sed -e 's|\\|\/|g' -e 's|^\([A-Za-z]\)\:/\(.*\)|/\L\1\E/\2|') mkdir -p "$OutDirNormal" mv ./.github/workflows/openvmm-pr.yaml "$OutDirNormal/pipeline.yaml" - mv target/aarch64-unknown-linux-gnu/flowey-ci/flowey_hvlite "$OutDirNormal/flowey" + mv target/aarch64-pc-windows-msvc/flowey-ci/flowey_hvlite.exe "$OutDirNormal/flowey.exe" working-directory: flowey_bootstrap shell: bash - run: echo "${{ runner.temp }}/bootstrapped-flowey" >> $GITHUB_PATH @@ -2344,7 +2448,7 @@ jobs: ${{ runner.temp }} EOF ) - flowey pipeline github --runtime $ESCAPED_AGENT_TEMPDIR/bootstrapped-flowey/pipeline.yaml --out .github/workflows/openvmm-pr.yaml ci checkin-gates --config=pr + flowey.exe pipeline github --runtime $ESCAPED_AGENT_TEMPDIR\\bootstrapped-flowey\\pipeline.yaml --out .github/workflows/openvmm-pr.yaml ci checkin-gates --config=pr shell: bash - name: πŸŒΌπŸ›« Initialize job run: | @@ -2352,31 +2456,31 @@ jobs: AgentTempDirNormal=$(echo "$AgentTempDirNormal" | sed -e 's|\\|\/|g' -e 's|^\([A-Za-z]\)\:/\(.*\)|/\L\1\E/\2|') echo "AgentTempDirNormal=$AgentTempDirNormal" >> $GITHUB_ENV - chmod +x $AgentTempDirNormal/bootstrapped-flowey/flowey + chmod +x $AgentTempDirNormal/bootstrapped-flowey/flowey.exe - echo '"debug"' | flowey v 17 'FLOWEY_LOG' update - echo "${{ runner.temp }}/work" | flowey v 17 '_internal_WORKING_DIR' --is-raw-string update + echo '"debug"' | flowey.exe v 17 'FLOWEY_LOG' update + echo "${{ runner.temp }}/work" | flowey.exe v 17 '_internal_WORKING_DIR' --is-raw-string update - cat <<'EOF' | flowey v 17 'verbose' update + cat <<'EOF' | flowey.exe v 17 'verbose' update ${{ inputs.verbose != '' && inputs.verbose || 'false' }} EOF shell: bash - name: add default cargo home to path - run: flowey e 17 flowey_lib_common::install_rust 0 + run: flowey.exe e 17 flowey_lib_common::install_rust 0 shell: bash - name: install Rust - run: flowey e 17 flowey_lib_common::install_rust 1 + run: flowey.exe e 17 flowey_lib_common::install_rust 1 shell: bash - name: detect active toolchain run: |- - flowey e 17 flowey_lib_common::install_rust 2 - flowey e 17 flowey_lib_common::cfg_cargo_common_flags 0 + flowey.exe e 17 flowey_lib_common::install_rust 2 + flowey.exe e 17 flowey_lib_common::cfg_cargo_common_flags 0 shell: bash - name: check if openvmm needs to be cloned run: |- - flowey e 17 flowey_lib_common::git_checkout 0 - flowey v 17 'flowey_lib_common::git_checkout:0:flowey_lib_common/src/git_checkout.rs:469:80' --is-raw-string --condvar flowey_lib_common::git_checkout:1:flowey_lib_common/src/git_checkout.rs:470:46 write-to-env github floweyvar6 - flowey v 17 'flowey_lib_common::git_checkout:1:flowey_lib_common/src/git_checkout.rs:470:46' write-to-env github FLOWEY_CONDITION + flowey.exe e 17 flowey_lib_common::git_checkout 0 + flowey.exe v 17 'flowey_lib_common::git_checkout:0:flowey_lib_common/src/git_checkout.rs:469:80' --is-raw-string --condvar flowey_lib_common::git_checkout:1:flowey_lib_common/src/git_checkout.rs:470:46 write-to-env github floweyvar6 + flowey.exe v 17 'flowey_lib_common::git_checkout:1:flowey_lib_common/src/git_checkout.rs:470:46' write-to-env github FLOWEY_CONDITION shell: bash - id: flowey_lib_common__git_checkout__1 uses: actions/checkout@v4 @@ -2388,27 +2492,23 @@ jobs: if: ${{ fromJSON(env.FLOWEY_CONDITION) }} - name: report cloned repo directories run: |- - flowey v 17 'flowey_lib_common::git_checkout:4:flowey_core/src/node/github_context.rs:55:41' --is-raw-string update --env-source github.workspace <> "$GITHUB_PATH" + rustup show + if: runner.os == 'Linux' + name: rustup (Linux) + shell: bash + - run: | + set -x + curl --fail -sSfLo rustup-init.exe https://win.rustup.rs/x86_64 --output rustup-init + ./rustup-init.exe -y --default-toolchain=1.91.1 + echo "$USERPROFILE\\.cargo\\bin" >> $GITHUB_PATH + if: runner.os == 'Windows' && runner.arch == 'X64' + name: rustup (Windows X64) + shell: bash + - run: | + set -x + curl --fail -sSfLo rustup-init.exe https://win.rustup.rs/aarch64 --output rustup-init + ./rustup-init.exe -y --default-toolchain=1.91.1 + echo "$USERPROFILE\\.cargo\\bin" >> $GITHUB_PATH + if: runner.os == 'Windows' && runner.arch == 'ARM64' + name: rustup (Windows ARM64) + shell: bash + - uses: actions/checkout@v4 with: - pattern: '{_internal-flowey-bootstrap-x86_64-windows-uid-14,x64-guest_test_uefi,x64-linux-musl-pipette,x64-linux-musl-tmk_vmm,x64-linux-tpm_guest_tests,x64-openhcl-igvm,x64-tmks,x64-windows-openvmm,x64-windows-pipette,x64-windows-prep_steps,x64-windows-tmk_vmm,x64-windows-tpm_guest_tests,x64-windows-vmgstool,x64-windows-vmm-tests-archive}' - path: ${{ runner.temp }}/used_artifacts/ - - run: echo "${{ runner.temp }}/used_artifacts/_internal-flowey-bootstrap-x86_64-windows-uid-14" >> $GITHUB_PATH + path: flowey_bootstrap + - name: Build flowey + run: | + set -x + CARGO_INCREMENTAL=0 cargo build -p flowey_hvlite --target aarch64-unknown-linux-gnu --profile flowey-ci + OutDirNormal=$(echo "${{ runner.temp }}/bootstrapped-flowey" | sed -e 's|\\|\/|g' -e 's|^\([A-Za-z]\)\:/\(.*\)|/\L\1\E/\2|') + mkdir -p "$OutDirNormal" + mv ./.github/workflows/openvmm-pr.yaml "$OutDirNormal/pipeline.yaml" + mv target/aarch64-unknown-linux-gnu/flowey-ci/flowey_hvlite "$OutDirNormal/flowey" + working-directory: flowey_bootstrap + shell: bash + - run: echo "${{ runner.temp }}/bootstrapped-flowey" >> $GITHUB_PATH shell: bash name: πŸŒΌπŸ“¦ Add flowey to PATH + - name: πŸŒΌπŸ”Ž Self-check YAML + run: |- + ESCAPED_AGENT_TEMPDIR=$( + cat <<'EOF' | sed 's/\\/\\\\/g' + ${{ runner.temp }} + EOF + ) + flowey pipeline github --runtime $ESCAPED_AGENT_TEMPDIR/bootstrapped-flowey/pipeline.yaml --out .github/workflows/openvmm-pr.yaml ci checkin-gates --config=pr + shell: bash - name: πŸŒΌπŸ›« Initialize job run: | AgentTempDirNormal="${{ runner.temp }}" AgentTempDirNormal=$(echo "$AgentTempDirNormal" | sed -e 's|\\|\/|g' -e 's|^\([A-Za-z]\)\:/\(.*\)|/\L\1\E/\2|') echo "AgentTempDirNormal=$AgentTempDirNormal" >> $GITHUB_ENV - chmod +x $AgentTempDirNormal/used_artifacts/_internal-flowey-bootstrap-x86_64-windows-uid-14/flowey.exe + chmod +x $AgentTempDirNormal/bootstrapped-flowey/flowey - echo '"debug"' | flowey.exe v 18 'FLOWEY_LOG' update - echo "${{ runner.temp }}/work" | flowey.exe v 18 '_internal_WORKING_DIR' --is-raw-string update + echo '"debug"' | flowey v 18 'FLOWEY_LOG' update + echo "${{ runner.temp }}/work" | flowey v 18 '_internal_WORKING_DIR' --is-raw-string update - cat <<'EOF' | flowey.exe v 18 'verbose' update + cat <<'EOF' | flowey v 18 'verbose' update ${{ inputs.verbose != '' && inputs.verbose || 'false' }} EOF - echo "${{ runner.temp }}\\used_artifacts\\x64-guest_test_uefi" | flowey.exe v 18 'artifact_use_from_x64-guest_test_uefi' --is-raw-string update - echo "${{ runner.temp }}\\used_artifacts\\x64-linux-musl-pipette" | flowey.exe v 18 'artifact_use_from_x64-linux-musl-pipette' --is-raw-string update - echo "${{ runner.temp }}\\used_artifacts\\x64-linux-musl-tmk_vmm" | flowey.exe v 18 'artifact_use_from_x64-linux-musl-tmk_vmm' --is-raw-string update - echo "${{ runner.temp }}\\used_artifacts\\x64-linux-tpm_guest_tests" | flowey.exe v 18 'artifact_use_from_x64-linux-tpm_guest_tests' --is-raw-string update - echo "${{ runner.temp }}\\used_artifacts\\x64-openhcl-igvm" | flowey.exe v 18 'artifact_use_from_x64-openhcl-igvm' --is-raw-string update - echo "${{ runner.temp }}\\used_artifacts\\x64-tmks" | flowey.exe v 18 'artifact_use_from_x64-tmks' --is-raw-string update - echo "${{ runner.temp }}\\used_artifacts\\x64-windows-openvmm" | flowey.exe v 18 'artifact_use_from_x64-windows-openvmm' --is-raw-string update - echo "${{ runner.temp }}\\used_artifacts\\x64-windows-pipette" | flowey.exe v 18 'artifact_use_from_x64-windows-pipette' --is-raw-string update - echo "${{ runner.temp }}\\used_artifacts\\x64-windows-prep_steps" | flowey.exe v 18 'artifact_use_from_x64-windows-prep_steps' --is-raw-string update - echo "${{ runner.temp }}\\used_artifacts\\x64-windows-tmk_vmm" | flowey.exe v 18 'artifact_use_from_x64-windows-tmk_vmm' --is-raw-string update - echo "${{ runner.temp }}\\used_artifacts\\x64-windows-tpm_guest_tests" | flowey.exe v 18 'artifact_use_from_x64-windows-tpm_guest_tests' --is-raw-string update - echo "${{ runner.temp }}\\used_artifacts\\x64-windows-vmgstool" | flowey.exe v 18 'artifact_use_from_x64-windows-vmgstool' --is-raw-string update - echo "${{ runner.temp }}\\used_artifacts\\x64-windows-vmm-tests-archive" | flowey.exe v 18 'artifact_use_from_x64-windows-vmm-tests-archive' --is-raw-string update - shell: bash - - name: creating new test content dir - run: |- - flowey.exe e 18 flowey_core::pipeline::artifact::resolve 5 - flowey.exe e 18 flowey_core::pipeline::artifact::resolve 6 - flowey.exe e 18 flowey_core::pipeline::artifact::resolve 1 - flowey.exe e 18 flowey_core::pipeline::artifact::resolve 0 - flowey.exe e 18 flowey_core::pipeline::artifact::resolve 8 - flowey.exe e 18 flowey_core::pipeline::artifact::resolve 2 - flowey.exe e 18 flowey_core::pipeline::artifact::resolve 4 - flowey.exe e 18 flowey_core::pipeline::artifact::resolve 10 - flowey.exe e 18 flowey_core::pipeline::artifact::resolve 9 - flowey.exe e 18 flowey_core::pipeline::artifact::resolve 3 - flowey.exe e 18 flowey_lib_hvlite::_jobs::consume_and_test_nextest_vmm_tests_archive 0 - shell: bash - - name: resolve OpenHCL igvm artifact - run: flowey.exe e 18 flowey_lib_hvlite::artifact_openhcl_igvm_from_recipe::resolve 0 shell: bash - - name: create azcopy cache dir - run: flowey.exe e 18 flowey_lib_common::download_azcopy 0 + - name: add default cargo home to path + run: flowey e 18 flowey_lib_common::install_rust 0 shell: bash - - name: Pre-processing cache vars - run: |- - flowey.exe e 18 flowey_lib_common::cache 0 - flowey.exe v 18 'flowey_lib_common::cache:2:flowey_lib_common/src/cache.rs:407:72' --is-raw-string write-to-env github floweyvar4 - flowey.exe v 18 'flowey_lib_common::cache:1:flowey_lib_common/src/cache.rs:406:72' --is-raw-string write-to-env github floweyvar5 + - name: install Rust + run: flowey e 18 flowey_lib_common::install_rust 1 shell: bash - - id: flowey_lib_common__cache__1 - uses: actions/cache@v4 - with: - key: ${{ env.floweyvar4 }} - path: ${{ env.floweyvar5 }} - name: 'Restore cache: azcopy' - - name: installing azcopy + - name: detect active toolchain run: |- - flowey.exe v 18 'flowey_lib_common::cache:4:flowey_lib_common/src/cache.rs:462:70' --is-raw-string update --env-source steps.flowey_lib_common__cache__1.outputs.cache-hit <> "$GITHUB_PATH" + rustup show + if: runner.os == 'Linux' + name: rustup (Linux) + shell: bash + - run: | + set -x + curl --fail -sSfLo rustup-init.exe https://win.rustup.rs/x86_64 --output rustup-init + ./rustup-init.exe -y --default-toolchain=1.91.1 + echo "$USERPROFILE\\.cargo\\bin" >> $GITHUB_PATH + if: runner.os == 'Windows' && runner.arch == 'X64' + name: rustup (Windows X64) + shell: bash + - run: | + set -x + curl --fail -sSfLo rustup-init.exe https://win.rustup.rs/aarch64 --output rustup-init + ./rustup-init.exe -y --default-toolchain=1.91.1 + echo "$USERPROFILE\\.cargo\\bin" >> $GITHUB_PATH + if: runner.os == 'Windows' && runner.arch == 'ARM64' + name: rustup (Windows ARM64) + shell: bash + - uses: actions/checkout@v4 with: - pattern: '{_internal-flowey-bootstrap-x86_64-windows-uid-14,x64-guest_test_uefi,x64-linux-musl-pipette,x64-linux-musl-tmk_vmm,x64-linux-tpm_guest_tests,x64-openhcl-igvm,x64-tmks,x64-windows-openvmm,x64-windows-pipette,x64-windows-prep_steps,x64-windows-tmk_vmm,x64-windows-tpm_guest_tests,x64-windows-vmgstool,x64-windows-vmm-tests-archive}' - path: ${{ runner.temp }}/used_artifacts/ - - run: echo "${{ runner.temp }}/used_artifacts/_internal-flowey-bootstrap-x86_64-windows-uid-14" >> $GITHUB_PATH + path: flowey_bootstrap + - name: Build flowey + run: | + set -x + CARGO_INCREMENTAL=0 cargo build -p flowey_hvlite --target aarch64-unknown-linux-gnu --profile flowey-ci + OutDirNormal=$(echo "${{ runner.temp }}/bootstrapped-flowey" | sed -e 's|\\|\/|g' -e 's|^\([A-Za-z]\)\:/\(.*\)|/\L\1\E/\2|') + mkdir -p "$OutDirNormal" + mv ./.github/workflows/openvmm-pr.yaml "$OutDirNormal/pipeline.yaml" + mv target/aarch64-unknown-linux-gnu/flowey-ci/flowey_hvlite "$OutDirNormal/flowey" + working-directory: flowey_bootstrap + shell: bash + - run: echo "${{ runner.temp }}/bootstrapped-flowey" >> $GITHUB_PATH shell: bash name: πŸŒΌπŸ“¦ Add flowey to PATH + - name: πŸŒΌπŸ”Ž Self-check YAML + run: |- + ESCAPED_AGENT_TEMPDIR=$( + cat <<'EOF' | sed 's/\\/\\\\/g' + ${{ runner.temp }} + EOF + ) + flowey pipeline github --runtime $ESCAPED_AGENT_TEMPDIR/bootstrapped-flowey/pipeline.yaml --out .github/workflows/openvmm-pr.yaml ci checkin-gates --config=pr + shell: bash - name: πŸŒΌπŸ›« Initialize job run: | AgentTempDirNormal="${{ runner.temp }}" AgentTempDirNormal=$(echo "$AgentTempDirNormal" | sed -e 's|\\|\/|g' -e 's|^\([A-Za-z]\)\:/\(.*\)|/\L\1\E/\2|') echo "AgentTempDirNormal=$AgentTempDirNormal" >> $GITHUB_ENV - chmod +x $AgentTempDirNormal/used_artifacts/_internal-flowey-bootstrap-x86_64-windows-uid-14/flowey.exe + chmod +x $AgentTempDirNormal/bootstrapped-flowey/flowey - echo '"debug"' | flowey.exe v 19 'FLOWEY_LOG' update - echo "${{ runner.temp }}/work" | flowey.exe v 19 '_internal_WORKING_DIR' --is-raw-string update + echo '"debug"' | flowey v 19 'FLOWEY_LOG' update + echo "${{ runner.temp }}/work" | flowey v 19 '_internal_WORKING_DIR' --is-raw-string update - cat <<'EOF' | flowey.exe v 19 'verbose' update + cat <<'EOF' | flowey v 19 'verbose' update ${{ inputs.verbose != '' && inputs.verbose || 'false' }} EOF - echo "${{ runner.temp }}\\used_artifacts\\x64-guest_test_uefi" | flowey.exe v 19 'artifact_use_from_x64-guest_test_uefi' --is-raw-string update - echo "${{ runner.temp }}\\used_artifacts\\x64-linux-musl-pipette" | flowey.exe v 19 'artifact_use_from_x64-linux-musl-pipette' --is-raw-string update - echo "${{ runner.temp }}\\used_artifacts\\x64-linux-musl-tmk_vmm" | flowey.exe v 19 'artifact_use_from_x64-linux-musl-tmk_vmm' --is-raw-string update - echo "${{ runner.temp }}\\used_artifacts\\x64-linux-tpm_guest_tests" | flowey.exe v 19 'artifact_use_from_x64-linux-tpm_guest_tests' --is-raw-string update - echo "${{ runner.temp }}\\used_artifacts\\x64-openhcl-igvm" | flowey.exe v 19 'artifact_use_from_x64-openhcl-igvm' --is-raw-string update - echo "${{ runner.temp }}\\used_artifacts\\x64-tmks" | flowey.exe v 19 'artifact_use_from_x64-tmks' --is-raw-string update - echo "${{ runner.temp }}\\used_artifacts\\x64-windows-openvmm" | flowey.exe v 19 'artifact_use_from_x64-windows-openvmm' --is-raw-string update - echo "${{ runner.temp }}\\used_artifacts\\x64-windows-pipette" | flowey.exe v 19 'artifact_use_from_x64-windows-pipette' --is-raw-string update - echo "${{ runner.temp }}\\used_artifacts\\x64-windows-prep_steps" | flowey.exe v 19 'artifact_use_from_x64-windows-prep_steps' --is-raw-string update - echo "${{ runner.temp }}\\used_artifacts\\x64-windows-tmk_vmm" | flowey.exe v 19 'artifact_use_from_x64-windows-tmk_vmm' --is-raw-string update - echo "${{ runner.temp }}\\used_artifacts\\x64-windows-tpm_guest_tests" | flowey.exe v 19 'artifact_use_from_x64-windows-tpm_guest_tests' --is-raw-string update - echo "${{ runner.temp }}\\used_artifacts\\x64-windows-vmgstool" | flowey.exe v 19 'artifact_use_from_x64-windows-vmgstool' --is-raw-string update - echo "${{ runner.temp }}\\used_artifacts\\x64-windows-vmm-tests-archive" | flowey.exe v 19 'artifact_use_from_x64-windows-vmm-tests-archive' --is-raw-string update shell: bash - - name: creating new test content dir - run: |- - flowey.exe e 19 flowey_core::pipeline::artifact::resolve 5 - flowey.exe e 19 flowey_core::pipeline::artifact::resolve 6 - flowey.exe e 19 flowey_core::pipeline::artifact::resolve 1 - flowey.exe e 19 flowey_core::pipeline::artifact::resolve 0 - flowey.exe e 19 flowey_core::pipeline::artifact::resolve 8 - flowey.exe e 19 flowey_core::pipeline::artifact::resolve 2 - flowey.exe e 19 flowey_core::pipeline::artifact::resolve 4 - flowey.exe e 19 flowey_core::pipeline::artifact::resolve 10 - flowey.exe e 19 flowey_core::pipeline::artifact::resolve 9 - flowey.exe e 19 flowey_core::pipeline::artifact::resolve 3 - flowey.exe e 19 flowey_lib_hvlite::_jobs::consume_and_test_nextest_vmm_tests_archive 0 + - name: add default cargo home to path + run: flowey e 19 flowey_lib_common::install_rust 0 shell: bash - - name: resolve OpenHCL igvm artifact - run: flowey.exe e 19 flowey_lib_hvlite::artifact_openhcl_igvm_from_recipe::resolve 0 + - name: install Rust + run: flowey e 19 flowey_lib_common::install_rust 1 shell: bash - - name: create azcopy cache dir - run: flowey.exe e 19 flowey_lib_common::download_azcopy 0 + - name: detect active toolchain + run: |- + flowey e 19 flowey_lib_common::install_rust 2 + flowey e 19 flowey_lib_common::cfg_cargo_common_flags 0 shell: bash - - name: Pre-processing cache vars + - name: check if openvmm needs to be cloned run: |- - flowey.exe e 19 flowey_lib_common::cache 0 - flowey.exe v 19 'flowey_lib_common::cache:2:flowey_lib_common/src/cache.rs:407:72' --is-raw-string write-to-env github floweyvar4 - flowey.exe v 19 'flowey_lib_common::cache:1:flowey_lib_common/src/cache.rs:406:72' --is-raw-string write-to-env github floweyvar5 + flowey e 19 flowey_lib_common::git_checkout 0 + flowey v 19 'flowey_lib_common::git_checkout:0:flowey_lib_common/src/git_checkout.rs:469:80' --is-raw-string --condvar flowey_lib_common::git_checkout:1:flowey_lib_common/src/git_checkout.rs:470:46 write-to-env github floweyvar6 + flowey v 19 'flowey_lib_common::git_checkout:1:flowey_lib_common/src/git_checkout.rs:470:46' write-to-env github FLOWEY_CONDITION shell: bash - - id: flowey_lib_common__cache__1 - uses: actions/cache@v4 + - id: flowey_lib_common__git_checkout__1 + uses: actions/checkout@v4 with: - key: ${{ env.floweyvar4 }} - path: ${{ env.floweyvar5 }} - name: 'Restore cache: azcopy' - - name: installing azcopy + fetch-depth: '1' + path: repo0 + persist-credentials: ${{ env.floweyvar6 }} + name: checkout repo openvmm + if: ${{ fromJSON(env.FLOWEY_CONDITION) }} + - name: report cloned repo directories run: |- - flowey.exe v 19 'flowey_lib_common::cache:4:flowey_lib_common/src/cache.rs:462:70' --is-raw-string update --env-source steps.flowey_lib_common__cache__1.outputs.cache-hit <> $GITHUB_PATH + - run: echo "${{ runner.temp }}/used_artifacts/_internal-flowey-bootstrap-x86_64-windows-uid-16" >> $GITHUB_PATH shell: bash name: πŸŒΌπŸ“¦ Add flowey to PATH - name: πŸŒΌπŸ›« Initialize job @@ -3352,7 +3387,7 @@ jobs: AgentTempDirNormal=$(echo "$AgentTempDirNormal" | sed -e 's|\\|\/|g' -e 's|^\([A-Za-z]\)\:/\(.*\)|/\L\1\E/\2|') echo "AgentTempDirNormal=$AgentTempDirNormal" >> $GITHUB_ENV - chmod +x $AgentTempDirNormal/used_artifacts/_internal-flowey-bootstrap-x86_64-windows-uid-14/flowey.exe + chmod +x $AgentTempDirNormal/used_artifacts/_internal-flowey-bootstrap-x86_64-windows-uid-16/flowey.exe echo '"debug"' | flowey.exe v 20 'FLOWEY_LOG' update echo "${{ runner.temp }}/work" | flowey.exe v 20 '_internal_WORKING_DIR' --is-raw-string update @@ -3376,29 +3411,588 @@ jobs: shell: bash - name: creating new test content dir run: |- - flowey.exe e 20 flowey_core::pipeline::artifact::resolve 5 - flowey.exe e 20 flowey_core::pipeline::artifact::resolve 6 - flowey.exe e 20 flowey_core::pipeline::artifact::resolve 1 - flowey.exe e 20 flowey_core::pipeline::artifact::resolve 0 - flowey.exe e 20 flowey_core::pipeline::artifact::resolve 8 - flowey.exe e 20 flowey_core::pipeline::artifact::resolve 2 - flowey.exe e 20 flowey_core::pipeline::artifact::resolve 4 - flowey.exe e 20 flowey_core::pipeline::artifact::resolve 10 - flowey.exe e 20 flowey_core::pipeline::artifact::resolve 9 - flowey.exe e 20 flowey_core::pipeline::artifact::resolve 3 - flowey.exe e 20 flowey_lib_hvlite::_jobs::consume_and_test_nextest_vmm_tests_archive 0 + flowey.exe e 20 flowey_core::pipeline::artifact::resolve 5 + flowey.exe e 20 flowey_core::pipeline::artifact::resolve 6 + flowey.exe e 20 flowey_core::pipeline::artifact::resolve 1 + flowey.exe e 20 flowey_core::pipeline::artifact::resolve 0 + flowey.exe e 20 flowey_core::pipeline::artifact::resolve 8 + flowey.exe e 20 flowey_core::pipeline::artifact::resolve 2 + flowey.exe e 20 flowey_core::pipeline::artifact::resolve 4 + flowey.exe e 20 flowey_core::pipeline::artifact::resolve 10 + flowey.exe e 20 flowey_core::pipeline::artifact::resolve 9 + flowey.exe e 20 flowey_core::pipeline::artifact::resolve 3 + flowey.exe e 20 flowey_lib_hvlite::_jobs::consume_and_test_nextest_vmm_tests_archive 0 + shell: bash + - name: resolve OpenHCL igvm artifact + run: flowey.exe e 20 flowey_lib_hvlite::artifact_openhcl_igvm_from_recipe::resolve 0 + shell: bash + - name: create azcopy cache dir + run: flowey.exe e 20 flowey_lib_common::download_azcopy 0 + shell: bash + - name: Pre-processing cache vars + run: |- + flowey.exe e 20 flowey_lib_common::cache 0 + flowey.exe v 20 'flowey_lib_common::cache:2:flowey_lib_common/src/cache.rs:407:72' --is-raw-string write-to-env github floweyvar4 + flowey.exe v 20 'flowey_lib_common::cache:1:flowey_lib_common/src/cache.rs:406:72' --is-raw-string write-to-env github floweyvar5 + shell: bash + - id: flowey_lib_common__cache__1 + uses: actions/cache@v4 + with: + key: ${{ env.floweyvar4 }} + path: ${{ env.floweyvar5 }} + name: 'Restore cache: azcopy' + - name: installing azcopy + run: |- + flowey.exe v 20 'flowey_lib_common::cache:4:flowey_lib_common/src/cache.rs:462:70' --is-raw-string update --env-source steps.flowey_lib_common__cache__1.outputs.cache-hit <> $GITHUB_PATH + shell: bash + name: πŸŒΌπŸ“¦ Add flowey to PATH + - name: πŸŒΌπŸ›« Initialize job + run: | + AgentTempDirNormal="${{ runner.temp }}" + AgentTempDirNormal=$(echo "$AgentTempDirNormal" | sed -e 's|\\|\/|g' -e 's|^\([A-Za-z]\)\:/\(.*\)|/\L\1\E/\2|') + echo "AgentTempDirNormal=$AgentTempDirNormal" >> $GITHUB_ENV + + chmod +x $AgentTempDirNormal/used_artifacts/_internal-flowey-bootstrap-x86_64-windows-uid-16/flowey.exe + + echo '"debug"' | flowey.exe v 21 'FLOWEY_LOG' update + echo "${{ runner.temp }}/work" | flowey.exe v 21 '_internal_WORKING_DIR' --is-raw-string update + + cat <<'EOF' | flowey.exe v 21 'verbose' update + ${{ inputs.verbose != '' && inputs.verbose || 'false' }} + EOF + echo "${{ runner.temp }}\\used_artifacts\\x64-guest_test_uefi" | flowey.exe v 21 'artifact_use_from_x64-guest_test_uefi' --is-raw-string update + echo "${{ runner.temp }}\\used_artifacts\\x64-linux-musl-pipette" | flowey.exe v 21 'artifact_use_from_x64-linux-musl-pipette' --is-raw-string update + echo "${{ runner.temp }}\\used_artifacts\\x64-linux-musl-tmk_vmm" | flowey.exe v 21 'artifact_use_from_x64-linux-musl-tmk_vmm' --is-raw-string update + echo "${{ runner.temp }}\\used_artifacts\\x64-linux-tpm_guest_tests" | flowey.exe v 21 'artifact_use_from_x64-linux-tpm_guest_tests' --is-raw-string update + echo "${{ runner.temp }}\\used_artifacts\\x64-openhcl-igvm" | flowey.exe v 21 'artifact_use_from_x64-openhcl-igvm' --is-raw-string update + echo "${{ runner.temp }}\\used_artifacts\\x64-tmks" | flowey.exe v 21 'artifact_use_from_x64-tmks' --is-raw-string update + echo "${{ runner.temp }}\\used_artifacts\\x64-windows-openvmm" | flowey.exe v 21 'artifact_use_from_x64-windows-openvmm' --is-raw-string update + echo "${{ runner.temp }}\\used_artifacts\\x64-windows-pipette" | flowey.exe v 21 'artifact_use_from_x64-windows-pipette' --is-raw-string update + echo "${{ runner.temp }}\\used_artifacts\\x64-windows-prep_steps" | flowey.exe v 21 'artifact_use_from_x64-windows-prep_steps' --is-raw-string update + echo "${{ runner.temp }}\\used_artifacts\\x64-windows-tmk_vmm" | flowey.exe v 21 'artifact_use_from_x64-windows-tmk_vmm' --is-raw-string update + echo "${{ runner.temp }}\\used_artifacts\\x64-windows-tpm_guest_tests" | flowey.exe v 21 'artifact_use_from_x64-windows-tpm_guest_tests' --is-raw-string update + echo "${{ runner.temp }}\\used_artifacts\\x64-windows-vmgstool" | flowey.exe v 21 'artifact_use_from_x64-windows-vmgstool' --is-raw-string update + echo "${{ runner.temp }}\\used_artifacts\\x64-windows-vmm-tests-archive" | flowey.exe v 21 'artifact_use_from_x64-windows-vmm-tests-archive' --is-raw-string update + shell: bash + - name: creating new test content dir + run: |- + flowey.exe e 21 flowey_core::pipeline::artifact::resolve 5 + flowey.exe e 21 flowey_core::pipeline::artifact::resolve 6 + flowey.exe e 21 flowey_core::pipeline::artifact::resolve 1 + flowey.exe e 21 flowey_core::pipeline::artifact::resolve 0 + flowey.exe e 21 flowey_core::pipeline::artifact::resolve 8 + flowey.exe e 21 flowey_core::pipeline::artifact::resolve 2 + flowey.exe e 21 flowey_core::pipeline::artifact::resolve 4 + flowey.exe e 21 flowey_core::pipeline::artifact::resolve 10 + flowey.exe e 21 flowey_core::pipeline::artifact::resolve 9 + flowey.exe e 21 flowey_core::pipeline::artifact::resolve 3 + flowey.exe e 21 flowey_lib_hvlite::_jobs::consume_and_test_nextest_vmm_tests_archive 0 + shell: bash + - name: resolve OpenHCL igvm artifact + run: flowey.exe e 21 flowey_lib_hvlite::artifact_openhcl_igvm_from_recipe::resolve 0 + shell: bash + - name: create azcopy cache dir + run: flowey.exe e 21 flowey_lib_common::download_azcopy 0 + shell: bash + - name: Pre-processing cache vars + run: |- + flowey.exe e 21 flowey_lib_common::cache 0 + flowey.exe v 21 'flowey_lib_common::cache:2:flowey_lib_common/src/cache.rs:407:72' --is-raw-string write-to-env github floweyvar4 + flowey.exe v 21 'flowey_lib_common::cache:1:flowey_lib_common/src/cache.rs:406:72' --is-raw-string write-to-env github floweyvar5 + shell: bash + - id: flowey_lib_common__cache__1 + uses: actions/cache@v4 + with: + key: ${{ env.floweyvar4 }} + path: ${{ env.floweyvar5 }} + name: 'Restore cache: azcopy' + - name: installing azcopy + run: |- + flowey.exe v 21 'flowey_lib_common::cache:4:flowey_lib_common/src/cache.rs:462:70' --is-raw-string update --env-source steps.flowey_lib_common__cache__1.outputs.cache-hit <> $GITHUB_PATH + shell: bash + name: πŸŒΌπŸ“¦ Add flowey to PATH + - name: πŸŒΌπŸ›« Initialize job + run: | + AgentTempDirNormal="${{ runner.temp }}" + AgentTempDirNormal=$(echo "$AgentTempDirNormal" | sed -e 's|\\|\/|g' -e 's|^\([A-Za-z]\)\:/\(.*\)|/\L\1\E/\2|') + echo "AgentTempDirNormal=$AgentTempDirNormal" >> $GITHUB_ENV + + chmod +x $AgentTempDirNormal/used_artifacts/_internal-flowey-bootstrap-x86_64-windows-uid-16/flowey.exe + + echo '"debug"' | flowey.exe v 22 'FLOWEY_LOG' update + echo "${{ runner.temp }}/work" | flowey.exe v 22 '_internal_WORKING_DIR' --is-raw-string update + + cat <<'EOF' | flowey.exe v 22 'verbose' update + ${{ inputs.verbose != '' && inputs.verbose || 'false' }} + EOF + echo "${{ runner.temp }}\\used_artifacts\\x64-guest_test_uefi" | flowey.exe v 22 'artifact_use_from_x64-guest_test_uefi' --is-raw-string update + echo "${{ runner.temp }}\\used_artifacts\\x64-linux-musl-pipette" | flowey.exe v 22 'artifact_use_from_x64-linux-musl-pipette' --is-raw-string update + echo "${{ runner.temp }}\\used_artifacts\\x64-linux-musl-tmk_vmm" | flowey.exe v 22 'artifact_use_from_x64-linux-musl-tmk_vmm' --is-raw-string update + echo "${{ runner.temp }}\\used_artifacts\\x64-linux-tpm_guest_tests" | flowey.exe v 22 'artifact_use_from_x64-linux-tpm_guest_tests' --is-raw-string update + echo "${{ runner.temp }}\\used_artifacts\\x64-openhcl-igvm" | flowey.exe v 22 'artifact_use_from_x64-openhcl-igvm' --is-raw-string update + echo "${{ runner.temp }}\\used_artifacts\\x64-tmks" | flowey.exe v 22 'artifact_use_from_x64-tmks' --is-raw-string update + echo "${{ runner.temp }}\\used_artifacts\\x64-windows-openvmm" | flowey.exe v 22 'artifact_use_from_x64-windows-openvmm' --is-raw-string update + echo "${{ runner.temp }}\\used_artifacts\\x64-windows-pipette" | flowey.exe v 22 'artifact_use_from_x64-windows-pipette' --is-raw-string update + echo "${{ runner.temp }}\\used_artifacts\\x64-windows-prep_steps" | flowey.exe v 22 'artifact_use_from_x64-windows-prep_steps' --is-raw-string update + echo "${{ runner.temp }}\\used_artifacts\\x64-windows-tmk_vmm" | flowey.exe v 22 'artifact_use_from_x64-windows-tmk_vmm' --is-raw-string update + echo "${{ runner.temp }}\\used_artifacts\\x64-windows-tpm_guest_tests" | flowey.exe v 22 'artifact_use_from_x64-windows-tpm_guest_tests' --is-raw-string update + echo "${{ runner.temp }}\\used_artifacts\\x64-windows-vmgstool" | flowey.exe v 22 'artifact_use_from_x64-windows-vmgstool' --is-raw-string update + echo "${{ runner.temp }}\\used_artifacts\\x64-windows-vmm-tests-archive" | flowey.exe v 22 'artifact_use_from_x64-windows-vmm-tests-archive' --is-raw-string update + shell: bash + - name: creating new test content dir + run: |- + flowey.exe e 22 flowey_core::pipeline::artifact::resolve 5 + flowey.exe e 22 flowey_core::pipeline::artifact::resolve 6 + flowey.exe e 22 flowey_core::pipeline::artifact::resolve 1 + flowey.exe e 22 flowey_core::pipeline::artifact::resolve 0 + flowey.exe e 22 flowey_core::pipeline::artifact::resolve 8 + flowey.exe e 22 flowey_core::pipeline::artifact::resolve 2 + flowey.exe e 22 flowey_core::pipeline::artifact::resolve 4 + flowey.exe e 22 flowey_core::pipeline::artifact::resolve 10 + flowey.exe e 22 flowey_core::pipeline::artifact::resolve 9 + flowey.exe e 22 flowey_core::pipeline::artifact::resolve 3 + flowey.exe e 22 flowey_lib_hvlite::_jobs::consume_and_test_nextest_vmm_tests_archive 0 shell: bash - name: resolve OpenHCL igvm artifact - run: flowey.exe e 20 flowey_lib_hvlite::artifact_openhcl_igvm_from_recipe::resolve 0 + run: flowey.exe e 22 flowey_lib_hvlite::artifact_openhcl_igvm_from_recipe::resolve 0 shell: bash - name: create azcopy cache dir - run: flowey.exe e 20 flowey_lib_common::download_azcopy 0 + run: flowey.exe e 22 flowey_lib_common::download_azcopy 0 shell: bash - name: Pre-processing cache vars run: |- - flowey.exe e 20 flowey_lib_common::cache 0 - flowey.exe v 20 'flowey_lib_common::cache:2:flowey_lib_common/src/cache.rs:407:72' --is-raw-string write-to-env github floweyvar4 - flowey.exe v 20 'flowey_lib_common::cache:1:flowey_lib_common/src/cache.rs:406:72' --is-raw-string write-to-env github floweyvar5 + flowey.exe e 22 flowey_lib_common::cache 0 + flowey.exe v 22 'flowey_lib_common::cache:2:flowey_lib_common/src/cache.rs:407:72' --is-raw-string write-to-env github floweyvar4 + flowey.exe v 22 'flowey_lib_common::cache:1:flowey_lib_common/src/cache.rs:406:72' --is-raw-string write-to-env github floweyvar5 shell: bash - id: flowey_lib_common__cache__1 uses: actions/cache@v4 @@ -3408,28 +4002,28 @@ jobs: name: 'Restore cache: azcopy' - name: installing azcopy run: |- - flowey.exe v 20 'flowey_lib_common::cache:4:flowey_lib_common/src/cache.rs:462:70' --is-raw-string update --env-source steps.flowey_lib_common__cache__1.outputs.cache-hit <> $GITHUB_PATH + - run: echo "${{ runner.temp }}/used_artifacts/_internal-flowey-bootstrap-x86_64-windows-uid-16" >> $GITHUB_PATH shell: bash name: πŸŒΌπŸ“¦ Add flowey to PATH - name: πŸŒΌπŸ›« Initialize job @@ -3630,53 +4224,53 @@ jobs: AgentTempDirNormal=$(echo "$AgentTempDirNormal" | sed -e 's|\\|\/|g' -e 's|^\([A-Za-z]\)\:/\(.*\)|/\L\1\E/\2|') echo "AgentTempDirNormal=$AgentTempDirNormal" >> $GITHUB_ENV - chmod +x $AgentTempDirNormal/used_artifacts/_internal-flowey-bootstrap-x86_64-windows-uid-14/flowey.exe + chmod +x $AgentTempDirNormal/used_artifacts/_internal-flowey-bootstrap-x86_64-windows-uid-16/flowey.exe - echo '"debug"' | flowey.exe v 21 'FLOWEY_LOG' update - echo "${{ runner.temp }}/work" | flowey.exe v 21 '_internal_WORKING_DIR' --is-raw-string update + echo '"debug"' | flowey.exe v 23 'FLOWEY_LOG' update + echo "${{ runner.temp }}/work" | flowey.exe v 23 '_internal_WORKING_DIR' --is-raw-string update - cat <<'EOF' | flowey.exe v 21 'verbose' update + cat <<'EOF' | flowey.exe v 23 'verbose' update ${{ inputs.verbose != '' && inputs.verbose || 'false' }} EOF - echo "${{ runner.temp }}\\used_artifacts\\x64-guest_test_uefi" | flowey.exe v 21 'artifact_use_from_x64-guest_test_uefi' --is-raw-string update - echo "${{ runner.temp }}\\used_artifacts\\x64-linux-musl-pipette" | flowey.exe v 21 'artifact_use_from_x64-linux-musl-pipette' --is-raw-string update - echo "${{ runner.temp }}\\used_artifacts\\x64-linux-musl-tmk_vmm" | flowey.exe v 21 'artifact_use_from_x64-linux-musl-tmk_vmm' --is-raw-string update - echo "${{ runner.temp }}\\used_artifacts\\x64-linux-tpm_guest_tests" | flowey.exe v 21 'artifact_use_from_x64-linux-tpm_guest_tests' --is-raw-string update - echo "${{ runner.temp }}\\used_artifacts\\x64-openhcl-igvm" | flowey.exe v 21 'artifact_use_from_x64-openhcl-igvm' --is-raw-string update - echo "${{ runner.temp }}\\used_artifacts\\x64-tmks" | flowey.exe v 21 'artifact_use_from_x64-tmks' --is-raw-string update - echo "${{ runner.temp }}\\used_artifacts\\x64-windows-openvmm" | flowey.exe v 21 'artifact_use_from_x64-windows-openvmm' --is-raw-string update - echo "${{ runner.temp }}\\used_artifacts\\x64-windows-pipette" | flowey.exe v 21 'artifact_use_from_x64-windows-pipette' --is-raw-string update - echo "${{ runner.temp }}\\used_artifacts\\x64-windows-prep_steps" | flowey.exe v 21 'artifact_use_from_x64-windows-prep_steps' --is-raw-string update - echo "${{ runner.temp }}\\used_artifacts\\x64-windows-tmk_vmm" | flowey.exe v 21 'artifact_use_from_x64-windows-tmk_vmm' --is-raw-string update - echo "${{ runner.temp }}\\used_artifacts\\x64-windows-tpm_guest_tests" | flowey.exe v 21 'artifact_use_from_x64-windows-tpm_guest_tests' --is-raw-string update - echo "${{ runner.temp }}\\used_artifacts\\x64-windows-vmgstool" | flowey.exe v 21 'artifact_use_from_x64-windows-vmgstool' --is-raw-string update - echo "${{ runner.temp }}\\used_artifacts\\x64-windows-vmm-tests-archive" | flowey.exe v 21 'artifact_use_from_x64-windows-vmm-tests-archive' --is-raw-string update + echo "${{ runner.temp }}\\used_artifacts\\x64-guest_test_uefi" | flowey.exe v 23 'artifact_use_from_x64-guest_test_uefi' --is-raw-string update + echo "${{ runner.temp }}\\used_artifacts\\x64-linux-musl-pipette" | flowey.exe v 23 'artifact_use_from_x64-linux-musl-pipette' --is-raw-string update + echo "${{ runner.temp }}\\used_artifacts\\x64-linux-musl-tmk_vmm" | flowey.exe v 23 'artifact_use_from_x64-linux-musl-tmk_vmm' --is-raw-string update + echo "${{ runner.temp }}\\used_artifacts\\x64-linux-tpm_guest_tests" | flowey.exe v 23 'artifact_use_from_x64-linux-tpm_guest_tests' --is-raw-string update + echo "${{ runner.temp }}\\used_artifacts\\x64-openhcl-igvm" | flowey.exe v 23 'artifact_use_from_x64-openhcl-igvm' --is-raw-string update + echo "${{ runner.temp }}\\used_artifacts\\x64-tmks" | flowey.exe v 23 'artifact_use_from_x64-tmks' --is-raw-string update + echo "${{ runner.temp }}\\used_artifacts\\x64-windows-openvmm" | flowey.exe v 23 'artifact_use_from_x64-windows-openvmm' --is-raw-string update + echo "${{ runner.temp }}\\used_artifacts\\x64-windows-pipette" | flowey.exe v 23 'artifact_use_from_x64-windows-pipette' --is-raw-string update + echo "${{ runner.temp }}\\used_artifacts\\x64-windows-prep_steps" | flowey.exe v 23 'artifact_use_from_x64-windows-prep_steps' --is-raw-string update + echo "${{ runner.temp }}\\used_artifacts\\x64-windows-tmk_vmm" | flowey.exe v 23 'artifact_use_from_x64-windows-tmk_vmm' --is-raw-string update + echo "${{ runner.temp }}\\used_artifacts\\x64-windows-tpm_guest_tests" | flowey.exe v 23 'artifact_use_from_x64-windows-tpm_guest_tests' --is-raw-string update + echo "${{ runner.temp }}\\used_artifacts\\x64-windows-vmgstool" | flowey.exe v 23 'artifact_use_from_x64-windows-vmgstool' --is-raw-string update + echo "${{ runner.temp }}\\used_artifacts\\x64-windows-vmm-tests-archive" | flowey.exe v 23 'artifact_use_from_x64-windows-vmm-tests-archive' --is-raw-string update shell: bash - name: creating new test content dir run: |- - flowey.exe e 21 flowey_core::pipeline::artifact::resolve 5 - flowey.exe e 21 flowey_core::pipeline::artifact::resolve 6 - flowey.exe e 21 flowey_core::pipeline::artifact::resolve 1 - flowey.exe e 21 flowey_core::pipeline::artifact::resolve 0 - flowey.exe e 21 flowey_core::pipeline::artifact::resolve 8 - flowey.exe e 21 flowey_core::pipeline::artifact::resolve 2 - flowey.exe e 21 flowey_core::pipeline::artifact::resolve 4 - flowey.exe e 21 flowey_core::pipeline::artifact::resolve 10 - flowey.exe e 21 flowey_core::pipeline::artifact::resolve 9 - flowey.exe e 21 flowey_core::pipeline::artifact::resolve 3 - flowey.exe e 21 flowey_lib_hvlite::_jobs::consume_and_test_nextest_vmm_tests_archive 0 + flowey.exe e 23 flowey_core::pipeline::artifact::resolve 5 + flowey.exe e 23 flowey_core::pipeline::artifact::resolve 6 + flowey.exe e 23 flowey_core::pipeline::artifact::resolve 1 + flowey.exe e 23 flowey_core::pipeline::artifact::resolve 0 + flowey.exe e 23 flowey_core::pipeline::artifact::resolve 8 + flowey.exe e 23 flowey_core::pipeline::artifact::resolve 2 + flowey.exe e 23 flowey_core::pipeline::artifact::resolve 4 + flowey.exe e 23 flowey_core::pipeline::artifact::resolve 10 + flowey.exe e 23 flowey_core::pipeline::artifact::resolve 9 + flowey.exe e 23 flowey_core::pipeline::artifact::resolve 3 + flowey.exe e 23 flowey_lib_hvlite::_jobs::consume_and_test_nextest_vmm_tests_archive 0 shell: bash - name: resolve OpenHCL igvm artifact - run: flowey.exe e 21 flowey_lib_hvlite::artifact_openhcl_igvm_from_recipe::resolve 0 + run: flowey.exe e 23 flowey_lib_hvlite::artifact_openhcl_igvm_from_recipe::resolve 0 shell: bash - name: create azcopy cache dir - run: flowey.exe e 21 flowey_lib_common::download_azcopy 0 + run: flowey.exe e 23 flowey_lib_common::download_azcopy 0 shell: bash - name: Pre-processing cache vars run: |- - flowey.exe e 21 flowey_lib_common::cache 0 - flowey.exe v 21 'flowey_lib_common::cache:2:flowey_lib_common/src/cache.rs:407:72' --is-raw-string write-to-env github floweyvar4 - flowey.exe v 21 'flowey_lib_common::cache:1:flowey_lib_common/src/cache.rs:406:72' --is-raw-string write-to-env github floweyvar5 + flowey.exe e 23 flowey_lib_common::cache 0 + flowey.exe v 23 'flowey_lib_common::cache:2:flowey_lib_common/src/cache.rs:407:72' --is-raw-string write-to-env github floweyvar4 + flowey.exe v 23 'flowey_lib_common::cache:1:flowey_lib_common/src/cache.rs:406:72' --is-raw-string write-to-env github floweyvar5 shell: bash - id: flowey_lib_common__cache__1 uses: actions/cache@v4 @@ -3686,28 +4280,28 @@ jobs: name: 'Restore cache: azcopy' - name: installing azcopy run: |- - flowey.exe v 21 'flowey_lib_common::cache:4:flowey_lib_common/src/cache.rs:462:70' --is-raw-string update --env-source steps.flowey_lib_common__cache__1.outputs.cache-hit <> $GITHUB_PATH + - run: echo "${{ runner.temp }}/used_artifacts/_internal-flowey-bootstrap-x86_64-linux-uid-10" >> $GITHUB_PATH shell: bash name: πŸŒΌπŸ“¦ Add flowey to PATH - name: πŸŒΌπŸ›« Initialize job @@ -3911,40 +4505,40 @@ jobs: AgentTempDirNormal=$(echo "$AgentTempDirNormal" | sed -e 's|\\|\/|g' -e 's|^\([A-Za-z]\)\:/\(.*\)|/\L\1\E/\2|') echo "AgentTempDirNormal=$AgentTempDirNormal" >> $GITHUB_ENV - chmod +x $AgentTempDirNormal/used_artifacts/_internal-flowey-bootstrap-x86_64-linux-uid-9/flowey + chmod +x $AgentTempDirNormal/used_artifacts/_internal-flowey-bootstrap-x86_64-linux-uid-10/flowey - echo '"debug"' | flowey v 22 'FLOWEY_LOG' update - echo "${{ runner.temp }}/work" | flowey v 22 '_internal_WORKING_DIR' --is-raw-string update + echo '"debug"' | flowey v 24 'FLOWEY_LOG' update + echo "${{ runner.temp }}/work" | flowey v 24 '_internal_WORKING_DIR' --is-raw-string update - cat <<'EOF' | flowey v 22 'verbose' update + cat <<'EOF' | flowey v 24 'verbose' update ${{ inputs.verbose != '' && inputs.verbose || 'false' }} EOF - echo "$AgentTempDirNormal/used_artifacts/x64-guest_test_uefi" | flowey v 22 'artifact_use_from_x64-guest_test_uefi' --is-raw-string update - echo "$AgentTempDirNormal/used_artifacts/x64-linux-musl-pipette" | flowey v 22 'artifact_use_from_x64-linux-musl-pipette' --is-raw-string update - echo "$AgentTempDirNormal/used_artifacts/x64-linux-musl-tmk_vmm" | flowey v 22 'artifact_use_from_x64-linux-musl-tmk_vmm' --is-raw-string update - echo "$AgentTempDirNormal/used_artifacts/x64-linux-openvmm" | flowey v 22 'artifact_use_from_x64-linux-openvmm' --is-raw-string update - echo "$AgentTempDirNormal/used_artifacts/x64-linux-vmm-tests-archive" | flowey v 22 'artifact_use_from_x64-linux-vmm-tests-archive' --is-raw-string update - echo "$AgentTempDirNormal/used_artifacts/x64-tmks" | flowey v 22 'artifact_use_from_x64-tmks' --is-raw-string update - echo "$AgentTempDirNormal/used_artifacts/x64-windows-pipette" | flowey v 22 'artifact_use_from_x64-windows-pipette' --is-raw-string update + echo "$AgentTempDirNormal/used_artifacts/x64-guest_test_uefi" | flowey v 24 'artifact_use_from_x64-guest_test_uefi' --is-raw-string update + echo "$AgentTempDirNormal/used_artifacts/x64-linux-musl-pipette" | flowey v 24 'artifact_use_from_x64-linux-musl-pipette' --is-raw-string update + echo "$AgentTempDirNormal/used_artifacts/x64-linux-musl-tmk_vmm" | flowey v 24 'artifact_use_from_x64-linux-musl-tmk_vmm' --is-raw-string update + echo "$AgentTempDirNormal/used_artifacts/x64-linux-openvmm" | flowey v 24 'artifact_use_from_x64-linux-openvmm' --is-raw-string update + echo "$AgentTempDirNormal/used_artifacts/x64-linux-vmm-tests-archive" | flowey v 24 'artifact_use_from_x64-linux-vmm-tests-archive' --is-raw-string update + echo "$AgentTempDirNormal/used_artifacts/x64-tmks" | flowey v 24 'artifact_use_from_x64-tmks' --is-raw-string update + echo "$AgentTempDirNormal/used_artifacts/x64-windows-pipette" | flowey v 24 'artifact_use_from_x64-windows-pipette' --is-raw-string update shell: bash - name: creating new test content dir run: |- - flowey e 22 flowey_core::pipeline::artifact::resolve 3 - flowey e 22 flowey_core::pipeline::artifact::resolve 6 - flowey e 22 flowey_core::pipeline::artifact::resolve 1 - flowey e 22 flowey_core::pipeline::artifact::resolve 0 - flowey e 22 flowey_core::pipeline::artifact::resolve 2 - flowey e 22 flowey_core::pipeline::artifact::resolve 5 - flowey e 22 flowey_lib_hvlite::_jobs::consume_and_test_nextest_vmm_tests_archive 0 + flowey e 24 flowey_core::pipeline::artifact::resolve 3 + flowey e 24 flowey_core::pipeline::artifact::resolve 6 + flowey e 24 flowey_core::pipeline::artifact::resolve 1 + flowey e 24 flowey_core::pipeline::artifact::resolve 0 + flowey e 24 flowey_core::pipeline::artifact::resolve 2 + flowey e 24 flowey_core::pipeline::artifact::resolve 5 + flowey e 24 flowey_lib_hvlite::_jobs::consume_and_test_nextest_vmm_tests_archive 0 shell: bash - name: create azcopy cache dir - run: flowey e 22 flowey_lib_common::download_azcopy 0 + run: flowey e 24 flowey_lib_common::download_azcopy 0 shell: bash - name: Pre-processing cache vars run: |- - flowey e 22 flowey_lib_common::cache 0 - flowey v 22 'flowey_lib_common::cache:2:flowey_lib_common/src/cache.rs:407:72' --is-raw-string write-to-env github floweyvar4 - flowey v 22 'flowey_lib_common::cache:1:flowey_lib_common/src/cache.rs:406:72' --is-raw-string write-to-env github floweyvar5 + flowey e 24 flowey_lib_common::cache 0 + flowey v 24 'flowey_lib_common::cache:2:flowey_lib_common/src/cache.rs:407:72' --is-raw-string write-to-env github floweyvar4 + flowey v 24 'flowey_lib_common::cache:1:flowey_lib_common/src/cache.rs:406:72' --is-raw-string write-to-env github floweyvar5 shell: bash - id: flowey_lib_common__cache__1 uses: actions/cache@v4 @@ -3954,34 +4548,34 @@ jobs: name: 'Restore cache: azcopy' - name: checking if packages need to be installed run: |- - flowey v 22 'flowey_lib_common::cache:4:flowey_lib_common/src/cache.rs:462:70' --is-raw-string update --env-source steps.flowey_lib_common__cache__1.outputs.cache-hit <> $GITHUB_PATH + - run: echo "${{ runner.temp }}/used_artifacts/_internal-flowey-bootstrap-x86_64-linux-uid-10" >> $GITHUB_PATH shell: bash name: πŸŒΌπŸ“¦ Add flowey to PATH - name: πŸŒΌπŸ›« Initialize job @@ -4635,17 +5231,17 @@ jobs: AgentTempDirNormal=$(echo "$AgentTempDirNormal" | sed -e 's|\\|\/|g' -e 's|^\([A-Za-z]\)\:/\(.*\)|/\L\1\E/\2|') echo "AgentTempDirNormal=$AgentTempDirNormal" >> $GITHUB_ENV - chmod +x $AgentTempDirNormal/used_artifacts/_internal-flowey-bootstrap-x86_64-linux-uid-9/flowey + chmod +x $AgentTempDirNormal/used_artifacts/_internal-flowey-bootstrap-x86_64-linux-uid-10/flowey - echo '"debug"' | flowey v 25 'FLOWEY_LOG' update - echo "${{ runner.temp }}/work" | flowey v 25 '_internal_WORKING_DIR' --is-raw-string update + echo '"debug"' | flowey v 27 'FLOWEY_LOG' update + echo "${{ runner.temp }}/work" | flowey v 27 '_internal_WORKING_DIR' --is-raw-string update - cat <<'EOF' | flowey v 25 'verbose' update + cat <<'EOF' | flowey v 27 'verbose' update ${{ inputs.verbose != '' && inputs.verbose || 'false' }} EOF shell: bash - name: Check if any jobs failed - run: flowey e 25 flowey_lib_hvlite::_jobs::all_good_job 0 + run: flowey e 27 flowey_lib_hvlite::_jobs::all_good_job 0 shell: bash job3: name: build artifacts (for VMM tests) [aarch64-windows] @@ -5469,7 +6065,7 @@ jobs: - name: 🌼πŸ₯Ύ Publish bootstrapped flowey uses: actions/upload-artifact@v4 with: - name: _internal-flowey-bootstrap-x86_64-windows-uid-14 + name: _internal-flowey-bootstrap-x86_64-windows-uid-16 path: ${{ runner.temp }}/bootstrapped-flowey job6: name: build artifacts [aarch64-linux] @@ -6348,7 +6944,7 @@ jobs: flowey e 8 flowey_lib_hvlite::build_openhcl_igvm_from_recipe 18 shell: bash - name: unpack openvmm-deps archive - run: flowey e 8 flowey_lib_hvlite::download_openvmm_deps 0 + run: flowey e 8 flowey_lib_hvlite::resolve_openvmm_deps 0 shell: bash - name: extract Aarch64 sysroot.tar.gz run: |- @@ -6491,12 +7087,17 @@ jobs: path: ${{ runner.temp }}/publish_artifacts/aarch64-openhcl-igvm-extras/ include-hidden-files: true job9: - name: verify openhcl binary size [aarch64] - runs-on: ubuntu-latest + name: build openhcl [aarch64-linux] with nix + runs-on: + - self-hosted + - 1ES.Pool=OpenVMM-GitHub-Linux-Pool-WestUS3 + - 1ES.ImageOverride=MMSUbuntu22.04-256GB permissions: contents: read id-token: write if: github.event.pull_request.draft == false + env: + USING_NIX: '1' steps: - run: | set -x @@ -6564,22 +7165,21 @@ jobs: cat <<'EOF' | flowey v 9 'verbose' update ${{ inputs.verbose != '' && inputs.verbose || 'false' }} EOF + mkdir -p "$AgentTempDirNormal/publish_artifacts/aarch64-openhcl-igvm-extras-nix" + echo "$AgentTempDirNormal/publish_artifacts/aarch64-openhcl-igvm-extras-nix" | flowey v 9 'artifact_publish_from_aarch64-openhcl-igvm-extras-nix' --is-raw-string update + mkdir -p "$AgentTempDirNormal/publish_artifacts/aarch64-openhcl-igvm-nix" + echo "$AgentTempDirNormal/publish_artifacts/aarch64-openhcl-igvm-nix" | flowey v 9 'artifact_publish_from_aarch64-openhcl-igvm-nix' --is-raw-string update shell: bash - - name: add default cargo home to path - run: flowey e 9 flowey_lib_common::install_rust 0 - shell: bash - - name: install Rust - run: flowey e 9 flowey_lib_common::install_rust 1 + - name: add nix profile to path + run: flowey e 9 flowey_lib_common::install_nix 0 shell: bash - - name: detect active toolchain - run: |- - flowey e 9 flowey_lib_common::install_rust 2 - flowey e 9 flowey_lib_common::cfg_cargo_common_flags 0 + - name: install Nix + run: flowey e 9 flowey_lib_common::install_nix 1 shell: bash - name: check if openvmm needs to be cloned run: |- flowey e 9 flowey_lib_common::git_checkout 0 - flowey v 9 'flowey_lib_common::git_checkout:0:flowey_lib_common/src/git_checkout.rs:469:80' --is-raw-string --condvar flowey_lib_common::git_checkout:1:flowey_lib_common/src/git_checkout.rs:470:46 write-to-env github floweyvar2 + flowey v 9 'flowey_lib_common::git_checkout:0:flowey_lib_common/src/git_checkout.rs:469:80' --is-raw-string --condvar flowey_lib_common::git_checkout:1:flowey_lib_common/src/git_checkout.rs:470:46 write-to-env github floweyvar1 flowey v 9 'flowey_lib_common::git_checkout:1:flowey_lib_common/src/git_checkout.rs:470:46' write-to-env github FLOWEY_CONDITION shell: bash - id: flowey_lib_common__git_checkout__1 @@ -6587,7 +7187,7 @@ jobs: with: fetch-depth: '1' path: repo0 - persist-credentials: ${{ env.floweyvar2 }} + persist-credentials: ${{ env.floweyvar1 }} name: checkout repo openvmm if: ${{ fromJSON(env.FLOWEY_CONDITION) }} - name: report cloned repo directories @@ -6597,134 +7197,156 @@ jobs: EOF flowey e 9 flowey_lib_common::git_checkout 3 flowey e 9 flowey_lib_hvlite::git_checkout_openvmm_repo 0 - flowey e 9 flowey_lib_hvlite::cfg_openvmm_magicpath 0 shell: bash - - name: checking if packages need to be installed - run: flowey e 9 flowey_lib_common::install_dist_pkg 0 + - name: resolve nix dependency paths + run: flowey e 9 flowey_lib_common::nix_deps_provider 0 shell: bash - - name: installing packages - run: flowey e 9 flowey_lib_common::install_dist_pkg 1 + - name: use local UEFI firmware + run: flowey e 9 flowey_lib_hvlite::download_uefi_mu_msvm 0 shell: bash - - name: create gh-release-download cache dir - run: flowey e 9 flowey_lib_common::download_gh_release 0 + - name: add default cargo home to path + run: flowey e 9 flowey_lib_common::install_rust 0 shell: bash - - name: Pre-processing cache vars - run: |- - flowey e 9 flowey_lib_common::cache 4 - flowey v 9 'flowey_lib_common::cache:10:flowey_lib_common/src/cache.rs:407:72' --is-raw-string write-to-env github floweyvar5 - flowey v 9 'flowey_lib_common::cache:9:flowey_lib_common/src/cache.rs:406:72' --is-raw-string write-to-env github floweyvar6 + - name: install Rust + run: flowey e 9 flowey_lib_common::install_rust 1 shell: bash - - id: flowey_lib_common__cache__5 - uses: actions/cache@v4 - with: - key: ${{ env.floweyvar5 }} - path: ${{ env.floweyvar6 }} - name: 'Restore cache: gh-release-download' - - name: download artifacts from github releases + - name: detect active toolchain run: |- - flowey v 9 'flowey_lib_common::cache:12:flowey_lib_common/src/cache.rs:462:70' --is-raw-string update --env-source steps.flowey_lib_common__cache__5.outputs.cache-hit < Result { + crate::node::shell::FloweyShell::new(self.platform) + } + /// Write a value. /// /// If this step has already read a secret value, then this will be diff --git a/flowey/flowey_core/src/node/shell.rs b/flowey/flowey_core/src/node/shell.rs new file mode 100644 index 0000000000..8f8b0f4b7b --- /dev/null +++ b/flowey/flowey_core/src/node/shell.rs @@ -0,0 +1,271 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +//! FloweyShell: A wrapper around xshell::Shell that automatically wraps commands +//! in nix-shell when running on the Nix platform. + +use crate::node::{FlowPlatform, FlowPlatformLinuxDistro}; +use std::collections::BTreeMap; +use std::ops::Deref; + +/// A wrapper around `xshell::Shell` that automatically wraps commands in nix-shell +/// when the platform is configured to use Nix. +/// +/// This allows pipeline-level configuration of nix usage without requiring individual +/// command sites to handle nix-shell wrapping manually. +pub struct FloweyShell { + inner: xshell::Shell, + platform: FlowPlatform, +} + +impl FloweyShell { + /// Create a new FloweyShell for the given platform. + /// + /// The shell will automatically wrap commands in nix-shell if the platform + /// is `FlowPlatform::Linux(FlowPlatformLinuxDistro::Nix)`. + pub fn new(platform: FlowPlatform) -> Result { + Ok(Self { + inner: xshell::Shell::new()?, + platform, + }) + } + + /// Get a reference to the inner xshell::Shell. + /// + /// This can be used for accessing the shell's methods directly, + /// such as `change_dir`, `current_dir`, etc. + pub fn inner(&self) -> &xshell::Shell { + &self.inner + } + + /// Check if commands should be wrapped in nix-shell. + fn needs_nix_wrapper(&self) -> bool { + // Only wrap in nix-shell if the platform is Nix AND we're not already in a nix-shell + // IN_NIX_SHELL is set when we're already inside a nix-shell (scenario 2) + // USING_NIX=1 is set when we want to use nix-shell but aren't in one yet (scenario 3) + matches!( + self.platform, + FlowPlatform::Linux(FlowPlatformLinuxDistro::Nix) + ) && std::env::var("IN_NIX_SHELL").is_err() + } + + /// Run a command with the given arguments and environment variables. + /// + /// If the platform is Nix, the command will be automatically wrapped in + /// `nix-shell --pure --run`. + /// + /// # Arguments + /// * `program` - The program to run (e.g., "cargo") + /// * `args` - Arguments to pass to the program + /// * `env_vars` - Environment variables to set for the command + pub fn run_cmd( + &self, + program: &str, + args: &[String], + env_vars: &BTreeMap, + ) -> Result<(), xshell::Error> { + if self.needs_nix_wrapper() { + self.run_in_nix_shell(program, args, env_vars) + } else { + let mut cmd = xshell::cmd!(self.inner, "{program} {args...}"); + cmd = cmd.envs(env_vars); + cmd.run() + } + } + + /// Run a command and capture its stdout as a String. + /// + /// If the platform is Nix, the command will be automatically wrapped in + /// `nix-shell --pure --run`. + /// + /// # Arguments + /// * `program` - The program to run (e.g., "cargo") + /// * `args` - Arguments to pass to the program + /// * `env_vars` - Environment variables to set for the command + pub fn read_cmd( + &self, + program: &str, + args: &[String], + env_vars: &BTreeMap, + ) -> Result { + if self.needs_nix_wrapper() { + self.read_in_nix_shell(program, args, env_vars) + } else { + let mut cmd = xshell::cmd!(self.inner, "{program} {args...}"); + cmd = cmd.envs(env_vars); + cmd.read() + } + } + + /// Run a command and capture its output (stdout, stderr, exit code). + /// + /// If the platform is Nix, the command will be automatically wrapped in + /// `nix-shell --pure --run`. + /// + /// # Arguments + /// * `program` - The program to run (e.g., "cargo") + /// * `args` - Arguments to pass to the program + /// * `env_vars` - Environment variables to set for the command + pub fn output_cmd( + &self, + program: &str, + args: &[String], + env_vars: &BTreeMap, + ) -> Result { + if self.needs_nix_wrapper() { + self.output_in_nix_shell(program, args, env_vars) + } else { + let mut cmd = xshell::cmd!(self.inner, "{program} {args...}"); + cmd = cmd.envs(env_vars); + cmd.output() + } + } + + /// Execute a command wrapped in nix-shell. + fn run_in_nix_shell( + &self, + program: &str, + args: &[String], + env_vars: &BTreeMap, + ) -> Result<(), xshell::Error> { + let full_cmd = build_shell_command(program, args, env_vars); + xshell::cmd!(self.inner, "nix-shell --pure --run {full_cmd}").run() + } + + /// Execute a command wrapped in nix-shell and capture stdout. + fn read_in_nix_shell( + &self, + program: &str, + args: &[String], + env_vars: &BTreeMap, + ) -> Result { + let full_cmd = build_shell_command(program, args, env_vars); + xshell::cmd!(self.inner, "nix-shell --pure --run {full_cmd}").read() + } + + /// Execute a command wrapped in nix-shell and capture output. + fn output_in_nix_shell( + &self, + program: &str, + args: &[String], + env_vars: &BTreeMap, + ) -> Result { + let full_cmd = build_shell_command(program, args, env_vars); + xshell::cmd!(self.inner, "nix-shell --pure --run {full_cmd}").output() + } +} + +impl Deref for FloweyShell { + type Target = xshell::Shell; + + fn deref(&self) -> &Self::Target { + &self.inner + } +} + +/// Build a shell command string from program, args, and environment variables. +/// +/// The resulting string is safe to pass to xshell::cmd! with {variable} interpolation, +/// which will handle proper shell escaping. +fn build_shell_command( + program: &str, + args: &[String], + env_vars: &BTreeMap, +) -> String { + // Build environment prefix (KEY=value KEY2=value2 ...) + let env_prefix = env_vars + .iter() + .map(|(k, v)| format!("{k}={v}")) + .collect::>() + .join(" "); + + // Build command line (program arg1 arg2 ...) + let cmd_line = if args.is_empty() { + program.to_string() + } else { + format!("{} {}", program, args.join(" ")) + }; + + // Combine environment and command + if env_prefix.is_empty() { + cmd_line + } else { + format!("{env_prefix} {cmd_line}") + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_build_shell_command_simple() { + let cmd = build_shell_command("cargo", &["build".into()], &BTreeMap::new()); + assert_eq!(cmd, "cargo build"); + } + + #[test] + fn test_build_shell_command_no_args() { + let cmd = build_shell_command("cargo", &[], &BTreeMap::new()); + assert_eq!(cmd, "cargo"); + } + + #[test] + fn test_build_shell_command_multiple_args() { + let args = vec!["build".into(), "--release".into(), "--verbose".into()]; + let cmd = build_shell_command("cargo", &args, &BTreeMap::new()); + assert_eq!(cmd, "cargo build --release --verbose"); + } + + #[test] + fn test_build_shell_command_with_env() { + let mut env = BTreeMap::new(); + env.insert("RUST_LOG".into(), "debug".into()); + let cmd = build_shell_command("cargo", &["build".into()], &env); + assert_eq!(cmd, "RUST_LOG=debug cargo build"); + } + + #[test] + fn test_build_shell_command_with_multiple_env() { + let mut env = BTreeMap::new(); + env.insert("RUST_LOG".into(), "debug".into()); + env.insert("CARGO_INCREMENTAL".into(), "0".into()); + let cmd = build_shell_command("cargo", &["build".into()], &env); + // BTreeMap maintains sorted order by key + assert_eq!(cmd, "CARGO_INCREMENTAL=0 RUST_LOG=debug cargo build"); + } + + #[test] + fn test_build_shell_command_env_and_args() { + let mut env = BTreeMap::new(); + env.insert("KEY".into(), "value".into()); + let args = vec!["test".into(), "--lib".into()]; + let cmd = build_shell_command("cargo", &args, &env); + assert_eq!(cmd, "KEY=value cargo test --lib"); + } + + #[test] + fn test_needs_nix_wrapper_for_nix_platform() { + let shell = FloweyShell::new(FlowPlatform::Linux(FlowPlatformLinuxDistro::Nix)).unwrap(); + assert!(shell.needs_nix_wrapper()); + } + + #[test] + fn test_needs_nix_wrapper_for_ubuntu() { + let shell = FloweyShell::new(FlowPlatform::Linux(FlowPlatformLinuxDistro::Ubuntu)).unwrap(); + assert!(!shell.needs_nix_wrapper()); + } + + #[test] + fn test_needs_nix_wrapper_for_windows() { + let shell = FloweyShell::new(FlowPlatform::Windows).unwrap(); + assert!(!shell.needs_nix_wrapper()); + } + + #[test] + fn test_deref_to_inner_shell() { + let shell = FloweyShell::new(FlowPlatform::Linux(FlowPlatformLinuxDistro::Ubuntu)).unwrap(); + // Test that we can call xshell::Shell methods via Deref + let _current_dir = shell.current_dir(); + } +} diff --git a/flowey/flowey_core/src/pipeline.rs b/flowey/flowey_core/src/pipeline.rs index 94c3363f8a..17389dbe77 100644 --- a/flowey/flowey_core/src/pipeline.rs +++ b/flowey/flowey_core/src/pipeline.rs @@ -61,6 +61,16 @@ pub mod user_facing { } fn linux_distro() -> FlowPlatformLinuxDistro { + // Check for Nix environment first + // USING_NIX=1 is set in CI, IN_NIX_SHELL is set when inside nix-shell + if std::env::var("USING_NIX") + .map(|v| v == "1") + .unwrap_or(false) + || std::env::var("IN_NIX_SHELL").is_ok() + { + return FlowPlatformLinuxDistro::Nix; + } + if let Ok(etc_os_release) = fs_err::read_to_string("/etc/os-release") { if etc_os_release.contains("ID=ubuntu") { FlowPlatformLinuxDistro::Ubuntu diff --git a/flowey/flowey_hvlite/src/pipelines/build_docs.rs b/flowey/flowey_hvlite/src/pipelines/build_docs.rs index 7d6ccdec7e..ee37fdb56d 100644 --- a/flowey/flowey_hvlite/src/pipelines/build_docs.rs +++ b/flowey/flowey_hvlite/src/pipelines/build_docs.rs @@ -89,7 +89,7 @@ impl IntoPipeline for BuildDocsCli { pipeline.inject_all_jobs_with(move |job| { job.dep_on(&cfg_common_params) - .dep_on(|_| flowey_lib_hvlite::_jobs::cfg_versions::Request {}) + .dep_on(|_| flowey_lib_hvlite::_jobs::cfg_versions::Request::Download) .dep_on( |_| flowey_lib_hvlite::_jobs::cfg_hvlite_reposource::Params { hvlite_repo_source: openvmm_repo_source.clone(), diff --git a/flowey/flowey_hvlite/src/pipelines/build_igvm.rs b/flowey/flowey_hvlite/src/pipelines/build_igvm.rs index 1d65300406..c12090d778 100644 --- a/flowey/flowey_hvlite/src/pipelines/build_igvm.rs +++ b/flowey/flowey_hvlite/src/pipelines/build_igvm.rs @@ -175,6 +175,19 @@ pub struct BuildIgvmCliCustomizations { /// to `trace` for debug builds and `debug` for release builds. #[clap(long)] pub max_trace_level: Option, + + /// (experimental) Only use local dependencies to build. Keeps flowey from + /// downloading any dependencies from the internet. + #[clap(long, requires_all = ["custom_openvmm_deps", "custom_protoc"])] + pub use_local_deps: bool, + + /// Use a custom openvmm_deps directory. + #[clap(long)] + pub custom_openvmm_deps: Option, + + /// Use a custom protoc directory. + #[clap(long)] + pub custom_protoc: Option, } #[derive(clap::ValueEnum, Copy, Clone, PartialEq, Eq, Debug)] @@ -294,6 +307,9 @@ impl IntoPipeline for BuildIgvmCli { custom_sidecar, mut custom_extra_rootfs, max_trace_level, + custom_openvmm_deps, + custom_protoc, + use_local_deps, }, } = self; @@ -305,80 +321,110 @@ impl IntoPipeline for BuildIgvmCli { let (pub_out_dir, _) = pipeline.new_artifact("build-igvm"); - pipeline - .new_job( - FlowPlatform::host(backend_hint), - FlowArch::host(backend_hint), - "build-igvm", - ) - .dep_on(|_| flowey_lib_hvlite::_jobs::cfg_versions::Request {}) - .dep_on( - |_| flowey_lib_hvlite::_jobs::cfg_hvlite_reposource::Params { - hvlite_repo_source: openvmm_repo, - }, - ) - .dep_on(|_| flowey_lib_hvlite::_jobs::cfg_common::Params { - local_only: Some(flowey_lib_hvlite::_jobs::cfg_common::LocalOnlyParams { - interactive: true, - auto_install: install_missing_deps, - force_nuget_mono: false, // no oss nuget packages - external_nuget_auth: false, - ignore_rust_version: true, - }), - verbose: ReadVar::from_static(verbose), - locked, - deny_warnings: false, - }) - .dep_on(|ctx| flowey_lib_hvlite::_jobs::local_build_igvm::Params { - artifact_dir: ctx.publish_artifact(pub_out_dir), - done: ctx.new_done_handle(), - - base_recipe: match recipe { - OpenhclRecipeCli::X64 => OpenhclIgvmRecipe::X64, - OpenhclRecipeCli::X64Devkern => OpenhclIgvmRecipe::X64Devkern, - OpenhclRecipeCli::X64TestLinuxDirect => OpenhclIgvmRecipe::X64TestLinuxDirect, - OpenhclRecipeCli::X64TestLinuxDirectDevkern => { - OpenhclIgvmRecipe::X64TestLinuxDirectDevkern - } - OpenhclRecipeCli::X64Cvm => OpenhclIgvmRecipe::X64Cvm, - OpenhclRecipeCli::X64CvmDevkern => OpenhclIgvmRecipe::X64CvmDevkern, - OpenhclRecipeCli::Aarch64 => OpenhclIgvmRecipe::Aarch64, - OpenhclRecipeCli::Aarch64Devkern => OpenhclIgvmRecipe::Aarch64Devkern, - }, - release, - release_cfg, + // Determine the architecture from the recipe + let recipe_arch = match recipe { + OpenhclRecipeCli::X64 + | OpenhclRecipeCli::X64Devkern + | OpenhclRecipeCli::X64Cvm + | OpenhclRecipeCli::X64CvmDevkern + | OpenhclRecipeCli::X64TestLinuxDirect + | OpenhclRecipeCli::X64TestLinuxDirectDevkern => CommonArch::X86_64, + OpenhclRecipeCli::Aarch64 | OpenhclRecipeCli::Aarch64Devkern => CommonArch::Aarch64, + }; + + let mut job = pipeline.new_job( + FlowPlatform::host(backend_hint), + FlowArch::host(backend_hint), + "build-igvm", + ); - customizations: flowey_lib_hvlite::_jobs::local_build_igvm::Customizations { - build_label, - override_arch: override_arch.map(|a| match a { - BuildIgvmArch::X86_64 => CommonArch::X86_64, - BuildIgvmArch::Aarch64 => CommonArch::Aarch64, - }), - with_perf_tools, - with_debuginfo, - override_kernel_pkg: override_kernel_pkg.map(|p| match p { - KernelPackageKindCli::Main => OpenhclKernelPackage::Main, - KernelPackageKindCli::Cvm => OpenhclKernelPackage::Cvm, - KernelPackageKindCli::Dev => OpenhclKernelPackage::Dev, - KernelPackageKindCli::CvmDev => OpenhclKernelPackage::CvmDev, - }), - with_sidecar, - custom_extra_rootfs, - override_openvmm_hcl_feature, - custom_sidecar, - override_manifest, - override_max_trace_level: max_trace_level.map(Into::into), - custom_openvmm_hcl, - custom_openhcl_boot, - custom_uefi, - custom_kernel, - custom_kernel_modules, - custom_vtl0_kernel, - custom_layer, - custom_directory, - }, + job = if use_local_deps { + let openvmm_deps_path = + custom_openvmm_deps.expect("must specify openvmm deps path to use local deps"); + let protoc_path = custom_protoc.expect("must specify protoc path to use local deps"); + + // Wrap with ReadVar::from_static() for CLI arguments + let openvmm_deps_var = ReadVar::from_static(openvmm_deps_path); + let protoc_var = ReadVar::from_static(protoc_path); + + job.dep_on(move |_| { + flowey_lib_hvlite::_jobs::cfg_versions::Request::Local( + recipe_arch, + openvmm_deps_var, + protoc_var, + ) }) - .finish(); + } else { + job.dep_on(|_| flowey_lib_hvlite::_jobs::cfg_versions::Request::Download) + }; + + job.dep_on( + |_| flowey_lib_hvlite::_jobs::cfg_hvlite_reposource::Params { + hvlite_repo_source: openvmm_repo, + }, + ) + .dep_on(|_| flowey_lib_hvlite::_jobs::cfg_common::Params { + local_only: Some(flowey_lib_hvlite::_jobs::cfg_common::LocalOnlyParams { + interactive: true, + auto_install: install_missing_deps, + force_nuget_mono: false, // no oss nuget packages + external_nuget_auth: false, + ignore_rust_version: true, + }), + verbose: ReadVar::from_static(verbose), + locked, + deny_warnings: false, + }) + .dep_on(|ctx| flowey_lib_hvlite::_jobs::local_build_igvm::Params { + artifact_dir: ctx.publish_artifact(pub_out_dir), + done: ctx.new_done_handle(), + + base_recipe: match recipe { + OpenhclRecipeCli::X64 => OpenhclIgvmRecipe::X64, + OpenhclRecipeCli::X64Devkern => OpenhclIgvmRecipe::X64Devkern, + OpenhclRecipeCli::X64TestLinuxDirect => OpenhclIgvmRecipe::X64TestLinuxDirect, + OpenhclRecipeCli::X64TestLinuxDirectDevkern => { + OpenhclIgvmRecipe::X64TestLinuxDirectDevkern + } + OpenhclRecipeCli::X64Cvm => OpenhclIgvmRecipe::X64Cvm, + OpenhclRecipeCli::X64CvmDevkern => OpenhclIgvmRecipe::X64CvmDevkern, + OpenhclRecipeCli::Aarch64 => OpenhclIgvmRecipe::Aarch64, + OpenhclRecipeCli::Aarch64Devkern => OpenhclIgvmRecipe::Aarch64Devkern, + }, + release, + release_cfg, + + customizations: flowey_lib_hvlite::_jobs::local_build_igvm::Customizations { + build_label, + override_arch: override_arch.map(|a| match a { + BuildIgvmArch::X86_64 => CommonArch::X86_64, + BuildIgvmArch::Aarch64 => CommonArch::Aarch64, + }), + with_perf_tools, + with_debuginfo, + override_kernel_pkg: override_kernel_pkg.map(|p| match p { + KernelPackageKindCli::Main => OpenhclKernelPackage::Main, + KernelPackageKindCli::Cvm => OpenhclKernelPackage::Cvm, + KernelPackageKindCli::Dev => OpenhclKernelPackage::Dev, + KernelPackageKindCli::CvmDev => OpenhclKernelPackage::CvmDev, + }), + with_sidecar, + custom_extra_rootfs, + override_openvmm_hcl_feature, + custom_sidecar, + override_manifest, + override_max_trace_level: max_trace_level.map(Into::into), + custom_openvmm_hcl, + custom_openhcl_boot, + custom_uefi, + custom_kernel, + custom_kernel_modules, + custom_vtl0_kernel, + custom_layer, + custom_directory, + }, + }) + .finish(); Ok(pipeline) } diff --git a/flowey/flowey_hvlite/src/pipelines/checkin_gates.rs b/flowey/flowey_hvlite/src/pipelines/checkin_gates.rs index 7c9796c045..8ac6bb7ff9 100644 --- a/flowey/flowey_hvlite/src/pipelines/checkin_gates.rs +++ b/flowey/flowey_hvlite/src/pipelines/checkin_gates.rs @@ -123,7 +123,7 @@ impl IntoPipeline for CheckinGatesCli { pipeline.inject_all_jobs_with(move |job| { let mut job = job .dep_on(&cfg_common_params) - .dep_on(|_| flowey_lib_hvlite::_jobs::cfg_versions::Request {}) + .dep_on(|_| flowey_lib_hvlite::_jobs::cfg_versions::Request::Download) .dep_on( |_| flowey_lib_hvlite::_jobs::cfg_hvlite_reposource::Params { hvlite_repo_source: openvmm_repo_source.clone(), @@ -598,6 +598,11 @@ impl IntoPipeline for CheckinGatesCli { let (pub_openhcl_igvm_extras, _use_openhcl_igvm_extras) = pipeline.new_artifact(format!("{arch_tag}-openhcl-igvm-extras")); + let (pub_openhcl_igvm_nix, _use_openhcl_igvm_nix) = + pipeline.new_artifact(format!("{arch_tag}-openhcl-igvm-nix")); + let (pub_openhcl_igvm_extras_nix, _use_openhcl_igvm_extras_nix) = + pipeline.new_artifact(format!("{arch_tag}-openhcl-igvm-extras-nix")); + let (pub_openhcl_baseline, _use_openhcl_baseline) = if matches!(config, PipelineConfig::Ci) { let (p, u) = pipeline.new_artifact(format!("{arch_tag}-openhcl-baseline")); @@ -705,6 +710,41 @@ impl IntoPipeline for CheckinGatesCli { all_jobs.push(job.finish()); + let job = pipeline + .new_job( + FlowPlatform::Linux(FlowPlatformLinuxDistro::Ubuntu), + FlowArch::X86_64, + format!("{} with nix", build_openhcl_job_tag(arch_tag)), + ) + .gh_set_pool(crate::pipelines_shared::gh_pools::linux_self_hosted_largedisk()) + .gh_dangerous_global_env_var("USING_NIX", "1") + .dep_on(|ctx| { + flowey_lib_common::install_nix::Request::EnsureInstalled(ctx.new_done_handle()) + }) + .dep_on(|_| flowey_lib_hvlite::_jobs::cfg_versions::Request::NixEnvironment) + .dep_on(|ctx| { + flowey_lib_hvlite::_jobs::build_and_publish_openhcl_igvm_from_recipe::Params { + igvm_files: igvm_recipes + .clone() + .into_iter() + .map(|recipe| OpenhclIgvmBuildParams { + profile: openvmm_hcl_profile, + recipe, + custom_target: Some(CommonTriple::Custom(openhcl_musl_target( + arch, + ))), + }) + .collect(), + artifact_dir_openhcl_igvm: ctx.publish_artifact(pub_openhcl_igvm_nix), + artifact_dir_openhcl_igvm_extras: ctx + .publish_artifact(pub_openhcl_igvm_extras_nix), + artifact_openhcl_verify_size_baseline: None, + done: ctx.new_done_handle(), + } + }); + + all_jobs.push(job.finish()); + if matches!(config, PipelineConfig::Pr) { let job = pipeline .new_job( diff --git a/flowey/flowey_hvlite/src/pipelines/custom_vmfirmwareigvm_dll.rs b/flowey/flowey_hvlite/src/pipelines/custom_vmfirmwareigvm_dll.rs index 2781b404c2..15288cf221 100644 --- a/flowey/flowey_hvlite/src/pipelines/custom_vmfirmwareigvm_dll.rs +++ b/flowey/flowey_hvlite/src/pipelines/custom_vmfirmwareigvm_dll.rs @@ -68,7 +68,7 @@ impl IntoPipeline for CustomVmfirmwareigvmDllCli { FlowArch::host(backend_hint), "custom-vmfirmwareigvm-dll", ) - .dep_on(|_| flowey_lib_hvlite::_jobs::cfg_versions::Request {}) + .dep_on(|_| flowey_lib_hvlite::_jobs::cfg_versions::Request::Download) .dep_on( |_| flowey_lib_hvlite::_jobs::cfg_hvlite_reposource::Params { hvlite_repo_source: openvmm_repo, diff --git a/flowey/flowey_hvlite/src/pipelines/mod.rs b/flowey/flowey_hvlite/src/pipelines/mod.rs index 4cf48a3036..54865221e2 100644 --- a/flowey/flowey_hvlite/src/pipelines/mod.rs +++ b/flowey/flowey_hvlite/src/pipelines/mod.rs @@ -13,7 +13,7 @@ pub mod restore_packages; pub mod vmm_tests; #[derive(clap::Subcommand)] -#[cfg_attr(windows, expect(clippy::large_enum_variant))] +#[expect(clippy::large_enum_variant)] pub enum OpenvmmPipelines { /// Alias for root-level `regen` command. // DEVNOTE: this enables the useful `cargo xflowey regen` alias diff --git a/flowey/flowey_hvlite/src/pipelines/restore_packages.rs b/flowey/flowey_hvlite/src/pipelines/restore_packages.rs index 7c929e2f8f..57a6c164e5 100644 --- a/flowey/flowey_hvlite/src/pipelines/restore_packages.rs +++ b/flowey/flowey_hvlite/src/pipelines/restore_packages.rs @@ -28,7 +28,7 @@ impl IntoPipeline for RestorePackagesCli { FlowArch::host(backend_hint), "restore packages", ) - .dep_on(|_| flowey_lib_hvlite::_jobs::cfg_versions::Request {}) + .dep_on(|_| flowey_lib_hvlite::_jobs::cfg_versions::Request::Download) .dep_on( |_| flowey_lib_hvlite::_jobs::cfg_hvlite_reposource::Params { hvlite_repo_source: openvmm_repo, diff --git a/flowey/flowey_hvlite/src/pipelines/vmm_tests.rs b/flowey/flowey_hvlite/src/pipelines/vmm_tests.rs index 8175fff573..0660e57ec6 100644 --- a/flowey/flowey_hvlite/src/pipelines/vmm_tests.rs +++ b/flowey/flowey_hvlite/src/pipelines/vmm_tests.rs @@ -135,7 +135,7 @@ impl IntoPipeline for VmmTestsCli { FlowArch::host(backend_hint), "build vmm test dependencies", ) - .dep_on(|_| flowey_lib_hvlite::_jobs::cfg_versions::Request {}) + .dep_on(|_| flowey_lib_hvlite::_jobs::cfg_versions::Request::Download) .dep_on( |_| flowey_lib_hvlite::_jobs::cfg_hvlite_reposource::Params { hvlite_repo_source: openvmm_repo.clone(), diff --git a/flowey/flowey_lib_common/src/_util/extract.rs b/flowey/flowey_lib_common/src/_util/extract.rs index 9244154101..388347a880 100644 --- a/flowey/flowey_lib_common/src/_util/extract.rs +++ b/flowey/flowey_lib_common/src/_util/extract.rs @@ -39,6 +39,7 @@ pub fn extract_zip_if_new_deps(ctx: &mut NodeCtx<'_>) -> ExtractZipDeps { FlowPlatformLinuxDistro::Fedora => vec!["bsdtar".into()], FlowPlatformLinuxDistro::Ubuntu => vec!["libarchive-tools".into()], FlowPlatformLinuxDistro::Arch => vec!["libarchive".into()], + FlowPlatformLinuxDistro::Nix => vec!["bsdtar".into()], FlowPlatformLinuxDistro::Unknown => vec![], }, _ => { @@ -68,7 +69,7 @@ pub fn extract_zip_if_new( bsdtar_installed: _, } = deps; - let sh = xshell::Shell::new()?; + let sh = rt.shell()?; let root_dir = match persistent_dir { Some(dir) => rt.read(dir), diff --git a/flowey/flowey_lib_common/src/download_azcopy.rs b/flowey/flowey_lib_common/src/download_azcopy.rs index 5a19791fc4..0dcc01864b 100644 --- a/flowey/flowey_lib_common/src/download_azcopy.rs +++ b/flowey/flowey_lib_common/src/download_azcopy.rs @@ -70,6 +70,7 @@ impl FlowNode for Node { FlowPlatformLinuxDistro::Fedora => vec!["bsdtar".into()], FlowPlatformLinuxDistro::Ubuntu => vec!["libarchive-tools".into()], FlowPlatformLinuxDistro::Arch => vec!["libarchive".into()], + FlowPlatformLinuxDistro::Nix => vec!["bsdtar".into()], FlowPlatformLinuxDistro::Unknown => vec![], }, _ => { diff --git a/flowey/flowey_lib_common/src/download_protoc.rs b/flowey/flowey_lib_common/src/download_protoc.rs index 18b3e2235c..f0601a2b19 100644 --- a/flowey/flowey_lib_common/src/download_protoc.rs +++ b/flowey/flowey_lib_common/src/download_protoc.rs @@ -13,6 +13,8 @@ pub struct ProtocPackage { flowey_request! { pub enum Request { + /// Use a locally downloaded protoc + LocalPath(ReadVar), /// What version to download (e.g: 27.1) Version(String), /// Return paths to items in the protoc package @@ -33,16 +35,26 @@ impl FlowNode for Node { fn emit(requests: Vec, ctx: &mut NodeCtx<'_>) -> anyhow::Result<()> { let mut version = None; + let mut local_path: Option> = None; let mut get_reqs = Vec::new(); for req in requests { match req { + Request::LocalPath(path) => { + same_across_all_reqs_backing_var("LocalPath", &mut local_path, path)? + } Request::Version(v) => same_across_all_reqs("Version", &mut version, v)?, Request::Get(v) => get_reqs.push(v), } } - let version = version.ok_or(anyhow::anyhow!("Missing essential request: Version"))?; + if version.is_some() && local_path.is_some() { + anyhow::bail!("Cannot specify both Version and LocalPath requests"); + } + + if version.is_none() && local_path.is_none() { + anyhow::bail!("Must specify a Version or LocalPath request"); + } // -- end of req processing -- // @@ -50,6 +62,43 @@ impl FlowNode for Node { return Ok(()); } + if let Some(local_path) = local_path { + ctx.emit_rust_step("use local protoc", |ctx| { + let get_reqs = get_reqs.claim(ctx); + let local_path = local_path.claim(ctx); + move |rt| { + // Read the ReadVar to get the actual PathBuf + let local_path = rt.read(local_path); + + let protoc_bin = local_path + .join("bin") + .join(rt.platform().binary("protoc")) + .absolute()?; + + assert!(protoc_bin.exists()); + + // Don't try to make executable - local paths (especially from nix store) + // should already be executable and may be read-only + + let protoc_includes = local_path.join("include").absolute()?; + assert!(protoc_includes.exists()); + + let pkg = ProtocPackage { + protoc_bin, + include_dir: protoc_includes, + }; + + rt.write_all(get_reqs, &pkg); + + Ok(()) + } + }); + + return Ok(()); + } + + let version = version.expect("local requests handled above"); + let tag = format!("v{version}"); let file_name = format!( "protoc-{}-{}.zip", diff --git a/flowey/flowey_lib_common/src/install_dist_pkg.rs b/flowey/flowey_lib_common/src/install_dist_pkg.rs index e456d7e9d3..9ff545f835 100644 --- a/flowey/flowey_lib_common/src/install_dist_pkg.rs +++ b/flowey/flowey_lib_common/src/install_dist_pkg.rs @@ -51,6 +51,23 @@ impl PackageManager { fn query_cmd(&self, packages_to_check: &BTreeSet) -> anyhow::Result> { let Self { distro, sh } = self; + // For Nix, check if commands exist on PATH + if matches!(distro, FlowPlatformLinuxDistro::Nix) { + let mut available = BTreeSet::new(); + for pkg in packages_to_check { + // Assume package name == binary name (works for python3, git, etc.) + if xshell::cmd!(sh, "command -v {pkg}") + .ignore_status() + .quiet() + .run() + .is_ok() + { + available.insert(pkg.clone()); + } + } + return Ok(available); + } + let output = match distro { FlowPlatformLinuxDistro::Ubuntu => { let fmt = "${binary:Package}\n"; @@ -63,6 +80,7 @@ impl PackageManager { FlowPlatformLinuxDistro::Arch => { xshell::cmd!(sh, "pacman -Qq {packages_to_check...}") } + FlowPlatformLinuxDistro::Nix => unreachable!("handled above"), FlowPlatformLinuxDistro::Unknown => anyhow::bail!("Unknown Linux distribution"), } .ignore_status() @@ -90,6 +108,8 @@ impl PackageManager { FlowPlatformLinuxDistro::Fedora => xshell::cmd!(sh, "sudo dnf update").run()?, // Running `pacman -Sy` without a full system update can break everything; do nothing FlowPlatformLinuxDistro::Arch => (), + // Nix doesn't need updates - packages are provided by shell.nix + FlowPlatformLinuxDistro::Nix => (), FlowPlatformLinuxDistro::Unknown => anyhow::bail!("Unknown Linux distribution"), } @@ -118,6 +138,17 @@ impl PackageManager { let auto_accept = (!interactive).then_some("--noconfirm"); xshell::cmd!(sh, "sudo pacman -S {auto_accept...} {packages...}").run()?; } + FlowPlatformLinuxDistro::Nix => { + log::warn!( + "===============================================================================" + ); + log::warn!("Missing packages in Nix environment: {:?}", packages); + log::warn!("Please add these to your shell.nix buildInputs"); + log::warn!("Continuing anyway - build may fail if these are actually needed"); + log::warn!( + "===============================================================================" + ); + } FlowPlatformLinuxDistro::Unknown => anyhow::bail!("Unknown Linux distribution"), } diff --git a/flowey/flowey_lib_common/src/install_nix.rs b/flowey/flowey_lib_common/src/install_nix.rs new file mode 100644 index 0000000000..ea6544e389 --- /dev/null +++ b/flowey/flowey_lib_common/src/install_nix.rs @@ -0,0 +1,114 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +//! Install the Nix package manager and ensure nix-shell is available on $PATH + +use flowey::node::prelude::*; +use std::io::Write; + +new_flow_node!(struct Node); + +flowey_request! { + pub enum Request { + /// Automatically install Nix package manager. + /// + /// Supported on Github and ADO backends. + AutoInstall(bool), + + /// Ensure that Nix was installed and is available on the $PATH + EnsureInstalled(WriteVar), + } +} + +impl FlowNode for Node { + type Request = Request; + + fn imports(_ctx: &mut ImportCtx<'_>) {} + + fn emit(requests: Vec, ctx: &mut NodeCtx<'_>) -> anyhow::Result<()> { + let mut ensure_installed = Vec::new(); + let mut auto_install = None; + + for req in requests { + match req { + Request::EnsureInstalled(v) => ensure_installed.push(v), + Request::AutoInstall(v) => { + same_across_all_reqs("AutoInstall", &mut auto_install, v)? + } + } + } + + let ensure_installed = ensure_installed; + let auto_install = + auto_install.ok_or(anyhow::anyhow!("Missing essential request: AutoInstall"))?; + + // -- end of req processing -- // + + if !ensure_installed.is_empty() && auto_install { + // Add nix profile bin to PATH first + let added_to_path = ctx.emit_rust_step("add nix profile to path", |ctx| { + let backend = ctx.backend(); + move |_| { + let nix_profile_bin = home::home_dir() + .context("Unable to get home dir")? + .join(".nix-profile") + .join("bin"); + + match backend { + FlowBackend::Github => { + let github_path = std::env::var("GITHUB_PATH")?; + let mut github_path = + fs_err::File::options().append(true).open(github_path)?; + github_path + .write_all(nix_profile_bin.as_os_str().as_encoded_bytes())?; + github_path.write_all(b"\n")?; + log::info!("Added {} to PATH (Github)", nix_profile_bin.display()); + } + FlowBackend::Ado => { + // ADO uses logging commands to update PATH + println!("##vso[task.prependpath]{}", nix_profile_bin.display()); + log::info!("Added {} to PATH (ADO)", nix_profile_bin.display()); + } + FlowBackend::Local => { + log::warn!("Cannot automatically add to PATH in local backend"); + log::warn!( + "Please add {} to your PATH manually", + nix_profile_bin.display() + ); + } + } + + Ok(()) + } + }); + + ctx.emit_rust_step("install Nix", |ctx| { + ensure_installed.claim(ctx); + added_to_path.claim(ctx); + + move |_rt: &mut RustRuntimeServices<'_>| { + // Check if nix-shell is already available + if which::which("nix-shell").is_ok() { + log::info!("nix-shell already available on PATH"); + return Ok(()); + } + + log::info!("Installing Nix package manager..."); + let sh = xshell::Shell::new()?; + + // Download and run the Nix installer script (single-user mode) + xshell::cmd!( + sh, + "sh -c 'curl --proto =https --tlsv1.2 -L https://nixos.org/nix/install | sh -s -- --no-daemon'" + ) + .run()?; + + log::info!("Nix installed successfully"); + Ok(()) + } + }); + } + + Ok(()) + } +} diff --git a/flowey/flowey_lib_common/src/lib.rs b/flowey/flowey_lib_common/src/lib.rs index c72ded3af4..6d9d0dceb2 100644 --- a/flowey/flowey_lib_common/src/lib.rs +++ b/flowey/flowey_lib_common/src/lib.rs @@ -42,9 +42,11 @@ pub mod install_azure_cli; pub mod install_cargo_nextest; pub mod install_dist_pkg; pub mod install_git; +pub mod install_nix; pub mod install_nodejs; pub mod install_nuget_azure_credential_provider; pub mod install_rust; +pub mod nix_deps_provider; pub mod nuget_install_package; pub mod publish_test_results; pub mod run_cargo_build; diff --git a/flowey/flowey_lib_common/src/nix_deps_provider.rs b/flowey/flowey_lib_common/src/nix_deps_provider.rs new file mode 100644 index 0000000000..1f98fc8338 --- /dev/null +++ b/flowey/flowey_lib_common/src/nix_deps_provider.rs @@ -0,0 +1,170 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +//! Provide paths to dependencies managed by Nix. +//! +//! This node reads Nix environment variables and converts them to flowey's +//! ReadVar system. It's used when building in a Nix environment +//! (USING_NIX=1) to pass Nix store paths to flowey jobs. + +use flowey::node::prelude::*; +use std::collections::BTreeMap; + +#[derive(Serialize, Deserialize, Copy, Clone, Debug, PartialEq, Eq, PartialOrd, Ord)] +pub enum OpenvmmDepsArch { + X86_64, + Aarch64, +} + +flowey_request! { + pub enum Request { + /// Set the path to the repo containing shell.nix + SetRepoPath(ReadVar), + GetOpenvmmDeps(OpenvmmDepsArch, WriteVar), + GetProtoc(WriteVar), + GetKernel(WriteVar), + GetUefiMuMsvm(WriteVar), + } +} + +new_flow_node!(struct Node); + +impl FlowNode for Node { + type Request = Request; + + fn imports(ctx: &mut ImportCtx<'_>) { + ctx.import::(); + } + + fn emit(requests: Vec, ctx: &mut NodeCtx<'_>) -> anyhow::Result<()> { + let mut repo_path = None; + let mut openvmm_deps_requests: BTreeMap>> = + BTreeMap::new(); + let mut protoc_requests = Vec::new(); + let mut openhcl_kernel_requests = Vec::new(); + let mut uefi_mu_msvm_requests = Vec::new(); + + let nix_installed = ctx.reqv(crate::install_nix::Request::EnsureInstalled); + + // Parse all requests and group by type + for req in requests { + match req { + Request::SetRepoPath(path) => { + same_across_all_reqs_backing_var("SetRepoPath", &mut repo_path, path)?; + } + Request::GetOpenvmmDeps(arch, var) => { + openvmm_deps_requests.entry(arch).or_default().push(var); + } + Request::GetProtoc(var) => protoc_requests.push(var), + Request::GetKernel(var) => openhcl_kernel_requests.push(var), + Request::GetUefiMuMsvm(var) => uefi_mu_msvm_requests.push(var), + } + } + + let repo_path = repo_path.context("Missing SetRepoPath request")?; + + // Only emit step if there are actual requests + if openvmm_deps_requests.is_empty() + && protoc_requests.is_empty() + && openhcl_kernel_requests.is_empty() + && uefi_mu_msvm_requests.is_empty() + { + return Ok(()); + } + + ctx.emit_rust_step("resolve nix dependency paths", |ctx| { + nix_installed.claim(ctx); + let repo_path = repo_path.claim(ctx); + let openvmm_deps_requests = openvmm_deps_requests.claim(ctx); + let protoc_requests = protoc_requests.claim(ctx); + let openhcl_kernel_vmlinux_requests = openhcl_kernel_requests.claim(ctx); + let uefi_mu_msvm_requests = uefi_mu_msvm_requests.claim(ctx); + + move |rt| { + let repo_path = rt.read(repo_path); + let sh = xshell::Shell::new()?; + sh.change_dir(&repo_path); + + // Helper function to get environment variable from nix-shell + let get_nix_env_var = |var_name: &str| -> anyhow::Result { + let cmd_str = format!("echo ${}", var_name); + let output = xshell::cmd!(sh, "nix-shell --pure --run {cmd_str}") + .output() + .context(format!("Failed to run nix-shell to get {}", var_name))?; + + if !output.status.success() { + anyhow::bail!( + "nix-shell command failed for {}: {}", + var_name, + String::from_utf8_lossy(&output.stderr) + ); + } + + let value = String::from_utf8(output.stdout)?.trim().to_string(); + + if value.is_empty() { + anyhow::bail!( + "{} not set in nix-shell environment. Check shell.nix", + var_name + ); + } + + Ok(value) + }; + + // Read and write openvmm_deps to all requesting vars + if !openvmm_deps_requests.is_empty() { + let openvmm_deps = get_nix_env_var("NIX_OPENVMM_DEPS")?; + let openvmm_deps_path = PathBuf::from(&openvmm_deps); + + // Note: In Nix, the same package is used for both x64 and aarch64 at build time + for (arch, vars) in openvmm_deps_requests { + log::info!( + "Resolved Nix openvmm_deps for {:?}: {}", + arch, + openvmm_deps_path.display() + ); + rt.write_all(vars, &openvmm_deps_path); + } + } + + // Read and write protoc path if requested + if !protoc_requests.is_empty() { + let protoc_path = get_nix_env_var("NIX_PROTOC_PATH")?; + let protoc_path = PathBuf::from(&protoc_path); + + log::info!("Resolved Nix protoc: {}", protoc_path.display()); + rt.write_all(protoc_requests, &protoc_path); + } + + // Read and write openhcl_kernel vmlinux path if requested + if !openhcl_kernel_vmlinux_requests.is_empty() { + let kernel_vmlinux = get_nix_env_var("NIX_OPENHCL_KERNEL")?; + let kernel_vmlinux_path = PathBuf::from(&kernel_vmlinux); + + log::info!( + "Resolved Nix openhcl_kernel: {}", + kernel_vmlinux_path.display() + ); + rt.write_all(openhcl_kernel_vmlinux_requests, &kernel_vmlinux_path); + } + + // Read and write UEFI firmware path if requested + if !uefi_mu_msvm_requests.is_empty() { + let uefi_mu_msvm = get_nix_env_var("NIX_UEFI_MU_MSVM")?; + let uefi_mu_msvm_path = PathBuf::from(&uefi_mu_msvm); + + log::info!( + "Resolved Nix UEFI firmware: {}", + uefi_mu_msvm_path.display() + ); + rt.write_all(uefi_mu_msvm_requests, &uefi_mu_msvm_path); + } + + Ok(()) + } + }); + + Ok(()) + } +} diff --git a/flowey/flowey_lib_common/src/run_cargo_build.rs b/flowey/flowey_lib_common/src/run_cargo_build.rs index ef4a3916bb..f423711bb8 100644 --- a/flowey/flowey_lib_common/src/run_cargo_build.rs +++ b/flowey/flowey_lib_common/src/run_cargo_build.rs @@ -261,12 +261,15 @@ impl FlowNode for Node { crate_type, } = cmd; - let sh = xshell::Shell::new()?; + let sh = rt.shell()?; let out_dir = sh.current_dir(); sh.change_dir(cargo_work_dir); - let mut cmd = xshell::cmd!(sh, "{argv0} {params...}"); + + // Prepare all parameters + let mut all_params = params.clone(); + if !matches!(rt.backend(), FlowBackend::Local) { // if running in CI, no need to waste time with incremental // build artifacts @@ -275,21 +278,28 @@ impl FlowNode for Node { // if build locally, use per-package target dirs // to avoid rebuilding // TODO: remove this once cargo's caching improves - cmd = cmd - .arg("--target-dir") - .arg(in_folder.join("target").join(&crate_name)); + all_params.push("--target-dir".to_owned()); + all_params.push( + in_folder + .join("target") + .join(&crate_name) + .to_string_lossy() + .to_string(), + ); } - cmd = cmd.envs(&with_env); log::info!( - "$ {}{cmd}", + "$ {}{} {}", with_env .iter() .map(|(k, v)| format!("{k}={v} ")) .collect::>() - .concat() + .concat(), + argv0, + all_params.join(" ") ); - let json = cmd.read()?; + + let json = sh.read_cmd(&argv0, &all_params, &with_env)?; let messages: Vec = serde_json::Deserializer::from_str(&json) .into_iter() diff --git a/flowey/flowey_lib_hvlite/src/_jobs/cfg_common.rs b/flowey/flowey_lib_hvlite/src/_jobs/cfg_common.rs index 4af63a27f4..3af7af0efc 100644 --- a/flowey/flowey_lib_hvlite/src/_jobs/cfg_common.rs +++ b/flowey/flowey_lib_hvlite/src/_jobs/cfg_common.rs @@ -43,7 +43,7 @@ impl SimpleFlowNode for Node { fn imports(ctx: &mut ImportCtx<'_>) { ctx.import::(); - ctx.import::(); + ctx.import::(); ctx.import::(); ctx.import::(); ctx.import::(); @@ -58,6 +58,8 @@ impl SimpleFlowNode for Node { ctx.import::(); ctx.import::(); ctx.import::(); + ctx.import::(); + ctx.import::(); ctx.import::(); ctx.import::(); ctx.import::(); @@ -86,6 +88,7 @@ impl SimpleFlowNode for Node { ctx.req(flowey_lib_common::install_rust::Request::IgnoreVersion( false, )); + ctx.req(flowey_lib_common::install_nix::Request::AutoInstall(true)); let token = ctx.get_gh_context_var().global().token(); ctx.req(flowey_lib_common::use_gh_cli::Request::WithAuth( flowey_lib_common::use_gh_cli::GhCliAuth::AuthToken(token), diff --git a/flowey/flowey_lib_hvlite/src/_jobs/cfg_versions.rs b/flowey/flowey_lib_hvlite/src/_jobs/cfg_versions.rs index 3f4103ecee..513b0ae724 100644 --- a/flowey/flowey_lib_hvlite/src/_jobs/cfg_versions.rs +++ b/flowey/flowey_lib_hvlite/src/_jobs/cfg_versions.rs @@ -6,6 +6,7 @@ //! pipelines. use crate::download_openhcl_kernel_package::OpenhclKernelPackageKind; +use crate::run_cargo_build::common::CommonArch; use flowey::node::prelude::*; // FUTURE: instead of hard-coding these values in-code, we might want to make @@ -34,7 +35,11 @@ pub const OPENVMM_DEPS: &str = "0.1.0-20250403.3"; pub const PROTOC: &str = "27.1"; flowey_request! { - pub struct Request {} + pub enum Request { + Download, + Local(CommonArch, ReadVar, ReadVar), + NixEnvironment, + } } new_flow_node!(struct Node); @@ -45,8 +50,9 @@ impl FlowNode for Node { fn imports(ctx: &mut ImportCtx<'_>) { ctx.import::(); ctx.import::(); - ctx.import::(); + ctx.import::(); ctx.import::(); + ctx.import::(); ctx.import::(); ctx.import::(); ctx.import::(); @@ -58,16 +64,121 @@ impl FlowNode for Node { ctx.import::(); ctx.import::(); ctx.import::(); + ctx.import::(); } #[rustfmt::skip] - fn emit(_requests: Vec, ctx: &mut NodeCtx<'_>) -> anyhow::Result<()> { - ctx.req(crate::download_openhcl_kernel_package::Request::Version(OpenhclKernelPackageKind::Dev, OPENHCL_KERNEL_DEV_VERSION.into())); - ctx.req(crate::download_openhcl_kernel_package::Request::Version(OpenhclKernelPackageKind::Main, OPENHCL_KERNEL_STABLE_VERSION.into())); - ctx.req(crate::download_openhcl_kernel_package::Request::Version(OpenhclKernelPackageKind::Cvm, OPENHCL_KERNEL_STABLE_VERSION.into())); - ctx.req(crate::download_openhcl_kernel_package::Request::Version(OpenhclKernelPackageKind::CvmDev, OPENHCL_KERNEL_DEV_VERSION.into())); - ctx.req(crate::download_openvmm_deps::Request::Version(OPENVMM_DEPS.into())); - ctx.req(crate::download_uefi_mu_msvm::Request::Version(MU_MSVM.into())); + fn emit(requests: Vec, ctx: &mut NodeCtx<'_>) -> anyhow::Result<()> { + use std::collections::BTreeMap; + + let mut has_local_requests = false; + let mut has_nix_requests = false; + let mut local_openvmm_deps: BTreeMap> = BTreeMap::new(); + let mut local_protoc: Option> = None; + let mut local_kernel: Option> = None; + let mut local_uefi: Option> = None; + + for req in requests { + match req { + Request::Download => { + // Download requests are always allowed and coexist with Local/Nix requests + } + Request::Local(arch, path, protoc_path) => { + has_local_requests = true; + + // Check that for every arch that shows up, the path is always the same + if let Some(existing_path) = local_openvmm_deps.get(&arch) { + if !existing_path.eq(&path) { + anyhow::bail!( + "OpenvmmDepsPath for {:?} must be consistent across requests", + arch + ); + } + } else { + local_openvmm_deps.insert(arch, path); + } + + same_across_all_reqs_backing_var("ProtocPath", &mut local_protoc, protoc_path)?; + } + Request::NixEnvironment => { + has_nix_requests = true; + } + } + } + + // If NixEnvironment was requested, get paths from nix_deps_provider + if has_nix_requests { + // Get the repo path so nix_deps_provider can access shell.nix + let repo_path = ctx.reqv(|v| crate::git_checkout_openvmm_repo::Request::GetRepoDir( + crate::git_checkout_openvmm_repo::req::GetRepoDir(v) + )); + ctx.req(flowey_lib_common::nix_deps_provider::Request::SetRepoPath(repo_path)); + + let nix_openvmm_deps_x64 = ctx.reqv(|v| flowey_lib_common::nix_deps_provider::Request::GetOpenvmmDeps( + flowey_lib_common::nix_deps_provider::OpenvmmDepsArch::X86_64, + v, + )); + local_openvmm_deps.insert(CommonArch::X86_64, nix_openvmm_deps_x64); + local_protoc = Some(ctx.reqv(flowey_lib_common::nix_deps_provider::Request::GetProtoc)); + local_kernel = Some(ctx.reqv(flowey_lib_common::nix_deps_provider::Request::GetKernel)); + local_uefi = Some(ctx.reqv(flowey_lib_common::nix_deps_provider::Request::GetUefiMuMsvm)); + } + + // Track whether we have local paths for openvmm_deps and protoc + let has_local_openvmm_deps = !local_openvmm_deps.is_empty(); + let has_local_protoc = local_protoc.is_some(); + let has_local_kernel = local_kernel.is_some(); + let has_local_uefi = local_uefi.is_some(); + + // If we have local requests, protoc must be provided + if has_local_requests && local_protoc.is_none() { + anyhow::bail!("Local mode requires protoc path to be specified"); + } + + // Set up local paths for openvmm_deps if provided + for (arch, path) in local_openvmm_deps { + let openvmm_deps_arch = match arch { + CommonArch::X86_64 => crate::resolve_openvmm_deps::OpenvmmDepsArch::X86_64, + CommonArch::Aarch64 => crate::resolve_openvmm_deps::OpenvmmDepsArch::Aarch64, + }; + + ctx.req(crate::resolve_openvmm_deps::Request::LocalPath( + openvmm_deps_arch, + path, + )); + } + + // Set up local path for protoc if provided + if let Some(protoc_path) = local_protoc { + ctx.req(flowey_lib_common::download_protoc::Request::LocalPath( + protoc_path, + )); + } + + if let Some(kernel_path) = local_kernel { + ctx.req(crate::download_openhcl_kernel_package::Request::LocalPath(kernel_path)); + } + + if let Some(uefi_path) = local_uefi { + ctx.req(crate::download_uefi_mu_msvm::Request::LocalPath(uefi_path)); + } + + // Set up version requests for everything + if !has_local_kernel { + ctx.req(crate::download_openhcl_kernel_package::Request::Version(OpenhclKernelPackageKind::Dev, OPENHCL_KERNEL_DEV_VERSION.into())); + ctx.req(crate::download_openhcl_kernel_package::Request::Version(OpenhclKernelPackageKind::Main, OPENHCL_KERNEL_STABLE_VERSION.into())); + ctx.req(crate::download_openhcl_kernel_package::Request::Version(OpenhclKernelPackageKind::Cvm, OPENHCL_KERNEL_STABLE_VERSION.into())); + ctx.req(crate::download_openhcl_kernel_package::Request::Version(OpenhclKernelPackageKind::CvmDev, OPENHCL_KERNEL_DEV_VERSION.into())); + } + + if !has_local_openvmm_deps { + ctx.req(crate::resolve_openvmm_deps::Request::Version(OPENVMM_DEPS.into())); + } + + if !has_local_uefi { + ctx.req(crate::download_uefi_mu_msvm::Request::Version(MU_MSVM.into())); + } + ctx.req(flowey_lib_common::download_azcopy::Request::Version(AZCOPY.into())); ctx.req(flowey_lib_common::download_cargo_fuzz::Request::Version(FUZZ.into())); ctx.req(flowey_lib_common::download_cargo_nextest::Request::Version(NEXTEST.into())); @@ -75,7 +186,9 @@ impl FlowNode for Node { ctx.req(flowey_lib_common::download_mdbook::Request::Version(MDBOOK.into())); ctx.req(flowey_lib_common::download_mdbook_admonish::Request::Version(MDBOOK_ADMONISH.into())); ctx.req(flowey_lib_common::download_mdbook_mermaid::Request::Version(MDBOOK_MERMAID.into())); - ctx.req(flowey_lib_common::download_protoc::Request::Version(PROTOC.into())); + if !has_local_protoc { + ctx.req(flowey_lib_common::download_protoc::Request::Version(PROTOC.into())); + } ctx.req(flowey_lib_common::install_azure_cli::Request::Version(AZURE_CLI.into())); ctx.req(flowey_lib_common::install_nodejs::Request::Version(NODEJS.into())); if !matches!(ctx.backend(), FlowBackend::Ado) { diff --git a/flowey/flowey_lib_hvlite/src/build_openhcl_igvm_from_recipe.rs b/flowey/flowey_lib_hvlite/src/build_openhcl_igvm_from_recipe.rs index af23a7025e..a28c0d0c79 100644 --- a/flowey/flowey_lib_hvlite/src/build_openhcl_igvm_from_recipe.rs +++ b/flowey/flowey_lib_hvlite/src/build_openhcl_igvm_from_recipe.rs @@ -15,8 +15,8 @@ use crate::build_openvmm_hcl::OpenvmmHclBuildProfile; use crate::build_openvmm_hcl::OpenvmmHclFeature; use crate::download_openhcl_kernel_package::OpenhclKernelPackageArch; use crate::download_openhcl_kernel_package::OpenhclKernelPackageKind; -use crate::download_openvmm_deps::OpenvmmDepsArch; use crate::download_uefi_mu_msvm::MuMsvmArch; +use crate::resolve_openvmm_deps::OpenvmmDepsArch; use crate::run_cargo_build::BuildProfile; use crate::run_cargo_build::common::CommonArch; use crate::run_cargo_build::common::CommonPlatform; @@ -275,7 +275,7 @@ impl SimpleFlowNode for Node { ctx.import::(); ctx.import::(); ctx.import::(); - ctx.import::(); + ctx.import::(); ctx.import::(); ctx.import::(); ctx.import::(); @@ -394,7 +394,7 @@ impl SimpleFlowNode for Node { } else { match typ { Vtl0KernelType::Example => ctx.reqv(|v| { - crate::download_openvmm_deps::Request::GetLinuxTestKernel( + crate::resolve_openvmm_deps::Request::GetLinuxTestKernel( match arch { CommonArch::X86_64 => OpenvmmDepsArch::X86_64, CommonArch::Aarch64 => OpenvmmDepsArch::Aarch64, @@ -407,7 +407,7 @@ impl SimpleFlowNode for Node { }; let initrd = ctx.reqv(|v| { - crate::download_openvmm_deps::Request::GetLinuxTestInitrd( + crate::resolve_openvmm_deps::Request::GetLinuxTestInitrd( match arch { CommonArch::X86_64 => OpenvmmDepsArch::X86_64, CommonArch::Aarch64 => OpenvmmDepsArch::Aarch64, diff --git a/flowey/flowey_lib_hvlite/src/build_openhcl_initrd.rs b/flowey/flowey_lib_hvlite/src/build_openhcl_initrd.rs index 64f8069a3d..1c8516927f 100644 --- a/flowey/flowey_lib_hvlite/src/build_openhcl_initrd.rs +++ b/flowey/flowey_lib_hvlite/src/build_openhcl_initrd.rs @@ -3,7 +3,7 @@ //! Wrapper around `update-rootfs.py` -use crate::download_openvmm_deps::OpenvmmDepsArch; +use crate::resolve_openvmm_deps::OpenvmmDepsArch; use crate::run_cargo_build::common::CommonArch; use flowey::node::prelude::*; use std::collections::BTreeMap; @@ -52,7 +52,7 @@ impl FlowNode for Node { type Request = Request; fn imports(ctx: &mut ImportCtx<'_>) { - ctx.import::(); + ctx.import::(); ctx.import::(); ctx.import::(); } @@ -64,6 +64,7 @@ impl FlowNode for Node { FlowPlatform::Linux(linux_distribution) => match linux_distribution { FlowPlatformLinuxDistro::Fedora | FlowPlatformLinuxDistro::Ubuntu => "python3", FlowPlatformLinuxDistro::Arch => "python", + FlowPlatformLinuxDistro::Nix => "python3", FlowPlatformLinuxDistro::Unknown => anyhow::bail!("Unknown Linux distribution"), }, _ => anyhow::bail!("Unsupported platform"), @@ -102,11 +103,11 @@ impl FlowNode for Node { let interactive_dep = if interactive { ctx.reqv(|v| { - crate::download_openvmm_deps::Request::GetOpenhclCpioDbgrd(openvmm_deps_arch, v) + crate::resolve_openvmm_deps::Request::GetOpenhclCpioDbgrd(openvmm_deps_arch, v) }) } else { ctx.reqv(|v| { - crate::download_openvmm_deps::Request::GetOpenhclCpioShell(openvmm_deps_arch, v) + crate::resolve_openvmm_deps::Request::GetOpenhclCpioShell(openvmm_deps_arch, v) }) }; @@ -116,6 +117,7 @@ impl FlowNode for Node { ctx.emit_rust_step("building openhcl initrd", |ctx| { pydeps.clone().claim(ctx); + let platform = ctx.platform(); let interactive_dep = interactive_dep.claim(ctx); let rootfs_config = rootfs_config.claim(ctx); let extra_env = extra_env.claim(ctx); @@ -131,7 +133,7 @@ impl FlowNode for Node { let openvmm_repo_path = rt.read(openvmm_repo_path); let kernel_package_root = rt.read(kernel_package_root); - let sh = xshell::Shell::new()?; + let sh = FloweyShell::new(platform)?; let initrd_path = sh.current_dir().join("openhcl.cpio.gz"); diff --git a/flowey/flowey_lib_hvlite/src/download_openhcl_kernel_package.rs b/flowey/flowey_lib_hvlite/src/download_openhcl_kernel_package.rs index 3058f73021..4537f90f54 100644 --- a/flowey/flowey_lib_hvlite/src/download_openhcl_kernel_package.rs +++ b/flowey/flowey_lib_hvlite/src/download_openhcl_kernel_package.rs @@ -22,6 +22,8 @@ pub enum OpenhclKernelPackageArch { flowey_request! { pub enum Request { + /// Use a locally provided kernel package directory + LocalPath(ReadVar), /// Specify version string to use for each package kind Version(OpenhclKernelPackageKind, String), /// Download the specified kernel package @@ -44,6 +46,7 @@ impl FlowNode for Node { } fn emit(requests: Vec, ctx: &mut NodeCtx<'_>) -> anyhow::Result<()> { + let mut local_path: Option> = None; let mut versions: BTreeMap = BTreeMap::new(); let mut reqs: BTreeMap< (OpenhclKernelPackageKind, OpenhclKernelPackageArch), @@ -52,6 +55,9 @@ impl FlowNode for Node { for req in requests { match req { + Request::LocalPath(path) => { + same_across_all_reqs_backing_var("LocalPath", &mut local_path, path)? + } Request::Version(arch, v) => { let mut old = versions.insert(arch, v.clone()); same_across_all_reqs("SetVersion", &mut old, v)? @@ -62,9 +68,15 @@ impl FlowNode for Node { } } - for req_kind in reqs.keys().map(|(k, _)| k) { - if !versions.contains_key(req_kind) { - anyhow::bail!("missing SetVersion for {:?}", req_kind) + if local_path.is_some() && !versions.is_empty() { + anyhow::bail!("Cannot specify both Version and LocalPath"); + } + + if local_path.is_none() { + for req_kind in reqs.keys().map(|(k, _)| k) { + if !versions.contains_key(req_kind) { + anyhow::bail!("missing SetVersion for {:?}", req_kind) + } } } @@ -74,6 +86,23 @@ impl FlowNode for Node { return Ok(()); } + // If local path provided, use it directly + if let Some(local_path) = local_path { + ctx.emit_rust_step("use local kernel package", |ctx| { + let local_path = local_path.claim(ctx); + let reqs = reqs.into_values().flatten().collect::>().claim(ctx); + + move |rt| { + let kernel_pkg_dir = rt.read(local_path); + // For local path, the directory itself is the package root + rt.write_all(reqs, &kernel_pkg_dir); + Ok(()) + } + }); + + return Ok(()); + } + let extract_zip_deps = flowey_lib_common::_util::extract::extract_zip_if_new_deps(ctx); for ((kind, arch), out_vars) in reqs { diff --git a/flowey/flowey_lib_hvlite/src/download_uefi_mu_msvm.rs b/flowey/flowey_lib_hvlite/src/download_uefi_mu_msvm.rs index 1798cf4ce2..82053f364d 100644 --- a/flowey/flowey_lib_hvlite/src/download_uefi_mu_msvm.rs +++ b/flowey/flowey_lib_hvlite/src/download_uefi_mu_msvm.rs @@ -14,6 +14,8 @@ pub enum MuMsvmArch { flowey_request! { pub enum Request { + /// Use a locally provided UEFI firmware file + LocalPath(ReadVar), /// Specify version of mu_msvm to use Version(String), /// Download the mu_msvm package for the given arch @@ -36,16 +38,26 @@ impl FlowNode for Node { fn emit(requests: Vec, ctx: &mut NodeCtx<'_>) -> anyhow::Result<()> { let mut version = None; + let mut local_path: Option> = None; let mut reqs: BTreeMap>> = BTreeMap::new(); for req in requests { match req { + Request::LocalPath(path) => { + same_across_all_reqs_backing_var("LocalPath", &mut local_path, path)? + } Request::Version(v) => same_across_all_reqs("Version", &mut version, v)?, Request::GetMsvmFd { arch, msvm_fd } => reqs.entry(arch).or_default().push(msvm_fd), } } - let version = version.ok_or(anyhow::anyhow!("Missing essential request: Version"))?; + if version.is_some() && local_path.is_some() { + anyhow::bail!("Cannot specify both Version and LocalPath"); + } + + if version.is_none() && local_path.is_none() { + anyhow::bail!("Must specify a Version or LocalPath request"); + } // -- end of req processing -- // @@ -53,6 +65,24 @@ impl FlowNode for Node { return Ok(()); } + // If local path provided, use it directly + if let Some(local_path) = local_path { + ctx.emit_rust_step("use local UEFI firmware", |ctx| { + let local_path = local_path.claim(ctx); + let reqs = reqs.into_values().flatten().collect::>().claim(ctx); + + move |rt| { + let msvm_fd = rt.read(local_path); + rt.write_all(reqs, &msvm_fd); + Ok(()) + } + }); + + return Ok(()); + } + + let version = version.expect("checked above"); + let extract_zip_deps = flowey_lib_common::_util::extract::extract_zip_if_new_deps(ctx); for (arch, out_vars) in reqs { diff --git a/flowey/flowey_lib_hvlite/src/init_cross_build.rs b/flowey/flowey_lib_hvlite/src/init_cross_build.rs index 0629ded775..f1649e9a76 100644 --- a/flowey/flowey_lib_hvlite/src/init_cross_build.rs +++ b/flowey/flowey_lib_hvlite/src/init_cross_build.rs @@ -60,37 +60,43 @@ impl FlowNode for Node { let (gcc_pkg, bin) = match target.architecture { Architecture::X86_64 => match platform { FlowPlatform::Linux(linux_distribution) => { - let pkg = match linux_distribution { - FlowPlatformLinuxDistro::Fedora => "gcc-x86_64-linux-gnu", - FlowPlatformLinuxDistro::Ubuntu => "gcc-x86-64-linux-gnu", + let (pkg, bin) = match linux_distribution { + FlowPlatformLinuxDistro::Fedora => { + ("gcc-x86_64-linux-gnu", "x86_64-linux-gnu-gcc") + } + FlowPlatformLinuxDistro::Ubuntu => { + ("gcc-x86-64-linux-gnu", "x86_64-linux-gnu-gcc") + } FlowPlatformLinuxDistro::Arch => { - match_arch!(host_arch, FlowArch::X86_64, "gcc") + match_arch!(host_arch, FlowArch::X86_64, ("gcc", "gcc")) } + FlowPlatformLinuxDistro::Nix => ("gcc", "gcc"), FlowPlatformLinuxDistro::Unknown => { anyhow::bail!("Unknown Linux distribution") } }; - (pkg.to_string(), "x86_64-linux-gnu-gcc".to_string()) + (pkg.to_string(), bin.to_string()) } _ => anyhow::bail!("Unsupported platform"), }, Architecture::Aarch64(_) => match platform { FlowPlatform::Linux(linux_distribution) => { - let pkg = match linux_distribution { + let (pkg, bin) = match linux_distribution { FlowPlatformLinuxDistro::Fedora | FlowPlatformLinuxDistro::Ubuntu => { - "gcc-aarch64-linux-gnu" + ("gcc-aarch64-linux-gnu", "aarch64-linux-gnu-gcc") } FlowPlatformLinuxDistro::Arch => match_arch!( host_arch, FlowArch::X86_64, - "aarch64-linux-gnu-gcc" + ("aarch64-linux-gnu-gcc", "aarch64-linux-gnu-gcc") ), + FlowPlatformLinuxDistro::Nix => ("gcc", "gcc"), FlowPlatformLinuxDistro::Unknown => { anyhow::bail!("Unknown Linux distribution") } }; - (pkg.to_string(), "aarch64-linux-gnu-gcc".to_string()) + (pkg.to_string(), bin.to_string()) } _ => anyhow::bail!("Unsupported platform"), }, diff --git a/flowey/flowey_lib_hvlite/src/init_openvmm_magicpath_linux_test_kernel.rs b/flowey/flowey_lib_hvlite/src/init_openvmm_magicpath_linux_test_kernel.rs index 4659ebbcf6..ad1f1e7a1c 100644 --- a/flowey/flowey_lib_hvlite/src/init_openvmm_magicpath_linux_test_kernel.rs +++ b/flowey/flowey_lib_hvlite/src/init_openvmm_magicpath_linux_test_kernel.rs @@ -5,7 +5,7 @@ //! correct "magic directory" set by the project-level `[env]` table in //! `.cargo/config.toml` -use crate::download_openvmm_deps::OpenvmmDepsArch; +use crate::resolve_openvmm_deps::OpenvmmDepsArch; use flowey::node::prelude::*; use std::collections::BTreeMap; @@ -29,7 +29,7 @@ impl FlowNode for Node { fn imports(ctx: &mut ImportCtx<'_>) { ctx.import::(); - ctx.import::(); + ctx.import::(); } fn emit(requests: Vec, ctx: &mut NodeCtx<'_>) -> anyhow::Result<()> { @@ -51,10 +51,10 @@ impl FlowNode for Node { OpenvmmLinuxTestKernelArch::X64 => OpenvmmDepsArch::X86_64, }; let openvmm_linux_test_kernel = ctx.reqv(|v| { - crate::download_openvmm_deps::Request::GetLinuxTestKernel(openvmm_deps_arch, v) + crate::resolve_openvmm_deps::Request::GetLinuxTestKernel(openvmm_deps_arch, v) }); let openvmm_linux_test_initrd = ctx.reqv(|v| { - crate::download_openvmm_deps::Request::GetLinuxTestInitrd(openvmm_deps_arch, v) + crate::resolve_openvmm_deps::Request::GetLinuxTestInitrd(openvmm_deps_arch, v) }); ctx.emit_rust_step(format!("extract {arch:?} sysroot.tar.gz"), |ctx| { diff --git a/flowey/flowey_lib_hvlite/src/init_openvmm_magicpath_openhcl_sysroot.rs b/flowey/flowey_lib_hvlite/src/init_openvmm_magicpath_openhcl_sysroot.rs index 60efae3ddb..9ba08bb5b5 100644 --- a/flowey/flowey_lib_hvlite/src/init_openvmm_magicpath_openhcl_sysroot.rs +++ b/flowey/flowey_lib_hvlite/src/init_openvmm_magicpath_openhcl_sysroot.rs @@ -4,7 +4,7 @@ //! Ensure the OpenHCL sysroot is extracted into the correct "magic directory" //! set by the project-level `[env]` table in `.cargo/config.toml` -use crate::download_openvmm_deps::OpenvmmDepsArch; +use crate::resolve_openvmm_deps::OpenvmmDepsArch; use flowey::node::prelude::*; use std::collections::BTreeMap; @@ -28,7 +28,7 @@ impl FlowNode for Node { fn imports(ctx: &mut ImportCtx<'_>) { ctx.import::(); - ctx.import::(); + ctx.import::(); } fn emit(requests: Vec, ctx: &mut NodeCtx<'_>) -> anyhow::Result<()> { @@ -50,7 +50,7 @@ impl FlowNode for Node { for (arch, out_vars) in sysroot_arch { let openhcl_sysroot_tar_gz = ctx.reqv(|v| { - crate::download_openvmm_deps::Request::GetOpenhclSysroot( + crate::resolve_openvmm_deps::Request::GetOpenhclSysroot( match arch { OpenvmmSysrootArch::Aarch64 => OpenvmmDepsArch::Aarch64, OpenvmmSysrootArch::X64 => OpenvmmDepsArch::X86_64, diff --git a/flowey/flowey_lib_hvlite/src/init_openvmm_magicpath_protoc.rs b/flowey/flowey_lib_hvlite/src/init_openvmm_magicpath_protoc.rs index 76cb05764a..e73125f825 100644 --- a/flowey/flowey_lib_hvlite/src/init_openvmm_magicpath_protoc.rs +++ b/flowey/flowey_lib_hvlite/src/init_openvmm_magicpath_protoc.rs @@ -45,10 +45,12 @@ impl FlowNode for Node { let protoc_pkg = rt.read(protoc_pkg); - flowey_lib_common::_util::copy_dir_all( - protoc_pkg.include_dir, - openvmm_magicpath.join("Google.Protobuf.Tools/tools/include"), - )?; + let include_dst = openvmm_magicpath.join("Google.Protobuf.Tools/tools/include"); + // Remove existing include directory to avoid permission issues + // (e.g., from previous runs or nix store) + let _ = fs_err::remove_dir_all(&include_dst); + + flowey_lib_common::_util::copy_dir_all(protoc_pkg.include_dir, &include_dst)?; let src = protoc_pkg.protoc_bin; let dst = dst_folder.join(expected_protoc_bin); @@ -56,7 +58,11 @@ impl FlowNode for Node { let _ = fs_err::remove_file(&dst); if !dst.exists() { - fs_err::hard_link(src.clone(), &dst)?; + // Try hardlinking first (faster), but fall back to copying if it fails + // (e.g., when source is in nix store or across filesystems) + if fs_err::hard_link(&src, &dst).is_err() { + fs_err::copy(&src, &dst)?; + } dst.make_executable()?; } diff --git a/flowey/flowey_lib_hvlite/src/init_vmm_tests_env.rs b/flowey/flowey_lib_hvlite/src/init_vmm_tests_env.rs index 97d0cee558..be74b998de 100644 --- a/flowey/flowey_lib_hvlite/src/init_vmm_tests_env.rs +++ b/flowey/flowey_lib_hvlite/src/init_vmm_tests_env.rs @@ -6,9 +6,9 @@ use crate::build_openhcl_igvm_from_recipe::OpenhclIgvmRecipe; use crate::build_tpm_guest_tests::TpmGuestTestsOutput; -use crate::download_openvmm_deps::OpenvmmDepsArch; use crate::download_release_igvm_files_from_gh::OpenhclReleaseVersion; use crate::download_uefi_mu_msvm::MuMsvmArch; +use crate::resolve_openvmm_deps::OpenvmmDepsArch; use flowey::node::prelude::*; use std::collections::BTreeMap; @@ -72,7 +72,7 @@ impl SimpleFlowNode for Node { type Request = Request; fn imports(ctx: &mut ImportCtx<'_>) { - ctx.import::(); + ctx.import::(); ctx.import::(); ctx.import::(); } @@ -106,10 +106,10 @@ impl SimpleFlowNode for Node { }; let test_linux_initrd = ctx.reqv(|v| { - crate::download_openvmm_deps::Request::GetLinuxTestInitrd(openvmm_deps_arch, v) + crate::resolve_openvmm_deps::Request::GetLinuxTestInitrd(openvmm_deps_arch, v) }); let test_linux_kernel = ctx.reqv(|v| { - crate::download_openvmm_deps::Request::GetLinuxTestKernel(openvmm_deps_arch, v) + crate::resolve_openvmm_deps::Request::GetLinuxTestKernel(openvmm_deps_arch, v) }); let mu_msvm_arch = match vmm_tests_target.architecture { diff --git a/flowey/flowey_lib_hvlite/src/lib.rs b/flowey/flowey_lib_hvlite/src/lib.rs index b1ddecbe91..83f469cbe2 100644 --- a/flowey/flowey_lib_hvlite/src/lib.rs +++ b/flowey/flowey_lib_hvlite/src/lib.rs @@ -35,7 +35,6 @@ pub mod build_vmgstool; pub mod build_xtask; pub mod cfg_openvmm_magicpath; pub mod download_openhcl_kernel_package; -pub mod download_openvmm_deps; pub mod download_openvmm_vmm_tests_artifacts; pub mod download_release_igvm_files_from_gh; pub mod download_uefi_mu_msvm; @@ -51,6 +50,7 @@ pub mod init_vmm_tests_env; pub mod install_git_credential_manager; pub mod install_openvmm_rust_build_essential; pub mod install_vmm_tests_deps; +pub mod resolve_openvmm_deps; pub mod run_cargo_build; pub mod run_cargo_nextest_run; pub mod run_igvmfilegen; diff --git a/flowey/flowey_lib_hvlite/src/download_openvmm_deps.rs b/flowey/flowey_lib_hvlite/src/resolve_openvmm_deps.rs similarity index 64% rename from flowey/flowey_lib_hvlite/src/download_openvmm_deps.rs rename to flowey/flowey_lib_hvlite/src/resolve_openvmm_deps.rs index b6cdbd8696..899bb11548 100644 --- a/flowey/flowey_lib_hvlite/src/download_openvmm_deps.rs +++ b/flowey/flowey_lib_hvlite/src/resolve_openvmm_deps.rs @@ -14,6 +14,8 @@ pub enum OpenvmmDepsArch { flowey_request! { pub enum Request { + /// Use a locally downloaded openvmm-deps for a specific architecture + LocalPath(OpenvmmDepsArch, ReadVar), /// Specify version of the github release to pull from Version(String), GetLinuxTestKernel(OpenvmmDepsArch, WriteVar), @@ -36,6 +38,7 @@ impl FlowNode for Node { fn emit(requests: Vec, ctx: &mut NodeCtx<'_>) -> anyhow::Result<()> { let mut version = None; + let mut local_paths: BTreeMap> = BTreeMap::new(); let mut linux_test_kernel: BTreeMap<_, Vec<_>> = BTreeMap::new(); let mut linux_test_initrd: BTreeMap<_, Vec<_>> = BTreeMap::new(); let mut openhcl_cpio_dbgrd: BTreeMap<_, Vec<_>> = BTreeMap::new(); @@ -45,7 +48,18 @@ impl FlowNode for Node { for req in requests { match req { Request::Version(v) => same_across_all_reqs("Version", &mut version, v)?, - + Request::LocalPath(arch, path) => { + if let Some(existing) = local_paths.get(&arch) { + if !existing.eq(&path) { + anyhow::bail!( + "Conflicting LocalPath requests for {:?}: different backing variables", + arch + ); + } + } else { + local_paths.insert(arch, path); + } + } Request::GetLinuxTestKernel(arch, var) => { linux_test_kernel.entry(arch).or_default().push(var) } @@ -64,7 +78,13 @@ impl FlowNode for Node { } } - let version = version.ok_or(anyhow::anyhow!("Missing essential request: Version"))?; + if version.is_some() && !local_paths.is_empty() { + anyhow::bail!("Cannot specify both Version and LocalPath requests"); + } + + if version.is_none() && local_paths.is_empty() { + anyhow::bail!("Must specify a Version or LocalPath request"); + } // -- end of req processing -- // @@ -77,6 +97,72 @@ impl FlowNode for Node { return Ok(()); } + if !local_paths.is_empty() { + ctx.emit_rust_step("use local openvmm-deps", |ctx| { + let linux_test_kernel = linux_test_kernel.claim(ctx); + let linux_test_initrd = linux_test_initrd.claim(ctx); + let openhcl_cpio_dbgrd = openhcl_cpio_dbgrd.claim(ctx); + let openhcl_cpio_shell = openhcl_cpio_shell.claim(ctx); + let openhcl_sysroot = openhcl_sysroot.claim(ctx); + let local_paths: BTreeMap> = local_paths + .into_iter() + .map(|(arch, var)| (arch, var.claim(ctx))) + .collect(); + move |rt| { + // Read all paths upfront for efficiency + let resolved_paths: BTreeMap = local_paths + .iter() + .map(|(arch, claimed_var)| (*arch, rt.read(claimed_var.clone()))) + .collect(); + + let get_base_dir = |arch: OpenvmmDepsArch| -> anyhow::Result<&PathBuf> { + resolved_paths.get(&arch).ok_or_else(|| { + anyhow::anyhow!("No local path specified for architecture {:?}", arch) + }) + }; + + let kernel_file_name = |arch| match arch { + OpenvmmDepsArch::X86_64 => "vmlinux", + OpenvmmDepsArch::Aarch64 => "Image", + }; + + for (arch, vars) in linux_test_kernel { + let base_dir = get_base_dir(arch)?; + let path = base_dir.join(kernel_file_name(arch)); + rt.write_all(vars, &path) + } + + for (arch, vars) in linux_test_initrd { + let base_dir = get_base_dir(arch)?; + let path = base_dir.join("initrd"); + rt.write_all(vars, &path) + } + + for (arch, vars) in openhcl_cpio_dbgrd { + let base_dir = get_base_dir(arch)?; + let path = base_dir.join("dbgrd.cpio.gz"); + rt.write_all(vars, &path) + } + + for (arch, vars) in openhcl_cpio_shell { + let base_dir = get_base_dir(arch)?; + let path = base_dir.join("shell.cpio.gz"); + rt.write_all(vars, &path) + } + + for (arch, vars) in openhcl_sysroot { + let base_dir = get_base_dir(arch)?; + let path = base_dir.join("sysroot.tar.gz"); + rt.write_all(vars, &path) + } + + Ok(()) + } + }); + + return Ok(()); + } + let extract_tar_bz2_deps = flowey_lib_common::_util::extract::extract_tar_bz2_if_new_deps(ctx); @@ -86,6 +172,7 @@ impl FlowNode for Node { || openhcl_cpio_shell.contains_key(&OpenvmmDepsArch::X86_64) || openhcl_sysroot.contains_key(&OpenvmmDepsArch::X86_64) { + let version = version.clone().expect("local requests handled above"); Some( ctx.reqv(|v| flowey_lib_common::download_gh_release::Request { repo_owner: "microsoft".into(), @@ -107,6 +194,7 @@ impl FlowNode for Node { || openhcl_cpio_shell.contains_key(&OpenvmmDepsArch::Aarch64) || openhcl_sysroot.contains_key(&OpenvmmDepsArch::Aarch64) { + let version = version.clone().expect("local requests handled above"); Some( ctx.reqv(|v| flowey_lib_common::download_gh_release::Request { repo_owner: "microsoft".into(), @@ -131,6 +219,7 @@ impl FlowNode for Node { let openhcl_cpio_dbgrd = openhcl_cpio_dbgrd.claim(ctx); let openhcl_cpio_shell = openhcl_cpio_shell.claim(ctx); let openhcl_sysroot = openhcl_sysroot.claim(ctx); + let version = version.clone().expect("local requests handled above"); move |rt| { let extract_dir_x64 = openvmm_deps_tar_bz2_x64 .map(|file| { diff --git a/flowey/flowey_lib_hvlite/src/run_split_debug_info.rs b/flowey/flowey_lib_hvlite/src/run_split_debug_info.rs index e35b27501d..3ae69beb80 100644 --- a/flowey/flowey_lib_hvlite/src/run_split_debug_info.rs +++ b/flowey/flowey_lib_hvlite/src/run_split_debug_info.rs @@ -46,6 +46,7 @@ impl SimpleFlowNode for Node { FlowPlatformLinuxDistro::Arch => { match_arch!(host_arch, FlowArch::X86_64, ("binutils", "objcopy")) } + FlowPlatformLinuxDistro::Nix => ("binutils", "objcopy"), FlowPlatformLinuxDistro::Unknown => anyhow::bail!("Unknown Linux distribution"), }, _ => anyhow::bail!("Unsupported platform"), @@ -59,6 +60,7 @@ impl SimpleFlowNode for Node { FlowPlatformLinuxDistro::Arch => { match_arch!(host_arch, FlowArch::X86_64, "aarch64-linux-gnu-binutils") } + FlowPlatformLinuxDistro::Nix => "binutils", FlowPlatformLinuxDistro::Unknown => { anyhow::bail!("Unknown Linux distribution") } diff --git a/lxutil.nix b/lxutil.nix new file mode 100644 index 0000000000..2c3ada0f73 --- /dev/null +++ b/lxutil.nix @@ -0,0 +1,27 @@ +{ system, stdenv, fetchzip, }: + +let + + arch = if system == "aarch64-linux" then "AARCH64" else "x64"; + hash = if system == "aarch64-linux" then + "sha256-ybBnZZssTXrOxqH6NudmYljn92ejCjrazBIv/GNQyn4=" + else + "sha256-ybBnZZssTXrOxqH6NudmYljn92ejCjrazBIv/GNQyn4="; + +in stdenv.mkDerivation { + pname = "lxutil"; + version = "10.0.26100.1-240331-1435.ge-release"; + + src = fetchzip { + url = + "https://github.com/microsoft/openvmm-deps/releases/download/Microsoft.WSL.LxUtil.10.0.26100.1-240331-1435.ge-release/Microsoft.WSL.LxUtil.${arch}.zip"; + inherit hash; + }; + + installPhase = '' + runHook preInstall + mkdir -p $out/bin + cp bin/lxutil.dll $out/bin/lxutil.dll + runHook postInstall + ''; +} diff --git a/mdbook.nix b/mdbook.nix new file mode 100644 index 0000000000..834ee022e3 --- /dev/null +++ b/mdbook.nix @@ -0,0 +1,27 @@ +{ stdenv, fetchzip, +#fetchFromGitHub +}: + +stdenv.mkDerivation { + pname = "mdBook"; + version = "0.4.40"; + + # src = fetchFromGitHub { + # owner = "rust-lang"; + # repo = "mdBook"; + # rev = "v0.4.40"; + # sha256 = "GGQK2Mf3EK1rwBMzQkAzWAaK6Fh0Qqqf8dtDjZPxOMA="; + # }; + src = fetchzip { + url = + "https://github.com/rust-lang/mdBook/releases/download/v0.4.40/mdbook-v0.4.40-x86_64-unknown-linux-gnu.tar.gz"; + hash = "sha256-ijQbAOvEcmKaoPMe+eZELxY8iCJvrMnk4R07+d5lGtQ="; + }; + + installPhase = '' + runHook preInstall + mkdir -p $out/bin + cp mdbook $out/bin/ + runHook postInstall + ''; +} diff --git a/mdbook_admonish.nix b/mdbook_admonish.nix new file mode 100644 index 0000000000..52d15198cc --- /dev/null +++ b/mdbook_admonish.nix @@ -0,0 +1,25 @@ +{ stdenv, fetchzip, }: + +stdenv.mkDerivation { + pname = "mdbook_admonish"; + version = "1.18.0"; + + # src = fetchFromGitHub { + # owner = "tommilligan"; + # repo = "mdbook-admonish"; + # rev = "v1.18.0"; + # sha256 = "GNQIOjgHCt3XPCzF0RjV9YStI8psLdHhTPuTkdgx8vA="; + # }; + src = fetchzip { + url = + "https://github.com/tommilligan/mdbook-admonish/releases/download/v1.18.0/mdbook-admonish-v1.18.0-x86_64-unknown-linux-gnu.tar.gz"; + hash = "sha256-L7Vt3a1vz1aO4ItCSpKqn+413JGZZ9R+ukqgsE38fMc="; + }; + + installPhase = '' + runHook preInstall + mkdir -p $out/bin + cp mdbook-admonish $out/bin/ + runHook postInstall + ''; +} diff --git a/mdbook_mermaid.nix b/mdbook_mermaid.nix new file mode 100644 index 0000000000..cf00f76ec3 --- /dev/null +++ b/mdbook_mermaid.nix @@ -0,0 +1,27 @@ +{ stdenv, fetchzip, +#fetchFromGitHub +}: + +stdenv.mkDerivation { + pname = "mdbook-mermaid"; + version = "0.14.0"; + + # src = fetchFromGitHub { + # owner = "badboy"; + # repo = "mdbook-mermaid"; + # rev = "v0.14.0"; + # sha256 = "elDKxtGMLka9Ss5CNnzw32ndxTUliNUgPXp7e4KUmBo="; + # }; + src = fetchzip { + url = + "https://github.com/badboy/mdbook-mermaid/releases/download/v0.14.0/mdbook-mermaid-v0.14.0-x86_64-unknown-linux-gnu.tar.gz"; + hash = "sha256-cbcPoLQ4b8cQ2xk0YnapC9L0Rayt0bblGXVfCzJLiGA="; + }; + + installPhase = '' + runHook preInstall + mkdir -p $out/bin + cp mdbook-mermaid $out/bin/ + runHook postInstall + ''; +} diff --git a/openhcl_kernel.nix b/openhcl_kernel.nix new file mode 100644 index 0000000000..04d272da67 --- /dev/null +++ b/openhcl_kernel.nix @@ -0,0 +1,49 @@ +{ system, stdenv, fetchzip, is_dev ? false, is_cvm ? false, }: + +let + version = if is_dev then "6.12.44.1" else "6.12.44.1"; + arch = if system == "aarch64-linux" then "arm64" else "x64"; + branch = if is_dev then "hcl-dev" else "hcl-main"; + build_type = if is_cvm then "cvm" else "std"; + # See https://github.com/microsoft/OHCL-Linux-Kernel/releases + url = + "https://github.com/microsoft/OHCL-Linux-Kernel/releases/download/rolling-lts/${branch}/${version}/Microsoft.OHCL.Kernel${ + if is_dev then ".Dev" else "" + }.${version}-${if is_cvm then "cvm-" else ""}${arch}.tar.gz"; + hash = { + hcl-main = { + std = { + x64 = "sha256-An1N76i1MPb+rrQ1nBpoiuxnNeD0E+VuwqXdkPzaZn0="; + arm64 = "sha256-ENjd+Pd9sQ/f0Gvyq0iB+IG7c4p+AxwxoWu87pZSXYQ="; + }; + cvm = { x64 = "sha256-pV/20epW9LYWzwA633MYxtaUCyMaLAWaaSEJyx+rniQ="; }; + }; + hcl-dev = { + std = { + x64 = "sha256-Ow9piuc2IDR4RPISKY5EAQ5ykjitem4CXS9974lvwPE="; + arm64 = ""; + }; + cvm = { + x64 = "sha256-IryjvoFDSghhVucKlIG9V0IzcVuf8m8Cmk5NhhWzTQM="; + }; + }; + }.${branch}.${build_type}.${arch}; + +in stdenv.mkDerivation { + pname = "openhcl-kernel"; + inherit version; + src = fetchzip { + inherit url; + stripRoot = false; + inherit hash; + }; + + installPhase = '' + runHook preInstall + mkdir -p $out/build/native/bin/${arch} + cp vmlinux* $out/build/native/bin/${arch}/ + cp kernel_build_metadata.json $out/build/native/bin/ + cp -r modules $out/build/native/bin/${arch}/ + runHook postInstall + ''; +} diff --git a/openvmm_deps.nix b/openvmm_deps.nix new file mode 100644 index 0000000000..e0c936ef72 --- /dev/null +++ b/openvmm_deps.nix @@ -0,0 +1,28 @@ +{ system, stdenv, fetchzip, }: + +let + + arch = if system == "aarch64-linux" then "aarch64" else "x86_64"; + hash = if system == "aarch64-linux" then + "sha256-yLGLoQrzA07jrG4G1HMb2P3fcmnGS3KF5H/4AtzDO4w=" + else + "sha256-uDCEo4wbHya3KEYVgFHxr+/OOkzyMCUwhLNX7kppojQ="; + +in stdenv.mkDerivation { + pname = "openvmm-deps"; + version = "0.1.0-20250403.3"; + + src = fetchzip { + url = + "https://github.com/microsoft/openvmm-deps/releases/download/0.1.0-20250403.3/openvmm-deps.${arch}.0.1.0-20250403.3.tar.bz2"; + stripRoot = false; + inherit hash; + }; + + installPhase = '' + runHook preInstall + mkdir $out + cp * $out + runHook postInstall + ''; +} diff --git a/protoc.nix b/protoc.nix new file mode 100644 index 0000000000..ab134c3ce2 --- /dev/null +++ b/protoc.nix @@ -0,0 +1,21 @@ +{ stdenv, fetchzip, }: + +stdenv.mkDerivation { + pname = "protoc"; + version = "27.1"; + + src = fetchzip { + url = + "https://github.com/protocolbuffers/protobuf/releases/download/v27.1/protoc-27.1-linux-x86_64.zip"; + stripRoot = false; + hash = "sha256-jk1VHYxOMo7C6mr1EVL97I2+osYz7lRtQLULv91gFH4="; + }; + + installPhase = '' + runHook preInstall + mkdir -p $out/bin + cp bin/protoc $out/bin/ + cp -r include $out + runHook postInstall + ''; +} diff --git a/shell.nix b/shell.nix new file mode 100644 index 0000000000..9eca4e56e3 --- /dev/null +++ b/shell.nix @@ -0,0 +1,60 @@ +let + rust_overlay = import (builtins.fetchTarball + "https://github.com/oxalica/rust-overlay/archive/master.tar.gz"); + # pkgs = import { crossSystem = { config = "aarch64-unknown-linux-gnu"; }; overlays = [ rust_overlay ]; }; + pkgs = import { overlays = [ rust_overlay ]; }; + + mdbook = pkgs.callPackage ./mdbook.nix { }; + mdbook_admonish = pkgs.callPackage ./mdbook_admonish.nix { }; + mdbook_mermaid = pkgs.callPackage ./mdbook_mermaid.nix { }; + + protoc = pkgs.callPackage ./protoc.nix { }; + + lxutil = pkgs.callPackage ./lxutil.nix { }; + openhcl_kernel = pkgs.callPackage ./openhcl_kernel.nix { }; + openvmm_deps = pkgs.callPackage ./openvmm_deps.nix { }; + uefi_mu_msvm = pkgs.callPackage ./uefi_mu_msvm.nix { }; + + glibc_2_39_52 = import (fetchTarball + "https://github.com/NixOS/nixpkgs/archive/ab7b6889ae9d484eed2876868209e33eb262511d.tar.gz") + { }; + + overrides = (builtins.fromTOML (builtins.readFile ./Cargo.toml)); + rustVersion = overrides.workspace.package.rust-version; + rust = pkgs.rust-bin.stable.${rustVersion}.default.override { + extensions = [ + "rust-src" # for rust-analyzer + "rust-analyzer" + ]; + targets = [ "x86_64-unknown-linux-musl" "x86_64-unknown-none" ]; + }; +in pkgs.mkShell.override { } { + nativeBuildInputs = [ + rust + mdbook + mdbook_admonish + mdbook_mermaid + protoc + ] ++ (with pkgs; [ + libarchive + git + perl + python3 + rustup + pkg-config + ]); + buildInputs = [ + pkgs.openssl.dev + ]; + CARGO_BUILD_ARGS = "--use-local-deps --custom-openvmm-deps ${openvmm_deps} --custom-uefi=${uefi_mu_msvm}/MSVM.fd --custom-kernel ${openhcl_kernel}/vmlinux --custom-kernel-modules ${openhcl_kernel}/modules --custom-protoc ${protoc}"; + + NIX_OPENVMM_DEPS = openvmm_deps; + NIX_PROTOC_PATH = protoc; + NIX_OPENHCL_KERNEL = openhcl_kernel; + NIX_UEFI_MU_MSVM = "${uefi_mu_msvm}/MSVM.fd"; + RUST_BACKTRACE = 1; + # will probably need more than one of these for local source + dependencies. + # RUSTFLAGS = "--remap-path-prefix =/src"; + SOURCE_DATE_EPOCH = 12345; + REALGCC = "gcc"; +} diff --git a/uefi_mu_msvm.nix b/uefi_mu_msvm.nix new file mode 100644 index 0000000000..faf3fad2b5 --- /dev/null +++ b/uefi_mu_msvm.nix @@ -0,0 +1,28 @@ +{ system, stdenv, fetchzip, }: + +let + + arch = if system == "aarch64-linux" then "AARCH64" else "x64"; + hash = if system == "aarch64-linux" then + "sha256-WFMMf9LdCd0X6jwPVhYScmoXjpQdJnswFwHjMWvmZz8=" + else + "sha256-wJeRZC6sd+tNSYHdyyN4Qj/sn5puT6R8eagFlHa6pP4="; + +in stdenv.mkDerivation { + pname = "openvmm-deps"; + version = "24.0.4"; + + src = fetchzip { + url = + "https://github.com/microsoft/mu_msvm/releases/download/v24.0.4/RELEASE-${arch}-artifacts.zip"; + stripRoot = false; + inherit hash; + }; + + installPhase = '' + runHook preInstall + mkdir $out + cp FV/MSVM.fd $out + runHook postInstall + ''; +}