diff --git a/test.sh b/test.sh index 52e8275c0..135acf088 100755 --- a/test.sh +++ b/test.sh @@ -1,127 +1,105 @@ -#!/bin/bash +#!/usr/bin/env bash # Run all known IncludeOS tests. # # A lot of these tests require vmrunner and a network bridge. # See https://github.com/includeos/vmrunner/pull/31 -: "${QUICK_SMOKE:=}" # Define this to only do a ~1-5 min. smoke test. -: "${DRY_RUN:=}" # Define this to expand all steps without running any -: "${CCACHE_FLAG:=}" # Define as "--arg withCcache true" to enable ccache. +: "${QUICK_SMOKE:=false}" # Set to "true" for a ~1–5 min smoke test. +: "${DRY_RUN:=false}" # Set to "true" to print steps without running them. +: "${USE_CCACHE:=false}" # Set to "true" to enable ccache. +: "${TESTS_STDOUT:=/dev/stdout}" # Set to /dev/null (or a file) to silence stdout of tests +: "${TESTS_STDERR:=/dev/stderr}" # Set to /dev/null (or a file) to silence stderr of tests +# +# counters +# steps=0 fails=0 failed_tests=() -success(){ - echo "" +# +# helpers +# +success() { if [[ $1 =~ ^[0-9]+$ ]]; then echo -n "👷💬 Step $1 succeeded " for ((i=1; i<=$1; i++)); do echo -n "👏" done else - echo "👷💬 Step $1 succeeded ✅" + echo -n "👷💬 Step $1 succeeded ✅" fi - echo "" } -fail(){ +fail() { echo "" echo "👷⛔ Step $1 failed ($2)" failed_tests+=("step $1: $2") } -run(){ - steps=$((steps + 1)) - echo "" - echo "🚧 Step $steps) $2" - echo "⚙️ Running this command:" - # This will print the body of a bash function, but won't expand variables - # inside. It works well for bundling simple commands together and allows us to - # print them without wrapping them in qotes. - declare -f $1 | sed '1d;2d;$d' | sed 's/^[[:space:]]*//' # Print the function body - echo "-------------------------------------- 💣 --------------------------------------" +# +# runners +# +run() { + if [ "$DRY_RUN" = true ]; then + echo "-------------------------------------- 💣 --------------------------------------" + return; + fi + echo "-------------------------------------- 🗲 --------------------------------------" + "${@}" >"${TESTS_STDOUT}" 2>"${TESTS_STDERR}" + errno=$? + echo "-------------------------------------- 󱦟 --------------------------------------" - if [ ! $DRY_RUN ] - then - $1 - fi - if [ $? -eq 0 ]; then - success $steps + if [ $errno -eq 0 ]; then + success "$steps.$substeps" else - echo "‼️ Error: Command failed with exit status $?" - fail $steps "$1" - fails=$((fails + 1)) - return $? + fail "$steps.$substeps" "${cmd[*]}" fi + printf "\n\n\n" } -unittests(){ - nix-build unittests.nix -} - -build_chainloader(){ - nix-build $CCACHE_FLAG chainloader.nix -} - -build_example(){ - nix-build $CCACHE_FLAG example.nix -} +run_single_test() { + subfolder="$1"; shift -multicore_subset(){ - nix-shell --pure --arg smp true $CCACHE_FLAG --argstr unikernel ./test/kernel/integration/smp --run ./test.py + cmd=( + nix-shell + --pure + --arg withCcache "$USE_CCACHE" + --argstr unikernel "$subfolder" + --run "./test.py" + ) - # The following tests are not using multiple CPU's, but have been equippedd with some anyway - # to make sure core functionality is not broken by missing locks etc. when waking up more cores. - nix-shell --pure --arg smp true $CCACHE_FLAG --argstr unikernel ./test/net/integration/udp --run ./test.py - nix-shell --pure --arg smp true $CCACHE_FLAG --argstr unikernel ./test/kernel/integration/paging --run ./test.py -} + echo "⚙️ Running this command:" + printf '%q ' "${cmd[@]}" + printf '\n' -smoke_tests(){ - nix-shell --pure $CCACHE_FLAG --argstr unikernel ./test/net/integration/udp --run ./test.py - nix-shell --pure $CCACHE_FLAG --argstr unikernel ./test/net/integration/tcp --run ./test.py - nix-shell --pure $CCACHE_FLAG --argstr unikernel ./test/kernel/integration/paging --run ./test.py - nix-shell --pure $CCACHE_FLAG --argstr unikernel ./test/kernel/integration/smp --run ./test.py + run "${cmd[@]}" + return $? } -run unittests "Build and run unit tests" - -run build_chainloader "Build the 32-bit chainloader" - -run build_example "Build the basic example" - -run multicore_subset "Run selected tests with multicore enabled" - -if [ "$QUICK_SMOKE" ]; then - - run smoke_tests "Build and run a few key smoke tests" - - if [ $fails -eq 0 ]; then - echo "" - echo "👷💬 A lot of things are working! 💪" - else - echo "" - echo "👷🧰 $fails / $steps steps failed. There's some work left to do. 🛠 " - echo "" - exit 1 - fi - exit 0 -fi +run_function() { + steps=$((steps + 1)) + echo "🚧 Step ${steps}: $2" + echo "⚙️ Running this command:" + # This will print the body of a bash function, but won't expand variables + # inside. It works well for bundling simple commands together and allows us to + # print them without wrapping them in qotes. + declare -f "$1" | sed '1d;2d;$d' | sed 's/^[[:space:]]*//' # Print the function body -# Continuing from here will run all integration tests. + run "$1" + return $? +} run_testsuite() { local base_folder="$1" shift local exclusion_list=("$@") - steps=$((steps + 1)) - substeps=1 + substeps=0 subfails=0 - echo "" echo "====================================== 🚜 ======================================" echo "" echo "🚧 $steps) Running integration tests in $base_folder" @@ -149,31 +127,14 @@ run_testsuite() { continue fi - - # The command to run, as string to be able to print the fully expanded command - cmd="nix-shell --pure $CCACHE_FLAG --argstr unikernel $subfolder --run ./test.py" - - echo "" + substeps=$((substeps + 1)) echo "🚧 Step $steps.$substeps" echo "📂 $subfolder" - echo "⚙️ Running this command:" - echo $cmd - echo "-------------------------------------- 💣 --------------------------------------" - - if [ ! $DRY_RUN ] - then - $cmd - fi - if [ $? -eq 0 ]; then - success "$steps.$substeps" - else - fail "$steps.$substeps" "$cmd" + if ! run_single_test "$subfolder"; then subfails=$((subfails + 1)) fi - substeps=$((substeps + 1)) - done if [ $subfails -eq 0 ]; then @@ -182,48 +143,176 @@ run_testsuite() { fail $steps fails=$((fails + 1)) fi + + echo "Test suite finished (ran ${substeps} tests)" + echo "--------------------------------------------------------------------------------" } # -# Kernel tests +# function suites +# used as targets for `./test.sh ...` # -exclusions=( - "LiveUpdate" # Missing includes - "context" # Outdated - references nonexisting OS::heap_end() - "fiber" # Crashes - "modules" # Requires 32-bit build, which our shell.nix is not set up for -) +unittests() { + nix-build unittests.nix +} + +build_chainloader() { + nix-build --arg withCcache "${USE_CCACHE}" chainloader.nix +} -run_testsuite "./test/kernel/integration" "${exclusions[@]}" +build_example() { + nix-build --arg withCcache "${USE_CCACHE}" example.nix +} + +multicore_subset() { + nix-shell --pure --arg smp true --arg withCcache "${USE_CCACHE}" --argstr unikernel ./test/kernel/integration/smp --run ./test.py + + # The following tests are not using multiple CPU's, but have been equippedd with some anyway + # to make sure core functionality is not broken by missing locks etc. when waking up more cores. + nix-shell --pure --arg smp true --arg withCcache "${USE_CCACHE}" --argstr unikernel ./test/net/integration/udp --run ./test.py + nix-shell --pure --arg smp true --arg withCcache "${USE_CCACHE}" --argstr unikernel ./test/kernel/integration/paging --run ./test.py +} + +smoke_tests() { + nix-shell --pure --arg withCcache "${USE_CCACHE}" --argstr unikernel ./test/net/integration/udp --run ./test.py + nix-shell --pure --arg withCcache "${USE_CCACHE}" --argstr unikernel ./test/net/integration/tcp --run ./test.py + nix-shell --pure --arg withCcache "${USE_CCACHE}" --argstr unikernel ./test/kernel/integration/paging --run ./test.py + nix-shell --pure --arg withCcache "${USE_CCACHE}" --argstr unikernel ./test/kernel/integration/smp --run ./test.py +} # -# C++ STL runtime tests +# wrappers for testsuites +# also acts as wrappers for `./test.sh ...` # -exclusions=( +kernel_tests() { + local exclusions=( + "LiveUpdate" # Missing includes + "context" # Outdated - references nonexisting OS::heap_end() + "fiber" # Crashes + "modules" # Requires 32-bit build, which our shell.nix is not set up for + ) + + run_testsuite "./test/kernel/integration" "${exclusions[@]}" +} + +stl_tests() { + local exclusions=() -) + run_testsuite "./test/stl/integration" "${exclusions[@]}" +} + +net_tests() { + local exclusions=( + "dhclient" # Times out because it requires DHCP server on the bridge. + "dhcpd" # Times out, requires certain routes to be set up. Seems easy. + "dhcpd_dhclient_linux" # We can't run userspace tests with this setup yet. + "gateway" # Requires NaCl which is currently not integrated + "http" # Linking fails, undefined ref to http_parser_parse_url, http_parser_execute + "microLB" # Missing dependencies: microLB, diskbuilder, os_add_os_library + "nat" # Times out after 3 / 6 tests seem to pass. Might be a legit bug here. + "router" # Times out, requies sudo and has complex network setup. + "router6" # Times out: iperf3: error - unable to connect to server + "vlan" # Times out. Looks similar to the nat test - maybe similar cause? + "websocket" # Linking fails, undefined ref to http_parser_parse_url, http_parser_execute + ) + + run_testsuite "./test/net/integration" "${exclusions[@]}" +} -run_testsuite "./test/stl/integration" "${exclusions[@]}" +custom_tests() { + # add your custom tests here + local exclusions=() + # for testing all subdirectories in path + : run_testsuite "./test/path/to/custom/tests*" "${exclusions[@]}" + + # for testing a single test + : run_test "./test/path/to/single_test*" +} # -# Networking tests +# entry points # -exclusions=( - "dhclient" # Times out because it requires DHCP server on the bridge. - "dhcpd" # Times out, requires certain routes to be set up. Seems easy. - "dhcpd_dhclient_linux" # We can't run userspace tests with this setup yet. - "gateway" # Requires NaCl which is currently not integrated - "http" # Linking fails, undefined ref to http_parser_parse_url, http_parser_execute - "microLB" # Missing dependencies: microLB, diskbuilder, os_add_os_library - "nat" # Times out after 3 / 6 tests seem to pass. Might be a legit bug here. - "router" # Times out, requies sudo and has complex network setup. - "router6" # Times out: iperf3: error - unable to connect to server - "vlan" # Times out. Looks similar to the nat test - maybe similar cause? - "websocket" # Linking fails, undefined ref to http_parser_parse_url, http_parser_execute -) - -run_testsuite "./test/net/integration" "${exclusions[@]}" + +run_all() { + # + # unit tests + # + run_function unittests "Build and run unit tests" + + # + # build tests + # + run_function build_chainloader "Build the 32-bit chainloader" + + run_function build_example "Build the basic example" + + run_function multicore_subset "Run selected tests with multicore enabled" + + if [ "$QUICK_SMOKE" = true ]; then + + run_function smoke_tests "Build and run a few key smoke tests" + + if [ $fails -eq 0 ]; then + echo "" + echo "👷💬 A lot of things are working! 💪" + else + echo "" + echo "👷🧰 $fails / $steps steps failed. There's some work left to do. 🛠 " + echo "" + exit 1 + fi + exit 0 + fi + + # all integration tests should go here + + run_function kernel_tests "Run kernel integration tests" + + run_function stl_tests "Run C++ STL integration tests" + + run_function net_tests "Run networking integration tests" + +} + +list_targets() { + cat <