From 74512f82d9c95aae5254fc25ae948b951b008ae2 Mon Sep 17 00:00:00 2001
From: Thorsten Klein <iwilltry42@gmail.com>
Date: Wed, 10 Nov 2021 13:09:10 +0100
Subject: [PATCH] [Enhancement] tests/e2e: speedup & cleanup (#848)

- run up to 4 tests in parallel
- send all logs to files
- output only logs of failed tests
---
 tests/common.sh                       |   3 +
 tests/extra_test_k3s_versions.sh      |   5 +-
 tests/runner.sh                       | 175 +++++++++++++++++++-------
 tests/test_basic.sh                   |  19 +--
 tests/test_config_file.sh             |  23 ++--
 tests/test_config_file_migration.sh   |   3 +
 tests/test_config_with_overrides.sh   |   6 +-
 tests/test_full_lifecycle.sh          |   3 +
 tests/test_ipam.sh                    |   3 +
 tests/test_loadbalancer.sh            |   3 +
 tests/test_memory_limits.sh           |   3 +
 tests/test_multi_master.sh            |  19 +--
 tests/test_multi_master_start_stop.sh |  24 ++--
 tests/test_node_edit.sh               |   3 +
 tests/test_registry.sh                |   2 +
 15 files changed, 212 insertions(+), 82 deletions(-)

diff --git a/tests/common.sh b/tests/common.sh
index e5deac1c..26ac8069 100755
--- a/tests/common.sh
+++ b/tests/common.sh
@@ -40,6 +40,9 @@ failed() {
   else
     log "${RED}$1${END}"
   fi
+  if [[ -n "$2" ]]; then
+    mv "$2" "$2.failed"
+  fi
   abort "test failed"
 }
 
diff --git a/tests/extra_test_k3s_versions.sh b/tests/extra_test_k3s_versions.sh
index 7be1d822..38fd44d3 100755
--- a/tests/extra_test_k3s_versions.sh
+++ b/tests/extra_test_k3s_versions.sh
@@ -3,12 +3,15 @@
 CURR_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
 [ -d "$CURR_DIR" ] || { echo "FATAL: no current dir (maybe running in zsh?)";  exit 1; }
 
-K3S_VERSIONS=("v1.18.19-k3s1" "v1.19.11-k3s1" "v1.20.7-k3s1" "v1.21.1-k3s1")
+K3S_VERSIONS=("v1.20.12-k3s1" "v1.21.6-k3s1" "v1.22.3-k3s1")
 FAILED_TESTS=()
 
 # shellcheck source=./common.sh
 source "$CURR_DIR/common.sh"
 
+LOG_FILE="$TEST_OUTPUT_DIR/$( basename "${BASH_SOURCE[0]}" ).log"
+exec >${LOG_FILE} 2>&1
+
 for version in "${K3S_VERSIONS[@]}"; do
 
   export CURRENT_STAGE="Suite | k3s-versions | $version"
diff --git a/tests/runner.sh b/tests/runner.sh
index b7a6c8ff..7937f658 100755
--- a/tests/runner.sh
+++ b/tests/runner.sh
@@ -1,7 +1,12 @@
 #!/bin/bash
 
-CURR_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
-[ -d "$CURR_DIR" ] || { echo "FATAL: no current dir (maybe running in zsh?)";  exit 1; }
+shopt -s nullglob # nullglob expands non-matching globs to zero arguments, rather than to themselves -> empty array when used to get array of filepaths
+
+CURR_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)"
+[ -d "$CURR_DIR" ] || {
+  echo "FATAL: no current dir (maybe running in zsh?)"
+  exit 1
+}
 
 : "${E2E_INCLUDE:=""}"
 : "${E2E_EXCLUDE:=""}"
@@ -20,64 +25,144 @@ info "Preparing filesystem and environment..."
 
 mkdir -p "$HOME"/.kube
 
-echo "Start time inside runner: $(date)"
+export TEST_OUTPUT_DIR="$HOME"/testoutput
+mkdir -p "$TEST_OUTPUT_DIR"
 
-section "BASIC TESTS"
+info "Start time inside runner: $(date)"
 
-for i in "$CURR_DIR"/test_*.sh ; do
-  base=$(basename "$i" .sh)
-  skip=false
-  included=false
-  excluded=false
+function run_tests() {
 
-  # prepare to skip test, if it's in the exclusion list
-  for excludetest in "${E2E_EXCLUDE[@]}"; do
-    [[ "$excludetest" =~ (^| )${base}($| ) ]] && excluded=true
-  done
+  local section_name="$1"
+  local prefix="$2"
 
-  # (re-)add test to list, if it's on inclusion list
-  for includetest in "${E2E_INCLUDE[@]}"; do
-    [[ "$includetest" =~ (^| )${base}($| ) ]] && included=true
-  done
+  local test_files=("$CURR_DIR/$prefix"*".sh")
+
+  section "$section_name ($CURR_DIR/$prefix*.sh)"
 
-  if  [[ -z "${E2E_INCLUDE}" ]]; then # no explicit include list given
-    if $excluded; then # test is on explicit exclude list
-      skip=true
+  num_total_tests=${#test_files[@]}
+
+  local included_tests=()
+  local excluded_tests=()
+
+  for i in "${test_files[@]}"; do
+
+    local base
+    base=$(basename "$i" .sh)
+    local skip=false
+    local included=false
+    local excluded=false
+
+    # prepare to skip test, if it's in the exclusion list
+    for excludetest in "${E2E_EXCLUDE[@]}"; do
+      [[ "$excludetest" =~ (^| )${base}($| ) ]] && excluded=true
+    done
+
+    # (re-)add test to list, if it's on inclusion list
+    for includetest in "${E2E_INCLUDE[@]}"; do
+      [[ "$includetest" =~ (^| )${base}($| ) ]] && included=true
+    done
+
+    if [[ -z "${E2E_INCLUDE}" ]]; then # no explicit include list given
+      if $excluded; then               # test is on explicit exclude list
+        skip=true
+      fi
+    else
+      if $included && $excluded; then # test is in both lists, so we include it
+        warn "Test ${base} is in both, exclude and include list. Include list takes precedence."
+        skip=false
+      fi
+      if ! $included; then # test is not in include list -> skip
+        skip=true
+      fi
     fi
-  else
-    if $included && $excluded; then # test is in both lists, so we include it
-      warn "Test ${base} is in both, exclude and include list. Include list takes precedence."
-      skip=false
+
+    # skip or run test
+    if [ "$skip" = true ]; then
+      highlight "***** Skipping $base *****"
+      excluded_tests+=("$i")
+    else
+      included_tests+=("$i")
     fi
-    if ! $included; then # test is not in include list -> skip
-      skip=true
+  done
+
+  local num_included_tests=${#included_tests[@]}
+  local num_excluded_tests=${#excluded_tests[@]}
+
+  #
+  # Run Tests
+  #
+  local max_batch_size=4
+  local current_batch_size=0
+  local current_batch_number=1
+  local total_batch_number=$(((num_included_tests + (max_batch_size - 1)) / max_batch_size))
+
+  info "Running $num_included_tests tests in $total_batch_number batches à max. $max_batch_size tests."
+
+  info "Starting test batch #$current_batch_number/$total_batch_number..."
+
+  for t in "${included_tests[@]}"; do
+
+    if [[ current_batch_size -eq max_batch_size ]]; then
+      info "Waiting for test batch #$current_batch_number/$total_batch_number to finish..."
+      current_batch_size=0
+      wait
+      ((current_batch_number = current_batch_number + 1))
+      info "Starting test batch #$current_batch_number/$total_batch_number..."
     fi
-  fi
 
-  # skip or run test
-  if [ "$skip" = true ]; then
-    highlight "***** Skipping $base *****"
-  else
+    local base
+    base=$(basename "$t" .sh)
     highlight "***** Running $base *****"
-    "$i" || abort "test $base failed"
+    ((current_batch_size = current_batch_size + 1))
+    "$t" &
+  done
+
+  wait
+
+  # Output logs of failed tests
+  local failed_logs=("$TEST_OUTPUT_DIR/$prefix"*".failed")
+  local num_failed_tests=${#failed_logs[@]}
+  if [[ num_failed_tests -gt 0 ]]; then
+    warn "FAILED LOGS: ${failed_logs[*]}"
+    for f in "${failed_logs[@]}"; do
+      info "FAILED -> $f"
+      local base
+      base=$(basename "$f" ".log.failed")
+      log "${RED}***************************************************${END}"
+      log "${RED}*** Failed Test ${base%%.*}${END}"
+      log "${RED}***************************************************${END}"
+      cat "$f"
+    done
+  fi
+
+  # Info Output about Test Results
+  info "FINISHED $section_name${END}
+  > ${WHT}Total:\t$num_total_tests${END}
+  > ${BLU}Run:\t$num_included_tests${END}
+  > ${YEL}Not Run:\t$num_excluded_tests${END}
+  > ${GRN}Passed:\t$((num_included_tests - num_failed_tests))${END}
+  > ${RED}Failed:\t$num_failed_tests${END}"
+
+  # Error out if we encountered any failed test
+  if [[ num_failed_tests -gt 0 ]]; then
+    warn "Failed $section_name: $num_failed_tests"
+    exit 1
   fi
-done
+}
 
-# Additional (extra) tests
+###############
+# Basic Tests #
+###############
+
+run_tests "BASIC TESTS" "test_"
+
+###############
+# Extra Tests #
+###############
 if [[ -n "$E2E_EXTRA" ]]; then
-  section "EXTRA TESTS"
-  for i in "$CURR_DIR"/extra_test_*.sh ; do
-    base=$(basename "$i" .sh)
-    if [[ $E2E_EXCLUDE =~ (^| )$base($| ) ]]; then
-      highlight "***** Skipping $base *****"
-    else
-      highlight "***** Running $base *****"
-      "$i" || abort "test $base failed"
-    fi
-  done
+  run_tests "EXTRA TESTS" "extra_test_"
 else
   info "NOT running EXTRA tests, please set E2E_EXTRA=1 if you wish to do so"
 fi
 
 exit 0
-
diff --git a/tests/test_basic.sh b/tests/test_basic.sh
index 1c934b5f..bf7bc9d4 100755
--- a/tests/test_basic.sh
+++ b/tests/test_basic.sh
@@ -6,28 +6,31 @@ CURR_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
 # shellcheck source=./common.sh
 source "$CURR_DIR/common.sh"
 
+LOG_FILE="$TEST_OUTPUT_DIR/$( basename "${BASH_SOURCE[0]}" ).log"
+exec >${LOG_FILE} 2>&1
+
 export CURRENT_STAGE="Test | basic"
 
 info "Creating two clusters..."
-$EXE cluster create c1 --wait --timeout 60s --api-port 6443 --env 'TEST_VAR=user\@pass\\@server:0' || failed "could not create cluster c1"
-$EXE cluster create c2 --wait --timeout 60s || failed "could not create cluster c2"
+$EXE cluster create c1 --wait --timeout 60s --api-port 6443 --env 'TEST_VAR=user\@pass\\@server:0' || failed_with_logfile "could not create cluster c1" "$LOG_FILE"
+$EXE cluster create c2 --wait --timeout 60s || failed_with_logfile "could not create cluster c2" "$LOG_FILE"
 
 info "Checking that we can get both clusters..."
 check_cluster_count 2
 
 info "Checking we have access to both clusters..."
-check_clusters "c1" "c2" || failed "error checking cluster"
+check_clusters "c1" "c2" || failed_with_logfile "error checking cluster" "$LOG_FILE"
 
 info "Checking cluster env var with escaped @ signs..."
-docker exec k3d-c1-server-0 env | grep -qE '^TEST_VAR=user@pass\\$' || failed "Failed to lookup proper env var in container"
+docker exec k3d-c1-server-0 env | grep -qE '^TEST_VAR=user@pass\\$' || failed_with_logfile "Failed to lookup proper env var in container" "$LOG_FILE"
 
 info "Check k3s token retrieval"
-check_cluster_token_exist "c1" || failed "could not find cluster token c1"
-check_cluster_token_exist "c2" || failed "could not find cluster token c2"
+check_cluster_token_exist "c1" || failed_with_logfile "could not find cluster token c1" "$LOG_FILE"
+check_cluster_token_exist "c2" || failed_with_logfile "could not find cluster token c2" "$LOG_FILE"
 
 info "Deleting clusters..."
-$EXE cluster delete c1 || failed "could not delete the cluster c1"
-$EXE cluster delete c2 || failed "could not delete the cluster c2"
+$EXE cluster delete c1 || failed_with_logfile "could not delete the cluster c1" "$LOG_FILE"
+$EXE cluster delete c2 || failed_with_logfile "could not delete the cluster c2" "$LOG_FILE"
 
 exit 0
 
diff --git a/tests/test_config_file.sh b/tests/test_config_file.sh
index 35062130..470109c0 100755
--- a/tests/test_config_file.sh
+++ b/tests/test_config_file.sh
@@ -6,6 +6,9 @@ CURR_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
 # shellcheck source=./common.sh
 source "$CURR_DIR/common.sh"
 
+LOG_FILE="$TEST_OUTPUT_DIR/$( basename "${BASH_SOURCE[0]}" ).log"
+exec >${LOG_FILE} 2>&1
+
 
 : "${EXTRA_FLAG:=""}"
 : "${EXTRA_TITLE:=""}"
@@ -26,46 +29,46 @@ sed -E "s/^name:.+/name: $clustername/g" < "$configfileoriginal" > "$configfile"
 highlight "[START] ConfigTest $EXTRA_TITLE"
 
 info "Creating cluster $clustername..."
-$EXE cluster create "$clustername" --config "$configfile" $EXTRA_FLAG || failed "could not create cluster $clustername $EXTRA_TITLE"
+$EXE cluster create "$clustername" --config "$configfile" $EXTRA_FLAG || failed_with_logfile "could not create cluster $clustername $EXTRA_TITLE" "$LOG_FILE"
 
 info "Sleeping for 5 seconds to give the cluster enough time to get ready..."
 sleep 5
 
 # 1. check initial access to the cluster
 info "Checking that we have access to the cluster..."
-check_clusters "$clustername" || failed "error checking cluster"
+check_clusters "$clustername" || failed_with_logfile "error checking cluster" "$LOG_FILE"
 
 info "Checking that we have 5 nodes online..."
-check_multi_node "$clustername" 5 || failed "failed to verify number of nodes"
+check_multi_node "$clustername" 5 || failed_with_logfile "failed to verify number of nodes" "$LOG_FILE"
 
 # 2. check some config settings
 
 ## Environment Variables
 info "Ensuring that environment variables are present in the node containers as set in the config (with comma)"
-exec_in_node "k3d-$clustername-server-0" "env" | grep -q "bar=baz,bob" || failed "Expected env var 'bar=baz,bob' is not present in node k3d-$clustername-server-0"
+exec_in_node "k3d-$clustername-server-0" "env" | grep -q "bar=baz,bob" || failed_with_logfile "Expected env var 'bar=baz,bob' is not present in node k3d-$clustername-server-0" "$LOG_FILE"
 
 ## Container Labels
 info "Ensuring that container labels have been set as stated in the config"
-docker_assert_container_label "k3d-$clustername-server-0" "foo=bar" || failed "Expected label 'foo=bar' not present on container/node k3d-$clustername-server-0"
+docker_assert_container_label "k3d-$clustername-server-0" "foo=bar" || failed_with_logfile "Expected label 'foo=bar' not present on container/node k3d-$clustername-server-0" "$LOG_FILE"
 
 ## K3s Node Labels
 info "Ensuring that k3s node labels have been set as stated in the config"
-k3s_assert_node_label "k3d-$clustername-server-0" "foo=bar" || failed "Expected label 'foo=bar' not present on node k3d-$clustername-server-0"
+k3s_assert_node_label "k3d-$clustername-server-0" "foo=bar" || failed_with_logfile "Expected label 'foo=bar' not present on node k3d-$clustername-server-0" "$LOG_FILE"
 
 ## Registry Node
 registryname="registry.localhost"
 info "Ensuring, that we have a registry node present"
-$EXE node list "$registryname" || failed "Expected registry node $registryname to be present"
+$EXE node list "$registryname" || failed_with_logfile "Expected registry node $registryname to be present" "$LOG_FILE"
 
 ## merged registries.yaml
 info "Ensuring, that the registries.yaml file contains both registries"
-exec_in_node "k3d-$clustername-server-0" "cat /etc/rancher/k3s/registries.yaml" | grep -qi "my.company.registry"  || failed "Expected 'my.company.registry' to be in the /etc/rancher/k3s/registries.yaml"
-exec_in_node "k3d-$clustername-server-0" "cat /etc/rancher/k3s/registries.yaml" | grep -qi "$registryname" || failed "Expected '$registryname' to be in the /etc/rancher/k3s/registries.yaml"
+exec_in_node "k3d-$clustername-server-0" "cat /etc/rancher/k3s/registries.yaml" | grep -qi "my.company.registry"  || failed_with_logfile "Expected 'my.company.registry' to be in the /etc/rancher/k3s/registries.yaml" "$LOG_FILE"
+exec_in_node "k3d-$clustername-server-0" "cat /etc/rancher/k3s/registries.yaml" | grep -qi "$registryname" || failed_with_logfile "Expected '$registryname' to be in the /etc/rancher/k3s/registries.yaml" "$LOG_FILE"
 
 # Cleanup
 
 info "Deleting cluster $clustername (using config file)..."
-$EXE cluster delete --config "$configfile" || failed "could not delete the cluster $clustername"
+$EXE cluster delete --config "$configfile" || failed_with_logfile "could not delete the cluster $clustername" "$LOG_FILE"
 
 rm "$configfile"
 
diff --git a/tests/test_config_file_migration.sh b/tests/test_config_file_migration.sh
index 8f4be093..22fdb726 100755
--- a/tests/test_config_file_migration.sh
+++ b/tests/test_config_file_migration.sh
@@ -6,6 +6,9 @@ CURR_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
 # shellcheck source=./common.sh
 source "$CURR_DIR/common.sh"
 
+LOG_FILE="$TEST_OUTPUT_DIR/$( basename "${BASH_SOURCE[0]}" ).log"
+exec >${LOG_FILE} 2>&1
+
 
 export CURRENT_STAGE="Test | config-file-migration"
 
diff --git a/tests/test_config_with_overrides.sh b/tests/test_config_with_overrides.sh
index c7731c54..1401ba3f 100755
--- a/tests/test_config_with_overrides.sh
+++ b/tests/test_config_with_overrides.sh
@@ -6,6 +6,9 @@ CURR_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
 # shellcheck source=./common.sh
 source "$CURR_DIR/common.sh"
 
+LOG_FILE="$TEST_OUTPUT_DIR/$( basename "${BASH_SOURCE[0]}" ).log"
+exec >${LOG_FILE} 2>&1
+
 
 : "${EXTRA_FLAG:=""}"
 : "${EXTRA_TITLE:=""}"
@@ -21,7 +24,7 @@ clustername="cfgoverridetest"
 highlight "[START] Config With Override $EXTRA_TITLE"
 
 info "Creating cluster $clustername..."
-$EXE cluster create "$clustername" --config "$CURR_DIR/assets/config_test_simple.yaml" --servers 4 -v /tmp/test:/tmp/test@loadbalancer --env "x=y@agent:1" $EXTRA_FLAG  || failed "could not create cluster $clustername $EXTRA_TITLE"
+$EXE cluster create "$clustername" --registry-create "newreg.localhost" --config "$CURR_DIR/assets/config_test_simple.yaml" --servers 4 -v /tmp/test:/tmp/test@loadbalancer --env "x=y@agent:1" $EXTRA_FLAG  || failed "could not create cluster $clustername $EXTRA_TITLE"
 
 info "Sleeping for 5 seconds to give the cluster enough time to get ready..."
 sleep 5
@@ -52,6 +55,7 @@ k3s_assert_node_label "k3d-$clustername-server-0" "foo=bar" || failed "Expected
 info "Ensuring, that we DO NOT have a registry node present"
 $EXE node list "k3d-$clustername-registry" && failed "Expected k3d-$clustername-registry to NOT be present"
 
+
 ## merged registries.yaml
 info "Ensuring, that the registries.yaml file contains both registries"
 exec_in_node "k3d-$clustername-server-0" "cat /etc/rancher/k3s/registries.yaml" | grep -qi "my.company.registry"  || failed "Expected 'my.company.registry' to be in the /etc/rancher/k3s/registries.yaml"
diff --git a/tests/test_full_lifecycle.sh b/tests/test_full_lifecycle.sh
index 53624f33..9ed9299d 100755
--- a/tests/test_full_lifecycle.sh
+++ b/tests/test_full_lifecycle.sh
@@ -6,6 +6,9 @@ CURR_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
 # shellcheck source=./common.sh
 source "$CURR_DIR/common.sh"
 
+LOG_FILE="$TEST_OUTPUT_DIR/$( basename "${BASH_SOURCE[0]}" ).log"
+exec >${LOG_FILE} 2>&1
+
 
 : "${EXTRA_FLAG:=""}"
 : "${EXTRA_TITLE:=""}"
diff --git a/tests/test_ipam.sh b/tests/test_ipam.sh
index bb80db69..3f7c37fd 100755
--- a/tests/test_ipam.sh
+++ b/tests/test_ipam.sh
@@ -6,6 +6,9 @@ CURR_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
 # shellcheck source=./common.sh
 source "$CURR_DIR/common.sh"
 
+LOG_FILE="$TEST_OUTPUT_DIR/$( basename "${BASH_SOURCE[0]}" ).log"
+exec >${LOG_FILE} 2>&1
+
 export CURRENT_STAGE="Test | IPAM"
 
 highlight "[START] IPAM $EXTRA_TITLE"
diff --git a/tests/test_loadbalancer.sh b/tests/test_loadbalancer.sh
index c9d17045..d12f9dc8 100755
--- a/tests/test_loadbalancer.sh
+++ b/tests/test_loadbalancer.sh
@@ -6,6 +6,9 @@ CURR_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
 # shellcheck source=./common.sh
 source "$CURR_DIR/common.sh"
 
+LOG_FILE="$TEST_OUTPUT_DIR/$( basename "${BASH_SOURCE[0]}" ).log"
+exec >${LOG_FILE} 2>&1
+
 export CURRENT_STAGE="Test | Loadbalancer"
 
 highlight "[START] LoadbalancerTest $EXTRA_TITLE"
diff --git a/tests/test_memory_limits.sh b/tests/test_memory_limits.sh
index 934a0f38..268bb3c1 100755
--- a/tests/test_memory_limits.sh
+++ b/tests/test_memory_limits.sh
@@ -6,6 +6,9 @@ CURR_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
 # shellcheck source=./common.sh
 source "$CURR_DIR/common.sh"
 
+LOG_FILE="$TEST_OUTPUT_DIR/$( basename "${BASH_SOURCE[0]}" ).log"
+exec >${LOG_FILE} 2>&1
+
 export CURRENT_STAGE="Test | MemoryLimits"
 
 highlight "[START] MemoryLimitTest $EXTRA_TITLE"
diff --git a/tests/test_multi_master.sh b/tests/test_multi_master.sh
index e11444b6..3099095d 100755
--- a/tests/test_multi_master.sh
+++ b/tests/test_multi_master.sh
@@ -6,6 +6,9 @@ CURR_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
 # shellcheck source=./common.sh
 source "$CURR_DIR/common.sh"
 
+LOG_FILE="$TEST_OUTPUT_DIR/$( basename "${BASH_SOURCE[0]}" ).log"
+exec >${LOG_FILE} 2>&1
+
 : "${EXTRA_FLAG:=""}"
 : "${EXTRA_TITLE:=""}"
 
@@ -16,21 +19,23 @@ fi
 
 export CURRENT_STAGE="Test | multi-server | $K3S_IMAGE_TAG"
 
-info "Creating cluster multiserver $EXTRA_TITLE ..."
-$EXE cluster create "multiserver" --servers 3 --api-port 6443 --wait --timeout 360s $EXTRA_FLAG || failed "could not create cluster multiserver $EXTRA_TITLE"
+clustername="multiserver"
+
+info "Creating cluster $clustername $EXTRA_TITLE ..."
+$EXE cluster create "$clustername" --servers 3 --api-port 6443 --wait --timeout 360s $EXTRA_FLAG || failed "could not create cluster $clustername $EXTRA_TITLE"
 info "Checking that we have access to the cluster..."
-check_clusters "multiserver" || failed "error checking cluster $EXTRA_TITLE"
+check_clusters "$clustername" || failed "error checking cluster $EXTRA_TITLE"
 
 info "Sleeping for 5 seconds to give the cluster enough time to get ready..."
 sleep 5
 
 info "Checking that we have 3 server nodes online..."
-check_multi_node "multiserver" 3 || failed "failed to verify number of nodes $EXTRA_TITLE"
+check_multi_node "$clustername" 3 || failed "failed to verify number of nodes $EXTRA_TITLE"
 
-info "Deleting cluster multiserver..."
-$EXE cluster delete "multiserver" || failed "could not delete the cluster multiserver $EXTRA_TITLE"
+info "Deleting cluster $clustername..."
+$EXE cluster delete "$clustername" || failed "could not delete the cluster $clustername $EXTRA_TITLE"
 
-passed "GOOD: multiserver cluster test successful $EXTRA_TITLE"
+passed "GOOD: $clustername cluster test successful $EXTRA_TITLE"
 
 exit 0
 
diff --git a/tests/test_multi_master_start_stop.sh b/tests/test_multi_master_start_stop.sh
index b32f040a..0b462bb0 100755
--- a/tests/test_multi_master_start_stop.sh
+++ b/tests/test_multi_master_start_stop.sh
@@ -8,6 +8,9 @@ KNOWN_TO_FAIL=("v1.17.17-k3s1" "v1.18.15-k3s1") # some versions of k3s don't wor
 # shellcheck source=./common.sh
 source "$CURR_DIR/common.sh"
 
+LOG_FILE="$TEST_OUTPUT_DIR/$( basename "${BASH_SOURCE[0]}" ).log"
+exec >${LOG_FILE} 2>&1
+
 : "${EXTRA_FLAG:=""}"
 : "${EXTRA_TITLE:=""}"
 
@@ -25,37 +28,38 @@ if [[ -n "$K3S_IMAGE_TAG" ]]; then
   EXTRA_TITLE="(rancher/k3s:$K3S_IMAGE_TAG)"
 fi
 
+clustername="multiserverstartstop"
 
-info "Creating cluster multiserver $EXTRA_TITLE ..."
-$EXE cluster create "multiserver" --servers 3 --api-port 6443 --wait --timeout 360s $EXTRA_FLAG || failed "could not create cluster multiserver $EXTRA_TITLE"
+info "Creating cluster $clustername $EXTRA_TITLE ..."
+$EXE cluster create "$clustername" --servers 3 --api-port 6443 --wait --timeout 360s $EXTRA_FLAG || failed "could not create cluster $clustername $EXTRA_TITLE"
 info "Checking that we have access to the cluster..."
-check_clusters "multiserver" || failed "error checking cluster $EXTRA_TITLE"
+check_clusters "$clustername" || failed "error checking cluster $EXTRA_TITLE"
 
 info "Sleeping for 5 seconds to give the cluster enough time to get ready..."
 sleep 5
 
 info "Checking that we have 3 server nodes online..."
-check_multi_node "multiserver" 3 || failed "failed to verify number of nodes $EXTRA_TITLE"
+check_multi_node "$clustername" 3 || failed "failed to verify number of nodes $EXTRA_TITLE"
 
 info "Stopping cluster..."
-$EXE cluster stop "multiserver" || failed "failed to stop cluster"
+$EXE cluster stop "$clustername" || failed "failed to stop cluster"
 
 info "Waiting for a bit..."
 sleep 5
 
 info "Restarting cluster (time: $(date -u +"%Y-%m-%d %H:%M:%S %Z"))..."
-$EXE cluster start multiserver --timeout 360s || failed "failed to restart cluster (timeout 360s)"
+$EXE cluster start $clustername --timeout 360s || failed "failed to restart cluster (timeout 360s)"
 
 info "Sleeping for 5 seconds to give the cluster enough time to get ready..."
 sleep 5
 
 info "Checking that we have access to the cluster..."
-check_clusters "multiserver" || failed "failed to verify that we have access to the cluster"
+check_clusters "$clustername" || failed "failed to verify that we have access to the cluster"
 
-info "Deleting cluster multiserver..."
-$EXE cluster delete "multiserver" || failed "could not delete the cluster multiserver $EXTRA_TITLE"
+info "Deleting cluster $clustername..."
+$EXE cluster delete "$clustername" || failed "could not delete the cluster $clustername $EXTRA_TITLE"
 
-passed "GOOD: multiserver cluster test successful $EXTRA_TITLE"
+passed "GOOD: $clustername cluster test successful $EXTRA_TITLE"
 
 exit 0
 
diff --git a/tests/test_node_edit.sh b/tests/test_node_edit.sh
index 2c6f7f2c..15435e35 100755
--- a/tests/test_node_edit.sh
+++ b/tests/test_node_edit.sh
@@ -6,6 +6,9 @@ CURR_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
 # shellcheck source=./common.sh
 source "$CURR_DIR/common.sh"
 
+LOG_FILE="$TEST_OUTPUT_DIR/$( basename "${BASH_SOURCE[0]}" ).log"
+exec >${LOG_FILE} 2>&1
+
 export CURRENT_STAGE="Test | NodeEdit"
 
 highlight "[START] NodeEdit"
diff --git a/tests/test_registry.sh b/tests/test_registry.sh
index 2426407e..fdbf6e8f 100755
--- a/tests/test_registry.sh
+++ b/tests/test_registry.sh
@@ -6,6 +6,8 @@ CURR_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
 # shellcheck source=./common.sh
 source "$CURR_DIR/common.sh"
 
+LOG_FILE="$TEST_OUTPUT_DIR/$( basename "${BASH_SOURCE[0]}" ).log"
+exec >${LOG_FILE} 2>&1
 
 : "${EXTRA_FLAG:=""}"
 : "${EXTRA_TITLE:=""}"