From b36ad1c933f6e8795c01924eb0f8365b7378afcb Mon Sep 17 00:00:00 2001 From: Samuel Hym Date: Fri, 18 Nov 2022 18:40:17 +0100 Subject: [PATCH] Add information to logs with a custom runner Add a bash script to use as runner for our test suite This script adds information to the logs: - explicit names of the test run when they start - some explicit messages for crashes (SIGSEGV, SIGBUS, ...) - and, maybe most importantly, anchors in CI logs, so that the main webpage contains the most important information and direct links to precise points of interest in the log It is written as a bash script but making sure it can be used as a runner also on Windows CI (notably by commenting all ends of lines, without which it fails with errors about '\r's that don't exist...) --- src/array/dune | 6 ++--- src/atomic/dune | 6 ++--- src/bigarray/dune | 3 +-- src/buffer/dune | 2 +- src/bytes/dune | 6 ++--- src/domain/dune | 4 +-- src/domainslib/dune | 8 +++--- src/ephemeron/dune | 4 +-- src/floatarray/dune | 6 ++--- src/hashtbl/dune | 6 ++--- src/internal/dune | 4 +-- src/lazy/dune | 6 ++--- src/lockfree/dune | 2 +- src/neg_tests/dune | 22 ++++++++--------- src/queue/dune | 4 +-- src/runner.sh | 59 ++++++++++++++++++++++++++++++++++++++++++++ src/semaphore/dune | 2 +- src/stack/dune | 4 +-- src/thread/dune | 4 +-- src/threadomain/dune | 2 +- 20 files changed, 107 insertions(+), 53 deletions(-) create mode 100755 src/runner.sh diff --git a/src/array/dune b/src/array/dune index bf98065f0..72d1cbc1d 100644 --- a/src/array/dune +++ b/src/array/dune @@ -19,7 +19,7 @@ (alias runtest) (package multicoretests) (deps stm_tests.exe) - (action (run ./%{deps} --verbose))) + (action (bash "%{dep:../runner.sh} %{deps}"))) (executable (name lin_tests) @@ -32,7 +32,7 @@ ; (alias runtest) ; (package multicoretests) ; (deps lin_tests.exe) -; (action (run ./%{deps} --verbose))) +; (action (bash "%{dep:../runner.sh} %{deps}"))) (executable (name lin_tests_dsl) @@ -43,4 +43,4 @@ (alias runtest) (package multicoretests) (deps lin_tests_dsl.exe) - (action (run ./%{deps} --verbose))) + (action (bash "%{dep:../runner.sh} %{deps}"))) diff --git a/src/atomic/dune b/src/atomic/dune index 887fac86c..01251f751 100644 --- a/src/atomic/dune +++ b/src/atomic/dune @@ -22,7 +22,7 @@ (alias runtest) (package multicoretests) (deps stm_tests.exe) - (action (run ./%{deps} --verbose))) + (action (bash "%{dep:../runner.sh} %{deps}"))) ;; Linearizability tests of Atomic, utilizing ppx_deriving_qcheck @@ -38,7 +38,7 @@ ; (alias runtest) ; (package multicoretests) ; (deps lin_tests.exe) -; (action (run ./%{deps} --verbose))) +; (action (bash "%{dep:../runner.sh} %{deps}"))) (executable (name lin_tests_dsl) @@ -49,4 +49,4 @@ (alias runtest) (package multicoretests) (deps lin_tests_dsl.exe) - (action (run ./%{deps} --verbose))) + (action (bash "%{dep:../runner.sh} %{deps}"))) diff --git a/src/bigarray/dune b/src/bigarray/dune index 9c9d1cc62..0f0b82a31 100644 --- a/src/bigarray/dune +++ b/src/bigarray/dune @@ -25,5 +25,4 @@ (alias runtest) (package multicoretests) (deps lin_tests_dsl.exe) - (action - (run ./%{deps} --verbose))) + (action (bash "%{dep:../runner.sh} %{deps}"))) diff --git a/src/buffer/dune b/src/buffer/dune index f0fb4a262..20b8d0d3a 100644 --- a/src/buffer/dune +++ b/src/buffer/dune @@ -17,4 +17,4 @@ (alias runtest) (package multicoretests) (deps stm_tests.exe) - (action (run ./%{deps} --verbose))) + (action (bash "%{dep:../runner.sh} %{deps}"))) diff --git a/src/bytes/dune b/src/bytes/dune index f5ea93fa2..19b27b94e 100644 --- a/src/bytes/dune +++ b/src/bytes/dune @@ -25,12 +25,10 @@ (alias runtest) (package multicoretests) (deps stm_tests.exe) - (action - (run ./%{deps} --verbose))) + (action (bash "%{dep:../runner.sh} %{deps}"))) (rule (alias runtest) (package multicoretests) (deps lin_tests_dsl.exe) - (action - (run ./%{deps} --verbose))) + (action (bash "%{dep:../runner.sh} %{deps}"))) diff --git a/src/domain/dune b/src/domain/dune index e1115f71c..0697f8703 100644 --- a/src/domain/dune +++ b/src/domain/dune @@ -19,7 +19,7 @@ (alias runtest) (package multicoretests) (deps domain_joingraph.exe) - (action (run ./%{deps} --verbose))) + (action (bash "%{dep:../runner.sh} %{deps}"))) (executable (name domain_spawntree) @@ -31,4 +31,4 @@ (alias runtest) (deps domain_spawntree.exe) (package multicoretests) - (action (run ./%{deps} --verbose))) + (action (bash "%{dep:../runner.sh} %{deps}"))) diff --git a/src/domainslib/dune b/src/domainslib/dune index 6de896e3a..b46ca4b6a 100644 --- a/src/domainslib/dune +++ b/src/domainslib/dune @@ -22,7 +22,7 @@ (alias runtest) (package multicoretests) (deps task_one_dep.exe) - (action (run ./%{deps} --verbose))) + (action (bash "%{dep:../runner.sh} %{deps}"))) (executable (name task_more_deps) @@ -34,7 +34,7 @@ (alias runtest) (deps task_more_deps.exe) (package multicoretests) - (action (run ./%{deps} --verbose))) + (action (bash "%{dep:../runner.sh} %{deps}"))) (executable (name task_parallel) @@ -45,7 +45,7 @@ (alias runtest) (package multicoretests) (deps task_parallel.exe) - (action (run ./%{deps} --verbose))) + (action (bash "%{dep:../runner.sh} %{deps}"))) ;; STM_seq and STM_domain test of Domainslib.Chan @@ -59,4 +59,4 @@ (rule (alias runtest) (deps chan_stm_tests.exe) - (action (run ./%{deps} --verbose))) + (action (bash "%{dep:../runner.sh} %{deps}"))) diff --git a/src/ephemeron/dune b/src/ephemeron/dune index cbc17cda1..cc5db17ce 100644 --- a/src/ephemeron/dune +++ b/src/ephemeron/dune @@ -15,7 +15,7 @@ (alias runtest) (package multicoretests) (deps stm_tests.exe) - (action (run ./%{deps} --verbose))) + (action (bash "%{dep:../runner.sh} %{deps}"))) (executable (name lin_tests_dsl) @@ -26,4 +26,4 @@ (alias runtest) (package multicoretests) (deps lin_tests_dsl.exe) - (action (run ./%{deps} --verbose))) + (action (bash "%{dep:../runner.sh} %{deps}"))) diff --git a/src/floatarray/dune b/src/floatarray/dune index dd05e8bdd..5113edb4b 100644 --- a/src/floatarray/dune +++ b/src/floatarray/dune @@ -18,8 +18,7 @@ (alias runtest) (package multicoretests) (deps stm_tests.exe) - (action - (run ./%{deps} --verbose))) + (action (bash "%{dep:../runner.sh} %{deps}"))) (executable (name lin_tests_dsl) @@ -30,5 +29,4 @@ (alias runtest) (package multicoretests) (deps lin_tests_dsl.exe) - (action - (run ./%{deps} --verbose))) + (action (bash "%{dep:../runner.sh} %{deps}"))) diff --git a/src/hashtbl/dune b/src/hashtbl/dune index 848668f5c..902d79e12 100644 --- a/src/hashtbl/dune +++ b/src/hashtbl/dune @@ -19,7 +19,7 @@ (alias runtest) (package multicoretests) (deps stm_tests.exe) - (action (run ./%{deps} --verbose))) + (action (bash "%{dep:../runner.sh} %{deps}"))) (executable (name lin_tests) @@ -32,7 +32,7 @@ ; (alias runtest) ; (package multicoretests) ; (deps lin_tests.exe) -; (action (run ./%{deps} --verbose))) +; (action (bash "%{dep:../runner.sh} %{deps}"))) (executable (name lin_tests_dsl) @@ -43,4 +43,4 @@ (alias runtest) (package multicoretests) (deps lin_tests_dsl.exe) - (action (run ./%{deps} --verbose))) + (action (bash "%{dep:../runner.sh} %{deps}"))) diff --git a/src/internal/dune b/src/internal/dune index 3c61a6921..3c2bedd6f 100644 --- a/src/internal/dune +++ b/src/internal/dune @@ -15,7 +15,7 @@ (alias runtest) (deps util_print_test.exe) (package multicoretests) - (action (run ./%{deps}))) + (action (bash "%{dep:../runner.sh} %{deps}"))) (executable @@ -28,4 +28,4 @@ (alias runtest) (deps cleanup.exe) (package multicoretests) - (action (run ./%{deps} --verbose))) + (action (bash "%{dep:../runner.sh} %{deps}"))) diff --git a/src/lazy/dune b/src/lazy/dune index ef7820e9a..c0575c468 100644 --- a/src/lazy/dune +++ b/src/lazy/dune @@ -19,7 +19,7 @@ (alias runtest) (package multicoretests) (deps stm_tests.exe) - (action (run ./%{deps} --verbose))) + (action (bash "%{dep:../runner.sh} %{deps}"))) (executable (name lin_tests) @@ -31,7 +31,7 @@ ; (alias runtest) ; (package multicoretests) ; (deps lin_tests.exe) -; (action (run ./%{deps} --verbose))) +; (action (bash "%{dep:../runner.sh} %{deps}"))) (executable (name lin_tests_dsl) @@ -42,4 +42,4 @@ ; (alias runtest) ; (package multicoretests) ; (deps lin_tests_dsl.exe) -; (action (run ./%{deps} --verbose))) +; (action (bash "%{dep:../runner.sh} %{deps}"))) diff --git a/src/lockfree/dune b/src/lockfree/dune index 342982c68..cbee0d906 100644 --- a/src/lockfree/dune +++ b/src/lockfree/dune @@ -18,4 +18,4 @@ (alias runtest) (package multicoretests) (deps ws_deque_test.exe) - (action (run ./%{deps} --verbose))) + (action (bash "%{dep:../runner.sh} %{deps}"))) diff --git a/src/neg_tests/dune b/src/neg_tests/dune index ba7046c13..716852839 100644 --- a/src/neg_tests/dune +++ b/src/neg_tests/dune @@ -45,19 +45,19 @@ (alias runtest) (package multicoretests) (deps stm_tests_sequential_ref.exe) - (action (run ./%{deps} --verbose))) + (action (bash "%{dep:../runner.sh} %{deps}"))) (rule (alias runtest) (package multicoretests) (deps stm_tests_domain_ref.exe) - (action (run ./%{deps} --verbose))) + (action (bash "%{dep:../runner.sh} %{deps}"))) (rule (alias runtest) (package multicoretests) (deps stm_tests_thread_ref.exe) - (action (run ./%{deps} --verbose))) + (action (bash "%{dep:../runner.sh} %{deps}"))) (library (name CList) @@ -73,7 +73,7 @@ (alias runtest) (package multicoretests) (deps stm_tests_conclist.exe) - (action (run ./%{deps} --verbose))) + (action (bash "%{dep:../runner.sh} %{deps}"))) ;; Linearizability tests of ref and Clist @@ -112,19 +112,19 @@ (alias runtest) (package multicoretests) (deps lin_tests_dsl_domain.exe) - (action (run ./%{deps} --verbose))) + (action (bash "%{dep:../runner.sh} %{deps}"))) ; (rule ; (alias runtest) ; (package multicoretests) ; (deps lin_tests_dsl_thread.exe) -; (action (run ./%{deps} --verbose))) +; (action (bash "%{dep:../runner.sh} %{deps}"))) (rule (alias runtest) (package multicoretests) (deps lin_tests_dsl_effect.exe) - (action (run ./%{deps} --verbose))) + (action (bash "%{dep:../runner.sh} %{deps}"))) (executable (name lin_tests_domain) @@ -149,22 +149,22 @@ ; (alias runtest) ; (package multicoretests) ; (deps lin_tests_domain.exe) -; (action (run ./%{deps} --verbose))) +; (action (bash "%{dep:../runner.sh} %{deps}"))) (rule (alias runtest) (package multicoretests) (deps lin_tests_thread_ref.exe) - (action (run ./%{deps} --verbose))) + (action (bash "%{dep:../runner.sh} %{deps}"))) (rule (alias runtest) (package multicoretests) (deps lin_tests_thread_conclist.exe) - (action (run ./%{deps} --verbose))) + (action (bash "%{dep:../runner.sh} %{deps}"))) ; (rule ; (alias runtest) ; (package multicoretests) ; (deps lin_tests_effect.exe) -; (action (run ./%{deps} --verbose))) +; (action (bash "%{dep:../runner.sh} %{deps}"))) diff --git a/src/queue/dune b/src/queue/dune index 749c59657..7f98a7f2b 100644 --- a/src/queue/dune +++ b/src/queue/dune @@ -18,7 +18,7 @@ (alias runtest) (package multicoretests) (deps lin_tests_dsl.exe) - (action (run ./%{deps} --verbose))) + (action (bash "%{dep:../runner.sh} %{deps}"))) (executable (name lin_tests) @@ -31,4 +31,4 @@ ; (alias runtest) ; (package multicoretests) ; (deps lin_tests.exe) -; (action (run ./%{deps} --verbose))) +; (action (bash "%{dep:../runner.sh} %{deps}"))) diff --git a/src/runner.sh b/src/runner.sh new file mode 100755 index 000000000..992b57c56 --- /dev/null +++ b/src/runner.sh @@ -0,0 +1,59 @@ +#!/bin/bash +# A runner for tests that adds timestamps and, in CI, anchors in logs +# All lines end as comments since bash on Windows CI seems to conjure +# some '\r' it doesn’t like into existence +test="$1" # +warning() { # +if [ "$CI" = true ] ; then # + printf '\n::warning title=%s in %s/%s::%s in src/%s/%s' "$1" "${PWD##*/}" "$test" "$1" "${PWD##*/}" "$test" # +else # + printf '\nWarning: %s in src/%s/%s' "$1" "${PWD##*/}" "$test" # +fi # +} # +error() { # +if [ "$CI" = true ] ; then # + printf '\n::error title=%s in %s/%s::%s in src/%s/%s' "$1" "${PWD##*/}" "$test" "$1" "${PWD##*/}" "$test" # +else # + printf '\nError: %s in src/%s/%s' "$1" "${PWD##*/}" "$test" # +fi # +} # + # +if [[ "$2" = "" ]] ; then # + printf '\n\nStarting "./%s --verbose" in %s\n' "$test" "${PWD##*/}" # + ./"$test" --verbose # + result="$?" # +else # + shift # + printf '\n\nStarting "./%s %s" in %s\n' "$test" "$*" "${PWD##*/}" # + ./"$test" "$@" # + result="$?" # +fi # +case "$result" in # + 0) # + exit 0 # + ;; # + 1) # + warning "Test failure" # + exit 1 # + ;; # + 139) # + error SIGSEGV # + kill -SIGSEGV $$ # + ;; # + 135) # + error SIGBUS # + kill -SIGBUS $$ # + ;; # + 137) # + error SIGKILL # + kill -SIGKILL $$ # + ;; # + 134) # + error SIGABRT # + kill -SIGABRT $$ # + ;; # + *) # + error "$result" # + exit "$result" # + ;; # +esac # diff --git a/src/semaphore/dune b/src/semaphore/dune index 8c1394c67..11d329758 100644 --- a/src/semaphore/dune +++ b/src/semaphore/dune @@ -16,4 +16,4 @@ (alias runtest) (package multicoretests) (deps stm_tests.exe) - (action (run ./%{deps} --verbose))) + (action (bash "%{dep:../runner.sh} %{deps}"))) diff --git a/src/stack/dune b/src/stack/dune index 71636f88d..3277a86ff 100644 --- a/src/stack/dune +++ b/src/stack/dune @@ -18,7 +18,7 @@ (alias runtest) (package multicoretests) (deps lin_tests_dsl.exe) - (action (run ./%{deps} --verbose))) + (action (bash "%{dep:../runner.sh} %{deps}"))) (executable (name lin_tests) @@ -31,5 +31,5 @@ ; (alias runtest) ; (package multicoretests) ; (deps lin_tests.exe) -; (action (run ./%{deps} --verbose))) +; (action (bash "%{dep:../runner.sh} %{deps}"))) diff --git a/src/thread/dune b/src/thread/dune index 310319a32..df96f882d 100644 --- a/src/thread/dune +++ b/src/thread/dune @@ -19,7 +19,7 @@ (alias runtest) (package multicoretests) (deps thread_joingraph.exe) - (action (run ./%{deps} --verbose))) + (action (bash "%{dep:../runner.sh} %{deps}"))) (executable (name thread_createtree) @@ -31,4 +31,4 @@ (alias runtest) (deps thread_createtree.exe) (package multicoretests) - (action (run ./%{deps} --verbose))) + (action (bash "%{dep:../runner.sh} %{deps}"))) diff --git a/src/threadomain/dune b/src/threadomain/dune index 83af6cf52..c770ca842 100644 --- a/src/threadomain/dune +++ b/src/threadomain/dune @@ -16,4 +16,4 @@ (alias runtest) (package multicoretests) (deps threadomain.exe) - (action (run ./%{deps} --verbose))) + (action (bash "%{dep:../runner.sh} %{deps}")))