diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 1a954835b1e4f6bdba39056f99763d5ac6d9ead8..245bc07f97b13c9913321286757aa8acbc3eec52 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -84,6 +84,52 @@ do-build-centos75:
OS: 'centos75'
+#
+# test_cmds.sh orig
+#
+.basic-pgms-orig: &basic-pgms-orig
+ stage: test
+ script:
+ - ./cicd_tests/basic-pgms-orig.sh
+
+basic-pgms-orig-ubuntu18:
+ <<: *basic-pgms-orig
+ tags:
+ - ubuntu18
+
+basic-pgms-orig-ubuntu16:
+ <<: *basic-pgms-orig
+ tags:
+ - ubuntu16
+
+basic-pgms-orig-centos75:
+ <<: *basic-pgms-orig
+ tags:
+ - centos75
+
+#
+# verify that test_cmds.sh detects test failure
+#
+.verify-fails: &verify-fails
+ stage: test
+ script:
+ - ./cicd_tests/verify-fails.sh
+
+verify-fails-ubuntu18:
+ <<: *verify-fails
+ tags:
+ - ubuntu18
+
+verify-fails-ubuntu16:
+ <<: *verify-fails
+ tags:
+ - ubuntu16
+
+verify-fails-centos75:
+ <<: *verify-fails
+ tags:
+ - centos75
+
#
# $PSZ ls
#
@@ -147,26 +193,4 @@ xform-cat-centos75:
variables:
OS: 'centos75'
-#
-# test_cmds.sh orig
-#
-.basic-pgms-orig: &basic-pgms-orig
- stage: test
- script:
- - ./cicd_tests/basic-pgms-orig.sh
-
-basic-pgms-orig-ubuntu18:
- <<: *basic-pgms-orig
- tags:
- - ubuntu18
-
-basic-pgms-orig-ubuntu16:
- <<: *basic-pgms-orig
- tags:
- - ubuntu16
-
-basic-pgms-orig-centos75:
- <<: *basic-pgms-orig
- tags:
- - centos75
diff --git a/cicd_tests/verify-fails.sh b/cicd_tests/verify-fails.sh
new file mode 100755
index 0000000000000000000000000000000000000000..a6cae1e3ec820d4e24a9d350b1a4a03e99f2d595
--- /dev/null
+++ b/cicd_tests/verify-fails.sh
@@ -0,0 +1,16 @@
+#!/bin/bash
+set -e
+set -x
+
+cd $CICD_MODULE_WORK_DIR/peasoup_umbrella
+source set_env_vars
+
+cd $PEASOUP_HOME/tests
+make clean
+
+# verify "fail" configuration
+./test_cmds.sh -l -c expect_fail -a ls
+./test_cmds.sh -l -c expect_fail -a grep
+
+exit 0
+
diff --git a/tests/ls/test_script.sh b/tests/ls/test_script.sh
index 424cb67c73932a94e94985217893637d6799f148..3fe317f0c01d92ff463abde547fd22b0b889556a 100755
--- a/tests/ls/test_script.sh
+++ b/tests/ls/test_script.sh
@@ -21,27 +21,27 @@ echo "TEST_PROG: $TEST_PROG"
run_basic_test 120 --help
run_basic_test 120 --version
run_basic_test 120 --doesnotexist
-run_basic_test 120 ls $DATA_DIR
-run_basic_test 120 ls $DATA_DIR/not-exist
-run_basic_test 120 ls -a $DATA_DIR
-run_basic_test 120 ls -l $DATA_DIR
-run_basic_test 120 ls -ltr $DATA_DIR
-run_basic_test 120 ls -ltrR $DATA_DIR
-run_basic_test 120 ls -ha $DATA_DIR
-run_basic_test 120 ls -Z $DATA_DIR
-run_basic_test 120 ls -Rd $DATA_DIR
-run_basic_test 120 ls -Rlg $DATA_DIR
-run_basic_test 120 ls -lC -s $DATA_DIR
-run_basic_test 120 ls -C --color=always $DATA_DIR
-run_basic_test 120 ls -F $DATA_DIR
-run_basic_test 120 ls -Fg $DATA_DIR
-run_basic_test 120 ls -ha --si $DATA_DIR
-run_basic_test 120 ls -ha --si -i $DATA_DIR
-run_basic_test 120 ls -lt --time-style=full-iso $DATA_DIR
-run_basic_test 120 ls -lt --time-style=long-iso $DATA_DIR
-run_basic_test 120 ls -lt --time-style=iso $DATA_DIR
-run_basic_test 120 ls -lt --tabsize=4 --time-style=iso $DATA_DIR
-run_basic_test 120 ls -ltZ $DATA_DIR
+run_basic_test 120 $DATA_DIR
+run_basic_test 120 $DATA_DIR/not-exist
+run_basic_test 120 -a $DATA_DIR
+run_basic_test 120 -l $DATA_DIR
+run_basic_test 120 -ltr $DATA_DIR
+run_basic_test 120 -ltrR $DATA_DIR
+run_basic_test 120 -ha $DATA_DIR
+run_basic_test 120 -Z $DATA_DIR
+run_basic_test 120 -Rd $DATA_DIR
+run_basic_test 120 -Rlg $DATA_DIR
+run_basic_test 120 -lC -s $DATA_DIR
+run_basic_test 120 -C --color=always $DATA_DIR
+run_basic_test 120 -F $DATA_DIR
+run_basic_test 120 -Fg $DATA_DIR
+run_basic_test 120 -ha --si $DATA_DIR
+run_basic_test 120 -ha --si -i $DATA_DIR
+run_basic_test 120 -lt --time-style=full-iso $DATA_DIR
+run_basic_test 120 -lt --time-style=long-iso $DATA_DIR
+run_basic_test 120 -lt --time-style=iso $DATA_DIR
+run_basic_test 120 -lt --tabsize=4 --time-style=iso $DATA_DIR
+run_basic_test 120 -ltZ $DATA_DIR
cleanup
diff --git a/tests/manual_test_lib.sh b/tests/manual_test_lib.sh
index e4b40decef5610e35f79ef45ce39d426c24a6a91..9c07c7fdcae18359fd028aa8fe8c1008d3c76021 100755
--- a/tests/manual_test_lib.sh
+++ b/tests/manual_test_lib.sh
@@ -31,23 +31,29 @@ if [[ ! -z "$IGNORE_RESULTS" ]]; then
echo "TEST SCRIPT COVERAGE RUN: running test script ignoring results."
fi
-TEST_PROG=$1
-BENCH=$2
+TEST_PROG=$(realpath $1)
+BENCH=$(realpath $2)
+
+DUMMY_NAME=DUMMY
+TEST_BASE=`basename $TEST_PROG`
+BENCH_BASE=`basename $BENCH`
+SCRATCH_DIR="$(dirname $BENCH)"
+
+TEST_PROG_ORIG=$SCRATCH_DIR/$TEST_BASE.orig
+BENCH_ORIG=$SCRATCH_DIR/$BENCH_BASE.xform
+CANONICAL=$SCRATCH_DIR/$TEST_BASE
+
+cp $TEST_PROG $TEST_PROG_ORIG
+cp $BENCH $BENCH_ORIG
if [ ! -z "$TEST_VERBOSE" ]; then
echo
- echo "Original program: $TEST_PROG"
- ls -lt $TEST_PROG
- echo "Test program: $BENCH"
- ls -lt $BENCH
+ echo "Original program : $TEST_PROG --> $TEST_PROG_ORIG"
+ echo "Test program : $BENCH --> $BENCH_ORIG"
+ echo "Canonical program: $CANONICAL"
echo
fi
-DUMMY_NAME=DUMMY
-
-BENCH_BASE=`basename $BENCH`
-TEST_BASE=`basename $TEST_PROG`
-
NAME_REGEX=`echo "($BENCH_BASE\|$TEST_BASE\|$ORIG_NAME\|.peasoup\|.ncexe\|.stratafied)" | sed 's/\./\\\./'`
usage()
@@ -128,6 +134,9 @@ run_test_prog_only()
TIMEOUT=$1
shift
+ cp $TEST_PROG_ORIG $CANONICAL
+ TEST_PROG=$CANONICAL
+
cmd_args="$@"
if [[ "$TEST_PROG" == "" ]]; then
@@ -220,6 +229,9 @@ run_server_test_prog_only()
run_bench_prog_only()
{
+ cp $BENCH_ORIG $CANONICAL
+ BENCH=$CANONICAL
+
TIMEOUT=$1
shift
@@ -399,11 +411,11 @@ compare_std_results()
diff orig_out test_out
if [ ! $? -eq 0 ]; then
echo "run_test Stdout Failure"
- report_failure
if [ ! -z "$TEST_VERBOSE" ]; then
show_log orig_out
show_log test_out
fi
+ report_failure
fi
}
diff --git a/tests/tcpdump/test_script.sh b/tests/tcpdump/test_script.sh
index 04bcea9b2dc609c4f4ce3cc4ef1453e7adf38b27..334d3239158acc90977f8af904deb6cc7da89cbb 100755
--- a/tests/tcpdump/test_script.sh
+++ b/tests/tcpdump/test_script.sh
@@ -1,24 +1,19 @@
#!/bin/bash
-TEST_LIB="$(dirname "$(realpath $0)")/../manual_test_lib.sh"
+TEST_DIR=$PEASOUP_HOME/tests/tcpdump
+TEST_LIB="$(realpath "$TEST_DIR/../manual_test_lib.sh")"
#used for filtering program names from output.
ORIG_NAME=tcpdump
-
#must import the library here, as it depends on some of the above variables
. $TEST_LIB
DELETE_FILTER="stonesoup|gcc|lib|DUMMY|exec|python|tcpdump"
-TEST_DIR=$PEASOUP_HOME/tests/tcpdump
-
-# debugging
-ls -lt $TEST_DIR/tcpd_tests/lmp.pcap
-
run_basic_test 20 -h
-run_basic_test 20 -$i -s0 -nr $TEST_DIR/tcpd_tests/print-flags.pcap
run_basic_test 20 -n -r $TEST_DIR/tcpd_tests/bgp_vpn_attrset.pcap -t -v
+run_basic_test 20 -$i -s0 -nr $TEST_DIR/tcpd_tests/print-flags.pcap
run_basic_test 20 -n -r $TEST_DIR/tcpd_tests/mpbgp-linklocal-nexthop.pcap -t -v
run_basic_test 20 -n -r $TEST_DIR/tcpd_tests/eapon1.pcap -t
run_basic_test 20 -n -r $TEST_DIR/tcpd_tests/02-sunrise-sunset-esp.pcap -t -n
diff --git a/tests/test_cmds.sh b/tests/test_cmds.sh
index ce3f0e479c71ae917a2bca98c082d46cf13b8b23..8e50df2f3a685e336c603b4da1ef9b4c698d89ae 100755
--- a/tests/test_cmds.sh
+++ b/tests/test_cmds.sh
@@ -162,6 +162,15 @@ do_tests()
orig)
cp $progpath $protected
;;
+ expect_fail)
+ set -x
+ base_prog=$(basename $progpath)
+ if [ $base_prog = "ls" ]; then
+ cp $(which diff) $protected
+ else
+ cp $(which ls) $protected
+ fi
+ ;;
*)
echo "Unknown configuration requested"
continue
@@ -186,13 +195,24 @@ do_tests()
echo "TEST ($config) ${prog}: Running tests..."
TEST_VERBOSE=1 timeout 300 ../$prog/test_script.sh $progpath ./$protected > test_${prog}.log 2>&1
if [ $? -eq 0 ]; then
- echo "TEST ($config) ${prog}: PASS"
- progs_pass="$progs_pass $prog.$config"
+ if [ "$config" != "expect_fail" ]; then
+ echo "TEST ($config) ${prog}: PASS"
+ progs_pass="$progs_pass $prog.$config"
+ else
+ progs_fail="$progs_fail $prog.$config"
+ if [ $show_logs_on_failure -eq 1 ]; then
+ cat test_${prog}.log
+ fi
+ fi
else
- echo "TEST ($config) ${prog}: FAIL"
- progs_fail="$progs_fail $prog.$config"
- if [ $show_logs_on_failure -eq 1 ]; then
- cat test_${prog}.log
+ if [ "$config" != "expect_fail" ]; then
+ echo "TEST ($config) ${prog}: FAIL"
+ progs_fail="$progs_fail $prog.$config"
+ if [ $show_logs_on_failure -eq 1 ]; then
+ cat test_${prog}.log
+ fi
+ else
+ progs_pass="$progs_pass $prog.$config"
fi
fi
done
@@ -226,6 +246,8 @@ usage()
echo
echo "config:"
echo " rida (default configuration)"
+ echo " orig test against self"
+ echo " fail deliberately induce failure"
echo " p1 P1 transform"
echo " mgx move globals"
echo " mgx_p1 move globals, followed by P1"