first working prototype

Former-commit-id: a79914f566a599e00313e82a73e52ec8388a066b

.gitattributes

@@ -2923,7 +2923,9 @@ tools/getsyms.sh -text
 tools/grace_utils.sh -text
 tools/integer_replay.sh -text
 tools/intxform_detect_benign_fp.sh -text
+tools/intxform_make_detector_binary.sh -text
 tools/intxform_replay.sh -text
+tools/intxform_run.sh -text
 tools/is_so.sh -text
 tools/libc_functions.txt -text
 tools/make_prog_signature.sh -text

tools/intxform_detect_benign_fp.sh

@@ -18,26 +18,28 @@ IDENTIFIED_PROG=$2
 INTEGER_WARNINGS_FILE=$3
 
 # configuration variables
-LIBC_FILTER=$PEASOUP_HOME/tools/libc_functions.txt   # libc and other system library functions
-
 TOP_DIR=`pwd`
+LIBC_FILTER=$PEASOUP_HOME/tools/libc_functions.txt   # libc and other system library functions
+ORIG_BINARY=a.ncexe
 INTEGER_ASPRI=a.irdb.integer.aspri
 INTEGER_BSPRI=a.irdb.integer.bspri
+REGRESSION_TESTS=$PEASOUP_HOME/tests/$IDENTIFIED_PROG/test_script.sh
 
 touch $INTEGER_WARNINGS_FILE
 
 echo "intxform(detect-benign-fp): transforming binary: cloneid=$CLONE_ID identifiedProg=$IDENTIFIED_PROG"
 
-if [ "$BENIGN_FP_DETECT" != "1" ]; then
-	echo "INTXFORM: Detection of benign false positives turned on for recognized program: $IDENTIFIED_PROG"
+if [ -f $REGRESSION_TESTS ]; then
+	echo "intxform(detect-benign-fp): manual regression tests detected for $IDENTIFIED_PROG"
+else
+	echo "intxform(detect-benign-fp): no manual regression tests detected for $IDENTIFIED_PROG"
+	exit 1
 fi
 
 echo "intxform(detect-benign-fp): Clone program"
 $SECURITY_TRANSFORMS_HOME/libIRDB/test/clone.exe $CLONE_ID clone.id
 tempcloneid=`cat clone.id`
 
-#    - Transform program and run against all Grace-generated inputs using a policy of continued execution when an integer detector triggers (we want to catch all detection messages)
-#    - Keep track of all inputs that trigger a C1 diagnostic and put in a list
 echo "intxform(detect-benign-fp): Integer transform on cloned copy"
 $SECURITY_TRANSFORMS_HOME/tools/transforms/integertransformdriver.exe $tempcloneid $LIBC_FILTER $INTEGER_WARNINGS_FILE --warning
 
@@ -46,19 +48,14 @@ echo "intxform(detect-benign-fp): Generate temporary aspri --> bspri for integer
 $SECURITY_TRANSFORMS_HOME/libIRDB/test/generate_spri.exe $($PEASOUP_HOME/tools/is_so.sh a.ncexe) $tempcloneid $INTEGER_ASPRI
 $SECURITY_TRANSFORMS_HOME/tools/spasm/spasm $INTEGER_ASPRI $INTEGER_BSPRI stratafier.o.exe libstrata.so.symbols
 
-#if [ $? -eq 0 ]; then
-# produce list of instruction addresses that trigger an integer detector
-#	echo "intxform(detect-benign-fp): false positives detection activated"
-#		timeout $TIMEOUT $PEASOUP_HOME/tools/integer_replay.sh $TOP_DIR/a.stratafied $CONCOLIC_DIR $TOP_DIR/$INTEGER_BSPRI $INTEGER_WARNINGS_FILE
-#		sort $INTEGER_WARNINGS_FILE | uniq > $INTEGER_WARNINGS_FILE.$$
-#		mv $INTEGER_WARNINGS_FILE.$$ $INTEGER_WARNINGS_FILE
-#
-#		cd $TOP_DIR   # restore working dir (just in case)
-#	else
-#		echo "Error generating integer transforms -- skip replay step to detect benign false positives"
-#	fi
-#fi
+# generate script to run instrumented binary
+DETECTOR_BINARY=benignfp.detector
+$PEASOUP_HOME/tools/intxform_make_detector_binary.sh $DETECTOR_BINARY
 
-$NUM_FP_DETECTED=`wc -l $INTEGER_WARNINGS_FILE`
-echo "------------ intxform: end detection of benign false positives: $NUM_FP_DETECTED benign false positives detected -----------------"
+# run regression tests
+rm -f $TOP_DIR/diagnostics.cumul.out
+touch $TOP_DIR/diagnostics.cumul.out
+$PEASOUP_HOME/tools/intxform_replay.sh $REGRESSION_TESTS $TOP_DIR/$DETECTOR_BINARY $TOP_DIR/$ORIG_BINARY $TOP_DIR/$INTEGER_BSPRI $TOP_DIR/diagnostics.cumul.out $INTEGER_WARNINGS_FILE
 
+NUM_FP_DETECTED=`wc -l $INTEGER_WARNINGS_FILE`
+echo "------------ intxform: end detection of benign false positives: $NUM_FP_DETECTED benign false positives detected -----------------"

tools/intxform_make_detector_binary.sh

+#!/bin/sh 
+
+name=$1
+
+current_dir=`pwd`
+#intxform_fp_detect_binary=$name.sh
+intxform_fp_detect_binary=$name
+
+echo "#!/bin/sh" >> $intxform_fp_detect_binary
+echo "" >> $intxform_fp_detect_binary
+echo "setsid $current_dir/intxform_run.sh $current_dir \"\$0\" \"\$@\"" >> $intxform_fp_detect_binary
+echo "SAVE_EXIT_CODE=\$?" >> $intxform_fp_detect_binary
+echo "datapath=$current_dir" >> $intxform_fp_detect_binary
+
+cat >> $intxform_fp_detect_binary <<"EOF"
+
+if [ -f $datapath/diagnostics.out ]; then
+	len=`cat $datapath/diagnostics.out | wc -l` 
+	if [ $len -gt 0 ]; then 
+
+        # make output more concise
+	    sort $datapath/diagnostics.out | uniq > tmp.$$
+		cat tmp.$$ >> $datapath/diagnostics.cumul.out
+	fi
+fi
+
+# final check, in case we couldn't catch the signal
+if [ $SAVE_EXIT_CODE = 139 ]; then
+	exit 200
+fi
+exit $SAVE_EXIT_CODE
+EOF
+
+chmod +x $intxform_fp_detect_binary
+
+cp $PEASOUP_HOME/tools/intxform_run.sh $current_dir
+
+

tools/intxform_replay.sh

@@ -11,11 +11,12 @@
 # Inputs
 REGRESSION_TEST_SCRIPT=$1            # path of regression test script
 STRATAFIED_BINARY=$2                 # stratafied subject program (a.stratafied)
-BSPRI=$3                             # bspri file with integer instrumention (warnings)
-CUMUL_DIAGNOSTICS=$4                 # path of file containing cumulated diagnostics
+ORIG_BINARY=$3                       # original binary (a.ncexe)
+BSPRI=$4                             # bspri file with integer instrumention (warnings)
+CUMUL_DIAGNOSTICS=$5                 # path of file containing cumulated diagnostics
 
 # Output
-INTEGER_WARN_INSTRUCTIONS=$5         # output file with addresses of benign errors
+INTEGER_WARN_INSTRUCTIONS=$6         # output file with addresses of benign errors
 
 TOP_LEVEL=`pwd`
 REGRESSION_TEST_SCRIPT_TIMEOUT=600   # timeout value for regression tests (seconds)
@@ -30,6 +31,9 @@ echo "   INTEGER_WARN_INSTRUCTIONS: $INTEGER_WARN_INSTRUCTIONS (output file)"
 echo "                         DIR: $TOP_LEVEL"
 echo "=========================================="
 
+touch $CUMUL_DIAGNOSTICS
+touch $INTEGER_WARN_INSTRUCTIONS
+
 #
 # Algorithm:
 # (1) run regression tests against integer transformed binary in diagnostics mode
@@ -41,10 +45,9 @@ echo "=========================================="
 #
 
 # (1) run regression tests against integer transformed binary in diagnostics mode
-LD_LIBRARY_PATH="$LD_LIBRARY_PATH:$TOP_LEVEL" STRATA_LOG=detectors STRATA_OUTPUT_FILE="$TOP_LEVEL/diagnostics.out" STRATA_IS_SO=0 STRATA_ANNOT_FILE="$TOP_LEVEL/a.ncexe.annot" STRATA_PC_CONFINE=1 STRATA_DETECTOR_POLICY="continue" STRATA_SPRI_FILE="$BSPRI" STRATA_NUM_HANDLE=1 STRATA_SIEVE=1 STRATA_RC=1 STRATA_PARTIAL_INLINING=0 STRATA_EXE_FILE="$TOP_LEVEL/a.stratafied" STRATA_DOUBLE_FREE=1 STRATA_MAX_WARNINGS=50000 timeout $REGRESSION_TEST_SCRIPT_TIMEOUT $REGRESSION_TEST_SCRIPT -i $STRATAFIED_BINARY $STRATAFIED_BINARY  
+timeout $REGRESSION_TEST_SCRIPT_TIMEOUT $REGRESSION_TEST_SCRIPT -i $STRATAFIED_BINARY $ORIG_BINARY  
 
 # Produce final output file containing addresses of detected benign false positive
 # (2) extract address from diagnostics  
-# (3) produce list of address where the instruction results in a benign false positive
-touch $INTEGER_WARN_INSTRUCTIONS
-cat $CUMUL_DIAGNOSTICS | grep -i diagnos | grep class | grep C1 | sed 's/.*diagnosis.*PC:\(.*\)/\1/' | sort | uniq | cut -d' ' -f1 >> $INTEGER_WARN_INSTRUCTIONS
+# (3) produce list of unique addresses where the instructions result in a benign false positive
+cat $CUMUL_DIAGNOSTICS | grep -i diagnos | grep class | grep C1 | sed 's/.*diagnosis.*PC:\(.*\)/\1/' | cut -d' ' -f1 | sort | uniq >> $INTEGER_WARN_INSTRUCTIONS

tools/intxform_run.sh

+#!/bin/bash
+######################################################################
+######################################################################
+# This file is used as a template, not actually for running the code #
+######################################################################
+######################################################################
+
+#
+# determine the directory that contains the files for peasoup
+#
+datapath=$1
+
+#
+# save original $0
+#
+
+origbinpath=$2
+
+#
+# grab the rest of the args in $*
+#
+shift 2;
+
+#
+# Run the program with the proper env. vars set., and the arguments to the program specified
+#
+
+command="
+LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$datapath
+STRATA_WATCHDOG=0
+STRATA_NUM_HANDLE=0
+STRATA_DOUBLE_FREE=0
+STRATA_HEAPRAND=0
+STRATA_CONTROLLED_EXIT=0
+STRATA_PC_CONFINE=0
+STRATA_PC_CONFINE_XOR=0				
+STRATA_REKEY_AFTER=5000
+STRATA_PC_CONFINE_XOR_KEY_LENGTH=1024		
+STRATA_ANNOT_FILE=$datapath/a.ncexe.annot 
+STRATA_IS_SO=0
+STRATA_SIEVE=1					
+STRATA_RC=1					
+STRATA_PARTIAL_INLINING=0			
+STRATA_EXE_FILE=$datapath/a.stratafied
+STRATA_MAX_WARNINGS=50000
+	exec -a $origbinpath $datapath/a.stratafied \"\$@\""
+
+command="STRATA_LOG=detectors STRATA_OUTPUT_FILE=$datapath/diagnostics.out $command"
+
+# make sure we pick up the BSPRI file genreated by intxform when it's trying to detect
+# benign false positives
+command="STRATA_SPRI_FILE=$datapath/a.irdb.integer.bspri $command"
+
+eval $command