Optionally keep track of job statistics

Copy correct libappfw.so for 32 or 64 bit (from appfw/lib/32 or appfw/lib/64)


Former-commit-id: e9b80511dc584e11691ec528f0f8f7c787ef1eef

.gitattributes

@@ -393,6 +393,11 @@ tools/bed_blackbox.sh -text
 tools/bed_manual.sh -text
 tools/cover.sh -text
 tools/db/drop_my_tables.sh -text
+tools/db/job.create.tbl -text
+tools/db/job.drop.tbl -text
+tools/db/job_spec_register.sh -text
+tools/db/job_spec_update.sh -text
+tools/db/job_status_report.sh -text
 tools/db/pdb.create.tbl -text
 tools/db/pdb.createprogram.tbl -text
 tools/db/pdb.drop.tbl -text

tools/db/job.create.tbl

+CREATE TABLE job_spec
+(
+	job_id		text PRIMARY KEY,
+	job_name	text,
+	variant_id	integer DEFAULT -1,
+	submitted_ts	timestamp,
+	start_ts	timestamp,
+	stop_ts		timestamp,
+	configuration	text,
+	status		text,
+	installer	text
+);
+
+CREATE TABLE job_status
+(
+	job_id		text,
+	step		text,
+	step_num	integer DEFAULT -1,
+	log		text,
+	start_ts	timestamp,
+	stop_ts		timestamp,
+	status		text
+);

tools/db/job.drop.tbl

+DROP TABLE job_spec;
+DROP TABLE job_status;

tools/db/job_spec_register.sh

+#!/bin/sh -x
+
+JOB_ID=$1
+NAME=$2
+VARIANT_ID=$3
+STATUS=$4
+SUBMITTED_TS=$5
+
+psql -q -t -c "INSERT INTO job_spec (job_id, job_name, variant_id, status, submitted_ts) VALUES ('$JOB_ID', '$NAME', '$VARIANT_ID', '$STATUS', '$SUBMITTED_TS')"

tools/db/job_spec_update.sh

+#!/bin/sh -x
+
+JOB_ID=$1
+STATUS=$2
+TIMESTAMP=$3
+INSTALLER=$4
+
+if [ $STATUS = 'pending' ]; then
+	psql -q -t -c "UPDATE job_spec SET status='$STATUS', start_ts='$TIMESTAMP' WHERE job_id='$JOB_ID'"
+elif [ $STATUS = 'error' ]; then
+	psql -q -t -c "UPDATE job_spec SET status='$STATUS', stop_ts='$TIMESTAMP' WHERE job_id='$JOB_ID'"
+else
+	psql -q -t -c "UPDATE job_spec SET status='$STATUS', stop_ts='$TIMESTAMP', installer='$INSTALLER' WHERE job_id='$JOB_ID'"
+fi

tools/db/job_status_report.sh

+#!/bin/sh -x
+
+JOB_ID=$1
+STEP=$2
+STEP_NUM=$3
+STATE=$4
+TIMESTAMP=$5
+STATUS=$6
+LOGFILE=$7
+
+#####################################################
+
+usage()
+{
+  echo "report_job_status <job_id> <step_name> <step_num> [ started | completed ] <timestamp> <status> <logFile>"
+}
+
+log_error()
+{
+  echo "report_job_status: ERROR: $1"
+  exit -1
+}
+
+log_message()
+{
+  echo "report_job_status: MESSAGE: $1"
+}
+
+#####################################################
+
+if [ -z $JOB_ID ]; then
+  usage
+fi
+
+if [ -z $STEP ]; then
+  usage
+fi
+
+if [ -z $STEP_NUM ]; then
+  usage
+fi
+
+if [ -z $TIMESTAMP ]; then
+  usage
+fi
+
+if [ -z $STATUS ]; then
+  usage
+fi
+
+if [ $STATE = "started" ]; then
+	psql -q -t -c "INSERT INTO job_status (job_id, step, step_num, status, start_ts) VALUES ('$JOB_ID', '$STEP', '$STEP_NUM', '$STATUS', '$TIMESTAMP')"
+else
+	if [ -z $LOGFILE ]; then
+		psql -q -t -c "UPDATE job_status SET status='$STATUS', stop_ts='$TIMESTAMP' WHERE job_id = '$JOB_ID' AND step='$STEP'"
+	else
+		attributes=$(grep ATTRIBUTE $LOGFILE | cut -d' ' -f3-)
+		psql -q -t -c "UPDATE job_status SET status='$STATUS', stop_ts='$TIMESTAMP', log='$attributes' WHERE job_id = '$JOB_ID' AND step='$STEP'"
+	fi
+fi
+
+if [ ! $? -eq 0 ]; then
+  log_error "Failed to register job status"
+fi
+
+exit 0

tools/db/pdb.drop.tbl

-DROP TABLE program_dependency;
-DROP TABLE program_info;
+DROP TABLE variant_dependency;
+DROP TABLE variant_info;
 DROP TABLE file_info;

tools/db/pdb_setup.sh

 #!/bin/sh
 
 psql -f $PEASOUP_HOME/tools/db/pdb.create.tbl
+psql -f $PEASOUP_HOME/tools/db/job.create.tbl

tools/db/pdb_teardown.sh

 #!/bin/sh 
 
 psql -f $PEASOUP_HOME/tools/db/pdb.drop.tbl
+psql -f $PEASOUP_HOME/tools/db/job.drop.tbl

tools/do_appfw.sh

@@ -11,7 +11,7 @@ $PEASOUP_HOME/tools/generate_string_signatures.sh "$program" "$program.sigs" $fi
 cp $program.sigs $program.sigs.orig
 
 # copy application firewall library 
-# for now, it's only SQL
-cp $SECURITY_TRANSFORMS_HOME/appfw/lib/libappfw.so${bits} libappfw.so
+cp $SECURITY_TRANSFORMS_HOME/appfw/lib/${bits}/libappfw.so libappfw.so
+#cp $SECURITY_TRANSFORMS_HOME/appfw/lib/${bits}/libappfw.so libappfw.so
 
 $PEASOUP_HOME/tools/update_env_var.sh DO_APPFW 1

tools/ps_analyze.sh

@@ -14,6 +14,9 @@ ulimit -s unlimited
 watchdog_val=30
 errors=0
 
+# record statistics in database?
+record_stats=0
+
 # DEFAULT TIMEOUT VALUE
 INTEGER_TRANSFORM_TIMEOUT_VALUE=1800
 TWITCHER_TRANSFORM_TIMEOUT_VALUE=1800
@@ -23,7 +26,7 @@ PN_TIMEOUT_VALUE=21600
 # 
 # set default values for 
 #
-initial_off_phases="isr ret_shadow_stack determine_program"
+initial_off_phases="isr ret_shadow_stack determine_program stats"
 
 #non-zero to use canaries in PN/P1, 0 to turn off canaries
 #DO_CANARIES=1
@@ -36,6 +39,9 @@ intxform_detect_fp=1      # default: detect benign false positives is on
                           #   but if determine_program is off, it's a no-op
 intxform_instrument_idioms=0  # default: do not instrument instructions marked as IDIOM by STARS
 
+# JOBID
+
+JOBID="$(basename $1)-$$"
 
 # 
 # By default, big data approach is off
@@ -158,7 +164,7 @@ check_options()
 	# Note that we use `"$@"' to let each command-line parameter expand to a 
 	# separate word. The quotes around `$@' are essential!
 	# We need TEMP as the `eval set --' would nuke the return value of getopt.
-	TEMP=`getopt -o s:t:w: --long step-option: --long integer_warnings_only --long integer_instrument_idioms --long integer_detect_fp --long no_integer_detect_fp --long step: --long timeout: --long manual_test_script: --long manual_test_coverage_file: --long watchdog: -n 'ps_analyze.sh' -- "$@"`
+	TEMP=`getopt -o s:t:w: --long step-option: --long integer_warnings_only --long integer_instrument_idioms --long integer_detect_fp --long no_integer_detect_fp --long step: --long timeout: --long id: --long manual_test_script: --long manual_test_coverage_file: --long watchdog: -n 'ps_analyze.sh' -- "$@"`
 
 	# error check #
 	if [ $? != 0 ] ; then echo "Terminating..." >&2 ; exit -1 ; fi
@@ -214,6 +220,10 @@ check_options()
 			set_timer $2 & TIMER_PID=$!
 			shift 2 
 			;;
+		--id) 
+			JOBID=$2
+			shift 2 
+			;;
 		--) 	shift 
 			break 
 			;;
@@ -257,6 +267,14 @@ check_options()
 	if [[ $? = 1 && "$TWITCHER_HOME" != "" ]]; then
 		phases_off="$phases_off heaprand=off double_free=off"
 	fi
+
+	#
+	# turn on/off recording of statistics
+	#
+	is_step_on stats
+	if [[ $? = 1 ]]; then
+		record_stats=1
+	fi
 }
 
 
@@ -342,6 +360,10 @@ perform_step()
 	echo -n Performing step "$step" [dependencies=$mandatory] ...
 	starttime=`date --iso-8601=seconds`
 
+	if [ $record_stats -eq 1 ]; then
+		$PEASOUP_HOME/tools/db/job_status_report.sh "$JOBID" "$step" "$stepnum" started "$starttime" inprogress
+	fi
+
 	# If verbose is on, tee to a file 
 	if [ ! -z "$DEBUG_STEPS" ]; then
 		$command 
@@ -353,14 +375,27 @@ perform_step()
 		$command > $logfile 2>&1 
 		command_exit=$?
 	fi
+
+	endtime=`date --iso-8601=seconds`
 	
 	echo "# ATTRIBUTE start_time=$starttime" >> $logfile
-	echo "# ATTRIBUTE end_time=`date --iso-8601=seconds`" >> $logfile
+	echo "# ATTRIBUTE end_time=$endtime" >> $logfile
 	echo "# ATTRIBUTE peasoup_step_name=$step" >> $logfile
 	echo "# ATTRIBUTE peasoup_step_number=$stepnum" >> $logfile
 	echo "# ATTRIBUTE peasoup_step_command=$command " >> $logfile
 	echo "# ATTRIBUTE peasoup_step_exitcode=$command_exit" >> $logfile
 
+	# report job status
+	if [ $command_exit -eq 0 ]; then
+		if [ $record_stats -eq 1 ]; then
+			$PEASOUP_HOME/tools/db/job_status_report.sh "$JOBID" "$step" "$stepnum" completed "$endtime" success $logfile
+		fi
+	else
+		if [ $record_stats -eq 1 ]; then
+			$PEASOUP_HOME/tools/db/job_status_report.sh "$JOBID" "$step" "$stepnum" completed "$endtime" error $logfile
+		fi
+	fi
+
 	is_step_error $step $command_exit
 	if [ $? -ne 0 ]; then
 		echo "Done.  Command failed! ***************************************"
@@ -393,7 +428,7 @@ report_logs()
 	logfile=logs/ps_analyze.log
 
 	echo "# ATTRIBUTE start_time=$ps_starttime" >> $logfile
-	echo "# ATTRIBUTE end_time=`date --iso-8601=seconds`" >> $logfile
+	echo "# ATTRIBUTE end_time=$ps_endtime" >> $logfile
 	echo "# ATTRIBUTE peasoup_step_name=all_peasoup" >> $logfile
 
 	for i in $all_logs
@@ -666,6 +701,8 @@ DB_PROGRAM_NAME=`basename $orig_exe.$$ | sed "s/[^a-zA-Z0-9]/_/g"`
 DB_PROGRAM_NAME="psprog_$DB_PROGRAM_NAME"
 MD5HASH=`md5sum $newname.ncexe | cut -f1 -d' '`
 
+INSTALLER=`pwd`
+
 #
 # register the program
 #
@@ -675,6 +712,16 @@ if [ ! $varid -gt 0 ]; then
 	fail_gracefully "Failed to write Variant into database. Exiting early.  Is postgres running?  Can $PGUSER access the db?"
 fi
 
+if [ $record_stats -eq 1 ]; then
+	$PEASOUP_HOME/tools/db/job_spec_register.sh "$JOBID" "$DB_PROGRAM_NAME" "$varid" 'submitted' "$ps_starttime"
+fi
+
+sleep 30
+
+if [ $record_stats -eq 1 ]; then
+	$PEASOUP_HOME/tools/db/job_spec_update.sh "$JOBID" 'pending' "$ps_starttime"
+fi
+
 # build basic IR
 perform_step fill_in_cfg mandatory $SECURITY_TRANSFORMS_HOME/libIRDB/test/fill_in_cfg.exe $varid	
 perform_step fill_in_indtargs mandatory $SECURITY_TRANSFORMS_HOME/libIRDB/test/fill_in_indtargs.exe $varid 
@@ -813,8 +860,10 @@ fi
 #
 # create a report for all of ps_analyze.
 #
+ps_endtime=`date --iso-8601=seconds`
 report_logs
 
+
 # go back to original directory
 cd - > /dev/null 2>&1
 
@@ -833,8 +882,19 @@ if [ -f $stratafied_exe ]; then
 		echo "*****************************"
 		echo "*Warning: Some steps failed!*"
 		echo "*****************************"
+		if [ $record_stats -eq 1 ]; then
+			$PEASOUP_HOME/tools/db/job_spec_update.sh "$JOBID" 'partial' "$ps_endtime" "$INSTALLER"
+		fi
+	else
+		if [ $record_stats -eq 1 ]; then
+			$PEASOUP_HOME/tools/db/job_spec_update.sh "$JOBID" 'success' "$ps_endtime" "$INSTALLER"
+		fi
 	fi
+
 	exit 0;
 else
+	if [ $record_stats -eq 1 ]; then
+		$PEASOUP_HOME/tools/db/job_spec_update.sh "$JOBID" 'error' "$ps_endtime"
+	fi
 	exit 255;
 fi