From 99a2953e84d10306ff5c0b61a97ba87b7008c2e5 Mon Sep 17 00:00:00 2001
From: an7s <an7s@git.zephyr-software.com>
Date: Fri, 22 Aug 2014 18:11:47 +0000
Subject: [PATCH] Optionally keep track of job statistics Copy correct
libappfw.so for 32 or 64 bit (from appfw/lib/32 or appfw/lib/64)
Former-commit-id: e9b80511dc584e11691ec528f0f8f7c787ef1eef
---
.gitattributes | 5 +++
tools/db/job.create.tbl | 23 ++++++++++++
tools/db/job.drop.tbl | 2 ++
tools/db/job_spec_register.sh | 9 +++++
tools/db/job_spec_update.sh | 14 ++++++++
tools/db/job_status_report.sh | 66 ++++++++++++++++++++++++++++++++++
tools/db/pdb.drop.tbl | 4 +--
tools/db/pdb_setup.sh | 1 +
tools/db/pdb_teardown.sh | 1 +
tools/do_appfw.sh | 4 +--
tools/ps_analyze.sh | 68 ++++++++++++++++++++++++++++++++---
11 files changed, 189 insertions(+), 8 deletions(-)
create mode 100644 tools/db/job.create.tbl
create mode 100644 tools/db/job.drop.tbl
create mode 100755 tools/db/job_spec_register.sh
create mode 100755 tools/db/job_spec_update.sh
create mode 100755 tools/db/job_status_report.sh
diff --git a/.gitattributes b/.gitattributes
index 0405eb530..b434e937d 100644
--- a/.gitattributes
+++ b/.gitattributes
@@ -393,6 +393,11 @@ tools/bed_blackbox.sh -text
tools/bed_manual.sh -text
tools/cover.sh -text
tools/db/drop_my_tables.sh -text
+tools/db/job.create.tbl -text
+tools/db/job.drop.tbl -text
+tools/db/job_spec_register.sh -text
+tools/db/job_spec_update.sh -text
+tools/db/job_status_report.sh -text
tools/db/pdb.create.tbl -text
tools/db/pdb.createprogram.tbl -text
tools/db/pdb.drop.tbl -text
diff --git a/tools/db/job.create.tbl b/tools/db/job.create.tbl
new file mode 100644
index 000000000..97bca29e6
--- /dev/null
+++ b/tools/db/job.create.tbl
@@ -0,0 +1,23 @@
+CREATE TABLE job_spec
+(
+ job_id text PRIMARY KEY,
+ job_name text,
+ variant_id integer DEFAULT -1,
+ submitted_ts timestamp,
+ start_ts timestamp,
+ stop_ts timestamp,
+ configuration text,
+ status text,
+ installer text
+);
+
+CREATE TABLE job_status
+(
+ job_id text,
+ step text,
+ step_num integer DEFAULT -1,
+ log text,
+ start_ts timestamp,
+ stop_ts timestamp,
+ status text
+);
diff --git a/tools/db/job.drop.tbl b/tools/db/job.drop.tbl
new file mode 100644
index 000000000..e2c7e0320
--- /dev/null
+++ b/tools/db/job.drop.tbl
@@ -0,0 +1,2 @@
+DROP TABLE job_spec;
+DROP TABLE job_status;
diff --git a/tools/db/job_spec_register.sh b/tools/db/job_spec_register.sh
new file mode 100755
index 000000000..b9466d96b
--- /dev/null
+++ b/tools/db/job_spec_register.sh
@@ -0,0 +1,9 @@
+#!/bin/sh -x
+
+JOB_ID=$1
+NAME=$2
+VARIANT_ID=$3
+STATUS=$4
+SUBMITTED_TS=$5
+
+psql -q -t -c "INSERT INTO job_spec (job_id, job_name, variant_id, status, submitted_ts) VALUES ('$JOB_ID', '$NAME', '$VARIANT_ID', '$STATUS', '$SUBMITTED_TS')"
diff --git a/tools/db/job_spec_update.sh b/tools/db/job_spec_update.sh
new file mode 100755
index 000000000..6c6b4b3ad
--- /dev/null
+++ b/tools/db/job_spec_update.sh
@@ -0,0 +1,14 @@
+#!/bin/sh -x
+
+JOB_ID=$1
+STATUS=$2
+TIMESTAMP=$3
+INSTALLER=$4
+
+if [ $STATUS = 'pending' ]; then
+ psql -q -t -c "UPDATE job_spec SET status='$STATUS', start_ts='$TIMESTAMP' WHERE job_id='$JOB_ID'"
+elif [ $STATUS = 'error' ]; then
+ psql -q -t -c "UPDATE job_spec SET status='$STATUS', stop_ts='$TIMESTAMP' WHERE job_id='$JOB_ID'"
+else
+ psql -q -t -c "UPDATE job_spec SET status='$STATUS', stop_ts='$TIMESTAMP', installer='$INSTALLER' WHERE job_id='$JOB_ID'"
+fi
diff --git a/tools/db/job_status_report.sh b/tools/db/job_status_report.sh
new file mode 100755
index 000000000..a77f6cc6a
--- /dev/null
+++ b/tools/db/job_status_report.sh
@@ -0,0 +1,66 @@
+#!/bin/sh -x
+
+JOB_ID=$1
+STEP=$2
+STEP_NUM=$3
+STATE=$4
+TIMESTAMP=$5
+STATUS=$6
+LOGFILE=$7
+
+#####################################################
+
+usage()
+{
+ echo "report_job_status <job_id> <step_name> <step_num> [ started | completed ] <timestamp> <status> <logFile>"
+}
+
+log_error()
+{
+ echo "report_job_status: ERROR: $1"
+ exit -1
+}
+
+log_message()
+{
+ echo "report_job_status: MESSAGE: $1"
+}
+
+#####################################################
+
+if [ -z $JOB_ID ]; then
+ usage
+fi
+
+if [ -z $STEP ]; then
+ usage
+fi
+
+if [ -z $STEP_NUM ]; then
+ usage
+fi
+
+if [ -z $TIMESTAMP ]; then
+ usage
+fi
+
+if [ -z $STATUS ]; then
+ usage
+fi
+
+if [ $STATE = "started" ]; then
+ psql -q -t -c "INSERT INTO job_status (job_id, step, step_num, status, start_ts) VALUES ('$JOB_ID', '$STEP', '$STEP_NUM', '$STATUS', '$TIMESTAMP')"
+else
+ if [ -z $LOGFILE ]; then
+ psql -q -t -c "UPDATE job_status SET status='$STATUS', stop_ts='$TIMESTAMP' WHERE job_id = '$JOB_ID' AND step='$STEP'"
+ else
+ attributes=$(grep ATTRIBUTE $LOGFILE | cut -d' ' -f3-)
+ psql -q -t -c "UPDATE job_status SET status='$STATUS', stop_ts='$TIMESTAMP', log='$attributes' WHERE job_id = '$JOB_ID' AND step='$STEP'"
+ fi
+fi
+
+if [ ! $? -eq 0 ]; then
+ log_error "Failed to register job status"
+fi
+
+exit 0
diff --git a/tools/db/pdb.drop.tbl b/tools/db/pdb.drop.tbl
index e75fc9e4c..b42f9199e 100644
--- a/tools/db/pdb.drop.tbl
+++ b/tools/db/pdb.drop.tbl
@@ -1,3 +1,3 @@
-DROP TABLE program_dependency;
-DROP TABLE program_info;
+DROP TABLE variant_dependency;
+DROP TABLE variant_info;
DROP TABLE file_info;
diff --git a/tools/db/pdb_setup.sh b/tools/db/pdb_setup.sh
index 7d489a046..5aadc308a 100755
--- a/tools/db/pdb_setup.sh
+++ b/tools/db/pdb_setup.sh
@@ -1,3 +1,4 @@
#!/bin/sh
psql -f $PEASOUP_HOME/tools/db/pdb.create.tbl
+psql -f $PEASOUP_HOME/tools/db/job.create.tbl
diff --git a/tools/db/pdb_teardown.sh b/tools/db/pdb_teardown.sh
index 4c48331df..5ebe4aa2b 100755
--- a/tools/db/pdb_teardown.sh
+++ b/tools/db/pdb_teardown.sh
@@ -1,3 +1,4 @@
#!/bin/sh
psql -f $PEASOUP_HOME/tools/db/pdb.drop.tbl
+psql -f $PEASOUP_HOME/tools/db/job.drop.tbl
diff --git a/tools/do_appfw.sh b/tools/do_appfw.sh
index 889aa6cca..ab0e196ec 100755
--- a/tools/do_appfw.sh
+++ b/tools/do_appfw.sh
@@ -11,7 +11,7 @@ $PEASOUP_HOME/tools/generate_string_signatures.sh "$program" "$program.sigs" $fi
cp $program.sigs $program.sigs.orig
# copy application firewall library
-# for now, it's only SQL
-cp $SECURITY_TRANSFORMS_HOME/appfw/lib/libappfw.so${bits} libappfw.so
+cp $SECURITY_TRANSFORMS_HOME/appfw/lib/${bits}/libappfw.so libappfw.so
+#cp $SECURITY_TRANSFORMS_HOME/appfw/lib/${bits}/libappfw.so libappfw.so
$PEASOUP_HOME/tools/update_env_var.sh DO_APPFW 1
diff --git a/tools/ps_analyze.sh b/tools/ps_analyze.sh
index 0e4e49326..92ce4c36b 100755
--- a/tools/ps_analyze.sh
+++ b/tools/ps_analyze.sh
@@ -14,6 +14,9 @@ ulimit -s unlimited
watchdog_val=30
errors=0
+# record statistics in database?
+record_stats=0
+
# DEFAULT TIMEOUT VALUE
INTEGER_TRANSFORM_TIMEOUT_VALUE=1800
TWITCHER_TRANSFORM_TIMEOUT_VALUE=1800
@@ -23,7 +26,7 @@ PN_TIMEOUT_VALUE=21600
#
# set default values for
#
-initial_off_phases="isr ret_shadow_stack determine_program"
+initial_off_phases="isr ret_shadow_stack determine_program stats"
#non-zero to use canaries in PN/P1, 0 to turn off canaries
#DO_CANARIES=1
@@ -36,6 +39,9 @@ intxform_detect_fp=1 # default: detect benign false positives is on
# but if determine_program is off, it's a no-op
intxform_instrument_idioms=0 # default: do not instrument instructions marked as IDIOM by STARS
+# JOBID
+
+JOBID="$(basename $1)-$$"
#
# By default, big data approach is off
@@ -158,7 +164,7 @@ check_options()
# Note that we use `"$@"' to let each command-line parameter expand to a
# separate word. The quotes around `$@' are essential!
# We need TEMP as the `eval set --' would nuke the return value of getopt.
- TEMP=`getopt -o s:t:w: --long step-option: --long integer_warnings_only --long integer_instrument_idioms --long integer_detect_fp --long no_integer_detect_fp --long step: --long timeout: --long manual_test_script: --long manual_test_coverage_file: --long watchdog: -n 'ps_analyze.sh' -- "$@"`
+ TEMP=`getopt -o s:t:w: --long step-option: --long integer_warnings_only --long integer_instrument_idioms --long integer_detect_fp --long no_integer_detect_fp --long step: --long timeout: --long id: --long manual_test_script: --long manual_test_coverage_file: --long watchdog: -n 'ps_analyze.sh' -- "$@"`
# error check #
if [ $? != 0 ] ; then echo "Terminating..." >&2 ; exit -1 ; fi
@@ -214,6 +220,10 @@ check_options()
set_timer $2 & TIMER_PID=$!
shift 2
;;
+ --id)
+ JOBID=$2
+ shift 2
+ ;;
--) shift
break
;;
@@ -257,6 +267,14 @@ check_options()
if [[ $? = 1 && "$TWITCHER_HOME" != "" ]]; then
phases_off="$phases_off heaprand=off double_free=off"
fi
+
+ #
+ # turn on/off recording of statistics
+ #
+ is_step_on stats
+ if [[ $? = 1 ]]; then
+ record_stats=1
+ fi
}
@@ -342,6 +360,10 @@ perform_step()
echo -n Performing step "$step" [dependencies=$mandatory] ...
starttime=`date --iso-8601=seconds`
+ if [ $record_stats -eq 1 ]; then
+ $PEASOUP_HOME/tools/db/job_status_report.sh "$JOBID" "$step" "$stepnum" started "$starttime" inprogress
+ fi
+
# If verbose is on, tee to a file
if [ ! -z "$DEBUG_STEPS" ]; then
$command
@@ -353,14 +375,27 @@ perform_step()
$command > $logfile 2>&1
command_exit=$?
fi
+
+ endtime=`date --iso-8601=seconds`
echo "# ATTRIBUTE start_time=$starttime" >> $logfile
- echo "# ATTRIBUTE end_time=`date --iso-8601=seconds`" >> $logfile
+ echo "# ATTRIBUTE end_time=$endtime" >> $logfile
echo "# ATTRIBUTE peasoup_step_name=$step" >> $logfile
echo "# ATTRIBUTE peasoup_step_number=$stepnum" >> $logfile
echo "# ATTRIBUTE peasoup_step_command=$command " >> $logfile
echo "# ATTRIBUTE peasoup_step_exitcode=$command_exit" >> $logfile
+ # report job status
+ if [ $command_exit -eq 0 ]; then
+ if [ $record_stats -eq 1 ]; then
+ $PEASOUP_HOME/tools/db/job_status_report.sh "$JOBID" "$step" "$stepnum" completed "$endtime" success $logfile
+ fi
+ else
+ if [ $record_stats -eq 1 ]; then
+ $PEASOUP_HOME/tools/db/job_status_report.sh "$JOBID" "$step" "$stepnum" completed "$endtime" error $logfile
+ fi
+ fi
+
is_step_error $step $command_exit
if [ $? -ne 0 ]; then
echo "Done. Command failed! ***************************************"
@@ -393,7 +428,7 @@ report_logs()
logfile=logs/ps_analyze.log
echo "# ATTRIBUTE start_time=$ps_starttime" >> $logfile
- echo "# ATTRIBUTE end_time=`date --iso-8601=seconds`" >> $logfile
+ echo "# ATTRIBUTE end_time=$ps_endtime" >> $logfile
echo "# ATTRIBUTE peasoup_step_name=all_peasoup" >> $logfile
for i in $all_logs
@@ -666,6 +701,8 @@ DB_PROGRAM_NAME=`basename $orig_exe.$$ | sed "s/[^a-zA-Z0-9]/_/g"`
DB_PROGRAM_NAME="psprog_$DB_PROGRAM_NAME"
MD5HASH=`md5sum $newname.ncexe | cut -f1 -d' '`
+INSTALLER=`pwd`
+
#
# register the program
#
@@ -675,6 +712,16 @@ if [ ! $varid -gt 0 ]; then
fail_gracefully "Failed to write Variant into database. Exiting early. Is postgres running? Can $PGUSER access the db?"
fi
+if [ $record_stats -eq 1 ]; then
+ $PEASOUP_HOME/tools/db/job_spec_register.sh "$JOBID" "$DB_PROGRAM_NAME" "$varid" 'submitted' "$ps_starttime"
+fi
+
+sleep 30
+
+if [ $record_stats -eq 1 ]; then
+ $PEASOUP_HOME/tools/db/job_spec_update.sh "$JOBID" 'pending' "$ps_starttime"
+fi
+
# build basic IR
perform_step fill_in_cfg mandatory $SECURITY_TRANSFORMS_HOME/libIRDB/test/fill_in_cfg.exe $varid
perform_step fill_in_indtargs mandatory $SECURITY_TRANSFORMS_HOME/libIRDB/test/fill_in_indtargs.exe $varid
@@ -813,8 +860,10 @@ fi
#
# create a report for all of ps_analyze.
#
+ps_endtime=`date --iso-8601=seconds`
report_logs
+
# go back to original directory
cd - > /dev/null 2>&1
@@ -833,8 +882,19 @@ if [ -f $stratafied_exe ]; then
echo "*****************************"
echo "*Warning: Some steps failed!*"
echo "*****************************"
+ if [ $record_stats -eq 1 ]; then
+ $PEASOUP_HOME/tools/db/job_spec_update.sh "$JOBID" 'partial' "$ps_endtime" "$INSTALLER"
+ fi
+ else
+ if [ $record_stats -eq 1 ]; then
+ $PEASOUP_HOME/tools/db/job_spec_update.sh "$JOBID" 'success' "$ps_endtime" "$INSTALLER"
+ fi
fi
+
exit 0;
else
+ if [ $record_stats -eq 1 ]; then
+ $PEASOUP_HOME/tools/db/job_spec_update.sh "$JOBID" 'error' "$ps_endtime"
+ fi
exit 255;
fi
--
GitLab