diff --git a/.gitattributes b/.gitattributes index 0405eb530827f18cd9a52ec36c99c783baf6aa7f..b434e937d4858ac53d2a653ea18bb61368e53d4b 100644 --- a/.gitattributes +++ b/.gitattributes @@ -393,6 +393,11 @@ tools/bed_blackbox.sh -text tools/bed_manual.sh -text tools/cover.sh -text tools/db/drop_my_tables.sh -text +tools/db/job.create.tbl -text +tools/db/job.drop.tbl -text +tools/db/job_spec_register.sh -text +tools/db/job_spec_update.sh -text +tools/db/job_status_report.sh -text tools/db/pdb.create.tbl -text tools/db/pdb.createprogram.tbl -text tools/db/pdb.drop.tbl -text diff --git a/tools/db/job.create.tbl b/tools/db/job.create.tbl new file mode 100644 index 0000000000000000000000000000000000000000..97bca29e62d5ea64aaf28faefa4888242bf53279 --- /dev/null +++ b/tools/db/job.create.tbl @@ -0,0 +1,23 @@ +CREATE TABLE job_spec +( + job_id text PRIMARY KEY, + job_name text, + variant_id integer DEFAULT -1, + submitted_ts timestamp, + start_ts timestamp, + stop_ts timestamp, + configuration text, + status text, + installer text +); + +CREATE TABLE job_status +( + job_id text, + step text, + step_num integer DEFAULT -1, + log text, + start_ts timestamp, + stop_ts timestamp, + status text +); diff --git a/tools/db/job.drop.tbl b/tools/db/job.drop.tbl new file mode 100644 index 0000000000000000000000000000000000000000..e2c7e0320a90faa5dd0a445b1b5f888fbe6cdace --- /dev/null +++ b/tools/db/job.drop.tbl @@ -0,0 +1,2 @@ +DROP TABLE job_spec; +DROP TABLE job_status; diff --git a/tools/db/job_spec_register.sh b/tools/db/job_spec_register.sh new file mode 100755 index 0000000000000000000000000000000000000000..b9466d96b67ed0de769bddfe139c5484de4e0cb3 --- /dev/null +++ b/tools/db/job_spec_register.sh @@ -0,0 +1,9 @@ +#!/bin/sh -x + +JOB_ID=$1 +NAME=$2 +VARIANT_ID=$3 +STATUS=$4 +SUBMITTED_TS=$5 + +psql -q -t -c "INSERT INTO job_spec (job_id, job_name, variant_id, status, submitted_ts) VALUES ('$JOB_ID', '$NAME', '$VARIANT_ID', '$STATUS', '$SUBMITTED_TS')" diff --git a/tools/db/job_spec_update.sh b/tools/db/job_spec_update.sh new file mode 100755 index 0000000000000000000000000000000000000000..6c6b4b3ad65278fac52711f2036d1fdf65650c68 --- /dev/null +++ b/tools/db/job_spec_update.sh @@ -0,0 +1,14 @@ +#!/bin/sh -x + +JOB_ID=$1 +STATUS=$2 +TIMESTAMP=$3 +INSTALLER=$4 + +if [ $STATUS = 'pending' ]; then + psql -q -t -c "UPDATE job_spec SET status='$STATUS', start_ts='$TIMESTAMP' WHERE job_id='$JOB_ID'" +elif [ $STATUS = 'error' ]; then + psql -q -t -c "UPDATE job_spec SET status='$STATUS', stop_ts='$TIMESTAMP' WHERE job_id='$JOB_ID'" +else + psql -q -t -c "UPDATE job_spec SET status='$STATUS', stop_ts='$TIMESTAMP', installer='$INSTALLER' WHERE job_id='$JOB_ID'" +fi diff --git a/tools/db/job_status_report.sh b/tools/db/job_status_report.sh new file mode 100755 index 0000000000000000000000000000000000000000..a77f6cc6a5eeddd4d76a55393ef53650b734dcf4 --- /dev/null +++ b/tools/db/job_status_report.sh @@ -0,0 +1,66 @@ +#!/bin/sh -x + +JOB_ID=$1 +STEP=$2 +STEP_NUM=$3 +STATE=$4 +TIMESTAMP=$5 +STATUS=$6 +LOGFILE=$7 + +##################################################### + +usage() +{ + echo "report_job_status <job_id> <step_name> <step_num> [ started | completed ] <timestamp> <status> <logFile>" +} + +log_error() +{ + echo "report_job_status: ERROR: $1" + exit -1 +} + +log_message() +{ + echo "report_job_status: MESSAGE: $1" +} + +##################################################### + +if [ -z $JOB_ID ]; then + usage +fi + +if [ -z $STEP ]; then + usage +fi + +if [ -z $STEP_NUM ]; then + usage +fi + +if [ -z $TIMESTAMP ]; then + usage +fi + +if [ -z $STATUS ]; then + usage +fi + +if [ $STATE = "started" ]; then + psql -q -t -c "INSERT INTO job_status (job_id, step, step_num, status, start_ts) VALUES ('$JOB_ID', '$STEP', '$STEP_NUM', '$STATUS', '$TIMESTAMP')" +else + if [ -z $LOGFILE ]; then + psql -q -t -c "UPDATE job_status SET status='$STATUS', stop_ts='$TIMESTAMP' WHERE job_id = '$JOB_ID' AND step='$STEP'" + else + attributes=$(grep ATTRIBUTE $LOGFILE | cut -d' ' -f3-) + psql -q -t -c "UPDATE job_status SET status='$STATUS', stop_ts='$TIMESTAMP', log='$attributes' WHERE job_id = '$JOB_ID' AND step='$STEP'" + fi +fi + +if [ ! $? -eq 0 ]; then + log_error "Failed to register job status" +fi + +exit 0 diff --git a/tools/db/pdb.drop.tbl b/tools/db/pdb.drop.tbl index e75fc9e4cde38fb4ff07f9750cabd36c80942d6a..b42f9199e2a3b5196b99f1454958a4c795726cc9 100644 --- a/tools/db/pdb.drop.tbl +++ b/tools/db/pdb.drop.tbl @@ -1,3 +1,3 @@ -DROP TABLE program_dependency; -DROP TABLE program_info; +DROP TABLE variant_dependency; +DROP TABLE variant_info; DROP TABLE file_info; diff --git a/tools/db/pdb_setup.sh b/tools/db/pdb_setup.sh index 7d489a04658a83b8ee3762e469f9931309ec6563..5aadc308a60c0645107b153316920bb74086b64c 100755 --- a/tools/db/pdb_setup.sh +++ b/tools/db/pdb_setup.sh @@ -1,3 +1,4 @@ #!/bin/sh psql -f $PEASOUP_HOME/tools/db/pdb.create.tbl +psql -f $PEASOUP_HOME/tools/db/job.create.tbl diff --git a/tools/db/pdb_teardown.sh b/tools/db/pdb_teardown.sh index 4c48331df85cce345f5003370c809da6eaadf4d0..5ebe4aa2b1c69e53a32d9b410d2d5f5b6a61df30 100755 --- a/tools/db/pdb_teardown.sh +++ b/tools/db/pdb_teardown.sh @@ -1,3 +1,4 @@ #!/bin/sh psql -f $PEASOUP_HOME/tools/db/pdb.drop.tbl +psql -f $PEASOUP_HOME/tools/db/job.drop.tbl diff --git a/tools/do_appfw.sh b/tools/do_appfw.sh index 889aa6cca40427683e125a8042c02ecf0fc2a99e..ab0e196ecd1eefd0254542ec5b16f854e217da6f 100755 --- a/tools/do_appfw.sh +++ b/tools/do_appfw.sh @@ -11,7 +11,7 @@ $PEASOUP_HOME/tools/generate_string_signatures.sh "$program" "$program.sigs" $fi cp $program.sigs $program.sigs.orig # copy application firewall library -# for now, it's only SQL -cp $SECURITY_TRANSFORMS_HOME/appfw/lib/libappfw.so${bits} libappfw.so +cp $SECURITY_TRANSFORMS_HOME/appfw/lib/${bits}/libappfw.so libappfw.so +#cp $SECURITY_TRANSFORMS_HOME/appfw/lib/${bits}/libappfw.so libappfw.so $PEASOUP_HOME/tools/update_env_var.sh DO_APPFW 1 diff --git a/tools/ps_analyze.sh b/tools/ps_analyze.sh index 0e4e4932680858526842e623667fc949970742aa..92ce4c36be5dbcc5db120191e96f561c188d3941 100755 --- a/tools/ps_analyze.sh +++ b/tools/ps_analyze.sh @@ -14,6 +14,9 @@ ulimit -s unlimited watchdog_val=30 errors=0 +# record statistics in database? +record_stats=0 + # DEFAULT TIMEOUT VALUE INTEGER_TRANSFORM_TIMEOUT_VALUE=1800 TWITCHER_TRANSFORM_TIMEOUT_VALUE=1800 @@ -23,7 +26,7 @@ PN_TIMEOUT_VALUE=21600 # # set default values for # -initial_off_phases="isr ret_shadow_stack determine_program" +initial_off_phases="isr ret_shadow_stack determine_program stats" #non-zero to use canaries in PN/P1, 0 to turn off canaries #DO_CANARIES=1 @@ -36,6 +39,9 @@ intxform_detect_fp=1 # default: detect benign false positives is on # but if determine_program is off, it's a no-op intxform_instrument_idioms=0 # default: do not instrument instructions marked as IDIOM by STARS +# JOBID + +JOBID="$(basename $1)-$$" # # By default, big data approach is off @@ -158,7 +164,7 @@ check_options() # Note that we use `"$@"' to let each command-line parameter expand to a # separate word. The quotes around `$@' are essential! # We need TEMP as the `eval set --' would nuke the return value of getopt. - TEMP=`getopt -o s:t:w: --long step-option: --long integer_warnings_only --long integer_instrument_idioms --long integer_detect_fp --long no_integer_detect_fp --long step: --long timeout: --long manual_test_script: --long manual_test_coverage_file: --long watchdog: -n 'ps_analyze.sh' -- "$@"` + TEMP=`getopt -o s:t:w: --long step-option: --long integer_warnings_only --long integer_instrument_idioms --long integer_detect_fp --long no_integer_detect_fp --long step: --long timeout: --long id: --long manual_test_script: --long manual_test_coverage_file: --long watchdog: -n 'ps_analyze.sh' -- "$@"` # error check # if [ $? != 0 ] ; then echo "Terminating..." >&2 ; exit -1 ; fi @@ -214,6 +220,10 @@ check_options() set_timer $2 & TIMER_PID=$! shift 2 ;; + --id) + JOBID=$2 + shift 2 + ;; --) shift break ;; @@ -257,6 +267,14 @@ check_options() if [[ $? = 1 && "$TWITCHER_HOME" != "" ]]; then phases_off="$phases_off heaprand=off double_free=off" fi + + # + # turn on/off recording of statistics + # + is_step_on stats + if [[ $? = 1 ]]; then + record_stats=1 + fi } @@ -342,6 +360,10 @@ perform_step() echo -n Performing step "$step" [dependencies=$mandatory] ... starttime=`date --iso-8601=seconds` + if [ $record_stats -eq 1 ]; then + $PEASOUP_HOME/tools/db/job_status_report.sh "$JOBID" "$step" "$stepnum" started "$starttime" inprogress + fi + # If verbose is on, tee to a file if [ ! -z "$DEBUG_STEPS" ]; then $command @@ -353,14 +375,27 @@ perform_step() $command > $logfile 2>&1 command_exit=$? fi + + endtime=`date --iso-8601=seconds` echo "# ATTRIBUTE start_time=$starttime" >> $logfile - echo "# ATTRIBUTE end_time=`date --iso-8601=seconds`" >> $logfile + echo "# ATTRIBUTE end_time=$endtime" >> $logfile echo "# ATTRIBUTE peasoup_step_name=$step" >> $logfile echo "# ATTRIBUTE peasoup_step_number=$stepnum" >> $logfile echo "# ATTRIBUTE peasoup_step_command=$command " >> $logfile echo "# ATTRIBUTE peasoup_step_exitcode=$command_exit" >> $logfile + # report job status + if [ $command_exit -eq 0 ]; then + if [ $record_stats -eq 1 ]; then + $PEASOUP_HOME/tools/db/job_status_report.sh "$JOBID" "$step" "$stepnum" completed "$endtime" success $logfile + fi + else + if [ $record_stats -eq 1 ]; then + $PEASOUP_HOME/tools/db/job_status_report.sh "$JOBID" "$step" "$stepnum" completed "$endtime" error $logfile + fi + fi + is_step_error $step $command_exit if [ $? -ne 0 ]; then echo "Done. Command failed! ***************************************" @@ -393,7 +428,7 @@ report_logs() logfile=logs/ps_analyze.log echo "# ATTRIBUTE start_time=$ps_starttime" >> $logfile - echo "# ATTRIBUTE end_time=`date --iso-8601=seconds`" >> $logfile + echo "# ATTRIBUTE end_time=$ps_endtime" >> $logfile echo "# ATTRIBUTE peasoup_step_name=all_peasoup" >> $logfile for i in $all_logs @@ -666,6 +701,8 @@ DB_PROGRAM_NAME=`basename $orig_exe.$$ | sed "s/[^a-zA-Z0-9]/_/g"` DB_PROGRAM_NAME="psprog_$DB_PROGRAM_NAME" MD5HASH=`md5sum $newname.ncexe | cut -f1 -d' '` +INSTALLER=`pwd` + # # register the program # @@ -675,6 +712,16 @@ if [ ! $varid -gt 0 ]; then fail_gracefully "Failed to write Variant into database. Exiting early. Is postgres running? Can $PGUSER access the db?" fi +if [ $record_stats -eq 1 ]; then + $PEASOUP_HOME/tools/db/job_spec_register.sh "$JOBID" "$DB_PROGRAM_NAME" "$varid" 'submitted' "$ps_starttime" +fi + +sleep 30 + +if [ $record_stats -eq 1 ]; then + $PEASOUP_HOME/tools/db/job_spec_update.sh "$JOBID" 'pending' "$ps_starttime" +fi + # build basic IR perform_step fill_in_cfg mandatory $SECURITY_TRANSFORMS_HOME/libIRDB/test/fill_in_cfg.exe $varid perform_step fill_in_indtargs mandatory $SECURITY_TRANSFORMS_HOME/libIRDB/test/fill_in_indtargs.exe $varid @@ -813,8 +860,10 @@ fi # # create a report for all of ps_analyze. # +ps_endtime=`date --iso-8601=seconds` report_logs + # go back to original directory cd - > /dev/null 2>&1 @@ -833,8 +882,19 @@ if [ -f $stratafied_exe ]; then echo "*****************************" echo "*Warning: Some steps failed!*" echo "*****************************" + if [ $record_stats -eq 1 ]; then + $PEASOUP_HOME/tools/db/job_spec_update.sh "$JOBID" 'partial' "$ps_endtime" "$INSTALLER" + fi + else + if [ $record_stats -eq 1 ]; then + $PEASOUP_HOME/tools/db/job_spec_update.sh "$JOBID" 'success' "$ps_endtime" "$INSTALLER" + fi fi + exit 0; else + if [ $record_stats -eq 1 ]; then + $PEASOUP_HOME/tools/db/job_spec_update.sh "$JOBID" 'error' "$ps_endtime" + fi exit 255; fi