Make sure ssh and scp timeout

SMP-analyze.sh

@@ -8,75 +8,101 @@ if [ -z "$IDA_PRO_SERVER_PORT" ]; then
     IDA_PRO_SERVER_PORT=22
 fi
 
-md5name=$(md5sum $1 | awk '{print $1}')
+file=$1
+md5name=$(md5sum $file | awk '{print $1}')
 directory=/tmp/remote-analyze/${md5name}
 
 copy_STARS_info()
 {
+	remotehost=$1
 	# Copy the answer back
-	scp -P $IDA_PRO_SERVER_PORT $IDA_PRO_SERVER_USER@$IDA_PRO_SERVER_HOST:$directory/a.ncexe.* .
+	scp -o ConnectTimeout=75 -o BatchMode=yes -P $IDA_PRO_SERVER_PORT $IDA_PRO_SERVER_USER@$remotehost:${directory}/a.ncexe.* .
+	return $?
 }
 
 server_has_cached_info()
 {
-	ssh -p $IDA_PRO_SERVER_PORT $IDA_PRO_SERVER_USER@$IDA_PRO_SERVER_HOST ls -l "$directory/a.ncexe.infoannot" | grep a.ncexe.infoannot
-	return $?
+	remotehost=$1
+	# Copy the answer back
+	ssh -o ConnectTimeout=10 -o BatchMode=yes -p $IDA_PRO_SERVER_PORT $IDA_PRO_SERVER_USER@$remotehost ls -l ${directory}/a.ncexe.infoannot > tmp.$$
+	if [ -f tmp.$$ ]; then
+		grep a.ncexe.infoannot tmp.$$
+		if [ ! $? -eq 0 ]; then
+			rm tmp.$$
+			return 1
+		else
+			return 0
+		fi
+	fi
+	return 1
 }
 
-server_has_cached_info
-if [ $? -eq 0 ]; then
-	echo SERVER HAS ALREADY ANALYZED $md5name, retrieving cached info
-	copy_STARS_info
-	exit 0
-else
-	echo SERVER HAS NOT YET ANALYZED $md5name, launch remote analysis 
-fi
-
-# Create unique directory on server
-ssh -p $IDA_PRO_SERVER_PORT $IDA_PRO_SERVER_USER@$IDA_PRO_SERVER_HOST mkdir -p "$directory"
-
-# Copy my files to it
-# The perl script will also include names of dependent shared object (.so) file
-# Start timing stuff
-copyStartTime=`date +%s`
-scp -P $IDA_PRO_SERVER_PORT -q $@ $IDA_PRO_SERVER_USER@$IDA_PRO_SERVER_HOST:$directory
-if [ -n "$SMPSA_PLUGIN" ]; then
-    scp -P $IDA_PRO_SERVER_PORT -q ${SMPSA_PLUGIN}* $IDA_PRO_SERVER_USER@$IDA_PRO_SERVER_HOST:$directory
-fi
-
-copyStopTime=`date +%s`
-# Check to see if the Ida Pro Server is too busy now and wait if necc.
-if [ -n "$MAX_IDA_PROCESSES" ]; then
-    while [ `ssh -p $IDA_PRO_SERVER_PORT $IDA_PRO_SERVER_USER@$IDA_PRO_SERVER_HOST pgrep idal|wc -l` -ge "$MAX_IDA_PROCESSES" ]; do
-        random=`od -An -N2 -tu2 /dev/urandom`
-        # Wait 10-30 seconds
-        seconds=`expr $random % 20 + 10`
-        echo Waiting $seconds seconds for an IDA process to exit...
-        sleep $seconds
-    done
-fi
+copy_and_verify_result()
+{
+	remotehost=$1
+	annotfile=$2
 
-# waitStartTime = copyStopTime
-waitStopTime=`date +%s`
+	copy_STARS_info $remotehost
 
-# Run ida pro on server
-# Assume remote server has proper plugin
-ssh -p $IDA_PRO_SERVER_PORT $IDA_PRO_SERVER_USER@$IDA_PRO_SERVER_HOST "cd ~/techx_umbrella/peasoup; source set_env_vars; export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:.; cd $directory; screen -D -L -ln -m -a -T xterm sh -x "'$SMPSA_HOME'"/SMP-analyze.sh $@" 2>&1
+	lines=`cat $annotfile | wc -l`
+	if [ $lines -lt 10 ]; then
+		echo "Failed to produce a valid annotations file for $annotfile."
+		return 1 
+	fi
 
-#copyAnswerStartTime=`date +%s`
+	return 0
+}
 
-copy_STARS_info
+run_remote_command()
+{
+	remotehost=$1
+	shift
+
+	echo "Remote analyze on host $remotehost"
+	ssh -o ConnectTimeout=10 -o BatchMode=yes -p $IDA_PRO_SERVER_PORT $IDA_PRO_SERVER_USER@$remotehost mkdir -p ${directory}
+
+	# Check to see if the Ida Pro Server is too busy now and wait if necc.
+	if [ -n "$MAX_IDA_PROCESSES" ]; then
+	    while [ `ssh -o ConnectTimeout=10 -o BatchMode=yes -p $IDA_PRO_SERVER_PORT $IDA_PRO_SERVER_USER@$remotehost pgrep idal|wc -l` -ge "$MAX_IDA_PROCESSES" ]; do
+	        random=`od -An -N2 -tu2 /dev/urandom`
+	        # Wait 10-30 seconds
+		seconds=`expr $random % 20 + 10`
+		echo Waiting $seconds seconds for an IDA process to exit...
+		sleep $seconds
+	    done
+	fi
+
+	scp -o ConnectTimeout=75 -o BatchMode=yes -P $IDA_PRO_SERVER_PORT -q $@ $IDA_PRO_SERVER_USER@$remotehost:$directory
+	if [ ! $? -eq 0 ]; then
+		return 1
+	fi
+
+	ssh -o ConnectTimeout=7200 -o BatchMode=yes -p $IDA_PRO_SERVER_PORT $IDA_PRO_SERVER_USER@$remotehost "cd ~/techx_umbrella/peasoup; source set_env_vars; export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:.; cd $directory; screen -D -L -ln -m -a -T xterm sh -x "'$SMPSA_HOME'"/SMP-analyze.sh $@" 2>&1
+	if [ ! $? -eq 0 ]; then
+		return 1
+	fi
+
+	copy_and_verify_result $remotehost $file.annot
+	return $?
+}
 
-#copyAnswerStopTime=`date +%s`
+# FIXME: need to handle multiple files to be analyzed?
 
-# Calculate times
-#copyTime=$(expr $copyStopTime - $copyStartTime)
-#waitTime=$(expr $waitStopTime - $copyStopTime)
-#copyAnswerTime=$(expr $copyAnswerStopTime - $copyAnswerStartTime)
+exit_code=1
+server_has_cached_info $IDA_PRO_SERVER_HOST
+if [ $? -eq 0 ]; then
+	echo SERVER HAS ALREADY ANALYZED $md5name, retrieving cached info
+	copy_and_verify_result $remotehost $file.annot
+	exit_code=$?
+else
+	run_remote_command $IDA_PRO_SERVER_HOST $@
+	exit_code=$?
+fi
 
-# write to file
-#echo "Copy Time, Wait Time, Execute Time, Copy Answer Time" >> /tmp/x.x
-#echo "`hostname`,$copyTime,$waitTime,$executeTime,$copyAnswerTime" >> /tmp/remote-analyze.${md5name}.results.txt
+if [ ! $exit_code -eq 0 ]; then
+	echo "Error detected on primary $IDA_PRO_SERVER_HOST, failing over to $IDA_PRO_SERVER_HOST2"
+	run_remote_command $IDA_PRO_SERVER_HOST2 $@
+	exit_code=$?
+fi
 
-# Cleanup
-#ssh -p $IDA_PRO_SERVER_PORT $IDA_PRO_SERVER_USER@$IDA_PRO_SERVER_HOST rm -rf $directory
+exit $exit_code