SMPDataFlowAnalysis.cpp

/*
 * SMPDataFlowAnalysis.cpp - <see below>.
 *
 * Copyright (c) 2000, 2001, 2010 - University of Virginia 
 *
 * This file is part of the Memory Error Detection System (MEDS) infrastructure.
 * This file may be used and modified for non-commercial purposes as long as 
 * all copyright, permission, and nonwarranty notices are preserved.  
 * Redistribution is prohibited without prior written consent from the University 
 * of Virginia.
 *
 * Please contact the authors for restrictions applying to commercial use.
 *
 * THIS SOURCE IS PROVIDED "AS IS" AND WITHOUT ANY EXPRESS OR IMPLIED
 * WARRANTIES, INCLUDING, WITHOUT LIMITATION, THE IMPLIED WARRANTIES OF
 * MERCHANTIBILITY AND FITNESS FOR A PARTICULAR PURPOSE.
 *
 * Author: University of Virginia
 * e-mail: jwd@virginia.com
 * URL   : http://www.cs.virginia.edu/
 *
 * Additional copyrights 2010, 2011, 2012, 2013, 2014, 2015 by Zephyr Software LLC
 * e-mail: {clc,jwd}@zephyr-software.com
 * URL   : http://www.zephyr-software.com/
 *
 */

//
// SMPDataFlowAnalysis.cpp
//
// This module contains common types an helper classes needed for the
//   SMP project (Software Memory Protection).
//

#include <list>
#include <set>
#include <vector>
#include <algorithm>
#include <string>

#include <cstring>
#include <cassert>

#include "interfaces/SMPDBInterface.h"
#include "base/SMPDataFlowAnalysis.h"
#include "base/SMPInstr.h"
#include "base/SMPBasicBlock.h"
#include "base/SMPFunction.h"

using namespace std;

// Set these to 1 for debugging output
#define SMP_DEBUG_CONTROLFLOW 0  // tells what processing stage is entered
#define SMP_DEBUG_CHUNKS 1  // tracking down tail chunks for functions
#define SMP_DEBUG_FRAMEFIXUP 0  // Fixing up stack frame info the way we want the offsets
#define SMP_DEBUG_OPERAND_TYPES 1  // leave on; warnings that should never happen
#define STARS_DEBUG_DUMP_IDENTIFY_HIDDEN_OPERANDS 0 // print HIDDEN if operand.showed() is false

#define MAX_IDA_REG STARS_x86_R_last

// Bit masks for extracting bits from a STARSBitSet unsigned char.
const unsigned char STARSBitMasks[8] = { 1, 2, 4, 8, 16, 32, 64, 128 };

const char *RegNames[MAX_IDA_REG + 1] =
	{ "EAX", "ECX", "EDX", "EBX", "ESP", "EBP", "ESI", "EDI",
	  "R8", "R9", "R10", "R11", "R12", "R13", "R14", "R15",
	  "AL", "CL", "DL", "BL", "AH", "CH", "DH", "BH",
	  "SPL", "BPL", "SIL", "DIL", "EIP", "ES", "CS", "SS",
	  "DS", "FS", "GS", "CF", "ZF", "SF", "OF", "PF",
	  "AF", "TF", "IF", "DF", "EFLAGS", "FPU_ST0", "FPU_ST1", "FPU_ST2",
	  "FPU_ST3", "FPU_ST4", "FPU_ST5", "FPU_ST6", "FPU_ST7", "FPU_CTRL", "FPU_STAT", "FPU_TAGS",
	  "MMX0", "MMX1", "MMX2", "MMX3", "MMX4", "MMX5", "MMX6", "MMX7",
	  "XMM0", "XMM1", "XMM2", "XMM3", "XMM4", "XMM5", "XMM6", "XMM7",
	  "XMM8", "XMM9", "XMM10", "XMM11", "XMM12", "XMM13", "XMM14", "XMM15",
	  "MXCSR",
	  "YMM0", "YMM1", "YMM2", "YMM3", "YMM4", "YMM5", "YMM6", "YMM7",
	  "YMM8", "YMM9", "YMM10", "YMM11", "YMM12", "YMM13", "YMM14", "YMM15",
	  "REG_ERROR"
	};

// NOTE: Review these sizes. Alter when annotation diffs can be isolated to the change.
//   !!!!****!!!! FP reg stack should be 10-byte registers, right?
const unsigned char RegSizes[MAX_IDA_REG + 1] =
	{ 4, 4, 4, 4, 4, 4, 4, 4,
	  8, 8, 8, 8, 8, 8, 8, 8,
	  1, 1, 1, 1, 1, 1, 1, 1,
	  1, 1, 1, 1, 4, 2, 2, 2,
	  2, 2, 2, 4, 4, 4, 4, 4,
	  4, 4, 4, 4, 4, 10, 10, 10,
	  10, 10, 10, 10, 10, 4, 4, 4,
	  16, 16, 16, 16, 16, 16, 16, 16,
	  16, 16, 16, 16, 16, 16, 16, 16,
	  16, 16, 16, 16, 16, 16, 16, 16,
	  4,
	  32, 32, 32, 32, 32, 32, 32, 32,
	  32, 32, 32, 32, 32, 32, 32, 32,
	  4
	};

unsigned char GetRegSize(uint16_t RegNum) {
	assert(RegNum != ((uint16_t) STARS_x86_R_none));
	return RegSizes[RegNum];
}

const char RegDtyps[MAX_IDA_REG + 1] =
{ STARS_dt_dword, STARS_dt_dword, STARS_dt_dword, STARS_dt_dword, STARS_dt_dword, STARS_dt_dword, STARS_dt_dword, STARS_dt_dword,
STARS_dt_qword, STARS_dt_qword, STARS_dt_qword, STARS_dt_qword, STARS_dt_qword, STARS_dt_qword, STARS_dt_qword, STARS_dt_qword,
STARS_dt_byte, STARS_dt_byte, STARS_dt_byte, STARS_dt_byte, STARS_dt_byte, STARS_dt_byte, STARS_dt_byte, STARS_dt_byte,
STARS_dt_byte, STARS_dt_byte, STARS_dt_byte, STARS_dt_byte, STARS_dt_dword, STARS_dt_dword, STARS_dt_dword, STARS_dt_dword,
STARS_dt_dword, STARS_dt_dword, STARS_dt_dword, STARS_dt_dword, STARS_dt_dword, STARS_dt_dword, STARS_dt_dword, STARS_dt_dword,
STARS_dt_dword, STARS_dt_dword, STARS_dt_dword, STARS_dt_dword, STARS_dt_dword, STARS_dt_tbyte, STARS_dt_tbyte, STARS_dt_tbyte,
STARS_dt_tbyte, STARS_dt_tbyte, STARS_dt_tbyte, STARS_dt_tbyte, STARS_dt_tbyte, STARS_dt_word, STARS_dt_word, STARS_dt_word,
STARS_dt_byte16, STARS_dt_byte16, STARS_dt_byte16, STARS_dt_byte16, STARS_dt_byte16, STARS_dt_byte16, STARS_dt_byte16, STARS_dt_byte16,
STARS_dt_byte16, STARS_dt_byte16, STARS_dt_byte16, STARS_dt_byte16, STARS_dt_byte16, STARS_dt_byte16, STARS_dt_byte16, STARS_dt_byte16,
STARS_dt_byte16, STARS_dt_byte16, STARS_dt_byte16, STARS_dt_byte16, STARS_dt_byte16, STARS_dt_byte16, STARS_dt_byte16, STARS_dt_byte16,
STARS_dt_word,
STARS_dt_byte32, STARS_dt_byte32, STARS_dt_byte32, STARS_dt_byte32, STARS_dt_byte32, STARS_dt_byte32, STARS_dt_byte32, STARS_dt_byte32,
STARS_dt_byte32, STARS_dt_byte32, STARS_dt_byte32, STARS_dt_byte32, STARS_dt_byte32, STARS_dt_byte32, STARS_dt_byte32, STARS_dt_byte32,
STARS_dt_word 
};

const char *ErrorStrings[1] = { "ERROR_REG" };

const char *WordRegStrings[8] = { "AX", "CX", "DX", "BX", "SP", "BP", "SI", "DI" };

const char *QWordRegStrings[8] = { "RAX", "RCX", "RDX", "RBX", "RSP", "RBP", "RSI", "RDI" };

const char *SignednessStrings[4] = { "UNKNOWNSIGN", "SIGNED", "UNSIGNED", "UNKNOWNSIGN" };

const char *LeaSignednessStrings[4] = { "NOFLAGUNKNOWNSIGN", "NOFLAGSIGNED", "NOFLAGUNSIGNED", "NOFLAGUNKNOWNSIGN" };

// Distinguishes subword regs from their parent regs
const char *MDGetRegNumName(uint16_t RegNum, uint16_t ByteWidth) {
	if ((STARS_x86_R_none == RegNum) || (MAX_IDA_REG < RegNum))
		return ErrorStrings[0];
	else if ((ByteWidth == 2) && (RegNum >= STARS_x86_R_ax) && (RegNum <= STARS_x86_R_di)) {
		// 16-bit registers
		return WordRegStrings[RegNum];
	}
	else if ((ByteWidth == 8) && (RegNum >= STARS_x86_R_ax) && (RegNum <= STARS_x86_R_di)) {
		// 64-bit registers
		return QWordRegStrings[RegNum];
	}
	else {
		return RegNames[RegNum];
	}
}

// Distinguishes subword regs from their parent regs
const char *MDGetRegName(const STARSOpndTypePtr &RegOp) {
	if (!RegOp->IsRegOp())
		return ErrorStrings[0];

	uint16_t RegNum = RegOp->GetReg();
	uint16_t ByteWidth = RegOp->GetByteWidth();
	return MDGetRegNumName(RegNum, ByteWidth);
}

// Define instruction categories for data flow analysis.
SMPitype DFACategory[STARS_NN_last+1];
// Define instruction categories for data type analysis.
int SMPTypeCategory[STARS_NN_last+1];

// Define which instructions define and use the CPU flags.
bool SMPDefsFlags[STARS_NN_last + 1];
bool SMPUsesFlags[STARS_NN_last + 1];

// Hash a global name and SSA number into an int, for use in SMPFunction.GlobalDefAddrBySSA map
int HashGlobalNameAndSSA(const STARSOpndTypePtr &DefOp, int SSANum) { 
	int HashValue = 0;
	if (DefOp->IsRegOp()) {
		HashValue = ((SSANum << 16) | ((int)(DefOp->GetReg())));
	}
	return HashValue;
}

// Hash a global name and SSA number into an int, for use in SMPFunction.GlobalDefAddrBySSA map
int64_t HashGlobalStackNameAndSSA(const STARSOpndTypePtr &DefOp, int SSANum, bool UseFP) {
	int64_t HashValue = 0;
	assert(MDIsDirectStackAccessOpnd(DefOp, UseFP));
	HashValue = ((((int64_t) SSANum) << 32) | (((uint64_t)(DefOp->GetAddr())) & 0xffffffff));
	return HashValue;
}

// Get the size in bytes of the data type of an operand.
size_t GetOpDataSize(const STARSOpndTypePtr &DataOp) {
	size_t DataSize;
	char OpDtyp = DataOp->GetOpDtyp();
	if (DataOp->IsRegOp()) {
		DataSize = RegSizes[DataOp->GetReg()];
		if (OpDtyp == STARS_dt_word) {
			DataSize = 2;
#if 0
			SMP_msg("Found 16-bit register using dtyp field.\n");
#endif
		}
		else if (OpDtyp == STARS_dt_qword) {
			DataSize = 8;
#if 0
			SMP_msg("Found 64-bit register using dtyp field.\n");