Skip to content
Snippets Groups Projects
SMPInstr.cpp 640 KiB
Newer Older
// Does instruction subtract an immediate value that is often used in ASCII computations,
//  such as the ASCII code for '0', 'a', or 'A' ?
bool SMPInstr::SubtractsImmedASCII(void) {
clc5q's avatar
clc5q committed
	op_t SubtrahendOp = this->GetUseOnlyAddSubOp();
	return (this->MDIsMaybeBenignUnderflowOpcode() && (o_imm == SubtrahendOp.type)
		&& (('0' == SubtrahendOp.value) || ('a' == SubtrahendOp.value)
		|| ('A' == SubtrahendOp.value)));
// Does instruction compare a location to an immediate value that is often used in ASCII computations,
//  such as the ASCII code for '0' or '9', 'a' or 'A', 'z' or 'Z', or carriage return?
#define SMP_ASCII_CARRIAGE_RETURN 13
bool SMPInstr::MDComparesImmedASCII(void) {
	bool ComparesToASCII = false;
	if (this->SMPcmd.itype == NN_cmp) {
		SMPRegTransfer *CurrRT = this->RTL.GetRT(0);
		if ((CurrRT != NULL) && (CurrRT->HasRightSubTree())) {
			CurrRT = CurrRT->GetRightTree();
			if (!(CurrRT->HasRightSubTree())) {
				op_t LeftOp = CurrRT->GetLeftOperand();
				op_t RightOp = CurrRT->GetRightOperand();
				if ((o_imm == RightOp.type) && ((SMP_ASCII_CARRIAGE_RETURN == RightOp.value)	
					|| ('0' == RightOp.value) || ('9' == RightOp.value) 
					|| ('a' == RightOp.value) || ('z' == RightOp.value)
					|| ('A' == RightOp.value) || ('Z' == RightOp.value))) {
					ComparesToASCII = true;
				}
			}
		}
	}
	return ComparesToASCII;
} // end of SMPInstr::MDComparesImmedASCII()

// Multiply by large constant; overflow is probably intentional.
#define STARS_LARGE_UNSIGNED_MUL_CONSTANT_THRESHOLD 0x20000000
#define STARS_LARGE_SIGNED_MUL_CONSTANT_THRESHOLD 0x10000000
#define STARS_SMALL_SIGNED_MUL_CONSTANT_THRESHOLD ((int)(-STARS_LARGE_SIGNED_MUL_CONSTANT_THRESHOLD))
bool SMPInstr::IsMultiplyByLargeConstant(uval_t &ConstValue, unsigned short SignMask) {
	bool LargeConstFound = false;

	if (this->MDIsMultiply() && (SignMask != FG_MASK_INCONSISTENT_SIGN) && (SignMask != 0)) {
		set<DefOrUse, LessDefUse>::iterator UseIter;
		for (UseIter = this->GetFirstUse(); UseIter != this->GetLastUse(); ++UseIter) {
			if (this->FindConstantValue(UseIter, ConstValue)) {
				if (FG_MASK_UNSIGNED == SignMask) {
					if (((uval_t) STARS_LARGE_UNSIGNED_MUL_CONSTANT_THRESHOLD) <= ConstValue) {
						LargeConstFound = true;
						break;
					}
				}
				else if (FG_MASK_SIGNED == SignMask) {
					int SignedConstValue = (int) ConstValue;
					if ( (((int) STARS_LARGE_SIGNED_MUL_CONSTANT_THRESHOLD) <= SignedConstValue)
						|| (((int) STARS_SMALL_SIGNED_MUL_CONSTANT_THRESHOLD) >= SignedConstValue)) {
						LargeConstFound = true;
						break;
					}
				}
			}
		}
	}

	return LargeConstFound;
} // end of SMPInstr::IsMultiplyByLargeConstant()

// Subtraction of large constant; underflow is probably intentional.
#define STARS_LARGE_UNSIGNED_SUB_CONSTANT_THRESHOLD 0x80000000
#define STARS_LARGE_SIGNED_SUB_CONSTANT_THRESHOLD 0x40000000
#define STARS_SMALL_SIGNED_SUB_CONSTANT_THRESHOLD ((int)(-STARS_LARGE_SIGNED_SUB_CONSTANT_THRESHOLD))
bool SMPInstr::IsSubtractionOfLargeConstant(uval_t &ConstValue, unsigned short SignMask) {
	bool LargeConstFound = false;

	if (this->MDIsUnderflowingOpcode() && (SignMask != FG_MASK_INCONSISTENT_SIGN) && (SignMask != 0)) {
clc5q's avatar
clc5q committed
		op_t SubtrahendOp = this->GetUseOnlyAddSubOp();
		if (o_void != SubtrahendOp.type) {
			set<DefOrUse, LessDefUse>::iterator UseIter = this->FindUse(SubtrahendOp);
			if (this->FindConstantValue(UseIter, ConstValue)) {
				if (FG_MASK_UNSIGNED == SignMask) {
					if (((uval_t) STARS_LARGE_UNSIGNED_SUB_CONSTANT_THRESHOLD) <= ConstValue) {
						LargeConstFound = true;
					}
				}
				else if (FG_MASK_SIGNED == SignMask) {
					int SignedConstValue = (int) ConstValue;
					if ( (((int) STARS_LARGE_SIGNED_SUB_CONSTANT_THRESHOLD) <= SignedConstValue)
						|| (((int) STARS_SMALL_SIGNED_SUB_CONSTANT_THRESHOLD) >= SignedConstValue)) {
						LargeConstFound = true;
					}
				}
			}
		}
	}

	return LargeConstFound;
} // end of SMPInstr::IsSubtractionOfLargeConstant()

// Subtraction of large constant; underflow is probably intentional.
#define STARS_LARGE_UNSIGNED_ADD_CONSTANT_THRESHOLD 0x80000000
#define STARS_LARGE_SIGNED_ADD_CONSTANT_THRESHOLD 0x40000000
#define STARS_SMALL_SIGNED_ADD_CONSTANT_THRESHOLD ((int)(-STARS_LARGE_SIGNED_ADD_CONSTANT_THRESHOLD))
bool SMPInstr::IsAdditionOfLargeConstant(uval_t &ConstValue, unsigned short SignMask) {
	bool LargeConstFound = false;

	if (this->MDIsOverflowingOpcode() && (SignMask != FG_MASK_INCONSISTENT_SIGN) && (SignMask != 0)) {
clc5q's avatar
clc5q committed
		op_t AddendOp = this->GetUseOnlyAddSubOp();
		if (o_void != AddendOp.type) {
			set<DefOrUse, LessDefUse>::iterator UseIter = this->FindUse(AddendOp);
			if (this->FindConstantValue(UseIter, ConstValue)) {
				if (FG_MASK_UNSIGNED == SignMask) {
					if (((uval_t) STARS_LARGE_UNSIGNED_ADD_CONSTANT_THRESHOLD) <= ConstValue) {
						LargeConstFound = true;
					}
				}
				else if (FG_MASK_SIGNED == SignMask) {
					int SignedConstValue = (int) ConstValue;
					if ( (((int) STARS_LARGE_SIGNED_ADD_CONSTANT_THRESHOLD) <= SignedConstValue)
						|| (((int) STARS_SMALL_SIGNED_ADD_CONSTANT_THRESHOLD) >= SignedConstValue)) {
						LargeConstFound = true;
					}
				}
			}
		}
	}

	return LargeConstFound;
} // end of SMPInstr::IsAdditionOfLargeConstant()

// MACHINE DEPENDENT: Opcode indicates 64-bit operands are the default.
bool SMPInstr::MDDefaultsTo64BitOperands(void) const {
	bool Default64;
	switch (this->SMPcmd.itype) {
		// use ss
		case NN_pop:
		case NN_popf:
		case NN_popfq:
		case NN_push:
		case NN_pushf:
		case NN_pushfq:
		case NN_retn:
		case NN_retf:
		case NN_retnq:
		case NN_retfq:
		case NN_call:
		case NN_callfi:
		case NN_callni:
		case NN_enter:
		case NN_enterq:
		case NN_leave:
		case NN_leaveq:

		// near branches
		case NN_ja:
		case NN_jae:
		case NN_jb:
		case NN_jbe:
		case NN_jc:
		case NN_je:
		case NN_jg:
		case NN_jge:
		case NN_jl:
		case NN_jle:
		case NN_jna:
		case NN_jnae:
		case NN_jnb:
		case NN_jnbe:
		case NN_jnc:
		case NN_jne:
		case NN_jng:
		case NN_jnge:
		case NN_jnl:
		case NN_jnle:
		case NN_jno:
		case NN_jnp:
		case NN_jns:
		case NN_jnz:
		case NN_jo:
		case NN_jp:
		case NN_jpe:
		case NN_jpo:
		case NN_js:
		case NN_jz:
		case NN_jcxz:
		case NN_jecxz:
		case NN_jrcxz:
		case NN_jmp:
		case NN_jmpni:
		case NN_jmpshort:
		case NN_loop:
		case NN_loopq:
		case NN_loope:
		case NN_loopqe:
		case NN_loopne:
		case NN_loopqne:
			Default64 = true;
			break;
		default:
			Default64 = false;
			break;
	}
	return Default64;
} // end of SMPInstr::MDDefaultsTo64BitOperands()

// MACHINE DEPENDENT: Inst has 64-bit operands
bool SMPInstr::MDHas64BitOperands(void) const {
#ifdef __EA64__
  return (((this->SMPcmd.auxpref & aux_use64) != 0)
      && ((this->SMPcmd.rex & REX_W) != 0
       || (((cmd.auxpref & aux_natop) != 0) && this->MDDefaultsTo64BitOperands())));
			// 64-bit segment, rex.w or insns-64
#else
  return false;
#endif
}

// MACHINE DEPENDENT: is current addressing 32-bit?
bool SMPInstr::MDIsAddressing32bit(void) const {
  int p = this->SMPcmd.auxpref & (aux_use32|aux_use64|aux_natad);
  return p == (aux_natad|aux_use32)
      || p == 0
      || p == aux_use64;
}

// MACHINE DEPENDENT: is current addressing 64-bit?
bool SMPInstr::MDIsAddressing64bit(void) const {
#ifdef __EA64__
  int p = this->SMPcmd.auxpref & (aux_use32|aux_use64|aux_natad);
  return p == (aux_natad|aux_use64);
#else
  return false;
#endif
}

// Fetch default bit-width op_t.dtyp field for current instruction's operands
char SMPInstr::GetOperandDtypField(void) const {
	if (this->MDHas64BitOperands()) {
		return dt_qword;
	}
	else {
		return dt_dword;
	}
}

bool SMPInstr::IsSetToZero(void) const {
	bool SetsToZero = false;
	SMPRegTransfer *CurrRT = this->RTL.GetRT(0);
	if ((!CurrRT->HasRightSubTree()) && (CurrRT->GetOperator() == SMP_ASSIGN)) {
		op_t RightOp = CurrRT->GetRightOperand();
		SetsToZero = ((o_imm == RightOp.type) && (0 == RightOp.value));
	}
	return SetsToZero;
}

// MACHINE DEPENDENT: Is instruction a return instruction?
bool SMPInstr::MDIsReturnInstr(void) const {
	return ((this->SMPcmd.itype == NN_retn) || (this->SMPcmd.itype == NN_retf));
}

// MACHINE DEPENDENT: Is instruction a POP instruction?
#define FIRST_POP_INST   NN_pop
#define LAST_POP_INST    NN_popfq
bool SMPInstr::MDIsPopInstr(void) const {
	return ((this->SMPcmd.itype >= FIRST_POP_INST)
			&& (this->SMPcmd.itype <= LAST_POP_INST));
}

// MACHINE DEPENDENT: Is instruction a PUSH instruction?
#define FIRST_PUSH_INST   NN_push
#define LAST_PUSH_INST    NN_pushfq
bool SMPInstr::MDIsPushInstr(void) const {
	return ((this->SMPcmd.itype >= FIRST_PUSH_INST)
			&& (this->SMPcmd.itype <= LAST_PUSH_INST));
}

// MACHINE DEPENDENT: Is instruction an ENTER instruction?
bool SMPInstr::MDIsEnterInstr(void) const {
	return ((this->SMPcmd.itype >= MD_FIRST_ENTER_INSTR)
			&& (this->SMPcmd.itype <= MD_LAST_ENTER_INSTR));
}

// MACHINE DEPENDENT: Is instruction a LEAVE instruction?
bool SMPInstr::MDIsLeaveInstr(void) const {
	return ((this->SMPcmd.itype >= MD_FIRST_LEAVE_INSTR)
			&& (this->SMPcmd.itype <= MD_LAST_LEAVE_INSTR));
// MACHINE DEPENDENT: Is instruction a HALT instruction?
bool SMPInstr::MDIsHaltInstr(void) const {
	return (NN_hlt == this->SMPcmd.itype);
}

#define MD_FIRST_COND_MOVE_INSTR NN_cmova
#define MD_LAST_COND_MOVE_INSTR  NN_fcmovnu
// MACHINE DEPENDENT: Is instruction a conditional move?
bool SMPInstr::MDIsConditionalMoveInstr(void) const {
	return ((this->SMPcmd.itype >= MD_FIRST_COND_MOVE_INSTR)
			&& (this->SMPcmd.itype <= MD_LAST_COND_MOVE_INSTR));
// MACHINE DEPENDENT: Is instruction any kind of move?
bool SMPInstr::MDIsMoveInstr(void) const {
	return ((NN_mov == this->SMPcmd.itype) || (NN_movsx == this->SMPcmd.itype)
		|| (NN_movzx == this->SMPcmd.itype) || this->MDIsConditionalMoveInstr());
}

// MACHINE DEPENDENT: Do opcode/operands definitely indicate signed arithmetic?
//  Generally, this is only true for certain variants of multiplication and division.
bool SMPInstr::MDIsSignedArithmetic(void) const {
	unsigned short opcode = this->SMPcmd.itype;
	if (NN_idiv == opcode)
		return true;
	if (NN_imul == opcode) {
		// If we discard the upper N bits of the multiplication result, then the
		//  lower N bits are the same for signed and unsigned multiplication, and
		//  gcc/g++ often use the IMUL opcode for both signed and unsigned multiplies
		//  when only N bits of result are retained. Therefore, the SIGNED nature of
		//  IMUL operands can only be inferred from the case in which 2N bits are kept.
		return (!(this->AreMultiplicationBitsDiscarded()));
	}
	else { // idiv and imul are only possible signed cases
		return false;
	}
} // end of SMPInstr::MDIsSignedArithmetic()

// MACHINE DEPENDENT: Is instruction a conditional jump based on an unsigned condition?
bool SMPInstr::MDIsUnsignedBranch(void) const {
	unsigned short opcode = this->SMPcmd.itype;
	return ((NN_ja == opcode) || (NN_jae == opcode) || (NN_jb == opcode) || (NN_jbe == opcode)
		|| (NN_jna == opcode) || (NN_jnae == opcode) || (NN_jnb == opcode) || (NN_jnbe == opcode));
}

// MACHINE DEPENDENT: Is instruction a conditional jump based on a signed condition?
bool SMPInstr::MDIsSignedBranch(void) const {
	unsigned short opcode = this->SMPcmd.itype;
	return ((NN_jg == opcode) || (NN_jge == opcode) || (NN_jl == opcode) || (NN_jle == opcode)
		|| (NN_jng == opcode) || (NN_jnge == opcode) || (NN_jnl == opcode) || (NN_jnle == opcode)
		|| (NN_js == opcode) || (NN_jns == opcode));
}

// MACHINE DEPENDENT: Is instruction a boolean set based on an unsigned condition?
bool SMPInstr::MDIsUnsignedSetValue(void) const {
	unsigned short opcode = this->SMPcmd.itype;
	return ((NN_seta == opcode) || (NN_setae == opcode) || (NN_setb == opcode) || (NN_setbe == opcode)
		|| (NN_setna == opcode) || (NN_setnae == opcode) || (NN_setnb == opcode) || (NN_setnbe == opcode));
}

// MACHINE DEPENDENT: Is instruction a boolean set based on a signed condition?
bool SMPInstr::MDIsSignedSetValue(void) const {
	unsigned short opcode = this->SMPcmd.itype;
	return ((NN_setg == opcode) || (NN_setge == opcode) || (NN_setl == opcode) || (NN_setle == opcode)
		|| (NN_setng == opcode) || (NN_setnge == opcode) || (NN_setnl == opcode) || (NN_setnle == opcode)
		|| (NN_sets == opcode) || (NN_setns == opcode));
}

// MACHINE DEPENDENT: Is instruction a boolean set based on any condition?
bool SMPInstr::MDIsAnySetValue(void) const {
	unsigned short opcode = this->SMPcmd.itype;
	return ((NN_seta <= opcode) && (NN_setz >= opcode));
}

// MACHINE DEPENDENT: Is instruction a left shift instruction?
bool SMPInstr::MDIsLeftShift(void) const {
	unsigned short opcode = this->SMPcmd.itype;
	return ((NN_sal == opcode) || (NN_shl == opcode));
}

// Is kind of shift or rotate that is used in hash functions
#define STARS_HASH_SHIFT_THRESHOLD 4
bool SMPInstr::MDIsHashingArithmetic(void) const {
	bool FoundHashShift = false;
	unsigned short opcode = this->SMPcmd.itype;
	// We are looking for shifts or rotates in the leftward direction.
	if ((opcode == NN_rcl) || (opcode == NN_rol) || (opcode == NN_shl) || (opcode == NN_shld)) {
		SMPRegTransfer *CurrRT = this->RTL.GetRT(0);
		assert(CurrRT->HasRightSubTree());
		CurrRT = CurrRT->GetRightTree();
		op_t ShiftCountOp = CurrRT->GetRightOperand();
		if (o_imm == ShiftCountOp.type) {
			uval_t CountValue = ShiftCountOp.value;
			FoundHashShift = (CountValue >= STARS_HASH_SHIFT_THRESHOLD);
		}
		else {
			// PEASOUP bug # 144: left shift count in CL reg, accumulates in checksum loop.
			FoundHashShift = true;
		}
	}
	return FoundHashShift;
} // end of SMPInstr::MDIsHashingArithmetic()

// Detect comparison of non-immediate to immediate. Return the two operands if true.
bool SMPInstr::MDIsCompareToPositiveConstant(op_t &NonConstOperand, uval_t &ConstValue) const {
	bool CompareToConst = false;

	if (NN_cmp == this->SMPcmd.itype) {
		SMPRegTransfer *CurrRT = this->RTL.GetRT(0);
		assert(CurrRT->HasRightSubTree());
		CurrRT = CurrRT->GetRightTree();
		assert(!(CurrRT->HasRightSubTree()));
		op_t LeftOp = CurrRT->GetLeftOperand();
		op_t RightOp = CurrRT->GetRightOperand();
		if (o_imm == RightOp.type) {
			CompareToConst = true;
			ConstValue = RightOp.value;
			NonConstOperand = LeftOp;
		}
		else if (o_imm == LeftOp.type) { // rare to see immediate first
			CompareToConst = true;
			ConstValue = LeftOp.value;
			NonConstOperand = RightOp;
		}
	}

	return CompareToConst;
} // end of SMPInstr::MDIsCompareToPositiveConstant()

bool SMPInstr::MDIsSubtractionOfConstant(op_t &NonConstOperand, uval_t &ConstValue) const {
	bool SubtractOfConst = false;

	SMPRegTransfer *CurrRT = this->RTL.GetRT(0);
	if (CurrRT->HasRightSubTree()) {
		CurrRT = CurrRT->GetRightTree();
		SMPoperator CurrOp = CurrRT->GetOperator();
		if (SMP_SUBTRACT == CurrOp) {
			assert(!(CurrRT->HasRightSubTree()));
			op_t LeftOp = CurrRT->GetLeftOperand();
			op_t RightOp = CurrRT->GetRightOperand();
			if (o_imm == RightOp.type) {
				SubtractOfConst = true;
				ConstValue = RightOp.value;
				NonConstOperand = LeftOp;
			}
		}
		else if (this->MDIsLoadEffectiveAddressInstr() && (SMP_ADD == CurrOp)) {
			// We could have an addition of a negative constant via an lea opcode,
			//  e.g. lea ecx,[eax-48] will look like ecx := eax+(-48) in the RTL.
			if (!(CurrRT->HasRightSubTree())) {
				op_t LeftOp = CurrRT->GetLeftOperand();
				op_t RightOp = CurrRT->GetRightOperand();
				if (o_imm == RightOp.type) {
					ConstValue = RightOp.value;
					int SignedConstValue = (int) ConstValue;
					if (0 > SignedConstValue) {
						SubtractOfConst = true;
						NonConstOperand = LeftOp;
						ConstValue = (uval_t)(-SignedConstValue); // make +(-x) into -(+x)
					}
				}
			}
		}
	}

	return SubtractOfConst;
} // end of SMPInstr::MDIsSubtractionOfConstant()

// is AND operation that masks off lower BytesMasked bytes
bool SMPInstr::MDIsSubregMaskInst(size_t &BytesMasked) {
	bool MaskingFound = false;

	if (this->MDIsBitwiseAndOpcode()) {
		SMPRegTransfer *CurrRT = this->RTL.GetRT(0);
		if (CurrRT->HasRightSubTree()) {
			CurrRT = CurrRT->GetRightTree();
			SMPoperator CurrOp = CurrRT->GetOperator();
			assert(SMP_BITWISE_AND == CurrOp);
			op_t RightOp = CurrRT->GetRightOperand();
			if (o_imm == RightOp.type) {
				uval_t MaskValue = RightOp.value;
				if (0x01000000 > MaskValue) {
					MaskingFound = true;
					if (0x100 > MaskValue) {
						BytesMasked = 1;
					}
					else if (0x10000 > MaskValue) {
						BytesMasked = 2;
					}
					else {
						BytesMasked = 3;
					}
				}
			}
		}
	}

	return MaskingFound;
}

// MACHINE DEPENDENT: Does instruction use a callee-saved register?
bool SMPInstr::MDUsesCalleeSavedReg(void) {
	set<DefOrUse, LessDefUse>::iterator CurrUse;
	for (CurrUse = this->GetFirstUse(); CurrUse != this->GetLastUse(); ++CurrUse) {
		op_t CurrOp = CurrUse->GetOp();
		if (CurrOp.is_reg(MD_FRAME_POINTER_REG) || CurrOp.is_reg(R_si)
			|| CurrOp.is_reg(R_di) || CurrOp.is_reg(R_bx)) {
			return true;
		}
	}
	return false;
} // end of SMPInstr::MDUsesCalleeSavedReg()

// Is the instruction a register to register copy of a stack pointer or frame pointer
//  into a general purpose register (which mmStrata will now need to track as a stack 
//  relative pointer)?
bool SMPInstr::MDIsStackPointerCopy(bool UseFP) {
	// OptType 3 indicates a move instruction
	// The lea instruction can perform three operand arithmetic, e.g.
	//  lea ebx,[esp+12] is just ebx:=esp+12, so it is a stack pointer copy.
	if (((this->OptType == 3) || (NN_lea == this->SMPcmd.itype))
		&& (this->GetFirstDef()->GetOp().type == o_reg)
		&& (!(this->GetFirstDef()->GetOp().is_reg(R_sp)))
		&& (!(this->HasSourceMemoryOperand()))) { // reg to reg move
			if (this->GetFirstUse()->GetOp().is_reg(MD_FRAME_POINTER_REG))
				// Move of base pointer EBP into a general register
				return true;
			else if ((this->GetFirstUse()->GetOp().is_reg(MD_STACK_POINTER_REG))
				&& !(this->GetFirstDef()->GetOp().is_reg(MD_FRAME_POINTER_REG)))
				// Move of ESP into something besides a base pointer
				return true;
		}
		else if (this->GetFirstUse()->GetOp().is_reg(MD_STACK_POINTER_REG)) {
			// Move of ESP into a register; no base pointer used in this function
			return true;
		}
	}
	return false;
} // end of SMPInstr::MDIsStackPointerCopy()

// Does any RTL fit the alloca() pattern: stack_pointer -= non-immediate-operand
bool SMPInstr::HasAllocaRTL(void) {
	bool FoundAlloca = false;
	size_t RTLCount = this->RTL.GetCount();
	size_t RTLIndex;

	for (RTLIndex = 0; RTLIndex < RTLCount; ++RTLIndex) {
		SMPRegTransfer *CurrRT = this->RTL.GetRT(RTLIndex);
		if (CurrRT->IsAllocaRTL()) {
			FoundAlloca = true;
			break;
		}
	}

	return FoundAlloca;
} // end of SMPInstr::HasAllocaRTL()

// Determine if the instruction saves or restores a pointer into the stack frame.
// If it saves a stack pointer, set Save to true, set the StackDelta saved, and set
//   the operand that received the saved stack pointer into CopyOp. and return true.
// If it restores a stack pointer, set Save to false, set CopyOp to the operand that
//   held the value being restored, set RestoreOp to the stack pointer or frame pointer
//   register (whichever was restored), leave StackDelta alone for later computation
//   based on reaching definitions, and return true.
// For most instructions, no save or restore of a stack pointer, so return false.
bool SMPInstr::MDIsStackPtrSaveOrRestore(bool UseFP, sval_t FPDelta, bool &Save, sval_t &StackDelta, op_t &CopyOp, bool &Error) {
	bool StackPointerSaveOrRestore;
	size_t RTLCount = this->RTL.GetCount();
	size_t RTLIndex;
	int BaseReg, IndexReg, CopyReg;
	ushort Scale;
	ea_t offset;
	SMPoperator CurrOper;
	bool LookUpStackDelta; // Get stack delta from reaching defs for TempOp
	sval_t DeltaAdjust; // add to StackDelta after computing from reaching defs, e.g. lea esp,[ecx-4] get TempOp of ecx
	                        //  and DeltaAdjust of -4

	Error = false;

	for (RTLIndex = 0; RTLIndex < RTLCount; ++RTLIndex) {
		bool FPRestore = false; // frame pointer is restored
		bool SPRestore = false; // stack pointer is restored
		StackPointerSaveOrRestore = false; // default unless we detect a save or restore of the stack or frame pointer
		TempOp = InitOp;
		LookUpStackDelta = false;
		DeltaAdjust = 0;
		Save = false; // default unless we detect a stack pointer save

		// The stack alignment instructions (SP := SP bitwise_and immediate_value)
		//  look like something that needs to be processed here, but we always ignore
		//  these instructions. They have a variable effect on the stack pointer, from zero
		//  to -15 delta, but we assume that the delta is zero. This works for us because
		//  no stack accesses will occur into the padding region.
		// Also, any instruction that definitely does not restore the stack pointer or
		//  frame pointer from an arbitrary register or memory location, e.g. a leave instruction
		//  in x86 CPUs, is already handled in normal stack delta computations and needs
		//  no lookups from reaching defs, etc.
		if (this->IsStackAlignmentInst() || this->MDIsLeaveInstr() || this->MDIsFrameAllocInstr()) {
			break; // exit and return false
		}

		SMPRegTransfer *CurrRT = this->RTL.GetRT(RTLIndex);
		CurrOper = CurrRT->GetOperator();
		if (SMP_ASSIGN != CurrOper) {
			break; // not a regular RTL
		}
		op_t LeftOp = CurrRT->GetLeftOperand();
		if (4 > GetOpDataSize(LeftOp)) {
			break; // Not tracking copies of less than the full stack or frame pointer
			// NOTE: We hard code 4 bytes because we will track bottom 32 bits, often seen in 64-bit code.
		if (LeftOp.is_reg(MD_STACK_POINTER_REG)) {
			SPRestore = true; // temporary; might just be a push or pop RTL, etc., in which case we will reset.
		}
		else if (UseFP && LeftOp.is_reg(MD_FRAME_POINTER_REG)) {
			FPRestore = true; // likewise temporary
		}
		if (!(SPRestore || FPRestore)) {
#if 0
			if (LeftOp.is_reg(MD_FLAGS_REG)) {
				break; // No point in looking for a save into the flags register
			}
#endif
			Save = true;  // Maybe; keep looking for save

		// If we are assigning to the stack pointer reg or the frame pointer reg, we need to analyze the right
		//  hand side of the RTL to see if it is a stack/frame pointer value, and not a simple push, pop, etc.
		if (!(CurrRT->HasRightSubTree())) {
			op_t RightOp = CurrRT->GetRightOperand();
			if ((o_reg <= RightOp.type) && (o_displ >= RightOp.type)) { // register or memory
				if (RightOp.is_reg(MD_STACK_POINTER_REG)) {
					// Stack pointer reg is being saved.
					StackDelta = this->GetStackPtrOffset(); // LeftOp := SP, so saved delta is just current delta
					CopyOp = LeftOp;
					StackPointerSaveOrRestore = true;
					FPRestore = false; // treat FP := SP as a save of SP rather than a restoration of FP
				else if (!SPRestore && UseFP && RightOp.is_reg(MD_FRAME_POINTER_REG)) {
					// Frame pointer is being saved
					StackDelta = FPDelta;
					CopyOp = LeftOp;
					StackPointerSaveOrRestore = true;
					break;
				}
				else if (SPRestore || FPRestore) {
					// stack or frame pointer is being restored; leave Save=false and set other outgoing arguments.
					TempOp = RightOp;
					CopyOp = RightOp;
					StackPointerSaveOrRestore = true;
					LookUpStackDelta = true;
				}
				else { // RightOp is register or non-stack-pointer memory expr; either might hold stack delta
					TempOp = RightOp;
					CopyOp = LeftOp;
					LookUpStackDelta = true; // See if RightOp is holding a stack delta
					StackPointerSaveOrRestore = true;  // Maybe; flag tells us to keep looking
				if (SPRestore || FPRestore) {
					SMP_msg("ERROR: Invalid operand type for assignment to stack or frame pointer at %lx\n",
						(unsigned long) this->GetAddr());
				StackPointerSaveOrRestore = false;
				break;
			}
		}
		else { // we have a right subtree in the CurrRT
			SMPRegTransfer *RightRT = CurrRT->GetRightTree();
			// In order to have a right subtree, we must have something like:
			//   lea esp,[ecx-4]  which produces the RTL: esp := ecx - 4
			// We should consider any other RTL structure besides a basic addition or
			//  subtraction on the right subtree to be invalid.
			CurrOper = RightRT->GetOperator();
			if ((SMP_ADD == CurrOper) || (SMP_SUBTRACT == CurrOper)) {
				op_t RightLeftOp = RightRT->GetLeftOperand();
				if (o_reg == RightLeftOp.type) {
					if (RightRT->HasRightSubTree()) {
						// Complex RTL such as lea esp,[ebx+ecx*4] ; cannot analyze
						StackPointerSaveOrRestore = false;
					}
					else {
						op_t RightRightOp = RightRT->GetRightOperand();
						if (o_imm != RightRightOp.type) {
							// Complex RTL such as lea esp,[ebx+ecx] ; cannot analyze
							StackPointerSaveOrRestore = false;
						}
						else {
							TempOp = RightLeftOp;
							DeltaAdjust = (sval_t) RightRightOp.value;
							if (DeltaAdjust > STARS_ESP_ADDITION_OVERFLOW_THRESHOLD) {
								// Really a subtraction via addition of a negative,
								//  or addition via subtraction of a negative.
								//  E.g. add esp, 0xfffffff0 is sub esp,16 and sub esp,0xfffffff0 is add esp,16
								int32 TempDelta = (int32) (DeltaAdjust & 0xffffffff);
								DeltaAdjust = (sval_t) TempDelta;
							}
							if (SMP_SUBTRACT == CurrOper) {
								// Negate the stack delta adjustment, e.g. lea esp,[ecx-4] needs DeltaAdjust of -4, not 4.
								DeltaAdjust = (0 - DeltaAdjust);
							}
							LookUpStackDelta = true;
							StackPointerSaveOrRestore = true;
							if (SPRestore || FPRestore) {
								CopyOp = RightLeftOp;
							}
							else {
								CopyOp = LeftOp;
							}
						}
					}
				}
				else { // weird RTL; LeftOp := (MemoryOp OPER ???)
					StackPointerSaveOrRestore = false;
				}
			}
			else { // not ADD or SUBTRACT
				StackPointerSaveOrRestore = false;
			}
		}

		if (LookUpStackDelta) {
			bool StackAccess = false;
			// We need to set StackDelta based on the reaching defs for TempOp
			// A reg is probably a general register, but could have lea ebx,[esp+4] so it could be stack or frame pointer.
			if (TempOp.is_reg(MD_STACK_POINTER_REG)) {
				// Weed out RTs that increment or decrement the stack pointer, e.g. SP := SP -4.
				//  These are not the kind of "save" or "restore" RTs that we are tracking.
				if (CopyOp.is_reg(MD_STACK_POINTER_REG)) {
					StackPointerSaveOrRestore = false;
					SPRestore = false;
					FPRestore = false;
					Save = false;
				}
				else {
					StackDelta = this->GetStackPtrOffset();
					StackDelta += DeltaAdjust;
					LookUpStackDelta = false; // just got it; no need for reaching defs
					StackPointerSaveOrRestore = true;
				}
			}
			else if (UseFP && TempOp.is_reg(MD_FRAME_POINTER_REG)) {
				StackDelta = FPDelta;
				StackDelta += DeltaAdjust;
				LookUpStackDelta = false; // just got it; no need for reaching defs
				StackPointerSaveOrRestore = true;
			else if (o_reg == TempOp.type) { // general reg, not frame or stack pointer reg
				CopyReg = TempOp.reg;
				MDExtractAddressFields(TempOp, BaseReg, IndexReg, Scale, offset);
				CopyReg = BaseReg;
				bool IndexedAccess = ((R_none != BaseReg) && (R_none != IndexReg));
				if (IndexedAccess) {
					StackPointerSaveOrRestore = false;  // Cannot analyze indexed accesses into the stack
				}
				else if (MDIsStackPtrReg(BaseReg, UseFP)) {
					StackAccess = true;
				}
				else {
					// memory expr that is not stack or frame pointer
					NonStackMemAccess = true;  // something like [ecx] might actually turn out to be stack access
					DeltaAdjust = (sval_t) TempOp.addr; // get normalized delta from addr field
				}
			if (StackPointerSaveOrRestore && LookUpStackDelta) {
				op_t FindOp = InitOp;
				if (StackAccess) {
					FindOp = TempOp;
				}
				else {
					FindOp.type = o_reg;
					FindOp.reg = CopyReg;
					FindOp.dtyp = this->GetOperandDtypField();
				}
				if (this->GetBlock()->GetFunc()->IsInStackPtrCopySet(FindOp)) {
					// Screened out time wasters that are not in copy set; now,
					//  look up reaching defs.
					// We need to find out which are the reaching definitions for the FindOp at the current InstAddr.
					this->GetBlock()->GetFunc()->ComputeTempReachingDefs(FindOp, this->GetAddr());
					this->GetBlock()->GetFunc()->ComputeTempStackDeltaReachesList(FindOp);
					// See if TempStackDeltaReachesList has a consistent delta value.
					StackPointerSaveOrRestore = this->GetBlock()->GetFunc()->FindReachingStackDelta(StackDelta); // consistent SavedDelta value across entire list
					StackDelta += DeltaAdjust;
					if (StackPointerSaveOrRestore && NonStackMemAccess) {
						// We have something like [ecx] or [ecx+DeltaAdjust]. It turns out that
						//  ECX has a copy of the stack pointer or frame pointer in it, but that
						//  does not mean that the memory location [ecx] has a copy of a stack or
						//  frame pointer. We need to look up the normalized stack address [esp+StackDelta]
						//  in the StackPtrCopySet just like we did for ECX before we conclude that a
						//  stack pointer save or restore is happening.
						FindOp = InitOp;
						FindOp.type = o_displ;
						FindOp.reg = MD_STACK_POINTER_REG;
						FindOp.addr = StackDelta;
						FindOp.dtyp = this->GetOperandDtypField();
						if (this->GetBlock()->GetFunc()->IsInStackPtrCopySet(FindOp)) {
							// Screened out time wasters that are not in copy set; now,
							//  look up reaching defs.
							// We need to find out which are the reaching definitions for the FindOp at the current InstAddr.
							this->GetBlock()->GetFunc()->ComputeTempReachingDefs(FindOp, this->GetAddr());
							this->GetBlock()->GetFunc()->ComputeTempStackDeltaReachesList(FindOp);
							// See if TempStackDeltaReachesList has a consistent delta value.
							StackPointerSaveOrRestore = this->GetBlock()->GetFunc()->FindReachingStackDelta(StackDelta); // consistent SavedDelta value across entire list
							// StackPointerSaveOrRestore will now be true only if [ecx] pointed to a saved stack or frame pointer with consistent delta.
						}
						else {
							// E.g. [ecx] pointed to stack location that was not holding a saved stack or frame pointer.
							StackPointerSaveOrRestore = false;
						}
					}
				}
				else {
					StackPointerSaveOrRestore = false; // reset, not in stack pointer copy set
				}
		} // end if (LookupStackDelta)

		if (!StackPointerSaveOrRestore && !Save && (SPRestore || FPRestore)) {
			// Any restore that could not be analyzed is an error.
			Error = true;
			break; // error exit
		}
		else if (StackPointerSaveOrRestore) {
			if (FPRestore) {
				// If we succeeded in looking up a stack delta that goes into the frame pointer reg,
				//  then we want to consider this instruction to be a save of a stack delta into
				//  a register (which happens to be the frame pointer reg in this case).
				FPRestore = false;
				Save = true;
			break; // assume only one save or restore in an instruction; exit with success
	} // end for all RTs in the RTL

	return StackPointerSaveOrRestore;
} // end of SMPInstr::MDIsStackPtrSaveOrRestore()

// Make list of regs and their stack offsets in RTLs for any push instruction
bool SMPInstr::GetPushedRegsList(map<uint32, sval_t> &PushedRegs) {
	size_t RTLCount = this->RTL.GetCount();
	size_t RTLIndex;
	op_t LeftOp, RightOp;
	ushort Scale;
	ea_t offset;
	SMPoperator CurrOper;

	for (RTLIndex = 0; RTLIndex < RTLCount; ++RTLIndex) {
		SMPRegTransfer *CurrRT = this->RTL.GetRT(RTLIndex);
		CurrOper = CurrRT->GetOperator();
		// Only process simple pushes of registers
		LeftOp = CurrRT->GetLeftOperand();
		if (MDIsStackAccessOpnd(LeftOp, false) && (!(CurrRT->HasRightSubTree())) && (SMP_ASSIGN == CurrOper)) {
			RightOp = CurrRT->GetRightOperand();
			if (RightOp.type == o_reg) {
				MDExtractAddressFields(LeftOp, BaseReg, IndexReg, Scale, offset);
				// Exclude indexed stack writes.
				if (R_none == IndexReg) {
					sval_t ESPOffset = (sval_t) offset;
					map<uint32, sval_t>::iterator FindIter = PushedRegs.find((uint32) RightOp.reg);
					if (FindIter == PushedRegs.end()) { // Not already pushed earlier
						pair<uint32, sval_t> TempPair((uint32) RightOp.reg, ESPOffset);
						pair<map<uint32, sval_t>::iterator, bool> InsertResult;
						InsertResult = PushedRegs.insert(TempPair);
						assert(InsertResult.second);
					}
				}
			}
		}
	} // end for all RTs in RTL

	return (!PushedRegs.empty());
} // end of SMPInstr::GetPushedRegsList()

// Make list of regs and their stack offsets in RTLs for any pop instruction
bool SMPInstr::GetPoppedRegsList(bool FirstReturnBlock, map<uint32, sval_t> &PoppedRegs) {
	size_t RTLCount = this->RTL.GetCount();
	size_t RTLIndex;
	size_t IncomingSize = PoppedRegs.size(); // from return blocks already processed
	size_t MatchCount = 0; // count of matches to incoming data
	op_t LeftOp, RightOp;
	ushort Scale;
	ea_t offset;
	SMPoperator CurrOper;

	for (RTLIndex = 0; RTLIndex < RTLCount; ++RTLIndex) {
		SMPRegTransfer *CurrRT = this->RTL.GetRT(RTLIndex);
		CurrOper = CurrRT->GetOperator();
		// Only process simple pushes of registers
		LeftOp = CurrRT->GetLeftOperand();
		if ((LeftOp.type == o_reg) && (!(CurrRT->HasRightSubTree())) && (SMP_ASSIGN == CurrOper)) {
			RightOp = CurrRT->GetRightOperand();
			if (MDIsStackAccessOpnd(RightOp, false)) {
				MDExtractAddressFields(RightOp, BaseReg, IndexReg, Scale, offset);
				// Exclude indexed stack reads.
				if (R_none == IndexReg) {
					sval_t ESPOffset = (sval_t) offset;
					map<uint32, sval_t>::iterator FindIter = PoppedRegs.find((uint32) LeftOp.reg);
					if (FindIter == PoppedRegs.end()) { // Not already popped in another block
						if (FirstReturnBlock) {
							pair<uint32, sval_t> TempPair((uint32) LeftOp.reg, ESPOffset);
							pair<map<uint32, sval_t>::iterator, bool> InsertResult;
							InsertResult = PoppedRegs.insert(TempPair);
							assert(InsertResult.second);
						}
						else {
							// We should not find a new pop in return block N not seen in return block N-1
							PoppedRegs.clear(); // causes a return value of false
							break;
						}
					}
					else {
						// Already in map; check for consistency across return points.
						if (ESPOffset != FindIter->second) { // inconsistent
							PoppedRegs.clear(); // causes a return value of false
							break;
						}
						else {
							++MatchCount;
						}
					}
				}
			}
		}
	} // end for all RTs in RTL

	if (FirstReturnBlock) {
		IncomingSize = 0; // Not trying to match within first return block
	}

	return ((MatchCount == IncomingSize) && (!PoppedRegs.empty()));
} // end of SMPInstr::GetPoppedRegsList()

// If call instruction is to malloc(), set the DEF register EAX type to
//  HEAPPTR and return true.
bool SMPInstr::MDFindMallocCall(op_t TargetOp) {
	bool changed = false;
	func_t *TargetFunc = get_func(TargetOp.addr);
	if (TargetFunc) {
		char FuncName[MAXSTR];
		get_func_name(TargetFunc->startEA, FuncName, sizeof(FuncName) - 1);
		if (0 == strcmp("malloc", FuncName)) {
			// NOTE: Some compilers might call it __malloc ; make this more robust !!!
#if SMP_VERBOSE_FIND_POINTERS
			SMP_msg("Found call to malloc at %x\n", this->addr);
#endif
			op_t SearchOp = InitOp;
			SearchOp.type = o_reg;
			SearchOp.reg = R_ax;
			SearchOp.dtyp = this->GetOperandDtypField();
			set<DefOrUse, LessDefUse>::iterator EAXDEF;
			EAXDEF = this->SetDefType(SearchOp, HEAPPTR);
			int SSANum = EAXDEF->GetSSANum();
			changed = true;
			if (this->BasicBlock->IsLocalName(SearchOp)) {
				(void) this->BasicBlock->PropagateLocalDefType(SearchOp, HEAPPTR,
						this->GetAddr(), SSANum, false);
			}
			else { // global name
				this->BasicBlock->GetFunc()->ResetProcessedBlocks(); // set Processed to false
				(void) this->BasicBlock->PropagateGlobalDefType(SearchOp, HEAPPTR,
						SSANum, false);
			}
		} // end if "malloc"
	} // end if (TargetFunc)
	return changed;
} // end of SMPInstr::MDFindMallocCall()

// Is instruction a branch (conditional or unconditional) to a
//  code target that is not in the current chunk?
bool SMPInstr::IsBranchToFarChunk(void) {
	if (this->IsFarBranchComputed()) { // answer is cached
		return this->IsBranchesToFarChunk();
	func_t *CurrChunk = get_fchunk(this->address);
	bool FarBranch = false;
	if ((JUMP | COND_BRANCH) & this->GetDataFlowType()) {
		// Instruction is a direct branch, conditional or unconditional
		if (this->NumUses() > 0) {
			set<DefOrUse, LessDefUse>::iterator CurrUse;
			for (CurrUse = this->GetFirstUse(); CurrUse != this->GetLastUse(); ++CurrUse) {
				op_t JumpTarget = CurrUse->GetOp();
				if ((o_near == JumpTarget.type) || (o_far == JumpTarget.type)) {
					// Branches to a code address
					// stdclib sometimes has jumps to zero and calls to zero. These are dead code.
					if (0 != JumpTarget.addr) {
						func_t *TargetChunk = get_fchunk(JumpTarget.addr);
						// Is target address within the same chunk as the branch?
						FarBranch = (NULL == TargetChunk) || (CurrChunk->startEA != TargetChunk->startEA);
						if (FarBranch) {
							this->FarBranchTarget = JumpTarget.addr;
						}
	if (FarBranch) {
		this->SetBranchesToFarChunk();
	}
	this->SetFarBranchComputed();