Skip to content
Snippets Groups Projects
SMPInstr.cpp 396 KiB
Newer Older
					// R_sp signifies no SIB index register. So, we have
					//  lea reg,[reg+0] with reg being the same in both place,
					//  once as Operands[0] and once as the base reg in Operands[1].
					IsNop = true;
				}
				else if (destreg == this->SMPcmd.Operands[1].reg) {
					IsNop = true;
				}
		}
	}
	return IsNop;
} // end of SMPInstr::MDIsNop()

// Is non-multiply arithmetic instruction that can possibly overflow?
bool SMPInstr::MDIsOverflowingOpcode(void) const {
	unsigned short opcode = this->SMPcmd.itype;

	return ((NN_adc == opcode) || (NN_add == opcode) || (NN_inc == opcode)
		|| (NN_neg == opcode) || (NN_xadd == opcode));
}

// Is non-multiply arithmetic instruction that can possibly underflow?
bool SMPInstr::MDIsUnderflowingOpcode(void) const {
	unsigned short opcode = this->SMPcmd.itype;

	return ((NN_dec == opcode) || (NN_sbb == opcode) || (NN_sub == opcode));
}

// Is potentially benign overflow instruction?
bool SMPInstr::MDIsMaybeBenignOverflowOpcode(void) const {
	unsigned short opcode = this->SMPcmd.itype;

	return ((NN_adc == opcode) || (NN_add == opcode));
}

// Is potentially benign underflow instruction?
bool SMPInstr::MDIsMaybeBenignUnderflowOpcode(void) const {
	unsigned short opcode = this->SMPcmd.itype;

	return ((NN_neg == opcode) || (NN_sbb == opcode) || (NN_sub == opcode));
}

// Is definitely benign underflow instruction?
//  NOTE: Overlaps with MDIsMaybeBenignUnderflowOpcode(), so call this one first.
bool SMPInstr::MDIsDefiniteBenignUnderflowOpcode(void) {
	unsigned short opcode = this->SMPcmd.itype;

	// gcc use:  sbb edx,edx as a tricky way to get all zeroes or all ones into edx.
	//  (Some sort of saturation?)
	//  The "underflow" on the subtraction is irrelevant and benign.
	return ((NN_sbb == opcode) && (this->SubtractsFromItself()));
}

// Does a subtraction operator get applied to same left and right operands?
bool SMPInstr::SubtractsFromItself(void) {
	bool SelfSubtract = false;
	size_t RTLCount = this->RTL.GetCount();
	for (size_t index = 0; index < RTLCount; ++index) {
		SMPRegTransfer *CurrRT = this->RTL.GetRT(index);
		if ((CurrRT != NULL) && (CurrRT->HasRightSubTree())) {
			CurrRT = CurrRT->GetRightTree();
			SMPoperator CurrOp = CurrRT->GetOperator();
			if ((SMP_SUBTRACT_BORROW == CurrOp) || (SMP_SUBTRACT == CurrOp)) {
				if (!(CurrRT->HasRightSubTree())) {
					// NOTE: Must change this code when we build more precise SMP_SUBTRACT_BORROW RTL.
					op_t LeftOp = CurrRT->GetLeftOperand();
					op_t RightOp = CurrRT->GetRightOperand();
					SelfSubtract = IsEqOp(RightOp, LeftOp);
				}
				break;
			}
		}
	}
	return SelfSubtract;
} // end of SMPInstr::SubtractsFromItself()

// MACHINE DEPENDENT: Is instruction a return instruction?
bool SMPInstr::MDIsReturnInstr(void) const {
	return ((this->SMPcmd.itype == NN_retn) || (this->SMPcmd.itype == NN_retf));
}

// MACHINE DEPENDENT: Is instruction a POP instruction?
#define FIRST_POP_INST   NN_pop
#define LAST_POP_INST    NN_popfq
bool SMPInstr::MDIsPopInstr(void) const {
	return ((this->SMPcmd.itype >= FIRST_POP_INST)
			&& (this->SMPcmd.itype <= LAST_POP_INST));
}

// MACHINE DEPENDENT: Is instruction a PUSH instruction?
#define FIRST_PUSH_INST   NN_push
#define LAST_PUSH_INST    NN_pushfq
bool SMPInstr::MDIsPushInstr(void) const {
	return ((this->SMPcmd.itype >= FIRST_PUSH_INST)
			&& (this->SMPcmd.itype <= LAST_PUSH_INST));
}

// MACHINE DEPENDENT: Is instruction an ENTER instruction?
bool SMPInstr::MDIsEnterInstr(void) const {
	return ((this->SMPcmd.itype >= MD_FIRST_ENTER_INSTR)
			&& (this->SMPcmd.itype <= MD_LAST_ENTER_INSTR));
}

// MACHINE DEPENDENT: Is instruction a LEAVE instruction?
bool SMPInstr::MDIsLeaveInstr(void) const {
	return ((this->SMPcmd.itype >= MD_FIRST_LEAVE_INSTR)
			&& (this->SMPcmd.itype <= MD_LAST_LEAVE_INSTR));
// MACHINE DEPENDENT: Is instruction a HALT instruction?
bool SMPInstr::MDIsHaltInstr(void) const {
	return (NN_hlt == this->SMPcmd.itype);
}

#define MD_FIRST_COND_MOVE_INSTR NN_cmova
#define MD_LAST_COND_MOVE_INSTR  NN_fcmovnu
// MACHINE DEPENDENT: Is instruction a conditional move?
bool SMPInstr::MDIsConditionalMoveInstr(void) const {
	return ((this->SMPcmd.itype >= MD_FIRST_COND_MOVE_INSTR)
			&& (this->SMPcmd.itype <= MD_LAST_COND_MOVE_INSTR));
// MACHINE DEPENDENT: Do opcode/operands definitely indicate signed arithmetic?
//  Generally, this is only true for certain variants of multiplication and division.
bool SMPInstr::MDIsSignedArithmetic(void) const {
	unsigned short opcode = this->SMPcmd.itype;
	if (NN_idiv == opcode)
		return true;
	if (NN_imul == opcode) {
		// If we discard the upper N bits of the multiplication result, then the
		//  lower N bits are the same for signed and unsigned multiplication, and
		//  gcc/g++ often use the IMUL opcode for both signed and unsigned multiplies
		//  when only N bits of result are retained. Therefore, the SIGNED nature of
		//  IMUL operands can only be inferred from the case in which 2N bits are kept.
		return (!(this->AreMultiplicationBitsDiscarded()));
	}
	else { // idiv and imul are only possible signed cases
		return false;
	}
} // end of SMPInstr::MDIsSignedArithmetic()

// MACHINE DEPENDENT: Is instruction a conditional jump based on an unsigned condition?
bool SMPInstr::MDIsUnsignedBranch(void) const {
	unsigned short opcode = this->SMPcmd.itype;
	return ((NN_ja == opcode) || (NN_jae == opcode) || (NN_jb == opcode) || (NN_jbe == opcode)
		|| (NN_jna == opcode) || (NN_jnae == opcode) || (NN_jnb == opcode) || (NN_jnbe == opcode));
}

// MACHINE DEPENDENT: Is instruction a conditional jump based on a signed condition?
bool SMPInstr::MDIsSignedBranch(void) const {
	unsigned short opcode = this->SMPcmd.itype;
	return ((NN_jg == opcode) || (NN_jge == opcode) || (NN_jl == opcode) || (NN_jle == opcode)
		|| (NN_jng == opcode) || (NN_jnge == opcode) || (NN_jnl == opcode) || (NN_jnle == opcode)
		|| (NN_js == opcode) || (NN_jns == opcode));
}

// MACHINE DEPENDENT: Is instruction a boolean set based on an unsigned condition?
bool SMPInstr::MDIsUnsignedSetValue(void) const {
	unsigned short opcode = this->SMPcmd.itype;
	return ((NN_seta == opcode) || (NN_setae == opcode) || (NN_setb == opcode) || (NN_setbe == opcode)
		|| (NN_setna == opcode) || (NN_setnae == opcode) || (NN_setnb == opcode) || (NN_setnbe == opcode));
}

// MACHINE DEPENDENT: Is instruction a boolean set based on a signed condition?
bool SMPInstr::MDIsSignedSetValue(void) const {
	unsigned short opcode = this->SMPcmd.itype;
	return ((NN_setg == opcode) || (NN_setge == opcode) || (NN_setl == opcode) || (NN_setle == opcode)
		|| (NN_setng == opcode) || (NN_setnge == opcode) || (NN_setnl == opcode) || (NN_setnle == opcode)
		|| (NN_sets == opcode) || (NN_setns == opcode));
}

// MACHINE DEPENDENT: Does instruction use a callee-saved register?
bool SMPInstr::MDUsesCalleeSavedReg(void) {
	set<DefOrUse, LessDefUse>::iterator CurrUse;
	for (CurrUse = this->GetFirstUse(); CurrUse != this->GetLastUse(); ++CurrUse) {
		op_t CurrOp = CurrUse->GetOp();
		if (CurrOp.is_reg(R_bp) || CurrOp.is_reg(R_si)
			|| CurrOp.is_reg(R_di) || CurrOp.is_reg(R_bx)) {
			return true;
		}
	}
	return false;
} // end of SMPInstr::MDUsesCalleeSavedReg()

// Is the instruction a register to register copy of a stack pointer or frame pointer
//  into a general purpose register (which mmStrata will now need to track as a stack 
//  relative pointer)?
bool SMPInstr::MDIsStackPointerCopy(bool UseFP) {
	// OptType 3 indicates a move instruction
	// The lea instruction can perform three operand arithmetic, e.g.
	//  lea ebx,[esp+12] is just ebx:=esp+12, so it is a stack pointer copy.
	if (((this->OptType == 3) || (NN_lea == this->SMPcmd.itype))
		&& (this->GetFirstDef()->GetOp().type == o_reg)
		&& (!(this->GetFirstDef()->GetOp().is_reg(R_sp)))
		&& (!(this->HasSourceMemoryOperand()))) { // reg to reg move
			if (this->GetFirstUse()->GetOp().is_reg(R_bp))
				// Move of base pointer EBP into a general register
				return true;
			else if ((this->GetFirstUse()->GetOp().is_reg(R_sp))
				&& !(this->GetFirstDef()->GetOp().is_reg(R_bp)))
				// Move of ESP into something besides a base pointer
				return true;
		}
		else if (this->GetFirstUse()->GetOp().is_reg(R_sp)) {
			// Move of ESP into a register; no base pointer used in this function
			return true;
		}
	}
	return false;
} // end of SMPInstr::MDIsStackPointerCopy()

// Determine if the instruction saves or restores a pointer into the stack frame.
// If it saves a stack pointer, set Save to true, set the StackDelta saved, and set
//   the operand that received the saved stack pointer into CopyOp. and return true.
// If it restores a stack pointer, set Save to false, set CopyOp to the operand that
//   held the value being restored, set RestoreOp to the stack pointer or frame pointer
//   register (whichever was restored), leave StackDelta alone for later computation
//   based on reaching definitions, and return true.
// For most instructions, no save or restore of a stack pointer, so return false.
bool SMPInstr::MDIsStackPtrSaveOrRestore(bool UseFP, bool &Save, sval_t &StackDelta, op_t &CopyOp, bool &Error) {
	bool StackPointerSaveOrRestore = false; // default unless we detect a save or restore of the stack or frame pointer
	size_t RTLCount = this->RTL.GetCount();
	size_t RTLIndex;
	sval_t FPDelta = BADADDR;
	op_t TempOp = InitOp;
	int BaseReg, IndexReg, CopyReg;
	ushort Scale;
	ea_t offset;
	SMPoperator CurrOper;
	bool LookUpStackDelta = false; // Get stack delta from reaching defs for TempOp
	sval_t DeltaAdjust = 0; // add to StackDelta after computing from reaching defs, e.g. lea esp,[ecx-4] get TempOp of ecx
	                        //  and DeltaAdjust of -4

	Save = false; // default unless we detect a stack pointer save
	Error = false;
	if (UseFP) {
		FPDelta = this->GetBlock()->GetFunc()->GetFramePtrStackDelta();
	}

	for (RTLIndex = 0; RTLIndex < RTLCount; ++RTLIndex) {
		bool FPRestore = false; // frame pointer is restored
		bool SPRestore = false; // stack pointer is restored
		SMPRegTransfer *CurrRT = this->RTL.GetRT(RTLIndex);
		op_t LeftOp = CurrRT->GetLeftOperand();
		if (LeftOp.is_reg(MD_STACK_POINTER_REG)) {
			SPRestore = true; // temporary; might just be a push or pop RTL, etc., in which case we will reset.
		}
		else if (UseFP && LeftOp.is_reg(MD_FRAME_POINTER_REG)) {
			FPRestore = true; // likewise temporary
		}
		Save = (!(SPRestore || FPRestore));

		// If we are assigning to the stack pointer reg or the frame pointer reg, we need to analyze the right
		//  hand side of the RTL to see if it is a stack/frame pointer value, and not a simple push, pop, etc.
		CurrOper = CurrRT->GetOperator();
		if (SMP_ASSIGN != CurrOper) {
			break; // not a regular RTL
		}
		if (!(CurrRT->HasRightSubTree())) {
			// Simple assignment to stack or frame pointer.
			op_t RightOp = CurrRT->GetRightOperand();
			if (RightOp.is_reg(MD_STACK_POINTER_REG)) {
				// Must be the move of stack pointer into frame pointer in function prologue.
				assert(FPRestore); // not really a restore, but we had temporarily marked it as such above
				Save = true;
				StackDelta = this->GetStackPtrOffset(); // FP := SP, so saved delta is just current delta
				CopyOp = RightOp;
				StackPointerSaveOrRestore = true;
				FPRestore = false;
				break;
			}
			else if ((o_reg <= RightOp.type) && (o_displ >= RightOp.type)) { // register or memory
				if (SPRestore || FPRestore) {
					// stack or frame pointer is being restored; leave Save=false and set other outgoing arguments.
					TempOp = RightOp;
					CopyOp = RightOp;
					StackPointerSaveOrRestore = true;
					LookUpStackDelta = true;
					break;
				}
				else if (RightOp.is_reg(MD_STACK_POINTER_REG)) {
					// Stack pointer reg is being saved.
					StackDelta = this->GetStackPtrOffset(); // LeftOp := SP, so saved delta is just current delta
					CopyOp = LeftOp;
					StackPointerSaveOrRestore = true;
					break;
				}
				else if (UseFP && RightOp.is_reg(MD_FRAME_POINTER_REG)) {
					// Frame pointer is being saved
					StackDelta = FPDelta;
					CopyOp = LeftOp;
					StackPointerSaveOrRestore = true;
					break;
				}
				else { // RightOp is register or non-stack-pointer memory expr; either might hold stack delta
					TempOp = RightOp;
					CopyOp = LeftOp;
					LookUpStackDelta = true; // See if RightOp is holding a stack delta
					break;
				}
			}
			else {
				SMP_msg("ERROR: Invalid operand type for assignment to stack or frame pointer at %x\n", this->GetAddr());
				StackPointerSaveOrRestore = false;
				break;
			}
		}
		else { // we have a right subtree in the CurrRT
			SMPRegTransfer *RightRT = CurrRT->GetRightTree();
			// In order to have a right subtree, we must have something like:
			//   lea esp,[ecx-4]  which produces the RTL: esp := ecx - 4
			// We should consider any other RTL structure besides a basic addition or
			//  subtraction on the right subtree to be invalid.
			CurrOper = RightRT->GetOperator();
			if ((SMP_ADD == CurrOper) || (SMP_SUBTRACT == CurrOper)) {
				op_t RightLeftOp = RightRT->GetLeftOperand();
				if (o_reg == RightLeftOp.type) {
					if (RightRT->HasRightSubTree()) {
						// Complex RTL such as lea esp,[ebx+ecx*4] ; cannot analyze
						StackPointerSaveOrRestore = false;
					}
					else {
						op_t RightRightOp = RightRT->GetRightOperand();
						if (o_imm != RightRightOp.type) {
							// Complex RTL such as lea esp,[ebx+ecx] ; cannot analyze
							StackPointerSaveOrRestore = false;
						}
						else {
							TempOp = RightLeftOp;
							DeltaAdjust = (sval_t) RightRightOp.value;
							if (SMP_SUBTRACT == CurrOper) {
								// Negate the stack delta adjustment, e.g. lea esp,[ecx-4] needs DeltaAdjust of -4, not 4.
								DeltaAdjust = (0 - DeltaAdjust);
							}
							LookUpStackDelta = true;
						}
					}
				}
				else { // weird RTL; LeftOp := (MemoryOp OPER ???)
					StackPointerSaveOrRestore = false;
				}
			}
			else { // not ADD or SUBTRACT
				StackPointerSaveOrRestore = false;
			}
		}
	} // end for all RTs in the RTL

	if (LookUpStackDelta) {
		bool StackAccess = false;
		// We need to set StackDelta based on the reaching defs for TempOp
		// A reg is probably a general register, but could have lea ebx,[esp+4] so it could be stack or frame pointer.
		if (TempOp.is_reg(MD_STACK_POINTER_REG)) {
			StackDelta = this->GetStackPtrOffset();
			StackDelta += DeltaAdjust;
			LookUpStackDelta = false; // just got it; no need for reaching defs
		}
		else if (UseFP && TempOp.is_reg(MD_FRAME_POINTER_REG)) {
			StackDelta = FPDelta;
			StackDelta += DeltaAdjust;
			LookUpStackDelta = false; // just got it; no need for reaching defs
		}
		else if (o_reg == TempOp.type) { // general reg, not frame or stack pointer reg
			CopyReg = TempOp.reg;
		}
		else {
			MDExtractAddressFields(TempOp, BaseReg, IndexReg, Scale, offset);
			CopyReg = BaseReg;
			bool IndexedAccess = ((R_none != BaseReg) && (R_none != IndexReg));
			if (IndexedAccess) {
				StackPointerSaveOrRestore = false;  // Cannot analyze indexed accesses into the stack
			}
			else if (MDIsStackPtrReg(BaseReg, UseFP)) {
				StackAccess = true;
			}
			else {
				// memory expr that is not stack or frame pointer
				DeltaAdjust = (sval_t) TempOp.addr; // get normalized delta from addr field
			}
		}

		if (StackPointerSaveOrRestore && LookUpStackDelta) {
			op_t FindOp = InitOp;
			if (StackAccess) {
				FindOp = TempOp;
			}
			else {
				FindOp.type = o_reg;
				FindOp.reg = CopyReg;
			}
			if (this->GetBlock()->GetFunc()->IsInStackPtrCopySet(FindOp)) {
				// Screened out time wasters that are not in copy set; now,
				//  look up reaching defs.
				// We need to find out which are the reaching definitions for the FindOp at the current InstAddr.
				this->GetBlock()->GetFunc()->ComputeTempReachingDefs(FindOp, this->GetAddr());
				this->GetBlock()->GetFunc()->ComputeTempStackDeltaReachesList(FindOp);
				// See if TempStackDeltaReachesList has a consistent delta value.
				StackPointerSaveOrRestore = this->GetBlock()->GetFunc()->FindReachingStackDelta(StackDelta); // consistent SavedDelta value across entire list
				StackDelta += DeltaAdjust;
			}
		}
	} // end if (LookupStackDelta)

	if (!StackPointerSaveOrRestore && !Save) {
		// Any restore that could not be analyzed is an error.
		Error = true;
	}

	return StackPointerSaveOrRestore;
} // end of SMPInstr::MDIsStackPtrSaveOrRestore()

// If call instruction is to malloc(), set the DEF register EAX type to
//  HEAPPTR and return true.
bool SMPInstr::MDFindMallocCall(op_t TargetOp) {
	bool changed = false;
	func_t *TargetFunc = get_func(TargetOp.addr);
	if (TargetFunc) {
		char FuncName[MAXSTR];
		get_func_name(TargetFunc->startEA, FuncName, sizeof(FuncName) - 1);
		if (0 == strcmp("malloc", FuncName)) {
			// NOTE: Some compilers might call it __malloc ; make this more robust !!!
#if SMP_VERBOSE_FIND_POINTERS
			SMP_msg("Found call to malloc at %x\n", this->addr);
#endif
			op_t SearchOp = InitOp;
			SearchOp.type = o_reg;
			SearchOp.reg = R_ax;
			set<DefOrUse, LessDefUse>::iterator EAXDEF;
			EAXDEF = this->SetDefType(SearchOp, HEAPPTR);
			int SSANum = EAXDEF->GetSSANum();
			changed = true;
			if (this->BasicBlock->IsLocalName(SearchOp)) {
				(void) this->BasicBlock->PropagateLocalDefType(SearchOp, HEAPPTR,
						this->GetAddr(), SSANum, false);
			}
			else { // global name
				this->BasicBlock->GetFunc()->ResetProcessedBlocks(); // set Processed to false
				(void) this->BasicBlock->PropagateGlobalDefType(SearchOp, HEAPPTR,
						SSANum, false);
			}
		} // end if "malloc"
	} // end if (TargetFunc)
	return changed;
} // end of SMPInstr::MDFindMallocCall()

// Is instruction a branch (conditional or unconditional) to a
//  code target that is not in the current chunk?
bool SMPInstr::IsBranchToFarChunk(void) {
	if (this->IsFarBranchComputed()) { // answer is cached
		return this->IsBranchesToFarChunk();
	func_t *CurrChunk = get_fchunk(this->address);
	bool FarBranch = false;
	if ((JUMP | COND_BRANCH) & this->GetDataFlowType()) {
		// Instruction is a direct branch, conditional or unconditional
		if (this->NumUses() > 0) {
			set<DefOrUse, LessDefUse>::iterator CurrUse;
			for (CurrUse = this->GetFirstUse(); CurrUse != this->GetLastUse(); ++CurrUse) {
				op_t JumpTarget = CurrUse->GetOp();
				if ((o_near == JumpTarget.type) || (o_far == JumpTarget.type)) {
					// Branches to a code address
					// stdclib sometimes has jumps to zero and calls to zero. These are dead code.
					if (0 != JumpTarget.addr) {
						func_t *TargetChunk = get_fchunk(JumpTarget.addr);
						// Is target address within the same chunk as the branch?
						FarBranch = (NULL == TargetChunk) || (CurrChunk->startEA != TargetChunk->startEA);
						if (FarBranch) {
							this->FarBranchTarget = JumpTarget.addr;
						}
	if (FarBranch) {
		this->SetBranchesToFarChunk();
	}
	this->SetFarBranchComputed();
	return FarBranch;
} // end of SMPInstr::IsBranchToFarChunk()

set<DefOrUse, LessDefUse>::iterator SMPInstr::SetUseSSA(op_t CurrOp, int SSASub) {
	return this->Uses.SetSSANum(CurrOp, SSASub);
};

set<DefOrUse, LessDefUse>::iterator SMPInstr::SetDefSSA(op_t CurrOp, int SSASub) {
	return this->Defs.SetSSANum(CurrOp, SSASub);
};

set<DefOrUse, LessDefUse>::iterator SMPInstr::SetUseType(op_t CurrOp, SMPOperandType CurrType) {
	return this->Uses.SetType(CurrOp, CurrType, this);
};

set<DefOrUse, LessDefUse>::iterator SMPInstr::SetDefType(op_t CurrOp, SMPOperandType CurrType) {
	return this->Defs.SetType(CurrOp, CurrType, this);
set<DefOrUse, LessDefUse>::iterator SMPInstr::SetDefMetadata(op_t CurrOp, SMPMetadataType Status) {
	return this->Defs.SetMetadata(CurrOp, Status);
};

set<DefOrUse, LessDefUse>::iterator SMPInstr::SetDefIndWrite(op_t CurrOp, bool IndWriteFlag) {
	return this->Defs.SetIndWrite(CurrOp, IndWriteFlag);
};

set<DefOrUse, LessDefUse>::iterator SMPInstr::SetUseNoTruncate(op_t CurrOp, bool NoTruncFlag) {
	return this->Uses.SetNoTruncation(CurrOp, NoTruncFlag);
};

set<DefOrUse, LessDefUse>::iterator SMPInstr::SetDefNoOverflow(op_t DefOp, bool NoOverflowFlag) {
	return this->Defs.SetNoOverflow(DefOp, NoOverflowFlag);
};

// Analyze the instruction and its operands.
void SMPInstr::Analyze(void) {
	bool DebugFlag = false;
	if (0x8049b00 == this->address) {
		// Setting up breakpoint line.
		DebugFlag = true;
	}

	// Fill cmd structure with disassembly of instr
	if (!SMPGetCmd(this->address, this->SMPcmd, this->features))

	// Record what type of instruction this is, simplified for the needs
	//  of data flow and type analysis.
	this->type = DFACategory[this->SMPcmd.itype];
	// Record optimization category.
	this->OptType = OptCategory[this->SMPcmd.itype];
	if ((NN_int == this->SMPcmd.itype) || (NN_into == this->SMPcmd.itype) || (NN_int3 == this->SMPcmd.itype)) {
		this->SetInterrupt();
	}
	else {
		this->ResetInterrupt();
	}
clc5q's avatar
clc5q committed
	// See if instruction is an ASM idiom for clearing a register.
	if (NN_xor == this->SMPcmd.itype) {
		ushort FirstReg;
		if (o_reg == this->SMPcmd.Operands[0].type) {
			FirstReg = this->SMPcmd.Operands[0].reg;
			if (this->SMPcmd.Operands[1].is_reg(FirstReg))
				this->SetRegClearIdiom();
	// See if instruction is simple nop or ASM idiom for nop.
	if (this->MDIsNop()) {
		this->SetNop();
	}
	// Build the DEF and USE lists for the instruction.
	this->BuildSMPDefUseLists();

	// Determine whether the instruction is a jump target by looking
	//  at its cross references and seeing if it has "TO" code xrefs.
	SMP_xref_t xrefs;
	for (bool ok = xrefs.SMP_first_to(this->address, XREF_FAR); ok; ok = xrefs.SMP_next_to()) {
		if ((xrefs.GetFrom() != 0) && (xrefs.GetIscode())) {
			this->SetJumpTarget();
	// If instruction is a call or indirect call, see if a call target has been recorded
	//  by IDA Pro.
	if (this->GetDataFlowType() == INDIR_CALL) {
		for (bool ok = xrefs.SMP_first_from(this->address, XREF_ALL);
			ok = xrefs.SMP_next_from()) {
			if ((xrefs.GetTo() != 0) && (xrefs.GetIscode())) {
				// Found a code target, with its address in CurrXrefs.to
				if (xrefs.GetTo() == (this->address + this->GetCmd().size)) {
					// A call instruction will have two targets: the fall through to the
					//  next instruction, and the called function. We want to find
					//  the called function.
					continue;
				}
				// We found a target, not the fall-through.
				this->CallTarget = xrefs.GetTo();
				SMP_msg("Found indirect call target %x at %x\n",
					xrefs.GetTo(), this->address);
				break;
			}
		} // end for all code xrefs
		if (BADADDR == this->CallTarget) {
			SMP_msg("WARNING: Did not find indirect call target at %x\n",
				this->address);
		}
	} // end if INDIR_CALL
	else if (this->GetDataFlowType() == CALL) {
		set<DefOrUse, LessDefUse>::iterator CurrUse;
		for (CurrUse = this->GetFirstUse(); CurrUse != this->GetLastUse(); ++CurrUse) {
			optype_t OpType = CurrUse->GetOp().type;
			if ((OpType == o_near) || (OpType == o_far)) {
				this->CallTarget = CurrUse->GetOp().addr;
			}
		}
		if (BADADDR == this->CallTarget) {
			SMP_msg("ERROR: Target not found for direct call at %x\n", this->address);
		SMP_msg("Analyzed debug instruction at %x\n", this->address);
	return;
} // end of SMPInstr::Analyze()

// Analyze the floating point NOP marker instruction at the top of the function.
void SMPInstr::AnalyzeMarker(void) {
	// Fill member variable SMPcmd structure with disassembly of instr
	(void) memset(&(this->SMPcmd), 0, sizeof(this->SMPcmd));
	this->SMPcmd.itype = NN_fnop;
	this->SMPcmd.size = 1;
	this->SMPcmd.ea = this->address;
	// Set the instr disassembly text.
	DisAsmText.SetMarkerInstText(this->GetAddr());

	// Record what type of instruction this is, simplified for the needs
	//  of data flow and type analysis.
	this->type = DFACategory[this->SMPcmd.itype];
	// Record optimization category.
	this->OptType = OptCategory[this->SMPcmd.itype];

	return;
} // end of SMPInstr::AnalyzeMarker()

// Detect oddities of call instructions, such as pseudo-calls that are
//  actually jumps within a function
void SMPInstr::AnalyzeCallInst(ea_t FirstFuncAddr, ea_t LastFuncAddr) {
	if (BADADDR != this->CallTarget) {
		if ((this->CallTarget > FirstFuncAddr)
				&& (this->CallTarget <= LastFuncAddr)) {
			this->SetCallUsedAsJump();
		}
		else {
			this->ResetCallUsedAsJump();
		}
		if (this->CallTarget == FirstFuncAddr) {
			this->SetDirectRecursiveCall();
		}
		else {
			this->ResetDirectRecursiveCall();
		}
		if (this->IsCallUsedAsJump())
sval_t SMPInstr::AnalyzeStackPointerDelta(sval_t IncomingDelta, sval_t PreAllocDelta) {
	uint16 InstType = this->SMPcmd.itype;
	sval_t InstDelta = StackAlteration[InstType];
	SMPitype FlowType = this->GetDataFlowType();
	if (this->IsCallUsedAsJump() || this->MDIsInterruptCall() || this->IsCondTailCall()) {
		// Call is used within function as a jump. Happens when setting up
		//  thunk offsets, for example; OR, call is an interrupt call, in which
		//  the interrupt return cleans up the stack, leaving a delta of zero, but
		//  we do not have the system call code to analyze, OR, the call is a conditional
		//  jump to another function (conditional tail call), in which case the current
		//  function must have a return statement to fall into which will clean up the
		//  only thing left on the stack (the return address) and the conditional jump
		//  has no effect on the stack pointer.
		; // leave InstDelta equal to negative or zero value from StackAlterationTable[]
	}
	else if ((CALL == FlowType) || (INDIR_CALL == FlowType) || TailCall) {
		// A real call instruction, which pushes a return address on the stack,
		//  not a call used as a branch within the function. A return instruction
		//  will usually cancel out the stack push that is implicit in the call, which 
		//  means that the function will have a net stack ptr delta of +4, which will
		//  cancel out the -4 value of the call instruction and set the delta to zero.
		//  However, this is not true in all cases, so we get the net stack ptr delta
		//  directly from the called function unless it is an unresolved indirect call,
		//  in which case we assume +4. !!!!****!!!! In the future, we could analyze
		//  the code around an unresolved indirect call to see if it seems to be
		//  removing items left on the stack by the callee.
		// SPECIAL CASE: A jump used as a tail call will have a stack ptr effect that is equal
		//  to the net stack ptr effect of its target function, usually +4, whereas a jump
		//  would otherwise have a net stack ptr effect of 0.
		ea_t CalledFuncAddr = this->GetCallTarget();
		if ((BADADDR == CalledFuncAddr) || (0 == CalledFuncAddr)) {
			InstDelta = 0;
		}
		else { // We have a call target
			SMPFunction *CalleeFunc = this->GetBlock()->GetFunc()->GetProg()->GetFuncFromAddr(CalledFuncAddr);
			sval_t AdjustmentDelta;
			if (CalleeFunc) {
				if (!CalleeFunc->HasSTARSStackPtrAnalysisCompleted()) {
					// Phase ordering issue in the call graph. A mutually recursive clique of functions has to
					//  be broken by starting processing somewhere, and all callees cannot be processed before
					//  we start. If we got our stack down to zero and then made a tail call, then we have to assume
					//  that the callee will use our return address, so we assume the default stack delta. If not a
					//  tail call, we ask our function to see if the information is available from IDA Pro analyses,
					//  or if it can be inferred from the fact that the call is followed by a stack adjustment.
					SMP_msg("WARNING: Callee stack ptr analysis not yet performed at inst %x ; normal delta assumed\n", this->GetAddr());
					if (TailCall) {
						InstDelta = CALLING_CONVENTION_DEFAULT_FUNCTION_STACK_DELTA;
					}
					else {
						AdjustmentDelta = this->GetBlock()->GetFunc()->GetStackDeltaForCallee(this->GetAddr());
						InstDelta += AdjustmentDelta;
					}
				}
				else if (!CalleeFunc->StackPtrAnalysisSucceeded()) {
					// Callee analyses were done, but they failed. In order to proceed, we have to assume
					//  the same situation as we just did in the case where analyses have not been performed.
					SMP_msg("WARNING: Callee stack ptr analysis failed at inst %x ; normal delta assumed\n", this->GetAddr());
					if (TailCall) {
						InstDelta = CALLING_CONVENTION_DEFAULT_FUNCTION_STACK_DELTA;
					}
					else {
						AdjustmentDelta = this->GetBlock()->GetFunc()->GetStackDeltaForCallee(this->GetAddr());
						InstDelta += AdjustmentDelta;
					}
				}
				else {
					// Callee's analyses have succeeded, so get delta straight from callee.
					InstDelta += CalleeFunc->GetNetStackPtrDelta();
				}
			}
			else {
#if 0
				SMP_msg("ERROR: SMPInstr::AnalyzeStackPointerDelta failed to find func at %x in inst %x\n",
					CalledFuncAddr, this->GetAddr());
				InstDelta = SMP_STACK_DELTA_ERROR_CODE;
#else
				SMP_msg("WARNING: SMPInstr::AnalyzeStackPointerDelta failed to find func at %x in inst %x\n",
					CalledFuncAddr, this->GetAddr());
				if (TailCall) {
					InstDelta = CALLING_CONVENTION_DEFAULT_FUNCTION_STACK_DELTA;
				}
				else {
					InstDelta = 0;
				}
#endif
			}
		}
	} // end CALL or INDIR_CALL or TailCall case
	else if (1 == InstDelta) { 
		// value of 1 is trigger to investigate the RTL for the 
		//  true value, which cannot be found simply by table lookup
		// In the special case of an x86 LEAVE instruction, the effect
		//  on the stack pointer is to deallocate the local frame size,
		//  plus pop the saved frame pointer into EBP. Helper functions
		//  need to know whether to look for this special case.
		bool IsLeaveInstr = this->MDIsLeaveInstr();
		InstDelta = this->RTL.TotalStackPointerAlteration(IsLeaveInstr, IncomingDelta, PreAllocDelta);
} // end of SMPInstr::AnalyzeStackPointerDelta()

// Total the stack adjustment bytes, as happens after a call to a function that leaves
//  outgoing args on the stack or swallows incoming args from the stack.
sval_t SMPInstr::FindStackAdjustment(void) {
	uint16 InstType = this->SMPcmd.itype;
	sval_t InstDelta = StackAlteration[InstType];

	if (1 == InstDelta) { 
		// value of 1 is trigger to investigate the RTL for the 
		//  true value, which cannot be found simply by table lookup
		// In the special case of an x86 LEAVE instruction, the effect
		//  on the stack pointer is to deallocate the local frame size,
		//  plus pop the saved frame pointer into EBP. Helper functions
		//  need to know whether to look for this special case.
		bool IsLeaveInstr = this->MDIsLeaveInstr();
		if (!IsLeaveInstr) {
			InstDelta = this->RTL.TotalStackPointerAlteration(IsLeaveInstr, 0, 0);
		}
		else {
			InstDelta = 0; // LEAVE is not the kind of instr we are looking for
		}
	}
	return InstDelta;
} // end of SMPInstr::FindStackAdjustment()

// Normalize stack operands to have a displacement from the stack pointer value on entry to the function,
//  rather than the current stack pointer value.
// UseFP indicates we are usign a frame pointer in the function.
// FPDelta holds the stack delta (normalized) for the frame pointer.
// DefOp comes in with the operand to be normalized, and contains the normalized operand upon return.
// Return true if operand is a register or stack location, false otherwise (true => include in data flow analysis sets and SSA.)
bool SMPInstr::MDComputeNormalizedDataFlowOp(bool UseFP, sval_t FPDelta, op_t &DefOp) {
	if (o_reg == DefOp.type) {
		return true;
	}
	else if (MDIsStackAccessOpnd(DefOp, UseFP)) {
		int SignedOffset = (int) DefOp.addr;
		sval_t NormalizedDelta;
		if (DefOp.reg == MD_FRAME_POINTER_REG) {
			// If FPDelta is -4 and SignedOffset is +8, then we have [ebp+8] as DefOp, and this
			//  is equivalent to [esp+4] where esp has its entry value, i.e. this would be the first incoming
			//  argument. If SignedOffset is -12, we have [ebp-12] as DefOp, and this is [esp-16] when
			//  normalized to the entry point value of the stack pointer. In both cases, we can see that the
			//  normalized stack delta is just FPDelta+SignedOffset.
			NormalizedDelta = FPDelta + (sval_t) SignedOffset;
			// Now, we simply convert the memory operand from EBP to ESP and replace the SignedOffset with the
			//  NormalizedDelta just computed.
			DefOp.reg = MD_STACK_POINTER_REG;
		}
		else {
			assert(DefOp.reg == MD_STACK_POINTER_REG);
			// We only need to adjust the offset to reflect the change in the stack pointer since the function
			//  was entered, e.g. [esp+4] is normalized to [esp-28] if the current esp value is 32 less than it
			//  was upon function entry. We get the value "-32" in that case from a member variable.
			NormalizedDelta = this->GetStackPtrOffset() + (sval_t) SignedOffset;
		}
		DefOp.addr = (ea_t) NormalizedDelta; // common to frame and stack pointer cases
		return true;
	}
	else {
		return false;
	}
} // end of SMPInstr::MDComputeNormalizedDataFlowOp()

// Normalize stack operands in all DEFs and USEs to have stack deltas relative to the function entry stack pointer.
// Return true if any stack DEFs or USEs were normalized.
bool SMPInstr::MDNormalizeStackOps(bool UseFP, sval_t FPDelta, bool Recomputing) {
	bool StackOpFound = false;
	bool OpNormalized;
	set<DefOrUse, LessDefUse>::iterator DefIter, UseIter;
	list<pair<set<DefOrUse, LessDefUse>::iterator, op_t> > DefWorkList, UseWorkList;
	list<pair<set<DefOrUse, LessDefUse>::iterator, op_t> >::iterator WorkIter;
	op_t OldOp, NewOp;
	
	// Find all the DEFs that need changing, and put their iterators into a list.
	// Normalizing stack ops could change their sort order, hence we could skip over
	//  a DEF in the set by erasing a DEF and reinserting a normalized DEF, so we
	//  make all the changes after we iterate through the DEFS set.
	for (DefIter = this->GetFirstDef(); DefIter != this->GetLastDef(); ++DefIter) {
		OldOp = DefIter->GetOp();
		NewOp = OldOp;
		if (o_reg != NewOp.type) {
			OpNormalized = this->MDComputeNormalizedDataFlowOp(UseFP, FPDelta, NewOp);
			if (OpNormalized) {
				StackOpFound = true;
				pair<set<DefOrUse, LessDefUse>::iterator, op_t> DefItem(DefIter, NewOp);
				DefWorkList.push_back(DefItem);
			}
		}
	}
	// Now go through the DEF worklist and change stack operands to normalized stack operands.
	for (WorkIter = DefWorkList.begin(); WorkIter != DefWorkList.end(); ++WorkIter) {
		DefIter = WorkIter->first;
		DefIter = this->Defs.SetOp(DefIter, NewOp);
	}

	// Find all USEs that need changing, and build a second work list.
	for (UseIter = this->GetFirstUse(); UseIter != this->GetLastUse(); ++UseIter) {
		OldOp = UseIter->GetOp();
		NewOp = OldOp;
		if (o_reg != NewOp.type) {
			OpNormalized = this->MDComputeNormalizedDataFlowOp(UseFP, FPDelta, NewOp);
			if (OpNormalized) {
				StackOpFound = true;
				pair<set<DefOrUse, LessDefUse>::iterator, op_t> UseItem(UseIter, NewOp);
				UseWorkList.push_back(UseItem);
			}
		}
	}

	// Now go through the USE worklist and change stack operands to normalized stack operands.
	for (WorkIter = UseWorkList.begin(); WorkIter != UseWorkList.end(); ++WorkIter) {
		UseIter = WorkIter->first;
		UseIter = this->Uses.SetOp(UseIter, NewOp);
	}

	return StackOpFound;
} // end of SMPInstr::MDNormalizeStackOps()

// Find USE-not-DEF operand that is not the flags register.
op_t SMPInstr::GetSourceOnlyOperand(void) {
	size_t OpNum;
	for (OpNum = 0; OpNum < UA_MAXOP; ++OpNum) {
		if (this->features & DefMacros[OpNum]) { // DEF
			;
		}
		else if (this->features & UseMacros[OpNum]) { // USE
			op_t CurrOp = this->SMPcmd.Operands[OpNum];
			if (!(CurrOp.is_reg(X86_FLAGS_REG))) {
				return CurrOp;
			}
		}
	}
clc5q's avatar
clc5q committed
	// It is expected that increment, decrement, and floating point stores
	//  will not have a USE-only operand. Increment and decrement have an
	//  operand that is both USEd and DEFed, while the floating point stack
	//  registers are implicit in most floating point opcodes. Also, exchange
	//  and exchange-and-add instructions have multiple DEF-and-USE operands.
	int TypeGroup = SMPTypeCategory[this->SMPcmd.itype];
	if ((TypeGroup != 2) && (TypeGroup != 4) && (TypeGroup != 9) && (TypeGroup != 12)
		&& (TypeGroup != 13)) {
		SMP_msg("ERROR: Could not find source only operand at %x in %s\n",
			this->address, DisAsmText.GetDisAsm(this->GetAddr()));
clc5q's avatar
clc5q committed
	}
} // end of SMPInstr::GetSourceOnlyOperand()

// Should apparent memory operands be ignored? e.g. lea opcode on x86
bool SMPInstr::MDIgnoreMemOps(void) {
	bool leaInst = (NN_lea == this->SMPcmd.itype);
	return leaInst;
}

// Find memory DEFs and USEs, store in DEFMemOp and USEMemOp
void SMPInstr::FindMemOps(void) {
	size_t OpNum;

	if (!(this->MDIgnoreMemOps())) {
		for (OpNum = 0; OpNum < UA_MAXOP; ++OpNum) {
			op_t TempOp = this->SMPcmd.Operands[OpNum];
			if ((TempOp.type >= o_mem) && (TempOp.type <= o_displ)) { // memory
				if (this->features & DefMacros[OpNum]) { // DEF
					if (this->DEFMemOp.type == o_void) { // only save first mem DEF
						this->DEFMemOp = TempOp;
					}
				}
				if (this->features & UseMacros[OpNum]) { // USE
					if (this->USEMemOp.type == o_void) { // only save first mem USE
						this->USEMemOp = TempOp;
					}
				}
			}
		} // end for (OpNum = 0; ...)
	}
	this->SetMemOpsFound();
	return;
} // end of SMPInstr::FindMemOps()

// Fill the Defs and Uses private data members.
void SMPInstr::BuildSMPDefUseLists(void) {
	size_t OpNum;
	bool DebugFlag = (0x8049b00 == this->GetAddr());
	bool WidthDoubler = this->MDDoublesWidth();
	this->Defs.clear();
	this->Uses.clear();

	// Start with the Defs.
	for (OpNum = 0; OpNum < UA_MAXOP; ++OpNum) {
		if (this->features & DefMacros[OpNum]) { // DEF
			op_t TempOp = this->SMPcmd.Operands[OpNum];
			if (WidthDoubler) {
				// Opcodes that sign-extend a byte to a word, or a word to a dword, 
				//  have only one operand. It is implicit, and it is the shorter USE.
				//  That means the DEF will have the same width as the USE, e.g. if
				//  we are sign-extending AX to EAX, the USE and DEF both be AX without
				//  a special fix. We fix this problem with the DEF operand now.
				if (TempOp.dtyp == dt_byte) {
					TempOp.dtyp = dt_word;
					TempOp.reg = MDCanonicalizeSubReg(TempOp.reg);
				}
				else if (TempOp.dtyp == dt_word) {
					TempOp.dtyp = dt_dword;
					TempOp.reg = MDCanonicalizeSubReg(TempOp.reg);
				}
				else if (TempOp.dtyp == dt_dword) {
					TempOp.dtyp = dt_qword;
				}
				else {
					SMP_msg("ERROR: Instruction operand %zu not 1,2, or 4 bytes at %x dtyp: %d\n", 
			if (MDKnownOperandType(TempOp)) {
				if (DebugFlag) {
					PrintOperand(TempOp);
				if (o_reg == TempOp.type) {
					// We want to map AH, AL, and AX to EAX, etc. throughout our data flow
					//  analysis and type inference systems.
					TempOp.reg = MDCanonicalizeSubReg(TempOp.reg);
				}
		}
	} // end for (OpNum = 0; ...)

	if (this->IsRegClearIdiom()) {
clc5q's avatar
clc5q committed
		// Something like xor eax,eax clears eax but does not really
		//  use eax. It is the same as mov eax,0 and we don't want to
		//  extend the prior def-use chain for eax to this instruction
		//  by treating the instruction as xor eax,eax. Instead, we
		//  build the DEF and USE lists and RTL as if it were mov eax,0.
clc5q's avatar
clc5q committed
		ImmOp.type = o_imm;
clc5q's avatar
clc5q committed
		this->Uses.SetRef(ImmOp, NUMERIC);
		return;
	}

	// Now, do the Uses. Uses have special case operations, because