Newer
Older
clc5q
committed
// Does instruction subtract an immediate value that is often used in ASCII computations,
// such as the ASCII code for '0', 'a', or 'A' ?
bool SMPInstr::SubtractsImmedASCII(void) {
op_t SubtrahendOp = this->GetUseOnlyAddSubOp();
return (this->MDIsMaybeBenignUnderflowOpcode() && (o_imm == SubtrahendOp.type)
&& (('0' == SubtrahendOp.value) || ('a' == SubtrahendOp.value)
|| ('A' == SubtrahendOp.value)));
2011
2012
2013
2014
2015
2016
2017
2018
2019
2020
2021
2022
2023
2024
2025
2026
2027
2028
2029
2030
2031
2032
2033
2034
// Does instruction compare a location to an immediate value that is often used in ASCII computations,
// such as the ASCII code for '0' or '9', 'a' or 'A', 'z' or 'Z', or carriage return?
#define SMP_ASCII_CARRIAGE_RETURN 13
bool SMPInstr::MDComparesImmedASCII(void) {
bool ComparesToASCII = false;
if (this->SMPcmd.itype == NN_cmp) {
SMPRegTransfer *CurrRT = this->RTL.GetRT(0);
if ((CurrRT != NULL) && (CurrRT->HasRightSubTree())) {
CurrRT = CurrRT->GetRightTree();
if (!(CurrRT->HasRightSubTree())) {
op_t LeftOp = CurrRT->GetLeftOperand();
op_t RightOp = CurrRT->GetRightOperand();
if ((o_imm == RightOp.type) && ((SMP_ASCII_CARRIAGE_RETURN == RightOp.value)
|| ('0' == RightOp.value) || ('9' == RightOp.value)
|| ('a' == RightOp.value) || ('z' == RightOp.value)
|| ('A' == RightOp.value) || ('Z' == RightOp.value))) {
ComparesToASCII = true;
}
}
}
}
return ComparesToASCII;
} // end of SMPInstr::MDComparesImmedASCII()
// Multiply by large constant; overflow is probably intentional.
#define STARS_LARGE_UNSIGNED_MUL_CONSTANT_THRESHOLD 0x20000000
#define STARS_LARGE_SIGNED_MUL_CONSTANT_THRESHOLD 0x10000000
#define STARS_SMALL_SIGNED_MUL_CONSTANT_THRESHOLD ((int)(-STARS_LARGE_SIGNED_MUL_CONSTANT_THRESHOLD))
bool SMPInstr::IsMultiplyByLargeConstant(uval_t &ConstValue, unsigned short SignMask) {
bool LargeConstFound = false;
if (this->MDIsMultiply() && (SignMask != FG_MASK_INCONSISTENT_SIGN) && (SignMask != 0)) {
set<DefOrUse, LessDefUse>::iterator UseIter;
for (UseIter = this->GetFirstUse(); UseIter != this->GetLastUse(); ++UseIter) {
if (this->FindConstantValue(UseIter, ConstValue)) {
if (FG_MASK_UNSIGNED == SignMask) {
if (((uval_t) STARS_LARGE_UNSIGNED_MUL_CONSTANT_THRESHOLD) <= ConstValue) {
LargeConstFound = true;
break;
}
}
else if (FG_MASK_SIGNED == SignMask) {
int SignedConstValue = (int) ConstValue;
if ( (((int) STARS_LARGE_SIGNED_MUL_CONSTANT_THRESHOLD) <= SignedConstValue)
|| (((int) STARS_SMALL_SIGNED_MUL_CONSTANT_THRESHOLD) >= SignedConstValue)) {
LargeConstFound = true;
break;
}
}
}
}
}
return LargeConstFound;
} // end of SMPInstr::IsMultiplyByLargeConstant()
// Subtraction of large constant; underflow is probably intentional.
#define STARS_LARGE_UNSIGNED_SUB_CONSTANT_THRESHOLD 0x80000000
#define STARS_LARGE_SIGNED_SUB_CONSTANT_THRESHOLD 0x40000000
#define STARS_SMALL_SIGNED_SUB_CONSTANT_THRESHOLD ((int)(-STARS_LARGE_SIGNED_SUB_CONSTANT_THRESHOLD))
bool SMPInstr::IsSubtractionOfLargeConstant(uval_t &ConstValue, unsigned short SignMask) {
bool LargeConstFound = false;
if (this->MDIsUnderflowingOpcode() && (SignMask != FG_MASK_INCONSISTENT_SIGN) && (SignMask != 0)) {
if (o_void != SubtrahendOp.type) {
set<DefOrUse, LessDefUse>::iterator UseIter = this->FindUse(SubtrahendOp);
if (this->FindConstantValue(UseIter, ConstValue)) {
if (FG_MASK_UNSIGNED == SignMask) {
if (((uval_t) STARS_LARGE_UNSIGNED_SUB_CONSTANT_THRESHOLD) <= ConstValue) {
LargeConstFound = true;
}
}
else if (FG_MASK_SIGNED == SignMask) {
int SignedConstValue = (int) ConstValue;
if ( (((int) STARS_LARGE_SIGNED_SUB_CONSTANT_THRESHOLD) <= SignedConstValue)
|| (((int) STARS_SMALL_SIGNED_SUB_CONSTANT_THRESHOLD) >= SignedConstValue)) {
LargeConstFound = true;
}
}
}
}
}
return LargeConstFound;
} // end of SMPInstr::IsSubtractionOfLargeConstant()
// Subtraction of large constant; underflow is probably intentional.
#define STARS_LARGE_UNSIGNED_ADD_CONSTANT_THRESHOLD 0x80000000
#define STARS_LARGE_SIGNED_ADD_CONSTANT_THRESHOLD 0x40000000
#define STARS_SMALL_SIGNED_ADD_CONSTANT_THRESHOLD ((int)(-STARS_LARGE_SIGNED_ADD_CONSTANT_THRESHOLD))
bool SMPInstr::IsAdditionOfLargeConstant(uval_t &ConstValue, unsigned short SignMask) {
bool LargeConstFound = false;
if (this->MDIsOverflowingOpcode() && (SignMask != FG_MASK_INCONSISTENT_SIGN) && (SignMask != 0)) {
2107
2108
2109
2110
2111
2112
2113
2114
2115
2116
2117
2118
2119
2120
2121
2122
2123
2124
2125
2126
2127
2128
if (o_void != AddendOp.type) {
set<DefOrUse, LessDefUse>::iterator UseIter = this->FindUse(AddendOp);
if (this->FindConstantValue(UseIter, ConstValue)) {
if (FG_MASK_UNSIGNED == SignMask) {
if (((uval_t) STARS_LARGE_UNSIGNED_ADD_CONSTANT_THRESHOLD) <= ConstValue) {
LargeConstFound = true;
}
}
else if (FG_MASK_SIGNED == SignMask) {
int SignedConstValue = (int) ConstValue;
if ( (((int) STARS_LARGE_SIGNED_ADD_CONSTANT_THRESHOLD) <= SignedConstValue)
|| (((int) STARS_SMALL_SIGNED_ADD_CONSTANT_THRESHOLD) >= SignedConstValue)) {
LargeConstFound = true;
}
}
}
}
}
return LargeConstFound;
} // end of SMPInstr::IsAdditionOfLargeConstant()
2129
2130
2131
2132
2133
2134
2135
2136
2137
2138
2139
2140
2141
2142
2143
2144
2145
2146
2147
2148
2149
2150
2151
2152
2153
2154
2155
2156
2157
2158
2159
2160
2161
2162
2163
2164
2165
2166
2167
2168
2169
2170
2171
2172
2173
2174
2175
2176
2177
2178
2179
2180
2181
2182
2183
2184
2185
2186
2187
2188
2189
2190
2191
2192
2193
2194
2195
2196
2197
2198
2199
2200
2201
2202
2203
2204
2205
2206
2207
2208
2209
2210
2211
2212
2213
2214
2215
// MACHINE DEPENDENT: Opcode indicates 64-bit operands are the default.
bool SMPInstr::MDDefaultsTo64BitOperands(void) const {
bool Default64;
switch (this->SMPcmd.itype) {
// use ss
case NN_pop:
case NN_popf:
case NN_popfq:
case NN_push:
case NN_pushf:
case NN_pushfq:
case NN_retn:
case NN_retf:
case NN_retnq:
case NN_retfq:
case NN_call:
case NN_callfi:
case NN_callni:
case NN_enter:
case NN_enterq:
case NN_leave:
case NN_leaveq:
// near branches
case NN_ja:
case NN_jae:
case NN_jb:
case NN_jbe:
case NN_jc:
case NN_je:
case NN_jg:
case NN_jge:
case NN_jl:
case NN_jle:
case NN_jna:
case NN_jnae:
case NN_jnb:
case NN_jnbe:
case NN_jnc:
case NN_jne:
case NN_jng:
case NN_jnge:
case NN_jnl:
case NN_jnle:
case NN_jno:
case NN_jnp:
case NN_jns:
case NN_jnz:
case NN_jo:
case NN_jp:
case NN_jpe:
case NN_jpo:
case NN_js:
case NN_jz:
case NN_jcxz:
case NN_jecxz:
case NN_jrcxz:
case NN_jmp:
case NN_jmpni:
case NN_jmpshort:
case NN_loop:
case NN_loopq:
case NN_loope:
case NN_loopqe:
case NN_loopne:
case NN_loopqne:
Default64 = true;
break;
default:
Default64 = false;
break;
}
return Default64;
} // end of SMPInstr::MDDefaultsTo64BitOperands()
// MACHINE DEPENDENT: Inst has 64-bit operands
bool SMPInstr::MDHas64BitOperands(void) const {
#ifdef __EA64__
return (((this->SMPcmd.auxpref & aux_use64) != 0)
&& ((this->SMPcmd.rex & REX_W) != 0
|| (((cmd.auxpref & aux_natop) != 0) && this->MDDefaultsTo64BitOperands())));
// 64-bit segment, rex.w or insns-64
#else
return false;
#endif
}
// MACHINE DEPENDENT: is current addressing 32-bit?
bool SMPInstr::MDIsAddressing32bit(void) const {
int p = this->SMPcmd.auxpref & (aux_use32|aux_use64|aux_natad);
return p == (aux_natad|aux_use32)
|| p == 0
|| p == aux_use64;
}
// MACHINE DEPENDENT: is current addressing 64-bit?
bool SMPInstr::MDIsAddressing64bit(void) const {
#ifdef __EA64__
int p = this->SMPcmd.auxpref & (aux_use32|aux_use64|aux_natad);
return p == (aux_natad|aux_use64);
#else
return false;
#endif
}
// Fetch default bit-width op_t.dtyp field for current instruction's operands
char SMPInstr::GetOperandDtypField(void) const {
if (this->MDHas64BitOperands()) {
return dt_qword;
}
else {
return dt_dword;
}
}
bool SMPInstr::IsSetToZero(void) const {
bool SetsToZero = false;
SMPRegTransfer *CurrRT = this->RTL.GetRT(0);
if ((!CurrRT->HasRightSubTree()) && (CurrRT->GetOperator() == SMP_ASSIGN)) {
op_t RightOp = CurrRT->GetRightOperand();
SetsToZero = ((o_imm == RightOp.type) && (0 == RightOp.value));
}
return SetsToZero;
}
// MACHINE DEPENDENT: Is instruction a return instruction?
bool SMPInstr::MDIsReturnInstr(void) const {
return ((this->SMPcmd.itype == NN_retn) || (this->SMPcmd.itype == NN_retf));
}
// MACHINE DEPENDENT: Is instruction a POP instruction?
#define FIRST_POP_INST NN_pop
#define LAST_POP_INST NN_popfq
bool SMPInstr::MDIsPopInstr(void) const {
return ((this->SMPcmd.itype >= FIRST_POP_INST)
&& (this->SMPcmd.itype <= LAST_POP_INST));
}
// MACHINE DEPENDENT: Is instruction a PUSH instruction?
#define FIRST_PUSH_INST NN_push
#define LAST_PUSH_INST NN_pushfq
bool SMPInstr::MDIsPushInstr(void) const {
return ((this->SMPcmd.itype >= FIRST_PUSH_INST)
&& (this->SMPcmd.itype <= LAST_PUSH_INST));
}
// MACHINE DEPENDENT: Is instruction an ENTER instruction?
bool SMPInstr::MDIsEnterInstr(void) const {
return ((this->SMPcmd.itype >= MD_FIRST_ENTER_INSTR)
&& (this->SMPcmd.itype <= MD_LAST_ENTER_INSTR));
}
// MACHINE DEPENDENT: Is instruction a LEAVE instruction?
bool SMPInstr::MDIsLeaveInstr(void) const {
return ((this->SMPcmd.itype >= MD_FIRST_LEAVE_INSTR)
&& (this->SMPcmd.itype <= MD_LAST_LEAVE_INSTR));
// MACHINE DEPENDENT: Is instruction a HALT instruction?
bool SMPInstr::MDIsHaltInstr(void) const {
return (NN_hlt == this->SMPcmd.itype);
}
#define MD_FIRST_COND_MOVE_INSTR NN_cmova
#define MD_LAST_COND_MOVE_INSTR NN_fcmovnu
// MACHINE DEPENDENT: Is instruction a conditional move?
bool SMPInstr::MDIsConditionalMoveInstr(void) const {
return ((this->SMPcmd.itype >= MD_FIRST_COND_MOVE_INSTR)
&& (this->SMPcmd.itype <= MD_LAST_COND_MOVE_INSTR));
}
clc5q
committed
// MACHINE DEPENDENT: Is instruction any kind of move?
bool SMPInstr::MDIsMoveInstr(void) const {
return ((NN_mov == this->SMPcmd.itype) || (NN_movsx == this->SMPcmd.itype)
|| (NN_movzx == this->SMPcmd.itype) || this->MDIsConditionalMoveInstr());
}
// MACHINE DEPENDENT: Do opcode/operands definitely indicate signed arithmetic?
// Generally, this is only true for certain variants of multiplication and division.
bool SMPInstr::MDIsSignedArithmetic(void) const {
unsigned short opcode = this->SMPcmd.itype;
if (NN_idiv == opcode)
return true;
if (NN_imul == opcode) {
// If we discard the upper N bits of the multiplication result, then the
// lower N bits are the same for signed and unsigned multiplication, and
// gcc/g++ often use the IMUL opcode for both signed and unsigned multiplies
// when only N bits of result are retained. Therefore, the SIGNED nature of
// IMUL operands can only be inferred from the case in which 2N bits are kept.
return (!(this->AreMultiplicationBitsDiscarded()));
}
else { // idiv and imul are only possible signed cases
return false;
}
} // end of SMPInstr::MDIsSignedArithmetic()
// MACHINE DEPENDENT: Is instruction a conditional jump based on an unsigned condition?
bool SMPInstr::MDIsUnsignedBranch(void) const {
unsigned short opcode = this->SMPcmd.itype;
return ((NN_ja == opcode) || (NN_jae == opcode) || (NN_jb == opcode) || (NN_jbe == opcode)
|| (NN_jna == opcode) || (NN_jnae == opcode) || (NN_jnb == opcode) || (NN_jnbe == opcode));
}
// MACHINE DEPENDENT: Is instruction a conditional jump based on a signed condition?
bool SMPInstr::MDIsSignedBranch(void) const {
unsigned short opcode = this->SMPcmd.itype;
return ((NN_jg == opcode) || (NN_jge == opcode) || (NN_jl == opcode) || (NN_jle == opcode)
|| (NN_jng == opcode) || (NN_jnge == opcode) || (NN_jnl == opcode) || (NN_jnle == opcode)
|| (NN_js == opcode) || (NN_jns == opcode));
}
// MACHINE DEPENDENT: Is instruction a boolean set based on an unsigned condition?
bool SMPInstr::MDIsUnsignedSetValue(void) const {
unsigned short opcode = this->SMPcmd.itype;
return ((NN_seta == opcode) || (NN_setae == opcode) || (NN_setb == opcode) || (NN_setbe == opcode)
|| (NN_setna == opcode) || (NN_setnae == opcode) || (NN_setnb == opcode) || (NN_setnbe == opcode));
}
// MACHINE DEPENDENT: Is instruction a boolean set based on a signed condition?
bool SMPInstr::MDIsSignedSetValue(void) const {
unsigned short opcode = this->SMPcmd.itype;
return ((NN_setg == opcode) || (NN_setge == opcode) || (NN_setl == opcode) || (NN_setle == opcode)
|| (NN_setng == opcode) || (NN_setnge == opcode) || (NN_setnl == opcode) || (NN_setnle == opcode)
|| (NN_sets == opcode) || (NN_setns == opcode));
}
// MACHINE DEPENDENT: Is instruction a boolean set based on any condition?
bool SMPInstr::MDIsAnySetValue(void) const {
unsigned short opcode = this->SMPcmd.itype;
return ((NN_seta <= opcode) && (NN_setz >= opcode));
}
clc5q
committed
// MACHINE DEPENDENT: Is instruction a left shift instruction?
bool SMPInstr::MDIsLeftShift(void) const {
unsigned short opcode = this->SMPcmd.itype;
return ((NN_sal == opcode) || (NN_shl == opcode));
}
// Is kind of shift or rotate that is used in hash functions
#define STARS_HASH_SHIFT_THRESHOLD 4
bool SMPInstr::MDIsHashingArithmetic(void) const {
bool FoundHashShift = false;
unsigned short opcode = this->SMPcmd.itype;
// We are looking for shifts or rotates in the leftward direction.
if ((opcode == NN_rcl) || (opcode == NN_rol) || (opcode == NN_shl) || (opcode == NN_shld)) {
SMPRegTransfer *CurrRT = this->RTL.GetRT(0);
assert(CurrRT->HasRightSubTree());
CurrRT = CurrRT->GetRightTree();
op_t ShiftCountOp = CurrRT->GetRightOperand();
if (o_imm == ShiftCountOp.type) {
uval_t CountValue = ShiftCountOp.value;
FoundHashShift = (CountValue >= STARS_HASH_SHIFT_THRESHOLD);
}
else {
// PEASOUP bug # 144: left shift count in CL reg, accumulates in checksum loop.
FoundHashShift = true;
}
}
return FoundHashShift;
} // end of SMPInstr::MDIsHashingArithmetic()
clc5q
committed
2390
2391
2392
2393
2394
2395
2396
2397
2398
2399
2400
2401
2402
2403
2404
2405
2406
2407
2408
2409
2410
2411
2412
2413
2414
2415
2416
2417
2418
2419
2420
2421
2422
2423
2424
2425
2426
2427
2428
2429
2430
2431
2432
// Detect comparison of non-immediate to immediate. Return the two operands if true.
bool SMPInstr::MDIsCompareToPositiveConstant(op_t &NonConstOperand, uval_t &ConstValue) const {
bool CompareToConst = false;
if (NN_cmp == this->SMPcmd.itype) {
SMPRegTransfer *CurrRT = this->RTL.GetRT(0);
assert(CurrRT->HasRightSubTree());
CurrRT = CurrRT->GetRightTree();
assert(!(CurrRT->HasRightSubTree()));
op_t LeftOp = CurrRT->GetLeftOperand();
op_t RightOp = CurrRT->GetRightOperand();
if (o_imm == RightOp.type) {
CompareToConst = true;
ConstValue = RightOp.value;
NonConstOperand = LeftOp;
}
else if (o_imm == LeftOp.type) { // rare to see immediate first
CompareToConst = true;
ConstValue = LeftOp.value;
NonConstOperand = RightOp;
}
}
return CompareToConst;
} // end of SMPInstr::MDIsCompareToPositiveConstant()
bool SMPInstr::MDIsSubtractionOfConstant(op_t &NonConstOperand, uval_t &ConstValue) const {
bool SubtractOfConst = false;
SMPRegTransfer *CurrRT = this->RTL.GetRT(0);
if (CurrRT->HasRightSubTree()) {
CurrRT = CurrRT->GetRightTree();
SMPoperator CurrOp = CurrRT->GetOperator();
if (SMP_SUBTRACT == CurrOp) {
assert(!(CurrRT->HasRightSubTree()));
op_t LeftOp = CurrRT->GetLeftOperand();
op_t RightOp = CurrRT->GetRightOperand();
if (o_imm == RightOp.type) {
SubtractOfConst = true;
ConstValue = RightOp.value;
NonConstOperand = LeftOp;
}
}
clc5q
committed
else if (this->MDIsLoadEffectiveAddressInstr() && (SMP_ADD == CurrOp)) {
// We could have an addition of a negative constant via an lea opcode,
// e.g. lea ecx,[eax-48] will look like ecx := eax+(-48) in the RTL.
if (!(CurrRT->HasRightSubTree())) {
op_t LeftOp = CurrRT->GetLeftOperand();
op_t RightOp = CurrRT->GetRightOperand();
if (o_imm == RightOp.type) {
ConstValue = RightOp.value;
int SignedConstValue = (int) ConstValue;
if (0 > SignedConstValue) {
SubtractOfConst = true;
NonConstOperand = LeftOp;
ConstValue = (uval_t)(-SignedConstValue); // make +(-x) into -(+x)
}
}
}
}
clc5q
committed
}
return SubtractOfConst;
} // end of SMPInstr::MDIsSubtractionOfConstant()
2455
2456
2457
2458
2459
2460
2461
2462
2463
2464
2465
2466
2467
2468
2469
2470
2471
2472
2473
2474
2475
2476
2477
2478
2479
2480
2481
2482
2483
2484
2485
2486
// is AND operation that masks off lower BytesMasked bytes
bool SMPInstr::MDIsSubregMaskInst(size_t &BytesMasked) {
bool MaskingFound = false;
if (this->MDIsBitwiseAndOpcode()) {
SMPRegTransfer *CurrRT = this->RTL.GetRT(0);
if (CurrRT->HasRightSubTree()) {
CurrRT = CurrRT->GetRightTree();
SMPoperator CurrOp = CurrRT->GetOperator();
assert(SMP_BITWISE_AND == CurrOp);
op_t RightOp = CurrRT->GetRightOperand();
if (o_imm == RightOp.type) {
uval_t MaskValue = RightOp.value;
if (0x01000000 > MaskValue) {
MaskingFound = true;
if (0x100 > MaskValue) {
BytesMasked = 1;
}
else if (0x10000 > MaskValue) {
BytesMasked = 2;
}
else {
BytesMasked = 3;
}
}
}
}
}
return MaskingFound;
}
// MACHINE DEPENDENT: Does instruction use a callee-saved register?
bool SMPInstr::MDUsesCalleeSavedReg(void) {
set<DefOrUse, LessDefUse>::iterator CurrUse;
for (CurrUse = this->GetFirstUse(); CurrUse != this->GetLastUse(); ++CurrUse) {
op_t CurrOp = CurrUse->GetOp();
if (CurrOp.is_reg(MD_FRAME_POINTER_REG) || CurrOp.is_reg(R_si)
|| CurrOp.is_reg(R_di) || CurrOp.is_reg(R_bx)) {
return true;
}
}
return false;
} // end of SMPInstr::MDUsesCalleeSavedReg()
// Is the instruction a register to register copy of a stack pointer or frame pointer
// into a general purpose register (which mmStrata will now need to track as a stack
// relative pointer)?
bool SMPInstr::MDIsStackPointerCopy(bool UseFP) {
// OptType 3 indicates a move instruction
// The lea instruction can perform three operand arithmetic, e.g.
// lea ebx,[esp+12] is just ebx:=esp+12, so it is a stack pointer copy.
if (((this->OptType == 3) || (NN_lea == this->SMPcmd.itype))
&& (this->GetFirstDef()->GetOp().type == o_reg)
&& (!(this->GetFirstDef()->GetOp().is_reg(R_sp)))
&& (!(this->HasSourceMemoryOperand()))) { // reg to reg move
if (this->GetFirstUse()->GetOp().is_reg(MD_FRAME_POINTER_REG))
// Move of base pointer EBP into a general register
return true;
else if ((this->GetFirstUse()->GetOp().is_reg(MD_STACK_POINTER_REG))
&& !(this->GetFirstDef()->GetOp().is_reg(MD_FRAME_POINTER_REG)))
// Move of ESP into something besides a base pointer
return true;
}
else if (this->GetFirstUse()->GetOp().is_reg(MD_STACK_POINTER_REG)) {
// Move of ESP into a register; no base pointer used in this function
return true;
}
}
return false;
} // end of SMPInstr::MDIsStackPointerCopy()
// Does any RTL fit the alloca() pattern: stack_pointer -= non-immediate-operand
bool SMPInstr::HasAllocaRTL(void) {
bool FoundAlloca = false;
size_t RTLCount = this->RTL.GetCount();
size_t RTLIndex;
for (RTLIndex = 0; RTLIndex < RTLCount; ++RTLIndex) {
SMPRegTransfer *CurrRT = this->RTL.GetRT(RTLIndex);
if (CurrRT->IsAllocaRTL()) {
FoundAlloca = true;
break;
}
}
return FoundAlloca;
} // end of SMPInstr::HasAllocaRTL()
// Determine if the instruction saves or restores a pointer into the stack frame.
// If it saves a stack pointer, set Save to true, set the StackDelta saved, and set
// the operand that received the saved stack pointer into CopyOp. and return true.
// If it restores a stack pointer, set Save to false, set CopyOp to the operand that
// held the value being restored, set RestoreOp to the stack pointer or frame pointer
// register (whichever was restored), leave StackDelta alone for later computation
// based on reaching definitions, and return true.
// For most instructions, no save or restore of a stack pointer, so return false.
bool SMPInstr::MDIsStackPtrSaveOrRestore(bool UseFP, sval_t FPDelta, bool &Save, sval_t &StackDelta, op_t &CopyOp, bool &Error) {
bool StackPointerSaveOrRestore;
size_t RTLCount = this->RTL.GetCount();
size_t RTLIndex;
int BaseReg, IndexReg, CopyReg;
ushort Scale;
ea_t offset;
SMPoperator CurrOper;
bool LookUpStackDelta; // Get stack delta from reaching defs for TempOp
sval_t DeltaAdjust; // add to StackDelta after computing from reaching defs, e.g. lea esp,[ecx-4] get TempOp of ecx
// and DeltaAdjust of -4
Error = false;
for (RTLIndex = 0; RTLIndex < RTLCount; ++RTLIndex) {
bool FPRestore = false; // frame pointer is restored
bool SPRestore = false; // stack pointer is restored
StackPointerSaveOrRestore = false; // default unless we detect a save or restore of the stack or frame pointer
TempOp = InitOp;
LookUpStackDelta = false;
DeltaAdjust = 0;
Save = false; // default unless we detect a stack pointer save
// The stack alignment instructions (SP := SP bitwise_and immediate_value)
// look like something that needs to be processed here, but we always ignore
// these instructions. They have a variable effect on the stack pointer, from zero
// to -15 delta, but we assume that the delta is zero. This works for us because
// no stack accesses will occur into the padding region.
// Also, any instruction that definitely does not restore the stack pointer or
// frame pointer from an arbitrary register or memory location, e.g. a leave instruction
// in x86 CPUs, is already handled in normal stack delta computations and needs
// no lookups from reaching defs, etc.
if (this->IsStackAlignmentInst() || this->MDIsLeaveInstr() || this->MDIsFrameAllocInstr()) {
break; // exit and return false
}
SMPRegTransfer *CurrRT = this->RTL.GetRT(RTLIndex);
CurrOper = CurrRT->GetOperator();
if (SMP_ASSIGN != CurrOper) {
break; // not a regular RTL
}
op_t LeftOp = CurrRT->GetLeftOperand();
if (4 > GetOpDataSize(LeftOp)) {
break; // Not tracking copies of less than the full stack or frame pointer
// NOTE: We hard code 4 bytes because we will track bottom 32 bits, often seen in 64-bit code.
if (LeftOp.is_reg(MD_STACK_POINTER_REG)) {
SPRestore = true; // temporary; might just be a push or pop RTL, etc., in which case we will reset.
}
else if (UseFP && LeftOp.is_reg(MD_FRAME_POINTER_REG)) {
FPRestore = true; // likewise temporary
}
if (!(SPRestore || FPRestore)) {
#if 0
if (LeftOp.is_reg(MD_FLAGS_REG)) {
break; // No point in looking for a save into the flags register
}
#endif
clc5q
committed
Save = true; // Maybe; keep looking for save
// If we are assigning to the stack pointer reg or the frame pointer reg, we need to analyze the right
// hand side of the RTL to see if it is a stack/frame pointer value, and not a simple push, pop, etc.
if (!(CurrRT->HasRightSubTree())) {
// Simple assignment.
op_t RightOp = CurrRT->GetRightOperand();
if ((o_reg <= RightOp.type) && (o_displ >= RightOp.type)) { // register or memory
if (RightOp.is_reg(MD_STACK_POINTER_REG)) {
// Stack pointer reg is being saved.
StackDelta = this->GetStackPtrOffset(); // LeftOp := SP, so saved delta is just current delta
CopyOp = LeftOp;
StackPointerSaveOrRestore = true;
FPRestore = false; // treat FP := SP as a save of SP rather than a restoration of FP
else if (!SPRestore && UseFP && RightOp.is_reg(MD_FRAME_POINTER_REG)) {
// Frame pointer is being saved
StackDelta = FPDelta;
CopyOp = LeftOp;
StackPointerSaveOrRestore = true;
break;
}
else if (SPRestore || FPRestore) {
// stack or frame pointer is being restored; leave Save=false and set other outgoing arguments.
TempOp = RightOp;
CopyOp = RightOp;
StackPointerSaveOrRestore = true;
LookUpStackDelta = true;
}
else { // RightOp is register or non-stack-pointer memory expr; either might hold stack delta
TempOp = RightOp;
CopyOp = LeftOp;
LookUpStackDelta = true; // See if RightOp is holding a stack delta
clc5q
committed
StackPointerSaveOrRestore = true; // Maybe; flag tells us to keep looking
}
}
else {
if (SPRestore || FPRestore) {
SMP_msg("ERROR: Invalid operand type for assignment to stack or frame pointer at %lx\n",
(unsigned long) this->GetAddr());
2657
2658
2659
2660
2661
2662
2663
2664
2665
2666
2667
2668
2669
2670
2671
2672
2673
2674
2675
2676
2677
2678
2679
2680
2681
2682
2683
StackPointerSaveOrRestore = false;
break;
}
}
else { // we have a right subtree in the CurrRT
SMPRegTransfer *RightRT = CurrRT->GetRightTree();
// In order to have a right subtree, we must have something like:
// lea esp,[ecx-4] which produces the RTL: esp := ecx - 4
// We should consider any other RTL structure besides a basic addition or
// subtraction on the right subtree to be invalid.
CurrOper = RightRT->GetOperator();
if ((SMP_ADD == CurrOper) || (SMP_SUBTRACT == CurrOper)) {
op_t RightLeftOp = RightRT->GetLeftOperand();
if (o_reg == RightLeftOp.type) {
if (RightRT->HasRightSubTree()) {
// Complex RTL such as lea esp,[ebx+ecx*4] ; cannot analyze
StackPointerSaveOrRestore = false;
}
else {
op_t RightRightOp = RightRT->GetRightOperand();
if (o_imm != RightRightOp.type) {
// Complex RTL such as lea esp,[ebx+ecx] ; cannot analyze
StackPointerSaveOrRestore = false;
}
else {
TempOp = RightLeftOp;
DeltaAdjust = (sval_t) RightRightOp.value;
if (DeltaAdjust > STARS_ESP_ADDITION_OVERFLOW_THRESHOLD) {
// Really a subtraction via addition of a negative,
// or addition via subtraction of a negative.
// E.g. add esp, 0xfffffff0 is sub esp,16 and sub esp,0xfffffff0 is add esp,16
int32 TempDelta = (int32) (DeltaAdjust & 0xffffffff);
DeltaAdjust = (sval_t) TempDelta;
}
if (SMP_SUBTRACT == CurrOper) {
// Negate the stack delta adjustment, e.g. lea esp,[ecx-4] needs DeltaAdjust of -4, not 4.
DeltaAdjust = (0 - DeltaAdjust);
}
LookUpStackDelta = true;
StackPointerSaveOrRestore = true;
if (SPRestore || FPRestore) {
CopyOp = RightLeftOp;
}
else {
CopyOp = LeftOp;
}
}
}
}
else { // weird RTL; LeftOp := (MemoryOp OPER ???)
StackPointerSaveOrRestore = false;
}
}
else { // not ADD or SUBTRACT
StackPointerSaveOrRestore = false;
}
}
if (LookUpStackDelta) {
bool StackAccess = false;
clc5q
committed
bool NonStackMemAccess = false;
2718
2719
2720
2721
2722
2723
2724
2725
2726
2727
2728
2729
2730
2731
2732
2733
2734
2735
2736
2737
2738
2739
2740
// We need to set StackDelta based on the reaching defs for TempOp
// A reg is probably a general register, but could have lea ebx,[esp+4] so it could be stack or frame pointer.
if (TempOp.is_reg(MD_STACK_POINTER_REG)) {
// Weed out RTs that increment or decrement the stack pointer, e.g. SP := SP -4.
// These are not the kind of "save" or "restore" RTs that we are tracking.
if (CopyOp.is_reg(MD_STACK_POINTER_REG)) {
StackPointerSaveOrRestore = false;
SPRestore = false;
FPRestore = false;
Save = false;
}
else {
StackDelta = this->GetStackPtrOffset();
StackDelta += DeltaAdjust;
LookUpStackDelta = false; // just got it; no need for reaching defs
StackPointerSaveOrRestore = true;
}
}
else if (UseFP && TempOp.is_reg(MD_FRAME_POINTER_REG)) {
StackDelta = FPDelta;
StackDelta += DeltaAdjust;
LookUpStackDelta = false; // just got it; no need for reaching defs
StackPointerSaveOrRestore = true;
else if (o_reg == TempOp.type) { // general reg, not frame or stack pointer reg
CopyReg = TempOp.reg;
MDExtractAddressFields(TempOp, BaseReg, IndexReg, Scale, offset);
CopyReg = BaseReg;
bool IndexedAccess = ((R_none != BaseReg) && (R_none != IndexReg));
if (IndexedAccess) {
StackPointerSaveOrRestore = false; // Cannot analyze indexed accesses into the stack
}
else if (MDIsStackPtrReg(BaseReg, UseFP)) {
StackAccess = true;
}
else {
// memory expr that is not stack or frame pointer
clc5q
committed
NonStackMemAccess = true; // something like [ecx] might actually turn out to be stack access
DeltaAdjust = (sval_t) TempOp.addr; // get normalized delta from addr field
}
if (StackPointerSaveOrRestore && LookUpStackDelta) {
op_t FindOp = InitOp;
if (StackAccess) {
FindOp = TempOp;
}
else {
FindOp.type = o_reg;
FindOp.reg = CopyReg;
FindOp.dtyp = this->GetOperandDtypField();
}
if (this->GetBlock()->GetFunc()->IsInStackPtrCopySet(FindOp)) {
// Screened out time wasters that are not in copy set; now,
// look up reaching defs.
// We need to find out which are the reaching definitions for the FindOp at the current InstAddr.
this->GetBlock()->GetFunc()->ComputeTempReachingDefs(FindOp, this->GetAddr());
this->GetBlock()->GetFunc()->ComputeTempStackDeltaReachesList(FindOp);
// See if TempStackDeltaReachesList has a consistent delta value.
StackPointerSaveOrRestore = this->GetBlock()->GetFunc()->FindReachingStackDelta(StackDelta); // consistent SavedDelta value across entire list
StackDelta += DeltaAdjust;
clc5q
committed
if (StackPointerSaveOrRestore && NonStackMemAccess) {
// We have something like [ecx] or [ecx+DeltaAdjust]. It turns out that
// ECX has a copy of the stack pointer or frame pointer in it, but that
// does not mean that the memory location [ecx] has a copy of a stack or
// frame pointer. We need to look up the normalized stack address [esp+StackDelta]
// in the StackPtrCopySet just like we did for ECX before we conclude that a
// stack pointer save or restore is happening.
FindOp = InitOp;
FindOp.type = o_displ;
FindOp.reg = MD_STACK_POINTER_REG;
FindOp.addr = StackDelta;
FindOp.dtyp = this->GetOperandDtypField();
clc5q
committed
if (this->GetBlock()->GetFunc()->IsInStackPtrCopySet(FindOp)) {
// Screened out time wasters that are not in copy set; now,
// look up reaching defs.
// We need to find out which are the reaching definitions for the FindOp at the current InstAddr.
this->GetBlock()->GetFunc()->ComputeTempReachingDefs(FindOp, this->GetAddr());
this->GetBlock()->GetFunc()->ComputeTempStackDeltaReachesList(FindOp);
// See if TempStackDeltaReachesList has a consistent delta value.
StackPointerSaveOrRestore = this->GetBlock()->GetFunc()->FindReachingStackDelta(StackDelta); // consistent SavedDelta value across entire list
// StackPointerSaveOrRestore will now be true only if [ecx] pointed to a saved stack or frame pointer with consistent delta.
}
else {
// E.g. [ecx] pointed to stack location that was not holding a saved stack or frame pointer.
StackPointerSaveOrRestore = false;
}
}
}
else {
StackPointerSaveOrRestore = false; // reset, not in stack pointer copy set
}
} // end if (LookupStackDelta)
if (!StackPointerSaveOrRestore && !Save && (SPRestore || FPRestore)) {
// Any restore that could not be analyzed is an error.
Error = true;
break; // error exit
}
else if (StackPointerSaveOrRestore) {
if (FPRestore) {
// If we succeeded in looking up a stack delta that goes into the frame pointer reg,
// then we want to consider this instruction to be a save of a stack delta into
// a register (which happens to be the frame pointer reg in this case).
FPRestore = false;
Save = true;
break; // assume only one save or restore in an instruction; exit with success
} // end for all RTs in the RTL
return StackPointerSaveOrRestore;
} // end of SMPInstr::MDIsStackPtrSaveOrRestore()
clc5q
committed
// Make list of regs and their stack offsets in RTLs for any push instruction
bool SMPInstr::GetPushedRegsList(map<uint32, sval_t> &PushedRegs) {
size_t RTLCount = this->RTL.GetCount();
size_t RTLIndex;
op_t LeftOp, RightOp;
int BaseReg, IndexReg;
clc5q
committed
2841
2842
2843
2844
2845
2846
2847
2848
2849
2850
2851
2852
2853
2854
2855
2856
2857
2858
2859
2860
2861
2862
2863
2864
2865
2866
2867
2868
2869
2870
2871
2872
2873
2874
2875
2876
2877
2878
ushort Scale;
ea_t offset;
SMPoperator CurrOper;
for (RTLIndex = 0; RTLIndex < RTLCount; ++RTLIndex) {
SMPRegTransfer *CurrRT = this->RTL.GetRT(RTLIndex);
CurrOper = CurrRT->GetOperator();
// Only process simple pushes of registers
LeftOp = CurrRT->GetLeftOperand();
if (MDIsStackAccessOpnd(LeftOp, false) && (!(CurrRT->HasRightSubTree())) && (SMP_ASSIGN == CurrOper)) {
RightOp = CurrRT->GetRightOperand();
if (RightOp.type == o_reg) {
MDExtractAddressFields(LeftOp, BaseReg, IndexReg, Scale, offset);
// Exclude indexed stack writes.
if (R_none == IndexReg) {
sval_t ESPOffset = (sval_t) offset;
map<uint32, sval_t>::iterator FindIter = PushedRegs.find((uint32) RightOp.reg);
if (FindIter == PushedRegs.end()) { // Not already pushed earlier
pair<uint32, sval_t> TempPair((uint32) RightOp.reg, ESPOffset);
pair<map<uint32, sval_t>::iterator, bool> InsertResult;
InsertResult = PushedRegs.insert(TempPair);
assert(InsertResult.second);
}
}
}
}
} // end for all RTs in RTL
return (!PushedRegs.empty());
} // end of SMPInstr::GetPushedRegsList()
// Make list of regs and their stack offsets in RTLs for any pop instruction
bool SMPInstr::GetPoppedRegsList(bool FirstReturnBlock, map<uint32, sval_t> &PoppedRegs) {
size_t RTLCount = this->RTL.GetCount();
size_t RTLIndex;
size_t IncomingSize = PoppedRegs.size(); // from return blocks already processed
size_t MatchCount = 0; // count of matches to incoming data
op_t LeftOp, RightOp;
int BaseReg, IndexReg;
clc5q
committed
2880
2881
2882
2883
2884
2885
2886
2887
2888
2889
2890
2891
2892
2893
2894
2895
2896
2897
2898
2899
2900
2901
2902
2903
2904
2905
2906
2907
2908
2909
2910
2911
2912
2913
2914
2915
2916
2917
2918
2919
2920
2921
2922
2923
2924
2925
2926
2927
2928
2929
2930
2931
ushort Scale;
ea_t offset;
SMPoperator CurrOper;
for (RTLIndex = 0; RTLIndex < RTLCount; ++RTLIndex) {
SMPRegTransfer *CurrRT = this->RTL.GetRT(RTLIndex);
CurrOper = CurrRT->GetOperator();
// Only process simple pushes of registers
LeftOp = CurrRT->GetLeftOperand();
if ((LeftOp.type == o_reg) && (!(CurrRT->HasRightSubTree())) && (SMP_ASSIGN == CurrOper)) {
RightOp = CurrRT->GetRightOperand();
if (MDIsStackAccessOpnd(RightOp, false)) {
MDExtractAddressFields(RightOp, BaseReg, IndexReg, Scale, offset);
// Exclude indexed stack reads.
if (R_none == IndexReg) {
sval_t ESPOffset = (sval_t) offset;
map<uint32, sval_t>::iterator FindIter = PoppedRegs.find((uint32) LeftOp.reg);
if (FindIter == PoppedRegs.end()) { // Not already popped in another block
if (FirstReturnBlock) {
pair<uint32, sval_t> TempPair((uint32) LeftOp.reg, ESPOffset);
pair<map<uint32, sval_t>::iterator, bool> InsertResult;
InsertResult = PoppedRegs.insert(TempPair);
assert(InsertResult.second);
}
else {
// We should not find a new pop in return block N not seen in return block N-1
PoppedRegs.clear(); // causes a return value of false
break;
}
}
else {
// Already in map; check for consistency across return points.
if (ESPOffset != FindIter->second) { // inconsistent
PoppedRegs.clear(); // causes a return value of false
break;
}
else {
++MatchCount;
}
}
}
}
}
} // end for all RTs in RTL
if (FirstReturnBlock) {
IncomingSize = 0; // Not trying to match within first return block
}
return ((MatchCount == IncomingSize) && (!PoppedRegs.empty()));
} // end of SMPInstr::GetPoppedRegsList()
// If call instruction is to malloc(), set the DEF register EAX type to
// HEAPPTR and return true.
bool SMPInstr::MDFindMallocCall(op_t TargetOp) {
bool changed = false;
func_t *TargetFunc = get_func(TargetOp.addr);
if (TargetFunc) {
char FuncName[MAXSTR];
get_func_name(TargetFunc->startEA, FuncName, sizeof(FuncName) - 1);
if (0 == strcmp("malloc", FuncName)) {
// NOTE: Some compilers might call it __malloc ; make this more robust !!!
#if SMP_VERBOSE_FIND_POINTERS
clc5q
committed
SMP_msg("Found call to malloc at %x\n", this->addr);
#endif
op_t SearchOp = InitOp;
SearchOp.type = o_reg;
SearchOp.reg = R_ax;
SearchOp.dtyp = this->GetOperandDtypField();
set<DefOrUse, LessDefUse>::iterator EAXDEF;
EAXDEF = this->SetDefType(SearchOp, HEAPPTR);
int SSANum = EAXDEF->GetSSANum();
changed = true;
if (this->BasicBlock->IsLocalName(SearchOp)) {
(void) this->BasicBlock->PropagateLocalDefType(SearchOp, HEAPPTR,
this->GetAddr(), SSANum, false);
}
else { // global name
this->BasicBlock->GetFunc()->ResetProcessedBlocks(); // set Processed to false
(void) this->BasicBlock->PropagateGlobalDefType(SearchOp, HEAPPTR,
SSANum, false);
}
} // end if "malloc"
} // end if (TargetFunc)
return changed;
} // end of SMPInstr::MDFindMallocCall()
// Is instruction a branch (conditional or unconditional) to a
// code target that is not in the current chunk?
bool SMPInstr::IsBranchToFarChunk(void) {
if (this->IsFarBranchComputed()) { // answer is cached
return this->IsBranchesToFarChunk();
func_t *CurrChunk = get_fchunk(this->address);
bool FarBranch = false;
if ((JUMP | COND_BRANCH) & this->GetDataFlowType()) {
// Instruction is a direct branch, conditional or unconditional
if (this->NumUses() > 0) {
set<DefOrUse, LessDefUse>::iterator CurrUse;
for (CurrUse = this->GetFirstUse(); CurrUse != this->GetLastUse(); ++CurrUse) {
op_t JumpTarget = CurrUse->GetOp();
if ((o_near == JumpTarget.type) || (o_far == JumpTarget.type)) {
// Branches to a code address
clc5q
committed
// stdclib sometimes has jumps to zero and calls to zero. These are dead code.
if (0 != JumpTarget.addr) {
func_t *TargetChunk = get_fchunk(JumpTarget.addr);
// Is target address within the same chunk as the branch?
FarBranch = (NULL == TargetChunk) || (CurrChunk->startEA != TargetChunk->startEA);
if (FarBranch) {
this->FarBranchTarget = JumpTarget.addr;
}
if (FarBranch) {
this->SetBranchesToFarChunk();
}
this->SetFarBranchComputed();