diff --git a/SMPDataFlowAnalysis.cpp b/SMPDataFlowAnalysis.cpp index eb982058a2dce6f9972e65ae2fedd5252e9fc2e8..17a8ad89deec4f816d071dd64784ae294d37040c 100644 --- a/SMPDataFlowAnalysis.cpp +++ b/SMPDataFlowAnalysis.cpp @@ -29,6 +29,7 @@ #define SMP_DEBUG_CONTROLFLOW 0 // tells what processing stage is entered #define SMP_DEBUG_XOR 0 #define SMP_DEBUG_CHUNKS 1 // tracking down tail chunks for functions +#define SMP_DEBUG_FRAMEFIXUP 1 // Used for binary search by function number in SMPStaticAnalyzer.cpp // to trigger debugging output and find which instruction in which @@ -358,8 +359,8 @@ int SMPInstr::GetOptType(void) const { } // Is this instruction the one that allocates space on the -// stack for the local variables of size LocSize? -bool SMPInstr::MDIsFrameAllocInstr(asize_t LocSize) const { +// stack for the local variables? +bool SMPInstr::MDIsFrameAllocInstr(void) const { // The frame allocating instruction should look like: // sub esp,48 or add esp,-64 etc. if ((SMPcmd.itype == NN_sub) || (SMPcmd.itype == NN_add)) { @@ -373,7 +374,23 @@ bool SMPInstr::MDIsFrameAllocInstr(asize_t LocSize) const { // and 16 bytes for outgoing arguments in a single // instruction: sub esp,80 // you cannot insist on finding sub esp,LocSize - return true; + // To make this more robust, we are going to insist that + // an allocation of stack space is either performed by + // adding a negative immediate value, or by subtracting + // a positive immediate value. We will throw in, free of + // charge, a subtraction of a register, which is how alloca() + // usually allocates stack space. + if (o_imm == Uses.GetRef(0).type) { + signed long TempImm = (signed long) Uses.GetRef(0).value; + if (((0 > TempImm) && (SMPcmd.itype == NN_add)) + || ((0 < TempImm) && (SMPcmd.itype == NN_sub))) { + return true; + } + } + else if ((o_reg == Uses.GetRef(0).type) + && (SMPcmd.itype == NN_sub)) { // alloca() ? + return true; + } } } return false; @@ -404,6 +421,8 @@ bool SMPInstr::MDIsFrameDeallocInstr(bool UseFP, asize_t LocalVarsSize) const { msg("Used imprecise LocalVarsSize to find dealloc instr.\n"); return true; } + else if (NN_leave == this->SMPcmd.itype) + return true; else return false; } // end of SMPInstr::MDIsFrameDeallocInstr() @@ -429,6 +448,18 @@ bool SMPInstr::MDIsPushInstr(void) const { && (SMPcmd.itype <= LAST_PUSH_INST)); } +// MACHINE DEPENDENT: Does instruction use a callee-saved register? +bool SMPInstr::MDUsesCalleeSavedReg(void) const { + for (size_t index = 0; index < this->Uses.GetSize(); ++index) { + op_t CurrUse = this->GetUse(index); + if (CurrUse.is_reg(R_bp) || CurrUse.is_reg(R_si) + || CurrUse.is_reg(R_di) || CurrUse.is_reg(R_bx)) { + return true; + } + } + return false; +} // end of SMPInstr::MDUsesCalleeSavedReg() + // Analyze the instruction and its operands. void SMPInstr::Analyze(void) { if (this->analyzed) @@ -595,6 +626,151 @@ void SMPInstr::AnnotateStackConstants(bool UseFP, FILE *AnnotFile) { return; } // end of SMPInstr::AnnotateStackConstants() +// Emit all annotations for the instruction. +void SMPInstr::EmitAnnotations(bool UseFP, bool AllocSeen, FILE *AnnotFile) { + ea_t addr = this->address; + flags_t InstrFlags = getFlags(addr); + bool MemDest = this->HasDestMemoryOperand(); + bool MemSrc = this->HasSourceMemoryOperand(); + bool SecondSrcOperandNum = this->IsSecondSrcOperandNumeric(InstrFlags); + + ++OptCount[OptType]; // keep count for debugging info + +#if SMP_DEBUG_MEM + if (MemDest || MemSrc) { + msg("OptType: %d %s", OptType, disasm); + this->PrintOperands(); + } +#endif + + // Emit appropriate optimization annotations. + bool SDTInstrumentation = false; + switch (OptType) { + case 0: // SDT will have to handle these + { +#if SMP_DEBUG_TYPE0 + msg("OptType 0: %x %s\n", addr, disasm); +#endif + // mmStrata wants to suppress warnings on the PUSH + // instructions that precede the LocalVarsAllocInstr + // (i.e. the PUSHes of callee-saved regs). + if (!AllocSeen && this->MDIsPushInstr()) { + qfprintf(AnnotFile, "%x %d INSTR LOCAL NoWarn %s \n", + addr, -3, disasm); + } + else { + SDTInstrumentation = true; + } + break; + } + + case 1: // nothing for SDT to do + { qfprintf(AnnotFile, "%x %d INSTR LOCAL NoMetaUpdate %s \n", + addr, -1, disasm); + ++AnnotationCount[OptType]; + break; + } + + case 4: // INC, DEC, etc.: no SDT work unless MemDest + { if (MemDest || MemSrc) { + SDTInstrumentation = true; + break; // treat as category 0 + } + qfprintf(AnnotFile, "%x %d INSTR LOCAL Always1stSrc %s \n", + addr, -1, disasm); + ++AnnotationCount[OptType]; + break; + } + + case 5: // ADD, etc.: If numeric 2nd src operand, no SDT work. + { if (MemDest || MemSrc) { + SDTInstrumentation = true; + break; // treat as category 0 + } + if (SecondSrcOperandNum) { // treat as category 1 + qfprintf(AnnotFile, "%x %d INSTR LOCAL %s %s \n", + addr, -1, OptExplanation[OptType], disasm); + ++AnnotationCount[OptType]; + } + break; + } + + case 6: // Only OS code should include these; problem for SDT + { if (MemDest) { + SDTInstrumentation = true; + break; // treat as category 0 + } + qfprintf(AnnotFile, "%x %d INSTR LOCAL AlwaysPTR %s \n", + addr, -OptType, disasm); + ++AnnotationCount[OptType]; + break; + } + + case 8: // Implicitly writes to EDX:EAX, always numeric. + { qfprintf(AnnotFile, "%x %d INSTR LOCAL n EDX EAX ZZ %s %s \n", + addr, -2, OptExplanation[OptType], disasm); + ++AnnotationCount[OptType]; + SDTInstrumentation = true; + break; + } + + case 9: // Either writes to FP reg (cat. 1) or memory (cat. 0) + { if (MemDest) { +#if SMP_DEBUG + // MemDest seems to happen too much. + msg("Floating point MemDest: %s \n", disasm); +#endif + SDTInstrumentation = true; + break; // treat as category 0 + } + qfprintf(AnnotFile, "%x %d INSTR LOCAL %s %s \n", + addr, -1, OptExplanation[OptType], disasm); + ++AnnotationCount[OptType]; + break; + } + + default: // 2,3,7: Optimization possibilities depend on operands + { +#if SMP_DEBUG2 + if (OptType == 3) { // MOV instr class + if (MemDest) { + msg("MemDest on MOV: %s\n", disasm); + } + else if (!SecondSrcOperandNum) { + msg("MOV: not 2nd op numeric: %s\n", disasm); + this->PrintOperands(); + } + } +#endif + SDTInstrumentation = true; + if (MemDest) { +#if SMP_DEBUG_XOR + if (OptType == 2) + msg("MemDest on OptType 2: %s\n", disasm); +#endif + break; // treat as category 0 + } + if ((OptType == 2) || (OptType == 7) || SecondSrcOperandNum) { + qfprintf(AnnotFile, "%x %d INSTR LOCAL n %s %s %s \n", + addr, -2, this->DestString(OptType), + OptExplanation[OptType], disasm); + ++AnnotationCount[OptType]; + } + break; + } + } // end switch (OptType) + + // If mmStrata is going to have to deal with the + // instruction, then we can annotate EBP and ESP + // relative constant offsets. If we have emitted + // an annotation of type -1, there is no point + // in telling mmStrata about these constants. + if (SDTInstrumentation) { + this->AnnotateStackConstants(UseFP, AnnotFile); + } + return; +} // end of SMPInstr::EmitAnnotations() + // ***************************************************************** // Class SMPBasicBlock // ***************************************************************** @@ -665,7 +841,6 @@ void SMPFunction::SetStackFrameInfo(void) { // to detect this and fix it. bool FrameInfoFixed = this->MDFixFrameInfo(); -#define SMP_DEBUG_FRAMEFIXUP 1 #if SMP_DEBUG_FRAMEFIXUP if (FrameInfoFixed) { msg("Fixed stack frame size info: %s\n", this->FuncName); @@ -702,7 +877,7 @@ void SMPFunction::SetStackFrameInfo(void) { } if (!FoundAllocInstr - && CurrInstr->MDIsFrameAllocInstr(this->LocalVarsSize)) { + && CurrInstr->MDIsFrameAllocInstr()) { this->LocalVarsAllocInstr = addr; FoundAllocInstr = true; } @@ -815,36 +990,163 @@ void SMPFunction::SetStackFrameInfo(void) { return; } // end of SMPFunction::SetStackFrameInfo() -// IDA Pro has trouble with functions that do not have any local -// variables. Unfortunately, the C library has plenty of these -// functions. IDA usually claims that frregs is zero and frsize -// is N, when the values should have been reversed. We can attempt -// to detect this and fix it. +// IDA Pro defines the sizes of regions in the stack frame in a way +// that suits its purposes but not ours. the frsize field of the func_info_t +// structure measures the distance between the stack pointer and the +// frame pointer (ESP and EBP in the x86). This region includes some +// of the callee-saved registers. So, the frregs field only includes +// the callee-saved registers that are above the frame pointer. +// x86 standard prologue on gcc/linux: +// push ebp ; save old frame pointer +// mov ebp,esp ; new frame pointer = current stack pointer +// push esi ; callee save reg +// push edi ; callee save reg +// sub esp,34h ; allocate 52 bytes for local variables +// +// Notice that EBP acquires its final frame pointer value AFTER the +// old EBP has been pushed. This means that, of the three callee saved +// registers, one is above where EBP points and two are below. +// IDA Pro is concerned with generating readable addressing expressions +// for items on the stack. None of the callee-saved regs will ever +// be addressed in the function; they will be dormant until they are popped +// off the stack in the function epilogue. In order to create readable +// disassembled code, IDA defines named constant offsets for locals. These +// offsets are negative values (x86 stack grows downward from EBP toward +// ESP). When ESP_relative addressing occurs, IDA converts a statement: +// mov eax,[esp+12] +// into the statement: +// mov eax,[esp+3Ch+var_30] +// Here, 3Ch == 60 decimal is the distance between ESP and EBP, and +// var_30 is defined to ahve the value -30h == -48 decimal. So, the +// "frame size" in IDA Pro is 60 bytes, and a certain local can be +// addressed in ESP-relative manner as shown, or as [ebp+var_30] for +// EBP-relative addressing. The interactive IDA user can then edit +// the name var_30 to something mnemonic, such as "virus_size", and IDA +// will replace all occurrences with the new name, so that code references +// automatically become [ebp+virus_size]. As the user proceeds +// interactively, he eventually produces very understandable code. +// This all makes sense for producing readable assembly text. However, +// our analyses have a compiler perspective as well as a memory access +// defense perspective. SMP distinguishes between callee saved regs, +// which should not be overwritten in the function body, and local +// variables, which can be written. We view the stack frame in logical +// pieces: here are the saved regs, here are the locals, here is the +// return address, etc. We don't care which direction from EBP the +// callee-saved registers lie; we don't want to lump them in with the +// local variables. We also don't like the fact that IDA Pro will take +// the function prologue code shown above and declare frregs=4 and +// frsize=60, because frsize no longer matches the stack allocation +// statement sub esp,34h == sub esp,52. We prefer frsize=52 and frregs=12. +// So, the task of this function is to fix these stack sizes in our +// private data members for the function, while leaving the IDA database +// alone because IDA needs to maintain its own definitions of these +// variables. // Fixing means we will update the data members LocalVarsSize and // CalleeSavedRegsSize. +// NOTE: This function is both machine dependent and platform dependent. +// The prologue and epilogue code generated by gcc-linux is as discussed +// above, while on Visual Studio and other Windows x86 compilers, the +// saving of registers other than EBP happens AFTER local stack allocation. +// A Windows version of the function would expect to see the pushing +// of ESI and EDI AFTER the sub esp,34h statement. bool SMPFunction::MDFixFrameInfo(void) { - // Does this function fit the problem pattern, with zero for saved - // regs and nonzero for local vars? - if ((LocalVarsSize == 0) || (CalleeSavedRegsSize != 0)) - return false; + int SavedRegsSize = 0; + int OtherPushesSize = 0; // besides callee-saved regs + int NewLocalsSize = 0; + int OldFrameTotal = this->CalleeSavedRegsSize + this->LocalVarsSize; + bool Changed = false; // Iterate through the first basic block in the function. If we find // a frame allocating Instr in it, then we have local vars. If not, - // we don't, and LocalVarsSize should have been zero, not - // CalleeSavedRegsSize, so swap them. + // we don't, and LocalVarsSize should have been zero. Count the callee + // register saves leading up to the local allocation. Set data members + // according to what we found if the values of the data members would + // change. SMPBasicBlock CurrBlock = this->Blocks.front(); for (list<SMPInstr>::iterator CurrInstr = CurrBlock.GetFirstInstr(); CurrInstr != CurrBlock.GetLastInstr(); // LastInstr is jump anyway ++CurrInstr) { - if (CurrInstr->MDIsFrameAllocInstr(LocalVarsSize)) - return false; + if (CurrInstr->MDIsPushInstr()) { + // We will make the gcc-linux assumption that a PUSH in + // the first basic block, prior to the stack allocating + // instruction, is a callee register save. To make this + // more robust, we should ensure that the register is from + // the callee saved group of registers, and that it has + // not been defined thus far in the function (else it might + // be a push of an outgoing argument to a call that happens + // in the first block when there are no locals). **!!!!** + if (CurrInstr->MDUsesCalleeSavedReg() + && !CurrInstr->HasSourceMemoryOperand()) { + SavedRegsSize += 4; // **!!** should check the size + } + else { + // Pushes of outgoing args can be scheduled so that + // they are mixed with the pushes of callee saved regs. + OtherPushesSize += 4; + } + } + else if (CurrInstr->MDIsFrameAllocInstr()) { + SavedRegsSize += OtherPushesSize; + // Get the size being allocated. + for (size_t index = 0; index < CurrInstr->NumUses(); ++index) { + // Find the immediate operand. + if (o_imm == CurrInstr->GetUse(index).type) { + // Get its value into LocalVarsSize. + long AllocValue = (signed long) CurrInstr->GetUse(index).value; + // One compiler might have sub esp,24 and another + // might have add esp,-24. Take the absolute value. + if (0 > AllocValue) + AllocValue = -AllocValue; + if (AllocValue != (long) this->LocalVarsSize) { + Changed = true; +#if SMP_DEBUG_FRAMEFIXUP + if (AllocValue + SavedRegsSize != OldFrameTotal) + msg("Total frame size changed: %s\n", this->FuncName); +#endif + this->LocalVarsSize = (asize_t) AllocValue; + this->CalleeSavedRegsSize = (ushort) SavedRegsSize; + NewLocalsSize = this->LocalVarsSize; + } + else { // Old value was correct; no change. + NewLocalsSize = this->LocalVarsSize; + if (SavedRegsSize != this->CalleeSavedRegsSize) { + this->CalleeSavedRegsSize = (ushort) SavedRegsSize; + Changed = true; +#if SMP_DEBUG_FRAMEFIXUP + msg("Only callee regs size changed: %s\n", this->FuncName); +#endif + } + } + } // end if (o_imm == ...) + } // end for all uses + break; // After frame allocation instr, we are done + } // end if (push) .. elsif frame allocating instr + } // end for all instructions in the first basic block + + // If we did not find an allocating instruction, see if it would keep + // the total size the same to set LocalVarsSize to 0 and to set + // CalleeSavedRegsSize to SavedRegsSize. If so, do it. If not, we + // might be better off to leave the numbers alone. + if (!Changed && (NewLocalsSize == 0)) { + if (OldFrameTotal == SavedRegsSize) { + this->CalleeSavedRegsSize = SavedRegsSize; + this->LocalVarsSize = 0; + Changed = true; + } +#if SMP_DEBUG_FRAMEFIXUP + else { + msg("Could not update frame sizes: %s\n", this->FuncName); + } +#endif } - // We did not find a stack frame allocation instruction in the first - // basic block, yet CalleeSavedRegsSize is zero and LocalVarsSize - // is not zero. Swap them. - this->CalleeSavedRegsSize = (ushort) this->LocalVarsSize; - this->LocalVarsSize = 0; - return true; + +#if SMP_DEBUG_FRAMEFIXUP + if ((0 < OtherPushesSize) && (0 < NewLocalsSize)) + msg("Extra pushes found of size %d in %s\n", OtherPushesSize, + this->FuncName); +#endif + + return Changed; } // end of SMPFunction::MDFixFrameInfo() // Emit the annotations describing the regions of the stack frame. @@ -1009,6 +1311,8 @@ void SMPFunction::Analyze(void) { return; } // end of SMPFunction::Analyze() +// Emit all annotations for the function, including all per-instruction +// annotations. void SMPFunction::EmitAnnotations(FILE *AnnotFile) { // Emit annotation for the function as a whole. if (this->StaticFunc) { @@ -1041,23 +1345,10 @@ void SMPFunction::EmitAnnotations(FILE *AnnotFile) { bool DeallocTrigger = false; for (CurrInst = Instrs.begin(); CurrInst != Instrs.end(); ++CurrInst) { ea_t addr = CurrInst->GetAddr(); - flags_t InstrFlags = getFlags(addr); - int OptType = CurrInst->GetOptType(); - char *disasm = CurrInst->GetDisasm(); - bool MemDest = CurrInst->HasDestMemoryOperand(); - bool MemSrc = CurrInst->HasSourceMemoryOperand(); - bool SecondSrcOperandNum = CurrInst->IsSecondSrcOperandNumeric(InstrFlags); - - ++OptCount[OptType]; // keep count for debugging info - - // If this is the instruction which allocated space - // for local variables, we want to emit annotations - // describing the stack frame. - if (addr == LocalVarsAllocInstr) { - EmitStackFrameAnnotations(AnnotFile, CurrInst); + if (this->LocalVarsAllocInstr == addr) { AllocSeen = true; + this->EmitStackFrameAnnotations(AnnotFile, CurrInst); } - // If this is the instruction which deallocated space // for local variables, we set a flag to remind us to // emit an annotation on the next instruction. @@ -1073,144 +1364,14 @@ void SMPFunction::EmitAnnotations(FILE *AnnotFile) { } else if (DeallocTrigger) { // Time for annotation qfprintf(AnnotFile, "%x %d DEALLOC STACK esp - %d %s\n", addr, - LocalVarsSize, LocalVarsSize, disasm); + LocalVarsSize, LocalVarsSize, CurrInst->GetDisasm()); DeallocTrigger = false; } -#if SMP_DEBUG_MEM - if (MemDest || MemSrc) { - msg("OptType: %d %s", OptType, disasm); - CurrInst->PrintOperands(); - } -#endif - - // Emit appropriate optimization annotations. - bool SDTInstrumentation = false; - switch (OptType) { - case 0: // SDT will have to handle these - { -#if SMP_DEBUG_TYPE0 - msg("OptType 0: %x %s\n", addr, disasm); -#endif - // mmStrata wants to suppress warnings on the PUSH - // instructions that precede the LocalVarsAllocInstr - // (i.e. the PUSHes of callee-saved regs). - if (!AllocSeen && CurrInst->MDIsPushInstr()) { - qfprintf(AnnotFile, "%x %d INSTR LOCAL NoWarn %s \n", - addr, -3, disasm); - } - else { - SDTInstrumentation = true; - } - break; - } - - case 1: // nothing for SDT to do - { qfprintf(AnnotFile, "%x %d INSTR LOCAL NoMetaUpdate %s \n", - addr, -1, disasm); - ++AnnotationCount[OptType]; - break; - } - - case 4: // INC, DEC, etc.: no SDT work unless MemDest - { if (MemDest || MemSrc) { - SDTInstrumentation = true; - break; // treat as category 0 - } - qfprintf(AnnotFile, "%x %d INSTR LOCAL Always1stSrc %s \n", - addr, -1, disasm); - ++AnnotationCount[OptType]; - break; - } - - case 5: // ADD, etc.: If numeric 2nd src operand, no SDT work. - { if (MemDest || MemSrc) { - SDTInstrumentation = true; - break; // treat as category 0 - } - if (SecondSrcOperandNum) { // treat as category 1 - qfprintf(AnnotFile, "%x %d INSTR LOCAL %s %s \n", - addr, -1, OptExplanation[OptType], disasm); - ++AnnotationCount[OptType]; - } - break; - } - - case 6: // Only OS code should include these; problem for SDT - { if (MemDest) { - SDTInstrumentation = true; - break; // treat as category 0 - } - qfprintf(AnnotFile, "%x %d INSTR LOCAL AlwaysPTR %s \n", - addr, -OptType, disasm); - ++AnnotationCount[OptType]; - break; - } - - case 8: // Implicitly writes to EDX:EAX, always numeric. - { qfprintf(AnnotFile, "%x %d INSTR LOCAL n EDX EAX ZZ %s %s \n", - addr, -2, OptExplanation[OptType], disasm); - ++AnnotationCount[OptType]; - SDTInstrumentation = true; - break; - } - - case 9: // Either writes to FP reg (cat. 1) or memory (cat. 0) - { if (MemDest) { -#if SMP_DEBUG - // MemDest seems to happen too much. - msg("Floating point MemDest: %s \n", disasm); -#endif - SDTInstrumentation = true; - break; // treat as category 0 - } - qfprintf(AnnotFile, "%x %d INSTR LOCAL %s %s \n", - addr, -1, OptExplanation[OptType], disasm); - ++AnnotationCount[OptType]; - break; - } - - default: // 2,3,7: Optimization possibilities depend on operands - { -#if SMP_DEBUG2 - if (OptType == 3) { // MOV instr class - if (MemDest) { - msg("MemDest on MOV: %s\n", disasm); - } - else if (!SecondSrcOperandNum) { - msg("MOV: not 2nd op numeric: %s\n", disasm); - CurrInst->PrintOperands(); - } - } -#endif - SDTInstrumentation = true; - if (MemDest) { -#if SMP_DEBUG_XOR - if (OptType == 2) - msg("MemDest on OptType 2: %s\n", disasm); -#endif - break; // treat as category 0 - } - if ((OptType == 2) || (OptType == 7) || SecondSrcOperandNum) { - qfprintf(AnnotFile, "%x %d INSTR LOCAL n %s %s %s \n", - addr, -2, CurrInst->DestString(OptType), - OptExplanation[OptType], disasm); - ++AnnotationCount[OptType]; - } - break; - } - } // end switch (OptType) - // If mmStrata is going to have to deal with the - // instruction, then we can annotate EBP and ESP - // relative constant offsets. If we have emitted - // an annotation of type -1, there is no point - // in telling mmStrata about these constants. - if (SDTInstrumentation) { - CurrInst->AnnotateStackConstants(UseFP, AnnotFile); - } + CurrInst->EmitAnnotations(this->UseFP, AllocSeen, AnnotFile); } // end for (ea_t addr = FuncInfo->startEA; ...) return; -} +} // end of SMPFunction::EmitAnnotations() // Initialize the DFACategory[] array to define instruction classes // for the purposes of data flow analysis.