SMPStaticAnalyzer.cpp

//
// SMPStaticAnalyzer.cpp
//
// This plugin performs the static analyses needed for the SMP project
//   (Software Memory Protection).
//

#include <list>
#include <vector>

#include <string>

#include <ida.hpp>
#include <idp.hpp>
#include <allins.hpp>
#include <auto.hpp>
#include <bytes.hpp>
#include <funcs.hpp>
#include <intel.hpp>
#include <loader.hpp>
#include <lines.hpp>
#include <nalt.hpp>
#include <name.hpp>
#include <ua.hpp>

#include "SMPStaticAnalyzer.h"
#include "SMPDataFlowAnalysis.h"


// Set to 1 for debugging output
#define SMP_DEBUG 1
#define SMP_DEBUG2 0   // verbose
#define SMP_DEBUG3 0   // verbose
#define SMP_DEBUG_MEM 0 // print memory operands
#define SMP_DEBUG_TYPE0 0 // Output instr info for OptType = 0
#define SMP_DEBUG_ORPHANS 1  // find code outside of functions
#define SMP_DEBUG_CHUNKS 0 // restructuring tail chunks, shared chunks, etc.
#define SMP_DEBUG_DATA_ONLY 0  // Find & fix data addresses in code segments

// Set to 1 when doing a binary search using SMP_DEBUG_COUNT to find
//  which function is causing a problem.
#define SMP_BINARY_DEBUG 0
#define SMP_DEBUG_COUNT 356  // How many funcs to process in problem search
int FuncsProcessed = 0;

#define SMP_FIXUP_IDB 1  // Try to fix the IDA database?
#define SMP_DEBUG_FIXUP_IDB 1  // debugging output for FixupIDB chain

// Define optimization categories for instructions.
int OptCategory[NN_last+1];
// Initialize the OptCategory[] array.
void InitOptCategory(void);

// Keep statistics on how many instructions we saw in each optimization
//  category, and how many optimizing annotations were emitted for
//  each category.
int OptCount[LAST_OPT_CATEGORY + 1];
int AnnotationCount[LAST_OPT_CATEGORY + 1];

static char *RegNames[R_of + 1] =
	{ "EAX", "ECX", "EDX", "EBX", "ESP", "EBP", "ESI", "EDI",
	  "R8", "R9", "R10", "R11", "R12", "R13", "R14", "R15",
	  "AL", "CL", "DL", "BL", "AH", "CH", "DH", "BH",
	  "SPL", "BPL", "SIL", "DIL", "EIP", "ES", "CS", "SS",
	  "DS", "FS", "GS", "CF", "ZF", "SF", "OF" 
	};

// The types of data objects based on their first operand flags.
static char *DataTypes[] = { "VOID", "NUMHEX", "NUMDEC", "CHAR",
		"SEG", "OFFSET", "NUMBIN", "NUMOCT", "ENUM", "FORCED", 
		"STRUCTOFFSET", "STACKVAR", "NUMFLOAT", "UNKNOWN", 
		"UNKNOWN", "UNKNOWN", 0};

// Filename (not including path) of executable being analyzed.
static char RootFileName[MAXSTR];

// Code addresses identified by a disassembler, such as objdump on
//  Linux. These can be used to improve the code vs. data identification
//  of IDA Pro.
vector<ea_t> DisasmLocs;
// Code addresses as identified by IDA Pro, to be compared to DisasmLocs.
vector<ea_t> IDAProLocs;

// Function start and end addresses (for function entry chunks only).
//  Kept here because IDA Pro 5.1 seems to have a memory overwriting
//  problem when iterating through all functions in the program. An existing
//  func_t *ChunkInfo data structure was getting overwritten by one of the 
//  function func_t data structures, causing changes of startEA and endEA among
//  other things.
struct SMP_bounds_t {
	ea_t startEA;
	ea_t endEA;
};

vector<SMP_bounds_t> FuncBounds;

// List of functions that need to be reanalyzed after all the code fixup
//  and code discovery is complete. Kept as a list of addresses; any address
//  within the function is good enough to designate it.
list<ea_t> FuncReanalyzeList;

// A code region that has been converted from data but has code addresses that
//  need to be reanalyzed. This is usually because a former data address is
//  now a jump to a code target that is still a data address. We have to wait
//  until the target has become code before IDA will accept the jump as valid.
class FixupRegion {
public:
	FixupRegion(SMP_bounds_t);
	inline ea_t GetStart(void) const { return CodeRegion.startEA; };
	inline ea_t GetEnd(void) const { return CodeRegion.endEA; };
	inline void SetStart(ea_t addr) { CodeRegion.startEA = addr; };
	list<ea_t> FixupInstrs; // easier to expose than to encapsulate
private:
	SMP_bounds_t CodeRegion;
};

FixupRegion::FixupRegion(SMP_bounds_t Range) {
	this->CodeRegion = Range;
	return;
}

// List of code regions that were not completely analysed because of jump to
//  data considerations.
list<FixupRegion> CodeReanalyzeList;

void IDAP_run(int);

// Functions for diagnosing and/or fixing problems in the IDA database.
void FixupIDB(void);  // Driver for all other fixing functions.
void FindDataInCode(void);
void AuditTailChunkOwnership(void);
void FindOrphanedCode(segment_t *, FILE *);
void FixCodeIdentification(void);
int FixupNewCodeChunks(void);
void AuditCodeTargets(void);
ea_t FindNewFuncLimit(ea_t);
void SpecialDebugOutput(void);
void RemoveIDACodeAddr(ea_t);

static int idaapi idp_callback(void *, int event_id, va_list va) {
	if (event_id == ph.auto_empty_finally) {   // IDA analysis is done
		IDAP_run(0);
		qexit(0);
	}
	return 0;
}

int IDAP_init(void) {
#if 0 // We are now calling from the SMP.idc script.
	// Skip this plugin if it was not specified by the user on the
	//  command line.
	if (get_plugin_options("SMPStaticAnalyzer") == NULL) {
		msg("IDAP_init point 2.\n");
		return PLUGIN_SKIP;
	}
#endif
	// Ensure correct working environment.
	if ((inf.filetype != f_ELF) && (inf.filetype != f_PE)) {
		error("Executable format must be PE or ELF.");
		return PLUGIN_SKIP;
	}
	if (ph.id != PLFM_386) {
		error("Processor must be x86.");
 		return PLUGIN_SKIP;
	}
	hook_to_notification_point(HT_IDP, idp_callback, NULL);
    InitOptCategory();
	InitDFACategory();
	return PLUGIN_KEEP;
} // end of IDAP_init

void IDAP_term(void) {
	unhook_from_notification_point(HT_IDP, idp_callback, NULL);
	return;
}

void IDAP_run(int arg) {
	segment_t *seg;
	char buf[MAXSTR];
	ea_t ea;
	flags_t ObjFlags;
	bool ReadOnlyFlag;
	FILE *SymsFile;
	SMPFunction *CurrFunc = NULL;
	bool FuncsDumped = false;

#if SMP_DEBUG2
	char FuncName[MAXSTR];			
#endif

#if SMP_DEBUG
	msg("Beginning IDAP_run.\n");
#endif
	// Open the output file.
	ssize_t FileLen;
	FileLen = get_root_filename(RootFileName, sizeof(RootFileName) - 1);
	string SymsFileName(RootFileName);
	string FileSuffix(".annot");
	SymsFileName += FileSuffix;
	SymsFile = qfopen(SymsFileName.c_str(), "w");