SMPStaticAnalyzer.cpp

//
// SMPStaticAnalyzer.cpp
//
// This plugin performs the static analyses needed for the SMP project
//   (Software Memory Protection).
//

#include <ida.hpp>
#include <idp.hpp>
#include <allins.hpp>
#include <auto.hpp>
#include <bytes.hpp>
#include <funcs.hpp>
#include <intel.hpp>
#include <loader.hpp>
#include <lines.hpp>
#include <name.hpp>
#include <ua.hpp>

#include "SMPStaticAnalyzer.h"
#include "SMPDataFlowAnalysis.h"


// Set to 1 for debugging output
#define SMP_DEBUG 1
#define SMP_DEBUG2 1   // verbose
#define SMP_DEBUG3 0   // verbose
#define SMP_DEBUG_MEM 0 // print memory operands
#define SMP_DEBUG_TYPE0 0 // Output instr info for OptType = 0
#define SMP_DEBUG_ORPHANS 1  // find code outside of functions

// Set to 1 when doing a binary search using SMP_DEBUG_COUNT to find
//  which function is causing a problem.
#define SMP_BINARY_DEBUG 0
#define SMP_DEBUG_COUNT 356  // How many funcs to process in problem search
int FuncsProcessed = 0;


// Define optimization categories for instructions.
int OptCategory[NN_last+1];
// Initialize the OptCategory[] array.
void InitOptCategory(void);

// Keep statistics on how many instructions we saw in each optimization
//  category, and how many optimizing annotations were emitted for
//  each category.
int OptCount[LAST_OPT_CATEGORY + 1];
int AnnotationCount[LAST_OPT_CATEGORY + 1];

static char *RegNames[R_of + 1] =
	{ "EAX", "ECX", "EDX", "EBX", "ESP", "EBP", "ESI", "EDI",
	  "R8", "R9", "R10", "R11", "R12", "R13", "R14", "R15",
	  "AL", "CL", "DL", "BL", "AH", "CH", "DH", "BH",
	  "SPL", "BPL", "SIL", "DIL", "EIP", "ES", "CS", "SS",
	  "DS", "FS", "GS", "CF", "ZF", "SF", "OF" 
	};

// The types of data objects based on their first operand flags.
static char *DataTypes[] = { "VOID", "NUMHEX", "NUMDEC", "CHAR",
		"SEG", "OFFSET", "NUMBIN", "NUMOCT", "ENUM", "FORCED", 
		"STRUCTOFFSET", "STACKVAR", "NUMFLOAT", "UNKNOWN", 
		"UNKNOWN", "UNKNOWN", 0};

void IDAP_run(int);
void FindOrphanedCode(segment_t *, FILE *);
void AuditCodeTargets(void);
ea_t FindNewFuncLimit(ea_t);
void SpecialDebugOutput(void);

static int idaapi idp_callback(void *, int event_id, va_list va) {
	if (event_id == ph.auto_empty_finally) {   // IDA analysis is done
		IDAP_run(0);
		qexit(0);
	}
	return 0;
}

int IDAP_init(void) {
#if 0 // We are now calling from the SMP.idc script.
	// Skip this plugin if it was not specified by the user on the
	//  command line.
	if (get_plugin_options("SMPStaticAnalyzer") == NULL) {
		msg("IDAP_init point 2.\n");
		return PLUGIN_SKIP;
	}
#endif
	// Ensure correct working environment.
	if ((inf.filetype != f_ELF) && (inf.filetype != f_PE)) {
		error("Executable format must be PE or ELF.");
		return PLUGIN_SKIP;
	}
	if (ph.id != PLFM_386) {
		error("Processor must be x86.");
 		return PLUGIN_SKIP;
	}
	hook_to_notification_point(HT_IDP, idp_callback, NULL);
    InitOptCategory();
	InitDFACategory();
	return PLUGIN_KEEP;
} // end of IDAP_init

void IDAP_term(void) {
	unhook_from_notification_point(HT_IDP, idp_callback, NULL);
	return;
}

void IDAP_run(int arg) {

	segment_t *seg;
	char buf[MAXSTR];
	ea_t ea;
	flags_t ObjFlags;
	bool ReadOnlyFlag;
	FILE *SymsFile;
	char FuncName[MAXSTR];			
	SMPFunction *CurrFunc = NULL;
	bool FuncsDumped = false;

#if SMP_DEBUG
	msg("Beginning IDAP_run.\n");
#endif
	// Open the output file.
	SymsFile = qfopen("SMP.annot", "w");
	if (NULL == SymsFile) {
		error("FATAL: Cannot open output file SMP.annot\n");
		return;
	}

	(void) memset(OptCount, 0, sizeof(OptCount));
	(void) memset(AnnotationCount, 0, sizeof(AnnotationCount));

	// Pre-audit the IDA database by seeing if all branches and calls
	//  have proper code targets and code cross references.
	SpecialDebugOutput();
	AuditCodeTargets();

	// First, examine the data segments and print info about static
	//   data, such as name/address/size. Do the same for functions in
	//   code segments.
	// Loop through all segments.
	for (int SegIndex = 0; SegIndex < get_segm_qty(); ++SegIndex) {
		char SegName[MAXSTR];
		seg = getnseg(SegIndex);
		ssize_t SegNameSize = get_segm_name(seg, SegName, sizeof(SegName) - 1);

		// We are only interested in the data segments of type
		// SEG_DATA, SEG_BSS and SEG_COMM.
		if ((seg->type == SEG_DATA) || (seg->type == SEG_BSS)
		    || (seg->type == SEG_COMM)) {
			// Loop through each of the segments we are interested in,
			//  examining all data objects (effective addresses).
			ReadOnlyFlag = ((seg->perm & SEGPERM_READ) && (!(seg->perm & SEGPERM_WRITE)));
#if SMP_DEBUG
			msg("Starting data segment of type %d", seg->type);
			if (SegNameSize > 0)
				msg(" SegName: %s\n", SegName);
			else
				msg("\n");
			if (ReadOnlyFlag) {
				msg("Read-only data segment.\n");
			}
#endif
			ea = seg->startEA;
			while (ea < seg->endEA) {
				ObjFlags = get_flags_novalue(ea);
				// Only process head bytes of data objects, i.e. isData().
				if (isData(ObjFlags)) {
				    // Compute the size of the data object.
					ea_t NextEA = ea;
				    do {
				       NextEA = nextaddr(NextEA);
					} while ((NextEA < seg->endEA) && (!isHead(get_flags_novalue(NextEA))));
				    size_t ObjSize = (size_t) (NextEA - ea);
					// Get the data object name using its address.
				    char *TrueName = get_true_name(BADADDR, ea, buf, sizeof(buf));
					if (NULL == TrueName) {
						qstrncpy(buf, "SMP_dummy0", 12);
					}
				    // Output the name, address, size, and type info.
					if (ReadOnlyFlag) {
						qfprintf(SymsFile, 
							"%x %d OBJECT GLOBAL %s  %s RO\n", ea, ObjSize,
				  			buf, DataTypes[get_optype_flags0(ObjFlags) >> 20]);
					}
					else {
						qfprintf(SymsFile, 
							"%x %d OBJECT GLOBAL %s  %s RW\n", ea, ObjSize,
				  			buf, DataTypes[get_optype_flags0(ObjFlags) >> 20]);
					}
					// Move on to next data object
					ea = NextEA;
				}
				else {
					ea = nextaddr(ea);
				}
			} // end while (ea < seg->endEA)
		} // end if (seg->type == SEG_DATA ...)
		else if (seg->type == SEG_CODE) {
#if SMP_DEBUG
			msg("Starting code segment");