SMPStaticAnalyzer.cpp

/*
 * SMPStaticAnalyzer.cpp - <see below>.
 *
 * Copyright (c) 2000, 2001, 2010 - University of Virginia 
 *
 * This file is part of the Memory Error Detection System (MEDS) infrastructure.
 * This file may be used and modified for non-commercial purposes as long as 
 * all copyright, permission, and nonwarranty notices are preserved.  
 * Redistribution is prohibited without prior written consent from the University 
 * of Virginia.
 *
 * Please contact the authors for restrictions applying to commercial use.
 *
 * THIS SOURCE IS PROVIDED "AS IS" AND WITHOUT ANY EXPRESS OR IMPLIED
 * WARRANTIES, INCLUDING, WITHOUT LIMITATION, THE IMPLIED WARRANTIES OF
 * MERCHANTIBILITY AND FITNESS FOR A PARTICULAR PURPOSE.
 *
 * Author: University of Virginia
 * e-mail: jwd@virginia.com
 * URL   : http://www.cs.virginia.edu/
 *
 * Additional copyrights 2010, 2011 by Zephyr Software LLC
 * e-mail: {clc,jwd}@zephyr-software.com
 * URL   : http://www.zephyr-software.com/
 */

//
// SMPStaticAnalyzer.cpp
//
// This plugin performs the static analyses needed for the SMP project
//   (Software Memory Protection).
//

using namespace std;

#include <list>
#include <vector>

#include <string>

#include <pro.h>
#include <ida.hpp>
#include <idp.hpp>
#include <allins.hpp>
#include <auto.hpp>
#include <bytes.hpp>
#include <funcs.hpp>
#include <intel.hpp>
#include <loader.hpp>
#include <lines.hpp>
#include <nalt.hpp>
#include <name.hpp>
#include <ua.hpp>

#include "SMPStaticAnalyzer.h"
#include "SMPDBInterface.h"
#include "SMPDataFlowAnalysis.h"
#include "SMPProgram.h"
#include "SMPFunction.h"
#include "SMPInstr.h"
#include "ProfilerInformation.h"


// Set to 1 for debugging output
#define SMP_DEBUG 1
#define SMP_DEBUG2 0   // verbose
#define SMP_DEBUG3 0   // verbose
#define SMP_DEBUG_MEM 0 // print memory operands
#define SMP_DEBUG_TYPE0 0 // Output instr info for OptType = 0
#define SMP_DEBUG_CHUNKS 0 // restructuring tail chunks, shared chunks, etc.
#define SMP_DEBUG_DATA_ONLY 0  // Find & fix data addresses in code segments

// Set to 1 when doing a binary search using SMP_DEBUG_COUNT to find
//  which function is causing a problem.
#define SMP_BINARY_DEBUG 0
#define SMP_DEBUG_COUNT 356  // How many funcs to process in problem search
int FuncsProcessed = 0;

#define SMP_FIXUP_IDB 0  // Try to fix the IDA database?
#define SMP_DEBUG_FIXUP_IDB 0  // debugging output for FixupIDB chain
#define SMP_FIND_ORPHANS 1  // find code outside of functions
#define SMP_DEBUG_CODE_ORPHANS 1 // Detect whether we are causing code to be orphaned

#if SMP_DEBUG_CODE_ORPHANS
set<ea_t> CodeOrphans;
#endif

// Define optimization categories for instructions.
int OptCategory[NN_last + 1];
// Initialize the OptCategory[] array.
void InitOptCategory(void);

// Record which opcodes change the stack pointer, and by how many
//  bytes up (reduction in stack size for stacks that grow downward)
//  or down (increase in stack size for stacks that grow downward).
sval_t StackAlteration[NN_last + 1];
// Initialize the StackAlteration[] array.
void InitStackAlteration(void);

// Keep statistics on how many instructions we saw in each optimization
//  category, and how many optimizing annotations were emitted for
//  each category.
int OptCount[LAST_OPT_CATEGORY + 1];
int AnnotationCount[LAST_OPT_CATEGORY + 1];

// Unique data referent number to use in data annotations.
unsigned long DataReferentID;

// Debugging counters for analyzing memory usage.
unsigned long UnusedInstrCount;
unsigned long UnusedBlockCount;
unsigned long UnusedStructCount;
unsigned long UnusedIntCount;

#if SMP_COUNT_MEMORY_ALLOCATIONS
// Counters for analyzing memory use for allocated and used objects.
unsigned long SMPInstCount;
unsigned long SMPBlockCount;
unsigned long SMPFuncCount;
unsigned long SMPGlobalVarCount;
unsigned long SMPLocalVarCount;
unsigned long SMPDefUseChainCount;
unsigned long SMPInstBytes;
unsigned long SMPDefUseChainBytes;
#endif

// The types of data objects based on their first operand flags.
const char *DataTypes[] = { "VOID", "NUMHEX", "NUMDEC", "CHAR",
		"SEG", "OFFSET", "NUMBIN", "NUMOCT", "ENUM", "FORCED", 
		"STRUCTOFFSET", "STACKVAR", "NUMFLOAT", "UNKNOWN", 
		"UNKNOWN", "UNKNOWN", 0};

// Filename (not including path) of executable being analyzed.
static char RootFileName[MAXSTR];

// strings for printing ZST_SysCallType
const char *CallTypeNames[4] = { "Unrestricted", "High-Privilege", "File-Access", "Network-Access" };

DisAsmString DisAsmText;

// Operand type that can have all fields initialized to o_void and zero
//  values, to be used to copy-initialize operands that we are adding to
//  RTLs and DEF and USE lists.
op_t InitOp; 

// File foo.exe.alarms for Zephyr Security Toolkit security alarm messages.
FILE *ZST_AlarmFile;

// Code addresses identified by a disassembler, such as objdump on
//  Linux. These can be used to improve the code vs. data identification
//  of IDA Pro.
vector<ea_t> DisasmLocs;
// Code addresses as identified by IDA Pro, to be compared to DisasmLocs.
vector<ea_t> IDAProLocs;

// Function start and end addresses (for function entry chunks only).
//  Kept here because IDA Pro 5.1 seems to have a memory overwriting
//  problem when iterating through all functions in the program. An existing
//  func_t *ChunkInfo data structure was getting overwritten by one of the 
//  function func_t data structures, causing changes of startEA and endEA among
//  other things.
struct SMP_bounds_t {
	ea_t startEA;
	ea_t endEA;
};

vector<SMP_bounds_t> FuncBounds;

// List of functions that need to be reanalyzed after all the code fixup
//  and code discovery is complete. Kept as a list of addresses; any address
//  within the function is good enough to designate it.
list<ea_t> FuncReanalyzeList;

// A code region that has been converted from data but has code addresses that
//  need to be reanalyzed. This is usually because a former data address is
//  now a jump to a code target that is still a data address. We have to wait
//  until the target has become code before IDA will accept the jump as valid.
class FixupRegion {
public:
	FixupRegion(SMP_bounds_t);
	inline ea_t GetStart(void) const { return CodeRegion.startEA; };
	inline ea_t GetEnd(void) const { return CodeRegion.endEA; };
	inline void SetStart(ea_t addr) { CodeRegion.startEA = addr; };
	list<ea_t> FixupInstrs; // easier to expose than to encapsulate
private:
	SMP_bounds_t CodeRegion;
};

FixupRegion::FixupRegion(SMP_bounds_t Range) {
	this->CodeRegion = Range;
	return;
}

// List of code regions that were not completely analysed because of jump to
//  data considerations.
list<FixupRegion> CodeReanalyzeList;

// Map library function names to their system call type.
map<string, ZST_SysCallType> ZST_FuncTypeMap;

// Map system call types to their Zephyr Security Toolkit security policy.
map<ZST_SysCallType, ZST_Policy> ZST_TypePolicyMap;

// Set of whitelisted file locations.
set<string> ZST_FileLocWhitelist;

// Set of whitelisted network locations.
set<string> ZST_NetworkLocWhitelist;

// Set of blacklisted file locations.
set<string> ZST_FileLocBlacklist;

// Set of blacklisted network locations.
set<string> ZST_NetworkLocBlacklist;


void IDAP_run(int);

// Functions for diagnosing and/or fixing problems in the IDA database.
void FixupIDB(void);  // Driver for all other fixing functions.
void FindDataInCode(void);
void AuditTailChunkOwnership(void);
void FindOrphanedCode(segment_t *, FILE *, FILE *);
void Debug_FindOrphanedCode(segment_t *, bool);
void FixCodeIdentification(void);
int FixupNewCodeChunks(void);
void AuditCodeTargets(void);
ea_t FindNewFuncLimit(ea_t);
void SpecialDebugOutput(void);
void RemoveIDACodeAddr(ea_t);
void ZST_InitPolicies(const char *);

static int idaapi idp_callback(void *, int event_id, va_list va) {
	if (event_id == ph.auto_empty_finally) {   // IDA analysis is done
		IDAP_run(0);
		qexit(0);
	}
	return 0;
}

int IDAP_init(void) {
#if 0 // We are now calling from the SMP.idc script.
	// Skip this plugin if it was not specified by the user on the
	//  command line.
	if (get_plugin_options("SMPStaticAnalyzer") == NULL) {
		SMP_msg("IDAP_init point 2.\n");
		return PLUGIN_SKIP;
	}
#endif
	// Ensure correct working environment.
	if ((inf.filetype != f_ELF) && (inf.filetype != f_PE)) {
		error("Executable format must be PE or ELF.");
		return PLUGIN_SKIP;
	}
	if (ph.id != PLFM_386) {
		error("Processor must be x86.");
 		return PLUGIN_SKIP;
	}
	hook_to_notification_point(HT_IDP, idp_callback, NULL);
	DataReferentID = 1;
	UnusedStructCount = 0;
	UnusedIntCount = 0;
#if SMP_COUNT_MEMORY_ALLOCATIONS
	SMPInstCount = 0;
	SMPBlockCount = 0;
	SMPDefUseChainCount = 0;
	SMPFuncCount = 0;
	SMPGlobalVarCount = 0;
	SMPLocalVarCount = 0;
	SMPInstBytes = 0;
	SMPDefUseChainBytes = 0;
#endif
	InitOp.type = o_void;
	InitOp.addr = 0;
	InitOp.dtyp = dt_dword;
	InitOp.flags = 0;
	InitOp.n = 0;
	InitOp.offb = 0;
	InitOp.offo = 0;
	InitOp.reg = R_none;
	InitOp.specflag1 = 0;
	InitOp.specflag2 = 0;
	InitOp.specflag3 = 0;
	InitOp.specflag4 = 0;
	InitOp.specval = 0;
	InitOp.value = 0;
	ZST_AlarmFile = NULL;
#ifdef STARS_IRDB_INTERFACE
	SMPLogFile = NULL;
#endif
    InitOptCategory();
	InitDFACategory();
	InitTypeCategory();
	InitSMPDefsFlags();
	InitSMPUsesFlags();
	InitLibFuncFGInfoMaps();
	InitIntegerErrorCallSinkMap();
	InitStackAlteration();
	return PLUGIN_KEEP;
} // end of IDAP_init

void IDAP_term(void) {
	unhook_from_notification_point(HT_IDP, idp_callback, NULL);
	return;
}

void IDAP_run(int arg) {
	FILE *AnnotFile;
	FILE *InfoAnnotFile;

#ifdef STARS_IRDB_INTERFACE
	string ZSTLogFileName(RootFileName);
	string LogFileSuffix(".STARSlog");
	ZSTLogFileName += LogFileSuffix;
	SMPLogFile = SMP_fopen(ZSTLogFileName.c_str(), "w");
	if (NULL == SMPLogFile) {
		error("ERROR: Cannot open STARS log file %s\n", ZSTLogFileName.c_str());
		error("Redirecting to stderr.\n");
		SMPLogFile = stderr;
	}
#endif

#if SMP_DEBUG
	SMP_msg("Beginning IDAP_run.\n");
#endif

	SMP_msg("IDA SDK version: %d \n", IDA_SDK_VERSION);

	// Open the output file.
	ssize_t FileLen;
	FileLen = get_root_filename(RootFileName, sizeof(RootFileName) - 1);
	string AnnotFileName(RootFileName);
	string FileSuffix(".annot");
	AnnotFileName += FileSuffix;
	string InfoAnnotFileName(RootFileName);
	string InfoFileSuffix(".infoannot");
	InfoAnnotFileName += InfoFileSuffix;
	string ZSTPolicyFileName(RootFileName);
	string PolicyFileSuffix(".policy");
	ZSTPolicyFileName += PolicyFileSuffix;
	string ZSTAlarmFileName(RootFileName);
	string AlarmFileSuffix(".alarms");
	ZSTAlarmFileName += AlarmFileSuffix;

	ea_t RecentAddr;
#if SMP_DEBUG_CODE_ORPHANS
	CodeOrphans.clear();
	RecentAddr = BADADDR;
#if IDA_SDK_VERSION < 600
	for (int SegIndex = 0; SegIndex < SMP_get_segm_qty(); ++SegIndex) {
		segment_t *seg = SMP_getnseg(SegIndex);
#else
	for (segment_t *seg = SMP_get_first_seg(); NULL != seg; seg = SMP_get_next_seg(RecentAddr)) {
		RecentAddr = seg->startEA;
#endif
		if (seg->type == SEG_CODE)
			Debug_FindOrphanedCode(seg, true);
	}
#endif

	SMPProgram *CurrProg = new SMPProgram();
	CurrProg->AnalyzeData(); // Analyze static data in the executable

	// read the Profiler generated information into a new prof_info class 
	ProfilerInformation *prof_info = new ProfilerInformation(AnnotFileName.c_str(), CurrProg);

	AnnotFile = SMP_fopen(AnnotFileName.c_str(), "w");
	if (NULL == AnnotFile) {
		error("FATAL ERROR: Cannot open output file %s\n", AnnotFileName.c_str());
		delete prof_info;
		return;
	}
	InfoAnnotFile = SMP_fopen(InfoAnnotFileName.c_str(), "w");
	if (NULL == InfoAnnotFile) {
		error("FATAL ERROR: Cannot open output file %s\n", InfoAnnotFileName.c_str());
		delete prof_info;
		return;
	}
	ZST_AlarmFile = SMP_fopen(ZSTAlarmFileName.c_str(), "w");
	if (NULL == ZST_AlarmFile) {
		error("FATAL ERROR: Cannot open security alarms file %s\n", ZSTAlarmFileName.c_str());
		delete prof_info;
		return;
	}

	// Read the Zephyr Security Toolkit system call security policies, if available.
	ZST_InitPolicies(ZSTPolicyFileName.c_str());

	(void) memset(OptCount, 0, sizeof(OptCount));
	(void) memset(AnnotationCount, 0, sizeof(AnnotationCount));

	// Record the start and end addresses for all function entry
	//  chunks in the program.
	FuncBounds.reserve(10 + get_func_qty());
	for (size_t FuncIndex = 0; FuncIndex < get_func_qty(); ++FuncIndex) {
		func_t *FuncInfo = getn_func(FuncIndex);
		SMP_bounds_t temp;
		temp.startEA = FuncInfo->startEA;
		temp.endEA = FuncInfo->endEA;
		FuncBounds.push_back(temp);
	}

#if SMP_DEBUG_DATA_ONLY
	FindDataInCode();
	FixCodeIdentification();
	SMP_fclose(SymsFile);
	delete prof_info;
	return;
#endif

	// Pre-audit the IDA database by seeing if the distinction
	//  between code and data can be improved, and if all branches
	//  and calls have proper code targets and code cross references.
#if SMP_FIXUP_IDB
	FixupIDB();
#endif

	if (0 < prof_info->GetProfilerAnnotationCount()) {
		SMP_msg("Calling InferDataGranularity\n");
		SMP_msg("ptr to MemoryAccessInfo: %p\n", prof_info->GetMemoryAccessInfo());
		prof_info->GetMemoryAccessInfo()->InferDataGranularity();
		SMP_msg("Returned from InferDataGranularity\n");
	}
	CurrProg->ProfGranularityFinished(AnnotFile, InfoAnnotFile);
	CurrProg->Analyze(prof_info);
	CurrProg->EmitAnnotations(AnnotFile, InfoAnnotFile);

#if SMP_DEBUG_CODE_ORPHANS
	RecentAddr = BADADDR;
#if IDA_SDK_VERSION < 600
	for (int SegIndex = 0; SegIndex < SMP_get_segm_qty(); ++SegIndex) {
		segment_t *seg = SMP_getnseg(SegIndex);
#else
	for (segment_t *seg = SMP_get_first_seg(); NULL != seg; seg = SMP_get_next_seg(RecentAddr)) {
		RecentAddr = seg->startEA;
#endif
		if (seg->type == SEG_CODE)
			Debug_FindOrphanedCode(seg, true);
	}
#endif

#if SMP_FIND_ORPHANS
	RecentAddr = BADADDR;
#if IDA_SDK_VERSION < 600
	for (int SegIndex = 0; SegIndex < SMP_get_segm_qty(); ++SegIndex) {
		segment_t *seg = SMP_getnseg(SegIndex);
#else
	for (segment_t *seg = SMP_get_first_seg(); NULL != seg; seg = SMP_get_next_seg(RecentAddr)) {
		RecentAddr = seg->startEA;
#endif
		if (seg->type == SEG_CODE)
			FindOrphanedCode(seg, AnnotFile, InfoAnnotFile);
	}
#endif

	for (int OptType = 0; OptType <= LAST_OPT_CATEGORY; ++OptType) {
		SMP_msg("Optimization Category Count %d:  %d Annotations: %d\n",
			OptType, OptCount[OptType], AnnotationCount[OptType]);
	}

	SMP_fclose(AnnotFile);
	SMP_fclose(InfoAnnotFile);
	SMP_fclose(ZST_AlarmFile);
	delete CurrProg;
	delete prof_info;
	return;
} // end IDAP_run()

char IDAP_comment[] = "ZephyrSoftware STARS (Static Analyzer for Reliability and Security)";
char IDAP_help[] = "Good luck";
char IDAP_name[] = "SMPStaticAnalyzer";
char IDAP_hotkey[] = "Alt-J";

plugin_t PLUGIN = {
	IDP_INTERFACE_VERSION,
	0,
	IDAP_init,
	IDAP_term,
	IDAP_run,
	IDAP_comment,
	IDAP_help,
	IDAP_name,
	IDAP_hotkey
};

// Find all code addresses in the IDA database and enter them into
//  IDAProLocs. Find all code addresses identified by the external
//  disassembler (e.g. objdump) and enter them into DisasmLocs.
void FindCodeAddresses(void) {
	// Read in code addresses as found by an external disassembler.
	ea_t CurrDisasmAddr;
	string DisasmFileName(RootFileName);
	string FileSuffix(".SMPobjdump");
	DisasmFileName += FileSuffix;
	FILE *DisasmFile = SMP_fopen(DisasmFileName.c_str(), "r");
	if (NULL == DisasmFile) {
		error("FATAL: Cannot open input file %s\n", DisasmFileName.c_str());
		return;
	}

#define DISASM_RESERVE_SIZE  50000
	DisasmLocs.reserve(DISASM_RESERVE_SIZE);
	int ScanReturn = qfscanf(DisasmFile, "%x", &CurrDisasmAddr);
	while (1 == ScanReturn) {
		int NextChar;
		DisasmLocs.push_back(CurrDisasmAddr);
		// Swallow the rest of the input line and get the next address.
		do {
			NextChar = qfgetc(DisasmFile);
		} while ((EOF != NextChar) && ('\n' != NextChar));
		ScanReturn = qfscanf(DisasmFile, "%x", &CurrDisasmAddr);
	} // end while (1 == ScanReturn)
	if (0 >= DisasmLocs.size()) {
		SMP_msg("ERROR: No addresses read from %s\n", DisasmFileName.c_str());
		SMP_fclose(DisasmFile);
		DisasmLocs.clear();
		return;
	}
	else {
		SMP_msg("%zu Disasm addresses read from %s\n", DisasmLocs.size(),
			DisasmFileName.c_str());
		SMP_fclose(DisasmFile);
	}

	// Find all the code locs in the IDA Pro database. As we find
	//  them, store them in IDAProLocs.
	ea_t RecentAddr = BADADDR;
#if IDA_SDK_VERSION < 600
	for (int SegIndex = 0; SegIndex < SMP_get_segm_qty(); ++SegIndex) {
		segment_t *seg = SMP_getnseg(SegIndex);
#else
	for (segment_t *seg = SMP_get_first_seg(); NULL != seg; seg = SMP_get_next_seg(RecentAddr)) {
		RecentAddr = seg->startEA;
#endif
		if (SEG_CODE != seg->type)
			continue;

		for (ea_t addr = seg->startEA; addr < seg->endEA; addr = get_item_end(addr)) {
			flags_t InstrFlags = getFlags(addr);
			if (isHead(InstrFlags) && isCode(InstrFlags)) {
				IDAProLocs.push_back(addr);
#if 0
				if ((0x806cda4 <= addr) && (0x806cf99 >= addr))
					SMP_msg("IDA code addr: %x\n", addr);
#endif
			} // end if (isHead(addr) && isCode(addr)
#if SMP_DEBUG_FIXUP_IDB
			else if ((0x806cda4 <= addr) && (0x806cf99 >= addr)) {
				if (!isHead(InstrFlags))
					SMP_msg("Weirdness: not isHead at %x\n", addr);
				if (isUnknown(InstrFlags)) {
					SMP_msg("Weirdness: isUnknown at %x\n", addr);
				}
			}
#endif
		} // end for (ea_t addr = seg->startEA; ...)
	} // end for all segments
	return;
} // end FindCodeAddresses()

// Return true if addr is not a proper beginning address for an instruction.
// Return false otherwise.
// Currently, we claim that an instruction is misaligned if DisasmLocs does
//  not contain it. This function is useful for dealing with errors in IDA
//  code identification, in which a large code section is identified as data,
//  but some instructions in the middle of the "data" are identified as
//  code but IDA often starts on the wrong boundary in these cases.
bool IsCodeMisaligned(ea_t addr) {
	// Do a binary search for addr within DisasmLocs, which is sorted
	//  in ascending address order because of the way in which it was
	//  generated.
	size_t min = 0;
	size_t max = DisasmLocs.size();  // don't access DisasmLocs[max]
	size_t index = (min + max) / 2;
	
	while (addr != DisasmLocs[index]) {
		if (min >= (max - 1))
			return true;
#if 0
		SMP_msg("min: %d max: %d index: %d\n", min, max, index);
#endif
		if (addr < DisasmLocs[index])
			max = index;
		else // must be addr > DisasmLocs[index];
			min = index;
	
		index = (min + max) / 2;
	}

	return false;
} // end of IsCodeMisaligned()

void RemoveIDACodeAddr(ea_t addr) {
	// Do a binary search for addr within IDAProLocs, which is sorted
	//  in ascending address order because of the way in which it was
	//  generated. Delete the element of IDAProLocs if found.
	size_t min = 0;
	size_t max = IDAProLocs.size();  // don't access IDAProLocs[max]
	size_t index = (min + max) / 2;
	
	while (addr != IDAProLocs[index]) {
		if (min >= (max - 1))
			return;
#if 0
		SMP_msg("min: %d max: %d index: %d\n", min, max, index);
#endif
		if (addr < IDAProLocs[index])
			max = index;
		else // must be addr > IDAProLocs[index];
			min = index;
	
		index = (min + max) / 2;
	}

	// IDAProLocs[index] contains addr.
	vector<ea_t>::iterator RemovalIterator = IDAProLocs.begin();
	RemovalIterator += index;
	RemovalIterator = IDAProLocs.erase(RemovalIterator);
	return;
} // end of RemoveIDACodeAddr()

// Driver for all other fixing functions. Upon its return, the IDA
//  database (IDB file) should be fixed up as much as we can fix it.
void FixupIDB(void) {
	FindCodeAddresses();
#if SMP_DEBUG_FIXUP_IDB
	SpecialDebugOutput();
#endif
	AuditCodeTargets();
	FindDataInCode();
	AuditTailChunkOwnership();
	if (DisasmLocs.size() > 0) {
		FixCodeIdentification();
		int fixes = FixupNewCodeChunks();
#if SMP_DEBUG_FIXUP_IDB
		SpecialDebugOutput();
#endif
	}
	DisasmLocs.clear();
	IDAProLocs.clear();
} // end of FixupIDB()

// Find and print all data head addresses in code segments. 
// If an isolated code instruction is found in the midst of a run
//  of data bytes and has no code xrefs jumping to it, it is not
//  reachable as code and is undoubtedly a mixup by IDA. Possibly
//  the whole data region will be converted to code later, in which
//  case the isolated code is not necessarily properly aligned and
//  parsed at its present address, so we are glad to convert it into
//  data anyway so that FindDataToConvert() will succeed on it later.
// Data to code conversion, and isolated code detection, are inhibited
//  by IDA identifying several consecutive instructions in the midst
//  of a data region, with the code addresses not agreeing with the
//  external disassembler's code addresses. We will convert these
//  misaligned instructions to data as we detect them. We will also
//  convert unexplored bytes (isUnknown(flags) == true) into data if
//  they are in the midst of a data sequence.
#define MIN_DATARUN_LEN 24  // #bytes on either side of "isolated" code
void FindDataInCode(void) {
	size_t DataRunLen = 0; // How many data bytes in a row have we seen?
	bool IsolatedCodeTrigger = false; // Have seen data, then isolated code
									// Now looking for data
	ea_t IsolatedCodeAddr;
	int IsolatedCodeLen;
	int InstrLen;
	bool InstOK;
	insn_t LocalCmd;
	ulong LocalFeatures;

	ea_t RecentAddr = BADADDR;
#if IDA_SDK_VERSION < 600
	for (int SegIndex = 0; SegIndex < SMP_get_segm_qty(); ++SegIndex) {
		segment_t *seg = SMP_getnseg(SegIndex);
#else
	for (segment_t *seg = SMP_get_first_seg(); NULL != seg; seg = SMP_get_next_seg(RecentAddr)) {
		RecentAddr = seg->startEA;
#endif
		if (SEG_CODE != seg->type)
			continue;
#if SMP_DEBUG_FIXUP_IDB
		char SegName[MAXSTR];
		ssize_t SegNameSize = SMP_get_segm_name(seg, SegName, sizeof(SegName) - 1);
		SMP_msg("Non-code addresses for code segment %s from %x to %x\n",
			SegName, seg->startEA, seg->endEA);
#endif
		for (ea_t addr = seg->startEA; addr < seg->endEA; addr = get_item_end(addr)) {
			flags_t AddrFlags = getFlags(addr);
			if (isHead(AddrFlags)) {
				if (isData(AddrFlags)) {
					DataRunLen += get_item_size(addr);
#if SMP_DEBUG_FIXUP_IDB
					SMP_msg("Data: %x\n", addr);
#endif
					if (MIN_DATARUN_LEN <= DataRunLen) {
						if (IsolatedCodeTrigger) {
							// Saw data, then one isolated code, then data
							do_unknown_range(IsolatedCodeAddr, IsolatedCodeLen, DOUNK_SIMPLE);
							RemoveIDACodeAddr(IsolatedCodeAddr);
							if (do_data_ex(IsolatedCodeAddr, byteflag(),
								IsolatedCodeLen, BADNODE)) {
									SMP_msg("Converted isolated code to data: %x\n",
										IsolatedCodeAddr);
							}
							else {
								SMP_msg("Failed to convert isolated code to data: %x len: %x\n",
									IsolatedCodeAddr, IsolatedCodeLen);
							}
							IsolatedCodeTrigger = false;
						} // end if (IsolatedCodeTrigger)
					} // end if (MIN_DATARUN_LEN <= DataRunLen)
				} // end if (isData(AddrFlags)
				else if (isUnknown(AddrFlags)) {
					// Just in case; unknown usually means not head or tail
					// If in a data run, convert to data.
					InstrLen = get_item_size(addr);
#if SMP_DEBUG_FIXUP_IDB
					SMP_msg("Unknown: %x len: %x\n", addr, InstrLen);
#endif
					if (0 < DataRunLen) {
						if (do_data_ex(addr, byteflag(), InstrLen, BADNODE)) {
#if SMP_DEBUG_FIXUP_IDB
							SMP_msg("Converted unknown to data at %x len: %x\n", addr, InstrLen);
#endif
							DataRunLen += InstrLen;
						}
						else {
#if SMP_DEBUG_FIXUP_IDB
							SMP_msg("Failed to convert unknown to data at %x len: %x\n", addr, InstrLen);
#endif
							DataRunLen = 0;
							IsolatedCodeTrigger = false;
						}
					}
				}
				else if (isCode(AddrFlags)) {  // must be true
					if (MIN_DATARUN_LEN <= DataRunLen) {
#if SMP_DEBUG_FIXUP_IDB
						SMP_msg("DataRunLen: %d at %x\n", DataRunLen, addr);
#endif
						InstOK = SMPGetCmd(addr, LocalCmd, LocalFeatures);
						assert(InstOK);

						InstrLen = (int) LocalCmd.size;

						// We don't check the returned InstrLen for validity because IsCodeMisaligned()
						//  will check for validity immediately below.
#if SMP_DEBUG_FIXUP_IDB
						SMP_msg("Calling IsCodeMisaligned: len %d\n", InstrLen);
#endif
						if (IsCodeMisaligned(addr)) {
#if SMP_DEBUG_FIXUP_IDB
							SMP_msg("Code was misaligned.\n");
#endif
							do_unknown_range(addr, InstrLen, DOUNK_SIMPLE);
							RemoveIDACodeAddr(addr);
							if (do_data_ex(addr, byteflag(), InstrLen, BADNODE)) {
#if SMP_DEBUG_FIXUP_IDB
								SMP_msg("Converted misaligned code to data at %x : len: %x\n",
									addr, InstrLen);
#endif
								// Step back so data gets processed.
								DataRunLen += get_item_size(addr);
								continue; // skip reset of DataRunLen
							}
							else {
#if SMP_DEBUG_FIXUP_IDB
								SMP_msg("Misaligned code left as unknown at %x : len: %x\n",
									addr, InstrLen);
#endif
								IsolatedCodeTrigger = false;
							}
						} // end if (IsCodeMisaligned() ...)
						else if (!hasRef(AddrFlags)) {
							// No references at all --> isolated code.
							IsolatedCodeTrigger = true;
							IsolatedCodeAddr = addr;
							IsolatedCodeLen = InstrLen;
						}
						else {
							SMP_xref_t xb;
							bool ok = xb.SMP_first_to(addr, XREF_ALL);
							if (!ok) {
								// No code xrefs to this target addr.
								IsolatedCodeTrigger = true;
								IsolatedCodeAddr = addr;
								IsolatedCodeLen = InstrLen;
							}
						}
					} // end if (MIN_DATARUN_LEN <= DataRunLen)
					else if (IsolatedCodeTrigger) {
						// Two instructions in a row does not fit the pattern.
						IsolatedCodeTrigger = false;
					}
					DataRunLen = 0;
				} // end if (isData) ... else if (isUnknown) ... else isCode
			} // end if (isHead)
			else if (isUnknown(AddrFlags)) {
				// If in a data run, convert to data.
				InstrLen = get_item_size(addr);
#if SMP_DEBUG_FIXUP_IDB
				SMP_msg("Unknown: %x len: %x\n", addr, InstrLen);
#endif
				if (0 < DataRunLen) {
					if (do_data_ex(addr, byteflag(), InstrLen, BADNODE)) {
#if SMP_DEBUG_FIXUP_IDB
						SMP_msg("Converted unknown to data at %x len: %x\n", addr, InstrLen);
#endif
						DataRunLen += InstrLen;
					}
					else {
#if SMP_DEBUG_FIXUP_IDB
						SMP_msg("Failed to convert unknown to data at %x len: %x\n", addr, InstrLen);
#endif
						DataRunLen = 0;
						IsolatedCodeTrigger = false;
					}
				}
			}
		} // end for (ea_t addr =  seg->startEA; ...)
	} // end for all segments
	return;
} // end of FindDataInCode()


// The choices that IDA makes for deciding which parent function of a
//  TAIL chunk is the primary owner of the tail can be counterintuitive.
//  A function entry can both fall into and jump to a tail chunk that
//  is contiguous with it, yet the "owner" might be a function that is
//  far below it in the executable address space. This function will
//  change the ownership to a more sensible arrangement.
void AuditTailChunkOwnership(void) {
	char FuncName[MAXSTR];
	// Iterate through all chunks in the program.
	size_t NumChunks = get_fchunk_qty();
	for (size_t ChunkIndex = 0; ChunkIndex < NumChunks; ++ChunkIndex) {
		func_t *ChunkInfo = getn_fchunk((int) ChunkIndex);
		if (is_func_tail(ChunkInfo)) {
			// For each TAIL chunk, find all the parent chunks. Find the last
			//  parent chunk with an address less than the TAIL chunk address.
			ea_t BestCandidate = 0;
			func_parent_iterator_t FuncParent(ChunkInfo);
#if SMP_DEBUG_CHUNKS
			SMP_msg("Tail chunk: %x ", ChunkInfo->startEA);
#endif
			for (bool ok = FuncParent.first(); ok; ok = FuncParent.next()) {
				ea_t parent = FuncParent.parent();
#if SMP_DEBUG_CHUNKS
				SMP_msg(" parent: %x ", parent);
#endif
				if ((parent > BestCandidate) && (parent < ChunkInfo->startEA))
					BestCandidate = parent;
			}
#if SMP_DEBUG_CHUNKS
			SMP_msg("\n");
#endif
			//  Make the best parent chunk the owner of the TAIL chunk if it is
			//  not already the owner.
			if (ChunkInfo->owner != BestCandidate) {
				if (0 < BestCandidate) {
					if (set_tail_owner(ChunkInfo, BestCandidate)) {
						func_t *FuncInfo = get_func(BestCandidate);
						SMP_msg("Set %x as new owner of tail %x\n",
							BestCandidate, ChunkInfo->startEA);
						// Reanalyze the parent function (and all its
						//  tail chunks) now that the structure has changed.
						reanalyze_function(FuncInfo);
					}
					else {
						SMP_msg("set_tail_owner failed for tail %x and parent %x\n",
							ChunkInfo->startEA, BestCandidate);
					}
				}
				else {
					func_t *FuncInfo = get_func(ChunkInfo->owner);
					get_func_name(FuncInfo->startEA, FuncName, sizeof(FuncName) - 1);
#if SMP_DEBUG_CHUNKS
					SMP_msg("No good parent candidate before tail at %x\n",
						ChunkInfo->startEA);
					SMP_msg("Current parent is %x: %s\n", FuncInfo->startEA, FuncName);
#endif
					// Find out if a function entry chunk that comes before the
					//  tail is a better candidate for the owner (i.e. it falls
					//  through to the tail, or jumps to it).
					BestCandidate = 0;
#if SMP_DEBUG_CHUNKS
					SMP_msg("Finding parent func candidates for %x:", ChunkInfo->startEA);
#endif
					SMP_bounds_t CurrFunc;
					for (size_t FuncIndex = 0; FuncIndex < FuncBounds.size(); ++FuncIndex) {
						CurrFunc = FuncBounds[FuncIndex];
						if ((CurrFunc.startEA < ChunkInfo->startEA)
							&& (CurrFunc.startEA > BestCandidate)) {
							BestCandidate = CurrFunc.startEA;
#if SMP_DEBUG_CHUNKS
							SMP_msg(" candidate: %x tail: %x", BestCandidate,
								ChunkInfo->startEA);
#endif
						}
						else {
#if SMP_DEBUG_CHUNKS
							SMP_msg(" not a candidate: %x tail: %x best: %x\n",
								CurrFunc.startEA, ChunkInfo->startEA, BestCandidate);
#endif
							break;
						}
					} // end for (size_t FuncIndex = 0; ...)
					if (0 >= BestCandidate) { // highly unlikely
						SMP_msg("No good func entry parent candidate.\n");
					}
					else {
						FuncInfo = get_func(BestCandidate);
						get_func_name(FuncInfo->startEA, FuncName, sizeof(FuncName) - 1);
#if SMP_DEBUG_CHUNKS
						SMP_msg("Best func entry parent candidate: %s at %x",
							FuncName, BestCandidate);
						if (FuncInfo->endEA == ChunkInfo->startEA)
							SMP_msg(" Function endEA == tail chunk startEA");
						SMP_msg("\n");
#endif
					}
				}
			} // end if (ChunkInfo->owner != BestCandidate)
#if SMP_DEBUG_CHUNKS
			else {
				SMP_msg("Already best parent for %x is %x\n", ChunkInfo->startEA,
					ChunkInfo->owner);
			}
#endif
		} // end if (is_func_tail(ChunkInfo))
	} // end for (size_t ChunkIndex = 0; ...)

	return;
} // end of AuditTailChunkOwnership()

// If the addresses signified from DisasmIndex to IDAProIndex are
//  all considered data and do NOT follow a return instruction,
//  return false and update AreaSize to reflect the area to be
//  converted.
// Return value: true -> skip to IDAProIndex; false -> convert AreaSize bytes.
bool FindDataToConvert(size_t IDAProIndex, size_t DisasmIndex, int &AreaSize) {
	ea_t PrevIDAAddr;
	ea_t NextIDAAddr;
	size_t ShadowDisasmIndex = DisasmIndex - 1;
	ea_t DisasmAddr = DisasmLocs[ShadowDisasmIndex];
	bool CannotConvert = false;  // return value
	bool DebugAddress = false;
#if SMP_DEBUG_FIXUP_IDB
	DebugAddress = (DisasmAddr == 0x806c19a);
#endif

	if (DebugAddress) {
		SMP_msg("IDAProIndex: %zu DisasmIndex: %zu\n", IDAProIndex, DisasmIndex);
		SMP_msg("IDA locs size %zu Disasm locs size %zu\n", IDAProLocs.size(),
			DisasmLocs.size());
	}
	if (IDAProIndex >= IDAProLocs.size()) {
		// Have already processed the last IDA address.
		if (DebugAddress) SMP_msg(" Already done with IDAProLocs.\n");
		return true;
	}
	else if (DisasmIndex >= DisasmLocs.size()) {
		// Strange. Last Disasm address is only one to convert, and
		//  IDA still has addresses after that?
		if (DebugAddress) SMP_msg(" Already done with DisasmLocs.\n");
		return true;
	}
	else if (IDAProIndex < 2) {
		// We have Disasm addrs before the very first IDA addr. We
		//  don't trust this boundary case.
		if (DebugAddress) SMP_msg(" Boundary case with IDAProLocs.\n");
		return true;
	}
	NextIDAAddr = IDAProLocs[IDAProIndex - 1];
	PrevIDAAddr = IDAProLocs[IDAProIndex - 2];
	if (DebugAddress) SMP_msg(" PrevIDAAddr: %x NextIDAAddr: %x\n", PrevIDAAddr, NextIDAAddr);

	// See if previous IDA address was a return.
	flags_t PrevFlags = getFlags(PrevIDAAddr);
	if (!isCode(PrevFlags) || !isHead(PrevFlags)) {
		SMP_msg("PrevIDAAddr %x not isCode or not isHead.\n", PrevIDAAddr);
		return true;
	}
	SMPInstr PrevInstr(PrevIDAAddr);
	PrevInstr.Analyze();
	if (DebugAddress) SMP_msg("Finished PrevInstr.Analyze()\n");
	if (PrevInstr.MDIsReturnInstr()) {
		// Right after a return come no-ops and 2-byte no-ops
		//  that are just for alignment. IDA does not seem to be
		//  happy when we convert all those to code.
		if (DebugAddress) SMP_msg(" Data followed a return instruction.\n");
		return true;
	}
	// Now, see if the area from DisasmAddr to NextIDAAddr is all data
	//  according to IDA.
	while (DisasmAddr < NextIDAAddr) {
		flags_t DataFlags = getFlags(DisasmAddr);
		if (isTail(DataFlags)) {
			if (DebugAddress) SMP_msg(" tail byte: %x\n", DisasmAddr);
			DisasmAddr = get_item_end(DisasmAddr);
		}