diff --git a/.gitignore b/.gitignore index f22b2653903c6d32e83fb1000814289c98b70da2..0688ae576a8209dbbbbc9131d856d14afc88a3d3 100644 --- a/.gitignore +++ b/.gitignore @@ -7,4 +7,5 @@ lib/ manifest.txt.config tools/meds2pdb/meds2pdb plugins_install/*.exe +plugins_install/*.so *.swp diff --git a/SConscript b/SConscript index 101e379d3f91b6603032800403378ad8f4d336be..afa8168e893ec3b7c765168429fb49dfd4cc094b 100644 --- a/SConscript +++ b/SConscript @@ -39,16 +39,17 @@ else: os.chdir(os.environ['SECURITY_TRANSFORMS_HOME']) -env['BASE_IRDB_LIBS']="IRDB-core", "pqxx", "pq", "EXEIO" pedi = Command( target = "./testoutput", source = "./SConscript", action = os.environ['PEDI_HOME']+"/pedi -m manifest.txt " ) +env['BASE_IRDB_LIBS']="IRDB-core", "pqxx", "pq", "EXEIO" + if sysname != "SunOS": libPEBLISS=SConscript("pebliss/trunk/pe_lib/SConscript", variant_dir='scons_build/libPEBLISS') # setup libraries needed for linking - env['BASE_IRDB_LIBS']="IRDB-core", "pqxx", "pq", "EXEIO", "pebliss" + env['BASE_IRDB_LIBS']=env['BASE_IRDB_LIBS']+("pebliss",) Depends(pedi,libPEBLISS) # pebliss requires iconv, which needs to be explicit on cygwin. @@ -58,20 +59,24 @@ if "CYGWIN" in sysname: Export('env') +env.Install("$SECURITY_TRANSFORMS_HOME/lib/", "$SECURITY_TRANSFORMS_HOME/libcapstone/libcapstone.so.4") +env.Command(os.environ['SECURITY_TRANSFORMS_HOME']+"/lib/libcapstone.so", os.environ['SECURITY_TRANSFORMS_HOME']+"/lib/libcapstone.so.4", "ln -s $SOURCE.abspath $TARGET.abspath") +libcapstone=os.environ['SECURITY_TRANSFORMS_HOME']+"/lib/libcapstone.so" + libehp=env.SConscript("libehp/SConscript", variant_dir='scons_build/libehp') libehp=env.Install("$SECURITY_TRANSFORMS_HOME/lib", libehp); - libtransform=SConscript("libtransform/SConscript", variant_dir='scons_build/libtransform') libEXEIO=SConscript("libEXEIO/SConscript", variant_dir='scons_build/libEXEIO') -libbea=SConscript("beaengine/SConscript", variant_dir='scons_build/beaengine') +#libbea=SConscript("beaengine/SConscript", variant_dir='scons_build/beaengine') libMEDSannotation=SConscript("libMEDSannotation/SConscript", variant_dir='scons_build/libMEDSannotation') libxform=SConscript("xform/SConscript", variant_dir='scons_build/libxform') libIRDB=SConscript("libIRDB/SConscript", variant_dir='scons_build/libIRDB') +Depends(libIRDB,libcapstone) libStructDiv=SConscript("libStructDiv/SConscript", variant_dir='scons_build/libStructDiv') libElfDep=SConscript("libElfDep/SConscript", variant_dir='scons_build/libElfDep') -Depends(pedi, (libehp,libtransform,libEXEIO,libbea,libMEDSannotation,libxform,libIRDB,libStructDiv,libElfDep)) +Depends(pedi, (libehp,libtransform,libEXEIO,libMEDSannotation,libxform,libIRDB,libStructDiv,libElfDep, libcapstone)) tools=None if 'build_tools' not in env or env['build_tools'] is None or int(env['build_tools']) == 1: diff --git a/SConstruct b/SConstruct index 0320809183fa2e033358695a0eb1cfb4ef10e7ad..0aebd26be2fc36817236fea21f095712beb6d0a9 100644 --- a/SConstruct +++ b/SConstruct @@ -24,28 +24,20 @@ env.Replace(build_tools=ARGUMENTS.get("build_tools", 1)) env.Replace(build_stars=ARGUMENTS.get("build_stars", 1)) env.Replace(build_cgc=ARGUMENTS.get("build_cgc", 0)) +env.Append(LINKFLAGS=" -Wl,-unresolved-symbols=ignore-in-shared-libs ") if int(env['debug']) == 1: print "Setting debug mode" env.Append(CFLAGS=" -g ") env.Append(CXXFLAGS=" -g ") - env.Append(LINKFLAGS=" -g ") + env.Append(LINKFLAGS=" -g ") + env.Append(SHLINKFLAGS=" -g ") else: print "Setting release mode" env.Append(CFLAGS=" -O3 ") env.Append(CXXFLAGS=" -O3 ") - env.Append(LINKFLAGS=" -O3 ") - -if 'build_cgc' in env and int(env['build_cgc']) == 1: - print "Setting debug mode" - env.Append(CFLAGS=" -DCGC ") - env.Append(CXXFLAGS=" -DCGC ") - env.Append(LINKFLAGS=" -DCGC ") - print 'Turn off appfw as we are building CGC' - env['build_appfw'] = 0 -elif env['build_appfw'] is None: # by default, turn on build of appfw, unless cgc is on - env['build_appfw'] = 1 - + env.Append(LINKFLAGS=" -O3 ") + env.Append(SHLINKFLAGS=" -O3 ") # set 32/64 bit build properly print "env[64bit]="+str(env['do_64bit_build']) @@ -69,6 +61,7 @@ else: # add extra flag for solaris. if sysname == "SunOS": env.Append(LINKFLAGS=" -L/opt/csw/lib -DSOLARIS ") + env.Append(SHLINKFLAGS=" -L/opt/csw/lib -DSOLARIS ") env.Append(CFLAGS=" -I/opt/csw/include -DSOLARIS ") env.Append(CXXFLAGS=" -I/opt/csw/include -DSOLARIS ") diff --git a/libEXEIO/src/SConscript b/libEXEIO/src/SConscript index 2082e8098785b66c926868216168447458aa1636..413aadb3a346d28327d1af0621ae11c140aa024a 100644 --- a/libEXEIO/src/SConscript +++ b/libEXEIO/src/SConscript @@ -21,7 +21,7 @@ cpppath=''' myenv=myenv.Clone(CPPPATH=Split(cpppath)) myenv.Append(CXXFLAGS=" -std=c++11 -Wall -Werror ") -lib=myenv.Library(libname, Split(files)) +lib=myenv.SharedLibrary(libname, Split(files), LIBS=Split("IRDB-core pebliss"), LIBPATH="$SECURITY_TRANSFORMS_HOME/lib" ) install=myenv.Install("$SECURITY_TRANSFORMS_HOME/lib/", lib) Default(install) diff --git a/libElfDep/src/SConscript b/libElfDep/src/SConscript index 848a762e591eb9a29d5555a7a065f0a5c31a1d78..9da08bd9c97e3e28caa1f2c475ec161be03df536 100644 --- a/libElfDep/src/SConscript +++ b/libElfDep/src/SConscript @@ -23,7 +23,7 @@ LIBS=Split("IRDB-core transform") myenv=myenv.Clone(CPPPATH=Split(cpppath)) myenv.Append(CXXFLAGS = " -std=c++11 ") -lib=myenv.Library("ElfDep", Split(files), LIBPATH=LIBPATH, LIBS=LIBS) +lib=myenv.SharedLibrary("ElfDep", Split(files), LIBPATH=LIBPATH, LIBS=LIBS) install=myenv.Install("$SECURITY_TRANSFORMS_HOME/lib/", lib) Default(install) diff --git a/libIRDB/include/core/IRDB_Objects.hpp b/libIRDB/include/core/IRDB_Objects.hpp new file mode 100644 index 0000000000000000000000000000000000000000..41f59ecf4ec35ffa28361c65d44acd860fcffaff --- /dev/null +++ b/libIRDB/include/core/IRDB_Objects.hpp @@ -0,0 +1,76 @@ +#ifndef IRDB_Objects_h +#define IRDB_Objects_h + +#include <map> +#include <utility> +#include <memory> + + +// TODO: Should really use unordered maps but was getting build errors + + +// *** A toolchain step should NOT delete pointers to any variant, file IR, or file object stored +// in a IRFiles_t object. I have made pointers shared where possible to communicate this. *** +class IRDBObjects_t +{ + public: + using FileSet_t = std::set<File_t*>; + IRDBObjects_t() + { + pqxx_interface = std::unique_ptr<pqxxDB_t>(new pqxxDB_t()); + // set up interface to the sql server + BaseObj_t::SetInterface(pqxx_interface.get()); + }; + ~IRDBObjects_t(); + + // Add/delete file IRs for a variant. + // Step does not have ownership of fileIR (can't make assumptions about its + // lifetime!), and a call to DeleteFileIR will render any pointers to that fileIR dangling. + // AddFileIR returns the added IR or the preexistent IR if it was already added. + FileIR_t* addFileIR(const db_id_t variant_id, const db_id_t file_id); // Returns NULL if no such file exists + void deleteFileIR(const db_id_t file_id); + // Add or delete a variant + // Step does not have ownership of variant (can't make assumptions about its lifetime!), and a call to DeleteVariant will render any pointers to that variant dangling. + // Deleting a variant does NOT write its files' IRs, but DOES delete them! + // AddVariant returns the added variant or the preexistent variant if it was already added. + VariantID_t* addVariant(const db_id_t variant_id); + void deleteVariant(db_id_t variant_id); + + // Get DB interface + pqxxDB_t* getDBInterface() const; + pqxxDB_t* resetDBInterface(); + + // Write back variants and file IRs. Does NOT commit changes. + int writeBackFileIR(const db_id_t file_id); + int writeBackVariant(const db_id_t variant_id); // Does NOT write back its files' IRs + int writeBackAll(void); // Returns -1 if any writes fail. + + void deleteAll(void); + + private: + std::unique_ptr<pqxxDB_t> pqxx_interface; + // type aliases of maps. maps allow speed of finding needed files, file IRs + // and/or variants that have already been read from the DB + using IdToVariantMap_t = std::map<db_id_t, std::unique_ptr<VariantID_t>>; + struct FileIRInfo_t + { + File_t * file; + std::unique_ptr<FileIR_t> fileIR; + + FileIRInfo_t() : file(nullptr), fileIR(nullptr) + { + } + + }; + using IdToFileIRInfoMap_t = std::map<db_id_t, FileIRInfo_t>; + + IdToVariantMap_t variant_map; + IdToFileIRInfoMap_t file_IR_map; + + // minor helper + bool filesAlreadyPresent(const std::set<File_t*>&) const; + +}; + +#endif + diff --git a/libIRDB/include/core/dbinterface.hpp b/libIRDB/include/core/dbinterface.hpp index a96f8d1cc5a9929674cf2d43183bf30b8226c2e0..96954be9e3e60d2cb254df905fa778ed055ca368 100644 --- a/libIRDB/include/core/dbinterface.hpp +++ b/libIRDB/include/core/dbinterface.hpp @@ -37,6 +37,7 @@ class DBinterface_t { public: DBinterface_t() {}; + virtual ~DBinterface_t() {}; virtual void IssueQuery(std::string query)=0; virtual void MoveToNextRow()=0; virtual std::string GetResultColumn(std::string colname)=0; diff --git a/libIRDB/include/core/fileir.hpp b/libIRDB/include/core/fileir.hpp index b540d5287a2fc1e00474365653a2e52dac72f7d2..7668dabaf3094753e0235894546627d1896356ea 100644 --- a/libIRDB/include/core/fileir.hpp +++ b/libIRDB/include/core/fileir.hpp @@ -123,9 +123,9 @@ class FileIR_t : public BaseObj_t RelocationSet_t relocs; TypeSet_t types; DataScoopSet_t scoops; - VariantID_t progid; + VariantID_t& progid; // Not owned by fileIR ICFSSet_t icfs_set; - File_t* fileptr; + File_t* fileptr; // Owned by variant, not fileIR EhProgramSet_t eh_pgms; EhCallSiteSet_t eh_css; diff --git a/libIRDB/include/core/pqxxdb.hpp b/libIRDB/include/core/pqxxdb.hpp index 819e50daf135d573e6757fa90be055d4f27c51da..9b2f3c1a4c023bab96110bbe79ad1a1ef53e56c4 100644 --- a/libIRDB/include/core/pqxxdb.hpp +++ b/libIRDB/include/core/pqxxdb.hpp @@ -24,6 +24,10 @@ class pqxxDB_t : public DBinterface_t { public: pqxxDB_t(); + ~pqxxDB_t() override + { + // do nothing + }; void IssueQuery(std::string query); void IssueQuery(std::stringstream & query); void MoveToNextRow(); diff --git a/libIRDB/include/core/transform_step.h b/libIRDB/include/core/transform_step.h new file mode 100644 index 0000000000000000000000000000000000000000..4069c60f51e4d49c1d502e79645c18c2cb9213c7 --- /dev/null +++ b/libIRDB/include/core/transform_step.h @@ -0,0 +1,35 @@ +#ifndef TransformStep_h +#define TransformStep_h + + +namespace Transform_SDK +{ + class TransformStep_t + { + public: + // Step names must be unique, allows arguments to + // be directed to their matching transform steps. + virtual std::string getStepName(void) const = 0; + + // Allows all steps to parse args before any step takes time to execute + virtual int parseArgs(const std::vector<std::string> step_args) + { + return 0; // success + } + + virtual int executeStep(libIRDB::IRDBObjects_t *const irdb_objects) + { + return 0; // success + } + + virtual ~TransformStep_t(void) + { + // do nothing + } + }; +} + +extern "C" +std::shared_ptr<Transform_SDK::TransformStep_t> GetTransformStep(void); + +#endif diff --git a/libIRDB/include/core/variantid.hpp b/libIRDB/include/core/variantid.hpp index 9e7285fab718b435c6c3ca3b4b1636ca06be0ff5..ce17a1f3683961122cbdf10e4ce2603a07dc5214 100644 --- a/libIRDB/include/core/variantid.hpp +++ b/libIRDB/include/core/variantid.hpp @@ -25,11 +25,14 @@ class VariantID_t; std::ostream& operator<<(std::ostream& out, const libIRDB::VariantID_t& pid); +using FileSet_t = std::set<File_t*>; + class VariantID_t : public BaseObj_t { public: VariantID_t(); // create a Variant ID not in the database - VariantID_t(db_id_t pid); // read from the DB + VariantID_t(db_id_t pid); // read from the DB + ~VariantID_t(); // Deletes the File_t objects -- beware dangling File_t* in FileIR_t objects! bool IsRegistered(); bool Register(); // accesses DB @@ -40,7 +43,8 @@ class VariantID_t : public BaseObj_t void DropFromDB(); - std::set<File_t*>& GetFiles() { return files; } + FileSet_t& GetFiles() { return files; } + const FileSet_t& GetFiles() const { return files; } std::string GetName() { return name; } void SetName(std::string newname) { name=newname;} @@ -48,14 +52,14 @@ class VariantID_t : public BaseObj_t File_t* GetMainFile() const; friend std::ostream& libIRDB::operator<<(std::ostream& out, const VariantID_t& pid); - friend class FileIR_T; - friend class Function_t; - friend class AddressID_t; - friend class Instruction_t; + //friend class FileIR_T; + //friend class Function_t; + //friend class AddressID_t; + //friend class Instruction_t; db_id_t GetOriginalVariantID() const { return orig_pid;} - void CloneFiles(std::set<File_t*>& files); + void CloneFiles(FileSet_t& files); File_t* CloneFile(File_t* fptr); private: @@ -67,12 +71,9 @@ class VariantID_t : public BaseObj_t void CreateTables(); // create the address, function and instruction tables - std::set<File_t*> files; + FileSet_t files; void ReadFilesFromDB(); - - - }; diff --git a/libIRDB/include/libIRDB-core.hpp b/libIRDB/include/libIRDB-core.hpp index 5680c2fe1a987e84bdcfba6cddd6ee4bb575ac7c..96d47e3053cbafb5bcbb0cebfd4b46e5039ea684 100644 --- a/libIRDB/include/libIRDB-core.hpp +++ b/libIRDB/include/libIRDB-core.hpp @@ -56,6 +56,8 @@ class Instruction_t; // forward decl for many classes #include <core/eh.hpp> #include <core/fileir.hpp> #include <core/pqxxdb.hpp> +#include <core/IRDB_Objects.hpp> +#include <core/transform_step.h> }; diff --git a/libIRDB/src/cfg/SConscript b/libIRDB/src/cfg/SConscript index 327f3031ef7b3242ecc8a218ada63da15d5dd62b..c9e8e715cd9b968b23bf9134009c81d655192826 100644 --- a/libIRDB/src/cfg/SConscript +++ b/libIRDB/src/cfg/SConscript @@ -14,11 +14,14 @@ cpppath=''' $SECURITY_TRANSFORMS_HOME/include/ $SECURITY_TRANSFORMS_HOME/libIRDB/include/ ''' +libpath=''' + $SECURITY_TRANSFORMS_HOME/lib + ''' myenv.Append(CCFLAGS=" -Wall -std=c++11") myenv=myenv.Clone(CPPPATH=Split(cpppath)) -lib=myenv.Library(libname, Split(files)) +lib=myenv.SharedLibrary(libname, Split(files), LIBS=Split("IRDB-core"), LIBPATH=Split(libpath)) install=myenv.Install("$SECURITY_TRANSFORMS_HOME/lib/", lib) Default(install) diff --git a/libIRDB/src/core/IRDB_Objects.cpp b/libIRDB/src/core/IRDB_Objects.cpp new file mode 100644 index 0000000000000000000000000000000000000000..17ea07d06eb9da606dd3038b6525043b5680ea49 --- /dev/null +++ b/libIRDB/src/core/IRDB_Objects.cpp @@ -0,0 +1,229 @@ +#include <map> +#include <libIRDB-core.hpp> +#include <utils.hpp> +#include <utility> +#include <memory> +#include <assert.h> + + +using namespace libIRDB; +using namespace std; + +#define ALLOF(a) begin(a),end(a) + + +IRDBObjects_t::~IRDBObjects_t() +{ + // All dynamically allocated DB objects + // are held as unique pointers and don't need to be + // explicitly deleted. +} + +FileIR_t* IRDBObjects_t::addFileIR(const db_id_t variant_id, const db_id_t file_id) +{ + const auto it = file_IR_map.find(file_id); + + if(it == file_IR_map.end()) + { + return nullptr; + } + else + { + const auto the_file = (it->second).file; + auto& the_fileIR = (it->second).fileIR; + + if(the_fileIR == NULL) + { + assert(the_file != NULL); + + assert(variant_map.find(variant_id) != variant_map.end()); + const auto & the_variant = *(variant_map.at(variant_id).get()); + + the_fileIR = unique_ptr<FileIR_t>(new FileIR_t(the_variant, the_file)); + assert(the_fileIR != NULL); + } + + // make sure static variable is set in the calling module -- IMPORTANT + the_fileIR->SetArchitecture(); + return the_fileIR.get(); + } +} + +int IRDBObjects_t::writeBackFileIR(const db_id_t file_id) +{ + const auto it = file_IR_map.find(file_id); + + if(it == file_IR_map.end()) + return 1; + + const auto& the_file = (it->second).file; + assert(the_file != NULL); + + try + { + // cout<<"Writing changes for "<<the_file->GetURL()<<endl; + // make sure static variable is set in the calling module -- IMPORTANT + const auto & the_fileIR = (it->second).fileIR; + the_fileIR->SetArchitecture(); + the_fileIR->WriteToDB(); + return 0; + } + catch (DatabaseError_t pnide) + { + cerr << "Unexpected database error: " << pnide << "file url: " << the_file->GetURL() << endl; + return -1; + } + catch (...) + { + cerr << "Unexpected error file url: " << the_file->GetURL() << endl; + return -1; + } + assert(0); // should not reach + +} + +void IRDBObjects_t::deleteFileIR(const db_id_t file_id) +{ + const auto it = file_IR_map.find(file_id); + + if(it == file_IR_map.end()) + return; + + auto& the_fileIR = (it->second).fileIR; + if(the_fileIR != NULL) + { + the_fileIR.reset(); + } +} + +bool IRDBObjects_t::filesAlreadyPresent(const set<File_t*>& the_files) const +{ + // look for a missing file + const auto missing_file_it=find_if(ALLOF(the_files), [&](const File_t* const f) + { + return (file_IR_map.find(f->GetBaseID()) == file_IR_map.end()); + }); + + // return true if no files missing + return missing_file_it==the_files.end(); +} + +VariantID_t* IRDBObjects_t::addVariant(const db_id_t variant_id) +{ + const auto var_it = variant_map.find(variant_id); + + if(var_it != variant_map.end()) + { + return (var_it->second).get(); + } + + variant_map[variant_id].reset(new VariantID_t(variant_id)); + auto the_variant = variant_map[variant_id].get(); + + assert(the_variant->IsRegistered()==true); + // disallow variants that share shallow copies to both be read in + // to prevent desynchronization. + assert(!filesAlreadyPresent(the_variant->GetFiles())); + + // add files + for(auto &curr_file : the_variant->GetFiles()) + { + file_IR_map[curr_file->GetBaseID()]=FileIRInfo_t(); + file_IR_map[curr_file->GetBaseID()].file = curr_file; + } + + return the_variant; +} + + +int IRDBObjects_t::writeBackVariant(const db_id_t variant_id) +{ + const auto it = variant_map.find(variant_id); + + if(it == variant_map.end()) + return 1; + + try + { + // cout<<"Writing changes for variant "<<variant_id<<endl; + it->second->WriteToDB(); + return 0; + } + catch (DatabaseError_t pnide) + { + cerr << "Unexpected database error: " << pnide << "variant ID: " << variant_id << endl; + return -1; + } + catch (...) + { + cerr << "Unexpected error variant ID: " << variant_id << endl; + return -1; + } + assert(0); // unreachable. + +} + +void IRDBObjects_t::deleteVariant(const db_id_t variant_id) +{ + const auto var_it = variant_map.find(variant_id); + + if(var_it == variant_map.end()) + return; + + // remove files and file IRs + for(const auto file : var_it->second->GetFiles()) + { + file_IR_map.erase(file->GetBaseID()); + } + + // remove variant + variant_map.erase(variant_id); +} + +int IRDBObjects_t::writeBackAll(void) +{ + int ret_status = 0; + + // Write back FileIRs + for(auto &file_pair : file_IR_map) + { + const int result = IRDBObjects_t::writeBackFileIR((file_pair.second.file)->GetBaseID()); + if(result != 0) + { + ret_status = -1; + } + } + + // Write back Variants + for(auto & variant_pair : variant_map) + { + const int result = IRDBObjects_t::writeBackVariant((variant_pair.second)->GetBaseID()); + if(result != 0) + { + ret_status = -1; + } + } + return ret_status; +} + +void IRDBObjects_t::deleteAll(void) +{ + // Delete Variants (also deletes all files) + for( auto &variant_pair : variant_map) + { + IRDBObjects_t::deleteVariant((variant_pair.second)->GetBaseID()); + } +} + +pqxxDB_t* IRDBObjects_t::getDBInterface() const +{ + return pqxx_interface.get(); +} + +pqxxDB_t* IRDBObjects_t::resetDBInterface() +{ + pqxx_interface.reset(new pqxxDB_t()); // Aborts if Commit() has not been called + BaseObj_t::SetInterface(pqxx_interface.get()); + return pqxx_interface.get(); +} + diff --git a/libIRDB/src/core/SConscript b/libIRDB/src/core/SConscript index c42f7419ef14a492131321c105f8738bef81618f..b538b5836f21a88d19043920906a184377752fa9 100644 --- a/libIRDB/src/core/SConscript +++ b/libIRDB/src/core/SConscript @@ -25,6 +25,7 @@ files= ''' reloc.cpp decode_cs.cpp operand_cs.cpp + IRDB_Objects.cpp ''' unused_files=''' decode_bea.cpp @@ -41,15 +42,17 @@ cpppath=''' $SECURITY_TRANSFORMS_HOME/libIRDB/include/ $SECURITY_TRANSFORMS_HOME/libcapstone/include/capstone/ ''' -# $SECURITY_TRANSFORMS_HOME/beaengine/include -# $SECURITY_TRANSFORMS_HOME/beaengine/beaengineSources/Includes/ +libpath=''' + $SECURITY_TRANSFORMS_HOME/lib + ''' -globs=glob.glob(os.environ['SECURITY_TRANSFORMS_HOME']+'/libcapstone/zipr_unpack/*.o') +#globs=glob.glob(os.environ['SECURITY_TRANSFORMS_HOME']+'/libcapstone/zipr_unpack/*.o') myenv.Append(CCFLAGS=" -Wall -std=c++11 ") +myenv.Append(LIBPATH=libpath) myenv=myenv.Clone(CPPPATH=Split(cpppath)) -mylib=myenv.Library(libname, Split(files) + globs) +mylib=myenv.SharedLibrary(libname, Split(files), LIBS=Split("pqxx capstone")) install=myenv.Install("$SECURITY_TRANSFORMS_HOME/lib/", mylib) Default(install) diff --git a/libIRDB/src/core/decode_cs.cpp b/libIRDB/src/core/decode_cs.cpp index 70dcc4c35477f73eb22ebc7f043e100ccbddc2c3..35119cd274dab619b230069c96c05f62e03c6d8c 100644 --- a/libIRDB/src/core/decode_cs.cpp +++ b/libIRDB/src/core/decode_cs.cpp @@ -18,7 +18,6 @@ DecodedInstructionCapstone_t::CapstoneHandle_t* DecodedInstructionCapstone_t::cs DecodedInstructionCapstone_t::CapstoneHandle_t::CapstoneHandle_t(FileIR_t* firp) { - const auto width=FileIR_t::GetArchitectureBitWidth(); const auto mode = (width==64) ? CS_MODE_64: CS_MODE_32; static_assert(sizeof(csh)==sizeof(handle), "Capstone handle size is unexpected. Has CS changed?"); @@ -197,7 +196,8 @@ DecodedInstructionCapstone_t::DecodedInstructionCapstone_t(const Instruction_t* if(!i) throw std::invalid_argument("No instruction given to DecodedInstruction_t(Instruction_t*)"); const auto length=i->GetDataBits().size(); - const auto data=i->GetDataBits().data(); + const auto &databits=i->GetDataBits(); + const auto data=databits.data(); const auto address=i->GetAddress()->GetVirtualOffset(); Disassemble(address,data,length); diff --git a/libIRDB/src/core/fileir.cpp b/libIRDB/src/core/fileir.cpp index 5225664cbdb3ab4271752ef0e624343e305ec019..7e93e6562928d909c63d44fe917b28702cf71997 100644 --- a/libIRDB/src/core/fileir.cpp +++ b/libIRDB/src/core/fileir.cpp @@ -82,11 +82,11 @@ static virtual_offset_t strtovo(std::string s) } // Create a Variant from the database -FileIR_t::FileIR_t(const VariantID_t &newprogid, File_t* fid) : BaseObj_t(NULL) +FileIR_t::FileIR_t(const VariantID_t &newprogid, File_t* fid) + : BaseObj_t(NULL), progid((VariantID_t&) newprogid) { orig_variant_ir_p=NULL; - progid=newprogid; if(fid==NULL) fileptr=newprogid.GetMainFile(); @@ -711,7 +711,7 @@ void FileIR_t::ReadRelocsFromDB void FileIR_t::WriteToDB() { - const auto WriteIRDB_start = clock(); +// const auto WriteIRDB_start = clock(); const auto pqIntr=dynamic_cast<pqxxDB_t*>(dbintr); assert(pqIntr); @@ -752,7 +752,7 @@ void FileIR_t::WriteToDB() } } dbintr->IssueQuery(q); - + // write out functions auto withHeader=true; @@ -777,7 +777,6 @@ void FileIR_t::WriteToDB() } W_addrs.complete(); - // write out instructions pqxx::tablewriter W(pqIntr->GetTransaction(),fileptr->instruction_table_name); @@ -823,7 +822,6 @@ void FileIR_t::WriteToDB() } } - const auto &insn_values=(*i)->WriteToDB(fileptr,j); W << insn_values; @@ -831,7 +829,6 @@ void FileIR_t::WriteToDB() W.complete(); - // icfs for (ICFSSet_t::iterator it = GetAllICFS().begin(); it != GetAllICFS().end(); ++it) { @@ -872,42 +869,46 @@ void FileIR_t::WriteToDB() // eh css relocs for(const auto& i : eh_css) { - for(auto& reloc : i->GetRelocations()) + const auto &relocs=i->GetRelocations(); + for(auto& reloc : relocs) W_reloc << reloc->WriteToDB(fileptr,i); } // eh pgms relocs for(const auto& i : eh_pgms) { - string r=""; - for(auto& reloc : i->GetRelocations()) + const auto &relocs=i->GetRelocations(); + for(auto& reloc : relocs) W_reloc << reloc->WriteToDB(fileptr,i); } // scoops relocs for(const auto& i : scoops) { - for(auto& reloc : i->GetRelocations()) + const auto &relocs=i->GetRelocations(); + for(auto& reloc : relocs) W_reloc << reloc->WriteToDB(fileptr,i); } // write out instruction's relocs for(const auto& i : insns) { - for(auto& reloc : i->GetRelocations()) + const auto &relocs=i->GetRelocations(); + for(auto& reloc : relocs) W_reloc << reloc->WriteToDB(fileptr,i); } W_reloc.complete(); +/* std::cout << std::dec; + const auto WriteIRDB_end = clock(); const auto read_time = (double)(ReadIRDB_end-ReadIRDB_start)/ CLOCKS_PER_SEC; const auto write_time = (double)(WriteIRDB_end-WriteIRDB_start)/ CLOCKS_PER_SEC; const auto wall_time = (double)(WriteIRDB_end-ReadIRDB_start)/ CLOCKS_PER_SEC; const auto transform_time=wall_time - read_time - write_time; - - std::cout << std::dec; std::cout << std::fixed << std::setprecision(2) << "#ATTRIBUTE ReadIRDB_WallClock=" << read_time <<endl; std::cout << std::fixed << std::setprecision(2) << "#ATTRIBUTE WriteIRDB_WallClock=" << write_time << endl; std::cout << std::fixed << std::setprecision(2) << "#ATTRIBUTE TotalIRDB_WallClock=" << wall_time << endl; std::cout << std::fixed << std::setprecision(2) << "#ATTRIBUTE TransformIRDB_WallClock=" << transform_time << endl; +*/ } @@ -1327,12 +1328,14 @@ void FileIR_t::GarbageCollectICFS() } } +/* int unused_icfs = this->GetAllICFS().size() - used_icfs.size(); if (unused_icfs > 0) { cerr << "FileIR_t::GarbageCollectICFS(): WARNING: " << dec << unused_icfs << " unused ICFS found. "; cerr << "Deleting before committing to IRDB" << endl; } +*/ ICFSSet_t to_erase; for(ICFSSet_t::const_iterator it=this->GetAllICFS().begin(); diff --git a/libIRDB/src/core/variantid.cpp b/libIRDB/src/core/variantid.cpp index 4daf529f1cc4d7e87a08322aceef47c9250a4014..df50ce94979590ef862885f5e08a32895c11a569 100644 --- a/libIRDB/src/core/variantid.cpp +++ b/libIRDB/src/core/variantid.cpp @@ -38,6 +38,15 @@ VariantID_t::VariantID_t() : } +VariantID_t::~VariantID_t() +{ + for(auto it : files) + { + delete it; + } +} + + void VariantID_t::CreateTables() { // note: this tables are now part of File_t. @@ -50,8 +59,7 @@ VariantID_t::VariantID_t(db_id_t pid) : BaseObj_t(NULL) q+=to_string(pid); q+=";"; - - try + try { BaseObj_t::dbintr->IssueQuery(q); } @@ -63,7 +71,7 @@ VariantID_t::VariantID_t(db_id_t pid) : BaseObj_t(NULL) throw DatabaseError_t(DatabaseError_t::VariantTableNotRegistered); }; - + if(BaseObj_t::dbintr->IsDone()) throw DatabaseError_t(DatabaseError_t::VariantNotInDatabase); @@ -148,9 +156,9 @@ VariantID_t* VariantID_t::Clone(bool deep) return ret; } -void VariantID_t::CloneFiles(set<File_t*> &files) +void VariantID_t::CloneFiles(FileSet_t &files) { - for(set<File_t*>::iterator fiter=files.begin(); fiter!=files.end(); ++fiter) + for(auto fiter=files.begin(); fiter!=files.end(); ++fiter) files.insert(CloneFile(*fiter)); } @@ -412,7 +420,7 @@ void VariantID_t::DropFromDB() File_t* VariantID_t::GetMainFile() const { for( - set<File_t*>::iterator it=files.begin(); + auto it=files.begin(); it!=files.end(); ++it ) @@ -465,8 +473,8 @@ void VariantID_t::ReadFilesFromDB() File_t *newfile=new File_t(file_id,orig_fid,url,hash,type,oid,atn,ftn,itn,icfs,icfs_map,rtn,typ,dtn,ehp,css,doipid); -std::cout<<"Found file "<<file_id<<"."<<std::endl; -std::cout<<" atn: " << atn << " ftn: " << ftn << " rtn: " << rtn << " typ: " << typ << std::endl; +// std::cout<<"Found file "<<file_id<<"."<<std::endl; +// std::cout<<" atn: " << atn << " ftn: " << ftn << " rtn: " << rtn << " typ: " << typ << std::endl; files.insert(newfile); diff --git a/libIRDB/src/syscall/SConscript b/libIRDB/src/syscall/SConscript index 358dafeb992a32b0b574f5a449c5d8cab3c566a2..2a5acdb1c9e4901dd7f686a81fadda160cbaa61f 100644 --- a/libIRDB/src/syscall/SConscript +++ b/libIRDB/src/syscall/SConscript @@ -20,7 +20,7 @@ myenv.Append(CXXFLAGS = " -std=c++11 ") myenv=myenv.Clone(CPPPATH=Split(cpppath)) -lib=myenv.Library(libname, Split(files)) +lib=myenv.SharedLibrary(libname, Split(files), LIBS=Split("IRDB-core IRDB-util"), LIBPATH=Split("$SECURITY_TRANSFORMS_HOME/lib")) install=myenv.Install("$SECURITY_TRANSFORMS_HOME/lib/", lib) Default(install) diff --git a/libIRDB/src/util/SConscript b/libIRDB/src/util/SConscript index 202ea107ef42cee0a01befee0d72f5adedc9c452..2e773c30efd67175ad565c6105cf6ff741e239f7 100644 --- a/libIRDB/src/util/SConscript +++ b/libIRDB/src/util/SConscript @@ -20,7 +20,7 @@ cpppath=''' myenv=myenv.Clone(CPPPATH=Split(cpppath)) myenv.Append(CXXFLAGS = " -std=c++11 ") -lib=myenv.Library(libname, Split(files)) +lib=myenv.SharedLibrary(libname, Split(files), LIBS=Split("IRDB-core"), LIBPATH=Split("$SECURITY_TRANSFORMS_HOME/lib")) install=env.Install("$SECURITY_TRANSFORMS_HOME/lib/", lib) Default(install) diff --git a/libIRDB/test/SConscript b/libIRDB/test/SConscript index 6a8156d2a984364bcc0ebecbc48bd86f0a1b2f8b..e01ec444ab8b37b14a3263868c3dff5620de359f 100644 --- a/libIRDB/test/SConscript +++ b/libIRDB/test/SConscript @@ -20,27 +20,27 @@ if 'build_tools' not in myenv or myenv['build_tools'] is None or int(myenv['buil ''' LIBPATH="$SECURITY_TRANSFORMS_HOME/lib" - LIBS=Split( 'IRDB-cfg IRDB-util ' + env.subst('$BASE_IRDB_LIBS')+ " MEDSannotation ehp") + LIBS=Split( 'IRDB-cfg IRDB-util ' + env.subst('$BASE_IRDB_LIBS')+ " MEDSannotation ehp transform") myenv=myenv.Clone(CPPPATH=Split(cpppath)) myenv.Append(CCFLAGS=" -std=c++11 -Wall") - ehframe=myenv.Object("read_ehframe.cpp"); - split_eh_frame=myenv.Object("split_eh_frame.cpp"); + ehframe=myenv.SharedObject("read_ehframe.cpp"); + split_eh_frame=myenv.SharedObject("split_eh_frame.cpp"); - pgm=myenv.Program("fill_in_indtargs.exe", ehframe+split_eh_frame+Split("fill_in_indtargs.cpp check_thunks.cpp"), LIBPATH=LIBPATH, LIBS=LIBS) - install=myenv.Install("$SECURITY_TRANSFORMS_HOME/bin/", pgm) + pgm=myenv.SharedLibrary("fill_in_indtargs.so", ehframe+split_eh_frame+Split("fill_in_indtargs.cpp check_thunks.cpp"), LIBPATH=LIBPATH, LIBS=LIBS) + install=myenv.Install("$SECURITY_TRANSFORMS_HOME/plugins_install", pgm) Default(install) installed=installed+install - pgm=myenv.Program("fill_in_cfg.exe", split_eh_frame+Split("fill_in_cfg.cpp"), LIBPATH=LIBPATH, LIBS=LIBS) - install=myenv.Install("$SECURITY_TRANSFORMS_HOME/bin/", pgm) + pgm=myenv.SharedLibrary("fill_in_cfg.so", Split("fill_in_cfg.cpp")+split_eh_frame, LIBPATH=LIBPATH, LIBS=LIBS) + install=myenv.Install("$SECURITY_TRANSFORMS_HOME/plugins_install", pgm) Default(install) installed=installed+install - pgm=myenv.Program("fix_calls.exe", ehframe+Split("fix_calls.cpp"), LIBPATH=LIBPATH, LIBS=LIBS) - install=myenv.Install("$SECURITY_TRANSFORMS_HOME/bin/", pgm) + pgm=myenv.SharedLibrary("fix_calls.so", ehframe+Split("fix_calls.cpp"), LIBPATH=LIBPATH, LIBS=LIBS) + install=myenv.Install("$SECURITY_TRANSFORMS_HOME/plugins_install", pgm) Default(install) installed=installed+install @@ -53,7 +53,7 @@ if 'build_tools' not in myenv or myenv['build_tools'] is None or int(myenv['buil for i in Split(pgms): # print "Registering pgm: "+ i pgm=myenv.Program(target=i+".exe", source=Split(i+".cpp"), LIBPATH=LIBPATH, LIBS=LIBS) - install=myenv.Install("$SECURITY_TRANSFORMS_HOME/bin/", pgm) + install=myenv.Install("$SECURITY_TRANSFORMS_HOME/plugins_install/", pgm) Default(install) installed=installed+install diff --git a/libIRDB/test/fill_in_cfg.cpp b/libIRDB/test/fill_in_cfg.cpp index e0719bcce5cd5f0a60ecaed62ec1fe1752715108..901638f121993e08248d05479da016e553e971b4 100644 --- a/libIRDB/test/fill_in_cfg.cpp +++ b/libIRDB/test/fill_in_cfg.cpp @@ -18,34 +18,21 @@ * */ -#include <libIRDB-core.hpp> +#include "fill_in_cfg.hpp" #include <iostream> #include <fstream> -#include <stdlib.h> #include <string.h> -#include <map> #include <assert.h> #include <sys/mman.h> #include <ctype.h> -#include <exeio.h> #include "elfio/elfio.hpp" #include "split_eh_frame.hpp" -int odd_target_count=0; -int bad_target_count=0; -int bad_fallthrough_count=0; -EXEIO::exeio *elfiop=NULL; - using namespace libIRDB; using namespace std; using namespace EXEIO; -set< pair<db_id_t,virtual_offset_t> > missed_instructions; -auto failed_target_count=0U; - -pqxxDB_t pqxx_interface; - -void populate_instruction_map +void PopulateCFG::populate_instruction_map ( map< pair<db_id_t,virtual_offset_t>, Instruction_t*> &insnMap, FileIR_t *firp @@ -74,7 +61,7 @@ void populate_instruction_map } -void set_fallthrough +void PopulateCFG::set_fallthrough ( map< pair<db_id_t,virtual_offset_t>, Instruction_t*> &insnMap, DecodedInstruction_t *disasm, Instruction_t *insn, FileIR_t *firp @@ -125,7 +112,7 @@ void set_fallthrough } -void set_target +void PopulateCFG::set_target ( map< pair<db_id_t,virtual_offset_t>, Instruction_t*> &insnMap, DecodedInstruction_t *disasm, Instruction_t *insn, FileIR_t *firp @@ -200,30 +187,14 @@ void set_target } } -static File_t* find_file(FileIR_t* firp, db_id_t fileid) +File_t* PopulateCFG::find_file(FileIR_t* firp, db_id_t fileid) { -#if 0 - set<File_t*> &files=firp->GetFiles(); - - for( - set<File_t*>::iterator it=files.begin(); - it!=files.end(); - ++it - ) - { - File_t* thefile=*it; - if(thefile->GetBaseID()==fileid) - return thefile; - } - return NULL; -#endif assert(firp->GetFile()->GetBaseID()==fileid); return firp->GetFile(); - } -void add_new_instructions(FileIR_t *firp) +void PopulateCFG::add_new_instructions(FileIR_t *firp) { int found_instructions=0; for( @@ -353,7 +324,7 @@ void add_new_instructions(FileIR_t *firp) } -void fill_in_cfg(FileIR_t *firp) +void PopulateCFG::fill_in_cfg(FileIR_t *firp) { int round=0; @@ -435,7 +406,7 @@ void fill_in_cfg(FileIR_t *firp) } -static bool is_in_relro_segment(const int secndx) +bool PopulateCFG::is_in_relro_segment(const int secndx) { ELFIO::elfio *real_elfiop = reinterpret_cast<ELFIO::elfio*>(elfiop->get_elfio()); if(!real_elfiop) @@ -477,7 +448,7 @@ static bool is_in_relro_segment(const int secndx) return false; } -void fill_in_scoops(FileIR_t *firp) +void PopulateCFG::fill_in_scoops(FileIR_t *firp) { auto max_base_id=firp->GetMaxBaseID(); @@ -551,7 +522,7 @@ void fill_in_scoops(FileIR_t *firp) } -void fill_in_landing_pads(FileIR_t *firp) +void PopulateCFG::fill_in_landing_pads(FileIR_t *firp) { const auto eh_frame_rep_ptr = split_eh_frame_t::factory(firp); // eh_frame_rep_ptr->parse(); already parsed now. @@ -609,74 +580,56 @@ void fill_in_landing_pads(FileIR_t *firp) } -void parse_args(int argc, char* argv[], bool &fix_landing_pads) -{ - for (int i = 0; i < argc; ++i) - { - if (strcmp("--fix-landing-pads", argv[i]) == 0) - { - fix_landing_pads = true; - } - else if (strcmp("--no-fix-landing-pads", argv[i]) == 0) - { - fix_landing_pads = false; - } - } +int PopulateCFG::parseArgs(const vector<string> step_args) +{ + if(step_args.size()<1) + { + cerr<<"Usage: <id> [--fix-landing-pads | --no-fix-landing-pads]"<<endl; + return -1; + } + + variant_id = stoi(step_args[0]); + + for (unsigned int i = 1; i < step_args.size(); ++i) + { + if (step_args[i]=="--fix-landing-pads") + { + fix_landing_pads = true; + } + else if (step_args[i]=="--no-fix-landing-pads") + { + fix_landing_pads = false; + } + } + + cout<<"fix_landing_pads="<<fix_landing_pads<<endl; + + return 0; } -int main(int argc, char* argv[]) +int PopulateCFG::executeStep(IRDBObjects_t *const irdb_objects) { - bool fix_landing_pads = true; // default - - if(argc<2) - { - cerr<<"Usage: fill_in_cfg <id> [--fix-landing-pads | --no-fix-landing-pads]"<<endl; - exit(-1); - } - - parse_args(argc, argv, fix_landing_pads); - - cout<<"fix_landing_pads="<<fix_landing_pads<<endl; - - VariantID_t *pidp=NULL; - FileIR_t * firp=NULL; - - try + try { - /* setup the interface to the sql server */ - BaseObj_t::SetInterface(&pqxx_interface); + const auto pqxx_interface = irdb_objects->getDBInterface(); + // now set the DB interface for THIS PLUGIN LIBRARY -- VERY IMPORTANT + BaseObj_t::SetInterface(pqxx_interface); - pidp=new VariantID_t(atoi(argv[1])); - - assert(pidp->IsRegistered()==true); - - cout<<"New Variant, after reading registration, is: "<<*pidp << endl; - - for(set<File_t*>::iterator it=pidp->GetFiles().begin(); - it!=pidp->GetFiles().end(); - ++it - ) + const auto variant = irdb_objects->addVariant(variant_id); + for(File_t* file : variant->GetFiles()) { - File_t* this_file=*it; - assert(this_file); - cout<<"Filling in cfg for "<<this_file->GetURL()<<endl; - - - // read the db - firp=new FileIR_t(*pidp, this_file); + const auto firp = irdb_objects->addFileIR(variant_id, file->GetBaseID()); assert(firp); + cout<<"Filling in cfg for "<<firp->GetFile()->GetURL()<<endl; /* get the OID of the file */ - int elfoid=this_file->GetELFOID(); + const int elfoid=firp->GetFile()->GetELFOID(); pqxx::largeobject lo(elfoid); - lo.to_file(pqxx_interface.GetTransaction(),"readeh_tmp_file.exe"); + lo.to_file(pqxx_interface->GetTransaction(),"readeh_tmp_file.exe"); - elfiop=new EXEIO::exeio; - assert(elfiop); + elfiop.reset(new exeio()); elfiop->load(string("readeh_tmp_file.exe")); - //EXEIO::dump::header(cout,*elfiop); - //EXEIO::dump::section_headers(cout,*elfiop); fill_in_cfg(firp); fill_in_scoops(firp); @@ -685,29 +638,26 @@ int main(int argc, char* argv[]) { fill_in_landing_pads(firp); } - - // write the DB back and commit our changes - firp->WriteToDB(); - delete firp; - delete elfiop; - firp=NULL; - elfiop=NULL; - } - - - pqxx_interface.Commit(); - } catch (DatabaseError_t pnide) { - cout<<"Unexpected database error: "<<pnide<<endl; - exit(-1); + cerr<<"Unexpected database error: "<<pnide<<endl; + return -1; } + catch(...) + { + cerr<<"Unexpected error"<<endl; + return -1; + } + + return 0; +} - assert(pidp); - delete pidp; - pidp=NULL; - return 0; +extern "C" +shared_ptr<Transform_SDK::TransformStep_t> GetTransformStep(void) +{ + const shared_ptr<Transform_SDK::TransformStep_t> the_step(new PopulateCFG()); + return the_step; } diff --git a/libIRDB/test/fill_in_cfg.hpp b/libIRDB/test/fill_in_cfg.hpp new file mode 100644 index 0000000000000000000000000000000000000000..4925bafda9045fc015ce8ca053ad0bf9bcb0090e --- /dev/null +++ b/libIRDB/test/fill_in_cfg.hpp @@ -0,0 +1,86 @@ +#ifndef fill_in_cfg_hpp +#define fill_in_cfg_hpp + +#include <libIRDB-core.hpp> +#include <stdlib.h> +#include <map> +#include <exeio.h> + +class PopulateCFG : public libIRDB::Transform_SDK::TransformStep_t +{ + public: + PopulateCFG(libIRDB::db_id_t p_variant_id = 0, + bool p_fix_landing_pads = true + ) + : + variant_id(p_variant_id), + fix_landing_pads(p_fix_landing_pads) + { + odd_target_count = 0; + bad_target_count = 0; + bad_fallthrough_count = 0; + failed_target_count = 0U; + + elfiop = std::unique_ptr<EXEIO::exeio>(nullptr); + } + + ~PopulateCFG(void) override + { + // do nothing (this class uses shared IRDB objects that + // are not managed by this class). + } + + std::string getStepName(void) const override + { + return std::string("fill_in_cfg"); + } + int parseArgs(const std::vector<std::string> step_args) override; + int executeStep(libIRDB::IRDBObjects_t *const) override; + + private: // methods + + // main workers + void fill_in_cfg(libIRDB::FileIR_t *); + void fill_in_scoops(libIRDB::FileIR_t *); + void fill_in_landing_pads(libIRDB::FileIR_t *); + + // helpers + void populate_instruction_map + ( + std::map< std::pair<libIRDB::db_id_t,libIRDB::virtual_offset_t>, libIRDB::Instruction_t*>&, + libIRDB::FileIR_t * + ); + + void set_fallthrough + ( + std::map< std::pair<libIRDB::db_id_t,libIRDB::virtual_offset_t>, libIRDB::Instruction_t*>&, + libIRDB::DecodedInstruction_t *, libIRDB::Instruction_t *, libIRDB::FileIR_t * + ); + + void set_target + ( + std::map< std::pair<libIRDB::db_id_t,libIRDB::virtual_offset_t>, libIRDB::Instruction_t*>&, + libIRDB::DecodedInstruction_t *, libIRDB::Instruction_t *, libIRDB::FileIR_t * + ); + + libIRDB::File_t* find_file(libIRDB::FileIR_t *, libIRDB::db_id_t); + void add_new_instructions(libIRDB::FileIR_t *); + bool is_in_relro_segment(const int); + + private: //data + + // stats + int odd_target_count; + int bad_target_count; + int bad_fallthrough_count; + unsigned int failed_target_count; + + // non-optional + libIRDB::db_id_t variant_id; + bool fix_landing_pads; + + std::unique_ptr<EXEIO::exeio> elfiop; + std::set< std::pair<libIRDB::db_id_t,libIRDB::virtual_offset_t> > missed_instructions; +}; + +#endif diff --git a/libIRDB/test/fill_in_indtargs.cpp b/libIRDB/test/fill_in_indtargs.cpp index 02046e3a444ccd345e8e0ac211873cd6376c0114..e796cb2690325bae0b2527458bc2b1b0bbff7ce2 100644 --- a/libIRDB/test/fill_in_indtargs.cpp +++ b/libIRDB/test/fill_in_indtargs.cpp @@ -25,6 +25,7 @@ #include <fstream> #include <limits> #include <string> +#include <algorithm> #include <stdlib.h> #include <string.h> #include <map> @@ -53,8 +54,16 @@ using namespace MEDS_Annotation; #define arch_ptr_bytes() (firp->GetArchitectureBitWidth()/8) #define ALLOF(a) begin(a),end(a) +extern void read_ehframe(FileIR_t* firp, EXEIO::exeio* ); + +class PopulateIndTargs_t : public libIRDB::Transform_SDK::TransformStep_t +{ + +public: + + /* - * global variables + * class variables */ // @@ -83,36 +92,8 @@ map<virtual_offset_t,Instruction_t*> lookupInstructionMap; // the set of things that are partially unpinned already. set<Instruction_t*> already_unpinned; -static long total_unpins=0; - - -/* - * Forward prototypes - */ - - -static void check_for_PIC_switch_table32_type2(FileIR_t *firp, Instruction_t* insn, DecodedInstruction_t disasm, EXEIO::exeio* elfiop, const set<virtual_offset_t>& thunk_bases); -static void check_for_PIC_switch_table32_type3(FileIR_t* firp, Instruction_t* insn, DecodedInstruction_t disasm, EXEIO::exeio* elfiop, const set<virtual_offset_t>& thunk_bases); -static void check_for_PIC_switch_table32(FileIR_t*, Instruction_t* insn, DecodedInstruction_t disasm, EXEIO::exeio* elfiop, const set<virtual_offset_t>& thunk_bases); -static void check_for_PIC_switch_table64(FileIR_t*, Instruction_t* insn, DecodedInstruction_t disasm, EXEIO::exeio* elfiop); -static void check_for_nonPIC_switch_table(FileIR_t*, Instruction_t* insn, DecodedInstruction_t disasm, EXEIO::exeio* elfiop); -static void check_for_nonPIC_switch_table_pattern2(FileIR_t*, Instruction_t* insn, DecodedInstruction_t disasm, EXEIO::exeio* elfiop); - -extern void read_ehframe(FileIR_t* firp, EXEIO::exeio* ); - - - - -template <class T> T MAX(T a, T b) -{ - return a>b ? a : b; -} - +long total_unpins=0; -/* - * range - record a new eh_frame range into the ranges global variable. - * this is called from read_ehframe. - */ void range(virtual_offset_t start, virtual_offset_t end) { pair<virtual_offset_t,virtual_offset_t> foo(start,end); @@ -627,7 +608,7 @@ bool backup_until(const string &insn_type_regex_str, Instruction_t *& prev, Inst /* * check_for_PIC_switch_table32 - look for switch tables in PIC code for 32-bit code. */ -static void check_for_PIC_switch_table32(FileIR_t *firp, Instruction_t* insn, DecodedInstruction_t disasm, EXEIO::exeio* elfiop, const set<virtual_offset_t> &thunk_bases) +void check_for_PIC_switch_table32(FileIR_t *firp, Instruction_t* insn, DecodedInstruction_t disasm, EXEIO::exeio* elfiop, const set<virtual_offset_t> &thunk_bases) { ibt_provenance_t prov=ibt_provenance_t::ibtp_switchtable_type1; @@ -817,7 +798,7 @@ I7: 08069391 <_gedit_app_ready+0x91> ret -static void check_for_PIC_switch_table32_type2(FileIR_t *firp, Instruction_t* insn, DecodedInstruction_t disasm, EXEIO::exeio* elfiop, const set<virtual_offset_t> &thunk_bases) +void check_for_PIC_switch_table32_type2(FileIR_t *firp, Instruction_t* insn, DecodedInstruction_t disasm, EXEIO::exeio* elfiop, const set<virtual_offset_t> &thunk_bases) { ibt_provenance_t prov=ibt_provenance_t::ibtp_switchtable_type2; auto ibtargets = InstructionSet_t(); @@ -954,7 +935,7 @@ cout<<hex<<"Found (type2) switch dispatch at "<<I5->GetAddress()->GetVirtualOffs * * nb: also works for 64-bit. */ -static void check_for_PIC_switch_table32_type3(FileIR_t* firp, Instruction_t* insn, DecodedInstruction_t disasm, EXEIO::exeio* elfiop, const set<virtual_offset_t> &thunk_bases) +void check_for_PIC_switch_table32_type3(FileIR_t* firp, Instruction_t* insn, DecodedInstruction_t disasm, EXEIO::exeio* elfiop, const set<virtual_offset_t> &thunk_bases) { uint32_t ptrsize=firp->GetArchitectureBitWidth()/8; ibt_provenance_t prov=ibt_provenance_t::ibtp_switchtable_type3; @@ -1181,7 +1162,7 @@ static void check_for_PIC_switch_table32_type3(FileIR_t* firp, Instruction_t* in * if so, see if we can trace back a few instructions to find a * the start of the table. */ -static void check_for_PIC_switch_table64(FileIR_t* firp, Instruction_t* insn, DecodedInstruction_t disasm, EXEIO::exeio* elfiop) +void check_for_PIC_switch_table64(FileIR_t* firp, Instruction_t* insn, DecodedInstruction_t disasm, EXEIO::exeio* elfiop) { ibt_provenance_t prov=ibt_provenance_t::ibtp_switchtable_type4; /* here's the pattern we're looking for */ @@ -1546,7 +1527,7 @@ Note: Here the operands of the add are reversed, so lookup code was not finding nb: handles both 32 and 64 bit */ -static void check_for_nonPIC_switch_table_pattern2(FileIR_t* firp, Instruction_t* insn, DecodedInstruction_t disasm, EXEIO::exeio* elfiop) +void check_for_nonPIC_switch_table_pattern2(FileIR_t* firp, Instruction_t* insn, DecodedInstruction_t disasm, EXEIO::exeio* elfiop) { ibt_provenance_t prov=ibt_provenance_t::ibtp_switchtable_type5; Instruction_t *I1 = nullptr; @@ -1654,7 +1635,7 @@ static void check_for_nonPIC_switch_table_pattern2(FileIR_t* firp, Instruction_t nb: handles both 32 and 64 bit */ -static void check_for_nonPIC_switch_table(FileIR_t* firp, Instruction_t* insn, DecodedInstruction_t disasm, EXEIO::exeio* elfiop) +void check_for_nonPIC_switch_table(FileIR_t* firp, Instruction_t* insn, DecodedInstruction_t disasm, EXEIO::exeio* elfiop) { ibt_provenance_t prov=ibt_provenance_t::ibtp_switchtable_type6; Instruction_t *I1 = nullptr; @@ -2816,7 +2797,7 @@ void unpin_well_analyzed_ibts(FileIR_t *firp, int64_t do_unpin_opt) /* * fill_in_indtargs - main driver routine for */ -void fill_in_indtargs(FileIR_t* firp, exeio* elfiop, std::list<virtual_offset_t> forced_pins, int64_t do_unpin_opt) +void fill_in_indtargs(FileIR_t* firp, exeio* elfiop, int64_t do_unpin_opt) { set<virtual_offset_t> thunk_bases; find_all_module_starts(firp,thunk_bases); @@ -2846,7 +2827,7 @@ void fill_in_indtargs(FileIR_t* firp, exeio* elfiop, std::list<virtual_offset_t> infer_targets(firp, elfiop->sections[secndx]); /* should move to separate function */ - std::list<virtual_offset_t>::iterator forced_iterator = forced_pins.begin(); + auto forced_iterator = forced_pins.begin(); for (; forced_iterator != forced_pins.end(); forced_iterator++) { possible_target(*forced_iterator, 0, ibt_provenance_t::ibtp_user); @@ -2895,37 +2876,42 @@ void fill_in_indtargs(FileIR_t* firp, exeio* elfiop, std::list<virtual_offset_t> } +bool split_eh_frame_opt=true; +int64_t do_unpin_opt=numeric_limits<int64_t>::max() ; +db_id_t variant_id=BaseObj_t::NOT_IN_DATABASE; +set<virtual_offset_t> forced_pins; -int main(int argc, char* argv[]) +int parseArgs(const vector<string> step_args) { - auto argc_iter = (int)2; - auto split_eh_frame_opt=true; - auto do_unpin_opt=numeric_limits<int64_t>::max() ; - auto forced_pins=std::list<virtual_offset_t> (); - if(argc<2) + if(step_args.size()<1) { - cerr<<"Usage: fill_in_indtargs <id> [--[no-]split-eh-frame] [--[no-]unpin] [addr,...]"<<endl; + cerr<<"Usage: <id> [--[no-]split-eh-frame] [--[no-]unpin] [addr,...]"<<endl; exit(-1); } + variant_id=stoi(step_args[0]); + cout<<"Parsing parameters with argc= " << step_args.size()<<endl; + // parse dash-style options. - while(argc_iter < argc && argv[argc_iter][0]=='-') + unsigned int argc_iter = 1; + while(argc_iter < step_args.size() && step_args[argc_iter][0]=='-') { - if(string(argv[argc_iter])=="--no-unpin") + cout<<"Parsing parameter: "<< step_args[argc_iter] << endl; + if(step_args[argc_iter]=="--no-unpin") { do_unpin_opt=-1; argc_iter++; } - else if(string(argv[argc_iter])=="--unpin") + else if(step_args[argc_iter]=="--unpin") { do_unpin_opt = numeric_limits<decltype(do_unpin_opt)>::max() ; argc_iter++; } - else if(string(argv[argc_iter])=="--max-unpin" || string(argv[argc_iter])=="--max-unpins") + else if(step_args[argc_iter]=="--max-unpin" || step_args[argc_iter]=="--max-unpins") { argc_iter++; - auto arg_as_str=argv[argc_iter]; + auto arg_as_str=step_args[argc_iter]; argc_iter++; try { @@ -2938,93 +2924,134 @@ int main(int argc, char* argv[]) } } - else if(string(argv[argc_iter])=="--no-split-eh-frame") + else if(step_args[argc_iter]=="--no-split-eh-frame") { split_eh_frame_opt=false; argc_iter++; } - else if(string(argv[argc_iter])=="--split-eh-frame") + else if(step_args[argc_iter]=="--split-eh-frame") { split_eh_frame_opt=true; argc_iter++; } else { - cerr<<"Unknown option: "<<argv[argc_iter]<<endl; - exit(2); + cerr<<"Unknown option: "<<step_args[argc_iter]<<endl; + return 2; } } // parse addr argumnets - for (; argc_iter < argc; argc_iter++) + for (; argc_iter < step_args.size(); argc_iter++) { char *end_ptr; - virtual_offset_t offset = strtol(argv[argc_iter], &end_ptr, 0); + virtual_offset_t offset = strtol(step_args[argc_iter].c_str(), &end_ptr, 0); if (*end_ptr == '\0') { cout << "force pinning: 0x" << std::hex << offset << endl; - forced_pins.push_back(offset); + forced_pins.insert(offset); } } - VariantID_t *pidp=nullptr; - FileIR_t * firp=nullptr; + cout<<"Setting unpin limit to: "<<dec<<do_unpin_opt<<endl; + return 0; +} + + +int executeStep(IRDBObjects_t *const irdb_objects) +{ + //VariantID_t *pidp=nullptr; + //FileIR_t * firp=nullptr; try { /* setup the interface to the sql server */ - pqxxDB_t pqxx_interface; - BaseObj_t::SetInterface(&pqxx_interface); + const auto pqxx_interface=irdb_objects->getDBInterface(); + BaseObj_t::SetInterface(pqxx_interface); + + auto pidp = irdb_objects->addVariant(variant_id); + assert(pidp); - pidp=new VariantID_t(atoi(argv[1])); + // pidp=new VariantID_t(atoi(argv[1])); assert(pidp->IsRegistered()==true); cout<<"New Variant, after reading registration, is: "<<*pidp << endl; - for(set<File_t*>::iterator it=pidp->GetFiles().begin(); - it!=pidp->GetFiles().end(); - ++it) + for(const auto &this_file : pidp->GetFiles()) { - File_t* this_file=*it; assert(this_file); cout<<"Analyzing file "<<this_file->GetURL()<<endl; // read the db - firp=new FileIR_t(*pidp, this_file); + auto firp = irdb_objects->addFileIR(variant_id, this_file->GetBaseID()); + // firp=new FileIR_t(*pidp, this_file); + assert(firp); + firp->SetBaseIDS(); + firp->AssembleRegistry(); // read the executeable file int elfoid=firp->GetFile()->GetELFOID(); pqxx::largeobject lo(elfoid); - lo.to_file(pqxx_interface.GetTransaction(),"readeh_tmp_file.exe"); - EXEIO::exeio* elfiop=new EXEIO::exeio; + lo.to_file(pqxx_interface->GetTransaction(),"readeh_tmp_file.exe"); + auto elfiop=unique_ptr<EXEIO::exeio>(new EXEIO::exeio); elfiop->load(string("readeh_tmp_file.exe")); // find all indirect branch targets - fill_in_indtargs(firp, elfiop, forced_pins, do_unpin_opt); + fill_in_indtargs(firp, elfiop.get(), do_unpin_opt); if(split_eh_frame_opt) split_eh_frame(firp); - - // write the DB back and commit our changes - firp->WriteToDB(); - - delete firp; } - if(getenv("FII_NOUPDATE")==nullptr) - pqxx_interface.Commit(); + if(getenv("FII_NOUPDATE")!=nullptr) + return -1; } catch (DatabaseError_t pnide) { cout<<"Unexpected database error: "<<pnide<<endl; - exit(-1); + return -1; + } + catch(...) + { + cerr<<"Unexpected error"<<endl; + return -1; } + return 0; +} - assert(firp && pidp); +std::string getStepName(void) const override +{ + return std::string("fill_in_indtargs"); +} - delete pidp; - return 0; + +}; + + +shared_ptr<Transform_SDK::TransformStep_t> curInvocation; + +bool possible_target(virtual_offset_t p, virtual_offset_t from_addr, ibt_provenance_t prov) +{ + assert(curInvocation); + return (dynamic_cast<PopulateIndTargs_t*>(curInvocation.get()))->possible_target(p,from_addr,prov); +} + +void range(virtual_offset_t start, virtual_offset_t end) +{ + assert(curInvocation); + return (dynamic_cast<PopulateIndTargs_t*>(curInvocation.get()))->range(start,end); } + +extern "C" +shared_ptr<Transform_SDK::TransformStep_t> GetTransformStep(void) +{ + curInvocation.reset(new PopulateIndTargs_t()); + return curInvocation; + + //return shared_ptr<Transform_SDK::TransformStep_t>(new PopulateIndTargs_t()); +} + + diff --git a/libIRDB/test/fix_calls.cpp b/libIRDB/test/fix_calls.cpp index 9e15b571e8c55c3ce145cf4517d3da73789014ed..84197c3410c857b226f96d4ea2e9c1b3a60308db 100644 --- a/libIRDB/test/fix_calls.cpp +++ b/libIRDB/test/fix_calls.cpp @@ -37,9 +37,18 @@ using namespace libIRDB; using namespace std; using namespace EXEIO; - +// macros #define ALLOF(a) begin(a),end(a) + +// externs +extern void read_ehframe(FileIR_t* firp, EXEIO::exeio* ); + +class FixCalls_t : public libIRDB::Transform_SDK::TransformStep_t +{ + +public: + class Range_t { public: @@ -64,7 +73,7 @@ struct Range_tCompare } }; -typedef std::set<Range_t, Range_tCompare> RangeSet_t; +using RangeSet_t = std::set<Range_t, Range_tCompare>; @@ -78,19 +87,9 @@ long long found_pattern=0; long long in_ehframe=0; long long no_fix_for_ib=0; long long no_fix_for_safefn=0; - -pqxxDB_t pqxx_interface; - bool opt_fix_icalls = false; bool opt_fix_safefn = true; -void fix_other_pcrel(FileIR_t* firp, Instruction_t *insn, uintptr_t offset); - -/* Read the exception handler frame so that those indirect branches are accounted for */ -void read_ehframe(FileIR_t* firp, EXEIO::exeio* ); - - - bool check_entry(bool &found, ControlFlowGraph_t* cfg) { @@ -138,6 +137,8 @@ bool check_entry(bool &found, ControlFlowGraph_t* cfg) return false; } +using ControlFlowGraphMap_t = map<Function_t*, ControlFlowGraph_t*>; +ControlFlowGraphMap_t cfg_optimizer; bool call_needs_fix(Instruction_t* insn) { @@ -242,9 +243,7 @@ bool call_needs_fix(Instruction_t* insn) return true; } - typedef map<Function_t*, ControlFlowGraph_t*> ControlFlowGraphMap_t; - static ControlFlowGraphMap_t cfg_optimizer; const auto is_found_it=cfg_optimizer.find(func); const auto is_found=(is_found_it!=end(cfg_optimizer)); @@ -424,7 +423,7 @@ string adjust_esp_offset(string newbits, int offset) * convert_to_jump - assume newbits is a call insn, convert newbits to a jump, and return it. * Also: if newbits is a call [esp+k], add "offset" to k. */ -static void convert_to_jump(Instruction_t* insn, int offset) +void convert_to_jump(Instruction_t* insn, int offset) { string newbits=insn->GetDataBits(); //DISASM d; @@ -956,15 +955,19 @@ void fix_other_pcrel(FileIR_t* firp) // // main rountine; convert calls into push/jump statements // -int main(int argc, char* argv[]) -{ +// int main(int argc, char* argv[]) + - bool fix_all=false; - bool do_eh_frame=true; +bool fix_all=false; +bool do_eh_frame=true; + + +int parseArgs(const vector<string> step_args) +{ - if(argc<2) + if(step_args.size()<1) { - cerr<<"Usage: fix_calls <id> [--fix-all | --no-fix-all ] [--eh-frame | --no-ehframe] "<<endl; + cerr<<"Usage: <id> [--fix-all | --no-fix-all ] [--eh-frame | --no-ehframe] "<<endl; cerr<<" --eh-frame " << endl; cerr<<" --no-eh-frame Use (or dont) the eh-frame section to be compatible with exception handling." << endl; cerr<<" --fix-all " << endl; @@ -974,81 +977,84 @@ int main(int argc, char* argv[]) exit(-1); } - for(int argc_iter=2; argc_iter<argc; argc_iter++) + for(unsigned int argc_iter=1; argc_iter<step_args.size(); argc_iter++) { - if(strcmp("--fix-all", argv[argc_iter])==0) + if("--fix-all"==step_args[argc_iter]) { fix_all=true; } - else if(strcmp("--no-fix-all", argv[argc_iter])==0) + else if("--no-fix-all"==step_args[argc_iter]) { fix_all=false; } - else if(strcmp("--eh-frame", argv[argc_iter])==0) + else if("--eh-frame"==step_args[argc_iter]) { do_eh_frame=true; } - else if(strcmp("--no-eh-frame", argv[argc_iter])==0) + else if("--no-eh-frame"==step_args[argc_iter]) { do_eh_frame=false; } - else if(strcmp("--fix-icalls", argv[argc_iter])==0) + else if("--fix-icalls"==step_args[argc_iter]) { opt_fix_icalls = true; } - else if(strcmp("--no-fix-icalls", argv[argc_iter])==0) + else if("--no-fix-icalls"==step_args[argc_iter]) { opt_fix_icalls = false; } - else if(strcmp("--fix-safefn", argv[argc_iter])==0) + else if("--fix-safefn"==step_args[argc_iter]) { opt_fix_safefn = true; } - else if(strcmp("--no-fix-safefn", argv[argc_iter])==0) + else if("--no-fix-safefn"==step_args[argc_iter]) { opt_fix_safefn = false; } else { - cerr<<"Unrecognized option: "<<argv[argc_iter]<<endl; - exit(-1); + cerr<<"Unrecognized option: "<<step_args[argc_iter]<<endl; + return -1; } } if(getenv("FIX_CALLS_FIX_ALL_CALLS")) fix_all=true; - VariantID_t *pidp=NULL; - FileIR_t *firp=NULL; + variant_id=stoi(step_args[0]); + return 0; +} + +db_id_t variant_id=BaseObj_t::NOT_IN_DATABASE; - /* setup the interface to the sql server */ - BaseObj_t::SetInterface(&pqxx_interface); - cout<<"Reading variant "<<string(argv[1])<<" from database." << endl; +int executeStep(IRDBObjects_t *const irdb_objects) +{ + + cout<<"Reading variant "<<variant_id<<" from database." << endl; try { + /* setup the interface to the sql server */ + const auto pqxx_interface=irdb_objects->getDBInterface(); + BaseObj_t::SetInterface(pqxx_interface); - pidp=new VariantID_t(atoi(argv[1])); + auto pidp = irdb_objects->addVariant(variant_id); cout<<"Fixing calls->push/jmp in variant "<<*pidp<< "." <<endl; assert(pidp->IsRegistered()==true); - for(set<File_t*>::iterator it=pidp->GetFiles().begin(); - it!=pidp->GetFiles().end(); - ++it - ) + for(const auto &this_file : pidp->GetFiles()) { - File_t* this_file=*it; assert(this_file); // read the db - firp=new FileIR_t(*pidp,this_file); + auto firp = irdb_objects->addFileIR(variant_id, this_file->GetBaseID()); assert(firp && pidp); eh_frame_ranges.clear(); int elfoid=firp->GetFile()->GetELFOID(); pqxx::largeobject lo(elfoid); - lo.to_file(pqxx_interface.GetTransaction(),"readeh_tmp_file.exe"); + lo.to_file(pqxx_interface->GetTransaction(),"readeh_tmp_file.exe"); EXEIO::exeio* elfiop=new EXEIO::exeio; elfiop->load(string("readeh_tmp_file.exe")); EXEIO::dump::header(cout,*elfiop); @@ -1059,24 +1065,22 @@ int main(int argc, char* argv[]) fix_all_calls(firp,true,fix_all); fix_other_pcrel(firp); - firp->WriteToDB(); cout<<"Done!"<<endl; - delete firp; } - cout<<"Writing variant "<<*pidp<<" back to database." << endl; - pqxx_interface.Commit(); - - } catch (DatabaseError_t pnide) { cout<<"Unexpected database error: "<<pnide<<endl; - exit(-1); + return -1; + } + catch(...) + { + cerr<<"Unexpected error"<<endl; + return -1; } - delete pidp; return 0; } @@ -1107,3 +1111,35 @@ bool possible_target(uintptr_t p, uintptr_t at, ibt_provenance_t prov) return false; } + +std::string getStepName(void) const override +{ + return std::string("fix_calls"); +} + +}; // end class FixCalls_t + +shared_ptr<Transform_SDK::TransformStep_t> curInvocation; + +bool possible_target(virtual_offset_t p, virtual_offset_t from_addr, ibt_provenance_t prov) +{ + assert(curInvocation); + return (dynamic_cast<FixCalls_t*>(curInvocation.get()))->possible_target(p,from_addr,prov); +} + +void range(virtual_offset_t start, virtual_offset_t end) +{ + assert(curInvocation); + return (dynamic_cast<FixCalls_t*>(curInvocation.get()))->range(start,end); +} + +extern "C" +shared_ptr<Transform_SDK::TransformStep_t> GetTransformStep(void) +{ + curInvocation.reset(new FixCalls_t()); + return curInvocation; + + //return shared_ptr<Transform_SDK::TransformStep_t>(new FixCalls_t()); +} + + diff --git a/libMEDSannotation/SConscript b/libMEDSannotation/SConscript index 31b5b9cea327e49ddcde3ed72f134038ddbfbb46..8ca4d129ed12299042ca4858702c025c6e8eb7f1 100644 --- a/libMEDSannotation/SConscript +++ b/libMEDSannotation/SConscript @@ -28,7 +28,7 @@ cpppath=''' myenv=myenv.Clone(CPPPATH=Split(cpppath)) myenv.Append(CXXFLAGS=" -std=c++11 -Wall -Werror ") -lib=myenv.Library(lib, Split(files)) +lib=myenv.SharedLibrary(lib, Split(files)) install=myenv.Install("$SECURITY_TRANSFORMS_HOME/lib/", lib) Default(install) diff --git a/libStructDiv/src/SConscript b/libStructDiv/src/SConscript index 8971803762bfb7ec6f46aad3f083ceea0492c129..11fce4d5b8a15fa99b3a6044b971a5bbbd3616c2 100644 --- a/libStructDiv/src/SConscript +++ b/libStructDiv/src/SConscript @@ -17,7 +17,7 @@ LIBPATH="$SECURITY_TRANSFORMS_HOME/lib" myenv=myenv.Clone(CPPPATH=Split(cpppath)) myenv.Append(CXXFLAGS="-fPIC") -lib=myenv.Library("StructDiv", Split(files)) +lib=myenv.SharedLibrary("StructDiv", Split(files)) install=myenv.Install("$SECURITY_TRANSFORMS_HOME/lib/", lib) Default(install) diff --git a/libcapstone b/libcapstone index 9408c0de4f3a4b2a3cc1cac3f22219301c561400..a6b87b7bc31499b9ec538701b438f7a60e9dda38 160000 --- a/libcapstone +++ b/libcapstone @@ -1 +1 @@ -Subproject commit 9408c0de4f3a4b2a3cc1cac3f22219301c561400 +Subproject commit a6b87b7bc31499b9ec538701b438f7a60e9dda38 diff --git a/libtransform/src/SConscript b/libtransform/src/SConscript index 81e16c5735b7682d28bbacab8ff27914dd7d59d4..d075920eb834ac1559db6c85db1387a88fd4111e 100644 --- a/libtransform/src/SConscript +++ b/libtransform/src/SConscript @@ -18,12 +18,12 @@ cpppath=''' LIBPATH="$SECURITY_TRANSFORMS_HOME/lib" -LIBS=Split("IRDB-core IRDB-cfg IRDB-util pqxx BeaEngine_s_d ") +LIBS=Split("IRDB-core IRDB-cfg IRDB-util MEDSannotation") myenv=myenv.Clone(CPPPATH=Split(cpppath)) myenv.Append(CXXFLAGS = " -std=c++11 ") -lib=myenv.Library("transform", Split(files), LIBPATH=LIBPATH, LIBS=LIBS) +lib=myenv.SharedLibrary("transform", Split(files), LIBPATH=LIBPATH, LIBS=LIBS) install=myenv.Install("$SECURITY_TRANSFORMS_HOME/lib/", lib) Default(install) Return('install') diff --git a/manifest.txt b/manifest.txt index f606f95ff5949f8681517c870ee31daa2c4d9a93..e0a42281f4ea89962b20b74251e882ecfc31397d 100644 --- a/manifest.txt +++ b/manifest.txt @@ -1,5 +1,6 @@ directory plugins_install ps +directory lib ps directory bin ps directory lib ps diff --git a/pebliss/trunk/pe_lib/SConscript b/pebliss/trunk/pe_lib/SConscript index 3a8a79d46d771df76ac6a01f146b44962f38fb60..6e58fd457d33b83d2ba007cb49085106469b062f 100644 --- a/pebliss/trunk/pe_lib/SConscript +++ b/pebliss/trunk/pe_lib/SConscript @@ -52,7 +52,7 @@ cpppath=''' myenv=myenv.Clone(CPPPATH=Split(cpppath)) myenv.Append(CCFLAGS=" -w ") -lib=myenv.Library(libname, Split(files)) +lib=myenv.SharedLibrary(libname, Split(files)) install=myenv.Install("$SECURITY_TRANSFORMS_HOME/lib/", lib) Default(install) diff --git a/plugins_install/zipr.sh b/plugins_install/zipr.sh new file mode 100755 index 0000000000000000000000000000000000000000..e37c8e25d4b4dc694940221a3cd62ec0f1aa845f --- /dev/null +++ b/plugins_install/zipr.sh @@ -0,0 +1,7 @@ +#/bin/bash + +cloneid=$1 +shift +other_opts="$@" + +(set -x ; LD_LIBRARY_PATH=$LD_LIBRARY_PATH:$ZIPR_INSTALL/lib $ZIPR_INSTALL/bin/zipr.exe --variant $cloneid --zipr:objcopy $PS_OBJCOPY $other_opts) diff --git a/tools/SConscript b/tools/SConscript index b5402aedebd806f99fb62cc0da5c346da290452a..a83cce77deeaf23e7148b274442ab2013ad11a4a 100644 --- a/tools/SConscript +++ b/tools/SConscript @@ -15,6 +15,7 @@ dirs=''' dump_insns hook_start rida + thanos ''' nobuild_dirs=''' diff --git a/tools/hook_start/SConscript b/tools/hook_start/SConscript index 58d5384d925f97cb476016d6bb682705c6042118..2018f047564baffef1abf48f20a47f131ac2068d 100644 --- a/tools/hook_start/SConscript +++ b/tools/hook_start/SConscript @@ -18,7 +18,7 @@ cpppath=''' # $SECURITY_TRANSFORMS_HOME/libtransform/include CPPFLAGS="--std=c++11" LIBPATH="$SECURITY_TRANSFORMS_HOME/lib" -LIBS=Split( env.subst('$BASE_IRDB_LIBS')+ " IRDB-core IRDB-cfg IRDB-util pqxx BeaEngine_s_d transform MEDSannotation ") +LIBS=Split( env.subst('$BASE_IRDB_LIBS')+ " IRDB-cfg IRDB-util transform MEDSannotation ") myenv=myenv.Clone(CPPPATH=Split(cpppath)) myenv.Append(CPPFLAGS=CPPFLAGS) diff --git a/tools/selective_cfi/zest_cfi_runtime/SConscript64 b/tools/selective_cfi/zest_cfi_runtime/SConscript64 index 9b14c4ec798020a8934afbc7e71eb0b46c8125b6..650a83866f068718da6cea07883de22bed4b5cff 100644 --- a/tools/selective_cfi/zest_cfi_runtime/SConscript64 +++ b/tools/selective_cfi/zest_cfi_runtime/SConscript64 @@ -1,8 +1,7 @@ import os -Import('env') -myenv=env.Clone() +myenv=Environment() myenv.Replace(SECURITY_TRANSFORMS_HOME=os.environ['SECURITY_TRANSFORMS_HOME']) myenv.Replace(ZEST_RUNTIME=os.environ['ZEST_RUNTIME']) diff --git a/tools/thanos/SConscript b/tools/thanos/SConscript new file mode 100644 index 0000000000000000000000000000000000000000..32e69a3b73e7aa730dec4997eb6fa438c3e8603c --- /dev/null +++ b/tools/thanos/SConscript @@ -0,0 +1,35 @@ +import os + + + +Import('env') +myenv=env.Clone() +myenv.Replace(SECURITY_TRANSFORMS_HOME=os.environ['SECURITY_TRANSFORMS_HOME']) + +cpppath=''' + $SECURITY_TRANSFORMS_HOME/include + $SECURITY_TRANSFORMS_HOME/libIRDB/include + $SECURITY_TRANSFORMS_HOME/libMEDSannotation/include + $SECURITY_TRANSFORMS_HOME/libtransform/include + $SECURITY_TRANSFORMS_HOME/libEXEIO/include + ''' + + +files=Glob( Dir('.').srcnode().abspath+"/*.cpp") + +myenv.Append(CXXFLAGS = " -std=c++11 -Wall ") + +pgm="thanos.exe" + +LIBPATH="$SECURITY_TRANSFORMS_HOME/lib" +LIBS=Split( env.subst('$BASE_IRDB_LIBS')+ " IRDB-cfg IRDB-util IRDB-core transform MEDSannotation pqxx pq dl") +myenv=myenv.Clone(CPPPATH=Split(cpppath)) +pgm=myenv.Program(pgm, files, LIBPATH=LIBPATH, LIBS=LIBS) +install=myenv.Install("$SECURITY_TRANSFORMS_HOME/plugins_install", pgm) +Default(install) + + + + +ret=[]+install +Return('ret') diff --git a/tools/thanos/SConstruct b/tools/thanos/SConstruct new file mode 100644 index 0000000000000000000000000000000000000000..44d3bd9e81dfb9812806072103183b71c0810436 --- /dev/null +++ b/tools/thanos/SConstruct @@ -0,0 +1,7 @@ + + + +env=Environment() +Export('env') +ret=SConscript("SConscript") +Return('ret') diff --git a/tools/thanos/thanos.cpp b/tools/thanos/thanos.cpp new file mode 100644 index 0000000000000000000000000000000000000000..fe1eeca1c37eb62a45229553df3d45b78effe0ea --- /dev/null +++ b/tools/thanos/thanos.cpp @@ -0,0 +1,397 @@ +#include <libIRDB-core.hpp> +#include <dlfcn.h> +#include <vector> +#include <memory> +#include <sstream> +#include <iostream> +#include <iomanip> +#include <unistd.h> +#include <fcntl.h> +#include <fstream> +#include <ctime> + + +using namespace std; +using namespace libIRDB; +using namespace Transform_SDK; + +#define ALLOF(a) begin(a),end(a) + +// global to be used like cout/cerr for writing to the logs +ofstream thanos_log; +ostream *real_cout; +ostream *real_cerr; +string thanos_path; +bool redirect_opt=true; + +class ThanosPlugin_t +{ + public: + static unique_ptr<ThanosPlugin_t> pluginFactory(const string plugin_details); + static int saveChanges(); + bool isOptional() + { + return step_optional; + } + string getStepName() + { + return step_name; + } + int runPlugin(); + + private: + // methods + ThanosPlugin_t(const string p_step_name, + const bool p_step_optional, + const vector<string> p_step_args + ) + : + step_name(p_step_name), + step_optional(p_step_optional), + step_args(p_step_args) + { + } + int executeStep(TransformStep_t& the_step, const bool are_debugging); + int commitAll(); + + // data + const string step_name; + const bool step_optional; + const vector<string> step_args; + static const unique_ptr<IRDBObjects_t> shared_objects; +}; +// initialize private static data member +const unique_ptr<IRDBObjects_t> ThanosPlugin_t::shared_objects(new IRDBObjects_t()); + +using PluginList_t = vector<unique_ptr<ThanosPlugin_t>>; +PluginList_t getPlugins(const int argc, char const *const argv[]); + +int main(int argc, char* argv[]) +{ + thanos_path=argv[0]; + ostream my_real_cerr(cerr.rdbuf()); + ostream my_real_cout(cout.rdbuf()); + real_cerr=&my_real_cerr; + real_cout=&my_real_cout; + + thanos_log.open("logs/thanos.log", ofstream::out); + + if(!thanos_log) + { + cerr<<"Cannot open logs/thanos.log"<<endl; + exit(1); + } + // get plugins + auto argv_iter=1; + while (true) + { + if(argv_iter >= argc) + { + break; + } + if(string(argv[argv_iter])=="--no-redirect") + { + redirect_opt=false; + argv_iter++; + } + else + break; + } + const auto plugin_argv_iter=argv_iter; + + auto thanos_plugins = getPlugins(argc-plugin_argv_iter, argv+plugin_argv_iter); + if(thanos_plugins.size() == 0) + { + // for now, usage is pretty strict to enable simple + // parsing, because this program is only used by an + // automated script + thanos_log << "Syntax error in arguments." << endl; + thanos_log << "USAGE: <thanos opts> (\"<step name> [-optional] [--step-args [ARGS]]\")+" << endl; + return 1; + } + + for(unsigned int i = 0; i < thanos_plugins.size(); ++i) + { + ThanosPlugin_t* plugin = thanos_plugins[i].get(); + + const int result = plugin->runPlugin(); + // if that returns failure AND the step is not optional + if(result != 0 && !plugin->isOptional()) + { + thanos_log << "A critical step failed: " << plugin->getStepName() << endl; + thanos_log << "If DEBUG_STEPS is not on, this failure could " + << "be due to an earlier critical step." << endl; + return 1; // critical step failed, abort + } + } + // write back final changes + const int result = ThanosPlugin_t::saveChanges(); + if(result != 0) + { + thanos_log << "A critical step failed: " << (thanos_plugins.back())->getStepName() + << endl; + thanos_log << "If DEBUG_STEPS is not on, this failure could " + << "be due to an earlier critical step." << endl; + return 1; // critical step failed, abort + } + else + { + return 0; // success :) + } +} + + +PluginList_t getPlugins(const int argc, char const *const argv[]) +{ + PluginList_t plugins; + + for(auto i = 0; i < argc; ++i) + { + auto the_plugin = ThanosPlugin_t::pluginFactory(string(argv[i])); + if(the_plugin == nullptr) + return PluginList_t(); + plugins.push_back(move(the_plugin)); + } + return plugins; +} + + +// assumes that tokens are always space-separated +// (cannot be delineated by quotes, for example) +const vector<string> getTokens(const string arg_string) +{ + vector<string> tokens; + istringstream arg_stream(arg_string); + string token; + while (getline(arg_stream, token, ' ')) + { + tokens.push_back(token); + } + return tokens; +} + + +unique_ptr<ThanosPlugin_t> ThanosPlugin_t::pluginFactory(const string plugin_details) +{ + auto tokens = getTokens(plugin_details); + if(tokens.size() < 1) + return unique_ptr<ThanosPlugin_t>(nullptr); + + const auto step_name = tokens[0]; + auto step_optional = false; + vector<string> step_args; + for(unsigned int i = 1; i < tokens.size(); ++i) + { + if(tokens[i] == "--step-args") + { + if(tokens.begin()+i+1 < tokens.end()) + step_args.assign(tokens.begin()+i+1, tokens.end()); + break; + } + else if(tokens[i] == "-optional") + { + step_optional = true; + } + else + { + return unique_ptr<ThanosPlugin_t>(nullptr); + } + } + return unique_ptr<ThanosPlugin_t>(new ThanosPlugin_t(step_name, step_optional, step_args)); +} + + +int ThanosPlugin_t::runPlugin() +{ + static const char *const base_path = getenv("SECURITY_TRANSFORMS_HOME"); + if(base_path == NULL) + { + thanos_log << "Environment variables not set." << endl; + return -1; + } + static const auto plugin_path (string(base_path).append("/plugins_install/")); + + void *const dlhdl = dlopen((plugin_path+"lib"+step_name+".so").c_str(), RTLD_NOW); + if(dlhdl == NULL) + { + const auto err=dlerror(); + thanos_log<<"Cannot open "<<step_name<<": "<<err<<endl; + return -1; + } + + const void *const sym = dlsym(dlhdl, "GetTransformStep"); + if(sym == NULL) + { + const auto err=dlerror(); + thanos_log<<"Cannot find GetTransformStep in "<<step_name<<": "<<err<<endl; + return -1; + } + + using GetTransformPtr_t = shared_ptr<TransformStep_t> (*)(void); // function pointer, takes void, returns TransformStep_t shared ptr + GetTransformPtr_t func=(GetTransformPtr_t)sym; + shared_ptr<TransformStep_t> the_step = (*func)(); + assert(the_step != NULL); + + static const char *const are_debugging = getenv("DEBUG_STEPS"); + + + auto saved_cerrbuf = cerr.rdbuf(); + auto saved_coutbuf = cout.rdbuf(); + ofstream logfile; + + auto are_logging = !((bool) are_debugging); + if(are_logging) + { + // setup logging + auto logfile_path = "./logs/"+step_name+".log"; + logfile.open(logfile_path,ofstream::out); + if(!logfile) + { + *real_cout<<"Cannot open log file "<<logfile_path<<endl; + exit(1); + } + if(redirect_opt) + { + cout.rdbuf(logfile.rdbuf()); + cerr.rdbuf(logfile.rdbuf()); + } + } + + + const auto start_time = clock(); + const auto start_t=time(nullptr); + const auto start_time_str = ctime(&start_t); + + const auto step_result = executeStep(*(the_step.get()), (bool) are_debugging); + + const auto end_time = clock(); + const auto end_t=time(nullptr); + const auto end_time_str = ctime(&end_t); + const auto elapsed_time = (double)(end_time-start_time)/ CLOCKS_PER_SEC; + + cout<< "#ATTRIBUTE start_time=" << start_time_str ; // endl in time_str + cout<< "#ATTRIBUTE end_time=" << end_time_str ; // endl in time_str + cout<< "#ATTRIBUTE elapsed_time=" << elapsed_time<<endl; + cout<< "#ATTRIBUTE step_name=" << step_name<<endl; + cout<< "#ATTRIBUTE step_command= " << thanos_path << " " << step_name + << " --step-args "; copy(ALLOF(step_args), ostream_iterator<string>(cout, " ")); cout<<endl; + cout<< "#ATTRIBUTE step_exitcode="<<dec<<step_result<<endl; + + + cerr.rdbuf(saved_cerrbuf); + cout.rdbuf(saved_coutbuf); + + the_step.reset(); // explicitly get rid of the handle to the library so we can close it. + dlclose(dlhdl); + + // return status of execute method + return step_result; +} + + +int ThanosPlugin_t::executeStep(TransformStep_t& the_step, const bool are_debugging) +{ + + *real_cout<<"Performing step "<<the_step.getStepName()<< " [dependencies=unknown] ..."; // no endl intentionally. + flush(*real_cout); + + + const int parse_retval = the_step.parseArgs(step_args); + if(parse_retval != 0) + { + return parse_retval; + } + + pqxxDB_t* pqxx_interface = shared_objects->getDBInterface(); + if(step_optional) + { + const int error = shared_objects->writeBackAll(); + if(error) + { + return 1; // the failure must be from a critical step, abort + } + else + { + // commit changes (in case this step fails) and reset interface + pqxx_interface->Commit(); + pqxx_interface = shared_objects->resetDBInterface(); + } + } + + const int step_error = the_step.executeStep(shared_objects.get()); + + if(step_error) + { + if(step_optional) + { + + *real_cout<<"Done. Command failed! ***************************************"<<endl; + // delete all shared items without writing + // next step will have to get the last "good" version from DB + shared_objects->deleteAll(); + } + else + { + *real_cout<<"Done. Command failed! ***************************************"<<endl; + *real_cout<<"ERROR: The "<<the_step.getStepName()<<" step is necessary, but failed. Exiting early."<<endl; + return 1; // critical step failed, abort + } + } + else + { + *real_cout<<"Done. Successful."<<endl; + } + + if(step_optional) + { + // write changes to DB to see if it succeeds + const int error = shared_objects->writeBackAll(); + if(error) + { + // abort changes by resetting DB interface + pqxx_interface = shared_objects->resetDBInterface(); + } + else if(are_debugging) + { + // commit changes (in case next step fails) and reset interface + pqxx_interface->Commit(); + pqxx_interface = shared_objects->resetDBInterface(); + } + } + else if(are_debugging) + { + // write changes to DB in case next step fails + const int error = shared_objects->writeBackAll(); + if(error) + { + return 1; // critical step failed, abort + } + else + { + // commit changes (in case next step fails) and reset interface + pqxx_interface->Commit(); + pqxx_interface = shared_objects->resetDBInterface(); + } + } + + return step_error; +} + + +int ThanosPlugin_t::saveChanges() +{ + pqxxDB_t* pqxx_interface = shared_objects->getDBInterface(); + const int error = shared_objects->writeBackAll(); + if(error) + { + return 1; // critical step failed, abort + } + else + { + // commit changes and reset interface + pqxx_interface->Commit(); + pqxx_interface = shared_objects->resetDBInterface(); + return 0; + } +} + diff --git a/xform/SConscript b/xform/SConscript index 26e7d6fd697a7e9ea443ebe409a6d4d69b83b60f..e1af37fa9cd25ac1e18962acd38edb281d8f18b1 100644 --- a/xform/SConscript +++ b/xform/SConscript @@ -30,7 +30,7 @@ CFLAGS="-fPIC -DUBUNTU" myenv=myenv.Clone(CC="$CXX", CPPPATH=Split(cpppath), CFLAGS=CFLAGS) myenv.Append(CXXFLAGS=" -std=c++11 ") -lib=myenv.Library(lib, Split(files)) +lib=myenv.SharedLibrary(lib, Split(files), LIBS=Split("IRDB-core EXEIO"), LIBPATH=Split("$SECURITY_TRANSFORMS_HOME/lib")) install=myenv.Install("$SECURITY_TRANSFORMS_HOME/lib/", lib)