diff --git a/libIRDB/include/core/address.hpp b/libIRDB/include/core/address.hpp index 7495975ef7e6340ef08316f96ac1946710f24318..b1e0da67da7ddfb6a24184712078c02581438f0e 100644 --- a/libIRDB/include/core/address.hpp +++ b/libIRDB/include/core/address.hpp @@ -48,7 +48,7 @@ class AddressID_t : public BaseObj_t virtual_offset_t GetVirtualOffset() { return virtual_offset; } void SetVirtualOffset(virtual_offset_t voff) { virtual_offset=voff; } - std::string WriteToDB(File_t *vid, db_id_t newid, bool p_withHeader); + std::vector<std::string> WriteToDB(File_t *vid, db_id_t newid, bool p_withHeader); inline bool operator<(const AddressID_t& cmp) const { return fileID < cmp.fileID || (fileID == cmp.fileID && virtual_offset < cmp.virtual_offset);} diff --git a/libIRDB/include/core/eh.hpp b/libIRDB/include/core/eh.hpp index 58b9999df359ffe8c0bf479e68a53e32aaa67b07..afdaa38917cbb841608505e31868b3b0d9b136ce 100644 --- a/libIRDB/include/core/eh.hpp +++ b/libIRDB/include/core/eh.hpp @@ -65,7 +65,7 @@ class EhProgram_t : public BaseObj_t int64_t GetReturnRegNumber() const { return return_register; } void SetReturnRegNumber(const int64_t rr) { return_register=rr; } - std::string WriteToDB(File_t* fid); // writes to DB, ID is not -1. + std::vector<std::string> WriteToDB(File_t* fid); // writes to DB, ID is not -1. friend bool operator<(const EhProgram_t&a, const EhProgram_t&b); diff --git a/libIRDB/include/core/fileir.hpp b/libIRDB/include/core/fileir.hpp index bc1e1587d2bbaf5d1a3eeab25a1022ce85d4a2a8..88c8884d93d256636f3780b2e9aeeb5f755f4aa6 100644 --- a/libIRDB/include/core/fileir.hpp +++ b/libIRDB/include/core/fileir.hpp @@ -20,6 +20,8 @@ #include "type.hpp" + + typedef std::set<Function_t*> FunctionSet_t; typedef std::set<AddressID_t*> AddressSet_t; @@ -169,5 +171,11 @@ class FileIR_t : public BaseObj_t void CleanupICFS(); void GarbageCollectICFS(); void DedupICFS(); + + + std::clock_t ReadIRDB_start; + std::clock_t ReadIRDB_end; + + }; diff --git a/libIRDB/include/core/instruction.hpp b/libIRDB/include/core/instruction.hpp index c52db17af84943c9ee6bda1abe86dfc2dc3608ee..e9e771652bb50b11c5cb7b83a9465bd61ebe3861 100644 --- a/libIRDB/include/core/instruction.hpp +++ b/libIRDB/include/core/instruction.hpp @@ -64,7 +64,7 @@ class Instruction_t : public BaseObj_t void SetIndirectBranchTargetAddress(AddressID_t* myIndTarg) { indTarg=myIndTarg; } void WriteToDB() { assert(0); } - std::string WriteToDB(File_t *fid, db_id_t newid, bool p_withHeader); + std::vector<std::string> WriteToDB(File_t *fid, db_id_t newid); int Disassemble(DISASM &d) const; std::string getDisassembly() const; bool Assemble(std::string assembly); diff --git a/libIRDB/include/core/reloc.hpp b/libIRDB/include/core/reloc.hpp index d8eb491dd0dc7010b7896006947c03c142dbe5a2..514018178d9bfdb9fcdd9339233247ce767fd145 100644 --- a/libIRDB/include/core/reloc.hpp +++ b/libIRDB/include/core/reloc.hpp @@ -33,7 +33,7 @@ class Relocation_t : public BaseObj_t // Relocation_t(db_id_t reloc_id) : BaseObj_t(NULL), type(""), wrt_obj(NULL), addend(0) { assert(0);} // read from DB void WriteToDB() { assert(0); } // writes to DB ID is not -1. - std::string WriteToDB(File_t* fid, BaseObj_t* insn); // writes to DB, ID is not -1. + std::vector<std::string> WriteToDB(File_t* fid, BaseObj_t* insn); // writes to DB, ID is not -1. void SetOffset(int off) { offset=off;} int GetOffset() const { return offset; } diff --git a/libIRDB/include/libIRDB-core.hpp b/libIRDB/include/libIRDB-core.hpp index af385b596becbb241937ab17195a8984d31aafee..42ee966f8eafc1315e39de3b52c3f4b3532c6caf 100644 --- a/libIRDB/include/libIRDB-core.hpp +++ b/libIRDB/include/libIRDB-core.hpp @@ -29,6 +29,7 @@ #include <iostream> #include <pqxx/pqxx> #include <beaengine/BeaEngine.h> +#include <ctime> namespace libIRDB { diff --git a/libIRDB/src/core/address.cpp b/libIRDB/src/core/address.cpp index 6942dd358f24365876e148c91ffd96d6e0ce2c2e..ea1cfe71e464c4dc51314cc29bb7ead62d6798d8 100644 --- a/libIRDB/src/core/address.cpp +++ b/libIRDB/src/core/address.cpp @@ -26,13 +26,14 @@ using namespace libIRDB; using namespace std; -string AddressID_t::WriteToDB(File_t *fid, db_id_t newid, bool p_withHeader) +vector<string> AddressID_t::WriteToDB(File_t *fid, db_id_t newid, bool p_withHeader) { assert(fid); if(GetBaseID()==NOT_IN_DATABASE) SetBaseID(newid); +/* string q; if (p_withHeader) q=string("insert into ")+fid->address_table_name + @@ -42,13 +43,13 @@ string AddressID_t::WriteToDB(File_t *fid, db_id_t newid, bool p_withHeader) q = ","; q += - string("(") + - string("'") + to_string(GetBaseID()) + string("', ") + - string("'") + to_string(fileID) + string("', ") + - string("'") + to_string(virtual_offset) + string("', ") + - string("'") + to_string(GetDoipID()) + string("' ") + - string(")"); +*/ + return { + to_string(GetBaseID()), + to_string(fileID), + to_string(virtual_offset), + to_string(GetDoipID()) + }; - return q; } diff --git a/libIRDB/src/core/eh.cpp b/libIRDB/src/core/eh.cpp index 0b39d6da51f7a190de2929c5f64311857c9cc246..664ad880532303351b31181b9b1a4b853a40003c 100644 --- a/libIRDB/src/core/eh.cpp +++ b/libIRDB/src/core/eh.cpp @@ -66,7 +66,7 @@ void libIRDB::EhProgram_t::print() const cout<<dec<<"CAF: "<<code_alignment_factor<<" DAF: "<<data_alignment_factor<<" ptrsize="<< +ptrsize<<" cie_hash: "<<hex<<cie_hash<<" fde_hash: "<<fde_hash<<endl; } -std::string EhProgram_t::WriteToDB(File_t* fid) // writes to DB, ID is not -1. +vector<std::string> EhProgram_t::WriteToDB(File_t* fid) // writes to DB, ID is not -1. { auto str_to_encoded_string=[](const string& data) -> string { @@ -92,18 +92,22 @@ std::string EhProgram_t::WriteToDB(File_t* fid) // writes to DB, ID is not -1 string encoded_cie_program=vec_to_encoded_string(cie_program); string encoded_fde_program=vec_to_encoded_string(fde_program); +/* string q; q ="insert into " + fid->GetEhProgramTableName(); q+="(eh_pgm_id,caf,daf,return_register,ptrsize,cie_program,fde_program) "+ string(" VALUES (") + - string("'") + to_string(GetBaseID()) + string("', ") + - string("'") + to_string(+code_alignment_factor) + string("', ") + - string("'") + to_string(+data_alignment_factor) + string("', ") + - string("'") + to_string(+return_register) + string("', ") + - string("'") + to_string(+ptrsize) + string("', ") + - string("'") + encoded_cie_program + string("', ") + - string("'") + encoded_fde_program + string("') ; "); - return q; +*/ + + return { + to_string(GetBaseID()), + to_string(+code_alignment_factor), + to_string(+data_alignment_factor), + to_string(+return_register), + to_string(+ptrsize), + encoded_cie_program, + encoded_fde_program + }; } diff --git a/libIRDB/src/core/fileir.cpp b/libIRDB/src/core/fileir.cpp index 7479725fa159a42c23706f563c956e177f90388b..80b5042ea5ebf4d2193993af37f4bb2c145ee74a 100644 --- a/libIRDB/src/core/fileir.cpp +++ b/libIRDB/src/core/fileir.cpp @@ -27,6 +27,9 @@ #include <elf.h> #include <stdlib.h> #include <sys/wait.h> +#include <iomanip> + + using namespace libIRDB; using namespace std; @@ -79,6 +82,7 @@ static virtual_offset_t strtovo(std::string s) // Create a Variant from the database FileIR_t::FileIR_t(const VariantID_t &newprogid, File_t* fid) : BaseObj_t(NULL) + { orig_variant_ir_p=NULL; progid=newprogid; @@ -130,6 +134,9 @@ FileIR_t::~FileIR_t() // DB operations void FileIR_t::ReadFromDB() { + ReadIRDB_start = clock(); + + auto entry_points=map<Function_t*,db_id_t>(); auto unresolvedICFS=std::map<Instruction_t*, db_id_t>(); auto unresolvedEhCallSites=std::map<EhCallSite_t*,db_id_t>(); @@ -157,6 +164,10 @@ void FileIR_t::ReadFromDB() UpdateEntryPoints(insnMap,entry_points); UpdateUnresolvedEhCallSites(insnMap,unresolvedEhCallSites); + + ReadIRDB_end = clock(); + + } @@ -524,6 +535,19 @@ std::map<db_id_t,AddressID_t*> FileIR_t::ReadAddrsFromDB } +static string encoded_to_raw_string(string encoded) +{ + int len = encoded.length(); + std::string raw; + for(int i=0; i< len; i+=2) + { + string byte = encoded.substr(i,2); + char chr = (char) (int)strtol(byte.c_str(), nullptr, 16); + raw.push_back(chr); + } + return raw; +} + std::map<db_id_t,Instruction_t*> FileIR_t::ReadInsnsFromDB ( const std::map<db_id_t,Function_t*> &funcMap, @@ -567,7 +591,8 @@ std::map<db_id_t,Instruction_t*> FileIR_t::ReadInsnsFromDB db_id_t icfs_id=atoi(dbintr->GetResultColumn("icfs_id").c_str()); db_id_t eh_pgm_id=atoi(dbintr->GetResultColumn("ehpgm_id").c_str()); db_id_t eh_cs_id=atoi(dbintr->GetResultColumn("ehcss_id").c_str()); - std::string data=(dbintr->GetResultColumn("data")); + std::string encoded_data=(dbintr->GetResultColumn("data")); + std::string data=encoded_to_raw_string(encoded_data); std::string callback=(dbintr->GetResultColumn("callback")); std::string comment=(dbintr->GetResultColumn("comment")); db_id_t indirect_branch_target_address_id = atoi(dbintr->GetResultColumn("ind_target_address_id").c_str()); @@ -670,6 +695,11 @@ void FileIR_t::ReadRelocsFromDB void FileIR_t::WriteToDB() { + const auto WriteIRDB_start = clock(); + + const auto pqIntr=dynamic_cast<pqxxDB_t*>(dbintr); + assert(pqIntr); + //Resolve (assemble) any instructions in the registry. AssembleRegistry(); @@ -694,7 +724,7 @@ void FileIR_t::WriteToDB() /* and now that everything has an ID, let's write to the DB */ - // write out the types +// write out the types string q=string(""); for(std::set<Type_t*>::const_iterator t=types.begin(); t!=types.end(); ++t) { @@ -707,9 +737,9 @@ void FileIR_t::WriteToDB() } dbintr->IssueQuery(q); - bool withHeader; - withHeader = true; +// write out functions + auto withHeader=true; q=string(""); for(std::set<Function_t*>::const_iterator f=funcs.begin(); f!=funcs.end(); ++f) { @@ -722,33 +752,25 @@ void FileIR_t::WriteToDB() } dbintr->IssueQuery(q); - withHeader = true; - q=string(""); - for(std::set<AddressID_t*>::const_iterator a=addrs.begin(); a!=addrs.end(); ++a) + +// write out addresses + pqxx::tablewriter W_addrs(pqIntr->GetTransaction(),fileptr->address_table_name); + for(const auto &a : addrs) { - q+=(*a)->WriteToDB(fileptr,j,withHeader); - withHeader = false; - if(q.size()>1024*1024) - { - q+=";"; - dbintr->IssueQuery(q); - q=string(""); - withHeader = true; - } + W_addrs << a->WriteToDB(fileptr,j,withHeader); } - dbintr->IssueQuery(q); + W_addrs.complete(); - withHeader = true; - q=string(""); + +// write out instructions + + pqxx::tablewriter W(pqIntr->GetTransaction(),fileptr->instruction_table_name); for(std::set<Instruction_t*>::const_iterator i=insns.begin(); i!=insns.end(); ++i) { Instruction_t const * const insnp=*i; DISASM disasm; insnp->Disassemble(disasm); - // we have a few new requirements for instructions that doesn't correspond to original program insns. -// cerr << "handling instruction:" << ((Instruction_t*)insnp)->getDisassembly() << " comment: " << ((Instruction_t*)insnp)->GetComment() << endl; - if(insnp->GetOriginalAddressID() == NOT_IN_DATABASE) { @@ -779,28 +801,16 @@ void FileIR_t::WriteToDB() } } - q+=(*i)->WriteToDB(fileptr,j,withHeader); - withHeader = false; - if(q.size()>1024*1024) - { - q+=";"; - dbintr->IssueQuery(q); - q=string(""); - withHeader = true; - } - string r=""; - std::set<Relocation_t*> irelocs = (*i)->GetRelocations(); - for(set<Relocation_t*>::iterator it=irelocs.begin(); it!=irelocs.end(); ++it) - { - Relocation_t* reloc=*it; - r+=reloc->WriteToDB(fileptr,*i); - } - if(r!="") - dbintr->IssueQuery(r); + const auto &insn_values=(*i)->WriteToDB(fileptr,j); + W << insn_values; + } - dbintr->IssueQuery(q); + W.complete(); + + +// icfs for (ICFSSet_t::iterator it = GetAllICFS().begin(); it != GetAllICFS().end(); ++it) { ICFS_t* icfs = *it; @@ -808,49 +818,75 @@ void FileIR_t::WriteToDB() string q = icfs->WriteToDB(fileptr); dbintr->IssueQuery(q); } + +// scoops for(DataScoopSet_t::const_iterator it=scoops.begin(); it!=scoops.end(); ++it) { DataScoop_t* scoop = *it; assert(scoop); string q = scoop->WriteToDB(fileptr,j); dbintr->IssueQuery(q); - - q=""; - const std::set<Relocation_t*> &the_relocs = scoop->GetRelocations(); - for(set<Relocation_t*>::const_iterator rit=the_relocs.begin(); rit!=the_relocs.end(); ++rit) - { - Relocation_t* reloc=*rit; - q+=reloc->WriteToDB(fileptr,scoop); - } - if(q!="") - dbintr->IssueQuery(q); - } + +// ehpgms + pqxx::tablewriter W_eh(pqIntr->GetTransaction(),fileptr->ehpgm_table_name); for(const auto& i : eh_pgms) + { + W_eh << i->WriteToDB(fileptr); + } + W_eh.complete(); + +// eh css + for(const auto& i : eh_css) { string q = i->WriteToDB(fileptr); dbintr->IssueQuery(q); + } - string r=""; + +// all relocs + pqxx::tablewriter W_reloc(pqIntr->GetTransaction(),fileptr->relocs_table_name); + +// eh css relocs + for(const auto& i : eh_css) + { for(auto& reloc : i->GetRelocations()) - { - r+=reloc->WriteToDB(fileptr,i); - } - if(r!="") - dbintr->IssueQuery(r); + W_reloc << reloc->WriteToDB(fileptr,i); } - for(const auto& i : eh_css) + +// eh pgms relocs + for(const auto& i : eh_pgms) { - string q = i->WriteToDB(fileptr); - dbintr->IssueQuery(q); string r=""; for(auto& reloc : i->GetRelocations()) - { - r+=reloc->WriteToDB(fileptr,i); - } - if(r!="") - dbintr->IssueQuery(r); + W_reloc << reloc->WriteToDB(fileptr,i); } +// scoops relocs + for(const auto& i : scoops) + { + for(auto& reloc : i->GetRelocations()) + W_reloc << reloc->WriteToDB(fileptr,i); + } +// write out instruction's relocs + for(const auto& i : insns) + { + for(auto& reloc : i->GetRelocations()) + W_reloc << reloc->WriteToDB(fileptr,i); + } + W_reloc.complete(); + + const auto WriteIRDB_end = clock(); + const auto read_time = (double)(ReadIRDB_end-ReadIRDB_start)/ CLOCKS_PER_SEC; + const auto write_time = (double)(WriteIRDB_end-WriteIRDB_start)/ CLOCKS_PER_SEC; + const auto wall_time = (double)(WriteIRDB_end-ReadIRDB_start)/ CLOCKS_PER_SEC; + const auto transform_time=wall_time - read_time - write_time; + + std::cout << std::dec; + std::cout << std::fixed << std::setprecision(2) << "#ATTRIBUTE ReadIRDB_WallClock=" << read_time <<endl; + std::cout << std::fixed << std::setprecision(2) << "#ATTRIBUTE WriteIRDB_WallClock=" << write_time << endl; + std::cout << std::fixed << std::setprecision(2) << "#ATTRIBUTE TotalIRDB_WallClock=" << wall_time << endl; + std::cout << std::fixed << std::setprecision(2) << "#ATTRIBUTE TransformIRDB_WallClock=" << transform_time << endl; + } diff --git a/libIRDB/src/core/instruction.cpp b/libIRDB/src/core/instruction.cpp index 106ffad5fb57cb87ac7318fef69255921b2c449c..1fab1950ef81e9eece8e12748affdb0f4a98f253 100644 --- a/libIRDB/src/core/instruction.cpp +++ b/libIRDB/src/core/instruction.cpp @@ -160,7 +160,7 @@ bool Instruction_t::Assemble(string assembly) } -string Instruction_t::WriteToDB(File_t *fid, db_id_t newid, bool p_withHeader) +vector<string> Instruction_t::WriteToDB(File_t *fid, db_id_t newid) { assert(fid); assert(my_address); @@ -189,36 +189,27 @@ string Instruction_t::WriteToDB(File_t *fid, db_id_t newid, bool p_withHeader) auto eh_css_id=NOT_IN_DATABASE; if(eh_cs) eh_css_id=eh_cs->GetBaseID(); - string q; - - if (p_withHeader) - q = string("insert into ")+fid->instruction_table_name + - string(" (instruction_id, address_id, parent_function_id, orig_address_id, fallthrough_address_id, " - "target_address_id, icfs_id, data, callback, comment, ind_target_address_id, ehpgm_id, ehcss_id, doip_id) VALUES "); - else - q = ","; - ostringstream hex_data; hex_data << setfill('0') << hex;; for (size_t i = 0; i < data.length(); ++i) hex_data << setw(2) << (int)(data[i]&0xff); - - q += string("('") + to_string(GetBaseID()) + string("', ") + - string("'") + to_string(my_address->GetBaseID()) + string("', ") + - string("'") + to_string(func_id) + string("', ") + - string("'") + to_string(orig_address_id) + string("', ") + - string("'") + to_string(ft_id) + string("', ") + - string("'") + to_string(targ_id) + string("', ") + - string("'") + to_string(icfs_id) + string("', ") + - string("decode('") + hex_data.str() + string("', 'hex'), ") + - string("'") + callback + string("', ") + - string("'") + comment + string("', ") + - string("'") + to_string(indirect_bt_id) + string("', ") + - string("'") + to_string(eh_pgm_id) + string("', ") + - string("'") + to_string(eh_css_id) + string("', ") + - string("'") + to_string(GetDoipID()) + string("') ") ; - - return q; + + + return { + to_string(GetBaseID()), + to_string(my_address->GetBaseID()), + to_string(func_id), + to_string(orig_address_id), + to_string(ft_id), + to_string(targ_id), + to_string(icfs_id), + to_string(eh_pgm_id), + to_string(eh_css_id), + hex_data.str(), + callback, + comment, + to_string(indirect_bt_id), + to_string(GetDoipID()) }; } @@ -269,7 +260,6 @@ bool Instruction_t::SetsStackPointer(DISASM* disasm) if(disasm->Instruction.ImplicitModifiedRegs==REGISTER_TYPE+GENERAL_REG+REG4) return true; - if(SetsStackPointer(&disasm->Argument1)) return true; if(SetsStackPointer(&disasm->Argument2)) return true; if(SetsStackPointer(&disasm->Argument3)) return true; diff --git a/libIRDB/src/core/reloc.cpp b/libIRDB/src/core/reloc.cpp index bc5587fc482434fbcf09bf02fd7f81bee5f6eaa9..4500d27b833d121c968de6c708b61050111c3b74 100644 --- a/libIRDB/src/core/reloc.cpp +++ b/libIRDB/src/core/reloc.cpp @@ -23,20 +23,23 @@ using namespace std; -std::string Relocation_t::WriteToDB(File_t* fid, BaseObj_t* myinsn) +vector<std::string> Relocation_t::WriteToDB(File_t* fid, BaseObj_t* myinsn) { - string q; db_id_t wrt_id=wrt_obj ? wrt_obj->GetBaseID() : BaseObj_t::NOT_IN_DATABASE; +/* + string q; q ="insert into " + fid->relocs_table_name; - q+="(reloc_id,reloc_offset,reloc_type,instruction_id,wrt_id,addend,doip_id) "+ + q+="(reloc_id,reloc_offset,reloc_type,instruction_id,addend,wrt_id,doip_id) "+ string(" VALUES (") + - string("'") + to_string(GetBaseID()) + string("', ") + - string("'") + to_string(offset) + string("', ") + - string("'") + (type) + string("', ") + - string("'") + to_string(myinsn->GetBaseID()) + string("', ") + - string("'") + to_string(wrt_id) + string("', ") + - string("'") + to_string(addend) + string("', ") + - string("'") + to_string(GetDoipID()) + string("') ; ") ; - return q; +*/ + return { + to_string(GetBaseID()), + to_string(offset), + (type), + to_string(myinsn->GetBaseID()), + to_string(addend), + to_string(wrt_id), + to_string(GetDoipID()) + }; } diff --git a/tools/meds2pdb/meds2pdb.cpp b/tools/meds2pdb/meds2pdb.cpp index e6a36c8d47cd9ea6a63fcee1daafe3dbadd21b0b..0239b33aa9da91a914f49d428189df2b3efa5a53 100644 --- a/tools/meds2pdb/meds2pdb.cpp +++ b/tools/meds2pdb/meds2pdb.cpp @@ -147,13 +147,15 @@ void insert_instructions(int fileID, const vector<wahoo::Instruction*> &instruct buf[0] = '\0'; if (data) { - sprintf(buf,"E'"); + sprintf(buf,"'"); for (int k = 0; k < instruction->getSize(); ++k) { unsigned char c = data[k]; - sprintf(&buf[strlen(buf)],"\\\\%03o", c); // octal encoding + char tmpbuf[10]; + sprintf(tmpbuf,"%02X", (unsigned int)c); // octal encoding + strcat(buf,tmpbuf); } - sprintf(&buf[strlen(buf)],"'::bytea"); + strcat(buf,"'"); } else {