Skip to content
Snippets Groups Projects

Compare revisions

Changes are shown as if the source revision was being merged into the target revision. Learn more about comparing revisions.

Source

Select target project
No results found

Target

Select target project
  • opensrc/zipr
  • whh8b/zipr
2 results
Show changes
Commits on Source (14)
Showing
with 3580 additions and 70 deletions
......@@ -26,3 +26,4 @@ irdb-libs/scons_build/
*.os
*.so
*.exe
.*.swp
......@@ -80,69 +80,46 @@ do-build-centos75:
#
# test_quick for EH handling
#
.basic-pgms-rida: &eh-tests
.basic_eh_tests: &basic_eh-tests
stage: test
script:
- ./cicd_testing/eh-tests.sh
eh-tests-ubuntu18:
<<: *eh-tests
basic_eh-tests-ubuntu18:
<<: *basic_eh-tests
tags:
- ubuntu18
eh-tests-ubuntu16:
<<: *eh-tests
basic_eh-tests-ubuntu16:
<<: *basic_eh-tests
tags:
- ubuntu16
eh-tests-centos75:
<<: *eh-tests
tags:
- centos75
#
# test_cmds.sh rida-p1
#
.basic-pgms-rida-p1: &basic-pgms-rida-p1
stage: test
script:
- ./cicd_testing/basic-pgms-rida-p1.sh
basic-pgms-rida-p1-ubuntu18:
<<: *basic-pgms-rida-p1
tags:
- ubuntu18
basic-pgms-rida-p1-ubuntu16:
<<: *basic-pgms-rida-p1
tags:
- ubuntu16
basic-pgms-rida-p1-centos75:
<<: *basic-pgms-rida-p1
basic_eh-tests-centos75:
<<: *basic_eh-tests
tags:
- centos75
#
# test_cmds.sh fixcalls
#
.basic-pgms-fixcalls: &basic-pgms-fixcalls
.builtin-xforms: &builtin-xforms
stage: test
script:
- ./cicd_testing/basic-pgms-fixcalls.sh
- ./cicd_testing/builtin-xforms.sh
basic-pgms-fixcalls-ubuntu18:
<<: *basic-pgms-fixcalls
builtin-xforms-ubuntu18:
<<: *builtin-xforms
tags:
- ubuntu18
basic-pgms-fixcalls-ubuntu16:
<<: *basic-pgms-fixcalls
builtin-xforms-ubuntu16:
<<: *builtin-xforms
tags:
- ubuntu16
basic-pgms-fixcalls-centos75:
<<: *basic-pgms-fixcalls
builtin-xforms-centos75:
<<: *builtin-xforms
tags:
- centos75
variables:
......@@ -179,27 +156,3 @@ elfdep-centos75:
variables:
OS: 'centos75'
#
# test_cmds.sh kill_deads
#
.basic-pgms-killdeads: &basic-pgms-killdeads
stage: test
script:
- ./cicd_testing/basic-pgms-killdeads.sh
basic-pgms-killdeads-ubuntu18:
<<: *basic-pgms-killdeads
tags:
- ubuntu18
basic-pgms-killdeads-ubuntu16:
<<: *basic-pgms-killdeads
tags:
- ubuntu16
basic-pgms-killdeads-centos75:
<<: *basic-pgms-killdeads
tags:
- centos75
variables:
OS: 'centos75'
......@@ -6,9 +6,6 @@
shallow = true
path = idaproCur_sdk
url = git@git.zephyr-software.com:allzp/idapro71_sdk
[submodule "irdb_transforms"]
path = irdb_transforms
url = git@git.zephyr-software.com:allnp/irdb_transforms.git
[submodule "SMPStaticAnalyzer"]
path = SMPStaticAnalyzer
url = git@git.zephyr-software.com:allnp/SMPStaticAnalyzer.git
......@@ -18,9 +15,6 @@
[submodule "zipr-sdk"]
path = zipr-sdk
url = git@git.zephyr-software.com:opensrc/zipr-sdk.git
[submodule "daffy"]
path = daffy
url = git@git.zephyr-software.com:allnp/daffy.git
[submodule "pedi"]
path = pedi
url = git@git.zephyr-software.com:allnp/pedi.git
......
......@@ -52,7 +52,10 @@ Export('env')
# list of zipr plugins and irdb xforms to build
transformDirs='''
irdb_transforms
builtin_xforms/add_lib
builtin_xforms/move_globals
builtin_xforms/p1transform
builtin_xforms/resolve_callbacks
zipr_push64_reloc_plugin
zipr
zipr_unpin_plugin
......
......@@ -14,6 +14,7 @@ env.Replace(SHLINKFLAGS="-fPIC -fmax-errors=2 -Wall -Werror -fmax-errors=2 -shar
env.Replace(SECURITY_TRANSFORMS_HOME=os.environ['SECURITY_TRANSFORMS_HOME'])
env.Replace(IRDB_SDK=os.environ['IRDB_SDK'])
env.Replace(SMPSA_HOME=os.environ['SMPSA_HOME'])
env.Replace(PEASOUP_HOME=os.environ['PEASOUP_HOME'])
env.Replace(debug=ARGUMENTS.get("debug",0))
env.Replace(PEDI_HOME=os.environ['PEDI_HOME'])
env.Replace(ZIPR_INSTALL=os.environ['ZIPR_INSTALL'])
......
Subproject commit a4b00a63158068efa2d9666d3254d1553bbf3be8
Subproject commit 8acd3ee30168df1c41e7f0eb565badb5bc17b86d
import os
Import('env')
# import and create a copy of the environment so we don't screw up anyone elses env.
myenv=env.Clone()
files=Glob( Dir('.').srcnode().abspath+"/*.cpp")
cpppath='''
$IRDB_SDK/include
'''
myenv.Append(CPPPATH=Split(cpppath))
pgm="add_lib.exe"
LIBPATH="$SECURITY_TRANSFORMS_HOME/lib"
LIBS=Split("stars "+ myenv.subst('$BASE_IRDB_LIBS')+ " irdb-elfdep irdb-transform")
pgm=myenv.Program(pgm, files, LIBPATH=LIBPATH, LIBS=LIBS)
install=myenv.Install("$SECURITY_TRANSFORMS_HOME/plugins_install/", pgm)
Default(install)
Return('install')
import os
env=Environment()
Export('env')
lib=SConscript("SConscript")
pedi = Command( target = "./testoutput",
source = "./SConscript",
action = "cd "+os.environ['SECURITY_TRANSFORMS_HOME']+" ; " + os.environ['PEDI_HOME']+"/pedi -m manifest.txt ; cd - " )
Depends(pedi,lib);
Default(pedi)
#include "al.hpp"
#include <irdb-elfdep>
using namespace IRDB_SDK;
using namespace std;
using namespace AddLib;
AddLib_t::AddLib_t(FileIR_t* firp, const StringSet_t &p_prepended, const StringSet_t &p_appended)
: Transform (firp), prepended(p_prepended), appended(p_appended)
{
}
int AddLib_t::execute()
{
auto ed=ElfDependencies_t::factory(getFileIR());
for(const auto &p : prepended)
ed->prependLibraryDepedencies(p);
for(const auto &a : appended)
ed->appendLibraryDepedencies(a);
return 0;
}
#include <string>
#include <set>
#include <irdb-core>
#include <irdb-transform>
namespace AddLib
{
using namespace IRDB_SDK;
using namespace std;
typedef set<string> StringSet_t;
class AddLib_t : Transform
{
public:
AddLib_t(FileIR_t* firp, const StringSet_t &prepended, const StringSet_t &appended);
int execute();
private:
StringSet_t prepended;
StringSet_t appended;
};
}
#include <stdlib.h>
#include <fstream>
#include <string>
#include "al.hpp"
#include <irdb-core>
#include <getopt.h>
using namespace std;
using namespace IRDB_SDK;
using namespace AddLib;
void usage(string programName)
{
cerr << "Usage: " << programName << " <variant id> <annotation file>" <<endl;
cout<<"--append lib Append named library" <<endl;
cout<<"-a lib" <<endl;
cout<<""<<endl;
cout<<"--prepend lib Prepend named library" <<endl;
cout<<"-p lib" <<endl;
cout<<""<<endl;
cout<<"--help,--usage,-?,-h Display this message"<<endl;
}
int main(int argc, char **argv)
{
string programName(argv[0]);
char *strtolError = NULL;
int transformExitCode = 0;
int variantID = -1;
auto pqxx_interface=pqxxDB_t::factory();
int exit_code=0;
/*
* Check that we've been called correctly:
* <program> <variant id> <annotation file>
*/
if(argc < 2)
{
usage(programName);
exit(1);
}
variantID = strtol(argv[1], &strtolError, 10);
if (*strtolError != '\0')
{
cerr << "Invalid variantID: " << argv[1] << endl;
exit(1);
}
// Parse some options for the transform
const static struct option long_options[] = {
{"prepend", required_argument, 0, 'p'},
{"append", required_argument, 0, 'a'},
{"help", no_argument, 0, 'h'},
{"usage", no_argument, 0, '?'},
{0,0,0,0}
};
auto short_opts="p:ah?";
auto prepended_libs=set<string>();
auto appended_libs=set<string>();
while(1) {
int index = 0;
int c = getopt_long(argc, argv,short_opts, long_options, &index);
if(c == -1)
break;
switch(c) {
case 0:
break;
case 'p':
prepended_libs.insert(string(optarg));
break;
case 'a':
appended_libs.insert(string(optarg));
break;
break;
case '?':
case 'h':
usage(argv[0]);
exit(1);
break;
default:
break;
}
}
/* setup the interface to the sql server */
BaseObj_t::setInterface(pqxx_interface.get());
auto pidp=VariantID_t::factory(variantID);
assert(pidp && pidp->isRegistered()==true);
for(set<File_t*>::iterator it=pidp->getFiles().begin();
it!=pidp->getFiles().end();
++it)
{
try
{
/* read the IR from the DB */
File_t* this_file = *it;
auto firp = FileIR_t::factory(pidp.get(), this_file);
cout<<"Transforming "<<this_file->getURL()<<endl;
assert(firp && pidp);
/*
* Create a transformation and then
* invoke its execution.
*/
AddLib_t al(firp.get(),prepended_libs,appended_libs);
transformExitCode = al.execute();
/*
* If everything about the transformation
* went okay, then we will write the updated
* set of instructions to the database.
*/
if (transformExitCode == 0)
{
if(getenv("MG_NOUPDATE")==NULL)
{
firp->writeToDB();
}
}
else
{
cerr << programName << ": transform failed. Check logs." << endl;
exit_code=2;
}
}
catch (DatabaseError_t pnide)
{
cerr << programName << ": Unexpected database error: " << pnide << endl;
exit(1);
}
catch (const std::exception &exc)
{
// catch anything thrown within try block that derives from std::exception
std::cerr << "Unexpected exception: " << exc.what();
exit(1);
}
catch (...)
{
cerr << programName << ": Unexpected error" << endl;
exit(1);
}
}
pqxx_interface->commit();
return exit_code;
}
#!/bin/bash
#benchmarks="473.astar"
# 447.dealII // broken build
benchmarks=" 400.perlbench 401.bzip2 403.gcc 410.bwaves 416.gamess 429.mcf 433.milc 434.zeusmp 435.gromacs 436.cactusADM 437.leslie3d 444.namd 445.gobmk 450.soplex 453.povray 454.calculix 456.hmmer 458.sjeng 459.GemsFDTD 462.libquantum 464.h264ref 465.tonto 470.lbm 471.omnetpp 473.astar 481.wrf 482.sphinx3 483.xalancbmk "
#benchmarks="bzip2"
#benchmarks="400.perlbench 403.gcc 445.gobmk 453.povray 458.sjeng 464.h264ref 464.tonto 471.omnetpp 481.wrf 482.sphinx3 483.xalancbmk"
number=1
setup()
{
if [ ! -d spec2006 ]; then
#svn co ^/spec2006/trunk spec2006
git clone --depth 1 http://git.zephyr-software.com/allzp/spec2006.git spec2006
fi
cd spec2006/
if [ ! -L bin ]; then
ln -s bin.power/ bin
fi
source shrc
bin/relocate
}
run_test()
{
local config_name=$1
local config=$2
all_configs_that_were_run="$all_configs_that_were_run $config_name"
cd $SPEC
if [ ! -d result.$config_name ]; then
echo "Re-doing runspec for test $config_name"
dropdb $PGDATABASE
createdb $PGDATABASE
$PEASOUP_HOME/tools/db/pdb_setup.sh
rm -Rf result/*
runspec --action scrub --config $config $benchmarks
runspec --action validate --config $config -n $number $benchmarks
cp benchspec/CPU2006/*/exe/* result
mv result result.$config_name
dropdb $PGDATABASE
createdb $PGDATABASE
$PEASOUP_HOME/tools/db/pdb_setup.sh
fi
}
get_size_result()
{
bench=$1
if [ -f $1 ]; then
size=$(stat --printf="%s" $bench)
echo -n $size
else
echo -n 0
fi
}
get_result()
{
bench=$1
config=$2
results=$(cat $SPEC/result.$config/CPU2006.002.log 2>/dev/null|grep Success|grep $bench|grep ratio=|sed 's/.*ratio=//'|sed 's/,.*//')
sum=0
count=0
for res in $results
do
sum=$(echo $sum + $res | bc)
count=$(echo $count + 1 | bc)
done
#echo sum=$sum
#echo count=$count
res=$(echo "scale=2; $sum / $count" | bc 2> /dev/null )
count=$(echo $res|wc -w)
if [ $count = 1 ]; then
echo -n $res
else
echo -n "N/A"
fi
}
# global
all_configs_that_were_run=""
get_raw_results()
{
echo "--------------------------------------------------------------"
echo "Performance results are:"
echo "--------------------------------------------------------------"
local configs="$all_configs_that_were_run"
echo benchmark $configs
for bench in $benchmarks
do
echo -n "$bench "
for config in $configs
do
get_result $bench $config
echo -n " "
done
echo
done
echo "--------------------------------------------------------------"
echo "Size results are:"
echo "--------------------------------------------------------------"
echo benchmark $configs
for bench in $SPEC/result.$config/*_base.amd64-m64-gcc42-nn
do
echo -n "$(basename $bench _base.amd64-m64-gcc42-nn) "
for config in $configs
do
file="$SPEC/result.$config/$(basename $bench)"
get_size_result $file
echo -n " "
done
echo
done
}
main()
{
start_dir=$(pwd)
setup
local zipr_flags="--backend zipr -o zipr:--add-sections -o zipr:true "
local hral_flags="-c move_globals=on -o move_globals:--elftables-only -c add_lib=on -o add_lib:--prepend -o add_lib:libheaprand.so"
if [[ -z $DIEHARD_HOME ]] ; then
echo "Cannot find DieHard for heap randomization test. Abborting now..."
exit 1
else
cp $DIEHARD_HOME/src/libdiehard-4k.so $PWD/libheaprand.so
export LD_LIBRARY_PATH="$PWD:$LD_LIBRARY_PATH"
fi
# zipr, etc.
PSOPTS="$zipr_flags $hral_flags " run_test hral $SPEC/config/ubuntu14.04lts-64bit-withps.cfg
PSOPTS="$zipr_flags" run_test zipr $SPEC/config/ubuntu14.04lts-64bit-withps.cfg
# no $PS -- aka, baseline.
run_test original $SPEC/config/ubuntu14.04lts-64bit.cfg
get_raw_results
}
main "$@"
import os
Import('env')
# import and create a copy of the environment so we don't screw up anyone elses env.
myenv=env.Clone()
cpppath='''
$SECURITY_TRANSFORMS_HOME/third_party/elfio-code
$PEASOUP_HOME/irdb-libs/libStructDiv/include
$PEASOUP_HOME/irdb-libs/libEXEIO/include
$IRDB_SDK/include
'''
files=Glob( Dir('.').srcnode().abspath+"/*.cpp")
pgm="move_globals.so"
LIBPATH="$SECURITY_TRANSFORMS_HOME/lib"
LIBS=Split("irdb-core irdb-cfg irdb-util irdb-transform irdb-deep StructDiv EXEIO ")
myenv.Append(CPPPATH=Split(cpppath))
pgm=myenv.SharedLibrary(pgm, files, LIBPATH=LIBPATH, LIBS=LIBS)
install=myenv.Install("$SECURITY_TRANSFORMS_HOME/plugins_install/", pgm)
Default(install)
Return('install')
env=Environment()
Export('env')
lib=SConscript("SConscript")
This diff is collapsed.
#ifndef _LIBTRANSFORM_INSTRUCTIONCOUNT_H_
#define _LIBTRANSFORM_INSTRUCTIONCOUNT_H_
#include <irdb-transform>
#include <irdb-deep>
#include <memory>
#include <map>
#include <set>
#include <tuple>
#include <libStructDiv.h>
#include <exeio.h>
#include <elfio/elfio.hpp>
#include <elfio/elfio_symbols.hpp>
#include <elf.h>
using Elf_Xword = uint64_t;
using Elf_Half = uint16_t;
template <class T_Sym, class T_Rela, class T_Rel, class T_Dyn, class Extractor>
class MoveGlobals_t : public IRDB_SDK::Transform
{
public:
MoveGlobals_t(IRDB_SDK::VariantID_t *p_variantID,
IRDB_SDK::FileIR_t*p_variantIR,
const std::string &p_dont_move,
const std::string &p_move_only,
const int p_max_moveables,
const bool p_random,
const bool p_aggrssive,
const bool p_use_stars=false);
int execute(IRDB_SDK::pqxxDB_t &pqxx_interface);
private:
// MEDS_Annotation::MEDS_Annotations_t& getAnnotations();
void ParseSyms(EXEIO::exeio * reader);
void SetupScoopMap();
void FilterScoops();
void TieScoops();
void FindInstructionReferences();
void FindDataReferences();
void UpdateScoopLocations();
void FilterAndCoalesceTiedScoops();
IRDB_SDK::Relocation_t* FindRelocationWithType(IRDB_SDK::BaseObj_t* obj, std::string type);
void PrintStats();
void HandleMemoryOperand(IRDB_SDK::DecodedInstruction_t &disasm, const IRDB_SDK::DecodedOperandVector_t::iterator the_arg,
IRDB_SDK::Instruction_t* insn, const IRDB_SDK::DecodedOperandVector_t &the_arg_container);
void HandleImmediateOperand(const IRDB_SDK::DecodedInstruction_t& disasm, const IRDB_SDK::DecodedOperandVector_t::iterator the_arg, IRDB_SDK::Instruction_t* insn);
IRDB_SDK::DataScoop_t* DetectAnnotationScoop(IRDB_SDK::Instruction_t* insn);
IRDB_SDK::DataScoop_t* DetectProperScoop(const IRDB_SDK::DecodedInstruction_t& disasm, const IRDB_SDK::DecodedOperandVector_t::iterator the_arg, IRDB_SDK::Instruction_t* insn,
IRDB_SDK::VirtualOffset_t immed_addr, bool immed, const IRDB_SDK::DecodedOperandVector_t &the_arg_container);
IRDB_SDK::DataScoop_t* DetectProperScoop_ConsiderEndOfPrev(const IRDB_SDK::DecodedInstruction_t& disasm, const IRDB_SDK::DecodedOperandVector_t::iterator the_arg, IRDB_SDK::Instruction_t* insn,
IRDB_SDK::VirtualOffset_t insn_addr, bool immed, IRDB_SDK::DataScoop_t* ret, const IRDB_SDK::DecodedOperandVector_t &the_arg_container);
IRDB_SDK::DataScoop_t* DetectProperScoop_ConsiderStartOfNext(const IRDB_SDK::DecodedInstruction_t& disasm, const IRDB_SDK::DecodedOperandVector_t::iterator the_arg, IRDB_SDK::Instruction_t* insn,
IRDB_SDK::VirtualOffset_t insn_addr, bool immed, IRDB_SDK::DataScoop_t* cand_scoop, const IRDB_SDK::DecodedOperandVector_t &the_arg_container);
void ApplyImmediateRelocation(IRDB_SDK::Instruction_t *insn, IRDB_SDK::DataScoop_t* to);
void ApplyAbsoluteMemoryRelocation(IRDB_SDK::Instruction_t *insn, IRDB_SDK::DataScoop_t* to);
void ApplyPcrelMemoryRelocation(IRDB_SDK::Instruction_t *insn, IRDB_SDK::DataScoop_t* to);
void ApplyDataRelocation(IRDB_SDK::DataScoop_t *from, unsigned int offset, IRDB_SDK::DataScoop_t* to);
IRDB_SDK::DataScoop_t* findScoopByAddress(const IRDB_SDK::VirtualOffset_t a) const;
bool AreScoopsAdjacent(const IRDB_SDK::DataScoop_t *a, const IRDB_SDK::DataScoop_t *b) const;
libStructDiv::StructuredDiversity_t *struct_div;
std::vector<T_Sym> static_symbols;
std::vector<T_Sym> dynamic_symbols;
EXEIO::exeio* exe_reader;
struct cmpByName {
bool operator()(const IRDB_SDK::DataScoop_t* a, const IRDB_SDK::DataScoop_t* b) const {
return (a->getName() < b->getName());
}
};
std::set<IRDB_SDK::DataScoop_t*> moveable_scoops;
std::map<IRDB_SDK::DataScoop_t*,unsigned int> reasons;
using ScoopPair_t = std::pair<IRDB_SDK::DataScoop_t*,IRDB_SDK::DataScoop_t*>;
std::set<ScoopPair_t> tied_scoops;
// sets to record what insns need to be fixed later.
struct Insn_fixup
{
IRDB_SDK::Instruction_t* from; IRDB_SDK::DataScoop_t* to;
bool operator <(const struct Insn_fixup& rhs) const
{
return std::tie(from, to) < std::tie(rhs.from, rhs.to);
}
};
using Insn_fixup_t = struct Insn_fixup;
std::set<Insn_fixup_t> pcrel_refs_to_scoops, absolute_refs_to_scoops, immed_refs_to_scoops;
// data references to scoops
struct Scoop_fixup
{
IRDB_SDK::DataScoop_t* from; unsigned int offset; IRDB_SDK::DataScoop_t* to;
bool operator <(const struct Scoop_fixup & rhs) const
{
return std::tie(from, offset, to) < std::tie(rhs.from, rhs.offset, rhs.to);
}
};
using Scoop_fixup_t = struct Scoop_fixup;;
std::set<Scoop_fixup_t> data_refs_to_scoops;
int tied_unpinned;
int tied_pinned;
int tied_nochange;
int ties_for_folded_constants;
const std::string dont_move;
const std::string move_only;
using RangePair_t = std::pair<IRDB_SDK::VirtualOffset_t,IRDB_SDK::VirtualOffset_t>;
struct cmpByRange
{
bool operator()(const RangePair_t& a, const RangePair_t& b) const {
return (a.second < b.first);
}
};
std::map<RangePair_t, IRDB_SDK::DataScoop_t*, cmpByRange> scoop_map;
const int max_moveables;
const bool random;
const bool aggressive;
const bool m_use_stars;
std::unique_ptr<IRDB_SDK::StaticGlobalStartMap_t > deep_global_static_ranges;
std::unique_ptr<IRDB_SDK::RangeSentinelSet_t> sentinels;
};
class Extractor64_t
{
public:
Elf_Xword elf_r_sym (Elf_Xword a) { return ELF64_R_SYM (a); }
Elf_Xword elf_r_type(Elf_Xword a) { return ELF64_R_TYPE(a); }
unsigned char elf_st_bind(unsigned char a) { return ELF64_ST_BIND(a); }
unsigned char elf_st_type(unsigned char a) { return ELF64_ST_TYPE(a); }
};
class Extractor32_t
{
public:
Elf32_Word elf_r_sym (Elf32_Word a) { return ELF32_R_SYM (a); }
Elf32_Word elf_r_type(Elf32_Word a) { return ELF32_R_TYPE(a); }
unsigned char elf_st_bind(unsigned char a) { return ELF32_ST_BIND(a); }
unsigned char elf_st_type(unsigned char a) { return ELF32_ST_TYPE(a); }
};
const static auto elftable_names= std::set<std::string> ({".dynamic",".got",".got.plt",".dynstr",".dynsym",".rel.dyn",".rela.dyn",".rel.plt",".rela.plt", ".gnu.version", ".gnu_version_r"});
const static auto elftable_nocodeptr_names= std::set<std::string> ({".dynamic"});
using MoveGlobals32_t = class MoveGlobals_t<Elf32_Sym, Elf32_Rela, Elf32_Rel, Elf32_Dyn, Extractor32_t>;
using MoveGlobals64_t = class MoveGlobals_t<Elf64_Sym, Elf64_Rela, Elf64_Rel, Elf64_Dyn, Extractor64_t>;
#endif
#include <stdlib.h>
#include <fstream>
#include <irdb-core>
#include "mg.hpp"
#include <getopt.h>
using namespace std;
using namespace IRDB_SDK;
#define ALLOF(a) begin(a),end(a)
class MoveGlobalsDriver_t : public TransformStep_t
{
void usage(string programName)
{
auto et_names=string();
for(const auto &str : elftable_names )
et_names+=str+" ";
cerr << "Usage: " << programName << " <variant id> <annotation file>" <<endl;
cout<<"--elftables-only,-o Move sections titled \""<<et_names<<"\""<<endl;
cout<<"--aggressive Use aggressive heuristics to move more variables (does not affect elftables) "<<endl;
cout<<"--no-conservative alias for --aggressive"<<endl;
cout<<"--conservative (default) Use conservative heuristics to increase reliability (does not affect elftables) "<<endl;
cout<<"--no-aggressive alias for --conservative"<<endl;
cout<<endl;
cout<<"---- debugging options (power users only) "<<endl;
cout<<"--use-stars Enable STARS deep analysis for more analysis precision (current default). "<<endl;
cout<<"--no-use-stars Disable STARS deep analysis for more analysis precision. "<<endl;
cout<<"--move,-m Move only the given objects."<<endl;
cout<<"--dont,-d Dont move the listed objects (overrides --only)."<<endl;
cout<<"--number,-n Max number of scoops to move."<<endl;
cout<<"--random,-r Randomly select scoops to move."<<endl;
cout<<"--help,--usage,-?,-h Display this message"<<endl;
}
DatabaseID_t variantID = BaseObj_t::NOT_IN_DATABASE;
string dont_move = "";
string move_only = "";
size_t max_moveables = 0;
bool random = false;
bool aggressive = false;
VariantID_t* pidp = nullptr;
const string programName=string("libmove_globals.so");
bool use_stars=true;
int parseArgs(const vector<string> step_args)
{
auto argv = vector<char*>();
transform(ALLOF(step_args), back_inserter(argv), [](const string &s) -> char* { return const_cast<char*>(s.c_str()); } );
const auto argc = step_args.size();
auto strtolError = (char*) nullptr;
/*
* Check that we've been called correctly:
* <program> <variant id> <annotation file>
*/
if(argc < 1)
{
usage(programName);
return 2;
}
variantID = strtol(step_args[0].c_str(), &strtolError, 10);
if (*strtolError != '\0')
{
cerr << "Invalid variantID: " << step_args[0] << endl;
return 1;
}
// Parse some options for the transform
const static struct option long_options[] = {
{"elftables-only", no_argument, 0, 'o'},
{"use-stars", no_argument, 0, 's'},
{"no-use-stars", no_argument, 0, 't'},
{"move", required_argument, 0, 'm'},
{"dont", required_argument, 0, 'd'},
{"number", required_argument, 0, 'n'},
{"aggressive", no_argument, 0, 'a'},
{"no-aggressive", no_argument, 0, 'A'},
{"conservative", no_argument, 0, 'A'},
{"no-conservative", no_argument, 0, 'a'},
{"random", no_argument, 0, 'r'},
{"help", no_argument, 0, 'h'},
{"usage", no_argument, 0, '?'},
{0,0,0,0}
};
auto short_opts="b:oh?m:d:n:aAst";
while(1)
{
int index = 0;
int c = getopt_long(argc, &argv[0], short_opts, long_options, &index);
if (c == -1)
break;
switch(c) {
case 0:
break;
case 'c':
case 'o':
// add elftables to move only list
for(const auto &str : elftable_names )
move_only+= str+" ";
break;
case 's':
use_stars=true;
break;
case 't':
use_stars=false;
break;
case 'm':
move_only+=string(" ") + optarg;
break;
case 'd':
dont_move+=string(" ") + optarg;
break;
case 'n':
max_moveables+=strtoll(optarg,nullptr,0);
break;
case 'r':
random=true;
break;
case 'a':
cout<<"Setting aggressive mode"<<endl;
aggressive=true;
break;
case 'A':
cout<<"Setting conservative mode"<<endl;
aggressive=false;
break;
case '?':
case 'h':
usage("libmove_globals.so");
return 1;
default:
break;
}
}
return 0;
}
int executeStep(IRDBObjects_t *const irdb_objects)
{
auto exit_code = (int) 0;
/* setup the interface to the sql server */
const auto pqxx_interface=irdb_objects->getDBInterface();
BaseObj_t::setInterface(pqxx_interface);
// get the variant info from the database
pidp=irdb_objects->addVariant(variantID); // new VariantID_t(variantID);
assert(pidp && pidp->isRegistered()==true);
auto transformExitCode = (int) 0;
for(auto this_file : pidp->getFiles())
{
try
{
/* read the IR from the DB */
auto firp = irdb_objects->addFileIR(variantID, this_file->getBaseID()); // new FileIR_t(*pidp, this_file);
cout<<"Transforming "<<this_file->getURL()<<endl;
assert(firp && pidp);
/*
* Create a transformation and then
* invoke its execution.
*/
if (firp->getArchitectureBitWidth() == 64)
{
MoveGlobals64_t mg(pidp, firp, dont_move, move_only, max_moveables, random, aggressive, use_stars);
transformExitCode = mg.execute(*pqxx_interface);
}
else
{
MoveGlobals32_t mg(pidp, firp, dont_move, move_only, max_moveables, random, aggressive, use_stars);
transformExitCode = mg.execute(*pqxx_interface);
}
/*
* If everything about the transformation
* went okay, then we will write the updated
* set of instructions to the database.
*/
if (transformExitCode != 0)
{
cerr << programName << ": transform failed. Check logs." << endl;
exit_code=2;
}
}
catch (DatabaseError_t pnide)
{
cerr << programName << ": Unexpected database error: " << pnide << endl;
return 1;
}
catch (const std::exception &exc)
{
// catch anything thrown within try block that derives from std::exception
std::cerr << "Unexpected exception: " << exc.what();
return 1;
}
catch (...)
{
cerr << programName << ": Unexpected error" << endl;
return 1;
}
}
return exit_code;
}
std::string getStepName(void) const override
{
return std::string("move_globals");
}
};
extern "C"
shared_ptr<TransformStep_t> getTransformStep(void)
{
return shared_ptr<TransformStep_t>(new MoveGlobalsDriver_t());
}
#!/bin/bash
# the bad boys
#benchmarks="
# 400.perlbench
# 403.gcc
# 445.gobmk
# 450.soplex
# 453.povray
# 458.sjeng
# 464.h264ref
# 465.tonto
# 471.omnetpp
# 481.wrf
# 482.sphinx3
# 483.xalancbmk
# "
# all
all_benchmarks="400.perlbench 401.bzip2 403.gcc 410.bwaves 416.gamess 429.mcf 433.milc 434.zeusmp 435.gromacs 436.cactusADM 437.leslie3d 444.namd 445.gobmk 450.soplex 453.povray 454.calculix 456.hmmer 458.sjeng 459.GemsFDTD 462.libquantum 464.h264ref 465.tonto 470.lbm 471.omnetpp 473.astar 481.wrf 482.sphinx3 483.xalancbmk"
number=1
setup()
{
if [ ! -d spec2006 ]; then
# svn co ^/spec2006/trunk spec2006
git clone --depth 1 http://git.zephyr-software.com/allzp/spec2006.git spec2006
fi
if [[ ! -f /usr/bin/gfortran ]]; then
sudo apt-get install gfortran -y
fi
cd spec2006/
if [ ! -L bin ]; then
ln -s bin.power/ bin
fi
source shrc
bin/relocate
}
run_test()
{
config_name=$1
config=$2
benchmarks="$3"
all_configs_that_were_run="$all_configs_that_were_run $config_name"
cd $SPEC
if [ ! -d result.$config_name ]; then
dropdb $PGDATABASE
createdb $PGDATABASE
$PEASOUP_HOME/tools/db/pdb_setup.sh
rm -Rf result/*
runspec --action scrub --config $config $benchmarks
echo
echo "**************************************************************************"
echo "Starting test of $config_name"
echo "**************************************************************************"
echo
runspec --action validate --config $config -n $number $benchmarks
cp benchspec/CPU2006/*/exe/* result
mv result result.$config_name
for bench in $benchmarks
do
mv benchspec/CPU2006/$bench/run/build*/peasoup*/logs result.$config_name/$bench.log
done
fi
}
get_size_result()
{
bench=$1
if [ -e $bench ]; then
size=$(stat --printf="%s" $bench)
#echo -n "$size"
#LC_ALL= numfmt --grouping $size
#LC_ALL= printf "%'d" $size
#LC_NUMERIC=en_US printf "%'d" $size
#LC_NUMERIC=en_US printf "%'f" $size
#LC_NUMERIC=en_US printf "%'.f" $size
#LC_NUMERIC=en_US printf "%'10.10f" $size
#LC_NUMERIC=en_US /usr/bin/printf "%'d" $size
echo $size
else
echo -n "0"
fi
}
get_result()
{
bench=$1
config=$2
results=$(cat $SPEC/result.$config/CPU2006.002.log|grep Success|grep $bench|grep ratio=|sed 's/.*ratio=//'|sed 's/,.*//')
sum=0
count=0
for res in $results
do
sum=$(echo $sum + $res | bc)
count=$(echo $count + 1 | bc)
done
#echo sum=$sum
#echo count=$count
res=$(echo "scale=2; $sum / $count" | bc 2> /dev/null )
count=$(echo $res|wc -w)
if [ $count = 1 ]; then
echo -n $res
else
echo -n "0"
fi
}
# global
all_configs_that_were_run=""
get_raw_results()
{
local configs="$all_configs_that_were_run"
get_raw_perf_results "$configs"
get_raw_size_results "$configs"
get_raw_mg_results "$configs"
#get_raw_fde_results "$configs"
}
get_raw_perf_results()
{
configs=$*
first_config=$1
echo "--------------------------------------------------------------"
echo "Performance results are:"
echo "--------------------------------------------------------------"
echo benchmark $configs
for bench in $benchmarks
do
echo -n "$bench "
for config in $*
do
get_result $bench $config
echo -n " "
done
echo
done
}
get_raw_size_results()
{
echo "--------------------------------------------------------------"
echo "Size results are:"
echo "--------------------------------------------------------------"
configs=$*
echo benchmark $configs
for bench in $SPEC/result.$first_config/*_base.amd64-m64-gcc42-nn
do
echo -n "$(basename $bench _base.amd64-m64-gcc42-nn) "
for config in $*
do
if [[ $config == "baseline" ]]; then
file="$SPEC/result.$config/$(basename $bench)"
cp $file /tmp/foo.exe
strip /tmp/foo.exe
file="/tmp/foo.exe"
else
file="$SPEC/result.$config/$(basename $bench)"
fi
res=$(get_size_result $file)
#printf "%15s" $res
echo -n " $res"
done
echo
done
}
get_raw_mg_results()
{
echo "--------------------------------------------------------------"
echo "Move_globals unmoveable scoop count:"
echo "--------------------------------------------------------------"
configs=$*
echo benchmark $configs
for bench in $benchmarks
do
echo -n $bench
for config in $*
do
file="$SPEC/result.$config/${bench}.log/move_globals.log"
res="N/A"
if [[ -f $file ]]; then
res=$(grep "ATTRIBUTE mg::unmoveable_scoops=" $file|sed "s|.*=||")
fi
echo -n " $res"
done
echo
done
echo "--------------------------------------------------------------"
echo "Move_globals moveable scoop count:"
echo "--------------------------------------------------------------"
configs=$*
echo benchmark $configs
for bench in $benchmarks
do
echo -n $bench
for config in $*
do
file="$SPEC/result.$config/${bench}.log/move_globals.log"
res="N/A"
if [[ -f $file ]]; then
res=$(grep "ATTRIBUTE mg::moveable_scoops=" $file|sed "s|.*=||")
fi
echo -n " $res"
done
echo
done
}
get_raw_fde_results()
{
echo "--------------------------------------------------------------"
echo "FDE results are:"
echo "--------------------------------------------------------------"
configs=$*
echo benchmark $configs
for bench in $SPEC/result.$first_config/*_base.amd64-m64-gcc42-nn
do
#printf "%-20s" $(basename $bench _base.amd64-m64-gcc42-nn)
echo -n $(basename $bench _base.amd64-m64-gcc42-nn)
for config in $*
do
file="$SPEC/result.$config/$(basename $bench)"
res=$(readelf -w $file |grep FDE|wc -l )
#if [[ $config == "baseline" ]]; then
#else
#fi
#printf "%15s" $res
echo -n " $res"
done
echo
done
}
main()
{
zipr_flags=" --backend zipr --step-option zipr:--add-sections --step-option zipr:true"
trace_flags=" --step-option zipr:--traceplacement:on --step-option zipr:true"
relax_flags=" --step-option zipr:--relax:on --step-option zipr:true --step-option zipr:--unpin:on --step-option zipr:false"
nounpin_flags=" --step-option zipr:--unpin:on --step-option zipr:false"
split_flags=" --step-option fill_in_indtargs:--split-eh-frame "
icall_flags=" --step-option fix_calls:--no-fix-icalls "
p1flags=" --critical-step p1transform=on "
mg_flags_elftables=" --critical-step move_globals=on --step-option move_globals:--elftables"
mg_flags_conservative=" --critical-step move_globals=on "
mg_aggressive_flags=" --critical-step move_globals=on --step-option move_globals:--aggressive "
start_dir=$(pwd)
setup
run_test baseline $SPEC/config/ubuntu14.04lts-64bit.cfg "$all_benchmarks"
run_test baseline-fpic $SPEC/config/ubuntu14.04lts-64bit-fpic.cfg "$all_benchmarks"
# should be 100% success, tested by jdh on 8/28/17 as 100% success.
#PSOPTS="$zipr_flags " run_test zipr $SPEC/config/ubuntu14.04lts-64bit-withps.cfg "$all_benchmarks"
PSOPTS="$zipr_flags $trace_flags " run_test zipr-trace $SPEC/config/ubuntu14.04lts-64bit-withps.cfg "$all_benchmarks"
PSOPTS="$zipr_flags $trace_flags " run_test zipr-trace-fpic $SPEC/config/ubuntu14.04lts-64bit-fpic-withps.cfg "$all_benchmarks"
PSOPTS="$zipr_flags $trace_flags $mg_flags_conservative" run_test mg-conservative-trace $SPEC/config/ubuntu14.04lts-64bit-withps.cfg "$all_benchmarks"
PSOPTS="$zipr_flags $trace_flags $mg_flags_conservative" run_test mg-conservative-trace-fpic $SPEC/config/ubuntu14.04lts-64bit-fpic-withps.cfg "$all_benchmarks"
PSOPTS="$zipr_flags $trace_flags $mg_flags_all $mg_aggressive_flags" run_test mg-aggressive-trace $SPEC/config/ubuntu14.04lts-64bit-withps.cfg "$all_benchmarks"
PSOPTS="$zipr_flags $trace_flags $mg_flags_all $mg_aggressive_flags" run_test mg-aggressive-trace-fpic $SPEC/config/ubuntu14.04lts-64bit-fpic-withps.cfg "$all_benchmarks"
PSOPTS="$zipr_flags $trace_flags $mg_flags_elftables" run_test mgelftables-trace $SPEC/config/ubuntu14.04lts-64bit-withps.cfg "$all_benchmarks"
get_raw_results
# get_raw_results baseline zipr zipr-trace split-no-fix-icalls split-no-fix-icalls-trace zipr-trace-p1 split-trace-p1 split-no-fix-icalls-trace-p1 zipr-trace-p1-rerun
}
main "$@"
/*
* Copyright (c) 2013, 2014 - University of Virginia
*
* This file may be used and modified for non-commercial purposes as long as
* all copyright, permission, and nonwarranty notices are preserved.
* Redistribution is prohibited without prior written consent from the University
* of Virginia.
*
* Please contact the authors for restrictions applying to commercial use.
*
* THIS SOURCE IS PROVIDED "AS IS" AND WITHOUT ANY EXPRESS OR IMPLIED
* WARRANTIES, INCLUDING, WITHOUT LIMITATION, THE IMPLIED WARRANTIES OF
* MERCHANTIBILITY AND FITNESS FOR A PARTICULAR PURPOSE.
*
* Author: University of Virginia
* e-mail: jwd@virginia.com
* URL : http://www.cs.virginia.edu/
*
*/
#include "AnnotationBoundaryGenerator.hpp"
#include <map>
#include <iostream>
using namespace std;
using namespace IRDB_SDK;
//using namespace MEDS_Annotation;
vector<Range> AnnotationBoundaryGenerator::GetBoundaries(IRDB_SDK::Function_t *func)
{
assert(0);
#if 0
vector<Range> ranges;
// std::multimap<VirtualOffset, MEDS_AnnotationBase>
MEDS_Annotations_t annotations = annotParser->getAnnotations();
for(
set<Instruction_t*>::const_iterator it=func->getInstructions().begin();
it!=func->getInstructions().end();
++it
)
{
Instruction_t* instr = *it;
VirtualOffset_t irdb_vo = instr->getAddress()->getVirtualOffset();
if (irdb_vo == 0) continue;
VirtualOffset vo(irdb_vo);
//std::pair<std::multimap<VirtualOffset, MEDS_AnnotationBase>::iterator,std::multimap<VirtualOffset, MEDS_AnnotationBase>::iterator> ret;
std::pair<MEDS_Annotations_t::iterator,MEDS_Annotations_t::iterator> ret;
ret = annotations.equal_range(vo);
MEDS_InstructionCheckAnnotation annotation;
MEDS_InstructionCheckAnnotation* p_annotation;
for (MEDS_Annotations_t::iterator it = ret.first; it != ret.second; ++it)
{
p_annotation=dynamic_cast<MEDS_InstructionCheckAnnotation*>(it->second);
if(p_annotation==NULL)
continue;
annotation = *p_annotation;
if (annotation.isValid() && annotation.isMemset())
{
//cerr<<"Memset annot found"<<endl;
int objectSize = annotation.getObjectSize();
int offset = annotation.getStackOffset();
Range cur;
cur.SetOffset(offset);
cur.SetSize(objectSize);
if (annotation.isEbpOffset())
{
if(offset < 0)
{
ranges.push_back(cur);
}
} else if (annotation.isEspOffset())
{
if(offset >= 0)
{
ranges.push_back(cur);
}
} else
{
// something went wrong
assert(0);
}
}
}
}
return ranges;
#endif
}
/*
* Copyright (c) 2013, 2014 - University of Virginia
*
* This file may be used and modified for non-commercial purposes as long as
* all copyright, permission, and nonwarranty notices are preserved.
* Redistribution is prohibited without prior written consent from the University
* of Virginia.
*
* Please contact the authors for restrictions applying to commercial use.
*
* THIS SOURCE IS PROVIDED "AS IS" AND WITHOUT ANY EXPRESS OR IMPLIED
* WARRANTIES, INCLUDING, WITHOUT LIMITATION, THE IMPLIED WARRANTIES OF
* MERCHANTIBILITY AND FITNESS FOR A PARTICULAR PURPOSE.
*
* Author: University of Virginia
* e-mail: jwd@virginia.com
* URL : http://www.cs.virginia.edu/
*
*/
#ifndef __ANNOTBOUNDGEN
#define __ANNOTBOUNDGEN
#include "PrecedenceBoundaryGenerator.hpp"
//#include "MEDS_AnnotationParser.hpp"
//#include "MEDS_AnnotationBase.hpp"
//#include "MEDS_InstructionCheckAnnotation.hpp"
// #include "VirtualOffset.hpp"
#include <fstream>
class AnnotationBoundaryGenerator : public PrecedenceBoundaryGenerator
{
protected:
// MEDS_Annotation::MEDS_AnnotationParser *annotParser;
public:
// AnnotationBoundaryGenerator(MEDS_Annotation::MEDS_AnnotationParser *annotParser) : annotParser(annotParser){}
virtual std::vector<Range> GetBoundaries(IRDB_SDK::Function_t *func);
};
#endif