Skip to content
Snippets Groups Projects

Compare revisions

Changes are shown as if the source revision was being merged into the target revision. Learn more about comparing revisions.

Source

Select target project
No results found

Target

Select target project
  • opensrc/zipr
  • whh8b/zipr
2 results
Show changes
Commits on Source (17)
Showing
with 126 additions and 4789 deletions
......@@ -21,28 +21,42 @@ stages:
- ./cicd_testing/do-clean.sh
# per os items
do-nightly-clean-ubuntu18:
<<: *do-nightly-clean
tags:
- ubuntu18
variables:
OS: 'ubuntu18'
# per os items
do-nightly-clean-ubuntu16:
#do-nightly-clean-ubuntu18:
# <<: *do-nightly-clean
# tags:
# - ubuntu18
# variables:
# OS: 'ubuntu18'
#
## per os items
#do-nightly-clean-ubuntu16:
# <<: *do-nightly-clean
# tags:
# - ubuntu16
# variables:
# OS: 'ubuntu16'
#
## per os items
#do-nightly-clean-centos75:
# <<: *do-nightly-clean
# tags:
# - centos75
# variables:
# OS: 'centos75'
#
do-nightly-clean-arm32:
<<: *do-nightly-clean
tags:
- ubuntu16
- arm32
variables:
OS: 'ubuntu16'
OS: 'arm32'
# per os items
do-nightly-clean-centos75:
do-nightly-clean-arm64:
<<: *do-nightly-clean
tags:
- centos75
- arm64
variables:
OS: 'centos75'
OS: 'arm64'
#
......@@ -56,26 +70,40 @@ do-nightly-clean-centos75:
# per os items
do-build-ubuntu18:
<<: *do-build
tags:
- ubuntu18
variables:
OS: 'ubuntu18'
do-build-ubuntu16:
#do-build-ubuntu18:
# <<: *do-build
# tags:
# - ubuntu18
# variables:
# OS: 'ubuntu18'
#
#do-build-ubuntu16:
# <<: *do-build
# tags:
# - ubuntu16
# variables:
# OS: 'ubuntu16'
#
#do-build-centos75:
# <<: *do-build
# tags:
# - centos75
# variables:
# OS: 'centos75'
#
do-build-arm32:
<<: *do-build
tags:
- ubuntu16
- arm32
variables:
OS: 'ubuntu16'
OS: 'arm32'
do-build-centos75:
do-build-arm64:
<<: *do-build
tags:
- centos75
- arm64
variables:
OS: 'centos75'
OS: 'arm64'
#
......@@ -86,20 +114,20 @@ do-build-centos75:
script:
- ./cicd_testing/eh-tests.sh
basic_eh-tests-ubuntu18:
<<: *basic_eh-tests
tags:
- ubuntu18
basic_eh-tests-ubuntu16:
<<: *basic_eh-tests
tags:
- ubuntu16
basic_eh-tests-centos75:
<<: *basic_eh-tests
tags:
- centos75
#basic_eh-tests-ubuntu18:
# <<: *basic_eh-tests
# tags:
# - ubuntu18
#
#basic_eh-tests-ubuntu16:
# <<: *basic_eh-tests
# tags:
# - ubuntu16
#
#basic_eh-tests-centos75:
# <<: *basic_eh-tests
# tags:
# - centos75
#
# test_cmds.sh fixcalls
......@@ -109,64 +137,64 @@ basic_eh-tests-centos75:
script:
- ./cicd_testing/builtin-xforms.sh
builtin-xforms-ubuntu18:
<<: *builtin-xforms
tags:
- ubuntu18
builtin-xforms-ubuntu16:
<<: *builtin-xforms
tags:
- ubuntu16
builtin-xforms-centos75:
<<: *builtin-xforms
tags:
- centos75
variables:
OS: 'centos75'
#builtin-xforms-ubuntu18:
# <<: *builtin-xforms
# tags:
# - ubuntu18
#
#builtin-xforms-ubuntu16:
# <<: *builtin-xforms
# tags:
# - ubuntu16
#
#builtin-xforms-centos75:
# <<: *builtin-xforms
# tags:
# - centos75
# variables:
# OS: 'centos75'
#
# elfdep test
#
# template
.elfdep: &elfdep
stage: test
script:
- ./cicd_testing/elfdep.sh
elfdep-ubuntu18:
<<: *elfdep
tags:
- ubuntu18
variables:
OS: 'ubuntu18'
elfdep-ubuntu16:
<<: *elfdep
tags:
- ubuntu16
variables:
OS: 'ubuntu16'
elfdep-centos75:
<<: *elfdep
tags:
- centos75
variables:
OS: 'centos75'
#.elfdep: &elfdep
# stage: test
# script:
# - ./cicd_testing/elfdep.sh
#
#elfdep-ubuntu18:
# <<: *elfdep
# tags:
# - ubuntu18
# variables:
# OS: 'ubuntu18'
#
#elfdep-ubuntu16:
# <<: *elfdep
# tags:
# - ubuntu16
# variables:
# OS: 'ubuntu16'
#
#elfdep-centos75:
# <<: *elfdep
# tags:
# - centos75
# variables:
# OS: 'centos75'
#
# deploy a docker image
#
deploy-u18:
stage: deploy
script:
- ./cicd_testing/deploy.sh
tags:
- ubuntu18
variables:
OS: 'ubuntu18'
#deploy-u18:
# stage: deploy
# script:
# - ./cicd_testing/deploy.sh
# tags:
# - ubuntu18
# variables:
# OS: 'ubuntu18'
......@@ -53,8 +53,6 @@ Export('env')
# list of zipr plugins and irdb xforms to build
transformDirs='''
builtin_xforms/add_lib
builtin_xforms/move_globals
builtin_xforms/p1transform
builtin_xforms/resolve_callbacks
zipr_push64_reloc_plugin
zipr
......
Subproject commit 73b402e2b8ad059acc19cbdd37c41a3ea30097f4
Subproject commit 3539f5fc632060d890c77b2b294f3edfc4691a2e
......@@ -6,7 +6,12 @@ if [[ "$*" =~ "--debug" ]]; then
SCONSDEBUG=" debug=1 "
fi
scons $SCONSDEBUG -j 3
if [[ $(uname -m) == 'armv7l' ]] || [[ $(uname -m) == 'aarch64' ]]; then
scons $SCONSDEBUG
else
scons $SCONSDEBUG -j 3
fi
exit
......
import os
Import('env')
# import and create a copy of the environment so we don't screw up anyone elses env.
myenv=env.Clone()
cpppath='''
$SECURITY_TRANSFORMS_HOME/third_party/elfio-code
$PEASOUP_HOME/irdb-libs/libEXEIO/include
$IRDB_SDK/include
'''
files=Glob( Dir('.').srcnode().abspath+"/*.cpp")
pgm="move_globals.so"
LIBPATH="$SECURITY_TRANSFORMS_HOME/lib"
LIBS=Split("irdb-core irdb-cfg irdb-util irdb-transform irdb-deep StructDiv EXEIO ")
myenv.Append(CPPPATH=Split(cpppath))
pgm=myenv.SharedLibrary(pgm, files, LIBPATH=LIBPATH, LIBS=LIBS)
install=myenv.Install("$SECURITY_TRANSFORMS_HOME/plugins_install/", pgm)
Default(install)
Return('install')
env=Environment()
Export('env')
lib=SConscript("SConscript")
This diff is collapsed.
#ifndef _LIBTRANSFORM_INSTRUCTIONCOUNT_H_
#define _LIBTRANSFORM_INSTRUCTIONCOUNT_H_
#include <irdb-transform>
#include <irdb-deep>
#include <memory>
#include <map>
#include <set>
#include <tuple>
#include <exeio.h>
#include <elfio/elfio.hpp>
#include <elfio/elfio_symbols.hpp>
#include <elf.h>
using Elf_Xword = uint64_t;
using Elf_Half = uint16_t;
template <class T_Sym, class T_Rela, class T_Rel, class T_Dyn, class Extractor>
class MoveGlobals_t : public IRDB_SDK::Transform_t
{
public:
MoveGlobals_t(IRDB_SDK::VariantID_t *p_variantID,
IRDB_SDK::FileIR_t*p_variantIR,
const std::string &p_dont_move,
const std::string &p_move_only,
const int p_max_moveables,
const bool p_random,
const bool p_aggrssive,
const bool p_use_stars=false);
int execute(IRDB_SDK::pqxxDB_t &pqxx_interface);
private:
// MEDS_Annotation::MEDS_Annotations_t& getAnnotations();
void ParseSyms(EXEIO::exeio * reader);
void SetupScoopMap();
void FilterScoops();
void TieScoops();
void FindInstructionReferences();
void FindDataReferences();
void UpdateScoopLocations();
void FilterAndCoalesceTiedScoops();
IRDB_SDK::Relocation_t* FindRelocationWithType(IRDB_SDK::BaseObj_t* obj, std::string type);
void PrintStats();
void HandleMemoryOperand(IRDB_SDK::DecodedInstruction_t &disasm, const IRDB_SDK::DecodedOperandVector_t::iterator the_arg,
IRDB_SDK::Instruction_t* insn, const IRDB_SDK::DecodedOperandVector_t &the_arg_container);
void HandleImmediateOperand(const IRDB_SDK::DecodedInstruction_t& disasm, const IRDB_SDK::DecodedOperandVector_t::iterator the_arg, IRDB_SDK::Instruction_t* insn);
IRDB_SDK::DataScoop_t* DetectAnnotationScoop(IRDB_SDK::Instruction_t* insn);
IRDB_SDK::DataScoop_t* DetectProperScoop(const IRDB_SDK::DecodedInstruction_t& disasm, const IRDB_SDK::DecodedOperandVector_t::iterator the_arg, IRDB_SDK::Instruction_t* insn,
IRDB_SDK::VirtualOffset_t immed_addr, bool immed, const IRDB_SDK::DecodedOperandVector_t &the_arg_container);
IRDB_SDK::DataScoop_t* DetectProperScoop_ConsiderEndOfPrev(const IRDB_SDK::DecodedInstruction_t& disasm, const IRDB_SDK::DecodedOperandVector_t::iterator the_arg, IRDB_SDK::Instruction_t* insn,
IRDB_SDK::VirtualOffset_t insn_addr, bool immed, IRDB_SDK::DataScoop_t* ret, const IRDB_SDK::DecodedOperandVector_t &the_arg_container);
IRDB_SDK::DataScoop_t* DetectProperScoop_ConsiderStartOfNext(const IRDB_SDK::DecodedInstruction_t& disasm, const IRDB_SDK::DecodedOperandVector_t::iterator the_arg, IRDB_SDK::Instruction_t* insn,
IRDB_SDK::VirtualOffset_t insn_addr, bool immed, IRDB_SDK::DataScoop_t* cand_scoop, const IRDB_SDK::DecodedOperandVector_t &the_arg_container);
void ApplyImmediateRelocation(IRDB_SDK::Instruction_t *insn, IRDB_SDK::DataScoop_t* to);
void ApplyAbsoluteMemoryRelocation(IRDB_SDK::Instruction_t *insn, IRDB_SDK::DataScoop_t* to);
void ApplyPcrelMemoryRelocation(IRDB_SDK::Instruction_t *insn, IRDB_SDK::DataScoop_t* to);
void ApplyDataRelocation(IRDB_SDK::DataScoop_t *from, unsigned int offset, IRDB_SDK::DataScoop_t* to);
IRDB_SDK::DataScoop_t* findScoopByAddress(const IRDB_SDK::VirtualOffset_t a) const;
bool AreScoopsAdjacent(const IRDB_SDK::DataScoop_t *a, const IRDB_SDK::DataScoop_t *b) const;
std::vector<T_Sym> static_symbols;
std::vector<T_Sym> dynamic_symbols;
EXEIO::exeio* exe_reader;
struct cmpByName {
bool operator()(const IRDB_SDK::DataScoop_t* a, const IRDB_SDK::DataScoop_t* b) const {
return (a->getName() < b->getName());
}
};
std::set<IRDB_SDK::DataScoop_t*> moveable_scoops;
std::map<IRDB_SDK::DataScoop_t*,unsigned int> reasons;
using ScoopPair_t = std::pair<IRDB_SDK::DataScoop_t*,IRDB_SDK::DataScoop_t*>;
std::set<ScoopPair_t> tied_scoops;
// sets to record what insns need to be fixed later.
struct Insn_fixup
{
IRDB_SDK::Instruction_t* from; IRDB_SDK::DataScoop_t* to;
bool operator <(const struct Insn_fixup& rhs) const
{
return std::tie(from, to) < std::tie(rhs.from, rhs.to);
}
};
using Insn_fixup_t = struct Insn_fixup;
std::set<Insn_fixup_t> pcrel_refs_to_scoops, absolute_refs_to_scoops, immed_refs_to_scoops;
// data references to scoops
struct Scoop_fixup
{
IRDB_SDK::DataScoop_t* from; unsigned int offset; IRDB_SDK::DataScoop_t* to;
bool operator <(const struct Scoop_fixup & rhs) const
{
return std::tie(from, offset, to) < std::tie(rhs.from, rhs.offset, rhs.to);
}
};
using Scoop_fixup_t = struct Scoop_fixup;;
std::set<Scoop_fixup_t> data_refs_to_scoops;
int tied_unpinned;
int tied_pinned;
int tied_nochange;
int ties_for_folded_constants;
const std::string dont_move;
const std::string move_only;
using RangePair_t = std::pair<IRDB_SDK::VirtualOffset_t,IRDB_SDK::VirtualOffset_t>;
struct cmpByRange
{
bool operator()(const RangePair_t& a, const RangePair_t& b) const {
return (a.second < b.first);
}
};
std::map<RangePair_t, IRDB_SDK::DataScoop_t*, cmpByRange> scoop_map;
const int max_moveables;
const bool random;
const bool aggressive;
const bool m_use_stars;
std::unique_ptr<IRDB_SDK::StaticGlobalStartMap_t > deep_global_static_ranges;
std::unique_ptr<IRDB_SDK::RangeSentinelSet_t> sentinels;
};
class Extractor64_t
{
public:
Elf_Xword elf_r_sym (Elf_Xword a) { return ELF64_R_SYM (a); }
Elf_Xword elf_r_type(Elf_Xword a) { return ELF64_R_TYPE(a); }
unsigned char elf_st_bind(unsigned char a) { return ELF64_ST_BIND(a); }
unsigned char elf_st_type(unsigned char a) { return ELF64_ST_TYPE(a); }
};
class Extractor32_t
{
public:
Elf32_Word elf_r_sym (Elf32_Word a) { return ELF32_R_SYM (a); }
Elf32_Word elf_r_type(Elf32_Word a) { return ELF32_R_TYPE(a); }
unsigned char elf_st_bind(unsigned char a) { return ELF32_ST_BIND(a); }
unsigned char elf_st_type(unsigned char a) { return ELF32_ST_TYPE(a); }
};
const static auto elftable_names= std::set<std::string> ({".dynamic",".got",".got.plt",".dynstr",".dynsym",".rel.dyn",".rela.dyn",".rel.plt",".rela.plt", ".gnu.version", ".gnu_version_r"});
const static auto elftable_nocodeptr_names= std::set<std::string> ({".dynamic"});
using MoveGlobals32_t = class MoveGlobals_t<Elf32_Sym, Elf32_Rela, Elf32_Rel, Elf32_Dyn, Extractor32_t>;
using MoveGlobals64_t = class MoveGlobals_t<Elf64_Sym, Elf64_Rela, Elf64_Rel, Elf64_Dyn, Extractor64_t>;
#endif
#include <stdlib.h>
#include <fstream>
#include <irdb-core>
#include "mg.hpp"
#include <getopt.h>
using namespace std;
using namespace IRDB_SDK;
#define ALLOF(a) begin(a),end(a)
class MoveGlobalsDriver_t : public TransformStep_t
{
void usage(string programName)
{
auto et_names=string();
for(const auto &str : elftable_names )
et_names+=str+" ";
cerr << "Usage: " << programName << " <variant id> <annotation file>" <<endl;
cout<<"--elftables-only,-o Move sections titled \""<<et_names<<"\""<<endl;
cout<<"--aggressive Use aggressive heuristics to move more variables (does not affect elftables) "<<endl;
cout<<"--no-conservative alias for --aggressive"<<endl;
cout<<"--conservative (default) Use conservative heuristics to increase reliability (does not affect elftables) "<<endl;
cout<<"--no-aggressive alias for --conservative"<<endl;
cout<<endl;
cout<<"---- debugging options (power users only) "<<endl;
cout<<"--use-stars Enable STARS deep analysis for more analysis precision (current default). "<<endl;
cout<<"--no-use-stars Disable STARS deep analysis for more analysis precision. "<<endl;
cout<<"--move,-m Move only the given objects."<<endl;
cout<<"--dont,-d Dont move the listed objects (overrides --only)."<<endl;
cout<<"--number,-n Max number of scoops to move."<<endl;
cout<<"--random,-r Randomly select scoops to move."<<endl;
cout<<"--help,--usage,-?,-h Display this message"<<endl;
}
DatabaseID_t variantID = BaseObj_t::NOT_IN_DATABASE;
string dont_move = "";
string move_only = "";
size_t max_moveables = 0;
bool random = false;
bool aggressive = false;
VariantID_t* pidp = nullptr;
const string programName=string("libmove_globals.so");
bool use_stars=true;
int parseArgs(const vector<string> step_args)
{
auto argv = vector<char*>({const_cast<char*>("libmove_globals.so")});
transform(ALLOF(step_args), back_inserter(argv), [](const string &s) -> char* { return const_cast<char*>(s.c_str()); } );
const auto argc = argv.size();
// Parse some options for the transform
const static struct option long_options[] = {
{"elftables-only", no_argument, 0, 'o'},
{"use-stars", no_argument, 0, 's'},
{"no-use-stars", no_argument, 0, 't'},
{"move", required_argument, 0, 'm'},
{"dont", required_argument, 0, 'd'},
{"number", required_argument, 0, 'n'},
{"aggressive", no_argument, 0, 'a'},
{"no-aggressive", no_argument, 0, 'A'},
{"conservative", no_argument, 0, 'A'},
{"no-conservative", no_argument, 0, 'a'},
{"random", no_argument, 0, 'r'},
{"help", no_argument, 0, 'h'},
{"usage", no_argument, 0, '?'},
{0,0,0,0}
};
auto short_opts="b:oh?m:d:n:aAst";
while(1)
{
int c = getopt_long(argc, &argv[0], short_opts, long_options, nullptr);
if (c == -1)
break;
switch(c) {
case 0:
break;
case 'c':
case 'o':
// add elftables to move only list
for(const auto &str : elftable_names )
move_only+= str+" ";
break;
case 's':
use_stars=true;
break;
case 't':
use_stars=false;
break;
case 'm':
move_only+=string(" ") + optarg;
break;
case 'd':
dont_move+=string(" ") + optarg;
break;
case 'n':
max_moveables+=strtoll(optarg,nullptr,0);
break;
case 'r':
random=true;
break;
case 'a':
cout<<"Setting aggressive mode"<<endl;
aggressive=true;
break;
case 'A':
cout<<"Setting conservative mode"<<endl;
aggressive=false;
break;
case '?':
case 'h':
usage("libmove_globals.so");
return 1;
default:
break;
}
}
return 0;
}
int executeStep()
{
variantID=getVariantID();
auto irdb_objects=getIRDBObjects();
auto exit_code = (int) 0;
/* setup the interface to the sql server */
const auto pqxx_interface=irdb_objects->getDBInterface();
BaseObj_t::setInterface(pqxx_interface);
// get the variant info from the database
pidp=irdb_objects->addVariant(variantID); // new VariantID_t(variantID);
assert(pidp && pidp->isRegistered()==true);
auto transformExitCode = (int) 0;
for(auto this_file : pidp->getFiles())
{
try
{
/* read the IR from the DB */
auto firp = irdb_objects->addFileIR(variantID, this_file->getBaseID()); // new FileIR_t(*pidp, this_file);
cout<<"Transforming "<<this_file->getURL()<<endl;
assert(firp && pidp);
/*
* Create a transformation and then
* invoke its execution.
*/
if (firp->getArchitectureBitWidth() == 64)
{
MoveGlobals64_t mg(pidp, firp, dont_move, move_only, max_moveables, random, aggressive, use_stars);
transformExitCode = mg.execute(*pqxx_interface);
}
else
{
MoveGlobals32_t mg(pidp, firp, dont_move, move_only, max_moveables, random, aggressive, use_stars);
transformExitCode = mg.execute(*pqxx_interface);
}
/*
* If everything about the transformation
* went okay, then we will write the updated
* set of instructions to the database.
*/
if (transformExitCode != 0)
{
cerr << programName << ": transform failed. Check logs." << endl;
exit_code=2;
}
}
catch (DatabaseError_t pnide)
{
cerr << programName << ": Unexpected database error: " << pnide << endl;
return 1;
}
catch (const std::exception &exc)
{
// catch anything thrown within try block that derives from std::exception
std::cerr << "Unexpected exception: " << exc.what();
return 1;
}
catch (...)
{
cerr << programName << ": Unexpected error" << endl;
return 1;
}
}
return exit_code;
}
std::string getStepName(void) const override
{
return std::string("move_globals");
}
};
extern "C"
shared_ptr<TransformStep_t> getTransformStep(void)
{
return shared_ptr<TransformStep_t>(new MoveGlobalsDriver_t());
}
#!/bin/bash
# the bad boys
#benchmarks="
# 400.perlbench
# 403.gcc
# 445.gobmk
# 450.soplex
# 453.povray
# 458.sjeng
# 464.h264ref
# 465.tonto
# 471.omnetpp
# 481.wrf
# 482.sphinx3
# 483.xalancbmk
# "
# all
all_benchmarks="400.perlbench 401.bzip2 403.gcc 410.bwaves 416.gamess 429.mcf 433.milc 434.zeusmp 435.gromacs 436.cactusADM 437.leslie3d 444.namd 445.gobmk 450.soplex 453.povray 454.calculix 456.hmmer 458.sjeng 459.GemsFDTD 462.libquantum 464.h264ref 465.tonto 470.lbm 471.omnetpp 473.astar 481.wrf 482.sphinx3 483.xalancbmk"
number=1
setup()
{
if [ ! -d spec2006 ]; then
# svn co ^/spec2006/trunk spec2006
git clone --depth 1 http://git.zephyr-software.com/allzp/spec2006.git spec2006
fi
if [[ ! -f /usr/bin/gfortran ]]; then
sudo apt-get install gfortran -y
fi
cd spec2006/
if [ ! -L bin ]; then
ln -s bin.power/ bin
fi
source shrc
bin/relocate
}
run_test()
{
config_name=$1
config=$2
benchmarks="$3"
all_configs_that_were_run="$all_configs_that_were_run $config_name"
cd $SPEC
if [ ! -d result.$config_name ]; then
dropdb $PGDATABASE
createdb $PGDATABASE
$PEASOUP_HOME/tools/db/pdb_setup.sh
rm -Rf result/*
runspec --action scrub --config $config $benchmarks
echo
echo "**************************************************************************"
echo "Starting test of $config_name"
echo "**************************************************************************"
echo
runspec --action validate --config $config -n $number $benchmarks
cp benchspec/CPU2006/*/exe/* result
mv result result.$config_name
for bench in $benchmarks
do
mv benchspec/CPU2006/$bench/run/build*/peasoup*/logs result.$config_name/$bench.log
done
fi
}
get_size_result()
{
bench=$1
if [ -e $bench ]; then
size=$(stat --printf="%s" $bench)
#echo -n "$size"
#LC_ALL= numfmt --grouping $size
#LC_ALL= printf "%'d" $size
#LC_NUMERIC=en_US printf "%'d" $size
#LC_NUMERIC=en_US printf "%'f" $size
#LC_NUMERIC=en_US printf "%'.f" $size
#LC_NUMERIC=en_US printf "%'10.10f" $size
#LC_NUMERIC=en_US /usr/bin/printf "%'d" $size
echo $size
else
echo -n "0"
fi
}
get_result()
{
bench=$1
config=$2
results=$(cat $SPEC/result.$config/CPU2006.002.log|grep Success|grep $bench|grep ratio=|sed 's/.*ratio=//'|sed 's/,.*//')
sum=0
count=0
for res in $results
do
sum=$(echo $sum + $res | bc)
count=$(echo $count + 1 | bc)
done
#echo sum=$sum
#echo count=$count
res=$(echo "scale=2; $sum / $count" | bc 2> /dev/null )
count=$(echo $res|wc -w)
if [ $count = 1 ]; then
echo -n $res
else
echo -n "0"
fi
}
# global
all_configs_that_were_run=""
get_raw_results()
{
local configs="$all_configs_that_were_run"
get_raw_perf_results "$configs"
get_raw_size_results "$configs"
get_raw_mg_results "$configs"
#get_raw_fde_results "$configs"
}
get_raw_perf_results()
{
configs=$*
first_config=$1
echo "--------------------------------------------------------------"
echo "Performance results are:"
echo "--------------------------------------------------------------"
echo benchmark $configs
for bench in $benchmarks
do
echo -n "$bench "
for config in $*
do
get_result $bench $config
echo -n " "
done
echo
done
}
get_raw_size_results()
{
echo "--------------------------------------------------------------"
echo "Size results are:"
echo "--------------------------------------------------------------"
configs=$*
echo benchmark $configs
for bench in $SPEC/result.$first_config/*_base.amd64-m64-gcc42-nn
do
echo -n "$(basename $bench _base.amd64-m64-gcc42-nn) "
for config in $*
do
if [[ $config == "baseline" ]]; then
file="$SPEC/result.$config/$(basename $bench)"
cp $file /tmp/foo.exe
strip /tmp/foo.exe
file="/tmp/foo.exe"
else
file="$SPEC/result.$config/$(basename $bench)"
fi
res=$(get_size_result $file)
#printf "%15s" $res
echo -n " $res"
done
echo
done
}
get_raw_mg_results()
{
echo "--------------------------------------------------------------"
echo "Move_globals unmoveable scoop count:"
echo "--------------------------------------------------------------"
configs=$*
echo benchmark $configs
for bench in $benchmarks
do
echo -n $bench
for config in $*
do
file="$SPEC/result.$config/${bench}.log/move_globals.log"
res="N/A"
if [[ -f $file ]]; then
res=$(grep "ATTRIBUTE mg::unmoveable_scoops=" $file|sed "s|.*=||")
fi
echo -n " $res"
done
echo
done
echo "--------------------------------------------------------------"
echo "Move_globals moveable scoop count:"
echo "--------------------------------------------------------------"
configs=$*
echo benchmark $configs
for bench in $benchmarks
do
echo -n $bench
for config in $*
do
file="$SPEC/result.$config/${bench}.log/move_globals.log"
res="N/A"
if [[ -f $file ]]; then
res=$(grep "ATTRIBUTE mg::moveable_scoops=" $file|sed "s|.*=||")
fi
echo -n " $res"
done
echo
done
}
get_raw_fde_results()
{
echo "--------------------------------------------------------------"
echo "FDE results are:"
echo "--------------------------------------------------------------"
configs=$*
echo benchmark $configs
for bench in $SPEC/result.$first_config/*_base.amd64-m64-gcc42-nn
do
#printf "%-20s" $(basename $bench _base.amd64-m64-gcc42-nn)
echo -n $(basename $bench _base.amd64-m64-gcc42-nn)
for config in $*
do
file="$SPEC/result.$config/$(basename $bench)"
res=$(readelf -w $file |grep FDE|wc -l )
#if [[ $config == "baseline" ]]; then
#else
#fi
#printf "%15s" $res
echo -n " $res"
done
echo
done
}
main()
{
zipr_flags=" --backend zipr --step-option zipr:--add-sections --step-option zipr:true"
trace_flags=" --step-option zipr:--traceplacement:on --step-option zipr:true"
relax_flags=" --step-option zipr:--relax:on --step-option zipr:true --step-option zipr:--unpin:on --step-option zipr:false"
nounpin_flags=" --step-option zipr:--unpin:on --step-option zipr:false"
split_flags=" --step-option fill_in_indtargs:--split-eh-frame "
icall_flags=" --step-option fix_calls:--no-fix-icalls "
p1flags=" --critical-step p1transform=on "
mg_flags_elftables=" --critical-step move_globals=on --step-option move_globals:--elftables"
mg_flags_conservative=" --critical-step move_globals=on "
mg_aggressive_flags=" --critical-step move_globals=on --step-option move_globals:--aggressive "
start_dir=$(pwd)
setup
run_test baseline $SPEC/config/ubuntu14.04lts-64bit.cfg "$all_benchmarks"
run_test baseline-fpic $SPEC/config/ubuntu14.04lts-64bit-fpic.cfg "$all_benchmarks"
# should be 100% success, tested by jdh on 8/28/17 as 100% success.
#PSOPTS="$zipr_flags " run_test zipr $SPEC/config/ubuntu14.04lts-64bit-withps.cfg "$all_benchmarks"
PSOPTS="$zipr_flags $trace_flags " run_test zipr-trace $SPEC/config/ubuntu14.04lts-64bit-withps.cfg "$all_benchmarks"
PSOPTS="$zipr_flags $trace_flags " run_test zipr-trace-fpic $SPEC/config/ubuntu14.04lts-64bit-fpic-withps.cfg "$all_benchmarks"
PSOPTS="$zipr_flags $trace_flags $mg_flags_conservative" run_test mg-conservative-trace $SPEC/config/ubuntu14.04lts-64bit-withps.cfg "$all_benchmarks"
PSOPTS="$zipr_flags $trace_flags $mg_flags_conservative" run_test mg-conservative-trace-fpic $SPEC/config/ubuntu14.04lts-64bit-fpic-withps.cfg "$all_benchmarks"
PSOPTS="$zipr_flags $trace_flags $mg_flags_all $mg_aggressive_flags" run_test mg-aggressive-trace $SPEC/config/ubuntu14.04lts-64bit-withps.cfg "$all_benchmarks"
PSOPTS="$zipr_flags $trace_flags $mg_flags_all $mg_aggressive_flags" run_test mg-aggressive-trace-fpic $SPEC/config/ubuntu14.04lts-64bit-fpic-withps.cfg "$all_benchmarks"
PSOPTS="$zipr_flags $trace_flags $mg_flags_elftables" run_test mgelftables-trace $SPEC/config/ubuntu14.04lts-64bit-withps.cfg "$all_benchmarks"
get_raw_results
# get_raw_results baseline zipr zipr-trace split-no-fix-icalls split-no-fix-icalls-trace zipr-trace-p1 split-trace-p1 split-no-fix-icalls-trace-p1 zipr-trace-p1-rerun
}
main "$@"
/*
* Copyright (c) 2013, 2014 - University of Virginia
*
* This file may be used and modified for non-commercial purposes as long as
* all copyright, permission, and nonwarranty notices are preserved.
* Redistribution is prohibited without prior written consent from the University
* of Virginia.
*
* Please contact the authors for restrictions applying to commercial use.
*
* THIS SOURCE IS PROVIDED "AS IS" AND WITHOUT ANY EXPRESS OR IMPLIED
* WARRANTIES, INCLUDING, WITHOUT LIMITATION, THE IMPLIED WARRANTIES OF
* MERCHANTIBILITY AND FITNESS FOR A PARTICULAR PURPOSE.
*
* Author: University of Virginia
* e-mail: jwd@virginia.com
* URL : http://www.cs.virginia.edu/
*
*/
#include "AnnotationBoundaryGenerator.hpp"
#include <map>
#include <iostream>
using namespace std;
using namespace IRDB_SDK;
//using namespace MEDS_Annotation;
vector<Range> AnnotationBoundaryGenerator::GetBoundaries(IRDB_SDK::Function_t *func)
{
assert(0);
#if 0
vector<Range> ranges;
// std::multimap<VirtualOffset, MEDS_AnnotationBase>
MEDS_Annotations_t annotations = annotParser->getAnnotations();
for(
set<Instruction_t*>::const_iterator it=func->getInstructions().begin();
it!=func->getInstructions().end();
++it
)
{
Instruction_t* instr = *it;
VirtualOffset_t irdb_vo = instr->getAddress()->getVirtualOffset();
if (irdb_vo == 0) continue;
VirtualOffset vo(irdb_vo);
//std::pair<std::multimap<VirtualOffset, MEDS_AnnotationBase>::iterator,std::multimap<VirtualOffset, MEDS_AnnotationBase>::iterator> ret;
std::pair<MEDS_Annotations_t::iterator,MEDS_Annotations_t::iterator> ret;
ret = annotations.equal_range(vo);
MEDS_InstructionCheckAnnotation annotation;
MEDS_InstructionCheckAnnotation* p_annotation;
for (MEDS_Annotations_t::iterator it = ret.first; it != ret.second; ++it)
{
p_annotation=dynamic_cast<MEDS_InstructionCheckAnnotation*>(it->second);
if(p_annotation==NULL)
continue;
annotation = *p_annotation;
if (annotation.isValid() && annotation.isMemset())
{
//cerr<<"Memset annot found"<<endl;
int objectSize = annotation.getObjectSize();
int offset = annotation.getStackOffset();
Range cur;
cur.SetOffset(offset);
cur.SetSize(objectSize);
if (annotation.isEbpOffset())
{
if(offset < 0)
{
ranges.push_back(cur);
}
} else if (annotation.isEspOffset())
{
if(offset >= 0)
{
ranges.push_back(cur);
}
} else
{
// something went wrong
assert(0);
}
}
}
}
return ranges;
#endif
}
/*
* Copyright (c) 2013, 2014 - University of Virginia
*
* This file may be used and modified for non-commercial purposes as long as
* all copyright, permission, and nonwarranty notices are preserved.
* Redistribution is prohibited without prior written consent from the University
* of Virginia.
*
* Please contact the authors for restrictions applying to commercial use.
*
* THIS SOURCE IS PROVIDED "AS IS" AND WITHOUT ANY EXPRESS OR IMPLIED
* WARRANTIES, INCLUDING, WITHOUT LIMITATION, THE IMPLIED WARRANTIES OF
* MERCHANTIBILITY AND FITNESS FOR A PARTICULAR PURPOSE.
*
* Author: University of Virginia
* e-mail: jwd@virginia.com
* URL : http://www.cs.virginia.edu/
*
*/
#ifndef __ANNOTBOUNDGEN
#define __ANNOTBOUNDGEN
#include "PrecedenceBoundaryGenerator.hpp"
//#include "MEDS_AnnotationParser.hpp"
//#include "MEDS_AnnotationBase.hpp"
//#include "MEDS_InstructionCheckAnnotation.hpp"
// #include "VirtualOffset.hpp"
#include <fstream>
class AnnotationBoundaryGenerator : public PrecedenceBoundaryGenerator
{
protected:
// MEDS_Annotation::MEDS_AnnotationParser *annotParser;
public:
// AnnotationBoundaryGenerator(MEDS_Annotation::MEDS_AnnotationParser *annotParser) : annotParser(annotParser){}
virtual std::vector<Range> GetBoundaries(IRDB_SDK::Function_t *func);
};
#endif
/*
* Copyright (c) 2013, 2014 - University of Virginia
*
* This file may be used and modified for non-commercial purposes as long as
* all copyright, permission, and nonwarranty notices are preserved.
* Redistribution is prohibited without prior written consent from the University
* of Virginia.
*
* Please contact the authors for restrictions applying to commercial use.
*
* THIS SOURCE IS PROVIDED "AS IS" AND WITHOUT ANY EXPRESS OR IMPLIED
* WARRANTIES, INCLUDING, WITHOUT LIMITATION, THE IMPLIED WARRANTIES OF
* MERCHANTIBILITY AND FITNESS FOR A PARTICULAR PURPOSE.
*
* Author: University of Virginia
* e-mail: jwd@virginia.com
* URL : http://www.cs.virginia.edu/
*
*/
#include "DirectOffsetInference.hpp"
using namespace std;
using namespace IRDB_SDK;
DirectOffsetInference::DirectOffsetInference(OffsetInference *offset_inference)
{
//TODO: throw exception
assert(offset_inference != NULL);
this->offset_inference = offset_inference;
}
PNStackLayout* DirectOffsetInference::GetPNStackLayout(Function_t *func)
{
return offset_inference->GetDirectAccessLayout(func);
}
std::string DirectOffsetInference::GetInferenceName() const
{
return "Direct Offset Inference";
}
/*
* Copyright (c) 2013, 2014 - University of Virginia
*
* This file may be used and modified for non-commercial purposes as long as
* all copyright, permission, and nonwarranty notices are preserved.
* Redistribution is prohibited without prior written consent from the University
* of Virginia.
*
* Please contact the authors for restrictions applying to commercial use.
*
* THIS SOURCE IS PROVIDED "AS IS" AND WITHOUT ANY EXPRESS OR IMPLIED
* WARRANTIES, INCLUDING, WITHOUT LIMITATION, THE IMPLIED WARRANTIES OF
* MERCHANTIBILITY AND FITNESS FOR A PARTICULAR PURPOSE.
*
* Author: University of Virginia
* e-mail: jwd@virginia.com
* URL : http://www.cs.virginia.edu/
*
*/
#ifndef __DIRECTOFFSETINFERENCE
#define __DIRECTOFFSETINFERENCE
#include "OffsetInference.hpp"
#include "PNStackLayoutInference.hpp"
class DirectOffsetInference : public PNStackLayoutInference
{
protected:
OffsetInference *offset_inference;
public:
DirectOffsetInference(OffsetInference *offset_inference);
virtual PNStackLayout* GetPNStackLayout(IRDB_SDK::Function_t *func);
virtual std::string GetInferenceName() const;
};
#endif
#include "EhUpdater.hpp"
#include <string>
#include <map>
using namespace std;
using namespace IRDB_SDK;
extern map<Function_t*, set<Instruction_t*> > inserted_instr; //used to undo inserted instructions
// see https://en.wikipedia.org/wiki/LEB128
static bool read_uleb128
( uint64_t &result,
uint32_t& position,
const uint8_t* const data,
const uint32_t max)
{
result = 0;
auto shift = 0;
while( position < max )
{
auto byte = data[position];
position++;
result |= ( ( byte & 0x7f ) << shift);
if ( ( byte & 0x80) == 0)
break;
shift += 7;
}
return ( position >= max );
}
static string to_uleb128 (uint64_t value)
{
auto output_str=string("");
do
{
auto byte = value&0x7f; // low order 7 bits of value;
value >>= 7;
if (value != 0) // more bytes to come
byte |= 0x80; // set high order bit of byte;
output_str.push_back(byte);
} while (value != 0);
return output_str;
}
/* transform any eh handling info for the FDE program*/
bool EhUpdater_t::update_program(EhProgram_t* ehpgm)
{
assert(ehpgm);
const auto daf=ehpgm->getDataAlignmentFactor();
const auto saved_reg_size=m_layout->GetSavedRegsSize();
const auto orig_frame_size=m_layout->GetOriginalAllocSize();
const auto altered_frame_size=m_layout->GetAlteredAllocSize();
/* change the offset, as needed, in a dwarf instruction. the offset is at location 'pos' */
const auto change_offset=[&](string &dwarf_insn, const uint32_t offset_pos, const bool factored) -> void
{
/* handle */
/* format: [(char)(opcode+reg#)] [(uleb128) offset/data_alignment_factor] */
/* we need to adjust factored offset if it's greater than the saved reg size */
auto factored_offset=uint64_t(0);
auto pos_to_read=(uint32_t)1;
const auto data=reinterpret_cast<const uint8_t*>(dwarf_insn.data());
const auto res=read_uleb128(factored_offset,pos_to_read,data, dwarf_insn.size());
assert(res);
auto offset=factored_offset;
if(factored)
offset*=daf;
if(offset>saved_reg_size)
{
const auto new_offset=offset+(altered_frame_size-orig_frame_size);
auto factored_new_offset=new_offset;
if(factored)
factored_new_offset/=daf;
const auto encoded_factored_new_offset=to_uleb128(factored_new_offset);
auto new_dwarf_insn=string("");
for(auto i=0U;i<offset_pos;i++)
new_dwarf_insn.push_back(dwarf_insn[i]);
new_dwarf_insn+=encoded_factored_new_offset;
dwarf_insn=new_dwarf_insn;
}
};
auto tmppgm = ehpgm->getFDEProgram();
for(auto &dwarf_insn : tmppgm)
{
auto opcode=dwarf_insn[0];
auto opcode_upper2=(uint8_t)(opcode >> 6);
auto opcode_lower6=(uint8_t)(opcode & (0x3f));
switch(opcode_upper2)
{
/* case DW_CFA_offset: */
/* reg should be restored from CFA+(offset*daf) */
case 0x2: /* DW_CFA_offset: */
{
change_offset(dwarf_insn,1, true);
break;
};
case 0:
{
switch(opcode_lower6)
{
/* sanitize */
case 0xd: /* DW_CFA_def_cfa_register: */
{
/* [ (char)opcode ] [ (uleb)register ] */
/* assert if register != sp (if bp not used) or bp (if bp is used) */
/* never update this insn */
assert(0);
}
/* handle */
case 0xe: /* DW_CFA_def_cfa_offset: */
{
/* [(char)opcode] [(uleb)offset] */
/* if offset > saved reg size, new_offset=offset+(new_frame_size-old_frame_size) */
change_offset(dwarf_insn,1,false);
break;
}
case 0x11: /*DW_CFA_def_cfa_offset_sf: */
{
/* [(char)opcode] [(sleb)offset/data_alignment_factor] */
/* if offset > saved reg size, new_offset=offset+(new_frame_size-old_frame_size) */
assert(0);
}
case 0x5: /*DW_CFA_offset_extended: */
{
/* [ (char)opcode ] [ (uleb)reg # ] [ (uleb) offset] */
/* we need to adjust factored offset if it's greater than the saved reg size */
change_offset(dwarf_insn,2,true);
break;
}
default:
break;
}
}
default:
break;
}
}
ehpgm->setFDEProgram(tmppgm);
return true;
}
/* transform any eh handling info for the instruction */
bool EhUpdater_t::update_instructions(Instruction_t* insn)
{
const auto ehpgm=insn->getEhProgram();
/* no program == no update */
if(ehpgm==NULL)
return true;
const auto new_eh_pgm=m_firp->copyEhProgram(*ehpgm);
// const auto new_eh_pgm=new EhProgram_t(*ehpgm);
// m_firp->GetAllEhPrograms().insert(new_eh_pgm);
insn->setEhProgram(new_eh_pgm);
return update_program(new_eh_pgm);
}
/* transform any eh handling info for each instruction */
bool EhUpdater_t::update_instructions()
{
// for all instructions in the fuunction.
for(const auto& i: m_func->getInstructions())
update_instructions(i);
// plus any instructions we allocated for the function
// inserted instrucctions are now part of m_func->getInstructions() -- were we updating this frame twice?
// for(const auto& i: inserted_instr[m_func])
// update_instructions(i);
return true;
}
/* transform any eh handling info for the function */
bool EhUpdater_t::execute()
{
if(m_func->getUseFramePointer()) /* no updates needed if a frame pointer is used */
return true;
/* can only update for p1 functions */
if( ! m_layout->IsP1() )
return false;
return update_instructions();
}
#ifndef EhUpdater
#define EhUpdater
#include <irdb-core>
#include "PNStackLayout.hpp"
class EhUpdater_t
{
public:
EhUpdater_t(IRDB_SDK::FileIR_t* p_firp, IRDB_SDK::Function_t* p_func, PNStackLayout* p_layout)
:
m_firp(p_firp),
m_func(p_func),
m_layout(p_layout)
{
}
bool execute();
private:
bool update_instructions();
bool update_instructions(IRDB_SDK::Instruction_t* insn);
bool update_program(IRDB_SDK::EhProgram_t* ehpgm);
IRDB_SDK::FileIR_t* m_firp;
IRDB_SDK::Function_t* m_func;
PNStackLayout* m_layout;
};
#endif
/*
* Copyright (c) 2013, 2014 - University of Virginia
*
* This file may be used and modified for non-commercial purposes as long as
* all copyright, permission, and nonwarranty notices are preserved.
* Redistribution is prohibited without prior written consent from the University
* of Virginia.
*
* Please contact the authors for restrictions applying to commercial use.
*
* THIS SOURCE IS PROVIDED "AS IS" AND WITHOUT ANY EXPRESS OR IMPLIED
* WARRANTIES, INCLUDING, WITHOUT LIMITATION, THE IMPLIED WARRANTIES OF
* MERCHANTIBILITY AND FITNESS FOR A PARTICULAR PURPOSE.
*
* Author: University of Virginia
* e-mail: jwd@virginia.com
* URL : http://www.cs.virginia.edu/
*
*/
#include "General_Utility.hpp"
#include <limits.h>
#include <cstdlib>
#include <cerrno>
using namespace std;
STR2NUM_ERROR str2int (int &i, char const *s, int base)
{
char *end;
long l;
errno = 0;
l = strtol(s, &end, base);
if ((errno == ERANGE && l == LONG_MAX) || l > INT_MAX) {
return STR2_OVERFLOW;
}
if ((errno == ERANGE && l == LONG_MIN) || l < INT_MIN) {
return STR2_UNDERFLOW;
}
if (*s == '\0' || *end != '\0') {
return STR2_INCONVERTIBLE;
}
i = l;
return STR2_SUCCESS;
}
//TODO: what if the string represents a negative number? Currently
//the number will be translated into an unsigned int. I could make this
//and incovertible situation.
STR2NUM_ERROR str2uint (unsigned int &i, char const *s, int base)
{
char *end;
unsigned long l;
errno = 0;
l = strtoul(s, &end, base);
if ((errno == ERANGE && l == ULONG_MAX) || l > UINT_MAX) {
return STR2_OVERFLOW;
}
if (*s == '\0' || *end != '\0') {
return STR2_INCONVERTIBLE;
}
i = l;
return STR2_SUCCESS;
}
/*
* Copyright (c) 2013, 2014 - University of Virginia
*
* This file may be used and modified for non-commercial purposes as long as
* all copyright, permission, and nonwarranty notices are preserved.
* Redistribution is prohibited without prior written consent from the University
* of Virginia.
*
* Please contact the authors for restrictions applying to commercial use.
*
* THIS SOURCE IS PROVIDED "AS IS" AND WITHOUT ANY EXPRESS OR IMPLIED
* WARRANTIES, INCLUDING, WITHOUT LIMITATION, THE IMPLIED WARRANTIES OF
* MERCHANTIBILITY AND FITNESS FOR A PARTICULAR PURPOSE.
*
* Author: University of Virginia
* e-mail: jwd@virginia.com
* URL : http://www.cs.virginia.edu/
*
*/
#ifndef _GENERAL_UTILITY
#define _GENERAL_UTILITY
enum STR2NUM_ERROR { STR2_SUCCESS, STR2_OVERFLOW, STR2_UNDERFLOW, STR2_INCONVERTIBLE };
STR2NUM_ERROR str2int (int &i, char const *s, int base = 0);
STR2NUM_ERROR str2uint (unsigned int &i, char const *s, int base = 0);
#endif
This diff is collapsed.
/*
* Copyright (c) 2013, 2014 - University of Virginia
*
* This file may be used and modified for non-commercial purposes as long as
* all copyright, permission, and nonwarranty notices are preserved.
* Redistribution is prohibited without prior written consent from the University
* of Virginia.
*
* Please contact the authors for restrictions applying to commercial use.
*
* THIS SOURCE IS PROVIDED "AS IS" AND WITHOUT ANY EXPRESS OR IMPLIED
* WARRANTIES, INCLUDING, WITHOUT LIMITATION, THE IMPLIED WARRANTIES OF
* MERCHANTIBILITY AND FITNESS FOR A PARTICULAR PURPOSE.
*
* Author: University of Virginia
* e-mail: jwd@virginia.com
* URL : http://www.cs.virginia.edu/
*
*/
#ifndef __OFFSETSTACKLAYOUTINFERENCE
#define __OFFSETSTACKLAYOUTINFERENCE
#include "PNStackLayoutInference.hpp"
#include "PNRegularExpressions.hpp"
#include <map>
#include <string>
class OffsetInference : public PNStackLayoutInference
{
protected:
std::map<IRDB_SDK::Instruction_t*, bool> in_prologue;
std::map<IRDB_SDK::Function_t*,PNStackLayout*> direct;
std::map<IRDB_SDK::Function_t*,PNStackLayout*> scaled;
std::map<IRDB_SDK::Function_t*,PNStackLayout*> all_offsets;
std::map<IRDB_SDK::Function_t*,PNStackLayout*> p1;
PNRegularExpressions *pn_regex;
virtual void FindAllOffsets(IRDB_SDK::Function_t *func);
virtual PNStackLayout* GetLayout(std::map<IRDB_SDK::Function_t*,PNStackLayout*> &mymap, IRDB_SDK::Function_t *func);
// virtual void getInstructions(std::vector<IRDB_SDK::Instruction_t*> &instructions,IRDB_SDK::libIRDB::BasicBlock_t *block,std::set<IRDB_SDK::libIRDB::BasicBlock_t*> &block_set);
virtual StackLayout* SetupLayout(IRDB_SDK::Function_t *func);
public:
OffsetInference() : pn_regex(NULL) {}
virtual ~OffsetInference();
virtual PNStackLayout* GetPNStackLayout(IRDB_SDK::Function_t *func);
virtual PNStackLayout* GetDirectAccessLayout(IRDB_SDK::Function_t *func);
virtual PNStackLayout* GetScaledAccessLayout(IRDB_SDK::Function_t *func);
virtual PNStackLayout* GetP1AccessLayout(IRDB_SDK::Function_t *func);
virtual std::string GetInferenceName() const;
};
#endif