diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml new file mode 100644 index 0000000000000000000000000000000000000000..7a49608fdaaece95fc42d9666a588f3dd352bde8 --- /dev/null +++ b/.gitlab-ci.yml @@ -0,0 +1,132 @@ +before_script: + - "source ~gitlab-runner/cicd_support/cicd_support.shinc" + +after_script: + - "source ~gitlab-runner/cicd_support/cicd_support.shinc" + +stages: + - clean + - build + - test + + +# +# Cleaning +# +.do-nightly-clean: &do-nightly-clean + stage: clean + script: + - ./cicd_testing/do-clean.sh + +do-nightly-clean-ubuntu18: + <<: *do-nightly-clean + tags: + - ubuntu18 + +do-nightly-clean-ubuntu16: + <<: *do-nightly-clean + tags: + - ubuntu16 + +do-nightly-clean-centos75: + <<: *do-nightly-clean + tags: + - centos75 + +# +# Building +# + +.do-build: &do-build + stage: build + script: + - ./cicd_testing/do-build.sh + +do-build-ubuntu18: + <<: *do-build + tags: + - ubuntu18 + +do-build-ubuntu16: + <<: *do-build + tags: + - ubuntu16 + +do-build-centos75: + <<: *do-build + tags: + - centos75 + +# +# $PSZ ls +# + +.xform-ls: &xform-ls + stage: test + script: + - ./cicd_testing/xform-ls.sh + + +xform-ls-ubuntu18: + <<: *xform-ls + tags: + - ubuntu18 + +xform-ls-ubuntu16: + <<: *xform-ls + tags: + - ubuntu16 + +xform-ls-centos75: + <<: *xform-ls + tags: + - centos75 + + + +# +# test_cmds.sh rida +# +.basic-pgms-rida: &basic-pgms-rida + stage: test + script: + - ./cicd_testing/basic-pgms-rida.sh + +basic-pgms-rida-ubuntu18: + <<: *basic-pgms-rida + tags: + - ubuntu18 + +basic-pgms-rida-ubuntu16: + <<: *basic-pgms-rida + tags: + - ubuntu16 + +basic-pgms-rida-centos75: + <<: *basic-pgms-rida + tags: + - centos75 + +# +# test_cmds.sh rida-p1 +# +.basic-pgms-rida-p1: &basic-pgms-rida-p1 + stage: test + script: + - ./cicd_testing/basic-pgms-rida-p1.sh + +basic-pgms-rida-p1-ubuntu18: + <<: *basic-pgms-rida-p1 + tags: + - ubuntu18 + +basic-pgms-rida-p1-ubuntu16: + <<: *basic-pgms-rida-p1 + tags: + - ubuntu16 + +basic-pgms-rida-p1-centos75: + <<: *basic-pgms-rida-p1 + tags: + - centos75 + diff --git a/SMPStaticAnalyzer b/SMPStaticAnalyzer index e22452d15f76d96049d278d70df5e669ce337f99..e86d0b03a4f8d4e3f4726f6cdccc440040eaa879 160000 --- a/SMPStaticAnalyzer +++ b/SMPStaticAnalyzer @@ -1 +1 @@ -Subproject commit e22452d15f76d96049d278d70df5e669ce337f99 +Subproject commit e86d0b03a4f8d4e3f4726f6cdccc440040eaa879 diff --git a/build-all.sh b/build-all.sh index 1f50eee9495340d27040b4823feefd0493250460..ffa2e82545b29ac728618c41901b027483ccd6c9 100755 --- a/build-all.sh +++ b/build-all.sh @@ -39,7 +39,7 @@ use_strata=0 if [[ $use_strata = 1 ]]; then # stratafier cd $PEASOUP_UMBRELLA_DIR/stratafier - make || exit + make || exit 1 # strata if [ ! "$STRATA_HOME" ]; then @@ -60,20 +60,20 @@ if [[ $use_strata = 1 ]]; then fi cd $STRATA_HOME32 - STRATA_HOME=$STRATA_HOME32 STRATA=$STRATA_HOME32 ./build -host=i386-linux || exit + STRATA_HOME=$STRATA_HOME32 STRATA=$STRATA_HOME32 ./build -host=i386-linux || exit 1 # build x86-64 strata cd $STRATA_HOME if [ -f Makefile -a Makefile -nt configure -a Makefile -nt Makefile.in ]; then echo Skipping Strata reconfigure step else - ./configure $cfar_mode || exit + ./configure $cfar_mode || exit 1 fi - make || exit + make || exit 1 else cd $STRATA_HOME - ./build $cfar_mode || exit + ./build $cfar_mode || exit 1 fi fi @@ -90,56 +90,56 @@ if [ ! "$SECURITY_TRANSFORMS_HOME" ]; then fi cd $SECURITY_TRANSFORMS_HOME -scons $SCONSDEBUG -j 3 || exit +scons $SCONSDEBUG -j 3 || exit 1 cd $SMPSA_HOME -scons $SCONSDEBUG -j 3 || exit +scons $SCONSDEBUG -j 3 || exit 1 cd $PEASOUP_HOME -make || exit +make || exit 1 if [ -d $ZIPR_CALLBACKS ]; then cd $ZIPR_CALLBACKS ./configure --enable-p1 --prefix=$ZIPR_INSTALL - make || exit - make install || exit + make || exit 1 + make install || exit 1 fi if [ -d $ZIPR_HOME ]; then cd $ZIPR_HOME - scons $SCONSDEBUG -j 3|| exit + scons $SCONSDEBUG -j 3|| exit 1 fi if [ -d $ZIPR_SCFI_PLUGIN ]; then cd $ZIPR_SCFI_PLUGIN - scons $SCONSDEBUG || exit + scons $SCONSDEBUG || exit 1 fi cd $PEASOUP_UMBRELLA_DIR/zipr_large_only_plugin/ -scons $SCONSDEBUG || exit +scons $SCONSDEBUG || exit 1 if [[ -e $PEASOUP_UMBRELLA_DIR/zipr ]] && [[ -e $PEASOUP_UMBRELLA_DIR/zipr_relax_plugin ]] ; then cd $PEASOUP_UMBRELLA_DIR/zipr_relax_plugin/ - scons $SCONSDEBUG || exit + scons $SCONSDEBUG || exit 1 fi if [[ -e $PEASOUP_UMBRELLA_DIR/zipr ]] && [[ -e $PEASOUP_UMBRELLA_DIR/zipr_trace_plugin ]] ; then cd $PEASOUP_UMBRELLA_DIR/zipr_trace_plugin/ - scons $SCONSDEBUG || exit + scons $SCONSDEBUG || exit 1 fi cd $PEASOUP_UMBRELLA_DIR/zipr_push64_reloc_plugin -scons $SCONSDEBUG || exit +scons $SCONSDEBUG || exit 1 cd $PEASOUP_UMBRELLA_DIR/zipr_unpin_plugin -scons $SCONSDEBUG || exit +scons $SCONSDEBUG || exit 1 cd $IRDB_TRANSFORMS -scons $SCONSDEBUG -j 3 || exit +scons $SCONSDEBUG -j 3 || exit 1 if [[ -d $DAFFY_HOME ]] && [[ $(uname -p) == 'x86_64' ]]; then cd $DAFFY_HOME - ./setup_cfar.sh + ./setup_cfar.sh || exit 1 fi cd $PEASOUP_UMBRELLA_DIR diff --git a/cicd_testing/basic-pgms-rida-p1.sh b/cicd_testing/basic-pgms-rida-p1.sh new file mode 100755 index 0000000000000000000000000000000000000000..9581d204be43941220f505d73ff97bd9a3e23f29 --- /dev/null +++ b/cicd_testing/basic-pgms-rida-p1.sh @@ -0,0 +1,9 @@ +#/bin/bash +set -e +set -x + +cd /tmp/peasoup_test +source set_env_vars +cd $CICD_TO_TEST_DIR/peasoup_examples/tests +cd $PEASOUP_HOME/tests; make clean; ./test_cmds.sh rida_p1 + diff --git a/cicd_testing/basic-pgms-rida.sh b/cicd_testing/basic-pgms-rida.sh new file mode 100755 index 0000000000000000000000000000000000000000..ac6ae5ec3967cd39ff71dce75c1b94b3e97567f9 --- /dev/null +++ b/cicd_testing/basic-pgms-rida.sh @@ -0,0 +1,11 @@ +#!/bin/bash +set -e +set -x + +cd /tmp/peasoup_test +source set_env_vars +cd $CICD_TO_TEST_DIR/peasoup_examples/tests +cd $PEASOUP_HOME/tests +make clean; ./test_cmds.sh orig +make clean; ./test_cmds.sh rida + diff --git a/cicd_testing/do-build.sh b/cicd_testing/do-build.sh new file mode 100755 index 0000000000000000000000000000000000000000..fa5bf13a3c5484d5632e98e9dc1ce3dea715252c --- /dev/null +++ b/cicd_testing/do-build.sh @@ -0,0 +1,21 @@ +#!/bin/bash +set -e +set -x + + +# update submodules +git submodule sync --recursive +git submodule update --recursive --init +# gather info for debugging later, probably not necessary +pwd +hostname +whoami +env|grep CICD + +time rsync -a --exclude='.git' $CICD_TO_TEST_DIR/ /tmp/peasoup_test +cd /tmp/peasoup_test +source set_env_vars +sudo ./get-peasoup-packages.sh all +./build-all.sh +./postgres_setup.sh + diff --git a/cicd_testing/do-clean.sh b/cicd_testing/do-clean.sh new file mode 100755 index 0000000000000000000000000000000000000000..9f9d4fd5016d189f24e361dc46f6f8a632cce1b0 --- /dev/null +++ b/cicd_testing/do-clean.sh @@ -0,0 +1,20 @@ +#!/bin/bash +set -e +set -x + +main() +{ + + if [[ $CICD_NIGHTLY == 1 ]] ; then + # gather info for debugging later, probably not necessary + pwd + hostname + whoami + env|grep CICD + + rm -rf /tmp/peasoup_test + fi +} + +main "$@" + diff --git a/cicd_testing/xform-ls.sh b/cicd_testing/xform-ls.sh new file mode 100755 index 0000000000000000000000000000000000000000..086ee0160722686534debd2832f6ae4f47ea77a0 --- /dev/null +++ b/cicd_testing/xform-ls.sh @@ -0,0 +1,12 @@ +#!/bin/bash +set -e +set -x + +cd /tmp/peasoup_test +source set_env_vars +cd /tmp +rm -rf xxx ped_ls; $PSZ /bin/ls ./xxx -c rida=on -s meds_static=off --tempdir ped_ls || true +if [[ ! -x ./xxx ]]; then cat ped_ls/logs/*; fi +rm -rf ped_ls +./xxx + diff --git a/clean-all.sh b/clean-all.sh index 17ba73ecd3bd451f06ee0fef1590dae91abba0b5..3597650a7e8dabd96c3488b6eefb255989eb692b 100755 --- a/clean-all.sh +++ b/clean-all.sh @@ -72,9 +72,13 @@ if [ -d $ZIPR_HOME ]; then scons -c || exit fi +cd $DAFFY_HOME +./clean_cfar.sh cd $PEASOUP_UMBRELLA_DIR + + # clean up installation if this module is the root of the install. # skip pedi cleanup if we are part of a larger project, as future builds # won't know how to install properly. diff --git a/daffy b/daffy index 4f46d9c1dd4447debbd4bf734a2faaade5a088ee..bb6639839d18db28ff2d11b6e8a4edbf4fbccd82 160000 --- a/daffy +++ b/daffy @@ -1 +1 @@ -Subproject commit 4f46d9c1dd4447debbd4bf734a2faaade5a088ee +Subproject commit bb6639839d18db28ff2d11b6e8a4edbf4fbccd82 diff --git a/get-peasoup-packages.sh b/get-peasoup-packages.sh index c7b99dfb01dc3fe75b8e9393f3d5720b89618fed..b9acaeff3f6878fbdc53c66fbe5d36b3774303b9 100755 --- a/get-peasoup-packages.sh +++ b/get-peasoup-packages.sh @@ -9,24 +9,24 @@ # Needed to build components BASE_PKGS=" scons - dwarfdump bison flex g++ nasm openjdk-6-jre sharutils - subversion - xdotool gcc-multilib g++-multilib autoconf realpath - libelf-dev - libxqilla-dev - libxerces-c-dev - screen - libxml2-dev + apt-libelf-dev + yum-libelf-devel + apt-libxqilla-dev + yum-libxqilla-devel + apt-libxerces-c-dev + yum-libxerces-c-devel + apt-libxml2-dev + yum-libxml2-devel libstdc++6:i386 coreutils makeself" @@ -35,15 +35,19 @@ BASE_PKGS=" # For clients of IRDB CLIENT_IRDB_PKGS=" postgresql-client + yum-postgresql-server + yum-postgresql-contrib pgadmin3 - libpqxx-dev - libmysqlclient-dev + apt-libpqxx-dev + yum-libpqxx-devel scons cmake automake1.9" # For IRDB server -SERVER_IRDB_PKGS="postgresql" +SERVER_IRDB_PKGS=" + postgresql +" ALL_PKGS="$BASE_PKGS $CLIENT_IRDB_PKGS $SERVER_IRDB_PKGS " @@ -52,8 +56,31 @@ install_packs() { for i in $* do - sudo apt-get install $i -y + which apt-get 1> /dev/null 2> /dev/null + if [[ $? == 0 ]]; then + if [[ $i =~ apt-* ]]; then + apters="$apters $(echo $i|sed "s/^apt-//")" + elif [[ $i =~ yum-* ]]; then + echo "Skipping install of $i for platform $(lsb_release -d -s)" + else + apters="$apters $i" + fi + else + if [[ $i =~ apt-* ]]; then + echo "Skipping install of $i for platform $(cat /etc/redhat-release)" + elif [[ $i =~ yum-* ]]; then + yummers="$yummers $(echo $i|sed "s/^yum-//")" + else + yummers="$yummers $i" + fi + fi done + which apt-get 1> /dev/null 2> /dev/null + if [[ $? == 0 ]]; then + sudo apt-get install -y --ignore-missing $apters + else + sudo yum install -y --skip-broken $yummers + fi } args="$@" @@ -91,4 +118,8 @@ cd irdb_transforms sudo ./get-packages.sh cd $orig_dir +cd daffy +sudo ./get-packages.sh +cd $orig_dir + echo Intsalling packages complete. diff --git a/irdb_transforms b/irdb_transforms index 9649375c418936fb5988a8a5eb715c8183973ea1..dde158362259e69be684d9e5a6ad9d215fff053b 160000 --- a/irdb_transforms +++ b/irdb_transforms @@ -1 +1 @@ -Subproject commit 9649375c418936fb5988a8a5eb715c8183973ea1 +Subproject commit dde158362259e69be684d9e5a6ad9d215fff053b diff --git a/peasoup_examples b/peasoup_examples index bea62e3fa07e41ccab4e51a9c1268c5ad521c5dc..c3032016a862840e44c337cf610334841959ddac 160000 --- a/peasoup_examples +++ b/peasoup_examples @@ -1 +1 @@ -Subproject commit bea62e3fa07e41ccab4e51a9c1268c5ad521c5dc +Subproject commit c3032016a862840e44c337cf610334841959ddac diff --git a/pedi b/pedi index 265af4e0849b05150e198d1a129b5c44f56ecf92..d4f06f290475e748c1fd9938c9f309292a1315da 160000 --- a/pedi +++ b/pedi @@ -1 +1 @@ -Subproject commit 265af4e0849b05150e198d1a129b5c44f56ecf92 +Subproject commit d4f06f290475e748c1fd9938c9f309292a1315da diff --git a/security_transforms b/security_transforms index c25b8a1e0f9761e7f01c3346e1cc2670d6571904..64b4c6dd8772e8aa01f637774a3cbfd7af48560c 160000 --- a/security_transforms +++ b/security_transforms @@ -1 +1 @@ -Subproject commit c25b8a1e0f9761e7f01c3346e1cc2670d6571904 +Subproject commit 64b4c6dd8772e8aa01f637774a3cbfd7af48560c diff --git a/zipr b/zipr index 709cd64ca1d3442cc396f14cfb38d2f6b1196351..fe312c083e050dbfc5979f6dce3cdec66e4c666e 160000 --- a/zipr +++ b/zipr @@ -1 +1 @@ -Subproject commit 709cd64ca1d3442cc396f14cfb38d2f6b1196351 +Subproject commit fe312c083e050dbfc5979f6dce3cdec66e4c666e diff --git a/zipr_install b/zipr_install index 4b59b7d6b7a3046860f1e55a7ae1e7590831ee1a..37f4db437647e7ff1dbd7c30fffcbad9d1dfd362 160000 --- a/zipr_install +++ b/zipr_install @@ -1 +1 @@ -Subproject commit 4b59b7d6b7a3046860f1e55a7ae1e7590831ee1a +Subproject commit 37f4db437647e7ff1dbd7c30fffcbad9d1dfd362