Merge branch 'master' into thanos_support

.gitlab-ci.yml

+before_script:
+  - "source ~gitlab-runner/cicd_support/cicd_support.shinc" 
+
+after_script:
+  - "source ~gitlab-runner/cicd_support/cicd_support.shinc" 
+
+stages:
+  - clean
+  - build
+  - test
+
+
+#
+# Cleaning
+#
+.do-nightly-clean: &do-nightly-clean
+  stage: clean
+  script:
+    - ./cicd_testing/do-clean.sh 
+
+do-nightly-clean-ubuntu18:
+  <<: *do-nightly-clean
+  tags:
+    - ubuntu18
+
+do-nightly-clean-ubuntu16:
+  <<: *do-nightly-clean
+  tags:
+    - ubuntu16
+
+do-nightly-clean-centos75:
+  <<: *do-nightly-clean
+  tags:
+    - centos75
+
+#
+# Building
+#
+
+.do-build: &do-build
+  stage: build
+  script:
+    - ./cicd_testing/do-build.sh 
+
+do-build-ubuntu18:
+  <<: *do-build
+  tags:
+    - ubuntu18
+
+do-build-ubuntu16:
+  <<: *do-build
+  tags:
+    - ubuntu16
+
+do-build-centos75:
+  <<: *do-build
+  tags:
+    - centos75
+
+#
+# $PSZ ls
+#
+
+.xform-ls: &xform-ls
+  stage: test
+  script:
+    - ./cicd_testing/xform-ls.sh
+
+
+xform-ls-ubuntu18:
+  <<: *xform-ls
+  tags:
+    - ubuntu18
+
+xform-ls-ubuntu16:
+  <<: *xform-ls
+  tags:
+    - ubuntu16
+
+xform-ls-centos75:
+  <<: *xform-ls
+  tags:
+    - centos75
+
+
+
+#
+# test_cmds.sh rida
+#
+.basic-pgms-rida: &basic-pgms-rida
+  stage: test
+  script:
+    - ./cicd_testing/basic-pgms-rida.sh
+
+basic-pgms-rida-ubuntu18:
+  <<: *basic-pgms-rida
+  tags:
+    - ubuntu18
+
+basic-pgms-rida-ubuntu16:
+  <<: *basic-pgms-rida
+  tags:
+    - ubuntu16
+
+basic-pgms-rida-centos75:
+  <<: *basic-pgms-rida
+  tags:
+    - centos75
+
+#
+# test_cmds.sh rida-p1
+#
+.basic-pgms-rida-p1: &basic-pgms-rida-p1
+  stage: test
+  script:
+    - ./cicd_testing/basic-pgms-rida-p1.sh
+
+basic-pgms-rida-p1-ubuntu18:
+  <<: *basic-pgms-rida-p1
+  tags:
+    - ubuntu18
+
+basic-pgms-rida-p1-ubuntu16:
+  <<: *basic-pgms-rida-p1
+  tags:
+    - ubuntu16
+
+basic-pgms-rida-p1-centos75:
+  <<: *basic-pgms-rida-p1
+  tags:
+    - centos75
+

SMPStaticAnalyzer @ e86d0b03

-Subproject commit e22452d15f76d96049d278d70df5e669ce337f99
+Subproject commit e86d0b03a4f8d4e3f4726f6cdccc440040eaa879

build-all.sh

@@ -39,7 +39,7 @@ use_strata=0
 if [[ $use_strata = 1 ]]; then
 	# stratafier
 	cd $PEASOUP_UMBRELLA_DIR/stratafier
-	make || exit
+	make || exit 1
 
 	# strata
 	if [ ! "$STRATA_HOME" ]; then 
@@ -60,20 +60,20 @@ if [[ $use_strata = 1 ]]; then
 		fi
 		
 		cd $STRATA_HOME32
-		STRATA_HOME=$STRATA_HOME32 STRATA=$STRATA_HOME32 ./build -host=i386-linux || exit
+		STRATA_HOME=$STRATA_HOME32 STRATA=$STRATA_HOME32 ./build -host=i386-linux || exit 1
 
 		# build x86-64 strata
 		cd $STRATA_HOME
 		if [ -f Makefile -a Makefile -nt configure -a Makefile -nt Makefile.in ]; then
 			echo Skipping Strata reconfigure step
 		else
-			./configure $cfar_mode || exit
+			./configure $cfar_mode || exit 1
 		fi
-		make || exit
+		make || exit 1
 
 	else
 		cd $STRATA_HOME
-		./build $cfar_mode || exit
+		./build $cfar_mode || exit 1
 	fi
 fi
 
@@ -90,56 +90,56 @@ if [ ! "$SECURITY_TRANSFORMS_HOME" ]; then
 fi
 
 cd $SECURITY_TRANSFORMS_HOME
-scons $SCONSDEBUG -j 3 || exit
+scons $SCONSDEBUG -j 3 || exit 1
 
 cd $SMPSA_HOME
-scons $SCONSDEBUG -j 3 || exit
+scons $SCONSDEBUG -j 3 || exit 1
 
 cd $PEASOUP_HOME
-make || exit
+make || exit 1
 
 if [ -d $ZIPR_CALLBACKS ]; then 
 	cd $ZIPR_CALLBACKS
 	./configure --enable-p1 --prefix=$ZIPR_INSTALL
-	make  || exit
-	make install || exit
+	make  || exit 1
+	make install || exit 1
 fi
 
 if [ -d $ZIPR_HOME ]; then
 	cd $ZIPR_HOME
-	scons $SCONSDEBUG -j 3|| exit
+	scons $SCONSDEBUG -j 3|| exit 1
 fi
 
 if [ -d $ZIPR_SCFI_PLUGIN ]; then
 	cd $ZIPR_SCFI_PLUGIN
-	scons  $SCONSDEBUG || exit
+	scons  $SCONSDEBUG || exit 1
 fi
 
 cd $PEASOUP_UMBRELLA_DIR/zipr_large_only_plugin/
-scons $SCONSDEBUG || exit
+scons $SCONSDEBUG || exit 1
 
 if [[ -e $PEASOUP_UMBRELLA_DIR/zipr ]] && [[ -e $PEASOUP_UMBRELLA_DIR/zipr_relax_plugin ]]  ; then
 	cd $PEASOUP_UMBRELLA_DIR/zipr_relax_plugin/
-	scons $SCONSDEBUG || exit
+	scons $SCONSDEBUG || exit 1
 fi
 
 if [[ -e $PEASOUP_UMBRELLA_DIR/zipr ]] && [[ -e $PEASOUP_UMBRELLA_DIR/zipr_trace_plugin ]]  ; then
 	cd $PEASOUP_UMBRELLA_DIR/zipr_trace_plugin/
-	scons $SCONSDEBUG || exit
+	scons $SCONSDEBUG || exit 1
 fi
 
 cd $PEASOUP_UMBRELLA_DIR/zipr_push64_reloc_plugin
-scons $SCONSDEBUG || exit
+scons $SCONSDEBUG || exit 1
 
 cd $PEASOUP_UMBRELLA_DIR/zipr_unpin_plugin
-scons $SCONSDEBUG || exit
+scons $SCONSDEBUG || exit 1
 
 cd $IRDB_TRANSFORMS
-scons $SCONSDEBUG -j 3 || exit
+scons $SCONSDEBUG -j 3 || exit 1
 
 if [[ -d $DAFFY_HOME ]] && [[ $(uname -p) == 'x86_64' ]]; then
 	cd $DAFFY_HOME
-	./setup_cfar.sh
+	./setup_cfar.sh || exit 1
 fi
 
 cd $PEASOUP_UMBRELLA_DIR

cicd_testing/basic-pgms-rida-p1.sh

+#/bin/bash 
+set -e
+set -x
+
+cd /tmp/peasoup_test
+source set_env_vars
+cd $CICD_TO_TEST_DIR/peasoup_examples/tests
+cd $PEASOUP_HOME/tests; make clean; ./test_cmds.sh rida_p1
+

cicd_testing/basic-pgms-rida.sh

+#!/bin/bash
+set -e
+set -x
+
+cd /tmp/peasoup_test
+source set_env_vars
+cd $CICD_TO_TEST_DIR/peasoup_examples/tests
+cd $PEASOUP_HOME/tests
+make clean; ./test_cmds.sh orig
+make clean; ./test_cmds.sh rida
+

cicd_testing/do-build.sh

+#!/bin/bash
+set -e
+set -x
+
+
+# update submodules
+git submodule sync --recursive
+git submodule update --recursive --init
+# gather info for debugging later, probably not necessary 
+pwd
+hostname
+whoami
+env|grep CICD
+
+time rsync -a --exclude='.git'  $CICD_TO_TEST_DIR/ /tmp/peasoup_test
+cd /tmp/peasoup_test
+source set_env_vars
+sudo ./get-peasoup-packages.sh all
+./build-all.sh 
+./postgres_setup.sh
+

cicd_testing/do-clean.sh

+#!/bin/bash
+set -e
+set -x
+
+main()
+{
+
+        if [[ $CICD_NIGHTLY == 1 ]] ; then
+		# gather info for debugging later, probably not necessary 
+		pwd
+		hostname
+		whoami
+		env|grep CICD
+
+		rm -rf /tmp/peasoup_test
+	fi
+}
+
+main "$@"
+

cicd_testing/xform-ls.sh

+#!/bin/bash
+set -e
+set -x
+
+cd /tmp/peasoup_test
+source set_env_vars
+cd /tmp
+rm -rf xxx ped_ls; $PSZ /bin/ls ./xxx -c rida=on -s meds_static=off --tempdir ped_ls || true
+if [[ ! -x ./xxx ]]; then cat ped_ls/logs/*; fi
+rm -rf ped_ls
+./xxx
+

clean-all.sh

@@ -72,9 +72,13 @@ if [ -d $ZIPR_HOME ]; then
 	scons -c || exit
 fi
 
+cd $DAFFY_HOME
+./clean_cfar.sh
 
 cd $PEASOUP_UMBRELLA_DIR
 
+
+
 # clean up installation if this module is the root of the install.
 # skip pedi cleanup if we are part of a larger project, as future builds
 # won't know how to install properly.  

daffy @ bb663983

-Subproject commit 4f46d9c1dd4447debbd4bf734a2faaade5a088ee
+Subproject commit bb6639839d18db28ff2d11b6e8a4edbf4fbccd82

get-peasoup-packages.sh

@@ -9,24 +9,24 @@
 # Needed to build components
 BASE_PKGS="
   scons
-  dwarfdump
   bison
   flex
   g++
   nasm
   openjdk-6-jre
   sharutils
-  subversion
-  xdotool
   gcc-multilib
   g++-multilib
   autoconf
   realpath
-  libelf-dev
-  libxqilla-dev
-  libxerces-c-dev
-  screen
-  libxml2-dev
+  apt-libelf-dev
+  yum-libelf-devel
+  apt-libxqilla-dev
+  yum-libxqilla-devel
+  apt-libxerces-c-dev
+  yum-libxerces-c-devel
+  apt-libxml2-dev
+  yum-libxml2-devel
   libstdc++6:i386
   coreutils
   makeself"
@@ -35,15 +35,19 @@ BASE_PKGS="
 # For clients of IRDB
 CLIENT_IRDB_PKGS="
   postgresql-client
+  yum-postgresql-server 
+  yum-postgresql-contrib
   pgadmin3
-  libpqxx-dev
-  libmysqlclient-dev
+  apt-libpqxx-dev
+  yum-libpqxx-devel
   scons
   cmake
   automake1.9"
 
 # For IRDB server
-SERVER_IRDB_PKGS="postgresql"
+SERVER_IRDB_PKGS="
+	postgresql
+"
 
 ALL_PKGS="$BASE_PKGS $CLIENT_IRDB_PKGS $SERVER_IRDB_PKGS "
 
@@ -52,8 +56,31 @@ install_packs()
 {
 	for i in $*
 	do
-		sudo apt-get install $i -y
+		which apt-get 1> /dev/null 2> /dev/null 
+		if [[ $? == 0  ]]; then
+			if [[ $i =~ apt-* ]]; then
+				apters="$apters $(echo $i|sed "s/^apt-//")"
+			elif [[ $i =~ yum-* ]]; then
+				echo "Skipping install of $i for platform  $(lsb_release -d -s)"
+			else
+				apters="$apters $i"
+			fi
+		else 
+			if [[ $i =~ apt-* ]]; then
+				echo "Skipping install of $i for platform  $(cat /etc/redhat-release)"
+			elif [[ $i =~ yum-* ]]; then
+				yummers="$yummers $(echo $i|sed "s/^yum-//")"
+			else
+				yummers="$yummers $i"
+			fi
+		fi
 	done
+	which apt-get 1> /dev/null 2> /dev/null 
+	if [[ $? == 0  ]]; then
+		sudo apt-get install -y --ignore-missing $apters
+	else
+		sudo yum install -y --skip-broken $yummers
+	fi
 }
 
 args="$@"
@@ -91,4 +118,8 @@ cd irdb_transforms
 sudo ./get-packages.sh
 cd $orig_dir
 
+cd daffy
+sudo ./get-packages.sh
+cd $orig_dir
+
 echo Intsalling packages complete.

irdb_transforms @ dde15836

-Subproject commit 9649375c418936fb5988a8a5eb715c8183973ea1
+Subproject commit dde158362259e69be684d9e5a6ad9d215fff053b

peasoup_examples @ c3032016

-Subproject commit bea62e3fa07e41ccab4e51a9c1268c5ad521c5dc
+Subproject commit c3032016a862840e44c337cf610334841959ddac

pedi @ d4f06f29

-Subproject commit 265af4e0849b05150e198d1a129b5c44f56ecf92
+Subproject commit d4f06f290475e748c1fd9938c9f309292a1315da

security_transforms @ 64b4c6dd

-Subproject commit c25b8a1e0f9761e7f01c3346e1cc2670d6571904
+Subproject commit 64b4c6dd8772e8aa01f637774a3cbfd7af48560c

zipr @ fe312c08

-Subproject commit 709cd64ca1d3442cc396f14cfb38d2f6b1196351
+Subproject commit fe312c083e050dbfc5979f6dce3cdec66e4c666e

zipr_install @ 37f4db43

-Subproject commit 4b59b7d6b7a3046860f1e55a7ae1e7590831ee1a
+Subproject commit 37f4db437647e7ff1dbd7c30fffcbad9d1dfd362