Merge branch '19-bring-up-to-date' into 'master'

Resolve "bring up to date"

Closes #19

See merge request !3

.gitlab-ci.yml

@@ -11,19 +11,6 @@ stages:
   - deploy
 
 
-#
-# Cleaning
-#
-.do-nightly-clean: &do-nightly-clean
-  stage: clean
-  script:
-    - ./cicd_testing/do-clean.sh 
-
-do-nightly-clean-ubuntu18:
-  <<: *do-nightly-clean
-  tags:
-    - ubuntu18
-
 
 
 #
@@ -33,26 +20,14 @@ do-nightly-clean-ubuntu18:
 .do-build: &do-build
   stage: build
   script:
+    - ./cicd_testing/do-clean.sh 
     - ./cicd_testing/do-build.sh 
-
-do-build-ubuntu18:
-  <<: *do-build
-  tags:
-    - ubuntu18
-
-#
-# testing
-#
-
-.do-test: &do-test
-  stage: test
-  script:
     - ./cicd_testing/do-test.sh 
 
-do-test-ubuntu18:
-  <<: *do-test
+do-build-ubuntu22:
+  <<: *do-build
   tags:
-    - ubuntu18
+    - ubuntu22
 
 #
 # testing docker container, plus weekly deployment
@@ -63,14 +38,14 @@ do-test-ubuntu18:
   script:
     - ./cicd_testing/do-deploy.sh 
 
-do-deploy-ubuntu18:
+do-deploy-ubuntu22:
   <<: *do-deploy
   tags:
-    - ubuntu18
+    - ubuntu22
 
 pages:
   tags:
-  - ubuntu18
+  - ubuntu22
   stage: deploy
   script:
   - ./cicd_testing/do-deploy-web.sh

cicd_testing/do-deploy.sh

@@ -28,7 +28,7 @@ function main()
 	# deploy the docker image
 	bash -x $TURBO_HOME/cicd_testing/turbo_docker/deploy.sh
 
-	docker run -d -p 55155:55155 --name $container_name git.zephyr-software.com:4567/allzp/turbo/turbo:latest
+	docker run -d -p 55155:55155 --name $container_name git.zephyr-software.com:4567/opensrc/turbo/turbo:latest
 
 	# sleep 5 seconds, then wait up to 5 minutes longer for turbo to start up.  avoid just waiting 5 minutes by
 	# using ping to check if server is up

cicd_testing/turbo_docker/Dockerfile

 
-FROM ubuntu:18.04
+FROM ubuntu:22.04
 RUN DEBIAN_FRONTEND=noninteractive apt-get update &&   \
     DEBIAN_FRONTEND=noninteractive apt-get install -yq \
 	postgresql  				\

cicd_testing/turbo_docker/deploy.sh

 #/bin/bash
 
-export TURBO_PATH=git.zephyr-software.com:4567/allzp/turbo/
+export TURBO_PATH=git.zephyr-software.com:4567/opensrc/turbo/
 export TURBO_TAG=turbo:latest
 export DOCKER_TURBO=${TURBO_PATH}${TURBO_TAG}
 

cli/SConscript

@@ -22,7 +22,7 @@ else:
 	env.Append(CXXFLAGS="  -O " )
 
 # setup link options with pkg-config
-pkg_config_str = subprocess.check_output("pkg-config --static --libs protobuf grpc++ grpc", shell=True)
+pkg_config_str = subprocess.check_output("pkg-config --static --libs protobuf grpc++ grpc", shell=True).decode()
 
 #print "pkg_config_str = " + pkg_config_str
 for option in Split(pkg_config_str):
@@ -33,7 +33,7 @@ for option in Split(pkg_config_str):
 	elif option.startswith("-pthread"):
 		env.Append(LINKFLAGS=Split(" "+option+" "))
 	else:
-		print "Unable to process pkg-config string: " + option
+		print("Unable to process pkg-config string: " + option)
 		sys.exit(1)
 
 
@@ -45,6 +45,6 @@ inst=env.Install("../bin", pgm)
 
 ret=[]+inst+pgm
 
-print "cli-ret = ", [str(s) for s in ret]
+print("cli-ret = ", [str(s) for s in ret])
 Default(ret)
 Return('ret')

get-packages.sh

@@ -16,6 +16,7 @@ function do_subdirs
 
 function main()
 {
+	set -x
 
 	local here_dir=$(pwd)
 	local grpc_packs="build-essential autoconf libtool pkg-config"
@@ -29,17 +30,10 @@ function main()
 	if [[ -d $here_dir/third-party/grpc ]]; then
 		cd $here_dir/third-party/grpc
 		if [[ ! -x third_party/protobuf/src/protoc ]] && [[ ! -x /usr/local/bin/protoc ]] ; then
-			if [[ ! -f Makefile ]]; then
-				cmake .
-			fi
+			mkdir cmake_build
+			cd cmake_build
+			cmake -DgRPC_INSTALL=ON -DCMAKE_INSTALL_PREFIX=/usr/local ..
 			make -j 3
-
-			cd $here_dir/third-party/grpc/third_party/protobuf
-			pwd
-			sudo make install
-
-			cd $here_dir/third-party/grpc
-			pwd
 			sudo make install
 		fi
 

prepd/prepd.cpp

@@ -50,7 +50,7 @@ void do_zafl(pqxx::connection &conn, int32_t log_id, const WorkUnit_t& work)
 
 	// start with some logging.
 	prepare();
-	txn->prepared("start_pipe")("Attempting to start zafl.\n")(log_id).exec();
+	txn->exec_prepared("start_pipe","Attempting to start zafl.\n",log_id);
 	txn->commit();
 	txn = make_unique<pqxx::work>(conn);
 
@@ -63,8 +63,8 @@ void do_zafl(pqxx::connection &conn, int32_t log_id, const WorkUnit_t& work)
 	if(file_write != raw.size())
 	{
 		prepare();
-		txn->prepared("update_log")("Couldn't create tmp output file.  Out of disk space?\n")(work.getVersionID()).exec();
-		txn->prepared("mark_err")(work.getVersionID()).exec();
+		txn->exec_prepared("update_log","Couldn't create tmp output file.  Out of disk space?\n",work.getVersionID());
+		txn->exec_prepared("mark_err",work.getVersionID());
 		txn->commit();
 		txn = make_unique<pqxx::work>(conn);
 		return;
@@ -82,8 +82,8 @@ void do_zafl(pqxx::connection &conn, int32_t log_id, const WorkUnit_t& work)
         {
                 cerr << "Couldn't start command." << endl;
 		prepare();
-		txn->prepared("update_log")("Couldn't start pipe.  Out of mem/processes?\n")(log_id).exec();
-		txn->prepared("mark_err")(work.getVersionID()).exec();
+		txn->exec_prepared("update_log","Couldn't start pipe.  Out of mem/processes?\n",log_id);
+		txn->exec_prepared("mark_err",work.getVersionID());
 		txn->commit();
 		txn = make_unique<pqxx::work>(conn);
 		return;
@@ -91,7 +91,7 @@ void do_zafl(pqxx::connection &conn, int32_t log_id, const WorkUnit_t& work)
 
 	// mark that we've started zafl.
 	prepare();
-	txn->prepared("update_log")("Pipe started with cmd: "+zafl_cmd+"\n")(log_id).exec();
+	txn->exec_prepared("update_log","Pipe started with cmd: "+zafl_cmd+"\n",log_id);
 	txn->commit();
 	txn = make_unique<pqxx::work>(conn);
 
@@ -124,7 +124,7 @@ void do_zafl(pqxx::connection &conn, int32_t log_id, const WorkUnit_t& work)
 			
 			// append the new log bytes to the db entry
 			prepare();
-			txn->prepared("update_log")(log_buffer)(log_id).exec();
+			txn->exec_prepared("update_log",log_buffer,log_id);
 			txn->commit();
 			txn = make_unique<pqxx::work>(conn);
 
@@ -139,7 +139,7 @@ void do_zafl(pqxx::connection &conn, int32_t log_id, const WorkUnit_t& work)
 	if(log_buffer!="")
 	{
 		prepare();
-		txn->prepared("update_log")(log_buffer)(log_id).exec();
+		txn->exec_prepared("update_log",log_buffer,log_id);
 		txn->commit();
 		txn = make_unique<pqxx::work>(conn);
 	}
@@ -150,8 +150,8 @@ void do_zafl(pqxx::connection &conn, int32_t log_id, const WorkUnit_t& work)
 	{	
 		// log the failure.
 		prepare();
-		txn->prepared("update_log")("Zafl failed with exit_code="+to_string(returnCode)+"!\n")(log_id).exec();
-		txn->prepared("mark_err"  )(work.getVersionID()).exec();
+		txn->exec_prepared("update_log","Zafl failed with exit_code="+to_string(returnCode)+"!\n",log_id);
+		txn->exec_prepared("mark_err"  ,work.getVersionID());
 		txn->commit();
 		txn = make_unique<pqxx::work>(conn);
 		return;
@@ -160,7 +160,7 @@ void do_zafl(pqxx::connection &conn, int32_t log_id, const WorkUnit_t& work)
 	{
 		// log success.
 		prepare();
-		txn->prepared("update_log")("Zafl seems to have worked!\n")(log_id).exec();
+		txn->exec_prepared("update_log","Zafl seems to have worked!\n",log_id);
 		txn->commit();
 		txn = make_unique<pqxx::work>(conn);
 	}
@@ -171,7 +171,7 @@ void do_zafl(pqxx::connection &conn, int32_t log_id, const WorkUnit_t& work)
 	const auto zafl_contents = ostrm.str();
 	const auto pqxx_zafl_contents = pqxx::binarystring(zafl_contents.c_str(), zafl_contents.size());
 	prepare();
-	txn->prepared("upload_zafl")(pqxx_zafl_contents)(work.getVersionID()).exec(); 
+	txn->exec_prepared("upload_zafl",pqxx_zafl_contents,work.getVersionID());
 	cout << "Zafl'd verison is " << zafl_contents.size() << " bytes " << endl;
 	txn->commit();
 
@@ -188,7 +188,7 @@ void do_work(const WorkUnit_t& work)
 	pqxx::connection conn{db_connect_options.c_str()};
 	pqxx::work txn{conn};
 	conn.prepare ("insert_log", "insert into zafl_logs (ver_id) values ($1) returning log_id");
-	const auto log_ids = txn.prepared("insert_log")(work.getVersionID()).exec();
+	const auto log_ids = txn.exec_prepared("insert_log",work.getVersionID());
 
 	assert(log_ids.size() == 1);
 	const auto log_id = log_ids[0]["log_id"].as<int>();
@@ -206,7 +206,7 @@ WorkUnit_t get_work()
 	pqxx::work txn{conn};
 
 	conn.prepare ("get_newest", "SELECT ver_id,orig_binary FROM ONLY boost_versions where start_time is NULL order by submit_time limit 1 ");
-	const auto check_res = txn.prepared("get_newest").exec();
+	const auto check_res = txn.exec_prepared("get_newest");
 
 	// no work to do.
 	if(check_res.size() == 0)
@@ -218,7 +218,7 @@ WorkUnit_t get_work()
 
 	cout << "Updating starting time " << endl;
 	conn.prepare ("update_start", "update boost_versions set start_time=NOW() where ver_id = $1 ");
-	txn .prepared("update_start")(ver_id).exec();
+	txn .exec_prepared("update_start",ver_id);
 	txn.commit();
 
 	return {ver_id,orig_binary_contents};

public/cli-install.sh

@@ -89,7 +89,7 @@ function install_files()
 			fi 
 		fi
 		echo "Installing $path/$file..."
-		curl --fail -s -o $path/$file https://allzp.pages.zephyr-software.com/turbo/$file 
+		curl --fail -s -o $path/$file https://opensrc.pages.zephyr-software.com/turbo/$file 
 		chmod $perms $path/$file
 		local curl_res=$?
 		if [[ $curl_res -ne 0 ]]; then

public/index.html

 <!DOCTYPE html>
 <html>
   <head>
-    <meta http-equiv="Refresh" content="7; url=//git.zephyr-software.com/allzp/turbo/wikis/home" />
+    <meta http-equiv="Refresh" content="7; url=//git.zephyr-software.com/opensrc/turbo/wikis/home" />
   </head>
   <body>
-    <p>Please follow <a href="//git.zephyr-software.com/allzp/turbo/wikis/home">this link</a>.</p>
+    <p>Please follow <a href="//git.zephyr-software.com/opensrc/turbo/wikis/home">this link</a>.</p>
   </body>
 </html>

grpc @ d2054ec6

-Subproject commit 7ec5b2d0c009fc83ebdfa5b21fad0bdc8581cb60
+Subproject commit d2054ec6c6e8abcecf0e24b0b4ee75035d80c3cc

turbo-rpc/SConscript

@@ -32,7 +32,7 @@ else:
 
 # build proto files
 proto_files = env.protoc( [], "turbo.proto")
-print "proto files = ", [str(s) for s in proto_files]
+print( "proto files = ", [str(s) for s in proto_files])
 
 for file in proto_files:
 	# print "file="+str(file);
@@ -48,7 +48,7 @@ Depends(lib,ret) # the library depends on having protoc headers installed
 ret = ret + env.Install("$TURBO_HOME/lib/", lib)
 
 
-print 'turbo-rpc ret=',  [str(s) for s in ret]
+print( 'turbo-rpc ret=',  [str(s) for s in ret])
 
 Default(ret)
 

turbod/SConscript

@@ -21,7 +21,7 @@ else:
         env.Append(CFLAGS="    -O " )
         env.Append(CXXFLAGS="  -O " )
 
-pkg_config_str = subprocess.check_output("pkg-config --static --libs protobuf grpc++ grpc", shell=True)
+pkg_config_str = subprocess.check_output("pkg-config --static --libs protobuf grpc++ grpc", shell=True).decode()
 
 extra_files = [env.File("libturbo-rpc.a", "../lib/")]
 #print "pkg_config_str = " + pkg_config_str
@@ -35,10 +35,10 @@ for option in Split(pkg_config_str):
 			env.Append(LIBS=Split(" " + option[2:] + " "))
 #		env.Append(LIBS=Split(" " + option[2:] + " "))
 	elif option.startswith("-pthread"):
-		print "adding -pthread: ", option
+		print( "adding -pthread: ", option)
 		env.Append(LINKFLAGS=Split(" "+option+" "))
 	else:
-		print "Unable to process pkg-config string: " + option
+		print( "Unable to process pkg-config string: " + option)
 		sys.exit(1)
 
 
@@ -49,6 +49,6 @@ pgm=env.Program('turbod.exe', ['turbod.cpp']+extra_files)
 inst=env.Install('../bin', pgm);
 ret=[]+inst+pgm
 
-print "turbod-ret = ", [str(s) for s in ret]
+print( "turbod-ret = ", [str(s) for s in ret])
 Default(ret)
 Return('ret')

turbod/turbod.cpp

@@ -46,7 +46,7 @@ class TurboServiceImpl final : public TurboService::Service
 				cout << "Requested creation of new boost named " << request->name() << endl;
 
 				conn.prepare("boostcheck", "select boost_id from boosts where boost_name = $1");
-				const auto check_res =  txn.prepared("boostcheck")(request->name()).exec();
+				const auto check_res =  txn.exec_prepared("boostcheck",request->name());
 
 				if(check_res.size() != 0)
 				{
@@ -54,7 +54,7 @@ class TurboServiceImpl final : public TurboService::Service
 				}
 
 				conn.prepare("boostadd", "insert into boosts (boost_name, create_time) values ($1, NOW()) returning boost_id ");
-				const auto add_res =  txn.prepared("boostadd")(request->name()).exec();
+				const auto add_res =  txn.exec_prepared("boostadd",request->name());
 
 				if(add_res.size() != 1)
 				{
@@ -88,7 +88,7 @@ class TurboServiceImpl final : public TurboService::Service
 							case when err is null then false else err end as err 
 						from boost_versions where ver_id = $1)"""
 						);
-				const auto res_table =  txn.prepared("getzafld")(request->ver_id()).exec();
+				const auto res_table =  txn.exec_prepared("getzafld",request->ver_id());
 
 				if(res_table.size() != 1)
 				{
@@ -118,7 +118,7 @@ class TurboServiceImpl final : public TurboService::Service
 				cout << "Requested boost list " << endl;
 
 				conn.prepare("boostlist", "select boost_id,encode(boost_name,'escape'), to_char(create_time, 'MM-DD-YYYY HH24:MI:SS') as create_time from boosts");
-				const auto res_table =  txn.prepared("boostlist").exec();
+				const auto res_table =  txn.exec_prepared("boostlist");
 
 				if(res_table.size() == 0)
 				{
@@ -164,7 +164,7 @@ class TurboServiceImpl final : public TurboService::Service
 								err 
 								from boost_versions where boost_id = $1)""" 
 						);
-				const auto res_table =  txn.prepared("boostlist")(request->boost_id()).exec();
+				const auto res_table =  txn.exec_prepared("boostlist",request->boost_id());
 
 				if(res_table.size() == 0)
 				{
@@ -207,7 +207,7 @@ class TurboServiceImpl final : public TurboService::Service
 				cout << "Requested creation of new version for boost id " << request->boost_id() << endl;
 
 				conn.prepare("boostcheck", "select boost_id,boost_name from boosts where boost_id = $1");
-				const auto check_res =  txn.prepared("boostcheck")(request->boost_id()).exec();
+				const auto check_res =  txn.exec_prepared("boostcheck",request->boost_id());
 
 				if(check_res.size() != 1)
 				{
@@ -219,7 +219,7 @@ class TurboServiceImpl final : public TurboService::Service
 
 				const auto boost_id    = check_res[0]["boost_id"].as<int>();
 				const auto binary_data = pqxx::binarystring(request->version_contents().c_str(), request->version_contents().size());
-				const auto add_res     = txn.prepared("versionadd")(boost_id)(binary_data).exec();
+				const auto add_res     = txn.exec_prepared("versionadd",boost_id,binary_data);
 
 				if(add_res.size() != 1)
 				{
@@ -251,7 +251,7 @@ class TurboServiceImpl final : public TurboService::Service
 							case when log is null then '' else encode(log,'escape') end as log 
 						from zafl_logs where ver_id = $1)"""
 						);
-				const auto res_table =  txn.prepared("getzafllog")(request->ver_id()).exec();
+				const auto res_table =  txn.exec_prepared("getzafllog",request->ver_id());
 
 				if(res_table.size() != 1)
 				{
@@ -283,7 +283,7 @@ class TurboServiceImpl final : public TurboService::Service
 							case when log is null then '' else encode(log,'escape') end as log 
 						from fuzz_logs where ver_id = $1)"""
 						);
-				const auto res_table =  txn.prepared("getFuzzlog")(request->ver_id()).exec();
+				const auto res_table =  txn.exec_prepared("getFuzzlog",request->ver_id());
 
 				if(res_table.size() != 1)
 				{
@@ -314,7 +314,7 @@ class TurboServiceImpl final : public TurboService::Service
 				// prepare and execute a command to insert an input
 				conn.prepare("input_add", 
 						R"""(insert into inputs ( boost_id, create_time, prov, input_data) values ($1, NOW(), 'User',  $2) returning input_id )""");
-				const auto res_input =  txn.prepared("input_add")(request->boost_id())(request->seed_file()).exec();
+				const auto res_input =  txn.exec_prepared("input_add",request->boost_id(),request->seed_file());
 				if(res_input.size() != 1)
 				{
 					return Status(ABORTED, "Invalid boost id");
@@ -324,7 +324,7 @@ class TurboServiceImpl final : public TurboService::Service
 
 				// prepare and execute a command to mark the input as a seed
 				conn.prepare("seed_add", R"""(insert into seeds ( input_id ) values ($1) returning seed_id )""");
-				const auto res_seed =  txn.prepared("seed_add")(input_id).exec();
+				const auto res_seed =  txn.exec_prepared("seed_add",input_id);
 				if(res_seed.size() != 1)
 				{
 					return Status(ABORTED, "Unexpected error");
@@ -357,7 +357,7 @@ class TurboServiceImpl final : public TurboService::Service
 
 
 				conn.prepare("findboost", "select boost_id from boost_versions where ver_id = $1");
-				const auto boost_id_res =  txn.prepared("findboost")(request->ver_id()).exec();
+				const auto boost_id_res =  txn.exec_prepared("findboost",request->ver_id());
 				if(boost_id_res.size() != 1)
 				{
 					return Status(ABORTED, "Invalid version id");
@@ -367,7 +367,7 @@ class TurboServiceImpl final : public TurboService::Service
 
 				// prepare and execute a command to insert an input
 				conn.prepare("aflinputadd", R"""(insert into inputs ( boost_id, create_time, prov, input_data) values ($1, NOW(), 'AFL',  $2) returning input_id )""");
-				const auto res_input =  txn.prepared("aflinputadd")(boost_id)(request->input_file()).exec();
+				const auto res_input =  txn.exec_prepared("aflinputadd",boost_id,request->input_file());
 				if(res_input.size() != 1)
 				{
 					return Status(ABORTED, "Invalid boost id");
@@ -398,7 +398,7 @@ class TurboServiceImpl final : public TurboService::Service
 				cout << "Requested boost list " << endl;
 
 				conn.prepare("input_download", "select input_id,input_data from inputs where boost_id in (select boost_id from boost_versions where ver_id = $1) ");
-				const auto res_table =  txn.prepared("input_download")(request->ver_id()).exec();
+				const auto res_table =  txn.exec_prepared("input_download",request->ver_id());
 
 				if(res_table.size() == 0)
 				{
@@ -439,7 +439,7 @@ class TurboServiceImpl final : public TurboService::Service
 				cout << "Requested boost list " << endl;
 
 				conn.prepare("input_download", "select input_id,input_data from inputs where input_id in (select input_id from forensics where ver_id = $1) ");
-				const auto res_table =  txn.prepared("input_download")(request->ver_id()).exec();
+				const auto res_table =  txn.exec_prepared("input_download",request->ver_id());
 
 				// OK to have 0 results, just means we haven't found a crash yet. 
 				for(const auto &res : res_table)
@@ -481,7 +481,7 @@ class TurboServiceImpl final : public TurboService::Service
 							insert into fuzz_logs (ver_id,log) select $1,$2 where not exists (select * from upsert)
 						    )""");
 
-				txn.prepared("update_fuzz_log")(request->ver_id())(request->log()).exec();
+				txn.exec_prepared("update_fuzz_log",request->ver_id(),request->log());
 				txn.commit();
 				return Status::OK;
 			}
@@ -505,7 +505,7 @@ class TurboServiceImpl final : public TurboService::Service
 							insert into forensics (ver_id,input_id) values ($1,$2) returning forensic_id
 						    )""");
 
-				const auto res = txn.prepared("addfor")(request->ver_id())(request->input_id()).exec();
+				const auto res = txn.exec_prepared("addfor",request->ver_id(),request->input_id());
 				txn.commit();
 				if(res.size() != 1)
 				{

zafl @ fccbeb34

-Subproject commit 13407ca1cc36bcd32e0a0bbcc93cd7ea60a1a46a
+Subproject commit fccbeb34d46b7c56e2d5a579b14d43b5d636c067

zipr @ 9556989f

-Subproject commit 96d8c0d55efb488e7d042c43b1f3d75f8b8b1f74
+Subproject commit 9556989fc278a0a26725ce418938913a16dfbdb0