diff --git a/build-scripts/unpack-tarballs b/build-scripts/unpack-tarballs index 9130c973b..01f67afb8 100755 --- a/build-scripts/unpack-tarballs +++ b/build-scripts/unpack-tarballs @@ -62,9 +62,11 @@ cd "$BASEDIR" gzip -dc $SOURCE_TARBALL | tar -xf - ln -s cfengine-3* core -log_debug "UNPACKING MASTERFILES TARBALL AND SYMLINKING masterfiles/" +log_debug "UNPACKING MASTERFILES TARBALL: $MASTERFILES_TARBALL AND SYMLINKING masterfiles/" # shellcheck disable=SC2086 # > Double quote to prevent globbing and word splitting. # We want globbing here gzip -dc $MASTERFILES_TARBALL | tar -xf - -ln -s cfengine-masterfiles-* masterfiles +# the masterfiles tarball would match cfengine-masterfiles-* so find the unpacked dir specifically +_cfmpfdir=$(find -type d -name 'cfengine-masterfiles-*') +ln -s "$_cfmpfdir" masterfiles diff --git a/ci/initialize-build-host.sh b/ci/initialize-build-host.sh index 90bbc9392..343e2cfd5 100644 --- a/ci/initialize-build-host.sh +++ b/ci/initialize-build-host.sh @@ -136,9 +136,10 @@ do sleep 10 done -echo '========================================= PRINTING CLOUD-INIT LOG ===================================================' -sudo sed 's/^.*/>>> &/' /var/log/cloud-init-output.log || true -echo '======================================= DONE PRINTING CLOUD-INIT LOG ================================================' +# TODO, instead of printing this out ALWAYS, print it out only in case of errors +#echo '========================================= PRINTING CLOUD-INIT LOG ===================================================' +#sudo sed 's/^.*/>>> &/' /var/log/cloud-init-output.log || true +#echo '======================================= DONE PRINTING CLOUD-INIT LOG ================================================' if [ $attempts -le 0 ] then @@ -147,9 +148,10 @@ then exit 1 fi -echo '=========================================== CURRENT ENVIRONMENT =====================================================' -export -echo '========================================= CURRENT ENVIRONMENT END ===================================================' +# TODO only print current environment on errors, maybe save the environment NOW and then show a diff at ERROR +#echo '=========================================== CURRENT ENVIRONMENT =====================================================' +#export +#echo '========================================= CURRENT ENVIRONMENT END ===================================================' # Disable TTY requirement. This normally happens in initialize-user-data.sh, but # for hosts that do not support cloud user data, it may not have happened diff --git a/ci/setup-cfengine-build-host.sh b/ci/setup-cfengine-build-host.sh index cac131fde..e6adec97b 100755 --- a/ci/setup-cfengine-build-host.sh +++ b/ci/setup-cfengine-build-host.sh @@ -26,6 +26,8 @@ rm -rf cfengine-masterfiles* function cleanup() { set -e + [ -f /var/log/messages ] && tail /var/log/messages + command -v journalctl >/dev/null && journalctl --lines=20 if command -v apt >/dev/null 2>&1; then # workaround for CFE-4544, remove scriptlets call systemctl even when systemctl is-system-running returns false # Replace systemctl with a no-op stub that always succeeds. We can't @@ -95,9 +97,12 @@ if [ -f /etc/os-release ]; then yum update --assumeyes alias software='yum install --assumeyes' elif grep -q debian /etc/os-release; then + DEBIAN_FRONTEND=noninteractive apt update + # sometimes the /boot partition is too small to handle kernel upgrade regenerations of initrd and related files on ubuntu, so allow failure first DEBIAN_FRONTEND=noninteractive apt upgrade --yes || true DEBIAN_FRONTEND=noninteractive apt autoremove --yes + # and now perform the upgrade a second time after hopefully autoremove cleans up /boot partition of kernel files that cause failure DEBIAN_FRONTEND=noninteractive apt upgrade --yes DEBIAN_FRONTEND=noninteractive apt autoremove --yes @@ -145,6 +150,20 @@ if [ -f /etc/cfengine-bootstrap-pr-host.flag ]; then exit fi +if [ -f /etc/cfengine-containers-host.flag ]; then + "$thisdir"/setup-ci-host.sh + exit +fi + +# platforms too old to support cf-remote, use scripts instead +if [ -f /etc/os-release ]; then + source /etc/os-release + if [ "$ID" = "centos" ] && [ "$VERSION_ID" = "7" ]; then + "$thisdir"/setup-ci-host.sh + exit + fi +fi + if grep -q ubuntu /etc/os-release; then if grep -qi version=\"16 /etc/os-release; then urlget https://cfengine-package-repos.s3.amazonaws.com/enterprise/Enterprise-3.21.8/agent/agent_ubuntu16_x86_64/cfengine-nova_3.21.8-1.ubuntu16_amd64.deb diff --git a/ci/setup-ci-host.sh b/ci/setup-ci-host.sh new file mode 100755 index 000000000..d3478569c --- /dev/null +++ b/ci/setup-ci-host.sh @@ -0,0 +1,208 @@ +#!/usr/bin/env bash +set -e +shopt -s expand_aliases +thisdir="$(dirname "$0")" + +packages="" # a space separated list of packages to install +function add-pkg() +{ + packages+=" $*" +} + +# we setup some vars for platform versions to make it easier to make choice later +# default version is 0 so that a check can be [ "$debian" -gt "12" ] and that will skip non-debians and such +redhat=0 +debian=0 +ubuntu=0 +suse=0 +# shellcheck disable=SC2034 +solaris=0 +# shellcheck disable=SC2034 +hpux=0 +# shellcheck disable=SC2034 +aix=0 + +if [ -f /etc/os-release ]; then + source /etc/os-release + if grep -q rhel /etc/os-release; then + yum update --assumeyes + alias packages='yum install --assumeyes' + redhat="$VERSION_ID" + elif grep -q debian /etc/os-release; then + alias packages='DEBIAN_FRONTEND=noninteractive apt install --yes' + debian="$VERSION_ID" + elif grep -q suse /etc/os-release; then + alias packages='zypper install -y' + # shellcheck disable=SC2034 + suse="$VERSION_ID" + else + echo "Unknown platform ID $ID. Need this information in order to update/upgrade distribution packages." + exit 1 + fi +elif [ -f /etc/redhat-release ]; then + alias packages='yum install --assumeyes' + # shellcheck disable=SC1091 + source /etc/redhat-release + redhat="$VERSION_ID" +else + echo "No /etc/os-release or /etc/redhat-release so cant determine platform." + exit 1 +fi + +if [ -f /etc/cfengine-containers-host.flag ]; then + if [ "$debian" -ge "12" ]; then + # in jenkins, CONTAINER labeled nodes are capable of running container builds like valgrind-check and static-check + add-pkg unzip # linux-install-groovy.sh needs unzip to unpack the groovy distribution archive + add-pkg buildah + add-pkg jq + add-pkg make + add-pkg parallel + add-pkg podman + if ! command -v groovy; then + bash "$thisdir"/linux-install-groovy.sh + fi + + # NOPASSWD is needed for various tools related to container jobs + rm -rf /etc/sudoers.d/999-local + cat >/etc/sudoers.d/999-local </dev/null; then + if [ ! -f /etc/cfengine-containers-host.flag ]; then + sysctl kernel.core_pattern='|/lib/systemd/systemd-coredump %p %u %g %s %t %e' + fi +fi + +"$thisdir"/linux-install-jdk.sh # the script should skip if sufficient java is already installed + +# leech2 build toolchain host +if [ "$ubuntu" -ge 20 ] || [ "$debian" -ge 12 ] || [ "$redhat" -ge 7 ]; then + "$thisdir"/linux-install-protobuf.sh + # TODO if mingw then pass along x86_64-pc-windows-gnu as an arg to install rust + "$thisdir"/linux-install-rust.sh +fi + +if [ "$redhat" -ge 7 ]; then + yum groups install -y 'Development Tools' +fi + +if [ "$redhat" = "8" ]; then + sudo rpm -iv https://kojipkgs.fedoraproject.org//packages/fakeroot/1.23/1.fc29/x86_64/fakeroot-1.23-1.fc29.x86_64.rpm https://kojipkgs.fedoraproject.org//packages/fakeroot/1.23/1.fc29/x86_64/fakeroot-libs-1.23-1.fc29.x86_64.rpm +fi + +if [ "$redhat" -gt 7 ]; then + yum install --assumeyes https://dl.fedoraproject.org/pub/epel/epel-release-latest-"$redhat".noarch.rpm +fi