diff --git a/build_library/board_options.sh b/build_library/board_options.sh index c454b47bcc6..7c80c447285 100644 --- a/build_library/board_options.sh +++ b/build_library/board_options.sh @@ -31,30 +31,3 @@ pkg_use_enabled() { pkg_version() { portageq-"${BOARD}" best_visible "${BOARD_ROOT}" "$1" "$2" || : } - -# Usage: pkg_provides [installed|binary] some-pkg/name-1.2.3 -# Prints: x86_32: libfoo.so.2 x86_64: libfoo.so.2 -pkg_provides() { - local provides p - provides=$(portageq-"${BOARD}" metadata "${BOARD_ROOT}" "$1" "$2" PROVIDES) - - if [[ -z "$provides" ]]; then - return - fi - - # convert: - # x86_32: libcom_err.so.2 libss.so.2 x86_64: libcom_err.so.2 libss.so.2 - # into: - # x86_32 libcom_err.so.2 libss.so.2 - # x86_64 libcom_err.so.2 libss.so.2 - echo -n "# $1:" - for p in ${provides}; do - if [[ "$p" == *: ]]; then - echo - echo -n "${p%:}" - else - echo -n " $p" - fi - done - echo -} diff --git a/build_library/build_image_util.sh b/build_library/build_image_util.sh index d942857eab0..de0bf19fd3a 100755 --- a/build_library/build_image_util.sh +++ b/build_library/build_image_util.sh @@ -176,54 +176,78 @@ systemd_enable() { sudo ln -sf "../${unit_file}" "${wants_dir}/${unit_alias}" } -# "equery list" a potentially uninstalled board package -query_available_package() { - local pkg="$1" - local format="${2:-\$cpv::\$repo}" - # Ignore masked versions. Assumes that sort --version-sort uses the - # same ordering as Portage. - equery-${BOARD} --no-color list -po --format "\$mask|$format" "$pkg" | \ - grep -E '^ +\|' | \ - cut -f2- -d\| | \ - sort --version-sort | \ - tail -n 1 +image_packages_portage_impl() { + local root=${1}; shift + local format=${1}; shift + + ROOT="${root}" PORTAGE_CONFIGROOT="${BUILD_DIR}/configroot" \ + equery --quiet --no-color list --format "${format}" '*' } # List packages installed directly in portages package database image_packages_portage() { - ROOT="$1" PORTAGE_CONFIGROOT="${BUILD_DIR}"/configroot \ - equery --no-color list --format '$cpv::$repo' '*' + image_packages_portage_impl "${1}" '$cpv::$repo' } -# List packages implicitly contained in rootfs, such as in initramfs. -image_packages_implicit() { - local profile="${BUILD_DIR}/configroot/etc/portage/profile" +# List packages installed directly in portages package database +image_packages_slots_repos_portage() { + image_packages_portage_impl "${1}" '$cp $fullversion $slot $repo' +} + +# run in a subshell to set nullglob shopt +image_packages_implicit_impl() ( + local root=${1}; shift + local format=${1}; shift # We also want to list packages that only exist in the initramfs. - # Approximate this by listing build dependencies of coreos-kernel that - # are specified with the "=" slot operator, excluding those already - # reported above. - local kernel_pkg=$(ROOT="$1" PORTAGE_CONFIGROOT="${BUILD_DIR}"/configroot \ - equery --no-color list --format '$cpv' sys-kernel/coreos-kernel) - # OEM ACIs have no kernel package. - if [[ -n "${kernel_pkg}" ]]; then - local depend_path="$1/var/db/pkg/$kernel_pkg/DEPEND" - local pkg - for pkg in $(awk 'BEGIN {RS=" "} /=$/ {print}' "$depend_path"); do - if ! ROOT="$1" PORTAGE_CONFIGROOT="${BUILD_DIR}"/configroot \ - equery -q list "$pkg" >/dev/null ; then - query_available_package "$pkg" - fi - done + # Approximate this by listing build dependencies of coreos-kernel, + # excluding those already existing in portage package database. + local kernel_pkg=$(ROOT="${root}" PORTAGE_CONFIGROOT="${BUILD_DIR}/configroot" \ + equery --quiet --no-color list --format '$cpv' sys-kernel/coreos-kernel) + # Sysexts have no kernel package. And no kernel means no initramfs. + if [[ -z ${kernel_pkg} ]]; then + return 0 fi - # In production images GCC libraries are extracted manually. - if [[ -f "${profile}/package.provided" ]]; then - local pkg - while read pkg; do - query_available_package "${pkg}" - done < "${profile}/package.provided" - fi + local -a pkgs to_check + mapfile -t -d ' ' pkgs <"${root}/var/db/pkg/${kernel_pkg}/DEPEND" + # Strip pkgs down to just category and name. First grep is to drop + # empty results, second grep is to drop blockers. + # sys-kernel/coreos-sources is not a package that gets installed + # into initramfs. Also add grub and shims - they get installed + # into image separately, much later, and not using emerge. + mapfile -t to_check < <( + printf '%s\n' "${pkgs[@]}" sys-boot/grub sys-boot/shim sys-boot/shim-signed | \ + sed -e 's/^[^a-z]*//' -e 's/\[.*\]$//' -e 's/:.*//' -e 's/-[0-9].*//' | \ + grep '.' | \ + grep -v '^!' | \ + grep -v -F 'sys-kernel/coreos-sources' | \ + sort -u) + shopt -s nullglob + local pkg skip p + for pkg in "${to_check[@]}"; do + skip='' + for p in "${root}/var/db/pkg/${pkg}-"[0-9]*; do + skip=x + break + done + if [[ -n ${skip} ]]; then continue; fi + if ! "equery-${BOARD}" --no-color --quiet list --format "${format}" "${pkg}"; then + warn "No information for ${pkg} found in ${BOARD}." + warn "This should not happen - by now, all the board packages should be built and installed in the board sysroot." + fi + # TODO: We should also check ${pkg} RDEPEND recursively. + done +) + +# List packages implicitly contained in rootfs, such as in initramfs. +image_packages_implicit() { + image_packages_implicit_impl "${1}" '$cpv::$repo' +} + +# List packages implicitly contained in rootfs, such as in initramfs. +image_packages_slots_repos_implicit() { + image_packages_implicit_impl "${1}" '$cp $fullversion $slot $repo' } # Generate a list of packages installed in an image. @@ -233,6 +257,13 @@ image_packages() { image_packages_implicit "$1" } +# Generate an extended list of packages installed in an image. +# Usage: image_packages /image/root +image_packages_slots_repos() { + image_packages_slots_repos_portage "${1}" + image_packages_slots_repos_implicit "${1}" +} + # Generate a list of installed packages in the format: # sys-apps/systemd-212-r8::coreos write_packages() { @@ -240,6 +271,15 @@ write_packages() { image_packages "$1" | sort > "$2" } +# Generate a list of installed packages in the format +# "$package $version $slot $repository": +# +# sys-apps/systemd 261.2 0/2 gentoo +write_packages_extended() { + info "Writing ${2##*/}" + image_packages_slots_repos "$1" | sort >"$2" +} + # Generate an SPDX SBOM using syft write_sbom() { info "Writing ${2##*/}" @@ -436,6 +476,33 @@ EOF sudo gzip -9 "${root_fs_dir}"/usr/share/licenses/common/* } +# Lookup the current version of a binary package, downloading it if needed. +# Usage: get_binary_pkg some-pkg/name +# Prints: some-pkg/name-1.2.3 +get_binary_pkg() { + local name="$1" version + + # If possible use the version installed in $BOARD_ROOT, + # fall back to any binary package that is available. + version=$(pkg_version installed "${name}") + if [[ -z "${version}" ]]; then + version=$(pkg_version binary "${name}") + fi + + # Nothing? Maybe we can fetch it. + if [[ -z "${version}" && ${FLAGS_getbinpkg} -eq ${FLAGS_TRUE} ]]; then + emerge-${BOARD} --verbose --getbinpkg --usepkgonly --fetchonly --nodeps "${name}" >&2 + version=$(pkg_version binary "${name}") + fi + + # Cry + if [[ -z "${version}" ]]; then + die "Binary package missing for ${name}" + fi + + echo "${version}" +} + # Add /usr/share/SLSA reports for packages indirectly contained within the rootfs # If the package is available in BOARD_ROOT accesses it from there, otherwise # needs to download binpkg. @@ -464,16 +531,6 @@ insert_extra_slsa() { done } -# Add an entry to the image's package.provided -package_provided() { - local p profile="${BUILD_DIR}/configroot/etc/portage/profile" - for p in "$@"; do - info "Writing $p to package.provided and soname.provided" - echo "$p" >> "${profile}/package.provided" - pkg_provides binary "$p" >> "${profile}/soname.provided" - done -} - assert_image_size() { local disk_img="$1" local disk_type="$2" diff --git a/build_library/catalyst.sh b/build_library/catalyst.sh index 698bb72d4c3..6421a122337 100644 --- a/build_library/catalyst.sh +++ b/build_library/catalyst.sh @@ -6,6 +6,7 @@ # common.sh should be sourced first [[ -n "${DEFAULT_BUILD_ROOT}" ]] || exit 1 . "${SCRIPTS_DIR}/sdk_lib/sdk_util.sh" || exit 1 +. "${BUILD_LIBRARY_DIR}/reports_util.sh" || exit 1 . "${BUILD_LIBRARY_DIR}/toolchain_util.sh" || exit 1 # Default option values, may be provided before including this file @@ -243,6 +244,36 @@ write_configs() { rm "${TEMPDIR}"/portage/package.env/qemu } +# runs in a subshell for a trap +generate_stage_listings() ( + local stage=${1}; shift + local listing_dir=${TEMPDIR}/${stage}-listing + mkdir -p "${listing_dir}" + trap 'rm -rf "${listing_dir}"' EXIT + # use lbzip2 for faster decompression + tar \ + --extract \ + --use-compress-program=lbzip2 \ + --file="${BUILDS}/${stage}-${ARCH}-${FLAGS_version}.tar.bz2" \ + --directory "${listing_dir}" + local cpvs=() cpv pkg version slot repo + local output_prefix=${BUILDS}/${TYPE}-${ARCH}-${FLAGS_version}-${stage} + local pkgs_file=${output_prefix}-packages_extended.txt + mapfile -t -d '' cpvs < <(find "${listing_dir}/var/db/pkg/" -mindepth 2 -maxdepth 2 -printf '%P\0') + info "Writing ${pkgs_file##*/}" + for cpv in "${cpvs[@]}"; do + # "-[0-9]*" is a glob pattern, not a regex, so it matches a + # dash, followed by a digit, followed by any number of any + # bytes + pkg=${cpv%-[0-9]*} + version=${cpv#"${pkg}-"} + slot=$(< "${listing_dir}/var/db/pkg/${cpv}/SLOT") + repo=$(< "${listing_dir}/var/db/pkg/${cpv}/repository") + echo "${pkg} ${version} ${slot} ${repo}" + done | LC_ALL=C sort >"${pkgs_file}" + write_contents_with_technical_details "${listing_dir}" "${output_prefix}-contents_wtd.txt" +) + build_stage() { local stage catalyst_conf target_tarball @@ -266,6 +297,7 @@ build_stage() { --file "$TEMPDIR/${stage}.spec" ln -sf "$stage-${ARCH}-${FLAGS_version}.tar.bz2" \ "$BUILDS/$stage-${ARCH}-latest.tar.bz2" + generate_stage_listings "${stage}" info "Finished building $target_tarball" } diff --git a/build_library/dev_container_util.sh b/build_library/dev_container_util.sh index b0e08474c0e..b1f31dbe319 100755 --- a/build_library/dev_container_util.sh +++ b/build_library/dev_container_util.sh @@ -82,11 +82,15 @@ create_dev_container() { info "Building developer image ${image_name}" # The "dev-image-rootfs" directory name is important - it is used to - # determine the package target in coreos/base/profile.bashrc + # determine the package target in coreos/base/profile.bashrc. The + # FLATCAR_GENERIC_BUILD_TYPE environment variable is also used + # there, but for different ebuild phase functions. local root_fs_dir="${BUILD_DIR}/dev-image-rootfs" + local -x FLATCAR_GENERIC_BUILD_TYPE=generic-dev local image_contents="${image_name%.bin}_contents.txt" local image_contents_wtd="${image_name%.bin}_contents_wtd.txt" local image_packages="${image_name%.bin}_packages.txt" + local image_packages_extended="${image_name%.bin}_packages_extended.txt" local image_licenses="${image_name%.bin}_licenses.json" start_image "${image_name}" "${disk_layout}" "${root_fs_dir}" "${update_group}" @@ -96,6 +100,7 @@ create_dev_container() { run_ldconfig "${root_fs_dir}" run_localedef "${root_fs_dir}" write_packages "${root_fs_dir}" "${BUILD_DIR}/${image_packages}" + write_packages_extended "${root_fs_dir}" "${BUILD_DIR}/${image_packages_extended}" write_licenses "${root_fs_dir}" "${BUILD_DIR}/${image_licenses}" insert_licenses "${BUILD_DIR}/${image_licenses}" "${root_fs_dir}" diff --git a/build_library/prod_image_util.sh b/build_library/prod_image_util.sh index 955ec5542be..c85169be78d 100755 --- a/build_library/prod_image_util.sh +++ b/build_library/prod_image_util.sh @@ -5,56 +5,6 @@ source "${BUILD_LIBRARY_DIR}/oem_sysexts.sh" || exit 1 -# Lookup the current version of a binary package, downloading it if needed. -# Usage: get_binary_pkg some-pkg/name -# Prints: some-pkg/name-1.2.3 -get_binary_pkg() { - local name="$1" version - - # If possible use the version installed in $BOARD_ROOT, - # fall back to any binary package that is available. - version=$(pkg_version installed "${name}") - if [[ -z "${version}" ]]; then - version=$(pkg_version binary "${name}") - fi - - # Nothing? Maybe we can fetch it. - if [[ -z "${version}" && ${FLAGS_getbinpkg} -eq ${FLAGS_TRUE} ]]; then - emerge-${BOARD} --verbose --getbinpkg --usepkgonly --fetchonly --nodeps "${name}" >&2 - version=$(pkg_version binary "${name}") - fi - - # Cry - if [[ -z "${version}" ]]; then - die "Binary package missing for ${name}" - fi - - echo "${version}" -} - -# The GCC package includes both its libraries and the compiler. -# In prod images we only need the shared libraries. -extract_prod_gcc() { - local root_fs_dir="$1" gcc pkg - gcc=$(get_binary_pkg sys-devel/gcc) - - # FIXME(marineam): Incompatible with FEATURES=binpkg-multi-instance - pkg="$(portageq-${BOARD} pkgdir)/${gcc}.tbz2" - [[ -f "${pkg}" ]] || die "${pkg} is missing" - - # Normally GCC's shared libraries are installed to: - # /usr/lib/gcc/x86_64-cros-linux-gnu/$version/* - # Instead we extract them to plain old /usr/lib - qtbz2 -O -t "${pkg}" | \ - lbzcat -d -c - | \ - sudo tar -C "${root_fs_dir}" -x \ - --transform 's#/usr/lib/.*/#/usr/lib64/#' \ - --wildcards './usr/lib/gcc/*.so*' \ - --wildcards './usr/share/SLSA' - - package_provided "${gcc}" -} - create_prod_image() { local image_name="$1" local disk_layout="$2" @@ -69,12 +19,16 @@ create_prod_image() { info "Building production image ${image_name}" # The "prod-image-rootfs" directory name is important - it is used - # to determine the package target in coreos/base/profile.bashrc + # to determine the package target in coreos/base/profile.bashrc. The + # FLATCAR_GENERIC_BUILD_TYPE environment variable is also used + # there, but for different ebuild phase functions. local root_fs_dir="${BUILD_DIR}/prod-image-rootfs" + local -x FLATCAR_GENERIC_BUILD_TYPE=generic-prod local root_fs_sysexts_output_dir="${BUILD_DIR}/rootfs-included-sysexts" local image_contents="${image_name%.bin}_contents.txt" local image_contents_wtd="${image_name%.bin}_contents_wtd.txt" local image_packages="${image_name%.bin}_packages.txt" + local image_packages_extended="${image_name%.bin}_packages_extended.txt" local image_sbom="${image_name%.bin}_sbom.json" local image_licenses="${image_name%.bin}_licenses.json" local image_kconfig="${image_name%.bin}_kernel_config.txt" @@ -91,9 +45,7 @@ create_prod_image() { start_image "${image_name}" "${disk_layout}" "${root_fs_dir}" "${update_group}" - # Install minimal GCC (libs only) and then everything else set_image_profile prod - extract_prod_gcc "${root_fs_dir}" emerge_to_image "${root_fs_dir}" "${base_pkg}" run_ldconfig "${root_fs_dir}" run_localedef "${root_fs_dir}" @@ -121,6 +73,7 @@ create_prod_image() { fi write_packages "${root_fs_dir}" "${BUILD_DIR}/${image_packages}" + write_packages_extended "${root_fs_dir}" "${BUILD_DIR}/${image_packages_extended}" insert_licenses "${BUILD_DIR}/${image_licenses}" "${root_fs_dir}" insert_extra_slsa "${root_fs_dir}" @@ -262,13 +215,19 @@ create_prod_sysexts() { # The --install_root_basename="${name}-extra-sysext-rootfs" flag # is important - it sets the name of a rootfs directory, which is # used to determine the package target in - # coreos/base/profile.bashrc - USE="${useflags_array[*]}" sudo -E "${SCRIPT_ROOT}/build_sysext" --board="${BOARD}" \ - --squashfs_base="${BUILD_DIR}/${image_sysext_base}" \ - --image_builddir="${BUILD_DIR}" \ - --install_root_basename="${name}-extra-sysext-rootfs" \ - ${mangle_script:+--manglefs_script=${mangle_script}} \ - "${name}" "${pkg_array[@]}" + # coreos/base/profile.bashrc. The FLATCAR_GENERIC_BUILD_TYPE + # environment variable is also used there, but for different + # ebuild phase functions. + sudo -E \ + "USE=${useflags_array[*]}" \ + "FLATCAR_GENERIC_BUILD_TYPE=generic-sysext-extra-${name}" \ + "${SCRIPT_ROOT}/build_sysext" \ + --board="${BOARD}" \ + --squashfs_base="${BUILD_DIR}/${image_sysext_base}" \ + --image_builddir="${BUILD_DIR}" \ + --install_root_basename="${name}-extra-sysext-rootfs" \ + ${mangle_script:+--manglefs_script=${mangle_script}} \ + "${name}" "${pkg_array[@]}" delta_generator \ -private_key "/usr/share/update_engine/update-payload-key.key.pem" \ -new_image "${BUILD_DIR}/${name}.raw" \ @@ -316,18 +275,24 @@ create_oem_sysexts() { # The --install_root_basename="${name}-oem-sysext-rootfs" flag is # important - it sets the name of a rootfs directory, which is # used to determine the package target in - # coreos/base/profile.bashrc + # coreos/base/profile.bashrc. The FLATCAR_GENERIC_BUILD_TYPE + # environment variable is also used there, but for different + # ebuild phase functions. # # OEM sysexts use no compression here since they will be stored # in a compressed OEM partition. - USE="${useflags}" sudo -E "${SCRIPT_ROOT}/build_sysext" --board="${BOARD}" \ - --squashfs_base="${BUILD_DIR}/${image_sysext_base}" \ - --image_builddir="${BUILD_DIR}" \ - --metapkgs="${metapkg}" \ - --install_root_basename="${name}-oem-sysext-rootfs" \ - --compression=none \ - ${mangle_script:+--manglefs_script="${mangle_script}"} \ - "${name}" + sudo -E \ + "USE=${useflags}" \ + "FLATCAR_GENERIC_BUILD_TYPE=generic-sysext-oem-${name}" \ + "${SCRIPT_ROOT}/build_sysext" \ + --board="${BOARD}" \ + --squashfs_base="${BUILD_DIR}/${image_sysext_base}" \ + --image_builddir="${BUILD_DIR}" \ + --metapkgs="${metapkg}" \ + --install_root_basename="${name}-oem-sysext-rootfs" \ + --compression=none \ + ${mangle_script:+--manglefs_script="${mangle_script}"} \ + "${name}" delta_generator \ -private_key "/usr/share/update_engine/update-payload-key.key.pem" \ -new_image "${BUILD_DIR}/${name}.raw" \ diff --git a/build_library/reports_util.sh b/build_library/reports_util.sh index b8548a94159..bcf4d902fe4 100644 --- a/build_library/reports_util.sh +++ b/build_library/reports_util.sh @@ -4,7 +4,7 @@ # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. -if [[ -n "${FLATCAR_REPORTS_UTIL_SH_INCLUDED:-}" ]]; then +if [[ -n ${FLATCAR_REPORTS_UTIL_SH_INCLUDED:-} ]]; then return 0 fi diff --git a/build_library/sysext_prod_builder b/build_library/sysext_prod_builder index 5406c884d7d..9731c2473d8 100755 --- a/build_library/sysext_prod_builder +++ b/build_library/sysext_prod_builder @@ -63,11 +63,16 @@ create_prod_sysext() { # # The --install_root_basename="${name}-base-sysext-rootfs" flag is # important - it sets the name of a rootfs directory, which is used - # to determine the package target in coreos/base/profile.bashrc + # to determine the package target in coreos/base/profile.bashrc. The + # FLATCAR_GENERIC_BUILD_TYPE environment variable is also used + # there, but for different ebuild phase functions. # # Built-in sysexts are stored in the compressed /usr partition, so we # disable compression to avoid double-compression. - sudo -E "FLATCAR_BUILD_ID=$FLATCAR_BUILD_ID" "${SCRIPTS_DIR}/build_sysext" \ + sudo -E \ + "FLATCAR_BUILD_ID=$FLATCAR_BUILD_ID" \ + "FLATCAR_GENERIC_BUILD_TYPE=generic-sysext-base-${name}" \ + "${SCRIPTS_DIR}/build_sysext" \ --board="${BOARD}" \ --image_builddir="${workdir}/sysext-build" \ --squashfs_base="${base_sysext}" \ diff --git a/build_sysext b/build_sysext index 6ecbdb5f500..f8c4d1d3b73 100755 --- a/build_sysext +++ b/build_sysext @@ -256,7 +256,11 @@ fi info "Writing ${SYSEXT_ID}_packages.txt" ROOT="${BUILD_DIR}/${FLAGS_install_root_basename}" PORTAGE_CONFIGROOT="/build/${FLAGS_board}" \ - equery --no-color list --format '$cpv::$repo' '*' > "${BUILD_DIR}/${SYSEXT_ID}_packages.txt" + equery --quiet --no-color list --format '$cpv::$repo' '*' > "${BUILD_DIR}/${SYSEXT_ID}_packages.txt" + +info "Writing ${SYSEXT_ID}_packages_extended.txt" +ROOT="${BUILD_DIR}/${FLAGS_install_root_basename}" PORTAGE_CONFIGROOT="/build/${FLAGS_board}" \ + equery --quiet --no-color list --format '$cp $fullversion $slot $repo' '*' > "${BUILD_DIR}/${SYSEXT_ID}_packages_extended.txt" if [[ "${FLAGS_strip_binaries}" = "${FLAGS_TRUE}" ]]; then diff --git a/ci-automation/sdk_bootstrap.sh b/ci-automation/sdk_bootstrap.sh index 0837e16a618..bc26f26c6b3 100644 --- a/ci-automation/sdk_bootstrap.sh +++ b/ci-automation/sdk_bootstrap.sh @@ -185,7 +185,8 @@ function _sdk_bootstrap_impl() { # Get Flatcar version number format (separator is '+' instead of '-', # equal to $(strip_version_prefix "$version") source sdk_container/.repo/manifests/version.txt - local dest_tarball="flatcar-sdk-${ARCH}-${FLATCAR_SDK_VERSION}.tar.bz2" + local name_prefix="flatcar-sdk-${ARCH}-${FLATCAR_SDK_VERSION}" + local dest_tarball="${name_prefix}.tar.bz2" local logs_tarball="sdk-bootstrap-logs-${ARCH}-$(date --utc '+%F-%H%M-%S').tar.xz" # change the owner of the files and directories in __build__ back @@ -200,9 +201,9 @@ function _sdk_bootstrap_impl() { if [[ -z ${failed} ]]; then ( cd "__build__/images/catalyst/builds/flatcar-sdk" - create_digests "${SIGNER}" "${dest_tarball}" - sign_artifacts "${SIGNER}" "${dest_tarball}"* - copy_to_buildcache "sdk/${ARCH}/${FLATCAR_SDK_VERSION}" "${dest_tarball}"* + create_digests "${SIGNER}" "${dest_tarball}" "${name_prefix}"-*-{packages_extended,contents_wtd}.txt + sign_artifacts "${SIGNER}" "${dest_tarball}"* "${name_prefix}"-*-{packages_extended,contents_wtd}.txt* + copy_to_buildcache "sdk/${ARCH}/${FLATCAR_SDK_VERSION}" "${dest_tarball}"* "${name_prefix}"-*-{packages_extended,contents_wtd}.txt* ) fi diff --git a/sdk_container/src/third_party/coreos-overlay/coreos-base/coreos/coreos-0.0.1-r321.ebuild b/sdk_container/src/third_party/coreos-overlay/coreos-base/coreos/coreos-0.0.1-r322.ebuild similarity index 100% rename from sdk_container/src/third_party/coreos-overlay/coreos-base/coreos/coreos-0.0.1-r321.ebuild rename to sdk_container/src/third_party/coreos-overlay/coreos-base/coreos/coreos-0.0.1-r322.ebuild diff --git a/sdk_container/src/third_party/coreos-overlay/coreos-base/coreos/coreos-0.0.1.ebuild b/sdk_container/src/third_party/coreos-overlay/coreos-base/coreos/coreos-0.0.1.ebuild index b6e64960492..2f83585b77d 100644 --- a/sdk_container/src/third_party/coreos-overlay/coreos-base/coreos/coreos-0.0.1.ebuild +++ b/sdk_container/src/third_party/coreos-overlay/coreos-base/coreos/coreos-0.0.1.ebuild @@ -186,6 +186,7 @@ RDEPEND="${RDEPEND} sys-boot/efibootmgr sys-boot/mokutil sys-cluster/ipvsadm + sys-devel/gcc sys-devel/gettext sys-fs/btrfs-progs sys-fs/cryptsetup diff --git a/sdk_container/src/third_party/coreos-overlay/coreos/config/env/app-admin/google-guest-configs b/sdk_container/src/third_party/coreos-overlay/coreos/config/env/app-admin/google-guest-configs index 06cf5a50af3..d1a02798791 100644 --- a/sdk_container/src/third_party/coreos-overlay/coreos/config/env/app-admin/google-guest-configs +++ b/sdk_container/src/third_party/coreos-overlay/coreos/config/env/app-admin/google-guest-configs @@ -1,5 +1,5 @@ cros_post_pkg_preinst_rm_from_board_sysroot() { - [[ $(flatcar_target) == generic-board ]] || return + [[ $(flatcar_target) = 'generic-board' ]] || return # Dracut copies all sysctl files unconditionally, but our initrd is shared, # and these rules are GCE-specific, so apply them from the OEM sysext. diff --git a/sdk_container/src/third_party/coreos-overlay/coreos/config/env/dev-libs/glib b/sdk_container/src/third_party/coreos-overlay/coreos/config/env/dev-libs/glib index e6388c51ad1..6f34398efb7 100644 --- a/sdk_container/src/third_party/coreos-overlay/coreos/config/env/dev-libs/glib +++ b/sdk_container/src/third_party/coreos-overlay/coreos/config/env/dev-libs/glib @@ -1,7 +1,4 @@ -# Do not install gobject-introspection binaries in production images. -if [[ $(flatcar_target) != 'sdk' ]] ; then - glib_mask="/usr/bin/gi-* /usr/lib*/libgirepository-2.0*" - PKG_INSTALL_MASK+=" ${glib_mask}" - INSTALL_MASK+=" ${glib_mask}" - unset glib_mask -fi +cros_post_pkg_preinst_drop_gobject_introspection() { + [[ $(flatcar_target) = 'generic-'* ]] || return + rm -r "${ED}"/usr/bin/gi-* "${ED}"/usr/lib*/libgirepository-2.0* || die +} diff --git a/sdk_container/src/third_party/coreos-overlay/coreos/config/env/net-misc/openssh b/sdk_container/src/third_party/coreos-overlay/coreos/config/env/net-misc/openssh index 3125f149361..8d7cb5d643b 100644 --- a/sdk_container/src/third_party/coreos-overlay/coreos/config/env/net-misc/openssh +++ b/sdk_container/src/third_party/coreos-overlay/coreos/config/env/net-misc/openssh @@ -1,15 +1,7 @@ -# Do not install the setuid file in production images. -# -# Do not install the config snippet that defines a subsystem. We have -# our own definition in coreos-init. -if [[ $(flatcar_target) != 'sdk' ]] ; then - INSTALL_MASK+=" /usr/lib*/misc/ssh-keysign /etc/ssh/sshd_config.d/*gentoo-subsystem.conf " -fi - cros_post_src_install_flatcar_modifications() { vendorize_pam_files - if [[ $(flatcar_target) != 'sdk' ]] ; then + if [[ $(flatcar_target) = 'generic-'* ]] ; then dosym -r /usr/share/flatcar/etc/ssh/ssh_config.d/50-flatcar-ssh.conf /usr/share/ssh/ssh_config insinto /usr/share/flatcar/etc/ssh/ssh_config.d/ newins - 50-flatcar-ssh.conf @@ -61,3 +53,12 @@ EOF ) fi } + +# Do not install the setuid file in production images. +# +# Do not install the config snippet that defines a subsystem. We have +# our own definition in coreos-init. +cros_post_pkg_preinst_drop_setuid_and_subsystem_config() { + [[ $(flatcar_target) = 'generic-'* ]] || return + rm -r "${ED}"/usr/lib*/misc/ssh-keysign "${ED}"/etc/ssh/sshd_config.d/*gentoo-subsystem.conf +} diff --git a/sdk_container/src/third_party/coreos-overlay/coreos/config/env/sys-devel/gcc b/sdk_container/src/third_party/coreos-overlay/coreos/config/env/sys-devel/gcc new file mode 100644 index 00000000000..dcfd740259e --- /dev/null +++ b/sdk_container/src/third_party/coreos-overlay/coreos/config/env/sys-devel/gcc @@ -0,0 +1,30 @@ +# The GCC package contains libraries, compilers, header files and so +# on. In production images we only need the shared libraries. Also, +# normally GCC's shared libraries are installed into +# /usr/lib/gcc/$triplet/$version. For production images, we move them +# to plain old /usr/lib64. +cros_post_pkg_preinst_move_and_keep_libs() { + if [[ $(flatcar_target) = 'generic-prod' ]]; then + local f + # GCCMAJOR comes from toolchain.eclass. + for f in "${ED}/usr/lib/gcc/${CTARGET}/${GCCMAJOR}/"*.so*; do + mv "${f}" "${EROOT}/usr/lib64/" || die + done + rm -r "${ED}/etc" || die + rm -r "${ED}/usr/${CTARGET}" || die + rm -r "${ED}/usr/bin" || die + rm -r "${ED}/usr/lib/gcc" || die + rm -r "${ED}/usr/libexec" || die + for f in "${ED}/usr/share/"*; do + [[ ${f} = "${ED}/usr/share/SLSA" ]] && continue + rm -r "${f}" || die + done + fi +} + +if [[ $(flatcar_target) = 'generic-prod' ]]; then + # Clobber the do_gcc_config function from toolchain.eclass to a + # no-op. We are not installing the compiler on prod images and + # gcc-config complains about some missing files. + do_gcc_config() { :; } +fi diff --git a/sdk_container/src/third_party/coreos-overlay/coreos/config/env/sys-fs/lvm2 b/sdk_container/src/third_party/coreos-overlay/coreos/config/env/sys-fs/lvm2 index 62455b50dd3..5f0b82673df 100644 --- a/sdk_container/src/third_party/coreos-overlay/coreos/config/env/sys-fs/lvm2 +++ b/sdk_container/src/third_party/coreos-overlay/coreos/config/env/sys-fs/lvm2 @@ -1,5 +1,5 @@ cros_post_src_install_tmpfiles_perms() { - if [[ $(flatcar_target) != 'sdk' ]]; then + if [[ $(flatcar_target) = 'generic-'* ]]; then # Upstream installs its tmpfiles config file with unnecessarilly # restrictive mode, relax it. It could be also fixed by passing # --enable-write_install option to configure script, but that diff --git a/sdk_container/src/third_party/coreos-overlay/profiles/coreos/base/profile.bashrc b/sdk_container/src/third_party/coreos-overlay/profiles/coreos/base/profile.bashrc index 72701a012f9..2c7a9bc0379 100644 --- a/sdk_container/src/third_party/coreos-overlay/profiles/coreos/base/profile.bashrc +++ b/sdk_container/src/third_party/coreos-overlay/profiles/coreos/base/profile.bashrc @@ -3,8 +3,8 @@ CROS_BUILD_BOARD_TREE="${SYSROOT}/build" CROS_ADDONS_TREE="/mnt/host/source/src/third_party/coreos-overlay/coreos" -# Are we merging for the board sysroot, or for the SDK, or for -# the images? Returns a string in a passed variable: +# Prints the type of image we are merging the package for, prints one +# of the following: # # - sdk (the SDK) # - generic-board (board sysroot) @@ -17,48 +17,29 @@ CROS_ADDONS_TREE="/mnt/host/source/src/third_party/coreos-overlay/coreos" # - generic-sysext-oem-${name} (OEM sysext image ${name}, like # azure, qemu_uefi) # - generic-unknown (something using generic profile, but otherwise -# unknown, probably something is messed up) +# unknown, may happen when config override is sourced between phase +# funcs and flatcar_target is called at the toplevel scope instead +# of a hook) # - unknown (unknown type of image, neither generic, nor sdk, # probably something is messed up) -flatcar_target_ref() { - local -n type_ref=${1}; shift - - local name - case ${FLATCAR_TYPE} in - sdk) type_ref='sdk';; - generic) - case ${ROOT} in - */prod-image-rootfs) type_ref='generic-prod';; - */dev-image-rootfs) type_ref='generic-dev';; - */*-base-sysext-rootfs) - name=${ROOT##*/} - name=${name%-base-sysext-rootfs} - type_ref="generic-sysext-base-${name}" - ;; - */*-extra-sysext-rootfs) - name=${ROOT##*/} - name=${name%-extra-sysext-rootfs} - type_ref="generic-sysext-extra-${name}" - ;; - */*-oem-sysext-rootfs) - name=${ROOT##*/} - name=${name%-oem-sysext-rootfs} - type_ref="generic-sysext-oem-${name}" - ;; - "${SYSROOT}") type_ref='generic-board';; - *) type_ref='generic-unknown' - esac - ;; - *) type_ref='unknown';; - esac -} - -# Prints the type of image we are merging the package for, see -# flatcar_target_ref for details. +# +# When using it with a string comparison, it is better to always use +# equality instead of unequality. So: +# +# if [[ $(flatcar_target) = 'generic-'* ]]; then …; fi +# +# instead of +# +# if [[ $(flatcar_target) != 'sdk' ]]; then …; fi +# +# The latter will be true if flatcar_target prints 'unknown'. flatcar_target() { local target_type - flatcar_target_ref target_type + local revert_shopt=$(shopt -p extglob) + shopt -s extglob + source "${BASH_SOURCE[0]}.flatcar-target" target_type echo "${target_type}" + ${revert_shopt} } # Load all additional bashrc files we have for this package. diff --git a/sdk_container/src/third_party/coreos-overlay/profiles/coreos/base/profile.bashrc.flatcar-target b/sdk_container/src/third_party/coreos-overlay/profiles/coreos/base/profile.bashrc.flatcar-target new file mode 100644 index 00000000000..5401207dcab --- /dev/null +++ b/sdk_container/src/third_party/coreos-overlay/profiles/coreos/base/profile.bashrc.flatcar-target @@ -0,0 +1,46 @@ +# In order to parse an extglob pattern, "shopt -s extglob" must be +# executed outside the outermost block that contains the pattern. In +# this case, any bash script sourcing this file must call "shopt -s +# extglob" beforehand. Failing to do so will result in parsing failure +# with error message like: +# +# line XX: syntax error near unexpected token `(' +local -n type_ref=${1}; shift +local name +case ${FLATCAR_TYPE} in + sdk) type_ref='sdk';; + generic) + if [[ ${EBUILD_PHASE_FUNC} = src_* ]]; then + type_ref=${FLATCAR_GENERIC_BUILD_TYPE:-generic-board} + elif [[ ${EBUILD_PHASE_FUNC} = pkg_* || -n ${ROOT:-} ]]; then + # Not using FLATCAR_GENERIC_BUILD_TYPE env var, because in + # most pkg_* phase funcs, a value of the variable does not + # come from the environment of emerge caller, but from the + # binary package, so it most likely will be generic-board. + case ${ROOT} in + */prod-image-rootfs) type_ref='generic-prod';; + */dev-image-rootfs) type_ref='generic-dev';; + */*-base-sysext-rootfs) + name=${ROOT##*/} + name=${name%-base-sysext-rootfs} + type_ref="generic-sysext-base-${name}" + ;; + */*-extra-sysext-rootfs) + name=${ROOT##*/} + name=${name%-extra-sysext-rootfs} + type_ref="generic-sysext-extra-${name}" + ;; + */*-oem-sysext-rootfs) + name=${ROOT##*/} + name=${name%-oem-sysext-rootfs} + type_ref="generic-sysext-oem-${name}" + ;; + /build/+([^/])-usr) type_ref='generic-board';; + *) type_ref='generic-unknown';; + esac + else + type_ref='generic-unknown' + fi + ;; + *) type_ref='unknown';; +esac diff --git a/sdk_container/src/third_party/coreos-overlay/profiles/coreos/targets/generic/prod/package.provided b/sdk_container/src/third_party/coreos-overlay/profiles/coreos/targets/generic/prod/package.provided new file mode 100644 index 00000000000..12f5bfc6be1 --- /dev/null +++ b/sdk_container/src/third_party/coreos-overlay/profiles/coreos/targets/generic/prod/package.provided @@ -0,0 +1,5 @@ +# Pulled in by sys-devel/gcc, they are needed only by the compiler, +# which we do not ship. +dev-libs/mpc-1.4.1 +dev-libs/mpfr-4.2.2 +sys-devel/gcc-config-2.12.2 diff --git a/sdk_lib/sdk_container_common.sh b/sdk_lib/sdk_container_common.sh index d1514a5f72b..06455a2f5de 100644 --- a/sdk_lib/sdk_container_common.sh +++ b/sdk_lib/sdk_container_common.sh @@ -80,34 +80,29 @@ function get_version_from_versionfile() { # return true if a given version number is an official build # function is_official() { - local vernum="$1" - - local official="$(echo "$vernum" | sed -n 's/^[0-9]\+\.[0-9]\+\.[0-9]\+$/true/p')" - - test -n "$official" + [[ ${1} =~ ^[0-9]+\.[0-9]+\.[0-9]+$ ]] } # -- # extract the build ID suffix from a version string ("alpha-3244.0.1-nightly2" => "nightly2") # function build_id_from_version() { - local version="$1" - # support vernums and versions ("alpha-"... is optional) - echo "${version}" | sed -n 's/^\([a-z]\+-\)\?[0-9.]\+[-+]\(.*\)$/\2/p' + if [[ ${1} =~ ^([a-z]+-)?[0-9.]+[-+](.*)$ ]]; then + echo "${BASH_REMATCH[2]}" + fi } # -- # Get channel from a version string ("alpha-3244.0.1-nightly2" => "alpha") # function channel_from_version() { - local version="$1" - local channel="" - - channel=$(echo "${version}" | cut -d - -f 1) - if [ "${channel}" != "alpha" ] && [ "${channel}" != "beta" ] && [ "${channel}" != "stable" ] && [ "${channel}" != "lts" ]; then - channel="developer" - fi + local version=${1} + local channel=${version%%-*} + case ${channel} in + alpha|beta|stable|lts) :;; + *) channel=developer + esac echo "${channel}" } # -- @@ -120,10 +115,9 @@ function get_git_channel() { # extract the version number (w/o build ID) from a version string ("alpha-3244.0.1-nightly2" => "3244.0.1") # function vernum_from_version() { - local version="$1" - - # support vernums and versions ("alpha-"... is optional) - echo "${version}" | sed -n 's/^\([a-z]\+-\)\?\([0-9.]\+\).*/\2/p' + if [[ ${1} =~ ^([a-z]+-)?([0-9.]+) ]]; then + echo "${BASH_REMATCH[2]}" + fi } # -- @@ -131,12 +125,13 @@ function vernum_from_version() { # and add a "+[build suffix]" if this is a non-official build. The "+" matches the version # string generation in the build scripts. function strip_version_prefix() { - local version="$1" + local version=${1}; shift + local build_id version_id - local build_id="$(build_id_from_version "${version}")" - local version_id="$(vernum_from_version "${version}")" + build_id=$(build_id_from_version "${version}") + version_id=$(vernum_from_version "${version}") - if [ -n "${build_id}" ] ; then + if [[ -n ${build_id} ]]; then echo "${version_id}+${build_id}" else echo "${version_id}" @@ -147,8 +142,8 @@ function strip_version_prefix() { # Derive docker-safe image version string from vernum. # Keep in sync with ci-automation/ci_automation_common.sh function vernum_to_docker_image_version() { - local vernum="$1" - echo "$vernum" | sed 's/[+]/-/g' + local vernum=${1}; shift + echo "${vernum//+/-}" } # -- @@ -162,16 +157,17 @@ function vernum_to_docker_image_version() { # the OS image is to be built with. # function create_versionfile() { - local sdk_version="$1" - local os_version="${2:-$sdk_version}" - local build_id="$(build_id_from_version "${os_version}")" - local version_id="$(vernum_from_version "${os_version}")" + local sdk_version=${1}; shift + local os_version=${1:-"${sdk_version}"} + local build_id version_id + build_id=$(build_id_from_version "${os_version}") + version_id=$(vernum_from_version "${os_version}") - sdk_version="$(strip_version_prefix "${sdk_version}")" - os_version="$(strip_version_prefix "${os_version}")" - yell "Writing versionfile '$sdk_container_common_versionfile' to SDK '$sdk_version', OS '$os_version'." + sdk_version=$(strip_version_prefix "${sdk_version}") + os_version=$(strip_version_prefix "${os_version}") + yell "Writing versionfile '${sdk_container_common_versionfile}' to SDK '${sdk_version}', OS '${os_version}'." - cat >"$sdk_container_common_versionfile" <"${sdk_container_common_versionfile}" <