Replace Portage-Stable with Gentoo

.github/workflows/cacerts-release.yaml

@@ -13,18 +13,21 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Check out scripts at ${{ github.ref }} for GHA scripts only
-        uses: actions/checkout@v4
+        uses: actions/checkout@v7
         with:
           token: ${{ secrets.BOT_PR_TOKEN }}
+          submodules: false
           path: gha
+          sparse-checkout: .github
       - name: Figure out branch
         id: figure-out-branch
         run: gha/.github/workflows/figure-out-branch.sh '${{ matrix.channel }}'
       - name: Check out work scripts branch for updating
         if: steps.figure-out-branch.outputs.SKIP == 0
-        uses: actions/checkout@v4
+        uses: actions/checkout@v7
         with:
           token: ${{ secrets.BOT_PR_TOKEN }}
+          submodules: true
           path: work
           ref: ${{ steps.figure-out-branch.outputs.BRANCH }}
       - name: Figure out latest ca-certificates release version

.github/workflows/ci.yaml

@@ -57,13 +57,14 @@ jobs:
         uses: docker/setup-docker-action@v4
 
       - name: Checkout scripts
-        uses: actions/checkout@v4
+        uses: actions/checkout@v7
         with:
+          submodules: true
           path: scripts
           fetch-depth: 0
 
       - name: Checkout build scripts
-        uses: actions/checkout@v4
+        uses: actions/checkout@v7
         with:
           repository: flatcar/flatcar-build-scripts
           path: flatcar-build-scripts

.github/workflows/firmware-release-main.yaml

@@ -9,9 +9,10 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Check out scripts
-        uses: actions/checkout@v4
+        uses: actions/checkout@v7
         with:
           token: ${{ secrets.BOT_PR_TOKEN }}
+          submodules: true
           path: scripts
       - name: Figure out latest Linux Firmware release version
         id: firmware-latest-release

.github/workflows/image_changes.sh

@@ -25,8 +25,6 @@ function github_ricj_callback() {
     show_changes_env+=(
         # Override the default locations of repositories.
         "SCRIPTS_REPO=."
-        "COREOS_OVERLAY_REPO=../coreos-overlay"
-        "PORTAGE_STABLE_REPO=../portage-stable"
     )
     show_changes_params+=(
         # We may not have a tag handy, so we tell show-changes

.github/workflows/kernel-release.yaml

@@ -13,18 +13,21 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Check out scripts at ${{ github.ref }} for GHA scripts only
-        uses: actions/checkout@v4
+        uses: actions/checkout@v7
         with:
           token: ${{ secrets.BOT_PR_TOKEN }}
+          submodules: false
           path: gha
+          sparse-checkout: .github
       - name: Figure out branch
         id: figure-out-branch
         run: gha/.github/workflows/figure-out-branch.sh '${{ matrix.channel }}'
       - name: Check out work scripts branch for updating
         if: steps.figure-out-branch.outputs.SKIP == 0
-        uses: actions/checkout@v4
+        uses: actions/checkout@v7
         with:
           token: ${{ secrets.BOT_PR_TOKEN }}
+          submodules: true
           path: work
           ref: ${{ steps.figure-out-branch.outputs.BRANCH }}
       - name: Figure out latest Linux release version

.github/workflows/mantle-releases-main.yml

@@ -13,18 +13,21 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Check out scripts at ${{ github.ref }} for GHA scripts only
-        uses: actions/checkout@v4
+        uses: actions/checkout@v7
         with:
           token: ${{ secrets.BOT_PR_TOKEN }}
+          submodules: false
           path: gha
+          sparse-checkout: .github
       - name: Figure out branch
         id: figure-out-branch
         run: gha/.github/workflows/figure-out-branch.sh '${{ matrix.channel }}'
       - name: Check out work scripts branch for updating
         if: steps.figure-out-branch.outputs.SKIP == 0
-        uses: actions/checkout@v4
+        uses: actions/checkout@v7
         with:
           token: ${{ secrets.BOT_PR_TOKEN }}
+          submodules: false
           path: work
           ref: ${{ steps.figure-out-branch.outputs.BRANCH }}
       - name: Fetch latest mantle hash

.github/workflows/run-kola-tests.yaml

@@ -42,8 +42,9 @@ jobs:
       - name: Set up Docker
         uses: docker/setup-docker-action@v4
 
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@v7
         with:
+          submodules: false
           path: scripts
           fetch-depth: 0
 
@@ -61,7 +62,6 @@ jobs:
 
           cd scripts
           git checkout ${{ github.event.pull_request.head.sha }}
-          git submodule update
 
       - name: Download binpkgs
         if: ${{ !inputs.workflow_run_id }}
@@ -246,11 +246,11 @@ jobs:
           sudo ln -s /bin/bash /bin/sh
           sudo apt update && sudo apt install -y ca-certificates curl gnupg lsb-release git bzip2 jq sqlite3
 
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@v7
         with:
+          submodules: false
           path: scripts
           fetch-depth: 0
-          submodules: true
 
       # Hack alert: actions/checkout will check out the (disjunct) merge commit of a PR
       #  instead of its head commit. That commit is not connected to any branch.
@@ -266,7 +266,6 @@ jobs:
 
           cd scripts
           git checkout ${{ github.event.pull_request.head.sha }}
-          git submodule update
 
       # This is clunky. Haven't figured out how to re-use matrix.arch here for downloads,
       #  so we download each arch individually.

.github/workflows/update-metadata-glsa.yaml

@@ -9,13 +9,14 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Check out scripts
-        uses: actions/checkout@v4
+        uses: actions/checkout@v7
         with:
           token: ${{ secrets.BOT_PR_TOKEN }}
+          submodules: false
       - name: Update GLSA metadata
         id: update-glsa-metadata
         run: |
-          glsa_dir='sdk_container/src/third_party/portage-stable/metadata/glsa'
+          glsa_dir='sdk_container/src/third_party/coreos-overlay/metadata/glsa'
           rm -rf "${glsa_dir}"
           mkdir --parents "${glsa_dir}"
           rsync --archive rsync://rsync.gentoo.org/gentoo-portage/metadata/glsa/* "${glsa_dir}"
@@ -30,7 +31,7 @@ jobs:
           base: main
           title: Monthly GLSA metadata ${{steps.update-glsa-metadata.outputs.TODAYDATE }}
           body: Updated GLSA metadata
-          commit-message: "portage-stable/metadata: Monthly GLSA metadata updates"
+          commit-message: "coreos-overlay/metadata: Monthly GLSA metadata updates"
           author: Flatcar Buildbot <buildbot@flatcar-linux.org>
           labels: main
           signoff: true

.github/workflows/update-portage-stable-packages-from-list.yaml

@@ -1,92 +1,53 @@
-name: Keep portage-stable packages updated
+name: Keep Gentoo submodule updated
 on:
   schedule:
-    - cron:  '0 7 * * 1'
+    - cron: '0 4 * * 0-4'
   workflow_dispatch:
 
 jobs:
-  keep-packages-updated:
+  keep-gentoo-updated:
     runs-on: ubuntu-latest
     steps:
       - name: Check out scripts
-        uses: actions/checkout@v4
+        uses: actions/checkout@v7
         with:
-          token: ${{ secrets.GITHUB_TOKEN }}
-          path: ./scripts
-      - name: Check out Gentoo
-        uses: actions/checkout@v4
-        with:
-          repository: gentoo/gentoo
-          path: gentoo
-          # Gentoo is quite a large repo, so limit ourselves to last
-          # quarter milion of commits. It is about two years worth of changes.
-          # Is is needed by the sync script to find out the hash of the last commit
-          # that made the changes to the package.
-          fetch-depth: 250000
-          ref: master
-      - name: Check out build scripts
-        uses: actions/checkout@v4
-        with:
-          repository: flatcar/flatcar-build-scripts
-          path: flatcar-build-scripts
-      - name: Update listed packages
-        id: update-listed-packages
+          token: ${{ secrets.BOT_PR_TOKEN }}
+          path: scripts
+          sparse-checkout: |
+            .gitmodules
+            .github/workflows/common.sh
+          sparse-checkout-cone-mode: false
+      - name: Update Gentoo submodule
+        id: update-gentoo
+        env:
+          WORK_SCRIPTS_DIR: "${{ github.workspace }}/scripts"
         run: |
-          git config --global user.name "Flatcar Buildbot"
-          git config --global user.email "buildbot@flatcar-linux.org"
-          old_head=$(git -C scripts rev-parse HEAD)
-          packages_list=$(realpath scripts/.github/workflows/portage-stable-packages-list)
-          gentoo_repo=$(realpath gentoo)
-          build_scripts=$(realpath flatcar-build-scripts)
-          pushd scripts/sdk_container/src/third_party/portage-stable
-          while read -r package; do
-              if [[ ! -e "${package}" ]]; then
-                  # If this happens, it means that the package was moved to overlay
-                  # or dropped, the list ought to be updated.
-                  echo "::warning title=${package}::Nonexistent package"
-                  continue
-              fi
-              if [[ ! -e "${gentoo_repo}/${package}" ]]; then
-                  # If this happens, it means that the package was obsoleted or moved
-                  # in Gentoo. The obsoletion needs to be handled in the case-by-case
-                  # manner, while move should be handled by doing the same move
-                  # in portage-stable. The build should not break because of the move,
-                  # because most likely it's already reflected in the profiles/updates
-                  # directory.
-                  echo "::warning title=${package}::Obsolete or moved package"
-                  continue
-              fi
-              GENTOO_REPO="${gentoo_repo}" "${build_scripts}/sync-with-gentoo" "${package}"
-          done < <(grep '^[^#]' "${packages_list}")
-          popd
-          new_head=$(git -C scripts rev-parse HEAD)
-          updated=0
-          if [[ "${new_head}" != "${old_head}" ]]; then
-              updated=1
+          set -euo pipefail
+          source scripts/.github/workflows/common.sh
+          prepare_git_repo
+          submodule=sdk_container/src/third_party/gentoo
+          url=$(git -C "${WORK_SCRIPTS_DIR}" config --file .gitmodules --get "submodule.${submodule}.url")
+          branch=$(git -C "${WORK_SCRIPTS_DIR}" config --file .gitmodules --get "submodule.${submodule}.branch")
+          read -r latest_commit _ < <(git ls-remote --exit-code "${url}" "refs/heads/${branch}")
+          short_commit=${latest_commit:0:12}
+          git -C "${WORK_SCRIPTS_DIR}" update-index --cacheinfo "160000,${latest_commit},${submodule}"
+          if git -C "${WORK_SCRIPTS_DIR}" diff --cached --quiet -- "${submodule}"; then
+              echo 'UPDATE_NEEDED=0' >>"${GITHUB_OUTPUT}"
+              exit 0
           fi
-          todaydate=$(date +%Y-%m-%d)
-          echo "UPDATED=${updated}" >>"${GITHUB_OUTPUT}"
-          echo "TODAYDATE=${todaydate}" >>"${GITHUB_OUTPUT}"
-      - name: Create pull request for main branch
-        uses: peter-evans/create-pull-request@v6
-        if: steps.update-listed-packages.outputs.UPDATED == 1
+          git -C "${WORK_SCRIPTS_DIR}" commit --signoff -m "gentoo: Update to ${short_commit}"
+          cleanup_repo
+          echo 'UPDATE_NEEDED=1' >>"${GITHUB_OUTPUT}"
+          echo "LATEST_COMMIT=${latest_commit}" >>"${GITHUB_OUTPUT}"
+          echo "SHORT_COMMIT=${short_commit}" >>"${GITHUB_OUTPUT}"
+      - name: Create pull request
+        uses: peter-evans/create-pull-request@v8
+        if: steps.update-gentoo.outputs.UPDATE_NEEDED == 1
         with:
-          token: ${{ secrets.GITHUB_TOKEN }}
+          token: ${{ secrets.BOT_PR_TOKEN }}
           path: scripts
-          branch: buildbot/weekly-portage-stable-package-updates-${{steps.update-listed-packages.outputs.TODAYDATE }}
+          branch: buildbot/update-gentoo-submodule
           delete-branch: true
-          base: main
-          title: Weekly portage-stable package updates ${{steps.update-listed-packages.outputs.TODAYDATE }}
-          body: |
-            CI: TODO
-
-            --
-
-            TODO: Changes.
-
-            --
-
-            - [ ] changelog
-            - [ ] image diff
-          labels: main
-          draft: true
+          title: Update gentoo submodule to ${{ steps.update-gentoo.outputs.SHORT_COMMIT }}
+          body: Update `sdk_container/src/third_party/gentoo` to flatcar-hub/gentoo@${{ steps.update-gentoo.outputs.LATEST_COMMIT }}.
+          labels: auto-ci,jenkins-ci,main

.github/workflows/update-sdk.yaml

@@ -63,9 +63,10 @@ jobs:
       - name: Set up Docker
         uses: docker/setup-docker-action@v4
 
-      - uses: actions/checkout@v4
+      - uses: actions/checkout@v7
         id: step2
         with:
+          submodules: true
           path: scripts
           fetch-depth: 0
 

.github/workflows/vmware-release-main.yaml

@@ -9,9 +9,10 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Check out scripts
-        uses: actions/checkout@v4
+        uses: actions/checkout@v7
         with:
           token: ${{ secrets.BOT_PR_TOKEN }}
+          submodules: true
           path: scripts
       - name: Figure out latest open-vm-tools release version
         id: openvmtools-latest-release

.gitmodules

@@ -0,0 +1,5 @@
+[submodule "sdk_container/src/third_party/gentoo"]
+	path = sdk_container/src/third_party/gentoo
+	url = https://github.com/flatcar-hub/gentoo.git
+	branch = master
+	shallow = true

PREFIX.md

@@ -1,3 +1,11 @@
+# !!! OUTDATED. This needs to be ported to match the changes after the Gentoo move !!!
+
+If anything, working with prefixes should be a lot easier now.
+But the below is outdated and will not work.
+`prefix-overlay` has been removed too, as it duplicated ebuilds now available via `gentoo`.
+
+# !!! OUTDATED. This needs to be ported to match the changes after the Gentoo move !!!
+
 # Prefix - build portable, distro-independent apps
 
 **!!! NOTE: Prefix support in the Flatcar SDK is EXPERIMENTAL at this time !!!**

README.md

@@ -28,18 +28,17 @@ The SDK can be used to
 # Using the scripts repository
 
 The repository is meant to be the entry point for Flatcar builds and development.
-Ebuilds for all packages reside in one of 2 subdirectories - [coreos-overlay](sdk_container/src/third_party/coreos-overlay) and [portage-stable](sdk_container/src/third_party/portage-stable/):
+Ebuilds for all packages reside in one of 2 subdirectories - [coreos-overlay](sdk_container/src/third_party/coreos-overlay) and [gentoo](sdk_container/src/third_party/gentoo/):
 ```
 scripts
    +--sdk_container
           +---------src
                      +--third_party
                              +------coreos-overlay
-                             +------portage-stable
+                             +------gentoo
 ```
 
-`portage-stable` is kept in alignment with upstream Gentoo and should not contain any modifications (with only minor, well-justified exceptions).
-Consider it a small sub-set of Gentoo.
+`gentoo` is a git submodule, pointing to Flatcar's fork of the main upstream Gentoo package repository. That fork is generally aligned with Gentoo and only occasionally diverges into alternative branches for brief periods to apply fixes shortly before upstream does. It includes far more packages than Flatcar actually uses.
 
 `coreos-overlay` contains significantly modified or even entirely self-written ebuilds.
 
@@ -66,7 +65,7 @@ While work on a native ARM64 native SDK is ongoing, it's unfortunately not ready
 
 The container can be run in one of two ways - "standalone", or integrated with the [scripts](https://github.com/flatcar/scripts) repo:
 * Standalone mode will use no host volumes and will allow you to play with the SDK in a sandboxed throw-away environment. In standalone mode, you interface with Docker directly to use the SDK container.
-* Integrated mode will closely integrate with the scripts repo directory and bind-mount it as well as the portage-stable and coreos-overlay directories into the container. Integrated mode uses wrapper scripts to interact with the SDK container. This is the recommended way for developing patches for Flatcar.
+* Integrated mode will closely integrate with the scripts repo directory and bind-mount it as well as the gentoo and coreos-overlay directories into the container. Integrated mode uses wrapper scripts to interact with the SDK container. This is the recommended way for developing patches for Flatcar.
 
 ## Standalone mode
 
@@ -96,7 +95,7 @@ To start a container in privileged mode with `/dev` available use:
 
 This is the preferred mode of working with the SDK.
 Interaction with the container happens via wrapper scripts from the scripts repository.
-Both the host's scripts repo as well as the ebuild paths (portage-stable and coreos-overlay) are made available in the container, allowing for work on these directly.
+Both the host's scripts repo as well as the ebuild directories (gentoo and coreos-overlay) are made available in the container, allowing for work on these directly.
 The wrapper scripts will re-use existing containers instead of creating new ones to preserve your work in the container, enabling consistency.
 
 To clone the scripts repo and pick a version: