From 65988b0c3270ee98f0f56daa043c1da9d3c2fdb9 Mon Sep 17 00:00:00 2001 From: "Dustin C. Hatch" Date: Thu, 5 Dec 2024 23:10:20 -0600 Subject: [PATCH] Rework images into three layers * Base * Crossdev * Build The *Base* layer is bootstrapped from a regular Gentoo stage 3 environment. The *Build* layer is the primary artifact, and is "squashed" into a single layer when built. --- README.md | 46 ++++++++++++----- build-image.sh | 63 ------------------------ common.sh | 15 ------ crossdev-image.sh | 36 -------------- image-build/base.sh | 39 +++++++++++++++ image-build/build.sh | 35 +++++++++++++ image-build/cross.sh | 31 ++++++++++++ image-build/gentoo-stage3.sh | 22 +++++++++ lib/bootstrap.sh | 29 +++++++++++ lib/common.sh | 17 +++++++ crossdev.sh => lib/crossdev.sh | 4 +- lib/profile.sh | 11 +++++ lib/sync.sh | 8 +++ lib/tools.sh | 15 ++++++ portage/env/nobuildpkg | 1 + portage/make.conf/15-keywords.conf | 1 + portage/make.conf/20-binpkgs.conf | 2 + portage/make.conf/80-quiet.conf | 2 + portage/package.accept_keywords/genimage | 1 + portage/package.accept_keywords/kernel | 1 + portage/package.env/kernel | 6 +++ portage/package.use/btrfs-progs | 1 + portage/package.use/git | 1 + portage/package.use/grub | 1 + portage/package.use/kernel | 2 + portage/package.use/selinux | 2 + portage/package.use/squashfs-tools | 1 + portage/package.use/systemd | 6 +++ sync.sh | 10 ---- stage3-image.sh => tools/fetch-stage3.sh | 17 +------ update.sh | 5 -- 31 files changed, 274 insertions(+), 157 deletions(-) delete mode 100644 build-image.sh delete mode 100644 common.sh delete mode 100644 crossdev-image.sh create mode 100644 image-build/base.sh create mode 100644 image-build/build.sh create mode 100644 image-build/cross.sh create mode 100644 image-build/gentoo-stage3.sh create mode 100755 lib/bootstrap.sh create mode 100644 lib/common.sh rename crossdev.sh => lib/crossdev.sh (83%) mode change 100644 => 100755 create mode 100755 lib/profile.sh create mode 100755 lib/sync.sh create mode 100755 lib/tools.sh create mode 100644 portage/env/nobuildpkg create mode 100644 portage/make.conf/15-keywords.conf create mode 100644 portage/make.conf/20-binpkgs.conf create mode 100644 portage/make.conf/80-quiet.conf create mode 100644 portage/package.accept_keywords/genimage create mode 100644 portage/package.accept_keywords/kernel create mode 100644 portage/package.env/kernel create mode 100644 portage/package.use/btrfs-progs create mode 100644 portage/package.use/git create mode 100644 portage/package.use/grub create mode 100644 portage/package.use/kernel create mode 100644 portage/package.use/selinux create mode 100644 portage/package.use/squashfs-tools create mode 100644 portage/package.use/systemd delete mode 100644 sync.sh rename stage3-image.sh => tools/fetch-stage3.sh (61%) delete mode 100644 update.sh diff --git a/README.md b/README.md index 4df386c..6ccd589 100644 --- a/README.md +++ b/README.md @@ -7,32 +7,56 @@ Prerequisites: ## Gentoo Stage 3 Image -This image forms the base layer for Aimee OS build images. - -Use the default stage3 tarball (`stage3-amd64-nomultilib-openrc`): +This image is used to bootstrap the base layer for Aimee OS build images. To +build it, use the `gentoo-stage3.sh` build script. This script takes a single +(optional) positional argument: the name of a Gentoo stage 3 tarball to use to +produce the container image. If none is specified, the latest +`stage3-amd64-nomultilib-openrc` tarball is downloaded and used. ```sh -sh stage3-image.sh +sh image-build/gentoo-stage3.sh ``` -Use a specific stage3 tarball: + +## Base Image + +The Aimee OS base build image is a minimal environment based on Gentoo. It +includes Portage and a basic build toolchain. Building it requires a Stage 3 +image to bootstrap the environment. By default, +_aimee-os.org/gentoo/stage3-amd64-nomultilib-openrc:latest_ is used, but an +alternative image can be specified using the `-b`/`--base` argument to the +`base.sh` script. ```sh -sh stage3-image.sh amd64-systemd +sh image-build/base.sh ``` ## Crossdev Image -This image includes a cross-compilation toolchain, generated by _crossdev_. +This image includes a cross-compilation toolchain, generated by _crossdev_. It +is built using the `cross.sh` script. This script takes a `-t`/`--target` +argument, which specifies the toolchain to generate. The default is +`aarch64-unknown-linux-gnu`. This image is layered on top of the base image, +which can be specified using the `-b`/`--base` argument. By default, +_aimee-os.org/build/base:latest_ is used. ```sh -sh crossdev-image.sh +sh image-build/cross.sh ``` -Use the `--base` and `--target` command-line arguments to specify a base Stage -3 image (see above) and crossdev target, respectively, e.g.: +## Build Image + +The top-level image is based on the Crossdev image, and includes additional +tools for building an Aimee OS project. It is built using the `build.sh` script. +As with the Crossdev image build script, this script accepts `-t`/`--target` +and `-b`/`--build` arguments, which specify the cross-compilation toolchain and +base image later, respectively. ```sh -sh crossdev-image.sh --target armv7a-unknown-linux-gnueabihf +sh image-build/build.sh ``` + +This image is "squashed," meaning it consists of only a single layer. The Base +and Crossdev image layers are merged into the final layer when the image is +built. This reduces the final size of the image. diff --git a/build-image.sh b/build-image.sh deleted file mode 100644 index bec0f26..0000000 --- a/build-image.sh +++ /dev/null @@ -1,63 +0,0 @@ -#!/bin/sh - -target=aarch64-unknown-linux-gnu - -MIRROR=http://mirror.leaseweb.com/gentoo -URLPATH=releases/amd64/autobuilds -GENTOO_KEY=13EBBDBEDE7A12775DFDB1BABB572E0E2D182910 -GENTOO_KEYSERVER=hkps://keys.gentoo.org - -target=aarch64-unknown-linux-gnu -stage3=amd64-nomultilib-openrc -while [ $# -gt 0 ]; do - case "$1" in - -t|--target) - shift - target="$1" - ;; - -b|--base) - shift - stage3="$1" - ;; - *) - printf 'Unknown argument: %s\n' "$1" >&2 - exit 2 - ;; - esac -done - -set -e - -gpg --keyserver ${GENTOO_KEYSERVER} --recv-keys ${GENTOO_KEY} -curl -fLO "${MIRROR}/${URLPATH}/latest-stage3-${stage3}.txt" -gpg --verify "latest-stage3-${stage3}.txt" - -tarball=$(gpg --decrypt "latest-stage3-${stage3}.txt" | awk '$1!="#"{print $1}') -if [ ! -f "${tarball##*/}" ]; then - curl -fLO "${MIRROR}/${URLPATH}/${tarball}" -fi -if [ ! -f "${tarball##*/}.asc" ]; then - curl -fLO "${MIRROR}/${URLPATH}/${tarball}.asc" -fi -gpg --verify "${tarball##*/}.asc" - -cid=$(buildah from scratch) -buildah add ${cid} "${tarball##*/}" -buildah run \ - --mount type=cache,target=/var/db/repos/gentoo \ - --mount type=cache,target=/var/cache\ - -- ${cid} \ - sh -s - "${target}" \ - < build.sh \ -|| { - rc=$? - buildah rm ${cid} - exit $rc -} - -buildah config \ - --add-history \ - --cmd /bin/bash \ - --label target=${target} \ - ${cid} -buildah commit --rm --squash ${cid} aimee-os.org/build/cross-${target} diff --git a/common.sh b/common.sh deleted file mode 100644 index 0626b7a..0000000 --- a/common.sh +++ /dev/null @@ -1,15 +0,0 @@ -#!/bin/sh - -buildah_run_script() { - local cid script - cid=$1 - shift - script=$1 - shift - buildah run \ - --mount type=cache,target=/var/db/repos/gentoo \ - --mount type=cache,target=/var/cache \ - -- "${cid}" \ - sh -s - "$@" \ - < "${script}" -} diff --git a/crossdev-image.sh b/crossdev-image.sh deleted file mode 100644 index ca17886..0000000 --- a/crossdev-image.sh +++ /dev/null @@ -1,36 +0,0 @@ -#!/bin/sh - -SELF=$(readlink -f "$0") -SRCDIR=${SELF%/*} -. "${SRCDIR}"/common.sh - -target=aarch64-unknown-linux-gnu -stage3=amd64-nomultilib-openrc -while [ $# -gt 0 ]; do - case "$1" in - -t|--target) - shift - target="$1" - ;; - -b|--base) - shift - stage3="$1" - ;; - *) - printf 'Unknown argument: %s\n' "$1" >&2 - exit 2 - ;; - esac -done - -set -e - -cid=$(buildah from "aimee-os.org/gentoo/stage3-${stage3}") -buildah_run_script ${cid} sync.sh -buildah_run_script ${cid} crossdev.sh "${target}" -buildah config \ - --add-history \ - --cmd /bin/bash \ - --label target=${target} \ - ${cid} -buildah commit --rm ${cid} aimee-os.org/build/cross-${target} diff --git a/image-build/base.sh b/image-build/base.sh new file mode 100644 index 0000000..dfa1191 --- /dev/null +++ b/image-build/base.sh @@ -0,0 +1,39 @@ +#!/bin/sh + +SELF=$(readlink -f "$0") +SRCDIR=${SELF%/*/*} +. "${SRCDIR}"/lib/common.sh + +base=aimee-os.org/gentoo/stage3-amd64-nomultilib-openrc:latest +while [ $# -gt 0 ]; do + case "$1" in + -b|--base) + shift + base="$1" + ;; + *) + printf 'Unknown argument: %s\n' "$1" >&2 + exit 2 + ;; + esac +done + +set -e + +builddir=$(export TMPDIR=${TMPDIR:-${PWD}} && mktemp -d gentoo.XXXXXX) +builddir=$(readlink -f "${builddir}") + +work=$(buildah from -v "${builddir}":/mnt/gentoo:rw,Z "${base}") +buildah add "${work}" portage /etc/portage/ +buildah_run_script "${work}" "${SRCDIR}"/lib/sync.sh +buildah_run_script "${work}" "${SRCDIR}"/lib/bootstrap.sh + +cid=$(buildah from scratch) +buildah copy "${cid}" "${builddir}" / +buildah_run_script "${cid}" "${SRCDIR}"/lib/profile.sh +buildah config --cmd /bin/bash "${cid}" +buildah commit --rm --squash "${cid}" aimee-os.org/build/base + +buildah run "${work}" find /mnt/gentoo -mindepth 1 -delete +buildah rm "${work}" +rmdir "${builddir}" diff --git a/image-build/build.sh b/image-build/build.sh new file mode 100644 index 0000000..b78396d --- /dev/null +++ b/image-build/build.sh @@ -0,0 +1,35 @@ +#!/bin/sh + +SELF=$(readlink -f "$0") +SRCDIR=${SELF%/*/*} +. "${SRCDIR}"/lib/common.sh + +target=aarch64-unknown-linux-gnu +while [ $# -gt 0 ]; do + case "$1" in + -t|--target) + shift + target="$1" + ;; + -b|--base) + shift + base="$1" + ;; + *) + printf 'Unknown argument: %s\n' "$1" >&2 + exit 2 + ;; + esac +done + +if [ -z "${base-}" ]; then + base=aimee-os.org/build/cross-"${target}" +fi + +set -e + +cid=$(buildah from "${base}") +buildah add "${cid}" portage /etc/portage +buildah_run_script "${cid}" "${SRCDIR}"/lib/sync.sh +buildah_run_script "${cid}" "${SRCDIR}"/lib/tools.sh +buildah commit --rm --squash "${cid}" "aimee-os.org/build/build-${target}" diff --git a/image-build/cross.sh b/image-build/cross.sh new file mode 100644 index 0000000..607f571 --- /dev/null +++ b/image-build/cross.sh @@ -0,0 +1,31 @@ +#!/bin/sh + +SELF=$(readlink -f "$0") +SRCDIR=${SELF%/*/*} +. "${SRCDIR}"/lib/common.sh + +target=aarch64-unknown-linux-gnu +base=aimee-os.org/build/base +while [ $# -gt 0 ]; do + case "$1" in + -t|--target) + shift + target="$1" + ;; + -b|--base) + shift + base="$1" + ;; + *) + printf 'Unknown argument: %s\n' "$1" >&2 + exit 2 + ;; + esac +done + +set -e + +cid=$(buildah from "${base}") +buildah_run_script "${cid}" "${SRCDIR}"/lib/sync.sh +buildah_run_script "${cid}" "${SRCDIR}"/lib/crossdev.sh "${target}" +buildah commit --rm "${cid}" aimee-os.org/build/cross-"${target}" diff --git a/image-build/gentoo-stage3.sh b/image-build/gentoo-stage3.sh new file mode 100644 index 0000000..ef1573a --- /dev/null +++ b/image-build/gentoo-stage3.sh @@ -0,0 +1,22 @@ +#!/bin/sh + +SELF=$(readlink -f "$0") +SRCDIR=${SELF%/*/*} + +tarball=$1 + +set -e + +if [ -z "${tarball}" ]; then + tarball=$(sh "${SRCDIR}"/tools/fetch-stage3.sh "$@") +fi +stage3=${tarball%.tar.*} +version=${stage3##*-} +stage3=${stage3%-*} + +cid=$(buildah from scratch) +buildah add "${cid}" "${tarball}" +buildah config --cmd /bin/bash "${cid}" +buildah commit --rm --squash "${cid}" "aimee-os.org/gentoo/${stage3}:${version}" +buildah tag "aimee-os.org/gentoo/${stage3}:${version}" \ + "aimee-os.org/gentoo/${stage3}:latest" diff --git a/lib/bootstrap.sh b/lib/bootstrap.sh new file mode 100755 index 0000000..9dd0c76 --- /dev/null +++ b/lib/bootstrap.sh @@ -0,0 +1,29 @@ +#!/bin/sh + +set -e + +mkdir -p /mnt/gentoo + +USE=build \ +emerge -kvnuUDj --rebuilt-binaries=y --root=/mnt/gentoo sys-apps/baselayout + +# Portage itself only requires Python and Bash. Actually building +# packages requires a lot more. These dependencies are implicit because +# they are listed in @system. We do not want everything from @system, +# though, so we have to explicitly install several packages. +# +# At some point, we may end up with an Aimee OS profile for Portage. +# If/when that happens, we can override @system to remove the stuff we +# don't want, like virtual/service-manager and virtual/openssh. +xargs emerge -kvnuUDj --rebuilt-binaries=y --root=/mnt/gentoo --color=y < /var/db/repos/crossdev/profiles/repo_name @@ -15,6 +15,6 @@ masters = gentoo auto-sync = no EOF -emerge -j sys-devel/crossdev +emerge -kvnuUDj --rebuilt-binaries=y --color=y sys-devel/crossdev crossdev --stable -t "$1" diff --git a/lib/profile.sh b/lib/profile.sh new file mode 100755 index 0000000..2c4cdfc --- /dev/null +++ b/lib/profile.sh @@ -0,0 +1,11 @@ +#!/bin/sh + +arch=$(uname -m) +case ${arch} in +x86_64) arch=amd64 ;; +aarch64) arch=arm64 ;; +esac + +mkdir -p /etc/portage + +eselect profile set default/linux/"${arch}"/23.0/no-multilib diff --git a/lib/sync.sh b/lib/sync.sh new file mode 100755 index 0000000..ca5fc97 --- /dev/null +++ b/lib/sync.sh @@ -0,0 +1,8 @@ +#!/bin/sh + +if [ ! -e /var/db/repos/gentoo/metadata ]; then + emerge-webrsync +fi +if [ "$(find /var/db/repos/gentoo/metadata -newermt '-24 hours' | wc -l)" -eq 0 ]; then + emaint sync +fi diff --git a/lib/tools.sh b/lib/tools.sh new file mode 100755 index 0000000..94d786b --- /dev/null +++ b/lib/tools.sh @@ -0,0 +1,15 @@ +#!/bin/sh + +xargs -r emerge -vnuUj --rebuilt-binaries=y --color=y < (09 Feb 2023) +# Do not build binary packages for kernel sources +sys-kernel/gentoo-sources nobuildpkg +sys-kernel/raspberrypi-sources nobuildpkg diff --git a/portage/package.use/btrfs-progs b/portage/package.use/btrfs-progs new file mode 100644 index 0000000..eca7fed --- /dev/null +++ b/portage/package.use/btrfs-progs @@ -0,0 +1 @@ +sys-fs/btrfs-progs -man diff --git a/portage/package.use/git b/portage/package.use/git new file mode 100644 index 0000000..46e62d2 --- /dev/null +++ b/portage/package.use/git @@ -0,0 +1 @@ +dev-vcs/git -perl diff --git a/portage/package.use/grub b/portage/package.use/grub new file mode 100644 index 0000000..e032517 --- /dev/null +++ b/portage/package.use/grub @@ -0,0 +1 @@ +sys-boot/grub -* GRUB_PLATFORMS: -* diff --git a/portage/package.use/kernel b/portage/package.use/kernel new file mode 100644 index 0000000..5f091c9 --- /dev/null +++ b/portage/package.use/kernel @@ -0,0 +1,2 @@ +sys-kernel/gentoo-sources symlink +sys-kernel/raspberrypi-sources symlink diff --git a/portage/package.use/selinux b/portage/package.use/selinux new file mode 100644 index 0000000..14ea6e0 --- /dev/null +++ b/portage/package.use/selinux @@ -0,0 +1,2 @@ +sys-libs/libselinux python +sys-process/audit python diff --git a/portage/package.use/squashfs-tools b/portage/package.use/squashfs-tools new file mode 100644 index 0000000..27ddfb0 --- /dev/null +++ b/portage/package.use/squashfs-tools @@ -0,0 +1 @@ +sys-fs/squashfs-tools zstd diff --git a/portage/package.use/systemd b/portage/package.use/systemd new file mode 100644 index 0000000..a0d6dce --- /dev/null +++ b/portage/package.use/systemd @@ -0,0 +1,6 @@ +sys-apps/dbus systemd +sys-apps/systemd -* PYTHON_SINGLE_TARGET: python3_12 +sec-policy/selinux-base systemd +sec-policy/selinux-base-policy systemd +virtual/tmpfiles systemd +virtual/udev systemd diff --git a/sync.sh b/sync.sh deleted file mode 100644 index 599eb24..0000000 --- a/sync.sh +++ /dev/null @@ -1,10 +0,0 @@ -#!/bin/sh - -export FEATURES='-ipc-sandbox -network-sandbox -pid-sandbox' - -if [ ! -e /var/db/repos/gentoo/metadata ]; then - emaint sync -fi -if [ "$(find /var/db/repos/gentoo -newermt '-24 hours' | wc -l)" -eq 0 ]; then - emerge-webrsync -fi diff --git a/stage3-image.sh b/tools/fetch-stage3.sh similarity index 61% rename from stage3-image.sh rename to tools/fetch-stage3.sh index dee7a98..011f1f5 100644 --- a/stage3-image.sh +++ b/tools/fetch-stage3.sh @@ -1,9 +1,5 @@ #!/bin/sh -SELF=$(readlink -f "$0") -SRCDIR=${SELF%/*} -. "${SRCDIR}"/common.sh - : "${GENTOO_MIRROR:=http://mirror.leaseweb.com/gentoo}" URLPATH=releases/amd64/autobuilds GENTOO_KEY=13EBBDBEDE7A12775DFDB1BABB572E0E2D182910 @@ -14,7 +10,7 @@ stage3="${1:-amd64-nomultilib-openrc}" set -e gpg --keyserver ${GENTOO_KEYSERVER} --recv-keys ${GENTOO_KEY} -curl -fLO "${MIRROR}/${URLPATH}/latest-stage3-${stage3}.txt" +curl -fLO "${GENTOO_MIRROR}/${URLPATH}/latest-stage3-${stage3}.txt" gpg --verify "latest-stage3-${stage3}.txt" tarball=$(gpg --decrypt "latest-stage3-${stage3}.txt" | awk '$1!="#"{print $1}') @@ -25,13 +21,4 @@ if [ ! -f "${tarball##*/}.asc" ]; then curl -fLO "${GENTOO_MIRROR}/${URLPATH}/${tarball}.asc" fi gpg --verify "${tarball##*/}.asc" - -cid=$(buildah from scratch) -buildah add ${cid} "${tarball##*/}" -buildah_run_script ${cid} sync.sh -buildah_run_script ${cid} update.sh -buildah config \ - --add-history \ - --cmd /bin/bash \ - ${cid} -buildah commit --rm --squash ${cid} "aimee-os.org/gentoo/stage3-${stage3}" +echo "${tarball##*/}" diff --git a/update.sh b/update.sh deleted file mode 100644 index 1893b73..0000000 --- a/update.sh +++ /dev/null @@ -1,5 +0,0 @@ -#!/bin/sh - -export FEATURES='-ipc-sandbox -network-sandbox -pid-sandbox' - -emerge -uUDj @world