Rework images into three layers

* Base
* Crossdev
* Build

The *Base* layer is bootstrapped from a regular Gentoo stage 3
environment.  The *Build* layer is the primary artifact, and is
"squashed" into a single layer when built.

README.md

@@ -7,32 +7,56 @@ Prerequisites:
 
 ## Gentoo Stage 3 Image
 
-This image forms the base layer for Aimee OS build images.
+This image is used to bootstrap the base layer for Aimee OS build images.  To
-
+build it, use the `gentoo-stage3.sh` build script.  This script takes a single
-Use the default stage3 tarball (`stage3-amd64-nomultilib-openrc`):
+(optional) positional argument: the name of a Gentoo stage 3 tarball to use to
+produce the container image.  If none is specified, the latest
+`stage3-amd64-nomultilib-openrc` tarball is downloaded and used.
 
 ```sh
-sh stage3-image.sh
+sh image-build/gentoo-stage3.sh
 ```
 
-Use a specific stage3 tarball:
+
+## Base Image
+
+The Aimee OS base build image is a minimal environment based on Gentoo.  It
+includes Portage and a basic build toolchain.  Building it requires a Stage 3
+image to bootstrap the environment.  By default,
+_aimee-os.org/gentoo/stage3-amd64-nomultilib-openrc:latest_ is used, but an
+alternative image can be specified using the `-b`/`--base` argument to the
+`base.sh` script.
 
 ```sh
-sh stage3-image.sh amd64-systemd
+sh image-build/base.sh
 ```
 
 
 ## Crossdev Image
 
-This image includes a cross-compilation toolchain, generated by _crossdev_.
+This image includes a cross-compilation toolchain, generated by _crossdev_.  It
+is built using the `cross.sh` script.  This script takes a `-t`/`--target`
+argument, which specifies the toolchain to generate.  The default is
+`aarch64-unknown-linux-gnu`.  This image is layered on top of the base image,
+which can be specified using the `-b`/`--base` argument.  By default,
+_aimee-os.org/build/base:latest_ is used.
 
 ```sh
-sh crossdev-image.sh
+sh image-build/cross.sh
 ```
 
-Use the `--base` and `--target` command-line arguments to specify a base Stage
+## Build Image
-3 image (see above) and crossdev target, respectively, e.g.:
+
+The top-level image is based on the Crossdev image, and includes additional
+tools for building an Aimee OS project.  It is built using the `build.sh` script.
+As with the Crossdev image build script, this script accepts `-t`/`--target`
+and `-b`/`--build` arguments, which specify the cross-compilation toolchain and
+base image later, respectively.
 
 ```sh
-sh crossdev-image.sh --target armv7a-unknown-linux-gnueabihf
+sh image-build/build.sh
 ```
+
+This image is "squashed," meaning it consists of only a single layer.  The Base
+and Crossdev image layers are merged into the final layer when the image is
+built.  This reduces the final size of the image.

build-image.sh

@@ -1,63 +0,0 @@
-#!/bin/sh
-
-target=aarch64-unknown-linux-gnu
-
-MIRROR=http://mirror.leaseweb.com/gentoo
-URLPATH=releases/amd64/autobuilds
-GENTOO_KEY=13EBBDBEDE7A12775DFDB1BABB572E0E2D182910
-GENTOO_KEYSERVER=hkps://keys.gentoo.org
-
-target=aarch64-unknown-linux-gnu
-stage3=amd64-nomultilib-openrc
-while [ $# -gt 0 ]; do
-    case "$1" in
-    -t|--target)
-        shift
-        target="$1"
-        ;;
-    -b|--base)
-        shift
-        stage3="$1"
-        ;;
-    *)
-        printf 'Unknown argument: %s\n' "$1" >&2
-        exit 2
-        ;;
-    esac
-done
-
-set -e
-
-gpg --keyserver ${GENTOO_KEYSERVER} --recv-keys ${GENTOO_KEY}
-curl -fLO "${MIRROR}/${URLPATH}/latest-stage3-${stage3}.txt"
-gpg --verify "latest-stage3-${stage3}.txt"
-
-tarball=$(gpg --decrypt "latest-stage3-${stage3}.txt" | awk '$1!="#"{print $1}')
-if [ ! -f "${tarball##*/}" ]; then
-    curl -fLO "${MIRROR}/${URLPATH}/${tarball}"
-fi
-if [ ! -f "${tarball##*/}.asc" ]; then
-    curl -fLO "${MIRROR}/${URLPATH}/${tarball}.asc"
-fi
-gpg --verify "${tarball##*/}.asc"
-
-cid=$(buildah from scratch)
-buildah add ${cid} "${tarball##*/}"
-buildah run \
-    --mount type=cache,target=/var/db/repos/gentoo \
-    --mount type=cache,target=/var/cache\
-    -- ${cid} \
-    sh -s - "${target}" \
-    < build.sh \
-|| {
-    rc=$?
-    buildah rm ${cid}
-    exit $rc
-}
-
-buildah config \
-    --add-history \
-    --cmd /bin/bash \
-    --label target=${target} \
-    ${cid}
-buildah commit --rm --squash  ${cid} aimee-os.org/build/cross-${target}

common.sh

@@ -1,15 +0,0 @@
-#!/bin/sh
-
-buildah_run_script() {
-    local cid script
-    cid=$1
-    shift
-    script=$1
-    shift
-    buildah run \
-        --mount type=cache,target=/var/db/repos/gentoo \
-        --mount type=cache,target=/var/cache \
-        -- "${cid}" \
-        sh -s - "$@" \
-        < "${script}"
-}

crossdev-image.sh

@@ -1,36 +0,0 @@
-#!/bin/sh
-
-SELF=$(readlink -f "$0")
-SRCDIR=${SELF%/*}
-. "${SRCDIR}"/common.sh
-
-target=aarch64-unknown-linux-gnu
-stage3=amd64-nomultilib-openrc
-while [ $# -gt 0 ]; do
-    case "$1" in
-    -t|--target)
-        shift
-        target="$1"
-        ;;
-    -b|--base)
-        shift
-        stage3="$1"
-        ;;
-    *)
-        printf 'Unknown argument: %s\n' "$1" >&2
-        exit 2
-        ;;
-    esac
-done
-
-set -e
-
-cid=$(buildah from "aimee-os.org/gentoo/stage3-${stage3}")
-buildah_run_script ${cid} sync.sh
-buildah_run_script ${cid} crossdev.sh "${target}"
-buildah config \
-    --add-history \
-    --cmd /bin/bash \
-    --label target=${target} \
-    ${cid}
-buildah commit --rm ${cid} aimee-os.org/build/cross-${target}

image-build/base.sh

@@ -0,0 +1,39 @@
+#!/bin/sh
+
+SELF=$(readlink -f "$0")
+SRCDIR=${SELF%/*/*}
+. "${SRCDIR}"/lib/common.sh
+
+base=aimee-os.org/gentoo/stage3-amd64-nomultilib-openrc:latest
+while [ $# -gt 0 ]; do
+    case "$1" in
+    -b|--base)
+        shift
+        base="$1"
+        ;;
+    *)
+        printf 'Unknown argument: %s\n' "$1" >&2
+        exit 2
+        ;;
+    esac
+done
+
+set -e
+
+builddir=$(export TMPDIR=${TMPDIR:-${PWD}} && mktemp -d gentoo.XXXXXX)
+builddir=$(readlink -f "${builddir}")
+
+work=$(buildah from -v "${builddir}":/mnt/gentoo:rw,Z "${base}")
+buildah add "${work}" portage /etc/portage/
+buildah_run_script "${work}" "${SRCDIR}"/lib/sync.sh
+buildah_run_script "${work}" "${SRCDIR}"/lib/bootstrap.sh
+
+cid=$(buildah from scratch)
+buildah copy "${cid}" "${builddir}" /
+buildah_run_script "${cid}" "${SRCDIR}"/lib/profile.sh
+buildah config --cmd /bin/bash "${cid}"
+buildah commit --rm --squash "${cid}" aimee-os.org/build/base
+
+buildah run "${work}" find /mnt/gentoo -mindepth 1 -delete
+buildah rm "${work}"
+rmdir "${builddir}"

image-build/build.sh

@@ -0,0 +1,35 @@
+#!/bin/sh
+
+SELF=$(readlink -f "$0")
+SRCDIR=${SELF%/*/*}
+. "${SRCDIR}"/lib/common.sh
+
+target=aarch64-unknown-linux-gnu
+while [ $# -gt 0 ]; do
+    case "$1" in
+    -t|--target)
+        shift
+        target="$1"
+        ;;
+    -b|--base)
+        shift
+        base="$1"
+        ;;
+    *)
+        printf 'Unknown argument: %s\n' "$1" >&2
+        exit 2
+        ;;
+    esac
+done
+
+if [ -z "${base-}" ]; then
+    base=aimee-os.org/build/cross-"${target}"
+fi
+
+set -e
+
+cid=$(buildah from "${base}")
+buildah add "${cid}" portage /etc/portage
+buildah_run_script "${cid}" "${SRCDIR}"/lib/sync.sh
+buildah_run_script "${cid}" "${SRCDIR}"/lib/tools.sh
+buildah commit --rm --squash "${cid}" "aimee-os.org/build/build-${target}"

image-build/cross.sh

@@ -0,0 +1,31 @@
+#!/bin/sh
+
+SELF=$(readlink -f "$0")
+SRCDIR=${SELF%/*/*}
+. "${SRCDIR}"/lib/common.sh
+
+target=aarch64-unknown-linux-gnu
+base=aimee-os.org/build/base
+while [ $# -gt 0 ]; do
+    case "$1" in
+    -t|--target)
+        shift
+        target="$1"
+        ;;
+    -b|--base)
+        shift
+        base="$1"
+        ;;
+    *)
+        printf 'Unknown argument: %s\n' "$1" >&2
+        exit 2
+        ;;
+    esac
+done
+
+set -e
+
+cid=$(buildah from "${base}")
+buildah_run_script "${cid}" "${SRCDIR}"/lib/sync.sh
+buildah_run_script "${cid}" "${SRCDIR}"/lib/crossdev.sh "${target}"
+buildah commit --rm "${cid}" aimee-os.org/build/cross-"${target}"

image-build/gentoo-stage3.sh

@@ -0,0 +1,22 @@
+#!/bin/sh
+
+SELF=$(readlink -f "$0")
+SRCDIR=${SELF%/*/*}
+
+tarball=$1
+
+set -e
+
+if [ -z "${tarball}" ]; then
+    tarball=$(sh "${SRCDIR}"/tools/fetch-stage3.sh "$@")
+fi
+stage3=${tarball%.tar.*}
+version=${stage3##*-}
+stage3=${stage3%-*}
+
+cid=$(buildah from scratch)
+buildah add "${cid}" "${tarball}"
+buildah config --cmd /bin/bash "${cid}"
+buildah commit --rm --squash "${cid}" "aimee-os.org/gentoo/${stage3}:${version}"
+buildah tag "aimee-os.org/gentoo/${stage3}:${version}" \
+    "aimee-os.org/gentoo/${stage3}:latest"

lib/bootstrap.sh

@@ -0,0 +1,29 @@
+#!/bin/sh
+
+set -e
+
+mkdir -p /mnt/gentoo
+
+USE=build \
+emerge -kvnuUDj --rebuilt-binaries=y --root=/mnt/gentoo sys-apps/baselayout
+
+# Portage itself only requires Python and Bash.  Actually building
+# packages requires a lot more.  These dependencies are implicit because
+# they are listed in @system.  We do not want everything from @system,
+# though, so we have to explicitly install several packages.
+#
+# At some point, we may end up with an Aimee OS profile for Portage.
+# If/when that happens, we can override @system to remove the stuff we
+# don't want, like virtual/service-manager and virtual/openssh.
+xargs emerge -kvnuUDj --rebuilt-binaries=y --root=/mnt/gentoo --color=y <<EOF
+app-arch/gzip
+dev-build/make
+net-misc/wget
+sys-apps/diffutils
+sys-apps/gawk
+sys-apps/grep
+sys-apps/portage
+sys-devel/binutils
+sys-devel/gcc
+sys-devel/patch
+EOF

lib/common.sh

@@ -0,0 +1,17 @@
+#!/bin/sh
+
+buildah_run_script() {
+    local cid script
+    cid=$1
+    shift
+    script=$(readlink -f "$1")
+    shift
+    buildah run \
+        --mount type=cache,target=/var/db/repos/gentoo \
+        --mount type=cache,target=/var/cache \
+        --mount type=bind,src="${script}",target=/run/script,ro,z \
+        --tty=false \
+        -e FEATURES='-ipc-sandbox -network-sandbox -pid-sandbox' \
+        "${cid}" \
+        /run/script "$@"
+}

lib/crossdev.sh

@@ -1,6 +1,6 @@
 #!/bin/sh
 
-export FEATURES='-ipc-sandbox -network-sandbox -pid-sandbox'
+set -e
 
 mkdir -p /var/db/repos/crossdev/profiles /var/db/repos/crossdev/metadata
 echo crossdev > /var/db/repos/crossdev/profiles/repo_name
@@ -15,6 +15,6 @@ masters = gentoo
 auto-sync = no
 EOF
 
-emerge -j sys-devel/crossdev
+emerge -kvnuUDj --rebuilt-binaries=y --color=y sys-devel/crossdev
 
 crossdev --stable -t "$1"

lib/profile.sh

@@ -0,0 +1,11 @@
+#!/bin/sh
+
+arch=$(uname -m)
+case ${arch} in
+x86_64) arch=amd64 ;;
+aarch64) arch=arm64 ;;
+esac
+
+mkdir -p /etc/portage
+
+eselect profile set default/linux/"${arch}"/23.0/no-multilib

lib/sync.sh

@@ -0,0 +1,8 @@
+#!/bin/sh
+
+if [ ! -e /var/db/repos/gentoo/metadata ]; then
+    emerge-webrsync
+fi
+if [ "$(find /var/db/repos/gentoo/metadata -newermt '-24 hours' | wc -l)" -eq 0 ]; then
+    emaint sync
+fi

lib/tools.sh

@@ -0,0 +1,15 @@
+#!/bin/sh
+
+xargs -r emerge -vnuUj --rebuilt-binaries=y --color=y <<EOF
+sec-policy/selinux-base
+sys-apps/policycoreutils
+sys-apps/kmod
+sys-apps/shadow
+sys-apps/systemd
+sys-boot/grub
+sys-fs/btrfs-progs
+sys-fs/dosfstools
+sys-fs/genimage
+sys-fs/mtools
+sys-fs/squashfs-tools
+EOF

portage/env/nobuildpkg

@@ -0,0 +1 @@
+FEATURES=-buildpkg

portage/make.conf/15-keywords.conf

@@ -0,0 +1 @@
+ACCEPT_KEYWORDS="${ARCH}"

portage/make.conf/20-binpkgs.conf

@@ -0,0 +1,2 @@
+FEATURES="${FEATURES} binpkg-multi-instance buildpkg"
+EMERGE_DEFAULT_OPTS="${EMERGE_DEFAULT_OPTS} --usepkg --binpkg-respect-use=y"

portage/make.conf/80-quiet.conf

@@ -0,0 +1,2 @@
+EMERGE_DEFAULT_OPTS="${EMERGE_DEFAULT_OPTS} --quiet-build=y --quiet-fail=y"
+FEATURES="${FEATURES} -news"

portage/package.accept_keywords/genimage

@@ -0,0 +1 @@
+sys-fs/genimage ~amd64

portage/package.accept_keywords/kernel

@@ -0,0 +1 @@
+sys-kernel/raspberrypi-sources **

portage/package.env/kernel

@@ -0,0 +1,6 @@
+# vim: set ft=gentoo-package-use :
+
+# Dustin C. Hatch <dustin@hatch.name> (09 Feb 2023)
+# Do not build binary packages for kernel sources
+sys-kernel/gentoo-sources nobuildpkg
+sys-kernel/raspberrypi-sources nobuildpkg

portage/package.use/btrfs-progs

@@ -0,0 +1 @@
+sys-fs/btrfs-progs -man

portage/package.use/git

@@ -0,0 +1 @@
+dev-vcs/git -perl

portage/package.use/grub

@@ -0,0 +1 @@
+sys-boot/grub -* GRUB_PLATFORMS: -*

portage/package.use/kernel

@@ -0,0 +1,2 @@
+sys-kernel/gentoo-sources symlink
+sys-kernel/raspberrypi-sources symlink

portage/package.use/selinux

@@ -0,0 +1,2 @@
+sys-libs/libselinux python
+sys-process/audit python

portage/package.use/squashfs-tools

@@ -0,0 +1 @@
+sys-fs/squashfs-tools zstd

portage/package.use/systemd

@@ -0,0 +1,6 @@
+sys-apps/dbus systemd
+sys-apps/systemd -* PYTHON_SINGLE_TARGET: python3_12
+sec-policy/selinux-base systemd
+sec-policy/selinux-base-policy systemd
+virtual/tmpfiles systemd
+virtual/udev systemd

sync.sh

@@ -1,10 +0,0 @@
-#!/bin/sh
-
-export FEATURES='-ipc-sandbox -network-sandbox -pid-sandbox'
-
-if [ ! -e /var/db/repos/gentoo/metadata ]; then
-    emaint sync
-fi
-if [ "$(find /var/db/repos/gentoo -newermt '-24 hours' | wc -l)" -eq 0 ]; then
-    emerge-webrsync
-fi

tools/fetch-stage3.sh

@@ -1,9 +1,5 @@
 #!/bin/sh
 
-SELF=$(readlink -f "$0")
-SRCDIR=${SELF%/*}
-. "${SRCDIR}"/common.sh
-
 : "${GENTOO_MIRROR:=http://mirror.leaseweb.com/gentoo}"
 URLPATH=releases/amd64/autobuilds
 GENTOO_KEY=13EBBDBEDE7A12775DFDB1BABB572E0E2D182910
@@ -14,7 +10,7 @@ stage3="${1:-amd64-nomultilib-openrc}"
 set -e
 
 gpg --keyserver ${GENTOO_KEYSERVER} --recv-keys ${GENTOO_KEY}
-curl -fLO "${MIRROR}/${URLPATH}/latest-stage3-${stage3}.txt"
+curl -fLO "${GENTOO_MIRROR}/${URLPATH}/latest-stage3-${stage3}.txt"
 gpg --verify "latest-stage3-${stage3}.txt"
 
 tarball=$(gpg --decrypt "latest-stage3-${stage3}.txt" | awk '$1!="#"{print $1}')
@@ -25,13 +21,4 @@ if [ ! -f "${tarball##*/}.asc" ]; then
     curl -fLO "${GENTOO_MIRROR}/${URLPATH}/${tarball}.asc"
 fi
 gpg --verify "${tarball##*/}.asc"
-
+echo "${tarball##*/}"
-cid=$(buildah from scratch)
-buildah add ${cid} "${tarball##*/}"
-buildah_run_script ${cid} sync.sh
-buildah_run_script ${cid} update.sh
-buildah config \
-    --add-history \
-    --cmd /bin/bash \
-    ${cid}
-buildah commit --rm --squash ${cid} "aimee-os.org/gentoo/stage3-${stage3}"

update.sh

@@ -1,5 +0,0 @@
-#!/bin/sh
-
-export FEATURES='-ipc-sandbox -network-sandbox -pid-sandbox'
-
-emerge -uUDj @world