build-sys: Many changes

.dockerignore

@@ -15,6 +15,8 @@
 !docs/
 # We use the spec file
 !contrib/
+# This is used to add content on top of our default base
+!hack/
 # The systemd units and baseimage bits end up in installs
 !systemd/
 !baseimage/

.github/workflows/build-and-publish.yml

@@ -42,7 +42,9 @@ jobs:
           fi
 
       - name: Build container
-        run: just build-integration-test-image
+        # TODO: Also consider building + publishing an image that is just "base + bootc"
+        # as this implicitly changed to also publish our integration test images.
+        run: just build
 
       - name: Login to ghcr.io
         uses: redhat-actions/podman-login@v1

.github/workflows/ci.yml

@@ -85,9 +85,9 @@ jobs:
             --tmpfs /var/lib/containers \
             -v /run/dbus:/run/dbus -v /run/systemd:/run/systemd localhost/bootc /src/crates/ostree-ext/ci/priv-integration.sh
           # Nondestructive but privileged tests
-          sudo bootc-integration-tests host-privileged localhost/bootc-integration-install
+          sudo bootc-integration-tests host-privileged localhost/bootc-install
           # Install tests
-          sudo bootc-integration-tests install-alongside localhost/bootc-integration-install
+          sudo bootc-integration-tests install-alongside localhost/bootc-install
 
           # system-reinstall-bootc tests
           cargo build --release -p system-reinstall-bootc
@@ -97,7 +97,7 @@ jobs:
 
           sudo install -m 0755 target/release/system-reinstall-bootc /usr/bin/system-reinstall-bootc
           # These tests may mutate the system live so we can't run in parallel
-          sudo bootc-integration-tests system-reinstall localhost/bootc-integration --test-threads=1
+          sudo bootc-integration-tests system-reinstall localhost/bootc --test-threads=1
 
           # And the fsverity case
           sudo podman run --privileged --pid=host localhost/bootc-fsverity bootc install to-existing-root --stateroot=other \
@@ -189,9 +189,9 @@ jobs:
 
       - name: Build container
         run: |
-          just build-integration-test-image-from-package target/packages
+          BOOTC_SKIP_PACKAGE=1 just build
           # Extra cross-check (duplicating the integration test) that we're using the right base
-          used_vid=$(podman run --rm localhost/bootc-integration bash -c '. /usr/lib/os-release && echo ${ID}-${VERSION_ID}')
+          used_vid=$(podman run --rm localhost/bootc bash -c '. /usr/lib/os-release && echo ${ID}-${VERSION_ID}')
           test ${{ matrix.test_os }} = "${used_vid}"
 
       - name: Unit and container integration tests

CONTRIBUTING.md

@@ -129,7 +129,7 @@ There's a small set of tests which are designed to run inside a bootc container
 and are built into the default container image:
 
 ```
-$ podman run --rm -ti localhost/bootc bootc-integration-tests container
+$ just test-container
 ```
 
 ## Submitting a patch

Dockerfile

@@ -15,14 +15,10 @@ COPY . /src
 FROM scratch as packaging
 COPY contrib/packaging /
 
-FROM $base as base
-# Mark this as a test image (moved from --label build flag to fix layer caching)
-LABEL bootc.testimage="1"
-
 # This image installs build deps, pulls in our source code, and installs updated
 # bootc binaries in /out. The intention is that the target rootfs is extracted from /out
 # back into a final stage (without the build deps etc) below.
-FROM base as buildroot
+FROM $base as buildroot
 # Flip this off to disable initramfs code
 ARG initramfs=1
 # This installs our buildroot, and we want to cache it independently of the rest.
@@ -40,8 +36,39 @@ FROM buildroot as sdboot-content
 # Writes to /out
 RUN /src/contrib/packaging/configure-systemdboot download
 
+# We always do a "from scratch" build
+# https://docs.fedoraproject.org/en-US/bootc/building-from-scratch/
+# because this fixes https://github.com/containers/composefs-rs/issues/132
+# NOTE: Until we have https://gitlab.com/fedora/bootc/base-images/-/merge_requests/317
+#       this stage will end up capturing whatever RPMs we find at this time.
+# NOTE: This is using the *stock* bootc binary, not the one we want to build from
+#       local sources. We'll override it later.
+# NOTE: All your base belong to me.
+FROM $base as target-base
+RUN /usr/libexec/bootc-base-imagectl build-rootfs --manifest=standard /target-rootfs
+
+FROM scratch as base
+COPY --from=target-base /target-rootfs/ /
+COPY --from=src /src/hack/ /run/hack/
+RUN cd /run/hack/ && ./provision-derived.sh
+# Note we don't do any customization here yet
+# Mark this as a test image
+LABEL bootc.testimage="1"
+# Otherwise standard metadata
+LABEL containers.bootc 1
+LABEL ostree.bootable 1
+# https://pagure.io/fedora-kiwi-descriptions/pull-request/52
+ENV container=oci
+# Optional labels that only apply when running this image as a container. These keep the default entry point running under systemd.
+STOPSIGNAL SIGRTMIN+3
+CMD ["/sbin/init"]
+
+# -------------
+# external dependency cutoff point:
 # NOTE: Every RUN instruction past this point should use `--network=none`; we want to ensure
 # all external dependencies are clearly delineated.
+# This is verified in `cargo xtask check-buildsys`.
+# -------------
 
 FROM buildroot as build
 # Version for RPM build (optional, computed from git in Justfile)
@@ -50,7 +77,7 @@ ARG pkgversion
 ARG SOURCE_DATE_EPOCH
 ENV SOURCE_DATE_EPOCH=${SOURCE_DATE_EPOCH}
 # Build RPM directly from source, using cached target directory
-RUN --mount=type=cache,target=/src/target --mount=type=cache,target=/var/roothome --network=none RPM_VERSION="${pkgversion}" /src/contrib/packaging/build-rpm
+RUN --network=none --mount=type=cache,target=/src/target --mount=type=cache,target=/var/roothome RPM_VERSION="${pkgversion}" /src/contrib/packaging/build-rpm
 
 FROM buildroot as sdboot-signed
 # The secureboot key and cert are passed via Justfile
@@ -66,11 +93,11 @@ FROM build as units
 # A place that we're more likely to be able to set xattrs
 VOLUME /var/tmp
 ENV TMPDIR=/var/tmp
-RUN --mount=type=cache,target=/src/target --mount=type=cache,target=/var/roothome --network=none make install-unit-tests
+RUN --network=none --mount=type=cache,target=/src/target --mount=type=cache,target=/var/roothome make install-unit-tests
 
 # This just does syntax checking
 FROM buildroot as validate
-RUN --mount=type=cache,target=/src/target --mount=type=cache,target=/var/roothome --network=none make validate
+RUN --network=none --mount=type=cache,target=/src/target --mount=type=cache,target=/var/roothome make validate
 
 # Common base for final images: configures variant, rootfs, and injects extra content
 FROM base as final-common
@@ -80,22 +107,12 @@ RUN --network=none --mount=type=bind,from=packaging,target=/run/packaging \
     --mount=type=bind,from=sdboot-signed,target=/run/sdboot-signed \
     /run/packaging/configure-variant "${variant}"
 ARG rootfs=""
-RUN --mount=type=bind,from=packaging,target=/run/packaging /run/packaging/configure-rootfs "${variant}" "${rootfs}"
+RUN --network=none --mount=type=bind,from=packaging,target=/run/packaging /run/packaging/configure-rootfs "${variant}" "${rootfs}"
 COPY --from=packaging /usr-extras/ /usr/
 
-# Default target for source builds (just build)
-# Installs packages from the internal build stage
+# Final target: installs pre-built packages from /run/packages volume mount.
+# Use with: podman build --target=final -v path/to/packages:/run/packages:ro
 FROM final-common as final
-RUN --mount=type=bind,from=packaging,target=/run/packaging \
-    --mount=type=bind,from=build,target=/build-output \
-    --network=none \
-    /run/packaging/install-rpm-and-setup /build-output/out
-RUN bootc container lint --fatal-warnings
-
-# Alternative target for pre-built packages (CI workflow)
-# Use with: podman build --target=final-from-packages -v path/to/packages:/run/packages:ro
-FROM final-common as final-from-packages
-RUN --mount=type=bind,from=packaging,target=/run/packaging \
-    --network=none \
+RUN --network=none --mount=type=bind,from=packaging,target=/run/packaging \
     /run/packaging/install-rpm-and-setup /run/packages
-RUN bootc container lint --fatal-warnings
+RUN --network=none bootc container lint --fatal-warnings

Justfile

@@ -17,10 +17,8 @@
 
 # This image is just the base image plus our updated bootc binary
 base_img := "localhost/bootc"
-# Derives from the above and adds nushell, cloudinit etc.
-integration_img := base_img + "-integration"
 # Has a synthetic upgrade
-integration_upgrade_img := integration_img + "-upgrade"
+upgrade_img := base_img + "-upgrade"
 
 # ostree: The default
 # composefs-sealeduki-sdboot: A system with a sealed composefs using systemd-boot
@@ -43,10 +41,34 @@ lbi_images := "quay.io/curl/curl:latest quay.io/curl/curl-base:latest registry.a
 generic_buildargs := ""
 # Args for package building (no secrets needed, just builds RPMs)
 base_buildargs := generic_buildargs + " --build-arg=base=" + base + " --build-arg=variant=" + variant
-buildargs := base_buildargs + " --secret=id=secureboot_key,src=target/test-secureboot/db.key --secret=id=secureboot_cert,src=target/test-secureboot/db.crt"
+# - scratch builds need extra perms per https://docs.fedoraproject.org/en-US/bootc/building-from-scratch/
+# - we do secure boot signing here, so provide the keys
+buildargs := base_buildargs \
+             + " --cap-add=all --security-opt=label=type:container_runtime_t --device /dev/fuse" \
+             + " --secret=id=secureboot_key,src=target/test-secureboot/db.key --secret=id=secureboot_cert,src=target/test-secureboot/db.crt"
 # Args for build-sealed (no base arg, it sets that itself)
 sealed_buildargs := "--build-arg=variant=" + variant + " --secret=id=secureboot_key,src=target/test-secureboot/db.key --secret=id=secureboot_cert,src=target/test-secureboot/db.crt"
 
+# The default target: build the container image from current sources.
+# Note commonly you might want to override the base image via e.g.
+# `just build --build-arg=base=quay.io/fedora/fedora-bootc:42`
+# into the container image.
+#
+# Note you can set `BOOTC_SKIP_PACKAGE=1` in the environment to bypass this stage. 
+build: package _keygen && _pull-lbi-images
+    #!/bin/bash
+    set -xeuo pipefail
+    test -d target/packages
+    # Resolve to absolute path for podman volume mount
+    # Use :z for SELinux relabeling
+    pkg_path=$(realpath target/packages)
+    podman build --target=final -v "${pkg_path}":/run/packages:ro,z -t {{base_img}}-bin {{buildargs}} .
+    ./hack/build-sealed {{variant}} {{base_img}}-bin {{base_img}} {{sealed_buildargs}}
+
+# Pull images used by hack/lbi
+_pull-lbi-images:
+    podman pull -q --retry 5 --retry-delay 5s {{lbi_images}}
+
 # Compute SOURCE_DATE_EPOCH and VERSION from git for reproducible builds.
 # Outputs shell variable assignments that can be eval'd.
 _git-build-vars:
@@ -66,22 +88,6 @@ _git-build-vars:
     echo "SOURCE_DATE_EPOCH=${SOURCE_DATE_EPOCH}"
     echo "VERSION=${VERSION}"
 
-# The default target: build the container image from current sources.
-# Note commonly you might want to override the base image via e.g.
-# `just build --build-arg=base=quay.io/fedora/fedora-bootc:42`
-#
-# The Dockerfile builds RPMs internally in its 'build' stage, so we don't need
-# to call 'package' first. This avoids cache invalidation from external files.
-build: _keygen
-    #!/bin/bash
-    set -xeuo pipefail
-    eval $(just _git-build-vars)
-    podman build {{base_buildargs}} --target=final \
-        --build-arg=SOURCE_DATE_EPOCH=${SOURCE_DATE_EPOCH} \
-        --build-arg=pkgversion=${VERSION} \
-        -t {{base_img}}-bin {{buildargs}} .
-    ./hack/build-sealed {{variant}} {{base_img}}-bin {{base_img}} {{sealed_buildargs}}
-
 # Generate Secure Boot keys (only for our own CI/testing)
 _keygen:
     ./hack/generate-secureboot-keys
@@ -90,74 +96,37 @@ _keygen:
 build-sealed:
     @just --justfile {{justfile()}} variant=composefs-sealeduki-sdboot build
 
-# Build packages (e.g. RPM) using a container buildroot
-_packagecontainer:
-    #!/bin/bash
-    set -xeuo pipefail
-    eval $(just _git-build-vars)
-    echo "Building RPM with version: ${VERSION}"
-    podman build {{base_buildargs}} --build-arg=SOURCE_DATE_EPOCH=${SOURCE_DATE_EPOCH} --build-arg=pkgversion=${VERSION} -t localhost/bootc-pkg --target=build .
-
 # Build packages (e.g. RPM) into target/packages/
 # Any old packages will be removed.
-package: _packagecontainer
-    mkdir -p target/packages
-    rm -vf target/packages/*.rpm
-    podman run --rm localhost/bootc-pkg tar -C /out/ -cf - . | tar -C target/packages/ -xvf -
-    chmod a+rx target target/packages
-    chmod a+r target/packages/*.rpm
-    podman rmi localhost/bootc-pkg
-
-# Copy pre-existing packages from PATH into target/packages/
-# Note: This is mainly for CI artifact extraction; build-from-package
-# now uses volume mounts directly instead of copying to target/packages/.
-copy-packages-from PATH:
+# Set BOOTC_SKIP_PACKAGE=1 in the environment to bypass this stage. We don't
+# yet have an accurate ability to avoid rebuilding this in CI yet.
+package:
     #!/bin/bash
     set -xeuo pipefail
-    if ! compgen -G "{{PATH}}/*.rpm" > /dev/null; then
-        echo "Error: No packages found in {{PATH}}" >&2
-        exit 1
+    packages=target/packages
+    if test -n "${BOOTC_SKIP_PACKAGE:-}"; then
+        if test '!' -d "${packages}"; then
+            echo "BOOTC_SKIP_PACKAGE is set, but missing ${packages}" 1>&2; exit 1
+        fi
+        exit 0
     fi
-    mkdir -p target/packages
-    rm -vf target/packages/*.rpm
-    cp -v {{PATH}}/*.rpm target/packages/
-    chmod a+rx target target/packages
-    chmod a+r target/packages/*.rpm
-
-# Build the container image using pre-existing packages from PATH
-# Uses the 'final-from-packages' target with a volume mount to inject packages,
-# avoiding Docker context cache invalidation issues.
-build-from-package PATH: _keygen
-    #!/bin/bash
-    set -xeuo pipefail
-    # Resolve to absolute path for podman volume mount
-    # Use :z for SELinux relabeling
-    pkg_path=$(realpath "{{PATH}}")
-    podman build {{base_buildargs}} --target=final-from-packages -v "${pkg_path}":/run/packages:ro,z -t {{base_img}}-bin {{buildargs}} .
-    ./hack/build-sealed {{variant}} {{base_img}}-bin {{base_img}} {{sealed_buildargs}}
-
-# Pull images used by hack/lbi
-_pull-lbi-images:
-    podman pull -q --retry 5 --retry-delay 5s {{lbi_images}}
-
-# This container image has additional testing content and utilities
-build-integration-test-image: build _pull-lbi-images
-    cd hack && podman build {{base_buildargs}} -t {{integration_img}}-bin -f Containerfile .
-    ./hack/build-sealed {{variant}} {{integration_img}}-bin {{integration_img}} {{sealed_buildargs}}
-
-# Build integration test image using pre-existing packages from PATH
-build-integration-test-image-from-package PATH: _pull-lbi-images
-    @just build-from-package {{PATH}}
-    cd hack && podman build {{base_buildargs}} -t {{integration_img}}-bin -f Containerfile .
-    ./hack/build-sealed {{variant}} {{integration_img}}-bin {{integration_img}} {{sealed_buildargs}}
+    eval $(just _git-build-vars)
+    echo "Building RPM with version: ${VERSION}"
+    podman build {{base_buildargs}} --build-arg=SOURCE_DATE_EPOCH=${SOURCE_DATE_EPOCH} --build-arg=pkgversion=${VERSION} -t localhost/bootc-pkg --target=build .
+    mkdir -p "${packages}"
+    rm -vf "${packages}"/*.rpm
+    podman run --rm localhost/bootc-pkg tar -C /out/ -cf - . | tar -C "${packages}"/ -xvf -
+    chmod a+rx target "${packages}"
+    chmod a+r "${packages}"/*.rpm
+    # Keep localhost/bootc-pkg for layer caching; use `just clean-local-images` to reclaim space
 
 # Build+test using the `composefs-sealeduki-sdboot` variant.
 test-composefs:
     just variant=composefs-sealeduki-sdboot test-tmt readonly local-upgrade-reboot
 
 # Only used by ci.yml right now
-build-install-test-image: build-integration-test-image
-    cd hack && podman build {{base_buildargs}} -t {{integration_img}}-install -f Containerfile.drop-lbis
+build-install-test-image: build
+    cd hack && podman build {{base_buildargs}} -t {{base_img}}-install -f Containerfile.drop-lbis
 
 # These tests accept the container image as input, and may spawn it.
 run-container-external-tests:
@@ -179,28 +148,29 @@ validate:
 #
 # To run an individual test, pass it as an argument like:
 # `just test-tmt readonly`
-test-tmt *ARGS: build-integration-test-image _build-upgrade-image
+test-tmt *ARGS: build
+    @just _build-upgrade-image
     @just test-tmt-nobuild {{ARGS}}
 
 # Generate a local synthetic upgrade
 _build-upgrade-image:
-    cat tmt/tests/Dockerfile.upgrade | podman build -t {{integration_upgrade_img}}-bin --from={{integration_img}}-bin -
-    ./hack/build-sealed {{variant}} {{integration_upgrade_img}}-bin {{integration_upgrade_img}} {{sealed_buildargs}}
+    cat tmt/tests/Dockerfile.upgrade | podman build -t {{upgrade_img}}-bin --from={{base_img}}-bin -
+    ./hack/build-sealed {{variant}} {{upgrade_img}}-bin {{upgrade_img}} {{sealed_buildargs}}
 
-# Assume the localhost/bootc-integration image is up to date, and just run tests.
+# Assume the localhost/bootc image is up to date, and just run tests.
 # Useful for iterating on tests quickly.
 test-tmt-nobuild *ARGS:
-    cargo xtask run-tmt --env=BOOTC_variant={{variant}} --upgrade-image={{integration_upgrade_img}} {{integration_img}} {{ARGS}}
+    cargo xtask run-tmt --env=BOOTC_variant={{variant}} --upgrade-image={{upgrade_img}} {{base_img}} {{ARGS}}
 
 # Cleanup all test VMs created by tmt tests
 tmt-vm-cleanup:
     bcvk libvirt rm --stop --force --label bootc.test=1
 
 # Run tests (unit and integration) that are containerized
-test-container: build-units build-integration-test-image
+test-container: build build-units
     podman run --rm --read-only localhost/bootc-units /usr/bin/bootc-units
     # Pass these through for cross-checking
-    podman run --rm --env=BOOTC_variant={{variant}} --env=BOOTC_base={{base}} {{integration_img}} bootc-integration-tests container
+    podman run --rm --env=BOOTC_variant={{variant}} --env=BOOTC_base={{base}} {{base_img}} bootc-integration-tests container
 
 # Remove all container images built (locally) via this Justfile, by matching a label
 clean-local-images:

crates/ostree-ext/ci/priv-integration.sh

@@ -8,6 +8,7 @@ set -euo pipefail
 mkdir -p /var/tmp
 
 sysroot=/run/host
+repo="${sysroot}/ostree/repo"
 # Current stable image fixture
 image=quay.io/fedora/fedora-coreos:testing-devel
 imgref=ostree-unverified-registry:${image}
@@ -111,10 +112,6 @@ derived_img_dir=dir:/var/tmp/derived.dir
 systemd-run -dP --wait skopeo copy containers-storage:localhost/fcos-derived "${derived_img}"
 systemd-run -dP --wait skopeo copy "${derived_img}" "${derived_img_dir}"
 
-# Prune to reset state
-ostree refs ostree/container/image --delete
-
-repo="${sysroot}/ostree/repo"
 images=$(ostree container image list --repo "${repo}" | wc -l)
 test "${images}" -eq 1
 ostree container image deploy --sysroot "${sysroot}" \