From 4c0c7ce42f4c3040c1d0f6dd9b4e7276432f82af Mon Sep 17 00:00:00 2001 From: cyteen Date: Sun, 14 Jul 2024 23:46:08 +0100 Subject: [PATCH] Apply davids fallocate patch. --- scripts/build-boot-rpi | 3 +- scripts/build-debian | 3 +- scripts/build-debian.orig | 305 ++++++++++++++++++++++++++++++++++++++ scripts/build-image | 6 +- test/qemu.sh | 3 +- 5 files changed, 315 insertions(+), 5 deletions(-) create mode 100755 scripts/build-debian.orig diff --git a/scripts/build-boot-rpi b/scripts/build-boot-rpi index 223eb4d..991bcb4 100755 --- a/scripts/build-boot-rpi +++ b/scripts/build-boot-rpi @@ -73,7 +73,8 @@ sed -i -e "s/BOOT_UART=0/BOOT_UART=1/" bootcode.bin # Create empty FAT partition rm -f vfat.img -fallocate -l 28MiB vfat.img +# fallocate fails on some (git CI) COW filesystems, so have a coreutils alternative. +fallocate -l 28MiB vfat.img || truncate --size 28MiB vfat.img mkfs.vfat vfat.img # Copy boot files to FAT partition diff --git a/scripts/build-debian b/scripts/build-debian index 2fb0fb2..19d6a3e 100755 --- a/scripts/build-debian +++ b/scripts/build-debian @@ -295,7 +295,8 @@ mv debian/tmp/versions.csv versions.csv # >>> ((3800000000 - 32 * 1024 * 1024) // (1024 * 1024)) * (1024 * 1024) # 3765436416 rm -f ext4.img -fallocate -l 3765436416 ext4.img +# fallocate fails on some (git CI) COW filesystems, so have a coreutils alternative. +fallocate -l 3765436416 ext4.img || truncate --size=3765436416 ext4.img mkfs.ext4 -d debian ext4.img pigz ext4.img diff --git a/scripts/build-debian.orig b/scripts/build-debian.orig new file mode 100755 index 0000000..2fb0fb2 --- /dev/null +++ b/scripts/build-debian.orig @@ -0,0 +1,305 @@ +#!/bin/sh +set -x + +# Build Debian root filesystem + +OS="$1" # For example "debian" +ARCH="$2" # For example "armhf" +DIST="$3" # For example "buster" + +set -ex + +case "${OS}-${ARCH}" in +devuan-armhf) + KERNEL=linux-image-armmp + URL=http://deb.devuan.org/merged + # https://pkginfo.devuan.org/cgi-bin/policy-query.html?c=package&q=${KERNEL} + SCRIPT=/usr/share/debootstrap/scripts/ceres + ;; +devuan-arm64) + KERNEL=linux-image-arm64 + URL=http://deb.devuan.org/merged + # https://pkginfo.devuan.org/cgi-bin/policy-query.html?c=package&q=${KERNEL} + SCRIPT=/usr/share/debootstrap/scripts/ceres + ;; +devuan-i386) + KERNEL=linux-image-686 + URL=http://deb.devuan.org/merged + # https://pkginfo.devuan.org/cgi-bin/policy-query.html?c=package&q=${KERNEL} + SCRIPT=/usr/share/debootstrap/scripts/ceres + ;; +devuan-amd64) + KERNEL=linux-image-amd64 + URL=http://deb.devuan.org/merged + # https://pkginfo.devuan.org/cgi-bin/policy-query.html?c=package&q=${KERNEL} + SCRIPT=/usr/share/debootstrap/scripts/ceres + ;; +debian-armhf) + KERNEL=linux-image-armmp + URL=http://deb.debian.org/debian + KERNEL_URL=http://packages.debian.org/"${DIST}"/"${KERNEL}" + SCRIPT=/usr/share/debootstrap/scripts/sid + ;; +debian-arm64) + KERNEL=linux-image-arm64 + URL=http://deb.debian.org/debian + KERNEL_URL=http://packages.debian.org/"${DIST}"/"${KERNEL}" + SCRIPT=/usr/share/debootstrap/scripts/sid + ;; +debian-i386) + KERNEL=linux-image-686 + URL=http://deb.debian.org/debian + KERNEL_URL=http://packages.debian.org/"${DIST}"/"${KERNEL}" + SCRIPT=/usr/share/debootstrap/scripts/sid + ;; +debian-amd64) + KERNEL=linux-image-amd64 + URL=http://deb.debian.org/debian + KERNEL_URL=http://packages.debian.org/"${DIST}"/"${KERNEL}" + SCRIPT=/usr/share/debootstrap/scripts/sid + ;; +ubuntu-amd64) + KERNEL=linux-image-generic + URL=http://archive.ubuntu.com/ubuntu + KERNEL_URL=http://packages.ubuntu.com/"${DIST}"/"${KERNEL}" + SCRIPT=/usr/share/debootstrap/scripts/gutsy + ;; +ubuntu-armhf | ubuntu-arm64) + KERNEL=linux-image-generic + URL=http://ports.ubuntu.com/ubuntu-ports + KERNEL_URL=http://packages.ubuntu.com/"${DIST}"/"${KERNEL}" + SCRIPT=/usr/share/debootstrap/scripts/gutsy + ;; +*) + echo "Can't decide kernel package for \"${ARCH}\"" + exit 1 + ;; +esac + +case "${OS}-${ARCH}-${DIST}" in +debian-*-rc-buggy | debian-*-experimental) + TARGET=sid + KERNELSUITE="-t experimental" + ;; +*) + TARGET="${DIST}" + KERNELSUITE="" + ;; +esac + +# Create fresh empty directory +TMP=$(mktemp -d -p "${PWD}" tmp-XXXXXX) +trap 'rm -rf "${TMP}"' EXIT INT TERM +cd "${TMP}" + +# Build a Debian root filesystem (first stage) +case ${OS} in +devuan) + debootstrap \ + --arch="${ARCH}" \ + --verbose \ + --variant=minbase \ + --foreign \ + --include=netbase,net-tools,sysvinit,u-boot-tools,initramfs-tools,openssh-server,nano,vim-tiny \ + "${TARGET}" \ + debian \ + "${URL}" \ + "${SCRIPT}" + ;; +*) + debootstrap \ + --arch="${ARCH}" \ + --verbose \ + --variant=minbase \ + --foreign \ + --include=netbase,net-tools,systemd-sysv,u-boot-tools,initramfs-tools,openssh-server,nano \ + "${TARGET}" \ + debian \ + "${URL}" \ + "${SCRIPT}" + ;; +esac + +# Randomly generated root password +PASSWORD="${PASSWORD_OVERRIDE:-$(pwgen -B -A 6 1)}" + +# Write apt sources config +case "${OS}-${ARCH}-${DIST}" in +debian-*-rc-buggy | debian-*-experimental) + tee debian/tmp/sources.list <<-EOF + deb http://deb.debian.org/debian ${TARGET} main + deb-src http://deb.debian.org/debian ${TARGET} main + deb http://deb.debian.org/debian ${DIST} main + deb-src http://deb.debian.org/debian ${DIST} main + EOF + ;; +debian-*-sid | debian-*-unstable) + tee debian/tmp/sources.list <<-EOF + deb http://deb.debian.org/debian ${DIST} main + deb-src http://deb.debian.org/debian ${DIST} main + EOF + ;; +debian-*) + tee debian/tmp/sources.list <<-EOF + deb http://deb.debian.org/debian ${DIST} main + deb-src http://deb.debian.org/debian ${DIST} main + deb http://deb.debian.org/debian ${DIST}-updates main + deb-src http://deb.debian.org/debian ${DIST}-updates main + deb http://security.debian.org/debian-security ${DIST}-security main + deb-src http://security.debian.org/debian-security ${DIST}-security main + EOF + ;; +devuan-*-excalibur | devuan-*-freia) + # https://pkginfo.devuan.org/sources.list.txt + tee debian/tmp/sources.list <<-EOF + deb http://deb.devuan.org/merged ${DIST} main + deb-src http://deb.devuan.org/merged ${DIST} main + EOF + ;; +devuan-*-ceres | devuan-*-unstable) + # https://pkginfo.devuan.org/sources.list.txt + tee debian/tmp/sources.list <<-EOF + deb http://deb.devuan.org/merged ${DIST} main + deb-src http://deb.devuan.org/merged ${DIST} main + EOF + ;; +devuan-*) + tee debian/tmp/sources.list <<-EOF + deb http://deb.devuan.org/merged ${DIST} main + deb-src http://deb.devuan.org/merged ${DIST} main + deb http://deb.devuan.org/merged ${DIST}-updates main + deb-src http://deb.devuan.org/merged ${DIST}-updates main + deb http://deb.devuan.org/merged ${DIST}-security main + deb-src http://deb.devuan.org/merged ${DIST}-security main + EOF + ;; +devuan-*-buster) + tee debian/tmp/sources.list <<-EOF + deb http://deb.devuan.org/merged ${DIST} main + deb-src http://deb.devuan.org/merged ${DIST} main + deb http://deb.devuan.org/merged ${DIST}-updates main + deb-src http://deb.devuan.org/merged ${DIST}-updates main + deb http://deb.devuan.org/merged ${DIST} main + deb-src http://deb.devuan.org/merged ${DIST} main + EOF + ;; +ubuntu-amd64-*) + tee debian/tmp/sources.list <<-EOF + deb http://archive.ubuntu.com/ubuntu ${DIST} main universe + deb-src http://archive.ubuntu.com/ubuntu ${DIST} main universe + deb http://archive.ubuntu.com/ubuntu ${DIST}-updates main universe + deb-src http://archive.ubuntu.com/ubuntu ${DIST}-updates main universe + deb http://archive.ubuntu.com/ubuntu ${DIST}-security main universe + deb-src http://archive.ubuntu.com/ubuntu ${DIST}-security main universe + EOF + ;; +ubuntu-*) + tee debian/tmp/sources.list <<-EOF + deb http://ports.ubuntu.com/ubuntu-ports ${DIST} main universe + deb-src http://ports.ubuntu.com/ubuntu-ports ${DIST} main universe + deb http://ports.ubuntu.com/ubuntu-ports ${DIST}-updates main universe + deb-src http://ports.ubuntu.com/ubuntu-ports ${DIST}-updates main universe + deb http://ports.ubuntu.com/ubuntu-ports ${DIST}-security main universe + deb-src http://ports.ubuntu.com/ubuntu-ports ${DIST}-security main universe + EOF + ;; +esac + +case "${OS}-${ARCH}-${DIST}" in +debian-*-rc-buggy | debian-*-experimental) + # For Debian Experimental, set lower priority to "experimental" + tee debian/tmp/priority-experimental <<-EOF + Package: * + Pin: release o=Debian,a=experimental + Pin-Priority: 300 + EOF + ;; +*) ;; +esac + +cp -rv --preserve=mode ../2nd-stage-files/pre-2nd-stage-files/* debian +cp -rv --preserve=mode ../2nd-stage-files/pre-2nd-stage-files-${ARCH}/* debian + +# Copy ARM emulation stuff +cp -v /usr/bin/qemu-*-static debian/usr/bin || : + +case "${OS}" in +devuan) + # Build a Devuan root filesystem (second stage) + chroot debian /bin/sh -ex <<-EOF + /debootstrap/debootstrap --second-stage + /bin/mv /tmp/sources.list /etc/apt/sources.list + # /usr/bin/apt-key add /etc/apt/trusted.gpg.d/devuan_key.gpg + /usr/bin/apt-get update + /usr/bin/apt-get -y upgrade + /usr/bin/apt-get -y --no-install-recommends ${KERNELSUITE} install ${KERNEL} + /usr/bin/apt-get -y install connman || : + /usr/bin/apt-get -y install openntpd || : + /usr/bin/apt-get clean + /bin/rm -rf /var/lib/apt/lists/* + /usr/bin/dpkg-query --showformat='kernel,\${Version}\n' --show ${KERNEL} > /tmp/versions.csv + /bin/rm -f /var/log/*.log + /bin/echo root:${PASSWORD} | /usr/sbin/chpasswd + /bin/sed -i "s/#*\s*PermitRootLogin .*/PermitRootLogin yes/" /etc/ssh/sshd_config + EOF + ;; +debian | ubuntu | *) + # Build a Debian root filesystem (second stage) + chroot debian /bin/sh -ex <<-EOF + /debootstrap/debootstrap --second-stage + /bin/mv /tmp/sources.list /etc/apt/sources.list + /bin/mv /tmp/priority-experimental /etc/apt/preferences.d/priority-experimental || : + /usr/bin/apt-get update + /usr/bin/apt-get -y upgrade + /usr/bin/apt-get -y --no-install-recommends ${KERNELSUITE} install ${KERNEL} + /usr/bin/apt-get -y install systemd-timesyncd || : + /usr/bin/apt-get -y install systemd-resolved || : + /usr/bin/apt-get clean + /bin/rm -rf /var/lib/apt/lists/* + /usr/bin/dpkg-query --showformat='kernel,\${Version}\n' --show ${KERNEL} > /tmp/versions.csv + /usr/bin/systemctl enable systemd-networkd.service + /usr/bin/systemctl enable systemd-resolved.service + /usr/bin/systemctl enable systemd-timesyncd.service + /bin/rm -f /var/log/*.log + /bin/echo root:${PASSWORD} | /usr/sbin/chpasswd + /bin/sed -i "s/#*\s*PermitRootLogin .*/PermitRootLogin yes/" /etc/ssh/sshd_config + EOF + ;; +esac + +# Remove ARM emulation stuff again +rm -v debian/usr/bin/qemu-*-static || : + +cp -rv --preserve=mode ../2nd-stage-files/post-2nd-stage-files/* debian + +# Set hostname +echo "${OS}" >debian/etc/hostname + +# Set resolv.conf +case "${OS}" in +debian | ubuntu) + ln -sf /run/systemd/resolve/stub-resolv.conf debian/etc/resolv.conf + ;; +devuan) + ln -sf /run/connman/resolv.conf debian/etc/resolv.conf + ;; +*) ;; +esac + +# List all files +find debian ! -type d -printf "/%P\n" | sort >files.txt + +mv debian/tmp/versions.csv versions.csv + +# Make a ext4 filesystem of this and put it into the image +# >>> ((3800000000 - 32 * 1024 * 1024) // (1024 * 1024)) * (1024 * 1024) +# 3765436416 +rm -f ext4.img +fallocate -l 3765436416 ext4.img +mkfs.ext4 -d debian ext4.img +pigz ext4.img + +mkdir -p "${ARTIFACTS_DIR:-/artifacts}" +cp -v files.txt "${ARTIFACTS_DIR:-/artifacts}/${OS}-${DIST}-${ARCH}-${PASSWORD}.bin.gz.files.txt" +cp -v versions.csv "${ARTIFACTS_DIR:-/artifacts}/${OS}-${DIST}-${ARCH}-${PASSWORD}.bin.gz.versions.csv" +cp -v ext4.img.gz "${ARTIFACTS_DIR:-/artifacts}/${OS}-${DIST}-${ARCH}-${PASSWORD}.bin.gz" diff --git a/scripts/build-image b/scripts/build-image index 42ecfeb..8347cb3 100755 --- a/scripts/build-image +++ b/scripts/build-image @@ -14,7 +14,8 @@ if [ $IMAGE_SIZE -lt 1000000000 ]; then exit 1 fi rm -f "${IMAGE}" -fallocate -v -l "${IMAGE_SIZE}" "${IMAGE}" # 3,800,000,000 bytes +# fallocate fails on some (git CI) COW filesystems, so have a coreutils alternative. +fallocate -v -l "${IMAGE_SIZE}" "${IMAGE}" || truncate --size "${IMAGE_SIZE}" "${IMAGE}" # 3,800,000,000 bytes parted -s "${IMAGE}" mklabel msdos parted -s "${IMAGE}" mkpart primary fat32 -- 4MiB 32MiB # 28 MiB (29,360,128 bytes) @@ -23,7 +24,8 @@ parted -s "${IMAGE}" set 2 boot on # Create empty FAT partition rm -f vfat.img -fallocate -l 28MiB vfat.img +# fallocate fails on some (git CI) COW filesystems, so have a coreutils alternative. +fallocate -l 28MiB vfat.img || truncate --size 28MiB vfat.img mkfs.vfat vfat.img # Leave a friendly note about this partition diff --git a/test/qemu.sh b/test/qemu.sh index a6b13d0..19a6696 100755 --- a/test/qemu.sh +++ b/test/qemu.sh @@ -31,7 +31,8 @@ LOOP1=$(losetup -f -P --show image.bin) && { } # Assume 8GB virtual disk -fallocate -l 8GB image.bin +# fallocate fails on some (git CI) COW filesystems, so have a coreutils alternative. +fallocate -l 8GB image.bin || truncate --size 8GB image.bin # Extend second partition parted -s -a opt image.bin "resizepart 2 100%"