archlinux/notes/archvps.sh

#!/usr/bin/env bash
set -a
set -E
revision=1.0

# Fill in details
hostname=
newuser=
port=

primaryip=
gateway=
primaryip6=
gateway6=
subnet=255.255.255.0

# ZFS key
zfsgpgkey=D0F2AE55C1BF11A026D155813A658A95B8CFCC51

# SSH keys
curl --silent --fail https://myvelabs.com/lab/archlinux/raw/branch/master/notes/sshkeys.pub -o sshkeys

# Static variables
originaliface=$(dmesg | grep $(ip r | grep 'default via' | awk '{print $5}') | grep 'renamed from' | awk '{print $NF}')
iface=$(ip r | grep 'default via' | awk '{print $5}')

# Exit function
trap '[ "${?}" -ne 77 ] || exit 77' ERR
function die
{
    local reset="$(tput sgr0)"
    local red="${reset}$(tput setaf 1)"
    local yellow="${reset}$(tput setaf 3)"

	cat <<- abort
	${red}
	Error encountered for the following reason:
	${yellow}
	    ${@}
	${red}
	Script aborted...
	${reset}
	abort

    exit 77
}

# Quit if zfs modules aren't loaded
modprobe zfs || die 'ZFS modules are not loaded'

##
## Declare functions
##

# Prompt function
function reformat
{
    case "${1}" in
        # Formatting
        bold) format+=($(tput bold))
            ;;
        italic) format+=($(tput sitm))
            ;;
        uline) format+=($(tput smul))
            ;;
        blink) format+=($(tput blink))
            ;;
        dim) format+=($(tput dim))
            ;;

        # Colours
        black) format+=($(tput setaf 0))
            ;;
        red) format+=($(tput setaf 1))
            ;;
        green) format+=($(tput setaf 2))
            ;;
        yellow) format+=($(tput setaf 3))
            ;;
        blue) format+=($(tput setaf 4))
            ;;
        magenta) format+=($(tput setaf 5))
            ;;
        cyan) format+=($(tput setaf 6))
            ;;
        white) format+=($(tput setaf 7))
            ;;

        # Read-specific
        array) array="-a"
            ;;
        secret) secret="-s"
            ;;
        press) press="-n 1"
            ;;

        # Everything else goes into line
        *) line="${@}"
            ;;
    esac
}

# Syntax: ask for var [in format] "statement"
function ask
{
    local format variable line secret press array

    case "${1}" in
        for)
            case "${2}" in
                in)
                    die "Syntax error detected (missing variable before $(tput sitm)$(tput smso)in$(tput rmso)$(tput ritm))
    Usage: ask for \${variable} in \${format[@]} \"\${line}\""                    
                    ;;
                *)
                    variable="${2}"
                    shift 2
                    ;;
            esac
            ;;
        *)
            die "Syntax error detected (misuse of $(tput sitm)$(tput smso)for$(tput rmso)$(tput ritm))
    Usage: ask for \${variable} in \${format[@]} \"\${line}\""
            ;;
    esac

    case "${1}" in
        in)
            shift
            for arg in "${@}"
            do
                reformat ${arg}
            done
            ;;
        *)
            line="${@}"
            ;;
    esac

    read ${press} -r -p "$(printf "%s" "${format[@]}" "${line}: " "$(tput sgr0)")" ${secret} ${array} ${variable}
    echo

    export ${variable}
}

# Syntax: say [as template] [in format] "statement"
function say
{
    local format line heading

    case "${1}" in
        as)
            # Templates
            case ${2} in
                title)
                    format+=($(tput setaf 6)) # cyan
                    ;;
                heading)
                    format+=($(tput setaf 5) $(tput bold)) # bold magenta
                    heading=":: "
                    ;;
                success)
                    format+=($(tput setaf 2)) # green
                    ;;
                warning)
                    format+=($(tput setaf 1)) # red
                    ;;
                *)
                    die "Syntax error detected (misuse of $(tput sitm)$(tput smso)as$(tput rmso)$(tput ritm))
    Usage: say as \${template} in \${format[@]} \"\${line}\""
                    ;;        
            esac
            shift 2
            ;;
    esac

    case "${1}" in
        in)
            shift
            for arg in "${@}"
            do
                reformat ${arg}
            done
            ;;
        *)
            line="${@}"
            ;;
    esac

    printf "%s" "${format[@]}" "${heading}" "${line}" "$(tput sgr0)"
    echo
}

function presskeytoresume
{
    read -n 1 -s -p "Press any key when ready..."
    echo
}

# Repeat a command until it exits with code 0
function repeat
{
    until ${@}
    do
        sleep 3
    done
}

# Fetch flags
while [ ${1} ]
do
    case ${1} in
        -s | --ssh-key )
            if [ "${2}" ]
            then
                echo "${2}" >>sshkeys
                if ! ssh-keygen -l -f sshkeys >/dev/null
                then
                    die 'Invalid SSH public key detected'
                fi
                shift
            fi
            ;;
        -p | --port )
            if [ "${2}" ]
            then
                port=${2}
                shift
            fi
            ;;
        -c | --cache )
            if [ "${2}" ]
            then
                cacheserver=${2}
                shift
            fi
            ;;
        -? | -h | --help )
            cat <<- help
    Parameters:
      -s, --ssh-key             Add SSH public key (enclosed in quotes)
      -p, --port                SSH port
      -c, --cache               Pacman cache server
      -?, -h, --help            This help screen
help
            exit 0
            ;;
        * )
            die "Unknown flag: ${1}"
            ;;
    esac
    shift
done

# Ensure pacman.conf settings are properly set
if [ ! -f /etc/pacman.conf.bkp ]
then
    mv /etc/pacman.conf /etc/pacman.conf.bkp
fi
sed -e '/ParallelDownloads/c ParallelDownloads = 10' \
    -e '/Color/c Color' \
    -e '/\[core\]/i [myvezfs]\
Server = https://repo.myvelabs.com/$repo\n' /etc/pacman.conf.bkp >/etc/pacman.conf

# Internet connection check
if ping -q -c 1 -W 5 archlinux.org >/dev/null || ! ping -q -c 1 -W 5 google.com >/dev/null
then
    timedatectl set-ntp true
    pacman -Sy --ask 4 >/dev/null 2>&1
else
    die 'No internet connectivity detected, plug in an ethernet cable and try again'
fi

# Detect needed firmwares
linux_firmware=(qemu-guest-agent)
virtualMachineService=qemu-guest-agent

# Virtual machine defaults
function virtualMachineDefaults
{
    hostname=testzfs
    userpass=123
    lukspass=12345678
    installation_drive_array=($(lsblk -d -n -o name | grep -v 'loop0\|sr0'))

    say in italic 'Troubleshooting defaults loaded'
    echo
}

# Grab system variables
function initialSetup
{
    # User password
    say as title "Set a password for user"
    until [ "${userpass}" = "${userpass2}" -a "${userpass}" ]
    do
        ask for userpass in secret "User password"
        ask for userpass2 in secret "Verify user password"
        echo
        if [ -z "${userpass}" ]
        then
            say as warning "Password field cannot be empty, try again"
        elif [ "${userpass}" != "${userpass2}" ]
        then
            say as warning "Passwords did not match, try again"
        fi
    done
    say as success "user@${hostname}'s password has been saved"
    echo

    # Encryption key
    say as title "Create an encryption passphrase for the system drive"
    until [ "${lukspass}" = "${lukspass2}" -a "${lukspass}" -a "${#lukspass}" -ge 8 ]
    do
        ask for lukspass in secret "Encryption password"
        ask for lukspass2 in secret "Verify encryption password"
        echo
        if [ -z "${lukspass}" ]
        then
            say as warning "Passphrase field cannot be empty, try again"
        elif [ "${lukspass}" != "${lukspass2}" ]
        then
            say as warning "Passphrases did not match, try again"
        elif [ "${#lukspass}" -lt 8 ]
        then
            say as warning "Passphrase needs to be at least 8 characters"
        fi
    done
    say as success "Encryption passphrase has been saved"
    echo

    # Assign system drive
    say as title "Identify the system drive from the list of available devices below"
    lsblk -d -o name,model,size,mountpoint | grep -v "archiso\|sr0"
    say in italic '## SSD and HDD device format begins with "sd" or "hd" (sda, sdb, sd[*])'
    say in italic '## NVME and PCI device format is "nvme[*]n1" (nvme0n1, nvme1n1, nvme[*]n1)'
    while true
    do
        ask for installation_drive "Installation device"
        if [ ${installation_drive} ] && \
            lsblk -o name | grep -q -w ${installation_drive}
        then
            installation_drive_array=${installation_drive}
            break
        else
            lsblk -d -o name,model,size,mountpoint | grep -v "archiso\|sr0"
            if [ -z ${installation_drive} ]
            then
                say as warning 'Field cannot be empty, try again'
            else
                say as warning 'Invalid selection or drive not available, try again'
            fi
        fi
    done
    say as success "Installation drive set as /dev/${installation_drive}"
    echo
}

# Generate SSH keys
function gensshid
{
    ssh-keygen -q \
        -t ed25519 \
        -P "" \
        -C "${USER}@${hostname}" \
        -f ~/.ssh/id_ed25519
    mkdir ~/.ssh/sockets/ 
}

clear

say in italic "## rev ${revision}"

echo
say in uline white dim "Full Disk Encryption"
echo
presskeytoresume
echo

# Setup
if ls -l /dev/disk/* | grep -q 'VBOX\|virtio\|QEMU'
then
    while true
    do
        ask for load_defaults in blue press "Would you like to load virtual machine troubleshooting defaults? (y/n)"
        echo

        case ${load_defaults} in
            [yY])
                virtualMachineDefaults
                break
                ;;
            [nN])
                initialSetup
                break
                ;;
            *)
                say as warning 'Not a valid answer, type "y" or "n"'
                ;;
        esac
    done
else
    initialSetup
fi

# Fetch disk ID of installation disks
for drive in ${installation_drive_array[@]}
do
    installation_disks+=(/dev/${drive})
done

# Default zroot layout
zrootlayout="$(printf '%s2 ' ${installation_disks[@]})"

# Device shredding
while true
do
    ask for shred_disk in blue press 'Would you like to shred your installation drive? (y/n)'
    echo

    case ${shred_disk} in
        [yY])
            say as title "Select an overwrite source"
			cat <<- shred
			    1) zero
			    2) urandom
			shred
            while true
            do
                ask for shred_source in press ":"
                case ${shred_source} in
                    1) shred=zero
                        break
                        ;;
                    2) shred=urandom
                        break
                        ;;
                    *) echo
                        say as warning "Invalid selection, type an option from 1 to 2"
                        ;;
                esac
            done
            echo
            # Disk shredding function
            function destroy_disk
            {
                say as title "Shredding ${1}"
                dd if=/dev/${shred} of=${1} bs=4M status=progress
                echo 3 >/proc/sys/vm/drop_caches
                say as success "${1} shredded"
                echo
            }
            break
            ;;
        [nN])
            say in yellow "Disk shredding skipped"
            echo

            # Empty disk shredding function
            function destroy_disk
            {
                true
            }
            break
            ;;
        *)
            say as warning 'Not a valid answer, type "y" or "n"'
            ;;
    esac

done

say as heading "Shredding drives"
cat <<- warning
$(tput setab 1)#############################################################$(tput sgr0)
$(tput setab 1)#############################################################$(tput sgr0)
$(tput setab 1)                                                             $(tput sgr0)
$(tput setab 1)                        $(tput blink)!!!WARNING!!!                        $(tput sgr0)
$(tput setab 1)          !!!ALL DATA WILL BE WIPED FROM THE DRIVE!!!        $(tput sgr0)
$(tput setab 1)                                                             $(tput sgr0)
$(tput setab 1)#############################################################$(tput sgr0)
$(tput setab 1)#############################################################$(tput sgr0)
## This will overwrite the drive by one pass
## Confirm by typing "Y" to proceed
## Anything other than capital Y will abort the installation
warning
ask for begin_install in press ":"

if ! [[ ${begin_install} = Y ]]
then
    echo
    say as warning in bold 'Aborting installation
Installer script stopped'
    echo
    exit 1
fi
echo

# Unmount if mounted
until ! mount | grep -q /mnt/boot
do
    umount /mnt/boot
done

# Export zpool
zpool export -a

# Clean slate
for drive in ${installation_disks[@]}
do
    destroy_disk ${drive} &
done
wait
for drive in ${installation_disks[@]}
do
    blkdiscard --quiet --force --secure ${drive} >/dev/null 2>&1
    wipefs --quiet --force --all ${drive}
done

# Partition each drive
say as heading "Partitioning system drive"
for drive in ${installation_disks[@]}
do
    parted --script --align=optimal ${drive} \
        mklabel msdos \
        mkpart primary 1MiB 300MiB \
        mkpart primary 300MiB 100% \
        set 1 boot on
    partprobe ${drive}
done
say as success "Partitioning completed!"
echo

# Format zfs
say as heading "Formatting zfs partitions"
until printf '%s' "${lukspass}" |\
    zpool create -f zroot \
    -o ashift=12 \
    -o autotrim=on \
    -R /mnt \
    -O acltype=posixacl \
    -O canmount=off \
    -O compression=lz4 \
    -O dnodesize=auto \
    -O normalization=formD \
    -O atime=off \
    -O xattr=sa \
    -O mountpoint=none \
    -O encryption=aes-256-gcm \
    -O keyformat=passphrase \
    -O keylocation=prompt \
    ${zrootlayout} \
    >/dev/null
do
    sleep 1
done

# Mount datasets
zfs create -o mountpoint=/      -o canmount=noauto                  zroot/ROOT
zfs create -o mountpoint=/.boot                                     zroot/BOOT
zfs create -o mountpoint=/home                                      zroot/HOME
zfs create -o mountpoint=/var/log                                   zroot/LOG
zfs create -o mountpoint=/var/tmp                                   zroot/TMP
zfs create -o mountpoint=/var/cache/pacman/pkg                      zroot/PKG
zfs create -o mountpoint=/etc/letsencrypt                           zroot/HTTPS
zfs create -o mountpoint=/docker                                    zroot/DOCKER
zfs create -o mountpoint=/docker/SQL             -o recordsize=16K  zroot/DOCKER/SQL
zpool export zroot
zpool import -R /mnt zroot -N -d ${installation_disks}2
printf '%s' "${lukspass}" | zfs load-key zroot
zfs mount zroot/ROOT
zfs mount -a

# Format efi partition
for drive in ${installation_disks[@]}
do
    yes | mkfs.fat -F 32 ${drive}1
done
echo

# Mount efi partition
install -d -m 755 /mnt/boot/
mount ${installation_disks}1 /mnt/boot

# Fix directory permissions
chmod 1777 /mnt/var/tmp

# # Confirm mounts
findmnt '/mnt' | grep -q -w 'zroot/ROOT' || die 'Root partition has not been mounted properly'
findmnt '/mnt/boot' | grep -q -w '/mnt/boot' || die 'Boot partition has not been mounted properly'

# Default packages
archpkgs="${linux_firmware[@]} \
    sudo openssh syslinux fakeroot \
    vim pacman-contrib bash-completion \
    mkinitcpio-netconf mkinitcpio-tinyssh tinyssh \
    reflector rsync \
    fail2ban \
    nginx-mainline certbot certbot-nginx apache \
    iptables-nft openbsd-netcat \
    docker docker-compose \
    wireguard-tools systemd-resolvconf \
    ufw unbound \
    git pv"

# Temporarily disable mkinitcpio
ln -s -f /dev/null /etc/pacman.d/hooks/90-mkinitcpio-install.hook

# Install archzfs
say as heading "Installing Arch Linux base packages"
repeat pacstrap -K /mnt --ask 4 \
    linux zfs-linux zfs-utils \
    base mkinitcpio dbus-broker-units \
    ${archpkgs}

# Configure zfs for pacman
sed -e '/ParallelDownloads/c ParallelDownloads = 10' \
    -e '/Color/c Color' \
    -e '/\[core\]/i [myvezfs]\
Server = https://repo.myvelabs.com/$repo\n' \
    -i /mnt/etc/pacman.conf

# Add archzfs keys
arch-chroot /mnt pacman-key -r ${zfsgpgkey} >/dev/null 2>&1
arch-chroot /mnt pacman-key --lsign-key ${zfsgpgkey} >/dev/null 2>&1

# Manually configure mkinitcpio
sed -e "s|%PKGBASE%|linux|g" \
    -e "s/^fallback/#&/g" \
    -e "s/ 'fallback'//" \
    /mnt/usr/share/mkinitcpio/hook.preset >/mnt/etc/mkinitcpio.d/linux.preset
rsync -a /mnt/usr/lib/modules/*/vmlinuz /mnt/boot/vmlinuz-linux

# Create custom system dirs
mkdir -p /mnt/etc/{zfs,pacman.d/hooks,nginx/sites-{available,enabled}} \
    /mnt/zfs/{bin,snapshots} \
    /mnt/etc/systemd/{resolved,journald,networkd}.conf.d \
    /mnt/local/{bin,systemd,hooks}

# Configure zfs for mkinitcpio
echo 'BINARIES+=(/usr/bin/zfs)' >/mnt/etc/mkinitcpio.conf.d/zz-binaries.conf
echo 'MODULES_DECOMPRESS="yes"' >/mnt/etc/mkinitcpio.conf.d/zz-modules.conf
grep '^HOOKS' /mnt/etc/mkinitcpio.conf | sed 's/filesystems fsck/netconf tinyssh zfsencryptssh zfs filesystems/' >/mnt/etc/mkinitcpio.conf.d/zz-hooks.conf

# ZFS properties
zpool set bootfs=zroot/ROOT zroot
zpool set cachefile=/etc/zfs/zpool.cache zroot
rsync -a /etc/zfs/zpool.cache /mnt/etc/zfs/zpool.cache

# Copy networking files
ln -s -f /run/systemd/resolve/stub-resolv.conf /mnt/etc/resolv.conf
rsync -a /etc/systemd/network/20-ethernet.network /mnt/etc/systemd/network/zz-fallback.network

cat >/mnt/etc/systemd/network/20-${iface}.network <<- network
[Match]
Name=${iface}

[Network]
Description=Static network
DHCP=no

# IPv4
Address=${primaryip}/24
Gateway=${gateway}

# IPv6
Address=${primaryip6}/64

[Route]
Gateway=${gateway6}
GatewayOnLink=yes
network

# SSH and Dropbear
cat sshkeys >/mnt/etc/tinyssh/root_key

    # Chroot into new root
    arch-chroot /mnt /usr/bin/bash <<"CHROOT"

# Global bashrc
tee -a /etc/skel/.bashrc >/dev/null <<'bashglobal'

# Add local functions folder to path
export PATH=${PATH}:${HOME}/.local/bin:/local/bin:/zfs/bin
export SUDO_PROMPT=$'\a'"$(tput rev)[sudo] password for %p:$(tput sgr0)"' '

# Colored prompts
alias diff='diff --color=auto'
alias ip='ip -color=auto'
export LESS='-R --use-color -Dd+r$Du+b$'

# Source bash functions
for file in $(find ~/.local/functions -type f)
do
    . ${file}
done

# Auto cd into directory
shopt -s autocd

# Enable tab complete for sudo
complete -c -f sudo

# Ignore duplicate and whitespace history entries
export HISTCONTROL=ignoreboth

#
# ~/.bash_aliases
#

# ZFS/btrfs
alias zfs='sudo zfs'
alias zpool='sudo zpool'

# Shutdown reboot
alias poweroff='sudo poweroff'
alias reboot='sudo reboot'

# Clear bash history
alias clearhistory='rm ${HISTFILE}; history -c -w'

# Miscellanous pacman
alias orphans='sudo pacman -Rcns $(pacman -Qtdq)'
alias unlock-pacman='sudo rm /var/lib/pacman/db.lck && sudo pacman -Syyu'

# Rsync
alias rsync='rsync -v -h --progress --info=progress2 --partial --append-verify'
# --log-file=
# --remove-source-files

#
# ~/.bash_functions
#

# Pacman tools
function installer
{
    sudo pacman -S ${@}
    echo
}

function uninstall
{
    sudo pacman -Rcns ${@}
    echo
}

function syur
{
    /local/bin/syu &&
    reboot
}

function syup
{
    /local/bin/syu &&
    poweroff
}

# Update bash
function update-bash
{
    vim ~/.bashrc &&
    source ~/.bashrc
}
bashglobal

# Root bashrc
rsync -a /etc/skel/.bashrc ~/
mkdir -p ~/.local/functions
cat > ~/.local/functions/root <<- 'rootbashrc'
#!/usr/bin/env bash
# Root shell color
PS1="$(tput setaf 1)[\u@\h \W \$?]\$$(tput sgr0) "

# Colored prompts
alias ll='ls --color=auto -l -a -h'
alias egrep='egrep --color=auto'
alias fgrep='fgrep --color=auto'

# Disable history
unset HISTFILE
rm -f ${HISTFILE}
history -c -w
rootbashrc

# Configure ssh
gensshid
ssh-keygen -A >/dev/null
cat >/etc/ssh/sshd_config.d/zz-homelab.conf <<- sshd
Port ${port:-22}
PermitRootLogin no
PasswordAuthentication no
AuthenticationMethods publickey
sshd

# tinyssh
rm -r -f /etc/tinyssh/sshkeydir/
tinyssh-convert /etc/tinyssh/sshkeydir/ </etc/ssh/ssh_host_ed25519_key

# ZFS setup
zpool set cachefile=/etc/zfs/zpool.cache zroot
zgenhostid $(hostid)

# Trim zroot monthly
cat >/etc/systemd/system/zfs-trim@.timer <<'TRIM'
[Unit]
Description=Monthly zpool trim on %i

[Timer]
OnCalendar=monthly
AccuracySec=1h
Persistent=true

[Install]
WantedBy=multi-user.target
TRIM

cat >/etc/systemd/system/zfs-trim@.service <<'TRIM'
[Unit]
Description=zpool trim on %i
Documentation=man:zpool-trim(8)
Requires=zfs.target
After=zfs.target
ConditionACPower=true
ConditionPathIsDirectory=/sys/module/zfs

[Service]
Nice=19
IOSchedulingClass=idle
KillSignal=SIGINT
ExecStart=/bin/sh -c '\
if /usr/bin/zpool status %i | grep "trimming"; then\
exec /usr/bin/zpool wait -t trim %i;\
else exec /usr/bin/zpool trim -w %i; fi'
ExecStop=-/bin/sh -c '/usr/bin/zpool trim -s %i 2>/dev/null || true'

[Install]
WantedBy=multi-user.target
TRIM

# Scrub zroot monthly
cat >/etc/systemd/system/zfs-scrub@.timer <<'SCRUB'
[Unit]
Description=Monthly zpool scrub on %i

[Timer]
OnCalendar=monthly
AccuracySec=1h
Persistent=true

[Install]
WantedBy=multi-user.target
SCRUB

cat >/etc/systemd/system/zfs-scrub@.service <<'SCRUB'
[Unit]
Description=zpool scrub on %i

[Service]
Nice=19
IOSchedulingClass=idle
KillSignal=SIGINT
ExecStart=/usr/bin/zpool scrub %i

[Install]
WantedBy=multi-user.target
SCRUB
echo

# mkinitcpio
say as heading "Regenerating cpio image"
mkinitcpio -P
echo

# Syslinux
say as heading "Configuring syslinux"
mkdir /boot/syslinux/
extlinux --install /boot/syslinux/
syslinux-install_update -i -a -m
# sed -i 's\linux ../vmlinuz-linux.*\linux ../vmlinuz-linux\g' /boot/syslinux/syslinux.cfg
# sed -i "s/APPEND root.*/APPEND zfs=bootfs ip=${primaryip}::${gateway}:${subnet}::${originaliface}:none rw quiet bgrt_disable/g" /boot/syslinux/syslinux.cfg
# sed -i 's/.*PROMPT.*/PROMPT 0/g' /boot/syslinux/syslinux.cfg
# sed -i 's/.*TIMEOUT.*/TIMEOUT 0/g' /boot/syslinux/syslinux.cfg
# sed -i 's/.*UI menu.*/#UI menu/g'
cat >/boot/syslinux/syslinux.cfg <<-syslinux
DEFAULT arch
PROMPT 0
TIMEOUT 0

MENU TITLE Arch Linux
MENU COLOR border       30;44   #40ffffff #a0000000 std
MENU COLOR title        1;36;44 #9033ccff #a0000000 std
MENU COLOR sel          7;37;40 #e0ffffff #20ffffff all
MENU COLOR unsel        37;44   #50ffffff #a0000000 std
MENU COLOR help         37;40   #c0ffffff #a0000000 std
MENU COLOR timeout_msg  37;40   #80ffffff #00000000 std
MENU COLOR timeout      1;37;40 #c0ffffff #00000000 std
MENU COLOR msg07        37;40   #90ffffff #a0000000 std
MENU COLOR tabmsg       31;40   #30ffffff #00000000 std

LABEL arch
        MENU LABEL Arch Linux
        LINUX ../vmlinuz-linux
        APPEND zfs=bootfs ip=${primaryip}::${gateway}:${subnet}::${originaliface}:none rw quiet bgrt_disable
        INITRD ../initramfs-linux.img
LABEL hdt
        MENU LABEL HDT (Hardware Detection Tool)
        COM32 hdt.c32
LABEL reboot
        MENU LABEL Reboot
        COM32 reboot.c32
LABEL poweroff
        MENU LABEL Poweroff
        COM32 poweroff.c32
syslinux
echo

# Locale
sed -i '/#en_US.UTF-8 UTF-8/ s/#//' /etc/locale.gen
locale-gen >/dev/null
echo 'LANG=en_US.UTF-8' >>/etc/locale.conf
say as heading "Locale configured"

# Time zone
ln -s -f /usr/share/zoneinfo/UTC /etc/localtime
hwclock --systohc --utc
say as heading "Time zone configured"

# Hostname
echo ${hostname} >/etc/hostname
cat >>/etc/hosts <<host
127.0.0.1        localhost
127.0.1.1        ${hostname}
host
say as heading "Hostname configured"

# User
useradd -m -g users -G wheel,docker -s /usr/bin/bash ${newuser} || die "User account creation has failed"
printf '%s\n' "${userpass}" "${userpass}" | passwd ${newuser} >/dev/null 2>&1
unset userpass userpass2

# Disable root account
passwd -l root >/dev/null 2>&1

# Sudoers
install -m 0440 /dev/stdin /etc/sudoers.d/01-DEFAULTS <<'DEFAULTS'
Defaults passwd_timeout=0
Defaults timestamp_type=global
Defaults insults
DEFAULTS

install -m 0440 /dev/stdin /etc/sudoers.d/02-COMMANDS <<'COMMANDS'
Cmnd_Alias POWER    =   /usr/bin/poweroff, /usr/bin/reboot
Cmnd_Alias ZFS      =   /usr/bin/zfs, /usr/bin/zpool
Cmnd_Alias FAIL2BAN =   /usr/bin/fail2ban-client
Cmnd_Alias PACMAN   =   /usr/bin/pacman
Cmnd_Alias IPTABLES =   /usr/bin/ufw
Cmnd_Alias MISC     =   /usr/bin/rsync
COMMANDS

install -m 0440 /dev/stdin /etc/sudoers.d/03-WHEEL <<'WHEEL'
%wheel ALL=(ALL:ALL) ALL
%wheel ALL=(ALL:ALL) NOPASSWD: POWER, ZFS, FAIL2BAN, PACMAN, IPTABLES, MISC
WHEEL

install -m 0440 /dev/stdin /etc/sudoers.d/.zz-NOPASSWD <<'NOPASSWD'
Defaults:${newuser} !authenticate
NOPASSWD

say as heading "Configured superuser and user"

# ZFS files
touch /zfs/snapshots/syu
chown ${newuser}:users /zfs/snapshots/syu

# Sysctl custom settings
cat >/etc/sysctl.d/zz-sysctl.conf <<- SYSCTL
net.core.netdev_max_backlog = 16384
net.core.somaxconn = 8192
net.core.rmem_default = 1048576
net.core.rmem_max = 16777216
net.core.wmem_default = 1048576
net.core.wmem_max = 16777216
net.core.optmem_max = 65536
net.ipv4.tcp_rmem = 4096 1048576 2097152
net.ipv4.tcp_wmem = 4096 65536 16777216
net.ipv4.udp_rmem_min = 8192
net.ipv4.udp_wmem_min = 8192
net.ipv4.tcp_fastopen = 3
net.ipv4.tcp_timestamps = 0
net.core.default_qdisc = cake
net.ipv4.tcp_congestion_control = bbr

# Swap
vm.swappiness=10
vm.vfs_cache_pressure=50

# Wireguard
net.ipv4.conf.all.forwarding = 1
net.ipv6.conf.all.forwarding = 1
SYSCTL

# zram
echo 'zram' >/etc/modules-load.d/zram.conf
echo 'options zram num_devices=1' >/etc/modprobe.d/zram.conf
echo 'KERNEL=="zram0", ATTR{comp_algorithm}="lz4", ATTR{disksize}="512M" RUN="/usr/bin/mkswap /dev/zram0", TAG+="systemd"' >/etc/udev/rules.d/99-zram.rules

# Docker
install /dev/stdin /usr/local/bin/update-dockerfiles <<'dockerfiles'
#!/usr/bin/env bash
for compose in $(find /docker -maxdepth 2 -type f -name docker-compose.yaml)
do
    docker compose -f ${compose} pull
    docker compose -f ${compose} up --detach
done

docker network prune -f
docker image prune -af
docker volume prune -af
dockerfiles

# Unbound
sed -i '/include-toplevel/c include-toplevel: "/etc/unbound/unbound.conf.d/*.conf"' /etc/unbound/unbound.conf
mkdir /etc/unbound/unbound.conf.d/
tee /etc/unbound/unbound.conf.d/unbound.conf >/dev/null <<- unbound.conf
server:
    # If no logfile is specified, syslog is used
    # logfile: "/var/log/unbound/unbound.log"
    verbosity: 0

    interface: 0.0.0.0@53
    do-ip4: yes
    do-udp: yes
    do-tcp: yes

    # May be set to yes if you have IPv6 connectivity
    do-ip6: yes

    # You want to leave this to no unless you have *native* IPv6. With 6to4 and
    # Terredo tunnels your web browser should favor IPv4 for the same reasons
    prefer-ip6: no

    # Use this only when you downloaded the list of primary root servers!
    # If you use the default dns-root-data package, unbound will find it automatically
    root-hints: "/etc/unbound/root.hints"

    # Trust glue only if it is within the server's authority
    harden-glue: yes

    # Require DNSSEC data for trust-anchored zones, if such data is absent, the zone becomes BOGUS
    harden-dnssec-stripped: yes

    # Don't use Capitalization randomization as it known to cause DNSSEC issues sometimes
    # see https://discourse.pi-hole.net/t/unbound-stubby-or-dnscrypt-proxy/9378 for further details
    use-caps-for-id: no

    # Reduce EDNS reassembly buffer size.
    # IP fragmentation is unreliable on the Internet today, and can cause
    # transmission failures when large DNS messages are sent via UDP. Even
    # when fragmentation does work, it may not be secure; it is theoretically
    # possible to spoof parts of a fragmented DNS message, without easy
    # detection at the receiving end. Recently, there was an excellent study
    # >>> Defragmenting DNS - Determining the optimal maximum UDP response size for DNS <<<
    # by Axel Koolhaas, and Tjeerd Slokker (https://indico.dns-oarc.net/event/36/contributions/776/)
    # in collaboration with NLnet Labs explored DNS using real world data from the
    # the RIPE Atlas probes and the researchers suggested different values for
    # IPv4 and IPv6 and in different scenarios. They advise that servers should
    # be configured to limit DNS messages sent over UDP to a size that will not
    # trigger fragmentation on typical network links. DNS servers can switch
    # from UDP to TCP when a DNS response is too big to fit in this limited
    # buffer size. This value has also been suggested in DNS Flag Day 2020.
    edns-buffer-size: 1232

    # Perform prefetching of close to expired message cache entries
    # This only applies to domains that have been frequently queried
    prefetch: yes

    # One thread should be sufficient, can be increased on beefy machines. In reality for most users running on small networks or on a single machine, it should be unnecessary to seek performance enhancement by increasing num-threads above 1.
    num-threads: 2

    # Ensure kernel buffer is large enough to not lose messages in traffic spikes
    so-rcvbuf: 1m

    # Ensure privacy of local IP ranges
    private-address: 10.6.22.0/24
    private-address: fd6e:4f68:5f03:fffe::/64

    # Only give access to recursion clients from LAN IPs
    access-control: 10.6.22.0/24 allow
    access-control: fd6e:4f68:5f03:fffe::/64 allow

    # Hide server info from clients
    hide-identity: yes
    hide-version: yes

    # Send minimum amount of information to upstream servers to enhance
    # privacy (best privacy).
    qname-minimisation: yes

    # Enable ratelimiting of queries (per second) sent to nameserver for
    # performing recursion. More queries are turned away with an error
    # (servfail). This stops recursive floods (e.g., random query names), but
    # not spoofed reflection floods. Cached responses are not rate limited by
    # this setting. Experimental option.
    ratelimit: 1000

    # Use this certificate bundle for authenticating connections made to
    # outside peers (e.g., auth-zone urls, DNS over TLS connections).
    tls-cert-bundle: "/etc/ssl/certs/ca-certificates.crt"
    tls-system-cert: yes

forward-zone:
    # Forward all queries (except those in cache and local zone) to
    # upstream recursive servers
    name: "."
    # Queries to this forward zone use TLS
    forward-tls-upstream: yes

    # https://dnsprivacy.org/wiki/display/DP/DNS+Privacy+Test+Servers

    ## Cloudflare
    forward-addr: 1.1.1.1@853#cloudflare-dns.com
    forward-addr: 1.0.0.1@853#cloudflare-dns.com

    ## Quad9
    # forward-addr: 9.9.9.9@853#dns.quad9.net
    # forward-addr: 149.112.112.112@853#dns.quad9.net

remote-control:
    # Enable remote control with unbound-control(8) here.
    control-enable: no

    # what interfaces are listened to for remote control.
    # give 0.0.0.0 and ::0 to listen to all interfaces.
    # set to an absolute path to use a unix local name pipe, certificates
    # are not used for that, so key and cert files need not be present.
    control-interface: 127.0.0.1
    # control-interface: ::1

    # port number for remote control operations.
    control-port: 8953
unbound.conf
curl --silent --fail --output /etc/unbound/root.hints https://www.internic.net/domain/named.cache
tee /etc/systemd/resolved.conf.d/zz-unbound.conf >/dev/null <<- 'unbound.conf'
[Resolve]
DNS=127.0.0.1
DNS=::1
DNSStubListener=no
unbound.conf

# Global ssh config
cat >/etc/ssh/ssh_config.d/zz-homelab.conf <<- sshconfig
# Preferred ciphers
Ciphers aes128-gcm@openssh.com,aes256-gcm@openssh.com,chacha20-poly1305@openssh.com

# Only use ipv4
AddressFamily inet

# Multiplex
ControlMaster auto
ControlPath ~/.ssh/sockets/%r@%h-%p
ControlPersist 10m
sshconfig

# Paccache hook
cat >/etc/pacman.d/hooks/zz-paccache.hook <<- PACCACHE
[Trigger]
Operation = Upgrade
Operation = Install
Operation = Remove
Type = Package
Target = *

[Action]
Description = Cleaning pacman cache...
When = PostTransaction
Exec = /usr/bin/paccache --remove
PACCACHE

# Locale.gen.pacnew hook
install /dev/stdin /local/hooks/localegen <<hook
#!/usr/bin/env bash
if [ -f /etc/locale.gen.pacnew ]
then
    sed -i '/#en_US.UTF-8 UTF-8/ s/#//' /etc/locale.gen.pacnew
    mv /etc/locale.gen.pacnew /etc/locale.gen
    locale-gen >/dev/null
fi
hook
# locale.gen.pacnew hook
cat >/etc/pacman.d/hooks/100-localegen.hook <<localegen
[Trigger]
Operation = Install
Operation = Upgrade
Type = Package
Target = glibc

[Action]
Description = Fixing locale.gen
When = PostTransaction
Exec = /local/hooks/localegen
localegen

# Environment
tee -a /etc/environment >/dev/null <<environment
EDITOR=vim
SUDO_EDITOR=vim
environment

# Permissions
chown -R ${newuser}:users /docker/

# Switch into new user
su ${newuser} <<"CHANGEUSER"
mkdir -p ~/.local/{bin,functions}

# Generate SSH identity
gensshid
cat /etc/tinyssh/root_key >~/.ssh/authorized_keys

cat >> ~/.bashrc <<- 'BASHRC'

# Custom shell color
PS1="$(tput setaf 8)[\u@\h \W \$?]\$$(tput sgr0) "

# Colored prompts
alias ll='ls --color=auto -l -a -h'
alias egrep='egrep --color=auto'
alias fgrep='fgrep --color=auto'

# Disable history
unset HISTFILE
rm -f ${HISTFILE}
history -c -w
BASHRC

# Startup script
cat > ~/.local/functions/init <<- 'init'
#!/usr/bin/env bash
if [ -f ~/.local/bin/startup ]
then
    echo -e "\n\e[1;31mInit script hasn't been run yet, executing script before proceeding...\e[0m\n"
    ~/.local/bin/startup || exit 1
fi
init

install /dev/stdin ~/.local/bin/startup <<'EOF'
#!/usr/bin/env bash

## Tput codes
reset=$(tput sgr0)
red=${reset}$(tput setaf 1)
yellow=${reset}$(tput setaf 3)

# Exit function
trap '[ "${?}" -ne 77 ] || exit 77' ERR
function die
{
	cat <<- abort
	${red}
	Error encountered for the following reason:
	${yellow}
	    "${1}"
	${red}
	Script aborted...
	${reset}
	abort
    exit 77
}

# Internet connection check
if ping -q -c 1 -W 3 archlinux.org >/dev/null
then
    sudo timedatectl set-ntp true
else
    die 'No internet connectivity detected, plug in an ethernet cable and try again'
fi

# Journal
sudo systemctl -q restart systemd-journald.service

rm -f ${0} ~/.local/functions/init
echo -e 'Supplementary installer completed, reboot one last time\e[0m\n'
EOF
CHANGEUSER

# fail2ban
install /dev/stdin /usr/local/bin/fail2ban-jails <<'ALL-JAILS'
#!/usr/bin/env bash
JAILS=$(sudo fail2ban-client status | grep "Jail list" | sed -E 's/^[^:]+:[ \t]+//' | sed 's/,//g')
for JAIL in ${JAILS}
do
    sudo fail2ban-client status ${JAIL}
done
ALL-JAILS
cat >/etc/fail2ban/jail.d/sshd.conf <<'SSHD'
[sshd]
enabled = true
filter = sshd
backend = systemd
maxretry = 5
findtime = 1d
bantime = 4w
ignoreip = 127.0.0.1/8
SSHD

# Nginx
sed '/^http {/a\
    include http_redirect;\
    include sites-enabled/\*.conf;\n\
    types_hash_max_size 4096;\
    server_names_hash_bucket_size 128;\n' \
    -i /etc/nginx/nginx.conf
cat >/etc/nginx/http_redirect <<- 'redirect'
server {
    listen 80;
    server_name _;
    if ($scheme = "http") {
        return 301 https://$host$request_uri;
    }
}
redirect
cat >/etc/nginx/proxy_params <<- 'proxy_params'
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header X-Forwarded-Host $host;
proxy_set_header X-Forwarded-Port $server_port;
proxy_set_header X-Forwarded-Scheme $scheme;
proxy_set_header X-Forwarded-Proto $scheme;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header Accept-Encoding "";
proxy_set_header Host $host;

proxy_next_upstream error timeout;

client_body_buffer_size 512k;
proxy_read_timeout 86400s;
client_max_body_size 0;

# Websocket
proxy_http_version 1.1;
proxy_cache_bypass  $http_upgrade;
proxy_set_header Upgrade $http_upgrade;
proxy_set_header Connection "upgrade";
proxy_params

cat >/etc/nginx/http_upgrade <<- 'http_upgrade'
# Security
server_tokens off;
add_header X-Frame-Options                   "SAMEORIGIN"               always;
add_header X-Content-Type-Options            "nosniff"                  always;
add_header X-XSS-Protection                  "1; mode=block"            always;
add_header Referrer-Policy                   "no-referrer"              always;
add_header X-Permitted-Cross-Domain-Policies "none"                     always;
add_header X-Robots-Tag                      "noindex, nofollow"        always;
# CSP breaks some webapps
# add_header Content-Security-Policy         "default-src 'self';"      always;

# http2
http2 on;

# http3
# Open port 443/udp to use http3
# Add reuseport to ONLY ONE virtual host: listen 443 quic reuseport;
listen 443 quic;
add_header Alt-Svc 'h3=":443"; ma=86400';
quic_retry on;
http3 on;

# Certbot defaults
listen 443 ssl;
include /etc/letsencrypt/options-ssl-nginx.conf;
ssl_dhparam /etc/letsencrypt/ssl-dhparams.pem;
add_header Strict-Transport-Security "max-age=31536000" always;
http_upgrade

# Firewall
ufw allow ${port:-22}/tcp comment SSH >/dev/null
ufw allow 80/tcp comment HTTP >/dev/null
ufw allow 443 comment HTTPS/3 >/dev/null
ufw allow 51820/udp comment Wireguard >/dev/null
ufw allow from 10.6.22.0/24 proto udp to any port 53 comment Unbound >/dev/null
ufw allow from fd6e:4f68:5f03:fffe::1/64 proto udp to any port 53 comment Unbound >/dev/null

# Enable system services
systemctl -q enable \
    systemd-resolved.service systemd-networkd.service sshd.service \
    ufw.service unbound.service \
    docker.service \
    nginx.service \
    fail2ban.service \
    zfs.target zfs-import-cache.service zfs-mount.service zfs-import.target \
    zfs-trim@zroot.timer zfs-scrub@zroot.timer \
    certbot-renew.timer

# Pacman hooks
# pacman.conf hook
cat >/etc/pacman.d/hooks/100-pacman.conf.hook <<pacman
[Trigger]
Operation = Install
Operation = Upgrade
Type = Package
Target = pacman

[Action]
Description = Fixing pacman.conf
When = PostTransaction
Exec = /local/hooks/pacman.conf
pacman
install /dev/stdin /local/hooks/pacman.conf <<'hook'
#!/usr/bin/env bash
if [ -f /etc/pacman.conf.pacnew ]
then
    sed -e '/ParallelDownloads/c ParallelDownloads = 10' \
        -e '/Color/c Color' \
        -e '/\[core\]/i [myvezfs]\
Server = https://repo.myvelabs.com/$repo\n' \
        -i /etc/pacman.conf.pacnew
    mv /etc/pacman.conf.pacnew /etc/pacman.conf
fi
hook

# mkinitcpio conf hook
cat >/etc/pacman.d/hooks/85-mkinitcpio.conf.hook <<preset
[Trigger]
Operation = Install
Operation = Upgrade
Type = Package
Target = mkinitcpio

[Action]
Description = Updating mkinitcpio.conf
When = PostTransaction
Exec = /local/hooks/mkinitcpio.conf
preset
install /dev/stdin /local/hooks/mkinitcpio.conf <<'hook'
#!/usr/bin/env bash
# Hooks
grep '^HOOKS' /etc/mkinitcpio.conf | sed 's/filesystems fsck/netconf tinyssh zfsencryptssh zfs filesystems/' >/etc/mkinitcpio.conf.d/zz-hooks.conf

# Linux preset
sed -e "s|%PKGBASE%|linux|g" \
    -e "s/^fallback/#&/g" \
    -e "s/ 'fallback'//" \
    /usr/share/mkinitcpio/hook.preset >/etc/mkinitcpio.d/linux.preset
hook

# Persistent journal logging
cat >/etc/systemd/journald.conf.d/zz-journald.conf <<- eof
[Journal]
Storage=persistent
SystemMaxUse=100M
eof

# Custom pacman update wrapper
install /dev/stdin /local/bin/syu <<'syu'
#!/usr/bin/env bash
set -e

# Record current time
echo $(date "+%Y-%m-%d-%H:%M:%S") >/zfs/snapshots/syu

# Check for new packages and continue if found
newpkg+=($(checkupdates --nocolor | awk '{print $1}'))
if [ "${newpkg}" ]
then
    # Sync pacman dbs
    sudo pacman --ask 4 --sync --refresh >/dev/null

    # Update archlinux-keyring first
    if [[ ${newpkg[@]} =~ "archlinux-keyring" ]]
    then
        sudo pacman --ask 4 --sync --needed archlinux-keyring
        echo
    fi
    if sudo pacman --ask 4 --sync --sysupgrade --needed
    then
        echo
        sudo pacdiff
        exit 0
    fi
fi
syu
CHROOT

## Pre and post update backup hooks
# Boot
cat >/mnt/etc/pacman.d/hooks/55-bootbackup_pre.hook <<pre
[Trigger]
Operation = Upgrade
Operation = Install
Operation = Remove
Type = Path
Target = usr/lib/modules/*/vmlinuz
Target = usr/lib/initcpio/*
Target = usr/lib/firmware/*
Target = usr/src/*/dkms.conf

[Action]
Depends = rsync
Description = Backing up pre /boot...
When = PreTransaction
Exec = /usr/bin/bash -c 'mount ${installation_disks}1; rsync -a --mkpath --delete /boot/ "/.boot/\$(cat /zfs/snapshots/syu)_pre"/'
AbortOnFail
pre
cat >/mnt/etc/pacman.d/hooks/95-bootbackup_post.hook <<post
[Trigger]
Operation = Upgrade
Operation = Install
Operation = Remove
Type = Path
Target = usr/lib/modules/*/vmlinuz
Target = usr/lib/initcpio/*
Target = usr/lib/firmware/*
Target = usr/src/*/dkms.conf

[Action]
Depends = rsync
Description = Backing up post /boot...
When = PostTransaction
Exec = /usr/bin/bash -c 'rsync -a --mkpath --delete /boot/ "/.boot/\$(cat /zfs/snapshots/syu)_post"/; /local/hooks/post-install-boot'
post
# zroot
cat >/mnt/etc/pacman.d/hooks/01-syu_pre.hook <<pre
[Trigger]
Type = Path
Operation = Upgrade
Operation = Install
Operation = Remove
Target = usr/lib/modules/*/vmlinuz
Target = usr/lib/initcpio/*
Target = usr/lib/firmware/*
Target = usr/src/*/dkms.conf

[Action]
Description = Creating pre zroot snapshot...
When = PreTransaction
Exec = /usr/bin/bash -c 'zfs snapshot zroot/ROOT@pre-\$(cat /zfs/snapshots/syu)'
AbortOnFail
pre
cat >/mnt/etc/pacman.d/hooks/zz-syu_post.hook <<post
[Trigger]
Type = Path
Operation = Upgrade
Operation = Install
Operation = Remove
Target = usr/lib/modules/*/vmlinuz
Target = usr/lib/initcpio/*
Target = usr/lib/firmware/*
Target = usr/src/*/dkms.conf

[Action]
Description = Creating post zroot snapshot...
When = PostTransaction
Exec = /usr/bin/bash -c 'zfs snapshot zroot/ROOT@post-\$(cat /zfs/snapshots/syu)'
post

# Generate fstab
genfstab -U -p /mnt/boot | sed "s|vfat.*rw|vfat            rw,x-systemd.idle-timeout=1min,x-systemd.automount,noauto,nofail|" >>/mnt/etc/fstab
sed -i '/UUID.*vfat/ s|/|/boot|' /mnt/etc/fstab

# Add zram to fstab
echo '/dev/zram0              none            swap            defaults        0 0' >>/mnt/etc/fstab
echo

# Reboot only if script succeeded
if /usr/bin/bash -c 'arch-chroot /mnt uname -a' | grep -q Linux
then
    until ! mount | grep -q /mnt/boot
    do
        umount /mnt/boot
    done
    zfs snapshot zroot/ROOT@fresh-installation
    zpool export -a
    say as success in bold "Installer has completed and system drive has been unmounted"
    say as success in bold "Boot into the new system, connect to a network and run $(tput smso)startup$(tput rmso) in the terminal"
    say as success in bold "Rebooting..."
    echo
    reboot
else
    die 'Something does not feel right'
fi