From 83d9b00e3f46cb3bd9c29523bbf8ffc098e65553 Mon Sep 17 00:00:00 2001 From: Charles Duffy Date: Thu, 3 Jan 2019 19:07:12 -0600 Subject: [PATCH] Introduce service wrapper permitting less stateful configuration Config files for the previous wrapper could not be shared across machines, and could not be generated independent of or prior to creation of local system state, as they needed to be aware of filesystem UUIDs. This wrapper permits more flexible configuration (including support for passing all parameters on the wrapper's command line, and not needing any on-disk files at all) while retaining support for the legacy configuration file format. --- scripts/beesd.in | 320 +++++++++++++++++++++++++++++------------------ 1 file changed, 196 insertions(+), 124 deletions(-) diff --git a/scripts/beesd.in b/scripts/beesd.in index 216bc413f..4e5009a60 100755 --- a/scripts/beesd.in +++ b/scripts/beesd.in @@ -1,146 +1,218 @@ -#!/bin/bash +#!/usr/bin/env bash +bees_config_dir=@ETC_PREFIX@/bees/ +bees_bin=$(realpath @LIBEXEC_PREFIX@/bees) +readonly bees_config_dir bees_bin -## Helpful functions -INFO(){ echo "INFO:" "$@"; } -ERRO(){ echo "ERROR:" "$@"; exit 1; } -YN(){ [[ "$1" =~ (1|Y|y) ]]; } +shopt -s extglob -## Global vars -export BEESHOME BEESSTATUS -export WORK_DIR CONFIG_DIR -export CONFIG_FILE -export UUID AL16M AL128K +# Prior wrapper required UUID to be used for configuration. Current one permits +# this legacy mode, but also permits reference to named configuration files, or +# `findmnt`-compatible mount point specifications with other arguments passed +# key/value style. -readonly AL128K="$((128*1024))" -readonly AL16M="$((16*1024*1024))" -readonly CONFIG_DIR=@ETC_PREFIX@/bees/ +[[ $bees_debug ]] && { PS4=':${BASH_SOURCE##*/}:$LINENO+'; set -x; } -readonly bees_bin=$(realpath @LIBEXEC_PREFIX@/bees) +usage() { + cat >&2 < /dev/null || ERRO "Missing 'bees' agent" + fsSpec should be in a format recognized by findmnt. Alternately, + "config-name" may refer to a file that exists in ${bees_config_dir:-/etc/bees} + with a .conf extension; if that file does not specify UUID, findmnt will be + used in addition. -uuid_valid(){ - if uuidparse -n -o VARIANT $1 | grep -i -q invalid; then - false - fi -} + Note that while config files may presently use shell arithmetic, use of this + functionality is not encouraged going forward: Setting ''idxSizeMB=4096'' is + preferred over ''DB_SIZE=$((1024*1024*1024*4))'' or ''DB_SIZE=$(( AL16M * 256 ))'', + although both of these are presently supported. -help(){ - echo "Usage: beesd [options] " - echo "- - -" - exec "$bees_bin" --help -} + If fsSpec contains a /, it assumed to be a mount point to be looked up by + findmnt, not a config file name. -get_bees_supp_opts(){ - "$bees_bin" --help |& awk '/--../ { gsub( ",", "" ); print $1 " " $2}' + daemon-options are passed directly through to the daemon on startup, as + documented at https://github.com/Zygo/bees/blob/master/docs/options.md. +EOF + exit 1 } -SUPPORTED_ARGS=( - $(get_bees_supp_opts) -) -NOT_SUPPORTED_ARGS=() -ARGUMENTS=() - -for arg in "${@}"; do - supp=false - for supp_arg in "${SUPPORTED_ARGS[@]}"; do - if [ "$arg" == "$supp_arg" ]; then - supp=true - break - fi - done - if $supp; then - ARGUMENTS+=($arg) - else - NOT_SUPPORTED_ARGS+=($arg) - fi -done - -for arg in "${ARGUMENTS[@]}"; do - case $arg in - -h) help;; - --help) help;; - esac -done - -for arg in "${NOT_SUPPORTED_ARGS[@]}"; do - if uuid_valid $arg; then - [ ! -z "$UUID" ] && help - UUID=$arg - fi -done +die() { echo "$*" >&2; exit 1; } -[ -z "$UUID" ] && help - - -FILE_CONFIG="$(egrep -l '^[^#]*UUID\s*=\s*"?'"$UUID" "$CONFIG_DIR"/*.conf | head -1)" -[ ! -f "$FILE_CONFIG" ] && ERRO "No config for $UUID" -INFO "Find $UUID in $FILE_CONFIG, use as conf" -source "$FILE_CONFIG" +allConfigNames=( blockdev fsSpec home idxSize idxSizeMB mntDir runDir status verbosity workDir ) +# Alternate names for configuration values; "bees_" will always be prepended +declare -A altConfigNames=( + # from original bees wrapper + [BEESHOME]=home + [BEESSTATUS]=status + [MNT_DIR]=mntDir + [UUID]=uuid + [WORK_DIR]=runDir + [DB_SIZE]=idxSize +) -## Pre checks -{ - [ ! -d "$CONFIG_DIR" ] && ERRO "Missing: $CONFIG_DIR" - [ "$UID" == "0" ] || ERRO "Must be run as root" +# legacy bees config files can be arbitrary shell scripts, so we need to actually evaluate them +sandboxedConfigFileEval() { + bash_exe=$(type -P bash) || exit + PATH=/var/empty ENV='' BASH_ENV='' AL128K="$((128*1024))" AL16M="$((16*1024*1024))" "$bash_exe" -r ${bees_debug+-x} \ + -c 'eval "$(&2; for var; do [[ ${!var} ]] && printf "%q=%s\\0" "$var" "${!var}"; done' \ + "${!altConfigNames[@]}" "${allConfigNames[@]}" \ + <"$1" } +readConfigFileIfExists() { + local line + [[ -s $1 ]] || return 1 + while IFS= read -r -d '' line; do + line=${line%%+([[:space:]])"#"*} + [[ $line ]] || continue + [[ $line = *=* ]] || { + printf 'WARNING: Config file line not recognized: %q\n' "$line" >&2 + continue + } + set_option "$line" + done < <(sandboxedConfigFileEval "$1") +} -WORK_DIR="${WORK_DIR:-/run/bees/}" -MNT_DIR="${MNT_DIR:-$WORK_DIR/mnt/$UUID}" -BEESHOME="${BEESHOME:-$MNT_DIR/.beeshome}" -BEESSTATUS="${BEESSTATUS:-$WORK_DIR/$UUID.status}" -DB_SIZE="${DB_SIZE:-$((8192*AL128K))}" - -INFO "Check: Disk exists" -if [ ! -b "/dev/disk/by-uuid/$UUID" ]; then - ERRO "Missing disk: /dev/disk/by-uuid/$UUID" -fi - -is_btrfs(){ [ "$(blkid -s TYPE -o value "$1")" == "btrfs" ]; } - -INFO "Check: Disk with btrfs" -if ! is_btrfs "/dev/disk/by-uuid/$UUID"; then - ERRO "Disk not contain btrfs: /dev/disk/by-uuid/$UUID" -fi - -INFO "WORK DIR: $WORK_DIR" -mkdir -p "$WORK_DIR" || exit 1 - -INFO "MOUNT DIR: $MNT_DIR" -mkdir -p "$MNT_DIR" || exit 1 - -umount_w(){ mountpoint -q "$1" && umount -l "$1"; } -force_umount(){ umount_w "$MNT_DIR"; } -trap force_umount SIGINT SIGTERM EXIT +set_option() { + local k v + k="${1%%=*}" v="${1#*=}" + [[ ${altConfigNames[$k]} ]] && k=${altConfigNames[$k]} + printf -v "bees_$k" %s "$v" +} -mount -osubvolid=5 /dev/disk/by-uuid/$UUID "$MNT_DIR" || exit 1 +uuid_re='^[[:xdigit:]]{8}-[[:xdigit:]]{4}-[[:xdigit:]]{4}-[[:xdigit:]]{4}-[[:xdigit:]]{12}$' + +# Shared code for setting configuration used by other operations. +# +# Reads from global associative array "opts" containing options passed in as +# key=value pairs on the command line, looks for config-file overrides, and +# sets individual global variables. +_setup() { + declare fstype + bees_fsSpec=$1; shift + + # Look for file-based configuration, additional to honoring configuration on the command line + bees_config_dir="${bees_config_dir:-/etc/bees}" + if [[ $bees_fsSpec =~ $uuid_re ]]; then + bees_uuid=$bees_fsSpec + # If our spec looks like a bare UUID, and no config file exists in the new + # format, fall back to legacy config file search mechanism (grep; ewww). + if ! readConfigFileIfExists "$bees_config_dir/UUID=$bees_fsSpec.conf"; then + # Legacy approach to finding a config file: Grep for a *.conf file + # containing the UUID within its text. Permitting spaces around the "=" + # appears to be a bug, but is retained for compatibility with the + # original upstream script. + allConfFiles=( "$bees_config_dir"/*.conf ) + if (( ${#allConfFiles[@]} )); then + # in read or readarray with -d '', the NUL terminating the empty string is used as delimiter character. + readarray -d '' -t matchingConfFiles < <(grep -E -l -Z "^[^#]*UUID[[:space:]]*=[[:space:]]*" "${allConfFiles[@]}") + else + matchingConfFiles=( ) + fi + if (( ${#matchingConfFiles[@]} == 1 )); then + # Exactly one configuration file exists in our target directory with a reference to the UUID given. + bees_config_file=${matchingConfFiles[0]} + readConfigFileIfExists "$bees_config_file" + echo "NOTE: Please consider renaming $bees_config_file to $bees_config_dir/UUID=$bees_fsSpec" >&2 + echo " ...and passing UUID=$bees_fsSpec on startup." >&2 + elif (( ${#matchingConfFiles[@]} > 1 )); then + # The legacy wrapper would silently use the first file and ignore + # others, but... no. + echo "ERROR: Passed a bare UUID, but multiple configuration files match it:" >&2 + printf ' - %q\n' "${matchingConfFiles[@]}" >&2 + die "Unable to continue." + fi + fi + else + # For a non-UUID fsSpec that is not a path, look only for a config file + # exactly matching its text. + # + # (Passing a mount point as a fsSpec is only supported with the new + # wrapper; all key=value pairs can be passed on the command line in this + # mode, so config file support is not needed). + [[ $bees_fsSpec = */* ]] || readConfigFileIfExists "$bees_config_dir/$bees_fsSpec.conf" + fi + + [[ $bees_uuid ]] || { + # if bees_uuid is not in our .conf file, look it up with findmnt + read -r bees_uuid fstype < <(findmnt -n -o uuid,fstype "$bees_fsSpec") && [[ $fstype ]] || exit + [[ $fstype = btrfs ]] || die "Device type is $fstype, not btrfs" + } + + [[ $bees_uuid = */* ]] || readConfigFileIfExists "$bees_config_dir/UUID=$bees_uuid.conf" + + # Honor any values read from config files above; otherwise, set defaults. + bees_workDir="${bees_workDir:-.beeshome}" + bees_runDir="${bees_runDir:-/run/bees}" + bees_mntDir="${bees_mntDir:-$bees_runDir/mnt/$bees_uuid}" + bees_home="${bees_home:-$bees_mntDir/$bees_workDir}" + bees_status="${bees_status:-${bees_runDir}/$bees_uuid.status}" + bees_verbosity="${bees_verbosity:-6}" + bees_idxSizeMB="${bees_idxSizeMB:-1024}" + bees_idxSize=${bees_idxSize:-"$(( bees_idxSizeMB * 1024 * 1024 ))"} + bees_blockdev=${bees_blockdev:-"/dev/disk/by-uuid/$bees_uuid"} + + [[ -b $bees_blockdev ]] || die "Block device $bees_blockdev missing" + (( bees_idxSize % (128 * 1024) == 0 )) || die "DB size must be divisible by 128KB" +} -if [ ! -d "$BEESHOME" ]; then - INFO "Create subvol $BEESHOME for store bees data" - btrfs sub cre "$BEESHOME" -else - btrfs sub show "$BEESHOME" &> /dev/null || ERRO "$BEESHOME MUST BE A SUBVOL!" -fi +do_run() { + local db old_db_size + + _setup "$1"; shift + mkdir -p -- "$bees_mntDir" || exit + + # subvol id 5 is reserved for the root subvolume of a btrfs filesystem. + mountpoint -q "$bees_mntDir" || mount -osubvolid=5 -- "$bees_blockdev" "$bees_mntDir" || exit + if [[ -d $bees_home ]]; then + btrfs subvolume show "$bees_home" >/dev/null 2>&1 || die "$bees_home exists but is not a subvolume" + else + btrfs subvolume create "$bees_home" || exit + sync # workaround for Zygo/bees#93 + fi + db=$bees_home/beeshash.dat + touch -- "$db" + + old_db_size=$(stat -c %s -- "$db") + new_db_size=$bees_idxSize + + if (( old_db_size != new_db_size )); then + rm -f -- "$bees_home"/beescrawl."$bees_uuid".dat + truncate -s "$new_db_size" -- "$db" || exit + fi + chmod 700 -- "$bees_home" + + # BEESSTATUS and BEESHOME are the only variables handled by the legacy + # wrapper for which getenv() is called in C code. + BEESSTATUS=$bees_status BEESHOME=$bees_home exec "${beesd_bin:-/lib/bees/bees}" \ + --verbose "$bees_verbosity" \ + "$@" "$bees_mntDir" || exit +} -# Check DB size -{ - DB_PATH="$BEESHOME/beeshash.dat" - touch "$DB_PATH" - OLD_SIZE="$(du -b "$DB_PATH" | sed 's/\t/ /g' | cut -d' ' -f1)" - NEW_SIZE="$DB_SIZE" - if (( "$NEW_SIZE"%AL128K > 0 )); then - ERRO "DB_SIZE Must be multiple of 128K" - fi - if (( "$OLD_SIZE" != "$NEW_SIZE" )); then - INFO "Resize db: $OLD_SIZE -> $NEW_SIZE" - [ -f "$BEESHOME/beescrawl.$UUID.dat" ] && rm "$BEESHOME/beescrawl.$UUID.dat" - truncate -s $NEW_SIZE $DB_PATH - fi - chmod 700 "$DB_PATH" +do_cleanup() { + _setup "$1"; shift + mountpoint -q "$bees_mntDir" && umount -l -- "$bees_mntDir" || exit } -MNT_DIR="$(realpath $MNT_DIR)" +(( $# >= 2 )) || usage +declare -f "do_$1" >/dev/null 2>&1 || usage +mode=$1; shift # must be a do_* function; currently "run" or "cleanup" + +declare -a args=( "$1" ); shift # pass first argument (config-name|fsSpec) through literally + +# parse other arguments as key=value pairs, or pass them through literally if they do not match that form. +# similarly, any option after "--" will be passed through literally. +while (( $# )); do + if [[ $1 = *=* ]]; then + set_option "$1" + elif [[ $1 = -- ]]; then + shift + args+=( "$@" ) + break + else + args+=( "$1" ) + fi + shift +done -cd "$MNT_DIR" -"$bees_bin" "${ARGUMENTS[@]}" $OPTIONS "$MNT_DIR" +"do_$mode" "${args[@]}"