Mercurial > hgrepos > FreeBSD > ports > sysutils > local-bsdtools

changeset 474:57f253106ed6

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
Implement fbhyve, a management system that runs bhyve virtual machines within tmux sessions
author Franz Glasner <fzglas.hg@dom66.de>
date Tue, 27 Aug 2024 21:47:57 +0200
parents c3125616d0ec
children 827371176fec
files .hgignore Makefile docs/man/man8/fbhyve.rst files/fbhyve.in pkg-plist tests/fbhyve/fbhyve_pc03.rc.conf.test tests/fbhyve/prepare-pc03.sh tests/fbhyve/vm_xxx_pc03.config
diffstat 8 files changed, 258 insertions(+), 126 deletions(-) [+]
line wrap: on
line diff
--- a/.hgignore	Mon Aug 26 17:22:26 2024 +0200
+++ b/.hgignore	Tue Aug 27 21:47:57 2024 +0200
@@ -4,3 +4,4 @@
 ^docs/_build/
 ^docs/_venv.*/
 ^_venv.*/
+^tests/fbhyve/bootvars.fd
--- a/Makefile	Mon Aug 26 17:22:26 2024 +0200
+++ b/Makefile	Tue Aug 27 21:47:57 2024 +0200
@@ -26,9 +26,13 @@
 USE_RC_SUBR=	fbhyve fwireguard
 
 SUB_LIST=	SIMPLEVERSIONTAG="${SIMPLEVERSIONTAG}" \
-		FWIREGUARD_ETCDIR="${FWIREGUARD_ETCDIR}"
+		FWIREGUARD_ETCDIR="${FWIREGUARD_ETCDIR}" \
+		FBHYVE_ETCDIR="${FBHYVE_ETCDIR}" \
+		FBHYVE_RUNDIR="${FBHYVE_RUNDIR}"
 
-PLIST_SUB=	FWIREGUARD_ETCDIR="${FWIREGUARD_ETCDIR}"
+PLIST_SUB=	FWIREGUARD_ETCDIR="${FWIREGUARD_ETCDIR}" \
+		FBHYVE_ETCDIR="${FBHYVE_ETCDIR}" \
+	=	FBHYVE_RUNDIR="${FBHYVE_RUNDIR}"
 
 .include <bsd.port.options.mk>
 
@@ -49,6 +53,9 @@
 
 FWIREGUARD_ETCDIR?=	"${PREFIX}/etc/fwireguard"
 
+FBHYVE_ETCDIR?=	"${PREFIX}/etc/fbhyve"
+FBHYVE_RUNDIR?=	/var/run/fbhyve
+
 do-extract:
 	${MKDIR} ${WRKSRC}/bin
 	${MKDIR} ${WRKSRC}/sbin
@@ -111,6 +118,8 @@
 	${INSTALL_DATA} ${WRKSRC}/share/examples/${PORTNAME}/${_exf} ${STAGEDIR}${EXAMPLESDIR}
 .endfor
 	${MKDIR} ${STAGEDIR}${FWIREGUARD_ETCDIR}
+	${MKDIR} ${STAGEDIR}${FBHYVE_ETCDIR}
+	${MKDIR} ${STAGEDIR}${FBHYVE_RUNDIR}
 
 post-install-DOCS-on:
 .for _mp in ${MANPAGES5:R}
--- a/docs/man/man8/fbhyve.rst	Mon Aug 26 17:22:26 2024 +0200
+++ b/docs/man/man8/fbhyve.rst	Tue Aug 27 21:47:57 2024 +0200
@@ -6,35 +6,60 @@
 Synopsis
 --------
 
-**service fbhyve** [ **start** | **stop** | **poll** | **status** ] [**name**]...
+**service fbhyve** [ **start** | **stop** | **restart** | **poll** | **status** ] [**name**]...
 
 
 Description
 -----------
 
-An :file:`rc.d` script to start :manpage:`bhyve(8)` virtual machines
+An :file:`rc.d` script to start :manpage:`bhyve(8)` virtual machines.
+
+Every VM is lives its own dedicated :manpage:`tmux(1)` session and has
+its console windows therein. The sessions have names that are derived from
+the name of the corresponding VM.
 
 
 Configuration Variables
 ~~~~~~~~~~~~~~~~~~~~~~~
 
 `fbhyve_enable`
+  Act as default for all listed VMs.
   Default: NO
 
 `fbhyve_list`
-  The name of VMs to start to (aka. "profiles" or "sessions")
+  The list of names of VMs to start to (aka. "profiles" or "sessions").
+  Default: empty
+
+`fbhyve_configdir`
+  Directory where by default the configuration files for all VMs live.
+  Default: :file:`/usr/local/etc/fbhyve`
+
+`fbhyve_tmux_session_prefix`
+  All tmux session names will have this prefix string. The effective session
+  name will have the VM's name appended.
+
+Each VM that is listed in `fbhyve_list` can be configured individually with:
+
+`fbhyve_<vm>_enable`
+  Enable or disable the virtual machine.
+  Default: ``$fbhyve_enable``
+
+`fbhyve_<vm>_config`
+  The :manpage:`bhyve_config(5)` style configuration file with configuration
+  variables for the virtual machine.
+  Default: ``${fbhyve_configdir}/<vm>.conf``
 
 
 Files
 -----
 
-- :file:`/usr/local/etc/fbhyve`
+- :file:`/var/run/fbhyve`
 
-    Default-directory where all the configuration files with relative
-    filenames are looked up.
+    Directory where some runtime data and PID files are stored.
 
 
 See also
 --------
 
-:manpage:`bhyve(8)`, :manpage:`rc.conf(5)`, :manpage:`tmux(1)`
+:manpage:`bhyve(8)`, :manpage:`bhyve_config(5)`, :manpage:`rc.conf(5)`,
+:manpage:`tmux(1)`
--- a/files/fbhyve.in	Mon Aug 26 17:22:26 2024 +0200
+++ b/files/fbhyve.in	Tue Aug 27 21:47:57 2024 +0200
@@ -1,7 +1,8 @@
 #!/bin/sh
+# -*- indent-tabs-mode: nil -*-
 
 # PROVIDE: fbhyve
-# REQUIRE: LOGIN
+# REQUIRE: LOGIN FILESYSTEMS
 # KEYWORD: shutdown nojail
 #
 
@@ -9,140 +10,145 @@
 
 #
 # Add the following lines to /etc/rc.conf to enable bhyve:
+#
 # fbhyve_enable (bool):  Set to "NO" by default.
-#                        Set it to "YES" to enable bhyve
-# bhyve_profiles (str): Set to "" by default.
-#                       Define your profiles here.
-# bhyve_tapdev (str):   Set to "tap0" by default.
-#                       Set to the tap(4) device to use.
-# bhyve_diskdev (str):  Must be set, no default.
-#                       Set to the disk device to use.
-# bhyve_ncpu (int):     Set to 1 by default.
-#                       Set to the number of CPUs for the VM.
-# bhyve_memsize (int):  Set to 512 by default.
-#                       Set to the number of MB of memory for the VM.
+#                        Acts as default for all listed VMs.
+#                        Set it to "YES" to enable bhyve.
+# fbhyve_list (str):     Set to "" by default.
+#                        Define the names of your VMs here.
+# fbhyve_tmux_session_prefix (str):  All tmux session names will have this
+#                                    prefix string.
+# fbhyve_configdir (str): Where by default config files for all VMs live.
+#                         Default: $PREFIX/etc/fbhyve
+#
+#
+# fbhyve_<vm>_enable (bool):  Allow to enable or disable a specific VM.
+#                             Set to $fbhyve_enable by default.
+# fbhyve_<vm>_config (str):   The bhyve configuration file to use to.
+#                             The default is $fbhyve_configdir/<vm>.conf
+#
 
 . /etc/rc.subr
 
 name="fbhyve"
-rcvar=fbhyve_enable
+desc="Manage system bhyve virtual machines"
+rcvar="fbhyve_enable"
 
-start_precmd="bhyve_prestart"
-status_cmd="bhyve_status"
-poll_cmd="bhyve_poll"
-stop_cmd="bhyve_stop"
-_session=$name
-command="/usr/local/bin/tmux"
-procname="-sh"
+start_precmd="fbhyve_pre_start"
+stop_postcmd="fbhyve_post_stop"
+status_cmd="fbhyve_status"
 
-[ -z "$bhyve_tapdev" ]  && bhyve_tapdev="tap0"
-[ -z "$bhyve_diskdev" ] && bhyve_diskdev="none"
-[ -z "$bhyve_ncpu" ]    && bhyve_ncpu="1"
-[ -z "$bhyve_memsize" ] && bhyve_memsize="512"
 
 load_rc_config $name
 
-: {fbhyve_enable:="NO"}
-: {fbhyve_configdir:="%%PREFIX%%/etc/fbhyve"}
+: ${fbhyve_enable:="NO"}
+: ${fbhyve_list=}
+: ${fbhyve_tmux_session_prefix:="${name}_"}
+: ${fbhyve_configdir:="%%FBHYVE_ETCDIR%%"}
+
+
+_fbhyve_vm_exists() {
+    local _p
+    for _p in ${fbhyve_list}; do
+        [ "${_p}" = "$1" ] && return 0;
+    done
+    return 1
+}
+
+
+if [ $# -eq 2 ]; then
+    _vm="$2"
+    if ! _fbhyve_vm_exists "${_vm}"; then
+        echo "ERROR: no VM named \`${_vm}' in \`fbhyve_list'" 1>&2
+        exit 1
+    fi
+    echo "-- VM: ${_vm} --"
+    _session="${fbhyve_tmux_session_prefix}${_vm}"
+    _window="${_session}_console"
+    eval fbhyve_enable="\${fbhyve_${_vm}_enable:-${fbhyve_enable}}"
+    eval fbhyve_config="\${fbhyve_${_vm}_config:-\"${fbhyve_configdir}/${_vm}.conf\"}"
+else
+    _ec=0
+    _swap=$*; shift; _vmarglist=$*
+    _vmlist=${_vmarglist:-${fbhyve_list}}
+    set -- ${_swap}
+    for _vm in ${_vmlist}; do
+        "$0" "$1" "${_vm}"
+        _vmec=$?
+        if [ ${_vmec} -gt ${_ec} ]; then
+            _ec=${_vmec}
+        fi
+    done
+    exit ${_ec}
+fi
+
+_rundir="%%FBHYVE_RUNDIR%%"
+pidfile="${_rundir}/${_vm}.pid"
+procname="bhyve:"         # something like bhyve: <vmname> (bhyve)
+
+required_dirs="${_rundir}"
+required_files="${fbhyve_config}"
+
+command="%%LOCALBASE%%/bin/tmux"
+
+command_args="new-session -ds ${_session} -n ${_window} \"sh -c 'echo \\\$\\\$ >\\\"${pidfile}\\\"; /usr/sbin/bhyve -k \\\"${fbhyve_config}\\\" \\\"${_vm}\\\"'\""
 
 
-if [ -n "$2" ]; then
-	profile="$2"
-	_session="${_session}_${profile}"
-	if [ "x${bhyve_profiles}" != "x" ]; then
-		eval fbhyve_enable="\${${_session}_enable:-${fbhyve_enable}}"
-		eval bhyve_tapdev="\${${_session}_tapdev:-${bhyve_tapdev}}"
-		eval bhyve_diskdev="\${${_session}_diskdev:-${bhyve_diskdev}}"
-		eval bhyve_ncpu="\${${_session}_ncpu:-${bhyve_ncpu}}"
-		eval bhyve_memsize="\${${_session}_memsize:-${bhyve_memsize}}"
-	else
-		echo "$0: extra argument ignored"
-	fi
-else
-	if [ "x${bhyve_profiles}" != "x" -a "x$1" != "x" ]; then
-		for profile in ${bhyve_profiles}; do
-			eval _enable="\${bhyve_${profile}_enable}"
-			case "x${_enable:-${fbhyve_enable}}" in
-			x|x[Nn][Oo]|x[Nn][Oo][Nn][Ee])
-				continue
-				;;
-			x[Yy][Ee][Ss])
-				;;
-			*)
-				if test -z "$_enable"; then
-					_var=fbhyve_enable
-				else
-					_var=bhyve_"${profile}"_enable
-				fi
-				echo "Bad value" \
-				    "'${_enable:-${fbhyve_enable}}'" \
-				    "for ${_var}. " \
-				    "Profile ${profile} skipped."
-				continue
-				;;
-			esac
-			echo "===> bhyve profile: ${profile}"
-			/usr/local/etc/rc.d/bhyve $1 ${profile}
-			retcode="$?"
-			if [ "0${retcode}" -ne 0 ]; then
-				failed="${profile} (${retcode}) ${failed:-}"
-			else
-				success="${profile} ${success:-}"
-			fi
-		done
-		exit 0
-	fi
-	profile=$name
-fi
+fbhyve_status()
+{
+    local _pid _rc
+
+    _rc=0
+    _pid=$(check_pidfile "$pidfile" "$procname")
+    if [ -n "${_pid}" ]; then
+        echo "VM ${_vm} is running as pid $_pid."
+    else
+        echo "VM ${_vm} is not running."
+        _rc=1
+    fi
 
-pidfile="/var/run/${_session}.pid"
+    if ${command} has-session -t ${_session} 2>/dev/null; then
+	echo "tmux session ${_session} exists."
+        if [ ${_rc} -gt 0 ]; then
+            _rc=2
+        fi
+    else
+	echo "tmux session ${_session} does not exist."
+        if [ ${_rc} -gt 0 ]; then
+            _rc=2
+        fi
+    fi
+    return ${_rc}
+}
 
 
-bhyve_prestart()
-{
-	case ${bhyve_diskdev} in
-	[Nn][Oo][Nn][Ee] | '')
-		echo "No ${_session}_diskdev set. Quitting." 1>&2
-		return 1;
-		;;
-	esac
-	if [ ! -c "${bhyve_diskdev}" -a ! -f "${bhyve_diskdev}" ]; then
-		echo "${bhyve_diskdev} doesn't exist or is not suitable as a diskdev" 1>&2
-		return 1;
-	fi
-}
-
-bhyve_status()
-{
-	if ${command} has-session -t ${_session} 2>/dev/null; then
-		echo "${_session} is running."
-	else
-		echo "${_session} is not running."
-		return 1
-	fi
+fbhyve_pre_start() {
+    if ! load_kld -m vmm vmm.ko; then
+        echo "ERROR: Cannot load kernel module \`vmm'" 1>&2
+        return 1
+    fi
+    if [ -e "/dev/vmm/${_vm}" ]; then
+        echo "ERROR: VM \`${_vm}' already created in the VM monitor" 1>&2
+        return 1
+    fi
+    if ${command} has-session -t "${_session}" 2>/dev/null; then
+        echo "ERROR: tmux session \`${_session}' already exists" 1>&2
+        return 1
+    fi
+    return 0
 }
 
-bhyve_poll()
-{
-	echo -n "Waiting for session: ${_session}"
-	while ${command} has-session -t ${_session} 2>/dev/null; do
-		sleep 1
-	done
-	echo
+
+fbhyve_post_stop() {
+    if [ -e "/dev/vmm/${_vm}" ]; then
+        /usr/sbin/bhyvectl --vm="${_vm}" --destroy
+    fi
+    if ${command} has-session -t "${_session}" 2>/dev/null; then
+        ${command} kill-session -t "${_session}"
+    fi
+    rm -f "${pidfile}"
+    return 0
 }
 
-bhyve_stop()
-{
-	if ${command} has-session -t ${_session} 2>/dev/null; then
-		echo "Stopping ${_session}."
-		${command} kill-session -t ${_session}
-		while ${command} has-session -t ${_session} 2>/dev/null; do
-			sleep 1
-		done
-	fi
-	rm -f ${pidfile}
-}
-
-command_args="new-session -ds ${_session} \"sh -c 'echo \\\$PPID >${pidfile}; while true; do /usr/sbin/bhyvectl --vm=${_session} --destroy; /usr/sbin/bhyveload -m ${bhyve_memsize} -d ${bhyve_diskdev} ${_session} && /usr/sbin/bhyve -c ${bhyve_ncpu} -m ${bhyve_memsize} -AI -H -P -g 0 -s 0:0,hostbridge -s 1:0,virtio-net,${bhyve_tapdev} -s 2:0,virtio-blk,${bhyve_diskdev} -s 31,lpc -l com1,stdio ${_session} || break; done'\""
 
 run_rc_command "$1"
--- a/pkg-plist	Mon Aug 26 17:22:26 2024 +0200
+++ b/pkg-plist	Tue Aug 27 21:47:57 2024 +0200
@@ -46,3 +46,5 @@
 @comment DIRECTORIES
 @dir %%ETCDIR%%
 @dir %%FWIREGUARD_ETCDIR%%
+@dir %%FBHYVE_ETCDIR%%
+@dir %%FBHYVE_RUNDIR%%
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/tests/fbhyve/fbhyve_pc03.rc.conf.test	Tue Aug 27 21:47:57 2024 +0200
@@ -0,0 +1,13 @@
+#!/bin/sh
+
+_TESTHOME="/home/fag/work/ports/sysutils/local-bsdtools"
+
+#ln -sf "${_TESTHOME}/tests/fbhyve/rc.conf.test /etc/rc.conf.d/fbhyve
+
+#rc_debug=YES
+
+fbhyve_enable=YES
+
+fbhyve_list="xxx yyy"
+
+fbhyve_xxx_config="${_TESTHOME}/tests/fbhyve/vm_xxx_pc03.config"
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/tests/fbhyve/prepare-pc03.sh	Tue Aug 27 21:47:57 2024 +0200
@@ -0,0 +1,20 @@
+#!/bin/sh
+
+#
+# Prepare disk devices/volumes
+#
+#   -> Intermediates
+zfs create -o canmount=off -o mountpoint=none zpool/bhyve
+zfs create -o canmount=off -o mountpoint=none -o primarycache=metadata -o secondarycache=none zpool/bhyve/test-local-bsdtools
+zfs create -o canmount=off zpool/bhyve/test-local-bsdtools/fbhyve
+
+
+#   -> Single disk setup
+zfs create -o volmode=dev -s -V 3G zpool/bhyve/test-local-bsdtools/fbhyve/disk0
+
+#
+# UEFI-Variables
+#
+#   -> Copy the template for the UEFI vars
+#
+cp -v /usr/local/share/uefi-firmware/BHYVE_UEFI_VARS.fd  /home/fag/work/ports/sysutils/local-bsdtools/tests/fbhyve/bootvars.fd
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/tests/fbhyve/vm_xxx_pc03.config	Tue Aug 27 21:47:57 2024 +0200
@@ -0,0 +1,56 @@
+# FreeBSD 14.1
+# config.dump=true
+name=test-fbhyve
+# -A
+acpi_tables=true
+# -c ...
+cpus=2
+cores=2
+threads=1
+sockets=1
+# -m
+memory.size=1G
+# -S
+memory.wired=false
+# -D
+destroy_on_poweroff=true
+# -K de
+keyboard.layout=de
+# -u
+rtc.use_localtime=false
+# -H
+x86.vmexit_on_hlt=true
+# - P
+x86.vmexit_on_pause=true
+# -w
+x86.strictmsr=false
+# -s 0,amd_hostbridge
+pci.0.0.0.device=hostbridge
+pci.0.0.0.pcireg.vendor=0x1022
+pci.0.0.0.pcireg.device=0x7432
+# -s 3,virtio-blk,/dev/xxx,sectorsize=512/2048
+pci.0.3.0.device=virtio-blk
+pci.0.3.0.path=/dev/zvol/zpool/bhyve/test-local-bsdtools/fbhyve/disk0
+pci.0.3.0.sectorsize=512/2048
+# -s 7,ahci-hd,/my/path,ro
+pci.0.7.0.device=ahci
+pci.0.7.0.port.0.type=hd
+pci.0.7.0.port.0.path=/home/test2/Downloads/OpenBSD/7.5/install75.img
+pci.0.7.0.port.0.ro=true
+# -s 10,virtio-net,tap1
+#pci.0.10.0.device=virtio-net
+#pci.0.10.0.backend=tap1
+# -s 11,virtio-net,tap3 -- bridge0 -- to easily have SSH access for SFTP
+#pci.0.11.0.device=virtio-net
+#pci.0.11.0.backend=tap3
+# -s 20,virtio-rnd
+pci.0.20.0.device=virtio-rnd
+# -s 31,lpc
+pci.0.31.0.device=lpc
+# -l com1,stdio
+lpc.com1.path=stdio
+# -l bootrom,/usr/local/share/uefi-firmware/BHYVE_UEFI.fd,/home/fag/bhyve/openbsd/bootvars.data
+lpc.bootrom=/usr/local/share/uefi-firmware/BHYVE_UEFI.fd
+lpc.bootvars=/home/fag/work/ports/sysutils/local-bsdtools/tests/fbhyve/bootvars.fd
+# fwcfg
+lpc.fwcfg=qemu