Add virtme-prep-kdir-mods and support it

This is still experimental.  With this patch, you can run
virtme-prep-kdir-mods, and then virtme-run --kdir will pick up
modules from the kernel directory.

Known issues:

 - There are no docs.

 - This is unlikely to work well if forced module signing is on.  An
   option to virtme-prep-kdir-mods could be added to work around
   that, but it would run slower.

 - There is no intelligent handling for the case where the kernel
   was modified and rebuilt but virtme-prep-kdir-mods was re-run.
   Because it uses symlinks, it will only load modules that still
   have their .ko files around, but virtme-run won't notice the
   discrepency.  Maybe file timestamps could be used to try to
   detect this.

Signed-off-by: Andy Lutomirski <luto@kernel.org>

4 files changed, 63 insertions, 18 deletions

diff --git a/bin/virtme-prep-kdir-mods b/bin/virtme-prep-kdir-mods
new file mode 100755
index 0000000..1906e81
--- /dev/null
+++ b/bin/virtme-prep-kdir-mods
@@ -0,0 +1,32 @@
+#!/bin/sh
+
+# This is still a bit of an experiment.
+
+if ! [ -d .tmp_versions ]; then
+    echo 'virtme-prep-kdir-mods must be run from a kernel build directory' >&2
+    exit 1
+fi
+
+FAKEVER=0.0.0
+MODDIR=".virtme_mods/lib/modules/$FAKEVER"
+
+# Set up .virtme_mods/lib/modules/0.0.0 as a module directory for this kernel,
+# but fill it with symlinks instead of actual modules.
+
+mkdir -p "$MODDIR/kernel"
+ln -srfT . "$MODDIR/build"
+
+# Remove all preexisting symlinks and add symlinks to all modules that belong
+# to the build kenrnel.
+find "$MODDIR/kernel" -type l -print0 |xargs -0 rm -f --
+grep -h '.ko$' .tmp_versions/*.mod |while read -r i; do
+    mkdir -p "$MODDIR/kernel/$(dirname "$i")"
+    ln -sr "$i" "$MODDIR/kernel/$i"
+done
+
+
+# Link in the files that make modules_install would copy
+ln -srf modules.builtin modules.builtin.modinfo modules.order "$MODDIR/"
+
+# Now run depmod to collect dependencies
+depmod -ae -F System.map -b .virtme_mods "$FAKEVER"
diff --git a/setup.py b/setup.py
index 9f7dc63..03c6061 100755
--- a/setup.py
+++ b/setup.py
@@ -29,6 +29,9 @@ setup(
             'virtme-configkernel = virtme.commands.configkernel:main',
         ]
     },
+    scripts = [
+        'bin/virtme-prep-kdir-mods',
+        ],
     package_data = {
         'virtme.guest': [
             'virtme-init',
diff --git a/virtme/commands/run.py b/virtme/commands/run.py
index 407f445..8c17592 100644
--- a/virtme/commands/run.py
+++ b/virtme/commands/run.py
@@ -118,12 +118,9 @@ def arg_fail(message):
     _ARGPARSER.print_usage()
     sys.exit(1)
 
-def get_kver_from_kdir(kdir):
-    kver_path = os.path.join(kdir, 'include/config/kernel.release')
-    with open(kver_path) as fd:
-        return fd.read().strip()
-
 def find_kernel_and_mods(arch, args):
+    use_root_mods = False
+
     if args.installed_kernel is not None:
         kver = args.installed_kernel
         modfiles = modfinder.find_modules_from_install(
@@ -133,14 +130,14 @@ def find_kernel_and_mods(arch, args):
         if not os.path.exists(kimg):
             kimg = '/boot/vmlinuz-%s' % kver
         dtb = None  # For now
+        use_root_mods = True
     elif args.kdir is not None:
         kimg = os.path.join(args.kdir, arch.kimg_path())
-        tmp_moddir = os.path.join(args.kdir, '.tmp_moddir')
-        if os.path.exists(tmp_moddir):
-            kver = get_kver_from_kdir(args.kdir)
-            moddir = os.path.join(tmp_moddir, 'lib/modules', kver)
+        virtme_mods = os.path.join(args.kdir, '.virtme_mods')
+        if os.path.exists(virtme_mods):
+            moddir = os.path.join(virtme_mods, 'lib/modules', '0.0.0')
             modfiles = modfinder.find_modules_from_install(
-                virtmods.MODALIASES, kver=kver)
+                virtmods.MODALIASES, kver='0.0.0')
         else:
             kver = None
             moddir = None
@@ -159,7 +156,7 @@ def find_kernel_and_mods(arch, args):
     else:
         arg_fail('You must specify a kernel to use.')
 
-    return kimg,dtb,modfiles,moddir
+    return kimg,dtb,modfiles,moddir,use_root_mods
 
 def export_virtfs(qemu, arch, qemuargs, path, mount_tag, security_model='none', readonly=True):
     # NB: We can't use -virtfs for this, because it can't handle a mount_tag
@@ -197,7 +194,7 @@ def main():
 
     config = mkinitramfs.Config()
 
-    kimg,dtb,modfiles,moddir = find_kernel_and_mods(arch, args)
+    kimg,dtb,modfiles,moddir,use_root_mods = find_kernel_and_mods(arch, args)
     config.modfiles = modfiles
     if config.modfiles:
         need_initramfs = True
@@ -224,9 +221,20 @@ def main():
                 '/bin/mount -n -t 9p -o ro,version=9p2000.L,trans=virtio,access=any virtme.guesttools /run/virtme/guesttools',
                 'exec /run/virtme/guesttools/virtme-init']
 
-    # Map modules
+    # Arrange for modules to end up in the right place
     if moddir is not None:
-        export_virtfs(qemu, arch, qemuargs, moddir, 'virtme.moddir')
+        if use_root_mods:
+            # Tell virtme-init to use the root /lib/modules
+            kernelargs.append("virtme_root_mods=1")
+        else:
+            # We're grabbing modules from somewhere other than /lib/modules.
+            # Rather than mounting it separately, symlink it in the guest.
+            # This allows symlinks within the module directory to resolve
+            # correctly in the guest.
+            kernelargs.append("virtme_link_mods=/%s" % qemu.quote_optarg(os.path.relpath(moddir, args.root)))
+    else:
+        # No modules are available.  virtme-init will hide /lib/modules/KVER
+        pass
 
     # Set up mounts
     mount_index = 0
diff --git a/virtme/guest/virtme-init b/virtme/guest/virtme-init
index cd3caa4..f60faa5 100755
--- a/virtme/guest/virtme-init
+++ b/virtme/guest/virtme-init
@@ -27,10 +27,12 @@ done
 
 kver="`uname -r`"
 
-if [[ -n "${mount_tags[virtme.moddir]}" ]]; then
-    mount -t tmpfs none /lib/modules
-    mkdir /lib/modules/"$kver"
-    mount -n -t 9p -o ro,version=9p2000.L,trans=virtio,access=any virtme.moddir /lib/modules/"$kver"
+if [[ -n "$virtme_root_mods" ]]; then
+    # /lib/modules is already set up
+    true
+elif [[ -n "$virtme_link_mods" ]]; then
+    mount -n -t tmpfs none /lib/modules
+    ln -s "$virtme_link_mods" "/lib/modules/$kver"
 elif [[ -d "/lib/modules/$kver" ]]; then
     # We may have mismatched modules.  Mask them off.
     mount -n -t tmpfs -o ro,mode=0000 disallow_modules "/lib/modules/$kver"