Merge branch 'issues-307-320' into 'master'

Allow to add overlays to resources

Closes #320 and #307

See merge request lava/lava!1004

doc/v2/actions-deploy.rst

@@ -49,6 +49,35 @@ a *****
    explanations. Ensure all information on options and possible values is in the
    reference guide.
 
+Overlays
+********
+
+LAVA can insert user provided overlays into your images right after the download step.
+
+In the url block, you can add a dictionary called `overlays` that will list the
+overlays to add to the given resource.
+
+.. code-block:: yaml
+
+  - deploy:
+      images:
+        rootfs:
+          url: http://example.com/rootfs.ext4.xz
+          compression: xz
+          format: ext4
+          overlays:
+            modules:
+              url: http://example.com/modules.tar.xz
+              compression: xz
+              format: tar
+              path: /
+
+In order to insert overlay into an image, you should specify the image format.
+Currently LAVA supports cpio (newc format) and ext4 images.
+
+The overlays should be archived using tar. The path is relative to the root of
+the image to update. This path is required.
+
 Parameter List
 **************
 

lava_common/schemas/deploy/__init__.py

@@ -25,8 +25,11 @@ from voluptuous import Any, Optional, Required
 from lava_common.schemas import action
 
 
-def url():
-    return {
+def url(extra=None):
+    if extra is None:
+        extra = {}
+
+    base_url = {
         Required("url"): str,
         Optional("compression"): Any("bz2", "gz", "xz", "zip", None),
         Optional("archive"): "tar",
@@ -34,6 +37,17 @@ def url():
         Optional("sha256sum"): str,
         Optional("sha512sum"): str,
     }
+    return Any(
+        {**base_url, **extra},
+        {
+            **base_url,
+            **extra,
+            Required("format"): Any("cpio.newc", "ext4"),
+            Required("overlays"): {
+                str: {**base_url, Required("format"): Any("tar"), Required("path"): str}
+            },
+        },
+    )
 
 
 def schema():

lava_common/schemas/deploy/download.py

@@ -28,6 +28,6 @@ from lava_common.schemas import deploy
 def schema():
     base = {
         Required("to"): "download",
-        Required("images"): {Required(str, "'images' is empty"): {**deploy.url()}},
+        Required("images"): {Required(str, "'images' is empty"): deploy.url()},
     }
     return {**deploy.schema(), **base}

lava_common/schemas/deploy/fastboot.py

@@ -26,18 +26,17 @@ from lava_common.schemas import deploy
 
 
 def schema():
+    extra = {
+        Optional("apply-overlay"): bool,
+        Optional("sparse"): bool,
+        Optional("reboot"): Any(
+            "hard-reset", "fastboot-reboot", "fastboot-reboot-bootloader"
+        ),
+    }
+
     base = {
         Required("to"): "fastboot",
-        Required("images"): {
-            Required(str, "'images' is empty"): {
-                **deploy.url(),
-                Optional("apply-overlay"): bool,
-                Optional("sparse"): bool,
-                Optional("reboot"): Any(
-                    "hard-reset", "fastboot-reboot", "fastboot-reboot-bootloader"
-                ),
-            }
-        },
+        Required("images"): {Required(str, "'images' is empty"): deploy.url(extra)},
         Optional("docker"): {Required("image"): str},
         Optional("connection"): "lxc",  # FIXME: other possible values?
     }

lava_common/schemas/deploy/iso_installer.py

@@ -29,10 +29,9 @@ def schema():
     base = {
         Required("to"): "iso-installer",
         Required("images"): {
-            Required("iso"): {
-                **deploy.url(),
-                Optional("image_arg"): str,  # TODO: is this optional?
-            },
+            Required("iso"): deploy.url(
+                {Optional("image_arg"): str}  # TODO: is this optional?
+            ),
             Required("preseed"): deploy.url(),
         },
         Required("iso"): {

lava_common/schemas/deploy/mps.py

@@ -31,10 +31,9 @@ def schema():
         Required("images"): All(
             {
                 Optional("recovery_image"): deploy.url(),
-                Optional(Match("test_binary(_\\w+)?$")): {
-                    **deploy.url(),
-                    Optional("rename"): str,
-                },
+                Optional(Match("test_binary(_\\w+)?$")): deploy.url(
+                    {Optional("rename"): str}
+                ),
             },
             Length(min=1),
         ),

lava_common/schemas/deploy/nbd.py

@@ -30,10 +30,9 @@ def schema():
 
     base = {
         Required("to"): "nbd",
-        Required("kernel", msg="needs a kernel to deploy"): {
-            **resource,
-            Optional("type"): Any("image", "uimage", "zimage"),
-        },
+        Required("kernel", msg="needs a kernel to deploy"): deploy.url(
+            {Optional("type"): Any("image", "uimage", "zimage")}
+        ),
         Required("nbdroot"): resource,
         Required("initrd"): resource,
         Optional("dtb"): resource,

lava_common/schemas/deploy/nfs.py

@@ -36,7 +36,9 @@ def schema():
         {
             Required("to"): "nfs",
             Required("images"): {
-                Required(str, "'images' is empty"): {**deploy.url(), "image_arg": str}
+                Required(str, "'images' is empty"): deploy.url(
+                    {Optional("image_arg"): str}
+                )
             },
             **deploy.schema(),
         },

lava_common/schemas/deploy/recovery.py

@@ -28,6 +28,6 @@ from lava_common.schemas import deploy
 def schema():
     base = {
         Required("to"): "recovery",
-        Required("images"): {Required(str, "'images' is empty"): {**deploy.url()}},
+        Required("images"): {Required(str, "'images' is empty"): deploy.url()},
     }
     return {**deploy.schema(), **base}

lava_common/schemas/deploy/tftp.py

@@ -27,23 +27,29 @@ from lava_common.schemas import deploy
 
 def schema():
     resource = deploy.url()
-    resource_ext = {
-        **resource,
-        Optional("install_modules"): bool,
-        Optional("install_overlay"): bool,
-    }
 
     base = {
         Required("to"): "tftp",
-        Required("kernel", msg="needs a kernel to deploy"): {
-            **resource,
-            Optional("type"): Any("image", "uimage", "zimage"),
-        },
+        Required("kernel", msg="needs a kernel to deploy"): deploy.url(
+            {Optional("type"): Any("image", "uimage", "zimage")}
+        ),
         Optional("dtb"): resource,
         Optional("modules"): resource,
         Optional("preseed"): resource,
-        Optional("ramdisk"): {**resource_ext, Optional("header"): "u-boot"},
-        Exclusive("nfsrootfs", "nfs"): {**resource_ext, Optional("prefix"): str},
+        Optional("ramdisk"): deploy.url(
+            {
+                Optional("install_modules"): bool,
+                Optional("install_overlay"): bool,
+                Optional("header"): "u-boot",
+            }
+        ),
+        Exclusive("nfsrootfs", "nfs"): deploy.url(
+            {
+                Optional("install_modules"): bool,
+                Optional("install_overlay"): bool,
+                Optional("prefix"): str,
+            }
+        ),
         Exclusive("persistent_nfs", "nfs"): {
             Required("address"): str,
             Optional("install_overlay"): bool,

lava_common/schemas/deploy/tmpfs.py

@@ -26,15 +26,14 @@ from lava_common.schemas import deploy
 
 
 def schema():
+    extra = {
+        Optional("format"): "qcow2",
+        Optional("image_arg"): str,  # TODO: is this optional?
+    }
+
     base = {
         Required("to"): "tmpfs",
-        Required("images"): {
-            Required(str, "'images' is empty"): {
-                **deploy.url(),
-                Optional("format"): "qcow2",
-                Optional("image_arg"): str,  # TODO: is this optional?
-            }
-        },
+        Required("images"): {Required(str, "'images' is empty"): deploy.url(extra)},
         Optional("type"): "monitor",
         Optional("uefi"): deploy.url(),  # TODO: check the exact syntax
     }

lava_common/schemas/deploy/u_boot_ums.py

@@ -28,6 +28,6 @@ from lava_common.schemas import deploy
 def schema():
     base = {
         Required("to"): "u-boot-ums",
-        Required("image"): {**deploy.url(), Optional("root_partition"): Range(min=0)},
+        Required("image"): deploy.url({Optional("root_partition"): Range(min=0)}),
     }
     return {**deploy.schema(), **base}

lava_common/schemas/deploy/uuu.py

@@ -31,11 +31,12 @@ def schema():
         Required("images"): All(
             {
                 Required("boot"): deploy.url(),
-                Any(str): {
-                    **deploy.url(),
-                    Optional("apply-overlay"): bool,
-                    Optional("root_partition"): Range(min=0),
-                },
+                Any(str): deploy.url(
+                    {
+                        Optional("apply-overlay"): bool,
+                        Optional("root_partition"): Range(min=0),
+                    }
+                ),
             },
             Length(min=1),
         ),

lava_dispatcher/actions/boot/qemu.py

@@ -194,7 +194,7 @@ class CallQemuAction(Action):
                 action="download-action", label=label, key="file"
             )
             if not image_arg or not action_arg:
-                self.errors = "Missing image_arg for %s. " % label
+                self.logger.warning("Missing image arg for %s", label)
                 continue
             self.commands.append(image_arg)
 
@@ -236,7 +236,8 @@ class CallQemuAction(Action):
             action_arg = self.get_namespace_data(
                 action="download-action", label=label, key="file"
             )
-            substitutions["{%s}" % label] = action_arg
+            if image_arg is not None:
+                substitutions["{%s}" % label] = action_arg
         substitutions["{NFS_SERVER_IP}"] = dispatcher_ip(
             self.job.parameters["dispatcher"], "nfs"
         )

lava_dispatcher/actions/deploy/apply_overlay.py

@@ -18,6 +18,7 @@
 # along
 # with this program; if not, see <http://www.gnu.org/licenses>.
 
+import guestfs
 import os
 import shutil
 import subprocess  # nosec - internal use.
@@ -845,3 +846,125 @@ class ConfigurePreseedFile(Action):
                     post_command[0],
                 )
         return connection
+
+
+class AppendOverlays(Action):
+
+    name = "append-overlays"
+    description = "append overlays to an image"
+    summary = "append overlays to an image"
+
+    # TODO: list libguestfs supported formats
+    IMAGE_FORMATS = ["cpio.newc", "ext4"]
+    OVERLAY_FORMATS = ["tar"]
+
+    def __init__(self, key, params):
+        super().__init__()
+        self.key = key
+        self.params = params
+
+    def validate(self):
+        super().validate()
+        # Check that we have "overlays" dict
+        if "overlays" not in self.params:
+            raise JobError("Missing 'overlays' dictionary")
+        if not isinstance(self.params["overlays"], dict):
+            raise JobError("'overlays' is not a dictionary")
+        for overlay, params in self.params["overlays"].items():
+            if params.get("format") not in self.OVERLAY_FORMATS:
+                raise JobError(
+                    "Invalid 'format' (%r) for 'overlays.%s'"
+                    % (params.get("format", ""), overlay)
+                )
+            path = params.get("path")
+            if path is None:
+                raise JobError("Missing 'path' for 'overlays.%s'" % overlay)
+            if not path.startswith("/") or ".." in path:
+                raise JobError("Invalid 'path': %r" % path)
+
+        # Check the image format
+        if self.params.get("format") not in self.IMAGE_FORMATS:
+            raise JobError("Unsupported image format %r" % self.params.get("format"))
+
+    def run(self, connection, max_end_time):
+        connection = super().run(connection, max_end_time)
+        if self.params["format"] == "cpio.newc":
+            self.update_cpio()
+        elif self.params["format"] == "ext4":
+            try:
+                self.update_guestfs()
+            except RuntimeError as exc:
+                self.logger.exception(str(exc))
+                raise JobError("Unable to update image %s: %r" % (self.key, str(exc)))
+        else:
+            raise LAVABug("Unknown format %r" % self.params["format"])
+        return connection
+
+    def update_cpio(self):
+        image = self.get_namespace_data(
+            action="download-action", label=self.key, key="file"
+        )
+        compression = self.get_namespace_data(
+            action="download-action", label=self.key, key="compression"
+        )
+        decompressed = self.get_namespace_data(
+            action="download-action", label=self.key, key="decompressed"
+        )
+        self.logger.info("Modifying %r", image)
+        tempdir = self.mkdtemp()
+        # Some images are kept compressed. We should decompress first
+        if compression and not decompressed:
+            self.logger.debug("* decompressing (%s)", compression)
+            image = decompress_file(image, compression)
+        # extract the archive
+        self.logger.debug("* extracting %r", image)
+        uncpio(image, tempdir)
+        os.unlink(image)
+
+        # Add overlays
+        self.logger.debug("Overlays:")
+        for overlay in self.params["overlays"]:
+            label = "%s.%s" % (self.key, overlay)
+            overlay_image = self.get_namespace_data(
+                action="download-action", label=label, key="file"
+            )
+            path = self.params["overlays"][overlay]["path"]
+            self.logger.debug("- %s: %r to %r", label, overlay_image, path)
+            # In the "validate" function, we check that path startswith '/'
+            # and does not contains '..'
+            untar_file(overlay_image, "." + path)
+
+        # Recreating the archive
+        self.logger.debug("* archiving %r", image)
+        cpio(tempdir, image)
+        if compression and not decompressed:
+            self.logger.debug("* compressing (%s)", compression)
+            image = compress_file(image, compression)
+
+    def update_guestfs(self):
+        image = self.get_namespace_data(
+            action="download-action", label=self.key, key="file"
+        )
+        self.logger.info("Modifying %r", image)
+        guest = guestfs.GuestFS(python_return_dict=True)
+        guest.add_drive(image)
+        try:
+            guest.launch()
+            device = guest.list_devices()[0]
+            guest.mount(device, "/")
+        except RuntimeError as exc:
+            self.logger.exception(str(exc))
+            raise JobError("Unable to update image %s: %r" % (self.key, str(exc)))
+
+        self.logger.debug("Overlays:")
+        for overlay in self.params["overlays"]:
+            label = "%s.%s" % (self.key, overlay)
+            overlay_image = self.get_namespace_data(
+                action="download-action", label=label, key="file"
+            )
+            path = self.params["overlays"][overlay]["path"]
+            self.logger.debug("- %s: %r to %r", label, overlay_image, path)
+            guest.mkdir_p(path)
+            guest.tar_in(overlay_image, path)
+        guest.umount(device)
+        guest.shutdown()

lava_dispatcher/actions/deploy/download.py

@@ -35,6 +35,7 @@ import subprocess  # nosec - verified.
 from lava_dispatcher.power import ResetDevice
 from lava_dispatcher.protocols.lxc import LxcProtocol
 from lava_dispatcher.actions.deploy import DeployAction
+from lava_dispatcher.actions.deploy.apply_overlay import AppendOverlays
 from lava_dispatcher.actions.deploy.overlay import OverlayAction
 from lava_dispatcher.connections.serial import ConnectDevice
 from lava_common.exceptions import InfrastructureError, JobError, LAVABug
@@ -103,6 +104,15 @@ class DownloaderAction(RetryAction):
         else:
             raise JobError("Unsupported url protocol scheme: %s" % url.scheme)
         self.pipeline.add_action(action)
+        overlays = self.params.get("overlays", [])
+        for overlay in overlays:
+            self.pipeline.add_action(
+                DownloaderAction(
+                    "%s.%s" % (self.key, overlay), self.path, params=overlays[overlay]
+                )
+            )
+        if overlays:
+            self.pipeline.add_action(AppendOverlays(self.key, params=self.params))
 
 
 class DownloadHandler(Action):
@@ -149,18 +159,24 @@ class DownloadHandler(Action):
         )
         super().cleanup(connection)
 
-    def _url_to_fname(self, path, compression):
+    def _compression(self):
+        if self.key == "ramdisk":
+            return False
+        return self.params.get("compression", False)
+
+    def _url_to_fname(self):
+        compression = self._compression()
         filename = os.path.basename(self.url.path)
 
         # Don't rename files we don't decompress during download
         if not compression or (compression not in self.decompress_command_map):
-            return os.path.join(path, filename)
+            return os.path.join(self.path, filename)
 
         parts = filename.split(".")
         # Files without suffixes, e.g. kernel images
         if len(parts) == 1:
-            return os.path.join(path, filename)
-        return os.path.join(path, ".".join(parts[:-1]))
+            return os.path.join(self.path, filename)
+        return os.path.join(self.path, ".".join(parts[:-1]))
 
     def validate(self):
         super().validate()
@@ -174,7 +190,7 @@ class DownloadHandler(Action):
         overlay = self.params.get("overlay", False)
         image_arg = self.params.get("image_arg")
 
-        self.fname = self._url_to_fname(self.path, compression)
+        self.fname = self._url_to_fname()
         if self.fname.endswith("/"):
             raise JobError("Cannot download a directory for %s" % self.key)
         # Save into the namespaced data
@@ -268,11 +284,17 @@ class DownloadHandler(Action):
                     "Unable to create %s: %s" % (self.path, str(exc))
                 )
 
-        compression = self.params.get("compression", False)
+        compression = self._compression()
         if self.key == "ramdisk":
-            compression = False
             self.logger.debug("Not decompressing ramdisk as can be used compressed.")
 
+        self.set_namespace_data(
+            action="download-action",
+            label=self.key,
+            key="decompressed",
+            value=bool(compression),
+        )
+
         md5sum = self.params.get("md5sum")
         sha256sum = self.params.get("sha256sum")
         sha512sum = self.params.get("sha512sum")
@@ -311,7 +333,7 @@ class DownloadHandler(Action):
                     "Compression %s specified but not decompressing during download",
                     compression,
                 )
-        else:
+        elif not self.params.get("compression", False):
             self.logger.debug("No compression specified")
 
         def update_progress():

tests/lava_dispatcher/actions/deploy/test_apply_overlay.py

+import logging
+import pytest
+
+from lava_common.exceptions import JobError, LAVABug
+from lava_dispatcher.actions.deploy.apply_overlay import AppendOverlays
+from lava_dispatcher.job import Job
+
+
+def test_append_overlays_validate():
+    # 1/ Working setup
+    params = {
+        "format": "cpio.newc",
+        "overlays": {
+            "modules": {
+                "url": "http://example.com/modules.tar.xz",
+                "compression": "xz",
+                "format": "tar",
+                "path": "/",
+            }
+        },
+    }
+    action = AppendOverlays("rootfs", params)
+    action.validate()
+
+    # 2/ Check errors
+    with pytest.raises(JobError) as exc:
+        del params["format"]
+        action.validate()
+    assert exc.match("Unsupported image format None")
+
+    with pytest.raises(JobError) as exc:
+        params["overlays"]["modules"]["path"] = "../../"
+        action.validate()
+    assert exc.match("Invalid 'path': '../../'")
+
+    with pytest.raises(JobError) as exc:
+        del params["overlays"]["modules"]["path"]
+        action.validate()
+    assert exc.match("Missing 'path' for 'overlays.modules'")
+
+    with pytest.raises(JobError) as exc:
+        params["overlays"]["modules"]["format"] = "git"
+        action.validate()
+    assert exc.match("Invalid 'format' \\('git'\\) for 'overlays.modules'")
+
+    with pytest.raises(JobError) as exc:
+        params["overlays"] = ""
+        action.validate()
+    assert exc.match("'overlays' is not a dictionary")
+
+    with pytest.raises(JobError) as exc:
+        del params["overlays"]
+        action.validate()
+    assert exc.match("Missing 'overlays' dictionary")
+
+
+def test_append_overlays_run(mocker):
+    params = {
+        "format": "cpio.newc",
+        "overlays": {
+            "modules": {
+                "url": "http://example.com/modules.tar.xz",
+                "compression": "xz",
+                "format": "tar",
+                "path": "/",
+            }
+        },
+    }
+    action = AppendOverlays("rootfs", params)
+    action.update_cpio = mocker.stub()
+    action.update_guestfs = mocker.stub()
+    assert action.run(None, 0) is None
+    action.update_cpio.assert_called_once_with()
+
+    params["format"] = "ext4"
+    assert action.run(None, 0) is None
+    action.update_guestfs.assert_called_once_with()
+
+    params["format"] = "tar"
+    with pytest.raises(LAVABug):
+        action.run(None, 0)
+
+
+def test_append_overlays_update_cpio(caplog, mocker, tmpdir):
+    caplog.set_level(logging.DEBUG)