Commit fbeaa15a authored by stevanradakovic's avatar stevanradakovic
Browse files

Merge branch 'issues-307-320' into 'master'

Allow to add overlays to resources

Closes #320 and #307

See merge request lava/lava!1004
parents 72d8b9fa 4d6617bc
......@@ -49,6 +49,35 @@ a *****
explanations. Ensure all information on options and possible values is in the
reference guide.
Overlays
********
LAVA can insert user provided overlays into your images right after the download step.
In the url block, you can add a dictionary called `overlays` that will list the
overlays to add to the given resource.
.. code-block:: yaml
- deploy:
images:
rootfs:
url: http://example.com/rootfs.ext4.xz
compression: xz
format: ext4
overlays:
modules:
url: http://example.com/modules.tar.xz
compression: xz
format: tar
path: /
In order to insert overlay into an image, you should specify the image format.
Currently LAVA supports cpio (newc format) and ext4 images.
The overlays should be archived using tar. The path is relative to the root of
the image to update. This path is required.
Parameter List
**************
......
......@@ -25,8 +25,11 @@ from voluptuous import Any, Optional, Required
from lava_common.schemas import action
def url():
return {
def url(extra=None):
if extra is None:
extra = {}
base_url = {
Required("url"): str,
Optional("compression"): Any("bz2", "gz", "xz", "zip", None),
Optional("archive"): "tar",
......@@ -34,6 +37,17 @@ def url():
Optional("sha256sum"): str,
Optional("sha512sum"): str,
}
return Any(
{**base_url, **extra},
{
**base_url,
**extra,
Required("format"): Any("cpio.newc", "ext4"),
Required("overlays"): {
str: {**base_url, Required("format"): Any("tar"), Required("path"): str}
},
},
)
def schema():
......
......@@ -28,6 +28,6 @@ from lava_common.schemas import deploy
def schema():
base = {
Required("to"): "download",
Required("images"): {Required(str, "'images' is empty"): {**deploy.url()}},
Required("images"): {Required(str, "'images' is empty"): deploy.url()},
}
return {**deploy.schema(), **base}
......@@ -26,18 +26,17 @@ from lava_common.schemas import deploy
def schema():
extra = {
Optional("apply-overlay"): bool,
Optional("sparse"): bool,
Optional("reboot"): Any(
"hard-reset", "fastboot-reboot", "fastboot-reboot-bootloader"
),
}
base = {
Required("to"): "fastboot",
Required("images"): {
Required(str, "'images' is empty"): {
**deploy.url(),
Optional("apply-overlay"): bool,
Optional("sparse"): bool,
Optional("reboot"): Any(
"hard-reset", "fastboot-reboot", "fastboot-reboot-bootloader"
),
}
},
Required("images"): {Required(str, "'images' is empty"): deploy.url(extra)},
Optional("docker"): {Required("image"): str},
Optional("connection"): "lxc", # FIXME: other possible values?
}
......
......@@ -29,10 +29,9 @@ def schema():
base = {
Required("to"): "iso-installer",
Required("images"): {
Required("iso"): {
**deploy.url(),
Optional("image_arg"): str, # TODO: is this optional?
},
Required("iso"): deploy.url(
{Optional("image_arg"): str} # TODO: is this optional?
),
Required("preseed"): deploy.url(),
},
Required("iso"): {
......
......@@ -31,10 +31,9 @@ def schema():
Required("images"): All(
{
Optional("recovery_image"): deploy.url(),
Optional(Match("test_binary(_\\w+)?$")): {
**deploy.url(),
Optional("rename"): str,
},
Optional(Match("test_binary(_\\w+)?$")): deploy.url(
{Optional("rename"): str}
),
},
Length(min=1),
),
......
......@@ -30,10 +30,9 @@ def schema():
base = {
Required("to"): "nbd",
Required("kernel", msg="needs a kernel to deploy"): {
**resource,
Optional("type"): Any("image", "uimage", "zimage"),
},
Required("kernel", msg="needs a kernel to deploy"): deploy.url(
{Optional("type"): Any("image", "uimage", "zimage")}
),
Required("nbdroot"): resource,
Required("initrd"): resource,
Optional("dtb"): resource,
......
......@@ -36,7 +36,9 @@ def schema():
{
Required("to"): "nfs",
Required("images"): {
Required(str, "'images' is empty"): {**deploy.url(), "image_arg": str}
Required(str, "'images' is empty"): deploy.url(
{Optional("image_arg"): str}
)
},
**deploy.schema(),
},
......
......@@ -28,6 +28,6 @@ from lava_common.schemas import deploy
def schema():
base = {
Required("to"): "recovery",
Required("images"): {Required(str, "'images' is empty"): {**deploy.url()}},
Required("images"): {Required(str, "'images' is empty"): deploy.url()},
}
return {**deploy.schema(), **base}
......@@ -27,23 +27,29 @@ from lava_common.schemas import deploy
def schema():
resource = deploy.url()
resource_ext = {
**resource,
Optional("install_modules"): bool,
Optional("install_overlay"): bool,
}
base = {
Required("to"): "tftp",
Required("kernel", msg="needs a kernel to deploy"): {
**resource,
Optional("type"): Any("image", "uimage", "zimage"),
},
Required("kernel", msg="needs a kernel to deploy"): deploy.url(
{Optional("type"): Any("image", "uimage", "zimage")}
),
Optional("dtb"): resource,
Optional("modules"): resource,
Optional("preseed"): resource,
Optional("ramdisk"): {**resource_ext, Optional("header"): "u-boot"},
Exclusive("nfsrootfs", "nfs"): {**resource_ext, Optional("prefix"): str},
Optional("ramdisk"): deploy.url(
{
Optional("install_modules"): bool,
Optional("install_overlay"): bool,
Optional("header"): "u-boot",
}
),
Exclusive("nfsrootfs", "nfs"): deploy.url(
{
Optional("install_modules"): bool,
Optional("install_overlay"): bool,
Optional("prefix"): str,
}
),
Exclusive("persistent_nfs", "nfs"): {
Required("address"): str,
Optional("install_overlay"): bool,
......
......@@ -26,15 +26,14 @@ from lava_common.schemas import deploy
def schema():
extra = {
Optional("format"): "qcow2",
Optional("image_arg"): str, # TODO: is this optional?
}
base = {
Required("to"): "tmpfs",
Required("images"): {
Required(str, "'images' is empty"): {
**deploy.url(),
Optional("format"): "qcow2",
Optional("image_arg"): str, # TODO: is this optional?
}
},
Required("images"): {Required(str, "'images' is empty"): deploy.url(extra)},
Optional("type"): "monitor",
Optional("uefi"): deploy.url(), # TODO: check the exact syntax
}
......
......@@ -28,6 +28,6 @@ from lava_common.schemas import deploy
def schema():
base = {
Required("to"): "u-boot-ums",
Required("image"): {**deploy.url(), Optional("root_partition"): Range(min=0)},
Required("image"): deploy.url({Optional("root_partition"): Range(min=0)}),
}
return {**deploy.schema(), **base}
......@@ -31,11 +31,12 @@ def schema():
Required("images"): All(
{
Required("boot"): deploy.url(),
Any(str): {
**deploy.url(),
Optional("apply-overlay"): bool,
Optional("root_partition"): Range(min=0),
},
Any(str): deploy.url(
{
Optional("apply-overlay"): bool,
Optional("root_partition"): Range(min=0),
}
),
},
Length(min=1),
),
......
......@@ -194,7 +194,7 @@ class CallQemuAction(Action):
action="download-action", label=label, key="file"
)
if not image_arg or not action_arg:
self.errors = "Missing image_arg for %s. " % label
self.logger.warning("Missing image arg for %s", label)
continue
self.commands.append(image_arg)
......@@ -236,7 +236,8 @@ class CallQemuAction(Action):
action_arg = self.get_namespace_data(
action="download-action", label=label, key="file"
)
substitutions["{%s}" % label] = action_arg
if image_arg is not None:
substitutions["{%s}" % label] = action_arg
substitutions["{NFS_SERVER_IP}"] = dispatcher_ip(
self.job.parameters["dispatcher"], "nfs"
)
......
......@@ -18,6 +18,7 @@
# along
# with this program; if not, see <http://www.gnu.org/licenses>.
import guestfs
import os
import shutil
import subprocess # nosec - internal use.
......@@ -845,3 +846,125 @@ class ConfigurePreseedFile(Action):
post_command[0],
)
return connection
class AppendOverlays(Action):
name = "append-overlays"
description = "append overlays to an image"
summary = "append overlays to an image"
# TODO: list libguestfs supported formats
IMAGE_FORMATS = ["cpio.newc", "ext4"]
OVERLAY_FORMATS = ["tar"]
def __init__(self, key, params):
super().__init__()
self.key = key
self.params = params
def validate(self):
super().validate()
# Check that we have "overlays" dict
if "overlays" not in self.params:
raise JobError("Missing 'overlays' dictionary")
if not isinstance(self.params["overlays"], dict):
raise JobError("'overlays' is not a dictionary")
for overlay, params in self.params["overlays"].items():
if params.get("format") not in self.OVERLAY_FORMATS:
raise JobError(
"Invalid 'format' (%r) for 'overlays.%s'"
% (params.get("format", ""), overlay)
)
path = params.get("path")
if path is None:
raise JobError("Missing 'path' for 'overlays.%s'" % overlay)
if not path.startswith("/") or ".." in path:
raise JobError("Invalid 'path': %r" % path)
# Check the image format
if self.params.get("format") not in self.IMAGE_FORMATS:
raise JobError("Unsupported image format %r" % self.params.get("format"))
def run(self, connection, max_end_time):
connection = super().run(connection, max_end_time)
if self.params["format"] == "cpio.newc":
self.update_cpio()
elif self.params["format"] == "ext4":
try:
self.update_guestfs()
except RuntimeError as exc:
self.logger.exception(str(exc))
raise JobError("Unable to update image %s: %r" % (self.key, str(exc)))
else:
raise LAVABug("Unknown format %r" % self.params["format"])
return connection
def update_cpio(self):
image = self.get_namespace_data(
action="download-action", label=self.key, key="file"
)
compression = self.get_namespace_data(
action="download-action", label=self.key, key="compression"
)
decompressed = self.get_namespace_data(
action="download-action", label=self.key, key="decompressed"
)
self.logger.info("Modifying %r", image)
tempdir = self.mkdtemp()
# Some images are kept compressed. We should decompress first
if compression and not decompressed:
self.logger.debug("* decompressing (%s)", compression)
image = decompress_file(image, compression)
# extract the archive
self.logger.debug("* extracting %r", image)
uncpio(image, tempdir)
os.unlink(image)
# Add overlays
self.logger.debug("Overlays:")
for overlay in self.params["overlays"]:
label = "%s.%s" % (self.key, overlay)
overlay_image = self.get_namespace_data(
action="download-action", label=label, key="file"
)
path = self.params["overlays"][overlay]["path"]
self.logger.debug("- %s: %r to %r", label, overlay_image, path)
# In the "validate" function, we check that path startswith '/'
# and does not contains '..'
untar_file(overlay_image, "." + path)
# Recreating the archive
self.logger.debug("* archiving %r", image)
cpio(tempdir, image)
if compression and not decompressed:
self.logger.debug("* compressing (%s)", compression)
image = compress_file(image, compression)
def update_guestfs(self):
image = self.get_namespace_data(
action="download-action", label=self.key, key="file"
)
self.logger.info("Modifying %r", image)
guest = guestfs.GuestFS(python_return_dict=True)
guest.add_drive(image)
try:
guest.launch()
device = guest.list_devices()[0]
guest.mount(device, "/")
except RuntimeError as exc:
self.logger.exception(str(exc))
raise JobError("Unable to update image %s: %r" % (self.key, str(exc)))
self.logger.debug("Overlays:")
for overlay in self.params["overlays"]:
label = "%s.%s" % (self.key, overlay)
overlay_image = self.get_namespace_data(
action="download-action", label=label, key="file"
)
path = self.params["overlays"][overlay]["path"]
self.logger.debug("- %s: %r to %r", label, overlay_image, path)
guest.mkdir_p(path)
guest.tar_in(overlay_image, path)
guest.umount(device)
guest.shutdown()
......@@ -35,6 +35,7 @@ import subprocess # nosec - verified.
from lava_dispatcher.power import ResetDevice
from lava_dispatcher.protocols.lxc import LxcProtocol
from lava_dispatcher.actions.deploy import DeployAction
from lava_dispatcher.actions.deploy.apply_overlay import AppendOverlays
from lava_dispatcher.actions.deploy.overlay import OverlayAction
from lava_dispatcher.connections.serial import ConnectDevice
from lava_common.exceptions import InfrastructureError, JobError, LAVABug
......@@ -103,6 +104,15 @@ class DownloaderAction(RetryAction):
else:
raise JobError("Unsupported url protocol scheme: %s" % url.scheme)
self.pipeline.add_action(action)
overlays = self.params.get("overlays", [])
for overlay in overlays:
self.pipeline.add_action(
DownloaderAction(
"%s.%s" % (self.key, overlay), self.path, params=overlays[overlay]
)
)
if overlays:
self.pipeline.add_action(AppendOverlays(self.key, params=self.params))
class DownloadHandler(Action):
......@@ -149,18 +159,24 @@ class DownloadHandler(Action):
)
super().cleanup(connection)
def _url_to_fname(self, path, compression):
def _compression(self):
if self.key == "ramdisk":
return False
return self.params.get("compression", False)
def _url_to_fname(self):
compression = self._compression()
filename = os.path.basename(self.url.path)
# Don't rename files we don't decompress during download
if not compression or (compression not in self.decompress_command_map):
return os.path.join(path, filename)
return os.path.join(self.path, filename)
parts = filename.split(".")
# Files without suffixes, e.g. kernel images
if len(parts) == 1:
return os.path.join(path, filename)
return os.path.join(path, ".".join(parts[:-1]))
return os.path.join(self.path, filename)
return os.path.join(self.path, ".".join(parts[:-1]))
def validate(self):
super().validate()
......@@ -174,7 +190,7 @@ class DownloadHandler(Action):
overlay = self.params.get("overlay", False)
image_arg = self.params.get("image_arg")
self.fname = self._url_to_fname(self.path, compression)
self.fname = self._url_to_fname()
if self.fname.endswith("/"):
raise JobError("Cannot download a directory for %s" % self.key)
# Save into the namespaced data
......@@ -268,11 +284,17 @@ class DownloadHandler(Action):
"Unable to create %s: %s" % (self.path, str(exc))
)
compression = self.params.get("compression", False)
compression = self._compression()
if self.key == "ramdisk":
compression = False
self.logger.debug("Not decompressing ramdisk as can be used compressed.")
self.set_namespace_data(
action="download-action",
label=self.key,
key="decompressed",
value=bool(compression),
)
md5sum = self.params.get("md5sum")
sha256sum = self.params.get("sha256sum")
sha512sum = self.params.get("sha512sum")
......@@ -311,7 +333,7 @@ class DownloadHandler(Action):
"Compression %s specified but not decompressing during download",
compression,
)
else:
elif not self.params.get("compression", False):
self.logger.debug("No compression specified")
def update_progress():
......
import logging
import pytest
from lava_common.exceptions import JobError, LAVABug
from lava_dispatcher.actions.deploy.apply_overlay import AppendOverlays
from lava_dispatcher.job import Job
def test_append_overlays_validate():
# 1/ Working setup
params = {
"format": "cpio.newc",
"overlays": {
"modules": {
"url": "http://example.com/modules.tar.xz",
"compression": "xz",
"format": "tar",
"path": "/",
}
},
}
action = AppendOverlays("rootfs", params)
action.validate()
# 2/ Check errors
with pytest.raises(JobError) as exc:
del params["format"]
action.validate()
assert exc.match("Unsupported image format None")
with pytest.raises(JobError) as exc:
params["overlays"]["modules"]["path"] = "../../"
action.validate()
assert exc.match("Invalid 'path': '../../'")
with pytest.raises(JobError) as exc:
del params["overlays"]["modules"]["path"]
action.validate()
assert exc.match("Missing 'path' for 'overlays.modules'")
with pytest.raises(JobError) as exc:
params["overlays"]["modules"]["format"] = "git"
action.validate()
assert exc.match("Invalid 'format' \\('git'\\) for 'overlays.modules'")
with pytest.raises(JobError) as exc:
params["overlays"] = ""
action.validate()
assert exc.match("'overlays' is not a dictionary")
with pytest.raises(JobError) as exc:
del params["overlays"]
action.validate()
assert exc.match("Missing 'overlays' dictionary")
def test_append_overlays_run(mocker):
params = {
"format": "cpio.newc",
"overlays": {
"modules": {
"url": "http://example.com/modules.tar.xz",
"compression": "xz",
"format": "tar",
"path": "/",
}
},
}
action = AppendOverlays("rootfs", params)
action.update_cpio = mocker.stub()
action.update_guestfs = mocker.stub()
assert action.run(None, 0) is None
action.update_cpio.assert_called_once_with()
params["format"] = "ext4"
assert action.run(None, 0) is None
action.update_guestfs.assert_called_once_with()
params["format"] = "tar"
with pytest.raises(LAVABug):
action.run(None, 0)
def test_append_overlays_update_cpio(caplog, mocker, tmpdir):
caplog.set_level(logging.DEBUG)