overlay.py 29.8 KB
Newer Older
Neil Williams's avatar
Neil Williams committed
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
# Copyright (C) 2014 Linaro Limited
#
# Author: Neil Williams <neil.williams@linaro.org>
#
# This file is part of LAVA Dispatcher.
#
# LAVA Dispatcher is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
# (at your option) any later version.
#
# LAVA Dispatcher is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along
# with this program; if not, see <http://www.gnu.org/licenses>.

import os
import stat
import glob
24
import shutil
Neil Williams's avatar
Neil Williams committed
25
import tarfile
Rémi Duraffort's avatar
Rémi Duraffort committed
26
from lava_dispatcher.actions.deploy import DeployAction
27
28
from lava_dispatcher.action import Action, Pipeline
from lava_common.exceptions import InfrastructureError, LAVABug
Rémi Duraffort's avatar
Rémi Duraffort committed
29
from lava_dispatcher.actions.deploy.testdef import TestDefinitionAction
30
from lava_dispatcher.logical import Deployment
Rémi Duraffort's avatar
Rémi Duraffort committed
31
32
from lava_dispatcher.utils.contextmanager import chdir
from lava_dispatcher.utils.filesystem import check_ssh_identity_file
33
from lava_dispatcher.utils.shell import which
Rémi Duraffort's avatar
Rémi Duraffort committed
34
35
36
from lava_dispatcher.utils.network import rpcinfo_nfs
from lava_dispatcher.protocols.multinode import MultinodeProtocol
from lava_dispatcher.protocols.vland import VlandProtocol
Neil Williams's avatar
Neil Williams committed
37
38


39
40
41
42
43
class Overlay(Deployment):
    compatibility = 4
    name = "overlay"

    def __init__(self, parent, parameters):
Rémi Duraffort's avatar
Rémi Duraffort committed
44
        super().__init__(parent)
45
46
47
48
49
50
51
        self.action = OverlayAction()
        self.action.section = self.action_type
        self.action.job = self.job
        parent.add_action(self.action, parameters)

    @classmethod
    def accepts(cls, device, parameters):
Neil Williams's avatar
Neil Williams committed
52
        if "overlay" not in device["actions"]["deploy"]["methods"]:
53
            return False, "'overlay' not in the device configuration deploy methods"
Neil Williams's avatar
Neil Williams committed
54
        if parameters["to"] != "overlay":
55
            return False, '"to" parameter is not "overlay"'
Neil Williams's avatar
Neil Williams committed
56
        return True, "accepted"
57
58


59
class CreateOverlay(DeployAction):
Neil Williams's avatar
Neil Williams committed
60
    """
Neil Williams's avatar
Neil Williams committed
61
62
63
64
65
66
67
68
69
70
71
72
73
74
    Creates a temporary location into which the lava test shell scripts are installed.
    The location remains available for the testdef actions to populate
    Multinode and LMP actions also populate the one location.
    CreateOverlay then creates a tarball of that location in the output directory
    of the job and removes the temporary location.
    ApplyOverlay extracts that tarball onto the image.

    Deployments which are for a job containing a 'test' action will have
    a TestDefinitionAction added to the job pipeline by this Action.

    The resulting overlay needs to be applied separately and custom classes
    exist for particular deployments, so that the overlay can be applied
    whilst the image is still mounted etc.

75
76
77
    This class handles parts of the overlay which are independent
    of the content of the test definitions themselves. Other
    overlays are handled by TestDefinitionAction.
Neil Williams's avatar
Neil Williams committed
78
79
    """

80
    name = "lava-create-overlay"
81
82
83
    description = "add lava scripts during deployment for test shell use"
    summary = "overlay the lava support scripts"

Neil Williams's avatar
Neil Williams committed
84
    def __init__(self):
Rémi Duraffort's avatar
Rémi Duraffort committed
85
        super().__init__()
Neil Williams's avatar
Neil Williams committed
86
        self.lava_test_dir = os.path.realpath(
Neil Williams's avatar
Neil Williams committed
87
88
            "%s/../../lava_test_shell" % os.path.dirname(__file__)
        )
Neil Williams's avatar
Neil Williams committed
89
        self.scripts_to_copy = []
Neil Williams's avatar
Neil Williams committed
90
        # 755 file permissions
Neil Williams's avatar
Neil Williams committed
91
92
93
94
95
96
97
        self.xmod = (
            stat.S_IRWXU | stat.S_IXGRP | stat.S_IRGRP | stat.S_IXOTH | stat.S_IROTH
        )
        self.target_mac = ""
        self.target_ip = ""
        self.probe_ip = ""
        self.probe_channel = ""
Neil Williams's avatar
Neil Williams committed
98

Neil Williams's avatar
Neil Williams committed
99
    def validate(self):
Rémi Duraffort's avatar
Rémi Duraffort committed
100
        super().validate()
Neil Williams's avatar
Neil Williams committed
101
102
103
        self.scripts_to_copy = sorted(
            glob.glob(os.path.join(self.lava_test_dir, "lava-*"))
        )
104

Neil Williams's avatar
Neil Williams committed
105
        lava_test_results_dir = self.get_constant("lava_test_results_dir", "posix")
Dean Arnold's avatar
Dean Arnold committed
106
        lava_test_results_dir = lava_test_results_dir % self.job.job_id
Neil Williams's avatar
Neil Williams committed
107
108
109
110
111
112
113
114
115
116
117
118
119
        self.set_namespace_data(
            action="test",
            label="results",
            key="lava_test_results_dir",
            value=lava_test_results_dir,
        )
        lava_test_sh_cmd = self.get_constant("lava_test_sh_cmd", "posix")
        self.set_namespace_data(
            action="test",
            label="shared",
            key="lava_test_sh_cmd",
            value=lava_test_sh_cmd,
        )
120

121
        # Add distro support scripts - only if deployment_data is set
Neil Williams's avatar
Neil Williams committed
122
        distro = self.parameters["deployment_data"].get("distro")
123
        if distro:
Neil Williams's avatar
Neil Williams committed
124
125
126
127
            distro_support_dir = "%s/distro/%s" % (self.lava_test_dir, distro)
            self.scripts_to_copy += sorted(
                glob.glob(os.path.join(distro_support_dir, "lava-*"))
            )
128

Neil Williams's avatar
Neil Williams committed
129
        if not self.scripts_to_copy:
130
            self.logger.debug("Skipping lava_test_shell support scripts.")
Neil Williams's avatar
Neil Williams committed
131
132
133
134
135
136
137
138
139
140
141
142
143
        if "parameters" in self.job.device:
            if "interfaces" in self.job.device["parameters"]:
                if "target" in self.job.device["parameters"]["interfaces"]:
                    self.target_mac = self.job.device["parameters"]["interfaces"][
                        "target"
                    ].get("mac", "")
                    self.target_ip = self.job.device["parameters"]["interfaces"][
                        "target"
                    ].get("ip", "")
        for device in self.job.device.get("static_info", []):
            if "probe_channel" in device and "probe_ip" in device:
                self.probe_channel = device["probe_channel"]
                self.probe_ip = device["probe_ip"]
144
                break
Neil Williams's avatar
Neil Williams committed
145
146

    def populate(self, parameters):
147
        self.pipeline = Pipeline(parent=self, job=self.job, parameters=parameters)
148
149
150
151
152
153
154
155
156
157
        if any(
            "ssh" in data for data in self.job.device["actions"]["deploy"]["methods"]
        ):
            # only devices supporting ssh deployments add this action.
            self.pipeline.add_action(SshAuthorize())
        self.pipeline.add_action(VlandOverlayAction())
        self.pipeline.add_action(MultinodeOverlayAction())
        self.pipeline.add_action(TestDefinitionAction())
        self.pipeline.add_action(CompressOverlay())
        self.pipeline.add_action(PersistentNFSOverlay())  # idempotent
Neil Williams's avatar
Neil Williams committed
158

159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
    def _export_data(self, fout, data, prefix):
        if isinstance(data, dict):
            if prefix:
                prefix += "_"
            for key, value in data.items():
                self._export_data(fout, value, "%s%s" % (prefix, key))
        elif isinstance(data, (list, tuple)):
            if prefix:
                prefix += "_"
            for index, value in enumerate(data):
                self._export_data(fout, value, "%s%s" % (prefix, index))
        else:
            if isinstance(data, bool):
                data = "1" if data else "0"
            elif isinstance(data, int):
                data = data
            else:
                data = "'%s'" % data
            self.logger.debug("- %s=%s", prefix, data)
            fout.write("export %s=%s\n" % (prefix, data))

180
    def run(self, connection, max_end_time):
181
        tmp_dir = self.mkdtemp()
Neil Williams's avatar
Neil Williams committed
182
183
184
185
186
187
        self.set_namespace_data(
            action="test", label="shared", key="location", value=tmp_dir
        )
        lava_test_results_dir = self.get_namespace_data(
            action="test", label="results", key="lava_test_results_dir"
        )
188
189
        if not lava_test_results_dir:
            raise LAVABug("Unable to identify top level lava test directory")
Neil Williams's avatar
Neil Williams committed
190
191
192
        shell = self.get_namespace_data(
            action="test", label="shared", key="lava_test_sh_cmd"
        )
193
        namespace = self.parameters.get("namespace")
194
        self.logger.debug("[%s] Preparing overlay tarball in %s", namespace, tmp_dir)
195
        lava_path = os.path.abspath("%s/%s" % (tmp_dir, lava_test_results_dir))
Neil Williams's avatar
Neil Williams committed
196
        for runner_dir in ["bin", "tests", "results"]:
Neil Williams's avatar
Neil Williams committed
197
            # avoid os.path.join as lava_test_results_dir startswith / so location is *dropped* by join.
Neil Williams's avatar
Neil Williams committed
198
199
            path = os.path.abspath("%s/%s" % (lava_path, runner_dir))
            if not os.path.exists(path):
200
                os.makedirs(path, 0o755)
Neil Williams's avatar
Neil Williams committed
201
                self.logger.debug("makedir: %s", path)
Neil Williams's avatar
Neil Williams committed
202
        for fname in self.scripts_to_copy:
Neil Williams's avatar
Neil Williams committed
203
            with open(fname, "r") as fin:
204
                foutname = os.path.basename(fname)
Neil Williams's avatar
Neil Williams committed
205
                output_file = "%s/bin/%s" % (lava_path, foutname)
206
207
208
209
210
                if "distro" in fname:
                    distribution = os.path.basename(os.path.dirname(fname))
                    self.logger.debug("Updating %s (%s)", output_file, distribution)
                else:
                    self.logger.debug("Creating %s", output_file)
Neil Williams's avatar
Neil Williams committed
211
                with open(output_file, "w") as fout:
212
                    fout.write("#!%s\n\n" % shell)
Neil Williams's avatar
Neil Williams committed
213
                    if foutname == "lava-target-mac":
214
                        fout.write("TARGET_DEVICE_MAC='%s'\n" % self.target_mac)
Neil Williams's avatar
Neil Williams committed
215
                    if foutname == "lava-target-ip":
216
                        fout.write("TARGET_DEVICE_IP='%s'\n" % self.target_ip)
Neil Williams's avatar
Neil Williams committed
217
                    if foutname == "lava-probe-ip":
218
                        fout.write("PROBE_DEVICE_IP='%s'\n" % self.probe_ip)
Neil Williams's avatar
Neil Williams committed
219
                    if foutname == "lava-probe-channel":
220
                        fout.write("PROBE_DEVICE_CHANNEL='%s'\n" % self.probe_channel)
Neil Williams's avatar
Neil Williams committed
221
                    if foutname == "lava-target-storage":
222
                        fout.write('LAVA_STORAGE="\n')
Neil Williams's avatar
Neil Williams committed
223
                        for method in self.job.device.get("storage_info", [{}]):
224
                            for key, value in method.items():
Neil Williams's avatar
Neil Williams committed
225
226
227
                                self.logger.debug(
                                    "storage methods:\t%s\t%s", key, value
                                )
228
229
                                fout.write(r"\t%s\t%s\n" % (key, value))
                        fout.write('"\n')
Neil Williams's avatar
Neil Williams committed
230
231
                    fout.write(fin.read())
                    os.fchmod(fout.fileno(), self.xmod)
232

233
234
235
        # Generate environment file
        self.logger.debug("Creating %s/environment", lava_path)
        with open(os.path.join(lava_path, "environment"), "w") as fout:
236
237
238
239
240
241
242
243
244
245
246
            sources = [
                ("environment", ""),
                ("device_info", "LAVA_DEVICE_INFO"),
                ("static_info", "LAVA_STATIC_INFO"),
                ("storage_info", "LAVA_STORAGE_INFO"),
            ]
            for source, prefix in sources:
                data = self.job.device.get(source, {})
                if data:
                    self.logger.debug("%s:", source)
                    self._export_data(fout, data, prefix)
247
248
249
250
251
252
253
254
255
256
257
258
            data = None
            if (
                "protocols" in self.job.parameters
                and "lava-multinode" in self.job.parameters["protocols"]
                and "environment" in self.job.parameters["protocols"]["lava-multinode"]
            ):
                data = self.job.parameters["protocols"]["lava-multinode"]["environment"]
            elif "environment" in self.job.parameters:
                data = self.job.parameters["environment"]
            if data:
                self.logger.debug("job environment:")
                self._export_data(fout, data, "")
259

260
        # Generate the file containing the secrets
Neil Williams's avatar
Neil Williams committed
261
        if "secrets" in self.job.parameters:
262
            self.logger.debug("Creating %s/secrets", lava_path)
Neil Williams's avatar
Neil Williams committed
263
264
            with open(os.path.join(lava_path, "secrets"), "w") as fout:
                for key, value in self.job.parameters["secrets"].items():
265
266
                    fout.write("%s=%s\n" % (key, value))

267
        connection = super().run(connection, max_end_time)
Neil Williams's avatar
Neil Williams committed
268
269
270
        return connection


271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
class OverlayAction(CreateOverlay):
    """
    Creates an overlay, but only if it has been requested by any of the test
    actions (in contrast with CreateOverlay, that creates the overlay
    unconditionally).
    """

    name = "lava-overlay"
    description = "add lava scripts during deployment for test shell use"
    summary = "overlay the lava support scripts"

    def validate(self):
        if not self.test_needs_overlay(self.parameters):
            return

        super().validate()

    def populate(self, parameters):
        if not self.test_needs_overlay(parameters):
            self.pipeline = Pipeline(parent=self, job=self.job, parameters=parameters)
            return

        super().populate(parameters)

    def run(self, connection, max_end_time):
        """
        Check if a lava-test-shell has been requested, implement the overlay
        * create test runner directories beneath the temporary location
        * copy runners into test runner directories
        """
        if not self.test_needs_overlay(self.parameters):
            namespace = self.parameters.get("namespace")
            self.logger.info("[%s] skipped %s - no test action.", namespace, self.name)
            return connection

        return super().run(connection, max_end_time)


Neil Williams's avatar
Neil Williams committed
309
class MultinodeOverlayAction(OverlayAction):
Neil Williams's avatar
Neil Williams committed
310

311
312
313
314
    name = "lava-multinode-overlay"
    description = "add lava scripts during deployment for multinode test shell use"
    summary = "overlay the lava multinode scripts"

Neil Williams's avatar
Neil Williams committed
315
    def __init__(self):
Rémi Duraffort's avatar
Rémi Duraffort committed
316
        super().__init__()
Neil Williams's avatar
Neil Williams committed
317
318
        # Multinode-only
        self.lava_multi_node_test_dir = os.path.realpath(
Neil Williams's avatar
Neil Williams committed
319
320
321
            "%s/../../lava_test_shell/multi_node" % os.path.dirname(__file__)
        )
        self.lava_multi_node_cache_file = (
322
323
            "/tmp/lava_multi_node_cache.txt"  # nosec - on the DUT
        )
Neil Williams's avatar
Neil Williams committed
324
325
        self.role = None
        self.protocol = MultinodeProtocol.name
Neil Williams's avatar
Neil Williams committed
326

Neil Williams's avatar
Neil Williams committed
327
    def populate(self, parameters):
Neil Williams's avatar
Neil Williams committed
328
329
        # override the populate function of overlay action which provides the
        # lava test directory settings etc.
Neil Williams's avatar
Neil Williams committed
330
331
332
        pass

    def validate(self):
Rémi Duraffort's avatar
Rémi Duraffort committed
333
        super().validate()
Neil Williams's avatar
Neil Williams committed
334
        # idempotency
Neil Williams's avatar
Neil Williams committed
335
        if "actions" not in self.job.parameters:
Neil Williams's avatar
Neil Williams committed
336
            return
Neil Williams's avatar
Neil Williams committed
337
338
339
340
        if "protocols" in self.job.parameters and self.protocol in [
            protocol.name for protocol in self.job.protocols
        ]:
            if "target_group" not in self.job.parameters["protocols"][self.protocol]:
Neil Williams's avatar
Neil Williams committed
341
                return
Neil Williams's avatar
Neil Williams committed
342
            if "role" not in self.job.parameters["protocols"][self.protocol]:
Neil Williams's avatar
Neil Williams committed
343
344
                self.errors = "multinode job without a specified role"
            else:
Neil Williams's avatar
Neil Williams committed
345
                self.role = self.job.parameters["protocols"][self.protocol]["role"]
Neil Williams's avatar
Neil Williams committed
346

347
    def run(self, connection, max_end_time):
Neil Williams's avatar
Neil Williams committed
348
        if self.role is None:
Neil Williams's avatar
Neil Williams committed
349
            self.logger.debug("skipped %s", self.name)
Neil Williams's avatar
Neil Williams committed
350
            return connection
Neil Williams's avatar
Neil Williams committed
351
352
353
354
355
356
357
358
359
        lava_test_results_dir = self.get_namespace_data(
            action="test", label="results", key="lava_test_results_dir"
        )
        shell = self.get_namespace_data(
            action="test", label="shared", key="lava_test_sh_cmd"
        )
        location = self.get_namespace_data(
            action="test", label="shared", key="location"
        )
360
        if not location:
Rémi Duraffort's avatar
Rémi Duraffort committed
361
            raise LAVABug("Missing lava overlay location")
362
        if not os.path.exists(location):
Rémi Duraffort's avatar
Rémi Duraffort committed
363
            raise LAVABug("Unable to find overlay location")
Neil Williams's avatar
Neil Williams committed
364

365
366
        # the roles list can only be populated after the devices have been assigned
        # therefore, cannot be checked in validate which is executed at submission.
Neil Williams's avatar
Neil Williams committed
367
368
369
370
        if "roles" not in self.job.parameters["protocols"][self.protocol]:
            raise LAVABug(
                "multinode definition without complete list of roles after assignment"
            )
371

Neil Williams's avatar
Neil Williams committed
372
        # Generic scripts
373
        lava_path = os.path.abspath("%s/%s" % (location, lava_test_results_dir))
Neil Williams's avatar
Neil Williams committed
374
375
376
        scripts_to_copy = glob.glob(
            os.path.join(self.lava_multi_node_test_dir, "lava-*")
        )
Neil Williams's avatar
Neil Williams committed
377
        self.logger.debug(self.lava_multi_node_test_dir)
378
379
        self.logger.debug("lava_path: %s", lava_path)
        self.logger.debug("scripts to copy %s", scripts_to_copy)
Neil Williams's avatar
Neil Williams committed
380
381

        for fname in scripts_to_copy:
Neil Williams's avatar
Neil Williams committed
382
            with open(fname, "r") as fin:
Neil Williams's avatar
Neil Williams committed
383
                foutname = os.path.basename(fname)
Neil Williams's avatar
Neil Williams committed
384
                output_file = "%s/bin/%s" % (lava_path, foutname)
Neil Williams's avatar
Neil Williams committed
385
                self.logger.debug("Creating %s", output_file)
Neil Williams's avatar
Neil Williams committed
386
                with open(output_file, "w") as fout:
Neil Williams's avatar
Neil Williams committed
387
388
                    fout.write("#!%s\n\n" % shell)
                    # Target-specific scripts (add ENV to the generic ones)
Neil Williams's avatar
Neil Williams committed
389
                    if foutname == "lava-group":
Neil Williams's avatar
Neil Williams committed
390
                        fout.write('LAVA_GROUP="\n')
Neil Williams's avatar
Neil Williams committed
391
392
393
394
395
396
397
398
399
                        for client_name in self.job.parameters["protocols"][
                            self.protocol
                        ]["roles"]:
                            role_line = self.job.parameters["protocols"][self.protocol][
                                "roles"
                            ][client_name]
                            self.logger.debug(
                                "group roles:\t%s\t%s", client_name, role_line
                            )
Neil Williams's avatar
Neil Williams committed
400
                            fout.write(r"\t%s\t%s\n" % (client_name, role_line))
Neil Williams's avatar
Neil Williams committed
401
                        fout.write('"\n')
Neil Williams's avatar
Neil Williams committed
402
403
404
405
406
407
                    elif foutname == "lava-role":
                        fout.write(
                            "TARGET_ROLE='%s'\n"
                            % self.job.parameters["protocols"][self.protocol]["role"]
                        )
                    elif foutname == "lava-self":
408
                        fout.write("LAVA_HOSTNAME='%s'\n" % self.job.job_id)
Neil Williams's avatar
Neil Williams committed
409
                    else:
410
                        fout.write("LAVA_TEST_BIN='%s/bin'\n" % lava_test_results_dir)
Neil Williams's avatar
Neil Williams committed
411
412
413
414
                        fout.write(
                            "LAVA_MULTI_NODE_CACHE='%s'\n"
                            % self.lava_multi_node_cache_file
                        )
Neil Williams's avatar
Neil Williams committed
415
416
                        # always write out full debug logs
                        fout.write("LAVA_MULTI_NODE_DEBUG='yes'\n")
417
418
                    fout.write(fin.read())
                    os.fchmod(fout.fileno(), self.xmod)
Neil Williams's avatar
Neil Williams committed
419
420
421
422
423
424
425
426
        self.call_protocols()
        return connection


class VlandOverlayAction(OverlayAction):
    """
    Adds data for vland interface locations, MAC addresses and vlan names
    """
427
428
429
430
431

    name = "lava-vland-overlay"
    description = "Populate specific vland scripts for tests to lookup vlan data."
    summary = "Add files detailing vlan configuration."

Neil Williams's avatar
Neil Williams committed
432
    def __init__(self):
Rémi Duraffort's avatar
Rémi Duraffort committed
433
        super().__init__()
Neil Williams's avatar
Neil Williams committed
434
435
        # vland-only
        self.lava_vland_test_dir = os.path.realpath(
Neil Williams's avatar
Neil Williams committed
436
437
438
            "%s/../../lava_test_shell/vland" % os.path.dirname(__file__)
        )
        self.lava_vland_cache_file = "/tmp/lava_vland_cache.txt"  # nosec - on the DUT
Neil Williams's avatar
Neil Williams committed
439
440
441
        self.params = {}
        self.sysfs = []
        self.tags = []
442
        self.names = []
Neil Williams's avatar
Neil Williams committed
443
444
445
446
447
448
449
450
        self.protocol = VlandProtocol.name

    def populate(self, parameters):
        # override the populate function of overlay action which provides the
        # lava test directory settings etc.
        pass

    def validate(self):
Rémi Duraffort's avatar
Rémi Duraffort committed
451
        super().validate()
Neil Williams's avatar
Neil Williams committed
452
        # idempotency
Neil Williams's avatar
Neil Williams committed
453
        if "actions" not in self.job.parameters:
Neil Williams's avatar
Neil Williams committed
454
            return
Neil Williams's avatar
Neil Williams committed
455
        if "protocols" not in self.job.parameters:
Neil Williams's avatar
Neil Williams committed
456
457
458
            return
        if self.protocol not in [protocol.name for protocol in self.job.protocols]:
            return
Neil Williams's avatar
Neil Williams committed
459
        if "parameters" not in self.job.device:
Neil Williams's avatar
Neil Williams committed
460
            self.errors = "Device lacks parameters"
Neil Williams's avatar
Neil Williams committed
461
        elif "interfaces" not in self.job.device["parameters"]:
Neil Williams's avatar
Neil Williams committed
462
463
464
465
            self.errors = "Device lacks vland interfaces data."
        if not self.valid:
            return
        # same as the parameters of the protocol itself.
Neil Williams's avatar
Neil Williams committed
466
467
468
469
470
471
472
        self.params = self.job.parameters["protocols"][self.protocol]
        device_params = self.job.device["parameters"]["interfaces"]
        vprotocol = [
            vprotocol
            for vprotocol in self.job.protocols
            if vprotocol.name == self.protocol
        ][0]
473
474
        # needs to be the configured interface for each vlan.
        for key, _ in self.params.items():
475
            if key not in vprotocol.params:
476
                continue
Neil Williams's avatar
Neil Williams committed
477
            self.names.append(",".join([key, vprotocol.params[key]["iface"]]))
Neil Williams's avatar
Neil Williams committed
478
        for interface in device_params:
Neil Williams's avatar
Neil Williams committed
479
480
481
482
483
484
485
486
487
            self.sysfs.append(
                ",".join(
                    [
                        interface,
                        device_params[interface]["mac"],
                        device_params[interface]["sysfs"],
                    ]
                )
            )
Neil Williams's avatar
Neil Williams committed
488
        for interface in device_params:
Neil Williams's avatar
Neil Williams committed
489
            if not device_params[interface]["tags"]:
490
491
                # skip primary interface
                continue
Neil Williams's avatar
Neil Williams committed
492
            for tag in device_params[interface]["tags"]:
493
                self.tags.append(",".join([interface, tag]))
Neil Williams's avatar
Neil Williams committed
494

495
    def run(self, connection, max_end_time):
Neil Williams's avatar
Neil Williams committed
496
497
        """
        Writes out file contents from lists, across multiple lines
Neil Williams's avatar
Neil Williams committed
498
499
        VAR="VAL1\n
        VAL2\n
Neil Williams's avatar
Neil Williams committed
500
        "
Neil Williams's avatar
Neil Williams committed
501
502
        The newline and escape characters are used to avoid unwanted whitespace.
        \n becomes \\n, a single escape gets expanded and itself then needs \n to output:
Neil Williams's avatar
Neil Williams committed
503
504
505
506
507
508
        VAL1
        VAL2
        """
        if not self.params:
            self.logger.debug("skipped %s", self.name)
            return connection
Neil Williams's avatar
Neil Williams committed
509
510
511
512
513
514
515
516
517
        location = self.get_namespace_data(
            action="test", label="shared", key="location"
        )
        lava_test_results_dir = self.get_namespace_data(
            action="test", label="results", key="lava_test_results_dir"
        )
        shell = self.get_namespace_data(
            action="test", label="shared", key="lava_test_sh_cmd"
        )
518
        if not location:
Rémi Duraffort's avatar
Rémi Duraffort committed
519
            raise LAVABug("Missing lava overlay location")
520
        if not os.path.exists(location):
Rémi Duraffort's avatar
Rémi Duraffort committed
521
            raise LAVABug("Unable to find overlay location")
Neil Williams's avatar
Neil Williams committed
522

523
        lava_path = os.path.abspath("%s/%s" % (location, lava_test_results_dir))
Neil Williams's avatar
Neil Williams committed
524
        scripts_to_copy = glob.glob(os.path.join(self.lava_vland_test_dir, "lava-*"))
Neil Williams's avatar
Neil Williams committed
525
526
527
528
        self.logger.debug(self.lava_vland_test_dir)
        self.logger.debug({"lava_path": lava_path, "scripts": scripts_to_copy})

        for fname in scripts_to_copy:
Neil Williams's avatar
Neil Williams committed
529
            with open(fname, "r") as fin:
Neil Williams's avatar
Neil Williams committed
530
                foutname = os.path.basename(fname)
Neil Williams's avatar
Neil Williams committed
531
                output_file = "%s/bin/%s" % (lava_path, foutname)
Neil Williams's avatar
Neil Williams committed
532
                self.logger.debug("Creating %s", output_file)
Neil Williams's avatar
Neil Williams committed
533
                with open(output_file, "w") as fout:
Neil Williams's avatar
Neil Williams committed
534
535
                    fout.write("#!%s\n\n" % shell)
                    # Target-specific scripts (add ENV to the generic ones)
Neil Williams's avatar
Neil Williams committed
536
                    if foutname == "lava-vland-self":
Neil Williams's avatar
Neil Williams committed
537
538
539
                        fout.write(r'LAVA_VLAND_SELF="')
                        for line in self.sysfs:
                            fout.write(r"%s\n" % line)
Neil Williams's avatar
Neil Williams committed
540
                    elif foutname == "lava-vland-names":
541
542
543
                        fout.write(r'LAVA_VLAND_NAMES="')
                        for line in self.names:
                            fout.write(r"%s\n" % line)
Neil Williams's avatar
Neil Williams committed
544
                    elif foutname == "lava-vland-tags":
Neil Williams's avatar
Neil Williams committed
545
                        fout.write(r'LAVA_VLAND_TAGS="')
546
547
548
549
550
                        if not self.tags:
                            fout.write(r"\n")
                        else:
                            for line in self.tags:
                                fout.write(r"%s\n" % line)
Neil Williams's avatar
Neil Williams committed
551
552
553
554
                    fout.write('"\n\n')
                    fout.write(fin.read())
                    os.fchmod(fout.fileno(), self.xmod)
        self.call_protocols()
Neil Williams's avatar
Neil Williams committed
555
        return connection
Neil Williams's avatar
Neil Williams committed
556
557


Neil Williams's avatar
Neil Williams committed
558
559
560
561
class CompressOverlay(Action):
    """
    Makes a tarball of the finished overlay and declares filename of the tarball
    """
Neil Williams's avatar
Neil Williams committed
562

563
564
565
    name = "compress-overlay"
    description = "Create a lava overlay tarball and store alongside the job"
    summary = "Compress the lava overlay files"
Neil Williams's avatar
Neil Williams committed
566

567
    def run(self, connection, max_end_time):
568
        output = os.path.join(self.mkdtemp(), "overlay-%s.tar.gz" % self.level)
Neil Williams's avatar
Neil Williams committed
569
570
571
572
573
574
575
576
577
        location = self.get_namespace_data(
            action="test", label="shared", key="location"
        )
        lava_test_results_dir = self.get_namespace_data(
            action="test", label="results", key="lava_test_results_dir"
        )
        self.set_namespace_data(
            action="test", label="shared", key="output", value=output
        )
578
        if not location:
Rémi Duraffort's avatar
Rémi Duraffort committed
579
            raise LAVABug("Missing lava overlay location")
580
        if not os.path.exists(location):
Rémi Duraffort's avatar
Rémi Duraffort committed
581
            raise LAVABug("Unable to find overlay location")
Neil Williams's avatar
Neil Williams committed
582
        if not self.valid:
583
            self.logger.error(self.errors)
Neil Williams's avatar
Neil Williams committed
584
            return connection
585
        connection = super().run(connection, max_end_time)
586
587
588
589
590
        with chdir(location):
            try:
                with tarfile.open(output, "w:gz") as tar:
                    tar.add(".%s" % lava_test_results_dir)
                    # ssh authorization support
Neil Williams's avatar
Neil Williams committed
591
592
                    if os.path.exists("./root/"):
                        tar.add(".%s" % "/root/")
593
            except tarfile.TarError as exc:
Neil Williams's avatar
Neil Williams committed
594
595
596
                raise InfrastructureError(
                    "Unable to create lava overlay tarball: %s" % exc
                )
597

Neil Williams's avatar
Neil Williams committed
598
599
600
        self.set_namespace_data(
            action=self.name, label="output", key="file", value=output
        )
Neil Williams's avatar
Neil Williams committed
601
        return connection
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616


class SshAuthorize(Action):
    """
    Handle including the authorization (ssh public key) into the
    deployment as a file in the overlay and writing to
    /root/.ssh/authorized_keys.
    if /root/.ssh/authorized_keys exists in the test image it will be overwritten
    when the overlay tarball is unpacked onto the test image.
    The key exists in the lava_test_results_dir to allow test writers to work around this
    after logging in via the identity_file set here.
    Hacking sessions already append to the existing file.
    Used by secondary connections only.
    Primary connections need the keys set up by admins.
    """
617
618

    name = "ssh-authorize"
Neil Williams's avatar
Neil Williams committed
619
620
    description = "include public key in overlay and authorize root user"
    summary = "add public key to authorized_keys"
621

622
    def __init__(self):
Rémi Duraffort's avatar
Rémi Duraffort committed
623
        super().__init__()
624
625
626
627
        self.active = False
        self.identity_file = None

    def validate(self):
Rémi Duraffort's avatar
Rémi Duraffort committed
628
        super().validate()
Neil Williams's avatar
Neil Williams committed
629
630
        if "to" in self.parameters:
            if self.parameters["to"] == "ssh":
631
                return
Neil Williams's avatar
Neil Williams committed
632
633
        if "authorize" in self.parameters:
            if self.parameters["authorize"] != "ssh":
634
                return
Neil Williams's avatar
Neil Williams committed
635
636
637
        if not any(
            "ssh" in data for data in self.job.device["actions"]["deploy"]["methods"]
        ):
638
639
            # idempotency - leave self.identity_file as None
            return
Neil Williams's avatar
Neil Williams committed
640
        params = self.job.device["actions"]["deploy"]["methods"]
641
642
643
644
645
646
        check = check_ssh_identity_file(params)
        if check[0]:
            self.errors = check[0]
        elif check[1]:
            self.identity_file = check[1]
        if self.valid:
Neil Williams's avatar
Neil Williams committed
647
648
649
650
651
652
653
            self.set_namespace_data(
                action=self.name,
                label="authorize",
                key="identity_file",
                value=self.identity_file,
            )
            if "authorize" in self.parameters:
654
655
656
                # only secondary connections set active.
                self.active = True

657
658
    def run(self, connection, max_end_time):
        connection = super().run(connection, max_end_time)
659
660
661
662
        if not self.identity_file:
            self.logger.debug("No authorisation required.")  # idempotency
            return connection
        # add the authorization keys to the overlay
Neil Williams's avatar
Neil Williams committed
663
664
665
666
667
668
        location = self.get_namespace_data(
            action="test", label="shared", key="location"
        )
        lava_test_results_dir = self.get_namespace_data(
            action="test", label="results", key="lava_test_results_dir"
        )
669
        if not location:
Rémi Duraffort's avatar
Rémi Duraffort committed
670
            raise LAVABug("Missing lava overlay location")
671
        if not os.path.exists(location):
Rémi Duraffort's avatar
Rémi Duraffort committed
672
            raise LAVABug("Unable to find overlay location")
673
        lava_path = os.path.abspath("%s/%s" % (location, lava_test_results_dir))
Neil Williams's avatar
Neil Williams committed
674
        output_file = "%s/%s" % (lava_path, os.path.basename(self.identity_file))
675
676
677
678
679
        shutil.copyfile(self.identity_file, output_file)
        shutil.copyfile("%s.pub" % self.identity_file, "%s.pub" % output_file)
        if not self.active:
            # secondary connections only
            return connection
Neil Williams's avatar
Neil Williams committed
680
681
682
683
        self.logger.info(
            "Adding SSH authorisation for %s.pub", os.path.basename(output_file)
        )
        user_sshdir = os.path.join(location, "root", ".ssh")
684
        os.makedirs(user_sshdir, 0o755, exist_ok=True)
685
686
687
        # if /root/.ssh/authorized_keys exists in the test image it will be overwritten
        # the key exists in the lava_test_results_dir to allow test writers to work around this
        # after logging in via the identity_file set here
Neil Williams's avatar
Neil Williams committed
688
        authorize = os.path.join(user_sshdir, "authorized_keys")
Neil Williams's avatar
Neil Williams committed
689
        self.logger.debug("Copying %s to %s", "%s.pub" % self.identity_file, authorize)
690
        shutil.copyfile("%s.pub" % self.identity_file, authorize)
691
        os.chmod(authorize, 0o600)
692
        return connection
693
694
695
696
697
698
699
700


class PersistentNFSOverlay(Action):
    """
    Instead of extracting, just populate the location of the persistent NFS
    so that it can be mounted later when the overlay is applied.
    """

701
702
703
704
    section = "deploy"
    name = "persistent-nfs-overlay"
    description = "unpack overlay into persistent NFS"
    summary = "add test overlay to NFS"
705
706

    def validate(self):
Rémi Duraffort's avatar
Rémi Duraffort committed
707
        super().validate()
Neil Williams's avatar
Neil Williams committed
708
        persist = self.parameters.get("persistent_nfs")
Neil Williams's avatar
Neil Williams committed
709
        if not persist:
710
            return
Neil Williams's avatar
Neil Williams committed
711
        if "address" not in persist:
Neil Williams's avatar
Neil Williams committed
712
713
            self.errors = "Missing address for persistent NFS"
            return
Neil Williams's avatar
Neil Williams committed
714
715
716
717
718
        if ":" not in persist["address"]:
            self.errors = (
                "Unrecognised NFS URL: '%s'"
                % self.parameters["persistent_nfs"]["address"]
            )
Neil Williams's avatar
Neil Williams committed
719
            return
Neil Williams's avatar
Neil Williams committed
720
721
        nfs_server, dirname = persist["address"].split(":")
        which("rpcinfo")
722
        self.errors = rpcinfo_nfs(nfs_server)
Neil Williams's avatar
Neil Williams committed
723
724
725
726
727
728
        self.set_namespace_data(
            action=self.name, label="nfs_address", key="nfsroot", value=dirname
        )
        self.set_namespace_data(
            action=self.name, label="nfs_address", key="serverip", value=nfs_server
        )
729
730
731

        self.job.device["dynamic_data"]["NFS_ROOTFS"] = dirname
        self.job.device["dynamic_data"]["NFS_SERVER_IP"] = nfs_server