download.py 30.9 KB
Newer Older
1
2
3
# Copyright (C) 2014 Linaro Limited
#
# Author: Neil Williams <neil.williams@linaro.org>
4
#         Remi Duraffort <remi.duraffort@linaro.org>
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
#
# This file is part of LAVA Dispatcher.
#
# LAVA Dispatcher is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
# (at your option) any later version.
#
# LAVA Dispatcher is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along
# with this program; if not, see <http://www.gnu.org/licenses>.

22
# This class is used for all downloads, including images and individual files for tftp.
23
# python2 only
24

25
import contextlib
Rémi Duraffort's avatar
Rémi Duraffort committed
26
import errno
27
import math
Neil Williams's avatar
Neil Williams committed
28
import os
29
import pathlib
30
import shutil
31
import time
Neil Williams's avatar
Neil Williams committed
32
import hashlib
33
import requests
34
import subprocess  # nosec - verified.
Rémi Duraffort's avatar
Rémi Duraffort committed
35
36
37
from lava_dispatcher.power import ResetDevice
from lava_dispatcher.protocols.lxc import LxcProtocol
from lava_dispatcher.actions.deploy import DeployAction
38
from lava_dispatcher.actions.deploy.overlay import OverlayAction
Rémi Duraffort's avatar
Rémi Duraffort committed
39
from lava_dispatcher.connections.serial import ConnectDevice
Neil Williams's avatar
Neil Williams committed
40
41
42
from lava_common.exceptions import InfrastructureError, JobError, LAVABug
from lava_dispatcher.action import Action, Pipeline
from lava_dispatcher.logical import Deployment, RetryAction
43
from lava_dispatcher.utils.compression import untar_file
Rémi Duraffort's avatar
Rémi Duraffort committed
44
from lava_dispatcher.utils.filesystem import (
45
46
    copy_to_lxc,
    lava_lxc_home,
47
    copy_overlay_to_lxc,
48
)
49
from lava_common.constants import (
50
51
52
53
    FILE_DOWNLOAD_CHUNK_SIZE,
    HTTP_DOWNLOAD_CHUNK_SIZE,
    SCP_DOWNLOAD_CHUNK_SIZE,
)
Rémi Duraffort's avatar
Rémi Duraffort committed
54
55
from lava_dispatcher.actions.boot.fastboot import EnterFastbootAction
from lava_dispatcher.actions.boot.u_boot import UBootEnterFastbootAction
Neil Williams's avatar
Neil Williams committed
56

57
from urllib.parse import quote_plus, urlparse
Neil Williams's avatar
Neil Williams committed
58

59

Neil Williams's avatar
Neil Williams committed
60
class DownloaderAction(RetryAction):
61
62
    """
    The retry pipeline for downloads.
Neil Williams's avatar
Neil Williams committed
63
    To allow any deploy action to work with multinode, each call *must* set a unique path.
64
    """
65
66
67
68
69

    name = "download-retry"
    description = "download with retry"
    summary = "download-retry"

70
    def __init__(self, key, path, params, uniquify=True):
Rémi Duraffort's avatar
Rémi Duraffort committed
71
        super().__init__()
72
        self.max_retries = 3
73
74
        self.key = key  # the key in the parameters of what to download
        self.path = path  # where to download
75
        self.uniquify = uniquify
76
        self.params = params
77

78
    def populate(self, parameters):
79
        self.pipeline = Pipeline(parent=self, job=self.job, parameters=parameters)
80
81

        # Find the right action according to the url
82
        url = self.params.get("url")
83
        if url is None:
Neil Williams's avatar
Neil Williams committed
84
85
86
            raise JobError(
                "Invalid deploy action: 'url' is missing for '%s'" % self.key
            )
87

88
        url = urlparse(url)
Neil Williams's avatar
Neil Williams committed
89
        if url.scheme == "scp":
90
91
92
            action = ScpDownloadAction(
                self.key, self.path, url, self.uniquify, params=self.params
            )
Neil Williams's avatar
Neil Williams committed
93
        elif url.scheme == "http" or url.scheme == "https":
94
95
96
            action = HttpDownloadAction(
                self.key, self.path, url, self.uniquify, params=self.params
            )
Neil Williams's avatar
Neil Williams committed
97
        elif url.scheme == "file":
98
99
100
            action = FileDownloadAction(
                self.key, self.path, url, self.uniquify, params=self.params
            )
Neil Williams's avatar
Neil Williams committed
101
        elif url.scheme == "lxc":
Neil Williams's avatar
Neil Williams committed
102
            action = LxcDownloadAction(self.key, self.path, url)
103
104
        else:
            raise JobError("Unsupported url protocol scheme: %s" % url.scheme)
105
        self.pipeline.add_action(action)
106
107


108
class DownloadHandler(Action):
Neil Williams's avatar
Neil Williams committed
109
110
111
112
113
114
115
116
117
118
    """
    The identification of which downloader and whether to
    decompress needs to be done in the validation stage,
    with the ScpAction or HttpAction or FileAction being
    selected as part of the deployment. All the information
    needed to make this selection is available before the
    job starts, so populate the pipeline as specifically
    as possible.
    """

119
120
121
    name = "download-action"
    description = "download action"
    summary = "download-action"
122
    timeout_exception = InfrastructureError
123

124
125
126
    # Supported decompression commands
    decompress_command_map = {"xz": "unxz", "gz": "gunzip", "bz2": "bunzip2"}

127
    def __init__(self, key, path, url, uniquify=True, params=None):
Rémi Duraffort's avatar
Rémi Duraffort committed
128
        super().__init__()
129
        self.url = url
130
        self.key = key
131
        self.size = -1
132
133
134
135
136

        self.path = path
        # Store the files in a sub-directory to keep the path unique.
        if uniquify:
            self.path = os.path.join(path, key)
137
        self.fname = None
138
        self.params = params
Neil Williams's avatar
Neil Williams committed
139

140
    def reader(self):
Rémi Duraffort's avatar
Rémi Duraffort committed
141
        raise LAVABug("'reader' function unimplemented")
Rémi Duraffort's avatar
Rémi Duraffort committed
142

143
    def cleanup(self, connection):
144
145
146
        if os.path.exists(self.path):
            self.logger.debug("Cleaning up download directory: %s", self.path)
            shutil.rmtree(self.path)
Neil Williams's avatar
Neil Williams committed
147
148
149
        self.set_namespace_data(
            action="download-action", label=self.key, key="file", value=""
        )
Rémi Duraffort's avatar
Rémi Duraffort committed
150
        super().cleanup(connection)
151

152
153
154
155
156
157
158
    def _compression(self):
        if self.key == "ramdisk":
            return False
        return self.params.get("compression", False)

    def _url_to_fname(self):
        compression = self._compression()
Neil Williams's avatar
Neil Williams committed
159
        filename = os.path.basename(self.url.path)
160
161
162

        # Don't rename files we don't decompress during download
        if not compression or (compression not in self.decompress_command_map):
163
            return os.path.join(self.path, filename)
164

Neil Williams's avatar
Neil Williams committed
165
        parts = filename.split(".")
166
167
        # Files without suffixes, e.g. kernel images
        if len(parts) == 1:
168
169
            return os.path.join(self.path, filename)
        return os.path.join(self.path, ".".join(parts[:-1]))
Neil Williams's avatar
Neil Williams committed
170

171
    def validate(self):
Rémi Duraffort's avatar
Rémi Duraffort committed
172
        super().validate()
173
174
175
176
        # Check that self.key is not a path
        if len(pathlib.Path(self.key).parts) != 1:
            raise JobError("Invalid key %r" % self.key)

177
178
179
180
181
        self.url = urlparse(self.params["url"])
        compression = self.params.get("compression")
        archive = self.params.get("archive")
        overlay = self.params.get("overlay", False)
        image_arg = self.params.get("image_arg")
182

183
        self.fname = self._url_to_fname()
184
185
186
187
188
189
190
191
192
193
194
195
196
        if self.fname.endswith("/"):
            raise JobError("Cannot download a directory for %s" % self.key)
        # Save into the namespaced data
        self.set_namespace_data(
            action="download-action", label=self.key, key="file", value=self.fname
        )
        self.set_namespace_data(
            action="download-action",
            label=self.key,
            key="compression",
            value=compression,
        )
        if image_arg is not None:
Neil Williams's avatar
Neil Williams committed
197
198
199
            self.set_namespace_data(
                action="download-action",
                label=self.key,
200
201
                key="image_arg",
                value=image_arg,
Neil Williams's avatar
Neil Williams committed
202
            )
Tyler Baker's avatar
Tyler Baker committed
203
        if overlay:
Neil Williams's avatar
Neil Williams committed
204
205
206
            self.set_namespace_data(
                action="download-action", label=self.key, key="overlay", value=overlay
            )
207

208
209
210
211
        if compression and compression not in ["gz", "bz2", "xz", "zip"]:
            self.errors = "Unknown 'compression' format '%s'" % compression
        if archive and archive not in ["tar"]:
            self.errors = "Unknown 'archive' format '%s'" % archive
212
        # pass kernel type to boot Action
Neil Williams's avatar
Neil Williams committed
213
        if self.key == "kernel" and ("kernel" in self.parameters):
214
            self.set_namespace_data(
Neil Williams's avatar
Neil Williams committed
215
216
217
218
219
                action="download-action",
                label="type",
                key=self.key,
                value=self.parameters[self.key].get("type"),
            )
220

221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
    def _check_checksum(self, algorithm, actual, expected):
        if expected is None:
            return
        if actual == expected:
            self.results = {"success": {algorithm: actual}}
            return

        self.logger.error(
            "%s sum for '%s' does not match", algorithm, self.url.geturl()
        )
        self.logger.info("actual  : %s", actual)
        self.logger.info("expected: %s", expected)

        self.results = {"fail": {algorithm: expected, "download": actual}}
        raise JobError("%s for '%s' does not match." % (algorithm, self.url.geturl()))

237
    def run(self, connection, max_end_time):
238
        def progress_unknown_total(downloaded_sz, last_val):
239
            """ Compute progress when the size is unknown """
240
            condition = downloaded_sz >= last_val + 25 * 1024 * 1024
Neil Williams's avatar
Neil Williams committed
241
242
243
244
245
246
247
            return (
                condition,
                downloaded_sz,
                "progress %dMB" % (int(downloaded_sz / (1024 * 1024)))
                if condition
                else "",
            )
248

249
        def progress_known_total(downloaded_sz, last_val):
250
            """ Compute progress when the size is known """
251
252
            percent = math.floor(downloaded_sz / float(self.size) * 100)
            condition = percent >= last_val + 5
Neil Williams's avatar
Neil Williams committed
253
254
255
256
257
258
259
            return (
                condition,
                percent,
                "progress %3d%% (%dMB)" % (percent, int(downloaded_sz / (1024 * 1024)))
                if condition
                else "",
            )
260

261
        connection = super().run(connection, max_end_time)
Neil Williams's avatar
Neil Williams committed
262
        # self.cookies = self.job.context.config.lava_cookies  # FIXME: work out how to restore
263
        md5 = hashlib.md5()  # nosec - not being used for cryptography.
264
        sha256 = hashlib.sha256()
Corentin LABBE's avatar
Corentin LABBE committed
265
        sha512 = hashlib.sha512()
Rémi Duraffort's avatar
Rémi Duraffort committed
266
267
268
269

        # Create a fresh directory if the old one has been removed by a previous cleanup
        # (when retrying inside a RetryAction)
        try:
270
            os.makedirs(self.path, 0o755)
Rémi Duraffort's avatar
Rémi Duraffort committed
271
272
        except OSError as exc:
            if exc.errno != errno.EEXIST:
Neil Williams's avatar
Neil Williams committed
273
274
275
                raise InfrastructureError(
                    "Unable to create %s: %s" % (self.path, str(exc))
                )
Rémi Duraffort's avatar
Rémi Duraffort committed
276

277
        compression = self._compression()
278
279
        if self.key == "ramdisk":
            self.logger.debug("Not decompressing ramdisk as can be used compressed.")
280

281
282
283
        md5sum = self.params.get("md5sum")
        sha256sum = self.params.get("sha256sum")
        sha512sum = self.params.get("sha512sum")
284

285
286
287
288
        if os.path.isdir(self.fname):
            raise JobError("Download '%s' is a directory, not a file" % self.fname)
        if os.path.exists(self.fname):
            os.remove(self.fname)
289

290
        self.logger.info("downloading %s", self.params["url"])
291
        self.logger.debug("saving as %s", self.fname)
292
293
294
295
296
297
298
299
300

        downloaded_size = 0
        beginning = time.time()
        # Choose the progress bar (is the size known?)
        if self.size == -1:
            self.logger.debug("total size: unknown")
            last_value = -25 * 1024 * 1024
            progress = progress_unknown_total
        else:
Neil Williams's avatar
Neil Williams committed
301
            self.logger.debug(
Rémi Duraffort's avatar
Rémi Duraffort committed
302
                "total size: %d (%dMB)", self.size, int(self.size / (1024 * 1024))
Neil Williams's avatar
Neil Williams committed
303
            )
304
305
306
307
308
            last_value = -5
            progress = progress_known_total

        decompress_command = None
        if compression:
309
310
            if compression in self.decompress_command_map:
                decompress_command = self.decompress_command_map[compression]
Neil Williams's avatar
Neil Williams committed
311
312
313
                self.logger.info(
                    "Using %s to decompress %s", decompress_command, compression
                )
314
            else:
Neil Williams's avatar
Neil Williams committed
315
316
317
318
                self.logger.info(
                    "Compression %s specified but not decompressing during download",
                    compression,
                )
319
        elif not self.params.get("compression", False):
320
            self.logger.debug("No compression specified")
321
322

        def update_progress():
Corentin LABBE's avatar
Corentin LABBE committed
323
            nonlocal downloaded_size, last_value, md5, sha256, sha512
324
            downloaded_size += len(buff)
Neil Williams's avatar
Neil Williams committed
325
            (printing, new_value, msg) = progress(downloaded_size, last_value)
326
327
328
329
330
            if printing:
                last_value = new_value
                self.logger.debug(msg)
            md5.update(buff)
            sha256.update(buff)
Corentin LABBE's avatar
Corentin LABBE committed
331
            sha512.update(buff)
332

333
        if compression and decompress_command:
334
            try:
335
                with open(self.fname, "wb") as dwnld_file:
336
337
                    proc = subprocess.Popen(  # nosec - internal.
                        [decompress_command], stdin=subprocess.PIPE, stdout=dwnld_file
Neil Williams's avatar
Neil Williams committed
338
                    )
Rémi Duraffort's avatar
Rémi Duraffort committed
339
            except OSError as exc:
340
                msg = "Unable to open %s: %s" % (self.fname, exc.strerror)
341
342
343
344
345
346
347
348
349
350
351
                self.logger.error(msg)
                raise InfrastructureError(msg)

            with proc.stdin as pipe:
                for buff in self.reader():
                    update_progress()
                    try:
                        pipe.write(buff)
                    except BrokenPipeError as exc:
                        error_message = str(exc)
                        self.logger.exception(error_message)
Neil Williams's avatar
Neil Williams committed
352
353
354
355
                        msg = (
                            "Make sure the 'compression' is corresponding "
                            "to the image file type."
                        )
356
357
358
359
                        self.logger.error(msg)
                        raise JobError(error_message)
            proc.wait()
        else:
360
            with open(self.fname, "wb") as dwnld_file:
361
362
363
364
365
366
                for buff in self.reader():
                    update_progress()
                    dwnld_file.write(buff)

        # Log the download speed
        ending = time.time()
Neil Williams's avatar
Neil Williams committed
367
        self.logger.info(
Rémi Duraffort's avatar
Rémi Duraffort committed
368
369
370
371
            "%dMB downloaded in %0.2fs (%0.2fMB/s)",
            downloaded_size / (1024 * 1024),
            round(ending - beginning, 2),
            round(downloaded_size / (1024 * 1024 * (ending - beginning)), 2),
Neil Williams's avatar
Neil Williams committed
372
        )
373

374
375
376
        # If the remote server uses "Content-Encoding: gzip", this calculation will be wrong
        # because requests will decompress the file on the fly, creating a larger file than
        # LAVA expects.
377
378
379
380
381
        if self.size > 0 and self.size != downloaded_size:
            raise InfrastructureError(
                "Download finished (%i bytes) but was not expected size (%i bytes), check your networking."
                % (downloaded_size, self.size)
            )
382

383
        # set the dynamic data into the context
Neil Williams's avatar
Neil Williams committed
384
        self.set_namespace_data(
385
            action="download-action", label=self.key, key="file", value=self.fname
Neil Williams's avatar
Neil Williams committed
386
        )
387
388
389
        self.set_namespace_data(
            action="download-action", label="file", key=self.key, value=self.fname
        )
Neil Williams's avatar
Neil Williams committed
390
391
392
393
394
395
396
397
398
        self.set_namespace_data(
            action="download-action", label=self.key, key="md5", value=md5.hexdigest()
        )
        self.set_namespace_data(
            action="download-action",
            label=self.key,
            key="sha256",
            value=sha256.hexdigest(),
        )
Corentin LABBE's avatar
Corentin LABBE committed
399
400
401
402
403
404
        self.set_namespace_data(
            action="download-action",
            label=self.key,
            key="sha512",
            value=sha512.hexdigest(),
        )
405

Senthil Kumaran S's avatar
Senthil Kumaran S committed
406
        # handle archive files
407
        archive = self.params.get("archive")
Senthil Kumaran S's avatar
Senthil Kumaran S committed
408
        if archive:
409
410
411
412
            if archive != "tar":
                raise JobError("Unknown archive format %r" % archive)

            target_fname_path = os.path.join(os.path.dirname(self.fname), self.key)
413
            self.logger.debug("Extracting %s archive in %s", archive, target_fname_path)
414
415
416
417
418
419
420
421
422
423
424
425
426
            untar_file(self.fname, target_fname_path)
            self.set_namespace_data(
                action="download-action",
                label=self.key,
                key="file",
                value=target_fname_path,
            )
            self.set_namespace_data(
                action="download-action",
                label="file",
                key=self.key,
                value=target_fname_path,
            )
Senthil Kumaran S's avatar
Senthil Kumaran S committed
427

428
429
430
        self._check_checksum("md5", md5.hexdigest(), md5sum)
        self._check_checksum("sha256", sha256.hexdigest(), sha256sum)
        self._check_checksum("sha512", sha512.hexdigest(), sha512sum)
Corentin LABBE's avatar
Corentin LABBE committed
431

432
        # certain deployments need prefixes set
Neil Williams's avatar
Neil Williams committed
433
434
435
436
437
438
439
440
        if self.parameters["to"] == "tftp" or self.parameters["to"] == "nbd":
            suffix = self.get_namespace_data(
                action="tftp-deploy", label="tftp", key="suffix"
            )
            self.set_namespace_data(
                action="download-action",
                label="file",
                key=self.key,
441
                value=os.path.join(suffix, self.key, os.path.basename(self.fname)),
Neil Williams's avatar
Neil Williams committed
442
443
444
445
446
447
448
449
450
            )
        elif self.parameters["to"] == "iso-installer":
            suffix = self.get_namespace_data(
                action="deploy-iso-installer", label="iso", key="suffix"
            )
            self.set_namespace_data(
                action="download-action",
                label="file",
                key=self.key,
451
                value=os.path.join(suffix, self.key, os.path.basename(self.fname)),
Neil Williams's avatar
Neil Williams committed
452
            )
453
454

        # xnbd protocoll needs to know the location
Neil Williams's avatar
Neil Williams committed
455
456
457
458
459
        nbdroot = self.get_namespace_data(
            action="download-action", label="file", key="nbdroot"
        )
        if "lava-xnbd" in self.parameters and nbdroot:
            self.parameters["lava-xnbd"]["nbdroot"] = nbdroot
460

461
        self.results = {
Neil Williams's avatar
Neil Williams committed
462
463
464
465
466
467
468
469
470
471
472
473
            "label": self.key,
            "size": downloaded_size,
            "md5sum": str(
                self.get_namespace_data(
                    action="download-action", label=self.key, key="md5"
                )
            ),
            "sha256sum": str(
                self.get_namespace_data(
                    action="download-action", label=self.key, key="sha256"
                )
            ),
Corentin LABBE's avatar
Corentin LABBE committed
474
475
476
477
478
            "sha512sum": str(
                self.get_namespace_data(
                    action="download-action", label=self.key, key="sha512"
                )
            ),
479
        }
Neil Williams's avatar
Neil Williams committed
480
481
482
        return connection


483
class FileDownloadAction(DownloadHandler):
484
485
486
    """
    Download a resource from file (copy)
    """
487

488
489
490
    name = "file-download"
    description = "copy a local file"
    summary = "local file copy"
491
492

    def validate(self):
Rémi Duraffort's avatar
Rémi Duraffort committed
493
        super().validate()
494
        try:
495
            self.logger.debug("Validating that %s exists", self.url.geturl())
496
497
            self.size = os.stat(self.url.path).st_size
        except OSError:
Neil Williams's avatar
Neil Williams committed
498
499
500
            self.errors = "Image file '%s' does not exist or is not readable" % (
                self.url.path
            )
501
502
503

    def reader(self):
        try:
504
            with open(self.url.path, "rb") as reader:
Neil Williams's avatar
Neil Williams committed
505
                buff = reader.read(FILE_DOWNLOAD_CHUNK_SIZE)
506
507
508
                while buff:
                    yield buff
                    buff = reader.read(FILE_DOWNLOAD_CHUNK_SIZE)
Rémi Duraffort's avatar
Rémi Duraffort committed
509
        except OSError as exc:
Neil Williams's avatar
Neil Williams committed
510
            raise InfrastructureError(
Rémi Duraffort's avatar
Rémi Duraffort committed
511
                "Unable to read from %s: %s" % (self.url.path, str(exc))
Neil Williams's avatar
Neil Williams committed
512
            )
513
514
515


class HttpDownloadAction(DownloadHandler):
516
517
518
    """
    Download a resource over http or https using requests module
    """
519

520
521
522
    name = "http-download"
    description = "use http to download the file"
    summary = "http download"
523
524

    def validate(self):
Rémi Duraffort's avatar
Rémi Duraffort committed
525
        super().validate()
526
        res = None
527
        try:
528
529
530
531
532
533
534
535
536
537
538
539
            http_cache = self.job.parameters["dispatcher"].get(
                "http_url_format_string", ""
            )
            if http_cache:
                self.logger.info("Using caching service: '%s'", http_cache)
                try:
                    self.url = urlparse(http_cache % quote_plus(self.url.geturl()))
                except TypeError as exc:
                    self.logger.error("Invalid http_url_format_string: '%s'", exc)
                    self.errors = "Invalid http_url_format_string: '%s'" % str(exc)
                    return

540
            self.logger.debug("Validating that %s exists", self.url.geturl())
541
542
543
544
            # Force the non-use of Accept-Encoding: gzip, this will permit to know the final size
            res = requests.head(
                self.url.geturl(), allow_redirects=True, headers={"Accept-Encoding": ""}
            )
545
            if res.status_code != requests.codes.OK:
546
                # try using (the slower) get for services with broken redirect support
547
                self.logger.debug("Using GET because HEAD is not supported properly")
548
                res.close()
549
550
551
552
553
554
555
                # Like for HEAD, we need get a size, so disable gzip
                res = requests.get(
                    self.url.geturl(),
                    allow_redirects=True,
                    stream=True,
                    headers={"Accept-Encoding": ""},
                )
556
                if res.status_code != requests.codes.OK:
Neil Williams's avatar
Neil Williams committed
557
558
559
560
                    self.errors = "Resource unavailable at '%s' (%d)" % (
                        self.url.geturl(),
                        res.status_code,
                    )
561
                    return
562

Neil Williams's avatar
Neil Williams committed
563
            self.size = int(res.headers.get("content-length", -1))
564
        except requests.Timeout:
565
            self.logger.error("Request timed out")
566
567
            self.errors = "'%s' timed out" % (self.url.geturl())
        except requests.RequestException as exc:
Senthil Kumaran S's avatar
Senthil Kumaran S committed
568
            self.logger.error("Resource not available")
569
            self.errors = "Unable to get '%s': %s" % (self.url.geturl(), str(exc))
570
571
572
        finally:
            if res is not None:
                res.close()
573
574

    def reader(self):
575
        res = None
576
        try:
577
578
            # FIXME: When requests 3.0 is released, use the enforce_content_length
            # parameter to raise an exception the file is not fully downloaded
Neil Williams's avatar
Neil Williams committed
579
            res = requests.get(self.url.geturl(), allow_redirects=True, stream=True)
580
            if res.status_code != requests.codes.OK:
581
582
                # This is an Infrastructure error because the validate function
                # checked that the file does exist.
Neil Williams's avatar
Neil Williams committed
583
584
585
                raise InfrastructureError(
                    "Unable to download '%s'" % (self.url.geturl())
                )
586
            for buff in res.iter_content(HTTP_DOWNLOAD_CHUNK_SIZE):
587
588
                yield buff
        except requests.RequestException as exc:
Neil Williams's avatar
Neil Williams committed
589
590
591
            raise InfrastructureError(
                "Unable to download '%s': %s" % (self.url.geturl(), str(exc))
            )
592
        finally:
593
594
            if res is not None:
                res.close()
595
596
597


class ScpDownloadAction(DownloadHandler):
598
599
600
    """
    Download a resource over scp
    """
601

602
603
604
    name = "scp-download"
    description = "Use scp to copy the file"
    summary = "scp download"
605
606

    def validate(self):
Rémi Duraffort's avatar
Rémi Duraffort committed
607
        super().validate()
Rémi Duraffort's avatar
Rémi Duraffort committed
608
        try:
609
            size = subprocess.check_output(  # nosec - internal.
Rémi Duraffort's avatar
Rémi Duraffort committed
610
                ["ssh", self.url.netloc, "stat", "-c", "%s", self.url.path],
Neil Williams's avatar
Neil Williams committed
611
612
                stderr=subprocess.STDOUT,
            )
613
            self.size = int(size)
Rémi Duraffort's avatar
Rémi Duraffort committed
614
615
        except subprocess.CalledProcessError as exc:
            self.errors = str(exc)
616
617
618
619

    def reader(self):
        process = None
        try:
620
            process = subprocess.Popen(  # nosec - internal.
Rémi Duraffort's avatar
Rémi Duraffort committed
621
                ["ssh", self.url.netloc, "cat", self.url.path], stdout=subprocess.PIPE
622
            )
623
            buff = process.stdout.read(SCP_DOWNLOAD_CHUNK_SIZE)
Rémi Duraffort's avatar
Rémi Duraffort committed
624
625
            while buff:
                yield buff
626
                buff = process.stdout.read(SCP_DOWNLOAD_CHUNK_SIZE)
Rémi Duraffort's avatar
Rémi Duraffort committed
627
            if process.wait() != 0:
Neil Williams's avatar
Neil Williams committed
628
629
630
631
                raise JobError(
                    "Dowloading '%s' failed with message '%s'"
                    % (self.url.geturl(), process.stderr.read())
                )
632
        finally:
Rémi Duraffort's avatar
Rémi Duraffort committed
633
            if process is not None:
634
                with contextlib.suppress(OSError):
Rémi Duraffort's avatar
Rémi Duraffort committed
635
                    process.kill()
636
637


638
639
640
641
642
class LxcDownloadAction(Action):
    """
    Map an already downloaded resource to the correct path.
    """

643
644
645
646
    name = "lxc-download"
    description = "Map to the correct lxc path"
    summary = "lxc download"

647
    def __init__(self, key, path, url):
Rémi Duraffort's avatar
Rémi Duraffort committed
648
        super().__init__()
649
650
651
652
653
        self.key = key
        self.path = path
        self.url = url

    def validate(self):
Rémi Duraffort's avatar
Rémi Duraffort committed
654
        super().validate()
Neil Williams's avatar
Neil Williams committed
655
        if self.url.scheme != "lxc":
656
657
658
659
            self.errors = "lxc:/// url scheme is invalid"
        if not self.url.path:
            self.errors = "Invalid path in lxc:/// url"

660
661
    def run(self, connection, max_end_time):
        connection = super().run(connection, max_end_time)
662
663
        # this is the device namespace - the lxc namespace is not accessible
        lxc_name = None
Neil Williams's avatar
Neil Williams committed
664
665
666
667
668
        protocol = [
            protocol
            for protocol in self.job.protocols
            if protocol.name == LxcProtocol.name
        ][0]
669
670
671
        if protocol:
            lxc_name = protocol.lxc_name
        if not lxc_name:
Neil Williams's avatar
Neil Williams committed
672
673
674
675
            raise JobError(
                "Erroneous lxc url '%s' without protocol %s" % self.url,
                LxcProtocol.name,
            )
676
677

        fname = os.path.basename(self.url.path)
Neil Williams's avatar
Neil Williams committed
678
        lxc_home = lava_lxc_home(lxc_name, self.job.parameters["dispatcher"])
679
        file_path = os.path.join(lxc_home, fname)
Rémi Duraffort's avatar
Rémi Duraffort committed
680
        self.logger.debug("Trying '%s' matching '%s'", file_path, fname)
681
        if os.path.exists(file_path):
Neil Williams's avatar
Neil Williams committed
682
683
684
            self.set_namespace_data(
                action="download-action", label=self.key, key="file", value=file_path
            )
685
686
687
688
689
        else:
            raise JobError("Resource unavailable: %s" % self.url.path)
        return connection


Neil Williams's avatar
Neil Williams committed
690
691
692
693
694
695
class QCowConversionAction(Action):
    """
    explicit action for qcow conversion to avoid reliance
    on filename suffix
    """

696
697
698
699
    name = "qcow2-convert"
    description = "convert qcow image using qemu-img"
    summary = "qcow conversion"

700
    def __init__(self, key):
Rémi Duraffort's avatar
Rémi Duraffort committed
701
        super().__init__()
702
        self.key = key
Neil Williams's avatar
Neil Williams committed
703

704
705
    def run(self, connection, max_end_time):
        connection = super().run(connection, max_end_time)
706
        fname = self.get_namespace_data(
Neil Williams's avatar
Neil Williams committed
707
            action="download-action", label=self.key, key="file"
708
        )
Rémi Duraffort's avatar
Rémi Duraffort committed
709
        origin = fname
710
        # Remove the '.qcow2' extension and add '.img'
Neil Williams's avatar
Neil Williams committed
711
        if fname.endswith(".qcow2"):
712
713
            fname = fname[:-6]
        fname += ".img"
Rémi Duraffort's avatar
Rémi Duraffort committed
714

715
        self.logger.debug("Converting downloaded image from qcow2 to raw")
716
        try:
717
718
            subprocess.check_output(  # nosec - checked.
                ["qemu-img", "convert", "-f", "qcow2", "-O", "raw", origin, fname],
Neil Williams's avatar
Neil Williams committed
719
720
                stderr=subprocess.STDOUT,
            )
721
722
723
724
725
        except subprocess.CalledProcessError as exc:
            self.logger.error("Unable to convert the qcow2 image")
            self.logger.error(exc.output)
            raise JobError(exc.output)

Neil Williams's avatar
Neil Williams committed
726
727
728
729
730
731
        self.set_namespace_data(
            action=self.name, label=self.key, key="file", value=fname
        )
        self.set_namespace_data(
            action=self.name, label="file", key=self.key, value=fname
        )
Rémi Duraffort's avatar
Rémi Duraffort committed
732
        return connection
733
734
735
736
737
738
739


class Download(Deployment):
    """
    Strategy class for a download deployment.
    Downloads the relevant parts, copies to LXC if available.
    """
Neil Williams's avatar
Neil Williams committed
740

741
    compatibility = 1
Neil Williams's avatar
Neil Williams committed
742
    name = "download"
743
744

    def __init__(self, parent, parameters):
Rémi Duraffort's avatar
Rémi Duraffort committed
745
        super().__init__(parent)
746
747
748
749
750
751
752
        self.action = DownloadAction()
        self.action.section = self.action_type
        self.action.job = self.job
        parent.add_action(self.action, parameters)

    @classmethod
    def accepts(cls, device, parameters):
Neil Williams's avatar
Neil Williams committed
753
        if "to" not in parameters:
754
            return False, '"to" is not in deploy parameters'
Neil Williams's avatar
Neil Williams committed
755
        if parameters["to"] != "download":
756
            return False, '"to" parameter is not "download"'
Neil Williams's avatar
Neil Williams committed
757
        return True, "accepted"
758
759
760
761


class DownloadAction(DeployAction):  # pylint:disable=too-many-instance-attributes

762
763
764
765
    name = "download-deploy"
    description = "download files and copy to LXC if available"
    summary = "download deployment"

766
    def __init__(self):
Rémi Duraffort's avatar
Rémi Duraffort committed
767
        super().__init__()
768
769
770
        self.download_dir = None

    def validate(self):
Rémi Duraffort's avatar
Rémi Duraffort committed
771
        super().validate()
Neil Williams's avatar
Neil Williams committed
772
773
774
        self.set_namespace_data(
            action=self.name, label="download-dir", key="dir", value=self.download_dir
        )
775
776

    def populate(self, parameters):
777
        self.pipeline = Pipeline(parent=self, job=self.job, parameters=parameters)
778
779
780
781
782
783
784
785
        # Check if the device has a power command such as HiKey, Dragonboard,
        # etc. against device that doesn't like Nexus, etc.
        # This is required in order to power on the device so that when the
        # test job writer wants to perform some operation using a
        # lava-test-shell action that follows, this becomes mandatory. Think of
        # issuing any fastboot commands on the powered on device.
        #
        # NOTE: Add more power on strategies, if required for specific devices.
Neil Williams's avatar
Neil Williams committed
786
        if self.job.device.get("fastboot_via_uboot", False):
787
788
            self.pipeline.add_action(ConnectDevice())
            self.pipeline.add_action(UBootEnterFastbootAction())
789
        elif self.job.device.hard_reset_command:
790
            self.force_prompt = True
791
792
            self.pipeline.add_action(ConnectDevice())
            self.pipeline.add_action(ResetDevice())
793
        else:
794
            self.pipeline.add_action(EnterFastbootAction())
795
796

        self.download_dir = self.mkdtemp()
797
798
799
800
801
802
        for image in sorted(parameters["images"].keys()):
            self.pipeline.add_action(
                DownloaderAction(
                    image, self.download_dir, params=parameters["images"][image]
                )
            )
803
        if self.test_needs_overlay(parameters):
804
805
            self.pipeline.add_action(OverlayAction())
        self.pipeline.add_action(CopyToLxcAction())
806
807
808
809
810
811
812


class CopyToLxcAction(DeployAction):
    """
    Copy downloaded files to LXC within LAVA_LXC_HOME.
    """

813
814
815
816
    name = "copy-to-lxc"
    description = "copy files to lxc"
    summary = "copy to lxc"

817
    def __init__(self):
Rémi Duraffort's avatar
Rémi Duraffort committed
818
        super().__init__()
819
820
821
        self.retries = 3
        self.sleep = 10

822
    def run(self, connection, max_end_time):
823
        connection = super().run(connection, max_end_time)
824
825
        # this is the device namespace - the lxc namespace is not accessible
        lxc_name = None
Neil Williams's avatar
Neil Williams committed
826
827
828
829
830
        protocol = [
            protocol
            for protocol in self.job.protocols
            if protocol.name == LxcProtocol.name
        ][0]
831
832
833
834
835
836
        if protocol:
            lxc_name = protocol.lxc_name
        else:
            return connection

        # Copy each file to LXC.
Neil Williams's avatar
Neil Williams committed
837
838
839
840
        for image in self.get_namespace_keys("download-action"):
            src = self.get_namespace_data(
                action="download-action", label=image, key="file"
            )
841
842
843
844
845
846
847
            # The archive extraction logic and some deploy logic in
            # DownloadHandler will set a label 'file' in the namespace but
            # that file would have been dealt with and the actual path may not
            # exist, though the key exists as part of the namespace, which we
            # can ignore safely, hence we continue on invalid src.
            if not src:
                continue
Neil Williams's avatar
Neil Williams committed
848
849
850
851
            copy_to_lxc(lxc_name, src, self.job.parameters["dispatcher"])
        overlay_file = self.get_namespace_data(
            action="compress-overlay", label="output", key="file"
        )
852
853
854
        if overlay_file is None:
            self.logger.debug("skipped %s", self.name)
        else:
Neil Williams's avatar
Neil Williams committed
855
856
857
858
859
860
            copy_overlay_to_lxc(
                lxc_name,
                overlay_file,
                self.job.parameters["dispatcher"],
                self.parameters["namespace"],
            )
861
        return connection