__init__.py 38 KB
Newer Older
Rémi Duraffort's avatar
Rémi Duraffort committed
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20
# -*- coding: utf-8 -*-
# Copyright (C) 2017-2018 Linaro Limited
#
# Author: Neil Williams <neil.williams@linaro.org>
#         Senthil Kumaran <senthil.kumaran@linaro.org>
#
# This file is part of LAVA.
#
# LAVA is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License version 3
# as published by the Free Software Foundation
#
# LAVA is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with LAVA.  If not, see <http://www.gnu.org/licenses/>.

21 22
from functools import wraps
from simplejson import JSONDecodeError
23
import os
24
import sys
Neil Williams's avatar
Neil Williams committed
25
import xmlrpc.client
26 27
import yaml

28
from django.conf import settings
29
from django.core.exceptions import PermissionDenied
30
from django.db.models import Count, Q
31
from django.db import transaction
32
from linaro_django_xmlrpc.models import ExposedV2API
33 34 35

from lava_common.compat import yaml_safe_dump, yaml_safe_load

36 37 38
from lava_scheduler_app.models import (
    Device,
    DeviceType,
39
    JSONDataError,
40
    DevicesUnavailableException,
41
    TestJob,
Neil Williams's avatar
Neil Williams committed
42
)
43
from lava_scheduler_app.views import get_restricted_job
44 45
from lava_scheduler_app.dbutils import (
    device_type_summary,
46 47
    testjob_submission,
    active_device_types,
48
)
49 50 51 52 53 54 55
from lava_scheduler_app.schema import (
    validate_submission,
    validate_device,
    SubmissionException,
)

# functions need to be members to be exposed in the API
56

Michael-Doyle Hudson's avatar
Michael-Doyle Hudson committed
57

Neil Williams's avatar
Neil Williams committed
58 59
# to make a function visible in the API, it must be a member of SchedulerAPI

Michael-Doyle Hudson's avatar
Michael-Doyle Hudson committed
60

61 62
def check_perm(perm):
    """ decorator to check that the caller has the given permission """
63

64 65 66 67 68 69 70
    def decorator(f):
        @wraps(f)
        def wrapper(self, *args, **kwargs):
            self._authenticate()
            if not self.user.has_perm(perm):
                raise xmlrpc.client.Fault(
                    403,
71
                    "User '%s' is missing permission %s ." % (self.user.username, perm),
72 73
                )
            return f(self, *args, **kwargs)
74

75
        return wrapper
76

77
    return decorator
78 79


80 81 82
def build_device_status_display(state, health):
    if state == Device.STATE_IDLE:
        if health in [Device.HEALTH_GOOD, Device.HEALTH_UNKNOWN]:
83
            return "idle"
84
        elif health == Device.HEALTH_RETIRED:
85
            return "retired"
86
        else:
87
            return "offline"
88
    elif state == Device.STATE_RESERVED:
89
        return "reserved"
90
    else:
91
        return "running"
92 93


94
class SchedulerAPI(ExposedV2API):
95
    def submit_job(self, job_data):
96 97 98 99 100 101 102
        """
        Name
        ----
        `submit_job` (`job_data`)

        Description
        -----------
103 104
        Submit the given job data which is in LAVA job JSON or YAML format as a
        new job to LAVA scheduler.
105 106 107 108

        Arguments
        ---------
        `job_data`: string
109
            Job JSON or YAML string.
110 111 112 113

        Return value
        ------------
        This function returns an XML-RPC integer which is the newly created
114
        job's id, provided the user is authenticated with an username and token.
115 116
        If the job is a multinode job, this function returns the list of created
        job IDs.
117
        """
118
        self._authenticate()
Neil Williams's avatar
Neil Williams committed
119
        try:
120 121
            job = testjob_submission(job_data, self.user)
        except SubmissionException as exc:
Neil Williams's avatar
Neil Williams committed
122
            raise xmlrpc.client.Fault(400, "Problem with submitted job data: %s" % exc)
Rémi Duraffort's avatar
Rémi Duraffort committed
123
        # FIXME: json error is not needed anymore
Neil Williams's avatar
Neil Williams committed
124
        except (JSONDataError, JSONDecodeError, ValueError) as exc:
Neil Williams's avatar
Neil Williams committed
125
            raise xmlrpc.client.Fault(400, "Decoding job submission failed: %s." % exc)
126
        except (Device.DoesNotExist, DeviceType.DoesNotExist):
Neil Williams's avatar
Neil Williams committed
127
            raise xmlrpc.client.Fault(404, "Specified device or device type not found.")
128
        except DevicesUnavailableException as exc:
Neil Williams's avatar
Neil Williams committed
129
            raise xmlrpc.client.Fault(400, "Device unavailable: %s" % str(exc))
130
        if isinstance(job, list):
131
            return [j.sub_id for j in job]
132 133
        else:
            return job.id
134

135
    def resubmit_job(self, job_id):
136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155
        """
        Name
        ----
        `resubmit_job` (`job_id`)

        Description
        -----------
        Resubmit the given job reffered by its id.

        Arguments
        ---------
        `job_id`: string
            The job's id which should be re-submitted.

        Return value
        ------------
        This function returns an XML-RPC integer which is the newly created
        job's id,  provided the user is authenticated with an username and
        token.
        """
156
        self._authenticate()
Paul Larson's avatar
Paul Larson committed
157
        try:
158
            job = get_restricted_job(self.user, job_id)
Paul Larson's avatar
Paul Larson committed
159
        except TestJob.DoesNotExist:
Neil Williams's avatar
Neil Williams committed
160
            raise xmlrpc.client.Fault(404, "Specified job not found.")
161

162
        if job.is_multinode:
163 164 165
            return self.submit_job(job.multinode_definition)
        else:
            return self.submit_job(job.definition)
166

167
    def cancel_job(self, job_id):
168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185
        """
        Name
        ----
        `cancel_job` (`job_id`)

        Description
        -----------
        Cancel the given job reffered by its id.

        Arguments
        ---------
        `job_id`: string
            Job id which should be canceled.

        Return value
        ------------
        None. The user should be authenticated with an username and token.
        """
186 187
        self._authenticate()
        if not job_id:
188
            raise xmlrpc.client.Fault(400, "Bad request: TestJob id was not specified.")
189

190 191 192 193
        with transaction.atomic():
            try:
                job = get_restricted_job(self.user, job_id, for_update=True)
            except PermissionDenied:
Neil Williams's avatar
Neil Williams committed
194
                raise xmlrpc.client.Fault(
195 196
                    401, "Permission denied for user to job %s" % job_id
                )
197
            except TestJob.DoesNotExist:
Neil Williams's avatar
Neil Williams committed
198
                raise xmlrpc.client.Fault(404, "Specified job not found.")
199

200 201 202
            try:
                job.cancel(self.user)
            except PermissionDenied:
Neil Williams's avatar
Neil Williams committed
203
                raise xmlrpc.client.Fault(403, "Permission denied.")
204
        return True
205

206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231
    def validate_yaml(self, yaml_string):
        """
        Name
        ----
        validate_yaml (yaml_job_data)

        Description
        -----------
        Validate the supplied pipeline YAML against the submission schema.

        Note: this does not validate the job itself, just the YAML against the
        submission schema. A job which validates against the schema can still be
        an invalid job for the dispatcher and such jobs will be accepted as Submitted,
        scheduled and then marked as Incomplete with a failure comment. Full validation
        only happens after a device has been assigned to the Submitted job.

        Arguments
        ---------
        'yaml_job_data': string

        Return value
        ------------
        Raises an Exception if the yaml_job_data is invalid.
        """
        try:
            # YAML can parse JSON as YAML, JSON cannot parse YAML at all
232
            yaml_data = yaml_safe_load(yaml_string)
233
        except yaml.YAMLError as exc:
Neil Williams's avatar
Neil Williams committed
234
            raise xmlrpc.client.Fault(400, "Decoding job submission failed: %s." % exc)
235 236 237 238
        try:
            # validate against the submission schema.
            validate_submission(yaml_data)  # raises SubmissionException if invalid.
        except SubmissionException as exc:
Neil Williams's avatar
Neil Williams committed
239
            raise xmlrpc.client.Fault(400, "Invalid YAML submission: %s" % exc)
240

241
    def job_output(self, job_id, offset=0):
242 243 244
        """
        Name
        ----
245
        `job_output` (`job_id`, `offset=0`)
246 247 248 249 250 251 252 253 254

        Description
        -----------
        Get the output of given job id.

        Arguments
        ---------
        `job_id`: string
            Job id for which the output is required.
255 256 257
        `offset`: integer
            Offset from which to start reading the output file specified in bytes.
            It defaults to 0.
258 259 260

        Return value
        ------------
261 262
        This function returns an XML-RPC binary data of output file, provided
        the user is authenticated with an username and token.
263
        """
264 265
        self._authenticate()
        if not job_id:
266
            raise xmlrpc.client.Fault(400, "Bad request: TestJob id was not specified.")
267
        try:
268
            job = get_restricted_job(self.user, job_id)
269
        except PermissionDenied:
Neil Williams's avatar
Neil Williams committed
270
            raise xmlrpc.client.Fault(
271 272
                401, "Permission denied for user to job %s" % job_id
            )
273
        except TestJob.DoesNotExist:
Neil Williams's avatar
Neil Williams committed
274
            raise xmlrpc.client.Fault(404, "Specified job not found.")
275

276 277 278 279 280 281 282 283
        # Open the logs
        output_path = os.path.join(job.output_dir, "output.yaml")
        try:
            with open(output_path, encoding="utf-8", errors="replace") as f_logs:
                if f_logs:
                    f_logs.seek(offset)
                return xmlrpc.client.Binary(f_logs.read().encode("UTF-8"))
        except OSError:
Neil Williams's avatar
Neil Williams committed
284
            raise xmlrpc.client.Fault(404, "Job output not found.")
285 286 287 288 289 290 291 292 293

    def all_devices(self):
        """
        Name
        ----
        `all_devices` ()

        Description
        -----------
294
        Get all the available devices with their state and type information.
295 296 297 298 299 300 301 302

        Arguments
        ---------
        None

        Return value
        ------------
        This function returns an XML-RPC array in which each item is a list of
303 304
        device hostname, device type, device state, current running job id and
        if device is pipeline. For example:
305

306
        [['panda01', 'panda', 'running', 'good', 164, False], ['qemu01', 'qemu', 'idle', 'unknwon', None, True]]
307 308
        """

Stevan Radakovic's avatar
Stevan Radakovic committed
309 310 311
        devices_list = Device.objects.visible_by_user(self.user).exclude(
            health=Device.HEALTH_RETIRED
        )
312

313 314 315 316 317
        devices_jobs = ((
            dev,
            dev.current_job() if dev.state == Device.STATE_RUNNING else None
        ) for dev in devices_list)

318 319 320 321 322
        return [
            [
                dev.hostname,
                dev.device_type.name,
                build_device_status_display(dev.state, dev.health),
323
                job.pk if job else None,
324 325
                True,
            ]
326
            for dev, job in devices_jobs
327
        ]
328 329 330 331 332 333 334 335 336 337

    def all_device_types(self):
        """
        Name
        ----
        `all_device_types` ()

        Description
        -----------
        Get all the available device types with their state and count
338
        information.
339 340 341 342 343 344 345 346 347 348 349 350 351 352 353

        Arguments
        ---------
        None

        Return value
        ------------
        This function returns an XML-RPC array in which each item is a dict
        which contains name (device type), idle, busy, offline counts.
        For example:

        [{'idle': 1, 'busy': 0, 'name': 'panda', 'offline': 0},
        {'idle': 1, 'busy': 0, 'name': 'qemu', 'offline': 0}]
        """

354
        all_device_types = []
355
        keys = ["busy", "idle", "offline"]
356

Stevan Radakovic's avatar
Stevan Radakovic committed
357
        device_types = device_type_summary(self.user)
358 359

        for dev_type in device_types:
360
            device_type = {"name": dev_type["device_type"]}
361
            for key in keys:
Neil Williams's avatar
Neil Williams committed
362
                device_type[key] = dev_type[key]
363
            all_device_types.append(device_type)
364

365
        return all_device_types
366

367 368 369
    def get_recent_jobs_for_device_type(
        self, device_type, count=1, restrict_to_user=False
    ):
370 371 372 373 374 375 376 377 378 379 380 381 382 383 384 385 386 387 388 389 390 391 392 393 394 395 396 397 398 399 400 401 402 403 404 405 406 407 408 409 410 411 412 413
        """
        Name
        ----

        `get_recent_jobs_for_device_type` (`device_type`, `count=1`, `restrict_to_user=False`)

        Description
        -----------
        Get details of recently finished jobs for a given device_type. Limits the list
        to test jobs submitted by the user making the query if restrict_to_user is set to
        True. Get only the most recent job by default, but count can be set higher to
        get for example the last 10 jobs.

        Arguments
        ---------
        `device_type`: string
            Name of the device_type for which you want the jobs
        `count`: integer (Optional, default=1)
            Number of last jobs you want
        `restrict_to_user`: boolean (Optional, default=False)
            Fetch only the jobs submitted by the user making the query if set to True

        Return value
        ------------
        This function returns a list of dictionaries, which correspond to the
        list of recently finished jobs informations (Complete or Incomplete)
        for this device, ordered from youngest to oldest.

        [
            {
                'description': 'ramdisk health check',
                'id': 359828,
                'status': 'Complete',
                'device': 'black01'
            },
            {
                'description': 'standard ARMMP NFS',
                'id': 359827
                'status': 'Incomplete',
                'device': 'black02'
            }
        ]
        """
        if not device_type:
Neil Williams's avatar
Neil Williams committed
414
            raise xmlrpc.client.Fault(
415 416 417
                400, "Bad request: device_type was not specified."
            )
        if count < 0:
418
            raise xmlrpc.client.Fault(400, "Bad request: count must not be negative.")
419 420 421
        try:
            dt = DeviceType.objects.get(name=device_type, display=True)
        except Device.DoesNotExist:
Neil Williams's avatar
Neil Williams committed
422
            raise xmlrpc.client.Fault(
423 424 425
                404, "DeviceType '%s' was not found." % device_type
            )

426 427 428
        job_qs = (
            TestJob.objects.filter(state=TestJob.STATE_FINISHED)
            .filter(requested_device_type=dt)
Stevan Radakovic's avatar
Stevan Radakovic committed
429
            .visible_by_user(self.user)
430 431
            .order_by("-id")
        )
432 433 434 435
        if restrict_to_user:
            job_qs = job_qs.filter(submitter=self.user)
        job_list = []
        for job in job_qs.all()[:count]:
436
            hostname = ""
437 438
            if job.actual_device:
                hostname = job.actual_device.hostname
439 440 441
            job_dict = {
                "id": job.id,
                "description": job.description,
Rémi Duraffort's avatar
Rémi Duraffort committed
442
                "status": job.get_legacy_status_display(),
443
                "device": hostname,
444 445 446 447
            }
            job_list.append(job_dict)
        return job_list

448 449 450 451 452 453 454 455 456 457 458 459 460 461 462 463 464 465 466 467 468 469 470 471 472 473 474 475 476 477 478 479 480 481 482 483 484 485 486 487 488 489 490
    def get_recent_jobs_for_device(self, device, count=1, restrict_to_user=False):
        """
        Name
        ----

        `get_recent_jobs_for_device` (`device`, `count=1`, `restrict_to_user=False`)

        Description
        -----------
        Get details of recently finished jobs for a given device. Limits the list
        to test jobs submitted by the user making the query if restrict_to_user is set to
        True. Get only the most recent job by default, but count can be set higher to
        get for example the last 10 jobs.

        Arguments
        ---------
        `device`: string
            Name of the device for which you want the jobs
        `count`: integer (Optional, default=1)
            Number of last jobs you want
        `restrict_to_user`: boolean (Optional, default=False)
            Fetch only the jobs submitted by the user making the query if set to True

        Return value
        ------------
        This function returns a list of dictionaries, which correspond to the
        list of recently finished jobs informations (Complete or Incomplete)
        for this device, ordered from youngest to oldest.

        [
            {
                'description': 'mainline--armada-370-db--multi_v7_defconfig--network',
                'id': 359828,
                'status': 'Complete'
            },
            {
                'description': 'mainline--armada-370-db--multi_v7_defconfig--sata',
                'id': 359827
                'status': 'Incomplete'
            }
        ]
        """
        if not device:
491
            raise xmlrpc.client.Fault(400, "Bad request: device was not specified.")
492
        if count < 0:
493
            raise xmlrpc.client.Fault(400, "Bad request: count must not be negative.")
494 495 496
        try:
            device_obj = Device.objects.get(hostname=device)
        except Device.DoesNotExist:
497
            raise xmlrpc.client.Fault(404, "Device '%s' was not found." % device)
498

Stevan Radakovic's avatar
Stevan Radakovic committed
499
        if not device_obj.can_view(self.user):
Neil Williams's avatar
Neil Williams committed
500
            raise xmlrpc.client.Fault(
501
                403, "Device '%s' not available to user '%s'." % (device, self.user)
502
            )
503 504 505
        job_qs = (
            TestJob.objects.filter(state=TestJob.STATE_FINISHED)
            .filter(actual_device=device_obj)
Stevan Radakovic's avatar
Stevan Radakovic committed
506
            .visible_by_user(self.user)
507 508
            .order_by("-id")
        )
509 510 511 512 513 514 515
        if restrict_to_user:
            job_qs = job_qs.filter(submitter=self.user)
        job_list = []
        for job in job_qs.all()[:count]:
            job_dict = {
                "id": job.id,
                "description": job.description,
Rémi Duraffort's avatar
Rémi Duraffort committed
516
                "status": job.get_legacy_status_display(),
517 518 519 520
            }
            job_list.append(job_dict)
        return job_list

521 522 523 524 525 526 527 528 529 530 531 532 533 534 535 536 537 538 539 540 541 542 543 544 545 546 547 548 549 550 551 552 553 554 555
    def get_device_type_by_alias(self, alias):
        """
        Name
        ----

        `get_device_type_by_alias` (`alias`)

        Description
        -----------
        Get the matching device-type(s) for the specified alias. It is
        possible that more than one device-type can be returned, depending
        on local admin configuration. An alias can be used to provide the
        link between the device-type name and the Device Tree name.
        It is possible for multiple device-types to have the same alias
        (to assist in transitions and migrations).
        The specified alias string can be a partial match, returning all
        device-types which have an alias name containing the requested
        string.

        Arguments
        ---------
        `alias`: string
            Name of the alias to lookup

        Return value
        ------------
        This function returns a dictionary containing the alias as the key
        and a list of device-types which use that alias as the value. If the
        specified alias does not match any device-type, the dictionary contains
        an empty list for the alias key.

        {'apq8016-sbc': ['dragonboard410c']}
        {'ompa4-panda': ['panda', 'panda-es']}
        """

Stevan Radakovic's avatar
Stevan Radakovic committed
556 557 558
        aliases = DeviceType.objects.filter(
            aliases__name__contains=alias
        ).visible_by_user(self.user)
559
        return {alias: [device_type.name for device_type in aliases]}
560

561 562 563 564 565 566 567 568 569 570 571 572 573 574 575 576 577 578 579 580 581 582 583 584 585 586 587 588 589 590
    def get_device_status(self, hostname):
        """
        Name
        ----
        `get_device_status` (`hostname`)

        Description
        -----------
        Get status, running job, date from which it is offline of the given
        device and the user who put it offline.

        Arguments
        ---------
        `hostname`: string
            Name of the device for which the status is asked.

        Return value
        ------------
        This function returns an XML-RPC dictionary which contains hostname,
        status, date from which the device is offline if the device is offline,
        the user who put the device offline if the device is offline and the
        job id of the running job.
        The device has to be visible to the user who requested device's status.

        Note that offline_since and offline_by can be empty strings if the device
        status is manually changed by an administrator in the database or from
        the admin site of LAVA even if device's status is offline.
        """

        if not hostname:
591
            raise xmlrpc.client.Fault(400, "Bad request: Hostname was not specified.")
592 593 594
        try:
            device = Device.objects.get(hostname=hostname)
        except Device.DoesNotExist:
595
            raise xmlrpc.client.Fault(404, "Device '%s' was not found." % hostname)
596 597

        device_dict = {}
Stevan Radakovic's avatar
Stevan Radakovic committed
598
        if device.can_view(self.user):
599
            device_dict["hostname"] = device.hostname
600 601 602
            device_dict["status"] = build_device_status_display(
                device.state, device.health
            )
603 604 605
            device_dict["job"] = None
            device_dict["offline_since"] = None
            device_dict["offline_by"] = None
Rémi Duraffort's avatar
Rémi Duraffort committed
606
            device_dict["is_pipeline"] = True
607

608 609 610
            current_job = device.current_job()
            if current_job is not None:
                device_dict["job"] = current_job.pk
611
        else:
Neil Williams's avatar
Neil Williams committed
612
            raise xmlrpc.client.Fault(
613 614 615 616
                403, "Permission denied for user to access %s information." % hostname
            )
        return device_dict

617 618 619 620 621 622 623 624 625 626 627 628 629 630 631 632 633 634 635 636 637 638 639 640 641 642 643 644 645
    def put_into_maintenance_mode(self, hostname, reason, notify=None):
        """
        Name
        ----
        `put_into_maintenance_mode` (`hostname`, `reason`, `notify`)

        Description
        -----------
        Put the given device in maintenance mode with the given reason and optionally
        notify the given mail address when the job has finished.

        Arguments
        ---------
        `hostname`: string
            Name of the device to put into maintenance mode.
        `reason`: string
            The reason given to justify putting the device into maintenance mode.
        `notify`: string
            Email address of the user to notify when the job has finished. Can be
            omitted.

        Return value
        ------------
        None. The user should be authenticated with a username and token and has
        sufficient permission.
        """

        self._authenticate()
        if not hostname:
646
            raise xmlrpc.client.Fault(400, "Bad request: Hostname was not specified.")
647
        if not reason:
648
            raise xmlrpc.client.Fault(400, "Bad request: Reason was not specified.")
649 650 651 652
        with transaction.atomic():
            try:
                device = Device.objects.select_for_update().get(hostname=hostname)
            except Device.DoesNotExist:
653
                raise xmlrpc.client.Fault(404, "Device '%s' was not found." % hostname)
654
            if device.can_change(self.user):
655 656 657
                device.health = Device.HEALTH_MAINTENANCE
                device.save()
            else:
Neil Williams's avatar
Neil Williams committed
658
                raise xmlrpc.client.Fault(
659 660 661
                    403,
                    "Permission denied for user to put %s into maintenance mode."
                    % hostname,
662
                )
663

664 665 666 667 668 669 670 671 672 673 674 675 676 677 678 679 680 681 682 683 684 685 686 687 688 689 690 691 692
    def put_into_online_mode(self, hostname, reason, skip_health_check=False):
        """
        Name
        ----
        `put_into_online_mode` (`hostname`, `reason`, `skip_health_check`)

        Description
        -----------
        Put the given device into online mode with the given reason ans skip health
        check if asked.

        Arguments
        ---------
        `hostname`: string
            Name of the device to put into online mode.
        `reason`: string
            The reason given to justify putting the device into online mode.
        `skip_health_check`: boolean
            Skip health check when putting the board into online mode. If
            omitted, health check is not skipped by default.

        Return value
        ------------
        None. The user should be authenticated with a username and token and has
        sufficient permission.
        """

        self._authenticate()
        if not hostname:
693
            raise xmlrpc.client.Fault(400, "Bad request: Hostname was not specified.")
694
        if not reason:
695
            raise xmlrpc.client.Fault(400, "Bad request: Reason was not specified.")
696 697 698 699
        with transaction.atomic():
            try:
                device = Device.objects.select_for_update().get(hostname=hostname)
            except Device.DoesNotExist:
700
                raise xmlrpc.client.Fault(404, "Device '%s' was not found." % hostname)
701
            if device.can_change(self.user):
702 703 704
                device.health = Device.HEALTH_UNKNOWN
                device.save()
            else:
Neil Williams's avatar
Neil Williams committed
705
                raise xmlrpc.client.Fault(
706 707
                    403,
                    "Permission denied for user to put %s into online mode." % hostname,
708
                )
709

710
    def pending_jobs_by_device_type(self, all=False):
711 712 713
        """
        Name
        ----
714
        `pending_jobs_by_device_type` ()
715 716 717

        Description
        -----------
718
        Get number of pending jobs in each device type.
719 720
        Private test jobs and hidden device types are
        excluded, except for authenticated superusers.
721 722 723

        Arguments
        ---------
724
        `all`: boolean - include retired devices and undisplayed device-types in the listing.
725 726 727

        Return value
        ------------
728 729
        This function returns a dict where the key is the device type and
        the value is the number of jobs pending in that device type.
730 731
        For example:

732
        {'qemu': 0, 'panda': 3}
733 734
        """

735
        pending_jobs_by_device = {}
736

Stevan Radakovic's avatar
Stevan Radakovic committed
737 738 739
        jobs_res = TestJob.objects.filter(
            state=TestJob.STATE_SUBMITTED
        ).visible_by_user(self.user)
740
        jobs_res = jobs_res.exclude(requested_device_type_id__isnull=True)
741 742
        jobs_res = jobs_res.values_list("requested_device_type_id")
        jobs_res = jobs_res.annotate(pending_jobs=(Count("id")))
743

744 745 746
        jobs = {}
        jobs_hash = dict(jobs_res)
        for job in jobs_hash:
747
            jobs[job] = jobs_hash[job]
748
        pending_jobs_by_device.update(jobs)
749

750
        # Get rest of the devices and put number of pending jobs as 0.
751
        if all:
752
            device_types = DeviceType.objects.all()
753
        else:
754
            device_types = active_device_types()
755

Stevan Radakovic's avatar
Stevan Radakovic committed
756
        device_types = device_types.visible_by_user(self.user)
757
        for device_type in device_types.values_list("name", flat=True):
758 759
            if device_type not in pending_jobs_by_device:
                pending_jobs_by_device[device_type] = 0
Neil Williams's avatar
Neil Williams committed
760

761
        return pending_jobs_by_device
762 763 764 765 766 767 768 769 770 771 772 773 774 775 776 777 778 779 780 781

    def job_details(self, job_id):
        """
        Name
        ----
        `job_details` (`job_id`)

        Description
        -----------
        Get the details of given job id.

        Arguments
        ---------
        `job_id`: string
            Job id for which the output is required.

        Return value
        ------------
        This function returns an XML-RPC structures of job details, provided
        the user is authenticated with an username and token.
782 783

        The elements available in XML-RPC structure include:
Rémi Duraffort's avatar
Rémi Duraffort committed
784
        _state, submitter_id, is_pipeline, id, failure_comment,
Stevan Radakovic's avatar
Stevan Radakovic committed
785
        multinode_definition, priority, _actual_device_cache,
786
        original_definition, status, health_check, description,
Stevan Radakovic's avatar
Stevan Radakovic committed
787 788 789
        start_time, target_group, pipeline_compatibility, submit_time,
        is_public, _old_status, actual_device_id, definition, sub_id,
        requested_device_type_id, end_time, absolute_url, submitter_username
790
        """
791
        self._authenticate()
792
        if not job_id:
793
            raise xmlrpc.client.Fault(400, "Bad request: TestJob id was not specified.")
794
        try:
795
            job = get_restricted_job(self.user, job_id)
Rémi Duraffort's avatar
Rémi Duraffort committed
796
            job.status = job.get_legacy_status_display()
797 798
            job.state = job.get_state_display()
            job.health = job.get_health_display()
799
            job.submitter_username = job.submitter.username
800
            job.absolute_url = job.get_absolute_url()
Rémi Duraffort's avatar
Rémi Duraffort committed
801
            job.is_pipeline = True
802
        except PermissionDenied:
Neil Williams's avatar
Neil Williams committed
803
            raise xmlrpc.client.Fault(
804 805
                401, "Permission denied for user to job %s" % job_id
            )
806
        except TestJob.DoesNotExist:
Neil Williams's avatar
Neil Williams committed
807
            raise xmlrpc.client.Fault(404, "Specified job not found.")
808 809

        return job
810

811 812 813 814 815 816 817 818 819 820 821 822 823 824 825 826 827 828 829 830 831 832 833 834 835 836
    def job_health(self, job_id):
        """
        Name
        ----
        `job_health` (`job_id`)

        Description
        -----------
        Get the health of given job id.

        Arguments
        ---------
        `job_id`: string
            Job id for which the output is required.

        Return value
        ------------
        This function returns an XML-RPC structures of job health with the
        following fields.
        The user is authenticated with an username and token.

        `job_health`: string
        ['Unknown'|'Complete'|'Incomplete'|'Canceled']
        """
        self._authenticate()
        if not job_id:
837
            raise xmlrpc.client.Fault(400, "Bad request: TestJob id was not specified.")
838 839 840
        try:
            job = get_restricted_job(self.user, job_id)
        except PermissionDenied:
Neil Williams's avatar
Neil Williams committed
841
            raise xmlrpc.client.Fault(
842 843
                401, "Permission denied for user to job %s" % job_id
            )
844
        except TestJob.DoesNotExist:
Neil Williams's avatar
Neil Williams committed
845
            raise xmlrpc.client.Fault(404, "Specified job not found.")
846

847
        job_health = {"job_id": job.id, "job_health": job.get_health_display()}
848 849

        if job.is_multinode:
850
            job_health.update({"sub_id": job.sub_id})
851 852 853 854 855 856 857 858 859 860 861 862 863 864 865 866 867 868 869 870 871 872 873 874 875 876 877 878 879

        return job_health

    def job_state(self, job_id):
        """
        Name
        ----
        `job_state` (`job_id`)

        Description
        -----------
        Get the state of given job id.

        Arguments
        ---------
        `job_id`: string
            Job id for which the output is required.

        Return value
        ------------
        This function returns an XML-RPC structures of job state with the
        following fields.
        The user is authenticated with an username and token.

        `job_state`: string
        ['Submitted'|'Scheduling'|'Scheduled'|'Running'|'Canceling'|'Finished']
        """
        self._authenticate()
        if not job_id:
880
            raise xmlrpc.client.Fault(400, "Bad request: TestJob id was not specified.")
881 882 883
        try:
            job = get_restricted_job(self.user, job_id)
        except PermissionDenied:
Neil Williams's avatar
Neil Williams committed
884
            raise xmlrpc.client.Fault(
885 886
                401, "Permission denied for user to job %s" % job_id
            )
887
        except TestJob.DoesNotExist:
Neil Williams's avatar
Neil Williams committed
888
            raise xmlrpc.client.Fault(404, "Specified job not found.")
889

890
        job_state = {"job_id": job.id, "job_state": job.get_state_display()}
891 892

        if job.is_multinode:
893
            job_state.update({"sub_id": job.sub_id})
894 895 896

        return job_state

897 898 899 900 901 902 903 904 905 906 907 908 909 910 911 912 913 914 915 916 917
    def all_jobs(self):
        """
        Name
        ----
        `all_jobs` ()

        Description
        -----------
        Get submitted or running jobs.

        Arguments
        ---------
        None

        Return value
        ------------
        This function returns a XML-RPC array of submitted and running jobs with their status and
        actual device for running jobs and requested device or device type for submitted jobs and
        job sub_id for multinode jobs.
        For example:

918 919 920
        [[73, 'multinode-job', 'submitted', None, 'kvm', '72.1'],
        [72, 'multinode-job', 'submitted', None, 'kvm', '72.0'],
        [71, 'test-job', 'running', 'kvm01', None, None]]
921 922
        """

Stevan Radakovic's avatar
Stevan Radakovic committed
923 924 925 926 927
        jobs = (
            TestJob.objects.exclude(state=TestJob.STATE_FINISHED)
            .visible_by_user(self.user)
            .order_by("-id")
        )
928 929 930 931 932 933 934 935 936 937 938
        jobs_list = [
            [
                job.id,
                job.description,
                job.get_legacy_status_display().lower(),
                job.actual_device,
                job.requested_device_type,
                job.sub_id,
            ]
            for job in jobs
        ]
939 940

        return jobs_list
941

942 943 944 945 946 947 948 949 950 951 952 953 954 955 956 957 958 959 960 961 962 963
    def get_device_config(self, device_hostname, context=None):
        """
        New in api_version 2 - see system.api_version()

        Name
        ----
        `get_device_config` (`device_hostname`, context=None)

        Description
        -----------
        Get the device configuration for given device hostname.

        Arguments
        ---------
        `device_hostname`: string
            Device hostname for which the configuration is required.

        Some device templates need a context specified when processing the
        device-type template. This can be specified as a YAML string:

        `get_device_config` `('qemu01', '{arch: amd64}')`

964 965 966 967 968
        Return value
        ------------
        This function returns an XML-RPC binary data of output file.
        """
        if not device_hostname:
Neil Williams's avatar
Neil Williams committed
969
            raise xmlrpc.client.Fault(
970 971
                400, "Bad request: Device hostname was not specified."
            )
972

973 974 975
        job_ctx = None
        if context is not None:
            try:
976
                job_ctx = yaml_safe_load(context)
977
            except yaml.YAMLError as exc:
Neil Williams's avatar
Neil Williams committed
978
                raise xmlrpc.client.Fault(
979 980
                    400, "Job context '%s' is not valid. %s" % (context, exc)
                )
981 982 983
        try:
            device = Device.objects.get(hostname=device_hostname)
        except Device.DoesNotExist:
Neil Williams's avatar
Neil Williams committed
984
            raise xmlrpc.client.Fault(404, "Specified device was not found.")
985

Stevan Radakovic's avatar
Stevan Radakovic committed
986 987 988 989 990
        if not device.can_view(self.user):
            raise xmlrpc.client.Fault(
                401, "Permission denied for user to device %s" % device.hostname
            )

991
        config = device.load_configuration(job_ctx=job_ctx, output_format="yaml")
992 993

        # validate against the device schema
994
        validate_device(yaml_safe_load(config))
995

996
        return xmlrpc.client.Binary(config.encode("UTF-8"))
997 998 999 1000 1001 1002 1003 1004 1005

    def import_device_dictionary(self, hostname, jinja_str):
        """
        Name
        ----
        `import_device_dictionary` (`device_hostname`, `jinja_string`)

        Description
        -----------
1006
        [user with change_device permission only]
1007 1008 1009 1010 1011 1012 1013 1014
        Import or update the device dictionary key value store for a
        pipeline device.

        Arguments
        ---------
        `device_hostname`: string
            Device hostname to update.
        `jinja_str`: string
1015
            Device configuration as Jinja2
1016 1017 1018 1019 1020 1021 1022

        Return value
        ------------
        This function returns an XML-RPC binary data of output file.
        """
        self._authenticate()
        try:
1023
            device = Device.objects.get(hostname=hostname)
1024
        except DeviceType.DoesNotExist:
1025
            raise xmlrpc.client.Fault(404, "Device '%s' was not found." % hostname)
1026
        if device.can_change(self.user):
Stevan Radakovic's avatar
Stevan Radakovic committed
1027 1028 1029 1030 1031
            if not device.save_configuration(jinja_str):
                raise xmlrpc.client.Fault(
                    400, "Unable to store the configuration for %s on disk" % hostname
                )
        else:
Neil Williams's avatar
Neil Williams committed
1032
            raise xmlrpc.client.Fault(