__init__.py 37.9 KB
Newer Older
Rémi Duraffort's avatar
Rémi Duraffort committed
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
# -*- coding: utf-8 -*-
# Copyright (C) 2017-2018 Linaro Limited
#
# Author: Neil Williams <neil.williams@linaro.org>
#         Senthil Kumaran <senthil.kumaran@linaro.org>
#
# This file is part of LAVA.
#
# LAVA is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License version 3
# as published by the Free Software Foundation
#
# LAVA is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with LAVA.  If not, see <http://www.gnu.org/licenses/>.

21
22
from functools import wraps
from simplejson import JSONDecodeError
23
import os
24
import sys
Neil Williams's avatar
Neil Williams committed
25
import xmlrpc.client
26
27
import yaml

28
from django.conf import settings
29
from django.core.exceptions import PermissionDenied
30
from django.db.models import Count, Q
31
from django.db import transaction
32
from linaro_django_xmlrpc.models import ExposedV2API
33
34
35

from lava_common.compat import yaml_safe_dump, yaml_safe_load

36
37
38
from lava_scheduler_app.models import (
    Device,
    DeviceType,
39
    JSONDataError,
40
    DevicesUnavailableException,
41
    TestJob,
Neil Williams's avatar
Neil Williams committed
42
)
43
from lava_scheduler_app.views import get_restricted_job
44
45
from lava_scheduler_app.dbutils import (
    device_type_summary,
46
47
    testjob_submission,
    active_device_types,
48
)
49
50
51
52
53
54
55
from lava_scheduler_app.schema import (
    validate_submission,
    validate_device,
    SubmissionException,
)

# functions need to be members to be exposed in the API
56

Michael-Doyle Hudson's avatar
Michael-Doyle Hudson committed
57

Neil Williams's avatar
Neil Williams committed
58
59
# to make a function visible in the API, it must be a member of SchedulerAPI

Michael-Doyle Hudson's avatar
Michael-Doyle Hudson committed
60

61
62
def check_perm(perm):
    """ decorator to check that the caller has the given permission """
63

64
65
66
67
68
69
70
    def decorator(f):
        @wraps(f)
        def wrapper(self, *args, **kwargs):
            self._authenticate()
            if not self.user.has_perm(perm):
                raise xmlrpc.client.Fault(
                    403,
71
                    "User '%s' is missing permission %s ." % (self.user.username, perm),
72
73
                )
            return f(self, *args, **kwargs)
74

75
        return wrapper
76

77
    return decorator
78
79


80
81
82
def build_device_status_display(state, health):
    if state == Device.STATE_IDLE:
        if health in [Device.HEALTH_GOOD, Device.HEALTH_UNKNOWN]:
83
            return "idle"
84
        elif health == Device.HEALTH_RETIRED:
85
            return "retired"
86
        else:
87
            return "offline"
88
    elif state == Device.STATE_RESERVED:
89
        return "reserved"
90
    else:
91
        return "running"
92
93


94
class SchedulerAPI(ExposedV2API):
95
    def submit_job(self, job_data):
96
97
98
99
100
101
102
        """
        Name
        ----
        `submit_job` (`job_data`)

        Description
        -----------
103
104
        Submit the given job data which is in LAVA job JSON or YAML format as a
        new job to LAVA scheduler.
105
106
107
108

        Arguments
        ---------
        `job_data`: string
109
            Job JSON or YAML string.
110
111
112
113

        Return value
        ------------
        This function returns an XML-RPC integer which is the newly created
114
        job's id, provided the user is authenticated with an username and token.
115
116
        If the job is a multinode job, this function returns the list of created
        job IDs.
117
        """
118
        self._authenticate()
Neil Williams's avatar
Neil Williams committed
119
        try:
120
121
            job = testjob_submission(job_data, self.user)
        except SubmissionException as exc:
Neil Williams's avatar
Neil Williams committed
122
            raise xmlrpc.client.Fault(400, "Problem with submitted job data: %s" % exc)
Rémi Duraffort's avatar
Rémi Duraffort committed
123
        # FIXME: json error is not needed anymore
Neil Williams's avatar
Neil Williams committed
124
        except (JSONDataError, JSONDecodeError, ValueError) as exc:
Neil Williams's avatar
Neil Williams committed
125
            raise xmlrpc.client.Fault(400, "Decoding job submission failed: %s." % exc)
126
        except (Device.DoesNotExist, DeviceType.DoesNotExist):
Neil Williams's avatar
Neil Williams committed
127
            raise xmlrpc.client.Fault(404, "Specified device or device type not found.")
128
        except DevicesUnavailableException as exc:
Neil Williams's avatar
Neil Williams committed
129
            raise xmlrpc.client.Fault(400, "Device unavailable: %s" % str(exc))
130
        if isinstance(job, list):
131
            return [j.sub_id for j in job]
132
133
        else:
            return job.id
134

135
    def resubmit_job(self, job_id):
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
        """
        Name
        ----
        `resubmit_job` (`job_id`)

        Description
        -----------
        Resubmit the given job reffered by its id.

        Arguments
        ---------
        `job_id`: string
            The job's id which should be re-submitted.

        Return value
        ------------
        This function returns an XML-RPC integer which is the newly created
        job's id,  provided the user is authenticated with an username and
        token.
        """
156
        self._authenticate()
Paul Larson's avatar
Paul Larson committed
157
        try:
158
            job = get_restricted_job(self.user, job_id)
Paul Larson's avatar
Paul Larson committed
159
        except TestJob.DoesNotExist:
Neil Williams's avatar
Neil Williams committed
160
            raise xmlrpc.client.Fault(404, "Specified job not found.")
161

162
        if job.is_multinode:
163
164
165
            return self.submit_job(job.multinode_definition)
        else:
            return self.submit_job(job.definition)
166

167
    def cancel_job(self, job_id):
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
        """
        Name
        ----
        `cancel_job` (`job_id`)

        Description
        -----------
        Cancel the given job reffered by its id.

        Arguments
        ---------
        `job_id`: string
            Job id which should be canceled.

        Return value
        ------------
        None. The user should be authenticated with an username and token.
        """
186
187
        self._authenticate()
        if not job_id:
188
            raise xmlrpc.client.Fault(400, "Bad request: TestJob id was not specified.")
189

190
191
192
193
        with transaction.atomic():
            try:
                job = get_restricted_job(self.user, job_id, for_update=True)
            except PermissionDenied:
Neil Williams's avatar
Neil Williams committed
194
                raise xmlrpc.client.Fault(
195
196
                    401, "Permission denied for user to job %s" % job_id
                )
197
            except TestJob.DoesNotExist:
Neil Williams's avatar
Neil Williams committed
198
                raise xmlrpc.client.Fault(404, "Specified job not found.")
199

200
201
202
            try:
                job.cancel(self.user)
            except PermissionDenied:
Neil Williams's avatar
Neil Williams committed
203
                raise xmlrpc.client.Fault(403, "Permission denied.")
204
        return True
205

206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
    def validate_yaml(self, yaml_string):
        """
        Name
        ----
        validate_yaml (yaml_job_data)

        Description
        -----------
        Validate the supplied pipeline YAML against the submission schema.

        Note: this does not validate the job itself, just the YAML against the
        submission schema. A job which validates against the schema can still be
        an invalid job for the dispatcher and such jobs will be accepted as Submitted,
        scheduled and then marked as Incomplete with a failure comment. Full validation
        only happens after a device has been assigned to the Submitted job.

        Arguments
        ---------
        'yaml_job_data': string

        Return value
        ------------
        Raises an Exception if the yaml_job_data is invalid.
        """
        try:
            # YAML can parse JSON as YAML, JSON cannot parse YAML at all
232
            yaml_data = yaml_safe_load(yaml_string)
233
        except yaml.YAMLError as exc:
Neil Williams's avatar
Neil Williams committed
234
            raise xmlrpc.client.Fault(400, "Decoding job submission failed: %s." % exc)
235
236
237
238
        try:
            # validate against the submission schema.
            validate_submission(yaml_data)  # raises SubmissionException if invalid.
        except SubmissionException as exc:
Neil Williams's avatar
Neil Williams committed
239
            raise xmlrpc.client.Fault(400, "Invalid YAML submission: %s" % exc)
240

241
    def job_output(self, job_id, offset=0):
242
243
244
        """
        Name
        ----
245
        `job_output` (`job_id`, `offset=0`)
246
247
248
249
250
251
252
253
254

        Description
        -----------
        Get the output of given job id.

        Arguments
        ---------
        `job_id`: string
            Job id for which the output is required.
255
256
257
        `offset`: integer
            Offset from which to start reading the output file specified in bytes.
            It defaults to 0.
258
259
260

        Return value
        ------------
261
262
        This function returns an XML-RPC binary data of output file, provided
        the user is authenticated with an username and token.
263
        """
264
265
        self._authenticate()
        if not job_id:
266
            raise xmlrpc.client.Fault(400, "Bad request: TestJob id was not specified.")
267
        try:
268
            job = get_restricted_job(self.user, job_id)
269
        except PermissionDenied:
Neil Williams's avatar
Neil Williams committed
270
            raise xmlrpc.client.Fault(
271
272
                401, "Permission denied for user to job %s" % job_id
            )
273
        except TestJob.DoesNotExist:
Neil Williams's avatar
Neil Williams committed
274
            raise xmlrpc.client.Fault(404, "Specified job not found.")
275

276
277
278
279
280
281
282
283
        # Open the logs
        output_path = os.path.join(job.output_dir, "output.yaml")
        try:
            with open(output_path, encoding="utf-8", errors="replace") as f_logs:
                if f_logs:
                    f_logs.seek(offset)
                return xmlrpc.client.Binary(f_logs.read().encode("UTF-8"))
        except OSError:
Neil Williams's avatar
Neil Williams committed
284
            raise xmlrpc.client.Fault(404, "Job output not found.")
285
286
287
288
289
290
291
292
293

    def all_devices(self):
        """
        Name
        ----
        `all_devices` ()

        Description
        -----------
294
        Get all the available devices with their state and type information.
295
296
297
298
299
300
301
302

        Arguments
        ---------
        None

        Return value
        ------------
        This function returns an XML-RPC array in which each item is a list of
303
304
        device hostname, device type, device state, current running job id and
        if device is pipeline. For example:
305

306
        [['panda01', 'panda', 'running', 'good', 164, False], ['qemu01', 'qemu', 'idle', 'unknwon', None, True]]
307
308
        """

Stevan Radakovic's avatar
Stevan Radakovic committed
309
310
311
        devices_list = Device.objects.visible_by_user(self.user).exclude(
            health=Device.HEALTH_RETIRED
        )
312

313
314
315
316
317
318
319
320
321
322
        return [
            [
                dev.hostname,
                dev.device_type.name,
                build_device_status_display(dev.state, dev.health),
                dev.current_job().pk if dev.current_job() else None,
                True,
            ]
            for dev in devices_list
        ]
323
324
325
326
327
328
329
330
331
332

    def all_device_types(self):
        """
        Name
        ----
        `all_device_types` ()

        Description
        -----------
        Get all the available device types with their state and count
333
        information.
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348

        Arguments
        ---------
        None

        Return value
        ------------
        This function returns an XML-RPC array in which each item is a dict
        which contains name (device type), idle, busy, offline counts.
        For example:

        [{'idle': 1, 'busy': 0, 'name': 'panda', 'offline': 0},
        {'idle': 1, 'busy': 0, 'name': 'qemu', 'offline': 0}]
        """

349
        all_device_types = []
350
        keys = ["busy", "idle", "offline"]
351

Stevan Radakovic's avatar
Stevan Radakovic committed
352
        device_types = device_type_summary(self.user)
353
354

        for dev_type in device_types:
355
            device_type = {"name": dev_type["device_type"]}
356
            for key in keys:
Neil Williams's avatar
Neil Williams committed
357
                device_type[key] = dev_type[key]
358
            all_device_types.append(device_type)
359

360
        return all_device_types
361

362
363
364
    def get_recent_jobs_for_device_type(
        self, device_type, count=1, restrict_to_user=False
    ):
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
        """
        Name
        ----

        `get_recent_jobs_for_device_type` (`device_type`, `count=1`, `restrict_to_user=False`)

        Description
        -----------
        Get details of recently finished jobs for a given device_type. Limits the list
        to test jobs submitted by the user making the query if restrict_to_user is set to
        True. Get only the most recent job by default, but count can be set higher to
        get for example the last 10 jobs.

        Arguments
        ---------
        `device_type`: string
            Name of the device_type for which you want the jobs
        `count`: integer (Optional, default=1)
            Number of last jobs you want
        `restrict_to_user`: boolean (Optional, default=False)
            Fetch only the jobs submitted by the user making the query if set to True

        Return value
        ------------
        This function returns a list of dictionaries, which correspond to the
        list of recently finished jobs informations (Complete or Incomplete)
        for this device, ordered from youngest to oldest.

        [
            {
                'description': 'ramdisk health check',
                'id': 359828,
                'status': 'Complete',
                'device': 'black01'
            },
            {
                'description': 'standard ARMMP NFS',
                'id': 359827
                'status': 'Incomplete',
                'device': 'black02'
            }
        ]
        """
        if not device_type:
Neil Williams's avatar
Neil Williams committed
409
            raise xmlrpc.client.Fault(
410
411
412
                400, "Bad request: device_type was not specified."
            )
        if count < 0:
413
            raise xmlrpc.client.Fault(400, "Bad request: count must not be negative.")
414
415
416
        try:
            dt = DeviceType.objects.get(name=device_type, display=True)
        except Device.DoesNotExist:
Neil Williams's avatar
Neil Williams committed
417
            raise xmlrpc.client.Fault(
418
419
420
                404, "DeviceType '%s' was not found." % device_type
            )

421
422
423
        job_qs = (
            TestJob.objects.filter(state=TestJob.STATE_FINISHED)
            .filter(requested_device_type=dt)
Stevan Radakovic's avatar
Stevan Radakovic committed
424
            .visible_by_user(self.user)
425
426
            .order_by("-id")
        )
427
428
429
430
        if restrict_to_user:
            job_qs = job_qs.filter(submitter=self.user)
        job_list = []
        for job in job_qs.all()[:count]:
431
            hostname = ""
432
433
            if job.actual_device:
                hostname = job.actual_device.hostname
434
435
436
            job_dict = {
                "id": job.id,
                "description": job.description,
Rémi Duraffort's avatar
Rémi Duraffort committed
437
                "status": job.get_legacy_status_display(),
438
                "device": hostname,
439
440
441
442
            }
            job_list.append(job_dict)
        return job_list

443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
    def get_recent_jobs_for_device(self, device, count=1, restrict_to_user=False):
        """
        Name
        ----

        `get_recent_jobs_for_device` (`device`, `count=1`, `restrict_to_user=False`)

        Description
        -----------
        Get details of recently finished jobs for a given device. Limits the list
        to test jobs submitted by the user making the query if restrict_to_user is set to
        True. Get only the most recent job by default, but count can be set higher to
        get for example the last 10 jobs.

        Arguments
        ---------
        `device`: string
            Name of the device for which you want the jobs
        `count`: integer (Optional, default=1)
            Number of last jobs you want
        `restrict_to_user`: boolean (Optional, default=False)
            Fetch only the jobs submitted by the user making the query if set to True

        Return value
        ------------
        This function returns a list of dictionaries, which correspond to the
        list of recently finished jobs informations (Complete or Incomplete)
        for this device, ordered from youngest to oldest.

        [
            {
                'description': 'mainline--armada-370-db--multi_v7_defconfig--network',
                'id': 359828,
                'status': 'Complete'
            },
            {
                'description': 'mainline--armada-370-db--multi_v7_defconfig--sata',
                'id': 359827
                'status': 'Incomplete'
            }
        ]
        """
        if not device:
486
            raise xmlrpc.client.Fault(400, "Bad request: device was not specified.")
487
        if count < 0:
488
            raise xmlrpc.client.Fault(400, "Bad request: count must not be negative.")
489
490
491
        try:
            device_obj = Device.objects.get(hostname=device)
        except Device.DoesNotExist:
492
            raise xmlrpc.client.Fault(404, "Device '%s' was not found." % device)
493

Stevan Radakovic's avatar
Stevan Radakovic committed
494
        if not device_obj.can_view(self.user):
Neil Williams's avatar
Neil Williams committed
495
            raise xmlrpc.client.Fault(
496
                403, "Device '%s' not available to user '%s'." % (device, self.user)
497
            )
498
499
500
        job_qs = (
            TestJob.objects.filter(state=TestJob.STATE_FINISHED)
            .filter(actual_device=device_obj)
Stevan Radakovic's avatar
Stevan Radakovic committed
501
            .visible_by_user(self.user)
502
503
            .order_by("-id")
        )
504
505
506
507
508
509
510
        if restrict_to_user:
            job_qs = job_qs.filter(submitter=self.user)
        job_list = []
        for job in job_qs.all()[:count]:
            job_dict = {
                "id": job.id,
                "description": job.description,
Rémi Duraffort's avatar
Rémi Duraffort committed
511
                "status": job.get_legacy_status_display(),
512
513
514
515
            }
            job_list.append(job_dict)
        return job_list

516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
    def get_device_type_by_alias(self, alias):
        """
        Name
        ----

        `get_device_type_by_alias` (`alias`)

        Description
        -----------
        Get the matching device-type(s) for the specified alias. It is
        possible that more than one device-type can be returned, depending
        on local admin configuration. An alias can be used to provide the
        link between the device-type name and the Device Tree name.
        It is possible for multiple device-types to have the same alias
        (to assist in transitions and migrations).
        The specified alias string can be a partial match, returning all
        device-types which have an alias name containing the requested
        string.

        Arguments
        ---------
        `alias`: string
            Name of the alias to lookup

        Return value
        ------------
        This function returns a dictionary containing the alias as the key
        and a list of device-types which use that alias as the value. If the
        specified alias does not match any device-type, the dictionary contains
        an empty list for the alias key.

        {'apq8016-sbc': ['dragonboard410c']}
        {'ompa4-panda': ['panda', 'panda-es']}
        """

Stevan Radakovic's avatar
Stevan Radakovic committed
551
552
553
        aliases = DeviceType.objects.filter(
            aliases__name__contains=alias
        ).visible_by_user(self.user)
554
        return {alias: [device_type.name for device_type in aliases]}
555

556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
    def get_device_status(self, hostname):
        """
        Name
        ----
        `get_device_status` (`hostname`)

        Description
        -----------
        Get status, running job, date from which it is offline of the given
        device and the user who put it offline.

        Arguments
        ---------
        `hostname`: string
            Name of the device for which the status is asked.

        Return value
        ------------
        This function returns an XML-RPC dictionary which contains hostname,
        status, date from which the device is offline if the device is offline,
        the user who put the device offline if the device is offline and the
        job id of the running job.
        The device has to be visible to the user who requested device's status.

        Note that offline_since and offline_by can be empty strings if the device
        status is manually changed by an administrator in the database or from
        the admin site of LAVA even if device's status is offline.
        """

        if not hostname:
586
            raise xmlrpc.client.Fault(400, "Bad request: Hostname was not specified.")
587
588
589
        try:
            device = Device.objects.get(hostname=hostname)
        except Device.DoesNotExist:
590
            raise xmlrpc.client.Fault(404, "Device '%s' was not found." % hostname)
591
592

        device_dict = {}
Stevan Radakovic's avatar
Stevan Radakovic committed
593
        if device.can_view(self.user):
594
            device_dict["hostname"] = device.hostname
595
596
597
            device_dict["status"] = build_device_status_display(
                device.state, device.health
            )
598
599
600
            device_dict["job"] = None
            device_dict["offline_since"] = None
            device_dict["offline_by"] = None
Rémi Duraffort's avatar
Rémi Duraffort committed
601
            device_dict["is_pipeline"] = True
602

603
604
605
            current_job = device.current_job()
            if current_job is not None:
                device_dict["job"] = current_job.pk
606
        else:
Neil Williams's avatar
Neil Williams committed
607
            raise xmlrpc.client.Fault(
608
609
610
611
                403, "Permission denied for user to access %s information." % hostname
            )
        return device_dict

612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
    def put_into_maintenance_mode(self, hostname, reason, notify=None):
        """
        Name
        ----
        `put_into_maintenance_mode` (`hostname`, `reason`, `notify`)

        Description
        -----------
        Put the given device in maintenance mode with the given reason and optionally
        notify the given mail address when the job has finished.

        Arguments
        ---------
        `hostname`: string
            Name of the device to put into maintenance mode.
        `reason`: string
            The reason given to justify putting the device into maintenance mode.
        `notify`: string
            Email address of the user to notify when the job has finished. Can be
            omitted.

        Return value
        ------------
        None. The user should be authenticated with a username and token and has
        sufficient permission.
        """

        self._authenticate()
        if not hostname:
641
            raise xmlrpc.client.Fault(400, "Bad request: Hostname was not specified.")
642
        if not reason:
643
            raise xmlrpc.client.Fault(400, "Bad request: Reason was not specified.")
644
645
646
647
        with transaction.atomic():
            try:
                device = Device.objects.select_for_update().get(hostname=hostname)
            except Device.DoesNotExist:
648
                raise xmlrpc.client.Fault(404, "Device '%s' was not found." % hostname)
649
            if device.can_change(self.user):
650
651
652
                device.health = Device.HEALTH_MAINTENANCE
                device.save()
            else:
Neil Williams's avatar
Neil Williams committed
653
                raise xmlrpc.client.Fault(
654
655
656
                    403,
                    "Permission denied for user to put %s into maintenance mode."
                    % hostname,
657
                )
658

659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
    def put_into_online_mode(self, hostname, reason, skip_health_check=False):
        """
        Name
        ----
        `put_into_online_mode` (`hostname`, `reason`, `skip_health_check`)

        Description
        -----------
        Put the given device into online mode with the given reason ans skip health
        check if asked.

        Arguments
        ---------
        `hostname`: string
            Name of the device to put into online mode.
        `reason`: string
            The reason given to justify putting the device into online mode.
        `skip_health_check`: boolean
            Skip health check when putting the board into online mode. If
            omitted, health check is not skipped by default.

        Return value
        ------------
        None. The user should be authenticated with a username and token and has
        sufficient permission.
        """

        self._authenticate()
        if not hostname:
688
            raise xmlrpc.client.Fault(400, "Bad request: Hostname was not specified.")
689
        if not reason:
690
            raise xmlrpc.client.Fault(400, "Bad request: Reason was not specified.")
691
692
693
694
        with transaction.atomic():
            try:
                device = Device.objects.select_for_update().get(hostname=hostname)
            except Device.DoesNotExist:
695
                raise xmlrpc.client.Fault(404, "Device '%s' was not found." % hostname)
696
            if device.can_change(self.user):
697
698
699
                device.health = Device.HEALTH_UNKNOWN
                device.save()
            else:
Neil Williams's avatar
Neil Williams committed
700
                raise xmlrpc.client.Fault(
701
702
                    403,
                    "Permission denied for user to put %s into online mode." % hostname,
703
                )
704

705
    def pending_jobs_by_device_type(self, all=False):
706
707
708
        """
        Name
        ----
709
        `pending_jobs_by_device_type` ()
710
711
712

        Description
        -----------
713
        Get number of pending jobs in each device type.
714
715
        Private test jobs and hidden device types are
        excluded, except for authenticated superusers.
716
717
718

        Arguments
        ---------
719
        `all`: boolean - include retired devices and undisplayed device-types in the listing.
720
721
722

        Return value
        ------------
723
724
        This function returns a dict where the key is the device type and
        the value is the number of jobs pending in that device type.
725
726
        For example:

727
        {'qemu': 0, 'panda': 3}
728
729
        """

730
        pending_jobs_by_device = {}
731

Stevan Radakovic's avatar
Stevan Radakovic committed
732
733
734
        jobs_res = TestJob.objects.filter(
            state=TestJob.STATE_SUBMITTED
        ).visible_by_user(self.user)
735
        jobs_res = jobs_res.exclude(requested_device_type_id__isnull=True)
736
737
        jobs_res = jobs_res.values_list("requested_device_type_id")
        jobs_res = jobs_res.annotate(pending_jobs=(Count("id")))
738

739
740
741
        jobs = {}
        jobs_hash = dict(jobs_res)
        for job in jobs_hash:
742
            jobs[job] = jobs_hash[job]
743
        pending_jobs_by_device.update(jobs)
744

745
        # Get rest of the devices and put number of pending jobs as 0.
746
        if all:
747
            device_types = DeviceType.objects.all()
748
        else:
749
            device_types = active_device_types()
750

Stevan Radakovic's avatar
Stevan Radakovic committed
751
        device_types = device_types.visible_by_user(self.user)
752
        for device_type in device_types.values_list("name", flat=True):
753
754
            if device_type not in pending_jobs_by_device:
                pending_jobs_by_device[device_type] = 0
Neil Williams's avatar
Neil Williams committed
755

756
        return pending_jobs_by_device
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776

    def job_details(self, job_id):
        """
        Name
        ----
        `job_details` (`job_id`)

        Description
        -----------
        Get the details of given job id.

        Arguments
        ---------
        `job_id`: string
            Job id for which the output is required.

        Return value
        ------------
        This function returns an XML-RPC structures of job details, provided
        the user is authenticated with an username and token.
777
778

        The elements available in XML-RPC structure include:
Rémi Duraffort's avatar
Rémi Duraffort committed
779
        _state, submitter_id, is_pipeline, id, failure_comment,
Stevan Radakovic's avatar
Stevan Radakovic committed
780
        multinode_definition, priority, _actual_device_cache,
781
        original_definition, status, health_check, description,
Stevan Radakovic's avatar
Stevan Radakovic committed
782
783
784
        start_time, target_group, pipeline_compatibility, submit_time,
        is_public, _old_status, actual_device_id, definition, sub_id,
        requested_device_type_id, end_time, absolute_url, submitter_username
785
        """
786
        self._authenticate()
787
        if not job_id:
788
            raise xmlrpc.client.Fault(400, "Bad request: TestJob id was not specified.")
789
        try:
790
            job = get_restricted_job(self.user, job_id)
Rémi Duraffort's avatar
Rémi Duraffort committed
791
            job.status = job.get_legacy_status_display()
792
793
            job.state = job.get_state_display()
            job.health = job.get_health_display()
794
            job.submitter_username = job.submitter.username
795
            job.absolute_url = job.get_absolute_url()
Rémi Duraffort's avatar
Rémi Duraffort committed
796
            job.is_pipeline = True
797
        except PermissionDenied:
Neil Williams's avatar
Neil Williams committed
798
            raise xmlrpc.client.Fault(
799
800
                401, "Permission denied for user to job %s" % job_id
            )
801
        except TestJob.DoesNotExist:
Neil Williams's avatar
Neil Williams committed
802
            raise xmlrpc.client.Fault(404, "Specified job not found.")
803
804

        return job
805

806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
    def job_health(self, job_id):
        """
        Name
        ----
        `job_health` (`job_id`)

        Description
        -----------
        Get the health of given job id.

        Arguments
        ---------
        `job_id`: string
            Job id for which the output is required.

        Return value
        ------------
        This function returns an XML-RPC structures of job health with the
        following fields.
        The user is authenticated with an username and token.

        `job_health`: string
        ['Unknown'|'Complete'|'Incomplete'|'Canceled']
        """
        self._authenticate()
        if not job_id:
832
            raise xmlrpc.client.Fault(400, "Bad request: TestJob id was not specified.")
833
834
835
        try:
            job = get_restricted_job(self.user, job_id)
        except PermissionDenied:
Neil Williams's avatar
Neil Williams committed
836
            raise xmlrpc.client.Fault(
837
838
                401, "Permission denied for user to job %s" % job_id
            )
839
        except TestJob.DoesNotExist:
Neil Williams's avatar
Neil Williams committed
840
            raise xmlrpc.client.Fault(404, "Specified job not found.")
841

842
        job_health = {"job_id": job.id, "job_health": job.get_health_display()}
843
844

        if job.is_multinode:
845
            job_health.update({"sub_id": job.sub_id})
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874

        return job_health

    def job_state(self, job_id):
        """
        Name
        ----
        `job_state` (`job_id`)

        Description
        -----------
        Get the state of given job id.

        Arguments
        ---------
        `job_id`: string
            Job id for which the output is required.

        Return value
        ------------
        This function returns an XML-RPC structures of job state with the
        following fields.
        The user is authenticated with an username and token.

        `job_state`: string
        ['Submitted'|'Scheduling'|'Scheduled'|'Running'|'Canceling'|'Finished']
        """
        self._authenticate()
        if not job_id:
875
            raise xmlrpc.client.Fault(400, "Bad request: TestJob id was not specified.")
876
877
878
        try:
            job = get_restricted_job(self.user, job_id)
        except PermissionDenied:
Neil Williams's avatar
Neil Williams committed
879
            raise xmlrpc.client.Fault(
880
881
                401, "Permission denied for user to job %s" % job_id
            )
882
        except TestJob.DoesNotExist:
Neil Williams's avatar
Neil Williams committed
883
            raise xmlrpc.client.Fault(404, "Specified job not found.")
884

885
        job_state = {"job_id": job.id, "job_state": job.get_state_display()}
886
887

        if job.is_multinode:
888
            job_state.update({"sub_id": job.sub_id})
889
890
891

        return job_state

892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
    def all_jobs(self):
        """
        Name
        ----
        `all_jobs` ()

        Description
        -----------
        Get submitted or running jobs.

        Arguments
        ---------
        None

        Return value
        ------------
        This function returns a XML-RPC array of submitted and running jobs with their status and
        actual device for running jobs and requested device or device type for submitted jobs and
        job sub_id for multinode jobs.
        For example:

913
914
915
        [[73, 'multinode-job', 'submitted', None, 'kvm', '72.1'],
        [72, 'multinode-job', 'submitted', None, 'kvm', '72.0'],
        [71, 'test-job', 'running', 'kvm01', None, None]]
916
917
        """

Stevan Radakovic's avatar
Stevan Radakovic committed
918
919
920
921
922
        jobs = (
            TestJob.objects.exclude(state=TestJob.STATE_FINISHED)
            .visible_by_user(self.user)
            .order_by("-id")
        )
923
924
925
926
927
928
929
930
931
932
933
        jobs_list = [
            [
                job.id,
                job.description,
                job.get_legacy_status_display().lower(),
                job.actual_device,
                job.requested_device_type,
                job.sub_id,
            ]
            for job in jobs
        ]
934
935

        return jobs_list
936

937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
    def get_device_config(self, device_hostname, context=None):
        """
        New in api_version 2 - see system.api_version()

        Name
        ----
        `get_device_config` (`device_hostname`, context=None)

        Description
        -----------
        Get the device configuration for given device hostname.

        Arguments
        ---------
        `device_hostname`: string
            Device hostname for which the configuration is required.

        Some device templates need a context specified when processing the
        device-type template. This can be specified as a YAML string:

        `get_device_config` `('qemu01', '{arch: amd64}')`

959
960
961
962
963
        Return value
        ------------
        This function returns an XML-RPC binary data of output file.
        """
        if not device_hostname:
Neil Williams's avatar
Neil Williams committed
964
            raise xmlrpc.client.Fault(
965
966
                400, "Bad request: Device hostname was not specified."
            )
967

968
969
970
        job_ctx = None
        if context is not None:
            try:
971
                job_ctx = yaml_safe_load(context)
972
            except yaml.YAMLError as exc:
Neil Williams's avatar
Neil Williams committed
973
                raise xmlrpc.client.Fault(
974
975
                    400, "Job context '%s' is not valid. %s" % (context, exc)
                )
976
977
978
        try:
            device = Device.objects.get(hostname=device_hostname)
        except Device.DoesNotExist:
Neil Williams's avatar
Neil Williams committed
979
            raise xmlrpc.client.Fault(404, "Specified device was not found.")
980

Stevan Radakovic's avatar
Stevan Radakovic committed
981
982
983
984
985
        if not device.can_view(self.user):
            raise xmlrpc.client.Fault(
                401, "Permission denied for user to device %s" % device.hostname
            )

986
        config = device.load_configuration(job_ctx=job_ctx, output_format="yaml")
987
988

        # validate against the device schema
989
        validate_device(yaml_safe_load(config))
990

991
        return xmlrpc.client.Binary(config.encode("UTF-8"))
992
993
994
995
996
997
998
999
1000

    def import_device_dictionary(self, hostname, jinja_str):
        """
        Name
        ----
        `import_device_dictionary` (`device_hostname`, `jinja_string`)

        Description
        -----------
For faster browsing, not all history is shown. View entire blame