models.py 78.2 KB
Newer Older
1
# -*- coding: utf-8 -*-
Rémi Duraffort's avatar
Rémi Duraffort committed
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
# Copyright (C) 2018 Linaro Limited
#
# Author: Neil Williams <neil.williams@linaro.org>
#         Remi Duraffort <remi.duraffort@linaro.org>
#
# This file is part of LAVA.
#
# LAVA is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License version 3
# as published by the Free Software Foundation
#
# LAVA is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with LAVA.  If not, see <http://www.gnu.org/licenses/>.

21

22
import contextlib
23
import datetime
24
import jinja2
25
import logging
26
import os
27
import uuid
28
import gzip
29
import simplejson
30
import yaml
Rémi Duraffort's avatar
Rémi Duraffort committed
31

32
from django.db import transaction
33
from django.db.models import Q
Michael Hudson-Doyle's avatar
Michael Hudson-Doyle committed
34
from django.conf import settings
Stevan Radakovic's avatar
Stevan Radakovic committed
35
from django.contrib.auth.models import User, Group, Permission
36
from django.contrib.contenttypes.models import ContentType
37
from django.contrib.admin.models import LogEntry, ADDITION, CHANGE
38
from django.contrib.postgres.fields import ArrayField
39
from django.contrib.sites.models import Site
40
41
42
43
44
from django.core.exceptions import (
    ImproperlyConfigured,
    PermissionDenied,
    ValidationError,
)
45
from django.urls import reverse
46
from django.db import models
47
from django.utils import timezone
48
from django.utils.translation import ugettext_lazy as _
49
from django.utils.safestring import mark_safe
50

51
from lava_common.compat import yaml_dump, yaml_safe_load, yaml_safe_dump
Rémi Duraffort's avatar
Rémi Duraffort committed
52
from lava_common.decorators import nottest
53
from lava_results_app.utils import export_testcase
54
from lava_scheduler_app import utils
55
from lava_scheduler_app.logutils import read_logs
56
import lava_scheduler_app.environment as environment
Stevan Radakovic's avatar
Stevan Radakovic committed
57
58
59
60
61
62
from lava_scheduler_app.managers import (
    RestrictedDeviceTypeQuerySet,
    RestrictedDeviceQuerySet,
    RestrictedTestJobQuerySet,
    GroupObjectPermissionManager,
)
63
from lava_scheduler_app.schema import SubmissionException, validate_device
64
from lava_server.compat import add_permissions
65

66
import requests
67

68

69
70
71
72
class JSONDataError(ValueError):
    """Error raised when JSON is syntactically valid but ill-formed."""


73
74
75
76
class DevicesUnavailableException(UserWarning):
    """Error raised when required number of devices are unavailable."""


77
78
class ExtendedUser(models.Model):

Rémi Duraffort's avatar
Rémi Duraffort committed
79
    user = models.OneToOneField(User, on_delete=models.CASCADE)
80
81

    irc_handle = models.CharField(
82
        max_length=40, default=None, null=True, blank=True, verbose_name="IRC handle"
83
84
85
    )

    irc_server = models.CharField(
86
        max_length=40, default=None, null=True, blank=True, verbose_name="IRC server"
87
88
    )

89
90
91
92
93
94
95
96
    table_length = models.PositiveSmallIntegerField(
        verbose_name="Table length",
        help_text="leave empty for system default",
        default=None,
        null=True,
        blank=True,
    )

97
98
99
    def __str__(self):
        return "%s: %s@%s" % (self.user, self.irc_handle, self.irc_server)

100

Stevan Radakovic's avatar
Stevan Radakovic committed
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
class GroupObjectPermission(models.Model):

    objects = GroupObjectPermissionManager()

    class Meta:
        abstract = True

    permission = models.ForeignKey(Permission, on_delete=models.CASCADE)
    group = models.ForeignKey(Group, on_delete=models.CASCADE)

    def save(self, *args, **kwargs):
        super().full_clean()
        super().save(*args, **kwargs)

    @classmethod
    def ensure_users_group(cls, user):
        # Get or create a group that matches the users' username.
        # Then ensure that only this user is belonging to his group.
        group, _ = Group.objects.get_or_create(name=user.username)
        group.user_set.set({user}, clear=True)
        return group


124
125
126
127
class Tag(models.Model):

    name = models.SlugField(unique=True)

128
129
    description = models.TextField(null=True, blank=True)

130
    def __str__(self):
131
        return self.name.lower()
132
133


134
135
136
class Architecture(models.Model):
    name = models.CharField(
        primary_key=True,
137
138
        verbose_name="Architecture version",
        help_text="e.g. ARMv7",
139
        max_length=100,
140
        editable=True,
141
142
    )

143
    def __str__(self):
144
145
146
147
148
149
        return self.pk


class ProcessorFamily(models.Model):
    name = models.CharField(
        primary_key=True,
150
151
        verbose_name="Processor Family",
        help_text="e.g. OMAP4, Exynos",
152
        max_length=100,
153
        editable=True,
154
155
    )

156
    def __str__(self):
157
158
159
        return self.pk


160
class Alias(models.Model):
161
162
163
    class Meta:
        verbose_name_plural = "Aliases"

164
165
    name = models.CharField(
        primary_key=True,
166
167
        verbose_name="Alias for this device-type",
        help_text="e.g. the device tree name(s)",
168
        max_length=200,
169
        editable=True,
170
    )
Rémi Duraffort's avatar
Rémi Duraffort committed
171
172
173
    device_type = models.ForeignKey(
        "DeviceType", related_name="aliases", null=True, on_delete=models.CASCADE
    )
174

175
    def __str__(self):
176
177
        return self.pk

178
    def full_clean(self, exclude=None, validate_unique=True):
179
        if DeviceType.objects.filter(name=self.name).exists():
180
181
182
183
184
            raise ValidationError(
                "DeviceType with name '%s' already exists." % self.name
            )
        super().full_clean(exclude=exclude, validate_unique=validate_unique)

185

186
187
188
class BitWidth(models.Model):
    width = models.PositiveSmallIntegerField(
        primary_key=True,
189
190
        verbose_name="Processor bit width",
        help_text="integer: e.g. 32 or 64",
191
        editable=True,
192
193
    )

194
    def __str__(self):
195
196
197
198
199
200
        return "%d" % self.pk


class Core(models.Model):
    name = models.CharField(
        primary_key=True,
201
202
        verbose_name="CPU core",
        help_text="Name of a specific CPU core, e.g. Cortex-A9",
203
        editable=True,
204
205
206
        max_length=100,
    )

207
    def __str__(self):
208
209
210
        return self.pk


Stevan Radakovic's avatar
Stevan Radakovic committed
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
class RestrictedObject(models.Model):
    class Meta:
        abstract = True

    def is_permission_restricted(self, perm):
        app_label, codename = perm.split(".", 1)
        perm_count = self.permissions.filter(
            permission__content_type__app_label=app_label, permission__codename=codename
        ).count()
        return perm_count > 0

    def has_any_permission_restrictions(self, perm):
        raise NotImplementedError("Should implement this")


class DeviceType(RestrictedObject):
227
228
229
    """
    A class of device, for example a pandaboard or a snowball.
    """
230

Stevan Radakovic's avatar
Stevan Radakovic committed
231
    class Meta:
232
233
        permissions = add_permissions(
            (("view_devicetype", "Can view device type"),),
234
            (("submit_to_devicetype", "Can submit jobs to device type"),),
Stevan Radakovic's avatar
Stevan Radakovic committed
235
236
237
        )

    VIEW_PERMISSION = "lava_scheduler_app.view_devicetype"
238
    CHANGE_PERMISSION = "lava_scheduler_app.change_devicetype"
Stevan Radakovic's avatar
Stevan Radakovic committed
239
240
241
    SUBMIT_PERMISSION = "lava_scheduler_app.submit_to_devicetype"

    # Order of permission importance from most to least.
242
    PERMISSIONS_PRIORITY = [CHANGE_PERMISSION, SUBMIT_PERMISSION, VIEW_PERMISSION]
Stevan Radakovic's avatar
Stevan Radakovic committed
243
244
245

    objects = RestrictedDeviceTypeQuerySet.as_manager()

246
    name = models.SlugField(
247
248
        primary_key=True, editable=True
    )  # read-only after create via admin.py
249

250
251
    architecture = models.ForeignKey(
        Architecture,
252
        related_name="device_types",
253
254
        blank=True,
        null=True,
255
        on_delete=models.SET_NULL,
256
257
258
259
    )

    processor = models.ForeignKey(
        ProcessorFamily,
260
        related_name="device_types",
261
262
        blank=True,
        null=True,
263
        on_delete=models.SET_NULL,
264
265
266
    )

    cpu_model = models.CharField(
267
268
        verbose_name="CPU model",
        help_text="e.g. a list of CPU model descriptive strings: OMAP4430 / OMAP4460",
269
270
271
272
273
274
275
276
        max_length=100,
        blank=True,
        null=True,
        editable=True,
    )

    bits = models.ForeignKey(
        BitWidth,
277
        related_name="device_types",
278
279
        blank=True,
        null=True,
280
        on_delete=models.SET_NULL,
281
282
    )

283
    cores = models.ManyToManyField(Core, related_name="device_types", blank=True)
284
285

    core_count = models.PositiveSmallIntegerField(
286
287
        verbose_name="Total number of cores",
        help_text="Must be an equal number of each type(s) of core(s).",
288
289
290
291
        blank=True,
        null=True,
    )

292
    def __str__(self):
293
294
        return self.name

295
    def full_clean(self, exclude=None, validate_unique=True):
296
        if Alias.objects.filter(name=self.name).exists():
297
298
299
            raise ValidationError("Alias with name '%s' already exists." % self.name)
        super().full_clean(exclude=exclude, validate_unique=validate_unique)

300
    description = models.TextField(
301
        verbose_name=_("Device Type Description"),
302
303
304
        max_length=200,
        null=True,
        blank=True,
305
        default=None,
306
307
    )

308
    health_frequency = models.IntegerField(
309
        verbose_name="How often to run health checks", default=24
310
311
    )

312
    disable_health_check = models.BooleanField(
313
314
        default=False, verbose_name="Disable health check for devices of this type"
    )
315

316
317
    HEALTH_PER_HOUR = 0
    HEALTH_PER_JOB = 1
318
    HEALTH_DENOMINATOR = ((HEALTH_PER_HOUR, "hours"), (HEALTH_PER_JOB, "jobs"))
319
    HEALTH_DENOMINATOR_REVERSE = {"hours": HEALTH_PER_HOUR, "jobs": HEALTH_PER_JOB}
320
321
322
323
    health_denominator = models.IntegerField(
        choices=HEALTH_DENOMINATOR,
        default=HEALTH_PER_HOUR,
        verbose_name="Initiate health checks by hours or by jobs.",
324
325
326
327
328
329
        help_text=(
            "Choose to submit a health check every N hours "
            "or every N jobs. Balance against the duration of "
            "a health check job and the average job duration."
        ),
    )
330

331
332
333
334
335
336
337
338
    display = models.BooleanField(
        default=True,
        help_text=(
            "Should this be displayed in the GUI or not. This can be "
            "useful if you are removing all devices of this type but don't "
            "want to loose the test results generated by the devices."
        ),
    )
339

Spring Zhang's avatar
Spring Zhang committed
340
    def get_absolute_url(self):
Rémi Duraffort's avatar
Rémi Duraffort committed
341
        return reverse("lava.scheduler.device_type.detail", args=[self.pk])
Spring Zhang's avatar
Spring Zhang committed
342

Stevan Radakovic's avatar
Stevan Radakovic committed
343
344
345
346
347
348
    def can_view(self, user):
        if user.has_perm(self.VIEW_PERMISSION, self):
            return True
        if not self.is_permission_restricted(self.VIEW_PERMISSION):
            return True
        return False
349

350
351
    def can_change(self, user):
        return user.has_perm(self.CHANGE_PERMISSION, self)
352

Stevan Radakovic's avatar
Stevan Radakovic committed
353
354
    def has_any_permission_restrictions(self, perm):
        return self.is_permission_restricted(perm)
Neil Williams's avatar
Neil Williams committed
355

356

Senthil Kumaran S's avatar
Senthil Kumaran S committed
357
358
359
360
361
362
class Worker(models.Model):
    """
    A worker node to which devices are attached.
    """

    hostname = models.CharField(
363
        verbose_name=_("Hostname"),
Senthil Kumaran S's avatar
Senthil Kumaran S committed
364
365
        max_length=200,
        primary_key=True,
366
        default=None,
367
        editable=True,
Senthil Kumaran S's avatar
Senthil Kumaran S committed
368
369
    )

Rémi Duraffort's avatar
Rémi Duraffort committed
370
    STATE_ONLINE, STATE_OFFLINE = range(2)
371
    STATE_CHOICES = ((STATE_ONLINE, "Online"), (STATE_OFFLINE, "Offline"))
372
    STATE_REVERSE = {"Online": STATE_ONLINE, "Offline": STATE_OFFLINE}
373
374
    state = models.IntegerField(
        choices=STATE_CHOICES, default=STATE_OFFLINE, editable=False
Rémi Duraffort's avatar
Rémi Duraffort committed
375
376
377
378
379
380
381
382
    )

    HEALTH_ACTIVE, HEALTH_MAINTENANCE, HEALTH_RETIRED = range(3)
    HEALTH_CHOICES = (
        (HEALTH_ACTIVE, "Active"),
        (HEALTH_MAINTENANCE, "Maintenance"),
        (HEALTH_RETIRED, "Retired"),
    )
383
384
385
386
387
    HEALTH_REVERSE = {
        "Active": HEALTH_ACTIVE,
        "Maintenance": HEALTH_MAINTENANCE,
        "Retired": HEALTH_RETIRED,
    }
388
    health = models.IntegerField(choices=HEALTH_CHOICES, default=HEALTH_ACTIVE)
389

Senthil Kumaran S's avatar
Senthil Kumaran S committed
390
    description = models.TextField(
391
        verbose_name=_("Worker Description"),
Senthil Kumaran S's avatar
Senthil Kumaran S committed
392
393
394
        max_length=200,
        null=True,
        blank=True,
395
        default=None,
396
        editable=True,
Senthil Kumaran S's avatar
Senthil Kumaran S committed
397
398
    )

399
    last_ping = models.DateTimeField(verbose_name=_("Last ping"), default=timezone.now)
400

401
    job_limit = models.PositiveIntegerField(default=0)
402

403
    def __str__(self):
Senthil Kumaran S's avatar
Senthil Kumaran S committed
404
405
        return self.hostname

406
    def can_change(self, user):
407
        return user.has_perm("lava_scheduler_app.change_worker")
Senthil Kumaran S's avatar
Senthil Kumaran S committed
408

409
    def can_update(self, user):
410
        if user.has_perm("lava_scheduler_app.change_worker"):
411
412
413
414
415
416
            return True
        elif user.username == "lava-health":
            return True
        else:
            return False

Senthil Kumaran S's avatar
Senthil Kumaran S committed
417
    def get_absolute_url(self):
Rémi Duraffort's avatar
Rémi Duraffort committed
418
        return reverse("lava.scheduler.worker.detail", args=[self.pk])
Senthil Kumaran S's avatar
Senthil Kumaran S committed
419
420

    def get_description(self):
421
        return mark_safe(self.description) if self.description else None
Senthil Kumaran S's avatar
Senthil Kumaran S committed
422

423
    def retired_devices_count(self):
424
        return self.device_set.filter(health=Device.HEALTH_RETIRED).count()
425

426
427
    def go_health_active(self, user, reason=None):
        if reason:
428
429
430
            self.log_admin_entry(
                user, "%s → Active (%s)" % (self.get_health_display(), reason)
            )
431
432
        else:
            self.log_admin_entry(user, "%s → Active" % self.get_health_display())
433
434
435
        for device in self.device_set.all().select_for_update():
            device.worker_signal("go_health_active", user, self.state, self.health)
            device.save()
Rémi Duraffort's avatar
Rémi Duraffort committed
436
437
        self.health = Worker.HEALTH_ACTIVE

438
439
    def go_health_maintenance(self, user, reason=None):
        if reason:
440
441
442
            self.log_admin_entry(
                user, "%s → Maintenance (%s)" % (self.get_health_display(), reason)
            )
443
444
        else:
            self.log_admin_entry(user, "%s → Maintenance" % self.get_health_display())
445
446
447
        for device in self.device_set.all().select_for_update():
            device.worker_signal("go_health_maintenance", user, self.state, self.health)
            device.save()
Rémi Duraffort's avatar
Rémi Duraffort committed
448
449
        self.health = Worker.HEALTH_MAINTENANCE

450
451
    def go_health_retired(self, user, reason=None):
        if reason:
452
453
454
            self.log_admin_entry(
                user, "%s → Retired (%s)" % (self.get_health_display(), reason)
            )
455
456
        else:
            self.log_admin_entry(user, "%s → Retired" % self.get_health_display())
457
458
459
        for device in self.device_set.all().select_for_update():
            device.worker_signal("go_health_retired", user, self.state, self.health)
            device.save()
Rémi Duraffort's avatar
Rémi Duraffort committed
460
461
462
463
464
465
466
467
        self.health = Worker.HEALTH_RETIRED

    def go_state_offline(self):
        self.state = Worker.STATE_OFFLINE

    def go_state_online(self):
        self.state = Worker.STATE_ONLINE

468
469
470
    def log_admin_entry(self, user, reason, addition=False):
        if user is None:
            user = User.objects.get(username="lava-health")
471
472
473
474
475
476
        worker_ct = ContentType.objects.get_for_model(Worker)
        LogEntry.objects.log_action(
            user_id=user.id,
            content_type_id=worker_ct.pk,
            object_id=self.pk,
            object_repr=self.hostname,
477
            action_flag=ADDITION if addition else CHANGE,
478
            change_message=reason,
479
480
        )

Senthil Kumaran S's avatar
Senthil Kumaran S committed
481

Stevan Radakovic's avatar
Stevan Radakovic committed
482
class Device(RestrictedObject):
483
    """
484
    A device that we can run tests on.
485
    """
486

Stevan Radakovic's avatar
Stevan Radakovic committed
487
    class Meta:
488
489
        permissions = add_permissions(
            (("view_device", "Can view device"),),
490
            (("submit_to_device", "Can submit jobs to device"),),
Stevan Radakovic's avatar
Stevan Radakovic committed
491
492
493
        )

    VIEW_PERMISSION = "lava_scheduler_app.view_device"
494
    CHANGE_PERMISSION = "lava_scheduler_app.change_device"
Stevan Radakovic's avatar
Stevan Radakovic committed
495
496
497
498
499
    SUBMIT_PERMISSION = "lava_scheduler_app.submit_to_device"

    # This maps the corresponding permissions for 'parent' dependencies.
    DEVICE_TYPE_PERMISSION_MAP = {
        VIEW_PERMISSION: DeviceType.VIEW_PERMISSION,
500
        CHANGE_PERMISSION: DeviceType.CHANGE_PERMISSION,
Stevan Radakovic's avatar
Stevan Radakovic committed
501
502
503
504
        SUBMIT_PERMISSION: DeviceType.SUBMIT_PERMISSION,
    }

    # Order of permission importance from most to least.
505
    PERMISSIONS_PRIORITY = [CHANGE_PERMISSION, SUBMIT_PERMISSION, VIEW_PERMISSION]
Stevan Radakovic's avatar
Stevan Radakovic committed
506
507
508

    objects = RestrictedDeviceQuerySet.as_manager()

509
    hostname = models.CharField(
510
        verbose_name=_("Hostname"),
Neil Williams's avatar
Neil Williams committed
511
512
        max_length=200,
        primary_key=True,
513
        editable=True,  # read-only after create via admin.py
514
515
    )

Rémi Duraffort's avatar
Rémi Duraffort committed
516
517
518
    device_type = models.ForeignKey(
        DeviceType, verbose_name=_("Device type"), on_delete=models.CASCADE
    )
519

520
    device_version = models.CharField(
521
        verbose_name=_("Device Version"),
Neil Williams's avatar
Neil Williams committed
522
523
524
525
        max_length=200,
        null=True,
        default=None,
        blank=True,
526
527
    )

528
    physical_owner = models.ForeignKey(
529
530
        User,
        related_name="physicalowner",
531
532
533
        null=True,
        blank=True,
        default=None,
534
        verbose_name=_("User with physical access"),
535
        on_delete=models.SET_NULL,
536
537
538
    )

    physical_group = models.ForeignKey(
539
540
        Group,
        related_name="physicalgroup",
541
542
543
        null=True,
        blank=True,
        default=None,
544
        verbose_name=_("Group with physical access"),
Rémi Duraffort's avatar
Rémi Duraffort committed
545
        on_delete=models.CASCADE,
546
547
548
    )

    description = models.TextField(
549
        verbose_name=_("Device Description"),
550
551
552
        max_length=200,
        null=True,
        blank=True,
553
        default=None,
554
555
    )

556
557
    tags = models.ManyToManyField(Tag, blank=True)

558
559
560
561
    # This state is a cache computed from the device health and jobs. So keep
    # it read only to the admins
    STATE_IDLE, STATE_RESERVED, STATE_RUNNING = range(3)
    STATE_CHOICES = (
562
563
564
565
        (STATE_IDLE, "Idle"),
        (STATE_RESERVED, "Reserved"),
        (STATE_RUNNING, "Running"),
    )
566
567
568
569
570
    STATE_REVERSE = {
        "Idle": STATE_IDLE,
        "Reserved": STATE_RESERVED,
        "Running": STATE_RUNNING,
    }
571
572
    state = models.IntegerField(
        choices=STATE_CHOICES, default=STATE_IDLE, editable=False
573
    )
574

575
    # The device health helps to decide what to do next with the device
576
577
578
    HEALTH_GOOD, HEALTH_UNKNOWN, HEALTH_LOOPING, HEALTH_BAD, HEALTH_MAINTENANCE, HEALTH_RETIRED = range(
        6
    )
579
    HEALTH_CHOICES = (
580
581
582
583
584
585
        (HEALTH_GOOD, "Good"),
        (HEALTH_UNKNOWN, "Unknown"),
        (HEALTH_LOOPING, "Looping"),
        (HEALTH_BAD, "Bad"),
        (HEALTH_MAINTENANCE, "Maintenance"),
        (HEALTH_RETIRED, "Retired"),
586
    )
587
588
589
590
591
592
593
594
    HEALTH_REVERSE = {
        "GOOD": HEALTH_GOOD,
        "UNKNOWN": HEALTH_UNKNOWN,
        "LOOPING": HEALTH_LOOPING,
        "BAD": HEALTH_BAD,
        "MAINTENANCE": HEALTH_MAINTENANCE,
        "RETIRED": HEALTH_RETIRED,
    }
595
    health = models.IntegerField(choices=HEALTH_CHOICES, default=HEALTH_MAINTENANCE)
596

Neil Williams's avatar
Neil Williams committed
597
    last_health_report_job = models.OneToOneField(
598
599
600
601
602
603
604
        "TestJob",
        blank=True,
        unique=True,
        null=True,
        related_name="+",
        on_delete=models.SET_NULL,
    )
605

606
    # TODO: make this mandatory
Senthil Kumaran S's avatar
Senthil Kumaran S committed
607
608
    worker_host = models.ForeignKey(
        Worker,
609
        verbose_name=_("Worker Host"),
610
611
        null=True,
        blank=True,
612
613
        default=None,
        on_delete=models.SET_NULL,
614
615
    )

616
    def __str__(self):
617
618
619
620
621
        return "%s (%s, health %s)" % (
            self.hostname,
            self.get_state_display(),
            self.get_health_display(),
        )
Zygmunt Krynicki's avatar
Zygmunt Krynicki committed
622

623
624
625
626
627
628
    def current_job(self):
        try:
            return self.testjobs.get(~Q(state=TestJob.STATE_FINISHED))
        except TestJob.DoesNotExist:
            return None

Zygmunt Krynicki's avatar
Zygmunt Krynicki committed
629
    def get_absolute_url(self):
Rémi Duraffort's avatar
Rémi Duraffort committed
630
        return reverse("lava.scheduler.device.detail", args=[self.pk])
Zygmunt Krynicki's avatar
Zygmunt Krynicki committed
631

632
633
634
635
636
637
    def get_simple_state_display(self):
        if self.state == Device.STATE_IDLE:
            if self.health in [Device.HEALTH_MAINTENANCE, Device.HEALTH_RETIRED]:
                return self.get_health_display()
        return self.get_state_display()

638
    def get_description(self):
639
        return mark_safe(self.description) if self.description else None
640

Stevan Radakovic's avatar
Stevan Radakovic committed
641
642
643
644
645
646
647
648
649
    def has_any_permission_restrictions(self, perm):
        if not self.is_permission_restricted(perm):
            return self.device_type.has_any_permission_restrictions(
                self.DEVICE_TYPE_PERMISSION_MAP[perm]
            )
        else:
            return True

    def can_view(self, user):
650
651
652
653
        """
        Checks if this device is visible to the specified user.
        Retired devices are deemed to be visible - filter these out
        explicitly where necessary.
Stevan Radakovic's avatar
Stevan Radakovic committed
654
        :param user: User trying to view the device
655
656
        :return: True if the user can see this device
        """
Stevan Radakovic's avatar
Stevan Radakovic committed
657
658
659
660
661
662
        if user.has_perm(self.VIEW_PERMISSION, self):
            return True
        if not self.is_permission_restricted(self.VIEW_PERMISSION):
            if self.device_type.can_view(user):
                return True
        return False
663

664
665
    def can_change(self, user):
        if user.has_perm(self.CHANGE_PERMISSION, self):
666
            return True
667
668
        if not self.is_permission_restricted(self.CHANGE_PERMISSION):
            if user.has_perm(self.device_type.CHANGE_PERMISSION, self.device_type):
Stevan Radakovic's avatar
Stevan Radakovic committed
669
                return True
670
        return False
671
672

    def can_submit(self, user):
673
        if self.health == Device.HEALTH_RETIRED:
674
675
676
            return False
        if user.username == "lava-health":
            return True
Stevan Radakovic's avatar
Stevan Radakovic committed
677
678
679
680
681
682
        if user.has_perm(self.SUBMIT_PERMISSION, self):
            return True
        if not self.is_permission_restricted(self.SUBMIT_PERMISSION):
            if not self.device_type.is_permission_restricted(
                DeviceType.SUBMIT_PERMISSION
            ):
683
                if user.is_authenticated:
Stevan Radakovic's avatar
Stevan Radakovic committed
684
685
686
687
688
                    return True
            elif user.has_perm(self.device_type.SUBMIT_PERMISSION, self.device_type):
                return True

        return False
689

690
    def is_valid(self):
691
        try:
692
            rendered = self.load_configuration()
693
            validate_device(rendered)
694
        except (SubmissionException, yaml.YAMLError):
695
696
697
            return False
        return True

698
    def log_admin_entry(self, user, reason):
699
700
        if user is None:
            user = User.objects.get(username="lava-health")
701
702
703
704
705
        device_ct = ContentType.objects.get_for_model(Device)
        LogEntry.objects.log_action(
            user_id=user.id,
            content_type_id=device_ct.pk,
            object_id=self.pk,
706
            object_repr=self.hostname,
707
            action_flag=CHANGE,
708
            change_message=reason,
709
710
        )

711
    def testjob_signal(self, signal, job, infrastructure_error=False):
712

713
714
        if signal == "go_state_scheduling":
            self.state = Device.STATE_RESERVED
715

716
717
        elif signal == "go_state_scheduled":
            self.state = Device.STATE_RESERVED
Antonio Terceiro's avatar
Antonio Terceiro committed
718

719
720
        elif signal == "go_state_running":
            self.state = Device.STATE_RUNNING
Antonio Terceiro's avatar
Antonio Terceiro committed
721

722
        elif signal == "go_state_canceling":
723
            pass
724

725
726
727
        elif signal == "go_state_finished":
            self.state = Device.STATE_IDLE

728
729
            prev_health_display = self.get_health_display()
            if job.health_check:
730
                self.last_health_report_job = job
731
732
733
734
735
                if self.health == Device.HEALTH_LOOPING:
                    if job.health == TestJob.HEALTH_INCOMPLETE:
                        # Looping is persistent until cancelled by the admin.
                        self.log_admin_entry(
                            None,
736
737
738
739
740
741
742
743
                            "%s → %s (Looping health-check [%s] failed)"
                            % (prev_health_display, self.get_health_display(), job.id),
                        )
                elif self.health in [
                    Device.HEALTH_GOOD,
                    Device.HEALTH_UNKNOWN,
                    Device.HEALTH_BAD,
                ]:
744
745
                    if job.health == TestJob.HEALTH_COMPLETE:
                        self.health = Device.HEALTH_GOOD
746
                        msg = "completed"
747
748
                    elif job.health == TestJob.HEALTH_INCOMPLETE:
                        self.health = Device.HEALTH_BAD
749
                        msg = "failed"
750
                    elif job.health == TestJob.HEALTH_CANCELED:
751
                        self.health = Device.HEALTH_BAD
752
                        msg = "canceled"
753
754
                    else:
                        raise NotImplementedError("Unexpected TestJob health")
755
                    self.log_admin_entry(
756
757
758
759
                        None,
                        "%s → %s (health-check [%s] %s)"
                        % (prev_health_display, self.get_health_display(), job.id, msg),
                    )
760
761
            elif infrastructure_error:
                self.health = Device.HEALTH_UNKNOWN
762
                self.log_admin_entry(
763
764
765
766
                    None,
                    "%s → %s (Infrastructure error after %s)"
                    % (prev_health_display, self.get_health_display(), job.display_id),
                )
Antonio Terceiro's avatar
Antonio Terceiro committed
767

768
        else:
769
770
771
772
773
774
775
776
777
778
779
780
            raise NotImplementedError("Unknown signal %s" % signal)

    def worker_signal(self, signal, user, prev_state, prev_health):
        # HEALTH_BAD and HEALTH_RETIRED are permanent states that are not
        # changed by worker signals
        if signal == "go_health_active":
            # When leaving retirement, don't cascade the change
            if prev_health == Worker.HEALTH_RETIRED:
                return
            # Only update health of devices in maintenance
            if self.health != Device.HEALTH_MAINTENANCE:
                return
781
782
783
            self.log_admin_entry(
                user, "%s → Unknown (worker going active)" % self.get_health_display()
            )
784
785
786
            self.health = Device.HEALTH_UNKNOWN

        elif signal == "go_health_maintenance":
787
788
789
790
791
            if self.health in [
                Device.HEALTH_BAD,
                Device.HEALTH_MAINTENANCE,
                Device.HEALTH_RETIRED,
            ]:
792
                return
793
794
795
796
797
            self.log_admin_entry(
                user,
                "%s → Maintenance (worker going maintenance)"
                % self.get_health_display(),
            )
798
799
800
            self.health = Device.HEALTH_MAINTENANCE

        elif signal == "go_health_retired":
801
            if self.health in [Device.HEALTH_BAD, Device.HEALTH_RETIRED]:
802
                return
803
804
805
            self.log_admin_entry(
                user, "%s → Retired (worker going retired)" % self.get_health_display()
            )
806
            self.health = Device.HEALTH_RETIRED
807

808
        else:
809
            raise NotImplementedError("Unknown signal %s" % signal)
810

811
    def load_configuration(self, job_ctx=None, output_format="dict"):
812
        """
813
        Maps the device dictionary to the static templates in /etc/.
Rémi Duraffort's avatar
Rémi Duraffort committed
814
        raise: this function can raise OSError, jinja2.TemplateError or yaml.YAMLError -
815
816
817
            handling these exceptions may be context-dependent, users will need
            useful messages based on these exceptions.
        """
818
819
820
821
        # The job_ctx should not be None while an empty dict is ok
        if job_ctx is None:
            job_ctx = {}

822
823
        if output_format == "raw":
            try:
824
                with open(
825
826
                    os.path.join(settings.DEVICES_PATH, "%s.jinja2" % self.hostname),
                    "r",
827
                ) as f_in:
828
                    return f_in.read()
Rémi Duraffort's avatar
Rémi Duraffort committed
829
            except OSError:
830
831
832
                return None

        try:
833
            template = environment.devices().get_template("%s.jinja2" % self.hostname)
834
835
            device_template = template.render(**job_ctx)
        except jinja2.TemplateError:
836
            return None
837
838
839
840

        if output_format == "yaml":
            return device_template
        else:
841
            return yaml_safe_load(device_template)
842

843
844
845
846
847
848
849
    def minimise_configuration(self, data):
        """
        Support for dynamic connections which only require
        critical elements of device configuration.
        Principally drop top level parameters and commands
        like power.
        """
850
851
852
853
854
855

        def get(data, keys):
            for key in keys:
                data = data.get(key, {})
            return data

856
        data["constants"]["kernel-start-message"] = ""
857
        device_configuration = {
858
            "hostname": self.hostname,
859
860
            "constants": data.get("constants", {}),
            "timeouts": data.get("timeouts", {}),
861
            "actions": {
862
                "deploy": {
863
864
                    "connections": get(data, ["actions", "deploy", "connections"]),
                    "methods": get(data, ["actions", "deploy", "methods"]),
865
866
                },
                "boot": {
867
868
                    "connections": get(data, ["actions", "boot", "connections"]),
                    "methods": get(data, ["actions", "boot", "methods"]),
869
                },
870
            },
871
872
873
        }
        return device_configuration

874
875
    def save_configuration(self, data):
        try:
876
            with open(
877
                os.path.join(settings.DEVICES_PATH, "%s.jinja2" % self.hostname), "w"
878
            ) as f_out:
879
880
                f_out.write(data)
            return True
Rémi Duraffort's avatar
Rémi Duraffort committed
881
        except OSError as exc:
882
            logger = logging.getLogger("lava_scheduler_app")
883
884
885
            logger.error(
                "Error saving device configuration for %s: %s", self.hostname, str(exc)
            )
886
887
888
889
890
891
892
            return False

    def get_extends(self):
        jinja_config = self.load_configuration(output_format="raw")
        if not jinja_config:
            return None

893
894
895
        env = jinja2.Environment(  # nosec - YAML, not HTML, no XSS scope.
            autoescape=False
        )
896
897
898
899
        try:
            ast = env.parse(jinja_config)
            extends = list(ast.find_all(jinja2.nodes.Extends))
            if len(extends) != 1:
900
                logger = logging.getLogger("lava_scheduler_app")
901
902
903
904
905
                logger.error("Found %d extends for %s", len(extends), self.hostname)
                return None
            else:
                return os.path.splitext(extends[0].template.value)[0]
        except jinja2.TemplateError as exc:
906
            logger = logging.getLogger("lava_scheduler_app")
907
            logger.error("Invalid template for %s: %s", self.hostname, str(exc))
908
            return None
909

910
911
    def get_health_check(self):
        # Get the device dictionary
912
913
        extends = self.get_extends()
        if not extends:
914
            return None
915

916
        filename = os.path.join(settings.HEALTH_CHECKS_PATH, "%s.yaml" % extends)
917
918
        # Try if health check file is having a .yml extension
        if not os.path.exists(filename):
919
            filename = os.path.join(settings.HEALTH_CHECKS_PATH, "%s.yml" % extends)
920
921
922
        try:
            with open(filename, "r") as f_in:
                return f_in.read()
Rémi Duraffort's avatar
Rémi Duraffort committed
923
        except OSError:
924
            return None
925

926

Andy Doan's avatar
Andy Doan committed
927
928
929
930
931
class JobFailureTag(models.Model):
    """
    Allows us to maintain a set of common ways jobs fail. These can then be
    associated with a TestJob so we can do easy data mining
    """
932

Andy Doan's avatar
Andy Doan committed
933
934
935
936
    name = models.CharField(unique=True, max_length=256)

    description = models.TextField(null=True, blank=True)

937
    def __str__(self):
Andy Doan's avatar
Andy Doan committed
938
939
        return self.name

940

941
def _get_tag_list(tags):
942
943
944
945
946
    """
    Creates a list of Tag objects for the specified device tags
    for singlenode and multinode jobs.
    :param tags: a list of strings from the JSON
    :return: a list of tags which match the strings
947
    :raise: yaml.YAMLError if a tag cannot be found in the database.
948
949
    """
    taglist = []
950
    if not isinstance(tags, list):
951
        msg = "'device_tags' needs to be a list - received %s" % type(tags)
952
        raise yaml.YAMLError(msg)
953
954
955
956
    for tag_name in tags:
        try:
            taglist.append(Tag.objects.get(name=tag_name))
        except Tag.DoesNotExist:
957
            msg = "Device tag '%s' does not exist in the database." % tag_name
958
            raise yaml.YAMLError(msg)
959

960
961
962
963
964
965
966
967
968
969
970