Commit 919c10c2 authored by Rémi Duraffort's avatar Rémi Duraffort
Browse files

Remove links to dashboard_app in the scheduler_app

Also remove unused functions.

Change-Id: I43d43c11139053a667f0cd6f42a363abf34988f8
parent f7a1c976
......@@ -10,7 +10,6 @@ import yaml
import jinja2
import datetime
import logging
import simplejson
from django.db.models import Q, Case, When, IntegerField, Sum
from django.db import IntegrityError, transaction
from django.contrib.auth.models import User
......@@ -20,7 +19,6 @@ from lava_scheduler_app.models import (
Device,
DevicesUnavailableException,
DeviceType,
is_deprecated_json,
JSONDataError,
TestJob,
TemporaryDevice,
......@@ -84,14 +82,6 @@ def initiate_health_check_job(device):
device.put_into_maintenance_mode(
user, "health check job not found in initiate_health_check_job")
raise JSONDataError("no health check job found for %r", device.hostname)
if is_deprecated_json(job_data):
# only JSON supports 'target' and that needs to be set by the health-check not the admin.
job_json = simplejson.loads(job_data)
if 'target' in job_json:
logger.error("[%s] JSON Health check definition must not specify a 'target'.", device.device_type.name)
device.put_into_maintenance_mode(
user, "target must not be defined in health check definitions.")
return None
try:
job = testjob_submission(job_data, user, check_device=device)
except (DevicesUnavailableException, SubmissionException) as exc:
......@@ -158,65 +148,31 @@ def submit_health_check_jobs():
logger.debug('submit health check for %s', device.hostname)
try:
initiate_health_check_job(device)
except (yaml.YAMLError, JSONDataError):
except yaml.YAMLError:
# already logged, don't allow the daemon to fail.
pass
def testjob_submission(job_definition, user, check_device=None, original_job=None):
"""
Single submission frontend for JSON or YAML
Single submission frontend for YAML
:param job_definition: string of the job submission
:param user: user attempting the submission
:param check_device: set specified device as the target
**and** thereby set job as a health check job. (JSON only)
:return: a job or a list of jobs
:raises: SubmissionException, Device.DoesNotExist,
DeviceType.DoesNotExist, DevicesUnavailableException,
JSONDataError, JSONDecodeError, ValueError
"""
if is_deprecated_json(job_definition):
allow_health = False
job_json = simplejson.loads(job_definition)
target_device = None
if 'target' in job_json:
target_device = Device.objects.get(hostname=job_json['target'])
if check_device:
job_json['target'] = check_device.hostname
job_json['health-check'] = True
job_definition = simplejson.dumps(job_json)
allow_health = True
try:
# returns a single job or a list (not a QuerySet) of job objects.
job = TestJob.from_json_and_user(job_definition, user, health_check=allow_health)
if isinstance(job, list):
# multinode health checks not supported
return job
job.health_check = allow_health
if check_device:
job.requested_device = check_device
elif target_device:
job.requested_device = target_device
job.save(update_fields=['health_check', 'requested_device'])
except (JSONDataError, ValueError) as exc:
if check_device:
check_device.put_into_maintenance_mode(
user, "Job submission failed for health job for %s: %s" % (check_device, exc))
raise JSONDataError("Health check job submission failed for %s: %s" % (check_device, exc))
else:
raise JSONDataError("Job submission failed: %s" % exc)
else:
validate_job(job_definition)
# returns a single job or a list (not a QuerySet) of job objects.
job = TestJob.from_yaml_and_user(job_definition, user, original_job=original_job)
if check_device and isinstance(check_device, Device) and not isinstance(job, list):
# the slave must neither know nor care if this is a health check,
# only the master cares and that has the database connection.
job.health_check = True
job.requested_device = check_device
job.save(update_fields=['health_check', 'requested_device'])
ValueError
"""
validate_job(job_definition)
# returns a single job or a list (not a QuerySet) of job objects.
job = TestJob.from_yaml_and_user(job_definition, user, original_job=original_job)
if check_device and isinstance(check_device, Device) and not isinstance(job, list):
# the slave must neither know nor care if this is a health check,
# only the master cares and that has the database connection.
job.health_check = True
job.requested_device = check_device
job.save(update_fields=['health_check', 'requested_device'])
return job
......
This diff is collapsed.
......@@ -86,226 +86,6 @@ class TestSchedulerAPI(TestCaseWithFactory): # pylint: disable=too-many-ancesto
else:
self.fail("fault not raised")
def test_submit_job_sets_definition(self):
user = self.factory.ensure_user('test', 'e@mail.invalid', 'test')
user.user_permissions.add(
Permission.objects.get(codename='add_testjob'))
user.save()
server = self.server_proxy('test', 'test')
definition = self.factory.make_job_json()
try:
job_id = server.scheduler.submit_job(definition)
except xmlrpclib.Fault as f:
self.assertEqual(400, f.faultCode)
else:
self.fail("v1 job was accepted")
def test_cancel_job_rejects_anonymous(self):
job = self.factory.make_testjob()
server = self.server_proxy()
try:
server.scheduler.cancel_job(job.id)
except xmlrpclib.Fault as f:
self.assertEqual(401, f.faultCode)
else:
self.fail("fault not raised")
def test_cancel_job_rejects_unpriv_user(self):
job = self.factory.make_testjob()
self.factory.ensure_user('test', 'e@mail.invalid', 'test')
server = self.server_proxy('test', 'test')
try:
server.scheduler.cancel_job(job.id)
except xmlrpclib.Fault as f:
self.assertEqual(403, f.faultCode)
else:
self.fail("fault not raised")
def test_cancel_job_cancels_job(self):
user = self.factory.ensure_user('test', 'e@mail.invalid', 'test')
job = self.factory.make_testjob(submitter=user)
server = self.server_proxy('test', 'test')
server.scheduler.cancel_job(job.id)
job = TestJob.objects.get(pk=job.pk)
self.assertIn(TestJob.STATUS_CHOICES[job.status], [
TestJob.STATUS_CHOICES[TestJob.CANCELED],
TestJob.STATUS_CHOICES[TestJob.CANCELING]
])
def test_cancel_job_user(self):
"""
tests whether the user who canceled the job is reflected properly.
See: https://bugs.linaro.org/show_bug.cgi?id=650
"""
user = self.factory.ensure_user('test', 'e@mail.invalid', 'test')
cancel_user = User.objects.create_user('test_cancel',
'cancel@mail.invalid',
'test_cancel')
cancel_user.save()
job = self.factory.make_testjob(submitter=user)
job.description = "sample job"
job.save()
job.cancel(user=cancel_user)
job = TestJob.objects.get(pk=job.pk)
self.assertIn(TestJob.STATUS_CHOICES[job.status], [
TestJob.STATUS_CHOICES[TestJob.CANCELED],
TestJob.STATUS_CHOICES[TestJob.CANCELING]
])
job = TestJob.objects.get(pk=job.pk) # reload
self.assertEqual(job.failure_comment,
"Canceled by %s" % cancel_user.username)
def test_json_vs_yaml(self):
"""
Test that invalid JSON gets rejected but valid YAML is accepted as pipeline
"""
user = self.factory.ensure_user('test', 'e@mail.invalid', 'test')
user.user_permissions.add(
Permission.objects.get(codename='add_testjob'))
user.save()
job = self.factory.make_testjob(submitter=user)
self.assertFalse(job.is_pipeline)
# "break" the JSON by dropping the closing } as JSON needs the complete file to validate
invalid_def = job.definition[:-2]
self.assertRaises(ValueError, json.loads, invalid_def)
server = self.server_proxy('test', 'test')
self.assertRaises(xmlrpclib.Fault, server.scheduler.submit_job, invalid_def)
invalid_yaml_def = """
# Sample JOB definition for a KVM
device_type: qemu
job_name: kvm-pipeline
priority: medium
"""
self.assertRaises(ValueError, json.loads, invalid_yaml_def)
self.assertRaises(xmlrpclib.Fault, server.scheduler.submit_job, invalid_yaml_def)
yaml_def = """
# Sample JOB definition for a KVM
device_type: qemu
job_name: kvm-pipeline
timeouts:
job:
minutes: 15
action:
minutes: 5
priority: medium
visibility: public
actions:
- deploy:
to: tmpfs
image: http://images.validation.linaro.org/kvm-debian-wheezy.img.gz
compression: gz
os: debian
- boot:
method: qemu
media: tmpfs
failure_retry: 2
- test:
name: kvm-basic-singlenode
definitions:
- repository: git://git.linaro.org/lava-team/lava-functional-tests.git
from: git
path: lava-test-shell/smoke-tests-basic.yaml
# name: if not present, use the name from the YAML. The name can
# also be overriden from the actual commands being run by
# calling the lava-test-suite-name API call (e.g.
# `lava-test-suite-name FOO`).
name: smoke-tests
"""
yaml_data = yaml.load(yaml_def)
validate_submission(yaml_data)
self.assertRaises(xmlrpclib.Fault, server.scheduler.submit_job, yaml_def)
device_type = self.factory.make_device_type('qemu')
device = self.factory.make_device(device_type=device_type, hostname="qemu1")
device.save()
self.assertFalse(device.is_pipeline)
self.assertRaises(xmlrpclib.Fault, server.scheduler.submit_job, yaml_def)
device = self.factory.make_device(device_type=device_type, hostname="qemu2", is_pipeline=True)
device.save()
self.assertTrue(device.is_pipeline)
job_id = server.scheduler.submit_job(yaml_def)
job = TestJob.objects.get(id=job_id)
self.assertTrue(job.is_pipeline)
def test_health_determination(self): # pylint: disable=too-many-statements
user = self.factory.ensure_user('test', 'e@mail.invalid', 'test')
user.user_permissions.add(
Permission.objects.get(codename='add_testjob'))
user.save()
device_type = self.factory.make_device_type('beaglebone-black')
device = self.factory.make_device(device_type=device_type, hostname="black01")
device.save()
filename = os.path.join(os.path.dirname(__file__), 'sample_jobs', 'master-check.json')
self.assertTrue(os.path.exists(filename))
with open(filename, 'r') as json_file:
definition = json_file.read()
# simulate UI submission
job = self.factory.make_testjob(definition=definition, submitter=user)
self.assertFalse(job.health_check)
job.save(update_fields=['health_check', 'requested_device'])
self.assertFalse(job.health_check)
job.delete()
# simulate API submission
job = testjob_submission(definition, user)
self.assertFalse(job.health_check)
self.assertIsNone(job.requested_device)
job.delete()
job = testjob_submission(definition, user, check_device=None)
self.assertFalse(job.health_check)
self.assertIsNone(job.requested_device)
job.delete()
# simulate initiating a health check
job = testjob_submission(definition, user, check_device=device)
self.assertTrue(job.health_check)
self.assertEqual(job.requested_device.hostname, device.hostname)
job.delete()
# modify definition to use the deprecated target support
device2 = self.factory.make_device(device_type=device_type, hostname="black02")
device2.save()
def_dict = json.loads(definition)
self.assertNotIn('target', def_dict)
def_dict['target'] = device2.hostname
definition = json.dumps(def_dict)
# simulate API submission with target set
job = testjob_submission(definition, user, check_device=None)
self.assertFalse(job.health_check)
self.assertEqual(job.requested_device.hostname, device2.hostname)
job.delete()
# healthcheck designation overrides target (although this is itself an admin error)
job = testjob_submission(definition, user, check_device=device)
self.assertTrue(job.health_check)
self.assertEqual(job.requested_device.hostname, device.hostname)
job.delete()
# check malformed JSON
self.assertRaises(SubmissionException, testjob_submission, definition[:100], user)
# check non-existent targets
def_dict['target'] = 'nosuchdevice'
definition = json.dumps(def_dict)
self.assertRaises(Device.DoesNotExist, testjob_submission, definition, user)
# check multinode API submission. bug #2130
filename = os.path.join(os.path.dirname(__file__), 'sample_jobs', 'master-multinode.json')
self.assertTrue(os.path.exists(filename))
with open(filename, 'r') as json_file:
definition = json_file.read()
job_list = testjob_submission(definition, user)
self.assertIsInstance(job_list, list)
for job in job_list:
self.assertIsNotNone(job.vm_group)
self.assertFalse(job.health_check)
if job.requested_device_type == device_type:
self.assertIsNone(job.requested_device)
else:
self.assertIsNotNone(job.requested_device)
self.assertIsInstance(job.requested_device, TemporaryDevice)
job.requested_device.delete()
job.delete()
def test_new_devices(self):
user = self.factory.ensure_user('test', 'e@mail.invalid', 'test')
user.user_permissions.add(
......
......@@ -5,23 +5,16 @@ import warnings
import yaml
from django.contrib.auth.models import Group, Permission, User
from django.core.exceptions import ValidationError
from django_testscenarios.ubertest import TestCase
from django.utils import timezone
from lava_scheduler_app.models import (
Device,
DeviceType,
JSONDataError,
Notification,
Tag,
TestJob,
DevicesUnavailableException,
_check_exclusivity,
is_deprecated_json,
)
from lava_scheduler_app.schema import SubmissionException
import simplejson
LOGGER = logging.getLogger()
LOGGER.level = logging.INFO # change to DEBUG to see *all* output
......@@ -136,27 +129,9 @@ class ModelFactory(object):
data['device_type'] = device_type.name
return data
def make_job_json(self, **kw):
return simplejson.dumps(self.make_job_data(**kw), sort_keys=True, indent=4 * ' ')
def make_invalid_job_json(self, **kw):
actions = [{'command': 'invalid_command'}]
return simplejson.dumps(self.make_job_data(actions, **kw), sort_keys=True, indent=4 * ' ')
def make_job_yaml(self, **kw):
return yaml.dump(self.make_job_data(**kw))
def make_testjob(self, definition=None, submitter=None, **kwargs):
if definition is None:
definition = self.make_job_json(**kwargs)
if submitter is None:
submitter = self.make_user()
if 'user' not in kwargs:
kwargs['user'] = submitter
testjob = TestJob.from_json_and_user(definition, submitter)
testjob.save()
return testjob
def make_notification(self, job):
notification = Notification()
notification.test_job = job
......@@ -180,21 +155,6 @@ class TestCaseWithFactory(TestCase): # pylint: disable=too-many-ancestors
class TestTestJob(TestCaseWithFactory): # pylint: disable=too-many-ancestors,too-many-public-methods
def test_from_json_and_user_sets_definition(self):
definition = self.factory.make_job_json()
job = TestJob.from_json_and_user(definition, self.factory.make_user())
self.assertEqual(definition, job.definition)
self.factory.cleanup()
def test_from_json_and_user_sets_outputdir(self):
definition = self.factory.make_job_json()
job = TestJob.from_json_and_user(definition, self.factory.make_user())
dir_list = job.output_dir.split('/')
self.assertIn("%02d" % job.submit_time.year, dir_list)
self.assertIn("%02d" % job.submit_time.month, dir_list)
self.assertIn("%02d" % job.submit_time.day, dir_list)
self.assertIn(str(job.id), dir_list)
def test_user_permission(self):
self.assertIn(
'cancel_resubmit_testjob',
......@@ -251,348 +211,6 @@ class TestTestJob(TestCaseWithFactory): # pylint: disable=too-many-ancestors,to
self.assertEqual(known_groups, list(job.viewing_groups.all()))
self.factory.cleanup()
def test_from_json_and_user_sets_submitter(self):
user = self.factory.make_user()
job = TestJob.from_json_and_user(
self.factory.make_job_json(), user)
self.assertEqual(user, job.submitter)
self.factory.cleanup()
def test_from_json_and_user_sets_device_type(self):
panda_type = self.factory.ensure_device_type(name='panda')
job = TestJob.from_json_and_user(
self.factory.make_job_json(device_type='panda'),
self.factory.make_user())
self.assertEqual(panda_type, job.requested_device_type)
self.factory.cleanup()
def test_from_json_and_user_sets_target(self):
panda_board = self.factory.make_device(hostname='panda02')
job = TestJob.from_json_and_user(
self.factory.make_job_json(target='panda02'),
self.factory.make_user())
self.assertEqual(panda_board, job.requested_device)
self.factory.cleanup()
def test_from_json_and_user_does_not_set_device_type_from_target(self):
panda_type = self.factory.ensure_device_type(name='panda')
self.factory.make_device(device_type=panda_type, hostname='panda02')
job = TestJob.from_json_and_user(
self.factory.make_job_json(target='panda02'),
self.factory.make_user())
self.assertEqual(None, job.requested_device_type)
self.factory.cleanup()
def test_from_json_and_user_sets_date_submitted(self):
before = timezone.now()
job = TestJob.from_json_and_user(
self.factory.make_job_json(),
self.factory.make_user())
after = timezone.now()
self.assertTrue(before < job.submit_time < after)
self.factory.cleanup()
def test_from_json_and_user_sets_status_to_SUBMITTED(self):
job = TestJob.from_json_and_user(
self.factory.make_job_json(),
self.factory.make_user())
self.assertEqual(job.status, TestJob.SUBMITTED)
self.factory.cleanup()
def test_from_json_and_user_sets_no_tags_if_no_tags(self):
job = TestJob.from_json_and_user(
self.factory.make_job_json(device_tags=[]),
self.factory.make_user())
self.assertEqual(set(job.tags.all()), set([]))
self.factory.cleanup()
def test_from_json_and_user_errors_on_unknown_tags(self):
"""
Tests that tags which are not already defined in the database
cause job submissions to be rejected.
"""
self.assertRaises(
JSONDataError, TestJob.from_json_and_user,
self.factory.make_job_json(tags=['unknown']),
self.factory.make_user())
self.factory.cleanup()
def test_from_json_and_user_errors_on_unsupported_tags(self):
"""
Tests that tags which do exist but are not defined for the
any of the devices of the requested type cause the submission
to be rejected with Devices Unavailable.
"""
device_type = self.factory.ensure_device_type(name='panda')
self.factory.make_device(device_type=device_type, hostname="panda2")
self.factory.ensure_tag('tag1')
self.factory.ensure_tag('tag2')
try:
TestJob.from_json_and_user(
self.factory.make_job_json(tags=['tag1', 'tag2']),
self.factory.make_user())
except DevicesUnavailableException:
pass
else:
self.fail("Device tags failure: job submitted without any devices supporting the requested tags")
self.factory.cleanup()
def test_from_json_and_user_sets_tag_from_device_tags(self):
device_type = self.factory.ensure_device_type(name='panda')
self.factory.ensure_tag('tag')
tags = list(Tag.objects.filter(name='tag'))
self.factory.make_device(device_type=device_type, hostname="panda1", tags=tags)
job = TestJob.from_json_and_user(
self.factory.make_job_json(tags=['tag']),
self.factory.make_user())
self.assertEqual(
set(tag.name for tag in job.tags.all()), {'tag'})
self.factory.cleanup()
def test_from_json_and_user_sets_multiple_tag_from_device_tags(self):
device_type = self.factory.ensure_device_type(name='panda')
tag_list = [
self.factory.ensure_tag('tag1'),
self.factory.ensure_tag('tag2')
]
self.factory.make_device(device_type=device_type, hostname="panda2", tags=tag_list)
job = TestJob.from_json_and_user(
self.factory.make_job_json(tags=['tag1', 'tag2']),
self.factory.make_user())
self.assertEqual(
set(tag.name for tag in job.tags.all()), {'tag1', 'tag2'})
self.factory.cleanup()
def test_from_json_and_user_reuses_tag_objects(self):
device_type = self.factory.ensure_device_type(name='panda')
self.factory.ensure_tag('tag')
tags = list(Tag.objects.filter(name='tag'))
self.factory.make_device(device_type=device_type, hostname="panda3", tags=tags)
job1 = TestJob.from_json_and_user(
self.factory.make_job_json(tags=['tag']),
self.factory.make_user())
job2 = TestJob.from_json_and_user(
self.factory.make_job_json(tags=['tag']),
self.factory.make_user())
self.assertEqual(
set(tag.pk for tag in job1.tags.all()),
set(tag.pk for tag in job2.tags.all()))
self.factory.cleanup()
def test_from_json_and_user_matches_available_tags(self):
"""
Test that with more than one device of the requested type supporting
tags, that the tag list set for the TestJob matches the list requested,
not a shorter list from a different device or a combined list of multiple
devices.
"""
device_type = self.factory.ensure_device_type(name='panda')
tag_list = [
self.factory.ensure_tag('common_tag1'),
self.factory.ensure_tag('common_tag2')
]
self.factory.make_device(device_type=device_type, hostname="panda4", tags=tag_list)
tag_list.append(self.factory.ensure_tag('unique_tag'))
self.factory.make_device(device_type=device_type, hostname="panda5", tags=tag_list)
job = TestJob.from_json_and_user(
self.factory.make_job_json(tags=['common_tag1', 'common_tag2', 'unique_tag']),
self.factory.make_user())
self.assertEqual(
set(tag for tag in job.tags.all()),
set(tag_list)
)
self.factory.cleanup()
def test_from_json_and_user_rejects_invalid_json(self):
self.assertRaises(
ValueError, TestJob.from_json_and_user, '{',
self.factory.make_user())
self.factory.cleanup()
def test_from_json_and_user_rejects_invalid_job(self):
# job data must have the 'actions' and 'timeout' properties, so this
# will be rejected.
self.assertRaises(
ValueError, TestJob.from_json_and_user, '{}',
self.factory.make_user())
self.factory.cleanup()
def test_from_json_rejects_exclusive(self):
panda_type = self.factory.ensure_device_type(name='panda')
panda_board = self.factory.make_device(device_type=panda_type, hostname='panda03')
job = TestJob.from_json_and_user(
self.factory.make_job_json(device_type='panda'),
self.factory.make_user())
self.assertEqual(panda_type, job.requested_device_type)
self.assertTrue(panda_board.is_exclusive)
self.assertRaises(
DevicesUnavailableException, _check_exclusivity, [panda_board], pipeline=False