Commit 4d0d867e authored by Rémi Duraffort's avatar Rémi Duraffort Committed by Neil Williams
Browse files

Remove non pipeline jobs support

Change-Id: I1aeeda2ba5db79d1b083d86b172b515811f5b45e
parent c44009fa
......@@ -128,14 +128,8 @@ class SchedulerAPI(ExposedAPI):
except TestJob.DoesNotExist:
raise xmlrpclib.Fault(404, "Specified job not found.")
# Reject v1 jobs
if not job.is_pipeline:
raise xmlrpclib.Fault(400, "v1 jobs cannot be submitted to this instance")
if job.is_multinode:
return self.submit_job(job.multinode_definition)
elif job.is_vmgroup:
return self.submit_job(job.vmgroup_definition)
else:
return self.submit_job(job.definition)
......@@ -180,9 +174,6 @@ class SchedulerAPI(ExposedAPI):
target_group=job.target_group)
for multinode_job in multinode_jobs:
multinode_job.cancel(self.user)
elif job.is_vmgroup:
for vmgroup_job in job.sub_jobs_list:
vmgroup_job.cancel(self.user)
else:
job.cancel(self.user)
return True
......@@ -777,7 +768,7 @@ class SchedulerAPI(ExposedAPI):
The elements available in XML-RPC structure include:
_results_link, _state, submitter_id, submit_token_id, is_pipeline,
id, failure_comment, multinode_definition, user_id, vmgroup_definition,
id, failure_comment, multinode_definition, user_id,
priority, _actual_device_cache, vm_group, original_definition,
status, health_check, description, admin_notifications, start_time,
target_group, visibility, requested_device_id, pipeline_compatibility,
......
......@@ -21,7 +21,6 @@ from lava_scheduler_app.models import (
DeviceType,
JSONDataError,
TestJob,
TemporaryDevice,
validate_job,
validate_device
)
......@@ -246,13 +245,6 @@ def find_device_for_job(job, device_list): # pylint: disable=too-many-branches
continue
if job.is_pipeline and not device.is_pipeline:
continue
if job.is_vmgroup:
# deprecated and slow!
# special handling, tied directly to the TestJob within the vmgroup
# mask to a Temporary Device to be able to see vm_group of the device
tmp_dev = TemporaryDevice.objects.filter(hostname=device.hostname)
if tmp_dev and job.vm_group != tmp_dev[0].vm_group:
continue
# these are the principal requirements and checks.
# for pipeline, requested_device is only used for automated health checks, handled above.
# device_type, requested_device and requested_device_type have been retrieved with select_related
......
DEFAULT_TEMPLATE = {
"job_name": "JOBNAME_PARAMETER",
"device_type": "DEVICE_TYPE_PARAMETER",
"tags": "TAGS_PARAMETER",
"timeout": "TIMEOUT_PARAMETER",
"logging_level": "DEBUG",
"notify_on_incomplete": "NOTIFY_ON_INCOMPLETE_PARAMETER",
"actions": "ACTIONS_PARAMETER"
}
ACTIONS_LINARO = [
{
"command": "DEPLOY_COMMAND_PARAMETER",
"parameters": "DEPLOY_PARAMETER",
},
"COMMAND_TEST_SHELL",
"COMMAND_SUBMIT_RESULTS"
]
ACTIONS_LINARO_BOOT = [
{
"command": "DEPLOY_COMMAND_PARAMETER",
"parameters": "DEPLOY_PARAMETER",
},
{
"command": "boot_linaro_image",
"parameters": {
"boot_cmds": "BOOT_OPTIONS_PARAMETER",
}
},
"COMMAND_TEST_SHELL",
"COMMAND_SUBMIT_RESULTS"
]
DEPLOY_IMAGE = {
"image": "PREBUILT_IMAGE_PARAMETER"
}
DEPLOY_IMAGE_HWPACK = {
"hwpack": "HWPACK_PARAMETER",
"rootfs": "ROOTFS_PARAMETER"
}
DEPLOY_IMAGE_KERNEL = {
"kernel": "KERNEL_PARAMETER",
"ramdisk": "RAMDISK_PARAMETER",
"dtb": "DTB_PARAMETER",
"rootfs": "ROOTFS_PARAMETER"
}
LAVA_TEST_SHELL_REPO = {
"testdef_repos": [
{
"git-repo": "REPO_PARAMETER",
"testdef": "TESTDEF_PARAMETER"
}
]
}
LAVA_TEST_SHELL_URL = {
"testdef_urls": "TESTDEF_URLS_PARAMETER"
}
COMMAND_SUBMIT_RESULTS = {
"command": "submit_results",
"parameters": {
"server": "SUBMIT_SERVER",
"stream": "BUNDLE_STREAM"
}
}
COMMAND_TEST_SHELL = {
"command": "lava_test_shell",
"parameters": "TEST_SHELL_PARAMETER",
}
ACTIONS_LINARO_ANDROID_IMAGE = [
{
"command": "deploy_linaro_android_image",
"parameters": {
"boot": "BOOT_IMAGE_PARAMETER",
"data": "DATA_IMAGE_PARAMETER",
"system": "SYSTEM_IMAGE_PARAMETER"
}
},
{
"command": "android_install_binaries"
},
"ANDROID_BOOT",
{
"command": "lava_android_test_install",
"parameters": {
"tests": "TESTS_PARAMETER"
}
},
{
"command": "lava_android_test_run",
"parameters": {
"test_name": "TEST_NAME_PARAMETER"
}
},
"COMMAND_SUBMIT_RESULTS"
]
ANDROID_BOOT_NO_CMDS = {
"command": "boot_linaro_android_image",
}
ANDROID_BOOT_WITH_CMDS = {
"command": "boot_linaro_android_image",
"parameters": {
"boot_cmds": "ANDROID_BOOT_OPTIONS_PARAMETER"
}
}
import re
# DEPRECATED
# pylint: disable=invalid-name,old-style-class,no-init
def getDispatcherErrors(logfile):
errors = []
error_types = ["Infrastructure Error:",
"Bootloader Error:",
"Kernel Error:",
"Userspace Error:",
"Test Shell Error:",
"Master Image Error:",
"OperationFailed:"]
for line in logfile:
for error in error_types:
index = line.find(error)
if index != -1:
try:
# decode the byte sequence to
# check that it's already unicode.
line[index:].decode('utf-8')
errors.append(line[index:])
except UnicodeError:
# string was not unicode, encode it.
errors.append(line[index:].encode('utf-8'))
return list(set(errors))
def getDispatcherLogMessages(logfile):
logs = []
log_prefix = '<LAVA_DISPATCHER>'
action_begin = '[ACTION-B]'
level_pattern = re.compile('....-..-.. (..:..:.. .. ([A-Z]+): .*)')
for line in logfile:
# log_prefix not always start at beginning of the line
pos = line.find(log_prefix)
if pos == -1: # log_prefix not found
continue
if pos > 0: # remove log_prefix leading characters
line = line[pos:-1]
line = line[len(log_prefix):].strip()
match = level_pattern.match(line)
if not match:
continue
line = match.group(1)
if len(line) > 120:
line = line[:120] + '...'
if line.find(action_begin) != -1:
logs.append((match.group(2), line, "action"))
else:
logs.append((match.group(2), line, ""))
return logs
class Sections:
def __init__(self):
self.sections = []
self.cur_section_type = None
self.cur_section = []
def push(self, sect_type, line):
if sect_type != self.cur_section_type:
self.close()
self.cur_section_type = sect_type
self.cur_section.append(line)
def close(self):
if self.cur_section_type is not None:
self.sections.append(
(self.cur_section_type,
len(self.cur_section),
''.join(self.cur_section)))
self.cur_section_type = None
self.cur_section = []
def formatLogFile(logfile):
if not logfile:
return [('log', 1, "Log file is missing")]
sections = Sections()
for line in logfile:
line = line.replace('\r', '')
if not line:
continue
if line == 'Traceback (most recent call last):\n':
sections.push('traceback', line)
elif sections.cur_section_type == 'traceback':
sections.push('traceback', line)
if not line.startswith(' '):
sections.close()
continue
elif line.find("<LAVA_DISPATCHER>") != -1 \
or line.find("lava_dispatcher") != -1 \
or line.find("CriticalError:") != -1:
sections.push('log', line)
else:
sections.push('console', line)
sections.close()
return sections.sections
......@@ -1673,7 +1673,7 @@ class TestJob(RestrictedResource):
@property
def essential_role(self): # pylint: disable=too-many-return-statements
if not (self.is_multinode or self.is_vmgroup or self.is_pipeline):
if not self.is_multinode:
return False
data = yaml.load(self.definition)
# would be nice to use reduce here but raising and catching TypeError is slower
......@@ -1690,21 +1690,16 @@ class TestJob(RestrictedResource):
@property
def device_role(self): # pylint: disable=too-many-return-statements
if not (self.is_multinode or self.is_vmgroup):
return "Error"
if self.is_pipeline:
data = yaml.load(self.definition)
if 'protocols' not in data:
return 'Error'
if 'lava-multinode' not in data['protocols']:
return 'Error'
if 'role' not in data['protocols']['lava-multinode']:
return 'Error'
return data['protocols']['lava-multinode']['role']
json_data = simplejson.loads(self.definition)
if 'role' not in json_data:
if not self.is_multinode:
return "Error"
return json_data['role']
data = yaml.load(self.definition)
if 'protocols' not in data:
return 'Error'
if 'lava-multinode' not in data['protocols']:
return 'Error'
if 'role' not in data['protocols']['lava-multinode']:
return 'Error'
return data['protocols']['lava-multinode']['role']
def log_admin_entry(self, user, reason):
testjob_ct = ContentType.objects.get_for_model(TestJob)
......@@ -2009,10 +2004,6 @@ class TestJob(RestrictedResource):
jobs = TestJob.objects.filter(
target_group=self.target_group).order_by('id')
return jobs
elif self.is_vmgroup:
jobs = TestJob.objects.filter(
vm_group=self.vm_group).order_by('id')
return jobs
else:
return None
......@@ -2040,13 +2031,6 @@ class TestJob(RestrictedResource):
return None
return parent.actual_device.worker_host
@property
def is_vmgroup(self):
if self.is_pipeline:
return False
else:
return bool(self.vm_group)
@property
def display_id(self):
if self.sub_id:
......@@ -2074,8 +2058,7 @@ class TestJob(RestrictedResource):
which do not have ORIGINAL_DEFINITION ie., jobs that were submitted
before this attribute was introduced, return the DEFINITION.
"""
if self.original_definition and\
not (self.is_multinode or self.is_vmgroup):
if self.original_definition and not self.is_multinode:
return self.original_definition
else:
return self.definition
......@@ -2114,7 +2097,7 @@ class TestJob(RestrictedResource):
def ready_or_running(job):
return job.status in [TestJob.SUBMITTED, TestJob.RUNNING] and device_ready(job)
if self.is_multinode or self.is_vmgroup:
if self.is_multinode:
# FIXME: bad use of map - use list comprehension
return ready_or_running(self) and all(map(ready_or_running, self.sub_jobs_list))
else:
......
/* Polling support for pipeline jobs.
Ensure that these variables are defined in pages using this support.
var job_status_url = {% url 'lava.scheduler.job_status' pk=job.pk %};
var section_url = '{% url 'lava.scheduler.job_pipeline_sections' pk=job.pk %}'
For complete log support:
var logs_url = '{% url 'lava.scheduler.job_pipeline_incremental' pk=job.pk %}{% querystring 'summary'=0 %}'
For summary support:
var logs_url = '{% url 'lava.scheduler.job_pipeline_incremental' pk=job.pk %}{% querystring 'summary'=1 %}'
*/
var pollTimer = null;
var device = 1;
var job = 1;
var logs = 1;
var timing = 1;
var priority = 1;
var section_data = 1;
function isElementInViewport(el) {
//special bonus for those using jQuery
if (typeof jQuery === "function" && el instanceof jQuery) {
el = el[0];
}
var rect = el.getBoundingClientRect();
return rect.bottom > 0 &&
rect.right > 0 &&
rect.left < (window.innerWidth || $(window).width()) &&
rect.top < (window.innerHeight || $(window).height());
}
function poll () {
var el = $('#bottom');
var scroll = isElementInViewport(el);
$.ajaxSetup({
dataType: 'html',
global: false
});
if (job) {
$.ajax({
dataType: 'json',
url: job_status_url,
success: function (res_data, success, xhr) {
var data = eval(res_data);
$('#jobstatusdef').html(data['job_status']);
$('#statusblock').html(data['device']);
$('#jobtiming').html(data['timing']);
if (!('priority' in data)) {
$('#priority-choice').css('display', 'none')
}
if ('failure_comment' in data) {
$('#failure_comment').html(data['failure_comment']);
$('#failure_block').css('display', 'block');
}
if ('X-JobStatus' in data) {
job = null;
}
}
});
}
if (logs) {
$.ajax({
dataType: 'html',
url: logs_url,
success: function (data, success, xhr) {
$('#log_data').html(data);
if (xhr.getResponseHeader('X-Is-Finished')) {
$('#log_progress').find('img').css('display', 'none');
logs = null;
} else {
if (scroll) {
document.getElementById('bottom').scrollIntoView();
}
}
}
});
}
if (section_data) {
$.ajax({
dataType: 'html',
url: section_url,
success: function (data, success, xhr) {
$('#sectionlogs').html(data);
if (xhr.getResponseHeader('X-Sections')) {
section_data = null;
}
}
});
}
if ((logs !== null) || (job !== null) || (section_data !== null)) {
pollTimer = setTimeout(poll, 6000);
}
}
$(document).ready(
function () {
pollTimer = setTimeout(poll, 6000);
}
);
import os
import json
import logging
from django.template import defaultfilters as filters
from django.utils.safestring import mark_safe
......
......@@ -10,7 +10,7 @@
</div>
{% endif %}
{% if can_admin and device.is_pipeline and template_mismatch %}
{% if can_admin and template_mismatch %}
<div class="alert alert-warning">
<strong>Configuration Error: template-mismatch.</strong>
<p>The '{{ device.device_type }}' device type has no matching jinja2 template.
......@@ -83,10 +83,8 @@
{% endif %}
<dt>Worker</dt>
<dd><a href="{{ device.worker_host.get_absolute_url }}">{{ device.worker_host.hostname }}</a></dd>
{% if device.is_pipeline %}
<dt>Device dictionary</dt>
<dd><a href="{% url 'lava.scheduler.device.dictionary' device.pk %}"><span class="glyphicon glyphicon-eye-open"></span></a></dd>
{% endif %}
</dl>
</div>
<div class="col-md-4">
......
{% load utils %}
{% load django_tables2 %}
{% spaceless %}
{% get_pipeline_sections pipeline_data as sections %}
<div class="col-md-6">
<h4>Sections</h4>
<ul class="nav nav-pills">
{% for item in sections %}
{% if 'section' in request.GET %}
{% for section, level in item.items %}
{% if section in request.GET.section %}
<li><a href="{% querystring "section"=section %}" class="btn btn-primary">{{ section }}</a></li>
{% else %}
<li><a href="{% querystring "section"=section %}" class="btn btn-info">{{ section }}</a></li>
{% endif %}
{% endfor %}
{% else %}
{% for section, level in item.items %}
{% if section == default_section %}
<li><a href="{% querystring "section"=section %}" class="btn btn-primary">{{ section }}</a></li>
{% else %}
<li><a href="{% querystring "section"=section %}" class="btn btn-info">{{ section }}</a></li>
{% endif %}
{% endfor %}
{% endif %}
{% endfor %}
</ul>
</div>
<div class="col-md-6">
<h4>Logging levels</h4>
<ul class="nav nav-pills">
<li>&nbsp;</li>
{% if 'off' in request.GET.warning %}
<li><a href="{% querystring "warning"="" %}" class="btn btn-info">Warning</a></li>
{% else %}
<li><a href="{% querystring "warning"="off" %}" class="btn btn-warning">Warning</a></li>
{% endif %}
{% if 'off' in request.GET.info %}
<li><a href="{% querystring "info"="" %}" class="btn btn-info">Info</a></li>
{% else %}
<li><a href="{% querystring "info"="off" %}" class="btn btn-primary">Info</a></li>
{% endif %}
{% if 'on' in request.GET.debug %}
<li><a href="{% querystring "debug"="" %}" class="btn btn-primary">Debug</a></li>
{% else %}
<li><a href="{% querystring "debug"="on" %}" class="btn btn-info">Debug</a></li>
{% endif %}
</ul>
</div>
{% endspaceless %}
{% extends "layouts/content-bootstrap.html" %}
{% block content %}
<h2>Expanded job definition file - {% if job.is_multinode %}{{ job.sub_id }}{% else %}{{ job.id }}{% endif %}
<a class="btn btn-xs btn-info" href="{% url 'lava.scheduler.job.definition.plain' job.pk %}" title="Download as text file"><span class="glyphicon glyphicon-download"></span> download</a></h2>
This job has repeated actions. Resubmit action will use Original Definition.
<div class="row">
<div class="col-md-4">
<h4 class="modal-header">Device</h4>
<dl class="dl-horizontal">
{% if job.actual_device %}
<dt>Name</dt>
<dd><a href="{{ job.actual_device.get_absolute_url }}">{{ job.actual_device.hostname }}</a>{% if not job.is_vmgroup %} <a
href="{% url 'lava.scheduler.device_report' job.actual_device.pk %}">(reports)</a>{% endif %}</dd>
{% endif %}
{% if job.requested_device %}
<dt>Type</dt>
<dd><a href="{{ job.requested_device.device_type.get_absolute_url }}">{{ job.requested_device.device_type }}</a>{% if not job.is_vmgroup %} <a
href="{% url 'lava.scheduler.device_type_report' job.requested_device.device_type %}">(reports)</a>{% endif %}</dd>
{% elif job.requested_device_type %}
<dt>Requested type</dt>
<dd><a href="{{ job.requested_device_type.get_absolute_url }}">{{ job.requested_device_type }}</a> <a href="{% url 'lava.scheduler.device_type_report' job.requested_device_type %}">(reports)</a></dd>
{% endif %}
</dl>
</div>
<div class="col-md-4">
<h4 class="modal-header">Logs</h4>
<div class="row">
<div class="col-md-6">
<ul class="nav nav-pills nav-stacked">
{% if job_file_present %}
<li><a href="{{ job.get_absolute_url }}" class="btn btn-primary">Summary</a></li>
{% endif %}
</ul>
</div>
<div class="col-md-6">
<ul class="nav nav-pills nav-stacked">
{% if job_file_present %}
<li><a href="{% url 'lava.scheduler.job.log_file' job.pk %}" class="btn btn-primary">Complete log</a></li>
{% endif %}
{% if job.results_link %}
<li><a href="{{ job.results_link }}" class="btn btn-success">Results Bundle</a></li>
{% endif %}
</ul>
</div>
</div>