Commit ae4d4776 authored by Rémi Duraffort's avatar Rémi Duraffort
Browse files

LAVA-757 Move device dictionaries to file system

Change-Id: Ibbc7fe21cf7972387df6486853edbfdce3c2dffa
parent 52eb1e1c
......@@ -598,10 +598,9 @@ can disable V1 test jobs on that worker.
This list may include ``lava-coordinator``, ``lava-server-doc``,
``libapache2-mod-uwsgi``, ``libapache2-mod-wsgi``, ``postgresql``,
``python-django-auth-ldap``, ``python-django-kvstore``,
``python-django-restricted-resource``, ``python-django-tables2``,
``python-ldap``, ``python-markdown``, ``uwsgi-core`` but may also remove
others. Check the list carefully.
``python-django-auth-ldap``, ``python-django-restricted-resource``,
``python-django-tables2``, ``python-ldap``, ``python-markdown``,
``uwsgi-core`` but may also remove others. Check the list carefully.
#. Check lava-slave is still pinging the master correctly.
......
import os
import unittest
from lava_scheduler_app.models import Device
def suite():
return unittest.TestLoader().discover(
......@@ -7,3 +10,8 @@ def suite():
pattern="*.py",
top_level_dir="lava_results_app"
)
Device.CONFIG_PATH = os.path.abspath(os.path.join(os.path.dirname(__file__),
"..", "..", "lava_scheduler_app",
"tests", "devices"))
......@@ -55,7 +55,7 @@ class TestMetaTypes(TestCaseWithFactory):
job_ctx = job_def.get('context', {})
job_ctx.update({'no_kvm': True}) # override to allow unit tests on all types of systems
device = Device.objects.get(hostname='fakeqemu1')
device_config = device.load_device_configuration(job_ctx, system=False) # raw dict
device_config = device.load_configuration(job_ctx) # raw dict
parser = JobParser()
obj = PipelineDevice(device_config, device.hostname)
pipeline_job = parser.parse(job.definition, obj, job.id, None, "", output_dir='/tmp')
......@@ -183,7 +183,7 @@ class TestMetaTypes(TestCaseWithFactory):
job_ctx = job_def.get('context', {})
job_ctx.update({'no_kvm': True}) # override to allow unit tests on all types of systems
device = Device.objects.get(hostname='fakeqemu1')
device_config = device.load_device_configuration(job_ctx, system=False) # raw dict
device_config = device.load_configuration(job_ctx) # raw dict
parser = JobParser()
obj = PipelineDevice(device_config, device.hostname)
pipeline_job = parser.parse(job.definition, obj, job.id, None, "", output_dir='/tmp')
......@@ -236,7 +236,7 @@ class TestMetaTypes(TestCaseWithFactory):
job_ctx = job_def.get('context', {})
job_ctx.update({'no_kvm': True}) # override to allow unit tests on all types of systems
device = Device.objects.get(hostname='fakeqemu1')
device_config = device.load_device_configuration(job_ctx, system=False) # raw dict
device_config = device.load_configuration(job_ctx) # raw dict
parser = JobParser()
obj = PipelineDevice(device_config, device.hostname)
pipeline_job = parser.parse(job.definition, obj, job.id, None, "", output_dir='/tmp')
......@@ -269,7 +269,7 @@ class TestMetaTypes(TestCaseWithFactory):
job_ctx = job_def.get('context', {})
job_ctx.update({'no_kvm': True}) # override to allow unit tests on all types of systems
device = Device.objects.get(hostname='fakeqemu1')
device_config = device.load_device_configuration(job_ctx, system=False) # raw dict
device_config = device.load_configuration(job_ctx) # raw dict
parser = JobParser()
obj = PipelineDevice(device_config, device.hostname)
pipeline_job = parser.parse(job.definition, obj, job.id, None, "", output_dir='/tmp')
......@@ -309,7 +309,7 @@ class TestMetaTypes(TestCaseWithFactory):
job_ctx = job_def.get('context', {})
job_ctx.update({'no_kvm': True}) # override to allow unit tests on all types of systems
device = Device.objects.get(hostname='fakeqemu1')
device_config = device.load_device_configuration(job_ctx, system=False) # raw dict
device_config = device.load_configuration(job_ctx) # raw dict
parser = JobParser()
obj = PipelineDevice(device_config, device.hostname)
pipeline_job = parser.parse(job.definition, obj, job.id, None, "", output_dir='/tmp')
......
......@@ -9,7 +9,7 @@ from lava_results_app.models import (
from lava_results_app.dbutils import map_scanned_results
from lava_scheduler_app.models import (
TestJob, Device,
DeviceType, DeviceDictionary,
DeviceType
)
from django_testscenarios.ubertest import TestCase as DjangoTestCase
......@@ -38,7 +38,6 @@ class ModelFactory(object):
# make sure the DB is in a clean state wrt devices and jobs
Device.objects.all().delete()
TestJob.objects.all().delete()
[item.delete() for item in DeviceDictionary.object_list()]
User.objects.all().delete()
def make_user(self):
......@@ -57,11 +56,6 @@ class ModelFactory(object):
def make_job_yaml(self, **kw):
return yaml.safe_dump(self.make_job_data(**kw))
def make_fake_qemu_device(self, hostname='fakeqemu1'): # pylint: disable=no-self-use
qemu = DeviceDictionary(hostname=hostname)
qemu.parameters = {'extends': 'qemu.jinja2', 'arch': 'amd64'}
qemu.save()
def make_device_type(self, name='qemu'):
(device_type, created) = DeviceType.objects.get_or_create(name=name)
if created:
......@@ -76,7 +70,6 @@ class ModelFactory(object):
if tags and type(tags) != list:
tags = []
device = Device(device_type=device_type, is_public=is_public, hostname=hostname, is_pipeline=True, **kw)
self.make_fake_qemu_device(hostname)
if tags:
device.tags = tags
logging.debug("making a device of type %s %s %s with tags '%s'"
......
......@@ -231,6 +231,9 @@ class DeviceAdmin(admin.ModelAdmin):
exclusive_device.boolean = True
exclusive_device.short_description = "v2 only"
def device_dictionary_jinja(self, obj):
return obj.load_configuration(output_format="raw")
fieldsets = (
('Properties', {
'fields': (('device_type', 'hostname'), 'worker_host', 'device_version')}),
......@@ -239,11 +242,11 @@ class DeviceAdmin(admin.ModelAdmin):
('Status', {
'fields': (('status', 'health_status'), ('last_health_report_job', 'current_job'))}),
('Advanced properties', {
'fields': ('description', 'tags', ('device_dictionary_yaml', 'device_dictionary_jinja')),
'fields': ('description', 'tags', ('device_dictionary_jinja')),
'classes': ('collapse', )
}),
)
readonly_fields = ('device_dictionary_yaml', 'device_dictionary_jinja')
readonly_fields = ('device_dictionary_jinja', )
list_display = ('hostname', 'device_type', 'current_job', 'worker_host',
'status', 'health_status', 'has_health_check',
'health_check_enabled', 'is_public', 'is_pipeline',
......
import xmlrpclib
import yaml
import jinja2
from simplejson import JSONDecodeError
from django.conf import settings
from django.core.exceptions import PermissionDenied
......@@ -13,23 +12,19 @@ from lava_scheduler_app.models import (
JSONDataError,
DevicesUnavailableException,
TestJob,
DeviceDictionary,
)
from lava_scheduler_app.views import (
get_restricted_job
)
from lava_scheduler_app.dbutils import device_type_summary
from lava_scheduler_app.utils import (
devicedictionary_to_jinja2,
jinja2_to_devicedictionary,
prepare_jinja_template,
from lava_scheduler_app.dbutils import (
device_type_summary,
testjob_submission
)
from lava_scheduler_app.schema import (
validate_submission,
validate_device,
SubmissionException,
)
from lava_scheduler_app.dbutils import testjob_submission
# functions need to be members to be exposed in the API
# pylint: disable=no-self-use
......@@ -1018,19 +1013,17 @@ class SchedulerAPI(ExposedAPI):
raise xmlrpclib.Fault(400, "Bad request: Device hostname was not "
"specified.")
element = DeviceDictionary.get(device_hostname)
if element is None:
try:
device = Device.objects.get(hostname=device_hostname)
except Device.DoesNotExist:
raise xmlrpclib.Fault(404, "Specified device not found.")
data = devicedictionary_to_jinja2(element.parameters,
element.parameters['extends'])
template = prepare_jinja_template(device_hostname, data, system_path=True)
device_configuration = template.render()
config = device.load_configuration(output_format="yaml")
# validate against the device schema
validate_device(yaml.load(device_configuration))
validate_device(yaml.load(config))
return xmlrpclib.Binary(device_configuration.encode('UTF-8'))
return xmlrpclib.Binary(config.encode('UTF-8'))
def import_device_dictionary(self, hostname, jinja_str):
"""
......@@ -1043,14 +1036,13 @@ class SchedulerAPI(ExposedAPI):
[superuser only]
Import or update the device dictionary key value store for a
pipeline device.
This action will be logged.
Arguments
---------
`device_hostname`: string
Device hostname to update.
`jinja_str`: string
Jinja2 settings to store in the DeviceDictionary
Device configuration as Jinja2
Return value
------------
......@@ -1068,35 +1060,12 @@ class SchedulerAPI(ExposedAPI):
raise xmlrpclib.Fault(
404, "Device '%s' was not found." % hostname
)
try:
device_data = jinja2_to_devicedictionary(jinja_str)
except (ValueError, KeyError, TypeError):
raise xmlrpclib.Fault(
400, "Unable to parse specified jinja string"
)
if not device_data or 'extends' not in device_data:
raise xmlrpclib.Fault(
400, "Invalid device dictionary content - %s - not updating." % jinja_str
)
try:
template = prepare_jinja_template(hostname, jinja_str, system_path=True)
except (jinja2.TemplateError, yaml.YAMLError, IOError) as exc:
raise xmlrpclib.Fault(
400, "Template error: %s" % exc
if not device.save_configuration(jinja_str):
raise xmlrpclib.Faul(
400, "Unable to store the configuration for %s on disk" % hostname
)
if not template:
raise xmlrpclib.Fault(400, "Empty template")
element = DeviceDictionary.get(hostname)
msg = ''
if element is None:
msg = "Adding new device dictionary for %s\n" % hostname
element = DeviceDictionary(hostname=hostname)
element.hostname = hostname
element.parameters = device_data
element.save()
msg += "Device dictionary updated for %s\n" % hostname
device.log_admin_entry(self.user, msg)
return msg
return "Device dictionary updated for %s" % hostname
def export_device_dictionary(self, hostname):
"""
......@@ -1136,14 +1105,14 @@ class SchedulerAPI(ExposedAPI):
raise xmlrpclib.Fault(
400, "Device '%s' is not a pipeline device" % hostname
)
device_dict = DeviceDictionary.get(hostname)
device_dict = device.load_configuration(output_format="raw")
if not device_dict:
raise xmlrpclib.Fault(
404, "Device '%s' does not have a device dictionary" % hostname
)
device_dict = device_dict.to_dict()
jinja_str = devicedictionary_to_jinja2(device_dict['parameters'], device_dict['parameters']['extends'])
return xmlrpclib.Binary(jinja_str.encode('UTF-8'))
return xmlrpclib.Binary(device_dict.encode('UTF-8'))
def validate_pipeline_devices(self, name=None):
"""
......@@ -1203,24 +1172,13 @@ class SchedulerAPI(ExposedAPI):
results = {}
for device in devices:
key = str(device.hostname)
element = DeviceDictionary.get(device.hostname)
if element is None:
config = device.load_configuration(output_format="yaml")
if config is None:
results[key] = {'Invalid': "Missing device dictionary"}
continue
data = devicedictionary_to_jinja2(element.parameters,
element.parameters['extends'])
if data is None:
results[key] = {'Invalid': 'Unable to convert device dictionary into jinja2'}
continue
try:
template = prepare_jinja_template(device.hostname, data, system_path=True)
device_configuration = template.render()
except jinja2.TemplateError as exc:
results[key] = {'Invalid': exc}
continue
try:
# validate against the device schema
validate_device(yaml.load(device_configuration))
validate_device(yaml.load(config))
except SubmissionException as exc:
results[key] = {'Invalid': exc}
continue
......
......@@ -18,7 +18,6 @@ from django.utils import timezone
from linaro_django_xmlrpc.models import AuthToken
from lava_scheduler_app.models import (
Device,
DeviceDictionary,
DevicesUnavailableException,
DeviceType,
is_deprecated_json,
......@@ -40,23 +39,25 @@ def match_vlan_interface(device, job_def):
return False
interfaces = []
logger = logging.getLogger('dispatcher-master')
device_dict = DeviceDictionary.get(device.hostname).to_dict()
if 'tags' not in device_dict['parameters']:
logger.error("%s has no tags in the device dictionary parameters", device.hostname)
device_dict = device.load_configuration()
if not device_dict or device_dict.get('parameters', {}).get('interfaces', None) is None:
return False
for vlan_name in job_def['protocols']['lava-vland']:
tag_list = job_def['protocols']['lava-vland'][vlan_name]['tags']
for interface, tags in device_dict['parameters']['tags'].iteritems():
for interface in device_dict['parameters']['interfaces']:
tags = device_dict['parameters']['interfaces'][interface]['tags']
if not tags:
continue
logger.info(
"Job requests %s for %s, device %s provides %s for %s",
tag_list, vlan_name, device.hostname, tags, interface)
# tags & job tags must equal job tags
# device therefore must support all job tags, not all job tags available on the device need to be specified
if set(tags) & set(tag_list) == set(tag_list) and interface not in interfaces:
logger.info("Matched vlan %s to interface %s on %s", vlan_name, interface, device)
interfaces.append(interface)
# matched, do not check any further interfaces of this device for this vlan
break
logger.info("Matched: %s", (len(interfaces) == len(job_def['protocols']['lava-vland'].keys())))
return len(interfaces) == len(job_def['protocols']['lava-vland'].keys())
......@@ -764,7 +765,7 @@ def select_device(job, dispatchers): # pylint: disable=too-many-return-statemen
device = job.actual_device
try:
device_config = device.load_device_configuration(job_ctx) # raw dict
device_config = device.load_configuration(job_ctx) # raw dict
except (jinja2.TemplateError, yaml.YAMLError, IOError) as exc:
logger.error("[%d] jinja2 error: %s", job.id, exc)
msg = "Administrative error. Unable to parse device configuration: '%s'" % exc
......@@ -856,3 +857,28 @@ def device_type_summary(visible=None):
)
),).order_by('device_type')
return devices
def load_devicetype_template(device_type_name):
"""
Loads the bare device-type template as a python dictionary object for
representation within the device_type templates.
No device-specific details are parsed - default values only, so some
parts of the dictionary may be unexpectedly empty. Not to be used when
rendering device configuration for a testjob.
:param device_type_name: DeviceType.name (string)
:param path: optional alternative path to templates
:return: None or a dictionary of the device type template.
"""
path = os.path.dirname(Device.CONFIG_PATH)
type_loader = jinja2.FileSystemLoader([os.path.join(path, 'device-types')])
env = jinja2.Environment(
loader=jinja2.ChoiceLoader([type_loader]),
trim_blocks=True)
try:
template = env.get_template("%s.jinja2" % device_type_name)
except jinja2.TemplateNotFound:
return None
if not template:
return None
return yaml.load(template.render())
# Copyright (C) 2015 Linaro Limited
#
# Author: Neil Williams <neil.williams@linaro.org>
#
# This file is part of LAVA Dispatcher.
#
# LAVA Dispatcher is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
# (at your option) any later version.
#
# LAVA Dispatcher is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along
# with this program; if not, see <http://www.gnu.org/licenses>.
# pylint gets confused: commands have no shebang, but the file is not a module.
# import order is a new pylint tag for putting system imports before package
# imports but there is no functional benefit.
# pylint: disable=invalid-name,wrong-import-order
import os
import sys
import yaml
from django.core.management.base import BaseCommand
from lava_scheduler_app.models import DeviceDictionary
from lava_scheduler_app.utils import (
devicedictionary_to_jinja2,
jinja2_to_devicedictionary,
prepare_jinja_template,
)
from lava_scheduler_app.schema import validate_device, SubmissionException
def parse_template(device_file):
if not os.path.exists(os.path.realpath(device_file)):
print "Unable to find file '%s'\n" % device_file
sys.exit(2)
with open(device_file, 'r') as fileh:
content = fileh.read()
return jinja2_to_devicedictionary(content)
class Command(BaseCommand):
logger = None
help = "LAVA Device Dictionary I/O tool"
def add_arguments(self, parser):
parser.add_argument('--hostname', help="Hostname of the device to use")
parser.add_argument('--import', help="create new or update existing entry")
parser.add_argument(
'--path',
default='/etc/lava-server/dispatcher-config/',
help='path to the lava-server jinja2 device type templates')
parser.add_argument(
'--export',
action="store_true",
help="export existing entry")
parser.add_argument(
'--review',
action="store_true",
help="review the generated device configuration")
def handle(self, *args, **options):
"""
Accept options via lava-server manage which provides access
to the database.
"""
hostname = options['hostname']
if hostname is None:
self.stderr.write("Please specify a hostname")
sys.exit(2)
if options['import']:
data = parse_template(options['import'])
element = DeviceDictionary.get(hostname)
if element is None:
self.stdout.write("Adding new device dictionary for %s" %
hostname)
element = DeviceDictionary(hostname=hostname)
element.hostname = hostname
element.parameters = data
element.save()
self.stdout.write("Device dictionary updated for %s" % hostname)
elif options['export'] or options['review']:
element = DeviceDictionary.get(hostname)
if element is None:
self.stderr.write("Unable to export - no dictionary found for '%s'" %
hostname)
sys.exit(2)
else:
data = devicedictionary_to_jinja2(
element.parameters,
element.parameters['extends']
)
if not options['review']:
self.stdout.write(data)
else:
template = prepare_jinja_template(hostname, data, system_path=False, path=options['path'])
device_configuration = template.render()
# validate against the device schema
try:
validate_device(yaml.load(device_configuration))
except (yaml.YAMLError, SubmissionException) as exc:
self.stderr.write("Invalid template: %s" % exc)
self.stdout.write(device_configuration)
else:
self.stderr.write("Please specify one of --import, --export or --review")
sys.exit(1)
# -*- coding: utf-8 -*-
from __future__ import unicode_literals
from django.db import migrations
import base64
import errno
import os
import pickle
import pprint
def devicedictionary_to_jinja2(data_dict, extends):
"""
Formats a DeviceDictionary as a jinja2 string dictionary
Arguments:
data_dict: the DeviceDictionary.to_dict()
extends: the name of the jinja2 device_type template file to extend.
(including file name extension / suffix) which jinja2 will later
assume to be in the jinja2 device_types folder
"""
if not isinstance(data_dict, dict):
return None
pp = pprint.PrettyPrinter(indent=0, width=80) # simulate human readable input
data = u'{%% extends \'%s\' %%}\n' % extends
for key, value in data_dict.items():
if key == 'extends':
continue
data += u'{%% set %s = %s %%}\n' % (str(key), pp.pformat(value).strip())
return data
def migrate_device_dict_to_filesystem(apps, schema_editor):
# Get the right version of the models
Device = apps.get_model("lava_scheduler_app", "Device")
DeviceDictionaryTable = apps.get_model("lava_scheduler_app", "DeviceDictionaryTable")
dd_dir = "/etc/lava-server/dispatcher-config/devices"
# Create the directory
try:
os.mkdir(dd_dir, 0o755)
except OSError as exc:
if exc.errno != errno.EEXIST:
pass
# Load the device dictionaries
DDT = {}
for device_dict in DeviceDictionaryTable.objects.all():
hostname = device_dict.kee.replace('__KV_STORE_::lava_scheduler_app.models.DeviceDictionary:', '')
value64 = device_dict.value
valuepickled = base64.b64decode(value64)
value = pickle.loads(valuepickled)
DDT[hostname] = devicedictionary_to_jinja2(value['parameters'], value['parameters']['extends'])
# Dump the device dictionaries to file system
for device in Device.objects.filter(is_pipeline=True).order_by('hostname'):
if device.hostname not in DDT:
print("Skip %s" % hostname)
continue
device_dict = DDT[device.hostname]
with open(os.path.join(dd_dir, "%s.jinja2" % device.hostname), "w") as f_out:
f_out.write(device_dict)
class Migration(migrations.Migration):
dependencies = [
('lava_scheduler_app', '0026_devicetype_disable_health_check'),
]
operations = [
migrations.RunPython(migrate_device_dict_to_filesystem),
migrations.DeleteModel(
name='PipelineStore',
),
migrations.DeleteModel(
name='DeviceDictionaryTable',
),
]
......@@ -33,8 +33,6 @@ from django.dispatch import receiver
from django.template.loader import render_to_string
from django.utils.translation import ugettext_lazy as _
from django.shortcuts import get_object_or_404
from django_kvstore import models as kvmodels
from django_kvstore import get_kvstore
from django_restricted_resource.models import (
......@@ -528,64 +526,6 @@ class Worker(models.Model):
raise ValueError("Worker node unavailable")
class DeviceDictionaryTable(models.Model):
kee = models.CharField(max_length=255)
value = models.TextField()
def __unicode__(self):
return self.kee.replace('__KV_STORE_::lava_scheduler_app.models.DeviceDictionary:', '')
def lookup_device_dictionary(self):
val = self.kee
msg = val.replace('__KV_STORE_::lava_scheduler_app.models.DeviceDictionary:', '')
return DeviceDictionary.get(msg)