Commit f5b8d1e4 authored by Rémi Duraffort's avatar Rémi Duraffort
Browse files

compat: use yaml C dumper/loader when available


Signed-off-by: default avatarRémi Duraffort <remi.duraffort@linaro.org>
parent 07a931e5
......@@ -45,6 +45,14 @@ try:
from yaml import CSafeLoader as SafeLoader
except ImportError:
from yaml import SafeLoader
try:
from yaml import CSafeDumper as SafeDumper
except ImportError:
from yaml import SafeDumper
try:
from yaml import CDumper as Dumper
except ImportError:
from yaml import Dumper
# handle compatibility for yaml.load
......@@ -55,3 +63,13 @@ def yaml_load(data):
# handle compatibility for yaml.safe_load
def yaml_safe_load(data):
return yaml.load(data, Loader=SafeLoader)
# handle compatibility for yaml.dump
def yaml_dump(data, *args, **kwargs):
return yaml.dump(data, *args, Dumper=Dumper, **kwargs)
# handle compatibility for yaml.safe_dump
def yaml_safe_dump(data, *args, **kwargs):
return yaml.dump(data, *args, Dumper=SafeDumper, **kwargs)
......@@ -18,8 +18,7 @@
# along
# with this program; if not, see <http://www.gnu.org/licenses>.
import yaml
from lava_common.compat import yaml_safe_dump
from lava_dispatcher.action import Pipeline
from lava_dispatcher.actions.deploy import DeployAction
from lava_dispatcher.actions.deploy.download import DownloaderAction
......@@ -101,10 +100,10 @@ class FlasherAction(DeployAction):
)
# Add some device configuration
substitutions["{DEVICE_INFO}"] = yaml.dump(
substitutions["{DEVICE_INFO}"] = yaml_safe_dump(
self.job.device.get("device_info", [])
)
substitutions["{STATIC_INFO}"] = yaml.dump(
substitutions["{STATIC_INFO}"] = yaml_safe_dump(
self.job.device.get("static_info", [])
)
......
......@@ -19,8 +19,8 @@
# with this program; if not, see <http://www.gnu.org/licenses>.
import os
import yaml
from lava_dispatcher.logical import Deployment
from lava_common.compat import yaml_safe_dump
from lava_common.exceptions import LAVABug
from lava_common.utils import debian_package_version
from lava_dispatcher.action import JobError, Pipeline
......@@ -29,6 +29,7 @@ from lava_dispatcher.actions.deploy.overlay import OverlayAction
from lava_dispatcher.actions.deploy.apply_overlay import ApplyLxcOverlay
from lava_dispatcher.actions.deploy.environment import DeployDeviceEnvironment
from lava_dispatcher.actions.boot.lxc import LxcStartAction, LxcStopAction
from lava_dispatcher.logical import Deployment
from lava_dispatcher.utils.shell import which
from lava_dispatcher.protocols.lxc import LxcProtocol
from lava_common.constants import (
......@@ -282,7 +283,7 @@ class LxcCreateUdevRuleAction(DeployAction):
device_info = self.job.device.get("device_info", [])
device_info_file = os.path.join(self.mkdtemp(), "device-info.yaml")
with open(device_info_file, "w") as device_info_obj:
yaml.dump(device_info, device_info_obj)
yaml_safe_dump(device_info, device_info_obj)
self.logger.debug(
"device info file '%s' created with:\n %s", device_info_file, device_info
)
......
......@@ -21,13 +21,12 @@
import os
import io
import re
import yaml
import base64
import hashlib
import tarfile
import shutil
from lava_common.compat import yaml_safe_load
from lava_common.compat import yaml_safe_dump, yaml_safe_load
from lava_common.decorators import nottest
from lava_common.exceptions import InfrastructureError, JobError, LAVABug, TestError
from lava_dispatcher.action import Action, Pipeline
......@@ -377,7 +376,7 @@ class InlineRepoAction(RepoAction):
if yaml_dirname != "":
os.makedirs(os.path.join(runner_path, yaml_dirname))
with open(yaml_file, "w") as test_file:
data = yaml.safe_dump(testdef)
data = yaml_safe_dump(testdef)
sha1.update(data.encode("utf-8"))
test_file.write(data)
......@@ -772,7 +771,7 @@ class TestOverlayAction(TestAction):
# FIXME: change lava-test-runner to accept a variable instead of duplicating the YAML?
with open("%s/testdef.yaml" % runner_path, "w") as run_file:
yaml.safe_dump(testdef, run_file)
yaml_safe_dump(testdef, run_file)
# write out the UUID of each test definition.
# FIXME: is this necessary any longer?
......@@ -784,7 +783,7 @@ class TestOverlayAction(TestAction):
content = self.get_namespace_data(
action="test", label=self.test_uuid, key="testdef_metadata"
)
metadata.write(yaml.safe_dump(content))
metadata.write(yaml_safe_dump(content))
# Need actions for the run.sh script (calling parameter support in base class)
# and install script (also calling parameter support here.)
......
......@@ -20,12 +20,12 @@
import re
import time
import yaml
import decimal
import logging
import pexpect
from collections import OrderedDict
from lava_common.compat import yaml_safe_dump
from lava_common.decorators import nottest
from lava_common.exceptions import JobError, InfrastructureError, TestError, LAVABug
from lava_dispatcher.actions.test import TestAction
......@@ -361,7 +361,7 @@ class TestShellAction(TestAction):
# Only print if the report is not empty
if self.report:
self.logger.debug(yaml.dump(self.report, default_flow_style=False))
self.logger.debug(yaml_safe_dump(self.report, default_flow_style=False))
if self.errors:
raise TestError(self.errors)
return connection
......
......@@ -21,11 +21,12 @@
import datetime
import logging
import yaml
import zmq
import zmq.auth
from zmq.utils.strtypes import b
from lava_common.compat import yaml_dump
class ZMQPushHandler(logging.Handler):
def __init__(self, logging_url, master_cert, slave_cert, job_id, socks_proxy, ipv6):
......@@ -107,12 +108,8 @@ class YAMLLogger(logging.Logger):
# Set width to a really large value in order to always get one line.
# But keep this reasonable because the logs will be loaded by CLoader
# that is limited to around 10**7 chars
data_str = yaml.dump(
data,
default_flow_style=True,
default_style='"',
width=10 ** 6,
Dumper=yaml.CDumper,
data_str = yaml_dump(
data, default_flow_style=True, default_style='"', width=10 ** 6
)[:-1]
# Test the limit and skip if the line is too long
if len(data_str) >= 10 ** 6:
......@@ -120,12 +117,8 @@ class YAMLLogger(logging.Logger):
data["msg"] = "<line way too long ...>"
else:
data["msg"] = {"skip": "line way too long ..."}
data_str = yaml.dump(
data,
default_flow_style=True,
default_style='"',
width=10 ** 6,
Dumper=yaml.CDumper,
data_str = yaml_dump(
data, default_flow_style=True, default_style='"', width=10 ** 6
)[:-1]
self._log(level, data_str, ())
......
......@@ -24,9 +24,8 @@ import time
import jinja2
import voluptuous
import unittest
import yaml
from lava_common.compat import yaml_safe_load
from lava_common.compat import yaml_safe_dump, yaml_safe_load
from lava_common.exceptions import (
InfrastructureError,
JobError,
......@@ -53,7 +52,7 @@ class StdoutTestCase(unittest.TestCase):
if cls.update_ref:
sys.stderr.write("WARNING: modifying pipeline references!")
with open(y_file, "w") as describe:
yaml.dump(
yaml_safe_dump(
job.pipeline.describe(False), describe, default_flow_style=None
)
with open(y_file, "r") as f_ref:
......@@ -226,7 +225,7 @@ class Factory:
print("#######")
try:
parser = JobParser()
job = parser.parse(yaml.dump(job_data), device, 4999, None, "")
job = parser.parse(yaml_safe_dump(job_data), device, 4999, None, "")
except (ConfigurationError, TypeError) as exc:
print("####### Parser exception ########")
print(device)
......@@ -264,7 +263,7 @@ class Factory:
if validate:
validate_job(job_data, strict=False)
try:
job = parser.parse(yaml.dump(job_data), device, 4212, None, "")
job = parser.parse(yaml_safe_dump(job_data), device, 4212, None, "")
job.logger = DummyLogger()
except LAVAError as exc:
print(exc)
......@@ -434,14 +433,14 @@ class TestPipeline(StdoutTestCase):
parser = JobParser()
(rendered, data) = factory.create_device("kvm01.jinja2")
device = yaml_safe_load(rendered)
job = parser.parse(yaml.dump(job_def), device, 4212, None, "")
job = parser.parse(yaml_safe_dump(job_def), device, 4212, None, "")
self.assertIsNotNone(job)
job_def["compatibility"] = job.compatibility + 1
self.assertRaises(
JobError, parser.parse, yaml.dump(job_def), device, 4212, None, ""
JobError, parser.parse, yaml_safe_dump(job_def), device, 4212, None, ""
)
job_def["compatibility"] = 0
job = parser.parse(yaml.dump(job_def), device, 4212, None, "")
job = parser.parse(yaml_safe_dump(job_def), device, 4212, None, "")
self.assertIsNotNone(job)
def test_pipeline_actions(self):
......
......@@ -22,7 +22,6 @@ import re
import os
import glob
import stat
import yaml
import shutil
import pexpect
import tempfile
......@@ -30,7 +29,7 @@ import unittest
import subprocess # nosec - unit test support.
from unittest.mock import patch
from lava_common.compat import yaml_safe_load
from lava_common.compat import yaml_safe_dump, yaml_safe_load
from lava_common.decorators import nottest
from lava_common.exceptions import InfrastructureError
from lava_dispatcher.power import FinalizeAction
......@@ -154,7 +153,7 @@ class TestDefinitionHandlers(StdoutTestCase):
data = [block["test"] for block in content["actions"] if "test" in block][0]
definitions = [block for block in data["definitions"] if "path" in block][0]
definitions["name"] = "smoke tests"
job = parser.parse(yaml.dump(content), device, 4212, None, "")
job = parser.parse(yaml_safe_dump(content), device, 4212, None, "")
deploy = [
action for action in job.pipeline.actions if action.name == "deployimages"
][0]
......
......@@ -20,10 +20,10 @@
import os
import yaml
import unittest
from unittest.mock import patch
from lava_common.compat import yaml_safe_load
from lava_common.compat import yaml_safe_dump, yaml_safe_load
from lava_dispatcher.device import NewDevice
from lava_dispatcher.parser import JobParser
from lava_dispatcher.actions.boot.ipxe import BootloaderAction
......@@ -339,7 +339,7 @@ class TestBootloaderAction(StdoutTestCase):
0
]
self.assertIsNotNone(boot)
sample_job_string = yaml.dump(sample_job_data)
sample_job_string = yaml_safe_dump(sample_job_data)
job = parser.parse(sample_job_string, device, 4212, None, "")
job.logger = DummyLogger()
job.validate()
......
......@@ -23,10 +23,9 @@ import os
import glob
import time
import unittest
import yaml
import pexpect
from lava_common.compat import yaml_safe_load
from lava_common.compat import yaml_safe_dump, yaml_safe_load
from lava_common.constants import SYS_CLASS_KVM
from lava_common.exceptions import JobError, InfrastructureError
from lava_dispatcher.utils.filesystem import mkdtemp
......@@ -364,7 +363,7 @@ class TestKVMInlineTestDeploy(StdoutTestCase):
device["actions"]["boot"]["methods"]["qemu"]["parameters"]["extra"][1], int
)
parser = JobParser()
job = parser.parse(yaml.dump(job_data), device, 4212, None, "")
job = parser.parse(yaml_safe_dump(job_data), device, 4212, None, "")
job.logger = DummyLogger()
job.validate()
boot_image = [
......
......@@ -19,9 +19,8 @@
# with this program; if not, see <http://www.gnu.org/licenses>.
import os
import yaml
import datetime
from lava_common.compat import yaml_safe_load
from lava_common.compat import yaml_safe_dump, yaml_safe_load
from lava_common.timeout import Timeout
from lava_common.exceptions import InfrastructureError, JobError
from lava_dispatcher.action import Action, Pipeline
......@@ -70,7 +69,7 @@ class TestDefinitionHandlers(StdoutTestCase):
data = yaml_safe_load(sample_job_data)
data["actions"][2]["test"]["definitions"][0]["from"] = "unusable-handler"
try:
job = parser.parse(yaml.dump(data), device, 4212, None, "")
job = parser.parse(yaml_safe_dump(data), device, 4212, None, "")
job.logger = DummyLogger()
except JobError:
pass
......
......@@ -19,9 +19,8 @@
# with this program; if not, see <http://www.gnu.org/licenses>.
import os
import yaml
import unittest
from lava_common.compat import yaml_safe_load
from lava_common.compat import yaml_safe_dump, yaml_safe_load
from lava_common.exceptions import JobError
from lava_dispatcher.device import NewDevice
from lava_dispatcher.parser import JobParser
......@@ -235,7 +234,7 @@ class TestLxcWithDevices(StdoutTestCase):
parser = JobParser()
(rendered, _) = self.factory.create_device("bbb-01.jinja2")
device = NewDevice(yaml_safe_load(rendered))
job = parser.parse(yaml.dump(data), device, 4577, None, "")
job = parser.parse(yaml_safe_dump(data), device, 4577, None, "")
job.logger = DummyLogger()
job.validate()
lxc_deploy = [
......@@ -293,7 +292,7 @@ class TestLxcWithDevices(StdoutTestCase):
parser = JobParser()
(rendered, _) = self.factory.create_device("bbb-01.jinja2")
device = NewDevice(yaml_safe_load(rendered))
job = parser.parse(yaml.dump(data), device, 4577, None, "")
job = parser.parse(yaml_safe_dump(data), device, 4577, None, "")
job.logger = DummyLogger()
job.validate()
lxc_deploy = [
......
......@@ -20,10 +20,9 @@
import os
import yaml
from unittest.mock import patch
from lava_common.compat import yaml_safe_load
from lava_common.compat import yaml_safe_dump, yaml_safe_load
from lava_common.decorators import nottest
from lava_dispatcher.tests.test_basic import StdoutTestCase
from lava_dispatcher.job import Job
......@@ -178,7 +177,7 @@ class TestMultiDefinition(StdoutTestCase):
self.assertEqual(len(block["definitions"]), 2)
self.assertEqual(block["definitions"][1], block["definitions"][0])
parser = JobParser()
job = parser.parse(yaml.dump(self.job_data), self.device, 4212, None, "")
job = parser.parse(yaml_safe_dump(self.job_data), self.device, 4212, None, "")
self.assertIsNotNone(job)
deploy = [
action for action in job.pipeline.actions if action.name == "tftp-deploy"
......
......@@ -20,11 +20,10 @@
import os
import yaml
import uuid
import json
from lava_common.compat import yaml_load
from lava_common.compat import yaml_dump, yaml_load
from lava_common.constants import LAVA_MULTINODE_SYSTEM_TIMEOUT
from lava_common.timeout import Timeout
from lava_common.exceptions import TestError, JobError, InfrastructureError
......@@ -284,7 +283,7 @@ class TestMultinode(StdoutTestCase):
# check that the description can be re-loaded as valid YAML
for action in self.client_job.pipeline.actions:
data = action.explode()
data_str = yaml.dump(data)
data_str = yaml_dump(data)
yaml_load(data_str) # nosec not suitable for safe_load
def test_multinode_timeout(self):
......
......@@ -20,10 +20,10 @@
import os
import yaml
import unittest
from unittest.mock import patch, MagicMock
from lava_common.compat import yaml_safe_load
from lava_common.compat import yaml_safe_dump, yaml_safe_load
from lava_common.exceptions import JobError
from lava_dispatcher.device import NewDevice
from lava_dispatcher.parser import JobParser
......@@ -675,7 +675,9 @@ class TestKernelConversion(StdoutTestCase):
)
def test_zimage_bootz(self, which_mock):
self.deploy_block["kernel"]["type"] = "zimage"
job = self.parser.parse(yaml.dump(self.base_data), self.device, 4212, None, "")
job = self.parser.parse(
yaml_safe_dump(self.base_data), self.device, 4212, None, ""
)
job.logger = DummyLogger()
job.validate()
deploy = [
......@@ -705,7 +707,9 @@ class TestKernelConversion(StdoutTestCase):
)
def test_image(self, which_mock):
self.deploy_block["kernel"]["type"] = "image"
job = self.parser.parse(yaml.dump(self.base_data), self.device, 4212, None, "")
job = self.parser.parse(
yaml_safe_dump(self.base_data), self.device, 4212, None, ""
)
job.logger = DummyLogger()
job.validate()
deploy = [
......@@ -736,7 +740,9 @@ class TestKernelConversion(StdoutTestCase):
)
def test_uimage(self, which_mock):
self.deploy_block["kernel"]["type"] = "uimage"
job = self.parser.parse(yaml.dump(self.base_data), self.device, 4212, None, "")
job = self.parser.parse(
yaml_safe_dump(self.base_data), self.device, 4212, None, ""
)
job.logger = DummyLogger()
job.validate()
deploy = [
......@@ -769,7 +775,9 @@ class TestKernelConversion(StdoutTestCase):
# drop bootz from the device for this part of the test
del self.device["parameters"]["bootz"]
self.deploy_block["kernel"]["type"] = "zimage"
job = self.parser.parse(yaml.dump(self.base_data), self.device, 4212, None, "")
job = self.parser.parse(
yaml_safe_dump(self.base_data), self.device, 4212, None, ""
)
job.logger = DummyLogger()
job.validate()
deploy = [
......
......@@ -20,10 +20,10 @@
import os
import yaml
import socket
from unittest.mock import patch
from lava_common.compat import yaml_safe_load
from lava_common.compat import yaml_safe_dump, yaml_safe_load
from lava_common.exceptions import JobError
from lava_dispatcher.device import NewDevice
from lava_dispatcher.parser import JobParser
......@@ -257,7 +257,7 @@ class TestVland(StdoutTestCase):
alpha_data["protocols"][VlandProtocol.name], {"vlan_one": {"tags": []}}
)
parser = JobParser()
job = parser.parse(yaml.dump(alpha_data), self.device, 4212, None, "")
job = parser.parse(yaml_safe_dump(alpha_data), self.device, 4212, None, "")
job.logger = DummyLogger()
job.validate()
tftp_deploy = [
......@@ -297,7 +297,7 @@ class TestVland(StdoutTestCase):
alpha_data["protocols"][VlandProtocol.name], {"vlan_one": {"tags": []}}
)
parser = JobParser()
job = parser.parse(yaml.dump(alpha_data), self.device, 4212, None, "")
job = parser.parse(yaml_safe_dump(alpha_data), self.device, 4212, None, "")
job.logger = DummyLogger()
job.validate()
vprotocol = [
......@@ -331,7 +331,7 @@ class TestVland(StdoutTestCase):
{"vlan_one": {"tags": ["spurious"]}},
)
parser = JobParser()
job = parser.parse(yaml.dump(alpha_data), self.device, 4212, None, "")
job = parser.parse(yaml_safe_dump(alpha_data), self.device, 4212, None, "")
job.logger = DummyLogger()
self.assertRaises(JobError, job.validate)
......@@ -346,7 +346,7 @@ class TestVland(StdoutTestCase):
if self.device["parameters"]["interfaces"][interface]["tags"] == []:
self.device["parameters"]["interfaces"][interface]["tags"] = None
parser = JobParser()
job = parser.parse(yaml.dump(alpha_data), self.device, 4212, None, "")
job = parser.parse(yaml_safe_dump(alpha_data), self.device, 4212, None, "")
deploy = [
action for action in job.pipeline.actions if action.name == "tftp-deploy"
][0]
......
......@@ -20,12 +20,12 @@
import json
import pytest
import tap
import yaml
from django.urls import reverse
from django.contrib.auth.models import User
from rest_framework.test import APIClient
from lava_common.compat import yaml_safe_dump
from lava_scheduler_app.models import Device, DeviceType, TestJob, Worker
from lava_results_app import models as result_models
from linaro_django_xmlrpc.models import AuthToken
......@@ -125,12 +125,12 @@ class TestRestApi:
# create testjobs
self.public_testjob1 = TestJob.objects.create(
definition=yaml.safe_dump(EXAMPLE_JOB),
definition=yaml_safe_dump(EXAMPLE_JOB),
submitter=self.user,
requested_device_type=self.public_device_type1,
)
self.private_testjob1 = TestJob.objects.create(
definition=yaml.safe_dump(EXAMPLE_JOB),
definition=yaml_safe_dump(EXAMPLE_JOB),
submitter=self.admin,
requested_device_type=self.public_device_type1,
)
......
......@@ -19,13 +19,13 @@
import csv
import io
import yaml
import xmlrpc.client
from linaro_django_xmlrpc.models import ExposedAPI
from django.db.models.fields import FieldDoesNotExist
from lava_common.compat import yaml_dump
from lava_results_app.dbutils import export_testsuite, testsuite_export_fields
from lava_results_app.models import (
Query,
......@@ -317,7 +317,7 @@ class ResultsAPI(ExposedAPI):
except TestJob.DoesNotExist:
raise xmlrpc.client.Fault(404, "Specified job not found.")
return yaml.dump(yaml_list, Dumper=yaml.CDumper)
return yaml_dump(yaml_list)
def get_testjob_metadata(self, job_id):
"""
......@@ -499,7 +499,7 @@ class ResultsAPI(ExposedAPI):
except TestJob.DoesNotExist:
raise xmlrpc.client.Fault(404, "Specified job not found.")
return yaml.dump(yaml_list, Dumper=yaml.CDumper)
return yaml_dump(yaml_list)
def get_testsuite_results_yaml(self, job_id, suite_name, limit=None, offset=None):
"""
......@@ -547,7 +547,7 @@ class ResultsAPI(ExposedAPI):
except TestSuite.DoesNotExist:
raise xmlrpc.client.Fault(404, "Specified test suite not found.")
return yaml.dump(yaml_list, Dumper=yaml.CDumper)
return yaml_dump(yaml_list)
def get_testsuite_results_csv(self, job_id, suite_name, limit=None, offset=None):
"""
......@@ -694,7 +694,7 @@ class ResultsAPI(ExposedAPI):
except TestCase.DoesNotExist:
raise xmlrpc.client.Fault(404, "Specified test case not found.")
return yaml.dump(yaml_list, Dumper=yaml.CDumper)
return yaml_dump(yaml_list)
def get_testcase_results_csv(self, job_id, suite_name, case_name):
"""
......
import os
import re
import yaml
import shutil
import decimal
......@@ -8,7 +7,7 @@ from django.core.exceptions import MultipleObjectsReturned
from django.urls.exceptions import NoReverseMatch