run.py 44.1 KB
Newer Older
Simon McVittie's avatar
Simon McVittie committed
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36
#!/usr/bin/python3

# flatdeb — build Flatpak runtimes from Debian packages
#
# Copyright © 2016-2017 Simon McVittie
# Copyright © 2017 Collabora Ltd.
#
# Partially derived from vectis, copyright © 2015-2017 Simon McVittie
#
# SPDX-License-Identifier: MIT
#
# Permission is hereby granted, free of charge, to any person obtaining
# a copy of this software and associated documentation files (the
# "Software"), to deal in the Software without restriction, including
# without limitation the rights to use, copy, modify, merge, publish,
# distribute, sublicense, and/or sell copies of the Software, and to
# permit persons to whom the Software is furnished to do so, subject to
# the following conditions:
#
# The above copyright notice and this permission notice shall be included
# in all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
# IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
# CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
# TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
# SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

"""
Create Flatpak runtimes from Debian packages.
"""

import argparse
import json
Simon McVittie's avatar
Simon McVittie committed
37
import logging
Simon McVittie's avatar
Simon McVittie committed
38 39 40
import os
import re
import subprocess
Simon McVittie's avatar
Simon McVittie committed
41
import sys
Simon McVittie's avatar
Simon McVittie committed
42
import urllib.parse
Simon McVittie's avatar
Simon McVittie committed
43
from contextlib import ExitStack
Simon McVittie's avatar
Simon McVittie committed
44 45 46 47 48
from tempfile import TemporaryDirectory

import yaml
from gi.repository import GLib

Simon McVittie's avatar
Simon McVittie committed
49
from flatdeb.worker import HostWorker
Simon McVittie's avatar
Simon McVittie committed
50 51


Simon McVittie's avatar
Simon McVittie committed
52 53 54
logger = logging.getLogger('flatdeb')


55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75
# TODO: When flatdeb is packaged/released, replace this with the released
# version in packages/releases
VERSION = None

if VERSION is None:
    _git_version = subprocess.check_output([
        'sh', '-c',
        'cd "$(dirname "$1")" && '
        'git describe '
        '--always '
        '--dirty '
        '--first-parent '
        '--long '
        '--tags '
        '--match="v[0-9]*" '
        '2>/dev/null || :',
        'sh',
        sys.argv[0],
    ])[1:].decode('utf-8').strip()
    VERSION = _git_version

Simon McVittie's avatar
Simon McVittie committed
76 77
_DEBOS_BASE_RECIPE = os.path.join(
    os.path.dirname(__file__), 'flatdeb', 'debos-base.yaml')
Simon McVittie's avatar
Simon McVittie committed
78 79
_DEBOS_RUNTIMES_RECIPE = os.path.join(
    os.path.dirname(__file__), 'flatdeb', 'debos-runtimes.yaml')
80

Simon McVittie's avatar
Simon McVittie committed
81 82 83 84 85 86
class Builder:

    """
    Main object
    """

87 88
    __multiarch_tuple_cache = {}

Simon McVittie's avatar
Simon McVittie committed
89 90 91
    def __init__(self):
        #: The Debian suite to use
        self.apt_suite = 'stretch'
Simon McVittie's avatar
Simon McVittie committed
92 93 94
        #: The Flatpak branch to use for the runtime, or None for apt_suite
        self.runtime_branch = None
        #: The Flatpak branch to use for the app
95
        self.app_branch = None
Simon McVittie's avatar
Simon McVittie committed
96 97
        #: The freedesktop.org cache directory
        self.xdg_cache_dir = os.getenv(
98
            'XDG_CACHE_HOME', os.path.expanduser('~/.cache'))
Simon McVittie's avatar
Simon McVittie committed
99 100 101 102 103 104
        #: Where to write output
        self.build_area = os.path.join(
            self.xdg_cache_dir, 'flatdeb',
        )
        self.repo = os.path.join(self.build_area, 'repo')

105
        self.__dpkg_archs = []
Simon McVittie's avatar
Simon McVittie committed
106 107
        self.flatpak_arch = None

108
        self.__primary_dpkg_arch_matches_cache = {}
Simon McVittie's avatar
Simon McVittie committed
109 110 111
        self.suite_details = {}
        self.runtime_details = {}
        self.worker = None
112
        self.host_worker = HostWorker()
113
        self.ostree_mode = 'archive-z2'
114
        self.export_bundles = False
115
        self.sources_required = set()
116
        self.strip_source_version_suffix = None
Simon McVittie's avatar
Simon McVittie committed
117

Simon McVittie's avatar
Simon McVittie committed
118 119
        self.logger = logger.getChild('Builder')

Simon McVittie's avatar
Simon McVittie committed
120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146
    @staticmethod
    def get_flatpak_arch(arch=None):
        """
        Return the Flatpak architecture name corresponding to uname
        result arch.

        If arch is None, return the Flatpak architecture name
        corresponding to the machine where this script is running.
        """

        if arch is None:
            arch = os.uname()[4]

        if re.match(r'^i.86$', arch):
            return 'i386'
        elif re.match(r'^arm.*', arch):
            if arch.endswith('b'):
                return 'armeb'
            else:
                return 'arm'
        elif arch in ('mips', 'mips64'):
            import struct
            if struct.pack('i', 1).startswith(b'\x01'):
                return arch + 'el'

        return arch

Simon McVittie's avatar
Simon McVittie committed
147 148 149 150 151 152 153 154 155 156 157 158 159 160
    @staticmethod
    def other_multiarch(arch):
        """
        Return the other architecture that accompanies the given Debian
        architecture in a multiarch setup, or None.
        """

        if arch == 'amd64':
            return 'i386'
        elif arch == 'arm64':
            return 'armhf'
        else:
            return None

161 162 163 164 165 166 167 168 169 170 171 172 173 174 175
    @staticmethod
    def multiarch_tuple(arch):
        """
        Return the multiarch tuple for the given dpkg architecture name.
        """

        if arch not in Builder.__multiarch_tuple_cache:
            Builder.__multiarch_tuple_cache[arch] = subprocess.check_output([
                'dpkg-architecture',
                '-qDEB_HOST_MULTIARCH',
                '-a{}'.format(arch),
            ]).decode('utf-8').strip()

        return Builder.__multiarch_tuple_cache[arch]

Simon McVittie's avatar
Simon McVittie committed
176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200
    @staticmethod
    def dpkg_to_flatpak_arch(arch):
        """
        Return the Flatpak architecture name corresponding to the given
        dpkg architecture name.
        """

        if arch == 'amd64':
            return 'x86_64'
        elif arch == 'arm64':
            return 'aarch64'
        elif arch in ('armel', 'armhf'):
            return 'arm'
        elif arch == 'powerpc':
            return 'ppc'
        elif arch == 'powerpc64':
            return 'ppc64'
        elif arch == 'powerpcel':
            return 'ppcle'
        elif arch == 'ppc64el':
            return 'ppc64le'

        return arch

    @property
201
    def primary_dpkg_arch(self):
Simon McVittie's avatar
Simon McVittie committed
202 203 204 205
        """
        The Debian architecture we are building a runtime for, such as
        i386 or amd64.
        """
206
        return self.__dpkg_archs[0]
Simon McVittie's avatar
Simon McVittie committed
207

208 209 210 211 212 213 214
    @property
    def dpkg_archs(self):
        """
        The Debian architectures we support via multiarch, such as
        ['amd64', 'i386'].
        """
        return self.__dpkg_archs
Simon McVittie's avatar
Simon McVittie committed
215

216 217 218 219 220 221
    @dpkg_archs.setter
    def dpkg_archs(self, value):
        self.__primary_dpkg_arch_matches_cache = {}
        self.__dpkg_archs = value

    def primary_dpkg_arch_matches(self, arch_spec):
Simon McVittie's avatar
Simon McVittie committed
222
        """
223 224
        Return True if arch_spec matches primary_dpkg_arch (or
        equivalently, if primary_dpkg_arch is one of the architectures
Simon McVittie's avatar
Simon McVittie committed
225 226 227
        described by arch_spec). For example, any-amd64 matches amd64
        but not i386.
        """
228
        if arch_spec not in self.__primary_dpkg_arch_matches_cache:
Simon McVittie's avatar
Simon McVittie committed
229
            exit_code = self.worker.call(
230
                ['dpkg-architecture', '--host-arch', self.primary_dpkg_arch,
Simon McVittie's avatar
Simon McVittie committed
231
                 '--is', arch_spec])
232
            self.__primary_dpkg_arch_matches_cache[arch_spec] = (exit_code == 0)
Simon McVittie's avatar
Simon McVittie committed
233

234
        return self.__primary_dpkg_arch_matches_cache[arch_spec]
Simon McVittie's avatar
Simon McVittie committed
235 236 237 238 239 240 241 242

    def run_command_line(self):
        """
        Run appropriate commands for the command-line arguments
        """
        parser = argparse.ArgumentParser(
            description='Build Flatpak runtimes',
        )
243
        parser.add_argument('--chdir', default=None)
244
        parser.add_argument(
Simon McVittie's avatar
Simon McVittie committed
245 246
            '--ostree-mode', default=self.ostree_mode,
        )
247 248 249
        parser.add_argument(
            '--export-bundles', action='store_true', default=False,
        )
Simon McVittie's avatar
Simon McVittie committed
250 251 252
        parser.add_argument('--build-area', default=self.build_area)
        parser.add_argument('--repo', default=self.repo)
        parser.add_argument('--suite', '-d', default=self.apt_suite)
253
        parser.add_argument('--architecture', '--arch', '-a')
Simon McVittie's avatar
Simon McVittie committed
254
        parser.add_argument('--runtime-branch', default=self.runtime_branch)
Simon McVittie's avatar
Simon McVittie committed
255 256 257 258 259 260 261 262 263 264 265
        subparsers = parser.add_subparsers(dest='command', metavar='command')

        subparser = subparsers.add_parser(
            'base',
            help='Build a fresh base tarball',
        )

        subparser = subparsers.add_parser(
            'runtimes',
            help='Build runtimes',
        )
266
        subparser.add_argument('yaml_file')
Simon McVittie's avatar
Simon McVittie committed
267 268 269 270 271

        subparser = subparsers.add_parser(
            'app',
            help='Build an app',
        )
272
        subparser.add_argument('--app-branch', default=self.app_branch)
273
        subparser.add_argument('yaml_manifest')
Simon McVittie's avatar
Simon McVittie committed
274 275 276 277 278 279 280 281

        subparser = subparsers.add_parser(
            'print-flatpak-architecture',
            help='Print the Flatpak architecture',
        )

        args = parser.parse_args()

282 283 284
        if args.chdir is not None:
            os.chdir(args.chdir)

Simon McVittie's avatar
Simon McVittie committed
285 286
        self.build_area = args.build_area
        self.apt_suite = args.suite
Simon McVittie's avatar
Simon McVittie committed
287
        self.runtime_branch = args.runtime_branch
Simon McVittie's avatar
Simon McVittie committed
288
        self.repo = args.repo
289
        self.export_bundles = args.export_bundles
Simon McVittie's avatar
Simon McVittie committed
290
        self.ostree_mode = args.ostree_mode
Simon McVittie's avatar
Simon McVittie committed
291
        self.worker = HostWorker()
Simon McVittie's avatar
Simon McVittie committed
292 293

        if args.architecture is None:
294 295 296 297 298
            self.dpkg_archs = [
                self.worker.check_output(
                    ['dpkg-architecture', '-q', 'DEB_HOST_ARCH'],
                ).decode('utf-8').rstrip('\n')
            ]
Simon McVittie's avatar
Simon McVittie committed
299
        else:
300
            self.dpkg_archs = args.architecture.split(',')
Simon McVittie's avatar
Simon McVittie committed
301

302
        self.flatpak_arch = self.dpkg_to_flatpak_arch(self.primary_dpkg_arch)
Simon McVittie's avatar
Simon McVittie committed
303 304 305 306 307 308 309

        os.makedirs(self.build_area, exist_ok=True)
        os.makedirs(os.path.dirname(self.repo), exist_ok=True)

        if args.command is None:
            parser.error('A command is required')

310 311 312
        with open(
                os.path.join('suites', self.apt_suite + '.yaml'),
                encoding='utf-8') as reader:
Simon McVittie's avatar
Simon McVittie committed
313 314
            self.suite_details = yaml.safe_load(reader)

Simon McVittie's avatar
Simon McVittie committed
315 316
        self.strip_source_version_suffix = self.suite_details.get(
            'strip_source_version_suffix', '')
317

Simon McVittie's avatar
Simon McVittie committed
318 319 320 321 322 323 324 325 326 327 328
        getattr(
            self, 'command_' + args.command.replace('-', '_'))(**vars(args))

    def command_print_flatpak_architecture(self, **kwargs):
        print(self.flatpak_arch)

    @property
    def apt_uris(self):
        for source in self.suite_details['sources']:
            yield source['apt_uri']

329
    def ensure_build_area(self):
Simon McVittie's avatar
Simon McVittie committed
330 331 332 333
        self.worker.check_call([
            'sh', '-euc',
            'mkdir -p "${XDG_CACHE_HOME:="$HOME/.cache"}/flatdeb"',
        ])
334

Simon McVittie's avatar
Simon McVittie committed
335 336 337
    def command_base(self, **kwargs):
        with ExitStack() as stack:
            stack.enter_context(self.worker)
338
            self.ensure_build_area()
Simon McVittie's avatar
Simon McVittie committed
339

Simon McVittie's avatar
Simon McVittie committed
340 341
            apt_suite = self.suite_details['sources'][0].get(
                'apt_suite', self.apt_suite)
Simon McVittie's avatar
Simon McVittie committed
342

Simon McVittie's avatar
Simon McVittie committed
343 344 345 346 347 348 349 350 351 352 353 354 355 356 357 358 359 360 361 362 363 364 365 366 367 368
            dest_recipe = '{}/{}'.format(
                self.worker.scratch,
                'flatdeb.yaml',
            )
            self.worker.install_file(_DEBOS_BASE_RECIPE, dest_recipe)

            for helper in (
                'add-foreign-architectures',
                'clean-up-base',
                'clean-up-before-pack',
                'disable-services',
                'usrmerge',
            ):
                dest = '{}/{}'.format(
                    self.worker.scratch,
                    helper,
                )
                self.worker.install_file(
                    os.path.join(
                        os.path.dirname(__file__),
                        'flatdeb',
                        helper,
                    ),
                    dest,
                    permissions=0o755,
                )
369

Simon McVittie's avatar
Simon McVittie committed
370 371 372 373 374 375 376 377 378 379 380 381 382 383 384
            self.worker.check_call([
                'mkdir', '-p',
                '{}/suites/{}/overlay/etc/apt/trusted.gpg.d'.format(
                    self.worker.scratch,
                    apt_suite,
                ),
            ])

            tarball = 'base-{}-{}.tar.gz'.format(
                self.apt_suite,
                ','.join(self.dpkg_archs),
            )
            output = os.path.join(self.build_area, tarball)

            script = self.suite_details.get('debootstrap_script')
Simon McVittie's avatar
Simon McVittie committed
385

Simon McVittie's avatar
Simon McVittie committed
386 387 388 389 390 391 392 393
            if script is not None:
                # TODO: flatdeb has historically used a configurable
                # debootstrap_script, but debos doesn't support scripts other
                # than 'unstable'. Does the Debian script work for precise and
                # produce the same results as the 'precise' script?
                # https://github.com/go-debos/debos/issues/16
                logger.debug(
                    'Ignoring /usr/share/debootstrap/scripts/%s', script)
Simon McVittie's avatar
Simon McVittie committed
394

Simon McVittie's avatar
Simon McVittie committed
395 396
            self.configure_apt(
                '{}/suites/{}/overlay'.format(self.worker.scratch, apt_suite))
Simon McVittie's avatar
Simon McVittie committed
397

Simon McVittie's avatar
Simon McVittie committed
398 399 400 401 402 403 404 405 406 407 408 409 410 411 412 413 414 415
            argv = [
                'debos',
                '--artifactdir={}'.format(self.build_area),
                '-t', 'architecture:{}'.format(self.primary_dpkg_arch),
                '-t', 'suite:{}'.format(apt_suite),
                '-t', 'mirror:{}'.format(
                    self.suite_details['sources'][0]['apt_uri'],
                ),
                '-t', 'ospack:{}'.format(tarball + '.new'),
                '-t', 'foreignarchs:{}'.format(
                    ' '.join(self.dpkg_archs[1:]),
                ),
                '-t', 'mergedusr:{}'.format(
                    str(
                        self.suite_details.get('can_merge_usr', False),
                    ).lower(),
                ),
            ]
416

417 418 419 420 421 422 423 424 425 426 427 428 429 430 431 432 433 434 435 436 437 438 439 440 441 442 443
            keyring = self.suite_details['sources'][0].get('keyring')

            if keyring is not None:
                if os.path.exists(os.path.join('suites', keyring)):
                    keyring = os.path.join('suites', keyring)
                elif os.path.exists(keyring):
                    pass
                else:
                    raise RuntimeError('Cannot open {}'.format(keyring))

                dest = '{}/suites/{}/overlay/etc/apt/trusted.gpg.d/{}'.format(
                    self.worker.scratch,
                    apt_suite,
                    os.path.basename(keyring),
                )
                self.worker.install_file(os.path.abspath(keyring), dest)

                argv.append('-t')
                argv.append(
                    'keyring:suites/{}/overlay/etc/apt/trusted.gpg.d/{}'.format(
                        apt_suite,
                        os.path.basename(keyring),
                    )
                )
            else:
                keyring = ''

Simon McVittie's avatar
Simon McVittie committed
444 445 446 447 448 449
            components = self.suite_details.get('apt_components', ['main'])

            if components:
                argv.append('-t')
                argv.append('components:{}'.format(yaml.dump(components)))

Simon McVittie's avatar
Simon McVittie committed
450 451
            argv.append(dest_recipe)
            self.worker.check_call(argv)
Simon McVittie's avatar
Simon McVittie committed
452

Simon McVittie's avatar
Simon McVittie committed
453
            os.rename(output + '.new', output)
Simon McVittie's avatar
Simon McVittie committed
454

Simon McVittie's avatar
Simon McVittie committed
455
    def ensure_local_repo(self):
456 457 458 459 460
        self.host_worker.check_call([
            'install',
            '-d',
            os.path.dirname(self.repo),
        ])
461
        self.host_worker.check_call([
Simon McVittie's avatar
Simon McVittie committed
462 463 464 465
            'ostree',
            '--repo=' + self.repo,
            'init',
            '--mode={}'.format(self.ostree_mode),
Simon McVittie's avatar
Simon McVittie committed
466 467
        ])

468
    def command_runtimes(self, *, yaml_file, **kwargs):
Simon McVittie's avatar
Simon McVittie committed
469
        self.ensure_local_repo()
Simon McVittie's avatar
Simon McVittie committed
470

Simon McVittie's avatar
Simon McVittie committed
471 472 473
        if self.runtime_branch is None:
            self.runtime_branch = self.apt_suite

474
        with open(yaml_file, encoding='utf-8') as reader:
Simon McVittie's avatar
Simon McVittie committed
475 476 477 478
            self.runtime_details = yaml.safe_load(reader)

        tarball = 'base-{}-{}.tar.gz'.format(
            self.apt_suite,
479
            ','.join(self.dpkg_archs),
Simon McVittie's avatar
Simon McVittie committed
480 481 482 483
        )

        with ExitStack() as stack:
            stack.enter_context(self.worker)
484
            self.ensure_build_area()
Simon McVittie's avatar
Simon McVittie committed
485

Simon McVittie's avatar
Simon McVittie committed
486 487 488 489 490
            dest_recipe = '{}/{}'.format(
                self.worker.scratch,
                'flatdeb.yaml',
            )
            self.worker.install_file(_DEBOS_RUNTIMES_RECIPE, dest_recipe)
Simon McVittie's avatar
Simon McVittie committed
491

Simon McVittie's avatar
Simon McVittie committed
492 493 494 495 496 497 498 499 500 501 502 503 504 505 506 507 508 509 510 511 512 513 514 515 516
            for helper in (
                'clean-up-base',
                'collect-source-code',
                'disable-services',
                'hard-link-alternatives',
                'make-flatpak-friendly',
                'platformize',
                'prepare-runtime',
                'put-ldconfig-in-path',
                'usrmerge',
                'write-manifest',
            ):
                dest = '{}/{}'.format(
                    self.worker.scratch,
                    helper,
                )
                self.worker.install_file(
                    os.path.join(
                        os.path.dirname(__file__),
                        'flatdeb',
                        helper,
                    ),
                    dest,
                    permissions=0o755,
                )
Simon McVittie's avatar
Simon McVittie committed
517

518 519
            prefix = self.runtime_details['id_prefix']

520 521
            # Do the Platform first, because we download its source
            # packages as part of preparing the Sdk
Simon McVittie's avatar
Simon McVittie committed
522 523 524 525 526 527 528 529 530 531 532 533 534 535 536 537 538 539 540 541 542 543 544 545 546 547 548 549 550 551 552 553 554 555 556 557 558 559 560 561 562 563 564 565 566 567 568 569 570 571 572 573 574 575 576 577 578 579 580 581 582 583 584 585 586 587 588 589 590 591 592 593 594 595 596 597 598 599 600 601 602 603 604 605 606 607 608 609 610 611 612 613 614 615 616 617 618 619 620 621 622 623 624 625 626 627 628 629 630 631 632 633 634 635 636 637 638 639 640 641 642 643 644 645 646 647 648 649 650 651 652 653 654 655 656 657 658 659 660 661 662 663 664
            for sdk in (False, True):
                packages = list(self.runtime_details.get('add_packages', []))

                for p in self.runtime_details.get('add_packages_multiarch', []):
                    for a in self.dpkg_archs:
                        packages.append(p + ':' + a)

                if sdk:
                    runtime = prefix + '.Sdk'
                else:
                    runtime = prefix + '.Platform'

                out_tarball = '{}-ostree-{}-{}.tar.gz'.format(
                    runtime,
                    self.flatpak_arch,
                    self.runtime_branch,
                )

                argv = [
                    'debos',
                    '--artifactdir={}'.format(self.build_area),
                    '-t', 'architecture:{}'.format(self.primary_dpkg_arch),
                    '-t', 'suite:{}'.format(self.apt_suite),
                    '-t', 'ospack:{}'.format(tarball),
                    '-t', 'ostree_tarball:{}'.format(out_tarball + '.new'),
                    '-t', 'runtime:{}'.format(runtime),
                    '-t', 'runtime_branch:{}'.format(self.runtime_branch),
                    '-t', 'strip_source_version_suffix:{}'.format(
                        self.strip_source_version_suffix),
                    '-t', 'repo:repo',
                ]

                if packages:
                    logger.info('Installing packages:')
                    packages.sort()

                    for p in packages:
                        logger.info('- %s', p)

                    argv.append('-t')
                    argv.append('packages:{}'.format(yaml.dump(packages)))

                script = self.runtime_details.get('post_script', '')

                if script:
                    dest = '{}/{}'.format(
                        self.worker.scratch,
                        'post_script',
                    )

                    with open(dest, encoding='utf-8') as writer:
                        writer.write('#!/bin/sh\n')
                        writer.write(script)
                        writer.write('\n')

                    argv.append('-t')
                    argv.append('post_script:"$RECIPEDIR/post_script"')

                if sdk:
                    sources_tarball = '{}-sources-{}-{}.tar.gz'.format(
                        runtime,
                        self.flatpak_arch,
                        self.runtime_branch,
                    )

                    sdk_details = self.runtime_details.get('sdk', {})
                    sdk_packages = list(sdk_details.get('add_packages', []))
                    argv.append('-t')
                    argv.append('sdk:yes')
                    argv.append('-t')
                    argv.append('sources_tarball:' + sources_tarball + '.new')

                    for p in sdk_details.get('add_packages_multiarch', []):
                        for a in self.dpkg_archs:
                            sdk_packages.append(p + ':' + a)

                    if sdk_packages:
                        logger.info('Installing extra packages for SDK:')
                        sdk_packages.sort()

                        for p in sdk_packages:
                            logger.info('- %s', p)

                        argv.append('-t')
                        argv.append(
                            'sdk_packages:{}'.format(yaml.dump(sdk_packages)))

                    script = sdk_details.get('post_script', '')

                    if script:
                        dest = '{}/{}'.format(
                            self.worker.scratch,
                            'sdk_post_script',
                        )

                        with open(dest, encoding='utf-8') as writer:
                            writer.write('#!/bin/sh\n')
                            writer.write(script)
                            writer.write('\n')

                        argv.append('-t')
                        argv.append('sdk_post_script:"$RECIPEDIR/sdk_post_script"')
                else:   # not sdk
                    platform_details = self.runtime_details.get('platform', {})
                    script = platform_details.get('post_script', '')

                    if script:
                        dest = '{}/{}'.format(
                            self.worker.scratch,
                            'platform_post_script',
                        )

                        with open(dest, encoding='utf-8') as writer:
                            writer.write('#!/bin/sh\n')
                            writer.write(script)
                            writer.write('\n')

                        argv.append('-t')
                        argv.append('platform_post_script:"$RECIPEDIR/platform_post_script"')

                self.create_flatpak_manifest_overlay(prefix, runtime, sdk=sdk)

                argv.append(dest_recipe)
                self.worker.check_call(argv)

                if sdk:
                    output = os.path.join(self.build_area, sources_tarball)
                    os.rename(output + '.new', output)

                output = os.path.join(self.build_area, out_tarball)
                os.rename(output + '.new', output)

            # Don't keep the history in this working repository:
            # if history is desired, mirror the commits into a public
            # repository and maintain history there.
            self.worker.check_call([
                'time',
                'ostree',
                '--repo=' + self.repo,
                'prune',
                '--refs-only',
                '--depth=1',
            ])
Simon McVittie's avatar
Simon McVittie committed
665 666

            self.worker.check_call([
Simon McVittie's avatar
Simon McVittie committed
667
                'time',
Simon McVittie's avatar
Simon McVittie committed
668 669
                'flatpak',
                'build-update-repo',
Simon McVittie's avatar
Simon McVittie committed
670
                self.repo,
Simon McVittie's avatar
Simon McVittie committed
671 672
            ])

673 674
            if self.export_bundles:
                for suffix in ('.Platform', '.Sdk'):
Simon McVittie's avatar
Simon McVittie committed
675 676 677 678 679 680 681
                    bundle = '{}-{}-{}.bundle'.format(
                        prefix + suffix,
                        self.flatpak_arch,
                        self.runtime_branch,
                    )
                    output = os.path.join(self.build_area, bundle)

Simon McVittie's avatar
Simon McVittie committed
682
                    self.worker.check_call([
683 684 685 686
                        'time',
                        'flatpak',
                        'build-bundle',
                        '--runtime',
Simon McVittie's avatar
Simon McVittie committed
687 688
                        self.repo,
                        output + '.new',
689
                        prefix + suffix,
Simon McVittie's avatar
Simon McVittie committed
690
                        self.runtime_branch,
691
                    ])
Simon McVittie's avatar
Simon McVittie committed
692

Simon McVittie's avatar
Simon McVittie committed
693
                    os.rename(output + '.new', output)
Simon McVittie's avatar
Simon McVittie committed
694

Simon McVittie's avatar
Simon McVittie committed
695
    def configure_apt(self, overlay):
Simon McVittie's avatar
Simon McVittie committed
696 697 698 699 700
        """
        Configure apt. We only do this once, so that all chroots
        created from the same base have their version numbers
        aligned.
        """
701
        with TemporaryDirectory(prefix='flatdeb-apt.') as t:
Simon McVittie's avatar
Simon McVittie committed
702 703 704 705
            # Set up the apt sources

            to_copy = os.path.join(t, 'sources.list')

706
            with open(to_copy, 'w', encoding='utf-8') as writer:
Simon McVittie's avatar
Simon McVittie committed
707 708 709 710 711 712 713 714 715
                for source in self.suite_details['sources']:
                    suite = source.get('apt_suite', self.apt_suite)
                    suite = suite.replace('*', self.apt_suite)
                    components = source.get(
                        'apt_components',
                        self.suite_details.get(
                            'apt_components',
                            ['main']))

716 717 718 719 720 721 722 723 724 725
                    options = []

                    if source.get('apt_trusted', False):
                        options.append('trusted=yes')

                    if options:
                        options_str = ' [' + ' '.join(options) + ']'
                    else:
                        options_str = ''

Simon McVittie's avatar
Simon McVittie committed
726
                    for prefix in ('deb', 'deb-src'):
727
                        writer.write('{}{} {} {} {}\n'.format(
Simon McVittie's avatar
Simon McVittie committed
728
                            prefix,
729
                            options_str,
Simon McVittie's avatar
Simon McVittie committed
730 731 732 733 734 735 736 737
                            source['apt_uri'],
                            suite,
                            ' '.join(components),
                        ))

                    keyring = source.get('keyring')

                    if keyring is not None:
738 739 740 741 742 743 744
                        if os.path.exists(os.path.join('suites', keyring)):
                            keyring = os.path.join('suites', keyring)
                        elif os.path.exists(keyring):
                            pass
                        else:
                            raise RuntimeError('Cannot open {}'.format(keyring))

Simon McVittie's avatar
Simon McVittie committed
745
                        self.worker.install_file(
Simon McVittie's avatar
Simon McVittie committed
746 747
                            os.path.abspath(keyring),
                            '{}/etc/apt/trusted.gpg.d/{}'.format(
Simon McVittie's avatar
Simon McVittie committed
748
                                overlay,
Simon McVittie's avatar
Simon McVittie committed
749 750 751 752
                                os.path.basename(keyring),
                            ),
                        )

Simon McVittie's avatar
Simon McVittie committed
753
            self.worker.install_file(
Simon McVittie's avatar
Simon McVittie committed
754
                to_copy,
Simon McVittie's avatar
Simon McVittie committed
755
                '{}/etc/apt/sources.list'.format(overlay),
Simon McVittie's avatar
Simon McVittie committed
756 757
            )

Simon McVittie's avatar
Simon McVittie committed
758 759 760 761
    def create_flatpak_manifest_overlay(self, prefix, runtime, sdk=False):
        overlay = '{}/runtimes/{}/overlay'.format(
            self.worker.scratch,
            runtime,
Simon McVittie's avatar
Simon McVittie committed
762 763
        )

764
        with TemporaryDirectory(prefix='flatdeb-ostreeify.') as t:
Simon McVittie's avatar
Simon McVittie committed
765 766 767 768 769 770
            metadata = os.path.join(t, 'metadata')

            keyfile = GLib.KeyFile()
            keyfile.set_string('Runtime', 'name', runtime)
            keyfile.set_string(
                'Runtime', 'runtime',
771
                '{}.Platform/{}/{}'.format(
Simon McVittie's avatar
Simon McVittie committed
772 773
                    prefix,
                    self.flatpak_arch,
Simon McVittie's avatar
Simon McVittie committed
774
                    self.runtime_branch,
Simon McVittie's avatar
Simon McVittie committed
775 776 777 778
                )
            )
            keyfile.set_string(
                'Runtime', 'sdk',
779
                '{}.Sdk/{}/{}'.format(
Simon McVittie's avatar
Simon McVittie committed
780 781
                    prefix,
                    self.flatpak_arch,
Simon McVittie's avatar
Simon McVittie committed
782
                    self.runtime_branch,
Simon McVittie's avatar
Simon McVittie committed
783 784 785
                )
            )

786 787 788 789 790 791 792 793 794
            keyfile.set_string(
                'Runtime', 'x-flatdeb-sources',
                '{}.Sdk.Sources/{}/{}'.format(
                    prefix,
                    self.flatpak_arch,
                    self.runtime_branch,
                ),
            )

Simon McVittie's avatar
Simon McVittie committed
795 796 797 798 799 800 801
            keyfile.set_string(
                'Environment', 'XDG_DATA_DIRS',
                ':'.join([
                    '/app/share', '/usr/share', '/usr/share/runtime/share',
                ]),
            )

802 803 804 805 806 807 808 809 810 811 812
            search_path = []

            for arch in self.dpkg_archs:
                search_path.append('/app/lib/{}'.format(self.multiarch_tuple(arch)))

            search_path.append('/app/lib')

            keyfile.set_string(
                'Environment', 'LD_LIBRARY_PATH', ':'.join(search_path),
            )

Simon McVittie's avatar
Simon McVittie committed
813
            if True:    # TODO: 'libgstreamer1.0-0' in installed:
814 815 816 817 818 819 820 821 822 823 824 825 826 827 828 829 830 831 832 833 834 835 836
                search_path = []

                for arch in self.dpkg_archs:
                    search_path.append(
                        '/app/lib/{}/gstreamer-1.0'.format(
                            self.multiarch_tuple(arch)))

                search_path.append('/app/lib/gstreamer-1.0')

                for arch in self.dpkg_archs:
                    search_path.append(
                        '/usr/lib/extensions/{}/gstreamer-1.0'.format(
                            self.multiarch_tuple(arch)))

                search_path.append('/usr/lib/extensions/gstreamer-1.0')

                for arch in self.dpkg_archs:
                    search_path.append(
                        '/usr/lib/{}/gstreamer-1.0'.format(
                            self.multiarch_tuple(arch)))

                search_path.append('/usr/lib/gstreamer-1.0')

Simon McVittie's avatar
Simon McVittie committed
837 838
                keyfile.set_string(
                    'Environment', 'GST_PLUGIN_SYSTEM_PATH',
839
                    ':'.join(search_path),
Simon McVittie's avatar
Simon McVittie committed
840 841
                )

Simon McVittie's avatar
Simon McVittie committed
842
            if True:    # TODO: 'libgirepository-1.0-1' in installed:
843 844 845 846 847 848 849 850 851
                search_path = []

                for arch in self.dpkg_archs:
                    search_path.append(
                        '/app/lib/{}/girepository-1.0'.format(
                            self.multiarch_tuple(arch)))

                search_path.append('/app/lib/girepository-1.0')

Simon McVittie's avatar
Simon McVittie committed
852 853
                keyfile.set_string(
                    'Environment', 'GI_TYPELIB_PATH',
854
                    ':'.join(search_path),
Simon McVittie's avatar
Simon McVittie committed
855 856
                )

857 858 859 860
            keyfile.set_string(
                'Runtime', 'x-flatdeb-version', VERSION,
            )

861 862 863 864 865
            for ext, detail in self.runtime_details.get(
                    'add-extensions', {}
                    ).items():
                group = 'Extension {}'.format(ext)

Simon McVittie's avatar
Simon McVittie committed
866
                self.worker.check_call([
867
                    'install', '-d',
Simon McVittie's avatar
Simon McVittie committed
868 869
                    '{}/ostree/main/files/{}'.format(
                        overlay, detail['directory']),
870 871
                ])

872 873 874 875 876 877 878 879 880
                for k, v in detail.items():
                    if isinstance(v, str):
                        keyfile.set_string(group, k, v)
                    elif isinstance(v, bool):
                        keyfile.set_boolean(group, k, v)
                    else:
                        raise RuntimeError(
                            'Unknown type {} in {}'.format(v, ext))

Simon McVittie's avatar
Simon McVittie committed
881 882
            keyfile.save_to_file(metadata)

Simon McVittie's avatar
Simon McVittie committed
883 884 885 886
            self.worker.check_call([
                'install', '-d', '{}/ostree/main'.format(overlay),
            ])
            self.worker.install_file(
Simon McVittie's avatar
Simon McVittie committed
887
                metadata,
Simon McVittie's avatar
Simon McVittie committed
888
                '{}/ostree/main/metadata'.format(overlay),
Simon McVittie's avatar
Simon McVittie committed
889 890
            )

891
        if sdk:
892 893 894 895 896 897 898 899 900 901 902 903 904 905 906 907 908 909 910 911 912 913 914 915 916 917 918 919
            with TemporaryDirectory(prefix='flatdeb-ostreeify.') as t:
                metadata = os.path.join(t, 'metadata')

                keyfile = GLib.KeyFile()
                keyfile.set_string('Runtime', 'name', runtime + '.Sources')
                keyfile.set_string(
                    'Runtime', 'runtime',
                    '{}.Platform/{}/{}'.format(
                        prefix,
                        self.flatpak_arch,
                        self.runtime_branch,
                    )
                )
                keyfile.set_string(
                    'Runtime', 'sdk',
                    '{}.Sdk/{}/{}'.format(
                        prefix,
                        self.flatpak_arch,
                        self.runtime_branch,
                    )
                )

                keyfile.set_string(
                    'Runtime', 'x-flatdeb-version', VERSION,
                )

                keyfile.save_to_file(metadata)

920 921 922 923 924
                self.worker.check_call([
                    'install', '-d',
                    '{}/ostree/source/metadata'.format(overlay),
                ])

Simon McVittie's avatar
Simon McVittie committed
925
                self.worker.install_file(
926
                    metadata,
Simon McVittie's avatar
Simon McVittie committed
927
                    '{}/ostree/source/metadata'.format(overlay),
928 929
                )

930
    def command_app(self, *, app_branch, yaml_manifest, **kwargs):
931
        self.worker.require_extended_attributes()
Simon McVittie's avatar
Simon McVittie committed
932
        self.ensure_local_repo()
Simon McVittie's avatar
Simon McVittie committed
933

934
        with open(yaml_manifest, encoding='utf-8') as reader:
Simon McVittie's avatar
Simon McVittie committed
935 936
            manifest = yaml.safe_load(reader)

Simon McVittie's avatar
Simon McVittie committed
937 938 939 940 941 942 943 944 945 946 947 948 949 950 951 952 953
        if self.runtime_branch is None:
            self.runtime_branch = manifest.get('runtime-version')

        if self.runtime_branch is None:
            self.runtime_branch = self.apt_suite

        self.app_branch = app_branch

        if self.app_branch is None:
            self.app_branch = manifest.get('branch')

        if self.app_branch is None:
            self.app_branch = 'master'

        manifest['branch'] = self.app_branch
        manifest['runtime-version'] = self.runtime_branch

Simon McVittie's avatar
Simon McVittie committed
954 955
        with ExitStack() as stack:
            stack.enter_context(self.worker)
956
            self.ensure_build_area()
Simon McVittie's avatar
Simon McVittie committed
957
            self.ensure_local_repo()
958 959 960
            t = stack.enter_context(
                TemporaryDirectory(prefix='flatpak-app.')
            )
Simon McVittie's avatar
Simon McVittie committed
961 962

            self.worker.check_call([
963
                'mkdir', '-p', '{}/home'.format(self.worker.scratch),
Simon McVittie's avatar
Simon McVittie committed
964
            ])
Simon McVittie's avatar
Simon McVittie committed
965

966
            self.worker.check_call([
967
                'env',
968
                'XDG_DATA_HOME={}/home'.format(self.worker.scratch),
969
                'flatpak', '--user',
970
                'remote-add', '--if-not-exists', '--no-gpg-verify',
971
                'flatdeb',
972
                'http://192.168.122.1:3142/local/flatdeb/repo',
973
            ])
Simon McVittie's avatar
Simon McVittie committed
974 975
            self.worker.check_call([
                'env',
976
                'XDG_DATA_HOME={}/home'.format(self.worker.scratch),
Simon McVittie's avatar
Simon McVittie committed
977
                'flatpak', '--user',
978
                'remote-modify', '--no-gpg-verify',
979
                '--url=http://192.168.122.1:3142/local/flatdeb/repo',
980
                'flatdeb',
Simon McVittie's avatar
Simon McVittie committed
981
            ])
982 983

            for runtime in (manifest['sdk'], manifest['runtime']):
984 985 986
                # This may fail: we might already have it.
                self.worker.call([
                    'env',
987
                    'XDG_DATA_HOME={}/home'.format(self.worker.scratch),
988 989 990 991 992 993 994 995 996 997
                    'flatpak', '--user',
                    'install', 'flatdeb',
                    '{}/{}/{}'.format(
                        runtime,
                        self.flatpak_arch,
                        self.runtime_branch,
                    ),
                ])
                self.worker.check_call([
                    'env',
998
                    'XDG_DATA_HOME={}/home'.format(self.worker.scratch),
999 1000 1001 1002 1003 1004 1005 1006
                    'flatpak', '--user',
                    'update',
                    '{}/{}/{}'.format(
                        runtime,
                        self.flatpak_arch,
                        self.runtime_branch,
                    ),
                ])
Simon McVittie's avatar
Simon McVittie committed
1007 1008 1009 1010 1011 1012 1013 1014 1015

            for module in manifest.get('modules', []):
                if isinstance(module, dict):
                    sources = module.setdefault('sources', [])

                    for source in sources:
                        if 'path' in source:
                            if source.get('type') == 'git':
                                clone = self.worker.check_output([
1016 1017 1018
                                    'mktemp', '-d',
                                    '-p', self.worker.scratch,
                                    'flatdeb-git.XXXXXX',
Simon McVittie's avatar
Simon McVittie committed
1019
                                ]).decode('utf-8').rstrip('\n')
1020
                                uploader = self.host_worker.Popen([
Simon McVittie's avatar
Simon McVittie committed
1021 1022 1023 1024 1025 1026 1027 1028 1029 1030 1031 1032 1033 1034
                                    'tar',
                                    '-cf-',
                                    '-C', source['path'],
                                    '.',
                                ], stdout=subprocess.PIPE)
                                self.worker.check_call([
                                    'tar',
                                    '-xf-',
                                    '-C', clone,
                                ], stdin=uploader.stdout)
                                uploader.wait()
                                source['path'] = clone
                            else:
                                d = self.worker.check_output([
1035 1036 1037
                                    'mktemp', '-d',
                                    '-p', self.worker.scratch,
                                    'flatdeb-path.XXXXXX',
Simon McVittie's avatar
Simon McVittie committed
1038 1039 1040 1041 1042 1043 1044 1045 1046 1047 1048 1049 1050 1051 1052 1053 1054 1055 1056 1057 1058 1059
                                ]).decode('utf-8').rstrip('\n')
                                clone = '{}/{}'.format(
                                    d, os.path.basename(source['path']),
                                )

                                permissions = 0o644

                                if GLib.file_test(
                                        source['path'],
                                        GLib.FileTest.IS_EXECUTABLE,
                                ):
                                    permissions = 0o755

                                self.worker.install_file(
                                    source['path'],
                                    clone,
                                    permissions,
                                )
                                source['path'] = clone

                    if 'x-flatdeb-apt-packages' in module:
                        packages = self.worker.check_output([
1060 1061 1062
                            'mktemp', '-d',
                            '-p', self.worker.scratch,
                            'flatdeb-debs.XXXXXX',
Simon McVittie's avatar
Simon McVittie committed
1063 1064 1065 1066
                        ]).decode('utf-8').rstrip('\n')

                        self.worker.check_call([
                            'env',
1067
                            'XDG_DATA_HOME={}/home'.format(self.worker.scratch),
Simon McVittie's avatar
Simon McVittie committed
1068 1069 1070 1071 1072 1073 1074
                            'flatpak', 'run',
                            '--filesystem={}'.format(packages),
                            '--share=network',
                            '--command=/usr/bin/env',
                            '{}/{}/{}'.format(
                                manifest['sdk'],
                                self.flatpak_arch,
Simon McVittie's avatar
Simon McVittie committed
1075
                                self.runtime_branch,
Simon McVittie's avatar
Simon McVittie committed
1076
                            ),
1077
                            'DEBIAN_FRONTEND=noninteractive',
Simon McVittie's avatar
Simon McVittie committed
1078 1079 1080 1081 1082 1083 1084 1085
                            'http_proxy=http://192.168.122.1:3142',
                            'export={}'.format(packages),
                            'sh',
                            '-euc',

                            'cp -a /usr/var /\n'
                            'install -d /var/cache/apt/archives/partial\n'
                            'fakeroot apt-get update\n'
1086 1087
                            'fakeroot apt-get -y --download-only \\\n'
                            '    --no-install-recommends install "$@"\n'
1088 1089 1090
                            'for x in /var/cache/apt/archives/*.deb; do\n'
                            '    package="$(dpkg-deb -f "$x" Package)"\n'
                            '    source="$(dpkg-deb -f "$x" Source)"\n'
1091
                            '    bu="$(dpkg-deb -f "$x" Built-Using)"\n'
1092 1093 1094 1095 1096 1097 1098 1099 1100 1101
                            '    version="$(dpkg-deb -f "$x" Version)"\n'
                            '    if [ -z "$source" ]; then\n'
                            '        source="$package"\n'
                            '    fi\n'
                            '    if [ "${source% (*}" != "$source" ]; then\n'
                            '        version="${source#* (}"\n'
                            '        version="${version%)}"\n'
                            '        source="${source% (*}"\n'
                            '    fi\n'
                            '    ( cd "$export" && \\\n'
1102 1103
                            '         apt-get -y --download-only \\\n'
                            '         -oAPT::Get::Only-Source=true source \\\n'
1104 1105
                            '         "$source=$version"\n'
                            '    )\n'
1106 1107 1108 1109 1110 1111 1112 1113 1114 1115 1116 1117 1118 1119 1120 1121
                            '    if [ -n "$bu" ]; then\n'
                            '        oldIFS="$IFS"\n'
                            '        IFS=","\n'
                            '        for dep in $bu; do\n'
                            '            bu="$(echo "$bu" | tr -d " ")"\n'
                            '            version="${bu#*(=}"\n'
                            '            version="${version%)}"\n'
                            '            source="${bu%(*}"\n'
                            '            ( cd "$export" && \\\n'
                            '                 apt-get -y --download-only \\\n'
                            '                 -oAPT::Get::Only-Source=true \\\n'
                            '                 source "$source=$version"\n'
                            '            )\n'
                            '        done\n'
                            '        IFS="$oldIFS"\n'
                            '    fi\n'
1122
                            'done\n'
Simon McVittie's avatar
Simon McVittie committed
1123 1124 1125 1126 1127 1128 1129 1130
                            'mv /var/cache/apt/archives/*.deb "$export"\n'
                            'mv /var/lib/apt/lists "$export"\n'
                            '',

                            'sh',   # argv[0]
                        ] + module['x-flatdeb-apt-packages'])

                        obtained = self.worker.check_output([
1131 1132 1133 1134 1135 1136
                            'sh', '-euc',
                            'cd "$1"\n'
                            'find * -type f -print0 | xargs -0 sha256sum -b\n'
                            '',
                            'sh',   # argv[0]
                            packages,
Simon McVittie's avatar
Simon McVittie committed
1137 1138
                        ]).decode('utf-8').splitlines()

1139 1140
                        for line in obtained:
                            sha256, f = line.split(' *', 1)
Simon McVittie's avatar
Simon McVittie committed
1141 1142
                            path = '{}/{}'.format(packages, f)

1143 1144 1145 1146 1147 1148 1149 1150 1151 1152 1153 1154
                            sources.append({
                                'dest': (os.path.dirname(f) or '.'),
                                'type': 'file',
                                'sha256': sha256,
                                'url': urllib.parse.urlunsplit((
                                    'file',
                                    '',
                                    urllib.parse.quote(path),
                                    '',
                                    '',
                                ))
                            })
Simon McVittie's avatar
Simon McVittie committed
1155

1156 1157 1158 1159
            remote_manifest = '{}/{}.json'.format(
                self.worker.scratch,
                manifest['id'],
            )
Simon McVittie's avatar
Simon McVittie committed
1160

1161 1162
            self.worker.check_call([
                'mkdir', '-p',
Simon McVittie's avatar
Simon McVittie committed
1163
                '{}/.flatpak-builder'.format(self.build_area),
1164
            ])
Simon McVittie's avatar
Simon McVittie committed
1165
            if self.build_area != self.worker.scratch:
1166 1167
                self.worker.check_call([
                    'ln', '-nsf',
Simon McVittie's avatar
Simon McVittie committed
1168
                    '{}/.flatpak-builder'.format(self.build_area),
1169 1170
                    '{}/'.format(self.worker.scratch),
                ])
1171

1172
            with TemporaryDirectory(prefix='flatdeb-manifest.') as t:
1173
                json_manifest = os.path.join(t, manifest['id'] + '.json')
Simon McVittie's avatar
Simon McVittie committed
1174 1175 1176 1177 1178 1179 1180 1181 1182 1183

                with open(
                        json_manifest, 'w', encoding='utf-8',
                ) as writer:
                    json.dump(manifest, writer, indent=2, sort_keys=True)

                self.worker.install_file(json_manifest, remote_manifest)

            self.worker.check_call([
                'env',
1184
                'DEBIAN_FRONTEND=noninteractive',
1185
                'XDG_DATA_HOME={}/home'.format(self.worker.scratch),
1186
                'http_proxy=http://192.168.122.1:3142',
1187 1188 1189 1190
                'sh', '-euc',
                'cd "$1"; shift; exec "$@"',
                'sh',                   # argv[0]
                self.worker.scratch,    # directory to cd into
Simon McVittie's avatar
Simon McVittie committed
1191
                'flatpak-builder',
1192
                '--arch={}'.format(self.flatpak_arch),
Simon McVittie's avatar
Simon McVittie committed
1193
                '--repo={}'.format(self.repo),
1194
                '--bundle-sources',
Simon McVittie's avatar
Simon McVittie committed
1195 1196 1197 1198
                '{}/workdir'.format(self.worker.scratch),
                remote_manifest,
            ])

1199
            if self.export_bundles:
Simon McVittie's avatar
Simon McVittie committed
1200
                self.worker.check_call([
1201 1202
                    'time',
                    'env',
1203
                    'XDG_DATA_HOME={}/home'.format(self.worker.scratch),
1204 1205
                    'flatpak',
                    'build-bundle',
Simon McVittie's avatar
Simon McVittie committed
1206
                    self.repo,
Simon McVittie's avatar
Simon McVittie committed
1207
                    '{}/bundle'.format(self.worker.scratch),
1208 1209 1210
                    manifest['id'],
                    manifest['branch'],
                ])
Simon McVittie's avatar
Simon McVittie committed
1211

1212 1213 1214 1215 1216 1217 1218 1219 1220 1221 1222 1223 1224 1225
                bundle = '{}-{}-{}.bundle'.format(
                    manifest['id'],
                    self.flatpak_arch,
                    manifest['branch'],
                )
                output = os.path.join(self.build_area, bundle)

                with open(output + '.new', 'wb') as writer:
                    self.worker.check_call([
                        'cat',
                        '{}/bundle'.format(self.worker.scratch),
                    ], stdout=writer)

                os.rename(output + '.new', output)
Simon McVittie's avatar
Simon McVittie committed
1226 1227

if __name__ == '__main__':
Simon McVittie's avatar
Simon McVittie committed
1228 1229 1230 1231 1232 1233 1234 1235 1236 1237 1238
    if sys.stderr.isatty():
        try:
            import colorlog
        except ImportError:
            pass
        else:
            formatter = colorlog.ColoredFormatter(
                '%(log_color)s%(levelname)s:%(name)s:%(reset)s %(message)s')
            handler = logging.StreamHandler()
            handler.setFormatter(formatter)
            logging.getLogger().addHandler(handler)