Automatically download debug symbols for everything in the Platform

This reduces the need to have a giant metapackage with all the shared
libraries' detached debug symbols; now we only need to do that for
libraries that are still using legacy debug symbols (foo-dbg).

This also means we tolerate transient uninstallability for debug
symbols: we'll simply produce a runtime without those debug symbols.
This is particularly important when taking binaries from
debian-security, which does not have a corresponding debug symbols
archive (although when security updates get mirrored into
proposed-updates, their detached debug symbols do appear there).

Finally, it also means we don't run into problems with version skew
between debug symbols and the actual libraries, particularly when
adding buster-proposed-updates-debug (which, in general, will be
strictly newer than buster + buster-security) in the hope of getting
detached debug symbols for buster-security.
Signed-off-by: Simon McVittie <smcv@collabora.com>

flatdeb/collect-dbgsym

+#!/usr/bin/python3
+
+# flatdeb — build Flatpak runtimes from Debian packages
+#
+# Copyright © 2016-2017 Simon McVittie
+# Copyright © 2017-2018 Collabora Ltd.
+#
+# SPDX-License-Identifier: MIT
+#
+# Permission is hereby granted, free of charge, to any person obtaining
+# a copy of this software and associated documentation files (the
+# "Software"), to deal in the Software without restriction, including
+# without limitation the rights to use, copy, modify, merge, publish,
+# distribute, sublicense, and/or sell copies of the Software, and to
+# permit persons to whom the Software is furnished to do so, subject to
+# the following conditions:
+#
+# The above copyright notice and this permission notice shall be included
+# in all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
+# IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
+# CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
+# TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
+# SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+
+"""
+Fetch source code for packages installed in the given sysroot.
+"""
+
+import argparse
+import logging
+import os
+import subprocess
+import sys
+
+try:
+    import typing
+except ImportError:
+    pass
+else:
+    typing  # silence "unused" warnings
+
+
+logger = logging.getLogger('flatdeb.collect-dbgsym')
+
+
+class InstalledPackage:
+    def __init__(self, fields):
+        # type: (typing.Sequence[str]) -> None
+        self.binary = fields[0]
+        self.binary_version = fields[1]
+
+    def __str__(self):
+        # type: () -> str
+        return '{}_{}'.format(self.binary, self.binary_version)
+
+    def __hash__(self):
+        # type: () -> int
+        return hash(self.binary) ^ hash(self.binary_version)
+
+    def __eq__(self, other):
+        # type: (typing.Any) -> bool
+        if isinstance(other, InstalledPackage):
+            return (
+                self.binary,
+                self.binary_version,
+            ) == (
+                other.binary,
+                other.binary_version,
+            )
+        else:
+            return NotImplemented
+
+    @property
+    def dbgsym_package(self):
+        # type: () -> str
+
+        if ':' in self.binary:
+            return self.binary.replace(':', '-dbgsym:')
+        else:
+            return self.binary + '-dbgsym'
+
+
+def read_manifest(path):
+    # type: (str) -> typing.List[InstalledPackage]
+
+    ret = []
+
+    with open(path, encoding='utf-8') as reader:
+        for line in reader:
+            line = line.rstrip('\n')
+
+            if not line:
+                continue
+
+            if line.startswith('#'):
+                continue
+
+            assert '\t' in line, repr(line)
+            ret.append(InstalledPackage(line.rstrip('\n').split('\t')))
+
+    return ret
+
+
+def main():
+    # type: (...) -> None
+    parser = argparse.ArgumentParser(
+        description='Install corresponding -dbgsym packages if available',
+    )
+    parser.add_argument('sysroot')
+
+    args = parser.parse_args()
+
+    in_chroot = [
+        'systemd-nspawn',
+        '--directory={}'.format(args.sysroot),
+        '--as-pid2',
+        '--tmpfs=/run/lock',
+    ]
+    in_chroot_quick = [
+        'chroot', args.sysroot,
+    ]
+
+    in_chroot.append('env')
+
+    for var in ('ftp_proxy', 'http_proxy', 'https_proxy', 'no_proxy'):
+        if var in os.environ:
+            in_chroot.append('{}={}'.format(var, os.environ[var]))
+
+    platform_manifest = os.path.join(
+        args.sysroot, 'usr', 'manifest.dpkg.platform')
+
+    if os.path.exists(platform_manifest):
+        platform_packages = read_manifest(platform_manifest)
+    else:
+        platform_packages = []
+
+    to_get = []     # type: typing.List[str]
+
+    for p in platform_packages:
+        logger.info('Package in Platform: %s', p)
+        to_get.append('{}={}'.format(p.dbgsym_package, p.binary_version))
+
+    subprocess.run(in_chroot + [
+        'sh', '-euc', '''
+        for x in "$@"; do
+            apt-get -y -q -q -m install "$x" || true
+        done
+        ''',
+        'sh',   # $0
+    ] + to_get)
+
+    parent = os.path.join(
+        args.sysroot, 'usr', 'lib', 'debug',
+    )
+    os.makedirs(parent, exist_ok=True)
+
+    with open(
+        os.path.join(parent, 'dbgsym-packages-installed.txt'),
+        'w',
+    ) as writer, open(
+        os.path.join(parent, 'dbgsym-packages-not-installed.txt'),
+        'w',
+    ) as missing_writer:
+        for p in platform_packages:
+            if subprocess.run(
+                in_chroot_quick + [
+                    'dpkg-query', '-W', p.dbgsym_package,
+                ],
+                stdout=writer,
+                stderr=subprocess.DEVNULL,
+            ).returncode == 0:
+                logger.info(
+                    'Installed -dbgsym package: %s_%s',
+                    p.dbgsym_package,
+                    p.binary_version,
+                )
+            else:
+                result = subprocess.run(
+                    in_chroot_quick + [
+                        'apt-cache', '-q', 'showpkg', p.dbgsym_package,
+                    ],
+                    stdout=subprocess.PIPE,
+                )
+
+                if result.returncode == 0 and result.stdout:
+                    logger.warning(
+                        'Unable to install -dbgsym package: %s_%s',
+                        p.dbgsym_package,
+                        p.binary_version,
+                    )
+                    logger.warning(
+                        'Versions available:\n%s',
+                        result.stdout.decode('utf-8', errors='replace'),
+                    )
+                    missing_writer.write('{}\t{}\n'.format(
+                        p.dbgsym_package,
+                        p.binary_version,
+                    ))
+                else:
+                    logger.info(
+                        'Skipped nonexistent -dbgsym package: %s',
+                        p.dbgsym_package,
+                    )
+
+
+if __name__ == '__main__':
+    if sys.stderr.isatty():
+        try:
+            import colorlog
+        except ImportError:
+            pass
+        else:
+            formatter = colorlog.ColoredFormatter(
+                '%(log_color)s%(levelname)s:%(name)s:%(reset)s %(message)s')
+            handler = logging.StreamHandler()
+            handler.setFormatter(formatter)
+            logging.getLogger().addHandler(handler)
+    else:
+        logging.basicConfig()
+
+    logging.getLogger().setLevel(logging.DEBUG)
+
+    try:
+        main()
+    except KeyboardInterrupt:
+        raise SystemExit(130)
+    except subprocess.CalledProcessError as e:
+        logger.error('%s', e)
+        raise SystemExit(1)

flatdeb/dbgsym-use-build-id

@@ -65,6 +65,9 @@ def main():
             subdirs.remove('.build-id')
 
         for f in files:
+            if f.endswith('.txt'):
+                continue
+
             # Scrape the output of readelf to find the build-ID for this
             # binary
             path = os.path.join(dirname, f)

flatdeb/debos-runtimes.yaml

@@ -60,14 +60,38 @@ actions:
 
   # TODO: Everything before this step is common to the Platform and SDK
 
+  {{ if $sdk }}
+  - action: run
+    label: copy manifest from platform
+    chroot: false
+    command: 'echo; cp "$RECIPEDIR/manifest.dpkg.platform" "$ROOTDIR/usr"'
+  - action: run
+    label: copy manifest from platform
+    chroot: false
+    command: 'echo; cp "$RECIPEDIR/manifest.dpkg.built-using.platform" "$ROOTDIR/usr"'
+  - action: run
+    label: copy manifest from platform
+    chroot: false
+    command: 'echo; cp "$RECIPEDIR/manifest.platform.deb822.gz" "$ROOTDIR/usr"'
+  {{ if $debug_symbols }}
+  - action: run
+    label: collect-dbgsym
+    chroot: false
+    command: |
+      set -e
+      echo
+      "$RECIPEDIR/collect-dbgsym" "$ROOTDIR"
+  {{ end }}
   # We do this as a separate step in case it involves removing any
-  # of the common packages
-  {{ if and $sdk $sdk_packages }}
+  # of the common packages, or replacing automatic debug symbols with
+  # legacy debug symbols that might have more coverage
+  {{ if $sdk_packages }}
   - action: run
     label: apt-install
     chroot: false
     command: echo; $RECIPEDIR/apt-install {{ if $apt_debug }}--debug{{ end }} "$ROOTDIR" "$RECIPEDIR/runtimes/{{ $runtime }}/sdk_packages.yaml"
   {{ end }}
+  {{ end }}
 
   {{ if not $sdk }}
   - action: run
@@ -136,18 +160,6 @@ actions:
       fi
 
   {{ if $sdk }}
-  - action: run
-    label: copy manifest from platform
-    chroot: false
-    command: 'echo; cp "$RECIPEDIR/manifest.dpkg.platform" "$ROOTDIR/usr"'
-  - action: run
-    label: copy manifest from platform
-    chroot: false
-    command: 'echo; cp "$RECIPEDIR/manifest.dpkg.built-using.platform" "$ROOTDIR/usr"'
-  - action: run
-    label: copy manifest from platform
-    chroot: false
-    command: 'echo; cp "$RECIPEDIR/manifest.platform.deb822.gz" "$ROOTDIR/usr"'
   - action: run
     label: collect-source-code
     chroot: false

run.py

@@ -895,6 +895,7 @@ class Builder:
                 'apt-install',
                 'clean-up-base',
                 'clean-up-before-pack',
+                'collect-dbgsym',
                 'collect-source-code',
                 'dbgsym-use-build-id',
                 'disable-services',

t/mypy.sh

@@ -13,6 +13,7 @@ i=0
 for script in \
     ./*.py \
     flatdeb/apt-install \
+    flatdeb/collect-dbgsym \
     flatdeb/collect-source-code \
     flatdeb/dbgsym-use-build-id \
     flatdeb/purge-conffiles \

t/pycodestyle.sh

@@ -13,6 +13,7 @@ if [ "x${PYCODESTYLE:=pycodestyle}" = xfalse ] || \
 elif "${PYCODESTYLE}" \
     ./*.py \
     flatdeb/apt-install \
+    flatdeb/collect-dbgsym \
     flatdeb/collect-source-code \
     flatdeb/dbgsym-use-build-id \
     flatdeb/purge-conffiles \

t/pyflakes.sh

@@ -13,6 +13,7 @@ if [ "x${PYFLAKES:=pyflakes3}" = xfalse ] || \
 elif "${PYFLAKES}" \
     ./*.py \
     flatdeb/apt-install \
+    flatdeb/collect-dbgsym \
     flatdeb/collect-source-code \
     flatdeb/dbgsym-use-build-id \
     flatdeb/purge-conffiles \