Merge branch 'wip/fdanis/strip_source_version_suffix_for_apps' into 'master'

apps: Add strip_source_version_suffix support

See merge request smcv/flatdeb!2

README

@@ -55,6 +55,9 @@ Then you can run:
     ./run.py --suite=stretch --arch=amd64 app \
         apps/org.debian.packages.mesa_utils.yaml
 
+To use the ./run.py app subcommand, you must include python3 and python3-debian
+in the SDK.
+
 On the same machine, or a test machine onto which you have copied
 `$HOME/.cache/flatdeb/repo` with `rsync` or similar:
 

flatdeb/collect-app-source-code

+#!/usr/bin/python3
+
+# flatdeb - build Flatpak runtimes from Debian packages
+#
+# Copyright (c) 2020 Collabora Ltd.
+#
+# SPDX-License-Identifier: MIT
+#
+# Permission is hereby granted, free of charge, to any person obtaining
+# a copy of this software and associated documentation files (the
+# "Software"), to deal in the Software without restriction, including
+# without limitation the rights to use, copy, modify, merge, publish,
+# distribute, sublicense, and/or sell copies of the Software, and to
+# permit persons to whom the Software is furnished to do so, subject to
+# the following conditions:
+#
+# The above copyright notice and this permission notice shall be included
+# in all copies or substantial portions of the Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
+# IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
+# CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
+# TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
+# SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+
+"""
+Fetch source code for app packages.
+"""
+
+import argparse
+import glob
+import logging
+import os
+import re
+import shutil
+import subprocess
+import sys
+from debian.debfile import DebFile
+
+logger = logging.getLogger('flatdeb.collect-app-source-code')
+
+
+def main():
+    parser = argparse.ArgumentParser(
+        description='Collect source code for app',
+    )
+    parser.add_argument('--strip-source-version-suffix', default='')
+    parser.add_argument('--export', default=os.getcwd())
+    parser.add_argument('packages', nargs='*')
+
+    args = parser.parse_args()
+
+    strip_source_version_suffix = None
+
+    if args.strip_source_version_suffix:
+        strip_source_version_suffix = re.compile(
+            '(?:' + args.strip_source_version_suffix + ')$')
+
+    subprocess.check_call(['cp', '-PRp', '/usr/var', '/'])
+    subprocess.check_call(['install', '-d', '/var/cache/apt/archives/partial'])
+    subprocess.check_call(['fakeroot', 'apt-get', 'update'])
+    subprocess.check_call(['fakeroot', 'apt-get', '-y', '--download-only',
+                          '--no-install-recommends', 'install'] +
+                          args.packages)
+
+    for file in glob.glob("/var/cache/apt/archives/*.deb"):
+        debian_control = DebFile(file).control.debcontrol()
+        package = debian_control.get('Package')
+        source = debian_control.get('Source')
+        bu = debian_control.get('Built-Using')
+        version = debian_control.get('Version')
+
+        if not source:
+            source = package
+        if source.endswith(')'):
+            source, version = source.rstrip(')').split(' (')
+        if strip_source_version_suffix is not None:
+            version = strip_source_version_suffix.sub('', version)
+        assert source
+        assert version
+
+        subprocess.check_call([
+            'apt-get', '-y', '--download-only',
+            '-oAPT::Get::Only-Source=true', 'source',
+            '{}={}'.format(source, version),
+        ], cwd=args.export)
+
+        if bu:
+            for dep in bu.split(', '):
+                source, version = dep.rstrip(')').split(' (= ')
+                if strip_source_version_suffix is not None:
+                    version = strip_source_version_suffix.sub('', version)
+                assert source
+                assert version
+
+                subprocess.check_call([
+                    'apt-get', '-y', '--download-only',
+                    '-oAPT::Get::Only-Source=true', 'source',
+                    '{}={}'.format(source, version),
+                ], cwd=args.export)
+
+    for file in glob.glob("/var/cache/apt/archives/*.deb"):
+        shutil.move(file, args.export)
+    shutil.move('/var/lib/apt/lists', args.export)
+
+
+if __name__ == '__main__':
+    if sys.stderr.isatty():
+        try:
+            import colorlog
+        except ImportError:
+            pass
+        else:
+            formatter = colorlog.ColoredFormatter(
+                '%(log_color)s%(levelname)s:%(name)s:%(reset)s %(message)s')
+            handler = logging.StreamHandler()
+            handler.setFormatter(formatter)
+            logging.getLogger().addHandler(handler)
+    else:
+        logging.basicConfig()
+
+    logging.getLogger().setLevel(logging.DEBUG)
+
+    try:
+        main()
+    except KeyboardInterrupt:
+        raise SystemExit(130)

run.py

@@ -1742,7 +1742,8 @@ class Builder:
         manifest['runtime-version'] = self.runtime_branch
 
         if self.remote_url is None:
-            self.remote_url = 'file://{}'.format(urllib.parse.quote(self.ostree_repo))
+            self.remote_url = 'file://{}'.format(
+                urllib.parse.quote(self.ostree_repo))
 
         with ExitStack() as stack:
             # We assume the build area has xattr support
@@ -1856,6 +1857,14 @@ class Builder:
                                 dir=scratch,
                             ),
                         )
+                        shutil.copy2(
+                            os.path.join(
+                                os.path.dirname(__file__),
+                                'flatdeb',
+                                'collect-app-source-code',
+                            ),
+                            packages
+                        )
                         subprocess.check_call([
                             'env',
                             'XDG_DATA_HOME={}/home'.format(scratch),
@@ -1869,56 +1878,14 @@ class Builder:
                                 self.runtime_branch,
                             ),
                             'DEBIAN_FRONTEND=noninteractive',
-                            'export={}'.format(packages),
-                            'sh',
-                            '-euc',
-
-                            'cp -PRp /usr/var /\n'
-                            'install -d /var/cache/apt/archives/partial\n'
-                            'fakeroot apt-get update\n'
-                            'fakeroot apt-get -y --download-only \\\n'
-                            '    --no-install-recommends install "$@"\n'
-                            'for x in /var/cache/apt/archives/*.deb; do\n'
-                            '    package="$(dpkg-deb -f "$x" Package)"\n'
-                            '    source="$(dpkg-deb -f "$x" Source)"\n'
-                            '    bu="$(dpkg-deb -f "$x" Built-Using)"\n'
-                            '    version="$(dpkg-deb -f "$x" Version)"\n'
-                            '    if [ -z "$source" ]; then\n'
-                            '        source="$package"\n'
-                            '    fi\n'
-                            '    if [ "${source% (*}" != "$source" ]; then\n'
-                            '        version="${source#* (}"\n'
-                            '        version="${version%)}"\n'
-                            '        source="${source% (*}"\n'
-                            '    fi\n'
-                            '    ( cd "$export" && \\\n'
-                            '         apt-get -y --download-only \\\n'
-                            '         -oAPT::Get::Only-Source=true source \\\n'
-                            '         "$source=$version"\n'
-                            '    )\n'
-                            '    if [ -n "$bu" ]; then\n'
-                            '        oldIFS="$IFS"\n'
-                            '        IFS=","\n'
-                            '        for dep in $bu; do\n'
-                            '            bu="$(echo "$bu" | tr -d " ")"\n'
-                            '            version="${bu#*(=}"\n'
-                            '            version="${version%)}"\n'
-                            '            source="${bu%(*}"\n'
-                            '            ( cd "$export" && \\\n'
-                            '               apt-get -y --download-only \\\n'
-                            '               -oAPT::Get::Only-Source=true \\\n'
-                            '               source "$source=$version"\n'
-                            '            )\n'
-                            '        done\n'
-                            '        IFS="$oldIFS"\n'
-                            '    fi\n'
-                            'done\n'
-                            'mv /var/cache/apt/archives/*.deb "$export"\n'
-                            'mv /var/lib/apt/lists "$export"\n'
-                            '',
-
-                            'sh',   # argv[0]
+                            '{}/collect-app-source-code'.format(packages),
+                            '--export={}'.format(packages),
+                            '--strip-source-version-suffix={}'.format(
+                                self.strip_source_version_suffix),
                         ] + module['x-flatdeb-apt-packages'])
+                        os.remove(
+                            os.path.join(packages, 'collect-app-source-code')
+                        )
 
                         obtained = subprocess.check_output([
                             'sh', '-euc',

runtimes/net.debian.flatpak.Base.yaml

@@ -17,6 +17,7 @@ sdk:
     - gdb
     - libc6-dbg
     - ltrace
+    - python3-debian
     - strace
     - valgrind
 ...

runtimes/net.debian.flatpak.Games.yaml

@@ -48,6 +48,7 @@ sdk:
     - libsdl2-dev
     - libvorbis-dev
     - ltrace
+    - python3-debian
     - strace
     - valgrind
     - zlib1g-dev

t/mypy.sh

@@ -14,6 +14,7 @@ for script in \
     ./*.py \
     flatdeb/apt-install \
     flatdeb/collect-dbgsym \
+    flatdeb/collect-app-source-code \
     flatdeb/collect-source-code \
     flatdeb/dbgsym-use-build-id \
     flatdeb/purge-conffiles \

t/pycodestyle.sh

@@ -14,6 +14,7 @@ elif "${PYCODESTYLE}" \
     ./*.py \
     flatdeb/apt-install \
     flatdeb/collect-dbgsym \
+    flatdeb/collect-app-source-code \
     flatdeb/collect-source-code \
     flatdeb/dbgsym-use-build-id \
     flatdeb/purge-conffiles \

t/pyflakes.sh

@@ -14,6 +14,7 @@ elif "${PYFLAKES}" \
     ./*.py \
     flatdeb/apt-install \
     flatdeb/collect-dbgsym \
+    flatdeb/collect-app-source-code \
     flatdeb/collect-source-code \
     flatdeb/dbgsym-use-build-id \
     flatdeb/purge-conffiles \