...
 
Commits (13)
......@@ -40,11 +40,19 @@ import subprocess
import sys
import yaml
try:
import typing
except ImportError:
pass
else:
typing # silence "unused" warnings
logger = logging.getLogger('flatdeb.apt-install')
def main():
# type: (...) -> None
parser = argparse.ArgumentParser(
description='Install the given packages'
)
......@@ -73,8 +81,8 @@ def main():
if var in os.environ:
in_chroot.append('{}={}'.format(var, os.environ[var]))
options = []
packages = []
options = [] # type: typing.List[str]
packages = [] # type: typing.List[str]
if args.debug:
options.append('-oDebug::pkgDepCache::AutoInstall=true')
......@@ -98,6 +106,7 @@ def main():
'apt-get', 'clean',
])
if __name__ == '__main__':
if sys.stderr.isatty():
try:
......
......@@ -37,18 +37,29 @@ import re
import subprocess
import sys
from debian.debian_support import Version
try:
import typing
except ImportError:
pass
else:
typing # silence "unused" warnings
logger = logging.getLogger('flatdeb.collect-source-code')
class InstalledPackage:
def __init__(self, fields):
# type: (typing.Sequence[str]) -> None
self.binary = fields[0]
self.binary_version = fields[1]
self.source = fields[2]
if self.source.endswith(')'):
self.source, self.source_version = self.source.rstrip(')').split(' (')
self.source, self.source_version = (
self.source.rstrip(')').split(' ('))
else:
self.source_version = self.binary_version
......@@ -58,12 +69,15 @@ class InstalledPackage:
self.installed_size = fields[3]
def __str__(self):
# type: () -> str
return '{}_{}'.format(self.binary, self.binary_version)
def __hash__(self):
# type: () -> int
return hash(self.binary) ^ hash(self.binary_version)
def __eq__(self, other):
# type: (typing.Any) -> bool
if isinstance(other, InstalledPackage):
return (
self.binary,
......@@ -78,16 +92,20 @@ class InstalledPackage:
class SourceRequired:
def __init__(self, source, source_version):
# type: (str, Version) -> None
self.source = source
self.source_version = source_version
def __str__(self):
# type: () -> str
return 'src:{}_{}'.format(self.source, self.source_version)
def __hash__(self):
# type: () -> int
return hash(self.source) ^ hash(self.source_version)
def __eq__(self, other):
# type: (typing.Any) -> bool
if isinstance(other, SourceRequired):
return (
self.source,
......@@ -99,8 +117,28 @@ class SourceRequired:
else:
return NotImplemented
def __lt__(self, other):
# type: (typing.Any) -> bool
if isinstance(other, SourceRequired):
return (
self.source,
Version(self.source_version),
) < (
other.source,
Version(other.source_version),
)
else:
return NotImplemented
@property
def get_source(self):
# type: () -> str
return '{}={}'.format(self.source, self.source_version)
def read_manifest(path):
# type: (str) -> typing.List[InstalledPackage]
ret = []
with open(path, encoding='utf-8') as reader:
......@@ -120,6 +158,8 @@ def read_manifest(path):
def read_built_using(path):
# type: (str) -> typing.Set[SourceRequired]
ret = set()
with open(path, encoding='utf-8') as reader:
......@@ -140,6 +180,7 @@ def read_built_using(path):
def main():
# type: (...) -> None
parser = argparse.ArgumentParser(
description='Collect source code',
)
......@@ -202,8 +243,10 @@ def main():
if os.path.exists(platform_built_using):
sources_required |= read_built_using(platform_built_using)
sources = []
missing_sources = set()
sources = set() # type: typing.Set[SourceRequired]
get_source = [] # type: typing.List[str]
included = set() # type: typing.Set[SourceRequired]
missing_sources = set() # type: typing.Set[SourceRequired]
for s in sources_required:
source = s.source
......@@ -216,7 +259,9 @@ def main():
source_version = strip_source_version_suffix.sub(
'', source_version)
sources.append('{}={}'.format(source, source_version))
s = SourceRequired(source, source_version)
sources.add(s)
get_source.append(s.get_source)
try:
subprocess.check_call(in_chroot + [
......@@ -226,13 +271,13 @@ def main():
'/src/files', # working directory
'apt-get', '-y', '--download-only', '-q', '-q',
'-oAPT::Get::Only-Source=true', 'source',
] + sources)
] + get_source)
except subprocess.CalledProcessError:
logger.warning(
'Unable to download some sources as a batch, trying '
'to download sources individually')
for source in sources:
for s in sources:
try:
subprocess.check_call(in_chroot + [
'sh', '-euc',
......@@ -241,30 +286,44 @@ def main():
'/src/files', # working directory
'apt-get', '-y', '--download-only', '-q', '-q',
'-oAPT::Get::Only-Source=true', 'source',
source,
s.get_source,
])
except subprocess.CalledProcessError:
# Non-fatal for now
logger.warning(
'Unable to get source code for %s', source)
missing_sources.add(source)
source_package = source.split('=', 1)[0]
missing_sources.add(s)
subprocess.call(in_chroot + [
'apt-cache', 'showsrc', source_package,
'apt-cache', 'showsrc', s.source,
])
else:
included.add(s)
else:
included = set(sources)
with open(
os.path.join(args.sysroot, 'src', 'files', 'sources.txt'), 'w'
) as writer:
writer.write('#Source\t#Version\n')
for s in sorted(included):
writer.write('{}\t{}\n'.format(s.source, s.source_version))
if missing_sources:
logger.warning('Missing source packages:')
with open(
os.path.join(args.sysroot, 'src', 'MISSING.txt'), 'w'
os.path.join(args.sysroot, 'src', 'files', 'MISSING.txt'), 'w'
) as writer:
for p in sorted(missing_sources):
writer.write('#Source\t#Version\n')
for s in sorted(missing_sources):
logger.warning('- %s', p)
writer.write('{}\n'.format(p))
writer.write('{}\t{}\n'.format(s.source, s.source_version))
logger.warning('Check that this runtime is GPL-compliant!')
if __name__ == '__main__':
if sys.stderr.isatty():
try:
......
......@@ -294,12 +294,19 @@ actions:
-C "$ROOTDIR/debug"
.
- action: run
label: list of included source code
chroot: false
command: >
cp -v "$ROOTDIR/src/files/sources.txt"
"$ARTIFACTDIR/{{ $sources_prefix }}.sources.txt"
- action: run
label: list of missing source code
chroot: false
command: >
test ! -e "$ROOTDIR/src/MISSING.txt" ||
cp -v "$ROOTDIR/src/MISSING.txt"
test ! -e "$ROOTDIR/src/files/MISSING.txt" ||
cp -v "$ROOTDIR/src/files/MISSING.txt"
"$ARTIFACTDIR/{{ $sources_prefix }}.MISSING.txt"
{{ end }}
......@@ -311,6 +318,8 @@ actions:
head -n10000
files/manifest.dpkg
files/manifest.dpkg.built-using
src/files/sources.txt
|| true
- action: run
label: metadata
......@@ -322,7 +331,7 @@ actions:
files/etc/apt/apt.conf.d/*
files/etc/apt/sources.list
files/etc/apt/sources.list.d/*
files/src/MISSING.txt
files/lib/os-release
src/files/MISSING.txt
metadata
|| true
......@@ -59,11 +59,15 @@ in_chroot () {
}
list_packages_ignore_arch () {
# ${} here is interpreted by dpkg-query
# shellcheck disable=SC2016
in_chroot dpkg-query --show -f '${Package}\n' | LC_ALL=C sort -u
}
is_installed () {
local status
# ${} here is interpreted by dpkg-query
# shellcheck disable=SC2016
if ! status="$(in_chroot dpkg-query --show -f '${Status}\n' "$@")"; then
return 1
fi
......
#!/bin/bash
#!/usr/bin/python3
# flatdeb — build Flatpak runtimes from Debian packages
#
# Copyright © 2016-2017 Simon McVittie
# Copyright © 2017-2019 Collabora Ltd.
#
......@@ -24,51 +26,105 @@
# TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
# SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
set -e
set -u
set -o pipefail
me="$(basename "$0")"
if [ "$#" != 1 ] || [ "$1" = '--help' ] || [ "x$(id -u)" != x0 ] ||
! [ -d "$1" ]; then
echo "$me: Usage:" >&2
echo " $me SYSROOT" >&2
echo "" >&2
echo "Run this script as root." >&2
exit 2
fi
sysroot="$1"
cd "$sysroot"
in_chroot () {
local var
for var in ftp_proxy http_proxy https_proxy no_proxy; do
if [ -n "${!var-}" ]; then
set -- "${var}=${!var}" "$@"
fi
done
systemd-nspawn --directory="$sysroot" --as-pid2 --tmpfs=/run/lock \
env DEBIAN_FRONTEND=noninteractive SUDO_FORCE_REMOVE=yes \
"$@"
}
echo "Purging leftover configuration files:"
in_chroot dpkg-query --show -f '${Status}:${Package}\n' \
| while read -r line; do
case "$line" in
(*\ config-files:*)
p="${line##*:}"
echo "- $p"
unwanted+=("$p")
;;
esac
done
if [ -n "${unwanted[*]}" ]; then
in_chroot dpkg --purge --force-remove-essential --force-depends \
"${unwanted[@]}"
fi
import argparse
import logging
import os
import subprocess
import sys
try:
import typing
except ImportError:
pass
else:
typing # silence "unused" warnings
logger = logging.getLogger('flatdeb.purge-conffiles')
def main():
# type: (...) -> None
parser = argparse.ArgumentParser(
description='Purge leftover configuration files'
)
parser.add_argument('sysroot')
args = parser.parse_args()
in_chroot = [
'systemd-nspawn',
'--directory={}'.format(args.sysroot),
'--as-pid2',
'--tmpfs=/run/lock',
'env',
'DEBIAN_FRONTEND=noninteractive',
'SUDO_FORCE_REMOVE=yes',
]
for var in ('ftp_proxy', 'http_proxy', 'https_proxy', 'no_proxy'):
if var in os.environ:
in_chroot.append('{}={}'.format(var, os.environ[var]))
logger.info('Purging leftover configuration files:')
unwanted = set() # type: typing.Set[str]
with subprocess.Popen(in_chroot + [
'dpkg-query',
'--show',
'-f', r'${Status}:${Package}\n',
], stdout=subprocess.PIPE, universal_newlines=True) as dpkg_query:
for line in dpkg_query.stdout:
if line == '\n':
continue
if ':' not in line:
raise AssertionError('dpkg-query produced {!r}'.format(line))
status, package = line.rsplit(':')
if status.endswith(' config-files'):
logger.info('- %s', package)
unwanted.add(package)
if dpkg_query.wait() != 0:
raise subprocess.CalledProcessError(
returncode=dpkg_query.returncode,
cmd=dpkg_query.args,
)
if unwanted:
subprocess.check_call(in_chroot + [
'dpkg',
'--purge',
'--force-depends',
'--force-remove-essential',
] + list(unwanted))
if __name__ == '__main__':
if sys.stderr.isatty():
try:
import colorlog
except ImportError:
pass
else:
formatter = colorlog.ColoredFormatter(
'%(log_color)s%(levelname)s:%(name)s:%(reset)s %(message)s')
handler = logging.StreamHandler()
handler.setFormatter(formatter)
logging.getLogger().addHandler(handler)
else:
logging.basicConfig()
logging.getLogger().setLevel(logging.DEBUG)
try:
main()
except KeyboardInterrupt:
raise SystemExit(130)
except subprocess.CalledProcessError as e:
logger.error('%s', e)
raise SystemExit(1)
......@@ -29,7 +29,6 @@ set -u
set -o pipefail
me="$(basename "$0")"
NULL=
if [ "$#" != 0 ] || [ "x$(id -u)" != x0 ]; then
echo "$me: Usage:" >&2
......
......@@ -29,7 +29,6 @@ set -u
set -o pipefail
me="$(basename "$0")"
NULL=
if [ "$#" != 0 ] || [ "x$(id -u)" != x0 ]; then
echo "$me: Usage:" >&2
......
......@@ -103,18 +103,23 @@ exec 3>"$sysroot/usr/manifest.dpkg"
printf '#Package[:Architecture]\t#Version\t#Source\t#Installed-Size\n' >&3
# ${} escapes here are for dpkg-query, not sh
# shellcheck disable=SC2016
dpkg_version="$(in_chroot dpkg-query -W -f '${Version}' dpkg)"
if in_chroot dpkg --compare-versions "$dpkg_version" ge 1.16.2; then
# shellcheck disable=SC2016
in_chroot dpkg-query -W -f \
'${binary:Package}\t${Version}\t${Source}\t${Installed-Size}\n' \
| LC_ALL=C sort -u >&3
# shellcheck disable=SC2016
in_chroot dpkg-query -W -f '${binary:Package}\n' \
| LC_ALL=C sort -u \
| in_chroot xargs -d '\n' dpkg-query -s \
| gzip -9nc \
> "$sysroot/usr/manifest.deb822.gz"
else
# shellcheck disable=SC2016
in_chroot dpkg-query -W -f \
'${Package}:${Architecture}\t${Version}\t${Source}\t${Installed-Size}\n' \
| LC_ALL=C sort -u >&3
......@@ -124,6 +129,7 @@ exec 3>"$sysroot/usr/manifest.dpkg.built-using"
printf '#Built-Binary\t#Built-Using-Source\t#Built-Using-Version\n' >&3
# shellcheck disable=SC2016
in_chroot dpkg-query -W -f '$Package\t${Built-Using}\n' | perl -ne '
chomp;
next unless /^\S+\t.+$/;
......
This diff is collapsed.
#!/bin/sh
# Copyright © 2016-2018 Simon McVittie
# Copyright © 2018 Collabora Ltd.
#
# SPDX-License-Identifier: MIT
set -e
set -u
export MYPYPATH="${PYTHONPATH:=$(pwd)}"
i=0
for script in \
./*.py \
flatdeb/apt-install \
flatdeb/collect-source-code \
flatdeb/purge-conffiles \
; do
i=$((i + 1))
if [ "x${MYPY:="$(command -v mypy || echo false)"}" = xfalse ]; then
echo "ok $i - $script # SKIP mypy not found"
elif "${MYPY}" \
--python-executable="${PYTHON:=python3}" \
--follow-imports=skip \
--ignore-missing-imports \
"$script"; then
echo "ok $i - $script"
else
echo "not ok $i - $script # TODO mypy issues reported"
fi
done
echo "1..$i"
# vim:set sw=4 sts=4 et:
#!/bin/sh
# Copyright © 2016-2018 Simon McVittie
# Copyright © 2018 Collabora Ltd.
#
# SPDX-License-Identifier: MIT
set -e
set -u
if [ "x${PYCODESTYLE:=pycodestyle}" = xfalse ] || \
[ -z "$(command -v "$PYCODESTYLE")" ]; then
echo "1..0 # SKIP pycodestyle not found"
elif "${PYCODESTYLE}" \
./*.py \
flatdeb/apt-install \
flatdeb/collect-source-code \
flatdeb/purge-conffiles \
>&2; then
echo "1..1"
echo "ok 1 - $PYCODESTYLE reported no issues"
else
echo "1..1"
echo "not ok 1 # TODO $PYCODESTYLE issues reported"
fi
# vim:set sw=4 sts=4 et:
#!/bin/sh
# Copyright © 2016-2018 Simon McVittie
# Copyright © 2018 Collabora Ltd.
#
# SPDX-License-Identifier: MIT
set -e
set -u
if [ "x${PYFLAKES:=pyflakes3}" = xfalse ] || \
[ -z "$(command -v "$PYFLAKES")" ]; then
echo "1..0 # SKIP pyflakes3 not found"
elif "${PYFLAKES}" \
./*.py \
flatdeb/apt-install \
flatdeb/collect-source-code \
flatdeb/purge-conffiles \
>&2; then
echo "1..1"
echo "ok 1 - $PYFLAKES reported no issues"
else
echo "1..1"
echo "not ok 1 # TODO $PYFLAKES issues reported"
fi
# vim:set sw=4 sts=4 et:
#!/bin/sh
#
# Copyright © 2018-2019 Collabora Ltd
#
# SPDX-License-Identifier: MIT
set -e
set -u
if ! command -v shellcheck >/dev/null 2>&1; then
echo "1..0 # SKIP shellcheck not available"
exit 0
fi
n=0
for shell_script in \
deb-buildapi/configure \
flatdeb/add-foreign-architectures \
flatdeb/clean-up-base \
flatdeb/clean-up-before-pack \
flatdeb/disable-services \
flatdeb/make-flatpak-friendly \
flatdeb/platformize \
flatdeb/prepare-runtime \
flatdeb/put-ldconfig-in-path \
flatdeb/symlink-alternatives \
flatdeb/usrmerge \
flatdeb/write-manifest \
run-in-fakemachine \
t/*.sh \
; do
n=$((n + 1))
# Ignore SC2039: we assume a Debian-style shell that has 'local'.
if shellcheck --exclude=SC2039 "$shell_script"; then
echo "ok $n - $shell_script"
else
echo "not ok $n # TODO - $shell_script"
fi
done
echo "1..$n"
# vim:set sw=4 sts=4 et ft=sh: