Commit bd8f60a9 authored by Simon McVittie's avatar Simon McVittie

collect-dbgsym: Install all -dbgsym packages in one transaction

If we have (for example) 400 -dbgsym packages to install, invoking apt
400 times is rather slow due to startup overhead (locking, reading
and updating the database). Instead, invoke apt-cache once to find out
what is available, and use that to compose a single very large apt-get
command-line.

Similarly, we can summarize what was and wasn't installed from the
information we already have, without having to invoke apt a lot.
Signed-off-by: Simon McVittie's avatarSimon McVittie <smcv@collabora.com>
parent 1435a5dc
......@@ -27,7 +27,7 @@
# SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
"""
Fetch source code for packages installed in the given sysroot.
Fetch detached debug symbols for packages installed in the given sysroot.
"""
import argparse
......@@ -43,6 +43,8 @@ except ImportError:
else:
typing # silence "unused" warnings
from debian.deb822 import Packages
logger = logging.getLogger('flatdeb.collect-dbgsym')
......@@ -50,38 +52,46 @@ logger = logging.getLogger('flatdeb.collect-dbgsym')
class InstalledPackage:
def __init__(self, fields):
# type: (typing.Sequence[str]) -> None
self.binary = fields[0]
self.qualified_binary = fields[0]
self.binary = self.qualified_binary.split(':')[0]
self.binary_version = fields[1]
self.source = fields[2] or self.binary
def __str__(self):
# type: () -> str
return '{}_{}'.format(self.binary, self.binary_version)
return '{}_{}'.format(self.qualified_binary, self.binary_version)
def __hash__(self):
# type: () -> int
return hash(self.binary) ^ hash(self.binary_version)
return hash(self.qualified_binary) ^ hash(self.binary_version)
def __eq__(self, other):
# type: (typing.Any) -> bool
if isinstance(other, InstalledPackage):
return (
self.binary,
self.qualified_binary,
self.binary_version,
self.source,
) == (
other.binary,
other.qualified_binary,
other.binary_version,
other.source,
)
else:
return NotImplemented
@property
def dbgsym_package(self):
return self.binary + '-dbgsym'
@property
def qualified_dbgsym_package(self):
# type: () -> str
if ':' in self.binary:
return self.binary.replace(':', '-dbgsym:')
if ':' in self.qualified_binary:
return self.qualified_binary.replace(':', '-dbgsym:')
else:
return self.binary + '-dbgsym'
return self.qualified_binary + '-dbgsym'
def read_manifest(path):
......@@ -138,20 +148,27 @@ def main():
else:
platform_packages = []
to_get = [] # type: typing.List[str]
to_inspect = set() # type: typing.Set[str]
for p in platform_packages:
logger.info('Package in Platform: %s', p)
to_get.append('{}={}'.format(p.dbgsym_package, p.binary_version))
for want in platform_packages:
logger.info('Package in Platform: %s from %s', want, want.source)
to_inspect.add(
'{}={}'.format(want.qualified_dbgsym_package, want.binary_version))
result = subprocess.run(
in_chroot_quick + ['apt-cache', 'show'] + list(to_inspect),
stdout=subprocess.PIPE,
)
subprocess.run(in_chroot + [
'sh', '-euc', '''
for x in "$@"; do
apt-get -y -q -q -m install "$x" || true
done
''',
'sh', # $0
] + to_get)
binaries = [] # type: typing.List[Packages]
to_get = set() # type: typing.Set[str]
for binary_stanza in Packages.iter_paragraphs(
sequence=result.stdout.splitlines(keepends=True),
encoding='utf-8',
):
if binary_stanza['Package'].endswith('-dbgsym'):
binaries.append(binary_stanza)
parent = os.path.join(
args.sysroot, 'usr', 'lib', 'debug',
......@@ -165,46 +182,70 @@ def main():
os.path.join(parent, 'dbgsym-packages-not-installed.txt'),
'w',
) as missing_writer:
for p in platform_packages:
if subprocess.run(
in_chroot_quick + [
'dpkg-query', '-W', p.dbgsym_package,
],
stdout=writer,
stderr=subprocess.DEVNULL,
).returncode == 0:
for want in platform_packages:
same_name = [] # type: typing.List[Packages]
for binary_stanza in binaries:
if want.dbgsym_package == binary_stanza['Package']:
same_name.append(binary_stanza)
if not same_name:
logger.info(
'Installed -dbgsym package: %s_%s',
p.dbgsym_package,
p.binary_version,
'Skipped nonexistent -dbgsym package: %s',
want.dbgsym_package,
)
else:
result = subprocess.run(
in_chroot_quick + [
'apt-cache', '-q', 'showpkg', p.dbgsym_package,
],
stdout=subprocess.PIPE,
continue
for binary_stanza in same_name:
source = binary_stanza.get(
'Source',
binary_stanza['Package'],
)
if result.returncode == 0 and result.stdout:
logger.warning(
'Unable to install -dbgsym package: %s_%s',
p.dbgsym_package,
p.binary_version,
)
logger.warning(
'Versions available:\n%s',
result.stdout.decode('utf-8', errors='replace'),
)
missing_writer.write('{}\t{}\n'.format(
p.dbgsym_package,
p.binary_version,
))
else:
logger.info(
'Skipped nonexistent -dbgsym package: %s',
p.dbgsym_package,
)
# It's important to check this to avoid including debug
# symbols that would require new source code that is not
# already required by the non-debug packages
if want.source != source:
continue
if want.binary_version != binary_stanza['Version']:
continue
logger.info(
'Installing -dbgsym package: %s_%s',
want.qualified_dbgsym_package,
want.binary_version,
)
writer.write('{}\t{}\n'.format(
want.qualified_dbgsym_package,
want.binary_version,
))
to_get.add('{}={}'.format(
want.qualified_dbgsym_package,
want.binary_version,
))
break
else:
logger.warning(
'Unable to install -dbgsym package: %s_%s from %s',
want.qualified_dbgsym_package,
want.binary_version,
want.source,
)
missing_writer.write('{}\t{}\n'.format(
want.qualified_dbgsym_package,
want.binary_version,
))
for binary_stanza in same_name:
logger.info('Available version:\n%s', binary_stanza)
subprocess.run(
in_chroot + [
'apt-get', '-y', '-q', '-q', '-m', 'install',
] + list(to_get),
check=True,
)
if __name__ == '__main__':
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment