Skip to content

GitLab

Commit 2cb972e8 authored by Simon McVittie's avatar Simon McVittie
Browse files

Add support for collecting source code directly into a directory 


Signed-off-by: Simon McVittie's avatarSimon McVittie <smcv@collabora.com>
parent 5cc6455e
Hide whitespace changes
Inline Side-by-side
Showing with 153 additions and 45 deletions
flatdeb/collect-source-code
View file @ 2cb972e8
......@@ -184,6 +184,7 @@ def main():
parser = argparse.ArgumentParser(
description='Collect source code',
)
parser.add_argument('--output', '-o', default='')
parser.add_argument('--strip-source-version-suffix', default='')
parser.add_argument('sysroot')
......@@ -200,9 +201,15 @@ def main():
'--directory={}'.format(args.sysroot),
'--as-pid2',
'--tmpfs=/run/lock',
'env',
]
if args.output:
in_chroot.append(
'--bind={}:/src/files'.format(os.path.abspath(args.output))
)
in_chroot.append('env')
for var in ('ftp_proxy', 'http_proxy', 'https_proxy', 'no_proxy'):
if var in os.environ:
in_chroot.append('{}={}'.format(var, os.environ[var]))
......@@ -291,7 +298,7 @@ def main():
except subprocess.CalledProcessError:
# Non-fatal for now
logger.warning(
'Unable to get source code for %s', source)
'Unable to get source code for %s', s.source)
missing_sources.add(s)
subprocess.call(in_chroot + [
'apt-cache', 'showsrc', s.source,
......@@ -301,8 +308,23 @@ def main():
else:
included = set(sources)
parent = args.output or os.path.join(args.sysroot, 'src', 'files')
try:
with open(
os.path.join(parent, 'sources.txt'), 'r'
) as reader:
for line in reader:
if line.startswith('#'):
continue
source, source_version = line.rstrip('\n').split('\t')[:2]
included.add(SourceRequired(source, source_version))
except OSError:
pass
with open(
os.path.join(args.sysroot, 'src', 'files', 'sources.txt'), 'w'
os.path.join(parent, 'sources.txt'), 'w'
) as writer:
writer.write('#Source\t#Version\n')
......@@ -312,8 +334,21 @@ def main():
if missing_sources:
logger.warning('Missing source packages:')
try:
with open(
os.path.join(parent, 'MISSING.txt'), 'r'
) as reader:
for line in reader:
if line.startswith('#'):
continue
source, source_version = line.rstrip('\n').split('\t')[:2]
missing_sources.add(SourceRequired(source, source_version))
except OSError:
pass
with open(
os.path.join(args.sysroot, 'src', 'files', 'MISSING.txt'), 'w'
os.path.join(parent, 'MISSING.txt'), 'w'
) as writer:
writer.write('#Source\t#Version\n')
......
flatdeb/debos-runtimes.yaml
View file @ 2cb972e8
......@@ -20,8 +20,9 @@
{{- $sysroot_tarball := or .sysroot_tarball "" -}}
{{- $ostree_prefix := or .ostree_prefix (printf "%s-runtime" $artifact_prefix) -}}
{{- $ostree_tarball := or .ostree_tarball (printf "%s.tar.gz" $ostree_prefix) -}}
{{- $sources_directory := or .sources_directory "" -}}
{{- $sources_prefix := or .sources_prefix (printf "%s-sources" $artifact_prefix) -}}
{{- $sources_tarball := or .sources_tarball (printf "%s.tar.gz" $sources_prefix) -}}
{{- $sources_tarball := or .sources_tarball "" -}}
{{- $debug_prefix := or .debug_prefix (printf "%s-debug" $artifact_prefix) -}}
{{- $debug_tarball := or .debug_tarball (printf "%s.tar.gz" $debug_prefix) -}}
......@@ -144,7 +145,13 @@ actions:
- action: run
label: collect-source-code
chroot: false
command: 'echo; "$RECIPEDIR/collect-source-code" --strip-source-version-suffix="{{ $strip_source_version_suffix }}" "$ROOTDIR"'
command: |
set -e
echo
"$RECIPEDIR/collect-source-code" \
{{ if $sources_directory }}--output="$ARTIFACTDIR/{{ $sources_directory }}"{{ end }} \
--strip-source-version-suffix="{{ $strip_source_version_suffix }}" \
"$ROOTDIR"
{{ else }}
- action: run
label: copy manifest for SDK
......@@ -277,6 +284,7 @@ actions:
files metadata
{{ if $sdk }}
{{ if $sources_tarball }}
- action: run
label: pack source code
chroot: false
......@@ -290,6 +298,7 @@ actions:
--use-compress-program="pigz -n --rsyncable"
-C "$ROOTDIR/src"
.
{{ end }}
- action: run
label: pack debug symbols
......@@ -309,17 +318,22 @@ actions:
label: list of included source code
chroot: false
command: >
cp -v "$ROOTDIR/src/files/sources.txt"
"$ARTIFACTDIR/{{ $sources_prefix }}.sources.txt"
set -e;
cd "$ARTIFACTDIR";
dir="{{ or $sources_directory "$ROOTDIR/src/files" }}";
cp -v "$dir/sources.txt"
"{{ $sources_prefix }}.sources.txt"
- action: run
label: list of missing source code
chroot: false
command: >
set -e;
test ! -e "$ROOTDIR/src/files/MISSING.txt" ||
cp -v "$ROOTDIR/src/files/MISSING.txt"
"$ARTIFACTDIR/{{ $sources_prefix }}.MISSING.txt"
cd "$ARTIFACTDIR";
dir="{{ or $sources_directory "$ROOTDIR/src/files" }}";
test ! -e "$dir/MISSING.txt" ||
cp -v "$dir/MISSING.txt"
"{{ $sources_prefix }}.MISSING.txt"
{{ end }}
- action: run
......@@ -327,11 +341,11 @@ actions:
chroot: false
command: >
set -e;
cd "$ROOTDIR";
cd "$ARTIFACTDIR";
head -n10000
files/manifest.dpkg
files/manifest.dpkg.built-using
src/files/sources.txt
"$ROOTDIR/files/manifest.dpkg"
"$ROOTDIR/files/manifest.dpkg.built-using"
{{ or $sources_directory "$ROOTDIR/src/files" }}/sources.txt
|| true
- action: run
......@@ -339,14 +353,14 @@ actions:
chroot: false
command: >
set -e;
cd "$ROOTDIR";
cd "$ARTIFACTDIR";
head -n10000
files/etc/apt/apt.conf
files/etc/apt/apt.conf.d/*
files/etc/apt/sources.list
files/etc/apt/sources.list.d/*
files/etc/debian_chroot
files/lib/os-release
src/files/MISSING.txt
metadata
"$ROOTDIR/files/etc/apt/apt.conf"
"$ROOTDIR/files/etc/apt/apt.conf.d"/*
"$ROOTDIR/files/etc/apt/sources.list"
"$ROOTDIR/files/etc/apt/sources.list.d"/*
"$ROOTDIR/files/etc/debian_chroot"
"$ROOTDIR/files/lib/os-release"
{{ or $sources_directory "ROOTDIR/src/files" }}/MISSING.txt
"$ROOTDIR/metadata"
|| true
run.py
View file @ 2cb972e8
......@@ -430,6 +430,31 @@ class Builder:
'--add-apt-keyring', action='append', default=[])
parser.add_argument(
'--generate-sysroot-tarball', action='store_true')
parser.add_argument(
'--no-generate-sysroot-tarball',
dest='generate_sysroot_tarball',
action='store_false',
)
parser.add_argument(
'--generate-source-tarball',
action='store_true',
default=None,
)
parser.add_argument(
'--no-generate-source-tarball',
dest='generate_source_tarball',
action='store_false',
)
parser.add_argument(
'--generate-source-directory',
default='',
)
parser.add_argument(
'--no-generate-source-directory',
dest='generate_source_directory',
action='store_const',
const='',
)
parser.add_argument(
'--build-id', default=None)
parser.add_argument(
......@@ -473,6 +498,15 @@ class Builder:
'--replace-apt-source argument must be in the form '
'"LABEL=deb http://ARCHIVE SUITE COMPONENT[...]"')
if (
'/' in args.generate_source_directory
or args.generate_source_directory == '..'
):
parser.error(
'--generate-source-directory must be a single '
'directory name'
)
if args.version:
print('flatdeb {}'.format(VERSION))
return
......@@ -795,6 +829,8 @@ class Builder:
self,
*,
yaml_file, # type: str
generate_source_directory='',
generate_source_tarball=True,
generate_sysroot_tarball=False,
**kwargs
):
......@@ -972,6 +1008,30 @@ class Builder:
'sysroot_tarball:{}'.format(
sysroot_tarball + '.new'))
if generate_source_directory:
os.makedirs(
os.path.join(
self.build_area,
generate_source_directory,
),
0o755,
exist_ok=True,
)
argv.append('-t')
argv.append(
'sources_directory:{}'.format(
generate_source_directory))
if generate_source_tarball is None:
generate_source_tarball = not generate_source_directory
if generate_source_tarball:
sources_tarball = sources_prefix + '.tar.gz'
argv.append('-t')
argv.append(
'sources_tarball:{}'.format(
sources_tarball + '.new'))
sdk_details = self.runtime_details.get('sdk', {})
sdk_packages = list(sdk_details.get('add_packages', []))
argv.append('-t')
......@@ -981,8 +1041,6 @@ class Builder:
argv.append('-t')
argv.append('debug_prefix:' + debug_prefix)
argv.append('-t')
argv.append('sources_tarball:' + sources_tarball + '.new')
argv.append('-t')
argv.append('sources_prefix:' + sources_prefix)
for p in sdk_details.get('add_packages_multiarch', []):
......@@ -1098,26 +1156,27 @@ class Builder:
'--tar-autocreate-parents',
])
output = os.path.join(self.build_area, sources_tarball)
os.rename(output + '.new', output)
if generate_source_tarball:
output = os.path.join(self.build_area, sources_tarball)
os.rename(output + '.new', output)
if self.ostree_commit:
logger.info('Committing %s to OSTree', sources_tarball)
subprocess.check_call([
'time',
'ostree',
'--repo=' + self.ostree_repo,
'commit',
'--branch=runtime/{}.Sources/{}/{}'.format(
runtime,
self.flatpak_arch,
self.runtime_branch,
),
'--subject=Update',
'--tree=tar={}'.format(output),
'--fsync=false',
'--tar-autocreate-parents',
])
if self.ostree_commit:
logger.info('Committing %s to OSTree', sources_tarball)
subprocess.check_call([
'time',
'ostree',
'--repo=' + self.ostree_repo,
'commit',
'--branch=runtime/{}.Sources/{}/{}'.format(
runtime,
self.flatpak_arch,
self.runtime_branch,
),
'--subject=Update',
'--tree=tar={}'.format(output),
'--fsync=false',
'--tar-autocreate-parents',
])
output = os.path.join(self.build_area, out_tarball)
os.rename(output + '.new', output)
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment