diff --git a/Dockerfile.backend b/Dockerfile.backend index 1851f375a6ff5a90afdf1ea2bb3b8f90aa52e885..1ace47bb24f71b31edff09b762114a38eed407f8 100644 --- a/Dockerfile.backend +++ b/Dockerfile.backend @@ -8,6 +8,7 @@ ARG WORKDIR=/tmp/sources RUN apt-get update \ && apt-get install -y \ apt-utils \ + aptly \ adduser \ ca-certificates \ curl \ @@ -58,6 +59,7 @@ COPY docker/ /opt/ VOLUME /etc/obs +COPY docker/services/aptly/aptly.conf /etc/aptly.conf RUN /opt/configure-backend-user.sh VOLUME /srv/obs diff --git a/docker-compose.yml b/docker-compose.yml index 38fd4c1eca7af0f44cdd3eaddeb097ea1988a85a..32ebaf3c356d950b176eea10a3cadb2dd6e9bf5e 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -19,11 +19,14 @@ services: hostname: obs-server restart: unless-stopped volumes: + - aptly-storage:/srv/aptly - backend-storage:/srv/obs - backend-logs:/srv/obs/log environment: OBS_FRONTEND_HOST: obs-frontend OBS_BACKEND_HOST: obs-server + ports: + - "127.0.0.1:8080:8080" obs-frontend: image: obs-frontend @@ -67,6 +70,7 @@ services: OBS_WORKER_INSTANCES: 1 volumes: + aptly-storage: backend-storage: backend-logs: frontend-logs: diff --git a/docker/configure-backend-user.sh b/docker/configure-backend-user.sh index 5b434692aaafe11e1522cdeae1d3813df10c1e70..b30c96535d95bf2435285f4149157d09d0022434 100755 --- a/docker/configure-backend-user.sh +++ b/docker/configure-backend-user.sh @@ -20,3 +20,6 @@ fi mkdir -p /srv/obs/repos chown obsrun:obsrun /srv/obs/repos + +mkdir -p /srv/aptly +chown obsrun:obsrun /srv/aptly diff --git a/docker/services/aptly/aptly.conf b/docker/services/aptly/aptly.conf new file mode 100644 index 0000000000000000000000000000000000000000..411e85ab1028bc7c2d20b9dce95363f26d87835d --- /dev/null +++ b/docker/services/aptly/aptly.conf @@ -0,0 +1,22 @@ +{ + "rootDir": "/srv/aptly", + "downloadConcurrency": 4, + "downloadSpeedLimit": 0, + "architectures": ["amd64", "arm64", "armhf", "source"], + "dependencyFollowSuggests": false, + "dependencyFollowRecommends": false, + "dependencyFollowAllVariants": false, + "dependencyFollowSource": false, + "dependencyVerboseResolve": false, + "gpgDisableSign": true, + "gpgDisableVerify": true, + "gpgProvider": "gpg", + "downloadSourcePackages": false, + "skipLegacyPool": true, + "ppaDistributorID": "ubuntu", + "ppaCodename": "", + "skipContentsPublishing": false, + "FileSystemPublishEndpoints": {}, + "S3PublishEndpoints": {}, + "SwiftPublishEndpoints": {} +} diff --git a/src/backend/BSAptly.pm b/src/backend/BSAptly.pm new file mode 100755 index 0000000000000000000000000000000000000000..f71e63e8f206411b2a9a11a1e0527f0a4d845635 --- /dev/null +++ b/src/backend/BSAptly.pm @@ -0,0 +1,22 @@ +#!/usr/bin/perl -w +# +# SPDX-License-Identifier: GPL-2.0+ +# +# Copyright 2022 Collabora Ltd. + +package BSAptly; + +use BSPublisher::Util; + +use strict; + +sub aptly_qsystem { + # By default, aptly will try to read configuration from ~/.aptly.conf + # OBS runs as `obsrun` user, but HOME is pointing to /root/, so this will raise an error. + # Let's set the default configuration to the system one. + my @args = ('aptly', '-config=/etc/aptly.conf', @_); + print("DEBUG >>> aptly_qsystem: ", join(' ', @args), "\n"); + return BSPublisher::Util::qsystem(@args); +} + +1; diff --git a/src/backend/bs_publish b/src/backend/bs_publish index 4776ef5e9a290196ce589a5931b0259369305335..b064c18cfd35d8a7340e9c78800ee1570bc5f678 100755 --- a/src/backend/bs_publish +++ b/src/backend/bs_publish @@ -59,6 +59,8 @@ use BSPublisher::Container; use BSPublisher::Util; use BSPublisher::Blobstore; +use BSAptly; + use strict; my $maxchild; @@ -866,73 +868,40 @@ sub deleterepo_susetags { qsystem('rm', '-rf', "$extrep/descr") if -d "$extrep/descr"; } -sub updaterepo_reprepro { - my ($prp, $extrep, @changed) = @_; - - my $repo = "$extrepodir/$BSConfig::reprepository->{$prp}{'repository'}"; - my $codename = $BSConfig::reprepository->{$prp}{'codename'}; - my $component = $BSConfig::reprepository->{$prp}{'component'}; - - for my $f (@changed) { - if ($f =~ /\.changes/) { - print " Updated changes file => $f\n" ; - my %types; - if (open (my $fh, '<', "$extrep/$f")) { - # Read the .changes file looking for binary packages - # (.deb, .udeb, .ddeb) - my $in_files = 0; - while (my $line = <$fh>) { - print " $line"; - - if ($in_files) { - if ($line =~ /^\s/) { - if ($line =~ /\.((?:|u|d)deb)$/) { - $types{$1} = 1; - } - } else { - $in_files = 0; - } - } elsif ($line =~ /^Files\s*:/) { - $in_files = 1; - } - } - } else { - print " unable to open $extrep/$f: $!\n"; - } +sub updaterepo_aptly { + my ($projid, $extrep, $added_ref, $removed_ref) = @_; + my @added = @{ $added_ref }; + my @removed = @{ $removed_ref }; + my ($pkg, $version, $arch, $pkgtype); + my @args; - if (!%types) { - # no binary packages at all? assume we must have mis-parsed it, - # and run reprepro for the .deb files so we get a better - # error report - print " warning: no .deb/.udeb/.ddeb found in .changes, assuming .deb only\n"; - %types = (deb => 1); - } + # Remove previous versions + for my $f (@removed) { + if ($f =~ /^.*\/(.*)_(.*)_(.*)\.([u]?deb)$/) { + ($pkg, $version, $arch, $pkgtype) = ($1, $2, $3, $4); + } elsif ($f =~ /^(.*)_(.*)\.dsc$/) { + ($pkg, $version, $arch, $pkgtype) = ($1, $2, "source", "source"); + } else { + next; + } + @args = ('repo', 'remove', $projid, + `echo 'Name (="$pkg"), Version (<="$version"), \$Architecture (="$arch"), \$PackageType (="$pkgtype")'`); + BSAptly::aptly_qsystem(@args); + } - foreach my $type (keys %types) { - - my @args = ('reprepro', '-b', $repo, - '--ignore=wrongdistribution', - '--ignore=conflictingarchall', - '--ignore=unusedarch', - '--ignore=surprisingbinary', - '-T', $type, - '-C', $component, - 'include', $codename, - "$extrep/$f"); - print(" importing .$type binaries: ", join(' ', @args), "\n"); - qsystem(@args); - } - } elsif ($f =~ /\.dsc/) { - print " Updated dsc file => $f\n" ; - my @args = ('reprepro', '-b', $repo, - '-C', $component, - '-P', 'standard', - '-S', 'main', - 'includedsc', $codename, "$extrep/$f"); - print(" importing sources: ", join(' ', @args), "\n"); - qsystem(@args); - } + # Add new packages + for my $f (@added) { + if (($f =~ /^.*\/(.*)_(.*)_(.*)\.([u]?deb)$/) or ($f =~ /^(.*)_(.*)\.dsc$/)) { + my $file = "$extrep/$f"; + @args = ('repo', 'add', $projid, $file); + BSAptly::aptly_qsystem(@args); + } } + + # Update published repo + my ($projname, $distribution, $component) = split(':', $projid, 3); + @args = ('publish', 'update', $distribution); + BSAptly::aptly_qsystem(@args); } sub compress_and_rename { @@ -2729,11 +2698,8 @@ sub publish { deleterepo_hdlist2($extrep, $projid, $xrepoid, $data); } if ($repotype{'debian'}) { - if ($BSConfig::reprepository && $BSConfig::reprepository->{$prp}) { - updaterepo_reprepro($prp, $extrep, @changed) - } else { - createrepo_debian($extrep, $projid, $xrepoid, $data, $repotype{'debian'}); - } + my @removed = (@db_deleted, @deleted); + updaterepo_aptly($projid, $extrep, \@changed, \@removed); } else { deleterepo_debian($extrep, $projid, $xrepoid, $data); } diff --git a/src/backend/bs_srcserver b/src/backend/bs_srcserver index 0274de67c889fadc508fc4d3dd55fa0e8a5a1057..889ead320a96b8a06c7c3faa6c10f9346608a11d 100755 --- a/src/backend/bs_srcserver +++ b/src/backend/bs_srcserver @@ -80,6 +80,8 @@ use BSSrcServer::Registry; use BSSrcServer::Signkey; use BSSrcServer::Notify; +use BSAptly; + # configure modules $BSSrcServer::Projlink::getrev = \&getrev; $BSSrcServer::Projlink::findpackages = \&findpackages; @@ -1871,6 +1873,34 @@ sub getpubkey { ######################################################################### +sub publishproject { + my ($projname, $distribution) = @_; + my @args; + + # Drop previous published repo for this distribution. + @args = ('publish', 'drop', $distribution); + eval { BSAptly::aptly_qsystem(@args) }; + + # Publish all components for this distribution. + my $components = []; + my $repos = []; + for my $i (findprojects()) { + my $p = BSRevision::readproj_local($i); + my $name = $p->{'name'}; + if ($name =~ /^$projname:$distribution:/) { + push @$components, (split(':', $name, 3))[2]; + push @$repos, $name; + } + } + if (!@$repos) { + return; + } + $components = join(',', @$components); + @args = ('publish', 'repo', + "-distribution=$distribution", "-component=$components", @$repos); + BSAptly::aptly_qsystem(@args); +} + sub putproject { my ($cgi, $projid) = @_; mkdir_p($uploaddir); @@ -1906,6 +1936,15 @@ sub putproject { } $proj = BSRevision::readproj_local($projid); + + # Assume project name convention: projname:distribution:component + # E.g. apertis:v2022:target + my ($projname, $distribution, $component) = split(':', $projid, 3); + my @args = ('repo', 'create', + "-distribution=$distribution", "-component=$component", $projid); + BSAptly::aptly_qsystem(@args); + publishproject($projname, $distribution); + return ($proj, $BSXML::proj); } @@ -1951,6 +1990,14 @@ sub delproject { } }; warn($@) if $@; + + # Assume project name convention: projname:distribution:component + # E.g. apertis:v2022:target + my ($projname, $distribution, $component) = split(':', $projid, 3); + publishproject($projname, $distribution); + my @args = ('repo', 'drop', $projid); + BSAptly::aptly_qsystem(@args); + return undef; }