From ce11d76a8e2d7b16a2ae4736574150f4fcb4307e Mon Sep 17 00:00:00 2001
From: Andrej Shadura <andrew.shadura@collabora.co.uk>
Date: Wed, 2 Jun 2021 15:22:55 +0200
Subject: [PATCH] Build Docker images
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
Was:
Create a Docker image with a Debian package built from the current Git
source. This eliminates an extra round trip with a manual upload to OBS
and the package getting published and fetched from apt repos.
Unfortunately, doing this in a way compatible with what was previously
done requires some non-trivial hacks.
Since we want fairly recent OmniAuth gems, we install them from external
sources directly into the resulting Docker image.
ruby-faraday is used by the OAuth2 auth backend, but new versions
require newer Ruby than what stretch has, so we preinstall it from
packages to avoid pinning it.
Signed-off-by: Andrej Shadura <andrew.shadura@collabora.co.uk>
Gbp-Pq: Topic collabora/sso
Gbp-Pq: Name Build-a-Docker-image-with-the-Web-UI-only.patch
Build Docker images in GitLab CI
Signed-off-by: Andrej Shadura <andrew.shadura@collabora.co.uk>
Gbp-Pq: Topic collabora/sso
Gbp-Pq: Name Build-Docker-images-in-GitLab-CI.patch
Split docker-entrypoint.sh into three separate files
Gbp-Pq: Topic collabora/sso
Gbp-Pq: Name Split-docker-entrypoint.sh-into-three-separate-files.patch
Move Docker-related files under docker/
Signed-off-by: Andrej Shadura <andrew.shadura@collabora.co.uk>
Gbp-Pq: Topic collabora/sso
Gbp-Pq: Name Move-Docker-related-files-under-docker.patch
Clean up stale pid files on start
Signed-off-by: Andrej Shadura <andrew.shadura@collabora.co.uk>
Gbp-Pq: Topic collabora
Gbp-Pq: Name Clean-up-stale-pid-files-on-start.patch
Install and configure mstmp
Signed-off-by: Andrej Shadura <andrew.shadura@collabora.co.uk>
Gbp-Pq: Topic collabora
Gbp-Pq: Name Install-and-configure-mstmp.patch
Allow changing the session lifetime
Signed-off-by: Andrej Shadura <andrew.shadura@collabora.co.uk>
Gbp-Pq: Topic collabora
Gbp-Pq: Name Allow-changing-the-session-lifetime.patch
Preinstall apt-transport-https ca-certificates build-essential
Signed-off-by: Andrej Shadura <andrew.shadura@collabora.co.uk>
Simplify the db config generation
Signed-off-by: Andrej Shadura <andrew.shadura@collabora.co.uk>
Remove MariaDB/MySQL from the API container
There’s no need to keep the database inside when it can be a separate
container.
Signed-off-by: Andrej Shadura <andrew.shadura@collabora.co.uk>
Switch to an external memcached
Signed-off-by: Andrej Shadura <andrew.shadura@collabora.co.uk>
Drop Apache and TLS termination, use Puma instead of Passenger
When the container is deployed, it will have a real HTTP server in front
of it, so there is no need to have Apache and TLS inside.
Since Passenger (at least of the version in Debian stretch) cannot be
easily used without Apache, use the standard solution for such cases
which is Puma, and expose OBS_FRONTEND_WORKERS (default: 4) to allow
scaling it.
Drop no longer necessary supervisord.
Signed-off-by: Andrej Shadura <andrew.shadura@collabora.co.uk>
Rename the frontend image to a more sensible name
A front-end is what it really is, and it’s also the name the upstream
uses for their container.
Signed-off-by: Andrej Shadura <andrew.shadura@collabora.co.uk>
Generalise the build process to enable building multiple images
Signed-off-by: Andrej Shadura <andrew.shadura@collabora.co.uk>
Teach deb-arch-to-obs-arch to work with an arbitrary argument
Signed-off-by: Andrej Shadura <andrew.shadura@collabora.co.uk>
Build the backend Docker image
This Docker image installs obs-build and obs-productconverter from
packages and everything else directly from the sources.
Signed-off-by: Andrej Shadura <andrew.shadura@collabora.co.uk>
WIP: update Docker stuff
Signed-off-by: Andrej Shadura <andrew.shadura@collabora.co.uk>
Pass TAG argument
Signed-off-by: Andrej Shadura <andrew.shadura@collabora.co.uk>
Don't purge or install unnecessary packages
Don't update packages
Install puma
Wip wup
Support buster
tags
more deps
unify log names with upstream
add cloud uploader
ensure correct permissions for log/tmp/sphinx
init app before db so that permissions are all correct
db setup needs command line mysql client
init or migrate database
remove more gem cruft
pre-create run dir with correct perms
provide our custom docker-compose
move services confs into a subdir because the worker also needs some
fix frontend
fix scripts
build worker image
add worker again
Build Docker images
Was:
Create a Docker image with a Debian package built from the current Git
source. This eliminates an extra round trip with a manual upload to OBS
and the package getting published and fetched from apt repos.
Unfortunately, doing this in a way compatible with what was previously
done requires some non-trivial hacks.
Since we want fairly recent OmniAuth gems, we install them from external
sources directly into the resulting Docker image.
ruby-faraday is used by the OAuth2 auth backend, but new versions
require newer Ruby than what stretch has, so we preinstall it from
packages to avoid pinning it.
Signed-off-by: Andrej Shadura <andrew.shadura@collabora.co.uk>
Gbp-Pq: Topic collabora/sso
Gbp-Pq: Name Build-a-Docker-image-with-the-Web-UI-only.patch
Build Docker images in GitLab CI
Signed-off-by: Andrej Shadura <andrew.shadura@collabora.co.uk>
Gbp-Pq: Topic collabora/sso
Gbp-Pq: Name Build-Docker-images-in-GitLab-CI.patch
Split docker-entrypoint.sh into three separate files
Gbp-Pq: Topic collabora/sso
Gbp-Pq: Name Split-docker-entrypoint.sh-into-three-separate-files.patch
Move Docker-related files under docker/
Signed-off-by: Andrej Shadura <andrew.shadura@collabora.co.uk>
Gbp-Pq: Topic collabora/sso
Gbp-Pq: Name Move-Docker-related-files-under-docker.patch
Clean up stale pid files on start
Signed-off-by: Andrej Shadura <andrew.shadura@collabora.co.uk>
Gbp-Pq: Topic collabora
Gbp-Pq: Name Clean-up-stale-pid-files-on-start.patch
Install and configure mstmp
Signed-off-by: Andrej Shadura <andrew.shadura@collabora.co.uk>
Gbp-Pq: Topic collabora
Gbp-Pq: Name Install-and-configure-mstmp.patch
Allow changing the session lifetime
Signed-off-by: Andrej Shadura <andrew.shadura@collabora.co.uk>
Gbp-Pq: Topic collabora
Gbp-Pq: Name Allow-changing-the-session-lifetime.patch
Preinstall apt-transport-https ca-certificates build-essential
Signed-off-by: Andrej Shadura <andrew.shadura@collabora.co.uk>
Simplify the db config generation
Signed-off-by: Andrej Shadura <andrew.shadura@collabora.co.uk>
Remove MariaDB/MySQL from the API container
There’s no need to keep the database inside when it can be a separate
container.
Signed-off-by: Andrej Shadura <andrew.shadura@collabora.co.uk>
Switch to an external memcached
Signed-off-by: Andrej Shadura <andrew.shadura@collabora.co.uk>
Drop Apache and TLS termination, use Puma instead of Passenger
When the container is deployed, it will have a real HTTP server in front
of it, so there is no need to have Apache and TLS inside.
Since Passenger (at least of the version in Debian stretch) cannot be
easily used without Apache, use the standard solution for such cases
which is Puma, and expose OBS_FRONTEND_WORKERS (default: 4) to allow
scaling it.
Drop no longer necessary supervisord.
Signed-off-by: Andrej Shadura <andrew.shadura@collabora.co.uk>
Rename the frontend image to a more sensible name
A front-end is what it really is, and it’s also the name the upstream
uses for their container.
Signed-off-by: Andrej Shadura <andrew.shadura@collabora.co.uk>
Generalise the build process to enable building multiple images
Signed-off-by: Andrej Shadura <andrew.shadura@collabora.co.uk>
Teach deb-arch-to-obs-arch to work with an arbitrary argument
Signed-off-by: Andrej Shadura <andrew.shadura@collabora.co.uk>
Build the backend Docker image
This Docker image installs obs-build and obs-productconverter from
packages and everything else directly from the sources.
Signed-off-by: Andrej Shadura <andrew.shadura@collabora.co.uk>
WIP: update Docker stuff
Signed-off-by: Andrej Shadura <andrew.shadura@collabora.co.uk>
Pass TAG argument
Signed-off-by: Andrej Shadura <andrew.shadura@collabora.co.uk>
Don't purge or install unnecessary packages
Don't update packages
Install puma
Wip wup
Support buster
tags
more deps
unify log names with upstream
add cloud uploader
ensure correct permissions for log/tmp/sphinx
init app before db so that permissions are all correct
db setup needs command line mysql client
init or migrate database
remove more gem cruft
pre-create run dir with correct perms
provide our custom docker-compose
move services confs into a subdir because the worker also needs some
fix frontend
fix scripts
build worker image
add worker again
depend on xml-structured
sphinxsearch is not in bullseye
adjust path
---
.gitlab-ci.yml | 70 ++++++++++++
Dockerfile.backend | 16 +++
Dockerfile.backend-base | 53 ++++++++++
Dockerfile.frontend | 49 +++++++++
Dockerfile.frontend-base | 74 +++++++++++++
Dockerfile.worker | 28 +++++
docker-compose.yml | 100 +++++++++++-------
docker/backend-docker-entrypoint.sh | 48 +++++++++
docker/configure-app.sh | 45 ++++++++
docker/configure-backend-user.sh | 22 ++++
docker/configure-db.sh | 34 ++++++
docker/configure-frontend-user.sh | 12 +++
docker/configure-sso.py | 38 +++++++
docker/deb-arch-to-obs-arch | 24 +++++
docker/frontend-docker-entrypoint.sh | 16 +++
.../backend/obsclouduploadserver.conf | 11 ++
.../backend/obsclouduploadworker.conf | 11 ++
docker/services/backend/obsdispatcher.conf | 11 ++
docker/services/backend/obsdodup.conf | 11 ++
docker/services/backend/obspublisher.conf | 11 ++
docker/services/backend/obsrepserver.conf | 11 ++
docker/services/backend/obsscheduler@.conf.in | 11 ++
docker/services/backend/obsservice.conf | 11 ++
.../services/backend/obsservicedispatch.conf | 11 ++
docker/services/backend/obssrcserver.conf | 11 ++
docker/services/backend/obswarden.conf | 11 ++
docker/services/worker/obsworker.conf | 13 +++
docker/worker-docker-entrypoint.sh | 72 +++++++++++++
src/api/config/environments/production.rb | 2 +-
src/api/config/options.yml.example | 2 +-
.../20191011000000_create_allowbuilddeps.rb | 2 +-
31 files changed, 802 insertions(+), 39 deletions(-)
create mode 100644 .gitlab-ci.yml
create mode 100644 Dockerfile.backend
create mode 100644 Dockerfile.backend-base
create mode 100644 Dockerfile.frontend
create mode 100644 Dockerfile.frontend-base
create mode 100644 Dockerfile.worker
create mode 100755 docker/backend-docker-entrypoint.sh
create mode 100755 docker/configure-app.sh
create mode 100755 docker/configure-backend-user.sh
create mode 100755 docker/configure-db.sh
create mode 100755 docker/configure-frontend-user.sh
create mode 100755 docker/configure-sso.py
create mode 100755 docker/deb-arch-to-obs-arch
create mode 100755 docker/frontend-docker-entrypoint.sh
create mode 100644 docker/services/backend/obsclouduploadserver.conf
create mode 100644 docker/services/backend/obsclouduploadworker.conf
create mode 100644 docker/services/backend/obsdispatcher.conf
create mode 100644 docker/services/backend/obsdodup.conf
create mode 100644 docker/services/backend/obspublisher.conf
create mode 100644 docker/services/backend/obsrepserver.conf
create mode 100644 docker/services/backend/obsscheduler@.conf.in
create mode 100644 docker/services/backend/obsservice.conf
create mode 100644 docker/services/backend/obsservicedispatch.conf
create mode 100644 docker/services/backend/obssrcserver.conf
create mode 100644 docker/services/backend/obswarden.conf
create mode 100644 docker/services/worker/obsworker.conf
create mode 100755 docker/worker-docker-entrypoint.sh
diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
new file mode 100644
index 0000000000..8cb609db38
--- /dev/null
+++ b/.gitlab-ci.yml
@@ -0,0 +1,70 @@
+stages:
+ - docker
+
+.build-docker-image:
+ stage: docker
+ image:
+ name: gcr.io/kaniko-project/executor:debug
+ entrypoint: [""]
+ script:
+ - |
+ cat << EOF > /kaniko/.docker/config.json
+ {
+ "auths":{
+ "$CI_REGISTRY": {
+ "username":"$CI_REGISTRY_USER",
+ "password":"$CI_REGISTRY_PASSWORD"
+ }
+ }
+ }
+ EOF
+ - >
+ /kaniko/executor
+ --context $CI_PROJECT_DIR
+ --dockerfile $CI_PROJECT_DIR/Dockerfile.$image
+ --destination $CI_REGISTRY_IMAGE/obs-$image:$CI_COMMIT_REF_SLUG
+ --build-arg REGISTRY=$CI_REGISTRY_IMAGE
+ --build-arg TAG=$CI_COMMIT_REF_SLUG
+ --single-snapshot
+ - echo Pushed $CI_REGISTRY_IMAGE/obs-$image:$CI_COMMIT_REF_SLUG
+
+frontend-base:
+ extends: .build-docker-image
+ only:
+ changes:
+ - Dockerfile.frontend-base
+ - src/api/Gemfile
+ - src/api/Gemfile.lock
+ variables:
+ image: frontend-base
+
+frontend:
+ extends: .build-docker-image
+ needs:
+ - frontend-base
+ variables:
+ image: frontend
+
+backend-base:
+ extends: .build-docker-image
+ only:
+ changes:
+ - Dockerfile.backend-base
+ - dist/**
+ - src/backend/**
+ variables:
+ image: backend-base
+
+backend:
+ extends: .build-docker-image
+ needs:
+ - backend-base
+ variables:
+ image: backend
+
+worker:
+ extends: .build-docker-image
+ needs:
+ - backend-base
+ variables:
+ image: worker
diff --git a/Dockerfile.backend b/Dockerfile.backend
new file mode 100644
index 0000000000..41bbf27328
--- /dev/null
+++ b/Dockerfile.backend
@@ -0,0 +1,16 @@
+ARG REGISTRY
+ARG TAG=latest
+FROM $REGISTRY/obs-backend-base:$TAG
+
+LABEL maintainer Andrej Shadura <andrew.shadura@collabora.co.uk>
+
+COPY docker/services/backend/*.conf /etc/supervisor/conf.d/
+COPY docker/ /opt/
+
+VOLUME /etc/obs
+
+RUN /opt/configure-backend-user.sh
+
+VOLUME /srv/obs
+
+ENTRYPOINT /opt/backend-docker-entrypoint.sh
diff --git a/Dockerfile.backend-base b/Dockerfile.backend-base
new file mode 100644
index 0000000000..f0ed5c44ec
--- /dev/null
+++ b/Dockerfile.backend-base
@@ -0,0 +1,53 @@
+FROM debian:bullseye-slim as server
+LABEL maintainer Andrej Shadura <andrew.shadura@collabora.co.uk>
+ENV LC_ALL=C.UTF-8
+ARG DEBIAN_FRONTEND=noninteractive
+ARG WORKDIR=/tmp/sources
+
+# Needs checking what’s actually needed
+RUN apt-get update \
+ && apt-get install -y \
+ apt-utils \
+ adduser \
+ ca-certificates \
+ curl \
+ diffutils \
+ dpkg-dev \
+ git \
+ locales \
+ libbssolv-perl \
+ libcompress-raw-zlib-perl \
+ libfile-sync-perl \
+ libio-compress-perl \
+ libjson-xs-perl \
+ libnet-ssleay-perl \
+ libsocket-msghdr-perl \
+ libtimedate-perl \
+ libxml-parser-perl \
+ libxml-simple-perl \
+ libxml-structured-perl \
+ libyaml-libyaml-perl \
+ make \
+ obs-build \
+ patch \
+ procps \
+ reprepro \
+ supervisor \
+ time \
+ tzdata \
+ zstd
+
+COPY . $WORKDIR
+
+RUN make -C $WORKDIR/dist install
+RUN make -C $WORKDIR/src/backend install
+
+RUN rm -rf $WORKDIR
+
+RUN mkdir -p /etc/obs
+RUN cp /usr/lib/obs/server/BSConfig.pm.template /etc/obs/BSConfig.pm
+
+RUN ln -sf /etc/obs/BSConfig.pm /usr/lib/obs/server/BSConfig.pm
+RUN ln -sf /usr/lib/obs-build /usr/lib/obs/server/build
+
+ENTRYPOINT /opt/backend-docker-entrypoint.sh
diff --git a/Dockerfile.frontend b/Dockerfile.frontend
new file mode 100644
index 0000000000..47248b2cd7
--- /dev/null
+++ b/Dockerfile.frontend
@@ -0,0 +1,49 @@
+ARG REGISTRY
+ARG TAG=latest
+FROM $REGISTRY/obs-frontend-base:$TAG as base
+
+ARG REGISTRY
+ARG TAG=latest
+FROM $REGISTRY/obs-frontend-base:$TAG
+ARG WORKDIR=/tmp/sources
+ARG INSTALLDIR=/obs
+
+ADD src/api/ $INSTALLDIR/src/api/
+
+COPY --from=base $INSTALLDIR/src/api/Gemfile* $INSTALLDIR/src/api/
+
+WORKDIR $INSTALLDIR/src/api
+
+RUN ls -la
+
+ARG BUNDLE_BUILD__SASSC=--disable-march-tune-native
+ARG NOKOGIRI_USE_SYSTEM_LIBRARIES=1
+
+RUN bundle install --jobs=$(nproc) --retry=3
+
+# Install extra gems for SSO
+RUN gem install omniauth omniauth-gitlab --no-doc
+RUN gem install omniauth-azure-oauth2 omniauth-azure-oauth2-v2 omniauth-phabricator --no-doc
+
+RUN rm -rf /var/lib/gems/*/cache
+
+ENV RAILS_ENV=production
+ENV RAILS_LOG_TO_STDOUT=true
+
+RUN echo nonce > config/secret.key \
+ && DATABASE_URL=mysql2://localhost/noncedb bundle exec rake assets:precompile RAILS_GROUPS=assets \
+ && rm config/secret.key
+
+RUN bundle config --local without test:assets:development
+
+RUN sed -i 's|^#!/usr/bin/ruby.ruby.*$|#!/usr/bin/ruby|' bin/* script/*
+RUN sed -i -e /mailcatcher:/d -e /web:/d Procfile
+
+COPY docker/ /opt/
+
+RUN /opt/configure-frontend-user.sh
+RUN mkdir -p log tmp db/sphinx \
+ && chown -R frontend /obs
+
+ENTRYPOINT /opt/frontend-docker-entrypoint.sh
+EXPOSE 3000
diff --git a/Dockerfile.frontend-base b/Dockerfile.frontend-base
new file mode 100644
index 0000000000..8c5ee5b323
--- /dev/null
+++ b/Dockerfile.frontend-base
@@ -0,0 +1,74 @@
+# FROM debian:buster-slim as frontend-base
+FROM debian:bookworm-slim as frontend-base
+ENV LC_ALL=C.UTF-8
+ARG DEBIAN_FRONTEND=noninteractive
+ARG INSTALLDIR=/obs
+
+RUN apt-get update \
+ && apt-get install -y \
+ apt-utils \
+ adduser \
+ ca-certificates \
+ curl \
+ diffutils \
+ dpkg-dev \
+ git \
+ locales \
+ make \
+ msmtp-mta \
+ mariadb-client \
+ npm \
+ pkgconf \
+ ruby \
+ ruby-dev \
+ ruby-bundler \
+ ruby-ffi \
+ ruby-foreman \
+ sphinxsearch \
+ patch \
+ supervisor \
+ time \
+ tzdata
+
+RUN apt-get update \
+ && apt-get install -y \
+ default-libmysqlclient-dev \
+ libldap2-dev \
+ libsasl2-dev \
+ libxml2-dev \
+ libxslt1-dev \
+ zlib1g-dev
+
+# Work around a bug in buster
+RUN [ $(readlink /usr/bin/ruby) != ruby2.5 ] \
+ || { ver=$(basename -s.gemspec $(echo /usr/share/rubygems-integration/all/specifications/rake-*.gemspec)); \
+ mkdir -p /usr/share/rubygems-integration/all/gems/$ver/exe; \
+ ln -s /usr/bin/rake /usr/share/rubygems-integration/all/gems/$ver/exe/rake; }
+
+RUN gem install --no-format-executable brakeman --version 5.0.2 --no-doc
+RUN gem install sassc --version 2.0.1 --no-doc
+
+ADD src/api/Gemfile* $INSTALLDIR/src/api/
+WORKDIR $INSTALLDIR/src/api/
+
+RUN sed -e "/gem 'puma'/d" Gemfile > Gemfile.new; \
+ echo "gem 'puma'" >> Gemfile.new; \
+ diff -u Gemfile Gemfile.new; \
+ mv Gemfile.new Gemfile
+
+RUN sed -e '/BUNDLED WITH/,+1 d' Gemfile.lock > Gemfile.lock.new; \
+ diff -u Gemfile.lock Gemfile.lock.new; \
+ mv Gemfile.lock.new Gemfile.lock
+
+ARG BUNDLE_BUILD__SASSC=--disable-march-tune-native
+ARG NOKOGIRI_USE_SYSTEM_LIBRARIES=1
+
+RUN bundle config --global without development:test
+
+RUN bundle install --jobs=$(nproc) --retry=3
+
+RUN rm -rf \
+ /var/lib/gems/*/cache/ \
+ /var/lib/gems/*/test/ \
+ /var/lib/gems/*/extensions/*/*/*/gem_make.out \
+ /var/lib/gems/*/extensions/*/*/*/*.log \
diff --git a/Dockerfile.worker b/Dockerfile.worker
new file mode 100644
index 0000000000..0bb994b36a
--- /dev/null
+++ b/Dockerfile.worker
@@ -0,0 +1,28 @@
+ARG REGISTRY
+ARG TAG=latest
+FROM $REGISTRY/obs-backend-base:$TAG
+
+LABEL maintainer Andrej Shadura <andrew.shadura@collabora.co.uk>
+ARG DEBIAN_FRONTEND=noninteractive
+
+# TODO: cleanup
+RUN apt-get update \
+ && apt-get install -y \
+ binutils \
+ cpio \
+ curl \
+ debootstrap \
+ fdisk \
+ libarchive-tools \
+ lsb-base \
+ lvm2 \
+ lzma \
+ psmisc \
+ rpm
+
+COPY docker/services/worker/*.conf /etc/supervisor/conf.d/
+COPY docker/ /opt/
+
+# RUN /opt/configure-worker-user.sh
+
+ENTRYPOINT /opt/worker-docker-entrypoint.sh
diff --git a/docker-compose.yml b/docker-compose.yml
index 90aa950244..69ebc276a2 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -1,44 +1,72 @@
-version: "2.1"
+---
+version: '2'
+
services:
- db:
- image: registry.opensuse.org/obs/server/unstable/container/leap151/containers/openbuildservice/mariadb
- ports:
- - "3306:3306"
- command: /usr/lib/mysql/mysql-systemd-helper start
+ obs-db:
+ image: mariadb:10.6
+ restart: unless-stopped
+ environment:
+ MARIADB_ROOT_PASSWORD: someobs
+ MARIADB_DATABASE: obsapi
+ MARIADB_USER: obs-api
+ MARIADB_PASSWORD: someobs
+
cache:
- image: registry.opensuse.org/obs/server/unstable/container/leap151/containers/openbuildservice/memcached
- ports:
- - "11211:11211"
- command: /usr/sbin/memcached -u memcached
- backend:
- image: registry.opensuse.org/obs/server/unstable/container/leap151/containers/openbuildservice/backend
- volumes:
- - .:/obs
- - ./dist/aws_credentials:/etc/obs/cloudupload/.aws/config
- - ./dist/ec2utils.conf:/etc/obs/cloudupload/.ec2utils.conf
- - ./dist/clouduploader.rb:/usr/bin/clouduploader
- command: /obs/contrib/start_development_backend -d /obs
- worker:
- image: registry.opensuse.org/obs/server/unstable/container/leap151/containers/openbuildservice/backend
+ image: memcached:1.6-alpine
+
+ obs-backend:
+ image: obs/obs-backend
+ hostname: obs-server
+ restart: unless-stopped
volumes:
- - .:/obs
- privileged: true
+ - backend-storage:/srv/obs
+ - backend-logs:/srv/obs/log
+ environment:
+ OBS_FRONTEND_HOST: obs-api
+
+ obs-frontend:
+ image: obs/obs-frontend
depends_on:
- - backend
- command: /obs/contrib/start_development_worker
- frontend:
- image: openbuildservice/frontend
- command: foreman start -p 3000
- build:
- dockerfile: docker-files/Dockerfile
- context: src/api
+ - obs-server
volumes:
- - .:/obs
+ - frontend-logs:/obs/src/api/log
+ - type: tmpfs
+ target: /tmp
+ tmpfs:
+ size: 4G
+ hostname: obs-api
+ restart: unless-stopped
+ environment:
+ DB_HOST: obs-db
+ DB_PORT: 3306
+ DB_ROOT_PASSWORD: someobs
+ DB_NAME: obsapi
+ DB_USER: obs-api
+ DB_PASSWORD: someobs
+ OBS_BACKEND_HOST: obs-server
+ OBS_FRONTEND_WORKERS: 4
ports:
- - "3000:3000"
- - "1080:1080"
+ - "127.0.0.1:3000:3000"
depends_on:
- - db
+ - obs-db
- cache
- - backend
- - worker
+
+ worker:
+ depends_on:
+ - obs-backend
+ image: obs/obs-worker
+ hostname: worker
+ restart: unless-stopped
+ privileged: true
+ volumes:
+ - worker-logs:/srv/obs/log
+ environment:
+ OBS_SRC_SERVER: obs-server:5352
+ OBS_REPO_SERVERS: obs-server:5252
+ OBS_WORKER_INSTANCES: 1
+
+volumes:
+ backend-storage:
+ backend-logs:
+ frontend-logs:
+ worker-logs:
diff --git a/docker/backend-docker-entrypoint.sh b/docker/backend-docker-entrypoint.sh
new file mode 100755
index 0000000000..eae9ba3725
--- /dev/null
+++ b/docker/backend-docker-entrypoint.sh
@@ -0,0 +1,48 @@
+#!/bin/sh
+
+if [ -z "$OBS_FRONTEND_HOST" ]; then
+ echo >&2 'error: OBS server frontend is unavailable and hostname option'
+ echo >&2 'is not specified '
+ echo >&2 ' You need to specify OBS_FRONTEND_HOST'
+ exit 1
+fi
+
+HOSTNAME=$(hostname)
+
+if [ "$(stat -c %U /srv/obs/run)" != obsrun ]
+then
+ echo "OBS files owned by the wrong user $(stat -c %U /srv/obs/run), re-owning..."
+ time chown obsrun:obsrun -R /srv/obs
+fi
+
+if [ ! -f /etc/obs/BSConfig.pm ]
+then
+ echo "OBS backend configuration not found, starting from scratch"
+ cp /usr/lib/obs/server/BSConfig.pm.template /etc/obs/BSConfig.pm
+fi
+
+echo "Configure OBS backend host: ${HOSTNAME}"
+sed -i "s/hostname = .*/hostname = '${HOSTNAME}';/g" /etc/obs/BSConfig.pm
+
+echo "Configure OBS frontend host: ${OBS_FRONTEND_HOST}"
+sed -i "s/frontend = undef/frontend = '${OBS_FRONTEND_HOST}'/g" /etc/obs/BSConfig.pm
+
+for arch in ${OBS_ARCHES:-amd64 i686 armhf arm64}
+do
+ obsarch=$(/opt/deb-arch-to-obs-arch "$arch")
+ if [ -z "$obsarch" ]
+ then
+ echo Failed to enable unsupported architecture $arch >&2
+ continue
+ fi
+ for template in /opt/services/backend/*@.conf.in
+ do
+ conf=$(echo $(basename $template) | sed -e "s|@|@$arch|" -e 's|.in$||')
+ sed -e "s|@ARCH@|$obsarch|g" $template > /etc/supervisor/conf.d/$conf
+ done
+done
+
+mkdir -p /srv/obs/log
+chmod ug=rwxt /srv/obs/run
+
+/usr/bin/supervisord -n
diff --git a/docker/configure-app.sh b/docker/configure-app.sh
new file mode 100755
index 0000000000..acaf3074f6
--- /dev/null
+++ b/docker/configure-app.sh
@@ -0,0 +1,45 @@
+#!/bin/sh -x
+
+if [ -z "$OBS_BACKEND_HOST" ]; then
+ echo >&2 'error: server backend is unavailable and hostname option is not specified '
+ echo >&2 ' You need to specify OBS_BACKEND_HOST'
+ exit 1
+fi
+
+if [ ! -z "$OBS_BACKEND_HOST" ]; then
+ sed -i s/"source_host: localhost"/"source_host: ${OBS_BACKEND_HOST}"/g config/options.yml
+fi
+
+for d in log tmp db/sphinx
+do
+ mkdir -p $d
+ chown -R frontend $d
+done
+
+# Allow overriding the secret key
+if [ -f /run/secrets/secretkey ]
+then
+ ln -sf /run/secrets/secretkey config/secret.key
+fi
+
+if [ ! -r config/secret.key ]
+then
+ bundle exec rake secret > config/secret.key
+fi
+
+for d in options.yml thinking_sphinx.yml
+do
+ [ -r config/$d ] || cp config/$d.example config/$d
+done
+
+# Set up msmtp if a configuration is supplied
+if [ -f /run/secrets/msmtprc ]
+then
+ ln -sf /run/secrets/msmtprc /etc/msmtprc
+fi
+
+# Set up SSO auth if a configuration is supplied
+if [ -f /run/secrets/ssoauth ]
+then
+ ln -sf /run/secrets/ssoauth config/auth.yml
+fi
diff --git a/docker/configure-backend-user.sh b/docker/configure-backend-user.sh
new file mode 100755
index 0000000000..5b434692aa
--- /dev/null
+++ b/docker/configure-backend-user.sh
@@ -0,0 +1,22 @@
+#!/bin/sh
+
+if ! getent group obsrun > /dev/null; then
+ addgroup --system --gid 999 obsrun
+fi
+
+if ! getent passwd obsrun > /dev/null; then
+ adduser --system --uid 999 \
+ --ingroup obsrun --shell /bin/false \
+ --home /usr/lib/obs --no-create-home obsrun
+ usermod -c "User for build service backend" obsrun
+fi
+
+if ! getent passwd obsservicerun > /dev/null; then
+ adduser --system --uid 998 \
+ --ingroup obsrun --shell /bin/false \
+ --home /usr/lib/obs/server --no-create-home obsservicerun
+ usermod -c "User for obs source service server" obsservicerun
+fi
+
+mkdir -p /srv/obs/repos
+chown obsrun:obsrun /srv/obs/repos
diff --git a/docker/configure-db.sh b/docker/configure-db.sh
new file mode 100755
index 0000000000..39feb5ee22
--- /dev/null
+++ b/docker/configure-db.sh
@@ -0,0 +1,34 @@
+#!/bin/sh
+
+if [ -z "$DB_HOST" -o -z "$DB_ROOT_PASSWORD" -o -z "$DB_NAME" -o -z "$DB_USER" -o -z "$DB_PASSWORD" ]; then
+ echo >&2 'error: database is uninitialized and password option is not specified or OBS'
+ echo >&2 ' You need to specify DB_HOST, DB_ROOT_PASSWORD, DB_NAME, DB_USER and DB_PASSWORD'
+ exit 1
+fi
+
+cat > config/database.yml <<EOF
+production:
+ adapter: mysql2
+ host: $DB_HOST
+ port: 3306
+ database: $DB_NAME
+ username: $DB_USER
+ password: $DB_PASSWORD
+ encoding: utf8mb4
+ collation: utf8mb4_unicode_ci
+ timeout: 15
+ pool: 30
+EOF
+
+rake() {
+ runuser -u frontend -- bundle exec rake "$@"
+}
+
+if ! rake db:migrate:status
+then
+ rake db:create || true
+ rake db:setup
+ rake writeconfiguration
+else
+ rake db:migrate:with_data
+fi
diff --git a/docker/configure-frontend-user.sh b/docker/configure-frontend-user.sh
new file mode 100755
index 0000000000..cd28b6af8d
--- /dev/null
+++ b/docker/configure-frontend-user.sh
@@ -0,0 +1,12 @@
+#!/bin/sh
+
+if ! getent group frontend > /dev/null; then
+ addgroup --system --gid 999 frontend
+fi
+
+if ! getent passwd frontend > /dev/null; then
+ adduser --system --uid 999 \
+ --ingroup frontend --shell /bin/false \
+ --home /obs --no-create-home frontend
+ usermod -c "User for build service frontend" frontend
+fi
diff --git a/docker/configure-sso.py b/docker/configure-sso.py
new file mode 100755
index 0000000000..13b373fa58
--- /dev/null
+++ b/docker/configure-sso.py
@@ -0,0 +1,38 @@
+#!/usr/bin/python3
+
+import yaml
+import os
+
+CONFIG_LOCATION='config/auth.yml'
+
+def parse_method(method: str):
+ for k, v in os.environ.items():
+ prefix = 'OBS_SSO_' + method.upper().replace('-', '_') + '_'
+ if k.startswith(prefix):
+ opt = k.replace(prefix, '').lower()
+ yield opt, v
+
+def reorder_options(options: dict):
+ new_options = {}
+ client_options = {}
+ for k, v in options.items():
+ if k.startswith('client_options_'):
+ client_options[k.replace('client_options_', '')] = v
+ else:
+ new_options[k] = v
+ if client_options:
+ new_options['client_options'] = client_options
+ return new_options
+
+def generate_yaml():
+ methods = os.environ['OBS_SSO_METHODS'].split()
+ config = {}
+ for method in methods:
+ options = reorder_options(dict(parse_method(method)))
+ config[method] = options
+ with open(CONFIG_LOCATION, 'w') as f:
+ yaml.safe_dump(config, stream=f)
+
+if __name__ == "__main__":
+ if os.environ.get('OBS_SSO_ENABLED') == 'true':
+ generate_yaml()
diff --git a/docker/deb-arch-to-obs-arch b/docker/deb-arch-to-obs-arch
new file mode 100755
index 0000000000..0d9d948781
--- /dev/null
+++ b/docker/deb-arch-to-obs-arch
@@ -0,0 +1,24 @@
+#!/bin/sh
+
+eval "$(dpkg-architecture ${1:+--host-arch $1} --print-set 2>/dev/null)"
+
+case "${DEB_HOST_ARCH:-$1}" in
+ (i?86)
+ echo "i586"
+ ;;
+
+ (armel)
+ echo "armv5el"
+ ;;
+
+ (armhf)
+ echo "armv7hl"
+ ;;
+
+ # add any more special cases here
+
+ (*)
+ echo "$DEB_HOST_GNU_CPU"
+ ;;
+esac
+
diff --git a/docker/frontend-docker-entrypoint.sh b/docker/frontend-docker-entrypoint.sh
new file mode 100755
index 0000000000..326e89ce10
--- /dev/null
+++ b/docker/frontend-docker-entrypoint.sh
@@ -0,0 +1,16 @@
+#!/bin/sh -e
+
+cd /obs/src/api
+
+# Make sure there are no stale files from previous runs
+rm -rfv tmp/pids/*
+chown -R frontend log tmp
+
+/opt/configure-app.sh
+/opt/configure-db.sh
+#/opt/configure-sso.py
+
+sed -i -e /web:/d Procfile
+echo "web: bundle exec puma -p 3000 -w ${OBS_FRONTEND_WORKERS:-4}" >> Procfile
+
+runuser -u frontend foreman start
diff --git a/docker/services/backend/obsclouduploadserver.conf b/docker/services/backend/obsclouduploadserver.conf
new file mode 100644
index 0000000000..e996e3a3d0
--- /dev/null
+++ b/docker/services/backend/obsclouduploadserver.conf
@@ -0,0 +1,11 @@
+[program:obsclouduploadserver]
+command=/usr/lib/obs/server/bs_clouduploadserver
+directory=/usr/lib/obs/server/
+stdout_logfile=/srv/obs/log/clouduploadserver.log
+redirect_stderr=true
+autostart=True
+priority=1
+stopsignal=KILL
+killasgroup=true
+stopasgroup=true
+
diff --git a/docker/services/backend/obsclouduploadworker.conf b/docker/services/backend/obsclouduploadworker.conf
new file mode 100644
index 0000000000..0cf718623a
--- /dev/null
+++ b/docker/services/backend/obsclouduploadworker.conf
@@ -0,0 +1,11 @@
+[program:obsclouduploadworker]
+command=/usr/lib/obs/server/bs_clouduploadworker
+directory=/usr/lib/obs/server/
+stdout_logfile=/srv/obs/log/clouduploadworker.log
+redirect_stderr=true
+autostart=True
+priority=1
+stopsignal=KILL
+killasgroup=true
+stopasgroup=true
+
diff --git a/docker/services/backend/obsdispatcher.conf b/docker/services/backend/obsdispatcher.conf
new file mode 100644
index 0000000000..e5db264923
--- /dev/null
+++ b/docker/services/backend/obsdispatcher.conf
@@ -0,0 +1,11 @@
+[program:obsdispatcher]
+command=/usr/lib/obs/server/bs_dispatch
+directory=/usr/lib/obs/server/
+stdout_logfile=/srv/obs/log/dispatcher.log
+redirect_stderr=true
+autostart=True
+priority=1
+stopsignal=KILL
+killasgroup=true
+stopasgroup=true
+
diff --git a/docker/services/backend/obsdodup.conf b/docker/services/backend/obsdodup.conf
new file mode 100644
index 0000000000..e515320dc9
--- /dev/null
+++ b/docker/services/backend/obsdodup.conf
@@ -0,0 +1,11 @@
+[program:obsdodup]
+command=/usr/lib/obs/server/bs_dodup
+directory=/usr/lib/obs/server/
+stdout_logfile=/srv/obs/log/dodup.log
+redirect_stderr=true
+autostart=True
+priority=1
+stopsignal=KILL
+killasgroup=true
+stopasgroup=true
+
diff --git a/docker/services/backend/obspublisher.conf b/docker/services/backend/obspublisher.conf
new file mode 100644
index 0000000000..6f4f13fff0
--- /dev/null
+++ b/docker/services/backend/obspublisher.conf
@@ -0,0 +1,11 @@
+[program:obspublisher]
+command=/usr/lib/obs/server/bs_publish
+directory=/usr/lib/obs/server/
+stdout_logfile=/srv/obs/log/publisher.log
+redirect_stderr=true
+autostart=True
+priority=1
+stopsignal=KILL
+killasgroup=true
+stopasgroup=true
+
diff --git a/docker/services/backend/obsrepserver.conf b/docker/services/backend/obsrepserver.conf
new file mode 100644
index 0000000000..5c3d9b8a03
--- /dev/null
+++ b/docker/services/backend/obsrepserver.conf
@@ -0,0 +1,11 @@
+[program:obsrepserver]
+command=/usr/lib/obs/server/bs_repserver
+directory=/usr/lib/obs/server/
+stdout_logfile=/srv/obs/log/rep_server.log
+redirect_stderr=true
+autostart=True
+priority=1
+stopsignal=KILL
+killasgroup=true
+stopasgroup=true
+
diff --git a/docker/services/backend/obsscheduler@.conf.in b/docker/services/backend/obsscheduler@.conf.in
new file mode 100644
index 0000000000..ee70976d20
--- /dev/null
+++ b/docker/services/backend/obsscheduler@.conf.in
@@ -0,0 +1,11 @@
+[program:obsscheduler@@ARCH@]
+command=/usr/lib/obs/server/bs_sched @ARCH@
+directory=/usr/lib/obs/server/
+stdout_logfile=/srv/obs/log/scheduler_@ARCH@.log
+redirect_stderr=true
+autostart=True
+priority=1
+stopsignal=KILL
+killasgroup=true
+stopasgroup=true
+
diff --git a/docker/services/backend/obsservice.conf b/docker/services/backend/obsservice.conf
new file mode 100644
index 0000000000..3ddbf9f48e
--- /dev/null
+++ b/docker/services/backend/obsservice.conf
@@ -0,0 +1,11 @@
+[program:obsservice]
+command=/usr/lib/obs/server/bs_service
+directory=/usr/lib/obs/server/
+stdout_logfile=/srv/obs/log/src_service.log
+redirect_stderr=true
+autostart=True
+priority=1
+stopsignal=KILL
+killasgroup=true
+stopasgroup=true
+
diff --git a/docker/services/backend/obsservicedispatch.conf b/docker/services/backend/obsservicedispatch.conf
new file mode 100644
index 0000000000..08fc6f673f
--- /dev/null
+++ b/docker/services/backend/obsservicedispatch.conf
@@ -0,0 +1,11 @@
+[program:obsservice]
+command=/usr/lib/obs/server/bs_servicedispatch
+directory=/usr/lib/obs/server/
+stdout_logfile=/srv/obs/log/servicedispatch.log
+redirect_stderr=true
+autostart=True
+priority=1
+stopsignal=KILL
+killasgroup=true
+stopasgroup=true
+
diff --git a/docker/services/backend/obssrcserver.conf b/docker/services/backend/obssrcserver.conf
new file mode 100644
index 0000000000..f67ac08195
--- /dev/null
+++ b/docker/services/backend/obssrcserver.conf
@@ -0,0 +1,11 @@
+[program:obssrcserver]
+command=/usr/lib/obs/server/bs_srcserver
+directory=/usr/lib/obs/server/
+stdout_logfile=/srv/obs/log/obssrcserver.log
+redirect_stderr=true
+autostart=True
+priority=1
+stopsignal=KILL
+killasgroup=true
+stopasgroup=true
+
diff --git a/docker/services/backend/obswarden.conf b/docker/services/backend/obswarden.conf
new file mode 100644
index 0000000000..e6cec6706e
--- /dev/null
+++ b/docker/services/backend/obswarden.conf
@@ -0,0 +1,11 @@
+[program:obswarden]
+command=/usr/lib/obs/server/bs_warden
+directory=/usr/lib/obs/server/
+stdout_logfile=/srv/obs/log/warden.log
+redirect_stderr=true
+autostart=True
+priority=1
+stopsignal=KILL
+killasgroup=true
+stopasgroup=true
+
diff --git a/docker/services/worker/obsworker.conf b/docker/services/worker/obsworker.conf
new file mode 100644
index 0000000000..1129707f4e
--- /dev/null
+++ b/docker/services/worker/obsworker.conf
@@ -0,0 +1,13 @@
+[program:obsworker]
+command=%(ENV_OBS_WORKER_PATH)s/bs_worker --hardstatus --root /var/cache/build/root_%(process_num)d --statedir /var/cache/build/state_%(process_num)d --id %(ENV_OBS_WORKER_NAME)s:%(process_num)d %(ENV_OBS_WORKER_OPT)s
+process_name=%(program_name)s_%(process_num)d
+directory=%(ENV_OBS_WORKER_PATH)s
+stdout_logfile=/srv/obs/log/worker_%(process_num)d.log
+redirect_stderr=true
+autostart=True
+priority=1
+stopsignal=KILL
+killasgroup=true
+stopasgroup=true
+numprocs=%(ENV_OBS_WORKER_INSTANCES)s
+numprocs_start=1
diff --git a/docker/worker-docker-entrypoint.sh b/docker/worker-docker-entrypoint.sh
new file mode 100755
index 0000000000..47027e23af
--- /dev/null
+++ b/docker/worker-docker-entrypoint.sh
@@ -0,0 +1,72 @@
+#!/bin/sh -ex
+
+obsrundir=/run/obs
+workerdir=/var/cache/build
+workerbootdir="$workerdir/boot"
+obslogdir=/var/log/obs
+
+: mkdir -p "$obsrundir"
+
+: ${OBS_REPO_SERVERS:=obs-server:5252}
+
+repo_param=
+for i in $OBS_REPO_SERVERS
+do
+ repo_param="$REPO_PARAM --reposerver http://$i"
+ WORKER_CODE="http://$i"
+done
+
+: ${OBS_WORKER_NICE_LEVEL:=18}
+
+OBS_WORKER_OPT="--hardstatus $repo_param ${OBS_WORKER_JOBS:+--jobs $OBS_WORKER_JOBS}\
+ ${OBS_WORKER_CLEANUP_CHROOT:+--cleanup-chroot}\
+ ${OBS_WORKER_WIPE_AFTER_BUILD:+--wipeafterbuild}\
+ ${OBS_SRC_SERVER:+--srcserver $OBS_SRC_SERVER}\
+ ${OBS_ARCH:+--arch $OBS_ARCH} ${OBS_WORKER_OPT}"
+
+export OBS_WORKER_OPT
+
+: ${OBS_WORKER_NAME:=$(hostname)}
+
+export OBS_WORKER_NAME
+
+: ${OBS_WORKER_INSTANCES:=$(nproc)}
+
+export OBS_WORKER_INSTANCES
+
+OBS_WORKER_PATH=/usr/lib/obs/server
+
+update_worker() {
+ echo "Fetching initial worker code from $WORKER_CODE/getworkercode"
+ mkdir -p "$workerbootdir"
+ cd "$workerbootdir"
+ for retry in $(seq 10)
+ do
+ if curl -sS "$WORKER_CODE/getworkercode" | cpio --extract
+ then
+ ln -sfn . XML
+ chmod 755 bs_worker
+ return 0
+ fi
+ # we need to wait for rep server maybe
+ echo "WARNING: Could not reach rep server $WORKER_CODE. Trying again." >&2
+ sleep 10
+ done
+ echo "ERROR: Unable to reach rep server $WORKER_CODE!" >&2
+ return 1
+}
+
+if [ -n "$WORKER_CODE" ]
+then
+ update_worker
+ OBS_WORKER_PATH="$workerbootdir"
+fi
+
+export OBS_WORKER_PATH
+
+for i in $(seq 1 $OBS_WORKER_INSTANCES)
+do
+ mkdir -p $workerdir/root_$i $workerdir/state_$i
+done
+
+nice -n "$OBS_WORKER_NICE_LEVEL" /usr/bin/supervisord -n
diff --git a/src/api/config/environments/production.rb b/src/api/config/environments/production.rb
index bf5bc55254..72d510d209 100644
--- a/src/api/config/environments/production.rb
+++ b/src/api/config/environments/production.rb
@@ -37,7 +37,7 @@ OBSApi::Application.configure do
config.action_controller.perform_caching = true
# Disable Rails's static asset server (Apache or nginx will already do this)
- config.public_file_server.enabled = false
+ config.public_file_server.enabled = true
# Compress JavaScripts and CSS
config.assets.compress = true
diff --git a/src/api/config/options.yml.example b/src/api/config/options.yml.example
index bc98f94f98..1a13f782fc 100644
--- a/src/api/config/options.yml.example
+++ b/src/api/config/options.yml.example
@@ -199,6 +199,7 @@ default: &default
production:
<<: *default
+ memcached_host: cache
test:
<<: *default
@@ -209,4 +210,3 @@ development:
<<: *default
source_host: backend
memcached_host: cache
-
diff --git a/src/api/db/migrate/20191011000000_create_allowbuilddeps.rb b/src/api/db/migrate/20191011000000_create_allowbuilddeps.rb
index f4de579749..acf2f28a56 100644
--- a/src/api/db/migrate/20191011000000_create_allowbuilddeps.rb
+++ b/src/api/db/migrate/20191011000000_create_allowbuilddeps.rb
@@ -1,4 +1,4 @@
-class CreateAllowbuilddeps < ActiveRecord::Migration
+class CreateAllowbuilddeps < ActiveRecord::Migration[4.2]
def self.up
create_table :allowbuilddeps do |t|
t.integer :db_project_id, :null => false
--
GitLab