diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml new file mode 100644 index 0000000000000000000000000000000000000000..8cb609db38300d5dd82bcd18d3ecae51b4788981 --- /dev/null +++ b/.gitlab-ci.yml @@ -0,0 +1,70 @@ +stages: + - docker + +.build-docker-image: + stage: docker + image: + name: gcr.io/kaniko-project/executor:debug + entrypoint: [""] + script: + - | + cat << EOF > /kaniko/.docker/config.json + { + "auths":{ + "$CI_REGISTRY": { + "username":"$CI_REGISTRY_USER", + "password":"$CI_REGISTRY_PASSWORD" + } + } + } + EOF + - > + /kaniko/executor + --context $CI_PROJECT_DIR + --dockerfile $CI_PROJECT_DIR/Dockerfile.$image + --destination $CI_REGISTRY_IMAGE/obs-$image:$CI_COMMIT_REF_SLUG + --build-arg REGISTRY=$CI_REGISTRY_IMAGE + --build-arg TAG=$CI_COMMIT_REF_SLUG + --single-snapshot + - echo Pushed $CI_REGISTRY_IMAGE/obs-$image:$CI_COMMIT_REF_SLUG + +frontend-base: + extends: .build-docker-image + only: + changes: + - Dockerfile.frontend-base + - src/api/Gemfile + - src/api/Gemfile.lock + variables: + image: frontend-base + +frontend: + extends: .build-docker-image + needs: + - frontend-base + variables: + image: frontend + +backend-base: + extends: .build-docker-image + only: + changes: + - Dockerfile.backend-base + - dist/** + - src/backend/** + variables: + image: backend-base + +backend: + extends: .build-docker-image + needs: + - backend-base + variables: + image: backend + +worker: + extends: .build-docker-image + needs: + - backend-base + variables: + image: worker diff --git a/Dockerfile.backend b/Dockerfile.backend new file mode 100644 index 0000000000000000000000000000000000000000..41bbf27328baa0f87e6f7a5793143eb7faa96207 --- /dev/null +++ b/Dockerfile.backend @@ -0,0 +1,16 @@ +ARG REGISTRY +ARG TAG=latest +FROM $REGISTRY/obs-backend-base:$TAG + +LABEL maintainer Andrej Shadura <andrew.shadura@collabora.co.uk> + +COPY docker/services/backend/*.conf /etc/supervisor/conf.d/ +COPY docker/ /opt/ + +VOLUME /etc/obs + +RUN /opt/configure-backend-user.sh + +VOLUME /srv/obs + +ENTRYPOINT /opt/backend-docker-entrypoint.sh diff --git a/Dockerfile.backend-base b/Dockerfile.backend-base new file mode 100644 index 0000000000000000000000000000000000000000..f0ed5c44ecc8d8db94bf1c5aeeae3079baab3f08 --- /dev/null +++ b/Dockerfile.backend-base @@ -0,0 +1,53 @@ +FROM debian:bullseye-slim as server +LABEL maintainer Andrej Shadura <andrew.shadura@collabora.co.uk> +ENV LC_ALL=C.UTF-8 +ARG DEBIAN_FRONTEND=noninteractive +ARG WORKDIR=/tmp/sources + +# Needs checking what’s actually needed +RUN apt-get update \ + && apt-get install -y \ + apt-utils \ + adduser \ + ca-certificates \ + curl \ + diffutils \ + dpkg-dev \ + git \ + locales \ + libbssolv-perl \ + libcompress-raw-zlib-perl \ + libfile-sync-perl \ + libio-compress-perl \ + libjson-xs-perl \ + libnet-ssleay-perl \ + libsocket-msghdr-perl \ + libtimedate-perl \ + libxml-parser-perl \ + libxml-simple-perl \ + libxml-structured-perl \ + libyaml-libyaml-perl \ + make \ + obs-build \ + patch \ + procps \ + reprepro \ + supervisor \ + time \ + tzdata \ + zstd + +COPY . $WORKDIR + +RUN make -C $WORKDIR/dist install +RUN make -C $WORKDIR/src/backend install + +RUN rm -rf $WORKDIR + +RUN mkdir -p /etc/obs +RUN cp /usr/lib/obs/server/BSConfig.pm.template /etc/obs/BSConfig.pm + +RUN ln -sf /etc/obs/BSConfig.pm /usr/lib/obs/server/BSConfig.pm +RUN ln -sf /usr/lib/obs-build /usr/lib/obs/server/build + +ENTRYPOINT /opt/backend-docker-entrypoint.sh diff --git a/Dockerfile.frontend b/Dockerfile.frontend new file mode 100644 index 0000000000000000000000000000000000000000..47248b2cd71f33a3952395f5d75ab864444155e9 --- /dev/null +++ b/Dockerfile.frontend @@ -0,0 +1,49 @@ +ARG REGISTRY +ARG TAG=latest +FROM $REGISTRY/obs-frontend-base:$TAG as base + +ARG REGISTRY +ARG TAG=latest +FROM $REGISTRY/obs-frontend-base:$TAG +ARG WORKDIR=/tmp/sources +ARG INSTALLDIR=/obs + +ADD src/api/ $INSTALLDIR/src/api/ + +COPY --from=base $INSTALLDIR/src/api/Gemfile* $INSTALLDIR/src/api/ + +WORKDIR $INSTALLDIR/src/api + +RUN ls -la + +ARG BUNDLE_BUILD__SASSC=--disable-march-tune-native +ARG NOKOGIRI_USE_SYSTEM_LIBRARIES=1 + +RUN bundle install --jobs=$(nproc) --retry=3 + +# Install extra gems for SSO +RUN gem install omniauth omniauth-gitlab --no-doc +RUN gem install omniauth-azure-oauth2 omniauth-azure-oauth2-v2 omniauth-phabricator --no-doc + +RUN rm -rf /var/lib/gems/*/cache + +ENV RAILS_ENV=production +ENV RAILS_LOG_TO_STDOUT=true + +RUN echo nonce > config/secret.key \ + && DATABASE_URL=mysql2://localhost/noncedb bundle exec rake assets:precompile RAILS_GROUPS=assets \ + && rm config/secret.key + +RUN bundle config --local without test:assets:development + +RUN sed -i 's|^#!/usr/bin/ruby.ruby.*$|#!/usr/bin/ruby|' bin/* script/* +RUN sed -i -e /mailcatcher:/d -e /web:/d Procfile + +COPY docker/ /opt/ + +RUN /opt/configure-frontend-user.sh +RUN mkdir -p log tmp db/sphinx \ + && chown -R frontend /obs + +ENTRYPOINT /opt/frontend-docker-entrypoint.sh +EXPOSE 3000 diff --git a/Dockerfile.frontend-base b/Dockerfile.frontend-base new file mode 100644 index 0000000000000000000000000000000000000000..8c5ee5b32336c0264430925c6ef067bcf5303533 --- /dev/null +++ b/Dockerfile.frontend-base @@ -0,0 +1,74 @@ +# FROM debian:buster-slim as frontend-base +FROM debian:bookworm-slim as frontend-base +ENV LC_ALL=C.UTF-8 +ARG DEBIAN_FRONTEND=noninteractive +ARG INSTALLDIR=/obs + +RUN apt-get update \ + && apt-get install -y \ + apt-utils \ + adduser \ + ca-certificates \ + curl \ + diffutils \ + dpkg-dev \ + git \ + locales \ + make \ + msmtp-mta \ + mariadb-client \ + npm \ + pkgconf \ + ruby \ + ruby-dev \ + ruby-bundler \ + ruby-ffi \ + ruby-foreman \ + sphinxsearch \ + patch \ + supervisor \ + time \ + tzdata + +RUN apt-get update \ + && apt-get install -y \ + default-libmysqlclient-dev \ + libldap2-dev \ + libsasl2-dev \ + libxml2-dev \ + libxslt1-dev \ + zlib1g-dev + +# Work around a bug in buster +RUN [ $(readlink /usr/bin/ruby) != ruby2.5 ] \ + || { ver=$(basename -s.gemspec $(echo /usr/share/rubygems-integration/all/specifications/rake-*.gemspec)); \ + mkdir -p /usr/share/rubygems-integration/all/gems/$ver/exe; \ + ln -s /usr/bin/rake /usr/share/rubygems-integration/all/gems/$ver/exe/rake; } + +RUN gem install --no-format-executable brakeman --version 5.0.2 --no-doc +RUN gem install sassc --version 2.0.1 --no-doc + +ADD src/api/Gemfile* $INSTALLDIR/src/api/ +WORKDIR $INSTALLDIR/src/api/ + +RUN sed -e "/gem 'puma'/d" Gemfile > Gemfile.new; \ + echo "gem 'puma'" >> Gemfile.new; \ + diff -u Gemfile Gemfile.new; \ + mv Gemfile.new Gemfile + +RUN sed -e '/BUNDLED WITH/,+1 d' Gemfile.lock > Gemfile.lock.new; \ + diff -u Gemfile.lock Gemfile.lock.new; \ + mv Gemfile.lock.new Gemfile.lock + +ARG BUNDLE_BUILD__SASSC=--disable-march-tune-native +ARG NOKOGIRI_USE_SYSTEM_LIBRARIES=1 + +RUN bundle config --global without development:test + +RUN bundle install --jobs=$(nproc) --retry=3 + +RUN rm -rf \ + /var/lib/gems/*/cache/ \ + /var/lib/gems/*/test/ \ + /var/lib/gems/*/extensions/*/*/*/gem_make.out \ + /var/lib/gems/*/extensions/*/*/*/*.log \ diff --git a/Dockerfile.worker b/Dockerfile.worker new file mode 100644 index 0000000000000000000000000000000000000000..0bb994b36a7967500f9186c60aa1f8c7e0f38570 --- /dev/null +++ b/Dockerfile.worker @@ -0,0 +1,28 @@ +ARG REGISTRY +ARG TAG=latest +FROM $REGISTRY/obs-backend-base:$TAG + +LABEL maintainer Andrej Shadura <andrew.shadura@collabora.co.uk> +ARG DEBIAN_FRONTEND=noninteractive + +# TODO: cleanup +RUN apt-get update \ + && apt-get install -y \ + binutils \ + cpio \ + curl \ + debootstrap \ + fdisk \ + libarchive-tools \ + lsb-base \ + lvm2 \ + lzma \ + psmisc \ + rpm + +COPY docker/services/worker/*.conf /etc/supervisor/conf.d/ +COPY docker/ /opt/ + +# RUN /opt/configure-worker-user.sh + +ENTRYPOINT /opt/worker-docker-entrypoint.sh diff --git a/docker-compose.yml b/docker-compose.yml index 90aa950244e4d2c157f621fe8cbff244e2b0057e..69ebc276a2087c2e8986a290381ed15bf81d7f88 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,44 +1,72 @@ -version: "2.1" +--- +version: '2' + services: - db: - image: registry.opensuse.org/obs/server/unstable/container/leap151/containers/openbuildservice/mariadb - ports: - - "3306:3306" - command: /usr/lib/mysql/mysql-systemd-helper start + obs-db: + image: mariadb:10.6 + restart: unless-stopped + environment: + MARIADB_ROOT_PASSWORD: someobs + MARIADB_DATABASE: obsapi + MARIADB_USER: obs-api + MARIADB_PASSWORD: someobs + cache: - image: registry.opensuse.org/obs/server/unstable/container/leap151/containers/openbuildservice/memcached - ports: - - "11211:11211" - command: /usr/sbin/memcached -u memcached - backend: - image: registry.opensuse.org/obs/server/unstable/container/leap151/containers/openbuildservice/backend - volumes: - - .:/obs - - ./dist/aws_credentials:/etc/obs/cloudupload/.aws/config - - ./dist/ec2utils.conf:/etc/obs/cloudupload/.ec2utils.conf - - ./dist/clouduploader.rb:/usr/bin/clouduploader - command: /obs/contrib/start_development_backend -d /obs - worker: - image: registry.opensuse.org/obs/server/unstable/container/leap151/containers/openbuildservice/backend + image: memcached:1.6-alpine + + obs-backend: + image: obs/obs-backend + hostname: obs-server + restart: unless-stopped volumes: - - .:/obs - privileged: true + - backend-storage:/srv/obs + - backend-logs:/srv/obs/log + environment: + OBS_FRONTEND_HOST: obs-api + + obs-frontend: + image: obs/obs-frontend depends_on: - - backend - command: /obs/contrib/start_development_worker - frontend: - image: openbuildservice/frontend - command: foreman start -p 3000 - build: - dockerfile: docker-files/Dockerfile - context: src/api + - obs-server volumes: - - .:/obs + - frontend-logs:/obs/src/api/log + - type: tmpfs + target: /tmp + tmpfs: + size: 4G + hostname: obs-api + restart: unless-stopped + environment: + DB_HOST: obs-db + DB_PORT: 3306 + DB_ROOT_PASSWORD: someobs + DB_NAME: obsapi + DB_USER: obs-api + DB_PASSWORD: someobs + OBS_BACKEND_HOST: obs-server + OBS_FRONTEND_WORKERS: 4 ports: - - "3000:3000" - - "1080:1080" + - "127.0.0.1:3000:3000" depends_on: - - db + - obs-db - cache - - backend - - worker + + worker: + depends_on: + - obs-backend + image: obs/obs-worker + hostname: worker + restart: unless-stopped + privileged: true + volumes: + - worker-logs:/srv/obs/log + environment: + OBS_SRC_SERVER: obs-server:5352 + OBS_REPO_SERVERS: obs-server:5252 + OBS_WORKER_INSTANCES: 1 + +volumes: + backend-storage: + backend-logs: + frontend-logs: + worker-logs: diff --git a/docker/backend-docker-entrypoint.sh b/docker/backend-docker-entrypoint.sh new file mode 100755 index 0000000000000000000000000000000000000000..eae9ba37250b3ef6f6cc95f27409d711d5c41da6 --- /dev/null +++ b/docker/backend-docker-entrypoint.sh @@ -0,0 +1,48 @@ +#!/bin/sh + +if [ -z "$OBS_FRONTEND_HOST" ]; then + echo >&2 'error: OBS server frontend is unavailable and hostname option' + echo >&2 'is not specified ' + echo >&2 ' You need to specify OBS_FRONTEND_HOST' + exit 1 +fi + +HOSTNAME=$(hostname) + +if [ "$(stat -c %U /srv/obs/run)" != obsrun ] +then + echo "OBS files owned by the wrong user $(stat -c %U /srv/obs/run), re-owning..." + time chown obsrun:obsrun -R /srv/obs +fi + +if [ ! -f /etc/obs/BSConfig.pm ] +then + echo "OBS backend configuration not found, starting from scratch" + cp /usr/lib/obs/server/BSConfig.pm.template /etc/obs/BSConfig.pm +fi + +echo "Configure OBS backend host: ${HOSTNAME}" +sed -i "s/hostname = .*/hostname = '${HOSTNAME}';/g" /etc/obs/BSConfig.pm + +echo "Configure OBS frontend host: ${OBS_FRONTEND_HOST}" +sed -i "s/frontend = undef/frontend = '${OBS_FRONTEND_HOST}'/g" /etc/obs/BSConfig.pm + +for arch in ${OBS_ARCHES:-amd64 i686 armhf arm64} +do + obsarch=$(/opt/deb-arch-to-obs-arch "$arch") + if [ -z "$obsarch" ] + then + echo Failed to enable unsupported architecture $arch >&2 + continue + fi + for template in /opt/services/backend/*@.conf.in + do + conf=$(echo $(basename $template) | sed -e "s|@|@$arch|" -e 's|.in$||') + sed -e "s|@ARCH@|$obsarch|g" $template > /etc/supervisor/conf.d/$conf + done +done + +mkdir -p /srv/obs/log +chmod ug=rwxt /srv/obs/run + +/usr/bin/supervisord -n diff --git a/docker/configure-app.sh b/docker/configure-app.sh new file mode 100755 index 0000000000000000000000000000000000000000..acaf3074f6242decfb76a4015341906ef8eb6c9b --- /dev/null +++ b/docker/configure-app.sh @@ -0,0 +1,45 @@ +#!/bin/sh -x + +if [ -z "$OBS_BACKEND_HOST" ]; then + echo >&2 'error: server backend is unavailable and hostname option is not specified ' + echo >&2 ' You need to specify OBS_BACKEND_HOST' + exit 1 +fi + +if [ ! -z "$OBS_BACKEND_HOST" ]; then + sed -i s/"source_host: localhost"/"source_host: ${OBS_BACKEND_HOST}"/g config/options.yml +fi + +for d in log tmp db/sphinx +do + mkdir -p $d + chown -R frontend $d +done + +# Allow overriding the secret key +if [ -f /run/secrets/secretkey ] +then + ln -sf /run/secrets/secretkey config/secret.key +fi + +if [ ! -r config/secret.key ] +then + bundle exec rake secret > config/secret.key +fi + +for d in options.yml thinking_sphinx.yml +do + [ -r config/$d ] || cp config/$d.example config/$d +done + +# Set up msmtp if a configuration is supplied +if [ -f /run/secrets/msmtprc ] +then + ln -sf /run/secrets/msmtprc /etc/msmtprc +fi + +# Set up SSO auth if a configuration is supplied +if [ -f /run/secrets/ssoauth ] +then + ln -sf /run/secrets/ssoauth config/auth.yml +fi diff --git a/docker/configure-backend-user.sh b/docker/configure-backend-user.sh new file mode 100755 index 0000000000000000000000000000000000000000..5b434692aaafe11e1522cdeae1d3813df10c1e70 --- /dev/null +++ b/docker/configure-backend-user.sh @@ -0,0 +1,22 @@ +#!/bin/sh + +if ! getent group obsrun > /dev/null; then + addgroup --system --gid 999 obsrun +fi + +if ! getent passwd obsrun > /dev/null; then + adduser --system --uid 999 \ + --ingroup obsrun --shell /bin/false \ + --home /usr/lib/obs --no-create-home obsrun + usermod -c "User for build service backend" obsrun +fi + +if ! getent passwd obsservicerun > /dev/null; then + adduser --system --uid 998 \ + --ingroup obsrun --shell /bin/false \ + --home /usr/lib/obs/server --no-create-home obsservicerun + usermod -c "User for obs source service server" obsservicerun +fi + +mkdir -p /srv/obs/repos +chown obsrun:obsrun /srv/obs/repos diff --git a/docker/configure-db.sh b/docker/configure-db.sh new file mode 100755 index 0000000000000000000000000000000000000000..39feb5ee22ab39d23845cd853ccf6625b49f2367 --- /dev/null +++ b/docker/configure-db.sh @@ -0,0 +1,34 @@ +#!/bin/sh + +if [ -z "$DB_HOST" -o -z "$DB_ROOT_PASSWORD" -o -z "$DB_NAME" -o -z "$DB_USER" -o -z "$DB_PASSWORD" ]; then + echo >&2 'error: database is uninitialized and password option is not specified or OBS' + echo >&2 ' You need to specify DB_HOST, DB_ROOT_PASSWORD, DB_NAME, DB_USER and DB_PASSWORD' + exit 1 +fi + +cat > config/database.yml <<EOF +production: + adapter: mysql2 + host: $DB_HOST + port: 3306 + database: $DB_NAME + username: $DB_USER + password: $DB_PASSWORD + encoding: utf8mb4 + collation: utf8mb4_unicode_ci + timeout: 15 + pool: 30 +EOF + +rake() { + runuser -u frontend -- bundle exec rake "$@" +} + +if ! rake db:migrate:status +then + rake db:create || true + rake db:setup + rake writeconfiguration +else + rake db:migrate:with_data +fi diff --git a/docker/configure-frontend-user.sh b/docker/configure-frontend-user.sh new file mode 100755 index 0000000000000000000000000000000000000000..cd28b6af8dc55a4b85e36837633b5004e581a906 --- /dev/null +++ b/docker/configure-frontend-user.sh @@ -0,0 +1,12 @@ +#!/bin/sh + +if ! getent group frontend > /dev/null; then + addgroup --system --gid 999 frontend +fi + +if ! getent passwd frontend > /dev/null; then + adduser --system --uid 999 \ + --ingroup frontend --shell /bin/false \ + --home /obs --no-create-home frontend + usermod -c "User for build service frontend" frontend +fi diff --git a/docker/configure-sso.py b/docker/configure-sso.py new file mode 100755 index 0000000000000000000000000000000000000000..13b373fa582df4bf7dbe9ea7a7a49dc1de55ee13 --- /dev/null +++ b/docker/configure-sso.py @@ -0,0 +1,38 @@ +#!/usr/bin/python3 + +import yaml +import os + +CONFIG_LOCATION='config/auth.yml' + +def parse_method(method: str): + for k, v in os.environ.items(): + prefix = 'OBS_SSO_' + method.upper().replace('-', '_') + '_' + if k.startswith(prefix): + opt = k.replace(prefix, '').lower() + yield opt, v + +def reorder_options(options: dict): + new_options = {} + client_options = {} + for k, v in options.items(): + if k.startswith('client_options_'): + client_options[k.replace('client_options_', '')] = v + else: + new_options[k] = v + if client_options: + new_options['client_options'] = client_options + return new_options + +def generate_yaml(): + methods = os.environ['OBS_SSO_METHODS'].split() + config = {} + for method in methods: + options = reorder_options(dict(parse_method(method))) + config[method] = options + with open(CONFIG_LOCATION, 'w') as f: + yaml.safe_dump(config, stream=f) + +if __name__ == "__main__": + if os.environ.get('OBS_SSO_ENABLED') == 'true': + generate_yaml() diff --git a/docker/deb-arch-to-obs-arch b/docker/deb-arch-to-obs-arch new file mode 100755 index 0000000000000000000000000000000000000000..0d9d948781b23bb7ae13f87399a1f6dcfb1afa17 --- /dev/null +++ b/docker/deb-arch-to-obs-arch @@ -0,0 +1,24 @@ +#!/bin/sh + +eval "$(dpkg-architecture ${1:+--host-arch $1} --print-set 2>/dev/null)" + +case "${DEB_HOST_ARCH:-$1}" in + (i?86) + echo "i586" + ;; + + (armel) + echo "armv5el" + ;; + + (armhf) + echo "armv7hl" + ;; + + # add any more special cases here + + (*) + echo "$DEB_HOST_GNU_CPU" + ;; +esac + diff --git a/docker/frontend-docker-entrypoint.sh b/docker/frontend-docker-entrypoint.sh new file mode 100755 index 0000000000000000000000000000000000000000..326e89ce10eb6d4f7cc377f2c52f40b9a0b950a9 --- /dev/null +++ b/docker/frontend-docker-entrypoint.sh @@ -0,0 +1,16 @@ +#!/bin/sh -e + +cd /obs/src/api + +# Make sure there are no stale files from previous runs +rm -rfv tmp/pids/* +chown -R frontend log tmp + +/opt/configure-app.sh +/opt/configure-db.sh +#/opt/configure-sso.py + +sed -i -e /web:/d Procfile +echo "web: bundle exec puma -p 3000 -w ${OBS_FRONTEND_WORKERS:-4}" >> Procfile + +runuser -u frontend foreman start diff --git a/docker/services/backend/obsclouduploadserver.conf b/docker/services/backend/obsclouduploadserver.conf new file mode 100644 index 0000000000000000000000000000000000000000..e996e3a3d0e42877b63f900f517dd554fa4ebc09 --- /dev/null +++ b/docker/services/backend/obsclouduploadserver.conf @@ -0,0 +1,11 @@ +[program:obsclouduploadserver] +command=/usr/lib/obs/server/bs_clouduploadserver +directory=/usr/lib/obs/server/ +stdout_logfile=/srv/obs/log/clouduploadserver.log +redirect_stderr=true +autostart=True +priority=1 +stopsignal=KILL +killasgroup=true +stopasgroup=true + diff --git a/docker/services/backend/obsclouduploadworker.conf b/docker/services/backend/obsclouduploadworker.conf new file mode 100644 index 0000000000000000000000000000000000000000..0cf718623a674176b6fe36f91659528209c32835 --- /dev/null +++ b/docker/services/backend/obsclouduploadworker.conf @@ -0,0 +1,11 @@ +[program:obsclouduploadworker] +command=/usr/lib/obs/server/bs_clouduploadworker +directory=/usr/lib/obs/server/ +stdout_logfile=/srv/obs/log/clouduploadworker.log +redirect_stderr=true +autostart=True +priority=1 +stopsignal=KILL +killasgroup=true +stopasgroup=true + diff --git a/docker/services/backend/obsdispatcher.conf b/docker/services/backend/obsdispatcher.conf new file mode 100644 index 0000000000000000000000000000000000000000..e5db264923013183e368e9e494f0e195aa95ff89 --- /dev/null +++ b/docker/services/backend/obsdispatcher.conf @@ -0,0 +1,11 @@ +[program:obsdispatcher] +command=/usr/lib/obs/server/bs_dispatch +directory=/usr/lib/obs/server/ +stdout_logfile=/srv/obs/log/dispatcher.log +redirect_stderr=true +autostart=True +priority=1 +stopsignal=KILL +killasgroup=true +stopasgroup=true + diff --git a/docker/services/backend/obsdodup.conf b/docker/services/backend/obsdodup.conf new file mode 100644 index 0000000000000000000000000000000000000000..e515320dc904f29153976cce449c73481ed42b94 --- /dev/null +++ b/docker/services/backend/obsdodup.conf @@ -0,0 +1,11 @@ +[program:obsdodup] +command=/usr/lib/obs/server/bs_dodup +directory=/usr/lib/obs/server/ +stdout_logfile=/srv/obs/log/dodup.log +redirect_stderr=true +autostart=True +priority=1 +stopsignal=KILL +killasgroup=true +stopasgroup=true + diff --git a/docker/services/backend/obspublisher.conf b/docker/services/backend/obspublisher.conf new file mode 100644 index 0000000000000000000000000000000000000000..6f4f13fff0153e81a4381661ce8793a7935010de --- /dev/null +++ b/docker/services/backend/obspublisher.conf @@ -0,0 +1,11 @@ +[program:obspublisher] +command=/usr/lib/obs/server/bs_publish +directory=/usr/lib/obs/server/ +stdout_logfile=/srv/obs/log/publisher.log +redirect_stderr=true +autostart=True +priority=1 +stopsignal=KILL +killasgroup=true +stopasgroup=true + diff --git a/docker/services/backend/obsrepserver.conf b/docker/services/backend/obsrepserver.conf new file mode 100644 index 0000000000000000000000000000000000000000..5c3d9b8a031b48763285d752ee2ebdc0839ba802 --- /dev/null +++ b/docker/services/backend/obsrepserver.conf @@ -0,0 +1,11 @@ +[program:obsrepserver] +command=/usr/lib/obs/server/bs_repserver +directory=/usr/lib/obs/server/ +stdout_logfile=/srv/obs/log/rep_server.log +redirect_stderr=true +autostart=True +priority=1 +stopsignal=KILL +killasgroup=true +stopasgroup=true + diff --git a/docker/services/backend/obsscheduler@.conf.in b/docker/services/backend/obsscheduler@.conf.in new file mode 100644 index 0000000000000000000000000000000000000000..ee70976d2095c2ef9c1bf90bba79064c9055b247 --- /dev/null +++ b/docker/services/backend/obsscheduler@.conf.in @@ -0,0 +1,11 @@ +[program:obsscheduler@@ARCH@] +command=/usr/lib/obs/server/bs_sched @ARCH@ +directory=/usr/lib/obs/server/ +stdout_logfile=/srv/obs/log/scheduler_@ARCH@.log +redirect_stderr=true +autostart=True +priority=1 +stopsignal=KILL +killasgroup=true +stopasgroup=true + diff --git a/docker/services/backend/obsservice.conf b/docker/services/backend/obsservice.conf new file mode 100644 index 0000000000000000000000000000000000000000..3ddbf9f48e12c795c1f49063b45cdabde0a4f58c --- /dev/null +++ b/docker/services/backend/obsservice.conf @@ -0,0 +1,11 @@ +[program:obsservice] +command=/usr/lib/obs/server/bs_service +directory=/usr/lib/obs/server/ +stdout_logfile=/srv/obs/log/src_service.log +redirect_stderr=true +autostart=True +priority=1 +stopsignal=KILL +killasgroup=true +stopasgroup=true + diff --git a/docker/services/backend/obsservicedispatch.conf b/docker/services/backend/obsservicedispatch.conf new file mode 100644 index 0000000000000000000000000000000000000000..08fc6f673f549b88f59cbe6e6a8689a40857d2bd --- /dev/null +++ b/docker/services/backend/obsservicedispatch.conf @@ -0,0 +1,11 @@ +[program:obsservice] +command=/usr/lib/obs/server/bs_servicedispatch +directory=/usr/lib/obs/server/ +stdout_logfile=/srv/obs/log/servicedispatch.log +redirect_stderr=true +autostart=True +priority=1 +stopsignal=KILL +killasgroup=true +stopasgroup=true + diff --git a/docker/services/backend/obssrcserver.conf b/docker/services/backend/obssrcserver.conf new file mode 100644 index 0000000000000000000000000000000000000000..f67ac08195fe5da6eff1b93fcc7153eac8a9eb59 --- /dev/null +++ b/docker/services/backend/obssrcserver.conf @@ -0,0 +1,11 @@ +[program:obssrcserver] +command=/usr/lib/obs/server/bs_srcserver +directory=/usr/lib/obs/server/ +stdout_logfile=/srv/obs/log/obssrcserver.log +redirect_stderr=true +autostart=True +priority=1 +stopsignal=KILL +killasgroup=true +stopasgroup=true + diff --git a/docker/services/backend/obswarden.conf b/docker/services/backend/obswarden.conf new file mode 100644 index 0000000000000000000000000000000000000000..e6cec6706e9668f93a05bf2293db25f40418cea1 --- /dev/null +++ b/docker/services/backend/obswarden.conf @@ -0,0 +1,11 @@ +[program:obswarden] +command=/usr/lib/obs/server/bs_warden +directory=/usr/lib/obs/server/ +stdout_logfile=/srv/obs/log/warden.log +redirect_stderr=true +autostart=True +priority=1 +stopsignal=KILL +killasgroup=true +stopasgroup=true + diff --git a/docker/services/worker/obsworker.conf b/docker/services/worker/obsworker.conf new file mode 100644 index 0000000000000000000000000000000000000000..1129707f4e11a0ee3879fa0ca1c0d56049fa2ed6 --- /dev/null +++ b/docker/services/worker/obsworker.conf @@ -0,0 +1,13 @@ +[program:obsworker] +command=%(ENV_OBS_WORKER_PATH)s/bs_worker --hardstatus --root /var/cache/build/root_%(process_num)d --statedir /var/cache/build/state_%(process_num)d --id %(ENV_OBS_WORKER_NAME)s:%(process_num)d %(ENV_OBS_WORKER_OPT)s +process_name=%(program_name)s_%(process_num)d +directory=%(ENV_OBS_WORKER_PATH)s +stdout_logfile=/srv/obs/log/worker_%(process_num)d.log +redirect_stderr=true +autostart=True +priority=1 +stopsignal=KILL +killasgroup=true +stopasgroup=true +numprocs=%(ENV_OBS_WORKER_INSTANCES)s +numprocs_start=1 diff --git a/docker/worker-docker-entrypoint.sh b/docker/worker-docker-entrypoint.sh new file mode 100755 index 0000000000000000000000000000000000000000..47027e23aff432c7838ebb9857fe8577967363d1 --- /dev/null +++ b/docker/worker-docker-entrypoint.sh @@ -0,0 +1,72 @@ +#!/bin/sh -ex + +obsrundir=/run/obs +workerdir=/var/cache/build +workerbootdir="$workerdir/boot" +obslogdir=/var/log/obs + +: mkdir -p "$obsrundir" + +: ${OBS_REPO_SERVERS:=obs-server:5252} + +repo_param= +for i in $OBS_REPO_SERVERS +do + repo_param="$REPO_PARAM --reposerver http://$i" + WORKER_CODE="http://$i" +done + +: ${OBS_WORKER_NICE_LEVEL:=18} + +OBS_WORKER_OPT="--hardstatus $repo_param ${OBS_WORKER_JOBS:+--jobs $OBS_WORKER_JOBS}\ + ${OBS_WORKER_CLEANUP_CHROOT:+--cleanup-chroot}\ + ${OBS_WORKER_WIPE_AFTER_BUILD:+--wipeafterbuild}\ + ${OBS_SRC_SERVER:+--srcserver $OBS_SRC_SERVER}\ + ${OBS_ARCH:+--arch $OBS_ARCH} ${OBS_WORKER_OPT}" + +export OBS_WORKER_OPT + +: ${OBS_WORKER_NAME:=$(hostname)} + +export OBS_WORKER_NAME + +: ${OBS_WORKER_INSTANCES:=$(nproc)} + +export OBS_WORKER_INSTANCES + +OBS_WORKER_PATH=/usr/lib/obs/server + +update_worker() { + echo "Fetching initial worker code from $WORKER_CODE/getworkercode" + mkdir -p "$workerbootdir" + cd "$workerbootdir" + for retry in $(seq 10) + do + if curl -sS "$WORKER_CODE/getworkercode" | cpio --extract + then + ln -sfn . XML + chmod 755 bs_worker + return 0 + fi + # we need to wait for rep server maybe + echo "WARNING: Could not reach rep server $WORKER_CODE. Trying again." >&2 + sleep 10 + done + echo "ERROR: Unable to reach rep server $WORKER_CODE!" >&2 + return 1 +} + +if [ -n "$WORKER_CODE" ] +then + update_worker + OBS_WORKER_PATH="$workerbootdir" +fi + +export OBS_WORKER_PATH + +for i in $(seq 1 $OBS_WORKER_INSTANCES) +do + mkdir -p $workerdir/root_$i $workerdir/state_$i +done + +nice -n "$OBS_WORKER_NICE_LEVEL" /usr/bin/supervisord -n diff --git a/src/api/config/environments/production.rb b/src/api/config/environments/production.rb index bf5bc55254076e42c2acb015f24d15cb566f2357..72d510d2097ed7313c1255b9ecd279830e434434 100644 --- a/src/api/config/environments/production.rb +++ b/src/api/config/environments/production.rb @@ -37,7 +37,7 @@ OBSApi::Application.configure do config.action_controller.perform_caching = true # Disable Rails's static asset server (Apache or nginx will already do this) - config.public_file_server.enabled = false + config.public_file_server.enabled = true # Compress JavaScripts and CSS config.assets.compress = true diff --git a/src/api/config/options.yml.example b/src/api/config/options.yml.example index bc98f94f9801f532b1b75db8bcf970e5ca6b26d8..1a13f782fceccf5a9854ab25008b06a3fcd18d9a 100644 --- a/src/api/config/options.yml.example +++ b/src/api/config/options.yml.example @@ -199,6 +199,7 @@ default: &default production: <<: *default + memcached_host: cache test: <<: *default @@ -209,4 +210,3 @@ development: <<: *default source_host: backend memcached_host: cache - diff --git a/src/api/db/migrate/20191011000000_create_allowbuilddeps.rb b/src/api/db/migrate/20191011000000_create_allowbuilddeps.rb index f4de5797495a1b326eb13a54679e85fc61ccb052..acf2f28a567163a4e748d8e248441eba9c184df5 100644 --- a/src/api/db/migrate/20191011000000_create_allowbuilddeps.rb +++ b/src/api/db/migrate/20191011000000_create_allowbuilddeps.rb @@ -1,4 +1,4 @@ -class CreateAllowbuilddeps < ActiveRecord::Migration +class CreateAllowbuilddeps < ActiveRecord::Migration[4.2] def self.up create_table :allowbuilddeps do |t| t.integer :db_project_id, :null => false