From ccdd3d0d7db801f78aca0ac3db0374b6c3c2161c Mon Sep 17 00:00:00 2001
From: Sjoerd Simons <sjoerd@collabora.com>
Date: Sat, 12 Nov 2022 20:25:37 +0100
Subject: [PATCH 1/4] Run most job on an elecronica specific runner

---
 .gitlab-ci.yml | 8 +-------
 1 file changed, 1 insertion(+), 7 deletions(-)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 857202f..c24fcac 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -4,7 +4,7 @@ variables:
 
 default:
   tags:
-    - kvm
+    - electronica2022
 
 image: $DOCKER_IMAGE
 
@@ -31,8 +31,6 @@ stages:
       - out/*
 
 .kernel:
-  tags:
-    - lightweight
   variables: &kernel_variables
     DEBIAN_FRONTEND: noninteractive
     GIT_STRATEGY: none
@@ -79,8 +77,6 @@ rootfs amd64:
 
 generate arm64 tests:
   stage: generate
-  tags:
-    - lightweight
   script:
     - ./generate_lava_job.py
         -e DEVICE_TYPE=bcm2711-rpi-4-b
@@ -104,8 +100,6 @@ generate arm64 tests:
 
 generate amd64 test:
   stage: generate
-  tags:
-    - lightweight
   script:
     - ./generate_lava_job.py
         -e DEVICE_TYPE=aaeon-UPN-EHLX4RE-A10-0864
-- 
GitLab


From 3195258e4d8f3e211e8c6d8adab68453842c06ac Mon Sep 17 00:00:00 2001
From: Sjoerd Simons <sjoerd@collabora.com>
Date: Sat, 12 Nov 2022 21:07:53 +0100
Subject: [PATCH 2/4] setup kernel in ospack build as well

---
 .gitlab-ci.yml | 46 ++++------------------------------------------
 ospack.yaml    |  9 ++++-----
 testjob.jinja2 | 12 +++---------
 3 files changed, 11 insertions(+), 56 deletions(-)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index c24fcac..bdf6671 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -26,50 +26,17 @@ stages:
     - pushd out && debos -t architecture:${ARCHITECTURE} ../ospack.yaml && popd
     - 'echo IMAGE_JOB_ID: \"${CI_JOB_ID}\" | tee -a out/image-build-job.yaml'
     - 'echo ARCHITECTURE: \"${ARCHITECTURE}\" | tee -a out/image-build-job.yaml'
+    - 'echo VMLINUZ: out/vmlinuz* | tee -a out/image-build-job.yaml'
+    - 'echo INITRD: out/initrd* | tee -a out/image-build-job.yaml'
   artifacts:
     paths:
       - out/*
 
-.kernel:
-  variables: &kernel_variables
-    DEBIAN_FRONTEND: noninteractive
-    GIT_STRATEGY: none
-  stage: build
-  script:
-    - mkdir dist
-    - "chdist -d dist create apertis https://repositories.apertis.org/apertis/ v2023pre target"
-    - cp /etc/apt/trusted.gpg.d/apertis-archive-keyring.gpg dist/apertis/etc/apt/trusted.gpg.d/
-    - "chdist -d dist -a ${ARCHITECTURE} apt apertis update"
-    - 'chdist -d dist -a ${ARCHITECTURE} apt apertis download "linux-image-*-${ARCHITECTURE}"'
-    - dpkg -x *.deb .
-    - mkdir -p out
-    - cp -v boot/vmlinuz* out
-    - if [ -d boot/dtbs ]  ; then cp -v boot/dtbs/*/*/*.dtb out ; fi
-    - depmod -b $(pwd)  $(basename $(echo lib/modules/*))
-    - tar cvzf out/modules.tar.gz lib/modules
-    - 'echo VMLINUZ: out/vmlinuz* | tee -a out/kernel-build-job.yaml'
-    - 'echo KERNEL_JOB_ID: \"${CI_JOB_ID}\" | tee -a out/kernel-build-job.yaml'
-  artifacts:
-    paths:
-      - out/*
-
-kernel arm64:
-  extends: .kernel
-  variables:
-    ARCHITECTURE: arm64
-    <<: *kernel_variables
-
 rootfs arm64:
   extends: .rootfs
   variables:
     ARCHITECTURE: arm64
 
-kernel amd64:
-  extends: .kernel
-  variables:
-    ARCHITECTURE: amd64
-    <<: *kernel_variables
-
 rootfs amd64:
   extends: .rootfs
   variables:
@@ -80,15 +47,13 @@ generate arm64 tests:
   script:
     - ./generate_lava_job.py
         -e DEVICE_TYPE=bcm2711-rpi-4-b
-        -e DTB=bcm2711-rpi-4-b.dtb
+        -e DTB=broadcom/bcm2711-rpi-4-b.dtb
         -e BOOT_METHOD=u-boot
-        --env-file out/kernel-build-job.yaml
         --env-file out/image-build-job.yaml > rpi.yaml
     - ./generate_lava_job.py
         -e DEVICE_TYPE=rk3399-roc-pc
-        -e DTB=rk3399-roc-pc.dtb
+        -e DTB=rockchip/rk3399-roc-pc.dtb
         -e BOOT_METHOD=u-boot
-        --env-file out/kernel-build-job.yaml
         --env-file out/image-build-job.yaml > renegade-elite.yaml
   artifacts:
     paths:
@@ -96,7 +61,6 @@ generate arm64 tests:
       - "renegade-elite.yaml"
   needs:
     - "rootfs arm64"
-    - "kernel arm64"
 
 generate amd64 test:
   stage: generate
@@ -104,14 +68,12 @@ generate amd64 test:
     - ./generate_lava_job.py
         -e DEVICE_TYPE=aaeon-UPN-EHLX4RE-A10-0864
         -e BOOT_METHOD=grub
-        --env-file out/kernel-build-job.yaml
         --env-file out/image-build-job.yaml > up-squared.yaml
   artifacts:
     paths:
       - "up-squared.yaml"
   needs:
     - "rootfs amd64"
-    - "kernel amd64"
 
 .lavatest:
   stage: lava
diff --git a/ospack.yaml b/ospack.yaml
index 777d532..94ae9ba 100644
--- a/ospack.yaml
+++ b/ospack.yaml
@@ -118,14 +118,13 @@ actions:
     description: "Disable systemd-timesyncd service by default"
     command: systemctl disable systemd-timesyncd
 
-  - action: run
-    description: Create a base kernel-less initramfs
-    chroot: true
-    command: mkinitramfs -o /boot/initramfs
+  - action: apt
+    packages:
+      - linux-image-{{ $architecture }}
 
   - action: run
     chroot: false
-    command: mv ${ROOTDIR}/boot/initramfs ${ARTIFACTDIR}/{{ $architecture }}-initramfs.gz
+    command: mv ${ROOTDIR}/boot/* ${ARTIFACTDIR}/
 
 {{- if eq $pack "true" }}
   - action: run
diff --git a/testjob.jinja2 b/testjob.jinja2
index 3a08403..e06e5eb 100644
--- a/testjob.jinja2
+++ b/testjob.jinja2
@@ -10,7 +10,6 @@ priority: high
 visibility: public
 
 {%- set gitlab_artifacts = '{{job.CI_API_V4_URL}}/projects/{{job.CI_PROJECT_ID}}/jobs/' -%}
-{%- set kernel_artifacts = gitlab_artifacts ~ env["KERNEL_JOB_ID"] ~ '/artifacts' -%}
 {%- set image_artifacts = gitlab_artifacts ~ env["IMAGE_JOB_ID"] ~ '/artifacts' %}
 
 actions:
@@ -19,23 +18,18 @@ actions:
         minutes: 2
       to: tftp
       kernel:
-        url: "{{kernel_artifacts}}/{{env['VMLINUZ']}}"
+        url: "{{image_artifacts}}/{{env['VMLINUZ']}}"
         type: image
         headers:
            JOB-TOKEN: {{ '{{job.CI_JOB_TOKEN}}' }}
-      modules:
-        url: "{{kernel_artifacts}}/out/modules.tar.gz"
-        compression: gz
-        headers:
-           JOB-TOKEN: {{ '{{job.CI_JOB_TOKEN}}' }}
 {%- if env['DTB'] is defined %}
       dtb:
-        url: "{{kernel_artifacts}}/out/{{env['DTB']}}"
+        url: "{{image_artifacts}}/out/dtbs/{{env['DTB']}}"
         headers:
            JOB-TOKEN: {{ '{{job.CI_JOB_TOKEN}}' }}
 {%- endif %}
       ramdisk:
-        url: "{{image_artifacts}}/out/{{env['ARCHITECTURE']}}-initramfs.gz"
+        url: "{{image_artifacts}}/{{env['INITRD']}}"
         compression: gz
         headers:
            JOB-TOKEN: {{ '{{job.CI_JOB_TOKEN}}' }}
-- 
GitLab


From 7a25f7d91dbedc406abe56df1cd71891b6a388f4 Mon Sep 17 00:00:00 2001
From: Sjoerd Simons <sjoerd@collabora.com>
Date: Sat, 12 Nov 2022 21:30:51 +0100
Subject: [PATCH 3/4] Split recipe in a base (cached) and a final recipe

To speedup builds use a base recipe that rarely changes with a main one
on top for the final additions; Also cache the base via gitlab to make
things really move quite fast
---
 .gitlab-ci.yml                  | 41 +++++++++++++++++++++++++--------
 ospack.yaml => base-rootfs.yaml | 19 ++-------------
 rootfs.yaml                     | 34 +++++++++++++++++++++++++++
 testjob.jinja2                  |  4 ++--
 4 files changed, 69 insertions(+), 29 deletions(-)
 rename ospack.yaml => base-rootfs.yaml (85%)
 create mode 100644 rootfs.yaml

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index bdf6671..61a6a1a 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -22,15 +22,36 @@ stages:
     - chmod -R og-w .
     - chmod -R a-w overlays/sudo-fqdn
   script:
-    - mkdir out
-    - pushd out && debos -t architecture:${ARCHITECTURE} ../ospack.yaml && popd
-    - 'echo IMAGE_JOB_ID: \"${CI_JOB_ID}\" | tee -a out/image-build-job.yaml'
-    - 'echo ARCHITECTURE: \"${ARCHITECTURE}\" | tee -a out/image-build-job.yaml'
-    - 'echo VMLINUZ: out/vmlinuz* | tee -a out/image-build-job.yaml'
-    - 'echo INITRD: out/initrd* | tee -a out/image-build-job.yaml'
+    - |
+      BASE=base-rootfs-${ARCHITECTURE}.tar.gz
+      if [ -f ${BASE} ] ; then
+        echo "Found cached base rootfs"
+        DEBOS_OPTIONS="-t base_rootfs:${BASE}"
+      fi
+    - "debos ${DEBOS_OPTIONS} -t architecture:${ARCHITECTURE} rootfs.yaml"
+    - 'echo IMAGE_JOB_ID: \"${CI_JOB_ID}\" | tee -a image-build-job.yaml'
+    - 'echo ARCHITECTURE: \"${ARCHITECTURE}\" | tee -a image-build-job.yaml'
+    - 'echo VMLINUZ: vmlinuz* | tee -a image-build-job.yaml'
+    - 'echo INITRD: initrd* | tee -a image-build-job.yaml'
+    - |
+      if [ -d dtbs ] ; then
+        echo DTB_BASE: dtbs/* | tee -a image-build-job.yaml
+      fi
   artifacts:
     paths:
-      - out/*
+      - rootfs*
+      - vmlinuz*
+      - config*
+      - initrd*
+      - dtbs
+      - image-build-job.yaml
+  cache:
+    key:
+      files:
+        - base-rootfs.yaml
+      prefix: ${ARCHITECTURE}
+    paths:
+      - base-rootfs-${ARCHITECTURE}.*
 
 rootfs arm64:
   extends: .rootfs
@@ -49,12 +70,12 @@ generate arm64 tests:
         -e DEVICE_TYPE=bcm2711-rpi-4-b
         -e DTB=broadcom/bcm2711-rpi-4-b.dtb
         -e BOOT_METHOD=u-boot
-        --env-file out/image-build-job.yaml > rpi.yaml
+        --env-file image-build-job.yaml > rpi.yaml
     - ./generate_lava_job.py
         -e DEVICE_TYPE=rk3399-roc-pc
         -e DTB=rockchip/rk3399-roc-pc.dtb
         -e BOOT_METHOD=u-boot
-        --env-file out/image-build-job.yaml > renegade-elite.yaml
+        --env-file image-build-job.yaml > renegade-elite.yaml
   artifacts:
     paths:
       - "rpi.yaml"
@@ -68,7 +89,7 @@ generate amd64 test:
     - ./generate_lava_job.py
         -e DEVICE_TYPE=aaeon-UPN-EHLX4RE-A10-0864
         -e BOOT_METHOD=grub
-        --env-file out/image-build-job.yaml > up-squared.yaml
+        --env-file image-build-job.yaml > up-squared.yaml
   artifacts:
     paths:
       - "up-squared.yaml"
diff --git a/ospack.yaml b/base-rootfs.yaml
similarity index 85%
rename from ospack.yaml
rename to base-rootfs.yaml
index 94ae9ba..acf3828 100644
--- a/ospack.yaml
+++ b/base-rootfs.yaml
@@ -4,7 +4,7 @@
 {{- $suite := or .suite "v2023pre" -}}
 {{- $timestamp := or .timestamp "" -}}
 {{- $snapshot := or .snapshot "" -}}
-{{- $ospack := or .ospack (printf "rootfs-%s" $architecture) -}}
+{{- $rootfs := or .rootfs (printf "base-rootfs-%s" $architecture) -}}
 {{- $pack := or .pack "true" -}}
 {{- $stable := or .stable "" -}}
 {{- $osname := or .osname "apertis" -}}
@@ -122,23 +122,8 @@ actions:
     packages:
       - linux-image-{{ $architecture }}
 
-  - action: run
-    chroot: false
-    command: mv ${ROOTDIR}/boot/* ${ARTIFACTDIR}/
-
 {{- if eq $pack "true" }}
-  - action: run
-    description: "Save installed package status"
-    chroot: false
-    command: gzip -c "${ROOTDIR}/var/lib/dpkg/status" > "${ARTIFACTDIR}/{{ $ospack }}.pkglist.gz"
-
-  - action: run
-    description: List files on {{ $ospack }}
-    chroot: false
-    script: scripts/list-files "$ROOTDIR" | gzip > "${ARTIFACTDIR}/{{ $ospack }}.filelist.gz"
-
-
   - action: pack
     compression: gz
-    file: {{ $ospack }}.tar.gz
+    file: {{ $rootfs }}.tar.gz
 {{- end }}
diff --git a/rootfs.yaml b/rootfs.yaml
new file mode 100644
index 0000000..3cd9d59
--- /dev/null
+++ b/rootfs.yaml
@@ -0,0 +1,34 @@
+{{- $architecture := or .architecture "amd64" }}
+{{- $rootfs := or .rootfs (printf "rootfs-%s" $architecture) -}}
+{{- $base_rootfs := or .base_rootfs "" -}}
+{{- $osname := or .osname "apertis" -}}
+
+architecture: {{ $architecture }}
+
+actions:
+{{- if eq $base_rootfs "" }}
+  - action: recipe
+    recipe: base-rootfs.yaml
+{{- else }}
+  - action: unpack
+    file: {{ $base_rootfs }}
+{{- end }}
+
+  - action: run
+    chroot: false
+    command: mv ${ROOTDIR}/boot/* ${ARTIFACTDIR}/
+
+  - action: run
+    description: "Save installed package status"
+    chroot: false
+    command: gzip -c "${ROOTDIR}/var/lib/dpkg/status" > "${ARTIFACTDIR}/{{ $rootfs }}.pkglist.gz"
+
+  - action: run
+    description: List files on {{ $rootfs }}
+    chroot: false
+    script: scripts/list-files "$ROOTDIR" | gzip > "${ARTIFACTDIR}/{{ $rootfs }}.filelist.gz"
+
+
+  - action: pack
+    compression: gz
+    file: {{ $rootfs }}.tar.gz
diff --git a/testjob.jinja2 b/testjob.jinja2
index e06e5eb..77c6bde 100644
--- a/testjob.jinja2
+++ b/testjob.jinja2
@@ -24,7 +24,7 @@ actions:
            JOB-TOKEN: {{ '{{job.CI_JOB_TOKEN}}' }}
 {%- if env['DTB'] is defined %}
       dtb:
-        url: "{{image_artifacts}}/out/dtbs/{{env['DTB']}}"
+        url: "{{image_artifacts}}/{{env['DTB_BASE']}}/{{env['DTB']}}"
         headers:
            JOB-TOKEN: {{ '{{job.CI_JOB_TOKEN}}' }}
 {%- endif %}
@@ -34,7 +34,7 @@ actions:
         headers:
            JOB-TOKEN: {{ '{{job.CI_JOB_TOKEN}}' }}
       nfsrootfs:
-        url: "{{image_artifacts}}/out/rootfs-{{env['ARCHITECTURE']}}.tar.gz"
+        url: "{{image_artifacts}}/rootfs-{{env['ARCHITECTURE']}}.tar.gz"
         compression: gz
         headers:
            JOB-TOKEN: {{ '{{job.CI_JOB_TOKEN}}' }}
-- 
GitLab


From 339004cbe06a35b8548bb1fef5560615ecfa76d9 Mon Sep 17 00:00:00 2001
From: Sjoerd Simons <sjoerd@collabora.com>
Date: Sat, 12 Nov 2022 23:55:15 +0100
Subject: [PATCH 4/4] Use pigz as gzip

Seems the apertis image builder, probably accidentally, use busybox-gzip
as gzip which is horrendously slow. Normal gzip would be a few times
faster and pigz is even faster; So for pigz to be gzip instead
---
 .gitlab-ci.yml | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 61a6a1a..e40aefc 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -21,6 +21,7 @@ stages:
   before_script: &gitlab_permissions_fixup
     - chmod -R og-w .
     - chmod -R a-w overlays/sudo-fqdn
+    - ln -sf  /usr/bin/pigz  /bin/gzip
   script:
     - |
       BASE=base-rootfs-${ARCHITECTURE}.tar.gz
@@ -28,7 +29,7 @@ stages:
         echo "Found cached base rootfs"
         DEBOS_OPTIONS="-t base_rootfs:${BASE}"
       fi
-    - "debos ${DEBOS_OPTIONS} -t architecture:${ARCHITECTURE} rootfs.yaml"
+    - "debos ${DEBOS_OPTIONS} -c $(nproc) -t architecture:${ARCHITECTURE} rootfs.yaml"
     - 'echo IMAGE_JOB_ID: \"${CI_JOB_ID}\" | tee -a image-build-job.yaml'
     - 'echo ARCHITECTURE: \"${ARCHITECTURE}\" | tee -a image-build-job.yaml'
     - 'echo VMLINUZ: vmlinuz* | tee -a image-build-job.yaml'
-- 
GitLab