ci: improve artifacts size and kernel fetches

.gitlab-image-tags.yml

 variables:
-  DEBIAN_TAG: '2023-12-22-virtme'
+  DEBIAN_TAG: '2024-01-12-preclone'

ci/ci-build-linux.sh

@@ -2,12 +2,19 @@
 
 set -x
 
-GIT_REPO=${TEST_GIT_REPOSITORY:-"https://git.linuxtv.org/media_tree.git"}
-GIT_BRANCH=${TEST_GIT_BRANCH:-"master"}
+source ci/linux-common.sh
 
-echo "Building branch ${GIT_BRANCH} from git repo ${GIT_REPO}"
-git clone --depth=1 --branch=${GIT_BRANCH} ${GIT_REPO} /opt/build/media-git
+# If the remote is not present, add and fetch it
+if [ -z "${TEST_REMOTE}" ]; then
+	TEST_REMOTE="test"
+	git remote add "test" "${GIT_REPO}"
+fi
+
+# Fetch the latest version of the branch
+git -C /opt/build/media-git fetch --depth=1 ${TEST_REMOTE} ${GIT_BRANCH}
+git -C /opt/build/media-git checkout ${TEST_REMOTE}/${GIT_BRANCH}
 
+# Apply the patchset if needed
 if [ ! -z "${TEST_PATCHSET_ID}" ]; then
 	echo "Getting patchset from Message-id: ${TEST_PATCHSET_ID}"
 	pushd /opt/build/media-git
@@ -17,7 +24,10 @@ if [ ! -z "${TEST_PATCHSET_ID}" ]; then
 	popd
 fi
 
-tar --zstd -cf media-git.tar.zstd /opt/build/media-git
+# Compress the source tree to use as artifact if the remote is not in the docker image
+if [ "${TEST_REMOTE}" == "test" ]; then
+	tar --zstd --exclude-vcs --exclude-vcs-ignore -cf media-git.tar.zstd /opt/build/media-git
+fi
 
 cp /opt/env.sh ./env.sh
 
@@ -27,5 +37,5 @@ echo "---------------"
 
 ./build.sh -virtme -build-only none
 
-tar --zstd -cf virtme.tar.zstd /opt/build/trees/virtme
+tar --zstd -cf virtme.tar.zstd /opt/build/trees/virtme/media-git/vmlinux /opt/build/trees/virtme/media-git/virtme-modules
 

ci/ci-run-tests.sh

@@ -2,7 +2,7 @@
 
 set -x
 
-tar -xf media-git.tar.zstd -C /
+source ci/linux-common.sh
 
 cp /opt/env.sh ./env.sh
 
@@ -15,6 +15,18 @@ if [ "${1}" = "virtme" ]; then
 	tar -xf virtme.tar.zstd -C /
 	./build.sh -virtme -virtme-quick none
 else
+	# If the remote is not present, get it from the artifact
+	if [ -z "${TEST_REMOTE}" ]; then
+		TEST_REMOTE="test"
+		git -C /opt/build/media-git remote add "test" "${GIT_REPO}"
+
+		tar -xf media-git.tar.zstd -C /
+	else
+		# Fetch the latest version of the repository
+		git -C /opt/build/media-git fetch --depth=1 ${TEST_REMOTE} ${GIT_BRANCH}
+		git -C /opt/build/media-git checkout ${TEST_REMOTE}/${GIT_BRANCH}
+	fi
+
 	./build.sh -test all
 fi
 

ci/docker-repos.sh

+
+GIT_REPOS="\
+	https://git.linuxtv.org/media_tree.git,master,media_tree \
+	https://git.linuxtv.org/media_stage.git,master,media_stage \
+	git://git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git,master,torvalds \
+	https://gitlab.collabora.com/sebastianfricke/linux.git,codec_staging,codec_staging \
+"
+

ci/docker_prepare.sh

@@ -30,6 +30,22 @@ git checkout v1.18
 popd
 rm -fr virtme-ng
 
+# Fetch main linux repositories
+source ci/docker-repos.sh
+
+mkdir -p /opt/build/media-git
+git init /opt/build/media-git
+
+for repo in ${GIT_REPOS}; do
+	REMOTE="$(echo $repo | cut -d, -f1)"
+	BRANCH="$(echo $repo | cut -d, -f2)"
+	NAME="$(echo $repo | cut -d, -f3)"
+
+	echo "Adding branch ${BRANCH} from git repo ${REMOTE} as ${NAME}"
+	git -C /opt/build/media-git remote add ${NAME} ${REMOTE}
+	git -C /opt/build/media-git fetch --depth=1 ${NAME} ${BRANCH}
+done
+
 cat <<EOF >> env.sh 
 #!/bin/bash
 cross_data=/opt/cross

ci/linux-common.sh

+
+source ci/docker-repos.sh
+
+GIT_REPO=${TEST_GIT_REPOSITORY:-"https://git.linuxtv.org/media_tree.git"}
+GIT_BRANCH=${TEST_GIT_BRANCH:-"master"}
+
+# Check if the remote is already present
+for repo in ${GIT_REPOS}; do
+	REMOTE="$(echo $repo | cut -d, -f1)"
+	BRANCH="$(echo $repo | cut -d, -f2)"
+	NAME="$(echo $repo | cut -d, -f3)"
+	
+	if [ "${REMOTE}" == "${GIT_REPO}" ] || [ "${NAME}" == "${GIT_REPO}" ]; then
+		TEST_REMOTE="${NAME}"
+	fi
+done
+
+