From 15f2c97dc6691c455c1784d0347ce70dd290233d Mon Sep 17 00:00:00 2001
From: Sjoerd Simons <sjoerd@collabora.com>
Date: Sat, 18 Jun 2022 19:58:09 +0200
Subject: [PATCH] Use parallel gzip instead of gzip for compression
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

golangs compress/gzip isn't a parallel implementation, so it's quite a
bit slower on most modern servers then pgzip. The below benchmark
run shows that publishing a debian bullseye mirror snapshot (amd64, arm64,
armhf, source) shows a gain of about 35% in publishing time (when skipping
bz2 using MR #1081)

```
 hyperfine -w 1 -m 3 -L aptly aptly-nobz2,aptly-nobz2-pgzip -p "{aptly} -config aptly.conf publish drop bullseye || true" "{aptly} -config aptly.conf  publish snapshot --skip-bz2=true --skip-contents --skip-signing bullseye"
Benchmark 1: aptly-nobz2 -config aptly.conf  publish snapshot --skip-bz2=true --skip-contents --skip-signing bullseye
  Time (mean ± σ):     35.548 s ±  0.378 s    [User: 39.465 s, System: 10.046 s]
  Range (min … max):   35.149 s … 35.902 s    3 runs

Benchmark 2: aptly-nobz2-pgzip -config aptly.conf  publish snapshot --skip-bz2=true --skip-contents --skip-signing bullseye
  Time (mean ± σ):     26.592 s ±  0.069 s    [User: 42.207 s, System: 9.676 s]
  Range (min … max):   26.521 s … 26.660 s    3 runs

Summary
  'aptly-nobz2-pgzip -config aptly.conf  publish snapshot --skip-bz2=true --skip-contents --skip-signing bullseye' ran
    1.34 ± 0.01 times faster than 'aptly-nobz2 -config aptly.conf  publish snapshot --skip-bz2=true --skip-contents --skip-signing bullseye'
```

Signed-off-by: Sjoerd Simons <sjoerd@collabora.com>
Upstream MR: https://github.com/aptly-dev/aptly/pull/1082
---
 go.mod            | 1 +
 go.sum            | 2 ++
 utils/compress.go | 5 +++--
 3 files changed, 6 insertions(+), 2 deletions(-)

diff --git a/go.mod b/go.mod
index 55b80ce2..ca3a2f6f 100644
--- a/go.mod
+++ b/go.mod
@@ -17,6 +17,7 @@ require (
 	github.com/jlaffaye/ftp v0.0.0-20180404123514-2403248fa8cc // indirect
 	github.com/kjk/lzma v0.0.0-20161016003348-3fd93898850d
 	github.com/klauspost/compress v1.13.6
+	github.com/klauspost/pgzip v1.2.5
 	github.com/mattn/go-colorable v0.1.2 // indirect
 	github.com/mattn/go-isatty v0.0.14 // indirect
 	github.com/mattn/go-runewidth v0.0.2 // indirect
diff --git a/go.sum b/go.sum
index 82ee111d..434cda23 100644
--- a/go.sum
+++ b/go.sum
@@ -201,6 +201,8 @@ github.com/kjk/lzma v0.0.0-20161016003348-3fd93898850d h1:RnWZeH8N8KXfbwMTex/KKM
 github.com/kjk/lzma v0.0.0-20161016003348-3fd93898850d/go.mod h1:phT/jsRPBAEqjAibu1BurrabCBNTYiVI+zbmyCZJY6Q=
 github.com/klauspost/compress v1.13.6 h1:P76CopJELS0TiO2mebmnzgWaajssP/EszplttgQxcgc=
 github.com/klauspost/compress v1.13.6/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk=
+github.com/klauspost/pgzip v1.2.5 h1:qnWYvvKqedOF2ulHpMG72XQol4ILEJ8k2wwRl/Km8oE=
+github.com/klauspost/pgzip v1.2.5/go.mod h1:Ch1tH69qFZu15pkjo5kYi6mth2Zzwzt50oCQKQE9RUs=
 github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
 github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
 github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc=
diff --git a/utils/compress.go b/utils/compress.go
index aef8f186..a4521b7e 100644
--- a/utils/compress.go
+++ b/utils/compress.go
@@ -1,10 +1,11 @@
 package utils
 
 import (
-	"compress/gzip"
 	"io"
 	"os"
 	"os/exec"
+
+	"github.com/klauspost/pgzip"
 )
 
 // CompressFile compresses file specified by source to .gz & .bz2
@@ -19,7 +20,7 @@ func CompressFile(source *os.File, onlyGzip bool) error {
 	}
 	defer gzFile.Close()
 
-	gzWriter := gzip.NewWriter(gzFile)
+	gzWriter := pgzip.NewWriter(gzFile)
 	defer gzWriter.Close()
 
 	source.Seek(0, 0)
-- 
GitLab