From b3eca700e7b23ad12374d1ee2d28eb110fecb400 Mon Sep 17 00:00:00 2001 From: Ryan Gonzalez <ryan.gonzalez@collabora.com> Date: Tue, 5 Sep 2023 15:20:47 -0500 Subject: [PATCH 1/6] Shut down cleanly when 'api serve' is interrupted This will properly close the db and, more particularly, flush out any profile files being written. Otherwise, they can end up empty or truncated. https://github.com/aptly-dev/aptly/pull/1219 Signed-off-by: Ryan Gonzalez <ryan.gonzalez@collabora.com> --- cmd/api_serve.go | 29 +++++++++++++++++++++-------- 1 file changed, 21 insertions(+), 8 deletions(-) diff --git a/cmd/api_serve.go b/cmd/api_serve.go index 7c19d2ef..9b6d3402 100644 --- a/cmd/api_serve.go +++ b/cmd/api_serve.go @@ -1,11 +1,15 @@ package cmd import ( + stdcontext "context" + "errors" "fmt" "net" "net/http" "net/url" "os" + "os/signal" + "syscall" "github.com/aptly-dev/aptly/api" "github.com/aptly-dev/aptly/systemd/activation" @@ -55,6 +59,17 @@ func aptlyAPIServe(cmd *commander.Command, args []string) error { listen := context.Flags().Lookup("listen").Value.String() fmt.Printf("\nStarting web server at: %s (press Ctrl+C to quit)...\n", listen) + server := http.Server{Handler: api.Router(context)} + + sigchan := make(chan os.Signal, 1) + signal.Notify(sigchan, syscall.SIGINT, syscall.SIGTERM) + go (func() { + if _, ok := <-sigchan; ok { + server.Shutdown(stdcontext.Background()) + } + })() + defer close(sigchan) + listenURL, err := url.Parse(listen) if err == nil && listenURL.Scheme == "unix" { file := listenURL.Path @@ -67,19 +82,17 @@ func aptlyAPIServe(cmd *commander.Command, args []string) error { } defer listener.Close() - err = http.Serve(listener, api.Router(context)) - if err != nil { - return fmt.Errorf("unable to serve: %s", err) - } - return nil + err = server.Serve(listener) + } else { + server.Addr = listen + err = server.ListenAndServe() } - err = http.ListenAndServe(listen, api.Router(context)) - if err != nil { + if err != nil && !errors.Is(err, http.ErrServerClosed) { return fmt.Errorf("unable to serve: %s", err) } - return err + return nil } func makeCmdAPIServe() *commander.Command { -- GitLab From 6a212a9ce0cdc4b1ca5dbd0475e89cca457681e9 Mon Sep 17 00:00:00 2001 From: Ryan Gonzalez <ryan.gonzalez@collabora.com> Date: Tue, 5 Sep 2023 15:22:24 -0500 Subject: [PATCH 2/6] Use github.com/saracen/walker for file walk operations In some local tests w/ a slowed down filesystem, this massively cut down on the time to clean up a repository by ~3x, bringing a total 'publish update' time from ~16s to ~13s. https://github.com/aptly-dev/aptly/pull/1222 Signed-off-by: Ryan Gonzalez <ryan.gonzalez@collabora.com> --- api/files.go | 20 ++++++++++---------- deb/changes.go | 11 +++++++---- deb/import.go | 14 ++++++++++---- files/package_pool.go | 11 +++++++---- files/public.go | 12 ++++++++---- go.mod | 1 + go.sum | 6 ++++-- 7 files changed, 47 insertions(+), 28 deletions(-) diff --git a/api/files.go b/api/files.go index 04242cc9..77a04042 100644 --- a/api/files.go +++ b/api/files.go @@ -6,8 +6,10 @@ import ( "os" "path/filepath" "strings" + "sync" "github.com/gin-gonic/gin" + "github.com/saracen/walker" ) func verifyPath(path string) bool { @@ -34,17 +36,16 @@ func verifyDir(c *gin.Context) bool { // GET /files func apiFilesListDirs(c *gin.Context) { list := []string{} + listLock := &sync.Mutex{} - err := filepath.Walk(context.UploadPath(), func(path string, info os.FileInfo, err error) error { - if err != nil { - return err - } - + err := walker.Walk(context.UploadPath(), func(path string, info os.FileInfo) error { if path == context.UploadPath() { return nil } if info.IsDir() { + listLock.Lock() + defer listLock.Unlock() list = append(list, filepath.Base(path)) return filepath.SkipDir } @@ -121,17 +122,16 @@ func apiFilesListFiles(c *gin.Context) { } list := []string{} + listLock := &sync.Mutex{} root := filepath.Join(context.UploadPath(), c.Params.ByName("dir")) - err := filepath.Walk(root, func(path string, info os.FileInfo, err error) error { - if err != nil { - return err - } - + err := walker.Walk(root, func(path string, info os.FileInfo) error { if path == root { return nil } + listLock.Lock() + defer listLock.Unlock() list = append(list, filepath.Base(path)) return nil diff --git a/deb/changes.go b/deb/changes.go index 6637034e..996b10a3 100644 --- a/deb/changes.go +++ b/deb/changes.go @@ -8,11 +8,13 @@ import ( "path/filepath" "sort" "strings" + "sync" "text/template" "github.com/aptly-dev/aptly/aptly" "github.com/aptly-dev/aptly/pgp" "github.com/aptly-dev/aptly/utils" + "github.com/saracen/walker" ) // Changes is a result of .changes file parsing @@ -247,6 +249,8 @@ func (c *Changes) GetArchitecture() string { // CollectChangesFiles walks filesystem collecting all .changes files func CollectChangesFiles(locations []string, reporter aptly.ResultReporter) (changesFiles, failedFiles []string) { + changesFilesLock := &sync.Mutex{} + for _, location := range locations { info, err2 := os.Stat(location) if err2 != nil { @@ -255,15 +259,14 @@ func CollectChangesFiles(locations []string, reporter aptly.ResultReporter) (cha continue } if info.IsDir() { - err2 = filepath.Walk(location, func(path string, info os.FileInfo, err3 error) error { - if err3 != nil { - return err3 - } + err2 = walker.Walk(location, func(path string, info os.FileInfo) error { if info.IsDir() { return nil } if strings.HasSuffix(info.Name(), ".changes") { + changesFilesLock.Lock() + defer changesFilesLock.Unlock() changesFiles = append(changesFiles, path) } diff --git a/deb/import.go b/deb/import.go index 53322f8d..8c16b331 100644 --- a/deb/import.go +++ b/deb/import.go @@ -5,14 +5,19 @@ import ( "path/filepath" "sort" "strings" + "sync" "github.com/aptly-dev/aptly/aptly" "github.com/aptly-dev/aptly/pgp" "github.com/aptly-dev/aptly/utils" + "github.com/saracen/walker" ) // CollectPackageFiles walks filesystem collecting all candidates for package files func CollectPackageFiles(locations []string, reporter aptly.ResultReporter) (packageFiles, otherFiles, failedFiles []string) { + packageFilesLock := &sync.Mutex{} + otherFilesLock := &sync.Mutex{} + for _, location := range locations { info, err2 := os.Stat(location) if err2 != nil { @@ -21,18 +26,19 @@ func CollectPackageFiles(locations []string, reporter aptly.ResultReporter) (pac continue } if info.IsDir() { - err2 = filepath.Walk(location, func(path string, info os.FileInfo, err3 error) error { - if err3 != nil { - return err3 - } + err2 = walker.Walk(location, func(path string, info os.FileInfo) error { if info.IsDir() { return nil } if strings.HasSuffix(info.Name(), ".deb") || strings.HasSuffix(info.Name(), ".udeb") || strings.HasSuffix(info.Name(), ".dsc") || strings.HasSuffix(info.Name(), ".ddeb") { + packageFilesLock.Lock() + defer packageFilesLock.Unlock() packageFiles = append(packageFiles, path) } else if strings.HasSuffix(info.Name(), ".buildinfo") { + otherFilesLock.Lock() + defer otherFilesLock.Unlock() otherFiles = append(otherFiles, path) } diff --git a/files/package_pool.go b/files/package_pool.go index 693a01ba..e82a7447 100644 --- a/files/package_pool.go +++ b/files/package_pool.go @@ -5,10 +5,12 @@ import ( "io" "os" "path/filepath" + "sort" "sync" "syscall" "github.com/pborman/uuid" + "github.com/saracen/walker" "github.com/aptly-dev/aptly/aptly" "github.com/aptly-dev/aptly/utils" @@ -97,13 +99,13 @@ func (pool *PackagePool) FilepathList(progress aptly.Progress) ([]string, error) } result := []string{} + resultLock := &sync.Mutex{} for _, dir := range dirs { - err = filepath.Walk(filepath.Join(pool.rootPath, dir.Name()), func(path string, info os.FileInfo, err error) error { - if err != nil { - return err - } + err = walker.Walk(filepath.Join(pool.rootPath, dir.Name()), func(path string, info os.FileInfo) error { if !info.IsDir() { + resultLock.Lock() + defer resultLock.Unlock() result = append(result, path[len(pool.rootPath)+1:]) } return nil @@ -117,6 +119,7 @@ func (pool *PackagePool) FilepathList(progress aptly.Progress) ([]string, error) } } + sort.Strings(result) return result, nil } diff --git a/files/public.go b/files/public.go index 991f5afd..0f349e50 100644 --- a/files/public.go +++ b/files/public.go @@ -5,11 +5,14 @@ import ( "io" "os" "path/filepath" + "sort" "strings" + "sync" "syscall" "github.com/aptly-dev/aptly/aptly" "github.com/aptly-dev/aptly/utils" + "github.com/saracen/walker" ) // PublishedStorage abstract file system with public dirs (published repos) @@ -248,12 +251,12 @@ func (storage *PublishedStorage) LinkFromPool(publishedDirectory, fileName strin func (storage *PublishedStorage) Filelist(prefix string) ([]string, error) { root := filepath.Join(storage.rootPath, prefix) result := []string{} + resultLock := &sync.Mutex{} - err := filepath.Walk(root, func(path string, info os.FileInfo, err error) error { - if err != nil { - return err - } + err := walker.Walk(root, func(path string, info os.FileInfo) error { if !info.IsDir() { + resultLock.Lock() + defer resultLock.Unlock() result = append(result, path[len(root)+1:]) } return nil @@ -264,6 +267,7 @@ func (storage *PublishedStorage) Filelist(prefix string) ([]string, error) { return []string{}, nil } + sort.Strings(result) return result, err } diff --git a/go.mod b/go.mod index 7eda3b1e..3a959a8f 100644 --- a/go.mod +++ b/go.mod @@ -28,6 +28,7 @@ require ( github.com/pkg/errors v0.9.1 github.com/prometheus/client_golang v1.12.1 github.com/rs/zerolog v1.28.0 + github.com/saracen/walker v0.1.3 github.com/smartystreets/gunit v1.0.4 // indirect github.com/smira/commander v0.0.0-20140515201010-f408b00e68d5 github.com/smira/flag v0.0.0-20170926215700-695ea5e84e76 diff --git a/go.sum b/go.sum index 97e45826..5a19ae3f 100644 --- a/go.sum +++ b/go.sum @@ -293,6 +293,8 @@ github.com/rogpeppe/go-internal v1.8.0/go.mod h1:WmiCO8CzOY8rg0OYDC4/i/2WRWAB6po github.com/rs/xid v1.4.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg= github.com/rs/zerolog v1.28.0 h1:MirSo27VyNi7RJYP3078AA1+Cyzd2GB66qy3aUHvsWY= github.com/rs/zerolog v1.28.0/go.mod h1:NILgTygv/Uej1ra5XxGf82ZFSLk58MFGAUS2o6usyD0= +github.com/saracen/walker v0.1.3 h1:YtcKKmpRPy6XJTHJ75J2QYXXZYWnZNQxPCVqZSHVV/g= +github.com/saracen/walker v0.1.3/go.mod h1:FU+7qU8DeQQgSZDmmThMJi93kPkLFgy0oVAcLxurjIk= github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88= @@ -308,8 +310,6 @@ github.com/smira/go-aws-auth v0.0.0-20180731211914-8b73995fd8d1 h1:VPv+J50mFyP42 github.com/smira/go-aws-auth v0.0.0-20180731211914-8b73995fd8d1/go.mod h1:KKhbssKjyR//TUP31t3ksE2b6oeAw328JzwmFJnzRCw= github.com/smira/go-ftp-protocol v0.0.0-20140829150050-066b75c2b70d h1:rvtR4+9N2LWPo0UHe6/aHvWpqD9Dhf10P2bfGFht74g= github.com/smira/go-ftp-protocol v0.0.0-20140829150050-066b75c2b70d/go.mod h1:Jm7yHrROA5tC42gyJ5EwiR8EWp0PUy0qOc4sE7Y8Uzo= -github.com/smira/go-xz v0.0.0-20220607140411-c2a07d4bedda h1:WWMF6Bz2r8/91uUs4ZYk10zSSflqHDE5Ot3/s1yz+x4= -github.com/smira/go-xz v0.0.0-20220607140411-c2a07d4bedda/go.mod h1:RdN8UkuBr4amSnXBHKWkn6p1mXqYjHw+Yvxz8gQfU5A= github.com/smira/go-xz v0.1.0 h1:1zVLT1sITUKcWNysfHMLZWJ2Yh7yJfhREsgmUdK4zb0= github.com/smira/go-xz v0.1.0/go.mod h1:OmdEWnIIkuLzRLHGF4YtjDzF9VFUevEcP6YxDPRqVrs= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= @@ -432,6 +432,8 @@ golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20220601150217-0de741cfad7f h1:Ax0t5p6N38Ga0dThY21weqDEyz2oklo4IvDkpigvkD8= +golang.org/x/sync v0.0.0-20220601150217-0de741cfad7f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -- GitLab From ee2b0129f58d521778816dc75bc5c8f16a27d136 Mon Sep 17 00:00:00 2001 From: Ryan Gonzalez <ryan.gonzalez@collabora.com> Date: Wed, 13 Sep 2023 10:41:00 -0500 Subject: [PATCH 3/6] Improve performance of simple reflist merges When merging reflists with ignoreConflicting set to true and overrideMatching set to false, the individual ref components are never examined, but the refs are still split anyway. Avoiding the split when we never use the components brings a massive speedup: on my system, the included benchmark goes from ~1500 us/it to ~180 us/it. https://github.com/aptly-dev/aptly/pull/1222 Signed-off-by: Ryan Gonzalez <ryan.gonzalez@collabora.com> --- deb/reflist.go | 53 +++++++++++++++++++++------------------ deb/reflist_bench_test.go | 30 ++++++++++++++++++++++ 2 files changed, 58 insertions(+), 25 deletions(-) create mode 100644 deb/reflist_bench_test.go diff --git a/deb/reflist.go b/deb/reflist.go index 187475dd..8a795da5 100644 --- a/deb/reflist.go +++ b/deb/reflist.go @@ -310,38 +310,41 @@ func (l *PackageRefList) Merge(r *PackageRefList, overrideMatching, ignoreConfli overridenName = nil overriddenArch = nil } else { - partsL := bytes.Split(rl, []byte(" ")) - archL, nameL, versionL := partsL[0][1:], partsL[1], partsL[2] + if !ignoreConflicting || overrideMatching { + partsL := bytes.Split(rl, []byte(" ")) + archL, nameL, versionL := partsL[0][1:], partsL[1], partsL[2] - partsR := bytes.Split(rr, []byte(" ")) - archR, nameR, versionR := partsR[0][1:], partsR[1], partsR[2] + partsR := bytes.Split(rr, []byte(" ")) + archR, nameR, versionR := partsR[0][1:], partsR[1], partsR[2] - if !ignoreConflicting && bytes.Equal(archL, archR) && bytes.Equal(nameL, nameR) && bytes.Equal(versionL, versionR) { - // conflicting duplicates with same arch, name, version, but different file hash - result.Refs = append(result.Refs, r.Refs[ir]) - il++ - ir++ - overridenName = nil - overriddenArch = nil - continue - } - - if overrideMatching { - if bytes.Equal(archL, overriddenArch) && bytes.Equal(nameL, overridenName) { - // this package has already been overridden on the right - il++ - continue - } - - if bytes.Equal(archL, archR) && bytes.Equal(nameL, nameR) { - // override with package from the right + if !ignoreConflicting && bytes.Equal(archL, archR) && + bytes.Equal(nameL, nameR) && bytes.Equal(versionL, versionR) { + // conflicting duplicates with same arch, name, version, but different file hash result.Refs = append(result.Refs, r.Refs[ir]) il++ ir++ - overriddenArch = archL - overridenName = nameL + overridenName = nil + overriddenArch = nil continue } + + if overrideMatching { + if bytes.Equal(archL, overriddenArch) && bytes.Equal(nameL, overridenName) { + // this package has already been overridden on the right + il++ + continue + } + + if bytes.Equal(archL, archR) && bytes.Equal(nameL, nameR) { + // override with package from the right + result.Refs = append(result.Refs, r.Refs[ir]) + il++ + ir++ + overriddenArch = archL + overridenName = nameL + continue + } + } } // otherwise append smallest of two diff --git a/deb/reflist_bench_test.go b/deb/reflist_bench_test.go new file mode 100644 index 00000000..367f3d09 --- /dev/null +++ b/deb/reflist_bench_test.go @@ -0,0 +1,30 @@ +package deb + +import ( + "fmt" + "sort" + "testing" +) + +func BenchmarkReflistSimpleMerge(b *testing.B) { + const count = 4096 + + l := NewPackageRefList() + r := NewPackageRefList() + + for i := 0; i < count; i++ { + if i%2 == 0 { + l.Refs = append(l.Refs, []byte(fmt.Sprintf("Pamd64 pkg%d %d", i, i))) + } else { + r.Refs = append(r.Refs, []byte(fmt.Sprintf("Pamd64 pkg%d %d", i, i))) + } + } + + sort.Sort(l) + sort.Sort(r) + + b.ResetTimer() + for i := 0; i < b.N; i++ { + l.Merge(r, false, true) + } +} -- GitLab From 9d7b3b3422d5b1bea44c6933bb27ebdb34c8305b Mon Sep 17 00:00:00 2001 From: Ryan Gonzalez <ryan.gonzalez@collabora.com> Date: Wed, 13 Sep 2023 12:26:12 -0500 Subject: [PATCH 4/6] Improve publish cleanup perf when sources share most of their packages The cleanup phase needs to list out all the files in each component in order to determine what's still in use. When there's a large number of sources (e.g. from having many snapshots), the time spent just loading the package information becomes substantial. However, in many cases, most of the packages being loaded are actually shared across the sources; if you're taking frequent snapshots, for instance, most of the packages in each snapshot will be the same as other snapshots. In these cases, re-reading the packages repeatedly is just a waste of time. To improve this, we maintain a list of refs that we know were processed for each component. When listing the refs from a source, only the ones that have not yet been processed will be examined. Some tests were also added specifically to check listing the files in a component. With this change, listing the files in components on a copy of our production database went from >10 minutes to ~10 seconds, and the newly added benchmark went from ~300ms to ~43ms. https://github.com/aptly-dev/aptly/pull/1222 Signed-off-by: Ryan Gonzalez <ryan.gonzalez@collabora.com> --- deb/publish.go | 54 ++++++++++++------ deb/publish_bench_test.go | 113 ++++++++++++++++++++++++++++++++++++++ deb/publish_test.go | 55 ++++++++++++++++++- deb/remote_test.go | 2 + 4 files changed, 205 insertions(+), 19 deletions(-) create mode 100644 deb/publish_bench_test.go diff --git a/deb/publish.go b/deb/publish.go index c64ad559..826f936a 100644 --- a/deb/publish.go +++ b/deb/publish.go @@ -1137,18 +1137,10 @@ func (collection *PublishedRepoCollection) Len() int { return len(collection.list) } -// CleanupPrefixComponentFiles removes all unreferenced files in published storage under prefix/component pair -func (collection *PublishedRepoCollection) CleanupPrefixComponentFiles(prefix string, components []string, - publishedStorage aptly.PublishedStorage, collectionFactory *CollectionFactory, progress aptly.Progress) error { - - collection.loadList() - - var err error +func (collection *PublishedRepoCollection) listReferencedFilesByComponent(prefix string, components []string, + collectionFactory *CollectionFactory, progress aptly.Progress) (map[string][]string, error) { referencedFiles := map[string][]string{} - - if progress != nil { - progress.Printf("Cleaning up prefix %#v components %s...\n", prefix, strings.Join(components, ", ")) - } + processedComponentRefs := map[string]*PackageRefList{} for _, r := range collection.list { if r.Prefix == prefix { @@ -1167,16 +1159,28 @@ func (collection *PublishedRepoCollection) CleanupPrefixComponentFiles(prefix st continue } - err = collection.LoadComplete(r, collectionFactory) - if err != nil { - return err + if err := collection.LoadComplete(r, collectionFactory); err != nil { + return nil, err } for _, component := range components { if utils.StrSliceHasItem(repoComponents, component) { - packageList, err := NewPackageListFromRefList(r.RefList(component), collectionFactory.PackageCollection(), progress) + unseenRefs := r.RefList(component) + processedRefs := processedComponentRefs[component] + if processedRefs != nil { + unseenRefs = unseenRefs.Subtract(processedRefs) + } else { + processedRefs = NewPackageRefList() + } + + if unseenRefs.Len() == 0 { + continue + } + processedComponentRefs[component] = processedRefs.Merge(unseenRefs, false, true) + + packageList, err := NewPackageListFromRefList(unseenRefs, collectionFactory.PackageCollection(), progress) if err != nil { - return err + return nil, err } packageList.ForEach(func(p *Package) error { @@ -1196,6 +1200,24 @@ func (collection *PublishedRepoCollection) CleanupPrefixComponentFiles(prefix st } } + return referencedFiles, nil +} + +// CleanupPrefixComponentFiles removes all unreferenced files in published storage under prefix/component pair +func (collection *PublishedRepoCollection) CleanupPrefixComponentFiles(prefix string, components []string, + publishedStorage aptly.PublishedStorage, collectionFactory *CollectionFactory, progress aptly.Progress) error { + + collection.loadList() + + if progress != nil { + progress.Printf("Cleaning up prefix %#v components %s...\n", prefix, strings.Join(components, ", ")) + } + + referencedFiles, err := collection.listReferencedFilesByComponent(prefix, components, collectionFactory, progress) + if err != nil { + return err + } + for _, component := range components { sort.Strings(referencedFiles[component]) diff --git a/deb/publish_bench_test.go b/deb/publish_bench_test.go new file mode 100644 index 00000000..b135a8b8 --- /dev/null +++ b/deb/publish_bench_test.go @@ -0,0 +1,113 @@ +package deb + +import ( + "fmt" + "os" + "sort" + "testing" + + "github.com/aptly-dev/aptly/database/goleveldb" +) + +func BenchmarkListReferencedFiles(b *testing.B) { + const defaultComponent = "main" + const repoCount = 16 + const repoPackagesCount = 1024 + const uniqPackagesCount = 64 + + tmpDir, err := os.MkdirTemp("", "aptly-bench") + if err != nil { + b.Fatal(err) + } + defer os.RemoveAll(tmpDir) + + db, err := goleveldb.NewOpenDB(tmpDir) + if err != nil { + b.Fatal(err) + } + defer db.Close() + + factory := NewCollectionFactory(db) + packageCollection := factory.PackageCollection() + repoCollection := factory.LocalRepoCollection() + publishCollection := factory.PublishedRepoCollection() + + sharedRefs := NewPackageRefList() + { + transaction, err := db.OpenTransaction() + if err != nil { + b.Fatal(err) + } + + for pkgIndex := 0; pkgIndex < repoPackagesCount-uniqPackagesCount; pkgIndex++ { + p := &Package{ + Name: fmt.Sprintf("pkg-shared_%d", pkgIndex), + Version: "1", + Architecture: "amd64", + } + p.UpdateFiles(PackageFiles{PackageFile{ + Filename: fmt.Sprintf("pkg-shared_%d.deb", pkgIndex), + }}) + + packageCollection.UpdateInTransaction(p, transaction) + sharedRefs.Refs = append(sharedRefs.Refs, p.Key("")) + } + + sort.Sort(sharedRefs) + + if err := transaction.Commit(); err != nil { + b.Fatal(err) + } + } + + for repoIndex := 0; repoIndex < repoCount; repoIndex++ { + refs := NewPackageRefList() + + transaction, err := db.OpenTransaction() + if err != nil { + b.Fatal(err) + } + + for pkgIndex := 0; pkgIndex < uniqPackagesCount; pkgIndex++ { + p := &Package{ + Name: fmt.Sprintf("pkg%d_%d", repoIndex, pkgIndex), + Version: "1", + Architecture: "amd64", + } + p.UpdateFiles(PackageFiles{PackageFile{ + Filename: fmt.Sprintf("pkg%d_%d.deb", repoIndex, pkgIndex), + }}) + + packageCollection.UpdateInTransaction(p, transaction) + refs.Refs = append(refs.Refs, p.Key("")) + } + + if err := transaction.Commit(); err != nil { + b.Fatal(err) + } + + sort.Sort(refs) + + repo := NewLocalRepo(fmt.Sprintf("repo%d", repoIndex), "comment") + repo.DefaultDistribution = fmt.Sprintf("dist%d", repoIndex) + repo.DefaultComponent = defaultComponent + repo.UpdateRefList(refs.Merge(sharedRefs, false, true)) + repoCollection.Add(repo) + + publish, err := NewPublishedRepo("", "test", "", nil, []string{defaultComponent}, []interface{}{repo}, factory) + if err != nil { + b.Fatal(err) + } + publishCollection.Add(publish) + } + + db.CompactDB() + + b.ResetTimer() + for i := 0; i < b.N; i++ { + _, err := publishCollection.listReferencedFilesByComponent("test", []string{defaultComponent}, factory, nil) + if err != nil { + b.Fatal(err) + } + } +} diff --git a/deb/publish_test.go b/deb/publish_test.go index 4850720f..15e3be92 100644 --- a/deb/publish_test.go +++ b/deb/publish_test.go @@ -7,6 +7,7 @@ import ( "io/ioutil" "os" "path/filepath" + "sort" "github.com/aptly-dev/aptly/aptly" "github.com/aptly-dev/aptly/database" @@ -450,13 +451,22 @@ type PublishedRepoCollectionSuite struct { var _ = Suite(&PublishedRepoCollectionSuite{}) func (s *PublishedRepoCollectionSuite) SetUpTest(c *C) { + s.SetUpPackages() + s.db, _ = goleveldb.NewOpenDB(c.MkDir()) s.factory = NewCollectionFactory(s.db) s.snapshotCollection = s.factory.SnapshotCollection() - s.snap1 = NewSnapshotFromPackageList("snap1", []*Snapshot{}, NewPackageList(), "desc1") - s.snap2 = NewSnapshotFromPackageList("snap2", []*Snapshot{}, NewPackageList(), "desc2") + snap1Refs := NewPackageRefList() + snap1Refs.Refs = [][]byte{s.p1.Key(""), s.p2.Key("")} + sort.Sort(snap1Refs) + s.snap1 = NewSnapshotFromRefList("snap1", []*Snapshot{}, snap1Refs, "desc1") + + snap2Refs := NewPackageRefList() + snap2Refs.Refs = [][]byte{s.p3.Key("")} + sort.Sort(snap2Refs) + s.snap2 = NewSnapshotFromRefList("snap2", []*Snapshot{}, snap2Refs, "desc2") s.snapshotCollection.Add(s.snap1) s.snapshotCollection.Add(s.snap2) @@ -534,7 +544,7 @@ func (s *PublishedRepoCollectionSuite) TestUpdateLoadComplete(c *C) { c.Assert(r.sourceItems["main"].snapshot, IsNil) c.Assert(s.collection.LoadComplete(r, s.factory), IsNil) c.Assert(r.Sources["main"], Equals, s.repo1.sourceItems["main"].snapshot.UUID) - c.Assert(r.RefList("main").Len(), Equals, 0) + c.Assert(r.RefList("main").Len(), Equals, 2) r, err = collection.ByStoragePrefixDistribution("", "ppa", "precise") c.Assert(err, IsNil) @@ -625,6 +635,45 @@ func (s *PublishedRepoCollectionSuite) TestByLocalRepo(c *C) { c.Check(s.collection.ByLocalRepo(s.localRepo), DeepEquals, []*PublishedRepo{s.repo4, s.repo5}) } +func (s *PublishedRepoCollectionSuite) TestListReferencedFiles(c *C) { + c.Check(s.factory.PackageCollection().Update(s.p1), IsNil) + c.Check(s.factory.PackageCollection().Update(s.p2), IsNil) + c.Check(s.factory.PackageCollection().Update(s.p3), IsNil) + + c.Check(s.collection.Add(s.repo1), IsNil) + c.Check(s.collection.Add(s.repo2), IsNil) + c.Check(s.collection.Add(s.repo4), IsNil) + c.Check(s.collection.Add(s.repo5), IsNil) + + files, err := s.collection.listReferencedFilesByComponent(".", []string{"main", "contrib"}, s.factory, nil) + c.Assert(err, IsNil) + c.Check(files, DeepEquals, map[string][]string{ + "contrib": { + "a/alien-arena/alien-arena-common_7.40-2_i386.deb", + "a/alien-arena/mars-invaders_7.40-2_i386.deb", + }, + "main": {"a/alien-arena/lonely-strangers_7.40-2_i386.deb"}, + }) + + snap3 := NewSnapshotFromRefList("snap3", []*Snapshot{}, s.snap2.RefList(), "desc3") + s.snapshotCollection.Add(snap3) + + // Ensure that adding a second publish point with matching files doesn't give duplicate results. + repo3, err := NewPublishedRepo("", "", "anaconda-2", []string{}, []string{"main"}, []interface{}{snap3}, s.factory) + c.Check(err, IsNil) + c.Check(s.collection.Add(repo3), IsNil) + + files, err = s.collection.listReferencedFilesByComponent(".", []string{"main", "contrib"}, s.factory, nil) + c.Assert(err, IsNil) + c.Check(files, DeepEquals, map[string][]string{ + "contrib": { + "a/alien-arena/alien-arena-common_7.40-2_i386.deb", + "a/alien-arena/mars-invaders_7.40-2_i386.deb", + }, + "main": {"a/alien-arena/lonely-strangers_7.40-2_i386.deb"}, + }) +} + type PublishedRepoRemoveSuite struct { PackageListMixinSuite db database.Storage diff --git a/deb/remote_test.go b/deb/remote_test.go index 91e3b224..c331579f 100644 --- a/deb/remote_test.go +++ b/deb/remote_test.go @@ -61,9 +61,11 @@ func (s *PackageListMixinSuite) SetUpPackages() { s.p1 = NewPackageFromControlFile(packageStanza.Copy()) stanza := packageStanza.Copy() stanza["Package"] = "mars-invaders" + stanza["Filename"] = "pool/contrib/m/mars-invaders/mars-invaders_7.40-2_i386.deb" s.p2 = NewPackageFromControlFile(stanza) stanza = packageStanza.Copy() stanza["Package"] = "lonely-strangers" + stanza["Filename"] = "pool/contrib/l/lonely-strangers/lonely-strangers_7.40-2_i386.deb" s.p3 = NewPackageFromControlFile(stanza) s.list.Add(s.p1) -- GitLab From 0903f4bdf338820d6a2624c5b71f26c81a914016 Mon Sep 17 00:00:00 2001 From: Ryan Gonzalez <ryan.gonzalez@collabora.com> Date: Thu, 14 Sep 2023 11:18:03 -0500 Subject: [PATCH 5/6] Use zero-copy decoding for reflists Reflists are basically stored as arrays of strings, which are quite space-efficient in MessagePack. Thus, using zero-copy decoding results in nice performance and memory savings, because the overhead of separate allocations ends up far exceeding the overhead of the original slice. With the included benchmark run for 20s with -benchmem, the runtime, memory usage, and allocations go from ~740us/op, ~192KiB/op, and 4100 allocs/op to ~240us/op, ~97KiB/op, and 13 allocs/op, respectively. https://github.com/aptly-dev/aptly/pull/1222 Signed-off-by: Ryan Gonzalez <ryan.gonzalez@collabora.com> --- deb/reflist.go | 4 +++- deb/reflist_bench_test.go | 17 +++++++++++++++++ 2 files changed, 20 insertions(+), 1 deletion(-) diff --git a/deb/reflist.go b/deb/reflist.go index 8a795da5..25cb0b6e 100644 --- a/deb/reflist.go +++ b/deb/reflist.go @@ -71,7 +71,9 @@ func (l *PackageRefList) Encode() []byte { // Decode decodes msgpack representation into PackageRefLit func (l *PackageRefList) Decode(input []byte) error { - decoder := codec.NewDecoderBytes(input, &codec.MsgpackHandle{}) + handle := &codec.MsgpackHandle{} + handle.ZeroCopy = true + decoder := codec.NewDecoderBytes(input, handle) return decoder.Decode(l) } diff --git a/deb/reflist_bench_test.go b/deb/reflist_bench_test.go index 367f3d09..b377574c 100644 --- a/deb/reflist_bench_test.go +++ b/deb/reflist_bench_test.go @@ -28,3 +28,20 @@ func BenchmarkReflistSimpleMerge(b *testing.B) { l.Merge(r, false, true) } } + +func BenchmarkReflistDecode(b *testing.B) { + const count = 4096 + + r := NewPackageRefList() + for i := 0; i < count; i++ { + r.Refs = append(r.Refs, []byte(fmt.Sprintf("Pamd64 pkg%d %d", i, i))) + } + + sort.Sort(r) + data := r.Encode() + + b.ResetTimer() + for i := 0; i < b.N; i++ { + (&PackageRefList{}).Decode(data) + } +} -- GitLab From a782df41ada44c6c2627497d71c1e1088a3039f7 Mon Sep 17 00:00:00 2001 From: Ryan Gonzalez <ryan.gonzalez@collabora.com> Date: Fri, 22 Sep 2023 15:01:53 -0500 Subject: [PATCH 6/6] Add dockerfile and helm chart from aptly-repository This imports the docker & helm setup, since having it all in one repo makes the update process a bit smoother. There are a few changes to the original docker setup: - The startup script has several improvements: - It actually forwards command-line arguments to aptly. - APTLY_PROFILE can be set at runtime to enable profiling, writing the data to /aptly/data/profile. - The dockerfile can build aptly w/ debugging enabled if APTLY_DEBUG=true is given, which can be passed over via GitLab CI variables. - GOFLAGS will be forwarded to the builder stage in the dockerfile, which is useful for passing down some development-related flags. The latter two points in particular make it easier to build and run versions of aptly w/ profiling enabled, for debugging performance and resource usage issues. Signed-off-by: Ryan Gonzalez <ryan.gonzalez@collabora.com> --- .gitlab-ci.yml | 40 +++++ chart/.helmignore | 23 +++ chart/Chart.yaml | 24 +++ chart/templates/NOTES.txt | 22 +++ chart/templates/_helpers.tpl | 90 ++++++++++++ chart/templates/configmap-publish.yaml | 23 +++ .../deployment-latest-snapshots.yaml | 68 +++++++++ chart/templates/hpa.yaml | 28 ++++ chart/templates/ingress-api.yaml | 61 ++++++++ chart/templates/ingress.yaml | 61 ++++++++ chart/templates/middleware.yaml | 13 ++ chart/templates/pvc.yaml | 18 +++ chart/templates/secret-config.yaml | 16 ++ chart/templates/secret-gpg.yaml | 13 ++ chart/templates/service-latest-snapshots.yaml | 16 ++ chart/templates/service.yaml | 20 +++ chart/templates/serviceaccount.yaml | 12 ++ chart/templates/statefulset.yaml | 126 ++++++++++++++++ chart/templates/tests/test-connection.yaml | 15 ++ chart/values.yaml | 139 ++++++++++++++++++ docker/Dockerfile | 46 ++++++ docker/aptly.conf | 3 + docker/start-aptly.sh | 19 +++ 23 files changed, 896 insertions(+) create mode 100644 .gitlab-ci.yml create mode 100644 chart/.helmignore create mode 100644 chart/Chart.yaml create mode 100644 chart/templates/NOTES.txt create mode 100644 chart/templates/_helpers.tpl create mode 100644 chart/templates/configmap-publish.yaml create mode 100644 chart/templates/deployment-latest-snapshots.yaml create mode 100644 chart/templates/hpa.yaml create mode 100644 chart/templates/ingress-api.yaml create mode 100644 chart/templates/ingress.yaml create mode 100644 chart/templates/middleware.yaml create mode 100644 chart/templates/pvc.yaml create mode 100644 chart/templates/secret-config.yaml create mode 100644 chart/templates/secret-gpg.yaml create mode 100644 chart/templates/service-latest-snapshots.yaml create mode 100644 chart/templates/service.yaml create mode 100644 chart/templates/serviceaccount.yaml create mode 100644 chart/templates/statefulset.yaml create mode 100644 chart/templates/tests/test-connection.yaml create mode 100644 chart/values.yaml create mode 100644 docker/Dockerfile create mode 100644 docker/aptly.conf create mode 100755 docker/start-aptly.sh diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml new file mode 100644 index 00000000..93b99e56 --- /dev/null +++ b/.gitlab-ci.yml @@ -0,0 +1,40 @@ +stages: + - docker + - tag + +variables: + TAG_SHA: $CI_COMMIT_SHORT_SHA-P$CI_PIPELINE_ID + GIT_SUBMODULE_STRATEGY: recursive + APTLY_DEBUG: 'false' + +aptly-image: + stage: docker + tags: + - lightweight + image: + name: gcr.io/kaniko-project/executor:debug + entrypoint: [""] + script: + - | + cat << EOF > /kaniko/.docker/config.json + { + "auths":{ + "$CI_REGISTRY": { + "username":"$CI_REGISTRY_USER", + "password":"$CI_REGISTRY_PASSWORD" + } + } + } + EOF + - > + /kaniko/executor + --context $CI_PROJECT_DIR + --dockerfile $CI_PROJECT_DIR/docker/Dockerfile + --destination $CI_REGISTRY_IMAGE:$TAG_SHA + --destination $CI_REGISTRY_IMAGE:$CI_COMMIT_REF_SLUG + --build-arg REGISTRY=$CI_REGISTRY_IMAGE + --build-arg TAG=$CI_COMMIT_REF_SLUG + --build-arg APTLY_DEBUG=$APTLY_DEBUG + --single-snapshot + - echo Pushed $CI_REGISTRY_IMAGE:$CI_COMMIT_REF_SLUG + and $CI_REGISTRY_IMAGE:$TAG_SHA diff --git a/chart/.helmignore b/chart/.helmignore new file mode 100644 index 00000000..0e8a0eb3 --- /dev/null +++ b/chart/.helmignore @@ -0,0 +1,23 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*.orig +*~ +# Various IDEs +.project +.idea/ +*.tmproj +.vscode/ diff --git a/chart/Chart.yaml b/chart/Chart.yaml new file mode 100644 index 00000000..0b1140ad --- /dev/null +++ b/chart/Chart.yaml @@ -0,0 +1,24 @@ +apiVersion: v2 +name: aptly +description: A Helm chart for Kubernetes + +# A chart can be either an 'application' or a 'library' chart. +# +# Application charts are a collection of templates that can be packaged into versioned archives +# to be deployed. +# +# Library charts provide useful utilities or functions for the chart developer. They're included as +# a dependency of application charts to inject those utilities and functions into the rendering +# pipeline. Library charts do not define any templates and therefore cannot be deployed. +type: application + +# This is the chart version. This version number should be incremented each time you make changes +# to the chart and its templates, including the app version. +# Versions are expected to follow Semantic Versioning (https://semver.org/) +version: 0.1.0 + +# This is the version number of the application being deployed. This version number should be +# incremented each time you make changes to the application. Versions are not expected to +# follow Semantic Versioning. They should reflect the version the application is using. +# It is recommended to use it with quotes. +appVersion: "1.16.0" diff --git a/chart/templates/NOTES.txt b/chart/templates/NOTES.txt new file mode 100644 index 00000000..b890d7ef --- /dev/null +++ b/chart/templates/NOTES.txt @@ -0,0 +1,22 @@ +1. Get the application URL by running these commands: +{{- if .Values.ingress.enabled }} +{{- range $host := .Values.ingress.hosts }} + {{- range .paths }} + http{{ if $.Values.ingress.tls }}s{{ end }}://{{ $host.host }}{{ .path }} + {{- end }} +{{- end }} +{{- else if contains "NodePort" .Values.service.type }} + export NODE_PORT=$(kubectl get --namespace {{ .Release.Namespace }} -o jsonpath="{.spec.ports[0].nodePort}" services {{ include "aptly.fullname" . }}) + export NODE_IP=$(kubectl get nodes --namespace {{ .Release.Namespace }} -o jsonpath="{.items[0].status.addresses[0].address}") + echo http://$NODE_IP:$NODE_PORT +{{- else if contains "LoadBalancer" .Values.service.type }} + NOTE: It may take a few minutes for the LoadBalancer IP to be available. + You can watch the status of by running 'kubectl get --namespace {{ .Release.Namespace }} svc -w {{ include "aptly.fullname" . }}' + export SERVICE_IP=$(kubectl get svc --namespace {{ .Release.Namespace }} {{ include "aptly.fullname" . }} --template "{{"{{ range (index .status.loadBalancer.ingress 0) }}{{.}}{{ end }}"}}") + echo http://$SERVICE_IP:{{ .Values.service.port }} +{{- else if contains "ClusterIP" .Values.service.type }} + export POD_NAME=$(kubectl get pods --namespace {{ .Release.Namespace }} -l "app.kubernetes.io/name={{ include "aptly.name" . }},app.kubernetes.io/instance={{ .Release.Name }}" -o jsonpath="{.items[0].metadata.name}") + export CONTAINER_PORT=$(kubectl get pod --namespace {{ .Release.Namespace }} $POD_NAME -o jsonpath="{.spec.containers[0].ports[0].containerPort}") + echo "Visit http://127.0.0.1:8080 to use your application" + kubectl --namespace {{ .Release.Namespace }} port-forward $POD_NAME 8080:$CONTAINER_PORT +{{- end }} diff --git a/chart/templates/_helpers.tpl b/chart/templates/_helpers.tpl new file mode 100644 index 00000000..c48a9dd4 --- /dev/null +++ b/chart/templates/_helpers.tpl @@ -0,0 +1,90 @@ +{{/* +Expand the name of the chart. +*/}} +{{- define "aptly.name" -}} +{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Create a default fully qualified app name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +If release name contains chart name it will be used as a full name. +*/}} +{{- define "aptly.fullname" -}} +{{- if .Values.fullnameOverride }} +{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- $name := default .Chart.Name .Values.nameOverride }} +{{- if contains $name .Release.Name }} +{{- .Release.Name | trunc 63 | trimSuffix "-" }} +{{- else }} +{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" }} +{{- end }} +{{- end }} +{{- end }} + +{{/* +Create chart name and version as used by the chart label. +*/}} +{{- define "aptly.chart" -}} +{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" }} +{{- end }} + +{{/* +Common labels +*/}} +{{- define "aptly.labels" -}} +helm.sh/chart: {{ include "aptly.chart" . }} +{{ include "aptly.selectorLabels" . }} +{{- if .Chart.AppVersion }} +app.kubernetes.io/version: {{ .Chart.AppVersion | quote }} +{{- end }} +app.kubernetes.io/managed-by: {{ .Release.Service }} +{{- end }} + +{{/* +Selector labels +*/}} +{{- define "aptly.selectorLabels" -}} +app.kubernetes.io/name: {{ include "aptly.name" . }} +app.kubernetes.io/instance: {{ .Release.Name }} +{{- end }} + +{{/* +Extra selector labels for the main aptly pod +*/}} +{{- define "aptly.extraPodSelectorLabels" -}} +service: aptly +{{- end }} + +{{/* +Extra selector labels for the aptly-latest-snapshots pod +*/}} +{{- define "aptly.latestSnapshots.extraPodSelectorLabels" -}} +service: aptly-latest-snapshots +{{- end }} + +{{/* +Create the name of the service account to use +*/}} +{{- define "aptly.serviceAccountName" -}} +{{- if .Values.serviceAccount.create }} +{{- default (include "aptly.fullname" .) .Values.serviceAccount.name }} +{{- else }} +{{- default "default" .Values.serviceAccount.name }} +{{- end }} +{{- end }} + +{{/* +Create the name of the configuration secret +*/}} +{{- define "aptly.configSecretName" -}} +{{ include "aptly.name" . }}-config +{{- end }} + +{{/* +Create the name of the GPG keys secret +*/}} +{{- define "aptly.gpgSecretName" -}} +{{ include "aptly.name" . }}-gpg +{{- end }} diff --git a/chart/templates/configmap-publish.yaml b/chart/templates/configmap-publish.yaml new file mode 100644 index 00000000..4b006eaa --- /dev/null +++ b/chart/templates/configmap-publish.yaml @@ -0,0 +1,23 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ include "aptly.fullname" . }}-publish + labels: + {{- include "aptly.labels" . | nindent 4 }} +data: + default.conf: | + server { + listen 80 default_server; + listen [::]:80 default_server; + + #access_log /var/log/nginx/host.access.log main; + location ~ /apertis/dists/(?<dist>[^/]+)/snapshots/latest\.txt$ { + resolver kube-dns.kube-system.svc.cluster.local; + proxy_pass http://{{ include "aptly.fullname" . }}-latest-snapshots.{{ .Release.Namespace }}.svc.cluster.local:{{ .Values.latestSnapshots.service.port }}/latest/$dist; + } + location / { + root /data; + autoindex on; + try_files $uri $uri/ =404; + } + } diff --git a/chart/templates/deployment-latest-snapshots.yaml b/chart/templates/deployment-latest-snapshots.yaml new file mode 100644 index 00000000..35ad3308 --- /dev/null +++ b/chart/templates/deployment-latest-snapshots.yaml @@ -0,0 +1,68 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: {{ include "aptly.fullname" . }}-latest-snapshots + labels: + {{- include "aptly.labels" . | nindent 4 }} +spec: + {{- if not .Values.autoscaling.enabled }} + replicas: {{ .Values.replicaCount }} + {{- end }} + selector: + matchLabels: + {{- include "aptly.selectorLabels" . | nindent 6 }} + {{- include "aptly.latestSnapshots.extraPodSelectorLabels" . | nindent 6 }} + template: + metadata: + {{- with .Values.podAnnotations }} + annotations: + {{- toYaml . | nindent 8 }} + {{- end }} + labels: + {{- include "aptly.selectorLabels" . | nindent 8 }} + {{- include "aptly.latestSnapshots.extraPodSelectorLabels" . | nindent 8 }} + spec: + {{- with .Values.imagePullSecrets }} + imagePullSecrets: + {{- toYaml . | nindent 8 }} + {{- end }} + serviceAccountName: {{ include "aptly.serviceAccountName" . }} + securityContext: + {{- toYaml .Values.podSecurityContext | nindent 8 }} + containers: + - name: {{ .Chart.Name }} + securityContext: + {{- toYaml .Values.securityContext | nindent 12 }} + image: "{{ .Values.latestSnapshots.image.repository }}:{{ .Values.latestSnapshots.image.tag | default .Chart.AppVersion }}" + imagePullPolicy: {{ .Values.latestSnapshots.image.pullPolicy }} + args: + - --api-url=http://{{ include "aptly.fullname" . }}:8080/ + - --refresh-interval-sec={{ .Values.latestSnapshots.refreshIntervalSec }} + ports: + - name: http + containerPort: 8080 + protocol: TCP + livenessProbe: + httpGet: + path: /healthz + port: http + readinessProbe: + httpGet: + path: /healthz + port: http + # Grabbing the latest snapshots can take a bit + initialDelaySeconds: 10 + resources: + {{- toYaml .Values.latestSnapshots.resources | nindent 12 }} + {{- with .Values.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.affinity }} + affinity: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.tolerations }} + tolerations: + {{- toYaml . | nindent 8 }} + {{- end }} diff --git a/chart/templates/hpa.yaml b/chart/templates/hpa.yaml new file mode 100644 index 00000000..194b07dc --- /dev/null +++ b/chart/templates/hpa.yaml @@ -0,0 +1,28 @@ +{{- if .Values.autoscaling.enabled }} +apiVersion: autoscaling/v2beta1 +kind: HorizontalPodAutoscaler +metadata: + name: {{ include "aptly.fullname" . }} + labels: + {{- include "aptly.labels" . | nindent 4 }} +spec: + scaleTargetRef: + apiVersion: apps/v1 + kind: Deployment + name: {{ include "aptly.fullname" . }} + minReplicas: {{ .Values.autoscaling.minReplicas }} + maxReplicas: {{ .Values.autoscaling.maxReplicas }} + metrics: + {{- if .Values.autoscaling.targetCPUUtilizationPercentage }} + - type: Resource + resource: + name: cpu + targetAverageUtilization: {{ .Values.autoscaling.targetCPUUtilizationPercentage }} + {{- end }} + {{- if .Values.autoscaling.targetMemoryUtilizationPercentage }} + - type: Resource + resource: + name: memory + targetAverageUtilization: {{ .Values.autoscaling.targetMemoryUtilizationPercentage }} + {{- end }} +{{- end }} diff --git a/chart/templates/ingress-api.yaml b/chart/templates/ingress-api.yaml new file mode 100644 index 00000000..fd0bc397 --- /dev/null +++ b/chart/templates/ingress-api.yaml @@ -0,0 +1,61 @@ +{{- if .Values.ingressApi.enabled -}} +{{- $fullName := include "aptly.fullname" . -}} +{{- $svcPort := .Values.service.portApi -}} +{{- if and .Values.ingressApi.className (not (semverCompare ">=1.18-0" .Capabilities.KubeVersion.GitVersion)) }} + {{- if not (hasKey .Values.ingressApi.annotations "kubernetes.io/ingress.class") }} + {{- $_ := set .Values.ingressApi.annotations "kubernetes.io/ingress.class" .Values.ingressApi.className}} + {{- end }} +{{- end }} +{{- if semverCompare ">=1.19-0" .Capabilities.KubeVersion.GitVersion -}} +apiVersion: networking.k8s.io/v1 +{{- else if semverCompare ">=1.14-0" .Capabilities.KubeVersion.GitVersion -}} +apiVersion: networking.k8s.io/v1beta1 +{{- else -}} +apiVersion: extensions/v1beta1 +{{- end }} +kind: Ingress +metadata: + name: {{ $fullName }}-api + labels: + {{- include "aptly.labels" . | nindent 4 }} + {{- with .Values.ingressApi.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + {{- if and .Values.ingressApi.className (semverCompare ">=1.18-0" .Capabilities.KubeVersion.GitVersion) }} + ingressClassName: {{ .Values.ingressApi.className }} + {{- end }} + {{- if .Values.ingressApi.tls }} + tls: + {{- range .Values.ingressApi.tls }} + - hosts: + {{- range .hosts }} + - {{ . | quote }} + {{- end }} + secretName: {{ .secretName }} + {{- end }} + {{- end }} + rules: + {{- range .Values.ingressApi.hosts }} + - host: {{ .host | quote }} + http: + paths: + {{- range .paths }} + - path: {{ .path }} + {{- if and .pathType (semverCompare ">=1.18-0" $.Capabilities.KubeVersion.GitVersion) }} + pathType: {{ .pathType }} + {{- end }} + backend: + {{- if semverCompare ">=1.19-0" $.Capabilities.KubeVersion.GitVersion }} + service: + name: {{ $fullName }} + port: + number: {{ $svcPort }} + {{- else }} + serviceName: {{ $fullName }} + servicePort: {{ $svcPort }} + {{- end }} + {{- end }} + {{- end }} +{{- end }} diff --git a/chart/templates/ingress.yaml b/chart/templates/ingress.yaml new file mode 100644 index 00000000..175872f7 --- /dev/null +++ b/chart/templates/ingress.yaml @@ -0,0 +1,61 @@ +{{- if .Values.ingress.enabled -}} +{{- $fullName := include "aptly.fullname" . -}} +{{- $svcPort := .Values.service.port -}} +{{- if and .Values.ingress.className (not (semverCompare ">=1.18-0" .Capabilities.KubeVersion.GitVersion)) }} + {{- if not (hasKey .Values.ingress.annotations "kubernetes.io/ingress.class") }} + {{- $_ := set .Values.ingress.annotations "kubernetes.io/ingress.class" .Values.ingress.className}} + {{- end }} +{{- end }} +{{- if semverCompare ">=1.19-0" .Capabilities.KubeVersion.GitVersion -}} +apiVersion: networking.k8s.io/v1 +{{- else if semverCompare ">=1.14-0" .Capabilities.KubeVersion.GitVersion -}} +apiVersion: networking.k8s.io/v1beta1 +{{- else -}} +apiVersion: extensions/v1beta1 +{{- end }} +kind: Ingress +metadata: + name: {{ $fullName }} + labels: + {{- include "aptly.labels" . | nindent 4 }} + {{- with .Values.ingress.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + {{- if and .Values.ingress.className (semverCompare ">=1.18-0" .Capabilities.KubeVersion.GitVersion) }} + ingressClassName: {{ .Values.ingress.className }} + {{- end }} + {{- if .Values.ingress.tls }} + tls: + {{- range .Values.ingress.tls }} + - hosts: + {{- range .hosts }} + - {{ . | quote }} + {{- end }} + secretName: {{ .secretName }} + {{- end }} + {{- end }} + rules: + {{- range .Values.ingress.hosts }} + - host: {{ .host | quote }} + http: + paths: + {{- range .paths }} + - path: {{ .path }} + {{- if and .pathType (semverCompare ">=1.18-0" $.Capabilities.KubeVersion.GitVersion) }} + pathType: {{ .pathType }} + {{- end }} + backend: + {{- if semverCompare ">=1.19-0" $.Capabilities.KubeVersion.GitVersion }} + service: + name: {{ $fullName }} + port: + number: {{ $svcPort }} + {{- else }} + serviceName: {{ $fullName }} + servicePort: {{ $svcPort }} + {{- end }} + {{- end }} + {{- end }} +{{- end }} diff --git a/chart/templates/middleware.yaml b/chart/templates/middleware.yaml new file mode 100644 index 00000000..74387ed2 --- /dev/null +++ b/chart/templates/middleware.yaml @@ -0,0 +1,13 @@ +apiVersion: traefik.io/v1alpha1 +kind: Middleware +metadata: + name: {{ include "aptly.name" . }}-auth + labels: + {{- include "aptly.labels" . | nindent 4 }} +spec: + forwardAuth: + address: http://oathkeeper-api:4456/decisions + authResponseHeaders: + - Authorization + # basicAuth: + # secret: {{ include "aptly.name" . }}-auth diff --git a/chart/templates/pvc.yaml b/chart/templates/pvc.yaml new file mode 100644 index 00000000..f16d1e53 --- /dev/null +++ b/chart/templates/pvc.yaml @@ -0,0 +1,18 @@ +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: {{ include "aptly.fullname" . }}-data + labels: + {{- include "aptly.labels" . | nindent 4 }} + annotations: + {{- toYaml .Values.persistence.annotations | nindent 14 }} +spec: + accessModes: + {{- toYaml .Values.persistence.accessModes | nindent 14 }} + +{{- if .Values.persistence.storageClass }} + storageClassName: {{ .Values.persistence.storageClass | quote }} +{{- end }} + resources: + requests: + storage: {{ .Values.persistence.size | quote }} diff --git a/chart/templates/secret-config.yaml b/chart/templates/secret-config.yaml new file mode 100644 index 00000000..aa059fc0 --- /dev/null +++ b/chart/templates/secret-config.yaml @@ -0,0 +1,16 @@ +{{- $persistencePath := printf "%s/%s" .Values.persistence.mountPath .Values.persistence.subPath | clean }} +{{- if ne (.Values.config.rootDir | clean) ($persistencePath | clean) }} + {{- fail + (printf "config.rootDir (%s) must match persistence $mountPath/$subPath (%s)" + .Values.config.rootDir $persistencePath) }} +{{- end }} + +apiVersion: v1 +kind: Secret +metadata: + name: {{ include "aptly.configSecretName" . }} + labels: + {{- include "aptly.labels" . | nindent 4 }} +stringData: + aptly.conf: | + {{ .Values.config | mustToPrettyJson | nindent 4 }} diff --git a/chart/templates/secret-gpg.yaml b/chart/templates/secret-gpg.yaml new file mode 100644 index 00000000..4ce8e331 --- /dev/null +++ b/chart/templates/secret-gpg.yaml @@ -0,0 +1,13 @@ +{{- if .Values.gpg.enabled }} +apiVersion: v1 +kind: Secret +metadata: + name: {{ include "aptly.gpgSecretName" . }} + labels: + {{- include "aptly.labels" . | nindent 4 }} +stringData: + {{- range $i, $key := .Values.gpg.keys }} + key{{$i}}.asc: | + {{ $key | nindent 4 }} + {{- end }} +{{- end }} diff --git a/chart/templates/service-latest-snapshots.yaml b/chart/templates/service-latest-snapshots.yaml new file mode 100644 index 00000000..1f98b52c --- /dev/null +++ b/chart/templates/service-latest-snapshots.yaml @@ -0,0 +1,16 @@ +apiVersion: v1 +kind: Service +metadata: + name: {{ include "aptly.fullname" . }}-latest-snapshots + labels: + {{- include "aptly.labels" . | nindent 4 }} +spec: + type: {{ .Values.latestSnapshots.service.type }} + ports: + - port: {{ .Values.latestSnapshots.service.port }} + targetPort: http + protocol: TCP + name: http + selector: + {{- include "aptly.selectorLabels" . | nindent 4 }} + {{- include "aptly.latestSnapshots.extraPodSelectorLabels" . | nindent 4 }} diff --git a/chart/templates/service.yaml b/chart/templates/service.yaml new file mode 100644 index 00000000..b9085638 --- /dev/null +++ b/chart/templates/service.yaml @@ -0,0 +1,20 @@ +apiVersion: v1 +kind: Service +metadata: + name: {{ include "aptly.fullname" . }} + labels: + {{- include "aptly.labels" . | nindent 4 }} +spec: + type: {{ .Values.service.type }} + ports: + - port: {{ .Values.service.port }} + targetPort: http + protocol: TCP + name: http + - port: {{ .Values.service.portApi }} + targetPort: api + protocol: TCP + name: api + selector: + {{- include "aptly.selectorLabels" . | nindent 4 }} + {{- include "aptly.extraPodSelectorLabels" . | nindent 4 }} diff --git a/chart/templates/serviceaccount.yaml b/chart/templates/serviceaccount.yaml new file mode 100644 index 00000000..8fcb3822 --- /dev/null +++ b/chart/templates/serviceaccount.yaml @@ -0,0 +1,12 @@ +{{- if .Values.serviceAccount.create -}} +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ include "aptly.serviceAccountName" . }} + labels: + {{- include "aptly.labels" . | nindent 4 }} + {{- with .Values.serviceAccount.annotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} +{{- end }} diff --git a/chart/templates/statefulset.yaml b/chart/templates/statefulset.yaml new file mode 100644 index 00000000..d126e591 --- /dev/null +++ b/chart/templates/statefulset.yaml @@ -0,0 +1,126 @@ +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: {{ include "aptly.fullname" . }} + labels: + {{- include "aptly.labels" . | nindent 4 }} +spec: + {{- if not .Values.autoscaling.enabled }} + replicas: {{ .Values.replicaCount }} + {{- end }} + serviceName: {{ include "aptly.fullname" . }} + selector: + matchLabels: + {{- include "aptly.selectorLabels" . | nindent 6 }} + {{- include "aptly.extraPodSelectorLabels" . | nindent 6 }} + template: + metadata: + {{- with .Values.podAnnotations }} + annotations: + {{- toYaml . | nindent 8 }} + {{- end }} + labels: + {{- include "aptly.selectorLabels" . | nindent 8 }} + {{- include "aptly.extraPodSelectorLabels" . | nindent 8 }} + spec: + {{- with .Values.imagePullSecrets }} + imagePullSecrets: + {{- toYaml . | nindent 8 }} + {{- end }} + serviceAccountName: {{ include "aptly.serviceAccountName" . }} + securityContext: + {{- toYaml .Values.podSecurityContext | nindent 8 }} + containers: + - name: {{ .Chart.Name }}-api + securityContext: + {{- toYaml .Values.securityContext | nindent 12 }} + image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}" + imagePullPolicy: {{ .Values.image.pullPolicy }} + args: + - -config=/aptly/config/aptly.conf + ports: + - name: api + containerPort: 8080 + protocol: TCP + livenessProbe: + httpGet: + path: /api/healthy + port: api + readinessProbe: + httpGet: + path: /api/ready + port: api + resources: + {{- toYaml (merge .Values.api.resources .Values.resources) | nindent 12 }} + volumeMounts: + - name: config + mountPath: /aptly/config + - name: data + mountPath: {{ .Values.persistence.mountPath }} + subPath: {{ .Values.persistence.subPath }} + {{- if .Values.gpg.enabled }} + - name: gpg + mountPath: /aptly/gpg + {{- end }} + {{- if .Values.extraVolumeMounts }} + {{- toYaml .Values.extraVolumeMounts | nindent 12 }} + {{- end }} + - name: {{ .Chart.Name }}-publish + securityContext: + {{- toYaml .Values.securityContext | nindent 12 }} + image: docker.io/library/nginx:1.25 + imagePullPolicy: {{ .Values.image.pullPolicy }} + ports: + - name: http + containerPort: 80 + protocol: TCP + livenessProbe: + httpGet: + path: / + port: http + readinessProbe: + httpGet: + path: / + port: http + resources: + {{- toYaml (merge .Values.publish.resources .Values.resources) | nindent 12 }} + volumeMounts: + - name: data + mountPath: "/data" + subPath: "public" + - name: "nginx-config" + mountPath: "/etc/nginx/conf.d" + {{- if .Values.extraVolumeMounts }} + {{- toYaml .Values.extraVolumeMounts | nindent 12 }} + {{- end }} + volumes: + - name: config + secret: + secretName: {{ include "aptly.configSecretName" . }} + {{- if .Values.gpg.enabled }} + - name: gpg + secret: + secretName: {{ include "aptly.gpgSecretName" . }} + {{- end }} + - name: nginx-config + configMap: + name: {{ include "aptly.fullname" . }}-publish + - name: data + persistentVolumeClaim: + {{- if .Values.persistence.existingClaim }} + claimName: {{ .Values.persistence.existingClaim | quote }} + {{- else }} + claimName: {{ include "aptly.fullname" . }}-data + {{- end }} + {{- with .Values.nodeSelector }} + nodeSelector: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.affinity }} + affinity: + {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.tolerations }} + tolerations: + {{- toYaml . | nindent 8 }} + {{- end }} diff --git a/chart/templates/tests/test-connection.yaml b/chart/templates/tests/test-connection.yaml new file mode 100644 index 00000000..69252d17 --- /dev/null +++ b/chart/templates/tests/test-connection.yaml @@ -0,0 +1,15 @@ +apiVersion: v1 +kind: Pod +metadata: + name: "{{ include "aptly.fullname" . }}-test-connection" + labels: + {{- include "aptly.labels" . | nindent 4 }} + annotations: + "helm.sh/hook": test +spec: + containers: + - name: wget + image: busybox + command: ['wget'] + args: ['{{ include "aptly.fullname" . }}:{{ .Values.service.port }}'] + restartPolicy: Never diff --git a/chart/values.yaml b/chart/values.yaml new file mode 100644 index 00000000..de4dc94e --- /dev/null +++ b/chart/values.yaml @@ -0,0 +1,139 @@ +# Default values for aptly. +# This is a YAML-formatted file. +# Declare variables to be passed into your templates. + +replicaCount: 1 + +# TODO: put 'image' in container-specific sections, for consistency +image: + repository: registry.gitlab.collabora.com/obs/aptly + pullPolicy: IfNotPresent + tag: main + +api: + resources: {} + +publish: + resources: {} + +latestSnapshots: + image: + repository: ghcr.io/collabora/aptly-rest-tools + pullPolicy: IfNotPresent + tag: main + refreshIntervalSec: 600 + + service: + type: ClusterIP + port: 80 + + resources: {} + +imagePullSecrets: [] +nameOverride: "" +fullnameOverride: "" + +# NOTE: rootDir must match ${persistence.mountPath}/${persistence.subPath} +# (this is checked by the chart) +config: + rootDir: /aptly/data + +persistence: + accessModes: [ReadWriteOnce] + annotations: {} + existingClaim: '' + size: 10Gi + storageClassName: '' + mountPath: /aptly/data + subPath: '' + +gpg: + enabled: false + # Expects a list of armored GPG private key strings. + keys: [] + +extraVolumeMounts: [] + +serviceAccount: + # Specifies whether a service account should be created + create: true + # Annotations to add to the service account + annotations: {} + # The name of the service account to use. + # If not set and create is true, a name is generated using the fullname template + name: "" + +podAnnotations: {} + +podSecurityContext: {} + # fsGroup: 2000 + +securityContext: {} + # capabilities: + # drop: + # - ALL + # readOnlyRootFilesystem: true + # runAsNonRoot: true + # runAsUser: 1000 + +service: + type: ClusterIP + port: 80 + portApi: 8080 + +ingress: + enabled: false + className: '' + annotations: {} + # kubernetes.io/ingress.class: nginx + # kubernetes.io/tls-acme: "true" + hosts: + - host: chart-example.local + paths: + - path: / + pathType: ImplementationSpecific + tls: [] + # - secretName: chart-example-tls + # hosts: + # - chart-example.local + +ingressApi: + enabled: false + className: '' + annotations: {} + # kubernetes.io/ingress.class: nginx + # kubernetes.io/tls-acme: "true" + hosts: + - host: chart-example-api.local + paths: + - path: / + pathType: ImplementationSpecific + tls: [] + # - secretName: chart-example-tls + # hosts: + # - chart-example.local + +resources: {} + # We usually recommend not to specify default resources and to leave this as a conscious + # choice for the user. This also increases chances charts run on environments with little + # resources, such as Minikube. If you do want to specify resources, uncomment the following + # lines, adjust them as necessary, and remove the curly braces after 'resources:'. + # limits: + # cpu: 100m + # memory: 128Mi + # requests: + # cpu: 100m + # memory: 128Mi + +autoscaling: + enabled: false + minReplicas: 1 + maxReplicas: 100 + targetCPUUtilizationPercentage: 80 + # targetMemoryUtilizationPercentage: 80 + +nodeSelector: {} + +tolerations: [] + +affinity: {} diff --git a/docker/Dockerfile b/docker/Dockerfile new file mode 100644 index 00000000..3a6b3c09 --- /dev/null +++ b/docker/Dockerfile @@ -0,0 +1,46 @@ +# Global ARGs shared by all stages +ARG DEBIAN_FRONTEND=noninteractive +ARG GOPATH=/usr/local/go + +# Build aptly +FROM debian:bookworm-slim as builder +ENV LC_ALL=C.UTF-8 +ARG APTLY_DEBUG=false +ARG DEBIAN_FRONTEND +# Useful for passing flags down for development purposes. +ARG GOFLAGS +ARG GOPATH + +RUN apt-get update && \ + apt-get install -y --no-install-recommends \ + build-essential \ + ca-certificates \ + gcc \ + git \ + golang-go \ + libc6-dev + +COPY . /work +WORKDIR /work +RUN sed -i "s/\\(EnableDebug = \\).*/\1$APTLY_DEBUG/" aptly/version.go \ + && make install + +FROM debian:bookworm-slim as server +ENV LC_ALL=C.UTF-8 +ARG DEBIAN_FRONTEND +ARG GOPATH + +RUN apt-get update && \ + apt-get install -y --no-install-recommends \ + bzip2 \ + ca-certificates \ + gnupg \ + gpgv \ + xz-utils + +COPY --from=builder $GOPATH/bin/aptly /usr/local/bin/aptly +COPY docker/aptly.conf /etc/aptly.conf +COPY docker/start-aptly.sh /usr/local/bin/ + +EXPOSE 8080 +ENTRYPOINT ["/usr/local/bin/start-aptly.sh"] diff --git a/docker/aptly.conf b/docker/aptly.conf new file mode 100644 index 00000000..d73938da --- /dev/null +++ b/docker/aptly.conf @@ -0,0 +1,3 @@ +{ + "rootDir": "/aptly/data" +} diff --git a/docker/start-aptly.sh b/docker/start-aptly.sh new file mode 100755 index 00000000..8205ba11 --- /dev/null +++ b/docker/start-aptly.sh @@ -0,0 +1,19 @@ +#!/bin/bash + +set -e + +shopt -s nullglob +for key in /aptly/gpg/*.asc; do + echo "NOTE: Importing gpg key: $key" + gpg --import $key +done + +profargs=() +if [ -n "$APTLY_PROFILE" ]; then + profdir=/aptly/data/profile/$(date -Isec) + mkdir -p $profdir + profargs=(-cpuprofile=$profdir/cpu.prof -memprofile=$profdir/mem.prof) +fi + +set -x +exec aptly "${profargs[@]}" api serve "$@" -- GitLab