From cc2e1a87296bf009407b8ed657f172750d9a3013 Mon Sep 17 00:00:00 2001 From: Jim Myhrberg Date: Fri, 24 Nov 2023 20:32:35 +0000 Subject: [PATCH] fix(badge/downloads): fix issues with total download counts Excluded patterns had a bug where they basically did not work. This means total downloads were inflated by also counting downloads for all sha256 files. Also the first 1000 limit when listing repository releases seems to no loner be a thing, so we can iterate over all pages. And finally, implement a caching mechanism so we can keep counting releases in case they are removed, or the first 1000 releases limit is enabled again in GitHub's API. --- go.mod | 2 +- go.sum | 1 + main.go | 212 +++++++++++++++++++++++++++++++++++++++++++++++++------- 3 files changed, 189 insertions(+), 26 deletions(-) diff --git a/go.mod b/go.mod index a74efc0..9b4666a 100644 --- a/go.mod +++ b/go.mod @@ -1,6 +1,6 @@ module emacs-builds-meta -go 1.20 +go 1.21 require ( github.com/dustin/go-humanize v1.0.1 diff --git a/go.sum b/go.sum index 04be3f4..980e331 100644 --- a/go.sum +++ b/go.sum @@ -9,6 +9,7 @@ github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiu github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= +github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/go-github v17.0.0+incompatible h1:N0LgJ1j65A7kfXrZnUDaYCs/Sf4rEjNlfyDHW9dolSY= github.com/google/go-github v17.0.0+incompatible/go.mod h1:zLgOLi98H3fifZn+44m+umXrS52loVEgC2AApnigrVQ= github.com/google/go-querystring v1.1.0 h1:AnCroh3fv4ZBgVIf1Iwtovgjaw/GiKJo8M8yD/fhyJ8= diff --git a/main.go b/main.go index 8614f2f..1c4d5bb 100644 --- a/main.go +++ b/main.go @@ -5,8 +5,8 @@ import ( "context" "encoding/json" "fmt" - "io/ioutil" "log" + "log/slog" "os" "path/filepath" "regexp" @@ -32,6 +32,49 @@ const downloadIcon = `` + `` + `` +type Cache struct { + AseetDownloads map[string]map[string]int `json:"asset_downloads,omitempty"` +} + +func (c *Cache) AssetsCount() int { + var count int + + for _, v := range c.AseetDownloads { + count += len(v) + } + + return count +} + +func (c *Cache) AssetReleasesCount() int { + return len(c.AseetDownloads) +} + +func (c *Cache) SetAssetDownloadCount( + releaseName, assetName string, + count int, +) { + if c.AseetDownloads == nil { + c.AseetDownloads = map[string]map[string]int{} + } + if c.AseetDownloads[releaseName] == nil { + c.AseetDownloads[releaseName] = map[string]int{} + } + c.AseetDownloads[releaseName][assetName] = count +} + +func (c *Cache) TotalAssetDownloads() int { + var count int + + for _, v := range c.AseetDownloads { + for _, v := range v { + count += v + } + } + + return count +} + type Badge struct { SchemaVersion int `json:"schemaVersion,omitempty"` Label string `json:"label,omitempty"` @@ -67,14 +110,19 @@ func NewApp() *cli.App { Name: "downloads", Flags: []cli.Flag{ &cli.StringSliceFlag{ - Name: "exclude", - Usage: "regexp asset filename pattern to exclude", - Value: cli.NewStringSlice(`.+\.sha\d+$`), + Name: "exclude", + Aliases: []string{"e"}, + Usage: "regexp asset filename pattern to exclude", + Value: cli.NewStringSlice(`.+\.sha\d+$`), }, &cli.StringFlag{ Name: "output", Aliases: []string{"o"}, }, + &cli.StringFlag{ + Name: "cache", + Aliases: []string{"c"}, + }, }, Action: badgesDownloadsAction, }, @@ -104,14 +152,36 @@ func badgesDownloadsAction(c *cli.Context) error { excludes = append(excludes, regexp.MustCompile(s)) } - parts := strings.SplitN(c.String("repository"), "/", 2) - owner := parts[0] - repo := parts[1] + owner, repo, err := getOwnerAndRepo(c) + if err != nil { + return err + } - count := 0 + cache := &Cache{} + cacheFile := c.String("cache") - for page := 1; page > 0 && page < 11; { - releases, resp, err := gh.Repositories.ListReleases( + if cacheFile != "" { + slog.Info("reading cache", slog.String("file", cacheFile)) + + cache, err = readCache(cacheFile) + if err != nil { + return err + } + + slog.Info("cache stats", + slog.Int("downloads", cache.TotalAssetDownloads()), + slog.Int("releases", cache.AssetReleasesCount()), + slog.Int("assets", cache.AssetsCount()), + ) + } + + lastPage := 1 + for page := 1; page > 0 && page <= lastPage; { + slog.Info("fetching releases", slog.Int("page", page)) + + var releases []*github.RepositoryRelease + var resp *github.Response + releases, resp, err = gh.Repositories.ListReleases( c.Context, owner, repo, &github.ListOptions{ Page: page, PerPage: 100, @@ -121,10 +191,27 @@ func badgesDownloadsAction(c *cli.Context) error { return err } - count += releaseAssetsDownlaodCount(releases, excludes) + updateAssetDownloadsCache(cache, releases, excludes) page = resp.NextPage + lastPage = resp.LastPage } + if cacheFile != "" { + slog.Info("writing cache", slog.String("file", cacheFile)) + slog.Info( + "cache stats", + slog.Int("downloads", cache.TotalAssetDownloads()), + slog.Int("releases", cache.AssetReleasesCount()), + slog.Int("assets", cache.AssetsCount()), + ) + + err = writeCache(cacheFile, cache) + if err != nil { + return err + } + } + + count := cache.TotalAssetDownloads() var humanCount string if count >= 1000 { v, sym := humanize.ComputeSI(float64(count)) @@ -133,6 +220,11 @@ func badgesDownloadsAction(c *cli.Context) error { humanCount = strconv.Itoa(count) } + slog.Info("total downloads", + slog.Int("count", count), + slog.String("humanized_count", humanCount), + ) + badge := &Badge{ SchemaVersion: 1, Style: "flat", @@ -142,7 +234,7 @@ func badgesDownloadsAction(c *cli.Context) error { Message: humanCount, } - b, err := jsonMarshal(badge) + b, err := jsonPrettyMarshal(badge) if err != nil { return err } @@ -155,7 +247,7 @@ func badgesDownloadsAction(c *cli.Context) error { return err } } - err := ioutil.WriteFile(filename, b, 0o644) //nolint:gosec + err := os.WriteFile(filename, b, 0o644) //nolint:gosec if err != nil { return err } @@ -166,30 +258,100 @@ func badgesDownloadsAction(c *cli.Context) error { return nil } -func releaseAssetsDownlaodCount( +func getOwnerAndRepo(c *cli.Context) (string, string, error) { + repository := c.String("repository") + if repository == "" { + return "", "", fmt.Errorf( + "No repository specified. Use --repository flag or set " + + "GITHUB_REPOSITORY environment variable.", + ) + } + parts := strings.SplitN(repository, "/", 2) + if len(parts) != 2 { + return "", "", fmt.Errorf( + "Invalid repository name. Expected format: owner/repo", + ) + } + + return parts[0], parts[1], nil +} + +func updateAssetDownloadsCache( + cache *Cache, releases []*github.RepositoryRelease, excludes []*regexp.Regexp, -) int { - count := 0 - +) { for _, release := range releases { for _, asset := range release.Assets { - for _, exclude := range excludes { - if exclude.MatchString(asset.GetName()) { - continue - } + name := asset.GetName() + if anyMatch(name, excludes) { + continue } if v := asset.GetDownloadCount(); v > 0 { - count += v + cache.SetAssetDownloadCount(release.GetName(), name, v) } } } - - return count } -func jsonMarshal(v interface{}) ([]byte, error) { +func anyMatch(s string, patterns []*regexp.Regexp) bool { + for _, p := range patterns { + if p.MatchString(s) { + return true + } + } + + return false +} + +func readCache(filename string) (*Cache, error) { + cache := &Cache{AseetDownloads: map[string]map[string]int{}} + + f, err := os.Open(filename) + if err != nil { + if os.IsNotExist(err) { + return cache, nil + } + + return nil, err + } + defer func() { + if e := f.Close(); e != nil { + err = e + } + }() + + err = json.NewDecoder(f).Decode(&cache) + if err != nil { + return nil, err + } + + return cache, nil +} + +func writeCache(filename string, cache *Cache) (err error) { + f, err := os.Create(filename) + if err != nil { + return err + } + defer func() { + if e := f.Close(); e != nil { + err = e + } + }() + + enc := json.NewEncoder(f) + enc.SetIndent("", " ") + err = enc.Encode(cache) + if err != nil { + return err + } + + return nil +} + +func jsonPrettyMarshal(v interface{}) ([]byte, error) { var buf bytes.Buffer enc := json.NewEncoder(&buf) enc.SetIndent("", " ")