fix(badge/downloads): fix issues with total download counts

Excluded patterns had a bug where they basically did not work. This
means total downloads were inflated by also counting downloads for all
sha256 files.

Also the first 1000 limit when listing repository releases seems to no
loner be a thing, so we can iterate over all pages.

And finally, implement a caching mechanism so we can keep counting
releases in case they are removed, or the first 1000 releases limit is
enabled again in GitHub's API.

go.mod

@@ -1,6 +1,6 @@
 module emacs-builds-meta
 
-go 1.20
+go 1.21
 
 require (
 	github.com/dustin/go-humanize v1.0.1

go.sum

@@ -9,6 +9,7 @@ github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiu
 github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38=
+github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
 github.com/google/go-github v17.0.0+incompatible h1:N0LgJ1j65A7kfXrZnUDaYCs/Sf4rEjNlfyDHW9dolSY=
 github.com/google/go-github v17.0.0+incompatible/go.mod h1:zLgOLi98H3fifZn+44m+umXrS52loVEgC2AApnigrVQ=
 github.com/google/go-querystring v1.1.0 h1:AnCroh3fv4ZBgVIf1Iwtovgjaw/GiKJo8M8yD/fhyJ8=

main.go

@@ -5,8 +5,8 @@ import (
 	"context"
 	"encoding/json"
 	"fmt"
-	"io/ioutil"
 	"log"
+	"log/slog"
 	"os"
 	"path/filepath"
 	"regexp"
@@ -32,6 +32,49 @@ const downloadIcon = `` +
 	`<line class="a" x1="3" y1="5.25" x2="7" y2="9.25"/>` +
 	`</svg>`
 
+type Cache struct {
+	AseetDownloads map[string]map[string]int `json:"asset_downloads,omitempty"`
+}
+
+func (c *Cache) AssetsCount() int {
+	var count int
+
+	for _, v := range c.AseetDownloads {
+		count += len(v)
+	}
+
+	return count
+}
+
+func (c *Cache) AssetReleasesCount() int {
+	return len(c.AseetDownloads)
+}
+
+func (c *Cache) SetAssetDownloadCount(
+	releaseName, assetName string,
+	count int,
+) {
+	if c.AseetDownloads == nil {
+		c.AseetDownloads = map[string]map[string]int{}
+	}
+	if c.AseetDownloads[releaseName] == nil {
+		c.AseetDownloads[releaseName] = map[string]int{}
+	}
+	c.AseetDownloads[releaseName][assetName] = count
+}
+
+func (c *Cache) TotalAssetDownloads() int {
+	var count int
+
+	for _, v := range c.AseetDownloads {
+		for _, v := range v {
+			count += v
+		}
+	}
+
+	return count
+}
+
 type Badge struct {
 	SchemaVersion int    `json:"schemaVersion,omitempty"`
 	Label         string `json:"label,omitempty"`
@@ -67,14 +110,19 @@ func NewApp() *cli.App {
 						Name: "downloads",
 						Flags: []cli.Flag{
 							&cli.StringSliceFlag{
-								Name:  "exclude",
-								Usage: "regexp asset filename pattern to exclude",
-								Value: cli.NewStringSlice(`.+\.sha\d+$`),
+								Name:    "exclude",
+								Aliases: []string{"e"},
+								Usage:   "regexp asset filename pattern to exclude",
+								Value:   cli.NewStringSlice(`.+\.sha\d+$`),
 							},
 							&cli.StringFlag{
 								Name:    "output",
 								Aliases: []string{"o"},
 							},
+							&cli.StringFlag{
+								Name:    "cache",
+								Aliases: []string{"c"},
+							},
 						},
 						Action: badgesDownloadsAction,
 					},
@@ -104,14 +152,36 @@ func badgesDownloadsAction(c *cli.Context) error {
 		excludes = append(excludes, regexp.MustCompile(s))
 	}
 
-	parts := strings.SplitN(c.String("repository"), "/", 2)
-	owner := parts[0]
-	repo := parts[1]
+	owner, repo, err := getOwnerAndRepo(c)
+	if err != nil {
+		return err
+	}
 
-	count := 0
+	cache := &Cache{}
+	cacheFile := c.String("cache")
 
-	for page := 1; page > 0 && page < 11; {
-		releases, resp, err := gh.Repositories.ListReleases(
+	if cacheFile != "" {
+		slog.Info("reading cache", slog.String("file", cacheFile))
+
+		cache, err = readCache(cacheFile)
+		if err != nil {
+			return err
+		}
+
+		slog.Info("cache stats",
+			slog.Int("downloads", cache.TotalAssetDownloads()),
+			slog.Int("releases", cache.AssetReleasesCount()),
+			slog.Int("assets", cache.AssetsCount()),
+		)
+	}
+
+	lastPage := 1
+	for page := 1; page > 0 && page <= lastPage; {
+		slog.Info("fetching releases", slog.Int("page", page))
+
+		var releases []*github.RepositoryRelease
+		var resp *github.Response
+		releases, resp, err = gh.Repositories.ListReleases(
 			c.Context, owner, repo, &github.ListOptions{
 				Page:    page,
 				PerPage: 100,
@@ -121,10 +191,27 @@ func badgesDownloadsAction(c *cli.Context) error {
 			return err
 		}
 
-		count += releaseAssetsDownlaodCount(releases, excludes)
+		updateAssetDownloadsCache(cache, releases, excludes)
 		page = resp.NextPage
+		lastPage = resp.LastPage
 	}
 
+	if cacheFile != "" {
+		slog.Info("writing cache", slog.String("file", cacheFile))
+		slog.Info(
+			"cache stats",
+			slog.Int("downloads", cache.TotalAssetDownloads()),
+			slog.Int("releases", cache.AssetReleasesCount()),
+			slog.Int("assets", cache.AssetsCount()),
+		)
+
+		err = writeCache(cacheFile, cache)
+		if err != nil {
+			return err
+		}
+	}
+
+	count := cache.TotalAssetDownloads()
 	var humanCount string
 	if count >= 1000 {
 		v, sym := humanize.ComputeSI(float64(count))
@@ -133,6 +220,11 @@ func badgesDownloadsAction(c *cli.Context) error {
 		humanCount = strconv.Itoa(count)
 	}
 
+	slog.Info("total downloads",
+		slog.Int("count", count),
+		slog.String("humanized_count", humanCount),
+	)
+
 	badge := &Badge{
 		SchemaVersion: 1,
 		Style:         "flat",
@@ -142,7 +234,7 @@ func badgesDownloadsAction(c *cli.Context) error {
 		Message:       humanCount,
 	}
 
-	b, err := jsonMarshal(badge)
+	b, err := jsonPrettyMarshal(badge)
 	if err != nil {
 		return err
 	}
@@ -155,7 +247,7 @@ func badgesDownloadsAction(c *cli.Context) error {
 				return err
 			}
 		}
-		err := ioutil.WriteFile(filename, b, 0o644) //nolint:gosec
+		err := os.WriteFile(filename, b, 0o644) //nolint:gosec
 		if err != nil {
 			return err
 		}
@@ -166,30 +258,100 @@ func badgesDownloadsAction(c *cli.Context) error {
 	return nil
 }
 
-func releaseAssetsDownlaodCount(
+func getOwnerAndRepo(c *cli.Context) (string, string, error) {
+	repository := c.String("repository")
+	if repository == "" {
+		return "", "", fmt.Errorf(
+			"No repository specified. Use --repository flag or set " +
+				"GITHUB_REPOSITORY environment variable.",
+		)
+	}
+	parts := strings.SplitN(repository, "/", 2)
+	if len(parts) != 2 {
+		return "", "", fmt.Errorf(
+			"Invalid repository name. Expected format: owner/repo",
+		)
+	}
+
+	return parts[0], parts[1], nil
+}
+
+func updateAssetDownloadsCache(
+	cache *Cache,
 	releases []*github.RepositoryRelease,
 	excludes []*regexp.Regexp,
-) int {
-	count := 0
-
+) {
 	for _, release := range releases {
 		for _, asset := range release.Assets {
-			for _, exclude := range excludes {
-				if exclude.MatchString(asset.GetName()) {
-					continue
-				}
+			name := asset.GetName()
+			if anyMatch(name, excludes) {
+				continue
 			}
 
 			if v := asset.GetDownloadCount(); v > 0 {
-				count += v
+				cache.SetAssetDownloadCount(release.GetName(), name, v)
 			}
 		}
 	}
-
-	return count
 }
 
-func jsonMarshal(v interface{}) ([]byte, error) {
+func anyMatch(s string, patterns []*regexp.Regexp) bool {
+	for _, p := range patterns {
+		if p.MatchString(s) {
+			return true
+		}
+	}
+
+	return false
+}
+
+func readCache(filename string) (*Cache, error) {
+	cache := &Cache{AseetDownloads: map[string]map[string]int{}}
+
+	f, err := os.Open(filename)
+	if err != nil {
+		if os.IsNotExist(err) {
+			return cache, nil
+		}
+
+		return nil, err
+	}
+	defer func() {
+		if e := f.Close(); e != nil {
+			err = e
+		}
+	}()
+
+	err = json.NewDecoder(f).Decode(&cache)
+	if err != nil {
+		return nil, err
+	}
+
+	return cache, nil
+}
+
+func writeCache(filename string, cache *Cache) (err error) {
+	f, err := os.Create(filename)
+	if err != nil {
+		return err
+	}
+	defer func() {
+		if e := f.Close(); e != nil {
+			err = e
+		}
+	}()
+
+	enc := json.NewEncoder(f)
+	enc.SetIndent("", "  ")
+	err = enc.Encode(cache)
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func jsonPrettyMarshal(v interface{}) ([]byte, error) {
 	var buf bytes.Buffer
 	enc := json.NewEncoder(&buf)
 	enc.SetIndent("", "  ")