From adbcfc6fc433fcc99b10dc5ccb51ba458333fa9c Mon Sep 17 00:00:00 2001 From: Jim Myhrberg Date: Mon, 28 Jun 2021 00:02:04 +0100 Subject: [PATCH] feat(cask): add cask update command to manage cask formula This will be used by the jimeh/homebrew-emacs-builds brew tap repository in combination with brew livecheck to automatically update cask formulas to the latest nightly builds from the jimeh/emacs-builds repository. --- go.mod | 1 + go.sum | 2 + pkg/cask/live_check.go | 13 ++ pkg/cask/release_info.go | 60 +++++ pkg/cask/update.go | 487 +++++++++++++++++++++++++++++++++++++++ pkg/cli/cask.go | 149 ++++++++++++ pkg/cli/cli.go | 1 + 7 files changed, 713 insertions(+) create mode 100644 pkg/cask/live_check.go create mode 100644 pkg/cask/release_info.go create mode 100644 pkg/cask/update.go create mode 100644 pkg/cli/cask.go diff --git a/go.mod b/go.mod index 8a6f862..fc79c9a 100644 --- a/go.mod +++ b/go.mod @@ -13,6 +13,7 @@ require ( github.com/hashicorp/go-hclog v0.16.1 github.com/hashicorp/go-multierror v1.1.1 // indirect github.com/hashicorp/go-retryablehttp v0.7.0 // indirect + github.com/hexops/gotextdiff v1.0.3 github.com/jimeh/undent v1.1.0 github.com/mattn/go-isatty v0.0.13 // indirect github.com/mitchellh/gon v0.2.3 diff --git a/go.sum b/go.sum index 9602a2c..7db7107 100644 --- a/go.sum +++ b/go.sum @@ -136,6 +136,8 @@ github.com/hashicorp/go-retryablehttp v0.7.0/go.mod h1:vAew36LZh98gCBJNLH42IQ1ER github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/hcl/v2 v2.0.0/go.mod h1:oVVDG71tEinNGYCxinCYadcmKU9bglqW9pV3txagJ90= +github.com/hexops/gotextdiff v1.0.3 h1:gitA9+qJrrTCsiCl7+kh75nPqQt1cx4ZkudSTLoUqJM= +github.com/hexops/gotextdiff v1.0.3/go.mod h1:pSWU5MAI3yDq+fZBTazCSJysOMbxWL1BSow5/V2vxeg= github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= github.com/jimeh/undent v1.1.0 h1:Cge7P4Ws6buy0SVuHBluY/aOKdFuJUMzoJswfAHZ4zE= diff --git a/pkg/cask/live_check.go b/pkg/cask/live_check.go new file mode 100644 index 0000000..a2d66aa --- /dev/null +++ b/pkg/cask/live_check.go @@ -0,0 +1,13 @@ +package cask + +type LiveCheck struct { + Cask string `json:"cask"` + Version LiveCheckVersion `json:"version"` +} + +type LiveCheckVersion struct { + Current string `json:"current"` + Latest string `json:"latest"` + Outdated bool `json:"outdated"` + NewerThanUpstream bool `json:"newer_than_upstream"` +} diff --git a/pkg/cask/release_info.go b/pkg/cask/release_info.go new file mode 100644 index 0000000..35d3ed4 --- /dev/null +++ b/pkg/cask/release_info.go @@ -0,0 +1,60 @@ +package cask + +import ( + "sort" + "strings" +) + +type ReleaseInfo struct { + Name string + Version string + Assets map[string]*ReleaseAsset +} + +func (s *ReleaseInfo) Asset(nameMatch string) *ReleaseAsset { + if a, ok := s.Assets[nameMatch]; ok { + return a + } + + // Dirty and inefficient way to ensure assets are searched in a predictable + // order. + var assets []*ReleaseAsset + for _, a := range s.Assets { + assets = append(assets, a) + } + sort.SliceStable(assets, func(i, j int) bool { + return assets[i].Filename < assets[j].Filename + }) + + for _, a := range assets { + if strings.Contains(a.Filename, nameMatch) { + return a + } + } + + return nil +} + +func (s *ReleaseInfo) DownloadURL(nameMatch string) string { + a := s.Asset(nameMatch) + if a == nil { + return "" + } + + return a.DownloadURL +} + +func (s *ReleaseInfo) SHA256(nameMatch string) string { + a := s.Asset(nameMatch) + if a == nil { + return "" + } + + return a.SHA256 +} + +type ReleaseAsset struct { + Filename string + DownloadURL string + SHA256 string +} diff --git a/pkg/cask/update.go b/pkg/cask/update.go new file mode 100644 index 0000000..ca77b03 --- /dev/null +++ b/pkg/cask/update.go @@ -0,0 +1,487 @@ +package cask + +import ( + "bytes" + "context" + "errors" + "fmt" + "io" + "net/http" + "os" + "path/filepath" + "strings" + "text/template" + "time" + + "github.com/google/go-github/v35/github" + "github.com/hashicorp/go-hclog" + "github.com/hexops/gotextdiff" + "github.com/hexops/gotextdiff/myers" + "github.com/hexops/gotextdiff/span" + "github.com/jimeh/build-emacs-for-macos/pkg/gh" + "github.com/jimeh/build-emacs-for-macos/pkg/repository" +) + +// Error vars +var ( + Err = errors.New("cask") + ErrReleaseNotFound = fmt.Errorf("%w: release not found", Err) + + ErrFailedSHA256Parse = fmt.Errorf( + "%w: failed to parse SHA256 from asset", Err, + ) + ErrFailedSHA256Download = fmt.Errorf( + "%w: failed to download SHA256 asset", Err, + ) + ErrNoTapOrOutput = fmt.Errorf( + "%w: no tap repository or output directory specified", Err, + ) +) + +type UpdateOptions struct { + // BuildsRepo is the GitHub repository containing binary releases. + BuildsRepo *repository.Repository + + // TapRepo is the GitHub repository to update the cask formula in. + TapRepo *repository.Repository + + // Ref is the git ref to apply cask formula updates on top of. Default + // branch will be used if empty. + Ref string + + // OutputDir specifies a directory to write cask files to. When set, tap + // repository is ignored and no changes will be committed directly against + // any specified tap repository. + OutputDir string + + // Force update will ignore the outdated live check flag, and process all + // casks regardless. But it will only update the cask in question if the + // resulting output cask formula is different. + Force bool + + // TemplatesDir is the directory where cask formula templates are located. + TemplatesDir string + + LiveChecks []*LiveCheck + + GithubToken string +} + +type Updater struct { + BuildsRepo *repository.Repository + TapRepo *repository.Repository + Ref string + OutputDir string + TemplatesDir string + + logger hclog.Logger + gh *github.Client +} + +func Update(ctx context.Context, opts *UpdateOptions) error { + updater := &Updater{ + BuildsRepo: opts.BuildsRepo, + TapRepo: opts.TapRepo, + Ref: opts.Ref, + OutputDir: opts.OutputDir, + TemplatesDir: opts.TemplatesDir, + logger: hclog.FromContext(ctx).Named("cask"), + gh: gh.New(ctx, opts.GithubToken), + } + + for _, chk := range opts.LiveChecks { + err := updater.Update(ctx, chk, opts.Force) + if err != nil { + return err + } + } + + return nil +} + +func (s *Updater) Update( + ctx context.Context, + chk *LiveCheck, + force bool, +) error { + if s.TapRepo == nil && s.OutputDir == "" { + return ErrNoTapOrOutput + } + + if !force && !chk.Version.Outdated { + s.logger.Info("skipping", "cask", chk.Cask, "reason", "up to date") + + return nil + } + + newCaskContent, err := s.renderCask(ctx, chk) + if err != nil { + return err + } + + caskFile := chk.Cask + ".rb" + + if s.OutputDir != "" { + _, err = s.putFile( + ctx, chk, filepath.Join(s.OutputDir, caskFile), newCaskContent, + ) + if err != nil { + return err + } + + return nil + } + + _, err = s.putRepoFile( + ctx, s.TapRepo, s.Ref, chk, + filepath.Join("Casks", caskFile), newCaskContent, + ) + if err != nil { + return err + } + + return nil +} + +func (s *Updater) putFile( + ctx context.Context, + chk *LiveCheck, + filename string, + content []byte, +) (bool, error) { + parent := filepath.Dir(filename) + s.logger.Info("processing formula update", + "output-directory", parent, "cask", chk.Cask, "file", filename, + ) + + err := os.MkdirAll(parent, 0o755) + if err != nil { + return false, err + } + + existingContent, err := os.ReadFile(filename) + if err != nil && !os.IsNotExist(err) { + return false, err + } + + infoMsg := "creating formula" + + if !os.IsNotExist(err) { + infoMsg = "updating formula" + if bytes.Equal(existingContent, content) { + s.logger.Info( + "skip update: no change to cask formula content", + "cask", chk.Cask, "file", filename, + ) + + s.logger.Debug( + "formula content", + "file", filename, "content", string(content), + ) + + return false, nil + } + } + + existing := string(existingContent) + edits := myers.ComputeEdits( + span.URIFromPath(filename), existing, string(content), + ) + diff := fmt.Sprint(gotextdiff.ToUnified( + filename, filename, existing, edits, + )) + + s.logger.Info( + infoMsg, + "cask", chk.Cask, "version", chk.Version.Latest, "file", filename, + "diff", diff, + ) + + s.logger.Debug( + "formula content", + "file", filename, "content", string(content), + ) + + err = os.WriteFile(filename, content, 0o644) //nolint:gosec + if err != nil { + return false, err + } + + return true, nil +} + +func (s *Updater) putRepoFile( + ctx context.Context, + repo *repository.Repository, + ref string, + chk *LiveCheck, + filename string, + content []byte, +) (bool, error) { + s.logger.Info("processing formula update", + "tap-repo", repo.Source, "cask", chk.Cask, "file", filename, + ) + repoContent, _, resp, err := s.gh.Repositories.GetContents( + ctx, repo.Owner(), repo.Name(), filename, + &github.RepositoryContentGetOptions{Ref: ref}, + ) + if err != nil && resp.StatusCode != http.StatusNotFound { + return false, err + } + + if resp.StatusCode == http.StatusNotFound { + err := s.createRepoFile(ctx, repo, chk, filename, content) + if err != nil { + return false, err + } + } else { + _, err := s.updateRepoFile( + ctx, repo, repoContent, chk, filename, content, + ) + if err != nil { + return false, err + } + } + + return true, nil +} + +func (s *Updater) createRepoFile( + ctx context.Context, + repo *repository.Repository, + chk *LiveCheck, + filename string, + content []byte, +) error { + commitMsg := fmt.Sprintf( + "feat(cask): create %s with version %s", + chk.Cask, chk.Version.Latest, + ) + + edits := myers.ComputeEdits( + span.URIFromPath(filename), "", string(content), + ) + diff := fmt.Sprint(gotextdiff.ToUnified(filename, filename, "", edits)) + + s.logger.Info( + "creating formula", + "cask", chk.Cask, "version", chk.Version.Latest, "file", filename, + "diff", diff, + ) + s.logger.Debug( + "formula content", + "file", filename, "content", string(content), + ) + contResp, _, err := s.gh.Repositories.CreateFile( + ctx, repo.Owner(), repo.Name(), filename, + &github.RepositoryContentFileOptions{ + Message: &commitMsg, + Content: content, + }, + ) + if err != nil { + return err + } + + s.logger.Info( + "new commit created", + "commit", contResp.GetSHA(), "message", contResp.GetMessage(), + "url", contResp.Commit.GetHTMLURL(), + ) + + return nil +} + +func (s *Updater) updateRepoFile( + ctx context.Context, + repo *repository.Repository, + repoContent *github.RepositoryContent, + chk *LiveCheck, + filename string, + content []byte, +) (bool, error) { + existingContent, err := repoContent.GetContent() + if err != nil { + return false, err + } + + if existingContent == string(content) { + s.logger.Info( + "skip update: no change to formula content", + "cask", chk.Cask, "file", filename, + ) + + return false, nil + } + + sha := repoContent.GetSHA() + + commitMsg := fmt.Sprintf( + "feat(cask): update %s to version %s", + chk.Cask, chk.Version.Latest, + ) + + edits := myers.ComputeEdits( + span.URIFromPath(filename), existingContent, string(content), + ) + diff := fmt.Sprint(gotextdiff.ToUnified( + filename, filename, existingContent, edits, + )) + + s.logger.Info( + "updating formula", + "cask", chk.Cask, "version", chk.Version.Latest, "file", filename, + "diff", diff, + ) + s.logger.Debug( + "formula content", + "file", filename, "content", string(content), + ) + + contResp, _, err := s.gh.Repositories.CreateFile( + ctx, repo.Owner(), repo.Name(), filename, + &github.RepositoryContentFileOptions{ + Message: &commitMsg, + Content: content, + SHA: &sha, + }, + ) + if err != nil { + return false, err + } + + s.logger.Info( + "new commit created", + "commit", contResp.GetSHA(), "message", contResp.GetMessage(), + "url", contResp.Commit.GetHTMLURL(), + ) + + return true, nil +} + +func (s *Updater) renderCask( + ctx context.Context, + chk *LiveCheck, +) ([]byte, error) { + releaseName := "Emacs." + chk.Version.Latest + + s.logger.Info("fetching release details", "release", releaseName) + release, resp, err := s.gh.Repositories.GetReleaseByTag( + ctx, s.BuildsRepo.Owner(), s.BuildsRepo.Name(), releaseName, + ) + if err != nil { + return nil, err + } + if release == nil || resp.StatusCode == http.StatusNotFound { + return nil, fmt.Errorf("%w: %s", ErrReleaseNotFound, releaseName) + } + + info := &ReleaseInfo{ + Name: release.GetName(), + Version: chk.Version.Latest, + Assets: map[string]*ReleaseAsset{}, + } + + s.logger.Info("processing release assets") + for _, asset := range release.Assets { + filename := asset.GetName() + s.logger.Debug("processing asset", "filename", filename) + + if strings.HasSuffix(filename, ".sha256") { + filename = strings.TrimSuffix(filename, ".sha256") + } + + if _, ok := info.Assets[filename]; !ok { + info.Assets[filename] = &ReleaseAsset{ + Filename: filename, + } + } + + if strings.HasSuffix(asset.GetName(), ".sha256") { + s.logger.Debug("downloading *.sha256 asset to extract SHA256 value") + r, err2 := s.downloadAssetContent(ctx, asset) + if err2 != nil { + return nil, err2 + } + defer r.Close() + + content := make([]byte, 64) + n, err2 := io.ReadAtLeast(r, content, 64) + if err2 != nil { + return nil, err2 + } + if n < 64 { + return nil, fmt.Errorf( + "%w: %s", ErrFailedSHA256Parse, asset.GetName(), + ) + } + + sha := string(content)[0:64] + if sha == "" { + return nil, fmt.Errorf( + "%w: %s", ErrFailedSHA256Parse, asset.GetName(), + ) + } + + info.Assets[filename].SHA256 = sha + } else { + info.Assets[filename].DownloadURL = asset.GetBrowserDownloadURL() + } + } + + templateFile := filepath.Join(s.TemplatesDir, chk.Cask+".rb.tpl") + tplContent, err := os.ReadFile(templateFile) + if err != nil { + return nil, err + } + + tpl, err := template.New(chk.Cask).Parse(string(tplContent)) + if err != nil { + return nil, err + } + + var buf bytes.Buffer + err = tpl.Execute(&buf, info) + if err != nil { + return nil, err + } + + return buf.Bytes(), nil +} + +func (s *Updater) downloadAssetContent( + ctx context.Context, + asset *github.ReleaseAsset, +) (io.ReadCloser, error) { + httpClient := &http.Client{Timeout: 60 * time.Second} + + r, downloadURL, err := s.gh.Repositories.DownloadReleaseAsset( + ctx, s.BuildsRepo.Owner(), s.BuildsRepo.Name(), + asset.GetID(), httpClient, + ) + if err != nil { + return nil, err + } + + if r == nil && downloadURL != "" { + req, err := http.NewRequestWithContext(ctx, "GET", downloadURL, nil) + if err != nil { + return nil, err + } + + //nolint:bodyclose + resp, err := httpClient.Do(req) + if err != nil { + return nil, err + } + r = resp.Body + } + + if r == nil { + return nil, fmt.Errorf( + "%s: %s", ErrFailedSHA256Download, asset.GetName(), + ) + } + + return r, nil +} diff --git a/pkg/cli/cask.go b/pkg/cli/cask.go new file mode 100644 index 0000000..4afc4ea --- /dev/null +++ b/pkg/cli/cask.go @@ -0,0 +1,149 @@ +package cli + +import ( + "encoding/json" + "errors" + "os" + + "github.com/jimeh/build-emacs-for-macos/pkg/cask" + "github.com/jimeh/build-emacs-for-macos/pkg/repository" + cli2 "github.com/urfave/cli/v2" +) + +type caskOptions struct { + BuildsRepo *repository.Repository + GithubToken string +} + +func caskCmd() *cli2.Command { + tokenDefaultText := "" + if len(os.Getenv("GITHUB_TOKEN")) > 0 { + tokenDefaultText = "***" + } + + return &cli2.Command{ + Name: "cask", + Usage: "manage Homebrew Cask formula", + Flags: []cli2.Flag{ + &cli2.StringFlag{ + Name: "builds-repository", + Aliases: []string{"builds-repo", "b"}, + Usage: "owner/name of GitHub repo for containing builds", + EnvVars: []string{"EMACS_BUILDS_REPOSITORY"}, + Value: "jimeh/emacs-builds", + }, + &cli2.StringFlag{ + Name: "github-token", + Usage: "GitHub API Token", + EnvVars: []string{"GITHUB_TOKEN"}, + DefaultText: tokenDefaultText, + Required: true, + }, + }, + Subcommands: []*cli2.Command{ + caskUpdateCmd(), + }, + } +} + +func caskActionWrapper( + f func(*cli2.Context, *Options, *caskOptions) error, +) func(*cli2.Context) error { + return actionWrapper(func(c *cli2.Context, opts *Options) error { + rOpts := &caskOptions{ + GithubToken: c.String("github-token"), + } + + if r := c.String("builds-repository"); r != "" { + var err error + rOpts.BuildsRepo, err = repository.NewGitHub(r) + if err != nil { + return err + } + } + + return f(c, opts, rOpts) + }) +} + +func caskUpdateCmd() *cli2.Command { + return &cli2.Command{ + Name: "update", + Usage: "update casks based on brew livecheck result in JSON format", + ArgsUsage: "", + Flags: []cli2.Flag{ + &cli2.StringFlag{ + Name: "ref", + Usage: "git ref to create/update casks on top of in the " + + "tap repository", + EnvVars: []string{"GITHUB_REF"}, + }, + &cli2.StringFlag{ + Name: "output", + Aliases: []string{"o"}, + Usage: "directory to write cask files to", + }, + &cli2.StringFlag{ + Name: "tap-repository", + Aliases: []string{"tap"}, + Usage: "owner/name of GitHub repo for Homebrew Tap to " + + "commit changes to if --output is not set", + EnvVars: []string{"GITHUB_REPOSITORY"}, + }, + &cli2.StringFlag{ + Name: "templates-dir", + Aliases: []string{"t"}, + Usage: "path to directory of cask templates", + EnvVars: []string{"CASK_TEMPLATE_DIR"}, + Required: true, + }, + }, + Action: caskActionWrapper(caskUpdateAction), + } +} + +func caskUpdateAction( + c *cli2.Context, + opts *Options, + cOpts *caskOptions, +) error { + updateOpts := &cask.UpdateOptions{ + BuildsRepo: cOpts.BuildsRepo, + GithubToken: cOpts.GithubToken, + Ref: c.String("ref"), + OutputDir: c.String("output"), + TemplatesDir: c.String("templates-dir"), + } + + if r := c.String("tap-repository"); r != "" { + var err error + updateOpts.TapRepo, err = repository.NewGitHub(r) + if err != nil { + return err + } + } + + arg := c.Args().First() + if arg == "" { + return errors.New("no livecheck argument given") + } + + if arg == "-" { + err := json.NewDecoder(c.App.Reader).Decode(&updateOpts.LiveChecks) + if err != nil { + return err + } + } else { + f, err := os.Open(arg) + if err != nil { + return err + } + + err = json.NewDecoder(f).Decode(&updateOpts.LiveChecks) + if err != nil { + return err + } + } + + return cask.Update(c.Context, updateOpts) +} diff --git a/pkg/cli/cli.go b/pkg/cli/cli.go index 9d54855..2e29336 100644 --- a/pkg/cli/cli.go +++ b/pkg/cli/cli.go @@ -49,6 +49,7 @@ func New(version, commit, date string) *CLI { notarizeCmd(), packageCmd(), releaseCmd(), + caskCmd(), { Name: "version", Usage: "print the version",