build-emacs-for-macos/build-emacs-for-macos

#!/usr/bin/env ruby
# frozen_string_literal: true

require 'English'
require 'date'
require 'erb'
require 'etc'
require 'fileutils'
require 'json'
require 'net/http'
require 'optparse'
require 'pathname'
require 'uri'

class Error < StandardError; end

module Output
  def info(msg, newline: true)
    out "INFO: #{msg}", newline: newline
  end

  def out(msg, newline: true)
    if newline
      puts "==> #{msg}"
    else
      print "==> #{msg}"
    end
  end

  def err(msg = nil)
    raise Error, msg
  end
end

class OS
  def self.version
    @version ||= OSVersion.new
  end

  def self.arch
    @arch ||= `uname -m`.strip
  end
end

class OSVersion
  def initialize
    @version = `sw_vers -productVersion`.match(
      /(?<major>\d+)(?:\.(?<minor>\d+)(:?\.(?<patch>\d+))?)?/
    )
  end

  def to_s
    @to_s ||= "#{major}.#{minor}"
  end

  def major
    @major ||= @version[:major]&.to_i
  end

  def minor
    @minor ||= @version[:minor]&.to_i
  end

  def patch
    @patch ||= @version[:patch]&.to_i
  end
end

class Build
  include Output

  DOWNLOAD_URL = 'https://github.com/emacs-mirror/emacs/tarball/%s'
  LATEST_URL = 'https://api.github.com/repos/emacs-mirror/emacs/commits/%s'
  NATIVE_COMP_REF_REGEXP = %r{^feature/native-comp}.freeze

  attr_reader :root_dir
  attr_reader :source_dir
  attr_reader :ref
  attr_reader :options

  def initialize(root_dir, ref = 'master', options = {})
    @root_dir = root_dir
    @ref = ref
    @options = options
  end

  def build
    unless meta[:sha] && meta[:date]
      err 'Failed to get commit info from GitHub API.'
    end

    tarball = download_tarball(meta[:sha])
    @source_dir = extract_tarball(tarball, patches(options))

    autogen
    detect_native_comp if options[:native_comp].nil?

    app = compile_source(@source_dir)
    symlink_internals(app)

    LibEmbedder.new(app, brew_dir, extra_libs).embed
    GccLibEmbedder.new(app, gcc_dir).embed if options[:native_comp]

    archive_app(app)
  end

  private

  def tarballs_dir
    @tarballs_dir ||= File.join(root_dir, 'tarballs')
  end

  def sources_dir
    @sources_dir ||= File.join(root_dir, 'sources')
  end

  def builds_dir
    @builds_dir ||= File.join(root_dir, 'builds')
  end

  def brew_dir
    @brew_dir ||= `brew --prefix`.chomp
  end

  def gcc_dir
    @gcc_dir ||= `brew --prefix gcc`.chomp
  end

  def libgccjit_dir
    @libgccjit_dir ||= `brew --prefix libgccjit`.chomp
  end

  def extra_libs
    @extra_libs ||= [
      "#{brew_dir}/opt/expat/lib/libexpat.1.dylib",
      "#{brew_dir}/opt/libiconv/lib/libiconv.2.dylib",
      "#{brew_dir}/opt/zlib/lib/libz.1.dylib"
    ]
  end

  def download_tarball(sha)
    FileUtils.mkdir_p(tarballs_dir)

    url = (DOWNLOAD_URL % sha)
    filename = "emacs-mirror-emacs-#{sha[0..6]}.tgz"
    target = File.join(tarballs_dir, filename)

    if File.exist?(target)
      info "#{filename} already exists locally, attempting to use."
      return target
    end

    info 'Downloading tarball from GitHub. This could take a while, ' \
         'please be patient.'
    result = run_cmd('curl', '-L', url, '-o', target)
    err 'Download failed.' unless result

    target
  end

  def extract_tarball(filename, patches = [])
    FileUtils.mkdir_p(sources_dir)

    dirname = File.basename(filename).gsub(/\.\w+$/, '')
    target = File.join(sources_dir, dirname)

    if File.exist?(target)
      info "#{dirname} source tree exists, attempting to use."
      return target
    end

    info 'Extracting tarball...'
    result = run_cmd('tar', '-xzf', filename, '-C', sources_dir)
    err 'Tarball extraction failed.' unless result

    patches.each { |patch| apply_patch(patch, target) }

    target
  end

  def configure_help
    return @configure_help if @configure_help

    FileUtils.cd(source_dir) { @configure_help = `./configure --help` }

    @configure_help
  end

  def supports_native_comp?
    @supports_native_comp ||= !!configure_help.match(/\s+--with-nativecomp\s+/)
  end

  def supports_xwidgets?
    @supports_xwidgets ||= !!configure_help.match(/\s+--with-xwidgets\s+/)
  end

  def detect_native_comp
    info 'Detecting native-comp support: ', newline: false
    options[:native_comp] = supports_native_comp?
    puts options[:native_comp] ? 'Supported' : 'Not supported'
  end

  def verify_native_comp
    return if supports_native_comp?

    err 'This emacs source tree does not support native-comp'
  end

  def verify_libgccjit
    err 'gcc not installed' unless Dir.exist?(gcc_dir)
    err 'libgccjit not installed' unless Dir.exist?(libgccjit_dir)

    if Dir["#{libgccjit_dir}/lib/**/libgccjit.so*"].empty?
      err "Detected libgccjit (#{libgccjit_dir}) does not have any " \
          'libgccjit.so* files. Please try reinstalling libgccjit: ' \
          'brew reinstall libgccjit'
    end

    dirs = Dir["{#{gcc_dir},#{libgccjit_dir}}/lib/gcc/*"]
    return if dirs.map { |dir| File.basename(dir) }.uniq == %w[10]

    err 'Detected gcc and libgccjit library paths do not belong to the ' \
        "same major version. Detected paths:\n  - #{dirs.join('  - ')}"
  end

  def gcc_library_paths
    @gcc_library_paths ||= Dir[
      "{#{gcc_dir},#{libgccjit_dir}}/lib/gcc/*",
      "{#{gcc_dir},#{libgccjit_dir}}/lib/gcc/*/gcc/*apple-darwin*/*"
    ].sort_by { |p| [p.size, p] }
  end

  def autogen
    FileUtils.cd(source_dir) do
      if File.exist?('autogen/copy_autogen')
        run_cmd 'autogen/copy_autogen'
      elsif File.exist?('autogen.sh')
        run_cmd './autogen.sh'
      end
    end
  end

  def compile_source(source)
    target = "#{source}/nextstep"
    emacs_app = "#{target}/Emacs.app"

    if File.exist?("#{target}/Emacs.app")
      info 'Emacs.app already exists in ' \
           "\"#{target.gsub(root_dir + '/', '')}\", attempting to use."
      return emacs_app
    end

    info 'Compiling from source. This will take a while...'

    FileUtils.cd(source) do
      if options[:native_comp]
        info 'Compiling with native-comp enabled'
        verify_native_comp
        verify_libgccjit

        apply_native_comp_env_setup_patch(source)

        ENV['CFLAGS'] = [
          "-I#{gcc_dir}/include",
          '-O2',
          '-march=native'
        ].compact.join(' ')

        ENV['LDFLAGS'] = [
          gcc_library_paths.map { |path| "-L#{path}" },
          "-I#{gcc_dir}/include",
          "-I#{libgccjit_dir}/include"
        ].flatten.compact.join(' ')

        ENV['LIBRARY_PATH'] = [
          gcc_library_paths,
          ENV['LIBRARY_PATH']
        ].flatten.compact.join(':')
      end

      ENV['CC'] = 'clang'
      ENV['PKG_CONFIG_PATH'] = [
        "#{brew_dir}/lib/pkgconfig",
        "#{brew_dir}/share/pkgconfig",
        "#{brew_dir}/opt/expat/lib/pkgconfig",
        "#{brew_dir}/opt/libxml2/lib/pkgconfig",
        "#{brew_dir}/opt/ncurses/lib/pkgconfig",
        "#{brew_dir}/opt/zlib/lib/pkgconfig",
        "#{brew_dir}/Homebrew/Library/Homebrew/os/mac/pkgconfig/#{OS.version}",
        ENV['PKG_CONFIG_PATH']
      ].compact.join(':')

      ENV['PATH'] = [
        "#{brew_dir}/opt/make/libexec/gnubin",
        "#{brew_dir}/opt/coreutils/libexec/gnubin",
        "#{brew_dir}/opt/gnu-sed/libexec/gnubin",
        "#{brew_dir}/bin",
        "#{brew_dir}/opt/texinfo/bin",
        ENV['PATH']
      ].compact.join(':')

      configure_flags = [
        '--with-ns',
        '--with-modules',
        '--enable-locallisppath=' \
        '/Library/Application Support/Emacs/${version}/site-lisp:' \
        '/Library/Application Support/Emacs/site-lisp'
      ]
      if options[:xwidgets] && supports_xwidgets?
        configure_flags << '--with-xwidgets'
      end
      configure_flags << '--with-nativecomp' if options[:native_comp]
      configure_flags << '--without-rsvg' unless options[:rsvg]

      run_cmd './configure', *configure_flags

      # Disable aligned_alloc on Mojave and below. See issue:
      # https://github.com/daviderestivo/homebrew-emacs-head/issues/15
      if OS.version.major <= 10 && OS.version.minor <= 14
        info 'Force disabling of aligned_alloc on macOS Mojave (10.14.x) ' \
             'and earlier'
        disable_alligned_alloc
      end

      make_flags = []
      make_flags += ['-j', options[:parallel].to_s] if options[:parallel]

      if options[:native_full_aot]
        info 'Using NATIVE_FULL_AOT=1'
        make_flags << 'NATIVE_FULL_AOT=1'
        ENV.delete('NATIVE_FAST_BOOT')
      else
        ENV.delete('NATIVE_FULL_AOT')
        ENV['NATIVE_FAST_BOOT'] = '1'
      end

      if options[:native_comp]
        make_flags << "BYTE_COMPILE_EXTRA_FLAGS=--eval '(setq comp-speed 2)'"
      end

      run_cmd 'make', *make_flags
      run_cmd 'make', 'install'
    end

    err 'Build failed.' unless File.exist?(emacs_app)

    emacs_app
  end

  def symlink_internals(app)
    return unless options[:native_comp]

    info 'Creating symlinks within Emacs.app needed for native-comp'

    FileUtils.cd(File.join(app, 'Contents')) do
      FileUtils.ln_s('Resources/lisp', 'lisp') unless File.exist?('lisp')

      source = Dir['MacOS/libexec/emacs/**/eln-cache',
                   'MacOS/lib/emacs/**/native-lisp'].first
      target = File.basename(source)
      FileUtils.ln_s(source, target) unless File.exist?(target)
    end
  end

  def archive_app(app)
    FileUtils.mkdir_p(builds_dir)

    metadata = [
      ref.gsub(/\W/, '-'),
      meta[:date],
      meta[:sha][0..6],
      "macOS-#{OS.version}",
      OS.arch
    ]

    filename = "Emacs.app-[#{metadata.join('][')}].tbz"
    target = "#{builds_dir}/#{filename}"

    app_base = File.basename(app)
    app_dir = File.dirname(app)

    if !File.exist?(target)
      info "Creating #{filename} archive in \"#{builds_dir}\"..."
      FileUtils.cd(app_dir) { system('tar', '-cjf', target, app_base) }
    else
      info "#{filename} archive exists in " \
           "#{builds_dir.gsub(root_dir + '/', '')}, skipping archving."
    end
  end

  def disable_alligned_alloc
    filename = 'src/config.h'
    content = File.read(filename)
                  .gsub('#define HAVE_ALIGNED_ALLOC 1',
                        '#undef HAVE_ALIGNED_ALLOC')
                  .gsub('#define HAVE_DECL_ALIGNED_ALLOC 1',
                        '#undef HAVE_DECL_ALIGNED_ALLOC')
                  .gsub('#define HAVE_ALLOCA 1',
                        '#undef HAVE_ALLOCA')
                  .gsub('#define HAVE_ALLOCA_H 1',
                        '#undef HAVE_ALLOCA_H')

    File.open(filename, 'w') { |f| f.write(content) }
  end

  def meta
    return @meta if @meta

    ref_sha = options[:git_sha] || ref
    url = format(LATEST_URL, ref_sha)
    commit_json = http_get(url)
    err "Failed to get commit info about: #{ref_sha}" if commit_json.nil?

    commit = JSON.parse(commit_json)

    @meta = {
      sha: commit['sha'],
      date: Date.parse(commit['commit']['committer']['date'])
    }
  end

  def http_get(url)
    response = Net::HTTP.get_response(URI.parse(url))
    return unless response.code == '200'

    response.body
  end

  def run_cmd(*args)
    out "CMD: #{args.join(' ')}"
    system(*args) || err("Exit code: #{$CHILD_STATUS.exitstatus}")
  end

  def apply_native_comp_env_setup_patch(source)
    term = 'native-compile-setup-environment-variables'
    file = 'lisp/emacs-lisp/comp.el'
    return if `grep '#{term}' '#{file}'`.strip.size.positive?

    apply_patch(
      { file: "#{__dir__}/patches/native-comp-env-setup.patch" },
      source
    )
  end

  def effective_version
    @effective_version ||= begin
      case ref
      when /^emacs-26.*/
        'emacs-26'
      when /^emacs-27.*/
        'emacs-27'
      else
        'emacs-28'
      end
    end
  end

  def patches(opts = {})
    p = []

    if %w[emacs-26 emacs-27 emacs-28].include?(effective_version)
      p << {
        url: 'https://github.com/d12frosted/homebrew-emacs-plus/raw/master/' \
             "patches/#{effective_version}/fix-window-role.patch"
      }
    end

    if %w[emacs-27 emacs-28].include?(effective_version)
      p << {
        url: 'https://github.com/d12frosted/homebrew-emacs-plus/raw/master/' \
             "patches/#{effective_version}/system-appearance.patch"
      }

      if options[:no_titlebar]
        p << {
          url: 'https://github.com/d12frosted/homebrew-emacs-plus/raw/master/' \
               "patches/#{effective_version}/no-titlebar.patch"
        }
      end

      if options[:no_frame_refocus]
        p << {
          url: 'https://github.com/d12frosted/homebrew-emacs-plus/raw/master/' \
               "patches/#{effective_version}/no-frame-refocus-cocoa.patch"
        }
      end
    end

    if effective_version == 'emacs-27'
      p << {
        url: 'https://github.com/d12frosted/homebrew-emacs-plus/raw/master/' \
             "patches/#{effective_version}/ligatures-freeze-fix.patch"
      }

      if opts[:xwidgets]
        p << {
          url: 'https://github.com/d12frosted/homebrew-emacs-plus/raw/master/' \
               "patches/#{effective_version}/xwidgets_webkit_in_cocoa.patch"
        }
      end
    end

    p
  end

  def apply_patch(patch, target)
    err "\"#{target}\" does not exist." unless File.exist?(target)

    if patch[:file]
      info 'Applying patch...'
      FileUtils.cd(target) { run_cmd('patch', '-f', '-p1', '-i', patch[:file]) }
    elsif patch[:url]
      patch_dir = "#{target}/macos_patches"
      run_cmd('mkdir', '-p', patch_dir)

      patch_file = "#{patch_dir}/patch-{num}.diff"
      num = 1
      while File.exist?(patch_file.gsub('{num}', num.to_s.rjust(3, '0')))
        num += 1
      end
      patch_file = patch_file.gsub('{num}', num.to_s.rjust(3, '0'))

      info "Downloading patch: #{patch[:url]}"
      run_cmd('curl', '-L#', patch[:url], '-o', patch_file)

      apply_patch({ file: patch_file }, target)
    elsif patch[:replace]
      err 'Patch replace input error' unless patch[:replace].size == 3

      file, before, after = patch[:replace]
      filepath = File.join(target, file)

      err "\"#{file}\" does not exist in #{target}" unless File.exist?(filepath)

      f = File.open(filepath, 'rb')
      s = f.read
      sub = s.gsub!(before, after)
      err "Replacement filed in #{file}" if sub.nil?

      f.reopen(filepath, 'wb').write(s)
      f.close
    end
  end
end

class AbstractEmbedder
  include Output

  attr_reader :app

  def initialize(app)
    err "#{app} does not exist" unless File.exist?(app)

    @app = app
  end

  private

  def bin
    "#{app}/Contents/MacOS/Emacs"
  end

  def lib_dir
    "#{app}/Contents/MacOS/#{lib_dir_name}"
  end

  def lib_dir_name
    'lib'
  end
end

class LibEmbedder < AbstractEmbedder
  attr_reader :lib_source
  attr_reader :extra_libs

  def initialize(app, lib_source, extra_libs = [])
    super(app)

    @lib_source = lib_source
    @extra_libs = extra_libs
  end

  def embed
    info 'Embedding libraries into Emacs.app'

    binary = "#{bin}-bin" if File.exist?("#{bin}-bin")
    binary ||= bin

    FileUtils.cd(File.dirname(app)) do
      copy_libs(binary)
      copy_extra_libs(extra_libs, binary) if extra_libs.any?
      self_ref_libs(binary)
    end
  end

  private

  def copy_libs(exe, rel_path = nil)
    exe_file = File.basename(exe)
    rel_path ||= Pathname.new(lib_dir).relative_path_from(
      Pathname.new(File.dirname(exe))
    ).to_s

    `otool -L "#{exe}"`.split("\n")[1..-1].each do |line|
      match = line.match(%r{^\s+(.+/(lib[^/ ]+))\s})
      next unless match && match[1].start_with?(lib_source)

      while_writable(exe) do
        if match[2] == exe_file
          system('install_name_tool', '-id',
                 "@executable_path/#{rel_path}/#{match[2]}", exe)
        else
          system('install_name_tool', '-change', match[1],
                 "@executable_path/#{rel_path}/#{match[2]}", exe)
        end
      end

      next if match[2] == exe_file || File.exist?("#{lib_dir}/#{match[2]}")

      FileUtils.mkdir_p(lib_dir)
      FileUtils.cp(match[1], lib_dir)
      copy_libs("#{lib_dir}/#{match[2]}", rel_path)
    end
  end

  def copy_extra_libs(extra_libs, exe, rel_path = nil)
    rel_path ||= Pathname.new(lib_dir).relative_path_from(
      Pathname.new(File.dirname(exe))
    ).to_s

    extra_libs.each do |lib|
      lib_file = File.basename(lib)
      target = "#{lib_dir}/#{lib_file}"
      unless File.exist?(target)
        FileUtils.mkdir_p(lib_dir)
        FileUtils.cp(lib, lib_dir)
      end

      while_writable(target) do
        system('install_name_tool', '-id',
               "@executable_path/#{rel_path}/#{lib_file}", target)
      end

      copy_libs(target, rel_path)
    end
  end

  def self_ref_libs(exe)
    rel_path = Pathname.new(lib_dir).relative_path_from(
      Pathname.new(File.dirname(exe))
    ).to_s
    lib_paths ||= Dir.glob("#{lib_dir}/*").select { |f| File.file?(f) }
    libs = lib_paths.map { |f| File.basename(f) }

    ([exe] + lib_paths).each do |bin_path|
      `otool -L "#{bin_path}"`.split("\n")[1..-1].each do |line|
        match = line.match(%r{^\s+(.+/(lib[^/ ]+))\s})
        next unless match
        next if match[1].start_with?('@executable_path/')
        next unless libs.include?(match[2])

        while_writable(bin_path) do
          system(
            'install_name_tool', '-change', match[1],
            "@executable_path/#{rel_path}/#{match[2]}",
            bin_path
          )
        end
      end
    end
  end

  def while_writable(file)
    mode = File.stat(file).mode
    File.chmod(0o775, file)
    yield
    File.chmod(mode, file)
  end
end

class GccLibEmbedder < AbstractEmbedder
  attr_reader :gcc_dir

  def initialize(app, gcc_dir)
    super(app)
    @gcc_dir = gcc_dir
  end

  def embed
    if embedded?
      info 'libgccjit already embedded in Emacs.app'
      return
    end

    info 'Embedding libgccjit into Emacs.app'
    if gcc_version.empty?
      err "No suitable GCC lib with libgccjit found in #{gcc_dir}"
    end

    FileUtils.mkdir_p(File.dirname(target_dir))
    FileUtils.cp_r(source_dir, target_dir)
    FileUtils.rm(Dir[File.join(target_dir, '**', '.DS_Store')], force: true)
  end

  private

  def embedded?
    Dir[File.join(target_dir, 'libgccjit.so*')].any?
  end

  def target_dir
    File.join(lib_dir, 'gcc', gcc_version)
  end

  def gcc_version
    @gcc_version ||= Dir[File.join(gcc_dir, 'lib', 'gcc', '*', 'libgcc*')]
                     .map { |path| File.basename(File.dirname(path)) }
                     .select { |path| path.match(/^\d+$/) }
                     .uniq.map(&:to_i).max.to_s
  end

  def source_dir
    @source_dir ||= File.join(gcc_dir, 'lib', 'gcc', gcc_version)
  end
end

if __FILE__ == $PROGRAM_NAME
  cli_options = {
    macos_fixes: true,
    native_full_aot: false,
    parallel: Etc.nprocessors,
    rsvg: false,
    xwidgets: true
  }

  begin
    OptionParser.new do |opts|
      opts.banner = <<~DOC
        Usage: ./build-emacs-for-macos [options] <branch/tag/sha>

        Branch, tag, and SHA are from the emacs-mirror/emacs/emacs Github repo,
        available here: https://github.com/emacs-mirror/emacs

        Options:
      DOC

      opts.on('-j', '--parallel COUNT',
              'Compile using COUNT parallel processes ' \
              "(detected: #{cli_options[:parallel]})") do |v|
        cli_options[:parallel] = v
      end

      opts.on('--git-sha SHA', 'Override detected git SHA of specified ' \
                               'branch allowing builds of old commits') do |v|
        cli_options[:git_sha] = v
      end

      opts.on('--[no-]xwidgets',
              'Enable/disable XWidgets ' \
              '(default: enabled if supported)') do |v|
        cli_options[:xwidgets] = v
      end

      opts.on('--[no-]native-comp',
              'Enable/disable native-comp ' \
              '(default: enabled if supported)') do |v|
        cli_options[:native_comp] = v
      end

      opts.on('--[no-]native-full-aot',
              'Enable/disable NATIVE_FULL_AOT / Ahead of Time compilation ' \
              '(default: disabled)') do |v|
        cli_options[:native_full_aot] = v
      end

      opts.on('--[no-]native-comp-macos-fixes',
              'Enable/disable fix based on feature/native-comp-macos-fixes ' \
              'branch (default: enabled if native-comp supported)') do |v|
        cli_options[:macos_fixes] = v
      end

      opts.on('--rsvg', 'Enable SVG image support via librsvg, ' \
                        'can yield a unstable build (default: disabled)') do
        cli_options[:rsvg] = true
      end

      opts.on('--no-titlebar', 'Apply no-titlebar patch (default: disabled)') do
        cli_options[:no_titlebar] = true
      end

      opts.on('--no-frame-refocus',
              'Apply no-frame-refocus patch (default: disabled)') do
        cli_options[:no_frame_refocus] = true
      end

      opts.on('--[no-]native-fast-boot',
              'DEPRECATED: use --[no-]native-full-aot instead') do |v|
        if v
          raise Error, '--native-fast-boot option is deprecated, ' \
                       'use --no-native-full-aot instead'
        else
          raise Error, '--no-native-fast-boot option is deprecated, ' \
                       'use --native-full-aot instead'
        end
      end

      opts.on('--[no-]launcher',
              'DEPRECATED: Launcher script is no longer used.') do |_|
        raise Error, '--[no-]launcher option is deprecated, launcher ' \
                     'script is no longer used.'
      end
    end.parse!

    Build.new(File.expand_path(__dir__), ARGV.shift, cli_options).build
  rescue Error => e
    warn "ERROR: #{e.message}"
    exit 1
  end
end