Merge pull request #17 from jimeh/support-libgccjit-homebrew-formula

feat(native_comp)!: use new libgccjit Homebrew formula

Brewfile

@@ -3,11 +3,13 @@
 brew 'coreutils'
 brew 'curl'
 brew 'expat'
+brew 'gcc'
 brew 'gmp'
 brew 'gnu-sed'
 brew 'gnutls'
 brew 'jansson'
 brew 'libffi'
+brew 'libgccjit'
 brew 'libiconv'
 brew 'librsvg'
 brew 'libtasn1'

Formula/gcc.rb.patch

@@ -1,21 +0,0 @@
-diff --git a/Formula/gcc.rb b/Formula/gcc.rb
-index 1bd636d496..03ad124218 100644
---- a/Formula/gcc.rb
-+++ b/Formula/gcc.rb
-@@ -53,7 +53,7 @@ class Gcc < Formula
-     #  - Ada, which requires a pre-existing GCC Ada compiler to bootstrap
-     #  - Go, currently not supported on macOS
-     #  - BRIG
--    languages = %w[c c++ objc obj-c++ fortran]
-+    languages = %w[c c++ objc obj-c++ fortran jit]
-
-     osmajor = `uname -r`.split(".").first
-     pkgversion = "Homebrew GCC #{pkg_version} #{build.used_options*" "}".strip
-@@ -73,6 +73,7 @@ class Gcc < Formula
-       --with-system-zlib
-       --with-pkgversion=#{pkgversion}
-       --with-bugurl=https://github.com/Homebrew/homebrew-core/issues
-+      --enable-host-shared
-     ]
-
-     # Xcode 10 dropped 32-bit support

README.md

@@ -18,7 +18,7 @@ Use this script at your own risk.
 
 ## Status
 
-As of writing (2020-08-19) it works for me on my machine. Your luck may vary.
+As of writing (2020-10-04) it works for me on my machine. Your luck may vary.
 
 I have successfully built:
 
@@ -30,7 +30,7 @@ For reference, my machine is:
 
 - 13-inch MacBook Pro (2020), 10th-gen 2.3 GHz Quad-Core Intel Core i7 (4c/8t)
 - macOS 10.15.6 (19G2021)
-- Xcode 11.7
+- Xcode 12.0
 
 ## Limitations
 
@@ -89,6 +89,9 @@ tarball.
 If you don't want the build process to eat all your CPU cores, pass in a `-j`
 value of how many CPU cores you want it to use.
 
+Re-building the same Git SHA again can yield weird results unless you first
+trash the corresponding directory from the `sources` directory.
+
 ### Examples
 
 To download a tarball of the `master` branch (Emacs 28.x as of writing) and
@@ -111,29 +114,16 @@ repository.
 
 ## Native-Comp
 
-To build a Emacs.app with native-comp support
+Building a Emacs.app with native-comp support
 ([gccemacs](https://akrl.sdf.org/gccemacs.html)) from the `feature/native-comp`
-branch, you will need to install a patched version of Homebrew's `gcc` formula
-that includes libgccjit.
+branch is now supported without much hassle thanks to the newly released
+`libgccjit` Homebrew formula.
 
-The patch itself is in `./Formula/gcc.rb.patch`, and comes from
-[this](https://gist.github.com/mikroskeem/0a5c909c1880408adf732ceba6d3f9ab#1-gcc-with-libgccjit-enabled)
-gist.
+Changes from the `feature/native-comp-macos-fixes` branch are also applied
+through a custom patch process which should be more future-proof compared to a
+regular git diff patch.
 
-You can install the patched formula by running the helper script:
-
-```
-./install-patched-gcc
-```
-
-The helper script will copy your local `gcc.rb` Forumla from Homebrew to
-`./Formula`, and apply the `./Formula/gcc.rb.patch` to it. After which it then
-proceed to install the patched gcc formula which includes libgccjit.
-
-As it requires installing and compiling GCC from source, it can take anywhere
-between 30-60 minutes or more depending on your machine.
-
-And finally to build a Emacs.app with native compilation enabled, run:
+To build a Emacs.app with native compilation enabled, simply run:
 
 ```
 ./build-emacs-for-macos feature/native-comp
@@ -142,7 +132,7 @@ And finally to build a Emacs.app with native compilation enabled, run:
 By default `NATIVE_FULL_AOT` is disabled which ensures a fast build by native
 compiling as few lisp source files as possible to build the app. Any remaining
 lisp files will be dynamically compiled in the background the first time you use
-them.
+them. To enable native full AoT, pass in the `--native-full-aot` option.
 
 On my machine it takes around 10 minutes to build Emacs.app with
 `NATIVE_FULL_AOT` disabled. With it enabled it takes around 20-25 minutes.

build-emacs-for-macos

@@ -99,7 +99,7 @@ class Build
     symlink_internals(app)
 
     LibEmbedder.new(app, brew_dir, extra_libs).embed
-    LibGccJitEmbedder.new(app, gcc_dir).embed if options[:native_comp]
+    GccLibEmbedder.new(app, gcc_dir).embed if options[:native_comp]
 
     archive_app(app)
   end
@@ -126,6 +126,10 @@ class Build
     @gcc_dir ||= `brew --prefix gcc`.chomp
   end
 
+  def libgccjit_dir
+    @libgccjit_dir ||= `brew --prefix libgccjit`.chomp
+  end
+
   def extra_libs
     @extra_libs ||= [
       "#{brew_dir}/opt/expat/lib/libexpat.1.dylib",
@@ -139,7 +143,7 @@ class Build
 
     url = (DOWNLOAD_URL % sha)
     filename = "emacs-mirror-emacs-#{sha[0..6]}.tgz"
-    target   = File.join(tarballs_dir, filename)
+    target = File.join(tarballs_dir, filename)
 
     if File.exist?(target)
       info "#{filename} already exists locally, attempting to use."
@@ -204,17 +208,25 @@ class Build
 
   def verify_libgccjit
     err 'gcc not installed' unless Dir.exist?(gcc_dir)
+    err 'libgccjit not installed' unless Dir.exist?(libgccjit_dir)
 
-    return if Dir["#{gcc_dir}/lib/**/libgccjit.so*"].any?
+    if Dir["#{libgccjit_dir}/lib/**/libgccjit.so*"].empty?
+      err "Detected libgccjit (#{libgccjit_dir}) does not have any " \
+          'libgccjit.so* files. Please try reinstalling libgccjit: ' \
+          'brew reinstall libgccjit'
+    end
 
-    err "Detected GCC (#{gcc_dir}) does not have libgccjit. Ensure patched " \
-        'gcc brew formula has been installed via ./install-patched-gcc'
+    dirs = Dir["{#{gcc_dir},#{libgccjit_dir}}/lib/gcc/*"]
+    return if dirs.map { |dir| File.basename(dir) }.uniq == %w[10]
+
+    err 'Detected gcc and libgccjit library paths do not belong to the ' \
+        "same major version. Detected paths:\n  - #{dirs.join('  - ')}"
   end
 
   def gcc_library_paths
     @gcc_library_paths ||= Dir[
-      "#{gcc_dir}/lib/gcc/*",
-      "#{gcc_dir}/lib/gcc/*/gcc/*apple-darwin*/*"
+      "{#{gcc_dir},#{libgccjit_dir}}/lib/gcc/*",
+      "{#{gcc_dir},#{libgccjit_dir}}/lib/gcc/*/gcc/*apple-darwin*/*"
     ].sort_by { |p| [p.size, p] }
   end
 
@@ -260,7 +272,8 @@ class Build
 
         ENV['LDFLAGS'] = [
           gcc_library_paths.map { |path| "-L#{path}" },
-          "-I#{gcc_dir}/include"
+          "-I#{gcc_dir}/include",
+          "-I#{libgccjit_dir}/include"
         ].flatten.compact.join(' ')
 
         ENV['LIBRARY_PATH'] = [
@@ -365,10 +378,10 @@ class Build
     ]
 
     filename = "Emacs.app-[#{metadata.join('][')}].tbz"
-    target   = "#{builds_dir}/#{filename}"
+    target = "#{builds_dir}/#{filename}"
 
     app_base = File.basename(app)
-    app_dir  = File.dirname(app)
+    app_dir = File.dirname(app)
 
     if !File.exist?(target)
       info "Creating #{filename} archive in \"#{builds_dir}\"..."
@@ -587,7 +600,7 @@ class AbstractEmbedder
   end
 
   def lib_dir_name
-    "lib-#{OS.arch}-#{OS.version.to_s.tr('.', '_')}"
+    'lib'
   end
 end
 
@@ -700,7 +713,7 @@ class LibEmbedder < AbstractEmbedder
   end
 end
 
-class LibGccJitEmbedder < AbstractEmbedder
+class GccLibEmbedder < AbstractEmbedder
   attr_reader :gcc_dir
 
   def initialize(app, gcc_dir)
@@ -721,6 +734,7 @@ class LibGccJitEmbedder < AbstractEmbedder
 
     FileUtils.mkdir_p(File.dirname(target_dir))
     FileUtils.cp_r(source_dir, target_dir)
+    FileUtils.rm(Dir[File.join(target_dir, '**', '.DS_Store')], force: true)
   end
 
   private
@@ -734,13 +748,10 @@ class LibGccJitEmbedder < AbstractEmbedder
   end
 
   def gcc_version
-    @gcc_version ||= Dir[File.join(gcc_dir, 'lib', 'gcc', '*', 'libgccjit.so*')]
-                     .map { |path| File.dirname(path) }
-                     .select { |path| path.match(%r{/\d+$}) }
-                     .uniq
-                     .map { |dir| File.basename(dir).to_i }
-                     .max
-                     .to_s
+    @gcc_version ||= Dir[File.join(gcc_dir, 'lib', 'gcc', '*', 'libgcc*')]
+                     .map { |path| File.basename(File.dirname(path)) }
+                     .select { |path| path.match(/^\d+$/) }
+                     .uniq.map(&:to_i).max.to_s
   end
 
   def source_dir

install-patched-gcc

@@ -1,27 +0,0 @@
-#! /usr/bin/env bash
-set -e
-
-brewdir="$(brew --prefix)"
-formula="${brewdir}/Homebrew/Library/Taps/homebrew/homebrew-core/Formula/gcc.rb"
-
-if [ ! -f "$formula" ]; then
-  echo "ERROR: ${formula} does not exist." 1>&2
-  exit 1
-fi
-
-gnubins=(
-  "${brewdir}/opt/coreutils/libexec/gnubin"
-  "${brewdir}/opt/make/libexec/gnubin"
-  "${brewdir}/opt/gnu-sed/libexec/gnubin"
-)
-
-for gnubin in "${gnubins[@]}"; do
-  if [ -d "$gnubin" ]; then
-    export PATH="${gnubin}:$PATH"
-  fi
-done
-
-cp "$formula" ./Formula/
-
-patch -f -p1 -i ./Formula/gcc.rb.patch
-brew install ./Formula/gcc.rb --build-from-source --force

patches/native-comp-env-setup.patch

@@ -1,8 +1,8 @@
 diff --git a/lisp/emacs-lisp/comp.el b/lisp/emacs-lisp/comp.el
-index 25e2de9..14ee6dc 100644
+index 25e2de9..bcedd31 100644
 --- a/lisp/emacs-lisp/comp.el
 +++ b/lisp/emacs-lisp/comp.el
-@@ -2801,6 +2801,36 @@ queued with LOAD %"
+@@ -2801,6 +2801,45 @@ queued with LOAD %"
        (comp-run-async-workers)
        (message "Compilation started."))))
  
@@ -13,15 +13,24 @@ index 25e2de9..14ee6dc 100644
 +             (string-match-p "\.app\/Contents\/MacOS\/?$"
 +                             invocation-directory))
 +    (let* ((library-path-env (getenv "LIBRARY_PATH"))
-+           (gcc-dir (car (file-expand-wildcards
-+                          (concat invocation-directory "lib-*/gcc/*"))))
-+           (gcc-darwin-dir (car (file-expand-wildcards
-+                                 (concat gcc-dir "/gcc/*apple-darwin*/*"))))
++           (gcc-base-dir (concat invocation-directory "lib/gcc"))
++           (gcc-version (car (seq-filter
++                              (lambda (dir) (string-match-p "^[0-9]+$" dir))
++                              (directory-files gcc-base-dir))))
++           (gcc-dir (concat gcc-base-dir "/" gcc-version))
++           (darwin-base-dir (car (file-expand-wildcards
++                                      (concat gcc-dir "/gcc/*apple-darwin*"))))
++           (darwin-version (car (seq-filter
++                                 (lambda (dir)
++                                   (string-match-p
++                                    "^[0-9]+\\(\.[0-9]+\\(\.[0-9]+\\)?\\)?$" dir))
++                                 (directory-files darwin-base-dir))))
++           (darwin-dir (concat darwin-base-dir "/" darwin-version))
 +           (lib-paths (append
-+                       (list gcc-dir gcc-darwin-dir)
++                       (list gcc-dir darwin-dir)
 +                       (if library-path-env (list library-path-env) (list)))))
 +
-+      (when (and gcc-dir gcc-darwin-dir)
++      (when (and gcc-dir darwin-dir)
 +        (setenv "LIBRARY_PATH" (mapconcat 'identity lib-paths ":")))))
 +
 +  ;; Remove advice, as it only needs to run once.