Parallelized Circle CI pipeline to build/deploy Re-Natal (w/ React Native & Om Next) project and upload DSYM to Sentry

.circleci___config.yml

version: 2
references:
  ios_config: &ios_config
    macos:
      xcode: "9.4.1"
    working_directory: ~/my-project
    environment:
      FL_OUTPUT_DIR: output # for Fastlane

  build_filter: &build_filter
    filters:
      branches:
        only: /^master.*/

  node_config: &node_config
  # Ideally everything builds on Linux then move to a mac box for XCode and deploy but 
    Circle restore workspace doesn't work between linux and mac
    docker:
      - image: circleci/node:10.10-stretch 
    working_directory: ~/my-project

  clojure_config: &clojure_config
    docker:
      - image: circleci/clojure:lein-2.8.1-node
    working_directory: ~/my-project

  workspace_root: &workspace_root ~/my-project
  ios_root: &ios_root ~/my-project/ios

  
  attach_workspace: &attach_workspace
    # Note: Workspaces don't persist outside of a workflow instance
    attach_workspace:
      at: *workspace_root
  
  # Faster access of git repo between jobs
  save_repo: &save_repo
    save_cache: # arch helps to ensure diffent systems don't build bad assests
      key: v1-repo-{{ arch }}-{{ .Branch }}-{{ .Revision }}
      paths:
        - .

  restore_repo: &restore_repo
    restore_cache:
      name: Restoring Repo
      keys:
        - v1-repo-{{ arch }}-{{ .Branch }}-{{ .Revision }}
  
  # Caching of deps for Clojure, Node, Gems and Cocoapods
  clojure_cache_key: &clojure_cache_key v1-dependency-clojure-{{ arch }}
  save_clojure: &save_clojure
    save_cache:
      key: v1-clojure-{{ arch }}
      paths:
        - /usr/local/Cellar
  restore_clojure: &restore_clojure
    restore_cache:
      name: Restoring Clojure
      keys:
        - v1-clojure-{{ arch }}

  jars_cache_key: &jars_cache_key v1-dependency-jars-{{ arch }}-{{ checksum "project.clj" }}
  restore_jars: &restore_jars
    restore_cache:
      name: Restoring Clojure Jars
      keys:
        - *jars_cache_key

  npm_cache_key: &npm_cache_key v1-dependency-npm-{{ arch }}-{{ checksum "package.json" }}
  restore_node_modules: &restore_node_modules
    restore_cache:
      name: Restoring Node Modules
      keys:
        - *npm_cache_key

  gem_cache_key: &gem_cache_key v6-dependency-gem-{{ arch }}-{{ checksum "~/my-project/ios/Gemfile.lock" }}
  restore_gems: &restore_gems
    restore_cache:
      name: Restoring Gems
      keys:
        - *gem_cache_key

  pod_cache_key: &pod_cache_key v6-dependency-pod-{{ arch }}-{{ checksum "~/my-project/ios/Podfile.lock" }}
  restore_pods: &restore_pods
    restore_cache:
      name: Restoring CocoaPods
      keys:
        - *pod_cache_key

jobs:
  # Install Clojure and Clojurescript on MacOS
  clj_install:
    <<: *ios_config
    steps:
      - *restore_clojure
      - run: |
          HOMEBREW_NO_AUTO_UPDATE=1 brew install leiningen; \
          brew link leiningen;
      - *save_clojure
      - persist_to_workspace:
          root: /usr/local/Cellar
          paths:
            - clojure
            - leiningen
  # Download repo and share with other jobs
  code_checkout:
    # <<: *node_config
    <<: *ios_config
    steps:
      - *restore_repo
      - checkout
      - *save_repo
  # Clojurescript deps
  lein_deps:
    # <<: *clojure_config
    <<: *ios_config
    steps:
      - *restore_repo
      - *restore_clojure
      - *restore_jars
      - run: brew link leiningen
      - run: lein deps
      - save_cache:
          key: *jars_cache_key
          paths:
            - .local-m2
  # Node deps
  npm_deps:
    # <<: *node_config
    <<: *ios_config
    steps:
      - *restore_repo
      - *restore_node_modules
      - run: yarn install
      - save_cache:
          key: *npm_cache_key
          paths:
            - node_modules
  # Fastlane and Cocoapod deps
  gem_deps:
    <<: *ios_config
    # <<: *node_config
    steps:
      - run: echo "ruby-2.5" > ~/.ruby-version
      - *restore_repo
      - *attach_workspace
      - *restore_gems
      - run:
          name: "Install Gems (Fastlane & Cocoapods)"
          working_directory: ios
          command: bundle install --local --no-cache --path=../.gems
      - save_cache:
          key: *gem_cache_key
          paths:
            - .gems
  # Pod Deps
  pod_deps:
    <<: *ios_config
    # <<: *node_config
    steps:
      - run: echo "ruby-2.5" > ~/.ruby-version
      - *restore_repo
      - *restore_gems
      - *restore_pods
      - run:
          name: "Slow Cocoapod setup"
          command: |
            # NEVER cache ~/.cocoapods, it takes 3 minutes to restore everytime b/c this repo is >1GB
            mkdir ~/.cocoapods/repos && \
            cd ~/.cocoapods/repos && \
            /usr/bin/git clone https://github.com/CocoaPods/Specs.git master --depth=1 || true
      - run:
          working_directory: ios
          command: pod install --no-repo-update
      - save_cache:
          key: *pod_cache_key
          paths:
            - ios/Pods
  # Build the index.ios.js from Cljs and bundle via React Native(v0.57)/Metro(custom transformer required)
  cljs_build:
    <<: *ios_config
    steps:
      - *restore_repo
      - *restore_node_modules
      - *restore_clojure
      - *restore_jars
      - run: brew link leiningen
      - run: lein prod-build
      - run: yarn transform_debug
      - persist_to_workspace:
          root: *workspace_root
          paths:
            - ios/main.jsbundle
            - main.jsbundle.map
  # Build IPA with Xcode/Fastlane
  ios_package:
    <<: *ios_config
    shell: /bin/bash --login -o pipefail
    steps:
      - run: echo "ruby-2.5" > ~/.ruby-version
      - checkout
      - *restore_gems
      - *restore_pods
      - *restore_node_modules
      - *attach_workspace
      - run:
          name: Activate Gems
          working_directory: ios
          command: bundle install --local --no-cache --path=../.gems
      - run: mv ios/test.jsbundle ios/main.jsbundle
      - run:
          name: Package IPA
          working_directory: ios
          no_output_timeout: 20m # xcode can take a while
          command: bundle exec fastlane package
      - run:
          # Deploy to Testflight, if you want to deploy to beta groups you must 
          # wait 10+ mins for the binary to generate. If you have an App Watch app or Today widget
          # you must also wait to get the real DSYM from iTunes Connect
          name: Deploy to Testflight
          working_directory: ios
          no_output_timeout: 20m 
          command: bundle exec fastlane deploy_beta
      - persist_to_workspace:
          root: *workspace_root
          paths:
            - ios/*/*.plist
            - ios/*/*.pbxproj
            - ios/dist/*
      - store_artifacts:
          path: ios/dist/fskl.ipa
  # Testflight (as of 9/2018) takes 10+ minutes to have the DSYM ready. Waiting on a Linux box is 
  # cheaper than waiting on a mac box. 
  sleep_600:
    <<: *node_config
    steps:
      - run: sleep 540; # b/c there's a 10 minute timeout on a task
  # Get the IPA, download Testflight DSYMs and upload to Sentry
  ios_dsym:
    <<: *ios_config
    shell: /bin/bash --login -o pipefail
    steps:
      - run: echo "ruby-2.5" > ~/.ruby-version
      - *restore_repo
      - *restore_gems # for fastlane
      - *restore_node_modules # for @sentry/cli
      - *attach_workspace # for dist/fskl.ipa
      - run:
          name: Install sentry-cli
          command: curl -sL https://sentry.io/get-cli/ | bash
      - run:
          name: Activate Gems
          working_directory: ios
          command: bundle install --local --no-cache --path=../.gems
      - run:
          # must wait for bitcode DSYM to upload to Sentry
          name: Send to Testflight
          working_directory: ios
          command: bundle exec fastlane beta



# Deploy Workflow takes 6 minutes to build the release.ios.js from Clojurescript,
# 9 minutes for Xcode to package and send to Testflight,
# 10 minutes waiting for Testflight
# 4 minutes to download the DSYMs/Upload to Sentry and commit version bump to Github
workflows:
  version: 2

  deploy:
    jobs:
      # env
      - clj_install: *build_filter
      # code
      - code_checkout: *build_filter
      # deps
      - lein_deps:
          <<: *build_filter
          requires:
            - clj_install
            - code_checkout
      - npm_deps:
          <<: *build_filter
          requires:
            - code_checkout
      - gem_deps:
          <<: *build_filter
          requires:
            - code_checkout
      - pod_deps:
          <<: *build_filter
          requires:
            - gem_deps
      # compile
      - cljs_build:
          <<: *build_filter
          requires:
            - lein_deps
            - npm_deps
      - ios_package:
          <<: *build_filter
          requires:
            - pod_deps
            - cljs_build
      - sleep_600:
          <<: *build_filter
          requires:
            - ios_package
      - ios_dsym:
          <<: *build_filter
          requires:
            - sleep_600

Fastfile

# This file contains the fastlane.tools configuration
# You can find the documentation at https://docs.fastlane.tools
#
# For a list of all available actions, check out
#
#     https://docs.fastlane.tools/actions
#
# For a list of all available plugins, check out
#
#     https://docs.fastlane.tools/plugins/available-plugins
#


default_platform(:ios)

PROJECT = './my-app.xcodeproj'
WORKSPACE = 'my-app.xcworkspace'
SCHEMA = 'my-app'
DSYM_PATH = '../.dsyms'

platform :ios do |options|
  desc 'Package iOS and send to TestFlight/AppStore'

  before_all do |options|
    # This fails the second run but it was already setup
    begin
      setup_circle_ci
    rescue => ex
      UI.error(ex)
    end
    
  end
  
  # Step 1. Build and send to Testflight
  lane :package_and_deploy_beta do |options|
    NEW_BETA = options[:increment_patch] == true
    package(options)
    deploy_beta(options)
  end 

  # Step 2. Download DSYM, send to Sentry and commit new version to release branch
  lane :beta do |options|
    upload_symbols()
    commit_version()
  end 

  lane :package do |options|
    desc 'Package iOS (after main.jsbundle is created)'

    NEW_BETA = options[:increment_patch] == true
    NEW_BUILD = latest_testflight_build_number + 1

    sh 'git checkout master'
    
    # download certs
    match(app_identifier: ['com.my-app', 'com.my-app.watchkitapp', 'com.my-app.watchkitapp.watchkitextension'], 
      git_url: 'my-repo',
      type: 'appstore')
    
    # bump build & patch numbers
    increment_build_number(
      build_number: NEW_BUILD,
      xcodeproj: PROJECT)
    
    if NEW_BETA
      puts 'Incrementing the patch triggers a new Testflight review.'
      increment_version_number(
        bump_type: 'patch',
        xcodeproj: PROJECT
      )
    end
    
    # build .ipa
    build_app(scheme: SCHEMA,
              workspace: WORKSPACE,
              clean: true,
              include_bitcode: true,
              output_directory: "dist",
              build_path: "dist",
              export_options: {
                method: "app-store",
                provisioningProfiles: {
                  "com.my-app" => "fastlane-match-will-generate-cert", 
                  "com.my-app.watchkitapp" => "fastlane-match-will-generate-cert",
                  "com.my-app.watchkitapp.watchkitextension" => "fastlane-match-will-generate-cert"
    }})
  end

  lane :deploy_beta do |options|
    NEW_BETA = options[:increment_patch] == true
   
    # upload to testflight, enable internal (optionally beta testers)
    NOTES = changelog_from_git_commits
    upload_to_testflight(
      ipa: 'dist/my-app.ipa',
      # distribute_external: NEW_BETA, # MUST wait 10+ minutes in order to auto-enable for beta testers
      skip_waiting_for_build_processing: true, # Don't send to beta users, instead CI will sleep and check for DSYM in 10 minutes
      groups: NEW_BETA ? '<My Beta Group>' : '',
      changelog: "New Changes:\n #{NOTES}",
      beta_app_description: "New Changes:\n #{NOTES}"
    )
  end
    
  lane :upload_symbols do
    # bit-code DSYMs MUST be downloaded from iTunes Connect
    download_dsyms(output_directory: DSYM_PATH)
    sentry_upload_dsym(
      auth_token: 'xxxxxxxxxxxxxxx',
      org_slug: 'my-org',
      project_slug: 'my-app',
      dsym_path: DSYM_PATH
    )
  end

  lane :commit_version do
    VERSION = get_version_number(xcodeproj: PROJECT, target: 'my-app')
    BUILD = get_build_number(xcodeproj: PROJECT)

    # commit new version and push to master
    commit_version_bump(
      message: "[BETA] v#{VERSION}(#{BUILD})",
      xcodeproj: PROJECT
    )
    add_git_tag(
      grouping: 'ios',
      prefix: 'v'
    )
    push_to_git_remote(
      remote: 'origin',
      local_branch: 'master',
      remote_branch: 'release',
      tags: true,
      force: true
    )
  end

  lane :certificates do
  desc 'Create the certs'
    match(app_identifier: ['com.my-app', 'com.my-app.watchkitapp', 'com.my-app.watchkitapp.watchkitextension'], readonly: true)
  end

end

error do |lane, exception|
 # Send error notification
end

rn-cli.config.js

/* 
  This is a custom React Native/Metro Transformer to fix issues 
  with bundling a large single JS file. It works with RNv0.57
  both locally and on Circle CI.
  
  
  Add the command below to package.json or project.clj.

  node --expose-gc --max_old_space_size=8192 \
    './node_modules/react-native/local-cli/cli.js' bundle \
    --sourcemap-output main.jsbundle.map \
    --bundle-output ios/main.jsbundle \
    --entry-file release.ios.js \
    --platform ios \
    --dev true \
    --assets-dest ios \
    --config=../../../../rn-cli.config.js",
*/

// Big files can slow down Metro, so we'll dump GC every 30 seconds to be safe.
(function scheduleGc() {
  if (!global.gc) {
    console.log("Garbage collection is not exposed");
    return;
  }

  var timeoutId = setTimeout(function() {
    global.gc();
    console.log("Manual gc", process.memoryUsage());
    scheduleGc();
  }, 30 * 1000);
  timeoutId.unref();
})();

// Custom Metro Transform (validated with React Native v0.57)
const path = require("path");
const fs = require("fs");
const defaultTransformer = require("./node_modules/metro/src/reactNativeTransformer");

function clojurescriptTransformer(code, filename) {
  console.log("Generating sourcemap for " + filename);
  var map = fs.readFileSync(filename + ".map", { encoding: "utf8" });
  var sourceMap = JSON.parse(map);

  var sourcesContent = [];
  sourceMap.sources.forEach(function(path) {
    var sourcePath = __dirname + "/" + path;
    if (path.indexOf(".") === -1) {
      console.log("ignore");
      return;
    }

    try {
      // try and find the corresponding `.cljs` file first
      const file = sourcePath.replace(".js", ".cljs");
      console.log(file);
      sourcesContent.push(fs.readFileSync(file, "utf8"));
    } catch (e) {
      // otherwise fallback to whatever is listed as the source
      console.log("sourcePath", path);
      try {
        sourcesContent.push(fs.readFileSync(sourcePath, "utf8"));
      } catch (e) {
        if (sourcePath.match(/\.js$/) !== null) {
          console.log("patch-cljs-metro-transformer error with ", sourcePath);
        }
      }
    }
  });
  sourceMap.sourcesContent = sourcesContent;

  return {
    filename: filename,
    code: code.replace("# sourceMappingURL=", ""),
    map: sourceMap
  };
}

// @see https://facebook.github.io/metro/docs/en/configuration
module.exports = {
  transform: data => {
    // When Clojurescript builds the release.ios.js file,
    //   don't transform it for the last cljs output. Instead
    //   just grab the soucemaps so they can be sent to
    //   Sentry or any crash/logging system.
    console.log(data.filename);
    if (data.filename.match(/release*/) !== null) {
      console.log("custom");
      var result = clojurescriptTransformer(data.src, data.filename);
      // console.log(result.map);
      return result;
    } else {
      return defaultTransformer.transform(data);
    }
  },
  projectRoot: path.resolve(__dirname, "../../.."),
  getTransformModulePath: () => require.resolve("./rn-cli.config") // yes, this is recursive. The transformer will get export.transform
};