Fetch Outreach call and email data

fetch_outreach_emails_and_calls.rb

require "bundler/inline"

gemfile do
  source "https://rubygems.org"
  gem "json_api_client"
  gem "oauth2"
  gem "faraday_middleware-oauth2_refresh"
end
require "yaml"

def get_access_token!
  config = YAML.load_file('v2_auth.yml').transform_keys(&:to_sym)
  config in {client_id:, client_secret:, site:, redirect_uri:, scope:}
  client = OAuth2::Client.new(client_id, client_secret, site: site)
  access_token = OAuth2::AccessToken.from_hash(client, config)

  if access_token.token && access_token.get("/api/v2", raise_errors: false).status == 200
    # do nothing
  elsif access_token.refresh_token
    access_token = access_token.refresh!(raise_errors: false)
  end

  if access_token.token.empty?
    authorize_url = client.auth_code.authorize_url(redirect_uri: redirect_uri, scope: scope)
    $stderr.print "1. Visit the following URL:\n\n  #{authorize_url}\n\n"
    $stderr.print "2. Authorize Outreach.\n\n"
    $stderr.print "3. After redirect, paste token here: "; code = gets.chomp; $stderr.print "\n"
    access_token = client.auth_code.get_token(code, redirect_uri: redirect_uri)
  end

  config.update(token: access_token.token, refresh_token: access_token.refresh_token)
  YAML.dump(config.transform_keys(&:to_s), File.open("v2_auth.yml", "w"))
  access_token
end

class Base < JsonApiClient::Resource
  self.site = "https://api.outreach.io/api/v2/"
  self.connection do |connection|
    connection.use FaradayMiddleware::OAuth2Refresh, get_access_token!, token_type: :bearer
  end

  def self.with_defaults
    includes(default_includes).select(default_selects)
  end

  def as_json(*)
    attributes.except(:type).merge(
      relationships.as_json.map do |relKey, relRef|
        [relKey, send(relKey)&.as_json] if send(relKey)
      end.compact.to_h
    )
  end
end

def flatten_json(object, path = [], path_joiner: :camelized)
  path_joiner = method(:"#{path_joiner}_path_joiner") if path_joiner.is_a? Symbol
  case object
  when Array
    object.each.with_index(0).with_object({}) do |(element, i), result|
      result.merge! flatten_json(element, path + [i], path_joiner: path_joiner)
    end
  when Hash
    object.each_with_object({}) do |(key, value), result|
      result.merge! flatten_json(value, path + [key], path_joiner: path_joiner)
    end
  else
    { path_joiner.call(path) => object }
  end
end

def camelized_path_joiner(path)
  path.reduce do |result = '', new_key|
    result.to_s + case new_key
      when Integer then "_#{new_key}_"
      else new_key.to_s.camelize(:upper)
    end
  end&.chomp("_")
end

class User < Base; end
class Prospect < Base; end
class CallPurpose < Base; end
class CallDisposition < Base; end
class Sequence < Base; end
class Mailbox < Base; end

class Call < Base
  def self.default_includes
    default_selects.keys
  end
  def self.default_selects
    {
      user: :name,
      prospect: [:externalId, :externalOwner, :externalSource],
      callPurpose: :name,
      callDisposition: [:name, :outcome],
      sequence: :name,
    }
  end
end

class Mailing < Base
  def self.default_includes
    [:'mailbox.user', :prospect]
  end
  def self.default_selects
    {
      user: :name,
      prospect: [:externalId, :externalOwner, :externalSource, :emails],
      mailbox: nil,
    }
  end
end

resources = {
  calls: Call.with_defaults,
  emails: Mailing.with_defaults,
}

resources.each do |resource|
  limit = 1000
  max_id = 0
  loop do
    page = resource
      .order(id: :asc)
      .where(id: "#{max_id}..inf")
      .with_params(page: { limit: limit })
      .all
    break if page.length == 0
    puts page.map(&:as_json).map(&method(:flatten_json)).map(&:to_json)
    max_id = page[-1].id + 1
  end
end

fetch_outreach_prospect_salesforce_ids.rb

require "bundler/inline"

gemfile do
  source "https://rubygems.org"
  gem "httparty"
  gem "subprocess"
end
require "yaml"

class Outreach
  include HTTParty
  base_uri "https://app1c.outreach.io/api/"
  cookies({
    "accounts_token" => YAML.load_file("v1_auth.yml")["jwt_token"]
  })
  headers({
    "X-Outreach-App-Mode": "app",
    "X-Requested-With": "XMLHttpRequest",
  })
rescue Errno::ENOENT
  $stderr.puts <<~ENDERR
    ERROR: no jwt_token set in v1_auth.yml.\n
    Please:
     1. Log into Outreach
     2. Open Devtools → Network tab
     3. Select a request → go to the Cookies tab in the request pane
     4. Copy the value of the `accounts_token` cookie
     5. Paste it into `v1_auth.yml` after the string `jwt_token: `
     6. Try again\n
  ENDERR
  exit 1
end

Subprocess.check_call(
  %w(in2csv -f ndjson),
  stdin: Subprocess::PIPE,
  stderr: $stderr,
  stdout: "data/outreach_prospect_salesforce_ids_thru_#{Time.now.strftime("%Y%m%d%H%M")}.csv",
) do |in2csv|

  page_num = 1
  per_page = 10
  loop do
    page = Outreach.get("/prospects", query: {
      "filter[order]" => "id",
      "filter[direction]" => "asc",
      "page" => page_num,
      "per_page" => per_page,
    })["prospects"]
    break if page&.length == 0
    page.each do |prospect|
      [$stdout, in2csv.stdin].each do |pipe|
        pipe.puts prospect.to_h.keep_if { |k| k =~ /^ id | plugin_mapping_.* $/x }.to_json
      end
    end
    page_num += 1
    break if page_num > 3
  end

end