code_punch.rb

require 'base64'
require 'find'

require 'dotenv'
Dotenv.load

require 'faraday'
require 'faraday_middleware'
require 'pry'

module CodePunch
  module Github
    LANGS = %w[ ruby go ]
    ACCESS_TOKEN = ENV['ACCESS_TOKEN']
    N = 5

    API = Faraday.new('https://api.github.com/v3') do |conn|
      conn.request :oauth2, ACCESS_TOKEN
      conn.request :json

      conn.response :json, content_type: /\bjson/

      conn.adapter Faraday.default_adapter
    end

    def self.run
      freqs = Hash.new(0)

      LANGS.each do |lang|
        lang = "language:#{lang}"
        result = API.get('/search/repositories', q: lang)
        repos = result.body['items'][0,N]
        repos.each do |repo|
          puts "Looking through #{repo['full_name']}"
          result = API.get('/search/code', q: ['the', lang, "repo:#{repo['full_name']}"].join(' '))
          files = result.body['items'][0,N]
          files.each do |file|
            puts "Looking at #{file['path']}"
            url = file['url']
            content = Base64.decode64(API.get(url).body['content'])
            content.chars.each {|char| freqs[char] += 1 unless char =~ /[\sa-zA-Z0-9]/ }
          end
        end
      end

      freqs.sort_by {|_,v| v}.reverse.each do |char, freq|
        puts "#{char} #{freq}"
      end
    end
  end

  module Source
    EXTS = %w[ .go .rb ]
    IGNORE = %w[ .bundle ]

    def self.run(dir)
      freqs = Hash.new(0)

      Find.find(File.expand_path(dir)) do |path|
        Find.prune if IGNORE.include?(File.basename(path))

        puts "Looking through #{path}" and next if File.directory?(path)

        if EXTS.include?(File.extname(path))
          puts "Looking at #{path}"
          content = File.read(path)
          content.chars.each {|char| freqs[char] += 1 unless char =~ /[\sa-zA-Z0-9]/ }
        end
      end

      freqs.sort_by {|_,v| v}.reverse.each do |char, freq|
        puts "#{char} #{freq}"
      end
    end
  end
end

Github:

/ 1068
. 1027
( 611
) 610
, 532
" 426
_ 338
' 321
= 279
#259
: 253
} 218
{ 218
- 177
] 83
[ 83
* 72
! 69
> 46
? 40
< 40
; 37
| 34
& 33
` 32
@ 31
% 24
\ 19
+ 12
$ 7
3
~ 3
^ 2
? 2
? 1

My src dir:

. 82138
' 64376
/ 63298
" 59139
_ 53435
- 47303
: 31036
, 28589
= 26147
#23531
( 18757
) 18723
< 13324
{ 12416
} 12307
@ 9982
> 9868
[ 7929
] 7825
+ 6780
? 6680
| 5791
* 4735
\ 3660
$ 3559
; 3532
` 2662
! 2012
& 1984
% 1563
~ 1052
^ 447
» 11
« 11
µ 9
’ 7
‘ 4