i18n_unused_scanner.rb

# License : WTF License
# for Rails project & put it at %RAILS_APP%/script/i18n_unused_scanner.rb and exec it
# just for fun , only test with OSX

require 'yaml'
require 'open3'
require 'awesome_print'

module I18nUnusedScanner
  SEARCH_PATTERNS = %w(js rb erb html)

  ###### keys diff start
  def self.flat_hash(source , flag = [] , global = {})
    if source.is_a?(Hash)
      source.each_pair do |key , value|
        global = flat_hash(source[key] , flag.dup.push(key) , global)
      end
    else
      global.merge!({flag.join("::") => true})
    end
    return global
  end

  # 這邊做 key 的刪除 ...
  def self.remove_key(locales , lang_full_key_set)
    remove_counter = 0
    lang_full_key_set.each do |lang_full_key|
      ary_lang_full_key = lang_full_key.split("::")
      lang_sym = ary_lang_full_key.shift.to_sym
      locals_datas = locales[lang_sym]
      unless locals_datas
        puts "QwQ no locals_datas : #{lang_sym}"
        next 
      end

      source = locals_datas[:body]
      unless source
        puts "QwQ no locals_body : #{lang_sym}"
        next 
      end
      
      is_success = false
      
      ori_ary_lang_full_key = ary_lang_full_key.dup

      begin
        flag = ary_lang_full_key.shift
        child_source = source[flag]
        if child_source.is_a?(String)
          source.delete(flag)
          remove_counter += 1
          is_success = true
          break
        elsif child_source.is_a?(Array)
          puts "skip remove Array : #{ori_ary_lang_full_key.join("::")}"
        elsif child_source.is_a?(Hash)
          source = child_source
        else
          puts "!!! #{ary_lang_full_key} : #{flag} , #{flag.class} , #{child_source} , #{child_source.class}"
          is_fail = true
          break
        end
      end while !ary_lang_full_key.empty?

      unless is_success
        puts "fail to remove(#{is_fail}) #{lang_full_key}"
        next
      end
    end
    return remove_counter > 0
  end
  def self.save_new_local_yaml(locales)
    locales.each_pair do |lang_sym , datas|
      full_filepath = "#{__dir__}/../config/locales/#{datas[:sub_filename]}_#{Time.now.strftime("%Y%m%d%H%M%S")}.yml"
      puts "save to : #{full_filepath}"
      file = File.open(full_filepath , 'w')
      file.puts(YAML.dump({datas[:sub_filename] => datas[:body]}))
      file.close
    end
  end

  # 這邊做 key 的互減 ... 有餘數 = 有多 key
  def self.show_diff(locales)
    all_keys = {}
    locales.each_pair do |lang_sym_main , datas_main|
      locales.each_pair do |lang_sym_sub , datas_sub|
        next if lang_sym_main == lang_sym_sub
        (datas_main[:full_key_set] - datas_sub[:full_key_set]).each do |addon_full_key|
          all_keys[lang_sym_main] ||= {}
          all_keys[lang_sym_main][addon_full_key] ||= []
          all_keys[lang_sym_main][addon_full_key] << lang_sym_sub
        end
      end
    end
    all_keys.each_pair do |lang_sym_main , addon_full_key_set|
      addon_full_key_set.each_pair do |addon_full_key , lang_sym_sub_set|
        puts "(#{lang_sym_main} - #{lang_sym_sub_set.join(',')}) => #{addon_full_key}"
      end
    end
    all_keys.clear # make GC
    return true
  end

  # 檢測無用的 key，其實就是 project 內做全文字串搜尋
  def self.show_unused(locales)
    # 這邊全展開為 tail_key => ["{lang}::{main_key}::{sub_key}...::{tail_key}" , ...]
    all_keys = {}
    locales.each_pair do |lang_sym , datas|
      datas[:full_key_set].each do |full_key|
        tail_key = full_key.split('::').pop
        all_keys[tail_key] ||= []
        all_keys[tail_key] << "#{lang_sym}::#{full_key}"
      end
    end

    total = all_keys.length

    puts "start_search : #{total} keys"

    flag = 0
    flag_char = %Q%-\\|/%

    # remove unsafe keys first
    all_keys.each_with_index do |(tail_key,lang_full_key_set),index|
      if tail_key.match(/["`]/)
        puts "\r#{index}/#{total} : safe_key_check_fail : #{tail_key}"
        all_keys.delete(tail_key)
      end
    end

    need_check_key_set = []
    all_keys.each_with_index do |(tail_key,lang_full_key_set),index|
      begin
        opts , errors = Open3.capture3(%Q% find -E #{__dir__}/../ -regex '.*\.(#{SEARCH_PATTERNS.join("|")})' -type f | xargs grep -m 1 "#{tail_key}" %)
        if !errors.empty?
          puts "\r#{index}/#{total} : search_error1 : #{tail_key}"
        elsif opts.empty?
          puts "\r#{index}/#{total} : maybe unused : #{tail_key}\n\t#{lang_full_key_set.join("\n\t")}"
          need_check_key_set << lang_full_key_set
        else
          print "\r#{flag_char[(flag += 1) % 4]}" # 轉轉轉
        end
      rescue
        puts $! , $@
        puts "\r#{index}/#{total} : search_error2 : #{tail_key}"
      end
    end

    # filter keyword loop
    filter_keywords = []
    loop do
      counter = 0
      need_check_key_set.each do |need_check_keys|
        need_check_keys.each do |need_check_key|
          is_next = false
          filter_keywords.each do |f|
            if need_check_key.match(f)
              is_next = true
              next
            end
          end
          next if is_next
          counter += 1
          puts "[#{counter}]\t #{need_check_key}"
        end
      end
      puts "now keywords : #{filter_keywords}"
      print "enter filter keyword , it will not to remove , and 'OKAY' to finish , 'QUIT' to abort , empty to show this again : "
      keyword = $stdin.gets.chomp
      next if keyword.upcase.strip == ""
      break if keyword.upcase == "OKAY"
      abort("QwQ QUIT!!") if keyword.upcase == "QUIT"
      filter_keywords << keyword
    end


    # search content
    remove_counter = 0
    need_check_key_set.each do |need_check_keys|
      need_check_keys.each do |need_check_key|
        filter_keywords.each do |f|
          next if need_check_key.match(f)
        end
        remove_counter += 1 if remove_key(locales , [need_check_key])
      end
    end
    # 存檔 ...
    if remove_counter > 0
      save_new_local_yaml(locales)
    else
      puts "no modify ..."
    end
  end

  def self.go!
    # make {en:{sub_filename:"en" ... }, tw:{sub_filename:"zh-tw" ... }, cn:{sub_filename:"zh-cn" ... }}
    locales = Dir["#{__dir__}/../config/locales/*.yml"].map do |filepath|
      sub_filename = filepath.split(/[\/\.]/)[-2]
      lang_sym = sub_filename.split('-')[-1].to_sym
      body = YAML.load_file("#{__dir__}/../config/locales/#{sub_filename}.yml")[sub_filename]
      full_key_set = flat_hash(body.dup).keys

      puts "#{lang_sym} : total : #{full_key_set.length}"

      [
        lang_sym , {
          lang: lang_sym ,
          sub_filename: sub_filename ,
          body: YAML.load_file("#{__dir__}/../config/locales/#{sub_filename}.yml")[sub_filename] ,
          full_key_set: full_key_set
        },
      ] # return
    end.to_h

    show_diff(locales)

    puts "start process all unless keys : search (#{SEARCH_PATTERNS.join(',')})"

    show_unused(locales)

    puts "[scan finished , please fix me ^.~]"
  end
end

I18nUnusedScanner.go!