unicode boms

gistfile1.rb

# test.txt
#
# ����sample text
# ��more text...
# ��sample more text...
# ��


# read in our multi-byte file
File.open('test.txt', 'r') do |f|
  @hfile = f.read
end

#change to unicode
unistring = @hfile.each_byte.map do |p|
  [p].pack('U')
end

# drop any null bytes
blah = []
unistring.each do |o|
  if !o.eql? "\000" then
    blah << o
  end
end

#convert array over to string
newstr = blah.to_s

# drop the bom
newstr = newstr.gsub(/\303\277\303\276/, '')

puts newstr