grid_fs.rb

require "mime/types"
require "digest/md5"
require "cgi"

### see: http://www.mongodb.org/display/DOCS/GridFS+Specification

class GridFS
##
#
  attr_accessor :prefix
  attr_accessor :namespace
  attr_accessor :file_model
  attr_accessor :chunk_model

##
#
  def initialize(options = {})
    options.to_options!
    @prefix = options[:prefix] || 'fs'
    @namespace = GridFS.namespace_for(@prefix)
    @file_model = @namespace.file_model
    @chunk_model = @file_model.chunk_model
  end

##
#
  def GridFS.namespace_for(prefix)
    prefix = prefix.to_s.downcase
    const = "::GridFS::#{ prefix.to_s.camelize }"
    namespace = const.split(/::/).last
    const_defined?(namespace) ? const_get(namespace) : build_namespace_for(namespace)
  end

##
#
  def GridFS.build_namespace_for(prefix)
    prefix = prefix.to_s.downcase
    const = prefix.camelize

    namespace =
      Module.new do
        module_eval(&NamespaceMixin)
        self
      end

    const_set(const, namespace)

    file_model = build_file_model_for(namespace)
    chunk_model = build_chunk_model_for(namespace)

    file_model.namespace = namespace
    chunk_model.namespace = namespace

    file_model.chunk_model = chunk_model
    chunk_model.file_model = file_model

    namespace.prefix = prefix
    namespace.file_model = file_model
    namespace.chunk_model = chunk_model

    namespace.send(:const_set, :File, file_model)
    namespace.send(:const_set, :Chunk, chunk_model)

    #at_exit{ file_model.create_indexes rescue nil }
    #at_exit{ chunk_model.create_indexes rescue nil }

    const_get(const)
  end

  NamespaceMixin = proc do
    class << self
      attr_accessor :prefix
      attr_accessor :file_model
      attr_accessor :chunk_model

      def to_s
        prefix
      end

      def namespace
        prefix
      end

      def put(arg, attributes = {})
        chunks = []
        file = file_model.new
        attributes.to_options!

        if attributes.has_key?(:id)
          file.id = attributes.delete(:id)
        end

        if attributes.has_key?(:_id)
          file.id = attributes.delete(:_id)
        end

        if attributes.has_key?(:content_type)
          attributes[:contentType] = attributes.delete(:content_type)
        end

        if attributes.has_key?(:upload_date)
          attributes[:uploadDate] = attributes.delete(:upload_date)
        end

        md5 = Digest::MD5.new
        length = 0
        chunkSize = file.chunkSize
        n = 0

        GridFS.reading(arg) do |io|
          filename = [file.id.to_s, GridFS.extract_basename(io)].join('/').squeeze('/')
          content_type = GridFS.extract_content_type(filename) || file.contentType

          attributes[:filename] ||= filename
          attributes[:contentType] ||= content_type

          while((buf = io.read(chunkSize)))
            md5 << buf
            length += buf.size
            chunk = file.chunks.build
            chunk.data = binary_for(buf)
            chunk.n = n
            n += 1
            chunk.save!
            chunks.push(chunk)
          end
        end

        attributes[:length] ||= length
        attributes[:uploadDate] ||= Time.now.utc
        attributes[:md5] ||= md5.hexdigest

        file.update_attributes(attributes)

        file.save!
        file
      ensure
        chunks.each{|chunk| chunk.destroy rescue nil} if $!
      end

      if defined?(Moped)
        def binary_for(*buf)
          Moped::BSON::Binary.new(:generic, buf.join)
        end
      else
        def binary_for(buf)
          BSON::Binary.new(buf.bytes.to_a)
        end
      end

      def get(id)
        file_model.find(id)
      end

      def delete(id)
        file_model.find(id).destroy
      rescue
        nil
      end

      def where(conditions = {})
        case conditions
          when String
            file_model.where(:filename => conditions)
          else
            file_model.where(conditions)
        end
      end

      def find(*args)
        where(*args).first
      end

      def [](filename)
        file_model.where(:filename => filename.to_s).first
      end

      def []=(filename, readable)
        file = self[filename]
        file.destroy if file
        put(readable, :filename => filename.to_s)
      end

    # TODO - opening with a mode = 'w' should return a GridIO::IOProxy
    # implementing a StringIO-like interface
    #
      def open(filename, mode = 'r', &block)
        raise NotImplementedError
      end
    end
  end

##
#
  def GridFS.build_file_model_for(namespace)
    prefix = namespace.name.split(/::/).last.downcase
    file_model_name = "#{ namespace.name }::File"
    chunk_model_name = "#{ namespace.name }::Chunk"

    Class.new do
      include Mongoid::Document

      singleton_class = class << self; self; end

      singleton_class.instance_eval do
        define_method(:name){ file_model_name }
        attr_accessor :chunk_model
        attr_accessor :namespace
      end

      self.default_collection_name = "#{ prefix }.files"

      field(:filename, :type => String)
      field(:contentType, :type => String, :default => 'application/octet-stream')

      field(:length, :type => Integer, :default => 0)
      field(:chunkSize, :type => Integer, :default => (256 * (2 ** 20)))
      field(:uploadDate, :type => Date, :default => Time.now.utc)
      field(:md5, :type => String, :default => Digest::MD5.hexdigest(''))

      %w( filename contentType length chunkSize uploadDate md5 ).each do |f|
        validates_presence_of(f)
      end
      validates_uniqueness_of(:filename)

      has_many(:chunks, :class_name => chunk_model_name, :inverse_of => :files, :dependent => :destroy, :order => [:n, :asc])

      index({:filename => 1}, :unique => true) 

      def basename
        ::File.basename(filename)
      end

      def prefix
        self.class.namespace.prefix
      end

      def each(&block)
        chunks.all.order_by([:n, :asc]).each do |chunk|
          block.call(chunk.data.to_s)
        end
      end

      def to_s
        to_s = ''
        each{|data| to_s << data}
        to_s
      end

      def base64
        Array(to_s).pack('m')
      end

      def data_uri(options = {})
        data = base64.chomp
        "data:#{ content_type };base64,".concat(data)
      end

      def bytes(&block)
        if block
          each{|data| block.call(data)}
          length
        else
          bytes = []
          each{|data| bytes.push(*data)}
          bytes
        end
      end

      def close
        self
      end

      def content_type
        contentType
      end

      def update_date 
        updateDate
      end

      def created_at
        updateDate
      end

      def namespace
        self.class.namespace
      end
    end
  end

##
#
  def GridFS.build_chunk_model_for(namespace)
    prefix = namespace.name.split(/::/).last.downcase
    file_model_name = "#{ namespace.name }::File"
    chunk_model_name = "#{ namespace.name }::Chunk"

    Class.new do
      include Mongoid::Document

      singleton_class = class << self; self; end

      singleton_class.instance_eval do
        define_method(:name){ chunk_model_name }
        attr_accessor :file_model
        attr_accessor :namespace
      end

      self.default_collection_name = "#{ prefix }.chunks"

      field(:n, :type => Integer, :default => 0)
      field(:data, :type => Moped::BSON::Binary)

      belongs_to(:file, :foreign_key => :files_id, :class_name => file_model_name)

      index({:files_id => 1, :n => -1}, :unique => true) 

      def namespace
        self.class.namespace
      end
    end
  end

##
#
  GridFS.build_namespace_for(:Fs)
  File = Fs.file_model
  Chunk = Fs.chunk_model

  to_delegate = %w(
    put
    get
    delete
    find
    []
    []=
  )
  
  to_delegate.each do |method|
    class_eval <<-__
      def GridFS.#{ method }(*args, &block)
        ::GridFS::Fs::#{ method }(*args, &block)
      end
    __
  end

##
#
  def GridFS.reading(arg, &block)
    if arg.respond_to?(:read)
      rewind(arg) do |io|
        block.call(io)
      end
    else
      open(arg.to_s) do |io|
        block.call(io)
      end
    end
  end

  def GridFS.rewind(io, &block)
    begin
      pos = io.pos
      io.flush
      io.rewind
    rescue
      nil
    end

    begin
      block.call(io)
    ensure
      begin
        io.pos = pos
      rescue
        nil
      end
    end
  end

  def GridFS.extract_basename(object)
    filename = nil
    [:original_path, :original_filename, :path, :filename, :pathname].each do |msg|
      if object.respond_to?(msg)
        filename = object.send(msg)
        break
      end
    end
    filename ? cleanname(filename) : nil
  end

  def GridFS.extract_content_type(filename)
    content_type = MIME::Types.type_for(::File.basename(filename.to_s)).first
    content_type.to_s if content_type
  end

  def GridFS.cleanname(pathname)
    basename = ::File.basename(pathname.to_s)
    CGI.unescape(basename).gsub(%r/[^0-9a-zA-Z_@)(~.-]/, '_').gsub(%r/_+/,'_')
  end
end

GridFs = GridFS

routes.rb

  match "#{ Upload.route }/:id(/:variant(/*path_info))" => "uploads#show", :as => :upload, :via => :get, :format => false

upload.rb

require "open-uri"
require "mime/types"
require "digest/md5"

class Upload
##
#
  include Mongoid::Document
  include Mongoid::Timestamps

##
#
  belongs_to(:context, :polymorphic => true)

##
#
  field(:basename)
  field(:tmp, :type => Boolean, :default => false)

##
#
  validates_presence_of(:basename)

##
#
  class Variant
  ##
  #
    include Mongoid::Document
    include Mongoid::Timestamps

  ##
  #
    field(:name, :default => 'original', :type => String)
    field(:default, :default => false)

    field(:content_type, :type => String)
    field(:length, :type => Integer)
    field(:md5, :type => String)

    field(:grid_fs, :type => Hash)
    field(:s3, :type => Hash)

  ##
  #
    validates_presence_of(:name)
    validates_presence_of(:content_type)
    validates_presence_of(:length)
    validates_presence_of(:md5)

  ##
  #
    embedded_in(:upload, :class_name => '::Upload')

  ##
  #
    before_destroy do |variant|
      if variant.grid_fs?
        variant.grid_fs_file.destroy rescue nil
      end

      if variant.s3?
        variant.s3_object.destroy rescue nil
      end

      true
    end

  ##
  #
    def grid_fs_file
      if grid_fs?
        namespace = GridFS.namespace_for(grid_fs['prefix'])
        namespace.get(grid_fs['file_id'])
      end
    end

    def grid_fs?
      !grid_fs.blank?
    end

  ##
  #
    def s3?
      !s3.blank?
    end

    def s3_object
      nil
    end

  ##
  #
    def url_for(*args, &block)
      case
        when grid_fs?
          "#{ Upload.route }/#{ upload.id }/#{ name }/#{ basename }"
      end
    end

    def url(*args, &block)
      url_for(*args, &block)
    end

    def to_s(*args, &block)
      if grid_fs?
        return grid_fs_file.to_s(*args, &block)
      end

      if s3?
        return s3_object.to_s(*args, &block)
      end
    end

    def image?
      content_type.to_s.split('/').include?('image')
    end
  end

##
#
  embeds_many(:variants, :class_name => '::Upload::Variant') do
    def find_by_name(name)
      all.detect{|variant| variant.name.to_s == name.to_s}
    end

    alias_method(:for, :find_by_name)

    def original
      find_by_name(:original)
    end

    def default
      all.detect{|variant| variant.default} || original
    end

    def url_for(*args, &block)
      options = args.extract_options!.to_options!
      name = args.shift || options.delete(:name)
      variant = find_by_name(name) || original
      variant.url_for(options) if variant
    end

    def url(*args, &block)
      url_for(*args, &block)
    end
  end

  before_destroy do |upload|
    upload.variants.destroy_all
    true
  end

##
# upload support
#
# /system/uploads/ 1234567890/original/logo.png
# /system/uploads/ 1234567890/small/logo.png
#

  def add_variant(name, io_or_path, options = {})
    name         = name.to_s.downcase.to_sym
    basename     = Upload.extract_basename(io_or_path) 
    content_type = Upload.extract_content_type(basename)

    filename = "#{ Upload.route }/#{ id }/#{ name }/#{ basename }"

    options.to_options!
    options[:filename] = filename

    grid_fs_file = Upload.grid.put(io_or_path, options)

    attributes = {
      'name'         => name,
      'basename'     => basename,

      'grid_fs' => {
        'prefix'  => grid_fs_file.namespace.prefix,
        'file_id' => grid_fs_file.id
      },

      'content_type' => grid_fs_file.content_type,
      'length'       => grid_fs_file.length,
      'md5'          => grid_fs_file.md5
    }

    variant = variants.build(attributes)

    self.basename = variant.basename if self.basename.blank?

    variant
  end

  def Upload.upload(*args, &block)
    new.tap{|u| u.upload(*args, &block)}
  end

  def Upload.upload!(*args, &block)
    new.tap{|u| u.upload!(*args, &block)}
  end

  def upload(*args, &block)
    variants.destroy_all
    add_variant(:original, *args, &block)
  end

  def upload!(*args, &block)
    upload(*args, &block)
  ensure
    save!
  end

  def io=(io)
    upload(io)
  end

  def path=(path)
    upload(path)
  end

  def url_for(*args, &block)
    variants.url_for(*args, &block)
  end

  def url(*args, &block)
    url_for(*args, &block)
  end

  def urls(*args, &block)
    variants.map{|variant| variant.url(*args, &block)}
  end

  def grid_fs_files
    variants.map{|variant| variant.grid_fs_file}.compact
  end

  def grid_fs_file_chunks
    grid_fs_files.map{|grid_fs_file| grid_fs_file.chunks}.compact.flatten
  end

  def image?
    variants.any?{|variant| variant.image?}
  end

  def Upload.route
    "/system/uploads"
  end

  def Upload.id
    new.id
  end

  def Upload.grid
    @grid ||= GridFS.namespace_for(:fs)
  end

  def Upload.extract_basename(object)
    filename = nil
    [:original_path, :original_filename, :path, :filename, :pathname].each do |msg|
      if object.respond_to?(msg)
        filename = object.send(msg)
        break
      end
    end
    cleanname(filename || object.to_s)
  end

  def Upload.cleanname(pathname)
    basename = ::File.basename(pathname.to_s)
    CGI.unescape(basename).gsub(%r/[^0-9a-zA-Z_@)(~.-]/, '_').gsub(%r/_+/,'_')
  end

  def Upload.extract_content_type(filename)
    content_type = MIME::Types.type_for(::File.basename(filename.to_s)).first
    content_type.to_s if content_type
  end

##
# tmpwatch support - nukes old files...
#
  def Upload.tmpwatch!(conditions = {})
    conditions.to_options!

    if conditions.empty?
      conditions.update(:updated_at.lt => 1.week.ago)
    end

    Upload.where(conditions.merge(:tmp => true)).each do |upload|
      upload.destroy unless upload.context_id
    end
  end
  at_exit{ Upload.tmpwatch! } if Rails.env.production?
end

upload_controller.rb

class UploadsController < ApplicationController

  def show
    begin
      upload = Upload.find(params[:id])

      variant =
        upload.variants.find_by_name(params[:variant]) ||
        upload.variants.original

      if variant.grid_fs?
        grid_fs_file = variant.grid_fs_file
        self.content_type = grid_fs_file.content_type
        self.response_body = grid_fs_file
        return
      end

    rescue Object => e
      raise unless Rails.env.production?
      head(:not_found)
    end
  end

  def create
    raise NotImplementedError.new('but needed for ajax uploads...')
  end

end