Skip to content

Instantly share code, notes, and snippets.

@ahoward
Created Jul 23, 2012
Embed
What would you like to do?
require "mime/types"
require "digest/md5"
require "cgi"
### see: http://www.mongodb.org/display/DOCS/GridFS+Specification
class GridFS
##
#
attr_accessor :prefix
attr_accessor :namespace
attr_accessor :file_model
attr_accessor :chunk_model
##
#
def initialize(options = {})
options.to_options!
@prefix = options[:prefix] || 'fs'
@namespace = GridFS.namespace_for(@prefix)
@file_model = @namespace.file_model
@chunk_model = @file_model.chunk_model
end
##
#
def GridFS.namespace_for(prefix)
prefix = prefix.to_s.downcase
const = "::GridFS::#{ prefix.to_s.camelize }"
namespace = const.split(/::/).last
const_defined?(namespace) ? const_get(namespace) : build_namespace_for(namespace)
end
##
#
def GridFS.build_namespace_for(prefix)
prefix = prefix.to_s.downcase
const = prefix.camelize
namespace =
Module.new do
module_eval(&NamespaceMixin)
self
end
const_set(const, namespace)
file_model = build_file_model_for(namespace)
chunk_model = build_chunk_model_for(namespace)
file_model.namespace = namespace
chunk_model.namespace = namespace
file_model.chunk_model = chunk_model
chunk_model.file_model = file_model
namespace.prefix = prefix
namespace.file_model = file_model
namespace.chunk_model = chunk_model
namespace.send(:const_set, :File, file_model)
namespace.send(:const_set, :Chunk, chunk_model)
#at_exit{ file_model.create_indexes rescue nil }
#at_exit{ chunk_model.create_indexes rescue nil }
const_get(const)
end
NamespaceMixin = proc do
class << self
attr_accessor :prefix
attr_accessor :file_model
attr_accessor :chunk_model
def to_s
prefix
end
def namespace
prefix
end
def put(arg, attributes = {})
chunks = []
file = file_model.new
attributes.to_options!
if attributes.has_key?(:id)
file.id = attributes.delete(:id)
end
if attributes.has_key?(:_id)
file.id = attributes.delete(:_id)
end
if attributes.has_key?(:content_type)
attributes[:contentType] = attributes.delete(:content_type)
end
if attributes.has_key?(:upload_date)
attributes[:uploadDate] = attributes.delete(:upload_date)
end
md5 = Digest::MD5.new
length = 0
chunkSize = file.chunkSize
n = 0
GridFS.reading(arg) do |io|
filename = [file.id.to_s, GridFS.extract_basename(io)].join('/').squeeze('/')
content_type = GridFS.extract_content_type(filename) || file.contentType
attributes[:filename] ||= filename
attributes[:contentType] ||= content_type
while((buf = io.read(chunkSize)))
md5 << buf
length += buf.size
chunk = file.chunks.build
chunk.data = binary_for(buf)
chunk.n = n
n += 1
chunk.save!
chunks.push(chunk)
end
end
attributes[:length] ||= length
attributes[:uploadDate] ||= Time.now.utc
attributes[:md5] ||= md5.hexdigest
file.update_attributes(attributes)
file.save!
file
ensure
chunks.each{|chunk| chunk.destroy rescue nil} if $!
end
if defined?(Moped)
def binary_for(*buf)
Moped::BSON::Binary.new(:generic, buf.join)
end
else
def binary_for(buf)
BSON::Binary.new(buf.bytes.to_a)
end
end
def get(id)
file_model.find(id)
end
def delete(id)
file_model.find(id).destroy
rescue
nil
end
def where(conditions = {})
case conditions
when String
file_model.where(:filename => conditions)
else
file_model.where(conditions)
end
end
def find(*args)
where(*args).first
end
def [](filename)
file_model.where(:filename => filename.to_s).first
end
def []=(filename, readable)
file = self[filename]
file.destroy if file
put(readable, :filename => filename.to_s)
end
# TODO - opening with a mode = 'w' should return a GridIO::IOProxy
# implementing a StringIO-like interface
#
def open(filename, mode = 'r', &block)
raise NotImplementedError
end
end
end
##
#
def GridFS.build_file_model_for(namespace)
prefix = namespace.name.split(/::/).last.downcase
file_model_name = "#{ namespace.name }::File"
chunk_model_name = "#{ namespace.name }::Chunk"
Class.new do
include Mongoid::Document
singleton_class = class << self; self; end
singleton_class.instance_eval do
define_method(:name){ file_model_name }
attr_accessor :chunk_model
attr_accessor :namespace
end
self.default_collection_name = "#{ prefix }.files"
field(:filename, :type => String)
field(:contentType, :type => String, :default => 'application/octet-stream')
field(:length, :type => Integer, :default => 0)
field(:chunkSize, :type => Integer, :default => (256 * (2 ** 20)))
field(:uploadDate, :type => Date, :default => Time.now.utc)
field(:md5, :type => String, :default => Digest::MD5.hexdigest(''))
%w( filename contentType length chunkSize uploadDate md5 ).each do |f|
validates_presence_of(f)
end
validates_uniqueness_of(:filename)
has_many(:chunks, :class_name => chunk_model_name, :inverse_of => :files, :dependent => :destroy, :order => [:n, :asc])
index({:filename => 1}, :unique => true)
def basename
::File.basename(filename)
end
def prefix
self.class.namespace.prefix
end
def each(&block)
chunks.all.order_by([:n, :asc]).each do |chunk|
block.call(chunk.data.to_s)
end
end
def to_s
to_s = ''
each{|data| to_s << data}
to_s
end
def base64
Array(to_s).pack('m')
end
def data_uri(options = {})
data = base64.chomp
"data:#{ content_type };base64,".concat(data)
end
def bytes(&block)
if block
each{|data| block.call(data)}
length
else
bytes = []
each{|data| bytes.push(*data)}
bytes
end
end
def close
self
end
def content_type
contentType
end
def update_date
updateDate
end
def created_at
updateDate
end
def namespace
self.class.namespace
end
end
end
##
#
def GridFS.build_chunk_model_for(namespace)
prefix = namespace.name.split(/::/).last.downcase
file_model_name = "#{ namespace.name }::File"
chunk_model_name = "#{ namespace.name }::Chunk"
Class.new do
include Mongoid::Document
singleton_class = class << self; self; end
singleton_class.instance_eval do
define_method(:name){ chunk_model_name }
attr_accessor :file_model
attr_accessor :namespace
end
self.default_collection_name = "#{ prefix }.chunks"
field(:n, :type => Integer, :default => 0)
field(:data, :type => Moped::BSON::Binary)
belongs_to(:file, :foreign_key => :files_id, :class_name => file_model_name)
index({:files_id => 1, :n => -1}, :unique => true)
def namespace
self.class.namespace
end
end
end
##
#
GridFS.build_namespace_for(:Fs)
File = Fs.file_model
Chunk = Fs.chunk_model
to_delegate = %w(
put
get
delete
find
[]
[]=
)
to_delegate.each do |method|
class_eval <<-__
def GridFS.#{ method }(*args, &block)
::GridFS::Fs::#{ method }(*args, &block)
end
__
end
##
#
def GridFS.reading(arg, &block)
if arg.respond_to?(:read)
rewind(arg) do |io|
block.call(io)
end
else
open(arg.to_s) do |io|
block.call(io)
end
end
end
def GridFS.rewind(io, &block)
begin
pos = io.pos
io.flush
io.rewind
rescue
nil
end
begin
block.call(io)
ensure
begin
io.pos = pos
rescue
nil
end
end
end
def GridFS.extract_basename(object)
filename = nil
[:original_path, :original_filename, :path, :filename, :pathname].each do |msg|
if object.respond_to?(msg)
filename = object.send(msg)
break
end
end
filename ? cleanname(filename) : nil
end
def GridFS.extract_content_type(filename)
content_type = MIME::Types.type_for(::File.basename(filename.to_s)).first
content_type.to_s if content_type
end
def GridFS.cleanname(pathname)
basename = ::File.basename(pathname.to_s)
CGI.unescape(basename).gsub(%r/[^0-9a-zA-Z_@)(~.-]/, '_').gsub(%r/_+/,'_')
end
end
GridFs = GridFS
match "#{ Upload.route }/:id(/:variant(/*path_info))" => "uploads#show", :as => :upload, :via => :get, :format => false
require "open-uri"
require "mime/types"
require "digest/md5"
class Upload
##
#
include Mongoid::Document
include Mongoid::Timestamps
##
#
belongs_to(:context, :polymorphic => true)
##
#
field(:basename)
field(:tmp, :type => Boolean, :default => false)
##
#
validates_presence_of(:basename)
##
#
class Variant
##
#
include Mongoid::Document
include Mongoid::Timestamps
##
#
field(:name, :default => 'original', :type => String)
field(:default, :default => false)
field(:content_type, :type => String)
field(:length, :type => Integer)
field(:md5, :type => String)
field(:grid_fs, :type => Hash)
field(:s3, :type => Hash)
##
#
validates_presence_of(:name)
validates_presence_of(:content_type)
validates_presence_of(:length)
validates_presence_of(:md5)
##
#
embedded_in(:upload, :class_name => '::Upload')
##
#
before_destroy do |variant|
if variant.grid_fs?
variant.grid_fs_file.destroy rescue nil
end
if variant.s3?
variant.s3_object.destroy rescue nil
end
true
end
##
#
def grid_fs_file
if grid_fs?
namespace = GridFS.namespace_for(grid_fs['prefix'])
namespace.get(grid_fs['file_id'])
end
end
def grid_fs?
!grid_fs.blank?
end
##
#
def s3?
!s3.blank?
end
def s3_object
nil
end
##
#
def url_for(*args, &block)
case
when grid_fs?
"#{ Upload.route }/#{ upload.id }/#{ name }/#{ basename }"
end
end
def url(*args, &block)
url_for(*args, &block)
end
def to_s(*args, &block)
if grid_fs?
return grid_fs_file.to_s(*args, &block)
end
if s3?
return s3_object.to_s(*args, &block)
end
end
def image?
content_type.to_s.split('/').include?('image')
end
end
##
#
embeds_many(:variants, :class_name => '::Upload::Variant') do
def find_by_name(name)
all.detect{|variant| variant.name.to_s == name.to_s}
end
alias_method(:for, :find_by_name)
def original
find_by_name(:original)
end
def default
all.detect{|variant| variant.default} || original
end
def url_for(*args, &block)
options = args.extract_options!.to_options!
name = args.shift || options.delete(:name)
variant = find_by_name(name) || original
variant.url_for(options) if variant
end
def url(*args, &block)
url_for(*args, &block)
end
end
before_destroy do |upload|
upload.variants.destroy_all
true
end
##
# upload support
#
# /system/uploads/ 1234567890/original/logo.png
# /system/uploads/ 1234567890/small/logo.png
#
def add_variant(name, io_or_path, options = {})
name = name.to_s.downcase.to_sym
basename = Upload.extract_basename(io_or_path)
content_type = Upload.extract_content_type(basename)
filename = "#{ Upload.route }/#{ id }/#{ name }/#{ basename }"
options.to_options!
options[:filename] = filename
grid_fs_file = Upload.grid.put(io_or_path, options)
attributes = {
'name' => name,
'basename' => basename,
'grid_fs' => {
'prefix' => grid_fs_file.namespace.prefix,
'file_id' => grid_fs_file.id
},
'content_type' => grid_fs_file.content_type,
'length' => grid_fs_file.length,
'md5' => grid_fs_file.md5
}
variant = variants.build(attributes)
self.basename = variant.basename if self.basename.blank?
variant
end
def Upload.upload(*args, &block)
new.tap{|u| u.upload(*args, &block)}
end
def Upload.upload!(*args, &block)
new.tap{|u| u.upload!(*args, &block)}
end
def upload(*args, &block)
variants.destroy_all
add_variant(:original, *args, &block)
end
def upload!(*args, &block)
upload(*args, &block)
ensure
save!
end
def io=(io)
upload(io)
end
def path=(path)
upload(path)
end
def url_for(*args, &block)
variants.url_for(*args, &block)
end
def url(*args, &block)
url_for(*args, &block)
end
def urls(*args, &block)
variants.map{|variant| variant.url(*args, &block)}
end
def grid_fs_files
variants.map{|variant| variant.grid_fs_file}.compact
end
def grid_fs_file_chunks
grid_fs_files.map{|grid_fs_file| grid_fs_file.chunks}.compact.flatten
end
def image?
variants.any?{|variant| variant.image?}
end
def Upload.route
"/system/uploads"
end
def Upload.id
new.id
end
def Upload.grid
@grid ||= GridFS.namespace_for(:fs)
end
def Upload.extract_basename(object)
filename = nil
[:original_path, :original_filename, :path, :filename, :pathname].each do |msg|
if object.respond_to?(msg)
filename = object.send(msg)
break
end
end
cleanname(filename || object.to_s)
end
def Upload.cleanname(pathname)
basename = ::File.basename(pathname.to_s)
CGI.unescape(basename).gsub(%r/[^0-9a-zA-Z_@)(~.-]/, '_').gsub(%r/_+/,'_')
end
def Upload.extract_content_type(filename)
content_type = MIME::Types.type_for(::File.basename(filename.to_s)).first
content_type.to_s if content_type
end
##
# tmpwatch support - nukes old files...
#
def Upload.tmpwatch!(conditions = {})
conditions.to_options!
if conditions.empty?
conditions.update(:updated_at.lt => 1.week.ago)
end
Upload.where(conditions.merge(:tmp => true)).each do |upload|
upload.destroy unless upload.context_id
end
end
at_exit{ Upload.tmpwatch! } if Rails.env.production?
end
class UploadsController < ApplicationController
def show
begin
upload = Upload.find(params[:id])
variant =
upload.variants.find_by_name(params[:variant]) ||
upload.variants.original
if variant.grid_fs?
grid_fs_file = variant.grid_fs_file
self.content_type = grid_fs_file.content_type
self.response_body = grid_fs_file
return
end
rescue Object => e
raise unless Rails.env.production?
head(:not_found)
end
end
def create
raise NotImplementedError.new('but needed for ajax uploads...')
end
end
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment