git-annex api for web remotes

api.py

import re, os, signal
import tornado.ioloop, tornado.web, tornado.httpserver
from subprocess import Popen, PIPE
from sys import exit

# DON'T FORGET TO SET THIS!
ANNEX_DIR = "/path/to/your/remote/repository"
API_PORT = 8888
NUM_PROCESSES = 10

# this is just so you don't get errors/exception cruft when you ctrl-c the server :)
def terminationHandler(signal, frame): exit(0)
signal.signal(signal.SIGINT, terminationHandler)

class Api(tornado.web.Application):
	def __init__(self):
		print "API started..."
		
		self.routes = [
			(r"/", self.MainHandler),(r"/sync/", self.SyncHandler),
			(r"/files/(\S+)", self.FileHandler)]
	
	def startup(self):
		tornado.web.Application.__init__(self, self.routes)
		
		server = tornado.httpserver.HTTPServer(self)
		server.bind(API_PORT)
		server.start(NUM_PROCESSES)
		
		tornado.ioloop.IOLoop.instance().start()
	
	def syncAnnex(self):
		create_rx = r'\s*create mode (?:\d+) (?:(?!\.data/.*))([a-zA-Z0-9_\-\./]+)'
		cmd = ['git', 'annex', 'sync']
		
		old_dir = os.getcwd()
		os.chdir(ANNEX_DIR)
		
		p = Popen(cmd, stdout=PIPE, close_fds=True)
		data = p.stdout.readline()
		
		new_files = []

		while data:
			print data.strip()
			create = re.findall(create_rx, data.strip())
			if len(create) == 1:
				print "INIT NEW FILE: %s" % create[0]
				new_files.append(create[0])
				
				'''
					OMG! WHAT SHOULD WE DO NOW THAT WE HAVE A NEW FILE?
				'''
				
			data = p.stdout.readline()
		p.stdout.close()
		
		os.chdir(old_dir)
		return new_files
	
	class SyncHandler(tornado.web.RequestHandler):
		@tornado.web.asynchronous
		def get(self):
			result = "HI!  Welcome to your ersatz dropbox!<br />"
			synced_files = self.application.syncAnnex()
			if len(synced_files) > 0:
				result += ("synced files:<br />%s" % synced_files)
			else:
				result += "no synced files..."
				
			self.finish(result)
	
	class MainHandler(tornado.web.RequestHandler):
		@tornado.web.asynchronous
		def get(self):
			self.finish("HI!  Welcome to your ersatz dropbox!")
	
	class FileHandler(tornado.web.RequestHandler):
		@tornado.web.asynchronous
		def get(self, file_path):
			# if file exists in git-annex, return the file
			# else, return 404 (file not found)

			if self.application.fileExistsInAnnex(file_path):
				with open(os.path.join(ANNEX_DIR, file_path), 'rb') as file:
					# TODO: set content-type, I.E.
					# self.set_header("Content-Type", "application/octet-stream")
					self.finish(file.read())
				return

			# TODO: log this: we want to know who/why is requesting non-entities
			else: self.set_status(404)
			self.finish("File not found, dummy.")

		@tornado.web.asynchronous
		def head(self, file_path):
			# if file exists in git-annex, return 200
			# else, return 405

			if self.application.fileExistsInAnnex(file_path): self.set_status(200)
			else: self.set_status(405)

			self.finish("OK")
	
	def fileExistsInAnnex(self, file_path, auto_add=True):
		old_dir = os.getcwd()
		os.chdir(ANNEX_DIR)

		cmd0 = ['git', 'annex', 'find', file_path]			
		p0 = Popen(cmd0, stdout=PIPE, close_fds=True)
		data0 = p0.stdout.readline()

		while data0:
			if data0.strip() == file_path:
				if auto_add:
					web_match_found = False
					m_path = re.compile("\s*web: http://localhost:8888/files/%s" % file_path)
					cmd1 = ['git', 'annex', 'whereis', file_path]
					p1 = Popen(cmd1, stdout=PIPE, close_fds=True)
					data1 = p1.stdout.readline()

					# if this file has not already been added to web remote, add it
					while data1:
						if re.match(m_path, data1) is not None:
							web_match_found = True
							p1.stdout.close()
							break

						data1 = p1.stdout.readline()

					p1.stdout.close()

					if not web_match_found:
						cmd2 = ['git', 'annex', 'addurl', '--file' , file_path,
							'http://localhost:8888/files/%s' % file_path,
							'--relaxed']
						p2 = Popen(cmd2)
						p2.wait()
						

				p0.stdout.close()
				os.chdir(old_dir)
				return True

			data0 = p0.stdout.readline()

		p0.stdout.close()
		os.chdir(old_dir)
		return False

	
if __name__ == "__main__":	
	api = Api()
	api.startup()