Skip to content

Instantly share code, notes, and snippets.

@echeipesh
Last active December 16, 2015 20:35
Show Gist options
  • Star 0 You must be signed in to star a gist
  • Fork 1 You must be signed in to fork a gist
  • Save echeipesh/812948930d6f5ec7b442 to your computer and use it in GitHub Desktop.
Save echeipesh/812948930d6f5ec7b442 to your computer and use it in GitHub Desktop.
trait LayerDeleter[ID] {
def delete(id: ID): Unit
}
class S3LayerDeleter(val store: AttributeStore[JsonFormat]) extends LayerDeleter[LayerId] {
/** Implement using S3 DELETE */
def delete(id: ID): Unit
}
class HadoopLayerDeleter(val store: AttributeStore[JsonFormat]) extends LayerDeleter[LayerId] {
/** Implement using Hadoop FileSystem */
def delete(id: ID): Unit
}
class AccumuloLayerDeleter(val store: AttributeStore[JsonFormat]) extends LayerDeleter[LayerId] {
/** Implement using BatchDeleter */
def delete(id: ID): Unit
}
trait LayerCopier[ID] {
def copy(from: ID, to: ID): Unit
}
class S3LayerCopier(val store: AttributeStore[JsonFormat]) extends LayerCopier[LayerId] {
/** Implement using S3 COPY */
// - Copy to different bucket/key/LayerId
// - Copy with different index? (Dfferent target file name in COPY)
def copy(from: ID, to: ID): Unit
def copy(from: LayerId, to: LayerId, key: String)
def copy(from: LayerId, to: LayerId, bucket: String, key: String)
}
class HadoopLayerCopier(val store: AttributeStore[JsonFormat]) extends LayerCopier[LayerId] {
/** Implement using Hadoop FileSystem */
// - Copy to different Path/LayerId
// - Copy with different index? (Can't do With FileSystem.copy)
def copy(from: ID, to: ID): Unit
def copy(from: LayerId, to: LayerId, path: Path)
}
class AccumuloLayerCopier(val store: AttributeStore[JsonFormat]) extends LayerCopier[LayerId] {
/** Must read/bulk import */
// - Copy to another table/LayerId
// - Copy with different index? (Sure, we have to read it anyway)
def copy(from: ID, to: ID): Unit
def copy(from: LayerId, to: LayerId, table: String)
}
trait LayerMover[ID] {
def move(from: ID, to: ID): Unit
}
class S3LayerMover(val store: AttributeStore[JsonFormat]) extends LayerMover[LayerId] {
/** Implement using S3 COPY/DELETE command */
def move(from: ID, to: ID): Unit
def move(from: LayerId, to: LayerId, key: String)
def move(from: LayerId, to: LayerId, bucket: String, key: String)
}
class HadoopLayerMover(val store: AttributeStore[JsonFormat]) extends LayerMover[LayerId] {
/** Implement using Hadoop FileSystem move */
def move(from: ID, to: ID): Unit
def move(from: LayerId, to: LayerId, path: Path)
}
class AccumuloLayerMover(val store: AttributeStore[JsonFormat]) extends LayerMover[LayerId] {
/** Maybe possible to use sublcass TransformingIterator, must be on Accumulo CLASS_PATH
* Fallback is to copy/delete */
def move(from: ID, to: ID): Unit
def move(from: LayerId, to: LayerId, table: String)
}
trait LayerReindexer[ID] {
def reindex(id: ID, keyIndex: KeyIndex[K])
def reindex(id: ID, indexMethod: KeyIndexMethod[K]): Unit
}
class S3LayerReindexer(val store: AttributeStore[JsonFormat]) extends LayerReindexer[LayerId] {
/** Implement using S3 Copy/Delete command */
// TODO: What happens when we reindex in place? How do we not stomp all over ourselves?
}
class HadoopLayerReindexer(val store: AttributeStore[JsonFormat]) extends LayerReindexer[LayerId] {
/** Read the layer, Backup old folder, write with new index, delete backup */
}
class AccumuloLayerReindexer(val store: AttributeStore[JsonFormat]) extends LayerReindexer[LayerId] {
/** Read the layer, delete old layer, write new layer. Very fragile. Can't backup for inplace reindex */
// TODO: What do we do if we reindex in place? Again we're stomping
}
trait LayerManager[ID] {
def delete(id: ID): Unit
def copy(from: ID, to: ID): Unit
def move(from: ID, to: ID): Unit
def reindex[K](id: ID, indexMethod: KeyIndexMethod[K])
def reindex[K](id: ID, keyIndex: KeyIndex[K])
}
class S3LayerManager(val store: AttributeStore[JsonFormat]) extends LayerManager[LayerId] {
def copy(from: LayerId, to: LayerId, bucket: String, key: String)
def move(from: ID, to: ID, bucket: String, key: String)
def reindex[K](id: ID, indexMethod: KeyIndexMethod[K], bucket: String, key: String)
def reindex[K](id: ID, keyIndex: KeyIndex[K], bucket: String, key: String)
}
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment