Create a gist now

Instantly share code, notes, and snippets.

What would you like to do?
compile rocksdb as backend for asyncstorage

Credits

All credit to @sahrens for sharing fb's internal implementation.

Setup

  • clone rocksdb from https://github.com/facebook/rocksdb.

  • edit MakeFile inside rocksdb. Search for Platform-specific compilation around line 1122. Make th next few lines to look like this:

ifeq ($(PLATFORM), IOS)
# For iOS, create universal object files to be used on both the simulator and
# a device.
PLATFORMSROOT=/Applications/Xcode.app/Contents/Developer/Platforms
SIMULATORROOT=$(PLATFORMSROOT)/iPhoneSimulator.platform/Developer
DEVICEROOT=$(PLATFORMSROOT)/iPhoneOS.platform/Developer
IOSVERSION=$(shell defaults read $(PLATFORMSROOT)/iPhoneOS.platform/version CFBundleShortVersionString)

.cc.o:
	# mkdir -p ios-x86/$(dir $@)
	# $(CXX) $(CXXFLAGS) -isysroot $(SIMULATORROOT)/SDKs/iPhoneSimulator$(IOSVERSION).sdk -arch i686 -arch x86_64 -c $< -o ios-x86/$@
	mkdir -p ios-arm/$(dir $@)
	export IPHONEOS_DEPLOYMENT_TARGET=7.0
	xcrun -sdk iphoneos $(CXX) $(CXXFLAGS) -isysroot $(DEVICEROOT)/SDKs/iPhoneOS$(IOSVERSION).sdk -arch armv6 -arch armv7 -arch armv7s -arch arm64 -c $< -o ios-arm/$@
	lipo ios-arm/$@ -create -output $@

.c.o:
	# mkdir -p ios-x86/$(dir $@)
	# $(CC) $(CFLAGS) -isysroot $(SIMULATORROOT)/SDKs/iPhoneSimulator$(IOSVERSION).sdk -arch i686 -arch x86_64 -c $< -o ios-x86/$@
	mkdir -p ios-arm/$(dir $@)
	export IPHONEOS_DEPLOYMENT_TARGET=7.0
	xcrun -sdk iphoneos $(CC) $(CFLAGS) -isysroot $(DEVICEROOT)/SDKs/iPhoneOS$(IOSVERSION).sdk -arch armv6 -arch armv7 -arch armv7s -arch arm64 -c $< -o ios-arm/$@
	lipo ios-arm/$@ -create -output $@
  • Run TARGET_OS=IOS -j4 make static_lib in terminal. This will generate librocksdb.a under rocksdb source directory.

  • Add librocksdb.a to libraries in your xcode project from rocksdb source directory. make sure to check copy files.

  • Add RCTAsyncRocksDBStorage.mm and RCTAsyncRocksDBStorage.h to your xcode project. Make sure to name it correctly. extension needs to be .mm and .h

  • Xcode -> project -> build settings -> search enable bitcode. Set it to NO.

  • Xcode -> project -> build settings -> header search path. Add rocksdb/include directory path. for me its: "/Users/chirag/Desktop/rocksdb/include" and set it to recursive.

Hoepfully it works for you :)

// Copyright 2004-present Facebook. All Rights Reserved.
#import "RCTAsyncRocksDBStorage.h"
#include <string>
#import <Foundation/Foundation.h>
#import <RCTConvert.h>
#import <RCTLog.h>
#import <RCTUtils.h>
#define ROCKSDB_LITE 1
#include <rocksdb/db.h>
#include <rocksdb/merge_operator.h>
#include <rocksdb/options.h>
#include <rocksdb/slice.h>
#include <rocksdb/status.h>
static NSString *const RKAsyncRocksDBStorageDirectory = @"RKAsyncRocksDBStorage";
namespace {
rocksdb::Slice SliceFromString(NSString *string)
{
return rocksdb::Slice((const char *)[string UTF8String], [string lengthOfBytesUsingEncoding:NSUTF8StringEncoding]);
}
void deepMergeInto(NSMutableDictionary *output, NSDictionary *input) {
for (NSString *key in input) {
id inputValue = input[key];
if ([inputValue isKindOfClass:[NSDictionary class]]) {
NSMutableDictionary *nestedOutput;
id outputValue = output[key];
if ([outputValue isKindOfClass:[NSMutableDictionary class]]) {
nestedOutput = outputValue;
} else {
if ([outputValue isKindOfClass:[NSDictionary class]]) {
nestedOutput = [outputValue mutableCopy];
} else {
output[key] = [inputValue copy];
}
}
if (nestedOutput) {
deepMergeInto(nestedOutput, inputValue);
output[key] = nestedOutput;
}
} else {
output[key] = inputValue;
}
}
}
class JSONMergeOperator : public rocksdb::AssociativeMergeOperator {
public:
virtual bool Merge(const rocksdb::Slice &key,
const rocksdb::Slice *existingValue,
const rocksdb::Slice &newValue,
std::string *mergedValue,
rocksdb::Logger *logger) const override {
NSError *error;
NSMutableDictionary *existingDict;
if (existingValue) {
NSString *existingString = [NSString stringWithUTF8String:existingValue->data()];
existingDict = RCTJSONParseMutable(existingString, &error);
if (error) {
RCTLogError(@"Parse error in RKAsyncRocksDBStorage merge operation. Error:\n%@\nString:\n%@", error, existingString);
return false;
}
} else {
// Nothing to merge, just assign the string without even parsing.
mergedValue->assign(newValue.data(), newValue.size());
return true;
}
NSString *newString = [NSString stringWithUTF8String:newValue.data()];
NSMutableDictionary *newDict = RCTJSONParse(newString, &error);
deepMergeInto(existingDict, newDict);
NSString *mergedNSString = RCTJSONStringify(existingDict, &error);
mergedValue->assign([mergedNSString UTF8String], [mergedNSString lengthOfBytesUsingEncoding:NSUTF8StringEncoding]);
return true;
}
virtual const char *Name() const override {
return "JSONMergeOperator";
}
};
} // namespace
@implementation RCTAsyncRocksDBStorage
{
rocksdb::DB *_db;
}
@synthesize methodQueue = _methodQueue;
static NSString *RCTGetStorageDirectory()
{
static NSString *storageDirectory = nil;
static dispatch_once_t onceToken;
dispatch_once(&onceToken, ^{
storageDirectory = NSSearchPathForDirectoriesInDomains(NSDocumentDirectory, NSUserDomainMask, YES).firstObject;
storageDirectory = [storageDirectory stringByAppendingPathComponent:RKAsyncRocksDBStorageDirectory];
});
return storageDirectory;
}
RCT_EXPORT_MODULE()
- (BOOL)ensureDirectorySetup:(NSError **)error
{
if (_db) {
return YES;
}
rocksdb::Options options;
options.create_if_missing = true;
RCTAssert(error != nil, @"Must provide error pointer.");
rocksdb::Status status = rocksdb::DB::Open(options, [RCTGetStorageDirectory() UTF8String], &_db);
if (!status.ok() || !_db) {
RCTLogError(@"Failed to open db at path %@.\n\nRocksDB Status: %s.\n\nNSError: %@", RCTGetStorageDirectory(), status.ToString().c_str(), *error);
*error = [NSError errorWithDomain:@"rocksdb" code:100 userInfo:@{NSLocalizedDescriptionKey:@"Failed to open db"}];
return NO;
}
return YES;
}
RCT_EXPORT_METHOD(multiGet:(NSStringArray *)keys
callback:(RCTResponseSenderBlock)callback)
{
NSDictionary *errorOut;
NSError *error;
NSMutableArray *result = [[NSMutableArray alloc] initWithCapacity:keys.count];
BOOL success = [self ensureDirectorySetup:&error];
if (!success || error) {
errorOut = RCTMakeError(@"Failed to setup directory", nil, nil);
} else {
std::vector<rocksdb::Slice> sliceKeys;
sliceKeys.reserve(keys.count);
for (NSString *key in keys) {
sliceKeys.push_back(SliceFromString(key));
}
std::vector<std::string> values;
std::vector<rocksdb::Status> statuses = _db->MultiGet(rocksdb::ReadOptions(), sliceKeys, &values);
RCTAssert(values.size() == keys.count, @"Key and value arrays should be equal size");
for (size_t ii = 0; ii < values.size(); ii++) {
id value = [NSNull null];
auto status = statuses[ii];
if (!status.IsNotFound()) {
if (!status.ok()) {
errorOut = RCTMakeError(@"RKAsyncRocksDB failed getting key: ", keys[ii], keys[ii]);
} else {
value = [NSString stringWithUTF8String:values[ii].c_str()];
}
}
[result addObject:@[keys[ii], value]];
}
}
if (callback) {
callback(@[errorOut ? @[errorOut] : [NSNull null], result]);
}
}
// kvPairs is a list of key-value pairs, e.g. @[@[key1, val1], @[key2, val2], ...]
// TODO: write custom RCTConvert category method for kvPairs
RCT_EXPORT_METHOD(multiSet:(NSArray *)kvPairs
callback:(RCTResponseSenderBlock)callback)
{
auto updates = rocksdb::WriteBatch();
for (NSArray *kvPair in kvPairs) {
NSStringArray *pair = [RCTConvert NSStringArray:kvPair];
if (pair.count == 2) {
updates.Put(SliceFromString(kvPair[0]), SliceFromString(kvPair[1]));
} else {
if (callback) {
callback(@[@[RCTMakeAndLogError(@"Input must be an array of [key, value] arrays, got: ", kvPair, nil)]]);
}
return;
}
}
[self _performWriteBatch:&updates callback:callback];
}
RCT_EXPORT_METHOD(multiMerge:(NSArray *)kvPairs
callback:(RCTResponseSenderBlock)callback)
{
auto updates = rocksdb::WriteBatch();
for (NSArray *kvPair in kvPairs) {
NSStringArray *pair = [RCTConvert NSStringArray:kvPair];
if (pair.count == 2) {
updates.Merge(SliceFromString(pair[0]), SliceFromString(pair[1]));
} else {
if (callback) {
callback(@[@[RCTMakeAndLogError(@"Input must be an array of [key, value] arrays, got: ", kvPair, nil)]]);
}
return;
}
}
[self _performWriteBatch:&updates callback:callback];
}
RCT_EXPORT_METHOD(multiRemove:(NSArray *)keys
callback:(RCTResponseSenderBlock)callback)
{
auto updates = rocksdb::WriteBatch();
for (NSString *key in keys) {
updates.Delete(SliceFromString(key));
}
[self _performWriteBatch:&updates callback:callback];
}
// TODO (#5906496): There's a lot of duplication in the error handling code here - can we refactor this?
- (void)_performWriteBatch:(rocksdb::WriteBatch *)updates callback:(RCTResponseSenderBlock)callback
{
NSDictionary *errorOut;
NSError *error;
BOOL success = [self ensureDirectorySetup:&error];
if (!success || error) {
errorOut = RCTMakeError(@"Failed to setup storage", nil, nil);
} else {
rocksdb::Status status = _db->Write(rocksdb::WriteOptions(), updates);
if (!status.ok()) {
errorOut = RCTMakeError(@"Failed to write to RocksDB database.", nil, nil);
}
}
if (callback) {
callback(@[errorOut ? @[errorOut] : [NSNull null]]);
}
}
RCT_EXPORT_METHOD(clear:(RCTResponseSenderBlock)callback)
{
// [self _nullOutDB];
// NSDictionary *errorOut;
// NSError *error;
// NSURL *userDirectory = getOrCreateRocksDBPath(&error);
// if (!userDirectory) {
// errorOut = RCTMakeError(@"Failed to setup storage", nil, nil);
// } else {
// rocksdb::Status status = rocksdb::DestroyDB([[userDirectory path] UTF8String], rocksdb::Options());
// if (!status.ok()) {
// errorOut = RCTMakeError(@"RocksDB:clear failed to destroy db at path ", [userDirectory path], nil);
// }
// }
// if (callback) {
// callback(@[errorOut ?: [NSNull null]]);
// }
}
RCT_EXPORT_METHOD(getAllKeys:(RCTResponseSenderBlock)callback)
{
NSError *error;
NSMutableArray *allKeys = [NSMutableArray new];
NSDictionary *errorOut;
BOOL success = [self ensureDirectorySetup:&error];
if (!success || error) {
errorOut = RCTMakeError(@"Failed to setup storage", nil, nil);
} else {
rocksdb::Iterator *it = _db->NewIterator(rocksdb::ReadOptions());
for (it->SeekToFirst(); it->Valid(); it->Next()) {
[allKeys addObject:[NSString stringWithUTF8String:it->key().data()]];
}
}
if (callback) {
callback(@[errorOut ?: [NSNull null], allKeys]);
}
}
@end
#import "RCTBridgeModule.h"
@interface RCTAsyncRocksDBStorage : NSObject <RCTBridgeModule>
- (void)multiGet:(NSArray *)keys callback:(RCTResponseSenderBlock)callback;
- (void)multiSet:(NSArray *)kvPairs callback:(RCTResponseSenderBlock)callback;
- (void)multiRemove:(NSArray *)keys callback:(RCTResponseSenderBlock)callback;
- (void)clear:(RCTResponseSenderBlock)callback;
- (void)getAllKeys:(RCTResponseSenderBlock)callback;
@end
@mvayngrib

should it be this?:

PLATFORM=IOS make -j4 static_lib
@mvayngrib

can't wait to use this, but having some problems :)

did you use the master branch? i'm getting a bunch of errors (pasted below)

i tried both PLATFORM=IOS make -j4 static_lib and TARGET_OS=IOS make -j4 static_lib

i also didn't find the Enable Bitcode option, but I'm running xcode 6.4, maybe it's only in 7. What version of Xcode are you using?

Undefined symbols for architecture x86_64:
  "_BZ2_bzCompress", referenced from:
      rocksdb::BlockBasedTableBuilder::WriteBlock(rocksdb::Slice const&, rocksdb::BlockHandle*) in librocksdb.a(block_based_table_builder.o)
  "_BZ2_bzCompressEnd", referenced from:
      rocksdb::BlockBasedTableBuilder::WriteBlock(rocksdb::Slice const&, rocksdb::BlockHandle*) in librocksdb.a(block_based_table_builder.o)
  "_BZ2_bzCompressInit", referenced from:
      rocksdb::BlockBasedTableBuilder::WriteBlock(rocksdb::Slice const&, rocksdb::BlockHandle*) in librocksdb.a(block_based_table_builder.o)
  "_BZ2_bzDecompress", referenced from:
      rocksdb::UncompressBlockContents(char const*, unsigned long, rocksdb::BlockContents*, unsigned int) in librocksdb.a(format.o)
  "_BZ2_bzDecompressEnd", referenced from:
      rocksdb::UncompressBlockContents(char const*, unsigned long, rocksdb::BlockContents*, unsigned int) in librocksdb.a(format.o)
  "_BZ2_bzDecompressInit", referenced from:
      rocksdb::UncompressBlockContents(char const*, unsigned long, rocksdb::BlockContents*, unsigned int) in librocksdb.a(format.o)
  "_deflate", referenced from:
      rocksdb::BlockBasedTableBuilder::WriteBlock(rocksdb::Slice const&, rocksdb::BlockHandle*) in librocksdb.a(block_based_table_builder.o)
  "_deflateEnd", referenced from:
      rocksdb::BlockBasedTableBuilder::WriteBlock(rocksdb::Slice const&, rocksdb::BlockHandle*) in librocksdb.a(block_based_table_builder.o)
  "_deflateInit2_", referenced from:
      rocksdb::BlockBasedTableBuilder::WriteBlock(rocksdb::Slice const&, rocksdb::BlockHandle*) in librocksdb.a(block_based_table_builder.o)
  "_inflate", referenced from:
      rocksdb::Zlib_Uncompress(char const*, unsigned long, int*, unsigned int, int) in librocksdb.a(format.o)
  "_inflateEnd", referenced from:
      rocksdb::Zlib_Uncompress(char const*, unsigned long, int*, unsigned int, int) in librocksdb.a(format.o)
  "_inflateInit2_", referenced from:
      rocksdb::Zlib_Uncompress(char const*, unsigned long, int*, unsigned int, int) in librocksdb.a(format.o)
  "_opendir$INODE64", referenced from:
      rocksdb::(anonymous namespace)::PosixEnv::GetChildren(std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::vector<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >, std::__1::allocator<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > > >*) in librocksdb.a(env_posix.o)
  "_readdir$INODE64", referenced from:
      rocksdb::(anonymous namespace)::PosixEnv::GetChildren(std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > const&, std::__1::vector<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> >, std::__1::allocator<std::__1::basic_string<char, std::__1::char_traits<char>, std::__1::allocator<char> > > >*) in librocksdb.a(env_posix.o)
ld: symbol(s) not found for architecture x86_64
clang: error: linker command failed with exit code 1 (use -v to see invocation)
@chirag04
Owner

@mvayngrib: I'm using xcode 7. use TARGET_OS=IOS make -j4 static_lib. Also, this only works on a real device. I get the same error for simulator.

@mvayngrib

@chirag04 i see, i had problems with react-native and xcode 7 a few weeks ago, but maybe they've been dealt with since

@chadwilken

I keep getting errors when using it with AsyncStorage with the error:

Failed to open db at path /Users/chadwilken/Library/Developer/CoreSimulator/Devices/818C47D2-ECF0-4003-865E-1FCAADCEF624/data/Containers/Data/Application/6C4F8F80-52E3-48B1-8ED5-84FB9F087514/Documents/RKAsyncRocksDBStorage.

RocksDB Status: IO error: lock /Users/chadwilken/Library/Developer/CoreSimulator/Devices/818C47D2-ECF0-4003-865E-1FCAADCEF624/data/Containers/Data/Application/6C4F8F80-52E3-48B1-8ED5-84FB9F087514/Documents/RKAsyncRocksDBStorage/LOCK: No locks available.

Any idea what would cause this?

@mvayngrib

@chadwilken _db isn't getting released on reload. Add delete _db in dealloc, as I did here: https://github.com/tradle/react-native-async-storage-rocks/blob/master/ios/RCTAsyncRocksDBStorage.mm#L115

Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment