alex.patch

From d065fd65b96419a26657a4df03447f9e04e3dff6 Mon Sep 17 00:00:00 2001
From: Alex Yang <alexyang@fb.com>
Date: Thu, 19 Jan 2017 17:01:22 -0800
Subject: [PATCH] add rocksdb_force_flush_memtable_and_lzero sys var which
 allows manually flushing memtable and L0

---
 mysql-test/r/mysqld--help-notwin-profiling.result  |  4 ++
 mysql-test/r/mysqld--help-notwin.result            |  4 ++
 mysql-test/suite/rocksdb/r/rocksdb.result          |  1 +
 ...force_flush_memtable_and_lzero_now_basic.result | 50 ++++++++++++++++++++++
 ...b_force_flush_memtable_and_lzero_now_basic.test | 17 ++++++++
 storage/rocksdb/ha_rocksdb.cc                      | 48 +++++++++++++++++++++
 6 files changed, 124 insertions(+)
 create mode 100644 mysql-test/suite/rocksdb_sys_vars/r/rocksdb_force_flush_memtable_and_lzero_now_basic.result
 create mode 100644 mysql-test/suite/rocksdb_sys_vars/t/rocksdb_force_flush_memtable_and_lzero_now_basic.test

diff --git a/mysql-test/r/mysqld--help-notwin-profiling.result b/mysql-test/r/mysqld--help-notwin-profiling.result
index a055e17..3f3b6f1 100644
--- a/mysql-test/r/mysqld--help-notwin-profiling.result
+++ b/mysql-test/r/mysqld--help-notwin-profiling.result
@@ -1142,6 +1142,9 @@ The following options may be given as the first argument:
  Forces memtable flush on ANALZYE table to get accurate
  cardinality
  (Defaults to on; use --skip-rocksdb-flush-memtable-on-analyze to disable.)
+ --rocksdb-force-flush-memtable-and-lzero-now 
+ Acts similar to force_flush_memtable_now, but also
+ compacts all L0 files.
  --rocksdb-force-flush-memtable-now 
  Forces memstore flush which may block all write requests
  so be careful
@@ -1961,6 +1964,7 @@ rocksdb-enable-thread-tracking FALSE
 rocksdb-enable-write-thread-adaptive-yield FALSE
 rocksdb-error-if-exists FALSE
 rocksdb-flush-memtable-on-analyze TRUE
+rocksdb-force-flush-memtable-and-lzero-now FALSE
 rocksdb-force-flush-memtable-now FALSE
 rocksdb-force-index-records-in-range 0
 rocksdb-global-info ON
diff --git a/mysql-test/r/mysqld--help-notwin.result b/mysql-test/r/mysqld--help-notwin.result
index b120a91..7e433b9 100644
--- a/mysql-test/r/mysqld--help-notwin.result
+++ b/mysql-test/r/mysqld--help-notwin.result
@@ -1140,6 +1140,9 @@ The following options may be given as the first argument:
  Forces memtable flush on ANALZYE table to get accurate
  cardinality
  (Defaults to on; use --skip-rocksdb-flush-memtable-on-analyze to disable.)
+ --rocksdb-force-flush-memtable-and-lzero-now 
+ Acts similar to force_flush_memtable_now, but also
+ compacts all L0 files.
  --rocksdb-force-flush-memtable-now 
  Forces memstore flush which may block all write requests
  so be careful
@@ -1958,6 +1961,7 @@ rocksdb-enable-thread-tracking FALSE
 rocksdb-enable-write-thread-adaptive-yield FALSE
 rocksdb-error-if-exists FALSE
 rocksdb-flush-memtable-on-analyze TRUE
+rocksdb-force-flush-memtable-and-lzero-now FALSE
 rocksdb-force-flush-memtable-now FALSE
 rocksdb-force-index-records-in-range 0
 rocksdb-global-info ON
diff --git a/mysql-test/suite/rocksdb/r/rocksdb.result b/mysql-test/suite/rocksdb/r/rocksdb.result
index d8ed776..63a748a 100644
--- a/mysql-test/suite/rocksdb/r/rocksdb.result
+++ b/mysql-test/suite/rocksdb/r/rocksdb.result
@@ -897,6 +897,7 @@ rocksdb_enable_thread_tracking	OFF
 rocksdb_enable_write_thread_adaptive_yield	OFF
 rocksdb_error_if_exists	OFF
 rocksdb_flush_memtable_on_analyze	ON
+rocksdb_force_flush_memtable_and_lzero_now	OFF
 rocksdb_force_flush_memtable_now	OFF
 rocksdb_force_index_records_in_range	0
 rocksdb_hash_index_allow_collision	ON
diff --git a/mysql-test/suite/rocksdb_sys_vars/r/rocksdb_force_flush_memtable_and_lzero_now_basic.result b/mysql-test/suite/rocksdb_sys_vars/r/rocksdb_force_flush_memtable_and_lzero_now_basic.result
new file mode 100644
index 0000000..68cfeb0
--- /dev/null
+++ b/mysql-test/suite/rocksdb_sys_vars/r/rocksdb_force_flush_memtable_and_lzero_now_basic.result
@@ -0,0 +1,50 @@
+CREATE TABLE valid_values (value varchar(255)) ENGINE=myisam;
+INSERT INTO valid_values VALUES(1);
+INSERT INTO valid_values VALUES(0);
+INSERT INTO valid_values VALUES('on');
+CREATE TABLE invalid_values (value varchar(255)) ENGINE=myisam;
+SET @start_global_value = @@global.ROCKSDB_FORCE_FLUSH_MEMTABLE_AND_LZERO_NOW;
+SELECT @start_global_value;
+@start_global_value
+0
+'# Setting to valid values in global scope#'
+"Trying to set variable @@global.ROCKSDB_FORCE_FLUSH_MEMTABLE_AND_LZERO_NOW to 1"
+SET @@global.ROCKSDB_FORCE_FLUSH_MEMTABLE_AND_LZERO_NOW   = 1;
+SELECT @@global.ROCKSDB_FORCE_FLUSH_MEMTABLE_AND_LZERO_NOW;
+@@global.ROCKSDB_FORCE_FLUSH_MEMTABLE_AND_LZERO_NOW
+0
+"Setting the global scope variable back to default"
+SET @@global.ROCKSDB_FORCE_FLUSH_MEMTABLE_AND_LZERO_NOW = DEFAULT;
+SELECT @@global.ROCKSDB_FORCE_FLUSH_MEMTABLE_AND_LZERO_NOW;
+@@global.ROCKSDB_FORCE_FLUSH_MEMTABLE_AND_LZERO_NOW
+0
+"Trying to set variable @@global.ROCKSDB_FORCE_FLUSH_MEMTABLE_AND_LZERO_NOW to 0"
+SET @@global.ROCKSDB_FORCE_FLUSH_MEMTABLE_AND_LZERO_NOW   = 0;
+SELECT @@global.ROCKSDB_FORCE_FLUSH_MEMTABLE_AND_LZERO_NOW;
+@@global.ROCKSDB_FORCE_FLUSH_MEMTABLE_AND_LZERO_NOW
+0
+"Setting the global scope variable back to default"
+SET @@global.ROCKSDB_FORCE_FLUSH_MEMTABLE_AND_LZERO_NOW = DEFAULT;
+SELECT @@global.ROCKSDB_FORCE_FLUSH_MEMTABLE_AND_LZERO_NOW;
+@@global.ROCKSDB_FORCE_FLUSH_MEMTABLE_AND_LZERO_NOW
+0
+"Trying to set variable @@global.ROCKSDB_FORCE_FLUSH_MEMTABLE_AND_LZERO_NOW to on"
+SET @@global.ROCKSDB_FORCE_FLUSH_MEMTABLE_AND_LZERO_NOW   = on;
+SELECT @@global.ROCKSDB_FORCE_FLUSH_MEMTABLE_AND_LZERO_NOW;
+@@global.ROCKSDB_FORCE_FLUSH_MEMTABLE_AND_LZERO_NOW
+0
+"Setting the global scope variable back to default"
+SET @@global.ROCKSDB_FORCE_FLUSH_MEMTABLE_AND_LZERO_NOW = DEFAULT;
+SELECT @@global.ROCKSDB_FORCE_FLUSH_MEMTABLE_AND_LZERO_NOW;
+@@global.ROCKSDB_FORCE_FLUSH_MEMTABLE_AND_LZERO_NOW
+0
+"Trying to set variable @@session.ROCKSDB_FORCE_FLUSH_MEMTABLE_AND_LZERO_NOW to 444. It should fail because it is not session."
+SET @@session.ROCKSDB_FORCE_FLUSH_MEMTABLE_AND_LZERO_NOW   = 444;
+ERROR HY000: Variable 'rocksdb_force_flush_memtable_and_lzero_now' is a GLOBAL variable and should be set with SET GLOBAL
+'# Testing with invalid values in global scope #'
+SET @@global.ROCKSDB_FORCE_FLUSH_MEMTABLE_AND_LZERO_NOW = @start_global_value;
+SELECT @@global.ROCKSDB_FORCE_FLUSH_MEMTABLE_AND_LZERO_NOW;
+@@global.ROCKSDB_FORCE_FLUSH_MEMTABLE_AND_LZERO_NOW
+0
+DROP TABLE valid_values;
+DROP TABLE invalid_values;
diff --git a/mysql-test/suite/rocksdb_sys_vars/t/rocksdb_force_flush_memtable_and_lzero_now_basic.test b/mysql-test/suite/rocksdb_sys_vars/t/rocksdb_force_flush_memtable_and_lzero_now_basic.test
new file mode 100644
index 0000000..7b55ac7
--- /dev/null
+++ b/mysql-test/suite/rocksdb_sys_vars/t/rocksdb_force_flush_memtable_and_lzero_now_basic.test
@@ -0,0 +1,17 @@
+--source include/have_rocksdb.inc
+
+CREATE TABLE valid_values (value varchar(255)) ENGINE=myisam;
+INSERT INTO valid_values VALUES(1);
+INSERT INTO valid_values VALUES(0);
+INSERT INTO valid_values VALUES('on');
+
+CREATE TABLE invalid_values (value varchar(255)) ENGINE=myisam;
+
+--let $sys_var=ROCKSDB_FORCE_FLUSH_MEMTABLE_AND_LZERO_NOW
+--let $read_only=0
+--let $session=0
+--let $sticky=1
+--source suite/sys_vars/inc/rocksdb_sys_var.inc
+
+DROP TABLE valid_values;
+DROP TABLE invalid_values;
diff --git a/storage/rocksdb/ha_rocksdb.cc b/storage/rocksdb/ha_rocksdb.cc
index b6cd351..1e207a3 100644
--- a/storage/rocksdb/ha_rocksdb.cc
+++ b/storage/rocksdb/ha_rocksdb.cc
@@ -324,6 +324,44 @@ rocksdb_force_flush_memtable_now(THD* const thd,
   return HA_EXIT_SUCCESS;
 }
 
+static void
+rocksdb_force_flush_memtable_and_lzero_now_stub(THD* const thd,
+    struct st_mysql_sys_var* const var, void* const var_ptr,
+    const void* const save)
+{
+}
+
+static int
+rocksdb_force_flush_memtable_and_lzero_now(THD* const thd,
+                                           struct st_mysql_sys_var* const var,
+                                           void* const var_ptr,
+                                           struct st_mysql_value* const value)
+{
+  sql_print_information("RocksDB: Manual memtable and L0 flush\n");
+  rocksdb_flush_all_memtables();
+
+  const Rdb_cf_manager& cf_manager= rdb_get_cf_manager();
+  for (const auto &cf_handle : cf_manager.get_all_cf())
+  {
+    rocksdb::ColumnFamilyMetaData metadata;
+    rdb->GetColumnFamilyMetaData(cf_handle, &metadata);
+
+    std::vector<std::string> file_names;
+    for (auto& file : metadata.levels[0].files)
+    {
+      DBUG_ASSERT(metadata.levels[0].level == 0);
+      file_names.emplace_back(file.db_path + "/" + file.name);
+    }
+
+    /* is this failing because its calling GetAndRefSuperVersion twice? */
+
+    /* how to figure out output level */
+    rdb->CompactFiles(rocksdb::CompactionOptions(), cf_handle, file_names, 1);
+  }
+
+  return HA_EXIT_SUCCESS;
+}
+
 static void rocksdb_drop_index_wakeup_thread(
     my_core::THD* const thd __attribute__((__unused__)),
     struct st_mysql_sys_var* const var __attribute__((__unused__)),
@@ -414,6 +452,7 @@ static my_bool rocksdb_enable_2pc= 0;
 static char * rocksdb_strict_collation_exceptions;
 static my_bool rocksdb_collect_sst_properties= 1;
 static my_bool rocksdb_force_flush_memtable_now_var= 0;
+static my_bool rocksdb_force_flush_memtable_and_lzero_now_var= 0;
 static uint64_t rocksdb_number_stat_computes= 0;
 static uint32_t rocksdb_seconds_between_stat_computes= 3600;
 static long long rocksdb_compaction_sequential_deletes= 0l;
@@ -1094,6 +1133,14 @@ static MYSQL_SYSVAR_BOOL(
   rocksdb_force_flush_memtable_now,
   rocksdb_force_flush_memtable_now_stub, FALSE);
 
+static MYSQL_SYSVAR_BOOL(
+  force_flush_memtable_and_lzero_now,
+  rocksdb_force_flush_memtable_and_lzero_now_var,
+  PLUGIN_VAR_RQCMDARG,
+  "Acts similar to force_flush_memtable_now, but also compacts all L0 files.",
+  rocksdb_force_flush_memtable_and_lzero_now,
+  rocksdb_force_flush_memtable_and_lzero_now_stub, FALSE);
+
 static MYSQL_THDVAR_BOOL(
   flush_memtable_on_analyze,
   PLUGIN_VAR_RQCMDARG,
@@ -1292,6 +1339,7 @@ static struct st_mysql_sys_var* rocksdb_system_variables[]= {
   MYSQL_SYSVAR(strict_collation_exceptions),
   MYSQL_SYSVAR(collect_sst_properties),
   MYSQL_SYSVAR(force_flush_memtable_now),
+  MYSQL_SYSVAR(force_flush_memtable_and_lzero_now),
   MYSQL_SYSVAR(flush_memtable_on_analyze),
   MYSQL_SYSVAR(seconds_between_stat_computes),
 
-- 
2.9.3