Handle large write batch for intrinsic tmp tables

Summary:
If the intrinsic tmp tables goes beyond some threshold, then we commit the write batch accumulated so far and start new transaction handling the write batch.

**Changes**
* Added a variable rocksdb_max_intrinsic_tmp_table_write_count which controls the size of write batch allowed for intrinsic tmp table. Once we cross that threshold, we will commit the current transaction and then start a new transaction.
* If there are valid open iterators before we commit intrinsic tmp table transaction, then we refresh all iterators back to previous position after committing transaction.  Recursive cte opens multiple iterators for same transaction and we need to refresh all iterators after we commit the current write batch for transaction.
     * I also refreshed the iterators after the first write to the write batch. This is done to handle the special case where we start rocksdb iterator on new transaction(with empty write batch). Then iterator will only see the already committed data, but ignores any new data added in write batch later. So we are pro-actively refresh the iterator after first write in write batch.
* Added a status variable rocksdb_intrinsic_tmp_table_commits to count number of intrinsic tmp table commits.
* create_ondisk_from_heap method uses table->record[1] for insertion. Myrocks encode_value_slice relies on table->record[0] for value encoding. We have reposition the fields in write_row specifically for intrinsic tmp table to point to record[1]. Better way to fix this is to remove the dependency on record[1] from encode_value_slice method and add dependency on input buf from write_row.

Reviewed By: luqun

Differential Revision: D37835016

---------------------------------------------------------------------------------------

Temporarily disable some of rocksdb.tmp_table test

Summary:
RocksDB tmp table needs bug fixes, and meanwhile
disabling some of the rocksdb.tmp_table tests so that it won't
report failures.

Reviewed By: lth

Differential Revision: D41391615

---------------------------------------------------------------------------------------

set m_read_opts.ignore_range_deletions properly (facebook#1218)

Summary:
ignore_range_deletions should be set to true when range del is not enabled

Pull Request resolved: facebook#1218

Reviewed By: Pushapgl

Differential Revision: D39115879

Pulled By: yoshinorim

mysql-test/combinations

@@ -1,8 +1,9 @@
 [innodb_intrinsic_table]
 enable_rocksdb_intrinsic_tmp_table= OFF
-loose-rocksdb_enable_tmp_table = OFF
+loose-rocksdb_enable_tmp_table= OFF
 
 [rocksdb_intrinsic_table]
 enable_rocksdb_intrinsic_tmp_table= ON
-loose-rocksdb_enable_tmp_table = ON
-loose-rocksdb_enable_delete_range_for_drop_index = ON
+loose-rocksdb_enable_tmp_table= ON
+loose-rocksdb_enable_delete_range_for_drop_index= ON
+loose-rocksdb_max_intrinsic_tmp_table_write_count= 3

mysql-test/r/mysqld--help-notwin.result

@@ -2167,6 +2167,10 @@ The following options may be given as the first argument:
  Track history for at most this many completed
  compactions. The history is in the
  INFORMATION_SCHEMA.ROCKSDB_COMPACTION_HISTORY table.
+ --rocksdb-max-intrinsic-tmp-table-write-count=# 
+ Intrinsic tmp table max allowed write batch size.After
+ this, current transaction holding write batch will commit
+ and newtransaction will be started.
  --rocksdb-max-latest-deadlocks=# 
  Maximum number of recent deadlocks to store
  --rocksdb-max-log-file-size=# 
@@ -3620,6 +3624,7 @@ rocksdb-max-background-flushes -1
 rocksdb-max-background-jobs 2
 rocksdb-max-bottom-pri-background-compactions 0
 rocksdb-max-compaction-history 64
+rocksdb-max-intrinsic-tmp-table-write-count 1000
 rocksdb-max-latest-deadlocks 5
 rocksdb-max-log-file-size 0
 rocksdb-max-manifest-file-size 1073741824

mysql-test/suite/json/r/json_table.result

@@ -1033,6 +1033,7 @@ Variable_name	Value
 Created_tmp_disk_tables	1
 Created_tmp_files	0
 Created_tmp_tables	1
+rocksdb_intrinsic_tmp_table_commits	0
 set @@max_heap_table_size= @save_heap_size;
 set @@internal_tmp_mem_storage_engine= @save_mem_se;
 #
@@ -1051,6 +1052,7 @@ Variable_name	Value
 Created_tmp_disk_tables	0
 Created_tmp_files	0
 Created_tmp_tables	1
+rocksdb_intrinsic_tmp_table_commits	0
 #
 # Bug#25525409: ASSERTION `TABLE_LIST->TABLE' FAILED IN SQL/SQL_BASE.CC
 #

mysql-test/suite/rocksdb/r/rocksdb.result

@@ -998,6 +998,7 @@ rocksdb_max_background_flushes	-1
 rocksdb_max_background_jobs	2
 rocksdb_max_bottom_pri_background_compactions	0
 rocksdb_max_compaction_history	64
+rocksdb_max_intrinsic_tmp_table_write_count	1000
 rocksdb_max_latest_deadlocks	5
 rocksdb_max_log_file_size	0
 rocksdb_max_manifest_file_size	1073741824
@@ -1599,6 +1600,7 @@ rocksdb_table_index_stats_success	#
 rocksdb_table_index_stats_failure	#
 rocksdb_table_index_stats_req_queue_length	#
 rocksdb_covered_secondary_key_lookups	#
+rocksdb_intrinsic_tmp_table_commits	#
 rocksdb_additional_compaction_triggers	#
 rocksdb_binlog_ttl_compaction_timestamp	#
 rocksdb_block_cache_add	#
@@ -1752,6 +1754,7 @@ ROCKSDB_GET_HIT_L1
 ROCKSDB_GET_HIT_L2_AND_UP
 ROCKSDB_GIT_DATE
 ROCKSDB_GIT_HASH
+ROCKSDB_INTRINSIC_TMP_TABLE_COMMITS
 ROCKSDB_ITER_BYTES_READ
 ROCKSDB_LAST_LEVEL_SEEK_DATA
 ROCKSDB_LAST_LEVEL_SEEK_DATA_USEFUL_FILTER_MATCH
@@ -1886,6 +1889,7 @@ ROCKSDB_GET_HIT_L1
 ROCKSDB_GET_HIT_L2_AND_UP
 ROCKSDB_GIT_DATE
 ROCKSDB_GIT_HASH
+ROCKSDB_INTRINSIC_TMP_TABLE_COMMITS
 ROCKSDB_ITER_BYTES_READ
 ROCKSDB_LAST_LEVEL_SEEK_DATA
 ROCKSDB_LAST_LEVEL_SEEK_DATA_USEFUL_FILTER_MATCH

mysql-test/suite/rocksdb/t/tmp_table.test

@@ -360,4 +360,22 @@ show create table tmp1;
 select * from t1;
 drop table t1;
 
+####################################################
+### [TODO] Fix bugs: Case 12: Intrinsic tmp table background commits
+####################################################
+#--echo Case 12: Intrinsic tmp table background commits
+#set big_tables = 1;
+#Set global rocksdb_max_intrinsic_tmp_table_write_count = 2;
+#let $old_value= query_get_value(show status like "rocksdb_intrinsic_tmp_table_commits", Value, 1);
+#create table t1(i int, c char(5));
+#insert into t1 values (0, "aaaa");
+#insert into t1 values (1, "aaaa");
+#select i, c, count(*) from t1 group by i, c having count(*) > 0;
+#let $new_value= query_get_value(show status like "rocksdb_intrinsic_tmp_table_commits", Value, 1);
+#--let $assert_text = Number of intrinsic tmp table commits should be 1
+#--let $assert_cond= $new_value - $old_value = 1
+#--source include/assert.inc
+#Set global rocksdb_max_intrinsic_tmp_table_write_count = default;
+#drop table t1;
+
 --source include/wait_until_count_sessions.inc

mysql-test/suite/rocksdb_sys_vars/r/rocksdb_max_intrinsic_tmp_table_write_count_basic.result

@@ -0,0 +1,46 @@
+CREATE TABLE valid_values (value varchar(255)) ENGINE=myisam;
+INSERT INTO valid_values VALUES(10);
+INSERT INTO valid_values VALUES(20);
+CREATE TABLE invalid_values (value varchar(255)) ENGINE=myisam;
+INSERT INTO invalid_values VALUES('\'aaa\'');
+SET @start_global_value = @@global.rocksdb_max_intrinsic_tmp_table_write_count;
+SELECT @start_global_value;
+@start_global_value
+1000
+'# Setting to valid values in global scope#'
+"Trying to set variable @@global.rocksdb_max_intrinsic_tmp_table_write_count to 10"
+SET @@global.rocksdb_max_intrinsic_tmp_table_write_count   = 10;
+SELECT @@global.rocksdb_max_intrinsic_tmp_table_write_count;
+@@global.rocksdb_max_intrinsic_tmp_table_write_count
+10
+"Setting the global scope variable back to default"
+SET @@global.rocksdb_max_intrinsic_tmp_table_write_count = DEFAULT;
+SELECT @@global.rocksdb_max_intrinsic_tmp_table_write_count;
+@@global.rocksdb_max_intrinsic_tmp_table_write_count
+1000
+"Trying to set variable @@global.rocksdb_max_intrinsic_tmp_table_write_count to 20"
+SET @@global.rocksdb_max_intrinsic_tmp_table_write_count   = 20;
+SELECT @@global.rocksdb_max_intrinsic_tmp_table_write_count;
+@@global.rocksdb_max_intrinsic_tmp_table_write_count
+20
+"Setting the global scope variable back to default"
+SET @@global.rocksdb_max_intrinsic_tmp_table_write_count = DEFAULT;
+SELECT @@global.rocksdb_max_intrinsic_tmp_table_write_count;
+@@global.rocksdb_max_intrinsic_tmp_table_write_count
+1000
+"Trying to set variable @@session.rocksdb_max_intrinsic_tmp_table_write_count to 444. It should fail because it is not session."
+SET @@session.rocksdb_max_intrinsic_tmp_table_write_count   = 444;
+ERROR HY000: Variable 'rocksdb_max_intrinsic_tmp_table_write_count' is a GLOBAL variable and should be set with SET GLOBAL
+'# Testing with invalid values in global scope #'
+"Trying to set variable @@global.rocksdb_max_intrinsic_tmp_table_write_count to 'aaa'"
+SET @@global.rocksdb_max_intrinsic_tmp_table_write_count   = 'aaa';
+Got one of the listed errors
+SELECT @@global.rocksdb_max_intrinsic_tmp_table_write_count;
+@@global.rocksdb_max_intrinsic_tmp_table_write_count
+1000
+SET @@global.rocksdb_max_intrinsic_tmp_table_write_count = @start_global_value;
+SELECT @@global.rocksdb_max_intrinsic_tmp_table_write_count;
+@@global.rocksdb_max_intrinsic_tmp_table_write_count
+1000
+DROP TABLE valid_values;
+DROP TABLE invalid_values;

mysql-test/suite/rocksdb_sys_vars/t/rocksdb_max_intrinsic_tmp_table_write_count_basic.test

@@ -0,0 +1,16 @@
+--source include/have_rocksdb.inc
+
+CREATE TABLE valid_values (value varchar(255)) ENGINE=myisam;
+INSERT INTO valid_values VALUES(10);
+INSERT INTO valid_values VALUES(20);
+
+CREATE TABLE invalid_values (value varchar(255)) ENGINE=myisam;
+INSERT INTO invalid_values VALUES('\'aaa\'');
+
+--let $sys_var=rocksdb_max_intrinsic_tmp_table_write_count
+--let $read_only=0
+--let $session=0
+--source ../include/rocksdb_sys_var.inc
+
+DROP TABLE valid_values;
+DROP TABLE invalid_values;

mysql-test/t/error_simulation.test

@@ -1,6 +1,4 @@
 -- source include/have_debug.inc
---source include/have_innodb_intrinsic_table.inc
-
 #
 # Bug #28499: crash for grouping query when tmp_table_size is too small
 #

mysql-test/t/with_recursive_innodb_tmp_table.test

@@ -1,4 +1,3 @@
---source include/have_innodb_intrinsic_table.inc
 --source include/no_valgrind_without_big.inc
 
 set session internal_tmp_mem_storage_engine='memory';

mysql-test/t/with_recursive_rocksdb-master.opt

@@ -0,0 +1 @@
+--loose-rocksdb_max_intrinsic_tmp_table_write_count=1000000