From b9e09453971eb3e611ca9d087bc6198a0f2080be Mon Sep 17 00:00:00 2001 From: Vladislav Vaintroub Date: Thu, 4 Jan 2018 16:52:15 +0000 Subject: [PATCH 001/106] update libmariadb --- libmariadb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libmariadb b/libmariadb index fe129ed39f33b..bc1777f8ce8cb 160000 --- a/libmariadb +++ b/libmariadb @@ -1 +1 @@ -Subproject commit fe129ed39f33ba2b430aac91473baee84de88b12 +Subproject commit bc1777f8ce8cb5fc3b4f0e184d56871d5b809c20 From 0de565a56475e355eb77225981256348c698b9b6 Mon Sep 17 00:00:00 2001 From: Igor Babaev Date: Thu, 4 Jan 2018 23:40:37 -0800 Subject: [PATCH 002/106] Fixed mdev-14852 Fails to reopen temp table within standard CTE If the specification of a CTE contains a reference to a temporary table then THD::open_temporary_table() must be called for this reference for any occurrence of the CTE in the query. By mistake this was done only for the first occurrences of CTEs. The patch fixes this problem in With_element::clone_parsed_spec(). It also moves there the call of check_dependencies_in_with_clauses() to its proper place before the call of check_table_access(). Additionally the patch optimizes the number of calls of the function check_dependencies_in_with_clauses(). --- mysql-test/r/cte_nonrecursive.result | 30 ++++++++++++++++++++++++++++ mysql-test/t/cte_nonrecursive.test | 29 +++++++++++++++++++++++++++ sql/sql_cte.cc | 7 +++++++ sql/sql_parse.cc | 22 +++++--------------- sql/sql_prepare.cc | 11 +++------- 5 files changed, 74 insertions(+), 25 deletions(-) diff --git a/mysql-test/r/cte_nonrecursive.result b/mysql-test/r/cte_nonrecursive.result index b1b41b1c5e72d..c1f4c9fd486f4 100644 --- a/mysql-test/r/cte_nonrecursive.result +++ b/mysql-test/r/cte_nonrecursive.result @@ -1265,3 +1265,33 @@ a 4 deallocate prepare stmt; drop table t1; +# +# MDEV-14852: CTE using temporary table in query +# with two references to the CTE +# +create temporary table t1 (i int); +insert into t1 values (5),(4),(1),(2),(3); +with +c1 as (select i from t1), +c2 as (select i from c1 where c1.i=2) +select i from c1 where i > 3 union select i from c2; +i +5 +4 +2 +drop table t1; +create table t1 (term char(10)); +create temporary table t2 (term char(10)); +insert into t1 values ('TERM01'),('TERM02'),('TERM03'); +insert into t2 values ('TERM02'),('TERM03'),('TERM04'); +with c1 as (select * from t1), c2 as (select * from t2) +(select * from c1 left outer join c2 on c1.term = c2.term) +union all +(select * from c1 right outer join c2 on c1.term = c2.term +where c1.term is null); +term term +TERM02 TERM02 +TERM03 TERM03 +TERM01 NULL +NULL TERM04 +drop table t1,t2; diff --git a/mysql-test/t/cte_nonrecursive.test b/mysql-test/t/cte_nonrecursive.test index 1f21dbcd36d7a..9436665bfee56 100644 --- a/mysql-test/t/cte_nonrecursive.test +++ b/mysql-test/t/cte_nonrecursive.test @@ -853,3 +853,32 @@ execute stmt; deallocate prepare stmt; drop table t1; + +--echo # +--echo # MDEV-14852: CTE using temporary table in query +--echo # with two references to the CTE +--echo # + +create temporary table t1 (i int); +insert into t1 values (5),(4),(1),(2),(3); + +with +c1 as (select i from t1), +c2 as (select i from c1 where c1.i=2) +select i from c1 where i > 3 union select i from c2; + +drop table t1; + +create table t1 (term char(10)); +create temporary table t2 (term char(10)); + +insert into t1 values ('TERM01'),('TERM02'),('TERM03'); +insert into t2 values ('TERM02'),('TERM03'),('TERM04'); + +with c1 as (select * from t1), c2 as (select * from t2) +(select * from c1 left outer join c2 on c1.term = c2.term) +union all +(select * from c1 right outer join c2 on c1.term = c2.term + where c1.term is null); + +drop table t1,t2; diff --git a/sql/sql_cte.cc b/sql/sql_cte.cc index 601c1928d5810..d12948fb624fe 100644 --- a/sql/sql_cte.cc +++ b/sql/sql_cte.cc @@ -817,12 +817,19 @@ st_select_lex_unit *With_element::clone_parsed_spec(THD *thd, parse_status= parse_sql(thd, &parser_state, 0); if (parse_status) goto err; + + if (check_dependencies_in_with_clauses(lex->with_clauses_list)) + goto err; + spec_tables= lex->query_tables; spec_tables_tail= 0; for (TABLE_LIST *tbl= spec_tables; tbl; tbl= tbl->next_global) { + if (!tbl->derived && !tbl->schema_table && + thd->open_temporary_table(tbl)) + goto err; spec_tables_tail= tbl; } if (check_table_access(thd, SELECT_ACL, spec_tables, FALSE, UINT_MAX, FALSE)) diff --git a/sql/sql_parse.cc b/sql/sql_parse.cc index f00c74b155c32..6d068eb99ac5e 100644 --- a/sql/sql_parse.cc +++ b/sql/sql_parse.cc @@ -2990,6 +2990,9 @@ mysql_execute_command(THD *thd) thd->get_stmt_da()->opt_clear_warning_info(thd->query_id); } + if (check_dependencies_in_with_clauses(thd->lex->with_clauses_list)) + DBUG_RETURN(1); + #ifdef HAVE_REPLICATION if (unlikely(thd->slave_thread)) { @@ -3446,14 +3449,6 @@ mysql_execute_command(THD *thd) ulong privileges_requested= lex->exchange ? SELECT_ACL | FILE_ACL : SELECT_ACL; - /* - The same function must be called for DML commands - when CTEs are supported in DML statements - */ - res= check_dependencies_in_with_clauses(thd->lex->with_clauses_list); - if (res) - break; - if (all_tables) res= check_table_access(thd, privileges_requested, @@ -3879,8 +3874,7 @@ mysql_execute_command(THD *thd) /* Copy temporarily the statement flags to thd for lock_table_names() */ uint save_thd_create_info_options= thd->lex->create_info.options; thd->lex->create_info.options|= create_info.options; - if (!(res= check_dependencies_in_with_clauses(lex->with_clauses_list))) - res= open_and_lock_tables(thd, create_info, lex->query_tables, TRUE, 0); + res= open_and_lock_tables(thd, create_info, lex->query_tables, TRUE, 0); thd->lex->create_info.options= save_thd_create_info_options; if (res) { @@ -4493,8 +4487,7 @@ mysql_execute_command(THD *thd) unit->set_limit(select_lex); - if (!(res= check_dependencies_in_with_clauses(lex->with_clauses_list)) && - !(res=open_and_lock_tables(thd, all_tables, TRUE, 0))) + if (!(res=open_and_lock_tables(thd, all_tables, TRUE, 0))) { MYSQL_INSERT_SELECT_START(thd->query()); /* @@ -4821,9 +4814,6 @@ mysql_execute_command(THD *thd) { List *lex_var_list= &lex->var_list; - if (check_dependencies_in_with_clauses(thd->lex->with_clauses_list)) - goto error; - if ((check_table_access(thd, SELECT_ACL, all_tables, FALSE, UINT_MAX, FALSE) || open_and_lock_tables(thd, all_tables, TRUE, 0))) goto error; @@ -6376,8 +6366,6 @@ static bool execute_sqlcom_select(THD *thd, TABLE_LIST *all_tables) new (thd->mem_root) Item_int(thd, (ulonglong) thd->variables.select_limit); } - if (check_dependencies_in_with_clauses(lex->with_clauses_list)) - return 1; if (!(res= open_and_lock_tables(thd, all_tables, TRUE, 0))) { diff --git a/sql/sql_prepare.cc b/sql/sql_prepare.cc index be78a76152e49..390b70877f5f3 100644 --- a/sql/sql_prepare.cc +++ b/sql/sql_prepare.cc @@ -1573,8 +1573,6 @@ static int mysql_test_select(Prepared_statement *stmt, lex->select_lex.context.resolve_in_select_list= TRUE; ulong privilege= lex->exchange ? SELECT_ACL | FILE_ACL : SELECT_ACL; - if (check_dependencies_in_with_clauses(lex->with_clauses_list)) - goto error; if (tables) { if (check_table_access(thd, privilege, tables, FALSE, UINT_MAX, FALSE)) @@ -1841,9 +1839,6 @@ static bool mysql_test_create_table(Prepared_statement *stmt) if (create_table_precheck(thd, tables, create_table)) DBUG_RETURN(TRUE); - if (check_dependencies_in_with_clauses(lex->with_clauses_list)) - DBUG_RETURN(TRUE); - if (select_lex->item_list.elements) { /* Base table and temporary table are not in the same name space. */ @@ -2234,9 +2229,6 @@ static bool mysql_test_insert_select(Prepared_statement *stmt, if (insert_precheck(stmt->thd, tables)) return 1; - if (check_dependencies_in_with_clauses(lex->with_clauses_list)) - return 1; - /* store it, because mysql_insert_select_prepare_tester change it */ first_local_table= lex->select_lex.table_list.first; DBUG_ASSERT(first_local_table != 0); @@ -2339,6 +2331,9 @@ static bool check_prepared_statement(Prepared_statement *stmt) if (tables) thd->get_stmt_da()->opt_clear_warning_info(thd->query_id); + if (check_dependencies_in_with_clauses(thd->lex->with_clauses_list)) + goto error; + if (sql_command_flags[sql_command] & CF_HA_CLOSE) mysql_ha_rm_tables(thd, tables); From 578345305e261d635bfc938a5dc2a22b839c2148 Mon Sep 17 00:00:00 2001 From: Igor Babaev Date: Fri, 5 Jan 2018 10:17:29 -0800 Subject: [PATCH 003/106] Added a test case for mdev-13454: Improper error in ONLY_FULL_GROUP_BY sql_mode with condition_pushdown_for_derived=on This bug is a consequence of the bug mdev-14368 fixed in 5.5.59. --- mysql-test/r/derived_cond_pushdown.result | 64 +++++++++++++++++++++++ mysql-test/t/derived_cond_pushdown.test | 25 +++++++++ 2 files changed, 89 insertions(+) diff --git a/mysql-test/r/derived_cond_pushdown.result b/mysql-test/r/derived_cond_pushdown.result index 42662b84a31e2..c588a953a2cb0 100644 --- a/mysql-test/r/derived_cond_pushdown.result +++ b/mysql-test/r/derived_cond_pushdown.result @@ -8807,3 +8807,67 @@ EXPLAIN } } drop table t1; +# +# MDEV-13454: consequence of mdev-14368 fixed for 5.5 +# +SET sql_mode = 'ONLY_FULL_GROUP_BY'; +create table t1 (id int, id2 int); +insert into t1 values (1,1),(2,3),(3,4),(7,2); +create table t2(id2 int); +insert t2 values (1),(2),(3); +SELECT * FROM t1 +LEFT OUTER JOIN +(SELECT id2, COUNT(*) as ct FROM t2 GROUP BY id2) vc USING (id2) +WHERE (vc.ct>0); +id2 id ct +1 1 1 +3 2 1 +2 7 1 +EXPLAIN FORMAT=JSON SELECT * FROM t1 +LEFT OUTER JOIN +(SELECT id2, COUNT(*) as ct FROM t2 GROUP BY id2) vc USING (id2) +WHERE (vc.ct>0); +EXPLAIN +{ + "query_block": { + "select_id": 1, + "table": { + "table_name": "", + "access_type": "ALL", + "rows": 3, + "filtered": 100, + "attached_condition": "vc.ct > 0", + "materialized": { + "query_block": { + "select_id": 2, + "having_condition": "ct > 0", + "filesort": { + "sort_key": "t2.id2", + "temporary_table": { + "table": { + "table_name": "t2", + "access_type": "ALL", + "rows": 3, + "filtered": 100 + } + } + } + } + } + }, + "block-nl-join": { + "table": { + "table_name": "t1", + "access_type": "ALL", + "rows": 4, + "filtered": 100 + }, + "buffer_type": "flat", + "buffer_size": "256Kb", + "join_type": "BNL", + "attached_condition": "t1.id2 = vc.id2" + } + } +} +DROP TABLE t1,t2; +SET sql_mode = DEFAULT; diff --git a/mysql-test/t/derived_cond_pushdown.test b/mysql-test/t/derived_cond_pushdown.test index 9eb332b72d198..4ea6214075bed 100644 --- a/mysql-test/t/derived_cond_pushdown.test +++ b/mysql-test/t/derived_cond_pushdown.test @@ -1565,3 +1565,28 @@ eval $q; eval explain format=json $q; drop table t1; + +--echo # +--echo # MDEV-13454: consequence of mdev-14368 fixed for 5.5 +--echo # + +SET sql_mode = 'ONLY_FULL_GROUP_BY'; + +create table t1 (id int, id2 int); +insert into t1 values (1,1),(2,3),(3,4),(7,2); + +create table t2(id2 int); +insert t2 values (1),(2),(3); + +let $q= +SELECT * FROM t1 + LEFT OUTER JOIN + (SELECT id2, COUNT(*) as ct FROM t2 GROUP BY id2) vc USING (id2) +WHERE (vc.ct>0); + +eval $q; +eval EXPLAIN FORMAT=JSON $q; + +DROP TABLE t1,t2; + +SET sql_mode = DEFAULT; From 3a22d6c136ff5a03012102d9a852c5e16ac612d4 Mon Sep 17 00:00:00 2001 From: Vladislav Vaintroub Date: Fri, 5 Jan 2018 18:22:57 +0000 Subject: [PATCH 004/106] Fix conf_to_src build. 2cd316911309e3db4007aa224f7874bfbeb14032 broke conf_to_src, because strings library is now dependend on mysys (my_alloc etc are used now directly in string lib) Fix by adding appropriate dependency. Also exclude conf_to_src from VS IDE builds. EXCLUDE_FROM_ALL is not enough for that. --- strings/CMakeLists.txt | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/strings/CMakeLists.txt b/strings/CMakeLists.txt index 65eeb457f2db1..2a7f0b71cc8d3 100644 --- a/strings/CMakeLists.txt +++ b/strings/CMakeLists.txt @@ -34,4 +34,5 @@ ADD_DEFINITIONS(-DDISABLE_MYSQL_THREAD_H) ADD_CONVENIENCE_LIBRARY(strings ${STRINGS_SOURCES}) ADD_EXECUTABLE(conf_to_src EXCLUDE_FROM_ALL conf_to_src.c) -TARGET_LINK_LIBRARIES(conf_to_src strings) +SET_TARGET_PROPERTIES(conf_to_src PROPERTIES EXCLUDE_FROM_DEFAULT_BUILD TRUE) +TARGET_LINK_LIBRARIES(conf_to_src mysys strings) From 15b1840f43be8c660382458826e9d5a47d148a67 Mon Sep 17 00:00:00 2001 From: Igor Babaev Date: Fri, 5 Jan 2018 12:13:23 -0800 Subject: [PATCH 005/106] Added the test case from for mdev-14777: Crash in MariaDB 10.2.12 on query using VIEW and WITH RECURSIVE. The cause of this crash was the same as of the crash reported in mdev-14755. --- mysql-test/r/cte_recursive.result | 28 +++++++++++++++++++++++++ mysql-test/t/cte_recursive.test | 34 +++++++++++++++++++++++++++++++ 2 files changed, 62 insertions(+) diff --git a/mysql-test/r/cte_recursive.result b/mysql-test/r/cte_recursive.result index ece5421e279b3..300539a75b224 100644 --- a/mysql-test/r/cte_recursive.result +++ b/mysql-test/r/cte_recursive.result @@ -2944,3 +2944,31 @@ limit 1 ); ERROR HY000: Unacceptable mutual recursion with anchored table 'cte_1' drop table t1; +# +# mdev-14777: crash caused by the same as in mdev-14755 +# +CREATE TABLE t1 (i1 int NOT NULL, i2 int); +CREATE TABLE t2 (d1 int NOT NULL PRIMARY KEY); +CREATE TABLE t3 (i int ); +insert into t1 select seq,seq from seq_1_to_100000; +insert into t2 select seq from seq_1000_to_100000; +insert into t3 select seq from seq_1_to_1000; +SELECT * +FROM +( +SELECT * +FROM +( +WITH RECURSIVE rt AS +( +SELECT i2 P, i1 C FROM t1 WHERE i1 IN (SELECT d1 FROM t2) +UNION +SELECT t1.i2 P, rt.C C FROM t1, rt +) +SELECT C,P +FROM ( SELECT P,C FROM rt WHERE NOT EXISTS (SELECT 1 FROM t1) ) Y +) X +WHERE 1 = 1 +) K, t3; +C P i +drop table t1,t2,t3; diff --git a/mysql-test/t/cte_recursive.test b/mysql-test/t/cte_recursive.test index 928f8a4334bb8..4e9a1e6aa32ba 100644 --- a/mysql-test/t/cte_recursive.test +++ b/mysql-test/t/cte_recursive.test @@ -1998,3 +1998,37 @@ set @var= ); drop table t1; + +--echo # +--echo # mdev-14777: crash caused by the same as in mdev-14755 +--echo # + +--source include/have_sequence.inc + +CREATE TABLE t1 (i1 int NOT NULL, i2 int); +CREATE TABLE t2 (d1 int NOT NULL PRIMARY KEY); +CREATE TABLE t3 (i int ); + +insert into t1 select seq,seq from seq_1_to_100000; +insert into t2 select seq from seq_1000_to_100000; +insert into t3 select seq from seq_1_to_1000; + +SELECT * +FROM +( + SELECT * + FROM + ( + WITH RECURSIVE rt AS + ( + SELECT i2 P, i1 C FROM t1 WHERE i1 IN (SELECT d1 FROM t2) + UNION + SELECT t1.i2 P, rt.C C FROM t1, rt + ) + SELECT C,P + FROM ( SELECT P,C FROM rt WHERE NOT EXISTS (SELECT 1 FROM t1) ) Y + ) X + WHERE 1 = 1 +) K, t3; + +drop table t1,t2,t3; From 73cf630ffc2e971fac68addc60c6fbf805665127 Mon Sep 17 00:00:00 2001 From: Sachin Setiya Date: Sat, 6 Jan 2018 23:51:37 +0530 Subject: [PATCH 006/106] Fix Compile Error while using Flag '-DUSE_ARIA_FOR_TMP_TABLES:BOOL=OFF' --- sql/sql_select.cc | 6 +++--- sql/sql_select.h | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/sql/sql_select.cc b/sql/sql_select.cc index 5d6945e866ba9..e80c02b52b4da 100644 --- a/sql/sql_select.cc +++ b/sql/sql_select.cc @@ -17799,7 +17799,7 @@ bool create_internal_tmp_table(TABLE *table, KEY *keyinfo, table->in_use->inc_status_created_tmp_tables(); table->in_use->query_plan_flags|= QPLAN_TMP_DISK; share->db_record_offset= 1; - table->created= TRUE; + table->set_created(); DBUG_RETURN(0); err: DBUG_RETURN(1); @@ -18299,8 +18299,8 @@ int rr_sequential_and_unpack(READ_RECORD *info) */ bool instantiate_tmp_table(TABLE *table, KEY *keyinfo, - MARIA_COLUMNDEF *start_recinfo, - MARIA_COLUMNDEF **recinfo, + TMP_ENGINE_COLUMNDEF *start_recinfo, + TMP_ENGINE_COLUMNDEF **recinfo, ulonglong options) { if (table->s->db_type() == TMP_ENGINE_HTON) diff --git a/sql/sql_select.h b/sql/sql_select.h index d4daa04f370ad..017f66a23b883 100644 --- a/sql/sql_select.h +++ b/sql/sql_select.h @@ -2264,8 +2264,8 @@ bool create_internal_tmp_table(TABLE *table, KEY *keyinfo, TMP_ENGINE_COLUMNDEF **recinfo, ulonglong options); bool instantiate_tmp_table(TABLE *table, KEY *keyinfo, - MARIA_COLUMNDEF *start_recinfo, - MARIA_COLUMNDEF **recinfo, + TMP_ENGINE_COLUMNDEF *start_recinfo, + TMP_ENGINE_COLUMNDEF **recinfo, ulonglong options); bool open_tmp_table(TABLE *table); void setup_tmp_table_column_bitmaps(TABLE *table, uchar *bitmaps); From 16d308e21d91b880f1a59ab964a86e39643cdf1e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Mon, 8 Jan 2018 09:24:13 +0200 Subject: [PATCH 007/106] MDEV-14874 innodb_encrypt_log corrupts the log when the LSN crosses 32-bit boundary This bug affects both writing and reading encrypted redo log in MariaDB 10.1, starting from version 10.1.3 which added support for innodb_encrypt_log. That is, InnoDB crash recovery and Mariabackup will sometimes fail when innodb_encrypt_log is used. MariaDB 10.2 or Mariabackup 10.2 or later versions are not affected. log_block_get_start_lsn(): Remove. This function would cause trouble if a log segment that is being read is crossing a 32-bit boundary of the LSN, because this function does not allow the most significant 32 bits of the LSN to change. log_blocks_crypt(), log_encrypt_before_write(), log_decrypt_after_read(): Add the parameter "lsn" for the start LSN of the block. log_blocks_encrypt(): Remove (unused function). --- extra/mariabackup/xtrabackup.cc | 2 +- mysql-test/suite/mariabackup/huge_lsn.opt | 6 +++ mysql-test/suite/mariabackup/huge_lsn.result | 18 +++++++ mysql-test/suite/mariabackup/huge_lsn.test | 51 ++++++++++++++++++ storage/innobase/include/log0crypt.h | 6 ++- storage/innobase/log/log0crypt.cc | 54 ++++++-------------- storage/innobase/log/log0log.cc | 7 +-- storage/xtradb/include/log0crypt.h | 6 ++- storage/xtradb/log/log0crypt.cc | 54 ++++++-------------- storage/xtradb/log/log0log.cc | 7 +-- 10 files changed, 124 insertions(+), 87 deletions(-) create mode 100644 mysql-test/suite/mariabackup/huge_lsn.opt create mode 100644 mysql-test/suite/mariabackup/huge_lsn.result create mode 100644 mysql-test/suite/mariabackup/huge_lsn.test diff --git a/extra/mariabackup/xtrabackup.cc b/extra/mariabackup/xtrabackup.cc index 437fc4aa7f917..38312086ec1ac 100644 --- a/extra/mariabackup/xtrabackup.cc +++ b/extra/mariabackup/xtrabackup.cc @@ -2698,7 +2698,7 @@ xtrabackup_scan_log_recs( if (srv_encrypt_log) { log_encrypt_before_write(scanned_checkpoint_no, - log_sys->buf, write_size); + log_sys->buf, start_lsn, write_size); } if (ds_write(dst_log_file, log_sys->buf, write_size)) { diff --git a/mysql-test/suite/mariabackup/huge_lsn.opt b/mysql-test/suite/mariabackup/huge_lsn.opt new file mode 100644 index 0000000000000..74a6450a1ef4e --- /dev/null +++ b/mysql-test/suite/mariabackup/huge_lsn.opt @@ -0,0 +1,6 @@ +--innodb-encrypt-log=ON +--plugin-load-add=$FILE_KEY_MANAGEMENT_SO +--loose-file-key-management +--loose-file-key-management-filekey=FILE:$MTR_SUITE_DIR/filekeys-data.key +--loose-file-key-management-filename=$MTR_SUITE_DIR/filekeys-data.enc +--loose-file-key-management-encryption-algorithm=aes_cbc diff --git a/mysql-test/suite/mariabackup/huge_lsn.result b/mysql-test/suite/mariabackup/huge_lsn.result new file mode 100644 index 0000000000000..740a436228c27 --- /dev/null +++ b/mysql-test/suite/mariabackup/huge_lsn.result @@ -0,0 +1,18 @@ +# +# MDEV-13416 mariabackup fails with EFAULT "Bad Address" +# +call mtr.add_suppression("InnoDB: New log files created"); +FOUND /InnoDB: .*started; log sequence number 17596481010700/ in mysqld.1.err +CREATE TABLE t(i INT) ENGINE INNODB; +INSERT INTO t VALUES(1); +# xtrabackup backup +INSERT INTO t VALUES(2); +# xtrabackup prepare +# shutdown server +# remove datadir +# xtrabackup move back +# restart server +SELECT * FROM t; +i +1 +DROP TABLE t; diff --git a/mysql-test/suite/mariabackup/huge_lsn.test b/mysql-test/suite/mariabackup/huge_lsn.test new file mode 100644 index 0000000000000..3a173d43a3c9d --- /dev/null +++ b/mysql-test/suite/mariabackup/huge_lsn.test @@ -0,0 +1,51 @@ +--source include/not_embedded.inc + +--echo # +--echo # MDEV-13416 mariabackup fails with EFAULT "Bad Address" +--echo # + +let INNODB_PAGE_SIZE=`select @@innodb_page_size`; +let MYSQLD_DATADIR=`select @@datadir`; +call mtr.add_suppression("InnoDB: New log files created"); + +--source include/shutdown_mysqld.inc + +perl; +my $file= "$ENV{MYSQLD_DATADIR}/ibdata1"; +open(FILE, "+<", $file) or die "Unable to open $file\n"; +binmode FILE; +my $ps= $ENV{INNODB_PAGE_SIZE}; +my $page; +die "Unable to read $file" unless sysread(FILE, $page, $ps) == $ps; +substr($page,26,8) = pack("NN", 4096, ~1024); +substr($page,0,4)=pack("N",0xdeadbeef); +substr($page,$ps-8,4)=pack("N",0xdeadbeef); +sysseek(FILE, 0, 0) || die "Unable to rewind $file\n"; +syswrite(FILE, $page, $ps)==$ps || die "Unable to write $file\n"; +close(FILE) || die "Unable to close $file\n"; +EOF + +--remove_files_wildcard $MYSQLD_DATADIR ib_logfile* + +--source include/start_mysqld.inc +let SEARCH_FILE= $MYSQLTEST_VARDIR/log/mysqld.1.err; +--let SEARCH_PATTERN= InnoDB: .*started; log sequence number 17596481010700 +--source include/search_pattern_in_file.inc + +CREATE TABLE t(i INT) ENGINE INNODB; +INSERT INTO t VALUES(1); + +echo # xtrabackup backup; +let $targetdir=$MYSQLTEST_VARDIR/tmp/backup; +--disable_result_log +exec $XTRABACKUP --defaults-file=$MYSQLTEST_VARDIR/my.cnf --backup --target-dir=$targetdir; +--enable_result_log +INSERT INTO t VALUES(2); +echo # xtrabackup prepare; +--disable_result_log +exec $XTRABACKUP --prepare --target-dir=$targetdir; +--source include/restart_and_restore.inc +--enable_result_log +SELECT * FROM t; +DROP TABLE t; +rmdir $targetdir; diff --git a/storage/innobase/include/log0crypt.h b/storage/innobase/include/log0crypt.h index 0ad7e7d7037a9..b7a221e0a8172 100644 --- a/storage/innobase/include/log0crypt.h +++ b/storage/innobase/include/log0crypt.h @@ -1,7 +1,7 @@ /***************************************************************************** Copyright (C) 2013, 2015, Google Inc. All Rights Reserved. -Copyright (C) 2014, 2017, MariaDB Corporation. All Rights Reserved. +Copyright (C) 2014, 2018, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -73,6 +73,8 @@ log_encrypt_before_write( /*=====================*/ ib_uint64_t next_checkpoint_no, /*!< in: log group to be flushed */ byte* block, /*!< in/out: pointer to a log block */ + lsn_t lsn, /*!< in: log sequence number of + the start of the buffer */ const ulint size); /*!< in: size of log blocks */ /******************************************************** @@ -83,6 +85,8 @@ void log_decrypt_after_read( /*===================*/ byte* frame, /*!< in/out: log segment */ + lsn_t lsn, /*!< in: log sequence number of the start + of the buffer */ const ulint size); /*!< in: log segment size */ /* Error codes for crypt info */ diff --git a/storage/innobase/log/log0crypt.cc b/storage/innobase/log/log0crypt.cc index 13d6911877990..ec3c72cab43b1 100644 --- a/storage/innobase/log/log0crypt.cc +++ b/storage/innobase/log/log0crypt.cc @@ -1,7 +1,7 @@ /***************************************************************************** Copyright (C) 2013, 2015, Google Inc. All Rights Reserved. -Copyright (C) 2014, 2017, MariaDB Corporation. All Rights Reserved. +Copyright (C) 2014, 2018, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -70,22 +70,6 @@ struct crypt_info_t { static std::deque crypt_info; -/*********************************************************************//** -Get a log block's start lsn. -@return a log block's start lsn */ -static inline -lsn_t -log_block_get_start_lsn( -/*====================*/ - lsn_t lsn, /*!< in: checkpoint lsn */ - ulint log_block_no) /*!< in: log block number */ -{ - lsn_t start_lsn = - (lsn & (lsn_t)0xffffffff00000000ULL) | - (((log_block_no - 1) & (lsn_t)0x3fffffff) << 9); - return start_lsn; -} - /*********************************************************************//** Get crypt info from checkpoint. @return a crypt info or NULL if not present. */ @@ -162,6 +146,8 @@ Crypt_result log_blocks_crypt( /*=============*/ const byte* block, /*!< in: blocks before encrypt/decrypt*/ + lsn_t lsn, /*!< in: log sequence number of the start + of the buffer */ ulint size, /*!< in: size of block */ byte* dst_block, /*!< out: blocks after encrypt/decrypt */ int what, /*!< in: encrypt or decrypt*/ @@ -171,21 +157,18 @@ log_blocks_crypt( Crypt_result rc = MY_AES_OK; uint dst_len; byte aes_ctr_counter[MY_AES_BLOCK_SIZE]; - byte is_encrypt= what == ENCRYPTION_FLAG_ENCRYPT; - lsn_t lsn = is_encrypt ? log_sys->lsn : srv_start_lsn; const uint src_len = OS_FILE_LOG_BLOCK_SIZE - LOG_BLOCK_HDR_SIZE; - for (ulint i = 0; i < size ; i += OS_FILE_LOG_BLOCK_SIZE) { + for (ulint i = 0; i < size ; i += OS_FILE_LOG_BLOCK_SIZE, + lsn += OS_FILE_LOG_BLOCK_SIZE) { ulint log_block_no = log_block_get_hdr_no(log_block); - lsn_t log_block_start_lsn = log_block_get_start_lsn( - lsn, log_block_no); const crypt_info_t* info = crypt_info == NULL ? get_crypt_info(log_block) : crypt_info; #ifdef DEBUG_CRYPT fprintf(stderr, "%s %lu chkpt: %lu key: %u lsn: %lu\n", - is_encrypt ? "crypt" : "decrypt", + what == ENCRYPTION_FLAG_ENCRYPT ? "crypt" : "decrypt", log_block_no, log_block_get_checkpoint_no(log_block), info ? info->key_version : 0, @@ -214,7 +197,7 @@ log_blocks_crypt( // (1-byte, only 5 bits are used). "+" means concatenate. bzero(aes_ctr_counter, MY_AES_BLOCK_SIZE); memcpy(aes_ctr_counter, info->crypt_nonce, 3); - mach_write_to_8(aes_ctr_counter + 3, log_block_start_lsn); + mach_write_to_8(aes_ctr_counter + 3, lsn); mach_write_to_4(aes_ctr_counter + 11, log_block_no); bzero(aes_ctr_counter + 15, 1); @@ -459,19 +442,6 @@ add_crypt_info( return true; } -/*********************************************************************//** -Encrypt log blocks. */ -UNIV_INTERN -Crypt_result -log_blocks_encrypt( -/*===============*/ - const byte* block, /*!< in: blocks before encryption */ - const ulint size, /*!< in: size of blocks, must be multiple of a log block */ - byte* dst_block) /*!< out: blocks after encryption */ -{ - return log_blocks_crypt(block, size, dst_block, ENCRYPTION_FLAG_ENCRYPT, NULL); -} - /*********************************************************************//** Set next checkpoint's key version to latest one, and generate current key. Key version 0 means no encryption. */ @@ -523,6 +493,8 @@ log_encrypt_before_write( /*=====================*/ ib_uint64_t next_checkpoint_no, /*!< in: log group to be flushed */ byte* block, /*!< in/out: pointer to a log block */ + lsn_t lsn, /*!< in: log sequence number of + the start of the buffer */ const ulint size) /*!< in: size of log blocks */ { ut_ad(size % OS_FILE_LOG_BLOCK_SIZE == 0); @@ -541,7 +513,8 @@ log_encrypt_before_write( byte* dst_frame = (byte*)malloc(size); //encrypt log blocks content - Crypt_result result = log_blocks_crypt(block, size, dst_frame, ENCRYPTION_FLAG_ENCRYPT, NULL); + Crypt_result result = log_blocks_crypt( + block, lsn, size, dst_frame, ENCRYPTION_FLAG_ENCRYPT, NULL); if (result == MY_AES_OK) { ut_ad(block[0] == dst_frame[0]); @@ -561,13 +534,16 @@ void log_decrypt_after_read( /*===================*/ byte* frame, /*!< in/out: log segment */ + lsn_t lsn, /*!< in: log sequence number of the start + of the buffer */ const ulint size) /*!< in: log segment size */ { ut_ad(size % OS_FILE_LOG_BLOCK_SIZE == 0); byte* dst_frame = (byte*)malloc(size); // decrypt log blocks content - Crypt_result result = log_blocks_crypt(frame, size, dst_frame, ENCRYPTION_FLAG_DECRYPT, NULL); + Crypt_result result = log_blocks_crypt( + frame, lsn, size, dst_frame, ENCRYPTION_FLAG_DECRYPT, NULL); if (result == MY_AES_OK) { memcpy(frame, dst_frame, size); diff --git a/storage/innobase/log/log0log.cc b/storage/innobase/log/log0log.cc index 1c9cbc3a219d2..7a242b76e854e 100644 --- a/storage/innobase/log/log0log.cc +++ b/storage/innobase/log/log0log.cc @@ -1380,7 +1380,7 @@ log_group_write_buf( ut_a(next_offset / UNIV_PAGE_SIZE <= ULINT_MAX); log_encrypt_before_write(log_sys->next_checkpoint_no, - buf, write_len); + buf, start_lsn, write_len); fil_io(OS_FILE_WRITE | OS_FILE_LOG, true, group->space_id, 0, (ulint) (next_offset / UNIV_PAGE_SIZE), @@ -2335,7 +2335,7 @@ log_group_read_log_seg( log_block_get_checksum(buf), source_offset); #endif - log_decrypt_after_read(buf, len); + log_decrypt_after_read(buf, start_lsn, len); #ifdef DEBUG_CRYPT fprintf(stderr, "AFTER DECRYPT: block: %lu checkpoint: %lu %.8lx %.8lx\n", @@ -2576,7 +2576,8 @@ log_group_archive( MONITOR_INC(MONITOR_LOG_IO); //TODO (jonaso): This must be dead code?? - log_encrypt_before_write(log_sys->next_checkpoint_no, buf, len); + log_encrypt_before_write(log_sys->next_checkpoint_no, + buf, start_lsn, len); fil_io(OS_FILE_WRITE | OS_FILE_LOG, false, group->archive_space_id, (ulint) (next_offset / UNIV_PAGE_SIZE), diff --git a/storage/xtradb/include/log0crypt.h b/storage/xtradb/include/log0crypt.h index 0ad7e7d7037a9..b7a221e0a8172 100644 --- a/storage/xtradb/include/log0crypt.h +++ b/storage/xtradb/include/log0crypt.h @@ -1,7 +1,7 @@ /***************************************************************************** Copyright (C) 2013, 2015, Google Inc. All Rights Reserved. -Copyright (C) 2014, 2017, MariaDB Corporation. All Rights Reserved. +Copyright (C) 2014, 2018, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -73,6 +73,8 @@ log_encrypt_before_write( /*=====================*/ ib_uint64_t next_checkpoint_no, /*!< in: log group to be flushed */ byte* block, /*!< in/out: pointer to a log block */ + lsn_t lsn, /*!< in: log sequence number of + the start of the buffer */ const ulint size); /*!< in: size of log blocks */ /******************************************************** @@ -83,6 +85,8 @@ void log_decrypt_after_read( /*===================*/ byte* frame, /*!< in/out: log segment */ + lsn_t lsn, /*!< in: log sequence number of the start + of the buffer */ const ulint size); /*!< in: log segment size */ /* Error codes for crypt info */ diff --git a/storage/xtradb/log/log0crypt.cc b/storage/xtradb/log/log0crypt.cc index a5fbbab17efcf..2a0a7abb6869e 100644 --- a/storage/xtradb/log/log0crypt.cc +++ b/storage/xtradb/log/log0crypt.cc @@ -1,7 +1,7 @@ /***************************************************************************** Copyright (C) 2013, 2015, Google Inc. All Rights Reserved. -Copyright (C) 2014, 2017, MariaDB Corporation. All Rights Reserved. +Copyright (C) 2014, 2018, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -69,22 +69,6 @@ struct crypt_info_t { static std::deque crypt_info; -/*********************************************************************//** -Get a log block's start lsn. -@return a log block's start lsn */ -static inline -lsn_t -log_block_get_start_lsn( -/*====================*/ - lsn_t lsn, /*!< in: checkpoint lsn */ - ulint log_block_no) /*!< in: log block number */ -{ - lsn_t start_lsn = - (lsn & (lsn_t)0xffffffff00000000ULL) | - (((log_block_no - 1) & (lsn_t)0x3fffffff) << 9); - return start_lsn; -} - /*********************************************************************//** Get crypt info from checkpoint. @return a crypt info or NULL if not present. */ @@ -161,6 +145,8 @@ Crypt_result log_blocks_crypt( /*=============*/ const byte* block, /*!< in: blocks before encrypt/decrypt*/ + lsn_t lsn, /*!< in: log sequence number of the start + of the buffer */ ulint size, /*!< in: size of block */ byte* dst_block, /*!< out: blocks after encrypt/decrypt */ int what, /*!< in: encrypt or decrypt*/ @@ -170,21 +156,18 @@ log_blocks_crypt( Crypt_result rc = MY_AES_OK; uint dst_len; byte aes_ctr_counter[MY_AES_BLOCK_SIZE]; - byte is_encrypt= what == ENCRYPTION_FLAG_ENCRYPT; - lsn_t lsn = is_encrypt ? log_sys->lsn : srv_start_lsn; const uint src_len = OS_FILE_LOG_BLOCK_SIZE - LOG_BLOCK_HDR_SIZE; - for (ulint i = 0; i < size ; i += OS_FILE_LOG_BLOCK_SIZE) { + for (ulint i = 0; i < size ; i += OS_FILE_LOG_BLOCK_SIZE, + lsn += OS_FILE_LOG_BLOCK_SIZE) { ulint log_block_no = log_block_get_hdr_no(log_block); - lsn_t log_block_start_lsn = log_block_get_start_lsn( - lsn, log_block_no); const crypt_info_t* info = crypt_info == NULL ? get_crypt_info(log_block) : crypt_info; #ifdef DEBUG_CRYPT fprintf(stderr, "%s %lu chkpt: %lu key: %u lsn: %lu\n", - is_encrypt ? "crypt" : "decrypt", + what == ENCRYPTION_FLAG_ENCRYPT ? "crypt" : "decrypt", log_block_no, log_block_get_checkpoint_no(log_block), info ? info->key_version : 0, @@ -213,7 +196,7 @@ log_blocks_crypt( // (1-byte, only 5 bits are used). "+" means concatenate. bzero(aes_ctr_counter, MY_AES_BLOCK_SIZE); memcpy(aes_ctr_counter, info->crypt_nonce, 3); - mach_write_to_8(aes_ctr_counter + 3, log_block_start_lsn); + mach_write_to_8(aes_ctr_counter + 3, lsn); mach_write_to_4(aes_ctr_counter + 11, log_block_no); bzero(aes_ctr_counter + 15, 1); @@ -458,19 +441,6 @@ add_crypt_info( return true; } -/*********************************************************************//** -Encrypt log blocks. */ -UNIV_INTERN -Crypt_result -log_blocks_encrypt( -/*===============*/ - const byte* block, /*!< in: blocks before encryption */ - const ulint size, /*!< in: size of blocks, must be multiple of a log block */ - byte* dst_block) /*!< out: blocks after encryption */ -{ - return log_blocks_crypt(block, size, dst_block, ENCRYPTION_FLAG_ENCRYPT, NULL); -} - /*********************************************************************//** Set next checkpoint's key version to latest one, and generate current key. Key version 0 means no encryption. */ @@ -522,6 +492,8 @@ log_encrypt_before_write( /*=====================*/ ib_uint64_t next_checkpoint_no, /*!< in: log group to be flushed */ byte* block, /*!< in/out: pointer to a log block */ + lsn_t lsn, /*!< in: log sequence number of + the start of the buffer */ const ulint size) /*!< in: size of log blocks */ { ut_ad(size % OS_FILE_LOG_BLOCK_SIZE == 0); @@ -540,7 +512,8 @@ log_encrypt_before_write( byte* dst_frame = (byte*)malloc(size); //encrypt log blocks content - Crypt_result result = log_blocks_crypt(block, size, dst_frame, ENCRYPTION_FLAG_ENCRYPT, NULL); + Crypt_result result = log_blocks_crypt( + block, lsn, size, dst_frame, ENCRYPTION_FLAG_ENCRYPT, NULL); if (result == MY_AES_OK) { ut_ad(block[0] == dst_frame[0]); @@ -560,13 +533,16 @@ void log_decrypt_after_read( /*===================*/ byte* frame, /*!< in/out: log segment */ + lsn_t lsn, /*!< in: log sequence number of the start + of the buffer */ const ulint size) /*!< in: log segment size */ { ut_ad(size % OS_FILE_LOG_BLOCK_SIZE == 0); byte* dst_frame = (byte*)malloc(size); // decrypt log blocks content - Crypt_result result = log_blocks_crypt(frame, size, dst_frame, ENCRYPTION_FLAG_DECRYPT, NULL); + Crypt_result result = log_blocks_crypt( + frame, lsn, size, dst_frame, ENCRYPTION_FLAG_DECRYPT, NULL); if (result == MY_AES_OK) { memcpy(frame, dst_frame, size); diff --git a/storage/xtradb/log/log0log.cc b/storage/xtradb/log/log0log.cc index 1542f4a646f74..8f8984f8880fd 100644 --- a/storage/xtradb/log/log0log.cc +++ b/storage/xtradb/log/log0log.cc @@ -1490,7 +1490,7 @@ log_group_write_buf( ut_a(next_offset / UNIV_PAGE_SIZE <= ULINT_MAX); log_encrypt_before_write(log_sys->next_checkpoint_no, - buf, write_len); + buf, start_lsn, write_len); #ifdef DEBUG_CRYPT fprintf(stderr, "WRITE: block: %lu checkpoint: %lu %.8lx %.8lx\n", @@ -2582,7 +2582,7 @@ log_group_read_log_seg( log_block_get_checksum(buf), source_offset); #endif - log_decrypt_after_read(buf, len); + log_decrypt_after_read(buf, start_lsn, len); #ifdef DEBUG_CRYPT fprintf(stderr, "AFTER DECRYPT: block: %lu checkpoint: %lu %.8lx %.8lx\n", @@ -2890,7 +2890,8 @@ log_group_archive( MONITOR_INC(MONITOR_LOG_IO); //TODO (jonaso): This must be dead code?? - log_encrypt_before_write(log_sys->next_checkpoint_no, buf, len); + log_encrypt_before_write(log_sys->next_checkpoint_no, + buf, start_lsn, len); fil_io(OS_FILE_WRITE | OS_FILE_LOG, false, group->archive_space_id, 0, From ae7e1b9b13ddc23167bd058ff4718ac116d17126 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20Lindstr=C3=B6m?= Date: Mon, 8 Jan 2018 12:25:31 +0200 Subject: [PATCH 008/106] MDEV-13262: innodb.deadlock_detect failed in buildbot There is a race condition on a test. con1 is the older transaction as it query started wait first. Test continues so that con1 gets lock wait timeout first. There is possibility that default connection gets lock timeout also or as con1 is rolled back it gets the locks it waited and does the update. Fixed by removing query outputs as they could vary and accepting success from default connection query. --- mysql-test/suite/innodb/r/deadlock_detect.result | 6 ------ mysql-test/suite/innodb/t/deadlock_detect.test | 14 +++++++++++++- 2 files changed, 13 insertions(+), 7 deletions(-) diff --git a/mysql-test/suite/innodb/r/deadlock_detect.result b/mysql-test/suite/innodb/r/deadlock_detect.result index c3e3794ed211b..4e14eff34b2b7 100644 --- a/mysql-test/suite/innodb/r/deadlock_detect.result +++ b/mysql-test/suite/innodb/r/deadlock_detect.result @@ -8,21 +8,15 @@ PRIMARY KEY(id) INSERT INTO t1 VALUES(1), (2), (3); BEGIN; SELECT * FROM t1 WHERE id = 1 FOR UPDATE; -id -1 connect con1,localhost,root,,; BEGIN; SELECT * FROM t1 WHERE id = 2 FOR UPDATE; -id -2 SELECT * FROM t1 WHERE id = 1 FOR UPDATE; connection default; SELECT * FROM t1 WHERE id = 2 FOR UPDATE; connection con1; -ERROR HY000: Lock wait timeout exceeded; try restarting transaction ROLLBACK; connection default; -ERROR HY000: Lock wait timeout exceeded; try restarting transaction ROLLBACK; DROP TABLE t1; disconnect con1; diff --git a/mysql-test/suite/innodb/t/deadlock_detect.test b/mysql-test/suite/innodb/t/deadlock_detect.test index 85d8c1f67f27f..babdb54719fde 100644 --- a/mysql-test/suite/innodb/t/deadlock_detect.test +++ b/mysql-test/suite/innodb/t/deadlock_detect.test @@ -18,6 +18,8 @@ CREATE TABLE t1( INSERT INTO t1 VALUES(1), (2), (3); +# We are not interested query results, only errors +--disable_result_log BEGIN; SELECT * FROM t1 WHERE id = 1 FOR UPDATE; @@ -39,12 +41,22 @@ reap; ROLLBACK; +# +# Note here that con1 is the older transaction as it +# query started wait first. Thus, con1 gets lock +# wait timeout first. There is possibility that +# default connection gets lock timeout also or +# as con1 is rolled back it gets the locks it waited +# and does the update. +# connection default; ---error ER_LOCK_WAIT_TIMEOUT +--error 0,ER_LOCK_WAIT_TIMEOUT reap; ROLLBACK; +--enable_result_log + DROP TABLE t1; disconnect con1; From 8099941b46dd5f7f79d361e6dcf378100305bd59 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Mon, 8 Jan 2018 10:58:36 +0200 Subject: [PATCH 009/106] MDEV-13487 Assertion failure in rec_get_trx_id() rec_get_trx_id(): Because rec is not necessarily residing in a buffer pool page (it could be an old version of a clustered index record, allocated from heap), remove the debug assertions that depend on page_align(rec). --- storage/innobase/rem/rem0rec.cc | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/storage/innobase/rem/rem0rec.cc b/storage/innobase/rem/rem0rec.cc index 37bf6e649a20c..6cd8db5d37bd5 100644 --- a/storage/innobase/rem/rem0rec.cc +++ b/storage/innobase/rem/rem0rec.cc @@ -1,7 +1,7 @@ /***************************************************************************** Copyright (c) 1994, 2016, Oracle and/or its affiliates. All Rights Reserved. -Copyright (c) 2017, MariaDB Corporation. +Copyright (c) 2017, 2018, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -2167,8 +2167,6 @@ rec_get_trx_id( const rec_t* rec, const dict_index_t* index) { - const page_t* page - = page_align(rec); ulint trx_id_col = dict_index_get_sys_col_pos(index, DATA_TRX_ID); const byte* trx_id; @@ -2179,11 +2177,7 @@ rec_get_trx_id( ulint* offsets = offsets_; ut_ad(trx_id_col <= MAX_REF_PARTS); - ut_ad(fil_page_index_page_check(page)); - ut_ad(mach_read_from_8(page + PAGE_HEADER + PAGE_INDEX_ID) - == index->id); ut_ad(dict_index_is_clust(index)); - ut_ad(page_rec_is_leaf(rec)); ut_ad(trx_id_col > 0); ut_ad(trx_id_col != ULINT_UNDEFINED); From 899c5899becbaf18651215693fb455001476b577 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Mon, 8 Jan 2018 12:58:25 +0200 Subject: [PATCH 010/106] MLOG-13101 Debug assertion failed in recv_parse_or_apply_log_rec_body() recv_parse_or_apply_log_rec_body(): Tolerate MLOG_4BYTES for dummy-writing the FIL_PAGE_SPACE_ID, written by fil_crypt_rotate_page(). --- storage/innobase/log/log0recv.cc | 2 ++ 1 file changed, 2 insertions(+) diff --git a/storage/innobase/log/log0recv.cc b/storage/innobase/log/log0recv.cc index 2e967a991216e..6bb2806fb7055 100644 --- a/storage/innobase/log/log0recv.cc +++ b/storage/innobase/log/log0recv.cc @@ -1166,6 +1166,8 @@ recv_parse_or_apply_log_rec_body( redo log been written with something older than InnoDB Plugin 1.0.4. */ ut_ad(0 + /* fil_crypt_rotate_page() writes this */ + || offs == FIL_PAGE_SPACE_ID || offs == IBUF_TREE_SEG_HEADER + IBUF_HEADER + FSEG_HDR_SPACE || offs == IBUF_TREE_SEG_HEADER From c903ba2f1ef5564d0eafd2803342f11cecb9bf01 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Mon, 8 Jan 2018 14:24:04 +0200 Subject: [PATCH 011/106] MDEV-13205 InnoDB: Failing assertion: !dict_index_is_online_ddl(index) upon ALTER TABLE dict_foreign_find_index(): Ignore incompletely created indexes. After a failed ADD UNIQUE INDEX, an incompletely created index could be left behind until the next ALTER TABLE statement. --- .../suite/innodb/r/innodb-index-online.result | 16 +++++++++++ .../suite/innodb/t/innodb-index-online.test | 27 +++++++++++++++++-- storage/innobase/dict/dict0dict.cc | 1 + storage/xtradb/dict/dict0dict.cc | 1 + 4 files changed, 43 insertions(+), 2 deletions(-) diff --git a/mysql-test/suite/innodb/r/innodb-index-online.result b/mysql-test/suite/innodb/r/innodb-index-online.result index 77b7bc365aad4..1dccc6ac4b829 100644 --- a/mysql-test/suite/innodb/r/innodb-index-online.result +++ b/mysql-test/suite/innodb/r/innodb-index-online.result @@ -339,6 +339,22 @@ ERROR 42000: Duplicate key name 'c2h' SET DEBUG_SYNC = 'RESET'; SET GLOBAL innodb_monitor_disable = module_ddl; DROP TABLE t1; +# +# MDEV-13205 assertion !dict_index_is_online_ddl(index) upon ALTER TABLE +# +CREATE TABLE t1 (c VARCHAR(64)) ENGINE=InnoDB; +SET DEBUG_SYNC = 'row_log_apply_before SIGNAL t1u_created WAIT_FOR dup_done'; +ALTER TABLE t1 ADD UNIQUE(c); +SET DEBUG_SYNC = 'now WAIT_FOR t1u_created'; +BEGIN; +INSERT INTO t1 VALUES('bar'),('bar'); +SET DEBUG_SYNC = 'now SIGNAL dup_done'; +ERROR 23000: Duplicate entry 'bar' for key 'c' +SET DEBUG_SYNC = 'RESET'; +CREATE TABLE t2 (c VARCHAR(64)) ENGINE=InnoDB; +ALTER TABLE t2 ADD FOREIGN KEY (c) REFERENCES t1 (c); +ERROR HY000: Can't create table `test`.`#sql-temporary` (errno: 150 "Foreign key constraint is incorrectly formed") +DROP TABLE t2,t1; SET GLOBAL innodb_file_per_table = @global_innodb_file_per_table_orig; SET GLOBAL innodb_monitor_enable = default; SET GLOBAL innodb_monitor_disable = default; diff --git a/mysql-test/suite/innodb/t/innodb-index-online.test b/mysql-test/suite/innodb/t/innodb-index-online.test index e09a65a878af1..331541ea099ec 100644 --- a/mysql-test/suite/innodb/t/innodb-index-online.test +++ b/mysql-test/suite/innodb/t/innodb-index-online.test @@ -382,8 +382,6 @@ connection default; reap; --enable_parsing #remove below con1 disconnect if above test case is enabled -connection con1; -disconnect con1; connection default; SHOW CREATE TABLE t1; @@ -399,6 +397,31 @@ SET GLOBAL innodb_monitor_disable = module_ddl; DROP TABLE t1; +--echo # +--echo # MDEV-13205 assertion !dict_index_is_online_ddl(index) upon ALTER TABLE +--echo # +CREATE TABLE t1 (c VARCHAR(64)) ENGINE=InnoDB; +SET DEBUG_SYNC = 'row_log_apply_before SIGNAL t1u_created WAIT_FOR dup_done'; +send ALTER TABLE t1 ADD UNIQUE(c); + +connection con1; +SET DEBUG_SYNC = 'now WAIT_FOR t1u_created'; +BEGIN; +INSERT INTO t1 VALUES('bar'),('bar'); +SET DEBUG_SYNC = 'now SIGNAL dup_done'; + +connection default; +--error ER_DUP_ENTRY +reap; + +SET DEBUG_SYNC = 'RESET'; +disconnect con1; +CREATE TABLE t2 (c VARCHAR(64)) ENGINE=InnoDB; +--replace_regex /#sql-[0-9a-f_]*/#sql-temporary/ +--error ER_CANT_CREATE_TABLE +ALTER TABLE t2 ADD FOREIGN KEY (c) REFERENCES t1 (c); +DROP TABLE t2,t1; + # Check that all connections opened by test cases in this file are really # gone so execution of other tests won't be affected by their presence. --source include/wait_until_count_sessions.inc diff --git a/storage/innobase/dict/dict0dict.cc b/storage/innobase/dict/dict0dict.cc index 11a4b89239740..0c8a278586d7a 100644 --- a/storage/innobase/dict/dict0dict.cc +++ b/storage/innobase/dict/dict0dict.cc @@ -3364,6 +3364,7 @@ dict_foreign_find_index( if (types_idx != index && !(index->type & DICT_FTS) && !index->to_be_dropped + && !dict_index_is_online_ddl(index) && dict_foreign_qualify_index( table, col_names, columns, n_cols, index, types_idx, diff --git a/storage/xtradb/dict/dict0dict.cc b/storage/xtradb/dict/dict0dict.cc index a23b297e9042f..fce293976bf85 100644 --- a/storage/xtradb/dict/dict0dict.cc +++ b/storage/xtradb/dict/dict0dict.cc @@ -3378,6 +3378,7 @@ dict_foreign_find_index( if (types_idx != index && !(index->type & DICT_FTS) && !index->to_be_dropped + && !dict_index_is_online_ddl(index) && dict_foreign_qualify_index( table, col_names, columns, n_cols, index, types_idx, From 18ccbf014adbc9eff74407ef15b3b6872196201e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Mon, 8 Jan 2018 17:09:21 +0200 Subject: [PATCH 012/106] Make the MDEV-14874 test case more robust --- mysql-test/suite/mariabackup/huge_lsn.result | 3 ++- mysql-test/suite/mariabackup/huge_lsn.test | 4 +++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/mysql-test/suite/mariabackup/huge_lsn.result b/mysql-test/suite/mariabackup/huge_lsn.result index 740a436228c27..f2202c20968f0 100644 --- a/mysql-test/suite/mariabackup/huge_lsn.result +++ b/mysql-test/suite/mariabackup/huge_lsn.result @@ -2,10 +2,11 @@ # MDEV-13416 mariabackup fails with EFAULT "Bad Address" # call mtr.add_suppression("InnoDB: New log files created"); -FOUND /InnoDB: .*started; log sequence number 17596481010700/ in mysqld.1.err +FOUND /InnoDB: New log files created, LSN=175964\d{8}/ in mysqld.1.err CREATE TABLE t(i INT) ENGINE INNODB; INSERT INTO t VALUES(1); # xtrabackup backup +SET GLOBAL innodb_flush_log_at_trx_commit=1; INSERT INTO t VALUES(2); # xtrabackup prepare # shutdown server diff --git a/mysql-test/suite/mariabackup/huge_lsn.test b/mysql-test/suite/mariabackup/huge_lsn.test index 3a173d43a3c9d..37be33916e7aa 100644 --- a/mysql-test/suite/mariabackup/huge_lsn.test +++ b/mysql-test/suite/mariabackup/huge_lsn.test @@ -28,8 +28,9 @@ EOF --remove_files_wildcard $MYSQLD_DATADIR ib_logfile* --source include/start_mysqld.inc +let SEARCH_RANGE= -50000; let SEARCH_FILE= $MYSQLTEST_VARDIR/log/mysqld.1.err; ---let SEARCH_PATTERN= InnoDB: .*started; log sequence number 17596481010700 +--let SEARCH_PATTERN= InnoDB: New log files created, LSN=175964\d{8} --source include/search_pattern_in_file.inc CREATE TABLE t(i INT) ENGINE INNODB; @@ -40,6 +41,7 @@ let $targetdir=$MYSQLTEST_VARDIR/tmp/backup; --disable_result_log exec $XTRABACKUP --defaults-file=$MYSQLTEST_VARDIR/my.cnf --backup --target-dir=$targetdir; --enable_result_log +SET GLOBAL innodb_flush_log_at_trx_commit=1; INSERT INTO t VALUES(2); echo # xtrabackup prepare; --disable_result_log From 919169e1f959744e609380132d8a561b96659811 Mon Sep 17 00:00:00 2001 From: Vladislav Vaintroub Date: Mon, 8 Jan 2018 17:01:55 +0000 Subject: [PATCH 013/106] Fix warning --- sql/handler.cc | 2 +- win/create_def_file.js | 5 +++++ 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/sql/handler.cc b/sql/handler.cc index 93db0b09abf08..10c2b65bbc0de 100644 --- a/sql/handler.cc +++ b/sql/handler.cc @@ -2068,7 +2068,7 @@ uint get_sql_xid(XID *xid, char *buf) MY_INT64_NUM_DECIMAL_DIGITS, -10, xid->formatID); } - return buf - orig_buf; + return (uint)(buf - orig_buf); } diff --git a/win/create_def_file.js b/win/create_def_file.js index 25bbbb4eb3d50..9a178510ca8c8 100644 --- a/win/create_def_file.js +++ b/win/create_def_file.js @@ -257,4 +257,9 @@ function addToResponseFile(filename, responseFile) responseFile.WriteLine("\""+fso.GetFile(filename).Path+"\""); } } + else + { + echo("file " + filename + " not found. Can't generate symbols file"); + WScript.Quit (1); + } } From 075f61a1d43c6805503735fc3dd118b90ae939de Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Tue, 9 Jan 2018 11:30:36 +0200 Subject: [PATCH 014/106] Revert part of commit fec844aca88e1c6b9c36bb0b811e92d9d023ffb9 row_insert_for_mysql(): Remove some duplicated code --- storage/innobase/row/row0mysql.cc | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/storage/innobase/row/row0mysql.cc b/storage/innobase/row/row0mysql.cc index 335ca35db95b8..7db855244ab44 100644 --- a/storage/innobase/row/row0mysql.cc +++ b/storage/innobase/row/row0mysql.cc @@ -1424,17 +1424,6 @@ row_insert_for_mysql( } else if (high_level_read_only) { return(DB_READ_ONLY); } - DBUG_EXECUTE_IF("mark_table_corrupted", { - /* Mark the table corrupted for the clustered index */ - dict_index_t* index = dict_table_get_first_index(table); - ut_ad(dict_index_is_clust(index)); - dict_set_corrupted(index, trx, "INSERT TABLE"); }); - - if (dict_table_is_corrupted(table)) { - - ib::error() << "Table " << table->name << " is corrupt."; - return(DB_TABLE_CORRUPT); - } DBUG_EXECUTE_IF("mark_table_corrupted", { /* Mark the table corrupted for the clustered index */ From 07aa98597984391a6e8c85f634106aef4b096a24 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jan=20Lindstr=C3=B6m?= Date: Tue, 9 Jan 2018 12:37:58 +0200 Subject: [PATCH 015/106] MDEV-14776: InnoDB Monitor output generated by specific error is flooding error logs innodb/buf_LRU_get_free_block Add debug instrumentation to produce error message about no free pages. Print error message only once and do not enable innodb monitor. xtradb/buf_LRU_get_free_block Add debug instrumentation to produce error message about no free pages. Print error message only once and do not enable innodb monitor. Remove code that does not seem to be used. innodb-lru-force-no-free-page.test New test case to force produce desired error message. --- .../r/innodb-lru-force-no-free-page.result | 9 ++ .../t/innodb-lru-force-no-free-page.test | 24 +++ storage/innobase/buf/buf0lru.cc | 78 +++++----- storage/xtradb/buf/buf0lru.cc | 138 ++++++------------ 4 files changed, 109 insertions(+), 140 deletions(-) create mode 100644 mysql-test/suite/innodb/r/innodb-lru-force-no-free-page.result create mode 100644 mysql-test/suite/innodb/t/innodb-lru-force-no-free-page.test diff --git a/mysql-test/suite/innodb/r/innodb-lru-force-no-free-page.result b/mysql-test/suite/innodb/r/innodb-lru-force-no-free-page.result new file mode 100644 index 0000000000000..d095f4e32e731 --- /dev/null +++ b/mysql-test/suite/innodb/r/innodb-lru-force-no-free-page.result @@ -0,0 +1,9 @@ +call mtr.add_suppression("\\[Warning\\] InnoDB: Difficult to find free blocks in the buffer pool."); +SET SESSION debug_dbug="+d,ib_lru_force_no_free_page"; +CREATE TABLE t1 (j LONGBLOB) ENGINE = InnoDB; +BEGIN; +INSERT INTO t1 VALUES (repeat('abcdefghijklmnopqrstuvwxyz',200)); +COMMIT; +SET SESSION debug_dbug=""; +DROP TABLE t1; +FOUND /InnoDB: Difficult to find free blocks / in mysqld.1.err diff --git a/mysql-test/suite/innodb/t/innodb-lru-force-no-free-page.test b/mysql-test/suite/innodb/t/innodb-lru-force-no-free-page.test new file mode 100644 index 0000000000000..e358446eca944 --- /dev/null +++ b/mysql-test/suite/innodb/t/innodb-lru-force-no-free-page.test @@ -0,0 +1,24 @@ +--source include/have_innodb.inc +--source include/have_debug.inc + +call mtr.add_suppression("\\[Warning\\] InnoDB: Difficult to find free blocks in the buffer pool."); + +SET SESSION debug_dbug="+d,ib_lru_force_no_free_page"; + +CREATE TABLE t1 (j LONGBLOB) ENGINE = InnoDB; +BEGIN; +INSERT INTO t1 VALUES (repeat('abcdefghijklmnopqrstuvwxyz',200)); +COMMIT; + +SET SESSION debug_dbug=""; + +DROP TABLE t1; + +# +# There should be only one message +# +let SEARCH_RANGE= -50000; +let SEARCH_FILE= $MYSQLTEST_VARDIR/log/mysqld.1.err; +--let SEARCH_PATTERN=InnoDB: Difficult to find free blocks +--source include/search_pattern_in_file.inc + diff --git a/storage/innobase/buf/buf0lru.cc b/storage/innobase/buf/buf0lru.cc index a047b28f4fd08..b1b63e72a45fe 100644 --- a/storage/innobase/buf/buf0lru.cc +++ b/storage/innobase/buf/buf0lru.cc @@ -1,7 +1,7 @@ /***************************************************************************** Copyright (c) 1995, 2016, Oracle and/or its affiliates. All Rights Reserved. -Copyright (c) 2017, MariaDB Corporation. +Copyright (c) 2017, 2018, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -90,6 +90,10 @@ during LRU eviction. */ frames in the buffer pool, we set this to TRUE */ static ibool buf_lru_switched_on_innodb_mon = FALSE; +/** True if diagnostic message about difficult to find free blocks +in the buffer bool has already printed. */ +static bool buf_lru_free_blocks_error_printed; + /******************************************************************//** These statistics are not 'of' LRU but 'for' LRU. We keep count of I/O and page_zip_decompress() operations. Based on the statistics, @@ -1045,8 +1049,6 @@ buf_LRU_get_free_block( ibool freed = FALSE; ulint n_iterations = 0; ulint flush_failures = 0; - ibool mon_value_was = FALSE; - ibool started_monitor = FALSE; MONITOR_INC(MONITOR_LRU_GET_FREE_SEARCH); loop: @@ -1054,6 +1056,11 @@ buf_LRU_get_free_block( buf_LRU_check_size_of_non_data_objects(buf_pool); + DBUG_EXECUTE_IF("ib_lru_force_no_free_page", + if (!buf_lru_free_blocks_error_printed) { + n_iterations = 21; + goto not_found;}); + /* If there is a block in the free list, take it */ block = buf_LRU_get_free_only(buf_pool); @@ -1062,16 +1069,11 @@ buf_LRU_get_free_block( buf_pool_mutex_exit(buf_pool); ut_ad(buf_pool_from_block(block) == buf_pool); memset(&block->page.zip, 0, sizeof block->page.zip); - - if (started_monitor) { - srv_print_innodb_monitor = - static_cast(mon_value_was); - } - return(block); } freed = FALSE; + if (buf_pool->try_LRU_scan || n_iterations > 0) { /* If no block was in the free list, search from the end of the LRU list and try to free a block there. @@ -1094,6 +1096,10 @@ buf_LRU_get_free_block( } } +#ifndef DBUG_OFF +not_found: +#endif + buf_pool_mutex_exit(buf_pool); if (freed) { @@ -1101,40 +1107,26 @@ buf_LRU_get_free_block( } - if (n_iterations > 20) { - ut_print_timestamp(stderr); - fprintf(stderr, - " InnoDB: Warning: difficult to find free blocks in\n" - "InnoDB: the buffer pool (%lu search iterations)!\n" - "InnoDB: %lu failed attempts to flush a page!" - " Consider\n" - "InnoDB: increasing the buffer pool size.\n" - "InnoDB: It is also possible that" - " in your Unix version\n" - "InnoDB: fsync is very slow, or" - " completely frozen inside\n" - "InnoDB: the OS kernel. Then upgrading to" - " a newer version\n" - "InnoDB: of your operating system may help." - " Look at the\n" - "InnoDB: number of fsyncs in diagnostic info below.\n" - "InnoDB: Pending flushes (fsync) log: %lu;" - " buffer pool: %lu\n" - "InnoDB: %lu OS file reads, %lu OS file writes," - " %lu OS fsyncs\n" - "InnoDB: Starting InnoDB Monitor to print further\n" - "InnoDB: diagnostics to the standard output.\n", - (ulong) n_iterations, - (ulong) flush_failures, - (ulong) fil_n_pending_log_flushes, - (ulong) fil_n_pending_tablespace_flushes, - (ulong) os_n_file_reads, (ulong) os_n_file_writes, - (ulong) os_n_fsyncs); - - mon_value_was = srv_print_innodb_monitor; - started_monitor = TRUE; - srv_print_innodb_monitor = TRUE; - os_event_set(srv_monitor_event); + if (n_iterations > 20 && !buf_lru_free_blocks_error_printed) { + ib_logf(IB_LOG_LEVEL_WARN, + "Difficult to find free blocks in" + " the buffer pool (" ULINTPF " search iterations)! " + ULINTPF " failed attempts to flush a page!", + n_iterations, flush_failures); + ib_logf(IB_LOG_LEVEL_INFO, + "Consider increasing the buffer pool size."); + ib_logf(IB_LOG_LEVEL_INFO, + "Pending flushes (fsync) log: " ULINTPF + " buffer pool: " ULINTPF + " OS file reads: " ULINTPF " OS file writes: " + ULINTPF " OS fsyncs: " ULINTPF "", + fil_n_pending_log_flushes, + fil_n_pending_tablespace_flushes, + os_n_file_reads, + os_n_file_writes, + os_n_fsyncs); + + buf_lru_free_blocks_error_printed = true; } /* If we have scanned the whole LRU and still are unable to diff --git a/storage/xtradb/buf/buf0lru.cc b/storage/xtradb/buf/buf0lru.cc index c71d45009e456..a84f2a832f35f 100644 --- a/storage/xtradb/buf/buf0lru.cc +++ b/storage/xtradb/buf/buf0lru.cc @@ -1,7 +1,7 @@ /***************************************************************************** Copyright (c) 1995, 2016, Oracle and/or its affiliates. All Rights Reserved. -Copyright (c) 2017, MariaDB Corporation. +Copyright (c) 2017, 2018, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -87,6 +87,10 @@ buffer pools. */ frames in the buffer pool, we set this to TRUE */ static ibool buf_lru_switched_on_innodb_mon = FALSE; +/** True if diagnostic message about difficult to find free blocks +in the buffer bool has already printed. */ +static bool buf_lru_free_blocks_error_printed; + /******************************************************************//** These statistics are not 'of' LRU but 'for' LRU. We keep count of I/O and page_zip_decompress() operations. Based on the statistics, @@ -1080,68 +1084,39 @@ buf_LRU_check_size_of_non_data_objects( } /** Diagnose failure to get a free page and request InnoDB monitor output in -the error log if more than two seconds have been spent already. +the error log if it has not yet printed. @param[in] n_iterations how many buf_LRU_get_free_page iterations already completed -@param[in] started_ms timestamp in ms of when the attempt to get the - free page started @param[in] flush_failures how many times single-page flush, if allowed, has failed -@param[out] mon_value_was previous srv_print_innodb_monitor value -@param[out] started_monitor whether InnoDB monitor print has been requested */ static void -buf_LRU_handle_lack_of_free_blocks(ulint n_iterations, ulint started_ms, - ulint flush_failures, - ibool *mon_value_was, - ibool *started_monitor) +buf_LRU_handle_lack_of_free_blocks( + ulint n_iterations, + ulint flush_failures) { - static ulint last_printout_ms = 0; - - /* Legacy algorithm started warning after at least 2 seconds, we - emulate this. */ - const ulint current_ms = ut_time_ms(); - - if ((current_ms > started_ms + 2000) - && (current_ms > last_printout_ms + 2000)) { - - ut_print_timestamp(stderr); - fprintf(stderr, - " InnoDB: Warning: difficult to find free blocks in\n" - "InnoDB: the buffer pool (%lu search iterations)!\n" - "InnoDB: %lu failed attempts to flush a page!" - " Consider\n" - "InnoDB: increasing the buffer pool size.\n" - "InnoDB: It is also possible that" - " in your Unix version\n" - "InnoDB: fsync is very slow, or" - " completely frozen inside\n" - "InnoDB: the OS kernel. Then upgrading to" - " a newer version\n" - "InnoDB: of your operating system may help." - " Look at the\n" - "InnoDB: number of fsyncs in diagnostic info below.\n" - "InnoDB: Pending flushes (fsync) log: %lu;" - " buffer pool: %lu\n" - "InnoDB: %lu OS file reads, %lu OS file writes," - " %lu OS fsyncs\n" - "InnoDB: Starting InnoDB Monitor to print further\n" - "InnoDB: diagnostics to the standard output.\n", - (ulong) n_iterations, - (ulong) flush_failures, - (ulong) fil_n_pending_log_flushes, - (ulong) fil_n_pending_tablespace_flushes, - (ulong) os_n_file_reads, (ulong) os_n_file_writes, - (ulong) os_n_fsyncs); - - last_printout_ms = current_ms; - *mon_value_was = srv_print_innodb_monitor; - *started_monitor = TRUE; - srv_print_innodb_monitor = TRUE; - os_event_set(lock_sys->timeout_event); + if (n_iterations > 20 && !buf_lru_free_blocks_error_printed) { + ib_logf(IB_LOG_LEVEL_WARN, + "Difficult to find free blocks in" + " the buffer pool (" ULINTPF " search iterations)! " + ULINTPF " failed attempts to flush a page!", + n_iterations, flush_failures); + ib_logf(IB_LOG_LEVEL_INFO, + "Consider increasing the buffer pool size."); + ib_logf(IB_LOG_LEVEL_INFO, + "Pending flushes (fsync) log: " ULINTPF + " buffer pool: " ULINTPF + " OS file reads: " ULINTPF " OS file writes: " + ULINTPF " OS fsyncs: " ULINTPF "", + fil_n_pending_log_flushes, + fil_n_pending_tablespace_flushes, + os_n_file_reads, + os_n_file_writes, + os_n_fsyncs); + + buf_lru_free_blocks_error_printed = true; } - } /** The maximum allowed backoff sleep time duration, microseconds */ @@ -1189,9 +1164,6 @@ buf_LRU_get_free_block( ibool freed = FALSE; ulint n_iterations = 0; ulint flush_failures = 0; - ibool mon_value_was = FALSE; - ibool started_monitor = FALSE; - ulint started_ms = 0; ut_ad(!mutex_own(&buf_pool->LRU_list_mutex)); @@ -1199,42 +1171,20 @@ buf_LRU_get_free_block( loop: buf_LRU_check_size_of_non_data_objects(buf_pool); - /* If there is a block in the free list, take it */ - if (DBUG_EVALUATE_IF("simulate_lack_of_pages", true, false)) { - - block = NULL; - - if (srv_debug_monitor_printed) - DBUG_SET("-d,simulate_lack_of_pages"); - - } else if (DBUG_EVALUATE_IF("simulate_recovery_lack_of_pages", - recv_recovery_on, false)) { - - block = NULL; - - if (srv_debug_monitor_printed) - DBUG_SUICIDE(); - } else { + DBUG_EXECUTE_IF("ib_lru_force_no_free_page", + if (!buf_lru_free_blocks_error_printed) { + n_iterations = 21; + goto not_found;}); - block = buf_LRU_get_free_only(buf_pool); - } + block = buf_LRU_get_free_only(buf_pool); if (block) { ut_ad(buf_pool_from_block(block) == buf_pool); memset(&block->page.zip, 0, sizeof block->page.zip); - - if (started_monitor) { - srv_print_innodb_monitor = - static_cast(mon_value_was); - } - return(block); } - if (!started_ms) - started_ms = ut_time_ms(); - if (srv_empty_free_list_algorithm == SRV_EMPTY_FREE_LIST_BACKOFF && buf_lru_manager_is_active && (srv_shutdown_state == SRV_SHUTDOWN_NONE @@ -1272,10 +1222,7 @@ buf_LRU_get_free_block( : FREE_LIST_BACKOFF_LOW_PRIO_DIVIDER)); } - buf_LRU_handle_lack_of_free_blocks(n_iterations, started_ms, - flush_failures, - &mon_value_was, - &started_monitor); + buf_LRU_handle_lack_of_free_blocks(n_iterations, flush_failures); n_iterations++; @@ -1314,13 +1261,8 @@ buf_LRU_get_free_block( mutex_exit(&buf_pool->flush_state_mutex); - if (DBUG_EVALUATE_IF("simulate_recovery_lack_of_pages", true, false) - || DBUG_EVALUATE_IF("simulate_lack_of_pages", true, false)) { - - buf_pool->try_LRU_scan = false; - } - freed = FALSE; + if (buf_pool->try_LRU_scan || n_iterations > 0) { /* If no block was in the free list, search from the @@ -1345,9 +1287,11 @@ buf_LRU_get_free_block( } - buf_LRU_handle_lack_of_free_blocks(n_iterations, started_ms, - flush_failures, &mon_value_was, - &started_monitor); +#ifndef DBUG_OFF +not_found: +#endif + + buf_LRU_handle_lack_of_free_blocks(n_iterations, flush_failures); /* If we have scanned the whole LRU and still are unable to find a free block then we should sleep here to let the From 7d201d7b3000bc7cf4e9a4360ad55ed4c85c60d8 Mon Sep 17 00:00:00 2001 From: Igor Babaev Date: Sat, 6 Jan 2018 09:32:47 -0800 Subject: [PATCH 016/106] Fixed mdev-14879 Lost rows for query using recursive CTE with recursive reference in subquery If a recursive CTE uses a subquery with recursive reference then the virtual function reset() must be called after each iteration performed at the execution of the CTE. --- mysql-test/r/cte_recursive.result | 78 +++++++++++++++++++++++++++++++ mysql-test/t/cte_recursive.test | 53 +++++++++++++++++++++ sql/sql_union.cc | 1 + 3 files changed, 132 insertions(+) diff --git a/mysql-test/r/cte_recursive.result b/mysql-test/r/cte_recursive.result index 300539a75b224..7bbdcb47f26c7 100644 --- a/mysql-test/r/cte_recursive.result +++ b/mysql-test/r/cte_recursive.result @@ -2972,3 +2972,81 @@ WHERE 1 = 1 ) K, t3; C P i drop table t1,t2,t3; +# +# mdev-14879: subquery with recursive reference in WHERE of CTE +# +create table flights +(departure varchar(32), +arrival varchar(32), +carrier varchar(20), +flight_number char(7)); +insert into flights values +('Seattle', 'Frankfurt', 'Lufthansa', 'LH 491'), +('Seattle', 'Chicago', 'American', 'AA 2573'), +('Seattle', 'Los Angeles', 'Alaska Air', 'AS 410'), +('Chicago', 'New York', 'American', 'AA 375'), +('Chicago', 'Montreal', 'Air Canada', 'AC 3053'), +('Los Angeles', 'New York', 'Delta', 'DL 1197'), +('Moscow', 'Tokyo', 'Aeroflot', 'SU 264'), +('New York', 'Paris', 'Air France', 'AF 23'), +('Frankfurt', 'Moscow', 'Lufthansa', 'LH 1444'), +('Tokyo', 'Seattle', 'ANA', 'NH 178'), +('Los Angeles', 'Tokyo', 'ANA', 'NH 175'), +('Moscow', 'Los Angeles', 'Aeroflot', 'SU 106'), +('Montreal', 'Paris', 'Air Canada', 'AC 870'), +('Cairo', 'Paris', 'Air France', 'AF 503'), +('New York', 'Seattle', 'American', 'AA 45'), +('Paris', 'Chicago', 'Air France', 'AF 6734'); +with recursive destinations (city) as +( select a.arrival from flights a where a.departure='Cairo' + union +select b.arrival from destinations r, flights b where r.city=b.departure) +select * from destinations; +city +Paris +Chicago +New York +Montreal +Seattle +Frankfurt +Los Angeles +Moscow +Tokyo +set standard_compliant_cte=0; +with recursive destinations (city, legs) as +( +select a.arrival, 1 from flights a where a.departure='Cairo' + union +select b.arrival, r.legs + 1 from destinations r, flights b +where r.city=b.departure and b.arrival not in (select city from destinations) +) +select * from destinations; +city legs +Paris 1 +Chicago 2 +New York 3 +Montreal 3 +Seattle 4 +Frankfurt 5 +Los Angeles 5 +Moscow 6 +Tokyo 6 +explain extended with recursive destinations (city, legs) as +( +select a.arrival, 1 from flights a where a.departure='Cairo' + union +select b.arrival, r.legs + 1 from destinations r, flights b +where r.city=b.departure and b.arrival not in (select city from destinations) +) +select * from destinations; +id select_type table type possible_keys key key_len ref rows filtered Extra +1 PRIMARY ALL NULL NULL NULL NULL 16 100.00 +2 DERIVED a ALL NULL NULL NULL NULL 16 100.00 Using where +3 RECURSIVE UNION b ALL NULL NULL NULL NULL 16 100.00 Using where +3 RECURSIVE UNION ref key0 key0 35 test.b.departure 2 100.00 +4 DEPENDENT SUBQUERY ALL NULL NULL NULL NULL 16 100.00 Using where +NULL UNION RESULT ALL NULL NULL NULL NULL NULL NULL +Warnings: +Note 1003 with recursive destinations as (select `test`.`a`.`arrival` AS `city`,1 AS `legs` from `test`.`flights` `a` where `test`.`a`.`departure` = 'Cairo' union select `test`.`b`.`arrival` AS `arrival`,`r`.`legs` + 1 AS `r.legs + 1` from `destinations` `r` join `test`.`flights` `b` where `r`.`city` = `test`.`b`.`departure` and !(`test`.`b`.`arrival`,(select `destinations`.`city` from `destinations` where trigcond(`test`.`b`.`arrival` = `destinations`.`city` or `destinations`.`city` is null) having trigcond(`destinations`.`city` is null))))select `destinations`.`city` AS `city`,`destinations`.`legs` AS `legs` from `destinations` +set standard_compliant_cte=default; +drop table flights; diff --git a/mysql-test/t/cte_recursive.test b/mysql-test/t/cte_recursive.test index 4e9a1e6aa32ba..332f6cfa49b04 100644 --- a/mysql-test/t/cte_recursive.test +++ b/mysql-test/t/cte_recursive.test @@ -2032,3 +2032,56 @@ FROM ) K, t3; drop table t1,t2,t3; + +--echo # +--echo # mdev-14879: subquery with recursive reference in WHERE of CTE +--echo # + +create table flights +(departure varchar(32), + arrival varchar(32), + carrier varchar(20), + flight_number char(7)); + +insert into flights values +('Seattle', 'Frankfurt', 'Lufthansa', 'LH 491'), +('Seattle', 'Chicago', 'American', 'AA 2573'), +('Seattle', 'Los Angeles', 'Alaska Air', 'AS 410'), +('Chicago', 'New York', 'American', 'AA 375'), +('Chicago', 'Montreal', 'Air Canada', 'AC 3053'), +('Los Angeles', 'New York', 'Delta', 'DL 1197'), +('Moscow', 'Tokyo', 'Aeroflot', 'SU 264'), +('New York', 'Paris', 'Air France', 'AF 23'), +('Frankfurt', 'Moscow', 'Lufthansa', 'LH 1444'), +('Tokyo', 'Seattle', 'ANA', 'NH 178'), +('Los Angeles', 'Tokyo', 'ANA', 'NH 175'), +('Moscow', 'Los Angeles', 'Aeroflot', 'SU 106'), +('Montreal', 'Paris', 'Air Canada', 'AC 870'), +('Cairo', 'Paris', 'Air France', 'AF 503'), +('New York', 'Seattle', 'American', 'AA 45'), +('Paris', 'Chicago', 'Air France', 'AF 6734'); + +with recursive destinations (city) as +( select a.arrival from flights a where a.departure='Cairo' + union + select b.arrival from destinations r, flights b where r.city=b.departure) +select * from destinations; + +set standard_compliant_cte=0; + +let $q= +with recursive destinations (city, legs) as +( + select a.arrival, 1 from flights a where a.departure='Cairo' + union + select b.arrival, r.legs + 1 from destinations r, flights b + where r.city=b.departure and b.arrival not in (select city from destinations) +) +select * from destinations; + +eval $q; +eval explain extended $q; + +set standard_compliant_cte=default; + +drop table flights; diff --git a/sql/sql_union.cc b/sql/sql_union.cc index 7cbc69f2ee70f..6f63cc5d0ebc1 100644 --- a/sql/sql_union.cc +++ b/sql/sql_union.cc @@ -1284,6 +1284,7 @@ bool st_select_lex_unit::exec_recursive() sq; sq= sq->next_with_rec_ref) { + sq->reset(); sq->engine->force_reexecution(); } From b132d4d749c29e558b8018f394e57713bab86a7d Mon Sep 17 00:00:00 2001 From: Vladislav Vaintroub Date: Tue, 9 Jan 2018 22:52:01 +0000 Subject: [PATCH 017/106] Windows, compilation : Treat warning as error, if MYSQL_MAINTAINER_MODE is set to ERR This matches gcc/clang handling. --- cmake/os/Windows.cmake | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/cmake/os/Windows.cmake b/cmake/os/Windows.cmake index 021f23a14f264..4fd01e5914ed5 100644 --- a/cmake/os/Windows.cmake +++ b/cmake/os/Windows.cmake @@ -142,6 +142,11 @@ IF(MSVC) SET(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} /wd4800 /wd4805 /wd4996 /we4700 /we4311 /we4477 /we4302 /we4090 /wd4267 ") SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} /wd4800 /wd4805 /wd4996 /wd4291 /wd4577 /we4099 /we4700 /we4311 /we4477 /we4302 /we4090 /wd4267") + IF(MYSQL_MAINTAINER_MODE MATCHES "ERR") + SET(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} /WX") + SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} /WX") + ENDIF() + ENDIF() # Always link with socket library From a408e881cf73d06fc92097fce6ef9584e16edf77 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Wed, 10 Jan 2018 09:17:43 +0200 Subject: [PATCH 018/106] MDEV-14174 crash on start with innodb-track-changed-pages The XtraDB option innodb_track_changed_pages causes the function log_group_read_log_seg() to be invoked even when recv_sys==NULL, leading to the SIGSEGV. This regression was caused by MDEV-11027 InnoDB log recovery is too noisy --- mysql-test/suite/innodb/t/innodb-master.opt | 1 + storage/xtradb/log/log0log.c | 16 +++++++++------- 2 files changed, 10 insertions(+), 7 deletions(-) diff --git a/mysql-test/suite/innodb/t/innodb-master.opt b/mysql-test/suite/innodb/t/innodb-master.opt index 5266978e4f0a7..8d9b0302a9eb1 100644 --- a/mysql-test/suite/innodb/t/innodb-master.opt +++ b/mysql-test/suite/innodb/t/innodb-master.opt @@ -2,3 +2,4 @@ --default-storage-engine=MyISAM --innodb-strict-mode=0 --innodb-file-per-table=0 +--loose-innodb-track-changed-pages diff --git a/storage/xtradb/log/log0log.c b/storage/xtradb/log/log0log.c index e327fa7ea9a6a..bed7e31da28bf 100644 --- a/storage/xtradb/log/log0log.c +++ b/storage/xtradb/log/log0log.c @@ -2503,7 +2503,6 @@ log_group_read_log_seg( ulint len; ulint source_offset; ibool sync; - ib_time_t time; ut_ad(mutex_own(&(log_sys->mutex))); @@ -2540,13 +2539,16 @@ log_group_read_log_seg( start_lsn += len; buf += len; - time = ut_time(); + if (recv_sys) { + ib_time_t time = ut_time(); - if (recv_sys->progress_time - time >= 15) { - recv_sys->progress_time = time; - ut_print_timestamp(stderr); - fprintf(stderr, " InnoDB: Read redo log up to LSN=%llu\n", - start_lsn); + if (recv_sys->progress_time - time >= 15) { + recv_sys->progress_time = time; + ut_print_timestamp(stderr); + fprintf(stderr, + " InnoDB: Read redo log up to LSN=%llu\n", + start_lsn); + } } if (start_lsn != end_lsn) { From a9c55c0059351220674a1dd3d3bf8d4e884b3285 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Wed, 10 Jan 2018 10:21:52 +0200 Subject: [PATCH 019/106] MDEV-13814 Extra logging when innodb_log_archive=ON Backport the fix from 10.0.33 to 5.5, in case someone compiles XtraDB with -DUNIV_LOG_ARCHIVE --- mysql-test/suite/innodb/t/innodb-master.opt | 1 + storage/xtradb/log/log0log.c | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/mysql-test/suite/innodb/t/innodb-master.opt b/mysql-test/suite/innodb/t/innodb-master.opt index 8d9b0302a9eb1..2e71d62206dbd 100644 --- a/mysql-test/suite/innodb/t/innodb-master.opt +++ b/mysql-test/suite/innodb/t/innodb-master.opt @@ -3,3 +3,4 @@ --innodb-strict-mode=0 --innodb-file-per-table=0 --loose-innodb-track-changed-pages +--loose-innodb-log-archive diff --git a/storage/xtradb/log/log0log.c b/storage/xtradb/log/log0log.c index bed7e31da28bf..1f9003aa66eb1 100644 --- a/storage/xtradb/log/log0log.c +++ b/storage/xtradb/log/log0log.c @@ -2539,7 +2539,7 @@ log_group_read_log_seg( start_lsn += len; buf += len; - if (recv_sys) { + if (recv_recovery_is_on()) { ib_time_t time = ut_time(); if (recv_sys->progress_time - time >= 15) { From d1cf9b167cb0da4987d898cf5d8b389419cae09e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Wed, 10 Jan 2018 13:18:02 +0200 Subject: [PATCH 020/106] MDEV-14909 MariaDB 10.2 refuses to start up after clean shutdown of MariaDB 10.3 recv_log_recover_10_3(): Determine if a log from MariaDB 10.3 is clean. recv_find_max_checkpoint(): Allow startup with a clean 10.3 redo log. srv_prepare_to_delete_redo_log_files(): When starting up with a 10.3 log, display a "Downgrading redo log" message instead of "Upgrading". --- .../r/innodb_encrypt_log_corruption.result | 27 ++++++- .../suite/innodb/r/log_corruption.result | 27 ++++++- mysql-test/suite/innodb/t/log_corruption.test | 62 +++++++++++++- storage/innobase/include/log0log.h | 4 +- storage/innobase/log/log0recv.cc | 80 ++++++++++++++++++- storage/innobase/srv/srv0start.cc | 8 +- 6 files changed, 191 insertions(+), 17 deletions(-) diff --git a/mysql-test/suite/encryption/r/innodb_encrypt_log_corruption.result b/mysql-test/suite/encryption/r/innodb_encrypt_log_corruption.result index 4a31f1ba4546c..e4ece7bc4ed34 100644 --- a/mysql-test/suite/encryption/r/innodb_encrypt_log_corruption.result +++ b/mysql-test/suite/encryption/r/innodb_encrypt_log_corruption.result @@ -17,6 +17,13 @@ WHERE engine = 'innodb' AND support IN ('YES', 'DEFAULT', 'ENABLED'); ENGINE SUPPORT COMMENT TRANSACTIONS XA SAVEPOINTS FOUND 1 /InnoDB: Upgrade after a crash is not supported. This redo log was created before MariaDB 10\.2\.2, and it appears corrupted/ in mysqld.1.err +# empty redo log from before MariaDB 10.2.2 +SELECT COUNT(*) FROM INFORMATION_SCHEMA.ENGINES +WHERE engine = 'innodb' +AND support IN ('YES', 'DEFAULT', 'ENABLED'); +COUNT(*) +1 +FOUND 1 /InnoDB: Upgrading redo log:/ in mysqld.1.err # redo log from "after" MariaDB 10.2.2, but with invalid header checksum SELECT * FROM INFORMATION_SCHEMA.ENGINES WHERE engine = 'innodb' @@ -28,7 +35,7 @@ SELECT * FROM INFORMATION_SCHEMA.ENGINES WHERE engine = 'innodb' AND support IN ('YES', 'DEFAULT', 'ENABLED'); ENGINE SUPPORT COMMENT TRANSACTIONS XA SAVEPOINTS -FOUND 1 /InnoDB: Unsupported redo log format. The redo log was created with malicious intentions, or perhaps\. Please follow the instructions at http://dev.mysql.com/doc/refman/5.7/en/upgrading-downgrading.html/ in mysqld.1.err +FOUND 1 /InnoDB: Unsupported redo log format. The redo log was created with malicious intentions, or perhaps\./ in mysqld.1.err # valid header, but old-format checkpoint blocks SELECT * FROM INFORMATION_SCHEMA.ENGINES WHERE engine = 'innodb' @@ -86,12 +93,26 @@ AND support IN ('YES', 'DEFAULT', 'ENABLED'); ENGINE SUPPORT COMMENT TRANSACTIONS XA SAVEPOINTS FOUND 1 /InnoDB: MLOG_FILE_NAME incorrect:bigot/ in mysqld.1.err FOUND 1 /len 22; hex 38000000000012860cb7809781e800066269676f7400; asc 8 bigot ;/ in mysqld.1.err -# missing MLOG_FILE_NAME or MLOG_FILE_DELETE before MLOG_CHECKPOINT +# 10.2 missing MLOG_FILE_NAME or MLOG_FILE_DELETE before MLOG_CHECKPOINT +SELECT * FROM INFORMATION_SCHEMA.ENGINES +WHERE engine = 'innodb' +AND support IN ('YES', 'DEFAULT', 'ENABLED'); +ENGINE SUPPORT COMMENT TRANSACTIONS XA SAVEPOINTS +FOUND 1 /InnoDB: Missing MLOG_FILE_NAME or MLOG_FILE_DELETE before MLOG_CHECKPOINT for tablespace 42/ in mysqld.1.err +# 10.3 missing MLOG_FILE_NAME or MLOG_FILE_DELETE before MLOG_CHECKPOINT SELECT * FROM INFORMATION_SCHEMA.ENGINES WHERE engine = 'innodb' AND support IN ('YES', 'DEFAULT', 'ENABLED'); ENGINE SUPPORT COMMENT TRANSACTIONS XA SAVEPOINTS -NOT FOUND /InnoDB: Missing MLOG_FILE_NAME or MLOG_FILE_DELETE before MLOG_CHECKPOINT for tablespace 42$/ in mysqld.1.err +FOUND 1 /InnoDB: Missing MLOG_FILE_NAME or MLOG_FILE_DELETE before MLOG_CHECKPOINT for tablespace 42/ in mysqld.1.err +FOUND 1 /Downgrade after a crash is not supported\. The redo log was created with MariaDB 10\.3\.1/ in mysqld.1.err +# Empty 10.3 redo log +SELECT COUNT(*) FROM INFORMATION_SCHEMA.ENGINES +WHERE engine = 'innodb' +AND support IN ('YES', 'DEFAULT', 'ENABLED'); +COUNT(*) +1 +FOUND 1 /InnoDB: Downgrading redo log:/ in mysqld.1.err # Minimal MariaDB 10.1.21 encrypted redo log SELECT COUNT(*) `1` FROM INFORMATION_SCHEMA.ENGINES WHERE engine='innodb' AND support IN ('YES', 'DEFAULT', 'ENABLED'); diff --git a/mysql-test/suite/innodb/r/log_corruption.result b/mysql-test/suite/innodb/r/log_corruption.result index 3a20a11cd8f1c..49f6e6b85d260 100644 --- a/mysql-test/suite/innodb/r/log_corruption.result +++ b/mysql-test/suite/innodb/r/log_corruption.result @@ -17,6 +17,13 @@ WHERE engine = 'innodb' AND support IN ('YES', 'DEFAULT', 'ENABLED'); ENGINE SUPPORT COMMENT TRANSACTIONS XA SAVEPOINTS FOUND 1 /InnoDB: Upgrade after a crash is not supported. This redo log was created before MariaDB 10\.2\.2, and it appears corrupted/ in mysqld.1.err +# empty redo log from before MariaDB 10.2.2 +SELECT COUNT(*) FROM INFORMATION_SCHEMA.ENGINES +WHERE engine = 'innodb' +AND support IN ('YES', 'DEFAULT', 'ENABLED'); +COUNT(*) +1 +FOUND 1 /InnoDB: Upgrading redo log:/ in mysqld.1.err # redo log from "after" MariaDB 10.2.2, but with invalid header checksum SELECT * FROM INFORMATION_SCHEMA.ENGINES WHERE engine = 'innodb' @@ -28,7 +35,7 @@ SELECT * FROM INFORMATION_SCHEMA.ENGINES WHERE engine = 'innodb' AND support IN ('YES', 'DEFAULT', 'ENABLED'); ENGINE SUPPORT COMMENT TRANSACTIONS XA SAVEPOINTS -FOUND 1 /InnoDB: Unsupported redo log format. The redo log was created with malicious intentions, or perhaps\. Please follow the instructions at http://dev.mysql.com/doc/refman/5.7/en/upgrading-downgrading.html/ in mysqld.1.err +FOUND 1 /InnoDB: Unsupported redo log format. The redo log was created with malicious intentions, or perhaps\./ in mysqld.1.err # valid header, but old-format checkpoint blocks SELECT * FROM INFORMATION_SCHEMA.ENGINES WHERE engine = 'innodb' @@ -86,12 +93,26 @@ AND support IN ('YES', 'DEFAULT', 'ENABLED'); ENGINE SUPPORT COMMENT TRANSACTIONS XA SAVEPOINTS FOUND 1 /InnoDB: MLOG_FILE_NAME incorrect:bigot/ in mysqld.1.err FOUND 1 /len 22; hex 38000000000012860cb7809781e800066269676f7400; asc 8 bigot ;/ in mysqld.1.err -# missing MLOG_FILE_NAME or MLOG_FILE_DELETE before MLOG_CHECKPOINT +# 10.2 missing MLOG_FILE_NAME or MLOG_FILE_DELETE before MLOG_CHECKPOINT +SELECT * FROM INFORMATION_SCHEMA.ENGINES +WHERE engine = 'innodb' +AND support IN ('YES', 'DEFAULT', 'ENABLED'); +ENGINE SUPPORT COMMENT TRANSACTIONS XA SAVEPOINTS +FOUND 1 /InnoDB: Missing MLOG_FILE_NAME or MLOG_FILE_DELETE before MLOG_CHECKPOINT for tablespace 42/ in mysqld.1.err +# 10.3 missing MLOG_FILE_NAME or MLOG_FILE_DELETE before MLOG_CHECKPOINT SELECT * FROM INFORMATION_SCHEMA.ENGINES WHERE engine = 'innodb' AND support IN ('YES', 'DEFAULT', 'ENABLED'); ENGINE SUPPORT COMMENT TRANSACTIONS XA SAVEPOINTS -NOT FOUND /InnoDB: Missing MLOG_FILE_NAME or MLOG_FILE_DELETE before MLOG_CHECKPOINT for tablespace 42$/ in mysqld.1.err +FOUND 1 /InnoDB: Missing MLOG_FILE_NAME or MLOG_FILE_DELETE before MLOG_CHECKPOINT for tablespace 42/ in mysqld.1.err +FOUND 1 /Downgrade after a crash is not supported\. The redo log was created with MariaDB 10\.3\.1/ in mysqld.1.err +# Empty 10.3 redo log +SELECT COUNT(*) FROM INFORMATION_SCHEMA.ENGINES +WHERE engine = 'innodb' +AND support IN ('YES', 'DEFAULT', 'ENABLED'); +COUNT(*) +1 +FOUND 1 /InnoDB: Downgrading redo log:/ in mysqld.1.err # Minimal MariaDB 10.1.21 encrypted redo log SELECT * FROM INFORMATION_SCHEMA.ENGINES WHERE engine = 'innodb' diff --git a/mysql-test/suite/innodb/t/log_corruption.test b/mysql-test/suite/innodb/t/log_corruption.test index 8013cc4583098..3db15bd43cc05 100644 --- a/mysql-test/suite/innodb/t/log_corruption.test +++ b/mysql-test/suite/innodb/t/log_corruption.test @@ -2,7 +2,7 @@ --source include/have_innodb_16k.inc --disable_query_log -call mtr.add_suppression("InnoDB: Upgrade after a crash is not supported"); +call mtr.add_suppression("InnoDB: (Up|Down)grade after a crash is not supported"); call mtr.add_suppression("InnoDB: Plugin initialization aborted"); call mtr.add_suppression("Plugin 'InnoDB' init function returned error"); call mtr.add_suppression("Plugin 'InnoDB' registration as a STORAGE ENGINE failed"); @@ -16,6 +16,7 @@ call mtr.add_suppression("InnoDB: Log scan aborted at LSN"); call mtr.add_suppression("InnoDB: Missing MLOG_FILE_NAME or MLOG_FILE_DELETE before MLOG_CHECKPOINT for tablespace 42\\r?$"); call mtr.add_suppression("InnoDB: Obtaining redo log encryption key version 1 failed"); call mtr.add_suppression("InnoDB: Decrypting checkpoint failed"); +call mtr.add_suppression("InnoDB: Are you sure you are using the right ib_logfiles to start up the database\\? Log sequence number in the ib_logfiles is 1213964,"); --enable_query_log let bugdir= $MYSQLTEST_VARDIR/tmp/log_corruption; @@ -140,6 +141,24 @@ eval $check_no_innodb; let SEARCH_PATTERN=InnoDB: Upgrade after a crash is not supported. This redo log was created before MariaDB 10\\.2\\.2, and it appears corrupted; --source include/search_pattern_in_file.inc +--echo # empty redo log from before MariaDB 10.2.2 +perl; +die unless open OUT, "+<", "$ENV{bugdir}/ib_logfile0"; +binmode OUT; +die unless seek(OUT, 0x800, 0); +print OUT pack("NnnNx[496]N", 0x80000944, 12, 12, 0, 0xb2a); +close OUT or die; +EOF +--let $restart_parameters= $dirs --innodb-force-recovery=5 --innodb-log-file-size=1m +--source include/start_mysqld.inc +SELECT COUNT(*) FROM INFORMATION_SCHEMA.ENGINES +WHERE engine = 'innodb' +AND support IN ('YES', 'DEFAULT', 'ENABLED'); +--source include/shutdown_mysqld.inc +--let SEARCH_PATTERN= InnoDB: Upgrading redo log: +--source include/search_pattern_in_file.inc +--let $restart_parameters= $dirs + --echo # redo log from "after" MariaDB 10.2.2, but with invalid header checksum perl; die unless open OUT, "+<", "$ENV{bugdir}/ib_logfile0"; @@ -165,7 +184,7 @@ EOF --source include/start_mysqld.inc eval $check_no_innodb; --source include/shutdown_mysqld.inc -let SEARCH_PATTERN=InnoDB: Unsupported redo log format. The redo log was created with malicious intentions, or perhaps\. Please follow the instructions at http://dev.mysql.com/doc/refman/5.7/en/upgrading-downgrading.html; +let SEARCH_PATTERN=InnoDB: Unsupported redo log format. The redo log was created with malicious intentions, or perhaps\.; --source include/search_pattern_in_file.inc --echo # valid header, but old-format checkpoint blocks @@ -321,7 +340,7 @@ let SEARCH_PATTERN=InnoDB: MLOG_FILE_NAME incorrect:bigot; --let SEARCH_PATTERN= len 22; hex 38000000000012860cb7809781e800066269676f7400; asc 8 bigot ; --source include/search_pattern_in_file.inc ---echo # missing MLOG_FILE_NAME or MLOG_FILE_DELETE before MLOG_CHECKPOINT +--echo # 10.2 missing MLOG_FILE_NAME or MLOG_FILE_DELETE before MLOG_CHECKPOINT perl; die unless open OUT, "+<", "$ENV{bugdir}/ib_logfile0"; binmode OUT; @@ -349,7 +368,42 @@ EOF --source include/start_mysqld.inc eval $check_no_innodb; --source include/shutdown_mysqld.inc ---let SEARCH_PATTERN= InnoDB: Missing MLOG_FILE_NAME or MLOG_FILE_DELETE before MLOG_CHECKPOINT for tablespace 42\$ +--let SEARCH_PATTERN= InnoDB: Missing MLOG_FILE_NAME or MLOG_FILE_DELETE before MLOG_CHECKPOINT for tablespace 42 +--source include/search_pattern_in_file.inc + +--echo # 10.3 missing MLOG_FILE_NAME or MLOG_FILE_DELETE before MLOG_CHECKPOINT +perl; +die unless open OUT, "+<", "$ENV{bugdir}/ib_logfile0"; +binmode OUT; +print OUT pack("Nx[5]nx[5]", 103, 0x1286), "MariaDB 10.3.1"; +print OUT pack("x[478]N", 0x85021a0f); +close OUT or die; +EOF + +--source include/start_mysqld.inc +eval $check_no_innodb; +--source include/shutdown_mysqld.inc +--let SEARCH_PATTERN= InnoDB: Missing MLOG_FILE_NAME or MLOG_FILE_DELETE before MLOG_CHECKPOINT for tablespace 42 +--source include/search_pattern_in_file.inc +--let SEARCH_PATTERN= Downgrade after a crash is not supported\. The redo log was created with MariaDB 10\.3\.1 +--source include/search_pattern_in_file.inc + +--echo # Empty 10.3 redo log +perl; +die unless open OUT, "+<", "$ENV{bugdir}/ib_logfile0"; +binmode OUT; +die unless seek(OUT, 0x800, 0); +print OUT pack("NnnNx[496]N", 0x80000944, 12, 12, 1, 0x46c8a2a2); +close OUT or die; +EOF + +--let $restart_parameters= $dirs --innodb-force-recovery=5 --innodb-log-file-size=1m +--source include/start_mysqld.inc +SELECT COUNT(*) FROM INFORMATION_SCHEMA.ENGINES +WHERE engine = 'innodb' +AND support IN ('YES', 'DEFAULT', 'ENABLED'); +--source include/shutdown_mysqld.inc +--let SEARCH_PATTERN= InnoDB: Downgrading redo log: --source include/search_pattern_in_file.inc --echo # Minimal MariaDB 10.1.21 encrypted redo log diff --git a/storage/innobase/include/log0log.h b/storage/innobase/include/log0log.h index 543302c52f0b3..91ca389df83fb 100644 --- a/storage/innobase/include/log0log.h +++ b/storage/innobase/include/log0log.h @@ -2,7 +2,7 @@ Copyright (c) 1995, 2017, Oracle and/or its affiliates. All rights reserved. Copyright (c) 2009, Google Inc. -Copyright (c) 2017, MariaDB Corporation. +Copyright (c) 2017, 2018, MariaDB Corporation. Portions of this file contain modifications contributed and copyrighted by Google, Inc. Those modifications are gratefully acknowledged and are described @@ -513,6 +513,8 @@ or the MySQL version that created the redo log file. */ /** The redo log format identifier corresponding to the current format version. Stored in LOG_HEADER_FORMAT. */ #define LOG_HEADER_FORMAT_CURRENT 1 +/** The MariaDB 10.3.2 log format */ +#define LOG_HEADER_FORMAT_10_3 103 /** Encrypted MariaDB redo log */ #define LOG_HEADER_FORMAT_ENCRYPTED (1U<<31) diff --git a/storage/innobase/log/log0recv.cc b/storage/innobase/log/log0recv.cc index 6bb2806fb7055..5ee6004fe55bf 100644 --- a/storage/innobase/log/log0recv.cc +++ b/storage/innobase/log/log0recv.cc @@ -2,7 +2,7 @@ Copyright (c) 1997, 2017, Oracle and/or its affiliates. All Rights Reserved. Copyright (c) 2012, Facebook Inc. -Copyright (c) 2013, 2017, MariaDB Corporation. +Copyright (c) 2013, 2018, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -907,6 +907,58 @@ recv_log_format_0_recover(lsn_t lsn) return(DB_SUCCESS); } +/** Determine if a redo log from MariaDB 10.3 is clean. +@return error code +@retval DB_SUCCESS if the redo log is clean +@retval DB_CORRUPTION if the redo log is corrupted +@retval DB_ERROR if the redo log is not empty */ +static +dberr_t +recv_log_recover_10_3() +{ + log_group_t* group = &log_sys->log; + const lsn_t lsn = group->lsn; + const lsn_t source_offset = log_group_calc_lsn_offset(lsn, group); + const ulint page_no + = (ulint) (source_offset / univ_page_size.physical()); + byte* buf = log_sys->buf; + + fil_io(IORequestLogRead, true, + page_id_t(SRV_LOG_SPACE_FIRST_ID, page_no), + univ_page_size, + (ulint) ((source_offset & ~(OS_FILE_LOG_BLOCK_SIZE - 1)) + % univ_page_size.physical()), + OS_FILE_LOG_BLOCK_SIZE, buf, NULL); + + if (log_block_calc_checksum(buf) != log_block_get_checksum(buf)) { + return(DB_CORRUPTION); + } + + if (group->is_encrypted()) { + log_crypt(buf, lsn, OS_FILE_LOG_BLOCK_SIZE, true); + } + + /* On a clean shutdown, the redo log will be logically empty + after the checkpoint lsn. */ + + if (log_block_get_data_len(buf) + != (source_offset & (OS_FILE_LOG_BLOCK_SIZE - 1))) { + return(DB_ERROR); + } + + /* Mark the redo log for downgrading. */ + srv_log_file_size = 0; + recv_sys->parse_start_lsn = recv_sys->recovered_lsn + = recv_sys->scanned_lsn + = recv_sys->mlog_checkpoint_lsn = lsn; + log_sys->last_checkpoint_lsn = log_sys->next_checkpoint_lsn + = log_sys->lsn = log_sys->write_lsn + = log_sys->current_flush_lsn = log_sys->flushed_to_disk_lsn + = lsn; + log_sys->next_checkpoint_no = 0; + return(DB_SUCCESS); +} + /** Find the latest checkpoint in the log header. @param[out] max_field LOG_CHECKPOINT_1 or LOG_CHECKPOINT_2 @return error code or DB_SUCCESS */ @@ -938,18 +990,24 @@ recv_find_max_checkpoint(ulint* max_field) return(DB_CORRUPTION); } + char creator[LOG_HEADER_CREATOR_END - LOG_HEADER_CREATOR + 1]; + + memcpy(creator, buf + LOG_HEADER_CREATOR, sizeof creator); + /* Ensure that the string is NUL-terminated. */ + creator[LOG_HEADER_CREATOR_END - LOG_HEADER_CREATOR] = 0; + switch (group->format) { case 0: return(recv_find_max_checkpoint_0(&group, max_field)); case LOG_HEADER_FORMAT_CURRENT: case LOG_HEADER_FORMAT_CURRENT | LOG_HEADER_FORMAT_ENCRYPTED: + case LOG_HEADER_FORMAT_10_3: + case LOG_HEADER_FORMAT_10_3 | LOG_HEADER_FORMAT_ENCRYPTED: break; default: - /* Ensure that the string is NUL-terminated. */ - buf[LOG_HEADER_CREATOR_END] = 0; ib::error() << "Unsupported redo log format." " The redo log was created" - " with " << buf + LOG_HEADER_CREATOR << + " with " << creator << ". Please follow the instructions at " REFMAN "upgrading-downgrading.html"; /* Do not issue a message about a possibility @@ -1018,6 +1076,20 @@ recv_find_max_checkpoint(ulint* max_field) return(DB_ERROR); } + switch (group->format) { + case LOG_HEADER_FORMAT_10_3: + case LOG_HEADER_FORMAT_10_3 | LOG_HEADER_FORMAT_ENCRYPTED: + dberr_t err = recv_log_recover_10_3(); + if (err != DB_SUCCESS) { + ib::error() + << "Downgrade after a crash is not supported." + " The redo log was created with " << creator + << (err == DB_ERROR + ? "." : ", and it appears corrupted."); + } + return(err); + } + return(DB_SUCCESS); } diff --git a/storage/innobase/srv/srv0start.cc b/storage/innobase/srv/srv0start.cc index 886d47e44c0a5..7bfd1715e9b7c 100644 --- a/storage/innobase/srv/srv0start.cc +++ b/storage/innobase/srv/srv0start.cc @@ -3,7 +3,7 @@ Copyright (c) 1996, 2017, Oracle and/or its affiliates. All rights reserved. Copyright (c) 2008, Google Inc. Copyright (c) 2009, Percona Inc. -Copyright (c) 2013, 2017, MariaDB Corporation. +Copyright (c) 2013, 2018, MariaDB Corporation. Portions of this file contain modifications contributed and copyrighted by Google, Inc. Those modifications are gratefully acknowledged and are described @@ -1413,7 +1413,11 @@ srv_prepare_to_delete_redo_log_files( { ib::info info; if (srv_log_file_size == 0) { - info << "Upgrading redo log: "; + info << ((log_sys->log.format + & ~LOG_HEADER_FORMAT_ENCRYPTED) + != LOG_HEADER_FORMAT_10_3 + ? "Upgrading redo log: " + : "Downgrading redo log: "); } else if (n_files != srv_n_log_files || srv_log_file_size != srv_log_file_size_requested) { From dfde5ae912d402ec68fd17eda92112d84a07ffdd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Wed, 10 Jan 2018 13:53:44 +0200 Subject: [PATCH 021/106] MDEV-14130 InnoDB messages should not refer to the MySQL 5.7 manual Replace most occurrences of the REFMAN macro. For some pages there is no replacement yet. --- .../suite/innodb/r/group_commit_crash.result | 2 +- ...oup_commit_crash_no_optimize_thread.result | 2 +- mysql-test/suite/innodb/r/innodb-16k.result | 4 ++-- mysql-test/suite/innodb/r/innodb.result | 12 +++++----- .../suite/innodb/r/innodb_bug47167.result | 8 +++---- .../suite/innodb/r/innodb_file_format.result | 18 +++++++------- .../innodb_prefix_index_restart_server.result | 4 ++-- .../suite/innodb/t/innodb_bug14147491.test | 10 ++++---- mysql-test/suite/innodb_zip/r/16k.result | 4 ++-- mysql-test/suite/innodb_zip/r/4k.result | 4 ++-- mysql-test/suite/innodb_zip/r/8k.result | 4 ++-- .../suite/innodb_zip/r/create_options.result | 20 ++++++++-------- .../innodb_zip/r/index_large_prefix.result | 24 +++++++++---------- .../innodb_zip/r/index_large_prefix_4k.result | 10 ++++---- .../innodb_zip/r/index_large_prefix_8k.result | 20 ++++++++-------- .../suite/innodb_zip/r/innodb-zip.result | 24 +++++++++---------- .../suite/innodb_zip/r/wl6501_scale_1.result | 6 ++--- .../r/innodb_file_format_basic.result | 6 ++--- .../r/innodb_file_format_max_basic.result | 6 ++--- .../r/innodb_large_prefix_basic.result | 10 ++++---- storage/innobase/buf/buf0buf.cc | 6 +---- storage/innobase/gis/gis0sea.cc | 7 +++--- storage/innobase/handler/ha_innodb.cc | 15 ++++++------ storage/innobase/log/log0recv.cc | 4 ++-- storage/innobase/ut/ut0dbg.cc | 4 ++-- 25 files changed, 115 insertions(+), 119 deletions(-) diff --git a/mysql-test/suite/innodb/r/group_commit_crash.result b/mysql-test/suite/innodb/r/group_commit_crash.result index f07df897453ae..c9daf347ee170 100644 --- a/mysql-test/suite/innodb/r/group_commit_crash.result +++ b/mysql-test/suite/innodb/r/group_commit_crash.result @@ -124,4 +124,4 @@ DROP TABLE t1; DROP TABLE t2; DROP PROCEDURE setcrash; Warnings: -Warning 131 Using innodb_file_format_max is deprecated and the parameter may be removed in future releases. See http://dev.mysql.com/doc/refman/5.7/en/innodb-file-format.html +Warning 131 Using innodb_file_format_max is deprecated and the parameter may be removed in future releases. See https://mariadb.com/kb/en/library/xtradbinnodb-file-format/ diff --git a/mysql-test/suite/innodb/r/group_commit_crash_no_optimize_thread.result b/mysql-test/suite/innodb/r/group_commit_crash_no_optimize_thread.result index 2cd9f01d7ed40..a3937b5ae3a98 100644 --- a/mysql-test/suite/innodb/r/group_commit_crash_no_optimize_thread.result +++ b/mysql-test/suite/innodb/r/group_commit_crash_no_optimize_thread.result @@ -124,4 +124,4 @@ DROP TABLE t1; DROP TABLE t2; DROP PROCEDURE setcrash; Warnings: -Warning 131 Using innodb_file_format_max is deprecated and the parameter may be removed in future releases. See http://dev.mysql.com/doc/refman/5.7/en/innodb-file-format.html +Warning 131 Using innodb_file_format_max is deprecated and the parameter may be removed in future releases. See https://mariadb.com/kb/en/library/xtradbinnodb-file-format/ diff --git a/mysql-test/suite/innodb/r/innodb-16k.result b/mysql-test/suite/innodb/r/innodb-16k.result index e825e6baa2420..a34640310c337 100644 --- a/mysql-test/suite/innodb/r/innodb-16k.result +++ b/mysql-test/suite/innodb/r/innodb-16k.result @@ -1,7 +1,7 @@ call mtr.add_suppression("InnoDB: Cannot add field .* in table .* because after adding it, the row size is .* which is greater than maximum allowed size (.*) for a record on index leaf page."); SET GLOBAL innodb_large_prefix = OFF; Warnings: -Warning 131 Using innodb_large_prefix is deprecated and the parameter may be removed in future releases. See http://dev.mysql.com/doc/refman/5.7/en/innodb-file-format.html +Warning 131 Using innodb_large_prefix is deprecated and the parameter may be removed in future releases. See https://mariadb.com/kb/en/library/xtradbinnodb-file-format/ # Test 1) Show the page size from Information Schema SELECT variable_value FROM information_schema.global_status WHERE LOWER(variable_name) = 'innodb_page_size'; @@ -971,4 +971,4 @@ COL197 TEXT) row_format=compact,ENGINE=INNODB; ERROR 42000: Row size too large (> 8126). Changing some columns to TEXT or BLOB or using ROW_FORMAT=DYNAMIC or ROW_FORMAT=COMPRESSED may help. In current row format, BLOB prefix of 768 bytes is stored inline. Warnings: -Warning 131 Using innodb_large_prefix is deprecated and the parameter may be removed in future releases. See http://dev.mysql.com/doc/refman/5.7/en/innodb-file-format.html +Warning 131 Using innodb_large_prefix is deprecated and the parameter may be removed in future releases. See https://mariadb.com/kb/en/library/xtradbinnodb-file-format/ diff --git a/mysql-test/suite/innodb/r/innodb.result b/mysql-test/suite/innodb/r/innodb.result index 6d635fbcdc070..656484af89b01 100644 --- a/mysql-test/suite/innodb/r/innodb.result +++ b/mysql-test/suite/innodb/r/innodb.result @@ -2360,13 +2360,13 @@ drop table t1; SET sql_mode = 'NO_ENGINE_SUBSTITUTION'; SET GLOBAL innodb_large_prefix=OFF; Warnings: -Warning 131 Using innodb_large_prefix is deprecated and the parameter may be removed in future releases. See http://dev.mysql.com/doc/refman/5.7/en/innodb-file-format.html +Warning 131 Using innodb_large_prefix is deprecated and the parameter may be removed in future releases. See https://mariadb.com/kb/en/library/xtradbinnodb-file-format/ create table t1 (v varchar(65530), key(v)); Warnings: Warning 1071 Specified key was too long; max key length is 767 bytes SET GLOBAL innodb_large_prefix=default; Warnings: -Warning 131 Using innodb_large_prefix is deprecated and the parameter may be removed in future releases. See http://dev.mysql.com/doc/refman/5.7/en/innodb-file-format.html +Warning 131 Using innodb_large_prefix is deprecated and the parameter may be removed in future releases. See https://mariadb.com/kb/en/library/xtradbinnodb-file-format/ drop table t1; create table t1 (v varchar(65536)); Warnings: @@ -2534,7 +2534,7 @@ drop table t1, t2, t3, t4, t5, t6, t7, t8, t9; SET sql_mode = 'NO_ENGINE_SUBSTITUTION'; SET GLOBAL innodb_large_prefix=OFF; Warnings: -Warning 131 Using innodb_large_prefix is deprecated and the parameter may be removed in future releases. See http://dev.mysql.com/doc/refman/5.7/en/innodb-file-format.html +Warning 131 Using innodb_large_prefix is deprecated and the parameter may be removed in future releases. See https://mariadb.com/kb/en/library/xtradbinnodb-file-format/ create table t1 (col1 varchar(768), index(col1)) character set = latin1 engine = innodb; Warnings: @@ -2553,7 +2553,7 @@ Warnings: Note 1071 Specified key was too long; max key length is 767 bytes SET GLOBAL innodb_large_prefix=default; Warnings: -Warning 131 Using innodb_large_prefix is deprecated and the parameter may be removed in future releases. See http://dev.mysql.com/doc/refman/5.7/en/innodb-file-format.html +Warning 131 Using innodb_large_prefix is deprecated and the parameter may be removed in future releases. See https://mariadb.com/kb/en/library/xtradbinnodb-file-format/ show create table t1; Table Create Table t1 CREATE TABLE `t1` ( @@ -2563,7 +2563,7 @@ t1 CREATE TABLE `t1` ( drop table t1, t2, t3, t4; set global innodb_large_prefix=OFF; Warnings: -Warning 131 Using innodb_large_prefix is deprecated and the parameter may be removed in future releases. See http://dev.mysql.com/doc/refman/5.7/en/innodb-file-format.html +Warning 131 Using innodb_large_prefix is deprecated and the parameter may be removed in future releases. See https://mariadb.com/kb/en/library/xtradbinnodb-file-format/ create table t1 (col1 varchar(768) primary key) character set = latin1 engine = innodb; ERROR 42000: Specified key was too long; max key length is 767 bytes @@ -2579,7 +2579,7 @@ ERROR 42000: Specified key was too long; max key length is 767 bytes SET sql_mode = default; set global innodb_large_prefix=default; Warnings: -Warning 131 Using innodb_large_prefix is deprecated and the parameter may be removed in future releases. See http://dev.mysql.com/doc/refman/5.7/en/innodb-file-format.html +Warning 131 Using innodb_large_prefix is deprecated and the parameter may be removed in future releases. See https://mariadb.com/kb/en/library/xtradbinnodb-file-format/ CREATE TABLE t1 ( id INT PRIMARY KEY diff --git a/mysql-test/suite/innodb/r/innodb_bug47167.result b/mysql-test/suite/innodb/r/innodb_bug47167.result index b678046e30889..14977539b3b51 100644 --- a/mysql-test/suite/innodb/r/innodb_bug47167.result +++ b/mysql-test/suite/innodb/r/innodb_bug47167.result @@ -4,19 +4,19 @@ select @old_innodb_file_format_max; Barracuda set global innodb_file_format_max = Barracuda; Warnings: -Warning 131 Using innodb_file_format_max is deprecated and the parameter may be removed in future releases. See http://dev.mysql.com/doc/refman/5.7/en/innodb-file-format.html +Warning 131 Using innodb_file_format_max is deprecated and the parameter may be removed in future releases. See https://mariadb.com/kb/en/library/xtradbinnodb-file-format/ select @@innodb_file_format_max; @@innodb_file_format_max Barracuda set global innodb_file_format_max = DEFAULT; Warnings: -Warning 131 Using innodb_file_format_max is deprecated and the parameter may be removed in future releases. See http://dev.mysql.com/doc/refman/5.7/en/innodb-file-format.html +Warning 131 Using innodb_file_format_max is deprecated and the parameter may be removed in future releases. See https://mariadb.com/kb/en/library/xtradbinnodb-file-format/ select @@innodb_file_format_max; @@innodb_file_format_max Antelope set global innodb_file_format_max = @old_innodb_file_format_max; Warnings: -Warning 131 Using innodb_file_format_max is deprecated and the parameter may be removed in future releases. See http://dev.mysql.com/doc/refman/5.7/en/innodb-file-format.html +Warning 131 Using innodb_file_format_max is deprecated and the parameter may be removed in future releases. See https://mariadb.com/kb/en/library/xtradbinnodb-file-format/ select @@innodb_file_format_max; @@innodb_file_format_max Barracuda @@ -29,4 +29,4 @@ ERROR 42000: Variable 'innodb_file_format_max' can't be set to the value of 'ON' set global innodb_file_format_max = off; ERROR 42000: Variable 'innodb_file_format_max' can't be set to the value of 'off' Warnings: -Warning 131 Using innodb_file_format_max is deprecated and the parameter may be removed in future releases. See http://dev.mysql.com/doc/refman/5.7/en/innodb-file-format.html +Warning 131 Using innodb_file_format_max is deprecated and the parameter may be removed in future releases. See https://mariadb.com/kb/en/library/xtradbinnodb-file-format/ diff --git a/mysql-test/suite/innodb/r/innodb_file_format.result b/mysql-test/suite/innodb/r/innodb_file_format.result index e489911afb5b6..48f330f90fed4 100644 --- a/mysql-test/suite/innodb/r/innodb_file_format.result +++ b/mysql-test/suite/innodb/r/innodb_file_format.result @@ -9,10 +9,10 @@ select @@innodb_file_format_max; Barracuda set global innodb_file_format=antelope; Warnings: -Warning 131 Using innodb_file_format is deprecated and the parameter may be removed in future releases. See http://dev.mysql.com/doc/refman/5.7/en/innodb-file-format.html +Warning 131 Using innodb_file_format is deprecated and the parameter may be removed in future releases. See https://mariadb.com/kb/en/library/xtradbinnodb-file-format/ set global innodb_file_format=barracuda; Warnings: -Warning 131 Using innodb_file_format is deprecated and the parameter may be removed in future releases. See http://dev.mysql.com/doc/refman/5.7/en/innodb-file-format.html +Warning 131 Using innodb_file_format is deprecated and the parameter may be removed in future releases. See https://mariadb.com/kb/en/library/xtradbinnodb-file-format/ set global innodb_file_format=cheetah; ERROR 42000: Variable 'innodb_file_format' can't be set to the value of 'cheetah' select @@innodb_file_format; @@ -20,7 +20,7 @@ select @@innodb_file_format; Barracuda set global innodb_file_format=default; Warnings: -Warning 131 Using innodb_file_format is deprecated and the parameter may be removed in future releases. See http://dev.mysql.com/doc/refman/5.7/en/innodb-file-format.html +Warning 131 Using innodb_file_format is deprecated and the parameter may be removed in future releases. See https://mariadb.com/kb/en/library/xtradbinnodb-file-format/ select @@innodb_file_format; @@innodb_file_format Barracuda @@ -33,10 +33,10 @@ select @@innodb_file_format; Barracuda set global innodb_file_format_max=antelope; Warnings: -Warning 131 Using innodb_file_format_max is deprecated and the parameter may be removed in future releases. See http://dev.mysql.com/doc/refman/5.7/en/innodb-file-format.html +Warning 131 Using innodb_file_format_max is deprecated and the parameter may be removed in future releases. See https://mariadb.com/kb/en/library/xtradbinnodb-file-format/ set global innodb_file_format_max=barracuda; Warnings: -Warning 131 Using innodb_file_format_max is deprecated and the parameter may be removed in future releases. See http://dev.mysql.com/doc/refman/5.7/en/innodb-file-format.html +Warning 131 Using innodb_file_format_max is deprecated and the parameter may be removed in future releases. See https://mariadb.com/kb/en/library/xtradbinnodb-file-format/ set global innodb_file_format_max=cheetah; ERROR 42000: Variable 'innodb_file_format_max' can't be set to the value of 'cheetah' select @@innodb_file_format_max; @@ -44,7 +44,7 @@ select @@innodb_file_format_max; Barracuda set global innodb_file_format_max=default; Warnings: -Warning 131 Using innodb_file_format_max is deprecated and the parameter may be removed in future releases. See http://dev.mysql.com/doc/refman/5.7/en/innodb-file-format.html +Warning 131 Using innodb_file_format_max is deprecated and the parameter may be removed in future releases. See https://mariadb.com/kb/en/library/xtradbinnodb-file-format/ select @@innodb_file_format_max; @@innodb_file_format_max Antelope @@ -57,10 +57,10 @@ select @@innodb_file_format_max; Antelope set global innodb_file_format_max=antelope; Warnings: -Warning 131 Using innodb_file_format_max is deprecated and the parameter may be removed in future releases. See http://dev.mysql.com/doc/refman/5.7/en/innodb-file-format.html +Warning 131 Using innodb_file_format_max is deprecated and the parameter may be removed in future releases. See https://mariadb.com/kb/en/library/xtradbinnodb-file-format/ set global innodb_file_format_check=off; ERROR HY000: Variable 'innodb_file_format_check' is a read only variable Warnings: -Warning 131 Using innodb_file_format is deprecated and the parameter may be removed in future releases. See http://dev.mysql.com/doc/refman/5.7/en/innodb-file-format.html +Warning 131 Using innodb_file_format is deprecated and the parameter may be removed in future releases. See https://mariadb.com/kb/en/library/xtradbinnodb-file-format/ Warnings: -Warning 131 Using innodb_file_format_max is deprecated and the parameter may be removed in future releases. See http://dev.mysql.com/doc/refman/5.7/en/innodb-file-format.html +Warning 131 Using innodb_file_format_max is deprecated and the parameter may be removed in future releases. See https://mariadb.com/kb/en/library/xtradbinnodb-file-format/ diff --git a/mysql-test/suite/innodb/r/innodb_prefix_index_restart_server.result b/mysql-test/suite/innodb/r/innodb_prefix_index_restart_server.result index a3ac78aadceec..6fe101c58d75a 100644 --- a/mysql-test/suite/innodb/r/innodb_prefix_index_restart_server.result +++ b/mysql-test/suite/innodb/r/innodb_prefix_index_restart_server.result @@ -1,6 +1,6 @@ set global innodb_large_prefix=1; Warnings: -Warning 131 Using innodb_large_prefix is deprecated and the parameter may be removed in future releases. See http://dev.mysql.com/doc/refman/5.7/en/innodb-file-format.html +Warning 131 Using innodb_large_prefix is deprecated and the parameter may be removed in future releases. See https://mariadb.com/kb/en/library/xtradbinnodb-file-format/ CREATE TABLE worklog5743 ( col_1_text TEXT(4000) , col_2_text TEXT(4000) , PRIMARY KEY (col_1_text(3072)) @@ -89,4 +89,4 @@ col_1_text = REPEAT("a", 3500) col_2_text = REPEAT("o", 3500) DROP TABLE worklog5743; SET GLOBAL innodb_large_prefix=1; Warnings: -Warning 131 Using innodb_large_prefix is deprecated and the parameter may be removed in future releases. See http://dev.mysql.com/doc/refman/5.7/en/innodb-file-format.html +Warning 131 Using innodb_large_prefix is deprecated and the parameter may be removed in future releases. See https://mariadb.com/kb/en/library/xtradbinnodb-file-format/ diff --git a/mysql-test/suite/innodb/t/innodb_bug14147491.test b/mysql-test/suite/innodb/t/innodb_bug14147491.test index 09f9d53eaff4f..d8f0cc6d4b602 100644 --- a/mysql-test/suite/innodb/t/innodb_bug14147491.test +++ b/mysql-test/suite/innodb/t/innodb_bug14147491.test @@ -8,12 +8,10 @@ -- source include/not_encrypted.inc --disable_query_log -call mtr.add_suppression("InnoDB: Table `test`.`t1` is corrupted. Please drop the table and recreate."); -call mtr.add_suppression("InnoDB: Cannot open table test/t1 from the internal data dictionary of InnoDB though the .frm file for the table exists. Please refer to http://dev.mysql.com/doc/refman/5.7/en/innodb-troubleshooting.html for how to resolve the issue."); -call mtr.add_suppression("InnoDB: Database page corruption on disk or a failed file read of tablespace test/t1 page \[page id: space=[0-9]+, page number=[0-9]+\]. You may have to recover from a backup."); -call mtr.add_suppression("InnoDB: We detected index corruption in an InnoDB type table.*"); -call mtr.add_suppression("mysqld: Index for table 't1' is corrupt; try to repair it"); -call mtr.add_suppression("mysqld.exe: Index for table 't1' is corrupt; try to repair it"); +call mtr.add_suppression("InnoDB: Table `test`\\.`t1` is corrupted\\. Please drop the table and recreate\\."); +call mtr.add_suppression("InnoDB: Database page corruption on disk or a failed file read of tablespace test/t1 page"); +call mtr.add_suppression("InnoDB: We detected index corruption in an InnoDB type table"); +call mtr.add_suppression("Index for table 't1' is corrupt; try to repair it"); --enable_query_log --echo # Create and populate the table to be corrupted diff --git a/mysql-test/suite/innodb_zip/r/16k.result b/mysql-test/suite/innodb_zip/r/16k.result index 0af34c0a738c6..7010cac182d9d 100644 --- a/mysql-test/suite/innodb_zip/r/16k.result +++ b/mysql-test/suite/innodb_zip/r/16k.result @@ -284,7 +284,7 @@ Warning 1030 Got error 140 "Wrong create options" from storage engine InnoDB SET GLOBAL innodb_file_per_table = ON; SET GLOBAL innodb_file_format = `Antelope`; Warnings: -Warning 131 Using innodb_file_format is deprecated and the parameter may be removed in future releases. See http://dev.mysql.com/doc/refman/5.7/en/innodb-file-format.html +Warning 131 Using innodb_file_format is deprecated and the parameter may be removed in future releases. See https://mariadb.com/kb/en/library/xtradbinnodb-file-format/ CREATE TABLE t4 (id int PRIMARY KEY) ENGINE=innodb KEY_BLOCK_SIZE=8; Got one of the listed errors SHOW WARNINGS; @@ -301,7 +301,7 @@ Error 1005 Can't create table `test`.`t5` (errno: 140 "Wrong create options") Warning 1030 Got error 140 "Wrong create options" from storage engine InnoDB SET GLOBAL innodb_file_format = `Barracuda`; Warnings: -Warning 131 Using innodb_file_format is deprecated and the parameter may be removed in future releases. See http://dev.mysql.com/doc/refman/5.7/en/innodb-file-format.html +Warning 131 Using innodb_file_format is deprecated and the parameter may be removed in future releases. See https://mariadb.com/kb/en/library/xtradbinnodb-file-format/ # Test 7) This series of tests were moved from innodb-index to here # because the second alter table t1 assumes a 16k page size. # Moving the test allows the rest of innodb-index to be run on all diff --git a/mysql-test/suite/innodb_zip/r/4k.result b/mysql-test/suite/innodb_zip/r/4k.result index 5b7a4955a7dfe..af67087f8771d 100644 --- a/mysql-test/suite/innodb_zip/r/4k.result +++ b/mysql-test/suite/innodb_zip/r/4k.result @@ -247,7 +247,7 @@ Warning 1030 Got error 140 "Wrong create options" from storage engine InnoDB SET GLOBAL innodb_file_per_table = ON; SET GLOBAL innodb_file_format = `Antelope`; Warnings: -Warning 131 Using innodb_file_format is deprecated and the parameter may be removed in future releases. See http://dev.mysql.com/doc/refman/5.7/en/innodb-file-format.html +Warning 131 Using innodb_file_format is deprecated and the parameter may be removed in future releases. See https://mariadb.com/kb/en/library/xtradbinnodb-file-format/ CREATE TABLE t4 (id int PRIMARY KEY) ENGINE=innodb KEY_BLOCK_SIZE=8; ERROR HY000: Can't create table `test`.`t4` (errno: 140 "Wrong create options") SHOW WARNINGS; @@ -266,7 +266,7 @@ Error 1005 Can't create table `test`.`t5` (errno: 140 "Wrong create options") Warning 1030 Got error 140 "Wrong create options" from storage engine InnoDB SET GLOBAL innodb_file_format = `Barracuda`; Warnings: -Warning 131 Using innodb_file_format is deprecated and the parameter may be removed in future releases. See http://dev.mysql.com/doc/refman/5.7/en/innodb-file-format.html +Warning 131 Using innodb_file_format is deprecated and the parameter may be removed in future releases. See https://mariadb.com/kb/en/library/xtradbinnodb-file-format/ # Test 7) Not included here; 16k only # Test 8) Test creating a table that could lead to undo log overflow. CREATE TABLE t1(a blob,b blob,c blob,d blob,e blob,f blob,g blob, diff --git a/mysql-test/suite/innodb_zip/r/8k.result b/mysql-test/suite/innodb_zip/r/8k.result index 2a0245b78599d..1b65daeaeacef 100644 --- a/mysql-test/suite/innodb_zip/r/8k.result +++ b/mysql-test/suite/innodb_zip/r/8k.result @@ -259,7 +259,7 @@ Warning 1030 Got error 140 "Wrong create options" from storage engine InnoDB SET GLOBAL innodb_file_per_table = ON; SET GLOBAL innodb_file_format = `Antelope`; Warnings: -Warning 131 Using innodb_file_format is deprecated and the parameter may be removed in future releases. See http://dev.mysql.com/doc/refman/5.7/en/innodb-file-format.html +Warning 131 Using innodb_file_format is deprecated and the parameter may be removed in future releases. See https://mariadb.com/kb/en/library/xtradbinnodb-file-format/ CREATE TABLE t4 (id int PRIMARY KEY) ENGINE=innodb KEY_BLOCK_SIZE=8; ERROR HY000: Can't create table `test`.`t4` (errno: 140 "Wrong create options") SHOW WARNINGS; @@ -277,7 +277,7 @@ Error 1005 Can't create table `test`.`t5` (errno: 140 "Wrong create options") Warning 1030 Got error 140 "Wrong create options" from storage engine InnoDB SET GLOBAL innodb_file_format = `Barracuda`; Warnings: -Warning 131 Using innodb_file_format is deprecated and the parameter may be removed in future releases. See http://dev.mysql.com/doc/refman/5.7/en/innodb-file-format.html +Warning 131 Using innodb_file_format is deprecated and the parameter may be removed in future releases. See https://mariadb.com/kb/en/library/xtradbinnodb-file-format/ # Test 7) Not included here; 16k only # Test 8) Test creating a table that could lead to undo log overflow. CREATE TABLE t1(a blob,b blob,c blob,d blob,e blob,f blob,g blob, diff --git a/mysql-test/suite/innodb_zip/r/create_options.result b/mysql-test/suite/innodb_zip/r/create_options.result index 4e38bec08e3bf..9165e9ee5ae20 100644 --- a/mysql-test/suite/innodb_zip/r/create_options.result +++ b/mysql-test/suite/innodb_zip/r/create_options.result @@ -1,7 +1,7 @@ SET default_storage_engine=InnoDB; SET GLOBAL innodb_file_format=`Barracuda`; Warnings: -Warning 131 Using innodb_file_format is deprecated and the parameter may be removed in future releases. See http://dev.mysql.com/doc/refman/5.7/en/innodb-file-format.html +Warning 131 Using innodb_file_format is deprecated and the parameter may be removed in future releases. See https://mariadb.com/kb/en/library/xtradbinnodb-file-format/ SET GLOBAL innodb_file_per_table=ON; SET SESSION innodb_strict_mode = ON; # Test 1) StrictMode=ON, CREATE and ALTER with each ROW_FORMAT & KEY_BLOCK_SIZE=0 @@ -264,7 +264,7 @@ Warning 1030 Got error 140 "Wrong create options" from storage engine InnoDB # and that they can be set to default values during strict mode. SET GLOBAL innodb_file_format=Antelope; Warnings: -Warning 131 Using innodb_file_format is deprecated and the parameter may be removed in future releases. See http://dev.mysql.com/doc/refman/5.7/en/innodb-file-format.html +Warning 131 Using innodb_file_format is deprecated and the parameter may be removed in future releases. See https://mariadb.com/kb/en/library/xtradbinnodb-file-format/ CREATE TABLE t1 ( i INT ) KEY_BLOCK_SIZE=4; Got one of the listed errors SHOW WARNINGS; @@ -318,12 +318,12 @@ Warning 1478 InnoDB: ROW_FORMAT=DYNAMIC requires innodb_file_format > Antelope. Error 1478 Table storage engine 'InnoDB' does not support the create option 'ROW_FORMAT' SET GLOBAL innodb_file_format=Barracuda; Warnings: -Warning 131 Using innodb_file_format is deprecated and the parameter may be removed in future releases. See http://dev.mysql.com/doc/refman/5.7/en/innodb-file-format.html +Warning 131 Using innodb_file_format is deprecated and the parameter may be removed in future releases. See https://mariadb.com/kb/en/library/xtradbinnodb-file-format/ DROP TABLE t1; CREATE TABLE t1 ( i INT ) ROW_FORMAT=COMPRESSED KEY_BLOCK_SIZE=4; SET GLOBAL innodb_file_format=Antelope; Warnings: -Warning 131 Using innodb_file_format is deprecated and the parameter may be removed in future releases. See http://dev.mysql.com/doc/refman/5.7/en/innodb-file-format.html +Warning 131 Using innodb_file_format is deprecated and the parameter may be removed in future releases. See https://mariadb.com/kb/en/library/xtradbinnodb-file-format/ ALTER TABLE t1 ADD COLUMN f1 INT; ERROR HY000: Table storage engine 'InnoDB' does not support the create option 'ROW_FORMAT' SHOW WARNINGS; @@ -344,7 +344,7 @@ SHOW WARNINGS; Level Code Message SET GLOBAL innodb_file_format=Barracuda; Warnings: -Warning 131 Using innodb_file_format is deprecated and the parameter may be removed in future releases. See http://dev.mysql.com/doc/refman/5.7/en/innodb-file-format.html +Warning 131 Using innodb_file_format is deprecated and the parameter may be removed in future releases. See https://mariadb.com/kb/en/library/xtradbinnodb-file-format/ # Test 8) StrictMode=ON, Make sure ROW_FORMAT=COMPRESSED # and a valid non-zero KEY_BLOCK_SIZE are rejected with # innodb_file_per_table=OFF and that they can be set to default @@ -754,7 +754,7 @@ TABLE_NAME ROW_FORMAT CREATE_OPTIONS t1 Compressed row_format=COMPRESSED key_block_size=1 SET GLOBAL innodb_file_format=Antelope; Warnings: -Warning 131 Using innodb_file_format is deprecated and the parameter may be removed in future releases. See http://dev.mysql.com/doc/refman/5.7/en/innodb-file-format.html +Warning 131 Using innodb_file_format is deprecated and the parameter may be removed in future releases. See https://mariadb.com/kb/en/library/xtradbinnodb-file-format/ ALTER TABLE t1 ADD COLUMN f1 INT; Warnings: Warning 1478 InnoDB: KEY_BLOCK_SIZE requires innodb_file_format > Antelope. @@ -772,7 +772,7 @@ TABLE_NAME ROW_FORMAT CREATE_OPTIONS t1 Dynamic row_format=COMPRESSED key_block_size=1 SET GLOBAL innodb_file_format=Barracuda; Warnings: -Warning 131 Using innodb_file_format is deprecated and the parameter may be removed in future releases. See http://dev.mysql.com/doc/refman/5.7/en/innodb-file-format.html +Warning 131 Using innodb_file_format is deprecated and the parameter may be removed in future releases. See https://mariadb.com/kb/en/library/xtradbinnodb-file-format/ ALTER TABLE t1 ADD COLUMN f2 INT; SHOW WARNINGS; Level Code Message @@ -788,7 +788,7 @@ TABLE_NAME ROW_FORMAT CREATE_OPTIONS t1 Dynamic row_format=DYNAMIC SET GLOBAL innodb_file_format=Antelope; Warnings: -Warning 131 Using innodb_file_format is deprecated and the parameter may be removed in future releases. See http://dev.mysql.com/doc/refman/5.7/en/innodb-file-format.html +Warning 131 Using innodb_file_format is deprecated and the parameter may be removed in future releases. See https://mariadb.com/kb/en/library/xtradbinnodb-file-format/ ALTER TABLE t1 ADD COLUMN f1 INT; SHOW WARNINGS; Level Code Message @@ -797,7 +797,7 @@ TABLE_NAME ROW_FORMAT CREATE_OPTIONS t1 Dynamic row_format=DYNAMIC SET GLOBAL innodb_file_format=Barracuda; Warnings: -Warning 131 Using innodb_file_format is deprecated and the parameter may be removed in future releases. See http://dev.mysql.com/doc/refman/5.7/en/innodb-file-format.html +Warning 131 Using innodb_file_format is deprecated and the parameter may be removed in future releases. See https://mariadb.com/kb/en/library/xtradbinnodb-file-format/ ALTER TABLE t1 ADD COLUMN f2 INT; SHOW WARNINGS; Level Code Message @@ -852,4 +852,4 @@ t1 Dynamic row_format=DYNAMIC # Cleanup DROP TABLE t1; Warnings: -Warning 131 Using innodb_file_format is deprecated and the parameter may be removed in future releases. See http://dev.mysql.com/doc/refman/5.7/en/innodb-file-format.html +Warning 131 Using innodb_file_format is deprecated and the parameter may be removed in future releases. See https://mariadb.com/kb/en/library/xtradbinnodb-file-format/ diff --git a/mysql-test/suite/innodb_zip/r/index_large_prefix.result b/mysql-test/suite/innodb_zip/r/index_large_prefix.result index df177e8ea2a0f..fe03586546aeb 100644 --- a/mysql-test/suite/innodb_zip/r/index_large_prefix.result +++ b/mysql-test/suite/innodb_zip/r/index_large_prefix.result @@ -96,7 +96,7 @@ create table worklog5743_8(a1 int, a2 TEXT, a3 TEXT) KEY_BLOCK_SIZE=8; create table worklog5743_16(a1 int, a2 TEXT, a3 TEXT) KEY_BLOCK_SIZE=16; set global innodb_large_prefix=0; Warnings: -Warning 131 Using innodb_large_prefix is deprecated and the parameter may be removed in future releases. See http://dev.mysql.com/doc/refman/5.7/en/innodb-file-format.html +Warning 131 Using innodb_large_prefix is deprecated and the parameter may be removed in future releases. See https://mariadb.com/kb/en/library/xtradbinnodb-file-format/ create index idx1 on worklog5743_1(a2(4000)); Got one of the listed errors show warnings; @@ -105,7 +105,7 @@ Note 1071 Specified key was too long; max key length is 767 bytes Error 1118 Row size too large. The maximum row size for the used table type, not counting BLOBs, is 8126. This includes storage overhead, check the manual. You have to change some columns to TEXT or BLOBs set global innodb_large_prefix=1; Warnings: -Warning 131 Using innodb_large_prefix is deprecated and the parameter may be removed in future releases. See http://dev.mysql.com/doc/refman/5.7/en/innodb-file-format.html +Warning 131 Using innodb_large_prefix is deprecated and the parameter may be removed in future releases. See https://mariadb.com/kb/en/library/xtradbinnodb-file-format/ create index idx2 on worklog5743_1(a2(4000)); Got one of the listed errors show warnings; @@ -130,7 +130,7 @@ show warnings; Level Code Message set global innodb_large_prefix=0; Warnings: -Warning 131 Using innodb_large_prefix is deprecated and the parameter may be removed in future releases. See http://dev.mysql.com/doc/refman/5.7/en/innodb-file-format.html +Warning 131 Using innodb_large_prefix is deprecated and the parameter may be removed in future releases. See https://mariadb.com/kb/en/library/xtradbinnodb-file-format/ SET sql_mode= ''; create index idx1 on worklog5743_2(a2(4000)); Warnings: @@ -140,7 +140,7 @@ Level Code Message Note 1071 Specified key was too long; max key length is 767 bytes set global innodb_large_prefix=1; Warnings: -Warning 131 Using innodb_large_prefix is deprecated and the parameter may be removed in future releases. See http://dev.mysql.com/doc/refman/5.7/en/innodb-file-format.html +Warning 131 Using innodb_large_prefix is deprecated and the parameter may be removed in future releases. See https://mariadb.com/kb/en/library/xtradbinnodb-file-format/ create index idx2 on worklog5743_2(a2(4000)); ERROR 42000: Row size too large. The maximum row size for the used table type, not counting BLOBs, is 8126. This includes storage overhead, check the manual. You have to change some columns to TEXT or BLOBs show warnings; @@ -165,7 +165,7 @@ show warnings; Level Code Message set global innodb_large_prefix=0; Warnings: -Warning 131 Using innodb_large_prefix is deprecated and the parameter may be removed in future releases. See http://dev.mysql.com/doc/refman/5.7/en/innodb-file-format.html +Warning 131 Using innodb_large_prefix is deprecated and the parameter may be removed in future releases. See https://mariadb.com/kb/en/library/xtradbinnodb-file-format/ create index idx1 on worklog5743_4(a2(4000)); Warnings: Note 1071 Specified key was too long; max key length is 767 bytes @@ -174,7 +174,7 @@ Level Code Message Note 1071 Specified key was too long; max key length is 767 bytes set global innodb_large_prefix=1; Warnings: -Warning 131 Using innodb_large_prefix is deprecated and the parameter may be removed in future releases. See http://dev.mysql.com/doc/refman/5.7/en/innodb-file-format.html +Warning 131 Using innodb_large_prefix is deprecated and the parameter may be removed in future releases. See https://mariadb.com/kb/en/library/xtradbinnodb-file-format/ create index idx2 on worklog5743_4(a2(4000)); ERROR 42000: Row size too large. The maximum row size for the used table type, not counting BLOBs, is 8126. This includes storage overhead, check the manual. You have to change some columns to TEXT or BLOBs show warnings; @@ -199,7 +199,7 @@ show warnings; Level Code Message set global innodb_large_prefix=0; Warnings: -Warning 131 Using innodb_large_prefix is deprecated and the parameter may be removed in future releases. See http://dev.mysql.com/doc/refman/5.7/en/innodb-file-format.html +Warning 131 Using innodb_large_prefix is deprecated and the parameter may be removed in future releases. See https://mariadb.com/kb/en/library/xtradbinnodb-file-format/ create index idx1 on worklog5743_8(a2(1000)); Warnings: Note 1071 Specified key was too long; max key length is 767 bytes @@ -208,7 +208,7 @@ Level Code Message Note 1071 Specified key was too long; max key length is 767 bytes set global innodb_large_prefix=1; Warnings: -Warning 131 Using innodb_large_prefix is deprecated and the parameter may be removed in future releases. See http://dev.mysql.com/doc/refman/5.7/en/innodb-file-format.html +Warning 131 Using innodb_large_prefix is deprecated and the parameter may be removed in future releases. See https://mariadb.com/kb/en/library/xtradbinnodb-file-format/ create index idx2 on worklog5743_8(a2(3073)); Warnings: Note 1071 Specified key was too long; max key length is 3072 bytes @@ -239,7 +239,7 @@ show warnings; Level Code Message set global innodb_large_prefix=0; Warnings: -Warning 131 Using innodb_large_prefix is deprecated and the parameter may be removed in future releases. See http://dev.mysql.com/doc/refman/5.7/en/innodb-file-format.html +Warning 131 Using innodb_large_prefix is deprecated and the parameter may be removed in future releases. See https://mariadb.com/kb/en/library/xtradbinnodb-file-format/ create index idx1 on worklog5743_16(a2(1000)); Warnings: Note 1071 Specified key was too long; max key length is 767 bytes @@ -248,7 +248,7 @@ Level Code Message Note 1071 Specified key was too long; max key length is 767 bytes set global innodb_large_prefix=1; Warnings: -Warning 131 Using innodb_large_prefix is deprecated and the parameter may be removed in future releases. See http://dev.mysql.com/doc/refman/5.7/en/innodb-file-format.html +Warning 131 Using innodb_large_prefix is deprecated and the parameter may be removed in future releases. See https://mariadb.com/kb/en/library/xtradbinnodb-file-format/ create index idx2 on worklog5743_16(a2(3073)); Warnings: Note 1071 Specified key was too long; max key length is 3072 bytes @@ -285,7 +285,7 @@ insert into worklog5743_8 values(9, repeat("a", 10000), repeat("a", 10000)); insert into worklog5743_16 values(9, repeat("a", 10000), repeat("a", 10000)); set global innodb_large_prefix=0; Warnings: -Warning 131 Using innodb_large_prefix is deprecated and the parameter may be removed in future releases. See http://dev.mysql.com/doc/refman/5.7/en/innodb-file-format.html +Warning 131 Using innodb_large_prefix is deprecated and the parameter may be removed in future releases. See https://mariadb.com/kb/en/library/xtradbinnodb-file-format/ insert into worklog5743_1 values(2, repeat("b", 10000)); insert into worklog5743_2 values(2, repeat("b", 10000)); insert into worklog5743_4 values(2, repeat("b", 10000)); @@ -293,7 +293,7 @@ insert into worklog5743_8 values(2, repeat("b", 10000), repeat("b", 10000)); insert into worklog5743_16 values(2, repeat("b", 10000), repeat("b", 10000)); set global innodb_large_prefix=1; Warnings: -Warning 131 Using innodb_large_prefix is deprecated and the parameter may be removed in future releases. See http://dev.mysql.com/doc/refman/5.7/en/innodb-file-format.html +Warning 131 Using innodb_large_prefix is deprecated and the parameter may be removed in future releases. See https://mariadb.com/kb/en/library/xtradbinnodb-file-format/ select a1, left(a2, 20) from worklog5743_1; a1 left(a2, 20) 9 aaaaaaaaaaaaaaaaaaaa diff --git a/mysql-test/suite/innodb_zip/r/index_large_prefix_4k.result b/mysql-test/suite/innodb_zip/r/index_large_prefix_4k.result index 50d4854154b18..c2929bf7937dd 100644 --- a/mysql-test/suite/innodb_zip/r/index_large_prefix_4k.result +++ b/mysql-test/suite/innodb_zip/r/index_large_prefix_4k.result @@ -99,7 +99,7 @@ create table worklog5743_2(a1 int, a2 TEXT not null) KEY_BLOCK_SIZE=2; create table worklog5743_4(a1 int, a2 TEXT not null) KEY_BLOCK_SIZE=4; set global innodb_large_prefix=0; Warnings: -Warning 131 Using innodb_large_prefix is deprecated and the parameter may be removed in future releases. See http://dev.mysql.com/doc/refman/5.7/en/innodb-file-format.html +Warning 131 Using innodb_large_prefix is deprecated and the parameter may be removed in future releases. See https://mariadb.com/kb/en/library/xtradbinnodb-file-format/ create index idx1 on worklog5743_1(a2(4000)); ERROR 42000: Row size too large. The maximum row size for the used table type, not counting BLOBs, is 1982. This includes storage overhead, check the manual. You have to change some columns to TEXT or BLOBs show warnings; @@ -124,7 +124,7 @@ show warnings; Level Code Message set global innodb_large_prefix=1; Warnings: -Warning 131 Using innodb_large_prefix is deprecated and the parameter may be removed in future releases. See http://dev.mysql.com/doc/refman/5.7/en/innodb-file-format.html +Warning 131 Using innodb_large_prefix is deprecated and the parameter may be removed in future releases. See https://mariadb.com/kb/en/library/xtradbinnodb-file-format/ SET sql_mode= ''; create index idx1 on worklog5743_2(a2(4000)); Warnings: @@ -161,7 +161,7 @@ show warnings; Level Code Message set global innodb_large_prefix=0; Warnings: -Warning 131 Using innodb_large_prefix is deprecated and the parameter may be removed in future releases. See http://dev.mysql.com/doc/refman/5.7/en/innodb-file-format.html +Warning 131 Using innodb_large_prefix is deprecated and the parameter may be removed in future releases. See https://mariadb.com/kb/en/library/xtradbinnodb-file-format/ create index idx1 on worklog5743_4(a2(4000)); Warnings: Note 1071 Specified key was too long; max key length is 767 bytes @@ -201,13 +201,13 @@ insert into worklog5743_2 values(9, repeat("a", 10000)); insert into worklog5743_4 values(9, repeat("a", 10000)); set global innodb_large_prefix=0; Warnings: -Warning 131 Using innodb_large_prefix is deprecated and the parameter may be removed in future releases. See http://dev.mysql.com/doc/refman/5.7/en/innodb-file-format.html +Warning 131 Using innodb_large_prefix is deprecated and the parameter may be removed in future releases. See https://mariadb.com/kb/en/library/xtradbinnodb-file-format/ insert into worklog5743_1 values(2, repeat("b", 10000)); insert into worklog5743_2 values(2, repeat("b", 10000)); insert into worklog5743_4 values(2, repeat("b", 10000)); set global innodb_large_prefix=1; Warnings: -Warning 131 Using innodb_large_prefix is deprecated and the parameter may be removed in future releases. See http://dev.mysql.com/doc/refman/5.7/en/innodb-file-format.html +Warning 131 Using innodb_large_prefix is deprecated and the parameter may be removed in future releases. See https://mariadb.com/kb/en/library/xtradbinnodb-file-format/ select a1, left(a2, 20) from worklog5743_1; a1 left(a2, 20) 9 aaaaaaaaaaaaaaaaaaaa diff --git a/mysql-test/suite/innodb_zip/r/index_large_prefix_8k.result b/mysql-test/suite/innodb_zip/r/index_large_prefix_8k.result index e99bb0876a56b..60dafa2ac460c 100644 --- a/mysql-test/suite/innodb_zip/r/index_large_prefix_8k.result +++ b/mysql-test/suite/innodb_zip/r/index_large_prefix_8k.result @@ -100,7 +100,7 @@ create table worklog5743_4(a1 int, a2 TEXT not null) KEY_BLOCK_SIZE=4; create table worklog5743_8(a1 int, a2 TEXT, a3 TEXT) KEY_BLOCK_SIZE=8; set global innodb_large_prefix=0; Warnings: -Warning 131 Using innodb_large_prefix is deprecated and the parameter may be removed in future releases. See http://dev.mysql.com/doc/refman/5.7/en/innodb-file-format.html +Warning 131 Using innodb_large_prefix is deprecated and the parameter may be removed in future releases. See https://mariadb.com/kb/en/library/xtradbinnodb-file-format/ create index idx1 on worklog5743_1(a2(4000)); ERROR 42000: Row size too large. The maximum row size for the used table type, not counting BLOBs, is 4030. This includes storage overhead, check the manual. You have to change some columns to TEXT or BLOBs show warnings; @@ -109,7 +109,7 @@ Note 1071 Specified key was too long; max key length is 767 bytes Error 1118 Row size too large. The maximum row size for the used table type, not counting BLOBs, is 4030. This includes storage overhead, check the manual. You have to change some columns to TEXT or BLOBs set global innodb_large_prefix=1; Warnings: -Warning 131 Using innodb_large_prefix is deprecated and the parameter may be removed in future releases. See http://dev.mysql.com/doc/refman/5.7/en/innodb-file-format.html +Warning 131 Using innodb_large_prefix is deprecated and the parameter may be removed in future releases. See https://mariadb.com/kb/en/library/xtradbinnodb-file-format/ create index idx2 on worklog5743_1(a2(4000)); ERROR 42000: Row size too large. The maximum row size for the used table type, not counting BLOBs, is 4030. This includes storage overhead, check the manual. You have to change some columns to TEXT or BLOBs show warnings; @@ -134,7 +134,7 @@ show warnings; Level Code Message set global innodb_large_prefix=0; Warnings: -Warning 131 Using innodb_large_prefix is deprecated and the parameter may be removed in future releases. See http://dev.mysql.com/doc/refman/5.7/en/innodb-file-format.html +Warning 131 Using innodb_large_prefix is deprecated and the parameter may be removed in future releases. See https://mariadb.com/kb/en/library/xtradbinnodb-file-format/ SET sql_mode= ''; create index idx1 on worklog5743_2(a2(4000)); Warnings: @@ -144,7 +144,7 @@ Level Code Message Note 1071 Specified key was too long; max key length is 767 bytes set global innodb_large_prefix=1; Warnings: -Warning 131 Using innodb_large_prefix is deprecated and the parameter may be removed in future releases. See http://dev.mysql.com/doc/refman/5.7/en/innodb-file-format.html +Warning 131 Using innodb_large_prefix is deprecated and the parameter may be removed in future releases. See https://mariadb.com/kb/en/library/xtradbinnodb-file-format/ create index idx2 on worklog5743_2(a2(4000)); ERROR 42000: Row size too large. The maximum row size for the used table type, not counting BLOBs, is 4030. This includes storage overhead, check the manual. You have to change some columns to TEXT or BLOBs show warnings; @@ -169,7 +169,7 @@ show warnings; Level Code Message set global innodb_large_prefix=0; Warnings: -Warning 131 Using innodb_large_prefix is deprecated and the parameter may be removed in future releases. See http://dev.mysql.com/doc/refman/5.7/en/innodb-file-format.html +Warning 131 Using innodb_large_prefix is deprecated and the parameter may be removed in future releases. See https://mariadb.com/kb/en/library/xtradbinnodb-file-format/ create index idx1 on worklog5743_4(a2(4000)); Warnings: Note 1071 Specified key was too long; max key length is 767 bytes @@ -178,7 +178,7 @@ Level Code Message Note 1071 Specified key was too long; max key length is 767 bytes set global innodb_large_prefix=1; Warnings: -Warning 131 Using innodb_large_prefix is deprecated and the parameter may be removed in future releases. See http://dev.mysql.com/doc/refman/5.7/en/innodb-file-format.html +Warning 131 Using innodb_large_prefix is deprecated and the parameter may be removed in future releases. See https://mariadb.com/kb/en/library/xtradbinnodb-file-format/ create index idx3 on worklog5743_4(a2(1537)); Warnings: Note 1071 Specified key was too long; max key length is 1536 bytes @@ -201,7 +201,7 @@ show warnings; Level Code Message set global innodb_large_prefix=0; Warnings: -Warning 131 Using innodb_large_prefix is deprecated and the parameter may be removed in future releases. See http://dev.mysql.com/doc/refman/5.7/en/innodb-file-format.html +Warning 131 Using innodb_large_prefix is deprecated and the parameter may be removed in future releases. See https://mariadb.com/kb/en/library/xtradbinnodb-file-format/ create index idx1 on worklog5743_8(a2(1000)); Warnings: Note 1071 Specified key was too long; max key length is 767 bytes @@ -210,7 +210,7 @@ Level Code Message Note 1071 Specified key was too long; max key length is 767 bytes set global innodb_large_prefix=1; Warnings: -Warning 131 Using innodb_large_prefix is deprecated and the parameter may be removed in future releases. See http://dev.mysql.com/doc/refman/5.7/en/innodb-file-format.html +Warning 131 Using innodb_large_prefix is deprecated and the parameter may be removed in future releases. See https://mariadb.com/kb/en/library/xtradbinnodb-file-format/ create index idx2 on worklog5743_8(a2(3073)); Warnings: Note 1071 Specified key was too long; max key length is 1536 bytes @@ -238,14 +238,14 @@ insert into worklog5743_4 values(9, repeat("a", 10000)); insert into worklog5743_8 values(9, repeat("a", 10000), repeat("a", 10000)); set global innodb_large_prefix=0; Warnings: -Warning 131 Using innodb_large_prefix is deprecated and the parameter may be removed in future releases. See http://dev.mysql.com/doc/refman/5.7/en/innodb-file-format.html +Warning 131 Using innodb_large_prefix is deprecated and the parameter may be removed in future releases. See https://mariadb.com/kb/en/library/xtradbinnodb-file-format/ insert into worklog5743_1 values(2, repeat("b", 10000)); insert into worklog5743_2 values(2, repeat("b", 10000)); insert into worklog5743_4 values(2, repeat("b", 10000)); insert into worklog5743_8 values(2, repeat("b", 10000), repeat("b", 10000)); set global innodb_large_prefix=1; Warnings: -Warning 131 Using innodb_large_prefix is deprecated and the parameter may be removed in future releases. See http://dev.mysql.com/doc/refman/5.7/en/innodb-file-format.html +Warning 131 Using innodb_large_prefix is deprecated and the parameter may be removed in future releases. See https://mariadb.com/kb/en/library/xtradbinnodb-file-format/ select a1, left(a2, 20) from worklog5743_1; a1 left(a2, 20) 9 aaaaaaaaaaaaaaaaaaaa diff --git a/mysql-test/suite/innodb_zip/r/innodb-zip.result b/mysql-test/suite/innodb_zip/r/innodb-zip.result index c715f77b9ba28..419671a6735eb 100644 --- a/mysql-test/suite/innodb_zip/r/innodb-zip.result +++ b/mysql-test/suite/innodb_zip/r/innodb-zip.result @@ -9,7 +9,7 @@ set session innodb_strict_mode=0; set global innodb_file_per_table=off; set global innodb_file_format=`0`; Warnings: -Warning 131 Using innodb_file_format is deprecated and the parameter may be removed in future releases. See http://dev.mysql.com/doc/refman/5.7/en/innodb-file-format.html +Warning 131 Using innodb_file_format is deprecated and the parameter may be removed in future releases. See https://mariadb.com/kb/en/library/xtradbinnodb-file-format/ SET @@global.innodb_stats_on_metadata=ON; create table t0(a int primary key) engine=innodb row_format=compressed; Warnings: @@ -45,7 +45,7 @@ Warning 1478 InnoDB: KEY_BLOCK_SIZE requires innodb_file_format > Antelope. Warning 1478 InnoDB: ignoring KEY_BLOCK_SIZE=1. set global innodb_file_format=`1`; Warnings: -Warning 131 Using innodb_file_format is deprecated and the parameter may be removed in future releases. See http://dev.mysql.com/doc/refman/5.7/en/innodb-file-format.html +Warning 131 Using innodb_file_format is deprecated and the parameter may be removed in future releases. See https://mariadb.com/kb/en/library/xtradbinnodb-file-format/ create table t7(a int primary key) engine=innodb key_block_size=1 row_format=redundant; Warnings: @@ -161,13 +161,13 @@ update t1 set c3 = repeat('E', 20000) where c1 = 1; drop table t1; set global innodb_file_format=`0`; Warnings: -Warning 131 Using innodb_file_format is deprecated and the parameter may be removed in future releases. See http://dev.mysql.com/doc/refman/5.7/en/innodb-file-format.html +Warning 131 Using innodb_file_format is deprecated and the parameter may be removed in future releases. See https://mariadb.com/kb/en/library/xtradbinnodb-file-format/ select @@innodb_file_format; @@innodb_file_format Antelope set global innodb_file_format=`1`; Warnings: -Warning 131 Using innodb_file_format is deprecated and the parameter may be removed in future releases. See http://dev.mysql.com/doc/refman/5.7/en/innodb-file-format.html +Warning 131 Using innodb_file_format is deprecated and the parameter may be removed in future releases. See https://mariadb.com/kb/en/library/xtradbinnodb-file-format/ select @@innodb_file_format; @@innodb_file_format Barracuda @@ -177,10 +177,10 @@ set global innodb_file_format=`-1`; ERROR 42000: Variable 'innodb_file_format' can't be set to the value of '-1' set global innodb_file_format=`Antelope`; Warnings: -Warning 131 Using innodb_file_format is deprecated and the parameter may be removed in future releases. See http://dev.mysql.com/doc/refman/5.7/en/innodb-file-format.html +Warning 131 Using innodb_file_format is deprecated and the parameter may be removed in future releases. See https://mariadb.com/kb/en/library/xtradbinnodb-file-format/ set global innodb_file_format=`Barracuda`; Warnings: -Warning 131 Using innodb_file_format is deprecated and the parameter may be removed in future releases. See http://dev.mysql.com/doc/refman/5.7/en/innodb-file-format.html +Warning 131 Using innodb_file_format is deprecated and the parameter may be removed in future releases. See https://mariadb.com/kb/en/library/xtradbinnodb-file-format/ set global innodb_file_format=`Cheetah`; ERROR 42000: Variable 'innodb_file_format' can't be set to the value of 'Cheetah' set global innodb_file_format=`abc`; @@ -192,7 +192,7 @@ ERROR 42000: Variable 'innodb_file_format' can't be set to the value of '' set global innodb_file_per_table = on; set global innodb_file_format = `1`; Warnings: -Warning 131 Using innodb_file_format is deprecated and the parameter may be removed in future releases. See http://dev.mysql.com/doc/refman/5.7/en/innodb-file-format.html +Warning 131 Using innodb_file_format is deprecated and the parameter may be removed in future releases. See https://mariadb.com/kb/en/library/xtradbinnodb-file-format/ set innodb_strict_mode = off; create table t1 (id int primary key) engine = innodb key_block_size = 0; drop table t1; @@ -328,7 +328,7 @@ drop table t7, t8, t9; set global innodb_file_per_table = on; set global innodb_file_format = `0`; Warnings: -Warning 131 Using innodb_file_format is deprecated and the parameter may be removed in future releases. See http://dev.mysql.com/doc/refman/5.7/en/innodb-file-format.html +Warning 131 Using innodb_file_format is deprecated and the parameter may be removed in future releases. See https://mariadb.com/kb/en/library/xtradbinnodb-file-format/ create table t1 (id int primary key) engine = innodb key_block_size = 1; ERROR HY000: Can't create table `mysqltest_innodb_zip`.`t1` (errno: 140 "Wrong create options") show warnings; @@ -374,14 +374,14 @@ drop table t8, t9; set global innodb_file_per_table=1; set global innodb_file_format=Barracuda; Warnings: -Warning 131 Using innodb_file_format is deprecated and the parameter may be removed in future releases. See http://dev.mysql.com/doc/refman/5.7/en/innodb-file-format.html +Warning 131 Using innodb_file_format is deprecated and the parameter may be removed in future releases. See https://mariadb.com/kb/en/library/xtradbinnodb-file-format/ set global innodb_file_per_table=on; set global innodb_file_format=`Barracuda`; Warnings: -Warning 131 Using innodb_file_format is deprecated and the parameter may be removed in future releases. See http://dev.mysql.com/doc/refman/5.7/en/innodb-file-format.html +Warning 131 Using innodb_file_format is deprecated and the parameter may be removed in future releases. See https://mariadb.com/kb/en/library/xtradbinnodb-file-format/ set global innodb_file_format_max=`Antelope`; Warnings: -Warning 131 Using innodb_file_format_max is deprecated and the parameter may be removed in future releases. See http://dev.mysql.com/doc/refman/5.7/en/innodb-file-format.html +Warning 131 Using innodb_file_format_max is deprecated and the parameter may be removed in future releases. See https://mariadb.com/kb/en/library/xtradbinnodb-file-format/ create table normal_table ( c1 int ) engine = innodb; @@ -396,7 +396,7 @@ select @@innodb_file_format_max; Barracuda set global innodb_file_format_max=`Antelope`; Warnings: -Warning 131 Using innodb_file_format_max is deprecated and the parameter may be removed in future releases. See http://dev.mysql.com/doc/refman/5.7/en/innodb-file-format.html +Warning 131 Using innodb_file_format_max is deprecated and the parameter may be removed in future releases. See https://mariadb.com/kb/en/library/xtradbinnodb-file-format/ select @@innodb_file_format_max; @@innodb_file_format_max Antelope diff --git a/mysql-test/suite/innodb_zip/r/wl6501_scale_1.result b/mysql-test/suite/innodb_zip/r/wl6501_scale_1.result index 0064cf2a469cb..3a74b6ebc1171 100644 --- a/mysql-test/suite/innodb_zip/r/wl6501_scale_1.result +++ b/mysql-test/suite/innodb_zip/r/wl6501_scale_1.result @@ -111,7 +111,7 @@ drop procedure populate; drop procedure populate_small; set global innodb_file_format = Barracuda; Warnings: -Warning 131 Using innodb_file_format is deprecated and the parameter may be removed in future releases. See http://dev.mysql.com/doc/refman/5.7/en/innodb-file-format.html +Warning 131 Using innodb_file_format is deprecated and the parameter may be removed in future releases. See https://mariadb.com/kb/en/library/xtradbinnodb-file-format/ set global innodb_file_per_table = 1; set innodb_strict_mode=OFF; create procedure populate() @@ -224,7 +224,7 @@ drop procedure populate; drop procedure populate_small; set global innodb_file_format = Barracuda; Warnings: -Warning 131 Using innodb_file_format is deprecated and the parameter may be removed in future releases. See http://dev.mysql.com/doc/refman/5.7/en/innodb-file-format.html +Warning 131 Using innodb_file_format is deprecated and the parameter may be removed in future releases. See https://mariadb.com/kb/en/library/xtradbinnodb-file-format/ set global innodb_file_per_table = 1; set innodb_strict_mode=OFF; create procedure populate() @@ -341,5 +341,5 @@ drop procedure populate; drop procedure populate_small; set global innodb_file_format = Barracuda; Warnings: -Warning 131 Using innodb_file_format is deprecated and the parameter may be removed in future releases. See http://dev.mysql.com/doc/refman/5.7/en/innodb-file-format.html +Warning 131 Using innodb_file_format is deprecated and the parameter may be removed in future releases. See https://mariadb.com/kb/en/library/xtradbinnodb-file-format/ set global innodb_file_per_table = 1; diff --git a/mysql-test/suite/sys_vars/r/innodb_file_format_basic.result b/mysql-test/suite/sys_vars/r/innodb_file_format_basic.result index c330bbf5c1645..0b2cbb97d250e 100644 --- a/mysql-test/suite/sys_vars/r/innodb_file_format_basic.result +++ b/mysql-test/suite/sys_vars/r/innodb_file_format_basic.result @@ -25,7 +25,7 @@ VARIABLE_NAME VARIABLE_VALUE INNODB_FILE_FORMAT Barracuda set global innodb_file_format='Antelope'; Warnings: -Warning 131 Using innodb_file_format is deprecated and the parameter may be removed in future releases. See http://dev.mysql.com/doc/refman/5.7/en/innodb-file-format.html +Warning 131 Using innodb_file_format is deprecated and the parameter may be removed in future releases. See https://mariadb.com/kb/en/library/xtradbinnodb-file-format/ select @@global.innodb_file_format; @@global.innodb_file_format Antelope @@ -37,7 +37,7 @@ VARIABLE_NAME VARIABLE_VALUE INNODB_FILE_FORMAT Antelope set @@global.innodb_file_format='Barracuda'; Warnings: -Warning 131 Using innodb_file_format is deprecated and the parameter may be removed in future releases. See http://dev.mysql.com/doc/refman/5.7/en/innodb-file-format.html +Warning 131 Using innodb_file_format is deprecated and the parameter may be removed in future releases. See https://mariadb.com/kb/en/library/xtradbinnodb-file-format/ select @@global.innodb_file_format; @@global.innodb_file_format Barracuda @@ -59,7 +59,7 @@ set global innodb_file_format='Salmon'; ERROR 42000: Variable 'innodb_file_format' can't be set to the value of 'Salmon' SET @@global.innodb_file_format = @start_global_value; Warnings: -Warning 131 Using innodb_file_format is deprecated and the parameter may be removed in future releases. See http://dev.mysql.com/doc/refman/5.7/en/innodb-file-format.html +Warning 131 Using innodb_file_format is deprecated and the parameter may be removed in future releases. See https://mariadb.com/kb/en/library/xtradbinnodb-file-format/ SELECT @@global.innodb_file_format; @@global.innodb_file_format Barracuda diff --git a/mysql-test/suite/sys_vars/r/innodb_file_format_max_basic.result b/mysql-test/suite/sys_vars/r/innodb_file_format_max_basic.result index 5402e16a42448..d36a17933d4a1 100644 --- a/mysql-test/suite/sys_vars/r/innodb_file_format_max_basic.result +++ b/mysql-test/suite/sys_vars/r/innodb_file_format_max_basic.result @@ -27,7 +27,7 @@ VARIABLE_NAME VARIABLE_VALUE INNODB_FILE_FORMAT_MAX Barracuda SET global innodb_file_format_max='Antelope'; Warnings: -Warning 131 Using innodb_file_format_max is deprecated and the parameter may be removed in future releases. See http://dev.mysql.com/doc/refman/5.7/en/innodb-file-format.html +Warning 131 Using innodb_file_format_max is deprecated and the parameter may be removed in future releases. See https://mariadb.com/kb/en/library/xtradbinnodb-file-format/ SELECT @@global.innodb_file_format_max; @@global.innodb_file_format_max Antelope @@ -41,7 +41,7 @@ VARIABLE_NAME VARIABLE_VALUE INNODB_FILE_FORMAT_MAX Antelope SET @@global.innodb_file_format_max='Barracuda'; Warnings: -Warning 131 Using innodb_file_format_max is deprecated and the parameter may be removed in future releases. See http://dev.mysql.com/doc/refman/5.7/en/innodb-file-format.html +Warning 131 Using innodb_file_format_max is deprecated and the parameter may be removed in future releases. See https://mariadb.com/kb/en/library/xtradbinnodb-file-format/ SELECT @@global.innodb_file_format_max; @@global.innodb_file_format_max Barracuda @@ -65,7 +65,7 @@ SET global innodb_file_format_max='Salmon'; ERROR 42000: Variable 'innodb_file_format_max' can't be set to the value of 'Salmon' SET @@global.innodb_file_format_max = @start_global_value; Warnings: -Warning 131 Using innodb_file_format_max is deprecated and the parameter may be removed in future releases. See http://dev.mysql.com/doc/refman/5.7/en/innodb-file-format.html +Warning 131 Using innodb_file_format_max is deprecated and the parameter may be removed in future releases. See https://mariadb.com/kb/en/library/xtradbinnodb-file-format/ SELECT @@global.innodb_file_format_max; @@global.innodb_file_format_max Barracuda diff --git a/mysql-test/suite/sys_vars/r/innodb_large_prefix_basic.result b/mysql-test/suite/sys_vars/r/innodb_large_prefix_basic.result index c6e803ffef855..17a23081096ba 100644 --- a/mysql-test/suite/sys_vars/r/innodb_large_prefix_basic.result +++ b/mysql-test/suite/sys_vars/r/innodb_large_prefix_basic.result @@ -25,7 +25,7 @@ VARIABLE_NAME VARIABLE_VALUE INNODB_LARGE_PREFIX ON set global innodb_large_prefix='OFF'; Warnings: -Warning 131 Using innodb_large_prefix is deprecated and the parameter may be removed in future releases. See http://dev.mysql.com/doc/refman/5.7/en/innodb-file-format.html +Warning 131 Using innodb_large_prefix is deprecated and the parameter may be removed in future releases. See https://mariadb.com/kb/en/library/xtradbinnodb-file-format/ select @@global.innodb_large_prefix; @@global.innodb_large_prefix 0 @@ -37,7 +37,7 @@ VARIABLE_NAME VARIABLE_VALUE INNODB_LARGE_PREFIX OFF set @@global.innodb_large_prefix=1; Warnings: -Warning 131 Using innodb_large_prefix is deprecated and the parameter may be removed in future releases. See http://dev.mysql.com/doc/refman/5.7/en/innodb-file-format.html +Warning 131 Using innodb_large_prefix is deprecated and the parameter may be removed in future releases. See https://mariadb.com/kb/en/library/xtradbinnodb-file-format/ select @@global.innodb_large_prefix; @@global.innodb_large_prefix 1 @@ -49,7 +49,7 @@ VARIABLE_NAME VARIABLE_VALUE INNODB_LARGE_PREFIX ON set global innodb_large_prefix=0; Warnings: -Warning 131 Using innodb_large_prefix is deprecated and the parameter may be removed in future releases. See http://dev.mysql.com/doc/refman/5.7/en/innodb-file-format.html +Warning 131 Using innodb_large_prefix is deprecated and the parameter may be removed in future releases. See https://mariadb.com/kb/en/library/xtradbinnodb-file-format/ select @@global.innodb_large_prefix; @@global.innodb_large_prefix 0 @@ -61,7 +61,7 @@ VARIABLE_NAME VARIABLE_VALUE INNODB_LARGE_PREFIX OFF set @@global.innodb_large_prefix='ON'; Warnings: -Warning 131 Using innodb_large_prefix is deprecated and the parameter may be removed in future releases. See http://dev.mysql.com/doc/refman/5.7/en/innodb-file-format.html +Warning 131 Using innodb_large_prefix is deprecated and the parameter may be removed in future releases. See https://mariadb.com/kb/en/library/xtradbinnodb-file-format/ select @@global.innodb_large_prefix; @@global.innodb_large_prefix 1 @@ -96,7 +96,7 @@ set global innodb_large_prefix='AUTO'; ERROR 42000: Variable 'innodb_large_prefix' can't be set to the value of 'AUTO' SET @@global.innodb_large_prefix = @start_global_value; Warnings: -Warning 131 Using innodb_large_prefix is deprecated and the parameter may be removed in future releases. See http://dev.mysql.com/doc/refman/5.7/en/innodb-file-format.html +Warning 131 Using innodb_large_prefix is deprecated and the parameter may be removed in future releases. See https://mariadb.com/kb/en/library/xtradbinnodb-file-format/ SELECT @@global.innodb_large_prefix; @@global.innodb_large_prefix 1 diff --git a/storage/innobase/buf/buf0buf.cc b/storage/innobase/buf/buf0buf.cc index ae081fa39da53..07d3f7efe2747 100644 --- a/storage/innobase/buf/buf0buf.cc +++ b/storage/innobase/buf/buf0buf.cc @@ -4360,11 +4360,7 @@ buf_page_get_gen( << ". The most probable cause" " of this error may be that the" " table has been corrupted." - " You can try to fix this" - " problem by using" - " innodb_force_recovery." - " Please see " REFMAN " for more" - " details. Aborting..."; + " See https://mariadb.com/kb/en/library/xtradbinnodb-recovery-modes/"; } #if defined UNIV_DEBUG || defined UNIV_BUF_DEBUG diff --git a/storage/innobase/gis/gis0sea.cc b/storage/innobase/gis/gis0sea.cc index 87ebd9ad34a47..dcf8cc6f7819e 100644 --- a/storage/innobase/gis/gis0sea.cc +++ b/storage/innobase/gis/gis0sea.cc @@ -1,7 +1,7 @@ /***************************************************************************** Copyright (c) 2016, Oracle and/or its affiliates. All Rights Reserved. -Copyright (c) 2017, MariaDB Corporation. +Copyright (c) 2017, 2018, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -773,8 +773,9 @@ rtr_page_get_father_node_ptr( error << ". You should dump + drop + reimport the table to" " fix the corruption. If the crash happens at" - " database startup, see " REFMAN - "forcing-innodb-recovery.html about forcing" + " database startup, see " + "https://mariadb.com/kb/en/library/xtradbinnodb-recovery-modes/" + " about forcing" " recovery. Then dump + drop + reimport."; } diff --git a/storage/innobase/handler/ha_innodb.cc b/storage/innobase/handler/ha_innodb.cc index 4a0b6dfe4b95c..b42da1c025e0d 100644 --- a/storage/innobase/handler/ha_innodb.cc +++ b/storage/innobase/handler/ha_innodb.cc @@ -1,7 +1,7 @@ /***************************************************************************** Copyright (c) 2000, 2017, Oracle and/or its affiliates. All Rights Reserved. -Copyright (c) 2013, 2017, MariaDB Corporation. +Copyright (c) 2013, 2018, MariaDB Corporation. Copyright (c) 2008, 2009 Google Inc. Copyright (c) 2009, Percona Inc. Copyright (c) 2012, Facebook Inc. @@ -3654,7 +3654,7 @@ static uint innobase_partition_flags() #define DEPRECATED_FORMAT_PARAMETER(x) \ "Using " x " is deprecated and the parameter" \ " may be removed in future releases." \ - " See " REFMAN "innodb-file-format.html" + " See https://mariadb.com/kb/en/library/xtradbinnodb-file-format/" /** Deprecation message about innodb_file_format */ static const char* deprecated_file_format @@ -22573,7 +22573,8 @@ const char* BUG_REPORT_MSG = "Submit a detailed bug report to https://jira.mariadb.org/"; const char* FORCE_RECOVERY_MSG = - "Please refer to " REFMAN "forcing-innodb-recovery.html" + "Please refer to " + "https://mariadb.com/kb/en/library/xtradbinnodb-recovery-modes/" " for information about forcing recovery."; const char* ERROR_CREATING_MSG = @@ -22581,17 +22582,17 @@ const char* ERROR_CREATING_MSG = const char* OPERATING_SYSTEM_ERROR_MSG = "Some operating system error numbers are described at" - " " REFMAN "operating-system-error-codes.html"; + " https://mariadb.com/kb/en/library/operating-system-error-codes/"; const char* FOREIGN_KEY_CONSTRAINTS_MSG = - "Please refer to " REFMAN "innodb-foreign-key-constraints.html" + "Please refer to https://mariadb.com/kb/en/library/foreign-keys/" " for correct foreign key definition."; const char* SET_TRANSACTION_MSG = - "Please refer to " REFMAN "set-transaction.html"; + "Please refer to https://mariadb.com/kb/en/library/set-transaction/"; const char* INNODB_PARAMETERS_MSG = - "Please refer to " REFMAN "innodb-parameters.html"; + "Please refer to https://mariadb.com/kb/en/library/xtradbinnodb-server-system-variables/"; /********************************************************************** Converts an identifier from my_charset_filename to UTF-8 charset. diff --git a/storage/innobase/log/log0recv.cc b/storage/innobase/log/log0recv.cc index 5ee6004fe55bf..818a0f0422eda 100644 --- a/storage/innobase/log/log0recv.cc +++ b/storage/innobase/log/log0recv.cc @@ -842,7 +842,7 @@ recv_find_max_checkpoint_0(log_group_t** max_group, ulint* max_field) " This redo log was created before MariaDB 10.2.2," " and we did not find a valid checkpoint." " Please follow the instructions at" - " " REFMAN "upgrading.html"; + " https://mariadb.com/kb/en/library/upgrading/"; return(DB_ERROR); } @@ -869,7 +869,7 @@ recv_log_format_0_recover(lsn_t lsn) " This redo log was created before MariaDB 10.2.2"; static const char* NO_UPGRADE_RTFM_MSG = ". Please follow the instructions at " - REFMAN "upgrading.html"; + "https://mariadb.com/kb/en/library/upgrading/"; fil_io(IORequestLogRead, true, page_id_t(SRV_LOG_SPACE_FIRST_ID, page_no), diff --git a/storage/innobase/ut/ut0dbg.cc b/storage/innobase/ut/ut0dbg.cc index 9e596dcda813d..7df189ac56011 100644 --- a/storage/innobase/ut/ut0dbg.cc +++ b/storage/innobase/ut/ut0dbg.cc @@ -1,7 +1,7 @@ /***************************************************************************** Copyright (c) 1994, 2016, Oracle and/or its affiliates. All Rights Reserved. -Copyright (c) 2017, MariaDB Corporation. +Copyright (c) 2017, 2018, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -53,7 +53,7 @@ ut_dbg_assertion_failed( " or crashes, even\n" "InnoDB: immediately after the mysqld startup, there may be\n" "InnoDB: corruption in the InnoDB tablespace. Please refer to\n" - "InnoDB: " REFMAN "forcing-innodb-recovery.html\n" + "InnoDB: https://mariadb.com/kb/en/library/xtradbinnodb-recovery-modes/\n" "InnoDB: about forcing recovery.\n", stderr); fflush(stderr); From 1a62c8a39647db00283b4e35fb6db3c7bc31e3ca Mon Sep 17 00:00:00 2001 From: Monty Date: Wed, 10 Jan 2018 14:41:10 +0200 Subject: [PATCH 022/106] MDEV-14822 binlog.binlog_killed fails with wrong result Problem was timing between the thread that was killed and reading the binary log. Updated the test to wait until the killed thread was properly terminated before checking what's in the binary log. To make check safe, I changed "threads_connected" to be updated after thd::cleanup() is done, to ensure that all binary logs updates are done before the variable is changed. This was mainly done to get the test deterministic and have now other real influence in how the server works. --- .../suite/binlog/r/binlog_killed.result | 42 +++++++++---------- mysql-test/suite/binlog/t/binlog_killed.test | 34 ++++++++++----- sql/mysqld.cc | 7 ++-- 3 files changed, 48 insertions(+), 35 deletions(-) diff --git a/mysql-test/suite/binlog/r/binlog_killed.result b/mysql-test/suite/binlog/r/binlog_killed.result index d751b1f900294..ae851c38e5f3e 100644 --- a/mysql-test/suite/binlog/r/binlog_killed.result +++ b/mysql-test/suite/binlog/r/binlog_killed.result @@ -185,90 +185,88 @@ connection con3; MI: MyISAM, INNODB BEGIN; INSERT INTO t2 VALUES (NULL, 1); -INSERT INTO t1 VALUES (NULL, 1); +INSERT INTO t1 VALUES (NULL, 2); connection con1; KILL ID; include/show_binlog_events.inc Log_name Pos Event_type Server_id End_log_pos Info master-bin.000002 # Format_desc # # SERVER_VERSION, BINLOG_VERSION master-bin.000002 # Gtid_list # # [#-#-#] -master-bin.000002 # Binlog_checkpoint # # master-bin.000001 -master-bin.000002 # Binlog_checkpoint # # master-bin.000002 master-bin.000002 # Gtid # # BEGIN GTID #-#-# master-bin.000002 # Intvar # # INSERT_ID=3 master-bin.000002 # Query # # use `test`; INSERT INTO t2 VALUES (NULL, 1) master-bin.000002 # Query # # COMMIT master-bin.000002 # Gtid # # BEGIN GTID #-#-# master-bin.000002 # Intvar # # INSERT_ID=4 -master-bin.000002 # Query # # use `test`; INSERT INTO t1 VALUES (NULL, 1) +master-bin.000002 # Query # # use `test`; INSERT INTO t1 VALUES (NULL, 2) master-bin.000002 # Query # # ROLLBACK disconnect con3; connect con3, localhost, root,,; connection con3; IM: INNODB, MyISAM BEGIN; -INSERT INTO t1 VALUES (NULL, 1); -INSERT INTO t2 VALUES (NULL, 1); +INSERT INTO t1 VALUES (NULL, 3); +INSERT INTO t2 VALUES (NULL, 4); connection con1; KILL ID; include/show_binlog_events.inc Log_name Pos Event_type Server_id End_log_pos Info master-bin.000002 # Gtid # # BEGIN GTID #-#-# master-bin.000002 # Intvar # # INSERT_ID=4 -master-bin.000002 # Query # # use `test`; INSERT INTO t2 VALUES (NULL, 1) +master-bin.000002 # Query # # use `test`; INSERT INTO t2 VALUES (NULL, 4) master-bin.000002 # Query # # COMMIT master-bin.000002 # Gtid # # BEGIN GTID #-#-# master-bin.000002 # Intvar # # INSERT_ID=5 -master-bin.000002 # Query # # use `test`; INSERT INTO t1 VALUES (NULL, 1) +master-bin.000002 # Query # # use `test`; INSERT INTO t1 VALUES (NULL, 3) master-bin.000002 # Query # # ROLLBACK disconnect con3; connect con3, localhost, root,,; connection con3; IMI: INNODB, MyISAM, INNODB BEGIN; -INSERT INTO t1 VALUES (NULL, 1); -INSERT INTO t2 VALUES (NULL, 1); -INSERT INTO t1 VALUES (NULL, 1); +INSERT INTO t1 VALUES (NULL, 5); +INSERT INTO t2 VALUES (NULL, 6); +INSERT INTO t1 VALUES (NULL, 7); connection con1; KILL ID; include/show_binlog_events.inc Log_name Pos Event_type Server_id End_log_pos Info master-bin.000002 # Gtid # # BEGIN GTID #-#-# master-bin.000002 # Intvar # # INSERT_ID=5 -master-bin.000002 # Query # # use `test`; INSERT INTO t2 VALUES (NULL, 1) +master-bin.000002 # Query # # use `test`; INSERT INTO t2 VALUES (NULL, 6) master-bin.000002 # Query # # COMMIT master-bin.000002 # Gtid # # BEGIN GTID #-#-# master-bin.000002 # Intvar # # INSERT_ID=6 -master-bin.000002 # Query # # use `test`; INSERT INTO t1 VALUES (NULL, 1) +master-bin.000002 # Query # # use `test`; INSERT INTO t1 VALUES (NULL, 5) master-bin.000002 # Intvar # # INSERT_ID=7 -master-bin.000002 # Query # # use `test`; INSERT INTO t1 VALUES (NULL, 1) +master-bin.000002 # Query # # use `test`; INSERT INTO t1 VALUES (NULL, 7) master-bin.000002 # Query # # ROLLBACK disconnect con3; connect con3, localhost, root,,; connection con3; MI2: MyISAM, INNODB, MyISAM, INNODB BEGIN; -INSERT INTO t2 VALUES (NULL, 1); -INSERT INTO t1 VALUES (NULL, 1); -INSERT INTO t2 VALUES (NULL, 1); -INSERT INTO t1 VALUES (NULL, 1); +INSERT INTO t2 VALUES (NULL, 8); +INSERT INTO t1 VALUES (NULL, 9); +INSERT INTO t2 VALUES (NULL, 10); +INSERT INTO t1 VALUES (NULL, 11); connection con1; KILL ID; include/show_binlog_events.inc Log_name Pos Event_type Server_id End_log_pos Info master-bin.000002 # Gtid # # BEGIN GTID #-#-# master-bin.000002 # Intvar # # INSERT_ID=6 -master-bin.000002 # Query # # use `test`; INSERT INTO t2 VALUES (NULL, 1) +master-bin.000002 # Query # # use `test`; INSERT INTO t2 VALUES (NULL, 8) master-bin.000002 # Query # # COMMIT master-bin.000002 # Gtid # # BEGIN GTID #-#-# master-bin.000002 # Intvar # # INSERT_ID=7 -master-bin.000002 # Query # # use `test`; INSERT INTO t2 VALUES (NULL, 1) +master-bin.000002 # Query # # use `test`; INSERT INTO t2 VALUES (NULL, 10) master-bin.000002 # Query # # COMMIT master-bin.000002 # Gtid # # BEGIN GTID #-#-# master-bin.000002 # Intvar # # INSERT_ID=8 -master-bin.000002 # Query # # use `test`; INSERT INTO t1 VALUES (NULL, 1) +master-bin.000002 # Query # # use `test`; INSERT INTO t1 VALUES (NULL, 9) master-bin.000002 # Intvar # # INSERT_ID=9 -master-bin.000002 # Query # # use `test`; INSERT INTO t1 VALUES (NULL, 1) +master-bin.000002 # Query # # use `test`; INSERT INTO t1 VALUES (NULL, 11) master-bin.000002 # Query # # ROLLBACK connection default; disconnect con1; diff --git a/mysql-test/suite/binlog/t/binlog_killed.test b/mysql-test/suite/binlog/t/binlog_killed.test index 8ac223603a376..7c3a262d2c153 100644 --- a/mysql-test/suite/binlog/t/binlog_killed.test +++ b/mysql-test/suite/binlog/t/binlog_killed.test @@ -10,6 +10,13 @@ call mtr.add_suppression("Unsafe statement written to the binary log using state -- source include/not_embedded.inc +# +# Avoid printing binlog checkpoints +# + +--let $skip_checkpoint_events=1 + + ### ### bug#22725 : incorrect killed error in binlogged query ### @@ -362,16 +369,20 @@ connect (con3, localhost, root,,); connection con3; let $ID= `select connection_id()`; +--let $threads_connected=`select variable_value from information_schema.global_status where variable_name="threads_connected"` +--let wait_condition=select variable_value < $threads_connected from information_schema.global_status where variable_name="threads_connected" + --echo MI: MyISAM, INNODB BEGIN; INSERT INTO t2 VALUES (NULL, 1); -INSERT INTO t1 VALUES (NULL, 1); +INSERT INTO t1 VALUES (NULL, 2); #Connection con1 as killer throughout the test connection con1; --replace_result $ID ID --eval KILL $ID let $binlog_file= query_get_value(SHOW MASTER STATUS, File, 1); +--source include/wait_condition.inc --let $binlog_start= 4 --source include/show_binlog_events.inc --let $binlog_killed_pos=query_get_value(SHOW MASTER STATUS, Position, 1) @@ -383,12 +394,13 @@ let $ID= `select connection_id()`; --echo IM: INNODB, MyISAM BEGIN; -INSERT INTO t1 VALUES (NULL, 1); -INSERT INTO t2 VALUES (NULL, 1); +INSERT INTO t1 VALUES (NULL, 3); +INSERT INTO t2 VALUES (NULL, 4); connection con1; --replace_result $ID ID --eval KILL $ID +--source include/wait_condition.inc --let $binlog_start= $binlog_killed_pos --source include/show_binlog_events.inc --let $binlog_killed_pos=query_get_value(SHOW MASTER STATUS, Position, 1) @@ -400,13 +412,14 @@ let $ID= `select connection_id()`; --echo IMI: INNODB, MyISAM, INNODB BEGIN; -INSERT INTO t1 VALUES (NULL, 1); -INSERT INTO t2 VALUES (NULL, 1); -INSERT INTO t1 VALUES (NULL, 1); +INSERT INTO t1 VALUES (NULL, 5); +INSERT INTO t2 VALUES (NULL, 6); +INSERT INTO t1 VALUES (NULL, 7); connection con1; --replace_result $ID ID --eval KILL $ID +--source include/wait_condition.inc --let $binlog_start= $binlog_killed_pos --source include/show_binlog_events.inc --let $binlog_killed_pos=query_get_value(SHOW MASTER STATUS, Position, 1) @@ -418,14 +431,15 @@ let $ID= `select connection_id()`; --echo MI2: MyISAM, INNODB, MyISAM, INNODB BEGIN; -INSERT INTO t2 VALUES (NULL, 1); -INSERT INTO t1 VALUES (NULL, 1); -INSERT INTO t2 VALUES (NULL, 1); -INSERT INTO t1 VALUES (NULL, 1); +INSERT INTO t2 VALUES (NULL, 8); +INSERT INTO t1 VALUES (NULL, 9); +INSERT INTO t2 VALUES (NULL, 10); +INSERT INTO t1 VALUES (NULL, 11); connection con1; --replace_result $ID ID --eval KILL $ID +--source include/wait_condition.inc --let $binlog_start= $binlog_killed_pos --source include/show_binlog_events.inc diff --git a/sql/mysqld.cc b/sql/mysqld.cc index 0ec99d6eb96ee..c2f0bbf2f1044 100644 --- a/sql/mysqld.cc +++ b/sql/mysqld.cc @@ -2919,16 +2919,17 @@ void unlink_thd(THD *thd) DBUG_ENTER("unlink_thd"); DBUG_PRINT("enter", ("thd: %p", thd)); + thd->cleanup(); + thd->add_status_to_global(); + unlink_not_visible_thd(thd); + /* Do not decrement when its wsrep system thread. wsrep_applier is set for applier as well as rollbacker threads. */ if (IF_WSREP(!thd->wsrep_applier, 1)) dec_connection_count(thd->scheduler); - thd->cleanup(); - thd->add_status_to_global(); - unlink_not_visible_thd(thd); thd->free_connection(); DBUG_VOID_RETURN; From 9c9cf556a15af84f9133ef859b50596085f7ae8e Mon Sep 17 00:00:00 2001 From: Oleksandr Byelkin Date: Fri, 6 Oct 2017 17:52:35 +0200 Subject: [PATCH 023/106] MDEV-13933: Wrong results in COUNT() query with EXISTS and exists_to_in Roll back to most general duplicate removing strategi in case of different stratagies for one position. --- mysql-test/r/subselect.result | 26 +++++++++ mysql-test/r/subselect_no_mat.result | 26 +++++++++ mysql-test/r/subselect_no_opts.result | 26 +++++++++ mysql-test/r/subselect_no_scache.result | 26 +++++++++ mysql-test/r/subselect_no_semijoin.result | 26 +++++++++ mysql-test/t/subselect.test | 28 ++++++++++ sql/opt_subselect.cc | 64 +++++++++++++++++------ 7 files changed, 206 insertions(+), 16 deletions(-) diff --git a/mysql-test/r/subselect.result b/mysql-test/r/subselect.result index 8fc9cd38728f9..e2f2b6521c873 100644 --- a/mysql-test/r/subselect.result +++ b/mysql-test/r/subselect.result @@ -7146,3 +7146,29 @@ SELECT (SELECT MAX(sq.f2) FROM t1) FROM (SELECT * FROM t2) AS sq WHERE f2 = 2; (SELECT MAX(sq.f2) FROM t1) NULL drop table t1, t2; +# +# MDEV-13933: Wrong results in COUNT() query with EXISTS and exists_to_in +# (5.5 test) +# +SET @optimiser_switch_save= @@optimizer_switch; +CREATE TABLE t1 (a INT NOT NULL); +INSERT INTO t1 VALUES (1),(1),(1),(5),(5); +CREATE TABLE t2 (b INT); +INSERT INTO t2 VALUES (5),(1); +CREATE TABLE t3 (c INT, KEY(c)); +INSERT INTO t3 VALUES (5),(5); +SET optimizer_switch='semijoin=on'; +select t1.a from t1 where t1.a in (select `test`.`t2`.`b` from `test`.`t2`) +and t1.a in (select `test`.`t3`.`c` from `test`.`t3`); +a +5 +5 +SET optimizer_switch='semijoin=off'; +select t1.a from t1 where t1.a in (select `test`.`t2`.`b` from `test`.`t2`) +and t1.a in (select `test`.`t3`.`c` from `test`.`t3`); +a +5 +5 +SET @@optimizer_switch= @optimiser_switch_save; +DROP TABLE t1, t2, t3; +End of 5.5 tests diff --git a/mysql-test/r/subselect_no_mat.result b/mysql-test/r/subselect_no_mat.result index a71df97e6bceb..25ef4a76962fd 100644 --- a/mysql-test/r/subselect_no_mat.result +++ b/mysql-test/r/subselect_no_mat.result @@ -7143,6 +7143,32 @@ SELECT (SELECT MAX(sq.f2) FROM t1) FROM (SELECT * FROM t2) AS sq WHERE f2 = 2; (SELECT MAX(sq.f2) FROM t1) NULL drop table t1, t2; +# +# MDEV-13933: Wrong results in COUNT() query with EXISTS and exists_to_in +# (5.5 test) +# +SET @optimiser_switch_save= @@optimizer_switch; +CREATE TABLE t1 (a INT NOT NULL); +INSERT INTO t1 VALUES (1),(1),(1),(5),(5); +CREATE TABLE t2 (b INT); +INSERT INTO t2 VALUES (5),(1); +CREATE TABLE t3 (c INT, KEY(c)); +INSERT INTO t3 VALUES (5),(5); +SET optimizer_switch='semijoin=on'; +select t1.a from t1 where t1.a in (select `test`.`t2`.`b` from `test`.`t2`) +and t1.a in (select `test`.`t3`.`c` from `test`.`t3`); +a +5 +5 +SET optimizer_switch='semijoin=off'; +select t1.a from t1 where t1.a in (select `test`.`t2`.`b` from `test`.`t2`) +and t1.a in (select `test`.`t3`.`c` from `test`.`t3`); +a +5 +5 +SET @@optimizer_switch= @optimiser_switch_save; +DROP TABLE t1, t2, t3; +End of 5.5 tests set optimizer_switch=default; select @@optimizer_switch like '%materialization=on%'; @@optimizer_switch like '%materialization=on%' diff --git a/mysql-test/r/subselect_no_opts.result b/mysql-test/r/subselect_no_opts.result index 2f8c67fa1670e..074874fbd5bb9 100644 --- a/mysql-test/r/subselect_no_opts.result +++ b/mysql-test/r/subselect_no_opts.result @@ -7141,4 +7141,30 @@ SELECT (SELECT MAX(sq.f2) FROM t1) FROM (SELECT * FROM t2) AS sq WHERE f2 = 2; (SELECT MAX(sq.f2) FROM t1) NULL drop table t1, t2; +# +# MDEV-13933: Wrong results in COUNT() query with EXISTS and exists_to_in +# (5.5 test) +# +SET @optimiser_switch_save= @@optimizer_switch; +CREATE TABLE t1 (a INT NOT NULL); +INSERT INTO t1 VALUES (1),(1),(1),(5),(5); +CREATE TABLE t2 (b INT); +INSERT INTO t2 VALUES (5),(1); +CREATE TABLE t3 (c INT, KEY(c)); +INSERT INTO t3 VALUES (5),(5); +SET optimizer_switch='semijoin=on'; +select t1.a from t1 where t1.a in (select `test`.`t2`.`b` from `test`.`t2`) +and t1.a in (select `test`.`t3`.`c` from `test`.`t3`); +a +5 +5 +SET optimizer_switch='semijoin=off'; +select t1.a from t1 where t1.a in (select `test`.`t2`.`b` from `test`.`t2`) +and t1.a in (select `test`.`t3`.`c` from `test`.`t3`); +a +5 +5 +SET @@optimizer_switch= @optimiser_switch_save; +DROP TABLE t1, t2, t3; +End of 5.5 tests set @optimizer_switch_for_subselect_test=null; diff --git a/mysql-test/r/subselect_no_scache.result b/mysql-test/r/subselect_no_scache.result index 8dda0a4fd3624..de49585b5628d 100644 --- a/mysql-test/r/subselect_no_scache.result +++ b/mysql-test/r/subselect_no_scache.result @@ -7152,6 +7152,32 @@ SELECT (SELECT MAX(sq.f2) FROM t1) FROM (SELECT * FROM t2) AS sq WHERE f2 = 2; (SELECT MAX(sq.f2) FROM t1) NULL drop table t1, t2; +# +# MDEV-13933: Wrong results in COUNT() query with EXISTS and exists_to_in +# (5.5 test) +# +SET @optimiser_switch_save= @@optimizer_switch; +CREATE TABLE t1 (a INT NOT NULL); +INSERT INTO t1 VALUES (1),(1),(1),(5),(5); +CREATE TABLE t2 (b INT); +INSERT INTO t2 VALUES (5),(1); +CREATE TABLE t3 (c INT, KEY(c)); +INSERT INTO t3 VALUES (5),(5); +SET optimizer_switch='semijoin=on'; +select t1.a from t1 where t1.a in (select `test`.`t2`.`b` from `test`.`t2`) +and t1.a in (select `test`.`t3`.`c` from `test`.`t3`); +a +5 +5 +SET optimizer_switch='semijoin=off'; +select t1.a from t1 where t1.a in (select `test`.`t2`.`b` from `test`.`t2`) +and t1.a in (select `test`.`t3`.`c` from `test`.`t3`); +a +5 +5 +SET @@optimizer_switch= @optimiser_switch_save; +DROP TABLE t1, t2, t3; +End of 5.5 tests set optimizer_switch=default; select @@optimizer_switch like '%subquery_cache=on%'; @@optimizer_switch like '%subquery_cache=on%' diff --git a/mysql-test/r/subselect_no_semijoin.result b/mysql-test/r/subselect_no_semijoin.result index 339e2b89786ad..46a46c91ddcb7 100644 --- a/mysql-test/r/subselect_no_semijoin.result +++ b/mysql-test/r/subselect_no_semijoin.result @@ -7141,5 +7141,31 @@ SELECT (SELECT MAX(sq.f2) FROM t1) FROM (SELECT * FROM t2) AS sq WHERE f2 = 2; (SELECT MAX(sq.f2) FROM t1) NULL drop table t1, t2; +# +# MDEV-13933: Wrong results in COUNT() query with EXISTS and exists_to_in +# (5.5 test) +# +SET @optimiser_switch_save= @@optimizer_switch; +CREATE TABLE t1 (a INT NOT NULL); +INSERT INTO t1 VALUES (1),(1),(1),(5),(5); +CREATE TABLE t2 (b INT); +INSERT INTO t2 VALUES (5),(1); +CREATE TABLE t3 (c INT, KEY(c)); +INSERT INTO t3 VALUES (5),(5); +SET optimizer_switch='semijoin=on'; +select t1.a from t1 where t1.a in (select `test`.`t2`.`b` from `test`.`t2`) +and t1.a in (select `test`.`t3`.`c` from `test`.`t3`); +a +5 +5 +SET optimizer_switch='semijoin=off'; +select t1.a from t1 where t1.a in (select `test`.`t2`.`b` from `test`.`t2`) +and t1.a in (select `test`.`t3`.`c` from `test`.`t3`); +a +5 +5 +SET @@optimizer_switch= @optimiser_switch_save; +DROP TABLE t1, t2, t3; +End of 5.5 tests set @optimizer_switch_for_subselect_test=null; set @join_cache_level_for_subselect_test=NULL; diff --git a/mysql-test/t/subselect.test b/mysql-test/t/subselect.test index 50d9517043a04..e6233e9de780e 100644 --- a/mysql-test/t/subselect.test +++ b/mysql-test/t/subselect.test @@ -6033,3 +6033,31 @@ CREATE TABLE t2 (f2 INT, KEY(f2)) ENGINE=MyISAM; INSERT t2 VALUES (6),(9); SELECT (SELECT MAX(sq.f2) FROM t1) FROM (SELECT * FROM t2) AS sq WHERE f2 = 2; drop table t1, t2; + +--echo # +--echo # MDEV-13933: Wrong results in COUNT() query with EXISTS and exists_to_in +--echo # (5.5 test) +--echo # +SET @optimiser_switch_save= @@optimizer_switch; + +CREATE TABLE t1 (a INT NOT NULL); +INSERT INTO t1 VALUES (1),(1),(1),(5),(5); + +CREATE TABLE t2 (b INT); +INSERT INTO t2 VALUES (5),(1); + +CREATE TABLE t3 (c INT, KEY(c)); +INSERT INTO t3 VALUES (5),(5); + +SET optimizer_switch='semijoin=on'; +select t1.a from t1 where t1.a in (select `test`.`t2`.`b` from `test`.`t2`) +and t1.a in (select `test`.`t3`.`c` from `test`.`t3`); + +SET optimizer_switch='semijoin=off'; +select t1.a from t1 where t1.a in (select `test`.`t2`.`b` from `test`.`t2`) +and t1.a in (select `test`.`t3`.`c` from `test`.`t3`); + +SET @@optimizer_switch= @optimiser_switch_save; +DROP TABLE t1, t2, t3; + +--echo End of 5.5 tests diff --git a/sql/opt_subselect.cc b/sql/opt_subselect.cc index 69a5367cdf1fb..028bf44bf79e4 100644 --- a/sql/opt_subselect.cc +++ b/sql/opt_subselect.cc @@ -2687,7 +2687,8 @@ void advance_sj_state(JOIN *join, table_map remaining_tables, uint idx, LooseScan detector in best_access_path) */ remaining_tables &= ~new_join_tab->table->map; - table_map dups_producing_tables; + table_map dups_producing_tables, prev_dups_producing_tables, + prev_sjm_lookup_tables; if (idx == join->const_tables) dups_producing_tables= 0; @@ -2698,7 +2699,7 @@ void advance_sj_state(JOIN *join, table_map remaining_tables, uint idx, if ((emb_sj_nest= new_join_tab->emb_sj_nest)) dups_producing_tables |= emb_sj_nest->sj_inner_tables; - Semi_join_strategy_picker **strategy; + Semi_join_strategy_picker **strategy, **prev_strategy; if (idx == join->const_tables) { /* First table, initialize pickers */ @@ -2750,23 +2751,54 @@ void advance_sj_state(JOIN *join, table_map remaining_tables, uint idx, 3. We have no clue what to do about fanount of semi-join Y. */ if ((dups_producing_tables & handled_fanout) || - (read_time < *current_read_time && + (read_time < *current_read_time && !(handled_fanout & pos->inner_tables_handled_with_other_sjs))) { - /* Mark strategy as used */ - (*strategy)->mark_used(); - pos->sj_strategy= sj_strategy; - if (sj_strategy == SJ_OPT_MATERIALIZE) - join->sjm_lookup_tables |= handled_fanout; + DBUG_ASSERT(pos->sj_strategy != sj_strategy); + /* + If the strategy choosen first time or + the strategy replace strategy which was used to exectly the same + tables + */ + if (pos->sj_strategy == SJ_OPT_NONE || + handled_fanout == + (prev_dups_producing_tables ^ dups_producing_tables)) + { + prev_strategy= strategy; + if (pos->sj_strategy == SJ_OPT_NONE) + { + prev_dups_producing_tables= dups_producing_tables; + prev_sjm_lookup_tables= join->sjm_lookup_tables; + } + /* Mark strategy as used */ + (*strategy)->mark_used(); + pos->sj_strategy= sj_strategy; + if (sj_strategy == SJ_OPT_MATERIALIZE) + join->sjm_lookup_tables |= handled_fanout; + else + join->sjm_lookup_tables &= ~handled_fanout; + *current_read_time= read_time; + *current_record_count= rec_count; + dups_producing_tables &= ~handled_fanout; + //TODO: update bitmap of semi-joins that were handled together with + // others. + if (is_multiple_semi_joins(join, join->positions, idx, + handled_fanout)) + pos->inner_tables_handled_with_other_sjs |= handled_fanout; + } else - join->sjm_lookup_tables &= ~handled_fanout; - *current_read_time= read_time; - *current_record_count= rec_count; - dups_producing_tables &= ~handled_fanout; - //TODO: update bitmap of semi-joins that were handled together with - // others. - if (is_multiple_semi_joins(join, join->positions, idx, handled_fanout)) - pos->inner_tables_handled_with_other_sjs |= handled_fanout; + { + /* Conflict fall to most general variant */ + (*prev_strategy)->set_empty(); + dups_producing_tables= prev_dups_producing_tables; + join->sjm_lookup_tables= prev_sjm_lookup_tables; + // mark it 'none' to avpoid loops + pos->sj_strategy= SJ_OPT_NONE; + // next skip to last; + strategy= pickers + + (sizeof(pickers)/sizeof(Semi_join_strategy_picker*) - 3); + continue; + } } else { From ec97aba28462aab18137f96317c4b441385df12a Mon Sep 17 00:00:00 2001 From: Monty Date: Wed, 10 Jan 2018 19:36:38 +0200 Subject: [PATCH 024/106] Fixed BUILD scripts - Skip 'clean' if not a git repository (Good for tar files) - Add configuration for ASAN builds --- BUILD/FINISH.sh | 12 ++++++---- BUILD/compile-pentium64-asan-max | 39 ++++++++++++++++++++++++++++++++ 2 files changed, 47 insertions(+), 4 deletions(-) create mode 100755 BUILD/compile-pentium64-asan-max diff --git a/BUILD/FINISH.sh b/BUILD/FINISH.sh index 20784c49f0b67..1faf62a9b62c3 100644 --- a/BUILD/FINISH.sh +++ b/BUILD/FINISH.sh @@ -14,7 +14,7 @@ # License along with this library; if not, write to the Free # Software Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, # MA 02110-1301, USA - +set -x -v cflags="$c_warnings $extra_flags $EXTRA_FLAGS $EXTRA_CFLAGS" cxxflags="$cxx_warnings $base_cxxflags $extra_flags $EXTRA_FLAGS $EXTRA_CXXFLAGS" extra_configs="$extra_configs $local_infile_configs $EXTRA_CONFIGS" @@ -32,15 +32,19 @@ then configure="$configure --verbose" fi +commands="" # git clean -fdX removes all ignored (build) files -commands="\ +if test -d .git +then + commands="\ git clean -fdX cd ./libmariadb git submodule update cd ../storage/rocksdb/rocksdb git submodule update -cd ../../.. - +cd ../../.." +fi +commands="$commands path=`dirname $0` . \"$path/autorun.sh\"" diff --git a/BUILD/compile-pentium64-asan-max b/BUILD/compile-pentium64-asan-max new file mode 100755 index 0000000000000..db595b418df35 --- /dev/null +++ b/BUILD/compile-pentium64-asan-max @@ -0,0 +1,39 @@ +#! /bin/sh +# Copyright (c) 2005, 2010, Oracle and/or its affiliates. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; version 2 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA + +# Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; version 2 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA + +path=`dirname $0` +. "$path/SETUP.sh" + +extra_flags="$pentium64_cflags $debug_cflags -lasan -O -g -fsanitize=address" +extra_configs="$pentium_configs $debug_configs $valgrind_configs $max_configs" +export LDFLAGS="-ldl" + +. "$path/FINISH.sh" From 79fc07471068dfe41d08cd23dff100039949836f Mon Sep 17 00:00:00 2001 From: Vladislav Vaintroub Date: Wed, 10 Jan 2018 16:43:25 +0000 Subject: [PATCH 025/106] Update CONC --- libmariadb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libmariadb b/libmariadb index bc1777f8ce8cb..8231a0add6a9c 160000 --- a/libmariadb +++ b/libmariadb @@ -1 +1 @@ -Subproject commit bc1777f8ce8cb5fc3b4f0e184d56871d5b809c20 +Subproject commit 8231a0add6a9c2e12a758db2e7072d42deef354b From cdb7a8fa6928f3fb103ed7f66486dc9130a60e89 Mon Sep 17 00:00:00 2001 From: Vladislav Vaintroub Date: Wed, 10 Jan 2018 20:16:52 +0000 Subject: [PATCH 026/106] Silence warning coming from Windows' own header dbghelp.h --- mysql-test/lib/My/SafeProcess/safe_kill_win.cc | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/mysql-test/lib/My/SafeProcess/safe_kill_win.cc b/mysql-test/lib/My/SafeProcess/safe_kill_win.cc index bb884cba11e50..6ca38ceee8115 100644 --- a/mysql-test/lib/My/SafeProcess/safe_kill_win.cc +++ b/mysql-test/lib/My/SafeProcess/safe_kill_win.cc @@ -26,7 +26,20 @@ #include #include #include + +#ifdef _MSC_VER +/* Silence warning in OS header dbghelp.h */ +#pragma warning(push) +#pragma warning(disable : 4091) +#endif + #include + +#ifdef _MSC_VER +/* Silence warning in OS header dbghelp.h */ +#pragma warning(pop) +#endif + #include #include From bdcd7f79e4a33e08bae31a86cee98e23ff6824ff Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Thu, 11 Jan 2018 09:33:26 +0200 Subject: [PATCH 027/106] MDEV-14916 InnoDB reports warning for "Purge reached the head of the history list" The warning was originally added in commit c67663054a7db1c77dd1dbc85b63595667a53500 (MySQL 4.1.12, 5.0.3) to trace claimed undo log corruption that was analyzed in https://lists.mysql.com/mysql/176250 on November 9, 2004. Originally, the limit was 20,000 undo log headers or transactions, but in commit 9d6d1902e091c868bb288e0ccf9f975ccb474db9 in MySQL 5.5.11 it was increased to 2,000,000. The message can be triggered when the progress of purge is prevented by a long-running transaction (or just an idle transaction whose read view was started a long time ago), by running many transactions that UPDATE or DELETE some records, then starting another transaction with a read view, and finally by executing more than 2,000,000 transactions that UPDATE or DELETE records in InnoDB tables. Finally, when the oldest long-running transaction is completed, purge would run up to the next-oldest transaction, and there would still be more than 2,000,000 transactions to purge. Because the message can be triggered when the database is obviously not corrupted, it should be removed. Heavy users of InnoDB should be monitoring the "History list length" in SHOW ENGINE INNODB STATUS; there is no need to spam the error log. --- storage/innobase/trx/trx0purge.c | 26 -------------------------- storage/xtradb/trx/trx0purge.c | 26 -------------------------- 2 files changed, 52 deletions(-) diff --git a/storage/innobase/trx/trx0purge.c b/storage/innobase/trx/trx0purge.c index 59f8cd52afa57..d23d1fc6a1fee 100644 --- a/storage/innobase/trx/trx0purge.c +++ b/storage/innobase/trx/trx0purge.c @@ -720,32 +720,6 @@ trx_purge_rseg_get_next_history_log( mutex_exit(&(rseg->mutex)); mtr_commit(&mtr); - - mutex_enter(&kernel_mutex); - - /* Add debug code to track history list corruption reported - on the MySQL mailing list on Nov 9, 2004. The fut0lst.c - file-based list was corrupt. The prev node pointer was - FIL_NULL, even though the list length was over 8 million nodes! - We assume that purge truncates the history list in large - size pieces, and if we here reach the head of the list, the - list cannot be longer than 2000 000 undo logs now. */ - - if (trx_sys->rseg_history_len > 2000000) { - ut_print_timestamp(stderr); - fprintf(stderr, - " InnoDB: Warning: purge reached the" - " head of the history list,\n" - "InnoDB: but its length is still" - " reported as %lu! Make a detailed bug\n" - "InnoDB: report, and submit it" - " to https://jira.mariadb.org/\n", - (ulong) trx_sys->rseg_history_len); - ut_ad(0); - } - - mutex_exit(&kernel_mutex); - return; } diff --git a/storage/xtradb/trx/trx0purge.c b/storage/xtradb/trx/trx0purge.c index bc094df0364cd..2a9e80fe66a2f 100644 --- a/storage/xtradb/trx/trx0purge.c +++ b/storage/xtradb/trx/trx0purge.c @@ -728,32 +728,6 @@ trx_purge_rseg_get_next_history_log( mutex_exit(&(rseg->mutex)); mtr_commit(&mtr); - - mutex_enter(&kernel_mutex); - - /* Add debug code to track history list corruption reported - on the MySQL mailing list on Nov 9, 2004. The fut0lst.c - file-based list was corrupt. The prev node pointer was - FIL_NULL, even though the list length was over 8 million nodes! - We assume that purge truncates the history list in large - size pieces, and if we here reach the head of the list, the - list cannot be longer than 2000 000 undo logs now. */ - - if (trx_sys->rseg_history_len > 2000000) { - ut_print_timestamp(stderr); - fprintf(stderr, - " InnoDB: Warning: purge reached the" - " head of the history list,\n" - "InnoDB: but its length is still" - " reported as %lu! Make a detailed bug\n" - "InnoDB: report, and submit it" - " to https://jira.mariadb.org/\n", - (ulong) trx_sys->rseg_history_len); - ut_ad(0); - } - - mutex_exit(&kernel_mutex); - return; } From 380069c235cf63cedff1855314766a545fb26846 Mon Sep 17 00:00:00 2001 From: Sergey Vojtovich Date: Wed, 13 Dec 2017 15:40:41 +0400 Subject: [PATCH 028/106] MDEV-14638 - Replace trx_sys_t::rw_trx_set with LF_HASH trx_sys_t::rw_trx_set is implemented as std::set, which does a few quite expensive operations under trx_sys_t::mutex protection: e.g. malloc/free when adding/removing elements. Traversing b-tree is not that cheap either. This has negative scalability impact, which is especially visible when running oltp_update_index.lua benchmark on a ramdisk. To reduce trx_sys_t::mutex contention std::set is replaced with LF_HASH. None of LF_HASH operations require trx_sys_t::mutex (nor any other global mutex) protection. Another interesting issue observed with std::set is reproducible ~2% performance decline after benchmark is ran for ~60 seconds. With LF_HASH results are stable. All in all this patch optimises away one of three trx_sys->mutex locks per oltp_update_index.lua query. The other two critical sections became smaller. Relevant clean-ups: Replaced rw_trx_set iteration at startup with local set. The latter is needed because values inserted to rw_trx_list must be ordered by trx->id. Removed redundant conditions from trx_reference(): it is (and even was) never called with transactions that have trx->state == TRX_STATE_COMMITTED_IN_MEMORY. do_ref_count doesn't (and probably even didn't) make any sense: now it is called only when reference counter increment is actually requested. Moved condition out of mutex in trx_erase_lists(). trx_rw_is_active(), trx_rw_is_active_low() and trx_get_rw_trx_by_id() were greatly simplified and replaced by appropriate trx_rw_hash_t methods. Compared to rw_trx_set, rw_trx_hash holds transactions only in PREPARED or ACTIVE states. Transactions in COMMITTED state were required to be found at InnoDB startup only. They are now looked up in the local set. Removed unused trx_assert_recovered(). Removed unused innobase_get_trx() declaration. Removed rather semantically incorrect trx_sys_rw_trx_add(). Moved information printout from trx_sys_init_at_db_start() to trx_lists_init_at_db_start(). --- storage/innobase/btr/btr0cur.cc | 6 +- storage/innobase/buf/buf0buf.cc | 3 - storage/innobase/handler/ha_innodb.cc | 14 +- storage/innobase/include/row0vers.h | 3 + storage/innobase/include/sync0sync.h | 1 + storage/innobase/include/sync0types.h | 2 + storage/innobase/include/trx0sys.h | 337 ++++++++++++++++++++++---- storage/innobase/include/trx0sys.ic | 142 ----------- storage/innobase/include/trx0trx.h | 20 +- storage/innobase/include/trx0trx.ic | 32 +-- storage/innobase/include/trx0types.h | 50 ---- storage/innobase/lock/lock0lock.cc | 61 ++--- storage/innobase/read/read0read.cc | 23 +- storage/innobase/row/row0row.cc | 4 +- storage/innobase/row/row0sel.cc | 10 +- storage/innobase/row/row0vers.cc | 38 ++- storage/innobase/sync/sync0debug.cc | 4 + storage/innobase/sync/sync0sync.cc | 1 + storage/innobase/trx/trx0sys.cc | 45 +--- storage/innobase/trx/trx0trx.cc | 173 ++++++++----- 20 files changed, 506 insertions(+), 463 deletions(-) diff --git a/storage/innobase/btr/btr0cur.cc b/storage/innobase/btr/btr0cur.cc index fc13537d36444..43163058418f2 100644 --- a/storage/innobase/btr/btr0cur.cc +++ b/storage/innobase/btr/btr0cur.cc @@ -482,10 +482,10 @@ btr_cur_instant_init_low(dict_index_t* index, mtr_t* mtr) /* In fact, because we only ever append fields to the 'default value' record, it is also OK to perform READ UNCOMMITTED and then ignore any extra fields, provided that - trx_rw_is_active(DB_TRX_ID). */ + trx_sys->rw_trx_hash.find(DB_TRX_ID). */ if (rec_offs_n_fields(offsets) > index->n_fields - && !trx_rw_is_active(row_get_rec_trx_id(rec, index, offsets), - NULL, false)) { + && !trx_sys->rw_trx_hash.find(row_get_rec_trx_id(rec, index, + offsets))) { goto inconsistent; } diff --git a/storage/innobase/buf/buf0buf.cc b/storage/innobase/buf/buf0buf.cc index df47a49152c44..93aa0a646d04e 100644 --- a/storage/innobase/buf/buf0buf.cc +++ b/storage/innobase/buf/buf0buf.cc @@ -443,9 +443,6 @@ bool buf_page_decrypt_after_read(buf_page_t* bpage, fil_space_t* space) MY_ATTRIBUTE((nonnull)); -/* prototypes for new functions added to ha_innodb.cc */ -trx_t* innobase_get_trx(); - /********************************************************************//** Gets the smallest oldest_modification lsn for any page in the pool. Returns zero if all modified pages have been flushed to disk. diff --git a/storage/innobase/handler/ha_innodb.cc b/storage/innobase/handler/ha_innodb.cc index 53aab5762022d..b01eb0463f14d 100644 --- a/storage/innobase/handler/ha_innodb.cc +++ b/storage/innobase/handler/ha_innodb.cc @@ -2819,13 +2819,17 @@ check_trx_exists( return(trx); } -/************************************************************************* -Gets current trx. */ -trx_t* -innobase_get_trx() +/** + Gets current trx. + + This function may be called during InnoDB initialisation, when + innodb_hton_ptr->slot is not yet set to meaningful value. +*/ + +trx_t *current_trx() { THD *thd=current_thd; - if (likely(thd != 0)) { + if (likely(thd != 0) && innodb_hton_ptr->slot != HA_SLOT_UNDEF) { trx_t*& trx = thd_to_trx(thd); return(trx); } else { diff --git a/storage/innobase/include/row0vers.h b/storage/innobase/include/row0vers.h index 576b53358f84f..ee0b1528683b8 100644 --- a/storage/innobase/include/row0vers.h +++ b/storage/innobase/include/row0vers.h @@ -41,6 +41,7 @@ class ReadView; /** Determine if an active transaction has inserted or modified a secondary index record. +@param[in,out] caller_trx trx of current thread @param[in] rec secondary index record @param[in] index secondary index @param[in] offsets rec_get_offsets(rec, index) @@ -48,6 +49,7 @@ index record. @retval NULL if the record was committed */ trx_t* row_vers_impl_x_locked( + trx_t* caller_trx, const rec_t* rec, dict_index_t* index, const ulint* offsets); @@ -126,6 +128,7 @@ which should be seen by a semi-consistent read. */ void row_vers_build_for_semi_consistent_read( /*====================================*/ + trx_t* caller_trx,/*!state to @@ -206,41 +198,6 @@ UNIV_INLINE trx_id_t trx_rw_min_trx_id(void); /*===================*/ -/****************************************************************//** -Checks if a rw transaction with the given id is active. -@return transaction instance if active, or NULL */ -UNIV_INLINE -trx_t* -trx_rw_is_active_low( -/*=================*/ - trx_id_t trx_id, /*!< in: trx id of the transaction */ - ibool* corrupt); /*!< in: NULL or pointer to a flag - that will be set if corrupt */ -/****************************************************************//** -Checks if a rw transaction with the given id is active. If the caller is -not holding trx_sys->mutex, the transaction may already have been -committed. -@return transaction instance if active, or NULL; */ -UNIV_INLINE -trx_t* -trx_rw_is_active( -/*=============*/ - trx_id_t trx_id, /*!< in: trx id of the transaction */ - ibool* corrupt, /*!< in: NULL or pointer to a flag - that will be set if corrupt */ - bool do_ref_count); /*!< in: if true then increment the - trx_t::n_ref_count */ -#if defined UNIV_DEBUG || defined UNIV_BLOB_LIGHT_DEBUG -/***********************************************************//** -Assert that a transaction has been recovered. -@return TRUE */ -UNIV_INLINE -ibool -trx_assert_recovered( -/*=================*/ - trx_id_t trx_id) /*!< in: transaction identifier */ - MY_ATTRIBUTE((warn_unused_result)); -#endif /* UNIV_DEBUG || UNIV_BLOB_LIGHT_DEBUG */ /*****************************************************************//** Updates the offset information about the end of the MySQL binlog entry which corresponds to the transaction just being committed. In a MySQL @@ -302,13 +259,6 @@ ulint trx_sys_any_active_transactions(void); /*=================================*/ -/** -Add the transaction to the RW transaction set -@param trx transaction instance to add */ -UNIV_INLINE -void -trx_sys_rw_trx_add(trx_t* trx); - #ifdef UNIV_DEBUG /*************************************************************//** Validate the trx_sys_t::rw_trx_list. @@ -497,6 +447,281 @@ FIL_PAGE_ARCH_LOG_NO_OR_SPACE_ID. */ #define TRX_SYS_DOUBLEWRITE_BLOCK_SIZE FSP_EXTENT_SIZE /* @} */ +trx_t* current_trx(); + +struct rw_trx_hash_element_t +{ + rw_trx_hash_element_t(): trx(0) + { + mutex_create(LATCH_ID_RW_TRX_HASH_ELEMENT, &mutex); + } + + + ~rw_trx_hash_element_t() + { + mutex_free(&mutex); + } + + + trx_id_t id; /* lf_hash_init() relies on this to be first in the struct */ + trx_t *trx; + ib_mutex_t mutex; +}; + + +/** + Wrapper around LF_HASH to store set of in memory read-write transactions. +*/ + +class rw_trx_hash_t +{ + LF_HASH hash; + + + /** + Constructor callback for lock-free allocator. + + Object is just allocated and is not yet accessible via rw_trx_hash by + concurrent threads. Object can be reused multiple times before it is freed. + Every time object is being reused initializer() callback is called. + */ + + static void rw_trx_hash_constructor(uchar *arg) + { + new(arg + LF_HASH_OVERHEAD) rw_trx_hash_element_t(); + } + + + /** + Destructor callback for lock-free allocator. + + Object is about to be freed and is not accessible via rw_trx_hash by + concurrent threads. + */ + + static void rw_trx_hash_destructor(uchar *arg) + { + reinterpret_cast + (arg + LF_HASH_OVERHEAD)->~rw_trx_hash_element_t(); + } + + + /** + Initializer callback for lock-free hash. + + Object is not yet accessible via rw_trx_hash by concurrent threads, but is + about to become such. Object id can be changed only by this callback and + remains the same until all pins to this object are released. + + Object trx can be changed to 0 by erase() under object mutex protection, + which indicates it is about to be removed from lock-free hash and become + not accessible by concurrent threads. + */ + + static void rw_trx_hash_initializer(LF_HASH *, + rw_trx_hash_element_t *element, + trx_t *trx) + { + element->trx= trx; + element->id= trx->id; + trx->rw_trx_hash_element= element; + } + + + /** + Gets LF_HASH pins. + + Pins are used to protect object from being destroyed or reused. They are + normally stored in trx object for quick access. If caller doesn't have trx + available, we try to get it using currnet_trx(). If caller doesn't have trx + at all, temporary pins are allocated. + */ + + LF_PINS *get_pins(trx_t *trx) + { + if (!trx->rw_trx_hash_pins) + { + trx->rw_trx_hash_pins= lf_hash_get_pins(&hash); + ut_a(trx->rw_trx_hash_pins); + } + return trx->rw_trx_hash_pins; + } + + +public: + void init() + { + lf_hash_init(&hash, sizeof(rw_trx_hash_element_t), LF_HASH_UNIQUE, 0, + sizeof(trx_id_t), 0, &my_charset_bin); + hash.alloc.constructor= rw_trx_hash_constructor; + hash.alloc.destructor= rw_trx_hash_destructor; + hash.initializer= + reinterpret_cast(rw_trx_hash_initializer); + } + + + void destroy() + { + lf_hash_destroy(&hash); + } + + + /** + Releases LF_HASH pins. + + Must be called by thread that owns trx_t object when the latter is being + "detached" from thread (e.g. released to the pool by trx_free()). Can be + called earlier if thread is expected not to use rw_trx_hash. + + Since pins are not allowed to be transferred to another thread, + initialisation thread calls this for recovered transactions. + */ + + void put_pins(trx_t *trx) + { + if (trx->rw_trx_hash_pins) + { + lf_hash_put_pins(trx->rw_trx_hash_pins); + trx->rw_trx_hash_pins= 0; + } + } + + + /** + Finds trx object in lock-free hash with given id. + + Only ACTIVE or PREPARED trx objects may participate in hash. Nevertheless + the transaction may get committed before this method returns. + + With do_ref_count == false the caller may dereference returned trx pointer + only if lock_sys->mutex was acquired before calling find(). + + With do_ref_count == true caller may dereference trx even if it is not + holding lock_sys->mutex. Caller is responsible for calling + trx_release_reference() when it is done playing with trx. + + Ideally this method should get caller rw_trx_hash_pins along with trx + object as a parameter, similar to insert() and erase(). However most + callers lose trx early in their call chains and it is not that easy to pass + them through. + + So we take more expensive approach: get trx through current_thd()->ha_data. + Some threads don't have trx attached to THD, and at least server + initialisation thread, fts_optimize_thread, srv_master_thread, + dict_stats_thread, srv_monitor_thread, btr_defragment_thread don't even + have THD at all. For such cases we allocate pins only for duration of + search and free them immediately. + + This has negative performance impact and should be fixed eventually (by + passing caller_trx as a parameter). Still stream of DML is more or less Ok. + + @return + @retval 0 not found + @retval pointer to trx + */ + + trx_t *find(trx_t *caller_trx, trx_id_t trx_id, bool do_ref_count= false) + { + /* + In MariaDB 10.3, purge will reset DB_TRX_ID to 0 + when the history is lost. Read/write transactions will + always have a nonzero trx_t::id; there the value 0 is + reserved for transactions that did not write or lock + anything yet. + */ + if (!trx_id) + return NULL; + + trx_t *trx= 0; + LF_PINS *pins= caller_trx ? get_pins(caller_trx) : lf_hash_get_pins(&hash); + ut_a(pins); + + rw_trx_hash_element_t *element= reinterpret_cast + (lf_hash_search(&hash, pins, reinterpret_cast(&trx_id), + sizeof(trx_id_t))); + if (element) + { + mutex_enter(&element->mutex); + lf_hash_search_unpin(pins); + if ((trx= element->trx)) + { + if (do_ref_count) + trx_reference(trx); +#ifdef UNIV_DEBUG + mutex_enter(&trx->mutex); + ut_ad(trx_state_eq(trx, TRX_STATE_ACTIVE) || + trx_state_eq(trx, TRX_STATE_PREPARED)); + mutex_exit(&trx->mutex); +#endif + } + mutex_exit(&element->mutex); + } + if (!caller_trx) + lf_hash_put_pins(pins); + return trx; + } + + + trx_t *find(trx_id_t trx_id, bool do_ref_count= false) + { + return find(current_trx(), trx_id, do_ref_count); + } + + + /** + Inserts trx to lock-free hash. + + Object becomes accessible via rw_trx_hash. + */ + + void insert(trx_t *trx) + { + ut_ad(trx_state_eq(trx, TRX_STATE_ACTIVE) || + trx_state_eq(trx, TRX_STATE_PREPARED)); + int res= lf_hash_insert(&hash, get_pins(trx), + reinterpret_cast(trx)); + ut_a(res == 0); + } + + + /** + Removes trx from lock-free hash. + + Object becomes not accessible via rw_trx_hash. But it still can be pinned + by concurrent find(), which is supposed to release it immediately after + it sees object trx is 0. + */ + + void erase(trx_t *trx) + { + ut_ad(trx_state_eq(trx, TRX_STATE_ACTIVE) || + trx_state_eq(trx, TRX_STATE_PREPARED)); + mutex_enter(&trx->rw_trx_hash_element->mutex); + trx->rw_trx_hash_element->trx= 0; + mutex_exit(&trx->rw_trx_hash_element->mutex); + int res= lf_hash_delete(&hash, get_pins(trx), + reinterpret_cast(&trx->id), + sizeof(trx_id_t)); + ut_a(res == 0); + } + + + /** + Returns the number of elements in the hash. + + The number is exact only if hash is protected against concurrent + modifications (e.g. single threaded startup or hash is protected + by some mutex). Otherwise the number may be used as a hint only, + because it may change even before this method returns. + */ + + int32_t size() + { + return my_atomic_load32_explicit(&hash.count, MY_MEMORY_ORDER_RELAXED); + } +}; + + /** The transaction system central memory data structure. */ struct trx_sys_t { @@ -569,8 +794,16 @@ struct trx_sys_t { transactions), protected by rseg->mutex */ - TrxIdSet rw_trx_set; /*!< Mapping from transaction id - to transaction instance */ + const char rw_trx_hash_pre_pad[CACHE_LINE_SIZE]; + + + /** + Lock-free hash of in memory read-write transactions. + Works faster when it is on it's own cache line (tested). + */ + + rw_trx_hash_t rw_trx_hash; + const char rw_trx_hash_post_pad[CACHE_LINE_SIZE]; ulint n_prepared_trx; /*!< Number of transactions currently in the XA PREPARED state */ diff --git a/storage/innobase/include/trx0sys.ic b/storage/innobase/include/trx0sys.ic index 020a46fb028a1..5c2a913554ac1 100644 --- a/storage/innobase/include/trx0sys.ic +++ b/storage/innobase/include/trx0sys.ic @@ -192,32 +192,6 @@ trx_write_trx_id( mach_write_to_6(ptr, id); } -/****************************************************************//** -Looks for the trx handle with the given id in rw_trx_list. -The caller must be holding trx_sys->mutex. -@return the trx handle or NULL if not found; -the pointer must not be dereferenced unless lock_sys->mutex was -acquired before calling this function and is still being held */ -UNIV_INLINE -trx_t* -trx_get_rw_trx_by_id( -/*=================*/ - trx_id_t trx_id) /*!< in: trx id to search for */ -{ - ut_ad(trx_id > 0); - ut_ad(trx_sys_mutex_own()); - - if (trx_sys->rw_trx_set.empty()) { - return(NULL); - } - - TrxIdSet::iterator it; - - it = trx_sys->rw_trx_set.find(TrxTrack(trx_id)); - - return(it == trx_sys->rw_trx_set.end() ? NULL : it->m_trx); -} - /****************************************************************//** Returns the minimum trx id in trx list. This is the smallest id for which the trx can possibly be active. (But, you must look at the trx->state @@ -245,29 +219,6 @@ trx_rw_min_trx_id_low(void) return(id); } -#if defined UNIV_DEBUG || defined UNIV_BLOB_LIGHT_DEBUG -/***********************************************************//** -Assert that a transaction has been recovered. -@return TRUE */ -UNIV_INLINE -ibool -trx_assert_recovered( -/*=================*/ - trx_id_t trx_id) /*!< in: transaction identifier */ -{ - const trx_t* trx; - - trx_sys_mutex_enter(); - - trx = trx_get_rw_trx_by_id(trx_id); - ut_a(trx->is_recovered); - - trx_sys_mutex_exit(); - - return(TRUE); -} -#endif /* UNIV_DEBUG || UNIV_BLOB_LIGHT_DEBUG */ - /****************************************************************//** Returns the minimum trx id in rw trx list. This is the smallest id for which the rw trx can possibly be active. (But, you must look at the trx->state @@ -288,86 +239,6 @@ trx_rw_min_trx_id(void) return(id); } -/****************************************************************//** -Checks if a rw transaction with the given id is active. If the caller is -not holding lock_sys->mutex, the transaction may already have been committed. -@return transaction instance if active, or NULL */ -UNIV_INLINE -trx_t* -trx_rw_is_active_low( -/*=================*/ - trx_id_t trx_id, /*!< in: trx id of the transaction */ - ibool* corrupt) /*!< in: NULL or pointer to a flag - that will be set if corrupt */ -{ - trx_t* trx; - - ut_ad(trx_sys_mutex_own()); - - if (trx_id < trx_rw_min_trx_id_low()) { - - trx = NULL; - } else if (trx_id >= trx_sys->max_trx_id) { - - /* There must be corruption: we let the caller handle the - diagnostic prints in this case. */ - - trx = NULL; - if (corrupt != NULL) { - *corrupt = TRUE; - } - } else { - trx = trx_get_rw_trx_by_id(trx_id); - - if (trx != NULL - && trx_state_eq(trx, TRX_STATE_COMMITTED_IN_MEMORY)) { - - trx = NULL; - } - } - - return(trx); -} - -/****************************************************************//** -Checks if a rw transaction with the given id is active. If the caller is -not holding lock_sys->mutex, the transaction may already have been -committed. -@return transaction instance if active, or NULL; */ -UNIV_INLINE -trx_t* -trx_rw_is_active( -/*=============*/ - trx_id_t trx_id, /*!< in: trx id of the transaction */ - ibool* corrupt, /*!< in: NULL or pointer to a flag - that will be set if corrupt */ - bool do_ref_count) /*!< in: if true then increment the - trx_t::n_ref_count */ -{ - if (!trx_id) { - /* In MariaDB 10.3, purge will reset DB_TRX_ID to 0 - when the history is lost. Read/write transactions will - always have a nonzero trx_t::id; there the value 0 is - reserved for transactions that did not write or lock - anything yet. */ - return NULL; - } - - trx_t* trx; - - trx_sys_mutex_enter(); - - trx = trx_rw_is_active_low(trx_id, corrupt); - - if (trx != 0) { - trx = trx_reference(trx, do_ref_count); - } - - trx_sys_mutex_exit(); - - return(trx); -} - /*****************************************************************//** Allocates a new transaction id. @return new, allocated trx id */ @@ -441,16 +312,3 @@ trx_sys_get_n_rw_trx(void) return(n_trx); } - -/** -Add the transaction to the RW transaction set -@param trx transaction instance to add */ -UNIV_INLINE -void -trx_sys_rw_trx_add(trx_t* trx) -{ - ut_ad(trx->id != 0); - - trx_sys->rw_trx_set.insert(TrxTrack(trx->id, trx)); - ut_d(trx->in_rw_trx_list = true); -} diff --git a/storage/innobase/include/trx0trx.h b/storage/innobase/include/trx0trx.h index 81ef6093e2ee7..9366d20b1f522 100644 --- a/storage/innobase/include/trx0trx.h +++ b/storage/innobase/include/trx0trx.h @@ -55,6 +55,8 @@ class ReadView; // Forward declaration class FlushObserver; +struct rw_trx_hash_element_t; + /** Dummy session used currently in MySQL interface */ extern sess_t* trx_dummy_sess; @@ -531,17 +533,12 @@ trx_set_rw_mode( trx_t* trx); /** -Increase the reference count. If the transaction is in state -TRX_STATE_COMMITTED_IN_MEMORY then the transaction is considered -committed and the reference count is not incremented. -@param trx Transaction that is being referenced -@param do_ref_count Increment the reference iff this is true -@return transaction instance if it is not committed */ +Increase the reference count. +@param trx Transaction that is being referenced */ UNIV_INLINE -trx_t* +void trx_reference( - trx_t* trx, - bool do_ref_count); + trx_t* trx); /** Release the transaction. Decrease the reference count. @@ -951,6 +948,9 @@ struct trx_t { Recovered XA: * NOT_STARTED -> PREPARED -> COMMITTED -> (freed) + Recovered XA followed by XA ROLLBACK: + * NOT_STARTED -> PREPARED -> ACTIVE -> COMMITTED -> (freed) + XA (2PC) (shutdown or disconnect before ROLLBACK or COMMIT): * NOT_STARTED -> PREPARED -> (freed) @@ -1277,6 +1277,8 @@ struct trx_t { os_event_t wsrep_event; /* event waited for in srv_conc_slot */ #endif /* WITH_WSREP */ + rw_trx_hash_element_t *rw_trx_hash_element; + LF_PINS *rw_trx_hash_pins; ulint magic_n; /** @return whether any persistent undo log has been generated */ diff --git a/storage/innobase/include/trx0trx.ic b/storage/innobase/include/trx0trx.ic index 6fa00c5333f27..cd0b812869db4 100644 --- a/storage/innobase/include/trx0trx.ic +++ b/storage/innobase/include/trx0trx.ic @@ -214,32 +214,14 @@ ok: } /** -Increase the reference count. If the transaction is in state -TRX_STATE_COMMITTED_IN_MEMORY then the transaction is considered -committed and the reference count is not incremented. -@param trx Transaction that is being referenced -@param do_ref_count Increment the reference iff this is true -@return transaction instance if it is not committed */ -UNIV_INLINE -trx_t* -trx_reference( - trx_t* trx, - bool do_ref_count) +Increase the reference count. +@param trx Transaction that is being referenced */ +UNIV_INLINE void trx_reference(trx_t *trx) { - trx_mutex_enter(trx); - - if (trx_state_eq(trx, TRX_STATE_COMMITTED_IN_MEMORY)) { - trx_mutex_exit(trx); - trx = NULL; - } else if (do_ref_count) { - ut_ad(trx->n_ref >= 0); - ++trx->n_ref; - trx_mutex_exit(trx); - } else { - trx_mutex_exit(trx); - } - - return(trx); + trx_mutex_enter(trx); + ut_ad(trx->n_ref >= 0); + ++trx->n_ref; + trx_mutex_exit(trx); } /** diff --git a/storage/innobase/include/trx0types.h b/storage/innobase/include/trx0types.h index 8092246c7fa43..2b080eea2d310 100644 --- a/storage/innobase/include/trx0types.h +++ b/storage/innobase/include/trx0types.h @@ -31,12 +31,9 @@ Created 3/26/1996 Heikki Tuuri #include "ut0mutex.h" #include "ut0new.h" -#include #include #include -//#include - /** printf(3) format used for printing DB_TRX_ID and other system fields */ #define TRX_ID_FMT IB_ID_FMT @@ -173,51 +170,4 @@ typedef ib_mutex_t PQMutex; typedef ib_mutex_t TrxSysMutex; typedef std::vector > trx_ids_t; - -/** Mapping read-write transactions from id to transaction instance, for -creating read views and during trx id lookup for MVCC and locking. */ -struct TrxTrack { - explicit TrxTrack(trx_id_t id, trx_t* trx = NULL) - : - m_id(id), - m_trx(trx) - { - // Do nothing - } - - trx_id_t m_id; - trx_t* m_trx; -}; - -struct TrxTrackHash { - size_t operator()(const TrxTrack& key) const - { - return(size_t(key.m_id)); - } -}; - -/** -Comparator for TrxMap */ -struct TrxTrackHashCmp { - - bool operator() (const TrxTrack& lhs, const TrxTrack& rhs) const - { - return(lhs.m_id == rhs.m_id); - } -}; - -/** -Comparator for TrxMap */ -struct TrxTrackCmp { - - bool operator() (const TrxTrack& lhs, const TrxTrack& rhs) const - { - return(lhs.m_id < rhs.m_id); - } -}; - -//typedef std::unordered_set TrxIdSet; -typedef std::set > - TrxIdSet; - #endif /* trx0types_h */ diff --git a/storage/innobase/lock/lock0lock.cc b/storage/innobase/lock/lock0lock.cc index 2f66e178daf7c..269f606250f35 100644 --- a/storage/innobase/lock/lock0lock.cc +++ b/storage/innobase/lock/lock0lock.cc @@ -1494,6 +1494,7 @@ static trx_t* lock_sec_rec_some_has_impl( /*=======================*/ + trx_t* caller_trx,/*!mutex); - if (trx_t* impl_trx = trx_rw_is_active(trx->id, NULL, false)) { - ulint heap_no = page_rec_get_heap_no(rec); - mutex_enter(&trx_sys->mutex); - - for (trx_t* t = UT_LIST_GET_FIRST(trx_sys->rw_trx_list); - t != NULL; - t = UT_LIST_GET_NEXT(trx_list, t)) { + for (trx_t* t = UT_LIST_GET_FIRST(trx_sys->rw_trx_list); + t != NULL; + t = UT_LIST_GET_NEXT(trx_list, t)) { - lock_t* expl_lock = lock_rec_has_expl( - precise_mode, block, heap_no, t); + lock_t* expl_lock = lock_rec_has_expl( + precise_mode, block, heap_no, t); - if (expl_lock && expl_lock->trx != impl_trx) { - /* An explicit lock is held by trx other than - the trx holding the implicit lock. */ - holds = expl_lock->trx; - break; - } + if (expl_lock && expl_lock->trx != trx) { + /* An explicit lock is held by trx other than + the trx holding the implicit lock. */ + holds = expl_lock->trx; + break; } - - mutex_exit(&trx_sys->mutex); } + mutex_exit(&trx_sys->mutex); + lock_mutex_exit(); return(holds); @@ -6223,7 +6226,6 @@ lock_rec_queue_validate( const dict_index_t* index, /*!< in: index, or NULL if not known */ const ulint* offsets)/*!< in: rec_get_offsets(rec, index) */ { - const trx_t* impl_trx; const lock_t* lock; ulint heap_no; @@ -6269,13 +6271,11 @@ lock_rec_queue_validate( /* Nothing we can do */ } else if (dict_index_is_clust(index)) { - trx_id_t trx_id; - /* Unlike the non-debug code, this invariant can only succeed if the check and assertion are covered by the lock mutex. */ - trx_id = lock_clust_rec_some_has_impl(rec, index, offsets); - impl_trx = trx_rw_is_active_low(trx_id, NULL); + const trx_t *impl_trx = trx_sys->rw_trx_hash.find( + lock_clust_rec_some_has_impl(rec, index, offsets)); ut_ad(lock_mutex_own()); /* impl_trx cannot be committed until lock_mutex_exit() @@ -6849,6 +6849,7 @@ static void lock_rec_convert_impl_to_expl( /*==========================*/ + trx_t* caller_trx,/*!rw_trx_hash.find(caller_trx, trx_id, true); } else { ut_ad(!dict_index_is_online_ddl(index)); - trx = lock_sec_rec_some_has_impl(rec, index, offsets); + trx = lock_sec_rec_some_has_impl(caller_trx, rec, index, + offsets); ut_ad(!trx || !lock_rec_other_trx_holds_expl( LOCK_S | LOCK_REC_NOT_GAP, trx, rec, block)); @@ -6934,7 +6936,8 @@ lock_clust_rec_modify_check_and_lock( /* If a transaction has no explicit x-lock set on the record, set one for it */ - lock_rec_convert_impl_to_expl(block, rec, index, offsets); + lock_rec_convert_impl_to_expl(thr_get_trx(thr), block, rec, index, + offsets); lock_mutex_enter(); @@ -7098,7 +7101,8 @@ lock_sec_rec_read_check_and_lock( || recv_recovery_is_on()) && !page_rec_is_supremum(rec)) { - lock_rec_convert_impl_to_expl(block, rec, index, offsets); + lock_rec_convert_impl_to_expl(thr_get_trx(thr), block, rec, + index, offsets); } lock_mutex_enter(); @@ -7173,7 +7177,8 @@ lock_clust_rec_read_check_and_lock( if (heap_no != PAGE_HEAP_NO_SUPREMUM) { - lock_rec_convert_impl_to_expl(block, rec, index, offsets); + lock_rec_convert_impl_to_expl(thr_get_trx(thr), block, rec, + index, offsets); } lock_mutex_enter(); diff --git a/storage/innobase/read/read0read.cc b/storage/innobase/read/read0read.cc index 2fb7083b0b2f8..8f4dd4f37b2d9 100644 --- a/storage/innobase/read/read0read.cc +++ b/storage/innobase/read/read0read.cc @@ -371,6 +371,7 @@ Copy the transaction ids from the source vector */ void ReadView::copy_trx_ids(const trx_ids_t& trx_ids) { + ut_ad(mutex_own(&trx_sys->mutex)); ulint size = trx_ids.size(); if (m_creator_trx_id > 0) { @@ -424,14 +425,24 @@ ReadView::copy_trx_ids(const trx_ids_t& trx_ids) } #ifdef UNIV_DEBUG - /* Assert that all transaction ids in list are active. */ + /* Original assertion was here to make sure that rw_trx_ids and + rw_trx_hash are in sync and they hold either ACTIVE or PREPARED + transaction. + + Now rw_trx_hash.find() does + ut_ad(trx_state_eq(trx, TRX_STATE_ACTIVE) || + trx_state_eq(trx, TRX_STATE_PREPARED)). + No need to repeat it here. We even can't repeat it here: it'll be race + condition because we need trx->element->mutex locked to perform this + check (see how it is done in find()). + + Now rw_trx_ids and rw_trx_hash may get out of sync for a short while: + when transaction is registered it first gets added into rw_trx_ids + under trx_sys->mutex protection and then to rw_trx_hash without mutex + protection. Thus we need repeat this lookup. */ for (trx_ids_t::const_iterator it = trx_ids.begin(); it != trx_ids.end(); ++it) { - - trx_t* trx = trx_get_rw_trx_by_id(*it); - ut_ad(trx != NULL); - ut_ad(trx->state == TRX_STATE_ACTIVE - || trx->state == TRX_STATE_PREPARED); + while (!trx_sys->rw_trx_hash.find(*it)); } #endif /* UNIV_DEBUG */ } diff --git a/storage/innobase/row/row0row.cc b/storage/innobase/row/row0row.cc index 3611b8c3fc7d6..3f6c8b9f918ad 100644 --- a/storage/innobase/row/row0row.cc +++ b/storage/innobase/row/row0row.cc @@ -415,8 +415,8 @@ row_build_low( times, and the cursor restore can happen multiple times for single insert or update statement. */ ut_a(!rec_offs_any_null_extern(rec, offsets) - || trx_rw_is_active(row_get_rec_trx_id(rec, index, offsets), - NULL, false)); + || trx_sys->rw_trx_hash.find(row_get_rec_trx_id(rec, index, + offsets))); #endif /* UNIV_DEBUG || UNIV_BLOB_LIGHT_DEBUG */ if (type != ROW_COPY_POINTERS) { diff --git a/storage/innobase/row/row0sel.cc b/storage/innobase/row/row0sel.cc index 53f60f2f70b66..eeb52bfeee316 100644 --- a/storage/innobase/row/row0sel.cc +++ b/storage/innobase/row/row0sel.cc @@ -799,7 +799,7 @@ row_sel_build_committed_vers_for_mysql( rec_offs_size(*offsets)); } - row_vers_build_for_semi_consistent_read( + row_vers_build_for_semi_consistent_read(prebuilt->trx, rec, mtr, clust_index, offsets, offset_heap, prebuilt->old_vers_heap, old_vers, vrow); } @@ -4972,17 +4972,17 @@ row_search_mvcc( /* In delete-marked records, DB_TRX_ID must always refer to an existing undo log record. */ ut_ad(trx_id); - if (!trx_rw_is_active(trx_id, NULL, false)) { + if (!trx_sys->rw_trx_hash.find(trx, trx_id)) { /* The clustered index record was delete-marked in a committed transaction. Ignore the record. */ goto locks_ok_del_marked; } - } else if (trx_t* trx = row_vers_impl_x_locked( - rec, index, offsets)) { + } else if (trx_t* t = row_vers_impl_x_locked( + trx, rec, index, offsets)) { /* The record belongs to an active transaction. We must acquire a lock. */ - trx_release_reference(trx); + trx_release_reference(t); } else { /* The secondary index record does not point to a delete-marked clustered index diff --git a/storage/innobase/row/row0vers.cc b/storage/innobase/row/row0vers.cc index 712c46cba35ad..0b8cf63f94091 100644 --- a/storage/innobase/row/row0vers.cc +++ b/storage/innobase/row/row0vers.cc @@ -47,6 +47,7 @@ Created 2/6/1997 Heikki Tuuri /** Check whether all non-virtual columns in a virtual index match that of in the cluster index +@param[in,out] caller_trx trx of current thread @param[in] index the secondary index @param[in] row the cluster index row in dtuple form @param[in] ext externally stored column prefix or NULL @@ -65,6 +66,7 @@ row_vers_non_vc_match( ulint* n_non_v_col); /** Determine if an active transaction has inserted or modified a secondary index record. +@param[in,out] caller_trx trx of current thread @param[in] clust_rec clustered index record @param[in] clust_index clustered index @param[in] rec secondary index record @@ -76,6 +78,7 @@ index record. UNIV_INLINE trx_t* row_vers_impl_x_locked_low( + trx_t* caller_trx, const rec_t* clust_rec, dict_index_t* clust_index, const rec_t* rec, @@ -84,7 +87,6 @@ row_vers_impl_x_locked_low( mtr_t* mtr) { trx_id_t trx_id; - ibool corrupt; ulint comp; ulint rec_del; const rec_t* version; @@ -118,13 +120,15 @@ row_vers_impl_x_locked_low( mem_heap_free(heap); DBUG_RETURN(0); } - corrupt = FALSE; - trx_t* trx = trx_rw_is_active(trx_id, &corrupt, true); + trx_t* trx = trx_sys->rw_trx_hash.find(caller_trx, trx_id, true); if (trx == 0) { /* The transaction that modified or inserted clust_rec is no longer active, or it is corrupt: no implicit lock on rec */ + trx_sys_mutex_enter(); + bool corrupt = trx_id >= trx_sys->max_trx_id; + trx_sys_mutex_exit(); if (corrupt) { lock_report_trx_id_insanity( trx_id, clust_rec, clust_index, clust_offsets, @@ -189,7 +193,7 @@ row_vers_impl_x_locked_low( inserting a delete-marked record. */ ut_ad(prev_version || !rec_get_deleted_flag(version, comp) - || !trx_rw_is_active(trx_id, NULL, false)); + || !trx_sys->rw_trx_hash.find(caller_trx, trx_id)); /* Free version and clust_offsets. */ mem_heap_free(old_heap); @@ -342,6 +346,7 @@ row_vers_impl_x_locked_low( /** Determine if an active transaction has inserted or modified a secondary index record. +@param[in,out] caller_trx trx of current thread @param[in] rec secondary index record @param[in] index secondary index @param[in] offsets rec_get_offsets(rec, index) @@ -349,6 +354,7 @@ index record. @retval NULL if the record was committed */ trx_t* row_vers_impl_x_locked( + trx_t* caller_trx, const rec_t* rec, dict_index_t* index, const ulint* offsets) @@ -389,7 +395,8 @@ row_vers_impl_x_locked( trx = 0; } else { trx = row_vers_impl_x_locked_low( - clust_rec, clust_index, rec, index, offsets, &mtr); + caller_trx, clust_rec, clust_index, rec, index, + offsets, &mtr); ut_ad(trx == 0 || trx_is_referenced(trx)); } @@ -1234,6 +1241,7 @@ which should be seen by a semi-consistent read. */ void row_vers_build_for_semi_consistent_read( /*====================================*/ + trx_t* caller_trx,/*!mutex. It may change from - ACTIVE to PREPARED or COMMITTED. */ - if (version_trx - && trx_state_eq(version_trx, - TRX_STATE_COMMITTED_IN_MEMORY)) { - version_trx = NULL; - } - trx_sys_mutex_exit(); - - if (!version_trx) { + if (!trx_sys->rw_trx_hash.find(caller_trx, version_trx_id)) { committed_version_trx: /* We found a version that belongs to a committed transaction: return it. */ diff --git a/storage/innobase/sync/sync0debug.cc b/storage/innobase/sync/sync0debug.cc index 76796f969132e..e61c01c044392 100644 --- a/storage/innobase/sync/sync0debug.cc +++ b/storage/innobase/sync/sync0debug.cc @@ -480,6 +480,7 @@ LatchDebug::LatchDebug() LEVEL_MAP_INSERT(SYNC_REC_LOCK); LEVEL_MAP_INSERT(SYNC_THREADS); LEVEL_MAP_INSERT(SYNC_TRX); + LEVEL_MAP_INSERT(SYNC_RW_TRX_HASH_ELEMENT); LEVEL_MAP_INSERT(SYNC_TRX_SYS); LEVEL_MAP_INSERT(SYNC_LOCK_SYS); LEVEL_MAP_INSERT(SYNC_LOCK_WAIT_SYS); @@ -761,6 +762,7 @@ LatchDebug::check_order( case SYNC_THREADS: case SYNC_LOCK_SYS: case SYNC_LOCK_WAIT_SYS: + case SYNC_RW_TRX_HASH_ELEMENT: case SYNC_TRX_SYS: case SYNC_IBUF_BITMAP_MUTEX: case SYNC_REDO_RSEG: @@ -1521,6 +1523,8 @@ sync_latch_meta_init() PFS_NOT_INSTRUMENTED); LATCH_ADD_MUTEX(FIL_CRYPT_THREADS_MUTEX, SYNC_NO_ORDER_CHECK, PFS_NOT_INSTRUMENTED); + LATCH_ADD_MUTEX(RW_TRX_HASH_ELEMENT, SYNC_RW_TRX_HASH_ELEMENT, + rw_trx_hash_element_mutex_key); latch_id_t id = LATCH_ID_NONE; diff --git a/storage/innobase/sync/sync0sync.cc b/storage/innobase/sync/sync0sync.cc index 5d98e49fd6dae..e78b2a033acb6 100644 --- a/storage/innobase/sync/sync0sync.cc +++ b/storage/innobase/sync/sync0sync.cc @@ -96,6 +96,7 @@ mysql_pfs_key_t sync_array_mutex_key; mysql_pfs_key_t thread_mutex_key; mysql_pfs_key_t zip_pad_mutex_key; mysql_pfs_key_t row_drop_list_mutex_key; +mysql_pfs_key_t rw_trx_hash_element_mutex_key; #endif /* UNIV_PFS_MUTEX */ #ifdef UNIV_PFS_RWLOCK mysql_pfs_key_t btr_search_latch_key; diff --git a/storage/innobase/trx/trx0sys.cc b/storage/innobase/trx/trx0sys.cc index c2e15fea95337..ef69bd3102bc9 100644 --- a/storage/innobase/trx/trx0sys.cc +++ b/storage/innobase/trx/trx0sys.cc @@ -429,8 +429,6 @@ void trx_sys_init_at_db_start() { trx_sysf_t* sys_header; - ib_uint64_t rows_to_undo = 0; - const char* unit = ""; /* VERY important: after the database is started, max_trx_id value is divisible by TRX_SYS_TRX_ID_WRITE_MARGIN, and the 'if' in @@ -455,43 +453,6 @@ trx_sys_init_at_db_start() trx_dummy_sess = sess_open(); trx_lists_init_at_db_start(); - - /* This mutex is not strictly required, it is here only to satisfy - the debug code (assertions). We are still running in single threaded - bootstrap mode. */ - - trx_sys_mutex_enter(); - - if (UT_LIST_GET_LEN(trx_sys->rw_trx_list) > 0) { - const trx_t* trx; - - for (trx = UT_LIST_GET_FIRST(trx_sys->rw_trx_list); - trx != NULL; - trx = UT_LIST_GET_NEXT(trx_list, trx)) { - - ut_ad(trx->is_recovered); - assert_trx_in_rw_list(trx); - - if (trx_state_eq(trx, TRX_STATE_ACTIVE)) { - rows_to_undo += trx->undo_no; - } - } - - if (rows_to_undo > 1000000000) { - unit = "M"; - rows_to_undo = rows_to_undo / 1000000; - } - - ib::info() << UT_LIST_GET_LEN(trx_sys->rw_trx_list) - << " transaction(s) which must be rolled back or" - " cleaned up in total " << rows_to_undo << unit - << " row operations to undo"; - - ib::info() << "Trx id counter is " << trx_sys->max_trx_id; - } - - trx_sys_mutex_exit(); - trx_sys->mvcc->clone_oldest_view(&purge_sys->view); } @@ -515,8 +476,7 @@ trx_sys_create(void) new(&trx_sys->rw_trx_ids) trx_ids_t(ut_allocator( mem_key_trx_sys_t_rw_trx_ids)); - - new(&trx_sys->rw_trx_set) TrxIdSet(); + trx_sys->rw_trx_hash.init(); } /*****************************************************************//** @@ -669,8 +629,7 @@ trx_sys_close(void) trx_sys->rw_trx_ids.~trx_ids_t(); - trx_sys->rw_trx_set.~TrxIdSet(); - + trx_sys->rw_trx_hash.destroy(); ut_free(trx_sys); trx_sys = NULL; diff --git a/storage/innobase/trx/trx0trx.cc b/storage/innobase/trx/trx0trx.cc index fde7443394e7c..d0f79bad3a337 100644 --- a/storage/innobase/trx/trx0trx.cc +++ b/storage/innobase/trx/trx0trx.cc @@ -236,6 +236,7 @@ struct TrxFactory { new(&trx->hit_list) hit_list_t(); + trx->rw_trx_hash_pins = 0; trx_init(trx); DBUG_LOG("trx", "Init: " << trx); @@ -446,6 +447,7 @@ trx_create_low() /* We just got trx from pool, it should be non locking */ ut_ad(trx->will_lock == 0); + ut_ad(!trx->rw_trx_hash_pins); /* Background trx should not be forced to rollback, we will unset the flag for user trx. */ @@ -483,6 +485,7 @@ trx_free(trx_t*& trx) { assert_trx_is_free(trx); + trx_sys->rw_trx_hash.put_pins(trx); trx->mysql_thd = 0; trx->mysql_log_file_name = 0; @@ -731,7 +734,9 @@ trx_resurrect_table_locks( trx_undo_rec_t* undo_rec; table_id_set tables; - if (trx_state_eq(trx, TRX_STATE_COMMITTED_IN_MEMORY) || undo->empty) { + ut_ad(trx_state_eq(trx, TRX_STATE_ACTIVE) || + trx_state_eq(trx, TRX_STATE_PREPARED)); + if (undo->empty) { return; } @@ -960,10 +965,65 @@ trx_resurrect_update( } } +/** Mapping read-write transactions from id to transaction instance, for +creating read views and during trx id lookup for MVCC and locking. */ +struct TrxTrack { + explicit TrxTrack(trx_id_t id, trx_t* trx = NULL) + : + m_id(id), + m_trx(trx) + { + // Do nothing + } + + trx_id_t m_id; + trx_t* m_trx; +}; + +/** +Comparator for TrxMap */ +struct TrxTrackCmp { + + bool operator() (const TrxTrack& lhs, const TrxTrack& rhs) const + { + return(lhs.m_id < rhs.m_id); + } +}; + +typedef std::set > + TrxIdSet; + +static inline void trx_sys_add_trx_at_init(trx_t *trx, trx_undo_t *undo, + uint64_t *rows_to_undo, + TrxIdSet *set) +{ + ut_ad(trx->id != 0); + ut_ad(trx->is_recovered); + + set->insert(TrxTrack(trx->id, trx)); + if (trx_state_eq(trx, TRX_STATE_ACTIVE) || + trx_state_eq(trx, TRX_STATE_PREPARED)) + { + trx_sys->rw_trx_hash.insert(trx); + trx_sys->rw_trx_hash.put_pins(trx); + trx_sys->rw_trx_ids.push_back(trx->id); + trx_resurrect_table_locks(trx, undo); + if (trx_state_eq(trx, TRX_STATE_ACTIVE)) + *rows_to_undo+= trx->undo_no; + } +#ifdef UNIV_DEBUG + trx->in_rw_trx_list= true; + if (trx->id > trx_sys->rw_max_trx_id) + trx_sys->rw_max_trx_id= trx->id; +#endif +} + /** Initialize (resurrect) transactions at startup. */ void trx_lists_init_at_db_start() { + TrxIdSet set; + uint64_t rows_to_undo = 0; ut_a(srv_is_being_started); ut_ad(!srv_was_started); ut_ad(!purge_sys); @@ -993,15 +1053,10 @@ trx_lists_init_at_db_start() for (undo = UT_LIST_GET_FIRST(rseg->old_insert_list); undo != NULL; undo = UT_LIST_GET_NEXT(undo_list, undo)) { - - trx_t* trx; - - trx = trx_resurrect_insert(undo, rseg); + trx_t* trx = trx_resurrect_insert(undo, rseg); trx->start_time = start_time; - - trx_sys_rw_trx_add(trx); - - trx_resurrect_table_locks(trx, undo); + trx_sys_add_trx_at_init(trx, undo, &rows_to_undo, + &set); } /* Ressurrect other transactions. */ @@ -1009,12 +1064,10 @@ trx_lists_init_at_db_start() undo != NULL; undo = UT_LIST_GET_NEXT(undo_list, undo)) { - /* Check the trx_sys->rw_trx_set first. */ - trx_sys_mutex_enter(); - - trx_t* trx = trx_get_rw_trx_by_id(undo->trx_id); - - trx_sys_mutex_exit(); + /* Check if trx_id was already registered first. */ + TrxIdSet::iterator it = + set.find(TrxTrack(undo->trx_id)); + trx_t *trx= it == set.end() ? 0 : it->m_trx; if (trx == NULL) { trx = trx_allocate_for_background(); @@ -1025,34 +1078,30 @@ trx_lists_init_at_db_start() } trx_resurrect_update(trx, undo, rseg); - - trx_sys_rw_trx_add(trx); - - trx_resurrect_table_locks(trx, undo); + trx_sys_add_trx_at_init(trx, undo, &rows_to_undo, + &set); } } - TrxIdSet::iterator end = trx_sys->rw_trx_set.end(); + if (set.size()) { - for (TrxIdSet::iterator it = trx_sys->rw_trx_set.begin(); - it != end; - ++it) { + ib::info() << set.size() + << " transaction(s) which must be rolled back or" + " cleaned up in total " << rows_to_undo + << " row operations to undo"; - ut_ad(it->m_trx->in_rw_trx_list); -#ifdef UNIV_DEBUG - if (it->m_trx->id > trx_sys->rw_max_trx_id) { - trx_sys->rw_max_trx_id = it->m_trx->id; - } -#endif /* UNIV_DEBUG */ + ib::info() << "Trx id counter is " << trx_sys->max_trx_id; + } - if (it->m_trx->state == TRX_STATE_ACTIVE - || it->m_trx->state == TRX_STATE_PREPARED) { + TrxIdSet::iterator end = set.end(); - trx_sys->rw_trx_ids.push_back(it->m_id); - } + for (TrxIdSet::iterator it = set.begin(); + it != end; + ++it) { UT_LIST_ADD_FIRST(trx_sys->rw_trx_list, it->m_trx); } + std::sort(trx_sys->rw_trx_ids.begin(), trx_sys->rw_trx_ids.end()); } /** Assign a persistent rollback segment in a round-robin fashion, @@ -1171,8 +1220,8 @@ trx_t::assign_temp_rseg() mutex_enter(&trx_sys->mutex); id = trx_sys_get_new_trx_id(); trx_sys->rw_trx_ids.push_back(id); - trx_sys->rw_trx_set.insert(TrxTrack(id, this)); mutex_exit(&trx_sys->mutex); + trx_sys->rw_trx_hash.insert(this); } ut_ad(!rseg->is_persistent()); @@ -1237,10 +1286,14 @@ trx_start_low( ut_ad(!trx->in_rw_trx_list); - /* We tend to over assert and that complicates the code somewhat. - e.g., the transaction state can be set earlier but we are forced to - set it under the protection of the trx_sys_t::mutex because some - trx list assertions are triggered unnecessarily. */ + /* No other thread can access this trx object through rw_trx_hash, thus + we don't need trx_sys->mutex protection for that purpose. Still this + trx can be found through trx_sys->mysql_trx_list, which means state + change must be protected by e.g. trx->mutex. + + For now we update it without mutex protection, because original code + did it this way. It has to be reviewed and fixed properly. */ + trx->state = TRX_STATE_ACTIVE; /* By default all transactions are in the read-only list unless they are non-locking auto-commit read only transactions or background @@ -1262,8 +1315,6 @@ trx_start_low( trx_sys->rw_trx_ids.push_back(trx->id); - trx_sys_rw_trx_add(trx); - ut_ad(trx->rsegs.m_redo.rseg != 0 || srv_read_only_mode || srv_force_recovery >= SRV_FORCE_NO_TRX_UNDO); @@ -1277,11 +1328,10 @@ trx_start_low( } #endif /* UNIV_DEBUG */ - trx->state = TRX_STATE_ACTIVE; - ut_ad(trx_sys_validate_trx_list()); trx_sys_mutex_exit(); + trx_sys->rw_trx_hash.insert(trx); } else { trx->id = 0; @@ -1302,17 +1352,11 @@ trx_start_low( trx_sys->rw_trx_ids.push_back(trx->id); - trx_sys->rw_trx_set.insert( - TrxTrack(trx->id, trx)); - trx_sys_mutex_exit(); + trx_sys->rw_trx_hash.insert(trx); } - - trx->state = TRX_STATE_ACTIVE; - } else { ut_ad(!read_write); - trx->state = TRX_STATE_ACTIVE; } } @@ -1642,24 +1686,14 @@ trx_erase_lists( bool serialised) { ut_ad(trx->id > 0); - trx_sys_mutex_enter(); - - if (serialised) { - UT_LIST_REMOVE(trx_sys->serialisation_list, trx); - } - - trx_ids_t::iterator it = std::lower_bound( - trx_sys->rw_trx_ids.begin(), - trx_sys->rw_trx_ids.end(), - trx->id); - ut_ad(*it == trx->id); - trx_sys->rw_trx_ids.erase(it); if (trx->read_only || trx->rsegs.m_redo.rseg == NULL) { + trx_sys_mutex_enter(); ut_ad(!trx->in_rw_trx_list); } else { + trx_sys_mutex_enter(); UT_LIST_REMOVE(trx_sys->rw_trx_list, trx); ut_d(trx->in_rw_trx_list = false); ut_ad(trx_sys_validate_trx_list()); @@ -1669,9 +1703,18 @@ trx_erase_lists( } } - trx_sys->rw_trx_set.erase(TrxTrack(trx->id)); + if (serialised) { + UT_LIST_REMOVE(trx_sys->serialisation_list, trx); + } + trx_ids_t::iterator it = std::lower_bound( + trx_sys->rw_trx_ids.begin(), + trx_sys->rw_trx_ids.end(), + trx->id); + ut_ad(*it == trx->id); + trx_sys->rw_trx_ids.erase(it); trx_sys_mutex_exit(); + trx_sys->rw_trx_hash.erase(trx); } /****************************************************************//** @@ -3036,6 +3079,7 @@ trx_set_rw_mode( ut_ad(!trx->in_rw_trx_list); ut_ad(!trx_is_autocommit_non_locking(trx)); ut_ad(!trx->read_only); + ut_ad(trx->id == 0); if (high_level_read_only) { return; @@ -3053,14 +3097,10 @@ trx_set_rw_mode( ut_ad(trx->rsegs.m_redo.rseg != 0); mutex_enter(&trx_sys->mutex); - - ut_ad(trx->id == 0); trx->id = trx_sys_get_new_trx_id(); trx_sys->rw_trx_ids.push_back(trx->id); - trx_sys->rw_trx_set.insert(TrxTrack(trx->id, trx)); - /* So that we can see our own changes. */ if (MVCC::is_view_active(trx->read_view)) { MVCC::set_view_creator_trx_id(trx->read_view, trx->id); @@ -3077,6 +3117,7 @@ trx_set_rw_mode( ut_d(trx->in_rw_trx_list = true); mutex_exit(&trx_sys->mutex); + trx_sys->rw_trx_hash.insert(trx); } /** From 0ca2ea1a6550bb718efc878ed451be69e8244cd4 Mon Sep 17 00:00:00 2001 From: Sergey Vojtovich Date: Wed, 20 Dec 2017 14:59:36 +0400 Subject: [PATCH 029/106] MDEV-14638 - Replace trx_sys_t::rw_trx_set with LF_HASH trx reference counter was updated under mutex and read without any protection. This is both slow and unsafe. Use atomic operations for reference counter accesses. --- storage/innobase/include/row0vers.h | 2 +- storage/innobase/include/trx0sys.h | 4 +- storage/innobase/include/trx0trx.h | 68 +++++++++++++++++------------ storage/innobase/include/trx0trx.ic | 28 ------------ storage/innobase/lock/lock0lock.cc | 12 ++--- storage/innobase/row/row0sel.cc | 2 +- storage/innobase/row/row0vers.cc | 10 ++--- 7 files changed, 55 insertions(+), 71 deletions(-) diff --git a/storage/innobase/include/row0vers.h b/storage/innobase/include/row0vers.h index ee0b1528683b8..645f11faaad70 100644 --- a/storage/innobase/include/row0vers.h +++ b/storage/innobase/include/row0vers.h @@ -45,7 +45,7 @@ index record. @param[in] rec secondary index record @param[in] index secondary index @param[in] offsets rec_get_offsets(rec, index) -@return the active transaction; trx_release_reference() must be invoked +@return the active transaction; trx->release_reference() must be invoked @retval NULL if the record was committed */ trx_t* row_vers_impl_x_locked( diff --git a/storage/innobase/include/trx0sys.h b/storage/innobase/include/trx0sys.h index 2e87d500896b5..e87aa1bef6ca3 100644 --- a/storage/innobase/include/trx0sys.h +++ b/storage/innobase/include/trx0sys.h @@ -598,7 +598,7 @@ class rw_trx_hash_t With do_ref_count == true caller may dereference trx even if it is not holding lock_sys->mutex. Caller is responsible for calling - trx_release_reference() when it is done playing with trx. + trx->release_reference() when it is done playing with trx. Ideally this method should get caller rw_trx_hash_pins along with trx object as a parameter, similar to insert() and erase(). However most @@ -646,7 +646,7 @@ class rw_trx_hash_t if ((trx= element->trx)) { if (do_ref_count) - trx_reference(trx); + trx->reference(); #ifdef UNIV_DEBUG mutex_enter(&trx->mutex); ut_ad(trx_state_eq(trx, TRX_STATE_ACTIVE) || diff --git a/storage/innobase/include/trx0trx.h b/storage/innobase/include/trx0trx.h index 9366d20b1f522..f6a997b8a0f2c 100644 --- a/storage/innobase/include/trx0trx.h +++ b/storage/innobase/include/trx0trx.h @@ -532,26 +532,6 @@ void trx_set_rw_mode( trx_t* trx); -/** -Increase the reference count. -@param trx Transaction that is being referenced */ -UNIV_INLINE -void -trx_reference( - trx_t* trx); - -/** -Release the transaction. Decrease the reference count. -@param trx Transaction that is being released */ -UNIV_INLINE -void -trx_release_reference( - trx_t* trx); - -/** -Check if the transaction is being referenced. */ -#define trx_is_referenced(t) ((t)->n_ref > 0) - /** @param[in] requestor Transaction requesting the lock @param[in] holder Transaction holding the lock @@ -889,6 +869,19 @@ struct TrxVersion { typedef std::list > hit_list_t; struct trx_t { +private: + /** + Count of references. + + We can't release the locks nor commit the transaction until this reference + is 0. We can change the state to TRX_STATE_COMMITTED_IN_MEMORY to signify + that it is no longer "active". + */ + + int32_t n_ref; + + +public: TrxMutex mutex; /*!< Mutex protecting the fields state and lock (except some fields of lock, which are protected by @@ -1235,14 +1228,6 @@ struct trx_t { const char* start_file; /*!< Filename where it was started */ #endif /* UNIV_DEBUG */ - lint n_ref; /*!< Count of references, protected - by trx_t::mutex. We can't release the - locks nor commit the transaction until - this reference is 0. We can change - the state to COMMITTED_IN_MEMORY to - signify that it is no longer - "active". */ - /** Version of this instance. It is incremented each time the instance is re-used in trx_start_low(). It is used to track whether a transaction has been restarted since it was tagged @@ -1311,6 +1296,33 @@ struct trx_t { return(assign_temp_rseg()); } + + bool is_referenced() + { + return my_atomic_load32_explicit(&n_ref, MY_MEMORY_ORDER_RELAXED) > 0; + } + + + void reference() + { +#ifdef UNIV_DEBUG + int32_t old_n_ref= +#endif + my_atomic_add32_explicit(&n_ref, 1, MY_MEMORY_ORDER_RELAXED); + ut_ad(old_n_ref >= 0); + } + + + void release_reference() + { +#ifdef UNIV_DEBUG + int32_t old_n_ref= +#endif + my_atomic_add32_explicit(&n_ref, -1, MY_MEMORY_ORDER_RELAXED); + ut_ad(old_n_ref > 0); + } + + private: /** Assign a rollback segment for modifying temporary tables. @return the assigned rollback segment */ diff --git a/storage/innobase/include/trx0trx.ic b/storage/innobase/include/trx0trx.ic index cd0b812869db4..c288da8bf0b0a 100644 --- a/storage/innobase/include/trx0trx.ic +++ b/storage/innobase/include/trx0trx.ic @@ -213,34 +213,6 @@ ok: trx->dict_operation = op; } -/** -Increase the reference count. -@param trx Transaction that is being referenced */ -UNIV_INLINE void trx_reference(trx_t *trx) -{ - trx_mutex_enter(trx); - ut_ad(trx->n_ref >= 0); - ++trx->n_ref; - trx_mutex_exit(trx); -} - -/** -Release the transaction. Decrease the reference count. -@param trx Transaction that is being released */ -UNIV_INLINE -void -trx_release_reference( - trx_t* trx) -{ - trx_mutex_enter(trx); - - ut_ad(trx->n_ref > 0); - --trx->n_ref; - - trx_mutex_exit(trx); -} - - /** @param trx Get the active view for this transaction, if one exists @return the transaction's read view or NULL if one not assigned. */ diff --git a/storage/innobase/lock/lock0lock.cc b/storage/innobase/lock/lock0lock.cc index 269f606250f35..2a6167ec809f9 100644 --- a/storage/innobase/lock/lock0lock.cc +++ b/storage/innobase/lock/lock0lock.cc @@ -6813,7 +6813,7 @@ lock_rec_convert_impl_to_expl_for_trx( trx_t* trx, /*!< in/out: active transaction */ ulint heap_no)/*!< in: rec heap number to lock */ { - ut_ad(trx_is_referenced(trx)); + ut_ad(trx->is_referenced()); ut_ad(page_rec_is_leaf(rec)); ut_ad(!rec_is_default_row(rec, index)); @@ -6837,7 +6837,7 @@ lock_rec_convert_impl_to_expl_for_trx( lock_mutex_exit(); - trx_release_reference(trx); + trx->release_reference(); DEBUG_SYNC_C("after_lock_rec_convert_impl_to_expl_for_trx"); } @@ -6883,7 +6883,7 @@ lock_rec_convert_impl_to_expl( if (trx != 0) { ulint heap_no = page_rec_get_heap_no(rec); - ut_ad(trx_is_referenced(trx)); + ut_ad(trx->is_referenced()); /* If the transaction is still active and has no explicit x-lock set on the record, set one for it. @@ -7655,13 +7655,13 @@ lock_trx_release_locks( trx->state = TRX_STATE_COMMITTED_IN_MEMORY; /*--------------------------------------*/ - if (trx_is_referenced(trx)) { + if (trx->is_referenced()) { ut_a(release_lock); lock_mutex_exit(); - while (trx_is_referenced(trx)) { + while (trx->is_referenced()) { trx_mutex_exit(trx); @@ -7681,7 +7681,7 @@ lock_trx_release_locks( trx_mutex_enter(trx); } - ut_ad(!trx_is_referenced(trx)); + ut_ad(!trx->is_referenced()); /* If the background thread trx_rollback_or_clean_recovered() is still active then there is a chance that the rollback diff --git a/storage/innobase/row/row0sel.cc b/storage/innobase/row/row0sel.cc index eeb52bfeee316..1395faaad9668 100644 --- a/storage/innobase/row/row0sel.cc +++ b/storage/innobase/row/row0sel.cc @@ -4982,7 +4982,7 @@ row_search_mvcc( trx, rec, index, offsets)) { /* The record belongs to an active transaction. We must acquire a lock. */ - trx_release_reference(t); + t->release_reference(); } else { /* The secondary index record does not point to a delete-marked clustered index diff --git a/storage/innobase/row/row0vers.cc b/storage/innobase/row/row0vers.cc index 0b8cf63f94091..fc8b1b2ec812d 100644 --- a/storage/innobase/row/row0vers.cc +++ b/storage/innobase/row/row0vers.cc @@ -73,7 +73,7 @@ index record. @param[in] index secondary index @param[in] offsets rec_get_offsets(rec, index) @param[in,out] mtr mini-transaction -@return the active transaction; trx_release_reference() must be invoked +@return the active transaction; trx->release_reference() must be invoked @retval NULL if the record was committed */ UNIV_INLINE trx_t* @@ -217,7 +217,7 @@ row_vers_impl_x_locked_low( or updated, the leaf page record always is created with a clear delete-mark flag. (We never insert a delete-marked record.) */ - trx_release_reference(trx); + trx->release_reference(); trx = 0; } @@ -328,7 +328,7 @@ row_vers_impl_x_locked_low( /* prev_version was the first version modified by the trx_id transaction: no implicit x-lock */ - trx_release_reference(trx); + trx->release_reference(); trx = 0; break; } @@ -350,7 +350,7 @@ index record. @param[in] rec secondary index record @param[in] index secondary index @param[in] offsets rec_get_offsets(rec, index) -@return the active transaction; trx_release_reference() must be invoked +@return the active transaction; trx->release_reference() must be invoked @retval NULL if the record was committed */ trx_t* row_vers_impl_x_locked( @@ -398,7 +398,7 @@ row_vers_impl_x_locked( caller_trx, clust_rec, clust_index, rec, index, offsets, &mtr); - ut_ad(trx == 0 || trx_is_referenced(trx)); + ut_ad(trx == 0 || trx->is_referenced()); } mtr_commit(&mtr); From 578ffcc5efad3da20a21056067755e0ae9e39267 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Thu, 11 Jan 2018 10:56:13 +0200 Subject: [PATCH 030/106] Skip mariabackup.huge_lsn if encryption is not available --- mysql-test/suite/mariabackup/huge_lsn.test | 1 + 1 file changed, 1 insertion(+) diff --git a/mysql-test/suite/mariabackup/huge_lsn.test b/mysql-test/suite/mariabackup/huge_lsn.test index 37be33916e7aa..baf577769c051 100644 --- a/mysql-test/suite/mariabackup/huge_lsn.test +++ b/mysql-test/suite/mariabackup/huge_lsn.test @@ -1,4 +1,5 @@ --source include/not_embedded.inc +--source include/have_file_key_management.inc --echo # --echo # MDEV-13416 mariabackup fails with EFAULT "Bad Address" From 30ecd2884a1464a0848dde1d34af84d194d1676a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Thu, 11 Jan 2018 12:12:31 +0200 Subject: [PATCH 031/106] Fix compilation warnings for libmariadb --- libmariadb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libmariadb b/libmariadb index 8231a0add6a9c..f3944bbd36af7 160000 --- a/libmariadb +++ b/libmariadb @@ -1 +1 @@ -Subproject commit 8231a0add6a9c2e12a758db2e7072d42deef354b +Subproject commit f3944bbd36af729b2f13313587018679c57de67d From 773c3ceb573ea5ae8506237d3648412251ce02e7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Thu, 11 Jan 2018 15:39:36 +0200 Subject: [PATCH 032/106] MDEV-14824 Assertion `!trx_is_started(trx)' failed in innobase_start_trx_and_assign_read_view In CREATE SEQUENCE or CREATE TEMPORARY SEQUENCE, we should not start an InnoDB transaction for inserting the sequence status record into the underlying no-rollback table. Because we did this, a debug assertion failure would fail in START TRANSACTION WITH CONSISTENT SNAPSHOT after CREATE TEMPORARY SEQUENCE was executed. row_ins_step(): Do not start the transaction. Let the caller do that. que_thr_step(): Start the transaction before calling row_ins_step(). row_ins_clust_index_entry(): Skip locking and undo logging for no-rollback tables, even for temporary no-rollback tables. row_ins_index_entry(): Allow trx->id==0 for no-rollback tables. row_insert_for_mysql(): Do not start a transaction for no-rollback tables. --- mysql-test/suite/sql_sequence/create.result | 1 + mysql-test/suite/sql_sequence/create.test | 1 + storage/innobase/que/que0que.cc | 3 ++- storage/innobase/row/row0ins.cc | 12 +++++------- storage/innobase/row/row0mysql.cc | 6 ++++-- 5 files changed, 13 insertions(+), 10 deletions(-) diff --git a/mysql-test/suite/sql_sequence/create.result b/mysql-test/suite/sql_sequence/create.result index 015b5ac762bf0..20d1682257513 100644 --- a/mysql-test/suite/sql_sequence/create.result +++ b/mysql-test/suite/sql_sequence/create.result @@ -482,6 +482,7 @@ select previous value for t1; previous value for t1 1 CREATE TEMPORARY SEQUENCE t1 start with 100 minvalue 100 maxvalue 200 increment by 1 cache 10 engine=innodb; +START TRANSACTION WITH CONSISTENT SNAPSHOT; select previous value for t1; previous value for t1 NULL diff --git a/mysql-test/suite/sql_sequence/create.test b/mysql-test/suite/sql_sequence/create.test index 6696e78db9263..91411bebfde0a 100644 --- a/mysql-test/suite/sql_sequence/create.test +++ b/mysql-test/suite/sql_sequence/create.test @@ -362,6 +362,7 @@ CREATE SEQUENCE t1 start with 1 minvalue 1 maxvalue 10 increment by 1 cache 10 e select next value for t1; select previous value for t1; CREATE TEMPORARY SEQUENCE t1 start with 100 minvalue 100 maxvalue 200 increment by 1 cache 10 engine=innodb; +START TRANSACTION WITH CONSISTENT SNAPSHOT; select previous value for t1; select next value for t1; select previous value for t1; diff --git a/storage/innobase/que/que0que.cc b/storage/innobase/que/que0que.cc index 5a3af9dfaebc7..a45ac188f2f0d 100644 --- a/storage/innobase/que/que0que.cc +++ b/storage/innobase/que/que0que.cc @@ -1,7 +1,7 @@ /***************************************************************************** Copyright (c) 1996, 2016, Oracle and/or its affiliates. All Rights Reserved. -Copyright (c) 2017, MariaDB Corporation. +Copyright (c) 2017, 2018, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -1030,6 +1030,7 @@ que_thr_step( } else if (type == QUE_NODE_SELECT) { thr = row_sel_step(thr); } else if (type == QUE_NODE_INSERT) { + trx_start_if_not_started_xa(thr_get_trx(thr), true); thr = row_ins_step(thr); } else if (type == QUE_NODE_UPDATE) { trx_start_if_not_started_xa(thr_get_trx(thr), true); diff --git a/storage/innobase/row/row0ins.cc b/storage/innobase/row/row0ins.cc index c111aa8e776a8..a62b888a8b4a4 100644 --- a/storage/innobase/row/row0ins.cc +++ b/storage/innobase/row/row0ins.cc @@ -1,7 +1,7 @@ /***************************************************************************** Copyright (c) 1996, 2016, Oracle and/or its affiliates. All Rights Reserved. -Copyright (c) 2016, 2017, MariaDB Corporation. +Copyright (c) 2016, 2018, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -3193,9 +3193,9 @@ row_ins_clust_index_entry( /* Try first optimistic descent to the B-tree */ log_free_check(); - const ulint flags = dict_table_is_temporary(index->table) - ? BTR_NO_LOCKING_FLAG - : index->table->no_rollback() ? BTR_NO_ROLLBACK : 0; + const ulint flags = index->table->no_rollback() ? BTR_NO_ROLLBACK + : dict_table_is_temporary(index->table) + ? BTR_NO_LOCKING_FLAG : 0; err = row_ins_clust_index_entry_low( flags, BTR_MODIFY_LEAF, index, n_uniq, entry, @@ -3304,7 +3304,7 @@ row_ins_index_entry( dtuple_t* entry, /*!< in/out: index entry to insert */ que_thr_t* thr) /*!< in: query thread */ { - ut_ad(thr_get_trx(thr)->id != 0); + ut_ad(thr_get_trx(thr)->id || index->table->no_rollback()); DBUG_EXECUTE_IF("row_ins_index_entry_timeout", { DBUG_SET("-d,row_ins_index_entry_timeout"); @@ -3764,8 +3764,6 @@ row_ins_step( trx = thr_get_trx(thr); - trx_start_if_not_started_xa(trx, true); - node = static_cast(thr->run_node); ut_ad(que_node_get_type(node) == QUE_NODE_INSERT); diff --git a/storage/innobase/row/row0mysql.cc b/storage/innobase/row/row0mysql.cc index 4b06c81ff6c2c..99f4d37dc6cde 100644 --- a/storage/innobase/row/row0mysql.cc +++ b/storage/innobase/row/row0mysql.cc @@ -1,7 +1,7 @@ /***************************************************************************** Copyright (c) 2000, 2017, Oracle and/or its affiliates. All Rights Reserved. -Copyright (c) 2015, 2017, MariaDB Corporation. +Copyright (c) 2015, 2018, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -1453,7 +1453,9 @@ row_insert_for_mysql( row_mysql_delay_if_needed(); - trx_start_if_not_started_xa(trx, true); + if (!table->no_rollback()) { + trx_start_if_not_started_xa(trx, true); + } row_get_prebuilt_insert_row(prebuilt); node = prebuilt->ins_node; From bf7719111fff5167e160abf869c03a81a3a39592 Mon Sep 17 00:00:00 2001 From: Monty Date: Thu, 11 Jan 2018 17:09:51 +0200 Subject: [PATCH 033/106] Removed duplicated copyright message --- BUILD/compile-pentium64-asan-max | 17 +---------------- 1 file changed, 1 insertion(+), 16 deletions(-) diff --git a/BUILD/compile-pentium64-asan-max b/BUILD/compile-pentium64-asan-max index db595b418df35..cd5a292d17c89 100755 --- a/BUILD/compile-pentium64-asan-max +++ b/BUILD/compile-pentium64-asan-max @@ -1,20 +1,5 @@ #! /bin/sh -# Copyright (c) 2005, 2010, Oracle and/or its affiliates. -# -# This program is free software; you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation; version 2 of the License. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program; if not, write to the Free Software -# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA - -# Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved. +# Copyright (c) 2018, MariaDB Corporation. # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by From 21239bb0fd2859968d3c42dcc56712a8978b6207 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Thu, 11 Jan 2018 22:54:22 +0200 Subject: [PATCH 034/106] After-merge fix to innodb.log_corruption --- .../encryption/r/innodb_encrypt_log_corruption.result | 4 ++-- mysql-test/suite/innodb/t/log_corruption.test | 11 +++++++++-- 2 files changed, 11 insertions(+), 4 deletions(-) diff --git a/mysql-test/suite/encryption/r/innodb_encrypt_log_corruption.result b/mysql-test/suite/encryption/r/innodb_encrypt_log_corruption.result index e85ad3c9db7f8..23e8a1c98c8af 100644 --- a/mysql-test/suite/encryption/r/innodb_encrypt_log_corruption.result +++ b/mysql-test/suite/encryption/r/innodb_encrypt_log_corruption.result @@ -124,8 +124,8 @@ SELECT COUNT(*) FROM INFORMATION_SCHEMA.ENGINES WHERE engine = 'innodb' AND support IN ('YES', 'DEFAULT', 'ENABLED'); COUNT(*) -0 -FOUND 1 /InnoDB: Upgrading redo log:/ in mysqld.1.err +1 +FOUND 2 /InnoDB: Upgrading redo log:/ in mysqld.1.err # Minimal MariaDB 10.1.21 encrypted redo log SELECT COUNT(*) `1` FROM INFORMATION_SCHEMA.ENGINES WHERE engine='innodb' AND support IN ('YES', 'DEFAULT', 'ENABLED'); diff --git a/mysql-test/suite/innodb/t/log_corruption.test b/mysql-test/suite/innodb/t/log_corruption.test index a1d85b9f6983d..ca5d945afb750 100644 --- a/mysql-test/suite/innodb/t/log_corruption.test +++ b/mysql-test/suite/innodb/t/log_corruption.test @@ -428,8 +428,15 @@ print OUT pack("Nx[5]nx[5]", 1, 0x1286); print OUT "ibbackup was here!!!1!"; print OUT pack("x[470]N", 0x52b54540); # In encryption.innodb_log_corruption the previous step would -# replace the block with an encrypted one. Rewrite it as it was. -die unless seek(OUT, 0x800, 0); +# replace the block with an encrypted one and update the checkpoint. +# Restore them. +# invalid (all-zero) checkpoint page 1 and an empty log page +print OUT chr(0) x 1024; +# valid checkpoint block 2 +print OUT pack("x[12]NNNx[264]", 0x12860c, 0, 0x80c); +# pointer to the MLOG_CHECKPOINT record, and checkpoint page checksum +print OUT pack("H*x[204]NNN", "590DBAACFE922582", 0x128612, 0, 0x101741b); +# log page print OUT pack("NnnNx[496]N", 0x80000944, 12, 12, 1, 0x46c8a2a2); close OUT or die; EOF From 5fce14dad02be5b23a8718a9a94fb7de00a7bb1e Mon Sep 17 00:00:00 2001 From: Monty Date: Thu, 11 Jan 2018 23:55:13 +0200 Subject: [PATCH 035/106] Removed wrong DBUG_DUMP that accessed not initialized memory. --- sql/field.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sql/field.cc b/sql/field.cc index e723b62163a0e..7b8ec18cad668 100644 --- a/sql/field.cc +++ b/sql/field.cc @@ -8294,6 +8294,7 @@ uchar *Field_blob::pack(uchar *to, const uchar *from, uint max_length) @return New pointer into memory based on from + length of the data */ + const uchar *Field_blob::unpack(uchar *to, const uchar *from, const uchar *from_end, uint param_data) { @@ -8311,7 +8312,6 @@ const uchar *Field_blob::unpack(uchar *to, const uchar *from, DBUG_RETURN(0); store(reinterpret_cast(from) + master_packlength, length, field_charset); - DBUG_DUMP("record", to, table->s->reclength); DBUG_RETURN(from + master_packlength + length); } From 3dc3ab1a3048484910ca8acccaf76c71b080e533 Mon Sep 17 00:00:00 2001 From: Andrei Elkin Date: Thu, 11 Jan 2018 23:56:54 +0200 Subject: [PATCH 036/106] Added checking that row events ends with a proper end block Problems -------- The slave io thread did not conduct integrity check for a group of row-based events. Specifically it tolerates missed terminal block event that must be flagged with STMT_END. Failure to react on its loss can confuse the applier thread in various ways. Another potential issue was that there were no check of impossible second in row Gtid-log-event while the slave io thread is receiving to be skipped events after reconnect. Fixes ----- The slave io thread is made by this patch to track the rows event STMT_END status. Whenever at next event reading the IO thread finds out that a preceding Rows event did not actually had the flag, an explicit error is issued. Replication can be resumed after the source of failure is eliminated, see a provided test. Note that currently the row-based group integrity check excludes the compressed version 2 Rows events (which are not generated by MariaDB master). Its uncompressed counterpart is manually tested. The 2nd issue is covered to produce an error in case the io thread receives a successive Gtid_log_event while it is post-reconnect skipping. --- .../r/rpl_row_end_of_statement_loss.result | 42 ++++ .../rpl_row_end_of_statement_loss-master.opt | 2 + .../rpl/t/rpl_row_end_of_statement_loss.test | 66 ++++++ sql/rpl_mi.h | 21 ++ sql/slave.cc | 200 +++++++++++++++++- 5 files changed, 321 insertions(+), 10 deletions(-) create mode 100644 mysql-test/suite/rpl/r/rpl_row_end_of_statement_loss.result create mode 100644 mysql-test/suite/rpl/t/rpl_row_end_of_statement_loss-master.opt create mode 100644 mysql-test/suite/rpl/t/rpl_row_end_of_statement_loss.test diff --git a/mysql-test/suite/rpl/r/rpl_row_end_of_statement_loss.result b/mysql-test/suite/rpl/r/rpl_row_end_of_statement_loss.result new file mode 100644 index 0000000000000..dc6a67b48d283 --- /dev/null +++ b/mysql-test/suite/rpl/r/rpl_row_end_of_statement_loss.result @@ -0,0 +1,42 @@ +include/master-slave.inc +[connection master] +connection slave; +call mtr.add_suppression("Slave IO thread did not receive an expected Rows-log end-of-statement"); +call mtr.add_suppression("Relay log write failure: could not queue event from master"); +SET @save_debug= @@global.debug; +SET GLOBAL debug_dbug="+d,simulate_stmt_end_rows_event_loss"; +include/stop_slave.inc +CHANGE MASTER TO master_host = '127.0.0.1', master_port = MASTER_PORT, MASTER_USE_GTID=SLAVE_POS; +connection master; +CREATE TABLE t (a INT, b text(8192));; +INSERT INTO t values (1, repeat('b', 8192)), (1, repeat('b', 8192)); +connection slave; +START SLAVE IO_THREAD; +include/wait_for_slave_io_error.inc [errno=1595] +SET GLOBAL debug_dbug="-d,simulate_stmt_end_rows_event_loss"; +include/start_slave.inc +connection master; +connection slave; +connection slave; +include/stop_slave.inc +connection master; +SET @save_log_bin_compress= @@GLOBAL.log_bin_compress; +SET @save_log_bin_compress_min_len= @@GLOBAL.log_bin_compress_min_len; +SET @@GLOBAL.log_bin_compress=ON; +SET @@GLOBAL.log_bin_compress_min_len=10; +INSERT INTO t values (2, repeat('b', 8192)), (2, repeat('b', 8192)); +connection slave; +SET GLOBAL debug_dbug="+d,simulate_stmt_end_rows_event_loss"; +START SLAVE IO_THREAD; +include/wait_for_slave_io_error.inc [errno=1595] +SET GLOBAL debug_dbug="-d,simulate_stmt_end_rows_event_loss"; +include/start_slave.inc +connection master; +connection slave; +connection master; +SET @@GLOBAL.log_bin_compress= @save_log_bin_compress; +SET @@GLOBAL.log_bin_compress_min_len= @save_log_bin_compress_min_len; +DROP TABLE t; +connection slave; +SET GLOBAL debug_dbug= @save_debug; +include/rpl_end.inc diff --git a/mysql-test/suite/rpl/t/rpl_row_end_of_statement_loss-master.opt b/mysql-test/suite/rpl/t/rpl_row_end_of_statement_loss-master.opt new file mode 100644 index 0000000000000..144bbca073039 --- /dev/null +++ b/mysql-test/suite/rpl/t/rpl_row_end_of_statement_loss-master.opt @@ -0,0 +1,2 @@ +--binlog-row-event-max-size=8192 + diff --git a/mysql-test/suite/rpl/t/rpl_row_end_of_statement_loss.test b/mysql-test/suite/rpl/t/rpl_row_end_of_statement_loss.test new file mode 100644 index 0000000000000..5b2d99f3bf1e5 --- /dev/null +++ b/mysql-test/suite/rpl/t/rpl_row_end_of_statement_loss.test @@ -0,0 +1,66 @@ +--source include/have_debug.inc +--source include/have_binlog_format_row.inc +--source include/master-slave.inc + +# Loss of STMT_END flagged event must error out the IO thread +--connection slave +call mtr.add_suppression("Slave IO thread did not receive an expected Rows-log end-of-statement"); +call mtr.add_suppression("Relay log write failure: could not queue event from master"); + +SET @save_debug= @@global.debug; +SET GLOBAL debug_dbug="+d,simulate_stmt_end_rows_event_loss"; +--source include/stop_slave.inc +--replace_result $MASTER_MYPORT MASTER_PORT +--eval CHANGE MASTER TO master_host = '127.0.0.1', master_port = $MASTER_MYPORT, MASTER_USE_GTID=SLAVE_POS + +--connection master +--let $max_row_size=8192 +--eval CREATE TABLE t (a INT, b text($max_row_size)); +--eval INSERT INTO t values (1, repeat('b', $max_row_size)), (1, repeat('b', $max_row_size)) + +# Prove that the missed STMT_END marked rows-event causes the io thread stop. +--connection slave +START SLAVE IO_THREAD; +--let $slave_io_errno=1595 +--source include/wait_for_slave_io_error.inc +SET GLOBAL debug_dbug="-d,simulate_stmt_end_rows_event_loss"; +--source include/start_slave.inc + +--connection master +sync_slave_with_master; + +# Compressed version of the above +--connection slave +--source include/stop_slave.inc + +--connection master +SET @save_log_bin_compress= @@GLOBAL.log_bin_compress; +SET @save_log_bin_compress_min_len= @@GLOBAL.log_bin_compress_min_len; + +SET @@GLOBAL.log_bin_compress=ON; +SET @@GLOBAL.log_bin_compress_min_len=10; + +--eval INSERT INTO t values (2, repeat('b', $max_row_size)), (2, repeat('b', $max_row_size)) + +# Prove that the missed STMT_END marked rows-event causes the io thread stop. +--connection slave +SET GLOBAL debug_dbug="+d,simulate_stmt_end_rows_event_loss"; +START SLAVE IO_THREAD; +--let $slave_io_errno=1595 +--source include/wait_for_slave_io_error.inc +SET GLOBAL debug_dbug="-d,simulate_stmt_end_rows_event_loss"; +--source include/start_slave.inc + +--connection master +sync_slave_with_master; + +# cleanup + +--connection master +SET @@GLOBAL.log_bin_compress= @save_log_bin_compress; +SET @@GLOBAL.log_bin_compress_min_len= @save_log_bin_compress_min_len; +DROP TABLE t; +sync_slave_with_master; +SET GLOBAL debug_dbug= @save_debug; + +--source include/rpl_end.inc diff --git a/sql/rpl_mi.h b/sql/rpl_mi.h index ccc1be6e5ce31..b304e45f86aaf 100644 --- a/sql/rpl_mi.h +++ b/sql/rpl_mi.h @@ -133,6 +133,19 @@ class Domain_id_filter extern TYPELIB slave_parallel_mode_typelib; +typedef struct st_rows_event_tracker +{ + char binlog_file_name[FN_REFLEN]; + my_off_t first_seen; + my_off_t last_seen; + bool stmt_end_seen; + void update(const char* file_name, size_t pos, + const char* buf, + const Format_description_log_event *fdle); + void reset(); + bool check_and_report(const char* file_name, size_t pos); +} Rows_event_tracker; + /***************************************************************************** Replication IO Thread @@ -301,6 +314,14 @@ class Master_info : public Slave_reporting_capability uint64 gtid_reconnect_event_skip_count; /* gtid_event_seen is false until we receive first GTID event from master. */ bool gtid_event_seen; + /** + The struct holds some history of Rows- log-event reading/queuing + by the receiver thread. Its fields are updated per each such event + at time of queue_event(), and they are checked to detect + the Rows- event group integrity violation at time of first non-Rows- + event gets handled. + */ + Rows_event_tracker rows_event_tracker; bool in_start_all_slaves, in_stop_all_slaves; bool in_flush_all_relay_logs; uint users; /* Active user for object */ diff --git a/sql/slave.cc b/sql/slave.cc index c686553fdd523..e5c502c2de56e 100644 --- a/sql/slave.cc +++ b/sql/slave.cc @@ -3349,7 +3349,8 @@ static ulong read_event(MYSQL* mysql, Master_info *mi, bool* suppress_warnings, we suppress prints to .err file as long as the reconnect happens without problems */ - *suppress_warnings= TRUE; + *suppress_warnings= + global_system_variables.log_warnings < 2 ? TRUE : FALSE; } else { @@ -4274,6 +4275,7 @@ pthread_handler_t handle_slave_io(void *arg) mi->abort_slave = 0; mysql_mutex_unlock(&mi->run_lock); mysql_cond_broadcast(&mi->start_cond); + mi->rows_event_tracker.reset(); DBUG_PRINT("master_info",("log_file_name: '%s' position: %llu", mi->master_log_name, mi->master_log_pos)); @@ -4356,6 +4358,10 @@ pthread_handler_t handle_slave_io(void *arg) */ mi->gtid_reconnect_event_skip_count= mi->events_queued_since_last_gtid; mi->gtid_event_seen= false; + /* + Reset stale state of the rows-event group tracker at reconnect. + */ + mi->rows_event_tracker.reset(); } #ifdef ENABLED_DEBUG_SYNC @@ -5752,7 +5758,7 @@ static int queue_event(Master_info* mi,const char* buf, ulong event_len) char* new_buf = NULL; char new_buf_arr[4096]; bool is_malloc = false; - + bool is_rows_event= false; /* FD_q must have been prepared for the first R_a event inside get_master_version_and_clock() @@ -6186,11 +6192,11 @@ static int queue_event(Master_info* mi,const char* buf, ulong event_len) got_gtid_event= true; if (mi->using_gtid == Master_info::USE_GTID_NO) goto default_action; - if (unlikely(!mi->gtid_event_seen)) + if (unlikely(mi->gtid_reconnect_event_skip_count)) { - mi->gtid_event_seen= true; - if (mi->gtid_reconnect_event_skip_count) + if (likely(!mi->gtid_event_seen)) { + mi->gtid_event_seen= true; /* If we are reconnecting, and we need to skip a partial event group already queued to the relay log before the reconnect, then we check @@ -6219,13 +6225,45 @@ static int queue_event(Master_info* mi,const char* buf, ulong event_len) rpl_slave_state_tostring_helper(&error_msg, &event_gtid, &first); goto err; } + if (global_system_variables.log_warnings > 1) + { + bool first= true; + StringBuffer<1024> gtid_text; + rpl_slave_state_tostring_helper(>id_text, &mi->last_queued_gtid, + &first); + sql_print_information("Slave IO thread is reconnected to " + "receive Gtid_log_event %s. It is to skip %llu " + "already received events including the gtid one", + gtid_text.ptr(), + mi->events_queued_since_last_gtid); + } + goto default_action; } - } + else + { + bool first; + StringBuffer<1024> gtid_text; - if (unlikely(mi->gtid_reconnect_event_skip_count)) - { - goto default_action; + gtid_text.append(STRING_WITH_LEN("Last received gtid: ")); + first= true; + rpl_slave_state_tostring_helper(>id_text, &mi->last_queued_gtid, + &first); + gtid_text.append(STRING_WITH_LEN(", currently received: ")); + first= true; + rpl_slave_state_tostring_helper(>id_text, &event_gtid, &first); + + error= ER_SLAVE_RELAY_LOG_WRITE_FAILURE; + sql_print_error("Slave IO thread has received a new Gtid_log_event " + "while skipping already logged events " + "after reconnect. %s. %llu remains to be skipped. " + "The number of originally read events was %llu", + gtid_text.ptr(), + mi->gtid_reconnect_event_skip_count, + mi->events_queued_since_last_gtid); + goto err; + } } + mi->gtid_event_seen= true; /* We have successfully queued to relay log everything before this GTID, so @@ -6292,8 +6330,34 @@ static int queue_event(Master_info* mi,const char* buf, ulong event_len) goto err; } } - buf = new_buf; is_compress_event = true; + buf = new_buf; + /* + As we are uncertain about compressed V2 rows events, we don't track + them + */ + if (LOG_EVENT_IS_ROW_V2((Log_event_type) buf[EVENT_TYPE_OFFSET])) + goto default_action; + /* fall through */ + case WRITE_ROWS_EVENT_V1: + case UPDATE_ROWS_EVENT_V1: + case DELETE_ROWS_EVENT_V1: + case WRITE_ROWS_EVENT: + case UPDATE_ROWS_EVENT: + case DELETE_ROWS_EVENT: + { + is_rows_event= true; + mi->rows_event_tracker.update(mi->master_log_name, + mi->master_log_pos, + buf, + mi->rli.relay_log. + description_event_for_queue); + + DBUG_EXECUTE_IF("simulate_stmt_end_rows_event_loss", + { + mi->rows_event_tracker.stmt_end_seen= false; + }); + } goto default_action; #ifndef DBUG_OFF @@ -6351,6 +6415,21 @@ static int queue_event(Master_info* mi,const char* buf, ulong event_len) break; } + /* + Integrity of Rows- event group check. + A sequence of Rows- events must end with STMT_END_F flagged one. + Even when Heartbeat event interrupts Rows- events flow this must indicate a + malfunction e.g logging on the master. + */ + if (((uchar) buf[EVENT_TYPE_OFFSET] != HEARTBEAT_LOG_EVENT) && + !is_rows_event && + mi->rows_event_tracker.check_and_report(mi->master_log_name, + mi->master_log_pos)) + { + error= ER_SLAVE_RELAY_LOG_WRITE_FAILURE; + goto err; + } + /* If we filter events master-side (eg. @@skip_replication), we will see holes in the event positions from the master. If we see such a hole, adjust @@ -6519,6 +6598,21 @@ static int queue_event(Master_info* mi,const char* buf, ulong event_len) The whole of the current event group is queued. So in case of reconnect we can start from after the current GTID. */ + if (mi->gtid_reconnect_event_skip_count) + { + bool first= true; + StringBuffer<1024> gtid_text; + + rpl_slave_state_tostring_helper(>id_text, &mi->last_queued_gtid, + &first); + sql_print_error("Slave IO thread received a terminal event from " + "group %s whose retrieval was interrupted " + "with reconnect. We still had %llu events to read. " + "The number of originally read events was %llu", + gtid_text.ptr(), + mi->gtid_reconnect_event_skip_count, + mi->events_queued_since_last_gtid); + } mi->gtid_current_pos.update(&mi->last_queued_gtid); mi->events_queued_since_last_gtid= 0; @@ -7518,6 +7612,92 @@ bool rpl_master_erroneous_autoinc(THD *thd) return FALSE; } + +static bool get_row_event_stmt_end(const char* buf, + const Format_description_log_event *fdle) +{ + uint8 const common_header_len= fdle->common_header_len; + Log_event_type event_type= (Log_event_type)(uchar)buf[EVENT_TYPE_OFFSET]; + + uint8 const post_header_len= fdle->post_header_len[event_type-1]; + const char *flag_start= buf + common_header_len; + /* + The term 4 below signifies that master is of 'an intermediate source', see + Rows_log_event::Rows_log_event. + */ + flag_start += RW_MAPID_OFFSET + (post_header_len == 6) ? 4 : RW_FLAGS_OFFSET; + + return (uint2korr(flag_start) & Rows_log_event::STMT_END_F) != 0; +} + + +/* + Reset log event tracking data. +*/ + +void Rows_event_tracker::reset() +{ + binlog_file_name[0]= 0; + first_seen= last_seen= 0; + stmt_end_seen= false; +} + + +/* + Update log event tracking data. + + The first- and last- seen event binlog position get memorized, as + well as the end-of-statement status of the last one. +*/ + +void Rows_event_tracker::update(const char* file_name, size_t pos, + const char* buf, + const Format_description_log_event *fdle) +{ + if (!first_seen) + { + first_seen= pos; + strmake(binlog_file_name, file_name, sizeof(binlog_file_name) - 1); + } + last_seen= pos; + DBUG_ASSERT(stmt_end_seen == 0); // We can only have one + stmt_end_seen= get_row_event_stmt_end(buf, fdle); +}; + + +/** + The function is called at next event reading + after a sequence of Rows- log-events. It checks the end-of-statement status + of the past sequence to report on any isssue. + In the positive case the tracker gets reset. + + @return true when the Rows- event group integrity found compromised, + false otherwise. +*/ +bool Rows_event_tracker::check_and_report(const char* file_name, + size_t pos) +{ + if (last_seen) + { + // there was at least one "block" event previously + if (!stmt_end_seen) + { + sql_print_error("Slave IO thread did not receive an expected " + "Rows-log end-of-statement for event starting " + "at log '%s' position %llu " + "whose last block was seen at log '%s' position %llu. " + "The end-of-statement should have been delivered " + "before the current one at log '%s' position %llu", + binlog_file_name, first_seen, + binlog_file_name, last_seen, file_name, pos); + return true; + } + reset(); + } + + return false; +} + /** @} (end of group Replication) */ From d32f5be307d5593bf36d243de275af88ff8ee92a Mon Sep 17 00:00:00 2001 From: Sergei Petrunia Date: Fri, 12 Jan 2018 15:11:56 +0300 Subject: [PATCH 037/106] MDEV-14372: Fix and enable rocksdb.information_schema test - Make Rdb_binlog_manager::unpack_value to not have a stack overrun when it is reading invalid data (which it currently does as we in MariaDB do not store binlog coordinates under BINLOG_INFO_INDEX_NUMBER, see comments in MDEV-14892 for details). - We may need to store these coordinates in the future, so instead of removing the call of this function, let's make it work properly for all possible inputs. --- .../rocksdb/r/information_schema.result | 12 ++++----- .../rocksdb/mysql-test/rocksdb/t/disabled.def | 1 - .../rocksdb/t/information_schema-master.opt | 2 +- .../rocksdb/t/information_schema.test | 12 +++++++-- storage/rocksdb/rdb_datadic.cc | 27 ++++++++++++++++++- storage/rocksdb/rdb_datadic.h | 5 +++- storage/rocksdb/rdb_i_s.cc | 1 - 7 files changed, 47 insertions(+), 13 deletions(-) diff --git a/storage/rocksdb/mysql-test/rocksdb/r/information_schema.result b/storage/rocksdb/mysql-test/rocksdb/r/information_schema.result index 291effa832c0c..717fb0b970c50 100644 --- a/storage/rocksdb/mysql-test/rocksdb/r/information_schema.result +++ b/storage/rocksdb/mysql-test/rocksdb/r/information_schema.result @@ -1,28 +1,28 @@ DROP TABLE IF EXISTS t1; DROP TABLE IF EXISTS t2; DROP TABLE IF EXISTS t3; +create table t1 (a int) engine=rocksdb; +drop table t1; select * from INFORMATION_SCHEMA.ROCKSDB_GLOBAL_INFO; TYPE NAME VALUE MAX_INDEX_ID MAX_INDEX_ID max_index_id CF_FLAGS 0 default [0] CF_FLAGS 1 __system__ [0] +DDL_DROP_INDEX_ONGOING cf_id:0,index_id:max_index_id select count(*) from INFORMATION_SCHEMA.ROCKSDB_GLOBAL_INFO; count(*) -3 +4 select VALUE into @keysIn from INFORMATION_SCHEMA.ROCKSDB_COMPACTION_STATS where CF_NAME = 'default' and LEVEL = 'Sum' and TYPE = 'KeyIn'; CREATE TABLE t1 (i1 INT, i2 INT, PRIMARY KEY (i1)) ENGINE = ROCKSDB; INSERT INTO t1 VALUES (1, 1), (2, 2), (3, 3); select * from INFORMATION_SCHEMA.ROCKSDB_GLOBAL_INFO; TYPE NAME VALUE -BINLOG FILE master-bin.000001 -BINLOG POS 1066 -BINLOG GTID uuid:5 MAX_INDEX_ID MAX_INDEX_ID max_index_id CF_FLAGS 0 default [0] CF_FLAGS 1 __system__ [0] select count(*) from INFORMATION_SCHEMA.ROCKSDB_GLOBAL_INFO; count(*) -6 +3 set global rocksdb_force_flush_memtable_now = true; set global rocksdb_compact_cf='default'; select case when VALUE-@keysIn >= 3 then 'true' else 'false' end from INFORMATION_SCHEMA.ROCKSDB_COMPACTION_STATS where CF_NAME = 'default' and LEVEL = 'Sum' and TYPE = 'KeyIn'; @@ -66,7 +66,7 @@ SHOW GLOBAL VARIABLES LIKE 'ROCKSDB_PAUSE_BACKGROUND_WORK'; Variable_name Value rocksdb_pause_background_work ON DROP TABLE t3; -cf_id:0,index_id:267 +cf_id:0,index_id:264 SET GLOBAL ROCKSDB_PAUSE_BACKGROUND_WORK=0; SHOW GLOBAL VARIABLES LIKE 'ROCKSDB_PAUSE_BACKGROUND_WORK'; Variable_name Value diff --git a/storage/rocksdb/mysql-test/rocksdb/t/disabled.def b/storage/rocksdb/mysql-test/rocksdb/t/disabled.def index dc2e04b93d79f..359d1d3e082c6 100644 --- a/storage/rocksdb/mysql-test/rocksdb/t/disabled.def +++ b/storage/rocksdb/mysql-test/rocksdb/t/disabled.def @@ -72,7 +72,6 @@ blind_delete_without_tx_api: MDEV-12286: rocksdb.blind_delete_without_tx_api tes ## Tests that fail for some other reason ## -information_schema : MariaRocks: requires GTIDs mysqlbinlog_gtid_skip_empty_trans_rocksdb : MariaRocks: requires GTIDs rpl_row_triggers : MariaRocks: requires GTIDs diff --git a/storage/rocksdb/mysql-test/rocksdb/t/information_schema-master.opt b/storage/rocksdb/mysql-test/rocksdb/t/information_schema-master.opt index 40b14167e17a5..863798476384d 100644 --- a/storage/rocksdb/mysql-test/rocksdb/t/information_schema-master.opt +++ b/storage/rocksdb/mysql-test/rocksdb/t/information_schema-master.opt @@ -1 +1 @@ ---binlog_format=row --gtid_mode=ON --enforce_gtid_consistency --log_slave_updates +--binlog_format=row --log_slave_updates diff --git a/storage/rocksdb/mysql-test/rocksdb/t/information_schema.test b/storage/rocksdb/mysql-test/rocksdb/t/information_schema.test index c8d9b538529ca..8746e589046f5 100644 --- a/storage/rocksdb/mysql-test/rocksdb/t/information_schema.test +++ b/storage/rocksdb/mysql-test/rocksdb/t/information_schema.test @@ -9,6 +9,13 @@ DROP TABLE IF EXISTS t2; DROP TABLE IF EXISTS t3; --enable_warnings +# MariaDB: the following is for handling the case where the tests +# is started on a totally empty datadir, where no MyRocks table has +# ever been created). In that case, there is no MAX_INDEX_ID. +# Create/drop a table so that we do have MAX_INDEX_ID. +create table t1 (a int) engine=rocksdb; +drop table t1; + --let $max_index_id = query_get_value(SELECT * from INFORMATION_SCHEMA.ROCKSDB_GLOBAL_INFO where type = 'MAX_INDEX_ID', VALUE, 1) --replace_result $max_index_id max_index_id select * from INFORMATION_SCHEMA.ROCKSDB_GLOBAL_INFO; @@ -19,9 +26,10 @@ select VALUE into @keysIn from INFORMATION_SCHEMA.ROCKSDB_COMPACTION_STATS where CREATE TABLE t1 (i1 INT, i2 INT, PRIMARY KEY (i1)) ENGINE = ROCKSDB; INSERT INTO t1 VALUES (1, 1), (2, 2), (3, 3); ---let $MASTER_UUID = query_get_value(SELECT @@SERVER_UUID, @@SERVER_UUID, 1) +# No binlog coordinates in MariaDB: --let $MASTER_UUID = query_get_value(SELECT @@SERVER_UUID, @@SERVER_UUID, 1) --let $max_index_id = query_get_value(SELECT * from INFORMATION_SCHEMA.ROCKSDB_GLOBAL_INFO where type = 'MAX_INDEX_ID', VALUE, 1) ---replace_result $MASTER_UUID uuid $max_index_id max_index_id +# No binlog coordinates in MariaDB: --replace_result $MASTER_UUID uuid $max_index_id max_index_id +--replace_result $max_index_id max_index_id select * from INFORMATION_SCHEMA.ROCKSDB_GLOBAL_INFO; select count(*) from INFORMATION_SCHEMA.ROCKSDB_GLOBAL_INFO; diff --git a/storage/rocksdb/rdb_datadic.cc b/storage/rocksdb/rdb_datadic.cc index f6f1406e80c46..b2f5af705a3a2 100644 --- a/storage/rocksdb/rdb_datadic.cc +++ b/storage/rocksdb/rdb_datadic.cc @@ -4261,7 +4261,7 @@ bool Rdb_binlog_manager::read(char *const binlog_name, std::string value; rocksdb::Status status = m_dict->get_value(m_key_slice, &value); if (status.ok()) { - if (!unpack_value((const uchar *)value.c_str(), binlog_name, binlog_pos, + if (!unpack_value((const uchar *)value.c_str(), value.size(), binlog_name, binlog_pos, binlog_gtid)) ret = true; } @@ -4331,35 +4331,60 @@ Rdb_binlog_manager::pack_value(uchar *const buf, const char *const binlog_name, @return true on error */ bool Rdb_binlog_manager::unpack_value(const uchar *const value, + size_t value_size_arg, char *const binlog_name, my_off_t *const binlog_pos, char *const binlog_gtid) const { uint pack_len = 0; + intmax_t value_size= value_size_arg; DBUG_ASSERT(binlog_pos != nullptr); + if ((value_size -= Rdb_key_def::VERSION_SIZE) < 0) + return true; // read version const uint16_t version = rdb_netbuf_to_uint16(value); + pack_len += Rdb_key_def::VERSION_SIZE; if (version != Rdb_key_def::BINLOG_INFO_INDEX_NUMBER_VERSION) return true; + if ((value_size -= sizeof(uint16)) < 0) + return true; + // read binlog file name length const uint16_t binlog_name_len = rdb_netbuf_to_uint16(value + pack_len); pack_len += sizeof(uint16); + + if (binlog_name_len >= (FN_REFLEN+1)) + return true; + + if ((value_size -= binlog_name_len) < 0) + return true; + if (binlog_name_len) { // read and set binlog name memcpy(binlog_name, value + pack_len, binlog_name_len); binlog_name[binlog_name_len] = '\0'; pack_len += binlog_name_len; + if ((value_size -= sizeof(uint32)) < 0) + return true; // read and set binlog pos *binlog_pos = rdb_netbuf_to_uint32(value + pack_len); pack_len += sizeof(uint32); + if ((value_size -= sizeof(uint16)) < 0) + return true; // read gtid length const uint16_t binlog_gtid_len = rdb_netbuf_to_uint16(value + pack_len); pack_len += sizeof(uint16); + + if (binlog_gtid_len >= GTID_BUF_LEN) + return true; + if ((value_size -= binlog_gtid_len) < 0) + return true; + if (binlog_gtid && binlog_gtid_len > 0) { // read and set gtid memcpy(binlog_gtid, value + pack_len, binlog_gtid_len); diff --git a/storage/rocksdb/rdb_datadic.h b/storage/rocksdb/rdb_datadic.h index 5796132de39e7..223f61edb4352 100644 --- a/storage/rocksdb/rdb_datadic.h +++ b/storage/rocksdb/rdb_datadic.h @@ -46,6 +46,8 @@ class Rdb_field_packing; class Rdb_cf_manager; class Rdb_ddl_manager; +const uint32_t GTID_BUF_LEN = 60; + /* @brief Field packing context. @@ -1154,7 +1156,8 @@ class Rdb_binlog_manager { rocksdb::Slice pack_value(uchar *const buf, const char *const binlog_name, const my_off_t &binlog_pos, const char *const binlog_gtid) const; - bool unpack_value(const uchar *const value, char *const binlog_name, + bool unpack_value(const uchar *const value, size_t value_size, + char *const binlog_name, my_off_t *const binlog_pos, char *const binlog_gtid) const; std::atomic m_slave_gtid_info_tbl; diff --git a/storage/rocksdb/rdb_i_s.cc b/storage/rocksdb/rdb_i_s.cc index 424a9e6c1f4a4..f8ddcb00fb3e0 100644 --- a/storage/rocksdb/rdb_i_s.cc +++ b/storage/rocksdb/rdb_i_s.cc @@ -729,7 +729,6 @@ static int rdb_i_s_global_info_fill_table( DBUG_ASSERT(tables != nullptr); static const uint32_t INT_BUF_LEN = 21; - static const uint32_t GTID_BUF_LEN = 60; static const uint32_t CF_ID_INDEX_BUF_LEN = 60; int ret = 0; From c481fc9ca788a3ac1d61ed6336208c0e2c1cde78 Mon Sep 17 00:00:00 2001 From: Sergei Petrunia Date: Fri, 12 Jan 2018 15:57:08 +0300 Subject: [PATCH 038/106] Change MyRocks maturity from Alpha to Beta --- storage/rocksdb/ha_rocksdb.h | 2 +- .../rocksdb/r/mariadb_port_fixes.result | 24 +++++++++---------- 2 files changed, 13 insertions(+), 13 deletions(-) diff --git a/storage/rocksdb/ha_rocksdb.h b/storage/rocksdb/ha_rocksdb.h index 82819bbf7b2d1..4432a4de8d1a9 100644 --- a/storage/rocksdb/ha_rocksdb.h +++ b/storage/rocksdb/ha_rocksdb.h @@ -1412,6 +1412,6 @@ struct Rdb_inplace_alter_ctx : public my_core::inplace_alter_handler_ctx { Rdb_inplace_alter_ctx &operator=(const Rdb_inplace_alter_ctx &); }; -const int MYROCKS_MARIADB_PLUGIN_MATURITY_LEVEL= MariaDB_PLUGIN_MATURITY_ALPHA; +const int MYROCKS_MARIADB_PLUGIN_MATURITY_LEVEL= MariaDB_PLUGIN_MATURITY_BETA; } // namespace myrocks diff --git a/storage/rocksdb/mysql-test/rocksdb/r/mariadb_port_fixes.result b/storage/rocksdb/mysql-test/rocksdb/r/mariadb_port_fixes.result index 9952314cd2cf8..021373be02a26 100644 --- a/storage/rocksdb/mysql-test/rocksdb/r/mariadb_port_fixes.result +++ b/storage/rocksdb/mysql-test/rocksdb/r/mariadb_port_fixes.result @@ -69,15 +69,15 @@ set global rocksdb_strict_collation_check=@tmp_rscc; # select plugin_name, plugin_maturity from information_schema.plugins where plugin_name like '%rocksdb%'; plugin_name plugin_maturity -ROCKSDB Alpha -ROCKSDB_CFSTATS Alpha -ROCKSDB_DBSTATS Alpha -ROCKSDB_PERF_CONTEXT Alpha -ROCKSDB_PERF_CONTEXT_GLOBAL Alpha -ROCKSDB_CF_OPTIONS Alpha -ROCKSDB_COMPACTION_STATS Alpha -ROCKSDB_GLOBAL_INFO Alpha -ROCKSDB_DDL Alpha -ROCKSDB_INDEX_FILE_MAP Alpha -ROCKSDB_LOCKS Alpha -ROCKSDB_TRX Alpha +ROCKSDB Beta +ROCKSDB_CFSTATS Beta +ROCKSDB_DBSTATS Beta +ROCKSDB_PERF_CONTEXT Beta +ROCKSDB_PERF_CONTEXT_GLOBAL Beta +ROCKSDB_CF_OPTIONS Beta +ROCKSDB_COMPACTION_STATS Beta +ROCKSDB_GLOBAL_INFO Beta +ROCKSDB_DDL Beta +ROCKSDB_INDEX_FILE_MAP Beta +ROCKSDB_LOCKS Beta +ROCKSDB_TRX Beta From 028e2ddc549fb7573b11392c87b9e8faf1cbf38b Mon Sep 17 00:00:00 2001 From: Varun Gupta Date: Fri, 12 Jan 2018 19:16:36 +0530 Subject: [PATCH 039/106] Added a missing result file to the rocksdb_sys_vars result suite --- .../rocksdb_sys_vars/include/rocksdb_sys_var.inc | 1 + .../rocksdb_sys_vars/r/rocksdb_git_hash_basic.result | 7 +++++++ 2 files changed, 8 insertions(+) create mode 100644 storage/rocksdb/mysql-test/rocksdb_sys_vars/r/rocksdb_git_hash_basic.result diff --git a/storage/rocksdb/mysql-test/rocksdb_sys_vars/include/rocksdb_sys_var.inc b/storage/rocksdb/mysql-test/rocksdb_sys_vars/include/rocksdb_sys_var.inc index 6ba9302667482..db0abc573589a 100644 --- a/storage/rocksdb/mysql-test/rocksdb_sys_vars/include/rocksdb_sys_var.inc +++ b/storage/rocksdb/mysql-test/rocksdb_sys_vars/include/rocksdb_sys_var.inc @@ -10,6 +10,7 @@ --eval SET @start_global_value = @@global.$sys_var if (!$suppress_default_value) { + --replace_regex /[a-f0-9]{40}/#/ SELECT @start_global_value; if ($session) { diff --git a/storage/rocksdb/mysql-test/rocksdb_sys_vars/r/rocksdb_git_hash_basic.result b/storage/rocksdb/mysql-test/rocksdb_sys_vars/r/rocksdb_git_hash_basic.result new file mode 100644 index 0000000000000..bbcfa1417eb72 --- /dev/null +++ b/storage/rocksdb/mysql-test/rocksdb_sys_vars/r/rocksdb_git_hash_basic.result @@ -0,0 +1,7 @@ +SET @start_global_value = @@global.ROCKSDB_GIT_HASH; +SELECT @start_global_value; +@start_global_value +# +"Trying to set variable @@global.ROCKSDB_GIT_HASH to 444. It should fail because it is readonly." +SET @@global.ROCKSDB_GIT_HASH = 444; +ERROR HY000: Variable 'rocksdb_git_hash' is a read only variable From c9c28bef3cb4cf21a6fee46f5ea3339867fbb07a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Otto=20Kek=C3=A4l=C3=A4inen?= Date: Wed, 15 Nov 2017 12:37:32 +0800 Subject: [PATCH 040/106] Minor spelling fixes in code comments, docs and output This commit does not touch any variable names or any other actual code, and thus should not in any way affect how the code works. --- BUILD-CMAKE | 2 +- Docs/sp-imp-spec.txt | 4 ++-- cmake/cpack_rpm.cmake | 2 +- configure.cmake | 2 +- debian/autobake-deb.sh | 2 +- debian/mariadb-server-10.3.mysql.init | 2 +- include/m_ctype.h | 4 ++-- include/my_atomic.h | 2 +- include/my_global.h | 8 ++++---- include/my_pthread.h | 4 ++-- include/mysql/service_thd_rnd.h | 2 +- include/mysql_com.h | 4 ++-- man/mysql-test-run.pl.1 | 4 ++-- man/mysql.1 | 2 +- mysys/my_atomic_writes.c | 2 +- mysys/my_context.c | 2 +- mysys/my_getopt.c | 2 +- mysys/thr_mutex.c | 4 ++-- strings/ctype-uca.c | 6 +++--- strings/decimal.c | 2 +- win/packaging/ca/CustomAction.cpp | 2 +- 21 files changed, 32 insertions(+), 32 deletions(-) diff --git a/BUILD-CMAKE b/BUILD-CMAKE index c95482cf619ac..aedce4400c6e8 100644 --- a/BUILD-CMAKE +++ b/BUILD-CMAKE @@ -97,7 +97,7 @@ shell>cd ../build shell>cmake ../src Note: if a directory was used for in-source build, out-of-source will -not work. To reenable out-of-source build, remove /CMakeCache.txt +not work. To re-enable out-of-source build, remove /CMakeCache.txt file. diff --git a/Docs/sp-imp-spec.txt b/Docs/sp-imp-spec.txt index f2794e1479351..0526cc43f811f 100644 --- a/Docs/sp-imp-spec.txt +++ b/Docs/sp-imp-spec.txt @@ -323,7 +323,7 @@ One "obvious" solution would be to simply push "mysql.proc" to the list of tables used by the query, but this implies a "join" with this table if the query is a select, so it doesn't work (and we can't exclude this - table easily; since a priviledged used might in fact want to search + table easily; since a privileged used might in fact want to search the proc table). Another solution would of course be to allow the opening and closing of the mysql.proc table during a query execution, but this it not @@ -400,7 +400,7 @@ instruction. Calling and returning from a CONTINUE handler poses some special - problems. Since we need to return to the point after its invokation, + problems. Since we need to return to the point after its invocation, we push the return location on a stack in the sp_rcontext (this is done by the exectution loop). The handler then ends with a special instruction, sp_instr_hreturn, which returns to this location. diff --git a/cmake/cpack_rpm.cmake b/cmake/cpack_rpm.cmake index 5b8aa36ff0de7..513f3671b5d4f 100644 --- a/cmake/cpack_rpm.cmake +++ b/cmake/cpack_rpm.cmake @@ -201,7 +201,7 @@ ALTERNATIVE_NAME("server" "mysql-server") ALTERNATIVE_NAME("test" "mysql-test") # Argh! Different distributions call packages differently, to be a drop-in -# replacement we have to fake distribution-speficic dependencies +# replacement we have to fake distribution-specificic dependencies IF(RPM MATCHES "(rhel|centos)6") ALTERNATIVE_NAME("client" "mysql") diff --git a/configure.cmake b/configure.cmake index 7258077d68b00..b62ac12e63488 100644 --- a/configure.cmake +++ b/configure.cmake @@ -488,7 +488,7 @@ int main() { # -# Test for endianess +# Test for endianness # INCLUDE(TestBigEndian) IF(APPLE) diff --git a/debian/autobake-deb.sh b/debian/autobake-deb.sh index cc5b04188f404..1a709eb82848b 100755 --- a/debian/autobake-deb.sh +++ b/debian/autobake-deb.sh @@ -33,7 +33,7 @@ fi # Look up distro-version specific stuff # # Always keep the actual packaging as up-to-date as possible following the latest -# Debian policy and targetting Debian Sid. Then case-by-case run in autobake-deb.sh +# Debian policy and targeting Debian Sid. Then case-by-case run in autobake-deb.sh # tests for backwards compatibility and strip away parts on older builders. # If iproute2 is not available (before Debian Jessie and Ubuntu Trusty) diff --git a/debian/mariadb-server-10.3.mysql.init b/debian/mariadb-server-10.3.mysql.init index d7f6d58e8ca29..35a52d5d8db12 100644 --- a/debian/mariadb-server-10.3.mysql.init +++ b/debian/mariadb-server-10.3.mysql.init @@ -25,7 +25,7 @@ SELF=$(cd $(dirname $0); pwd -P)/$(basename $0) MYADMIN="/usr/bin/mysqladmin --defaults-file=/etc/mysql/debian.cnf" -# priority can be overriden and "-s" adds output to stderr +# priority can be overridden and "-s" adds output to stderr ERR_LOGGER="logger -p daemon.err -t /etc/init.d/mysql -i" if [ -f /etc/default/mysql ]; then diff --git a/include/m_ctype.h b/include/m_ctype.h index b78443f88f898..da52b4c15a0ed 100644 --- a/include/m_ctype.h +++ b/include/m_ctype.h @@ -376,7 +376,7 @@ typedef size_t (*my_charset_conv_case)(CHARSET_INFO *, A structure to return the statistics of a native string copying, when no Unicode conversion is involved. - The stucture is OK to be unitialized before calling a copying routine. + The stucture is OK to be uninitialized before calling a copying routine. A copying routine must populate the structure as follows: - m_source_end_pos must be set by to a non-NULL value in the range of the input string. @@ -425,7 +425,7 @@ struct my_charset_handler_st my_charset_conv_case caseup; my_charset_conv_case casedn; - /* Charset dependant snprintf() */ + /* Charset dependent snprintf() */ size_t (*snprintf)(CHARSET_INFO *, char *to, size_t n, const char *fmt, ...) ATTRIBUTE_FORMAT_FPTR(printf, 4, 5); diff --git a/include/my_atomic.h b/include/my_atomic.h index 896dc2b5c3325..1f5d85fcf125b 100644 --- a/include/my_atomic.h +++ b/include/my_atomic.h @@ -101,7 +101,7 @@ sequentially-consistent operation ordering. We choose implementation as follows: on Windows using Visual C++ the native - implementation should be preferrable. When using gcc we prefer the Solaris + implementation should be preferable. When using gcc we prefer the Solaris implementation before the gcc because of stability preference, we choose gcc builtins if available. */ diff --git a/include/my_global.h b/include/my_global.h index 7986a0e21dd92..a96c947cf44f9 100644 --- a/include/my_global.h +++ b/include/my_global.h @@ -178,7 +178,7 @@ /* The macros below are borrowed from include/linux/compiler.h in the - Linux kernel. Use them to indicate the likelyhood of the truthfulness + Linux kernel. Use them to indicate the likelihood of the truthfulness of a condition. This serves two purposes - newer versions of gcc will be able to optimize for branch predication, which could yield siginficant performance gains in frequently executed sections of the code, and the @@ -527,7 +527,7 @@ typedef SOCKET my_socket; typedef int my_socket; /* File descriptor for sockets */ #define INVALID_SOCKET -1 #endif -/* Type for fuctions that handles signals */ +/* Type for functions that handles signals */ #define sig_handler RETSIGTYPE C_MODE_START #ifdef HAVE_SIGHANDLER_T @@ -857,7 +857,7 @@ static inline double log2(double x) /* Max size that must be added to a so that we know Size to make - adressable obj. + addressable obj. */ #if SIZEOF_CHARP == 4 typedef long my_ptrdiff_t; @@ -869,7 +869,7 @@ typedef long long my_ptrdiff_t; #define MY_ALIGN_DOWN(A,L) ((A) & ~((L) - 1)) #define ALIGN_SIZE(A) MY_ALIGN((A),sizeof(double)) #define ALIGN_MAX_UNIT (sizeof(double)) -/* Size to make adressable obj. */ +/* Size to make addressable obj. */ #define ALIGN_PTR(A, t) ((t*) MY_ALIGN((A), sizeof(double))) #define ADD_TO_PTR(ptr,size,type) (type) ((uchar*) (ptr)+size) #define PTR_BYTE_DIFF(A,B) (my_ptrdiff_t) ((uchar*) (A) - (uchar*) (B)) diff --git a/include/my_pthread.h b/include/my_pthread.h index 037ac0a062f42..ecb201a60da03 100644 --- a/include/my_pthread.h +++ b/include/my_pthread.h @@ -196,7 +196,7 @@ int sigwait(sigset_t *set, int *sig); #endif #if !defined(HAVE_SIGWAIT) && !defined(HAVE_rts_threads) && !defined(sigwait) && !defined(alpha_linux_port) && !defined(_AIX) -int sigwait(sigset_t *setp, int *sigp); /* Use our implemention */ +int sigwait(sigset_t *setp, int *sigp); /* Use our implementation */ #endif @@ -393,7 +393,7 @@ typedef struct st_safe_mutex_deadlock_t #ifdef SAFE_MUTEX_DETECT_DESTROY /* - Used to track the destroying of mutexes. This needs to be a seperate + Used to track the destroying of mutexes. This needs to be a separate structure because the safe_mutex_t structure could be freed before the mutexes are destroyed. */ diff --git a/include/mysql/service_thd_rnd.h b/include/mysql/service_thd_rnd.h index 21133c7889ff6..13fc1c1683917 100644 --- a/include/mysql/service_thd_rnd.h +++ b/include/mysql/service_thd_rnd.h @@ -18,7 +18,7 @@ @file This service provides access to the thd-local random number generator. - It's preferrable over the global one, because concurrent threads + It's preferable over the global one, because concurrent threads can generate random numbers without fighting each other over the access to the shared rnd state. */ diff --git a/include/mysql_com.h b/include/mysql_com.h index 71858c93046fe..7f445502228ff 100644 --- a/include/mysql_com.h +++ b/include/mysql_com.h @@ -152,7 +152,7 @@ enum enum_indicator_type */ #define SCRAMBLE_LENGTH 20 #define SCRAMBLE_LENGTH_323 8 -/* length of password stored in the db: new passwords are preceeded with '*' */ +/* length of password stored in the db: new passwords are preceded with '*' */ #define SCRAMBLED_PASSWORD_CHAR_LENGTH (SCRAMBLE_LENGTH*2+1) #define SCRAMBLED_PASSWORD_CHAR_LENGTH_323 (SCRAMBLE_LENGTH_323*2) @@ -288,7 +288,7 @@ enum enum_indicator_type #endif /* - Gather all possible capabilites (flags) supported by the server + Gather all possible capabilities (flags) supported by the server MARIADB_* flags supported only by MariaDB connector(s). */ diff --git a/man/mysql-test-run.pl.1 b/man/mysql-test-run.pl.1 index 93fb42e251ae5..b928ec665873a 100644 --- a/man/mysql-test-run.pl.1 +++ b/man/mysql-test-run.pl.1 @@ -276,7 +276,7 @@ T}:T{ Setting of a timeout in minutes or seconds, corresponding to command line option \fB\-\-\fR\fB\fIname\fR\fR\fB\-timeout\fR\&. - Avaliable timeout names are TESTCASE, + Available timeout names are TESTCASE, SUITE (both in minutes) and START, SHUTDOWN (both in seconds)\&. These variables are supported from @@ -1350,7 +1350,7 @@ for a description\&. .\" noreorder option: mysql-test-run.pl \fB\-\-noreorder\fR .sp -Do not reorder tests to reduce number of restarts, but run them in exactly the order given\&. If a whole suite is to be run, the tests are run in alphabetical order, though similiar combinations will be grouped together\&. If more than one suite is listed, the tests are run one suite at a time, in the order listed\&. +Do not reorder tests to reduce number of restarts, but run them in exactly the order given\&. If a whole suite is to be run, the tests are run in alphabetical order, though similar combinations will be grouped together\&. If more than one suite is listed, the tests are run one suite at a time, in the order listed\&. .RE .sp .RS 4 diff --git a/man/mysql.1 b/man/mysql.1 index b7f7a63bff47d..f925fee454b32 100644 --- a/man/mysql.1 +++ b/man/mysql.1 @@ -698,7 +698,7 @@ the section called \(lqMYSQL COMMANDS\(rq\&. Disabled by default\&. .\} .\" mysql: net-buffer-length option .\" net-buffer-length option: mysql -\fB\-\-net\-buffer\-lenght=\fR\fB\fIsize\fR\fR +\fB\-\-net\-buffer\-length=\fR\fB\fIsize\fR\fR .sp Set the buffer size for TCP/IP and socket communication\&. (Default value is 16KB\&.) .RE diff --git a/mysys/my_atomic_writes.c b/mysys/my_atomic_writes.c index 0b54a20771338..7f1e353c121fd 100644 --- a/mysys/my_atomic_writes.c +++ b/mysys/my_atomic_writes.c @@ -259,7 +259,7 @@ static my_bool shannon_has_atomic_write(File file, int page_size) ************************************************************************/ /* - Initalize automic write sub systems. + Initialize automic write sub systems. Checks if we have any devices that supports atomic write */ diff --git a/mysys/my_context.c b/mysys/my_context.c index cf10738bdbdbc..5423f59d19bd9 100644 --- a/mysys/my_context.c +++ b/mysys/my_context.c @@ -707,7 +707,7 @@ my_context_continue(struct my_context *c) { /* This seems to be a common trick to run ConvertThreadToFiber() only on the - first occurence in a thread, in a way that works on multiple Windows + first occurrence in a thread, in a way that works on multiple Windows versions. */ void *current_fiber= GetCurrentFiber(); diff --git a/mysys/my_getopt.c b/mysys/my_getopt.c index 0ea062988f68f..9e617366ed523 100644 --- a/mysys/my_getopt.c +++ b/mysys/my_getopt.c @@ -236,7 +236,7 @@ int handle_options(int *argc, char ***argv, { is_cmdline_arg= 1; - /* save the separator too if skip unkown options */ + /* save the separator too if skip unknown options */ if (my_getopt_skip_unknown) (*argv)[argvpos++]= cur_arg; else diff --git a/mysys/thr_mutex.c b/mysys/thr_mutex.c index 49cb3ea600fc6..268a2b80f63a0 100644 --- a/mysys/thr_mutex.c +++ b/mysys/thr_mutex.c @@ -240,7 +240,7 @@ int safe_mutex_lock(safe_mutex_t *mp, myf my_flags, const char *file, if (!mp->file) { fprintf(stderr, - "safe_mutex: Trying to lock unitialized mutex at %s, line %d\n", + "safe_mutex: Trying to lock uninitialized mutex at %s, line %d\n", file, line); fflush(stderr); abort(); @@ -585,7 +585,7 @@ int safe_mutex_destroy(safe_mutex_t *mp, const char *file, uint line) if (!mp->file) { fprintf(stderr, - "safe_mutex: Trying to destroy unitialized mutex at %s, line %d\n", + "safe_mutex: Trying to destroy uninitialized mutex at %s, line %d\n", file, line); fflush(stderr); abort(); diff --git a/strings/ctype-uca.c b/strings/ctype-uca.c index e3ed531df9360..4386b6aa432d9 100644 --- a/strings/ctype-uca.c +++ b/strings/ctype-uca.c @@ -34034,7 +34034,7 @@ apply_one_rule(MY_CHARSET_LOADER *loader, /** Check if collation rules are valid, - i.e. characters are not outside of the collation suported range. + i.e. characters are not outside of the collation supported range. */ static int check_rules(MY_CHARSET_LOADER *loader, @@ -34170,7 +34170,7 @@ init_weight_level(MY_CHARSET_LOADER *loader, MY_COLL_RULES *rules, memcpy(dst->weights, src->weights, npages * sizeof(uint16 *)); /* - Calculate maximum lenghts for the pages which will be overwritten. + Calculate maximum lengths for the pages which will be overwritten. Mark pages that will be otherwriten as NULL. We'll allocate their own memory. */ @@ -34188,7 +34188,7 @@ init_weight_level(MY_CHARSET_LOADER *loader, MY_COLL_RULES *rules, { /* Not an expansion and not a contraction. - The page correspoding to r->curr[0] in "dst" + The page corresponding to r->curr[0] in "dst" will need at least the same amount of weights that r->base[0] has in "src". */ diff --git a/strings/decimal.c b/strings/decimal.c index 7db5111fc84e0..d7fd9a5722b2f 100644 --- a/strings/decimal.c +++ b/strings/decimal.c @@ -1694,7 +1694,7 @@ decimal_round(const decimal_t *from, decimal_t *to, int scale, scale increment for '/' NOTE - returned valued may be larger than the actual buffer requred + returned valued may be larger than the actual buffer required in the operation, as decimal_result_size, by design, operates on precision/scale values only and not on the actual decimal number diff --git a/win/packaging/ca/CustomAction.cpp b/win/packaging/ca/CustomAction.cpp index c0062ddcdd1b7..9803b0b07a2a7 100644 --- a/win/packaging/ca/CustomAction.cpp +++ b/win/packaging/ca/CustomAction.cpp @@ -524,7 +524,7 @@ extern "C" UINT CheckDBInUse(MSIHANDLE hInstall) or 4GB(x64 OS). Fragmentation due to loaded modules, heap and stack - limit maximum size of continous memory block further, + limit maximum size of continuous memory block further, so that limit for 32 bit process is about 1200 on 32 bit OS or 2000 MB on 64 bit OS(found experimentally). */ From 2da191791260d5ca3adcb6857404912d52926ee2 Mon Sep 17 00:00:00 2001 From: Sergei Petrunia Date: Fri, 12 Jan 2018 16:04:29 +0000 Subject: [PATCH 041/106] Attempt to eliminate race conditions in rocksdb.information_schema --- storage/rocksdb/mysql-test/rocksdb/r/information_schema.result | 2 ++ storage/rocksdb/mysql-test/rocksdb/t/information_schema.test | 2 ++ 2 files changed, 4 insertions(+) diff --git a/storage/rocksdb/mysql-test/rocksdb/r/information_schema.result b/storage/rocksdb/mysql-test/rocksdb/r/information_schema.result index 717fb0b970c50..6850d8dff1696 100644 --- a/storage/rocksdb/mysql-test/rocksdb/r/information_schema.result +++ b/storage/rocksdb/mysql-test/rocksdb/r/information_schema.result @@ -1,6 +1,7 @@ DROP TABLE IF EXISTS t1; DROP TABLE IF EXISTS t2; DROP TABLE IF EXISTS t3; +SET GLOBAL ROCKSDB_PAUSE_BACKGROUND_WORK=1; create table t1 (a int) engine=rocksdb; drop table t1; select * from INFORMATION_SCHEMA.ROCKSDB_GLOBAL_INFO; @@ -12,6 +13,7 @@ DDL_DROP_INDEX_ONGOING cf_id:0,index_id:max_index_id select count(*) from INFORMATION_SCHEMA.ROCKSDB_GLOBAL_INFO; count(*) 4 +SET GLOBAL ROCKSDB_PAUSE_BACKGROUND_WORK=0; select VALUE into @keysIn from INFORMATION_SCHEMA.ROCKSDB_COMPACTION_STATS where CF_NAME = 'default' and LEVEL = 'Sum' and TYPE = 'KeyIn'; CREATE TABLE t1 (i1 INT, i2 INT, PRIMARY KEY (i1)) ENGINE = ROCKSDB; INSERT INTO t1 VALUES (1, 1), (2, 2), (3, 3); diff --git a/storage/rocksdb/mysql-test/rocksdb/t/information_schema.test b/storage/rocksdb/mysql-test/rocksdb/t/information_schema.test index 8746e589046f5..2ffc186dd8fb6 100644 --- a/storage/rocksdb/mysql-test/rocksdb/t/information_schema.test +++ b/storage/rocksdb/mysql-test/rocksdb/t/information_schema.test @@ -13,6 +13,7 @@ DROP TABLE IF EXISTS t3; # is started on a totally empty datadir, where no MyRocks table has # ever been created). In that case, there is no MAX_INDEX_ID. # Create/drop a table so that we do have MAX_INDEX_ID. +SET GLOBAL ROCKSDB_PAUSE_BACKGROUND_WORK=1; create table t1 (a int) engine=rocksdb; drop table t1; @@ -20,6 +21,7 @@ drop table t1; --replace_result $max_index_id max_index_id select * from INFORMATION_SCHEMA.ROCKSDB_GLOBAL_INFO; select count(*) from INFORMATION_SCHEMA.ROCKSDB_GLOBAL_INFO; +SET GLOBAL ROCKSDB_PAUSE_BACKGROUND_WORK=0; select VALUE into @keysIn from INFORMATION_SCHEMA.ROCKSDB_COMPACTION_STATS where CF_NAME = 'default' and LEVEL = 'Sum' and TYPE = 'KeyIn'; From 3e6fcb6ac8e39593d441f3b60fbc21a88330f2c7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Fri, 12 Jan 2018 16:44:27 +0200 Subject: [PATCH 042/106] MDEV-14935 Remove bogus conditions related to not redo-logging PAGE_MAX_TRX_ID changes InnoDB originally skipped the redo logging of PAGE_MAX_TRX_ID changes until I enabled it in commit e76b873f24cd7cfc4b2d69b33103a6dd9601cbb8 that was part of MySQL 5.5.5 already. Later, when a more complete history of the InnoDB Plugin for MySQL 5.1 (aka branches/zip in the InnoDB subversion repository) and of the planned-to-be closed-source branches/innodb+ that became the basis of InnoDB in MySQL 5.5 was pushed to the MySQL source repository, the change was part of commit 509e761f06d6d7902bd5fdd0955447ed772af768: ------------------------------------------------------------------------ r5038 | marko | 2009-05-19 22:59:07 +0300 (Tue, 19 May 2009) | 30 lines branches/zip: Write PAGE_MAX_TRX_ID to the redo log. Otherwise, transactions that are started before the rollback of incomplete transactions has finished may have an inconsistent view of the secondary indexes. dict_index_is_sec_or_ibuf(): Auxiliary function for controlling updates and checks of PAGE_MAX_TRX_ID: check whether an index is a secondary index or the insert buffer tree. page_set_max_trx_id(), page_update_max_trx_id(), lock_rec_insert_check_and_lock(), lock_sec_rec_modify_check_and_lock(), btr_cur_ins_lock_and_undo(), btr_cur_upd_lock_and_undo(): Add the parameter mtr. page_set_max_trx_id(): Allow mtr to be NULL. When mtr==NULL, do not attempt to write to the redo log. This only occurs when creating a page or reorganizing a compressed page. In these cases, the PAGE_MAX_TRX_ID will be set correctly during the application of redo log records, even though there is no explicit log record about it. btr_discard_only_page_on_level(): Preserve PAGE_MAX_TRX_ID. This function should be unreachable, though. btr_cur_pessimistic_update(): Update PAGE_MAX_TRX_ID. Add some assertions for checking that PAGE_MAX_TRX_ID is set on all secondary index leaf pages. rb://115 tested by Michael, fixes Issue #211 ------------------------------------------------------------------------ After this fix, some bogus references to recv_recovery_is_on() remained. Also, some references could be replaced with references to index->is_dummy to prepare us for MDEV-14481 (background redo log apply). --- storage/innobase/lock/lock0lock.cc | 15 +++++---------- storage/innobase/page/page0cur.cc | 17 ++++++++++------- storage/innobase/row/row0upd.cc | 4 ++-- 3 files changed, 17 insertions(+), 19 deletions(-) diff --git a/storage/innobase/lock/lock0lock.cc b/storage/innobase/lock/lock0lock.cc index 2a6167ec809f9..09b900d15543c 100644 --- a/storage/innobase/lock/lock0lock.cc +++ b/storage/innobase/lock/lock0lock.cc @@ -1,7 +1,7 @@ /***************************************************************************** Copyright (c) 1996, 2016, Oracle and/or its affiliates. All Rights Reserved. -Copyright (c) 2014, 2017, MariaDB Corporation. +Copyright (c) 2014, 2018, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -459,11 +459,7 @@ lock_sec_rec_cons_read_sees( /* NOTE that we might call this function while holding the search system latch. */ - if (recv_recovery_is_on()) { - - return(false); - - } else if (dict_table_is_temporary(index->table)) { + if (dict_table_is_temporary(index->table)) { /* Temp-tables are not shared across connections and multiple transactions from different connections cannot simultaneously @@ -1518,7 +1514,7 @@ lock_sec_rec_some_has_impl( max trx id to the log, and therefore during recovery, this value for a page may be incorrect. */ - if (max_trx_id < trx_rw_min_trx_id() && !recv_recovery_is_on()) { + if (max_trx_id < trx_rw_min_trx_id()) { trx = 0; @@ -7097,9 +7093,8 @@ lock_sec_rec_read_check_and_lock( if the max trx id for the page >= min trx id for the trx list or a database recovery is running. */ - if ((page_get_max_trx_id(block->frame) >= trx_rw_min_trx_id() - || recv_recovery_is_on()) - && !page_rec_is_supremum(rec)) { + if (!page_rec_is_supremum(rec) + && page_get_max_trx_id(block->frame) >= trx_rw_min_trx_id()) { lock_rec_convert_impl_to_expl(thr_get_trx(thr), block, rec, index, offsets); diff --git a/storage/innobase/page/page0cur.cc b/storage/innobase/page/page0cur.cc index 5ede7faf3bacd..d9a04ba939323 100644 --- a/storage/innobase/page/page0cur.cc +++ b/storage/innobase/page/page0cur.cc @@ -2,6 +2,7 @@ Copyright (c) 1994, 2016, Oracle and/or its affiliates. All Rights Reserved. Copyright (c) 2012, Facebook Inc. +Copyright (c) 2018, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -1279,7 +1280,7 @@ page_cur_insert_rec_low( == (ibool) !!page_is_comp(page)); ut_ad(fil_page_index_page_check(page)); ut_ad(mach_read_from_8(page + PAGE_HEADER + PAGE_INDEX_ID) == index->id - || recv_recovery_is_on() + || index->is_dummy || (mtr ? mtr->is_inside_ibuf() : dict_index_is_ibuf(index))); ut_ad(!page_rec_is_supremum(current_rec)); @@ -1506,8 +1507,8 @@ page_cur_insert_rec_zip( ut_ad(page_is_comp(page)); ut_ad(fil_page_index_page_check(page)); ut_ad(mach_read_from_8(page + PAGE_HEADER + PAGE_INDEX_ID) == index->id - || (mtr ? mtr->is_inside_ibuf() : dict_index_is_ibuf(index)) - || recv_recovery_is_on()); + || index->is_dummy + || (mtr ? mtr->is_inside_ibuf() : dict_index_is_ibuf(index))); ut_ad(!page_get_instant(page)); ut_ad(!page_cur_is_after_last(cursor)); #ifdef UNIV_ZIP_DEBUG @@ -1626,6 +1627,7 @@ page_cur_insert_rec_zip( because the MLOG_COMP_REC_INSERT should only be logged after a successful operation. */ ut_ad(!recv_recovery_is_on()); + ut_ad(!index->is_dummy); } else if (recv_recovery_is_on()) { /* This should be followed by MLOG_ZIP_PAGE_COMPRESS_NO_DATA, @@ -1967,6 +1969,8 @@ page_parse_copy_rec_list_to_created_page( page_t* page; page_zip_des_t* page_zip; + ut_ad(index->is_dummy); + if (ptr + 4 > end_ptr) { return(NULL); @@ -1992,8 +1996,7 @@ page_parse_copy_rec_list_to_created_page( page_copy_rec_list_end_to_created_page() which was logged by. page_copy_rec_list_to_created_page_write_log(). For other pages, this field must be zero-initialized. */ - ut_ad(!page_get_instant(block->frame) - || (page_is_root(block->frame) && index->is_dummy)); + ut_ad(!page_get_instant(block->frame) || page_is_root(block->frame)); while (ptr < rec_end) { ptr = page_cur_parse_insert_rec(TRUE, ptr, end_ptr, @@ -2320,8 +2323,8 @@ page_cur_delete_rec( ut_ad(!!page_is_comp(page) == dict_table_is_comp(index->table)); ut_ad(fil_page_index_page_check(page)); ut_ad(mach_read_from_8(page + PAGE_HEADER + PAGE_INDEX_ID) == index->id - || (mtr ? mtr->is_inside_ibuf() : dict_index_is_ibuf(index)) - || recv_recovery_is_on()); + || index->is_dummy + || (mtr ? mtr->is_inside_ibuf() : dict_index_is_ibuf(index))); ut_ad(mtr == NULL || mtr->is_named_space(index->space)); /* The record must not be the supremum or infimum record. */ diff --git a/storage/innobase/row/row0upd.cc b/storage/innobase/row/row0upd.cc index c2ae828b1e0bc..60cc54f8a9b33 100644 --- a/storage/innobase/row/row0upd.cc +++ b/storage/innobase/row/row0upd.cc @@ -1,7 +1,7 @@ /***************************************************************************** Copyright (c) 1996, 2017, Oracle and/or its affiliates. All Rights Reserved. -Copyright (c) 2015, 2017, MariaDB Corporation. +Copyright (c) 2015, 2018, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -715,7 +715,7 @@ row_upd_rec_in_place( ut_ad(index->is_instant()); break; case REC_STATUS_NODE_PTR: - if (recv_recovery_is_on() + if (index->is_dummy && fil_page_get_type(page_align(rec)) == FIL_PAGE_RTREE) { /* The function rtr_update_mbr_field_in_place() From 4cafd8e66f77798168aecd9bb239f97d7cd1ee3b Mon Sep 17 00:00:00 2001 From: Sergei Petrunia Date: Sat, 13 Jan 2018 01:26:06 +0300 Subject: [PATCH 043/106] rocksdb.information_schema testcase is not stable --- storage/rocksdb/mysql-test/rocksdb/t/disabled.def | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/storage/rocksdb/mysql-test/rocksdb/t/disabled.def b/storage/rocksdb/mysql-test/rocksdb/t/disabled.def index 359d1d3e082c6..c2ca6efe46c79 100644 --- a/storage/rocksdb/mysql-test/rocksdb/t/disabled.def +++ b/storage/rocksdb/mysql-test/rocksdb/t/disabled.def @@ -67,6 +67,7 @@ lock_wait_timeout_stats: MDEV-13404 compact_deletes: MDEV-12663 : rocksdb.compact_deletes times out and causes other tests to fail blind_delete_without_tx_api: MDEV-12286: rocksdb.blind_delete_without_tx_api test fails +information_schema: MDEV-14372: unstable testcase ## ## Tests that fail for some other reason @@ -75,4 +76,3 @@ blind_delete_without_tx_api: MDEV-12286: rocksdb.blind_delete_without_tx_api tes mysqlbinlog_gtid_skip_empty_trans_rocksdb : MariaRocks: requires GTIDs rpl_row_triggers : MariaRocks: requires GTIDs - From 93e8ee4ae102c60719b0c2cb467aec52024efd58 Mon Sep 17 00:00:00 2001 From: Aleksey Midenkov Date: Fri, 12 Jan 2018 12:54:07 +0300 Subject: [PATCH 044/106] MDEV-14923 Assertion upon INSERT into locked versioned partitioned table --- .../suite/versioning/r/partition.result | 8 ++++ mysql-test/suite/versioning/t/partition.test | 9 +++++ sql/partition_info.cc | 39 +++++++++++-------- sql/partition_info.h | 2 +- sql/sql_partition.cc | 7 +--- 5 files changed, 41 insertions(+), 24 deletions(-) diff --git a/mysql-test/suite/versioning/r/partition.result b/mysql-test/suite/versioning/r/partition.result index 78d09c5c01925..2b275538fb9f6 100644 --- a/mysql-test/suite/versioning/r/partition.result +++ b/mysql-test/suite/versioning/r/partition.result @@ -407,6 +407,14 @@ Warning 4112 Versioned table `test`.`t1`: partition `p2` is full, add more HISTO delete from t1 where x = 2; Warnings: Warning 4112 Versioned table `test`.`t1`: partition `p2` is full, add more HISTORY partitions +# MDEV-14923 Assertion upon INSERT into locked versioned partitioned table +create or replace table t1 (x int) with system versioning +partition by system_time (partition p1 history, partition pn current); +lock table t1 write; +alter table t1 add partition (partition p1 history); +ERROR HY000: Duplicate partition name p1 +insert into t1 values (1); +unlock tables; # Test cleanup drop database test; create database test; diff --git a/mysql-test/suite/versioning/t/partition.test b/mysql-test/suite/versioning/t/partition.test index 9422d5d336d69..167a5b2f7a4d8 100644 --- a/mysql-test/suite/versioning/t/partition.test +++ b/mysql-test/suite/versioning/t/partition.test @@ -351,6 +351,15 @@ alter table t1 partition by system_time limit 1 ( delete from t1 where x = 1; delete from t1 where x = 2; +--echo # MDEV-14923 Assertion upon INSERT into locked versioned partitioned table +create or replace table t1 (x int) with system versioning +partition by system_time (partition p1 history, partition pn current); +lock table t1 write; +--error ER_SAME_NAME_PARTITION +alter table t1 add partition (partition p1 history); +insert into t1 values (1); +unlock tables; + --echo # Test cleanup drop database test; create database test; diff --git a/sql/partition_info.cc b/sql/partition_info.cc index 8133f15012c64..65df87bee041f 100644 --- a/sql/partition_info.cc +++ b/sql/partition_info.cc @@ -1940,7 +1940,7 @@ static void warn_if_dir_in_part_elem(THD *thd, partition_element *part_elem) bool partition_info::check_partition_info(THD *thd, handlerton **eng_type, handler *file, HA_CREATE_INFO *info, - bool add_or_reorg_part) + partition_info *add_or_reorg_part) { handlerton *table_engine= default_engine_type; uint i, tot_partitions; @@ -2174,6 +2174,24 @@ bool partition_info::check_partition_info(THD *thd, handlerton **eng_type, goto end; } + if (hist_parts > 1) + { + if (unlikely(vers_info->limit == 0 && vers_info->interval == 0)) + { + push_warning_printf(thd, + Sql_condition::WARN_LEVEL_WARN, + WARN_VERS_PARAMETERS, + ER_THD(thd, WARN_VERS_PARAMETERS), + "no rotation condition for multiple HISTORY partitions."); + } + } + if (unlikely(now_parts > 1)) + { + my_error(ER_VERS_WRONG_PARTS, MYF(0), info->alias); + goto end; + } + + DBUG_ASSERT(table_engine != partition_hton && default_engine_type == table_engine); if (eng_type) @@ -2188,6 +2206,9 @@ bool partition_info::check_partition_info(THD *thd, handlerton **eng_type, if (add_or_reorg_part) { + if (unlikely(part_type == VERSIONING_PARTITION && + vers_setup_expression(thd, add_or_reorg_part->partitions.elements))) + goto end; if (unlikely(((part_type == RANGE_PARTITION || part_type == VERSIONING_PARTITION) && check_range_constants(thd)) || (part_type == LIST_PARTITION && @@ -2195,22 +2216,6 @@ bool partition_info::check_partition_info(THD *thd, handlerton **eng_type, goto end; } - if (hist_parts > 1) - { - if (vers_info->limit == 0 && vers_info->interval == 0) - { - push_warning_printf(thd, - Sql_condition::WARN_LEVEL_WARN, - WARN_VERS_PARAMETERS, - ER_THD(thd, WARN_VERS_PARAMETERS), - "no rotation condition for multiple HISTORY partitions."); - } - } - if (now_parts > 1) - { - my_error(ER_VERS_WRONG_PARTS, MYF(0), info->alias); - goto end; - } result= FALSE; end: DBUG_RETURN(result); diff --git a/sql/partition_info.h b/sql/partition_info.h index 6f1414efb02bf..23e94661e6881 100644 --- a/sql/partition_info.h +++ b/sql/partition_info.h @@ -348,7 +348,7 @@ class partition_info : public Sql_alloc bool check_list_constants(THD *thd); bool check_partition_info(THD *thd, handlerton **eng_type, handler *file, HA_CREATE_INFO *info, - bool check_partition_function); + partition_info *add_or_reorg_part= NULL); void print_no_partition_found(TABLE *table, myf errflag); void print_debug(const char *str, uint*); Item* get_column_item(Item *item, Field *field); diff --git a/sql/sql_partition.cc b/sql/sql_partition.cc index d1da9823aa46f..0249fa1694ed4 100644 --- a/sql/sql_partition.cc +++ b/sql/sql_partition.cc @@ -5436,13 +5436,8 @@ the generated partition syntax in a correct manner. tab_part_info->use_default_num_subpartitions= FALSE; } - if (alter_info->flags & Alter_info::ALTER_ADD_PARTITION && - tab_part_info->part_type == VERSIONING_PARTITION && - tab_part_info->vers_setup_expression(thd, alt_part_info->partitions.elements)) - goto err; - if (tab_part_info->check_partition_info(thd, (handlerton**)NULL, - table->file, 0, TRUE)) + table->file, 0, alt_part_info)) { goto err; } From 57fd548d1035b07de1ec3e1a99881d5869ce0c44 Mon Sep 17 00:00:00 2001 From: Eugene Kosov Date: Fri, 12 Jan 2018 22:45:35 +0300 Subject: [PATCH 045/106] SQL: uninitialized read [#387] rpl.rpl_events sometimes fails in release mode after be81b00c843b4a3335a630afa7200fda6eab8e89 This could be prevented with MSAN --- sql/sql_class.cc | 2 ++ 1 file changed, 2 insertions(+) diff --git a/sql/sql_class.cc b/sql/sql_class.cc index 84fdaf8b1b6e4..9f6ff7cde3450 100644 --- a/sql/sql_class.cc +++ b/sql/sql_class.cc @@ -813,6 +813,8 @@ THD::THD(my_thread_id id, bool is_wsrep_applier, bool skip_global_sys_var_lock) // Must be reset to handle error with THD's created for init of mysqld lex->current_select= 0; start_utime= utime_after_query= 0; + system_time= 0; + system_time_sec_part= 0; utime_after_lock= 0L; progress.arena= 0; progress.report_to_client= 0; From 7e1738c3c4d11829b00092fbf9daa8840c1aa83c Mon Sep 17 00:00:00 2001 From: Sergei Golubchik Date: Wed, 10 Jan 2018 14:05:56 +0100 Subject: [PATCH 046/106] fix debian builds --- debian/mariadb-server-10.3.install | 1 - debian/mariadb-test.install | 1 + 2 files changed, 1 insertion(+), 1 deletion(-) diff --git a/debian/mariadb-server-10.3.install b/debian/mariadb-server-10.3.install index bc700ab248602..604179c7dca44 100644 --- a/debian/mariadb-server-10.3.install +++ b/debian/mariadb-server-10.3.install @@ -55,7 +55,6 @@ usr/lib/mysql/plugin/query_response_time.so usr/lib/mysql/plugin/server_audit.so usr/lib/mysql/plugin/simple_password_check.so usr/lib/mysql/plugin/sql_errlog.so -usr/lib/mysql/plugin/versioning.so usr/lib/mysql/plugin/wsrep_info.so usr/share/apport/package-hooks/source_mariadb-10.3.py usr/share/doc/mariadb-server-10.3/mysqld.sym.gz diff --git a/debian/mariadb-test.install b/debian/mariadb-test.install index 605620dc28c7c..75f52b3f52641 100644 --- a/debian/mariadb-test.install +++ b/debian/mariadb-test.install @@ -16,6 +16,7 @@ usr/lib/mysql/plugin/mypluglib.so usr/lib/mysql/plugin/qa_auth_client.so usr/lib/mysql/plugin/qa_auth_interface.so usr/lib/mysql/plugin/qa_auth_server.so +usr/lib/mysql/plugin/test_versioning.so usr/share/mysql/mysql-test/README usr/share/mysql/mysql-test/README-gcov usr/share/mysql/mysql-test/README.stress From a544f920e3fe9e24b20db434ca15bb9ebfe0137e Mon Sep 17 00:00:00 2001 From: Sergei Golubchik Date: Wed, 10 Jan 2018 16:12:24 +0100 Subject: [PATCH 047/106] remove "Transaction-based system versioning is EXPERIMENTAL" warning and the system_versioning_transaction_registry variable. The user enables transaction registry by specifying BIGINT for row_start/row_end columns. check mysql.transaction_registry structure on the first open, not on startup. Avoid warnings unless transaction_registry is actually used. --- mysql-test/r/mysqld--help.result | 4 --- .../mariabackup/system_versioning.result | 4 --- .../suite/mariabackup/system_versioning.test | 2 -- .../sys_vars/r/sysvars_server_embedded.result | 14 -------- .../r/sysvars_server_notembedded.result | 14 -------- mysql-test/suite/versioning/common.opt | 1 - mysql-test/suite/versioning/r/insert.result | 6 ---- mysql-test/suite/versioning/r/online.result | 4 --- mysql-test/suite/versioning/r/trx_id.result | 24 -------------- mysql-test/suite/versioning/t/insert.test | 4 --- mysql-test/suite/versioning/t/online.test | 3 -- mysql-test/suite/versioning/t/trx_id.test | 22 ------------- sql/handler.cc | 4 +-- sql/mysqld.cc | 32 ------------------- sql/mysqld.h | 2 -- sql/share/errmsg-utf8.txt | 5 +-- sql/sql_table.cc | 2 +- sql/sys_vars.cc | 18 ----------- sql/table.cc | 23 ++++++------- sql/table.h | 6 +++- sql/vtmd.cc | 2 +- 21 files changed, 22 insertions(+), 174 deletions(-) diff --git a/mysql-test/r/mysqld--help.result b/mysql-test/r/mysqld--help.result index 2ca655939a953..f980018b86b6b 100644 --- a/mysql-test/r/mysqld--help.result +++ b/mysql-test/r/mysqld--help.result @@ -1227,9 +1227,6 @@ The following options may be given as the first argument: Use simple algorithm of timestamp handling in InnoDB instead of TRX_SEES (Defaults to on; use --skip-system-versioning-innodb-algorithm-simple to disable.) - --system-versioning-transaction-registry - Enable or disable update of - `mysql`.`transaction_registry` --table-cache=# Deprecated; use --table-open-cache instead. --table-definition-cache=# The number of cached table definitions @@ -1660,7 +1657,6 @@ sync-relay-log-info 10000 sysdate-is-now FALSE system-versioning-alter-history ERROR system-versioning-innodb-algorithm-simple TRUE -system-versioning-transaction-registry FALSE table-cache 431 table-definition-cache 400 table-open-cache 431 diff --git a/mysql-test/suite/mariabackup/system_versioning.result b/mysql-test/suite/mariabackup/system_versioning.result index 3b3cd8f938cf2..0e1e9253dd0af 100644 --- a/mysql-test/suite/mariabackup/system_versioning.result +++ b/mysql-test/suite/mariabackup/system_versioning.result @@ -18,9 +18,6 @@ select a from t for system_time all; a 2 1 -set global system_versioning_transaction_registry=on; -Warnings: -Warning 4143 Transaction-based system versioning is EXPERIMENTAL and is subject to change in future. create or replace table t ( a int, s bigint unsigned as row start invisible, @@ -50,4 +47,3 @@ a 2 1 drop table t; -set global system_versioning_transaction_registry=off; diff --git a/mysql-test/suite/mariabackup/system_versioning.test b/mysql-test/suite/mariabackup/system_versioning.test index 99392c3fc5b6a..1ced00b4588b1 100644 --- a/mysql-test/suite/mariabackup/system_versioning.test +++ b/mysql-test/suite/mariabackup/system_versioning.test @@ -22,7 +22,6 @@ select a from t for system_time all; rmdir $targetdir; -set global system_versioning_transaction_registry=on; create or replace table t ( a int, s bigint unsigned as row start invisible, @@ -48,5 +47,4 @@ select * from t; select a from t for system_time all; drop table t; -set global system_versioning_transaction_registry=off; rmdir $targetdir; diff --git a/mysql-test/suite/sys_vars/r/sysvars_server_embedded.result b/mysql-test/suite/sys_vars/r/sysvars_server_embedded.result index c80733322767d..49c384164c9e6 100644 --- a/mysql-test/suite/sys_vars/r/sysvars_server_embedded.result +++ b/mysql-test/suite/sys_vars/r/sysvars_server_embedded.result @@ -4100,20 +4100,6 @@ NUMERIC_BLOCK_SIZE NULL ENUM_VALUE_LIST OFF,ON READ_ONLY NO COMMAND_LINE_ARGUMENT OPTIONAL -VARIABLE_NAME SYSTEM_VERSIONING_TRANSACTION_REGISTRY -SESSION_VALUE NULL -GLOBAL_VALUE OFF -GLOBAL_VALUE_ORIGIN COMPILE-TIME -DEFAULT_VALUE OFF -VARIABLE_SCOPE GLOBAL -VARIABLE_TYPE BOOLEAN -VARIABLE_COMMENT Enable or disable update of `mysql`.`transaction_registry` -NUMERIC_MIN_VALUE NULL -NUMERIC_MAX_VALUE NULL -NUMERIC_BLOCK_SIZE NULL -ENUM_VALUE_LIST OFF,ON -READ_ONLY NO -COMMAND_LINE_ARGUMENT OPTIONAL VARIABLE_NAME TABLE_DEFINITION_CACHE SESSION_VALUE NULL GLOBAL_VALUE 400 diff --git a/mysql-test/suite/sys_vars/r/sysvars_server_notembedded.result b/mysql-test/suite/sys_vars/r/sysvars_server_notembedded.result index 0ebf5d71f4e8e..9ee1d72d40685 100644 --- a/mysql-test/suite/sys_vars/r/sysvars_server_notembedded.result +++ b/mysql-test/suite/sys_vars/r/sysvars_server_notembedded.result @@ -5052,20 +5052,6 @@ NUMERIC_BLOCK_SIZE NULL ENUM_VALUE_LIST OFF,ON READ_ONLY NO COMMAND_LINE_ARGUMENT OPTIONAL -VARIABLE_NAME SYSTEM_VERSIONING_TRANSACTION_REGISTRY -SESSION_VALUE NULL -GLOBAL_VALUE OFF -GLOBAL_VALUE_ORIGIN COMPILE-TIME -DEFAULT_VALUE OFF -VARIABLE_SCOPE GLOBAL -VARIABLE_TYPE BOOLEAN -VARIABLE_COMMENT Enable or disable update of `mysql`.`transaction_registry` -NUMERIC_MIN_VALUE NULL -NUMERIC_MAX_VALUE NULL -NUMERIC_BLOCK_SIZE NULL -ENUM_VALUE_LIST OFF,ON -READ_ONLY NO -COMMAND_LINE_ARGUMENT OPTIONAL VARIABLE_NAME TABLE_DEFINITION_CACHE SESSION_VALUE NULL GLOBAL_VALUE 400 diff --git a/mysql-test/suite/versioning/common.opt b/mysql-test/suite/versioning/common.opt index 391f1a5cba965..412290a7585cd 100644 --- a/mysql-test/suite/versioning/common.opt +++ b/mysql-test/suite/versioning/common.opt @@ -1,2 +1 @@ ---system-versioning-transaction-registry=1 --plugin-load-add=test_versioning diff --git a/mysql-test/suite/versioning/r/insert.result b/mysql-test/suite/versioning/r/insert.result index 943175de6c583..493f34a97f315 100644 --- a/mysql-test/suite/versioning/r/insert.result +++ b/mysql-test/suite/versioning/r/insert.result @@ -284,13 +284,7 @@ rollback to a; commit; call verify_vtq; No A B C D -set global system_versioning_transaction_registry= off; insert into t2(x) values (1); -insert into t1(x) values (1); -ERROR HY000: Temporal operation requires `mysql.transaction_registry` (@@system_versioning_transaction_registry). -set global system_versioning_transaction_registry= on; -Warnings: -Warning 4134 Transaction-based system versioning is EXPERIMENTAL and is subject to change in future. create or replace table t1 ( x int, y int as (x) virtual, diff --git a/mysql-test/suite/versioning/r/online.result b/mysql-test/suite/versioning/r/online.result index 267da5db8fada..b2a34481d6390 100644 --- a/mysql-test/suite/versioning/r/online.result +++ b/mysql-test/suite/versioning/r/online.result @@ -12,9 +12,6 @@ alter table t drop system versioning, lock=none; ERROR 0A000: LOCK=NONE is not supported. Reason: Not implemented for system-versioned tables. Try LOCK=SHARED alter table t drop system versioning, algorithm=inplace; ERROR 0A000: ALGORITHM=INPLACE is not supported. Reason: Not implemented for system-versioned tables. Try ALGORITHM=COPY -set global system_versioning_transaction_registry=on; -Warnings: -Warning 4134 Transaction-based system versioning is EXPERIMENTAL and is subject to change in future. create or replace table t (a int, b int) engine=innodb; alter table t add s bigint unsigned as row start, @@ -34,5 +31,4 @@ alter table t add index idx(a), lock=none; alter table t drop column s, drop column e; alter table t drop system versioning, lock=none; ERROR 0A000: LOCK=NONE is not supported. Reason: Not implemented for system-versioned tables. Try LOCK=SHARED -set global system_versioning_transaction_registry=off; drop table t; diff --git a/mysql-test/suite/versioning/r/trx_id.result b/mysql-test/suite/versioning/r/trx_id.result index 800d61beba784..a88ba7eec2b4c 100644 --- a/mysql-test/suite/versioning/r/trx_id.result +++ b/mysql-test/suite/versioning/r/trx_id.result @@ -1,16 +1,3 @@ -select @@system_versioning_transaction_registry; -@@system_versioning_transaction_registry -0 -create or replace table t1 ( -x int, -sys_trx_start bigint(20) unsigned as row start invisible, -sys_trx_end bigint(20) unsigned as row end invisible, -period for system_time (sys_trx_start, sys_trx_end) -) with system versioning engine innodb; -ERROR HY000: Temporal operation requires `mysql.transaction_registry` (@@system_versioning_transaction_registry). -set global system_versioning_transaction_registry= 1; -Warnings: -Warning 4134 Transaction-based system versioning is EXPERIMENTAL and is subject to change in future. create or replace table t1 ( x int, sys_trx_start bigint(20) unsigned as row start invisible, @@ -18,21 +5,11 @@ sys_trx_end bigint(20) unsigned as row end invisible, period for system_time (sys_trx_start, sys_trx_end) ) with system versioning engine innodb; insert into t1 (x) values (1); -set global system_versioning_transaction_registry= 0; -insert into t1 (x) values (2); -ERROR HY000: Temporal operation requires `mysql.transaction_registry` (@@system_versioning_transaction_registry). -delete from t1; -ERROR HY000: Temporal operation requires `mysql.transaction_registry` (@@system_versioning_transaction_registry). -update t1 set x= 3; -ERROR HY000: Temporal operation requires `mysql.transaction_registry` (@@system_versioning_transaction_registry). # ALTER ADD SYSTEM VERSIONING should write to mysql.transaction_registry create function check_result (cond boolean) returns char(50) deterministic return if(cond = 1, '[CORRECT]', '[INCORRECT]'); set @@system_versioning_alter_history=keep; -set global system_versioning_transaction_registry=on; -Warnings: -Warning 4134 Transaction-based system versioning is EXPERIMENTAL and is subject to change in future. create or replace table t1 (x int) engine innodb; insert into t1 values (1); alter table t1 @@ -92,5 +69,4 @@ select sys_end = 18446744073709551615 as transaction_based from t1 for system_ti transaction_based 1 drop table t1; -set global system_versioning_transaction_registry=off; drop function check_result; diff --git a/mysql-test/suite/versioning/t/insert.test b/mysql-test/suite/versioning/t/insert.test index c7c0e0a7c0b59..f2fbe31e1d0f2 100644 --- a/mysql-test/suite/versioning/t/insert.test +++ b/mysql-test/suite/versioning/t/insert.test @@ -188,11 +188,7 @@ rollback to a; commit; call verify_vtq; -set global system_versioning_transaction_registry= off; insert into t2(x) values (1); ---error ER_VERS_TRT_IS_DISABLED -insert into t1(x) values (1); -set global system_versioning_transaction_registry= on; # virtual columns create or replace table t1 ( diff --git a/mysql-test/suite/versioning/t/online.test b/mysql-test/suite/versioning/t/online.test index 3b1c20792d263..4fbd5d85100e3 100644 --- a/mysql-test/suite/versioning/t/online.test +++ b/mysql-test/suite/versioning/t/online.test @@ -18,7 +18,6 @@ alter table t drop system versioning, lock=none; alter table t drop system versioning, algorithm=inplace; -set global system_versioning_transaction_registry=on; create or replace table t (a int, b int) engine=innodb; --error ER_ALTER_OPERATION_NOT_SUPPORTED_REASON @@ -40,6 +39,4 @@ alter table t drop column s, drop column e; --error ER_ALTER_OPERATION_NOT_SUPPORTED_REASON alter table t drop system versioning, lock=none; -set global system_versioning_transaction_registry=off; - drop table t; diff --git a/mysql-test/suite/versioning/t/trx_id.test b/mysql-test/suite/versioning/t/trx_id.test index d42333d56d9de..ee4e927b50698 100644 --- a/mysql-test/suite/versioning/t/trx_id.test +++ b/mysql-test/suite/versioning/t/trx_id.test @@ -1,18 +1,6 @@ -- source include/have_innodb.inc -- source include/not_embedded.inc -select @@system_versioning_transaction_registry; - ---error ER_VERS_TRT_IS_DISABLED -create or replace table t1 ( - x int, - sys_trx_start bigint(20) unsigned as row start invisible, - sys_trx_end bigint(20) unsigned as row end invisible, - period for system_time (sys_trx_start, sys_trx_end) -) with system versioning engine innodb; - -set global system_versioning_transaction_registry= 1; - create or replace table t1 ( x int, sys_trx_start bigint(20) unsigned as row start invisible, @@ -22,21 +10,12 @@ create or replace table t1 ( insert into t1 (x) values (1); -set global system_versioning_transaction_registry= 0; ---error ER_VERS_TRT_IS_DISABLED -insert into t1 (x) values (2); ---error ER_VERS_TRT_IS_DISABLED -delete from t1; ---error ER_VERS_TRT_IS_DISABLED -update t1 set x= 3; - --echo # ALTER ADD SYSTEM VERSIONING should write to mysql.transaction_registry create function check_result (cond boolean) returns char(50) deterministic return if(cond = 1, '[CORRECT]', '[INCORRECT]'); set @@system_versioning_alter_history=keep; -set global system_versioning_transaction_registry=on; create or replace table t1 (x int) engine innodb; insert into t1 values (1); @@ -93,5 +72,4 @@ alter table t1 drop column sys_start, drop column sys_end; select sys_end = 18446744073709551615 as transaction_based from t1 for system_time all; drop table t1; -set global system_versioning_transaction_registry=off; drop function check_result; diff --git a/sql/handler.cc b/sql/handler.cc index 36d4879e242d1..5a008f0dbeef7 100644 --- a/sql/handler.cc +++ b/sql/handler.cc @@ -1432,7 +1432,7 @@ int ha_commit_trans(THD *thd, bool all) if (trx_end_id) { - if (!use_transaction_registry) + if (!TR_table::use_transaction_registry) { my_error(ER_VERS_TRT_IS_DISABLED, MYF(0)); goto err; @@ -7517,7 +7517,7 @@ bool Vers_parse_info::check_sys_fields(const char *table_name, { if (found == check_unit) { - if (found == VERS_TRX_ID && !use_transaction_registry) + if (found == VERS_TRX_ID && !TR_table::use_transaction_registry) { my_error(ER_VERS_TRT_IS_DISABLED, MYF(0)); return true; diff --git a/sql/mysqld.cc b/sql/mysqld.cc index c0d5530687174..321264e8d7b09 100644 --- a/sql/mysqld.cc +++ b/sql/mysqld.cc @@ -536,8 +536,6 @@ ulonglong slave_skipped_errors; ulong feature_files_opened_with_delayed_keys= 0, feature_check_constraint= 0; ulonglong denied_connections; my_decimal decimal_zero; -my_bool opt_transaction_registry= 1; -my_bool use_transaction_registry= 1; /* Maximum length of parameter value which can be set through @@ -6023,36 +6021,6 @@ int mysqld_main(int argc, char **argv) if (Events::init((THD*) 0, opt_noacl || opt_bootstrap)) unireg_abort(1); - if (opt_transaction_registry) - { - use_transaction_registry= true; - if (opt_bootstrap) - { - use_transaction_registry= false; - } - else - { - THD *thd = new THD(0); - thd->thread_stack= (char*) &thd; - thd->store_globals(); - { - TR_table trt(thd); - if (trt.check()) - { - use_transaction_registry= false; - } - } - - trans_commit_stmt(thd); - delete thd; - } - } - else - use_transaction_registry= false; - - if (opt_transaction_registry && !use_transaction_registry) - sql_print_information("Disabled transaction registry."); - if (WSREP_ON) { if (opt_bootstrap) diff --git a/sql/mysqld.h b/sql/mysqld.h index 3175a1a385cfc..88a0fa000715c 100644 --- a/sql/mysqld.h +++ b/sql/mysqld.h @@ -315,8 +315,6 @@ extern my_bool encrypt_tmp_disk_tables, encrypt_tmp_files; extern ulong encryption_algorithm; extern const char *encryption_algorithm_names[]; extern const char *quoted_string; -extern my_bool opt_transaction_registry; -extern my_bool use_transaction_registry; #ifdef HAVE_PSI_INTERFACE #ifdef HAVE_MMAP diff --git a/sql/share/errmsg-utf8.txt b/sql/share/errmsg-utf8.txt index 5a404997a5698..a6b399810eae3 100644 --- a/sql/share/errmsg-utf8.txt +++ b/sql/share/errmsg-utf8.txt @@ -7889,7 +7889,7 @@ ER_NOT_LOG_TABLE eng "Table %`s.%`s is not a log table" ER_VERS_TRT_IS_DISABLED - eng "Temporal operation requires `mysql.transaction_registry` (@@system_versioning_transaction_registry)." + eng "Transaction registry is disabled" ER_VERS_DUPLICATE_ROW_START_END eng "Duplicate ROW %s column %`s" @@ -7897,9 +7897,6 @@ ER_VERS_DUPLICATE_ROW_START_END ER_VERS_ALREADY_VERSIONED eng "Table %`s is already system-versioned" -WARN_VERS_TRT_EXPERIMENTAL - eng "Transaction-based system versioning is EXPERIMENTAL and is subject to change in future." - ER_VERS_TRUNCATE_VIEW eng "DELETE HISTORY from VIEW is prohibited" diff --git a/sql/sql_table.cc b/sql/sql_table.cc index 58e29ad9b1812..d1bb661c3b4df 100644 --- a/sql/sql_table.cc +++ b/sql/sql_table.cc @@ -7609,7 +7609,7 @@ static bool mysql_inplace_alter_table(THD *thd, ulonglong trx_end_id= table->file->ht->prepare_commit_versioned(thd, &trx_start_id); if (trx_end_id) { - if (!use_transaction_registry) + if (!TR_table::use_transaction_registry) { my_error(ER_VERS_TRT_IS_DISABLED, MYF(0)); goto rollback; diff --git a/sql/sys_vars.cc b/sql/sys_vars.cc index 3b37fac5c39c4..d00f35e2028fb 100644 --- a/sql/sys_vars.cc +++ b/sql/sys_vars.cc @@ -427,24 +427,6 @@ static Sys_var_enum Sys_vers_alter_history( SESSION_VAR(vers_alter_history), CMD_LINE(REQUIRED_ARG), vers_alter_history_keywords, DEFAULT(VERS_ALTER_HISTORY_ERROR)); -static bool update_transaction_registry(sys_var *self, THD *thd, enum_var_type type) -{ - use_transaction_registry= opt_transaction_registry; - if (use_transaction_registry) - { - push_warning(thd, Sql_condition::WARN_LEVEL_WARN, WARN_VERS_TRT_EXPERIMENTAL, - ER_THD(thd, WARN_VERS_TRT_EXPERIMENTAL)); - } - return false; -} - -static Sys_var_mybool Sys_vers_transaction_registry( - "system_versioning_transaction_registry", - "Enable or disable update of `mysql`.`transaction_registry`", - GLOBAL_VAR(opt_transaction_registry), CMD_LINE(OPT_ARG), - DEFAULT(FALSE), NO_MUTEX_GUARD, NOT_IN_BINLOG, - 0, ON_UPDATE(update_transaction_registry)); - static Sys_var_ulonglong Sys_binlog_cache_size( "binlog_cache_size", "The size of the transactional cache for " "updates to transactional engines for the binary log. " diff --git a/sql/table.cc b/sql/table.cc index d7e7e3f0a8947..5a87acd4ed42f 100644 --- a/sql/table.cc +++ b/sql/table.cc @@ -8579,6 +8579,8 @@ void TABLE_SHARE::vers_destroy() } } +enum TR_table::enabled TR_table::use_transaction_registry= TR_table::MAYBE; + TR_table::TR_table(THD* _thd, bool rw) : thd(_thd), open_tables_backup(NULL) { @@ -8601,6 +8603,11 @@ bool TR_table::open() bool error= !open_log_table(thd, this, open_tables_backup); thd->temporary_tables= temporary_tables; + if (use_transaction_registry == MAYBE) + error= check(error); + + use_transaction_registry= error ? NO : YES; + return error; } @@ -8787,27 +8794,21 @@ void TR_table::warn_schema_incorrect(const char *reason) { if (MYSQL_VERSION_ID == table->s->mysql_version) { - sql_print_error("`%s.%s` schema is incorrect: %s.", db, table_name, reason); + sql_print_error("%`s.%`s schema is incorrect: %s.", db, table_name, reason); } else { - sql_print_error("`%s.%s` schema is incorrect: %s. Created with MariaDB %d, " + sql_print_error("%`s.%`s schema is incorrect: %s. Created with MariaDB %d, " "now running %d.", db, table_name, reason, MYSQL_VERSION_ID, static_cast(table->s->mysql_version)); } } -bool TR_table::check() +bool TR_table::check(bool error) { - if (!ha_resolve_by_legacy_type(thd, DB_TYPE_INNODB)) - { - sql_print_information("`%s.%s` requires InnoDB storage engine.", db, table_name); - return true; - } - - if (open()) + if (error) { - sql_print_warning("`%s.%s` does not exist (open failed).", db, table_name); + sql_print_warning("%`s.%`s does not exist (open failed).", db, table_name); return true; } diff --git a/sql/table.h b/sql/table.h index 6899b65eb573e..71858be6229bb 100644 --- a/sql/table.h +++ b/sql/table.h @@ -2992,6 +2992,10 @@ class TR_table: public TABLE_LIST FLD_ISO_LEVEL, FIELD_COUNT }; + + enum enabled {NO, MAYBE, YES}; + static enum enabled use_transaction_registry; + /** @param[in,out] Thread handle @param[in] Current transaction is read-write. @@ -3084,7 +3088,7 @@ class TR_table: public TABLE_LIST @retval true if schema is incorrect and false otherwise */ - bool check(); + bool check(bool error); TABLE * operator-> () const { diff --git a/sql/vtmd.cc b/sql/vtmd.cc index 566638bbc4677..a78fdd75a2a00 100644 --- a/sql/vtmd.cc +++ b/sql/vtmd.cc @@ -257,7 +257,7 @@ VTMD_table::update(THD *thd, const char* archive_name) quit: if (!result && vtmd.table->file->ht->prepare_commit_versioned) { - DBUG_ASSERT(use_transaction_registry); // FIXME: disable survival mode while TRT is disabled + DBUG_ASSERT(TR_table::use_transaction_registry); // FIXME: disable survival mode while TRT is disabled TR_table trt(thd, true); ulonglong trx_start_id= 0; ulonglong trx_end_id= vtmd.table->file->ht->prepare_commit_versioned(thd, &trx_start_id); From 376b0ea1da1514ca645ae4e275a412c1bc9d4b7d Mon Sep 17 00:00:00 2001 From: Sergei Golubchik Date: Wed, 10 Jan 2018 19:26:31 +0100 Subject: [PATCH 048/106] Revert "SQL: Backup_query_start_time RAII" This reverts commit 07b1a7743096b01ea55407bada7bfad8c609397c. --- sql/sp_head.cc | 7 +++++-- sql/sql_class.h | 30 ++++-------------------------- 2 files changed, 9 insertions(+), 28 deletions(-) diff --git a/sql/sp_head.cc b/sql/sp_head.cc index decf007b2d894..07e760f44384d 100644 --- a/sql/sp_head.cc +++ b/sql/sp_head.cc @@ -3272,7 +3272,7 @@ sp_instr_stmt::execute(THD *thd, uint *nextp) int res; bool save_enable_slow_log; const CSET_STRING query_backup= thd->query_string; - Backup_query_start_time time_info; + QUERY_START_TIME_INFO time_info; Sub_statement_state backup_state; DBUG_ENTER("sp_instr_stmt::execute"); DBUG_PRINT("info", ("command: %d", m_lex_keeper.sql_command())); @@ -3288,7 +3288,7 @@ sp_instr_stmt::execute(THD *thd, uint *nextp) Save start time info for the CALL statement and overwrite it with the current time for log_slow_statement() to log the individual query timing. */ - time_info.backup(*thd); + thd->backup_query_start_time(&time_info); thd->set_time(); } thd->store_slow_query_state(&backup_state); @@ -3354,6 +3354,9 @@ sp_instr_stmt::execute(THD *thd, uint *nextp) thd->get_stmt_da()->reset_diagnostics_area(); } } + /* Restore the original query start time */ + if (thd->enable_slow_log) + thd->restore_query_start_time(&time_info); DBUG_RETURN(res || thd->is_error()); } diff --git a/sql/sql_class.h b/sql/sql_class.h index c15bbe54a1b48..647f254a117aa 100644 --- a/sql/sql_class.h +++ b/sql/sql_class.h @@ -2089,36 +2089,14 @@ struct QUERY_START_TIME_INFO my_time_t start_time; ulong start_time_sec_part; ulonglong start_utime, utime_after_lock; -}; - -class Backup_query_start_time : public QUERY_START_TIME_INFO -{ - QUERY_START_TIME_INFO *m_origin; -public: - Backup_query_start_time() : m_origin(NULL) - {} - Backup_query_start_time(QUERY_START_TIME_INFO &origin) + void backup_query_start_time(QUERY_START_TIME_INFO *backup) { - backup(origin); + *backup= *this; } - ~Backup_query_start_time() + void restore_query_start_time(QUERY_START_TIME_INFO *backup) { - restore(); - } - void backup(QUERY_START_TIME_INFO &origin) - { - m_origin= &origin; - QUERY_START_TIME_INFO *backup_= this; - *backup_= origin; - } - void restore() - { - if (m_origin) - { - *m_origin= *this; - m_origin= NULL; - } + *this= *backup; } }; From 5d3bae242c6756fa57a34adb4ad56232566ef385 Mon Sep 17 00:00:00 2001 From: Sergei Golubchik Date: Wed, 10 Jan 2018 20:09:06 +0100 Subject: [PATCH 049/106] remove dead VERS_EXPERIMENTAL code changed to use DBUG keywords instead, so that the code is compiled and tested added tests. --- mysql-test/suite/versioning/r/alter.result | 2 +- mysql-test/suite/versioning/r/debug.result | 39 ++++++++++++++++++++++ mysql-test/suite/versioning/t/alter.test | 2 +- mysql-test/suite/versioning/t/debug.test | 29 ++++++++++++++++ sql/field.h | 4 --- sql/handler.cc | 16 +++------ sql/mysqld.cc | 5 --- sql/mysqld.h | 9 ----- sql/sql_base.cc | 36 +------------------- sql/sql_class.h | 4 --- sql/sql_show.cc | 32 +++--------------- sql/sql_yacc.yy | 16 +-------- sql/sys_vars.cc | 15 --------- 13 files changed, 80 insertions(+), 129 deletions(-) create mode 100644 mysql-test/suite/versioning/r/debug.result create mode 100644 mysql-test/suite/versioning/t/debug.test diff --git a/mysql-test/suite/versioning/r/alter.result b/mysql-test/suite/versioning/r/alter.result index 8ebfb96ebc15f..04d7139b207a5 100644 --- a/mysql-test/suite/versioning/r/alter.result +++ b/mysql-test/suite/versioning/r/alter.result @@ -388,7 +388,7 @@ alter table t modify column row_start bigint unsigned; ERROR HY000: Can not change system versioning field `row_start` set system_versioning_alter_history= SURVIVE; ERROR 42000: Variable 'system_versioning_alter_history' can't be set to the value of 'SURVIVE' -set system_versioning_alter_history= DROP; +set system_versioning_alter_history= 'DROP'; ERROR 42000: Variable 'system_versioning_alter_history' can't be set to the value of 'DROP' create or replace table t (a int) with system versioning; alter table t add system versioning; diff --git a/mysql-test/suite/versioning/r/debug.result b/mysql-test/suite/versioning/r/debug.result new file mode 100644 index 0000000000000..6e704011002b6 --- /dev/null +++ b/mysql-test/suite/versioning/r/debug.result @@ -0,0 +1,39 @@ +create table t1 (a int); +show create table t1; +Table Create Table +t1 CREATE TABLE `t1` ( + `a` int(11) DEFAULT NULL +) ENGINE=MyISAM DEFAULT CHARSET=latin1 +create temporary table tt1 (a int) with system versioning; +ERROR HY000: System versioning prohibited for TEMPORARY tables +set @old_dbug=@@global.debug_dbug; +set global debug_dbug='+d,sysvers_force'; +create table t2 (a int); +show create table t2; +Table Create Table +t2 CREATE TABLE `t2` ( + `a` int(11) DEFAULT NULL +) ENGINE=MyISAM DEFAULT CHARSET=latin1 WITH SYSTEM VERSIONING +create temporary table tt2 (a int) with system versioning; +show create table tt2; +Table Create Table +tt2 CREATE TEMPORARY TABLE `tt2` ( + `a` int(11) DEFAULT NULL +) ENGINE=MyISAM DEFAULT CHARSET=latin1 +connect con1, localhost, root; +create table t3 (a int); +show create table t3; +Table Create Table +t3 CREATE TABLE `t3` ( + `a` int(11) DEFAULT NULL +) ENGINE=MyISAM DEFAULT CHARSET=latin1 WITH SYSTEM VERSIONING +create temporary table tt3 (a int) with system versioning; +show create table tt3; +Table Create Table +tt3 CREATE TEMPORARY TABLE `tt3` ( + `a` int(11) DEFAULT NULL +) ENGINE=MyISAM DEFAULT CHARSET=latin1 +disconnect con1; +connection default; +set global debug_dbug=@old_dbug; +drop table t1, t2, t3; diff --git a/mysql-test/suite/versioning/t/alter.test b/mysql-test/suite/versioning/t/alter.test index 1394c3bc7910f..5e4efb036c689 100644 --- a/mysql-test/suite/versioning/t/alter.test +++ b/mysql-test/suite/versioning/t/alter.test @@ -272,7 +272,7 @@ alter table t modify column row_start bigint unsigned; set system_versioning_alter_history= SURVIVE; --error ER_WRONG_VALUE_FOR_VAR -set system_versioning_alter_history= DROP; +set system_versioning_alter_history= 'DROP'; if (0) { diff --git a/mysql-test/suite/versioning/t/debug.test b/mysql-test/suite/versioning/t/debug.test new file mode 100644 index 0000000000000..027af48fff55a --- /dev/null +++ b/mysql-test/suite/versioning/t/debug.test @@ -0,0 +1,29 @@ +--source include/have_debug.inc + +create table t1 (a int); +show create table t1; + +--error ER_VERS_TEMPORARY +create temporary table tt1 (a int) with system versioning; + +set @old_dbug=@@global.debug_dbug; +set global debug_dbug='+d,sysvers_force'; + +create table t2 (a int); +show create table t2; + +create temporary table tt2 (a int) with system versioning; +show create table tt2; + +--connect con1, localhost, root + +create table t3 (a int); +show create table t3; + +create temporary table tt3 (a int) with system versioning; +show create table tt3; +--disconnect con1 +--connection default + +set global debug_dbug=@old_dbug; +drop table t1, t2, t3; diff --git a/sql/field.h b/sql/field.h index 9fd584e88c2fa..549dc3d00d5ec 100644 --- a/sql/field.h +++ b/sql/field.h @@ -1474,10 +1474,6 @@ class Field: public Value_source return flags & VERS_UPDATE_UNVERSIONED_FLAG; } -#ifdef VERS_EXPERIMENTAL - bool vers_sys_invisible(THD *thd) const; -#endif - virtual bool vers_trx_id() const { return false; diff --git a/sql/handler.cc b/sql/handler.cc index 5a008f0dbeef7..84cb550143cc3 100644 --- a/sql/handler.cc +++ b/sql/handler.cc @@ -6962,13 +6962,9 @@ bool Table_scope_and_contents_source_st::vers_fix_system_fields( } } -#ifdef VERS_EXPERIMENTAL - if (thd->variables.vers_force) - { - alter_info->flags|= Alter_info::ALTER_ADD_SYSTEM_VERSIONING; - options|= HA_VERSIONED_TABLE; - } -#endif + DBUG_EXECUTE_IF("sysvers_force", if (!tmp_table()) { + alter_info->flags|= Alter_info::ALTER_ADD_SYSTEM_VERSIONING; + options|= HA_VERSIONED_TABLE; }); // Possibly override default storage engine to match one used in source table. if (vers_tables && alter_info->flags & Alter_info::ALTER_ADD_SYSTEM_VERSIONING && @@ -7183,11 +7179,7 @@ bool Vers_parse_info::fix_alter_info(THD *thd, Alter_info *alter_info, if (!need_check(alter_info) && !share->versioned) return false; - if (share->tmp_table && share->tmp_table != INTERNAL_TMP_TABLE -#ifdef VERS_EXPERIMENTAL - && !thd->variables.vers_force -#endif - ) + if (DBUG_EVALUATE_IF("sysvers_force", 0, share->tmp_table)) { my_error(ER_VERS_TEMPORARY, MYF(0)); return true; diff --git a/sql/mysqld.cc b/sql/mysqld.cc index 321264e8d7b09..067b2e70ba6e8 100644 --- a/sql/mysqld.cc +++ b/sql/mysqld.cc @@ -9852,11 +9852,6 @@ static int get_options(int *argc_ptr, char ***argv_ptr) if (thread_cache_size > max_connections) SYSVAR_AUTOSIZE(thread_cache_size, max_connections); -#ifdef VERS_EXPERIMENTAL - if (opt_bootstrap) - global_system_variables.vers_force= 0; -#endif - return 0; } diff --git a/sql/mysqld.h b/sql/mysqld.h index 88a0fa000715c..dd84adfdff387 100644 --- a/sql/mysqld.h +++ b/sql/mysqld.h @@ -200,15 +200,6 @@ struct vers_asof_timestamp_t {} }; -#ifdef VERS_EXPERIMENTAL -enum vers_show_enum -{ - VERS_SHOW_OFF= 0, - VERS_SHOW_RANGE, - VERS_SHOW_ALWAYS -}; -#endif - enum vers_alter_history_enum { VERS_ALTER_HISTORY_ERROR= 0, diff --git a/sql/sql_base.cc b/sql/sql_base.cc index 69b867da0cbde..3c913f8353092 100644 --- a/sql/sql_base.cc +++ b/sql/sql_base.cc @@ -7617,35 +7617,6 @@ bool get_key_map_from_key_list(key_map *map, TABLE *table, } -#ifdef VERS_EXPERIMENTAL -inline -bool Field::vers_sys_invisible(THD *thd) const -{ - DBUG_ASSERT(vers_sys_field()); - DBUG_ASSERT(table); - DBUG_ASSERT(table->versioned()); - DBUG_ASSERT(table->pos_in_table_list); - - if (thd->lex->sql_command != SQLCOM_SELECT) - return invisible; - - switch (thd->variables.vers_show) - { - case VERS_SHOW_RANGE: - { - vers_system_time_t vers_type= table->pos_in_table_list->vers_conditions.type; - return vers_type > SYSTEM_TIME_AS_OF ? false : invisible; - } - case VERS_SHOW_ALWAYS: - return false; - default: - break; - }; - return invisible; -} -#endif - - /* Drops in all fields instead of current '*' field @@ -7775,12 +7746,7 @@ insert_fields(THD *thd, Name_resolution_context *context, const char *db_name, Field_iterator_natural_join). But view fields can never be invisible. */ - if ((field= field_iterator.field()) && ( -#ifdef VERS_EXPERIMENTAL - field->vers_sys_field() && field->table->versioned() ? - field->vers_sys_invisible(thd) : -#endif - field->invisible != VISIBLE)) + if ((field= field_iterator.field()) && field->invisible != VISIBLE) continue; Item *item; diff --git a/sql/sql_class.h b/sql/sql_class.h index 647f254a117aa..5469171f0e368 100644 --- a/sql/sql_class.h +++ b/sql/sql_class.h @@ -708,10 +708,6 @@ typedef struct system_variables uint in_subquery_conversion_threshold; vers_asof_timestamp_t vers_asof_timestamp; -#ifdef VERS_EXPERIMENTAL - my_bool vers_force; - ulong vers_show; -#endif my_bool vers_innodb_algorithm_simple; ulong vers_alter_history; } SV; diff --git a/sql/sql_show.cc b/sql/sql_show.cc index 41cf8556e8771..3fc0568d2a954 100644 --- a/sql/sql_show.cc +++ b/sql/sql_show.cc @@ -2108,9 +2108,6 @@ int show_create_table(THD *thd, TABLE_LIST *table_list, String *packet, TABLE *table= table_list->table; TABLE_SHARE *share= table->s; sql_mode_t sql_mode= thd->variables.sql_mode; -#ifdef VERS_EXPERIMENTAL - ulong vers_show= thd->variables.vers_show; -#endif bool explicit_fields= false; bool foreign_db_mode= sql_mode & (MODE_POSTGRESQL | MODE_ORACLE | MODE_MSSQL | MODE_DB2 | @@ -2192,11 +2189,7 @@ int show_create_table(THD *thd, TABLE_LIST *table_list, String *packet, uint flags = field->flags; - if (field->invisible > INVISIBLE_USER -#ifdef VERS_EXPERIMENTAL - && !(field->vers_sys_field() && vers_show == VERS_SHOW_ALWAYS) -#endif - ) + if (field->invisible > INVISIBLE_USER) continue; if (not_the_first_field) packet->append(STRING_WITH_LEN(",\n")); @@ -2344,11 +2337,7 @@ int show_create_table(THD *thd, TABLE_LIST *table_list, String *packet, for (uint j=0 ; j < key_info->user_defined_key_parts ; j++,key_part++) { Field *field= key_part->field; - if (field->invisible > INVISIBLE_USER -#ifdef VERS_EXPERIMENTAL - && !(field->vers_sys_field() && vers_show == VERS_SHOW_ALWAYS) -#endif - ) + if (field->invisible > INVISIBLE_USER) continue; if (j) @@ -2387,11 +2376,7 @@ int show_create_table(THD *thd, TABLE_LIST *table_list, String *packet, DBUG_ASSERT(fe); explicit_fields= fs->invisible < INVISIBLE_SYSTEM; DBUG_ASSERT(!explicit_fields || fe->invisible < INVISIBLE_SYSTEM); - if (explicit_fields -#ifdef VERS_EXPERIMENTAL - || vers_show == VERS_SHOW_ALWAYS -#endif - ) + if (explicit_fields) { packet->append(STRING_WITH_LEN(",\n PERIOD FOR SYSTEM_TIME (")); append_identifier(thd,packet,fs->field_name.str, fs->field_name.length); @@ -2444,11 +2429,7 @@ int show_create_table(THD *thd, TABLE_LIST *table_list, String *packet, add_table_options(thd, table, create_info_arg, table_list->schema_table != 0, 0, packet); - if (table->versioned() -#ifdef VERS_EXPERIMENTAL - && (!thd->variables.vers_force || explicit_fields) -#endif - ) + if (table->versioned()) packet->append(STRING_WITH_LEN(" WITH SYSTEM VERSIONING")); #ifdef WITH_PARTITION_STORAGE_ENGINE @@ -5037,11 +5018,6 @@ class Warnings_only_error_handler : public Internal_error_handler static bool get_all_archive_tables(THD *thd, Dynamic_array &all_archive_tables) { -#ifdef VERS_EXPERIMENTAL - if (thd->variables.vers_show == VERS_SHOW_ALWAYS) - return false; -#endif - if (thd->variables.vers_alter_history != VERS_ALTER_HISTORY_SURVIVE) return false; diff --git a/sql/sql_yacc.yy b/sql/sql_yacc.yy index 6eb53f70aaf96..895248f4824dd 100644 --- a/sql/sql_yacc.yy +++ b/sql/sql_yacc.yy @@ -6229,9 +6229,7 @@ versioning_option: { if (Lex->create_info.options & HA_LEX_CREATE_TMP_TABLE) { -#ifdef VERS_EXPERIMENTAL - if (!thd->variables.vers_force) -#endif + if (DBUG_EVALUATE_IF("sysvers_force", 0, 1)) { my_error(ER_VERS_TEMPORARY, MYF(0)); MYSQL_YYABORT; @@ -16161,18 +16159,6 @@ set_expr_or_default: if ($$ == NULL) MYSQL_YYABORT; } - | DROP - { - $$=new (thd->mem_root) Item_string_sys(thd, "DROP", 4); - if ($$ == NULL) - MYSQL_YYABORT; - } - | RANGE_SYM - { - $$=new (thd->mem_root) Item_string_sys(thd, C_STRING_WITH_LEN("RANGE")); - if ($$ == NULL) - MYSQL_YYABORT; - } ; /* Lock function */ diff --git a/sql/sys_vars.cc b/sql/sys_vars.cc index d00f35e2028fb..94c771369937c 100644 --- a/sql/sys_vars.cc +++ b/sql/sys_vars.cc @@ -396,21 +396,6 @@ static Sys_var_vers_asof Sys_vers_asof_timestamp( SESSION_VAR(vers_asof_timestamp.type), NO_CMD_LINE, Sys_var_vers_asof::asof_keywords, DEFAULT(SYSTEM_TIME_UNSPECIFIED)); -#ifdef VERS_EXPERIMENTAL -static Sys_var_mybool Sys_vers_force( - "debug_system_versioning_force", "Force system versioning for all created tables", - SESSION_VAR(vers_force), CMD_LINE(OPT_ARG), DEFAULT(FALSE)); - -static const char *vers_show_keywords[]= {"OFF", "RANGE", "ALWAYS", NULL}; -static Sys_var_enum Sys_vers_show( - "debug_system_versioning_show", "Show system fields special rules. " - "OFF: don't use special show rules; " - "RANGE: show implicit system fields only in range versioned queries; " - "ALWAYS: show system fields in all queries", - SESSION_VAR(vers_show), CMD_LINE(REQUIRED_ARG), - vers_show_keywords, DEFAULT(VERS_SHOW_OFF)); -#endif - static Sys_var_mybool Sys_vers_innodb_algorithm_simple( "system_versioning_innodb_algorithm_simple", "Use simple algorithm of timestamp handling in InnoDB instead of TRX_SEES", From 28bed454693dc9605194137cfd736db500a5cf2e Mon Sep 17 00:00:00 2001 From: Sergei Golubchik Date: Thu, 11 Jan 2018 14:52:26 +0100 Subject: [PATCH 050/106] debug: don't hide row_start/row_end columns for testing --- mysql-test/suite/versioning/r/debug.result | 17 ++++++++++++++++- mysql-test/suite/versioning/t/debug.test | 8 +++++++- sql/handler.cc | 2 +- 3 files changed, 24 insertions(+), 3 deletions(-) diff --git a/mysql-test/suite/versioning/r/debug.result b/mysql-test/suite/versioning/r/debug.result index 6e704011002b6..406717f8b5d14 100644 --- a/mysql-test/suite/versioning/r/debug.result +++ b/mysql-test/suite/versioning/r/debug.result @@ -35,5 +35,20 @@ tt3 CREATE TEMPORARY TABLE `tt3` ( ) ENGINE=MyISAM DEFAULT CHARSET=latin1 disconnect con1; connection default; +set debug_dbug='+d,sysvers_show'; +show create table t3; +Table Create Table +t3 CREATE TABLE `t3` ( + `a` int(11) DEFAULT NULL +) ENGINE=MyISAM DEFAULT CHARSET=latin1 WITH SYSTEM VERSIONING +create table t4 (a int); +show create table t4; +Table Create Table +t4 CREATE TABLE `t4` ( + `a` int(11) DEFAULT NULL, + `row_start` timestamp(6) GENERATED ALWAYS AS ROW START, + `row_end` timestamp(6) GENERATED ALWAYS AS ROW END, + PERIOD FOR SYSTEM_TIME (`row_start`, `row_end`) +) ENGINE=MyISAM DEFAULT CHARSET=latin1 WITH SYSTEM VERSIONING set global debug_dbug=@old_dbug; -drop table t1, t2, t3; +drop table t1, t2, t3, t4; diff --git a/mysql-test/suite/versioning/t/debug.test b/mysql-test/suite/versioning/t/debug.test index 027af48fff55a..c6d5bd6086191 100644 --- a/mysql-test/suite/versioning/t/debug.test +++ b/mysql-test/suite/versioning/t/debug.test @@ -25,5 +25,11 @@ show create table tt3; --disconnect con1 --connection default +set debug_dbug='+d,sysvers_show'; + +show create table t3; +create table t4 (a int); +show create table t4; + set global debug_dbug=@old_dbug; -drop table t1, t2, t3; +drop table t1, t2, t3, t4; diff --git a/sql/handler.cc b/sql/handler.cc index 84cb550143cc3..c030cea033e8e 100644 --- a/sql/handler.cc +++ b/sql/handler.cc @@ -6874,7 +6874,7 @@ static Create_field *vers_init_sys_field(THD *thd, const char *field_name, int f f->set_handler(&type_handler_timestamp2); f->length= MAX_DATETIME_PRECISION; } - f->invisible= INVISIBLE_SYSTEM; + f->invisible= DBUG_EVALUATE_IF("sysvers_show", VISIBLE, INVISIBLE_SYSTEM); if (f->check(thd)) return NULL; From 6a8cf407d29a522b6837622dba4a583b038cf7a5 Mon Sep 17 00:00:00 2001 From: Sergei Golubchik Date: Thu, 11 Jan 2018 16:34:32 +0100 Subject: [PATCH 051/106] fix tests on windows --- mysql-test/r/mysqld--help,win.rdiff | 30 +++++++++---------- mysql-test/suite/versioning/r/insert.result | 1 + mysql-test/suite/versioning/t/insert.test | 4 +++ mysql-test/suite/versioning/t/partition.test | 8 +++++ .../suite/versioning/wait_system_clock.inc | 10 +++++++ 5 files changed, 38 insertions(+), 15 deletions(-) create mode 100644 mysql-test/suite/versioning/wait_system_clock.inc diff --git a/mysql-test/r/mysqld--help,win.rdiff b/mysql-test/r/mysqld--help,win.rdiff index b93adcc89d441..33374105fb590 100644 --- a/mysql-test/r/mysqld--help,win.rdiff +++ b/mysql-test/r/mysqld--help,win.rdiff @@ -1,6 +1,6 @@ --- a/mysql-test/r/mysqld--help.result +++ b/mysql-test/r/mysqld--help.result -@@ -381,7 +381,6 @@ +@@ -381,7 +381,6 @@ The following options may be given as the first argument: The number of segments in a key cache -L, --language=name Client error messages in given language. May be given as a full path. Deprecated. Use --lc-messages-dir instead. @@ -8,7 +8,7 @@ --lc-messages=name Set the language used for the error messages. -L, --lc-messages-dir=name Directory where error messages are -@@ -602,6 +601,7 @@ +@@ -603,6 +602,7 @@ The following options may be given as the first argument: Use MySQL-5.6 (instead of MariaDB-5.3) format for TIME, DATETIME, TIMESTAMP columns. (Defaults to on; use --skip-mysql56-temporal-format to disable.) @@ -16,7 +16,7 @@ --net-buffer-length=# Buffer length for TCP/IP and socket communication --net-read-timeout=# -@@ -1020,6 +1020,9 @@ +@@ -1048,6 +1048,9 @@ The following options may be given as the first argument: characteristics (isolation level, read only/read write,snapshot - but not any work done / data modified within the transaction). @@ -26,7 +26,7 @@ --show-slave-auth-info Show user and password in SHOW SLAVE HOSTS on this master. -@@ -1132,6 +1135,10 @@ +@@ -1171,6 +1174,10 @@ The following options may be given as the first argument: Log slow queries to given log file. Defaults logging to 'hostname'-slow.log. Must be enabled to activate other slow log options @@ -37,7 +37,7 @@ --socket=name Socket file to use for connection --sort-buffer-size=# Each thread that needs to do a sort allocates a buffer of -@@ -1151,6 +1158,7 @@ +@@ -1190,6 +1197,7 @@ The following options may be given as the first argument: EMPTY_STRING_IS_NULL --stack-trace Print a symbolic stack trace on failure (Defaults to on; use --skip-stack-trace to disable.) @@ -45,7 +45,7 @@ --standard-compliant-cte Allow only CTEs compliant to SQL standard (Defaults to on; use --skip-standard-compliant-cte to disable.) -@@ -1214,6 +1222,11 @@ +@@ -1261,6 +1269,11 @@ The following options may be given as the first argument: --thread-pool-max-threads=# Maximum allowed number of worker threads in the thread pool @@ -57,7 +57,7 @@ --thread-pool-oversubscribe=# How many additional active worker threads in a group are allowed. -@@ -1252,8 +1265,8 @@ +@@ -1299,8 +1312,8 @@ The following options may be given as the first argument: automatically convert it to an on-disk MyISAM or Aria table. -t, --tmpdir=name Path for temporary files. Several paths may be specified, @@ -68,7 +68,7 @@ --transaction-alloc-block-size=# Allocation block size for transactions to be stored in binary log -@@ -1387,7 +1400,6 @@ +@@ -1434,7 +1447,6 @@ key-cache-block-size 1024 key-cache-division-limit 100 key-cache-file-hash-size 512 key-cache-segments 0 @@ -76,7 +76,7 @@ lc-messages en_US lc-messages-dir MYSQL_SHAREDIR/ lc-time-names en_US -@@ -1459,6 +1471,7 @@ +@@ -1506,6 +1518,7 @@ myisam-sort-buffer-size 134216704 myisam-stats-method NULLS_UNEQUAL myisam-use-mmap FALSE mysql56-temporal-format TRUE @@ -84,7 +84,7 @@ net-buffer-length 16384 net-read-timeout 30 net-retry-count 10 -@@ -1561,6 +1574,8 @@ +@@ -1616,6 +1629,8 @@ session-track-schema TRUE session-track-state-change FALSE session-track-system-variables autocommit,character_set_client,character_set_connection,character_set_results,time_zone session-track-transaction-info OFF @@ -93,7 +93,7 @@ show-slave-auth-info FALSE silent-startup FALSE skip-grant-tables TRUE -@@ -1585,6 +1600,7 @@ +@@ -1642,6 +1657,7 @@ slave-transaction-retry-interval 0 slave-type-conversions slow-launch-time 2 slow-query-log FALSE @@ -101,10 +101,10 @@ sort-buffer-size 2097152 sql-mode STRICT_TRANS_TABLES,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION stack-trace TRUE -@@ -1598,9 +1614,9 @@ - sync-relay-log 10000 - sync-relay-log-info 10000 +@@ -1657,9 +1673,9 @@ sync-relay-log-info 10000 sysdate-is-now FALSE + system-versioning-alter-history ERROR + system-versioning-innodb-algorithm-simple TRUE -table-cache 431 +table-cache 2000 table-definition-cache 400 @@ -113,7 +113,7 @@ table-open-cache-instances 8 tc-heuristic-recover OFF tcp-keepalive-interval 0 -@@ -1609,6 +1625,8 @@ +@@ -1668,6 +1684,8 @@ tcp-keepalive-time 0 thread-cache-size 151 thread-pool-idle-timeout 60 thread-pool-max-threads 65536 diff --git a/mysql-test/suite/versioning/r/insert.result b/mysql-test/suite/versioning/r/insert.result index 493f34a97f315..e99d45011d57a 100644 --- a/mysql-test/suite/versioning/r/insert.result +++ b/mysql-test/suite/versioning/r/insert.result @@ -347,6 +347,7 @@ now() < sysdate() 1 create table t1 (a int) with system versioning; insert t1 values (1); +set @a=sysdate(6); select * from t1 for system_time as of now(6); a select * from t1 for system_time as of sysdate(6); diff --git a/mysql-test/suite/versioning/t/insert.test b/mysql-test/suite/versioning/t/insert.test index f2fbe31e1d0f2..f70ba344b138e 100644 --- a/mysql-test/suite/versioning/t/insert.test +++ b/mysql-test/suite/versioning/t/insert.test @@ -245,6 +245,10 @@ set timestamp=1000000019; select now() < sysdate(); create table t1 (a int) with system versioning; insert t1 values (1); + +--source suite/versioning/wait_system_clock.inc +set @a=sysdate(6); + select * from t1 for system_time as of now(6); select * from t1 for system_time as of sysdate(6); diff --git a/mysql-test/suite/versioning/t/partition.test b/mysql-test/suite/versioning/t/partition.test index 167a5b2f7a4d8..1dea9f7c69cc5 100644 --- a/mysql-test/suite/versioning/t/partition.test +++ b/mysql-test/suite/versioning/t/partition.test @@ -146,6 +146,8 @@ explain partitions select * from t1; set @str= concat('select row_start from t1 partition (pn) into @ts0'); prepare stmt from @str; execute stmt; drop prepare stmt; +--source suite/versioning/wait_system_clock.inc + set @now= now(6); delete from t1; execute select_p0; @@ -158,6 +160,9 @@ select @ts0 = @ts1; set @now= now(6); insert into t1 values (2); + +--source suite/versioning/wait_system_clock.inc + execute select_p0; execute select_pn; @@ -166,6 +171,9 @@ prepare stmt from @str; execute stmt; drop prepare stmt; set @now= now(6); update t1 set x = x + 1; + +--source suite/versioning/wait_system_clock.inc + execute select_p0; execute select_pn; diff --git a/mysql-test/suite/versioning/wait_system_clock.inc b/mysql-test/suite/versioning/wait_system_clock.inc new file mode 100644 index 0000000000000..21bbe7aea218d --- /dev/null +++ b/mysql-test/suite/versioning/wait_system_clock.inc @@ -0,0 +1,10 @@ +# +# windows has a rather low-resolution system clock +# wait until the event from the past will actually be in the past +# +if (`select @@version_compile_os in ("win32","win64","windows")`) +{ + let $_past=`select max(row_start) from t1`; + --let $wait_condition=select TIMESTAMP'$_past' < sysdate(6) + --source include/wait_condition.inc +} From 826f615fc9b69a0e1383034c8d38cb057a45a79f Mon Sep 17 00:00:00 2001 From: Sergei Golubchik Date: Thu, 11 Jan 2018 19:41:32 +0100 Subject: [PATCH 052/106] MDEV-14788 System versioning cannot be based on local timestamps, as it is now followup to be81b00c843 Fix updates and deletes too --- mysql-test/suite/versioning/r/insert.result | 6 ++++++ mysql-test/suite/versioning/r/update.result | 6 ++---- mysql-test/suite/versioning/t/insert.test | 4 ++++ mysql-test/suite/versioning/t/update.test | 5 ++--- sql/sql_delete.cc | 2 +- sql/sql_insert.cc | 9 ++------- sql/table.cc | 9 +++++++++ sql/table.h | 1 + 8 files changed, 27 insertions(+), 15 deletions(-) diff --git a/mysql-test/suite/versioning/r/insert.result b/mysql-test/suite/versioning/r/insert.result index e99d45011d57a..0c9718c106b24 100644 --- a/mysql-test/suite/versioning/r/insert.result +++ b/mysql-test/suite/versioning/r/insert.result @@ -353,6 +353,12 @@ a select * from t1 for system_time as of sysdate(6); a 1 +update t1 set a=2; +delete from t1; +select *, row_start > @a, row_end > @a from t1 for system_time all; +a row_start > @a row_end > @a +1 0 1 +2 1 1 # # MDEV-14871 Server crashes in fill_record / fill_record_n_invoke_before_triggers upon inserting into versioned table with trigger # diff --git a/mysql-test/suite/versioning/r/update.result b/mysql-test/suite/versioning/r/update.result index 08be6825b7f3a..e0daf1d6588b1 100644 --- a/mysql-test/suite/versioning/r/update.result +++ b/mysql-test/suite/versioning/r/update.result @@ -616,12 +616,10 @@ No A B C D 2 1 1 1 1 3 1 1 1 1 ### Issue #365, bug 7 (duplicate of historical row) -set timestamp= 1000000019; create or replace table t1 (a int primary key, b int) with system versioning engine myisam; insert into t1 (a) values (1); -update t1 set b= 2; -insert into t1 (a) values (1) on duplicate key update a= 2; -ERROR 23000: Duplicate entry '1-2001-09-09 01:46:59.000000' for key 'PRIMARY' +replace t1 values (1,2),(1,3),(2,4); +ERROR 23000: Duplicate entry '1-YYYY-MM-DD hh:mm:ss.uuuuuu' for key 'PRIMARY' drop database test; create database test; diff --git a/mysql-test/suite/versioning/t/insert.test b/mysql-test/suite/versioning/t/insert.test index f70ba344b138e..482f0dd77cea1 100644 --- a/mysql-test/suite/versioning/t/insert.test +++ b/mysql-test/suite/versioning/t/insert.test @@ -251,6 +251,10 @@ set @a=sysdate(6); select * from t1 for system_time as of now(6); select * from t1 for system_time as of sysdate(6); +update t1 set a=2; +delete from t1; +--sorted_result +select *, row_start > @a, row_end > @a from t1 for system_time all; --echo # --echo # MDEV-14871 Server crashes in fill_record / fill_record_n_invoke_before_triggers upon inserting into versioned table with trigger diff --git a/mysql-test/suite/versioning/t/update.test b/mysql-test/suite/versioning/t/update.test index bda908f4b54d6..60f2070527655 100644 --- a/mysql-test/suite/versioning/t/update.test +++ b/mysql-test/suite/versioning/t/update.test @@ -277,13 +277,12 @@ call test_07('bigint unsigned', 'innodb', 'vtq_commit_ts(sys_trx_end)'); call verify_vtq; --echo ### Issue #365, bug 7 (duplicate of historical row) -set timestamp= 1000000019; create or replace table t1 (a int primary key, b int) with system versioning engine myisam; insert into t1 (a) values (1); -update t1 set b= 2; +--replace_regex /'1-[- .\d:]+'/'1-YYYY-MM-DD hh:mm:ss.uuuuuu'/ --error ER_DUP_ENTRY -insert into t1 (a) values (1) on duplicate key update a= 2; +replace t1 values (1,2),(1,3),(2,4); drop database test; create database test; diff --git a/sql/sql_delete.cc b/sql/sql_delete.cc index a382fdb7c031b..14d04dcc2ad71 100644 --- a/sql/sql_delete.cc +++ b/sql/sql_delete.cc @@ -253,7 +253,7 @@ int TABLE::delete_row() return file->ha_delete_row(record[0]); store_record(this, record[1]); - vers_end_field()->set_time(); + vers_update_end(); return file->ha_update_row(record[1], record[0]); } diff --git a/sql/sql_insert.cc b/sql/sql_insert.cc index 203838dac101e..cb46b4847bdc3 100644 --- a/sql/sql_insert.cc +++ b/sql/sql_insert.cc @@ -1638,8 +1638,7 @@ int vers_insert_history_row(TABLE *table) restore_record(table,record[1]); // Set Sys_end to now() - if (table->vers_end_field()->set_time()) - DBUG_ASSERT(0); + table->vers_update_end(); return table->file->ha_write_row(table->record[0]); } @@ -1971,11 +1970,7 @@ int write_record(THD *thd, TABLE *table,COPY_INFO *info) DBUG_ASSERT(table->insert_values); store_record(table,insert_values); restore_record(table,record[1]); - if (table->vers_end_field()->set_time()) - { - error= 1; - goto err; - } + table->vers_update_end(); error= table->file->ha_update_row(table->record[1], table->record[0]); restore_record(table,insert_values); diff --git a/sql/table.cc b/sql/table.cc index 5a87acd4ed42f..cd635d32b7148 100644 --- a/sql/table.cc +++ b/sql/table.cc @@ -7800,6 +7800,15 @@ void TABLE::vers_update_fields() } +void TABLE::vers_update_end() +{ + vers_end_field()->set_notnull(); + if (vers_end_field()->store_timestamp(in_use->system_time, + in_use->system_time_sec_part)) + DBUG_ASSERT(0); +} + + bool TABLE_LIST::vers_vtmd_name(String& out) const { static const char *vtmd_suffix= "_vtmd"; diff --git a/sql/table.h b/sql/table.h index 71858be6229bb..3882bfad27916 100644 --- a/sql/table.h +++ b/sql/table.h @@ -1588,6 +1588,7 @@ struct TABLE int delete_row(); void vers_update_fields(); + void vers_update_end(); /** Number of additional fields used in versioned tables */ #define VERSIONING_FIELDS 2 From 8ee071a22d6ba69177d8f75509425c06a630d6f5 Mon Sep 17 00:00:00 2001 From: Sergei Golubchik Date: Thu, 11 Jan 2018 23:04:49 +0100 Subject: [PATCH 053/106] cleanup: remove redundant check it's already checked before the update loop (see check_fields()), and this condition cannot suddenly become false in the middle of the statement --- sql/sql_base.cc | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/sql/sql_base.cc b/sql/sql_base.cc index 3c913f8353092..6178dd3ac6a1f 100644 --- a/sql/sql_base.cc +++ b/sql/sql_base.cc @@ -8115,11 +8115,8 @@ fill_record(THD *thd, TABLE *table_arg, List &fields, List &values, thus we safely can take table from the first field. */ fld= (Item_field*)f++; - if (!(field= fld->field_for_view_update())) - { - my_error(ER_NONUPDATEABLE_COLUMN, MYF(0), fld->name.str); - goto err; - } + field= fld->field_for_view_update(); + DBUG_ASSERT(field); DBUG_ASSERT(field->field->table == table_arg); table_arg->auto_increment_field_not_null= FALSE; f.rewind(); From 558ee2ee84513e9037fdf9ff516e8b05ac5c9717 Mon Sep 17 00:00:00 2001 From: Sergei Golubchik Date: Fri, 12 Jan 2018 14:47:33 +0100 Subject: [PATCH 054/106] fix --embedded tests --- mysql-test/suite/versioning/r/create.result | 5 +---- mysql-test/suite/versioning/t/create.test | 2 +- 2 files changed, 2 insertions(+), 5 deletions(-) diff --git a/mysql-test/suite/versioning/r/create.result b/mysql-test/suite/versioning/r/create.result index 3cf159ac3d000..7fe1db5964fbc 100644 --- a/mysql-test/suite/versioning/r/create.result +++ b/mysql-test/suite/versioning/r/create.result @@ -28,7 +28,7 @@ table_collation latin1_swedish_ci checksum NULL create_options table_comment -select table_catalog,table_schema,table_name,column_name,ordinal_position,column_default,character_maximum_length,character_octet_length,character_set_name,collation_name,column_key,extra,privileges,column_comment,is_generated,generation_expression from information_schema.columns where table_name='t1'; +select table_catalog,table_schema,table_name,column_name,ordinal_position,column_default,character_maximum_length,character_octet_length,character_set_name,collation_name,column_key,extra,column_comment,is_generated,generation_expression from information_schema.columns where table_name='t1'; table_catalog def table_schema test table_name t1 @@ -41,7 +41,6 @@ character_set_name NULL collation_name NULL column_key extra -privileges select,insert,update,references column_comment is_generated NEVER generation_expression NULL @@ -57,7 +56,6 @@ character_set_name NULL collation_name NULL column_key extra INVISIBLE -privileges select,insert,update,references column_comment start is_generated ALWAYS generation_expression ROW START @@ -73,7 +71,6 @@ character_set_name NULL collation_name NULL column_key extra INVISIBLE -privileges select,insert,update,references column_comment end is_generated ALWAYS generation_expression ROW END diff --git a/mysql-test/suite/versioning/t/create.test b/mysql-test/suite/versioning/t/create.test index f1ffd7fcf680e..24d8d3350206d 100644 --- a/mysql-test/suite/versioning/t/create.test +++ b/mysql-test/suite/versioning/t/create.test @@ -16,7 +16,7 @@ eval create table t1 ( show create table t1; --query_vertical select table_catalog,table_schema,table_name,table_type,version,table_rows,avg_row_length,data_free,auto_increment,check_time,table_collation,checksum,create_options,table_comment from information_schema.tables where table_name='t1' ---query_vertical select table_catalog,table_schema,table_name,column_name,ordinal_position,column_default,character_maximum_length,character_octet_length,character_set_name,collation_name,column_key,extra,privileges,column_comment,is_generated,generation_expression from information_schema.columns where table_name='t1' +--query_vertical select table_catalog,table_schema,table_name,column_name,ordinal_position,column_default,character_maximum_length,character_octet_length,character_set_name,collation_name,column_key,extra,column_comment,is_generated,generation_expression from information_schema.columns where table_name='t1' --echo # Implicit fields test create or replace table t1 ( From bd87c872c07c39ed0ca521d92ca464b924650f41 Mon Sep 17 00:00:00 2001 From: Sergei Golubchik Date: Fri, 12 Jan 2018 21:32:07 +0100 Subject: [PATCH 055/106] cleanup: don't generate tests with SP or PS unless it's a test of SP or PS --- .../suite/versioning/r/select2,myisam.rdiff | 38 ++++ .../r/{select_sp.result => select2.result} | 109 ++++------ .../t/{select_sp.test => select2.test} | 193 ++++++++---------- 3 files changed, 163 insertions(+), 177 deletions(-) create mode 100644 mysql-test/suite/versioning/r/select2,myisam.rdiff rename mysql-test/suite/versioning/r/{select_sp.result => select2.result} (82%) rename mysql-test/suite/versioning/t/{select_sp.test => select2.test} (56%) diff --git a/mysql-test/suite/versioning/r/select2,myisam.rdiff b/mysql-test/suite/versioning/r/select2,myisam.rdiff new file mode 100644 index 0000000000000..b24682fbb9f36 --- /dev/null +++ b/mysql-test/suite/versioning/r/select2,myisam.rdiff @@ -0,0 +1,38 @@ +--- suite/versioning/r/select_sp.result ++++ suite/versioning/r/select_sp.result +@@ -22,8 +22,6 @@ + delete from t1 where x > 7; + insert into t1(x, y) values(3, 33); + select sys_start from t1 where x = 3 and y = 33 into @t1; +-set @x1= @t1; +-select vtq_commit_ts(@x1) into @t1; + select x, y from t1; + x y + 0 100 +@@ -84,7 +82,7 @@ + 8 108 + 9 109 + 3 33 +-select x as ASOF2_x, y from t1 for system_time as of @x0; ++select x as ASOF2_x, y from t1 for system_time as of @t0; + ASOF2_x y + 0 100 + 1 101 +@@ -96,7 +94,7 @@ + 7 107 + 8 108 + 9 109 +-select x as FROMTO2_x, y from t1 for system_time from @x0 to @x1; ++select x as FROMTO2_x, y from t1 for system_time from '0-0-0 0:0:0' to @t1; + FROMTO2_x y + 0 100 + 1 101 +@@ -108,7 +106,7 @@ + 7 107 + 8 108 + 9 109 +-select x as BETWAND2_x, y from t1 for system_time between transaction @x0 and transaction @x1; ++select x as BETWAND2_x, y from t1 for system_time between timestamp '0-0-0 0:0:0' and timestamp @t1; + BETWAND2_x y + 0 100 + 1 101 diff --git a/mysql-test/suite/versioning/r/select_sp.result b/mysql-test/suite/versioning/r/select2.result similarity index 82% rename from mysql-test/suite/versioning/r/select_sp.result rename to mysql-test/suite/versioning/r/select2.result index 6bfe26d0ef779..79c4416a29153 100644 --- a/mysql-test/suite/versioning/r/select_sp.result +++ b/mysql-test/suite/versioning/r/select2.result @@ -1,17 +1,10 @@ -create procedure test_01() -begin -declare engine varchar(255) default default_engine(); -declare sys_type varchar(255) default sys_datatype(default_engine()); -set @str= concat(' - create table t1( - x int unsigned, - y int unsigned, - sys_start ', sys_type, ' as row start invisible, - sys_end ', sys_type, ' as row end invisible, - period for system_time (sys_start, sys_end)) - with system versioning - engine ', engine); -prepare stmt from @str; execute stmt; drop prepare stmt; +create table t1( +x int unsigned, +y int unsigned, +sys_start SYS_TYPE as row start invisible, +sys_end SYS_TYPE as row end invisible, +period for system_time (sys_start, sys_end)) +with system versioning engine=ENGINE; insert into t1 (x, y) values (0, 100), (1, 101), @@ -24,67 +17,14 @@ insert into t1 (x, y) values (8, 108), (9, 109); set @t0= now(6); -if engine = 'innodb' then select sys_start from t1 limit 1 into @x0; -end if; delete from t1 where x = 3; delete from t1 where x > 7; insert into t1(x, y) values(3, 33); select sys_start from t1 where x = 3 and y = 33 into @t1; -if engine = 'innodb' then set @x1= @t1; select vtq_commit_ts(@x1) into @t1; -end if; select x, y from t1; -select x as ASOF_x, y from t1 for system_time as of timestamp @t0; -select x as FROMTO_x, y from t1 for system_time from '0-0-0 0:0:0' to timestamp @t1; -select x as BETWAND_x, y from t1 for system_time between '0-0-0 0:0:0' and timestamp @t1; -select x as ALL_x, y from t1 for system_time all; -if engine = 'innodb' then -select x as ASOF2_x, y from t1 for system_time as of @x0; -select x as FROMTO2_x, y from t1 for system_time from @x0 to @x1; -select x as BETWAND2_x, y from t1 for system_time between transaction @x0 and transaction @x1; -else -select x as ASOF2_x, y from t1 for system_time as of @t0; -select x as FROMTO2_x, y from t1 for system_time from '0-0-0 0:0:0' to @t1; -select x as BETWAND2_x, y from t1 for system_time between timestamp '0-0-0 0:0:0' and timestamp @t1; -end if; -drop table t1; -end~~ -create or replace procedure test_02() -begin -declare engine varchar(255) default default_engine(); -declare sys_type varchar(255) default sys_datatype(default_engine()); -set @str0= concat('( - x int, - y int, - sys_start ', sys_type, ' as row start invisible, - sys_end ', sys_type, ' as row end invisible, - period for system_time (sys_start, sys_end)) - with system versioning - engine ', engine); -set @str= concat('create or replace table t1', @str0); -prepare stmt from @str; execute stmt; drop prepare stmt; -set @str= concat('create or replace table t2', @str0); -prepare stmt from @str; execute stmt; drop prepare stmt; -insert into t1 values (1, 1), (1, 2), (1, 3), (4, 4), (5, 5); -insert into t2 values (1, 2), (2, 1), (3, 1); -set @t0= now(6); -select t1.x as IJ1_x1, t1.y as y1, t2.x as x2, t2.y as y2 from t1 inner join t2 on t1.x = t2.x; -select t1.x as LJ1_x1, t1.y as y1, t2.x as x2, t2.y as y2 from t1 left join t2 on t1.x = t2.x; -select t1.x as RJ1_x1, t1.y as y1, t2.x as x2, t2.y as y2 from t1 right join t2 on t1.x = t2.x; -delete from t1; -delete from t2; -select IJ2_x1,y1,x2,y2 from (select t1.x as IJ2_x1, t1.y as y1, t2.x as x2, t2.y as y2 from t1 inner join t2 on t1.x = t2.x) -for system_time as of timestamp @t0 as t; -select LJ2_x1,y1,x2,y2 from (select t1.x as LJ2_x1, t1.y as y1, t2.x as x2, t2.y as y2 from t1 left join t2 on t1.x = t2.x) -for system_time as of timestamp @t0 as t; -select RJ2_x1,y1,x2,y2 from (select t1.x as RJ2_x1, t1.y as y1, t2.x as x2, t2.y as y2 from t1 right join t2 on t1.x = t2.x) -for system_time as of timestamp @t0 as t; -drop table t1; -drop table t2; -end~~ -call test_01(); x y 0 100 1 101 @@ -94,6 +34,7 @@ x y 6 106 7 107 3 33 +select x as ASOF_x, y from t1 for system_time as of timestamp @t0; ASOF_x y 0 100 1 101 @@ -105,6 +46,7 @@ ASOF_x y 7 107 8 108 9 109 +select x as FROMTO_x, y from t1 for system_time from '0-0-0 0:0:0' to timestamp @t1; FROMTO_x y 0 100 1 101 @@ -116,6 +58,7 @@ FROMTO_x y 7 107 8 108 9 109 +select x as BETWAND_x, y from t1 for system_time between '0-0-0 0:0:0' and timestamp @t1; BETWAND_x y 0 100 1 101 @@ -128,6 +71,7 @@ BETWAND_x y 8 108 9 109 3 33 +select x as ALL_x, y from t1 for system_time all; ALL_x y 0 100 1 101 @@ -140,6 +84,7 @@ ALL_x y 8 108 9 109 3 33 +select x as ASOF2_x, y from t1 for system_time as of @x0; ASOF2_x y 0 100 1 101 @@ -151,6 +96,7 @@ ASOF2_x y 7 107 8 108 9 109 +select x as FROMTO2_x, y from t1 for system_time from @x0 to @x1; FROMTO2_x y 0 100 1 101 @@ -162,6 +108,7 @@ FROMTO2_x y 7 107 8 108 9 109 +select x as BETWAND2_x, y from t1 for system_time between transaction @x0 and transaction @x1; BETWAND2_x y 0 100 1 101 @@ -174,39 +121,63 @@ BETWAND2_x y 8 108 9 109 3 33 -call test_02(); +drop table t1; +create table t1( +x int, +y int, +sys_start SYS_TYPE as row start invisible, +sys_end SYS_TYPE as row end invisible, +period for system_time (sys_start, sys_end)) +with system versioning engine=ENGINE; +create table t2 like t1; +insert into t1 values (1, 1), (1, 2), (1, 3), (4, 4), (5, 5); +insert into t2 values (1, 2), (2, 1), (3, 1); +set @t0= now(6); +select t1.x as IJ1_x1, t1.y as y1, t2.x as x2, t2.y as y2 from t1 inner join t2 on t1.x = t2.x; IJ1_x1 y1 x2 y2 1 1 1 2 1 2 1 2 1 3 1 2 +select t1.x as LJ1_x1, t1.y as y1, t2.x as x2, t2.y as y2 from t1 left join t2 on t1.x = t2.x; LJ1_x1 y1 x2 y2 1 1 1 2 1 2 1 2 1 3 1 2 4 4 NULL NULL 5 5 NULL NULL +select t1.x as RJ1_x1, t1.y as y1, t2.x as x2, t2.y as y2 from t1 right join t2 on t1.x = t2.x; RJ1_x1 y1 x2 y2 1 1 1 2 1 2 1 2 1 3 1 2 NULL NULL 2 1 NULL NULL 3 1 +delete from t1; +delete from t2; +select IJ2_x1,y1,x2,y2 from (select t1.x as IJ2_x1, t1.y as y1, t2.x as x2, t2.y as y2 from t1 inner join t2 on t1.x = t2.x) +for system_time as of timestamp @t0 as t; IJ2_x1 y1 x2 y2 1 1 1 2 1 2 1 2 1 3 1 2 +select LJ2_x1,y1,x2,y2 from (select t1.x as LJ2_x1, t1.y as y1, t2.x as x2, t2.y as y2 from t1 left join t2 on t1.x = t2.x) +for system_time as of timestamp @t0 as t; LJ2_x1 y1 x2 y2 1 1 1 2 1 2 1 2 1 3 1 2 4 4 NULL NULL 5 5 NULL NULL +select RJ2_x1,y1,x2,y2 from (select t1.x as RJ2_x1, t1.y as y1, t2.x as x2, t2.y as y2 from t1 right join t2 on t1.x = t2.x) +for system_time as of timestamp @t0 as t; RJ2_x1 y1 x2 y2 1 1 1 2 1 2 1 2 1 3 1 2 NULL NULL 2 1 NULL NULL 3 1 +drop table t1; +drop table t2; # MDEV-14686 Server crashes in Item_field::used_tables on 2nd call of SP [#422] create or replace table t1 (called int, bad int) with system versioning; create or replace procedure bad() select * from t1 where bad in (select called from t1); @@ -365,5 +336,3 @@ select * from (t1 for system_time all join t2 for system_time all) for system_ti ERROR 42000: You have an error in your SQL syntax; check the manual that corresponds to your MariaDB server version for the right syntax to use near '' at line 1 drop view v1; drop table t1, t2; -drop procedure test_01; -drop procedure test_02; diff --git a/mysql-test/suite/versioning/t/select_sp.test b/mysql-test/suite/versioning/t/select2.test similarity index 56% rename from mysql-test/suite/versioning/t/select_sp.test rename to mysql-test/suite/versioning/t/select2.test index 41022259b4fac..81070a960ee6c 100644 --- a/mysql-test/suite/versioning/t/select_sp.test +++ b/mysql-test/suite/versioning/t/select2.test @@ -1,110 +1,92 @@ --source suite/versioning/engines.inc --source suite/versioning/common.inc -delimiter ~~; -create procedure test_01() -begin - declare engine varchar(255) default default_engine(); - declare sys_type varchar(255) default sys_datatype(default_engine()); - - set @str= concat(' - create table t1( - x int unsigned, - y int unsigned, - sys_start ', sys_type, ' as row start invisible, - sys_end ', sys_type, ' as row end invisible, - period for system_time (sys_start, sys_end)) - with system versioning - engine ', engine); - prepare stmt from @str; execute stmt; drop prepare stmt; - insert into t1 (x, y) values - (0, 100), - (1, 101), - (2, 102), - (3, 103), - (4, 104), - (5, 105), - (6, 106), - (7, 107), - (8, 108), - (9, 109); - set @t0= now(6); - if engine = 'innodb' then - select sys_start from t1 limit 1 into @x0; - end if; - - delete from t1 where x = 3; - delete from t1 where x > 7; - - insert into t1(x, y) values(3, 33); - select sys_start from t1 where x = 3 and y = 33 into @t1; - if engine = 'innodb' then - set @x1= @t1; - select vtq_commit_ts(@x1) into @t1; - end if; - - select x, y from t1; - select x as ASOF_x, y from t1 for system_time as of timestamp @t0; - select x as FROMTO_x, y from t1 for system_time from '0-0-0 0:0:0' to timestamp @t1; - select x as BETWAND_x, y from t1 for system_time between '0-0-0 0:0:0' and timestamp @t1; - select x as ALL_x, y from t1 for system_time all; - - if engine = 'innodb' then - select x as ASOF2_x, y from t1 for system_time as of @x0; - select x as FROMTO2_x, y from t1 for system_time from @x0 to @x1; - select x as BETWAND2_x, y from t1 for system_time between transaction @x0 and transaction @x1; - else - select x as ASOF2_x, y from t1 for system_time as of @t0; - select x as FROMTO2_x, y from t1 for system_time from '0-0-0 0:0:0' to @t1; - select x as BETWAND2_x, y from t1 for system_time between timestamp '0-0-0 0:0:0' and timestamp @t1; - end if; - - drop table t1; -end~~ - -create or replace procedure test_02() -begin - declare engine varchar(255) default default_engine(); - declare sys_type varchar(255) default sys_datatype(default_engine()); - - set @str0= concat('( - x int, - y int, - sys_start ', sys_type, ' as row start invisible, - sys_end ', sys_type, ' as row end invisible, - period for system_time (sys_start, sys_end)) - with system versioning - engine ', engine); - set @str= concat('create or replace table t1', @str0); - prepare stmt from @str; execute stmt; drop prepare stmt; - set @str= concat('create or replace table t2', @str0); - prepare stmt from @str; execute stmt; drop prepare stmt; - - insert into t1 values (1, 1), (1, 2), (1, 3), (4, 4), (5, 5); - insert into t2 values (1, 2), (2, 1), (3, 1); - set @t0= now(6); - - select t1.x as IJ1_x1, t1.y as y1, t2.x as x2, t2.y as y2 from t1 inner join t2 on t1.x = t2.x; - select t1.x as LJ1_x1, t1.y as y1, t2.x as x2, t2.y as y2 from t1 left join t2 on t1.x = t2.x; - select t1.x as RJ1_x1, t1.y as y1, t2.x as x2, t2.y as y2 from t1 right join t2 on t1.x = t2.x; - - delete from t1; - delete from t2; - - select IJ2_x1,y1,x2,y2 from (select t1.x as IJ2_x1, t1.y as y1, t2.x as x2, t2.y as y2 from t1 inner join t2 on t1.x = t2.x) - for system_time as of timestamp @t0 as t; - select LJ2_x1,y1,x2,y2 from (select t1.x as LJ2_x1, t1.y as y1, t2.x as x2, t2.y as y2 from t1 left join t2 on t1.x = t2.x) - for system_time as of timestamp @t0 as t; - select RJ2_x1,y1,x2,y2 from (select t1.x as RJ2_x1, t1.y as y1, t2.x as x2, t2.y as y2 from t1 right join t2 on t1.x = t2.x) - for system_time as of timestamp @t0 as t; - - drop table t1; - drop table t2; -end~~ -delimiter ;~~ - -call test_01(); -call test_02(); +let $engine=`select default_engine()`; +let $sys_type=`select sys_datatype(default_engine())`; + +replace_result $engine ENGINE $sys_type SYS_TYPE; +eval create table t1( + x int unsigned, + y int unsigned, + sys_start $sys_type as row start invisible, + sys_end $sys_type as row end invisible, + period for system_time (sys_start, sys_end)) +with system versioning engine=$engine; + +insert into t1 (x, y) values + (0, 100), + (1, 101), + (2, 102), + (3, 103), + (4, 104), + (5, 105), + (6, 106), + (7, 107), + (8, 108), + (9, 109); +set @t0= now(6); +select sys_start from t1 limit 1 into @x0; + +delete from t1 where x = 3; +delete from t1 where x > 7; + +insert into t1(x, y) values(3, 33); +select sys_start from t1 where x = 3 and y = 33 into @t1; +if(`select '$engine' = 'innodb'`) { + set @x1= @t1; + select vtq_commit_ts(@x1) into @t1; +} + +select x, y from t1; +select x as ASOF_x, y from t1 for system_time as of timestamp @t0; +select x as FROMTO_x, y from t1 for system_time from '0-0-0 0:0:0' to timestamp @t1; +select x as BETWAND_x, y from t1 for system_time between '0-0-0 0:0:0' and timestamp @t1; +select x as ALL_x, y from t1 for system_time all; + +if(`select '$engine' = 'innodb'`) { + select x as ASOF2_x, y from t1 for system_time as of @x0; + select x as FROMTO2_x, y from t1 for system_time from @x0 to @x1; + select x as BETWAND2_x, y from t1 for system_time between transaction @x0 and transaction @x1; +} +if(`select '$engine' != 'innodb'`) { + select x as ASOF2_x, y from t1 for system_time as of @t0; + select x as FROMTO2_x, y from t1 for system_time from '0-0-0 0:0:0' to @t1; + select x as BETWAND2_x, y from t1 for system_time between timestamp '0-0-0 0:0:0' and timestamp @t1; +} + +drop table t1; + +replace_result $engine ENGINE $sys_type SYS_TYPE; +eval create table t1( + x int, + y int, + sys_start $sys_type as row start invisible, + sys_end $sys_type as row end invisible, + period for system_time (sys_start, sys_end)) +with system versioning engine=$engine; + +create table t2 like t1; + +insert into t1 values (1, 1), (1, 2), (1, 3), (4, 4), (5, 5); +insert into t2 values (1, 2), (2, 1), (3, 1); +set @t0= now(6); + +select t1.x as IJ1_x1, t1.y as y1, t2.x as x2, t2.y as y2 from t1 inner join t2 on t1.x = t2.x; +select t1.x as LJ1_x1, t1.y as y1, t2.x as x2, t2.y as y2 from t1 left join t2 on t1.x = t2.x; +select t1.x as RJ1_x1, t1.y as y1, t2.x as x2, t2.y as y2 from t1 right join t2 on t1.x = t2.x; + +delete from t1; +delete from t2; + +select IJ2_x1,y1,x2,y2 from (select t1.x as IJ2_x1, t1.y as y1, t2.x as x2, t2.y as y2 from t1 inner join t2 on t1.x = t2.x) +for system_time as of timestamp @t0 as t; +select LJ2_x1,y1,x2,y2 from (select t1.x as LJ2_x1, t1.y as y1, t2.x as x2, t2.y as y2 from t1 left join t2 on t1.x = t2.x) +for system_time as of timestamp @t0 as t; +select RJ2_x1,y1,x2,y2 from (select t1.x as RJ2_x1, t1.y as y1, t2.x as x2, t2.y as y2 from t1 right join t2 on t1.x = t2.x) +for system_time as of timestamp @t0 as t; + +drop table t1; +drop table t2; --echo # MDEV-14686 Server crashes in Item_field::used_tables on 2nd call of SP [#422] create or replace table t1 (called int, bad int) with system versioning; @@ -226,7 +208,4 @@ select * from (t1 for system_time all join t2 for system_time all) for system_ti drop view v1; drop table t1, t2; -drop procedure test_01; -drop procedure test_02; - -- source suite/versioning/common_finish.inc From 2484a2e4c85240fcbb71241a8b48a2155bf5d0c0 Mon Sep 17 00:00:00 2001 From: Sergei Golubchik Date: Fri, 12 Jan 2018 22:21:38 +0100 Subject: [PATCH 056/106] cleanup: remove include/rpl_events.inc introduces 10 years ago, still included only once. no need to complicate debugging by unnecessary includes. --- mysql-test/include/rpl_events.inc | 159 ----------------------- mysql-test/suite/rpl/r/rpl_events.result | 17 +-- mysql-test/suite/rpl/t/rpl_events.test | 154 +++++++++++++++++++++- 3 files changed, 155 insertions(+), 175 deletions(-) delete mode 100644 mysql-test/include/rpl_events.inc diff --git a/mysql-test/include/rpl_events.inc b/mysql-test/include/rpl_events.inc deleted file mode 100644 index 0effa8c4e5c28..0000000000000 --- a/mysql-test/include/rpl_events.inc +++ /dev/null @@ -1,159 +0,0 @@ -################################################################## -# Author: Giuseppe, Chuck Bell # -# Date: 17-January-2007 # -# Purpose: To test that event effects are replicated # -# in both row based and statement based format # -################################################################## - ---disable_warnings -DROP EVENT IF EXISTS test.justonce; -drop table if exists t1,t2; ---enable_warnings - -# first, we need a table to record something from an event - -eval CREATE TABLE `t1` ( - `id` INT(10) UNSIGNED NOT NULL, - `c` VARCHAR(50) NOT NULL, - `ts` TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, - PRIMARY KEY (`id`) -) ENGINE=$engine_type DEFAULT CHARSET=utf8; - -INSERT INTO t1 (id, c) VALUES (1, 'manually'); - -# We create the event so that it inserts exactly 1 row in the table -# A recuring event is used so that we can be sure the event will -# fire regardless of timing delays on the server. Otherwise, it is -# possible for the event to timeout before it has inserted a row. ---echo "Creating event test.justonce on the master" -CREATE EVENT test.justonce ON SCHEDULE EVERY 2 SECOND DO - INSERT IGNORE INTO t1 (id, c) VALUES (2, 'from justonce'); - -# Show the event is alive and present on master ---echo "Checking event is active on master" -SELECT db, name, status, originator FROM mysql.event WHERE db = 'test' AND name = 'justonce'; - -# Wait until event has fired. We know this because t1 will contain -# the row from the event. -let $wait_condition= - SELECT COUNT(*) = 1 FROM t1 WHERE c = 'from justonce'; ---source include/wait_condition.inc - -# check that table t1 contains something ---echo "Checking event data on the master" -let $events_done=`SELECT count(*) FROM t1 id`; ---disable_query_log -eval SELECT $events_done > 0 as ONE; ---enable_query_log - -sync_slave_with_master; - ---echo "Checking event data on the slave" ---disable_query_log -eval SELECT count(*) - $events_done as ZERO FROM t1 id; ---enable_query_log - ---echo "Checking event is inactive on slave" -SELECT db, name, status, originator FROM mysql.event WHERE db = 'test' AND name = 'justonce'; - -# Create an event on the slave and check to see what the originator is. ---echo "Dropping event test.slave_once on the slave" ---disable_warnings -DROP EVENT IF EXISTS test.slave_once; ---enable_warnings - -# Create an event on slave and check its state. An event shouldn't be executed -# so set start time in 1 hour. -CREATE EVENT test.slave_once ON SCHEDULE EVERY 5 MINUTE STARTS CURRENT_TIMESTAMP + INTERVAL 1 HOUR DO - INSERT IGNORE INTO t1(id, c) VALUES (3, 'from slave_once'); - ---echo "Checking event status on the slave for originator value = slave's server_id" -SELECT db, name, status, originator FROM mysql.event WHERE db = 'test' AND name = 'slave_once'; - ---echo "Dropping event test.slave_once on the slave" ---disable_warnings -DROP EVENT IF EXISTS test.slave_once; ---enable_warnings - -connection master; - -# BUG#20384 - disable events on slave ---echo "Dropping event test.justonce on the master" ---disable_warnings -DROP EVENT IF EXISTS test.justonce; ---enable_warnings - -# Create an event on master and check its state on slave. An event shouldn't be executed -# so set start time in 1 hour. Check that changes of event statement replicated to slave - ---echo "Creating event test.er on the master" -CREATE EVENT test.er ON SCHEDULE EVERY 3 SECOND STARTS CURRENT_TIMESTAMP + INTERVAL 1 HOUR DO - INSERT IGNORE INTO t1(id, c) VALUES (4, 'from er'); - ---echo "Checking event status on the master" -SELECT db, name, status, originator, body FROM mysql.event WHERE db = 'test' AND name = 'er'; - -sync_slave_with_master; - ---echo "Checking event status on the slave" -SELECT db, name, status, originator, body FROM mysql.event WHERE db = 'test' AND name = 'er'; - -connection master; ---echo "Altering event test.er on the master" -ALTER EVENT test.er ON SCHEDULE EVERY 5 SECOND STARTS CURRENT_TIMESTAMP + INTERVAL 1 HOUR DO - INSERT IGNORE INTO t1(id, c) VALUES (5, 'from alter er'); - ---echo "Checking event status on the master" -SELECT db, name, status, originator, body FROM mysql.event WHERE db = 'test' AND name = 'er'; - -sync_slave_with_master; - ---echo "Checking event status on the slave" -SELECT db, name, status, originator, body FROM mysql.event WHERE db = 'test' AND name = 'er'; - -connection master; ---echo "Dropping event test.er on the master" -DROP EVENT test.er; - ---echo "Checking event status on the master" -SELECT db, name, status, originator FROM mysql.event WHERE db = 'test'; - ---disable_info - -sync_slave_with_master; - ---echo "Checking event status on the slave" -SELECT db, name, status, originator FROM mysql.event WHERE db = 'test'; - -# test the DISABLE ON SLAVE for setting event SLAVESIDE_DISABLED as status -# on CREATE EVENT - -# Create an event on slave and check its status. An event shouldn't be executed -# so set start time in 1 hour. - ---echo "Creating event test.slave_terminate on the slave" -CREATE EVENT test.slave_terminate ON SCHEDULE EVERY 3 SECOND STARTS CURRENT_TIMESTAMP + INTERVAL 1 HOUR DO - INSERT IGNORE INTO t1(id, c) VALUES (6, 'from slave_terminate'); - ---echo "Checking event status on the slave" -SELECT db, name, status, originator FROM mysql.event WHERE db = 'test' AND name = 'slave_terminate'; - ---echo "Dropping event test.slave_terminate on the slave" -DROP EVENT test.slave_terminate; - ---echo "Creating event test.slave_terminate with DISABLE ON SLAVE on the slave" -CREATE EVENT test.slave_terminate ON SCHEDULE EVERY 3 SECOND DISABLE ON SLAVE DO - INSERT IGNORE INTO t1(c) VALUES (7, 'from slave_terminate'); - ---echo "Checking event status on the slave" -SELECT db, name, status, originator FROM mysql.event WHERE db = 'test' AND name = 'slave_terminate'; - ---echo "Dropping event test.slave_terminate on the slave" -DROP EVENT test.slave_terminate; - ---echo "Cleanup" -connection master; -DROP TABLE t1; -sync_slave_with_master; -connection master; - diff --git a/mysql-test/suite/rpl/r/rpl_events.result b/mysql-test/suite/rpl/r/rpl_events.result index f7f802a57e6af..4b2226109d906 100644 --- a/mysql-test/suite/rpl/r/rpl_events.result +++ b/mysql-test/suite/rpl/r/rpl_events.result @@ -2,17 +2,15 @@ include/master-slave.inc [connection master] SET @old_event_scheduler = @@global.event_scheduler; set global event_scheduler=1; -DROP EVENT IF EXISTS test.justonce; -drop table if exists t1,t2; CREATE TABLE `t1` ( `id` INT(10) UNSIGNED NOT NULL, `c` VARCHAR(50) NOT NULL, `ts` TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, PRIMARY KEY (`id`) -) ENGINE=MyISAM DEFAULT CHARSET=utf8; +) DEFAULT CHARSET=utf8; INSERT INTO t1 (id, c) VALUES (1, 'manually'); "Creating event test.justonce on the master" -CREATE EVENT test.justonce ON SCHEDULE EVERY 2 SECOND DO +CREATE EVENT test.justonce ON SCHEDULE EVERY 2 SECOND DO INSERT IGNORE INTO t1 (id, c) VALUES (2, 'from justonce'); "Checking event is active on master" SELECT db, name, status, originator FROM mysql.event WHERE db = 'test' AND name = 'justonce'; @@ -31,7 +29,7 @@ db name status originator test justonce SLAVESIDE_DISABLED 1 "Dropping event test.slave_once on the slave" DROP EVENT IF EXISTS test.slave_once; -CREATE EVENT test.slave_once ON SCHEDULE EVERY 5 MINUTE STARTS CURRENT_TIMESTAMP + INTERVAL 1 HOUR DO +CREATE EVENT test.slave_once ON SCHEDULE EVERY 5 MINUTE STARTS CURRENT_TIMESTAMP + INTERVAL 1 HOUR DO INSERT IGNORE INTO t1(id, c) VALUES (3, 'from slave_once'); "Checking event status on the slave for originator value = slave's server_id" SELECT db, name, status, originator FROM mysql.event WHERE db = 'test' AND name = 'slave_once'; @@ -43,7 +41,7 @@ connection master; "Dropping event test.justonce on the master" DROP EVENT IF EXISTS test.justonce; "Creating event test.er on the master" -CREATE EVENT test.er ON SCHEDULE EVERY 3 SECOND STARTS CURRENT_TIMESTAMP + INTERVAL 1 HOUR DO +CREATE EVENT test.er ON SCHEDULE EVERY 3 SECOND STARTS CURRENT_TIMESTAMP + INTERVAL 1 HOUR DO INSERT IGNORE INTO t1(id, c) VALUES (4, 'from er'); "Checking event status on the master" SELECT db, name, status, originator, body FROM mysql.event WHERE db = 'test' AND name = 'er'; @@ -56,7 +54,7 @@ db name status originator body test er SLAVESIDE_DISABLED 1 INSERT IGNORE INTO t1(id, c) VALUES (4, 'from er') connection master; "Altering event test.er on the master" -ALTER EVENT test.er ON SCHEDULE EVERY 5 SECOND STARTS CURRENT_TIMESTAMP + INTERVAL 1 HOUR DO +ALTER EVENT test.er ON SCHEDULE EVERY 5 SECOND STARTS CURRENT_TIMESTAMP + INTERVAL 1 HOUR DO INSERT IGNORE INTO t1(id, c) VALUES (5, 'from alter er'); "Checking event status on the master" SELECT db, name, status, originator, body FROM mysql.event WHERE db = 'test' AND name = 'er'; @@ -78,7 +76,7 @@ connection slave; SELECT db, name, status, originator FROM mysql.event WHERE db = 'test'; db name status originator "Creating event test.slave_terminate on the slave" -CREATE EVENT test.slave_terminate ON SCHEDULE EVERY 3 SECOND STARTS CURRENT_TIMESTAMP + INTERVAL 1 HOUR DO +CREATE EVENT test.slave_terminate ON SCHEDULE EVERY 3 SECOND STARTS CURRENT_TIMESTAMP + INTERVAL 1 HOUR DO INSERT IGNORE INTO t1(id, c) VALUES (6, 'from slave_terminate'); "Checking event status on the slave" SELECT db, name, status, originator FROM mysql.event WHERE db = 'test' AND name = 'slave_terminate'; @@ -87,7 +85,7 @@ test slave_terminate ENABLED 2 "Dropping event test.slave_terminate on the slave" DROP EVENT test.slave_terminate; "Creating event test.slave_terminate with DISABLE ON SLAVE on the slave" -CREATE EVENT test.slave_terminate ON SCHEDULE EVERY 3 SECOND DISABLE ON SLAVE DO +CREATE EVENT test.slave_terminate ON SCHEDULE EVERY 3 SECOND DISABLE ON SLAVE DO INSERT IGNORE INTO t1(c) VALUES (7, 'from slave_terminate'); "Checking event status on the slave" SELECT db, name, status, originator FROM mysql.event WHERE db = 'test' AND name = 'slave_terminate'; @@ -100,7 +98,6 @@ connection master; DROP TABLE t1; connection slave; connection master; -connection master; CREATE TABLE t28953 (a INT); CREATE EVENT event1 ON SCHEDULE EVERY 1 YEAR DO BEGIN diff --git a/mysql-test/suite/rpl/t/rpl_events.test b/mysql-test/suite/rpl/t/rpl_events.test index 5bdf8ad241232..3e73fc7a3eeaa 100644 --- a/mysql-test/suite/rpl/t/rpl_events.test +++ b/mysql-test/suite/rpl/t/rpl_events.test @@ -9,16 +9,158 @@ SET @old_event_scheduler = @@global.event_scheduler; set global event_scheduler=1; -let $engine_type= MyISAM; +# first, we need a table to record something from an event + +eval CREATE TABLE `t1` ( + `id` INT(10) UNSIGNED NOT NULL, + `c` VARCHAR(50) NOT NULL, + `ts` TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP, + PRIMARY KEY (`id`) +) DEFAULT CHARSET=utf8; + +INSERT INTO t1 (id, c) VALUES (1, 'manually'); + +# We create the event so that it inserts exactly 1 row in the table +# A recuring event is used so that we can be sure the event will +# fire regardless of timing delays on the server. Otherwise, it is +# possible for the event to timeout before it has inserted a row. +--echo "Creating event test.justonce on the master" +CREATE EVENT test.justonce ON SCHEDULE EVERY 2 SECOND DO + INSERT IGNORE INTO t1 (id, c) VALUES (2, 'from justonce'); + +# Show the event is alive and present on master +--echo "Checking event is active on master" +SELECT db, name, status, originator FROM mysql.event WHERE db = 'test' AND name = 'justonce'; + +# Wait until event has fired. We know this because t1 will contain +# the row from the event. +let $wait_condition= + SELECT COUNT(*) = 1 FROM t1 WHERE c = 'from justonce'; +--source include/wait_condition.inc + +# check that table t1 contains something +--echo "Checking event data on the master" +let $events_done=`SELECT count(*) FROM t1 id`; +--disable_query_log +eval SELECT $events_done > 0 as ONE; +--enable_query_log ---source include/rpl_events.inc +sync_slave_with_master; + +--echo "Checking event data on the slave" +--disable_query_log +eval SELECT count(*) - $events_done as ZERO FROM t1 id; +--enable_query_log + +--echo "Checking event is inactive on slave" +SELECT db, name, status, originator FROM mysql.event WHERE db = 'test' AND name = 'justonce'; + +# Create an event on the slave and check to see what the originator is. +--echo "Dropping event test.slave_once on the slave" +--disable_warnings +DROP EVENT IF EXISTS test.slave_once; +--enable_warnings + +# Create an event on slave and check its state. An event shouldn't be executed +# so set start time in 1 hour. +CREATE EVENT test.slave_once ON SCHEDULE EVERY 5 MINUTE STARTS CURRENT_TIMESTAMP + INTERVAL 1 HOUR DO + INSERT IGNORE INTO t1(id, c) VALUES (3, 'from slave_once'); + +--echo "Checking event status on the slave for originator value = slave's server_id" +SELECT db, name, status, originator FROM mysql.event WHERE db = 'test' AND name = 'slave_once'; + +--echo "Dropping event test.slave_once on the slave" +--disable_warnings +DROP EVENT IF EXISTS test.slave_once; +--enable_warnings + +connection master; + +# BUG#20384 - disable events on slave +--echo "Dropping event test.justonce on the master" +--disable_warnings +DROP EVENT IF EXISTS test.justonce; +--enable_warnings + +# Create an event on master and check its state on slave. An event shouldn't be executed +# so set start time in 1 hour. Check that changes of event statement replicated to slave + +--echo "Creating event test.er on the master" +CREATE EVENT test.er ON SCHEDULE EVERY 3 SECOND STARTS CURRENT_TIMESTAMP + INTERVAL 1 HOUR DO + INSERT IGNORE INTO t1(id, c) VALUES (4, 'from er'); + +--echo "Checking event status on the master" +SELECT db, name, status, originator, body FROM mysql.event WHERE db = 'test' AND name = 'er'; + +sync_slave_with_master; + +--echo "Checking event status on the slave" +SELECT db, name, status, originator, body FROM mysql.event WHERE db = 'test' AND name = 'er'; + +connection master; +--echo "Altering event test.er on the master" +ALTER EVENT test.er ON SCHEDULE EVERY 5 SECOND STARTS CURRENT_TIMESTAMP + INTERVAL 1 HOUR DO + INSERT IGNORE INTO t1(id, c) VALUES (5, 'from alter er'); + +--echo "Checking event status on the master" +SELECT db, name, status, originator, body FROM mysql.event WHERE db = 'test' AND name = 'er'; + +sync_slave_with_master; + +--echo "Checking event status on the slave" +SELECT db, name, status, originator, body FROM mysql.event WHERE db = 'test' AND name = 'er'; + +connection master; +--echo "Dropping event test.er on the master" +DROP EVENT test.er; + +--echo "Checking event status on the master" +SELECT db, name, status, originator FROM mysql.event WHERE db = 'test'; + +--disable_info + +sync_slave_with_master; + +--echo "Checking event status on the slave" +SELECT db, name, status, originator FROM mysql.event WHERE db = 'test'; + +# test the DISABLE ON SLAVE for setting event SLAVESIDE_DISABLED as status +# on CREATE EVENT + +# Create an event on slave and check its status. An event shouldn't be executed +# so set start time in 1 hour. + +--echo "Creating event test.slave_terminate on the slave" +CREATE EVENT test.slave_terminate ON SCHEDULE EVERY 3 SECOND STARTS CURRENT_TIMESTAMP + INTERVAL 1 HOUR DO + INSERT IGNORE INTO t1(id, c) VALUES (6, 'from slave_terminate'); + +--echo "Checking event status on the slave" +SELECT db, name, status, originator FROM mysql.event WHERE db = 'test' AND name = 'slave_terminate'; + +--echo "Dropping event test.slave_terminate on the slave" +DROP EVENT test.slave_terminate; + +--echo "Creating event test.slave_terminate with DISABLE ON SLAVE on the slave" +CREATE EVENT test.slave_terminate ON SCHEDULE EVERY 3 SECOND DISABLE ON SLAVE DO + INSERT IGNORE INTO t1(c) VALUES (7, 'from slave_terminate'); + +--echo "Checking event status on the slave" +SELECT db, name, status, originator FROM mysql.event WHERE db = 'test' AND name = 'slave_terminate'; + +--echo "Dropping event test.slave_terminate on the slave" +DROP EVENT test.slave_terminate; + +--echo "Cleanup" + +connection master; +DROP TABLE t1; +sync_slave_with_master; +connection master; # # Bug #28953 Using events in a replication let the slave crash. # -connection master; - CREATE TABLE t28953 (a INT); DELIMITER |; @@ -39,7 +181,7 @@ DROP EVENT event2; # # BUG#44331 # This test verifies if the definer is consistent between master and slave, -# when the event is created without the DEFINER clause set explicitly or the +# when the event is created without the DEFINER clause set explicitly or the # DEFINER is set to CURRENT_USER # CREATE TABLE test.t1(details CHAR(30)); @@ -53,7 +195,7 @@ CREATE DEFINER=CURRENT_USER /*!50000 EVENT event44331_2 */ ON SCHEDULE AT CURRENT_TIMESTAMP ON COMPLETION PRESERVE DISABLE DO INSERT INTO test.t1 VALUES('event event44331_2 fired - DEFINER=CURRENT_USER'); - + CREATE DEFINER=CURRENT_USER() EVENT event44331_3 ON SCHEDULE AT CURRENT_TIMESTAMP ON COMPLETION PRESERVE DISABLE From 0a63b50c7a2dcab6fe99885cfcbd4a35d322c63a Mon Sep 17 00:00:00 2001 From: Sergey Vojtovich Date: Tue, 28 Nov 2017 16:34:31 +0400 Subject: [PATCH 057/106] Cleanup UT_LOW_PRIORITY_CPU/UT_RESUME_PRIORITY_CPU Server already has HMT_low/HMT_medium. --- config.h.cmake | 1 - configure.cmake | 11 ----------- include/my_cpu.h | 13 +++++++------ storage/innobase/include/ut0ut.h | 9 --------- storage/innobase/ut/ut0ut.cc | 5 +++-- 5 files changed, 10 insertions(+), 29 deletions(-) diff --git a/config.h.cmake b/config.h.cmake index 51abd4b11d53f..31a563037a0cd 100644 --- a/config.h.cmake +++ b/config.h.cmake @@ -191,7 +191,6 @@ #cmakedefine HAVE_PREAD 1 #cmakedefine HAVE_PAUSE_INSTRUCTION 1 #cmakedefine HAVE_FAKE_PAUSE_INSTRUCTION 1 -#cmakedefine HAVE_HMT_PRIORITY_INSTRUCTION 1 #cmakedefine HAVE_RDTSCLL 1 #cmakedefine HAVE_READ_REAL_TIME 1 #cmakedefine HAVE_PTHREAD_ATTR_CREATE 1 diff --git a/configure.cmake b/configure.cmake index ffcb9ba83b9c2..40aca55dc47dc 100644 --- a/configure.cmake +++ b/configure.cmake @@ -783,17 +783,6 @@ IF(NOT CMAKE_CROSSCOMPILING AND NOT MSVC) } " HAVE_FAKE_PAUSE_INSTRUCTION) ENDIF() - IF (NOT HAVE_PAUSE_INSTRUCTION) - CHECK_C_SOURCE_COMPILES(" - #include - int main() - { - __ppc_set_ppr_low(); - __ppc_set_ppr_med(); - return 0; - } - " HAVE_HMT_PRIORITY_INSTRUCTION) - ENDIF() ENDIF() CHECK_SYMBOL_EXISTS(tcgetattr "termios.h" HAVE_TCGETATTR 1) diff --git a/include/my_cpu.h b/include/my_cpu.h index 856a8e9b04a5f..f2e26fca70cbd 100644 --- a/include/my_cpu.h +++ b/include/my_cpu.h @@ -23,17 +23,18 @@ The defines are the same ones used by the linux kernel */ -#if defined(__powerpc__) +#ifdef _ARCH_PWR8 +#include /* Very low priority */ -#define HMT_very_low() asm volatile("or 31,31,31") +#define HMT_very_low() __ppc_set_ppr_very_low() /* Low priority */ -#define HMT_low() asm volatile("or 1,1,1") +#define HMT_low() __ppc_set_ppr_low() /* Medium low priority */ -#define HMT_medium_low() asm volatile("or 6,6,6") +#define HMT_medium_low() __ppc_set_ppr_med_low() /* Medium priority */ -#define HMT_medium() asm volatile("or 2,2,2") +#define HMT_medium() __ppc_set_ppr_med() /* Medium high priority */ -#define HMT_medium_high() asm volatile("or 5,5,5") +#define HMT_medium_high() __ppc_set_ppr_med_high() /* High priority */ #define HMT_high() asm volatile("or 3,3,3") #else diff --git a/storage/innobase/include/ut0ut.h b/storage/innobase/include/ut0ut.h index 352f5cce83da3..c91e24ad314c5 100644 --- a/storage/innobase/include/ut0ut.h +++ b/storage/innobase/include/ut0ut.h @@ -60,15 +60,6 @@ typedef time_t ib_time_t; # define UT_COMPILER_BARRIER() #endif -#if defined(HAVE_HMT_PRIORITY_INSTRUCTION) -# include -# define UT_LOW_PRIORITY_CPU() __ppc_set_ppr_low() -# define UT_RESUME_PRIORITY_CPU() __ppc_set_ppr_med() -#else -# define UT_LOW_PRIORITY_CPU() ((void)0) -# define UT_RESUME_PRIORITY_CPU() ((void)0) -#endif - /*********************************************************************//** Delays execution for at most max_wait_us microseconds or returns earlier if cond becomes true. diff --git a/storage/innobase/ut/ut0ut.cc b/storage/innobase/ut/ut0ut.cc index 88c5c889c8d2f..25705cb0e2c25 100644 --- a/storage/innobase/ut/ut0ut.cc +++ b/storage/innobase/ut/ut0ut.cc @@ -37,6 +37,7 @@ Created 5/11/1994 Heikki Tuuri #include "trx0trx.h" #include #include "log.h" +#include "my_cpu.h" #ifdef _WIN32 /*****************************************************************//** @@ -290,14 +291,14 @@ ut_delay( { ulint i; - UT_LOW_PRIORITY_CPU(); + HMT_low(); for (i = 0; i < delay * 50; i++) { MY_RELAX_CPU(); UT_COMPILER_BARRIER(); } - UT_RESUME_PRIORITY_CPU(); + HMT_medium(); } /*************************************************************//** From 09ef28abd7c3c80a80512bb4bba7f8e8f0194abe Mon Sep 17 00:00:00 2001 From: Monty Date: Sat, 13 Jan 2018 16:38:43 +0200 Subject: [PATCH 058/106] Fixed BUILD scripts - Removed extra set -x -v used for debugging - Fixed that that gcc version tests works for gcc 7 --- BUILD/FINISH.sh | 2 +- BUILD/check-cpu | 6 ++++++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/BUILD/FINISH.sh b/BUILD/FINISH.sh index 1faf62a9b62c3..ea64183253de9 100644 --- a/BUILD/FINISH.sh +++ b/BUILD/FINISH.sh @@ -14,7 +14,7 @@ # License along with this library; if not, write to the Free # Software Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, # MA 02110-1301, USA -set -x -v + cflags="$c_warnings $extra_flags $EXTRA_FLAGS $EXTRA_CFLAGS" cxxflags="$cxx_warnings $base_cxxflags $extra_flags $EXTRA_FLAGS $EXTRA_CXXFLAGS" extra_configs="$extra_configs $local_infile_configs $EXTRA_CONFIGS" diff --git a/BUILD/check-cpu b/BUILD/check-cpu index ad8816dc42135..c1c85cfd9086d 100755 --- a/BUILD/check-cpu +++ b/BUILD/check-cpu @@ -40,6 +40,12 @@ check_compiler_cpu_flags () { cc_major=$1 cc_minor=$2 cc_patch=$3 + if test -z "$cc_minor"; then + cc_minor="0"; + fi + if test -z "$cc_patch"; then + cc_minor="0"; + fi cc_comp=`expr $cc_major '*' 100 '+' $cc_minor` fi From fc65577873967ecbba93b75260cc54275f5900d2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Sat, 13 Jan 2018 18:04:03 +0200 Subject: [PATCH 059/106] MDEV-14887 On a 32-bit system, MariaDB 10.2 mishandles data file sizes exceeding 4GiB This is a regression that was introduced in MySQL 5.7.6 in https://github.com/mysql/mysql-server/commit/19855664de0245ff24e0753dc82723fc4e2fb7a5 fil_node_open_file(): Use proper 64-bit arithmetics for truncating size_bytes to a multiple of a file extent size. --- storage/innobase/fil/fil0fil.cc | 24 ++++++++++-------------- 1 file changed, 10 insertions(+), 14 deletions(-) diff --git a/storage/innobase/fil/fil0fil.cc b/storage/innobase/fil/fil0fil.cc index 3b496e3959f2c..86c0499eca444 100644 --- a/storage/innobase/fil/fil0fil.cc +++ b/storage/innobase/fil/fil0fil.cc @@ -1,7 +1,7 @@ /***************************************************************************** Copyright (c) 1995, 2017, Oracle and/or its affiliates. All Rights Reserved. -Copyright (c) 2014, 2017, MariaDB Corporation. +Copyright (c) 2014, 2018, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -566,7 +566,6 @@ bool fil_node_open_file( fil_node_t* node) { - os_offset_t size_bytes; bool success; bool read_only_mode; fil_space_t* space = node->space; @@ -611,7 +610,7 @@ fil_node_open_file( return(false); } - size_bytes = os_file_get_size(node->handle); + os_offset_t size_bytes = os_file_get_size(node->handle); ut_a(size_bytes != (os_offset_t) -1); ut_a(space->purpose != FIL_TYPE_LOG); @@ -694,20 +693,17 @@ fil_node_open_file( space->free_len = free_len; if (first_time_open) { - ulint extent_size; - - extent_size = psize * FSP_EXTENT_SIZE; - - /* After apply-incremental, tablespaces are not extended - to a whole megabyte. Do not cut off valid data. */ - /* Truncate the size to a multiple of extent size. */ - if (size_bytes >= extent_size) { - size_bytes = ut_2pow_round(size_bytes, - extent_size); + ulint mask = psize * FSP_EXTENT_SIZE - 1; + + if (size_bytes <= mask) { + /* .ibd files start smaller than an + extent size. Do not truncate valid data. */ + } else { + size_bytes &= ~os_offset_t(mask); } - node->size = static_cast(size_bytes / psize); + node->size = ulint(size_bytes / psize); space->size += node->size; } } From 68e5d6a9416b2923e78123cbc67e1755b8a21806 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Sat, 13 Jan 2018 20:27:46 +0200 Subject: [PATCH 060/106] Do not truncate integers on 32-bit systems in Rows_event_tracker commit 3dc3ab1a3048484910ca8acccaf76c71b080e533 introduced Rows_event_tracker, using a mismatch of size_t (the native register width) and my_off_t (the file offset width, usually 64 bits). Use my_off_t both in member fields and member functions. --- sql/rpl_mi.h | 4 ++-- sql/slave.cc | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/sql/rpl_mi.h b/sql/rpl_mi.h index b304e45f86aaf..b81c1fb398a91 100644 --- a/sql/rpl_mi.h +++ b/sql/rpl_mi.h @@ -139,11 +139,11 @@ typedef struct st_rows_event_tracker my_off_t first_seen; my_off_t last_seen; bool stmt_end_seen; - void update(const char* file_name, size_t pos, + void update(const char* file_name, my_off_t pos, const char* buf, const Format_description_log_event *fdle); void reset(); - bool check_and_report(const char* file_name, size_t pos); + bool check_and_report(const char* file_name, my_off_t pos); } Rows_event_tracker; /***************************************************************************** diff --git a/sql/slave.cc b/sql/slave.cc index e5c502c2de56e..d124b15e47c2d 100644 --- a/sql/slave.cc +++ b/sql/slave.cc @@ -7650,7 +7650,7 @@ void Rows_event_tracker::reset() well as the end-of-statement status of the last one. */ -void Rows_event_tracker::update(const char* file_name, size_t pos, +void Rows_event_tracker::update(const char* file_name, my_off_t pos, const char* buf, const Format_description_log_event *fdle) { @@ -7675,7 +7675,7 @@ void Rows_event_tracker::update(const char* file_name, size_t pos, false otherwise. */ bool Rows_event_tracker::check_and_report(const char* file_name, - size_t pos) + my_off_t pos) { if (last_seen) { From 72136ae75c69cee5230e7be286cf12663d3d15ef Mon Sep 17 00:00:00 2001 From: Eugene Kosov Date: Sun, 14 Jan 2018 19:50:45 +0300 Subject: [PATCH 061/106] Compilation speed (#546) Speed up compilation Standard C++ headers contribute a lot to compilation time. Avoid algorithm and sstream in frequently used headers. --- include/my_dbug.h | 6 +++++- sql/item_func.cc | 2 +- sql/item_sum.cc | 4 ++-- sql/key.cc | 19 ++++++++----------- sql/log.cc | 2 +- sql/log_event.cc | 6 ++---- sql/mdl.h | 5 +---- storage/perfschema/pfs.cc | 3 +-- storage/perfschema/table_threads.cc | 4 ++-- 9 files changed, 23 insertions(+), 28 deletions(-) diff --git a/include/my_dbug.h b/include/my_dbug.h index ba9e8a025d77b..003290afb9671 100644 --- a/include/my_dbug.h +++ b/include/my_dbug.h @@ -195,10 +195,14 @@ void debug_sync_point(const char* lock_name, uint lock_timeout); #ifdef __cplusplus } +/* + DBUG_LOG() was initially intended for InnoDB. To be able to use it elsewhere + one should #include . We intentially avoid including it here to save + compilation time. +*/ # ifdef DBUG_OFF # define DBUG_LOG(keyword, v) do {} while (0) # else -# include # define DBUG_LOG(keyword, v) do { \ if (_db_pargs_(__LINE__, keyword)) { \ std::ostringstream _db_s; _db_s << v; \ diff --git a/sql/item_func.cc b/sql/item_func.cc index b1970be2b298e..de740cbcece13 100644 --- a/sql/item_func.cc +++ b/sql/item_func.cc @@ -2889,7 +2889,7 @@ bool Item_func_min_max::get_date(MYSQL_TIME *ltime, ulonglong fuzzy_date) ltime->hour+= (ltime->month * 32 + ltime->day) * 24; ltime->year= ltime->month= ltime->day= 0; if (adjust_time_range_with_warn(ltime, - std::min(decimals, TIME_SECOND_PART_DIGITS))) + MY_MIN(decimals, TIME_SECOND_PART_DIGITS))) return (null_value= true); } diff --git a/sql/item_sum.cc b/sql/item_sum.cc index f5ea2a5df9361..94569aa66c9fa 100644 --- a/sql/item_sum.cc +++ b/sql/item_sum.cc @@ -2257,7 +2257,7 @@ bool Item_sum_bit::remove_as_window(ulonglong value) } // Prevent overflow; - num_values_added = std::min(num_values_added, num_values_added - 1); + num_values_added = MY_MIN(num_values_added, num_values_added - 1); set_bits_from_counters(); return 0; } @@ -2270,7 +2270,7 @@ bool Item_sum_bit::add_as_window(ulonglong value) bit_counters[i]+= (value & (1ULL << i)) ? 1 : 0; } // Prevent overflow; - num_values_added = std::max(num_values_added, num_values_added + 1); + num_values_added = MY_MAX(num_values_added, num_values_added + 1); set_bits_from_counters(); return 0; } diff --git a/sql/key.cc b/sql/key.cc index cdf5c8c4a7438..af473d85847bb 100644 --- a/sql/key.cc +++ b/sql/key.cc @@ -21,9 +21,6 @@ #include "key.h" // key_rec_cmp #include "field.h" // Field -using std::min; -using std::max; - /* Search after a key that starts with 'field' @@ -135,7 +132,7 @@ void key_copy(uchar *to_key, uchar *from_record, KEY *key_info, Don't copy data for null values The -1 below is to subtract the null byte which is already handled */ - length= min(key_length, key_part->store_length-1); + length= MY_MIN(key_length, key_part->store_length-1); if (with_zerofill) bzero((char*) to_key, length); continue; @@ -145,7 +142,7 @@ void key_copy(uchar *to_key, uchar *from_record, KEY *key_info, key_part->key_part_flag & HA_VAR_LENGTH_PART) { key_length-= HA_KEY_BLOB_LENGTH; - length= min(key_length, key_part->length); + length= MY_MIN(key_length, key_part->length); uint bytes= key_part->field->get_key_image(to_key, length, Field::itRAW); if (with_zerofill && bytes < length) bzero((char*) to_key + bytes, length - bytes); @@ -153,7 +150,7 @@ void key_copy(uchar *to_key, uchar *from_record, KEY *key_info, } else { - length= min(key_length, key_part->length); + length= MY_MIN(key_length, key_part->length); Field *field= key_part->field; CHARSET_INFO *cs= field->charset(); uint bytes= field->get_key_image(to_key, length, Field::itRAW); @@ -205,7 +202,7 @@ void key_restore(uchar *to_record, const uchar *from_key, KEY *key_info, Don't copy data for null bytes The -1 below is to subtract the null byte which is already handled */ - length= min(key_length, key_part->store_length-1); + length= MY_MIN(key_length, key_part->store_length-1); continue; } } @@ -247,7 +244,7 @@ void key_restore(uchar *to_record, const uchar *from_key, KEY *key_info, my_ptrdiff_t ptrdiff= to_record - field->table->record[0]; field->move_field_offset(ptrdiff); key_length-= HA_KEY_BLOB_LENGTH; - length= min(key_length, key_part->length); + length= MY_MIN(key_length, key_part->length); old_map= dbug_tmp_use_all_columns(field->table, field->table->write_set); field->set_key_image(from_key, length); dbug_tmp_restore_column_map(field->table->write_set, old_map); @@ -256,7 +253,7 @@ void key_restore(uchar *to_record, const uchar *from_key, KEY *key_info, } else { - length= min(key_length, key_part->length); + length= MY_MIN(key_length, key_part->length); /* skip the byte with 'uneven' bits, if used */ memcpy(to_record + key_part->offset, from_key + used_uneven_bits , (size_t) length - used_uneven_bits); @@ -314,7 +311,7 @@ bool key_cmp_if_same(TABLE *table,const uchar *key,uint idx,uint key_length) return 1; continue; } - length= min((uint) (key_end-key), store_length); + length= MY_MIN((uint) (key_end-key), store_length); if (!(key_part->key_type & (FIELDFLAG_NUMBER+FIELDFLAG_BINARY+ FIELDFLAG_PACK))) { @@ -392,7 +389,7 @@ void field_unpack(String *to, Field *field, const uchar *rec, uint max_length, tmp.length(charpos); } if (max_length < field->pack_length()) - tmp.length(min(tmp.length(),max_length)); + tmp.length(MY_MIN(tmp.length(),max_length)); ErrConvString err(&tmp); to->append(err.ptr()); } diff --git a/sql/log.cc b/sql/log.cc index 238739223683e..f1a0834217a34 100644 --- a/sql/log.cc +++ b/sql/log.cc @@ -7044,7 +7044,7 @@ int MYSQL_BIN_LOG::write_cache(THD *thd, IO_CACHE *cache) int4store(ev + EVENT_LEN_OFFSET, ev_len + writer.checksum_len); writer.remains= ev_len; - if (writer.write(ev, std::min(ev_len, length - hdr_offs))) + if (writer.write(ev, MY_MIN(ev_len, length - hdr_offs))) DBUG_RETURN(ER_ERROR_ON_WRITE); /* next event header at ... */ diff --git a/sql/log_event.cc b/sql/log_event.cc index bd6d325b3efdc..d80f45dffc1c1 100644 --- a/sql/log_event.cc +++ b/sql/log_event.cc @@ -55,8 +55,6 @@ #define my_b_write_string(A, B) my_b_write((A), (uchar*)(B), (uint) (sizeof(B) - 1)) -using std::max; - /** BINLOG_CHECKSUM variable. */ @@ -1777,8 +1775,8 @@ int Log_event::read_log_event(IO_CACHE* file, String* packet, if (data_len < LOG_EVENT_MINIMAL_HEADER_LEN) DBUG_RETURN(LOG_READ_BOGUS); - if (data_len > max(max_allowed_packet, - opt_binlog_rows_event_max_size + MAX_LOG_EVENT_HEADER)) + if (data_len > MY_MAX(max_allowed_packet, + opt_binlog_rows_event_max_size + MAX_LOG_EVENT_HEADER)) DBUG_RETURN(LOG_READ_TOO_LARGE); if (likely(data_len > LOG_EVENT_MINIMAL_HEADER_LEN)) diff --git a/sql/mdl.h b/sql/mdl.h index 97216b8e7b10c..2aeaef89f264b 100644 --- a/sql/mdl.h +++ b/sql/mdl.h @@ -21,8 +21,6 @@ #include #include -#include - class THD; class MDL_context; @@ -373,8 +371,7 @@ class MDL_key character set is utf-8, we can safely assume that no character starts with a zero byte. */ - using std::min; - return memcmp(m_ptr, rhs->m_ptr, min(m_length, rhs->m_length)); + return memcmp(m_ptr, rhs->m_ptr, MY_MIN(m_length, rhs->m_length)); } MDL_key(const MDL_key *rhs) diff --git a/storage/perfschema/pfs.cc b/storage/perfschema/pfs.cc index f781a83b3246d..624c7f4697b06 100644 --- a/storage/perfschema/pfs.cc +++ b/storage/perfschema/pfs.cc @@ -41,7 +41,6 @@ #include "sp_head.h" #include "pfs_digest.h" -using std::min; /** @page PAGE_PERFORMANCE_SCHEMA The Performance Schema main page MySQL PERFORMANCE_SCHEMA implementation. @@ -2022,7 +2021,7 @@ static void set_thread_account_v1(const char *user, int user_len, DBUG_ASSERT((host != NULL) || (host_len == 0)); DBUG_ASSERT(host_len >= 0); - host_len= min(host_len, sizeof(pfs->m_hostname)); + host_len= MY_MIN(host_len, static_cast(sizeof(pfs->m_hostname))); if (unlikely(pfs == NULL)) return; diff --git a/storage/perfschema/table_threads.cc b/storage/perfschema/table_threads.cc index 5c78b567b8c99..211865c0f63d3 100644 --- a/storage/perfschema/table_threads.cc +++ b/storage/perfschema/table_threads.cc @@ -257,8 +257,8 @@ int table_threads::read_row_values(TABLE *table, changed to less than or equal to 64 characters. */ set_field_varchar_utf8(f, m_row.m_processlist_state_ptr, - std::min(m_row.m_processlist_state_length, - f->char_length())); + MY_MIN(m_row.m_processlist_state_length, + f->char_length())); } else f->set_null(); From 3d798be1d481835b60a811148a10ea14c37d5674 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Mon, 15 Jan 2018 10:49:03 +0200 Subject: [PATCH 062/106] MDEV-14655 Assertion `!fts_index' failed in prepare_inplace_alter_table_dict MariaDB inherits the MySQL limitation that ALGORITHM=INPLACE cannot create more than one FULLTEXT INDEX at a time. As part of the MDEV-11369 Instant ADD COLUMN refactoring, MariaDB 10.3.2 accidentally stopped enforcing the restriction. Actually, it is a bug in MySQL 5.6 and MariaDB 10.0 that an ALTER TABLE statement with multiple ADD FULLTEXT INDEX but without explicit ALGORITHM=INPLACE would return in an error message, rather than executing the operation with ALGORITHM=COPY. ha_innobase::check_if_supported_inplace_alter(): Enforce the restriction on multiple FULLTEXT INDEX. prepare_inplace_alter_table_dict(): Replace some code with debug assertions. A "goto error_handled" at this point would result in another error, because the reference count of ctx->new_table would be 0. --- .../suite/innodb_fts/r/innodb-fts-ddl.result | 14 +++-- .../suite/innodb_fts/t/innodb-fts-ddl.test | 13 ++--- storage/innobase/handler/handler0alter.cc | 51 +++++++------------ 3 files changed, 35 insertions(+), 43 deletions(-) diff --git a/mysql-test/suite/innodb_fts/r/innodb-fts-ddl.result b/mysql-test/suite/innodb_fts/r/innodb-fts-ddl.result index 983f254cf9036..cd7d8f03923a8 100644 --- a/mysql-test/suite/innodb_fts/r/innodb-fts-ddl.result +++ b/mysql-test/suite/innodb_fts/r/innodb-fts-ddl.result @@ -162,11 +162,15 @@ INSERT INTO articles (FTS_DOC_ID, title, body) VALUES (14,'1001 MySQL Tricks','1. Never run mysqld as root. 2. ...'), (19, 'MySQL vs. YourSQL','In the following database comparison ...'), (20, 'MySQL Security','When configured properly, MySQL ...'); -ALTER TABLE articles ADD FULLTEXT INDEX idx3 (title), -ADD FULLTEXT INDEX idx5 (title); -ERROR HY000: InnoDB presently supports one FULLTEXT index creation at a time -CREATE FULLTEXT INDEX idx on articles (title); -ALTER TABLE articles ADD FULLTEXT INDEX idx3 (title); +ALTER TABLE articles ADD FULLTEXT INDEX idx (title), +ADD FULLTEXT INDEX idx3 (title), ALGORITHM=INPLACE; +ERROR 0A000: ALGORITHM=INPLACE is not supported. Reason: InnoDB presently supports one FULLTEXT index creation at a time. Try ALGORITHM=COPY +ALTER TABLE articles ADD FULLTEXT INDEX idx (title), +ADD FULLTEXT INDEX idx3 (title); +affected rows: 6 +info: Records: 6 Duplicates: 0 Warnings: 1 +Warnings: +Note 1831 Duplicate index `idx3`. This is deprecated and will be disallowed in a future release ALTER TABLE articles ADD INDEX t20 (title(20)), LOCK=NONE; ALTER TABLE articles DROP INDEX t20; INSERT INTO articles (FTS_DOC_ID, title, body) VALUES diff --git a/mysql-test/suite/innodb_fts/t/innodb-fts-ddl.test b/mysql-test/suite/innodb_fts/t/innodb-fts-ddl.test index 10dc1462c987e..23065a97002e8 100644 --- a/mysql-test/suite/innodb_fts/t/innodb-fts-ddl.test +++ b/mysql-test/suite/innodb_fts/t/innodb-fts-ddl.test @@ -195,12 +195,13 @@ INSERT INTO articles (FTS_DOC_ID, title, body) VALUES (19, 'MySQL vs. YourSQL','In the following database comparison ...'), (20, 'MySQL Security','When configured properly, MySQL ...'); ---error ER_INNODB_FT_LIMIT -ALTER TABLE articles ADD FULLTEXT INDEX idx3 (title), - ADD FULLTEXT INDEX idx5 (title); - -CREATE FULLTEXT INDEX idx on articles (title); -ALTER TABLE articles ADD FULLTEXT INDEX idx3 (title); +--error ER_ALTER_OPERATION_NOT_SUPPORTED_REASON +ALTER TABLE articles ADD FULLTEXT INDEX idx (title), + ADD FULLTEXT INDEX idx3 (title), ALGORITHM=INPLACE; +--enable_info +ALTER TABLE articles ADD FULLTEXT INDEX idx (title), + ADD FULLTEXT INDEX idx3 (title); +--disable_info ALTER TABLE articles ADD INDEX t20 (title(20)), LOCK=NONE; ALTER TABLE articles DROP INDEX t20; diff --git a/storage/innobase/handler/handler0alter.cc b/storage/innobase/handler/handler0alter.cc index 7d4b15fc40e05..55962e1c6d1b1 100644 --- a/storage/innobase/handler/handler0alter.cc +++ b/storage/innobase/handler/handler0alter.cc @@ -1162,6 +1162,7 @@ ha_innobase::check_if_supported_inplace_alter( /* If the table already contains fulltext indexes, refuse to rebuild the table natively altogether. */ if (m_prebuilt->table->fts) { +cannot_create_many_fulltext_index: ha_alter_info->unsupported_reason = innobase_get_err_msg( ER_INNODB_FT_LIMIT); DBUG_RETURN(HA_ALTER_INPLACE_NOT_SUPPORTED); @@ -1191,6 +1192,7 @@ ha_innobase::check_if_supported_inplace_alter( We could also do ADD SPATIAL INDEX by implementing row_log_apply() for it. */ + bool add_fulltext = false; for (uint i = 0; i < ha_alter_info->index_add_count; i++) { const KEY* key = @@ -1202,16 +1204,18 @@ ha_innobase::check_if_supported_inplace_alter( | HA_PACK_KEY | HA_GENERATED_KEY | HA_BINARY_PACK_KEY))); + if (add_fulltext) { + goto cannot_create_many_fulltext_index; + } + add_fulltext = true; ha_alter_info->unsupported_reason = innobase_get_err_msg( ER_ALTER_OPERATION_NOT_SUPPORTED_REASON_FTS); online = false; - break; } - if (key->flags & HA_SPATIAL) { + if (online && (key->flags & HA_SPATIAL)) { ha_alter_info->unsupported_reason = innobase_get_err_msg( ER_ALTER_OPERATION_NOT_SUPPORTED_REASON_GIS); online = false; - break; } } } @@ -5291,35 +5295,18 @@ prepare_inplace_alter_table_dict( ctx->prepare_instant(); } - if (!ctx->is_instant()) { - if (num_fts_index > 1) { - my_error(ER_INNODB_FT_LIMIT, MYF(0)); - goto error_handled; - } - - if (!ctx->online) { - /* This is not an online operation (LOCK=NONE). */ - } else if (ctx->add_autoinc == ULINT_UNDEFINED - && num_fts_index == 0 - && (!innobase_need_rebuild(ha_alter_info, old_table) - || !innobase_fulltext_exist(altered_table))) { - /* InnoDB can perform an online operation - (LOCK=NONE). */ - } else { - size_t query_length; - /* This should have been blocked in - check_if_supported_inplace_alter(). */ - ut_ad(0); - my_error(ER_NOT_SUPPORTED_YET, MYF(0), - innobase_get_stmt_unsafe( - ctx->prebuilt->trx->mysql_thd, - &query_length)); - goto error_handled; - } - } - if (ctx->need_rebuild()) { not_instant_add_column: + DBUG_ASSERT(ctx->need_rebuild()); + DBUG_ASSERT(!ctx->is_instant()); + DBUG_ASSERT(num_fts_index <= 1); + DBUG_ASSERT(!ctx->online || num_fts_index == 0); + DBUG_ASSERT(!ctx->online + || ctx->add_autoinc == ULINT_UNDEFINED); + DBUG_ASSERT(!ctx->online + || !innobase_need_rebuild(ha_alter_info, old_table) + || !innobase_fulltext_exist(altered_table)); + uint32_t key_id = FIL_DEFAULT_ENCRYPTION_KEY; fil_encryption_t mode = FIL_ENCRYPTION_DEFAULT; @@ -5419,7 +5406,7 @@ prepare_inplace_alter_table_dict( ut_ad(index->trx_id == ctx->trx->id); if (index->type & DICT_FTS) { - DBUG_ASSERT(num_fts_index); + DBUG_ASSERT(num_fts_index == 1); DBUG_ASSERT(!fts_index); DBUG_ASSERT(index->type == DICT_FTS); fts_index = ctx->add_index[a]; @@ -5504,7 +5491,7 @@ prepare_inplace_alter_table_dict( /* If ADD INDEX with LOCK=NONE has been requested, allocate a modification log. */ if (index->type & DICT_FTS) { - DBUG_ASSERT(num_fts_index); + DBUG_ASSERT(num_fts_index == 1); DBUG_ASSERT(!fts_index); DBUG_ASSERT(index->type == DICT_FTS); fts_index = ctx->add_index[a]; From ec062c618186681504e79e6f568f98406020ab01 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Mon, 15 Jan 2018 15:26:02 +0200 Subject: [PATCH 063/106] MDEV-12121 follow-up: Unbreak the WITH_INNODB_AHI=OFF build --- storage/innobase/buf/buf0lru.cc | 12 ++++++++---- storage/innobase/dict/dict0dict.cc | 8 ++++++-- storage/innobase/fil/fil0fil.cc | 20 ++++++++++++++++---- storage/innobase/handler/ha_xtradb.h | 4 +++- storage/innobase/include/buf0lru.h | 12 +++++++----- storage/innobase/include/fil0fil.h | 10 +++++++--- storage/innobase/include/ha0ha.h | 3 ++- 7 files changed, 49 insertions(+), 20 deletions(-) diff --git a/storage/innobase/buf/buf0lru.cc b/storage/innobase/buf/buf0lru.cc index 10ee106137dda..2d68b8e9ccd7a 100644 --- a/storage/innobase/buf/buf0lru.cc +++ b/storage/innobase/buf/buf0lru.cc @@ -696,22 +696,26 @@ buf_flush_dirty_pages( /** Empty the flush list for all pages belonging to a tablespace. @param[in] id tablespace identifier @param[in] observer flush observer, - or NULL if nothing is to be written -@param[in] drop_ahi whether to drop the adaptive hash index */ + or NULL if nothing is to be written */ void buf_LRU_flush_or_remove_pages( ulint id, - FlushObserver* observer, - bool drop_ahi) + FlushObserver* observer +#ifdef BTR_CUR_HASH_ADAPT + , bool drop_ahi /*!< whether to drop the adaptive hash index */ +#endif /* BTR_CUR_HASH_ADAPT */ + ) { /* Pages in the system tablespace must never be discarded. */ ut_ad(id || observer); for (ulint i = 0; i < srv_buf_pool_instances; i++) { buf_pool_t* buf_pool = buf_pool_from_array(i); +#ifdef BTR_CUR_HASH_ADAPT if (drop_ahi) { buf_LRU_drop_page_hash_for_tablespace(buf_pool, id); } +#endif /* BTR_CUR_HASH_ADAPT */ buf_flush_dirty_pages(buf_pool, id, observer); } diff --git a/storage/innobase/dict/dict0dict.cc b/storage/innobase/dict/dict0dict.cc index 49c60e5df56bd..b93eb4d6b1760 100644 --- a/storage/innobase/dict/dict0dict.cc +++ b/storage/innobase/dict/dict0dict.cc @@ -2,7 +2,7 @@ Copyright (c) 1996, 2016, Oracle and/or its affiliates. All Rights Reserved. Copyright (c) 2012, Facebook Inc. -Copyright (c) 2013, 2017, MariaDB Corporation. +Copyright (c) 2013, 2018, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -1641,7 +1641,11 @@ dict_table_rename_in_cache( return(DB_OUT_OF_MEMORY); } - fil_delete_tablespace(table->space, true); + fil_delete_tablespace(table->space +#ifdef BTR_CUR_HASH_ADAPT + , true +#endif /* BTR_CUR_HASH_ADAPT */ + ); /* Delete any temp file hanging around. */ if (os_file_status(filepath, &exists, &ftype) diff --git a/storage/innobase/fil/fil0fil.cc b/storage/innobase/fil/fil0fil.cc index 86c0499eca444..9e31cb8bd81b3 100644 --- a/storage/innobase/fil/fil0fil.cc +++ b/storage/innobase/fil/fil0fil.cc @@ -2960,10 +2960,14 @@ fil_table_accessible(const dict_table_t* table) /** Delete a tablespace and associated .ibd file. @param[in] id tablespace identifier -@param[in] drop_ahi whether to drop the adaptive hash index @return DB_SUCCESS or error */ dberr_t -fil_delete_tablespace(ulint id, bool drop_ahi) +fil_delete_tablespace( + ulint id +#ifdef BTR_CUR_HASH_ADAPT + , bool drop_ahi /*!< whether to drop the adaptive hash index */ +#endif /* BTR_CUR_HASH_ADAPT */ + ) { char* path = 0; fil_space_t* space = 0; @@ -3006,7 +3010,11 @@ fil_delete_tablespace(ulint id, bool drop_ahi) To deal with potential read requests, we will check the ::stop_new_ops flag in fil_io(). */ - buf_LRU_flush_or_remove_pages(id, NULL, drop_ahi); + buf_LRU_flush_or_remove_pages(id, NULL +#ifdef BTR_CUR_HASH_ADAPT + , drop_ahi +#endif /* BTR_CUR_HASH_ADAPT */ + ); /* If it is a delete then also delete any generated files, otherwise when we drop the database the remove directory will fail. */ @@ -3286,7 +3294,11 @@ fil_discard_tablespace( { dberr_t err; - switch (err = fil_delete_tablespace(id, true)) { + switch (err = fil_delete_tablespace(id +#ifdef BTR_CUR_HASH_ADAPT + , true +#endif /* BTR_CUR_HASH_ADAPT */ + )) { case DB_SUCCESS: break; diff --git a/storage/innobase/handler/ha_xtradb.h b/storage/innobase/handler/ha_xtradb.h index 09628b7ec48bb..d2d1f3613826d 100644 --- a/storage/innobase/handler/ha_xtradb.h +++ b/storage/innobase/handler/ha_xtradb.h @@ -1,7 +1,7 @@ /***************************************************************************** Copyright (c) 2000, 2016, Oracle and/or its affiliates. All Rights Reserved. -Copyright (c) 2017, MariaDB Corporation. +Copyright (c) 2017, 2018, MariaDB Corporation. Copyright (c) 2009, Percona Inc. Portions of this file contain modifications contributed and copyrighted @@ -957,7 +957,9 @@ innodb_check_deprecated(void) #ifdef HA_XTRADB_SYSVARS /* XtraDB compatibility system variables */ +#ifdef BTR_CUR_HASH_ADAPT MYSQL_SYSVAR(adaptive_hash_index_partitions), +#endif /* BTR_CUR_HASH_ADAPT */ MYSQL_SYSVAR(buffer_pool_populate), #if defined UNIV_DEBUG || defined UNIV_PERF_DEBUG MYSQL_SYSVAR(cleaner_eviction_factor), diff --git a/storage/innobase/include/buf0lru.h b/storage/innobase/include/buf0lru.h index 7b739fc033212..f6a7695a2b5b4 100644 --- a/storage/innobase/include/buf0lru.h +++ b/storage/innobase/include/buf0lru.h @@ -1,7 +1,7 @@ /***************************************************************************** Copyright (c) 1995, 2016, Oracle and/or its affiliates. All Rights Reserved. -Copyright (c) 2017, MariaDB Corporation. +Copyright (c) 2017, 2018, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -53,13 +53,15 @@ These are low-level functions /** Empty the flush list for all pages belonging to a tablespace. @param[in] id tablespace identifier @param[in,out] observer flush observer, - or NULL if nothing is to be written -@param[in] drop_ahi whether to drop the adaptive hash index */ + or NULL if nothing is to be written */ void buf_LRU_flush_or_remove_pages( ulint id, - FlushObserver* observer, - bool drop_ahi = false); + FlushObserver* observer +#ifdef BTR_CUR_HASH_ADAPT + , bool drop_ahi = false /*!< whether to drop the adaptive hash index */ +#endif /* BTR_CUR_HASH_ADAPT */ + ); #if defined UNIV_DEBUG || defined UNIV_BUF_DEBUG /********************************************************************//** diff --git a/storage/innobase/include/fil0fil.h b/storage/innobase/include/fil0fil.h index 9fa507c2114fd..c2152ce11d00d 100644 --- a/storage/innobase/include/fil0fil.h +++ b/storage/innobase/include/fil0fil.h @@ -1,7 +1,7 @@ /***************************************************************************** Copyright (c) 1995, 2017, Oracle and/or its affiliates. All Rights Reserved. -Copyright (c) 2013, 2017, MariaDB Corporation. +Copyright (c) 2013, 2018, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -931,10 +931,14 @@ fil_table_accessible(const dict_table_t* table) /** Delete a tablespace and associated .ibd file. @param[in] id tablespace identifier -@param[in] drop_ahi whether to drop the adaptive hash index @return DB_SUCCESS or error */ dberr_t -fil_delete_tablespace(ulint id, bool drop_ahi = false); +fil_delete_tablespace( + ulint id +#ifdef BTR_CUR_HASH_ADAPT + , bool drop_ahi = false /*!< whether to drop the adaptive hash index */ +#endif /* BTR_CUR_HASH_ADAPT */ + ); /** Truncate the tablespace to needed size. @param[in] space_id id of tablespace to truncate diff --git a/storage/innobase/include/ha0ha.h b/storage/innobase/include/ha0ha.h index ca4cb0a5f8fec..f5be654f49007 100644 --- a/storage/innobase/include/ha0ha.h +++ b/storage/innobase/include/ha0ha.h @@ -1,6 +1,7 @@ /***************************************************************************** Copyright (c) 1994, 2016, Oracle and/or its affiliates. All Rights Reserved. +Copyright (c) 2018, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -210,7 +211,7 @@ struct ha_node_t { }; #endif /* BTR_CUR_HASH_ADAPT */ -#ifdef UNIV_DEBUG +#if defined UNIV_DEBUG && defined BTR_CUR_HASH_ADAPT /********************************************************************//** Assert that the synchronization object in a hash operation involving possible change in the hash table is held. From 850702da6bed7e18145be7688d95225241ac29bc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Mon, 15 Jan 2018 15:37:22 +0200 Subject: [PATCH 064/106] MDEV-13626: Merge InnoDB test cases from MySQL 5.7 (part 6) innodb.truncate_inject: Replacement for innodb_zip.wl6501_error_1 Note: unlike MySQL, in some cases TRUNCATE does not return an error in MariaDB. This should be fixed in the scope of MDEV-13564 or similar. --- .../suite/innodb/r/truncate_inject.result | 114 +++++++++ .../suite/innodb/t/truncate_inject.test | 97 ++++++++ .../include/innodb_wl6501_error.inc | 226 ------------------ 3 files changed, 211 insertions(+), 226 deletions(-) create mode 100644 mysql-test/suite/innodb/r/truncate_inject.result create mode 100644 mysql-test/suite/innodb/t/truncate_inject.test delete mode 100644 mysql-test/suite/innodb_zip/include/innodb_wl6501_error.inc diff --git a/mysql-test/suite/innodb/r/truncate_inject.result b/mysql-test/suite/innodb/r/truncate_inject.result new file mode 100644 index 0000000000000..5ec532a0f835a --- /dev/null +++ b/mysql-test/suite/innodb/r/truncate_inject.result @@ -0,0 +1,114 @@ +SET @save_dbug = @@SESSION.debug_dbug; +call mtr.add_suppression("InnoDB: Flagged corruption of .* in table `test`\\.`t` in TRUNCATE TABLE"); +# 1. Error in assigning undo logs for truncate action +CREATE TABLE t (i int PRIMARY KEY, f float UNIQUE, c char(100), INDEX ck(c)) +ENGINE = InnoDB; +insert into t values (1, 1.1, 'a'), (2, 2.2, 'b'), (3, 3.3, 'c'); +check table t; +Table Op Msg_type Msg_text +test.t check status OK +SET debug_dbug = '+d,ib_err_trunc_assigning_undo_log'; +truncate table t; +ERROR HY000: Got error 168 "Unknown (generic) error from engine" from storage engine InnoDB +SET debug_dbug = @save_dbug; +check table t; +Table Op Msg_type Msg_text +test.t check status OK +select * from t; +i f c +1 1.1 a +2 2.2 b +3 3.3 c +# 2. Error while preparing for truncate +SET debug_dbug = '+d,ib_err_trunc_preparing_for_truncate'; +truncate table t; +ERROR HY000: Got error 168 "Unknown (generic) error from engine" from storage engine InnoDB +SET debug_dbug = @save_dbug; +check table t; +Table Op Msg_type Msg_text +test.t check status OK +select * from t; +i f c +1 1.1 a +2 2.2 b +3 3.3 c +# 3. Error while dropping/creating indexes +SET debug_dbug = '+d,ib_err_trunc_drop_index'; +truncate table t; +ERROR HY000: Got error 168 "Unknown (generic) error from engine" from storage engine InnoDB +SET debug_dbug = @save_dbug; +check table t; +Table Op Msg_type Msg_text +test.t check Warning InnoDB: Index PRIMARY is marked as corrupted +test.t check error Corrupt +select * from t; +Got one of the listed errors +drop table t; +CREATE TABLE t (i int PRIMARY KEY, f float UNIQUE, c char(100), INDEX ck(c)) +ENGINE = InnoDB; +insert into t values (1, 1.1, 'a'), (2, 2.2, 'b'), (3, 3.3, 'c'); +check table t; +Table Op Msg_type Msg_text +test.t check status OK +SET debug_dbug = '+d,ib_err_trunc_create_index'; +truncate table t; +ERROR HY000: Got error 168 "Unknown (generic) error from engine" from storage engine InnoDB +SET debug_dbug = @save_dbug; +check table t; +Table Op Msg_type Msg_text +test.t check Warning InnoDB: Index PRIMARY is marked as corrupted +test.t check error Corrupt +select * from t; +Got one of the listed errors +drop table t; +CREATE TABLE t (i int PRIMARY KEY, f float UNIQUE, c char(100), INDEX ck(c)) +ENGINE = InnoDB; +insert into t values (1, 1.1, 'a'), (2, 2.2, 'b'), (3, 3.3, 'c'); +check table t; +Table Op Msg_type Msg_text +test.t check status OK +SET debug_dbug = '+d,ib_err_trunc_temp_recreate_index'; +truncate table t; +SET debug_dbug = @save_dbug; +check table t; +Table Op Msg_type Msg_text +test.t check status OK +select * from t; +i f c +drop table t; +# 4. Error while completing truncate of table involving FTS +CREATE TABLE t (i int PRIMARY KEY, f float UNIQUE, c char(100), +FULLTEXT INDEX(c)) ENGINE = InnoDB; +insert into t values (1, 1.1, 'mysql is now oracle company'), +(2, 2.2, 'innodb is part of mysql'), +(3, 3.3, 'innodb is default storage engine of mysql'); +check table t; +Table Op Msg_type Msg_text +test.t check status OK +SET debug_dbug = '+d,ib_err_trunc_temp_recreate_index'; +truncate table t; +SET debug_dbug = @save_dbug; +check table t; +Table Op Msg_type Msg_text +test.t check status OK +select * from t; +i f c +drop table t; +# 5. Error while updating sys-tables +CREATE TABLE t (i int PRIMARY KEY, f float UNIQUE, c char(100), +FULLTEXT INDEX(c)) ENGINE = InnoDB; +insert into t values (1, 1.1, 'mysql is now oracle company'), +(2, 2.2, 'innodb is part of mysql'), +(3, 3.3, 'innodb is default storage engine of mysql'); +check table t; +Table Op Msg_type Msg_text +test.t check status OK +SET debug_dbug = '+d,ib_err_trunc_temp_recreate_index'; +truncate table t; +SET debug_dbug = @save_dbug; +check table t; +Table Op Msg_type Msg_text +test.t check status OK +select * from t order by i; +i f c +drop table t; diff --git a/mysql-test/suite/innodb/t/truncate_inject.test b/mysql-test/suite/innodb/t/truncate_inject.test new file mode 100644 index 0000000000000..35e516324bb67 --- /dev/null +++ b/mysql-test/suite/innodb/t/truncate_inject.test @@ -0,0 +1,97 @@ +# This test is based on innodb_zip.wl6501_error_1 in MySQL 5.7. + +--source include/have_innodb.inc +--source include/innodb_row_format.inc +--source include/have_debug.inc + +SET @save_dbug = @@SESSION.debug_dbug; + +call mtr.add_suppression("InnoDB: Flagged corruption of .* in table `test`\\.`t` in TRUNCATE TABLE"); + +--echo # 1. Error in assigning undo logs for truncate action +CREATE TABLE t (i int PRIMARY KEY, f float UNIQUE, c char(100), INDEX ck(c)) +ENGINE = InnoDB; +insert into t values (1, 1.1, 'a'), (2, 2.2, 'b'), (3, 3.3, 'c'); +check table t; +# +SET debug_dbug = '+d,ib_err_trunc_assigning_undo_log'; +--error ER_GET_ERRNO +truncate table t; +SET debug_dbug = @save_dbug; +check table t; +select * from t; + +--echo # 2. Error while preparing for truncate +SET debug_dbug = '+d,ib_err_trunc_preparing_for_truncate'; +--error ER_GET_ERRNO +truncate table t; +SET debug_dbug = @save_dbug; +check table t; +select * from t; + +--echo # 3. Error while dropping/creating indexes +SET debug_dbug = '+d,ib_err_trunc_drop_index'; +--error ER_GET_ERRNO +truncate table t; +SET debug_dbug = @save_dbug; +check table t; +--error ER_TABLE_CORRUPT,ER_GET_ERRNO +select * from t; +drop table t; + +CREATE TABLE t (i int PRIMARY KEY, f float UNIQUE, c char(100), INDEX ck(c)) +ENGINE = InnoDB; +insert into t values (1, 1.1, 'a'), (2, 2.2, 'b'), (3, 3.3, 'c'); +check table t; + +SET debug_dbug = '+d,ib_err_trunc_create_index'; +--error ER_GET_ERRNO +truncate table t; +SET debug_dbug = @save_dbug; +check table t; +--error ER_TABLE_CORRUPT,ER_GET_ERRNO +select * from t; +drop table t; + +CREATE TABLE t (i int PRIMARY KEY, f float UNIQUE, c char(100), INDEX ck(c)) +ENGINE = InnoDB; +insert into t values (1, 1.1, 'a'), (2, 2.2, 'b'), (3, 3.3, 'c'); +check table t; + +SET debug_dbug = '+d,ib_err_trunc_temp_recreate_index'; +truncate table t; +SET debug_dbug = @save_dbug; + +check table t; +select * from t; +drop table t; + +--echo # 4. Error while completing truncate of table involving FTS +CREATE TABLE t (i int PRIMARY KEY, f float UNIQUE, c char(100), +FULLTEXT INDEX(c)) ENGINE = InnoDB; +insert into t values (1, 1.1, 'mysql is now oracle company'), + (2, 2.2, 'innodb is part of mysql'), + (3, 3.3, 'innodb is default storage engine of mysql'); +check table t; +SET debug_dbug = '+d,ib_err_trunc_temp_recreate_index'; +truncate table t; +SET debug_dbug = @save_dbug; + +check table t; +select * from t; +drop table t; + +--echo # 5. Error while updating sys-tables +CREATE TABLE t (i int PRIMARY KEY, f float UNIQUE, c char(100), +FULLTEXT INDEX(c)) ENGINE = InnoDB; +insert into t values (1, 1.1, 'mysql is now oracle company'), + (2, 2.2, 'innodb is part of mysql'), + (3, 3.3, 'innodb is default storage engine of mysql'); +check table t; +SET debug_dbug = '+d,ib_err_trunc_temp_recreate_index'; +truncate table t; +SET debug_dbug = @save_dbug; + +check table t; +select * from t order by i; +drop table t; diff --git a/mysql-test/suite/innodb_zip/include/innodb_wl6501_error.inc b/mysql-test/suite/innodb_zip/include/innodb_wl6501_error.inc deleted file mode 100644 index 424608f251b11..0000000000000 --- a/mysql-test/suite/innodb_zip/include/innodb_wl6501_error.inc +++ /dev/null @@ -1,226 +0,0 @@ -# -# WL#6501: make truncate table atomic -# - ---source include/have_innodb.inc ---source include/have_debug.inc - ---disable_query_log -# suppress expected warnings -call mtr.add_suppression("Unable to truncate FTS index for table"); -call mtr.add_suppression("Unable to assign a new identifier to table " - "`.*`\.`.*` after truncating it"); -call mtr.add_suppression("Flagged corruption of .* in table " - "`.*`\.`.*` in TRUNCATE TABLE"); -call mtr.add_suppression("Parent table of FTS auxiliary table " - ".*\/.* not found"); ---enable_query_log -################################################################################ -# -# Will test following scenarios: -# 1. Error in assigning undo logs for truncate action. -# 2. Error while preparing for truncate. -# 3. Error while dropping/creating indexes. -# 4. Error while completing truncate of table involving FTS. -# 5. Error while updating sys-tables. -# -################################################################################ - -#----------------------------------------------------------------------------- -# -# create test-bed -# -let $per_table = `select @@innodb_file_per_table`; -let $format = `select @@innodb_file_format`; - -eval set global innodb_file_per_table = on; -let $WL6501_TMP_DIR = `select @@tmpdir`; -let $WL6501_DATA_DIR = `select @@datadir`; -set innodb_strict_mode=off; - -#----------------------------------------------------------------------------- -# -# 1. Error in assigning undo logs for truncate action. -# ---echo "1. Error in assigning undo logs for truncate action." -eval set global innodb_file_per_table = $wl6501_file_per_table; ---disable_warnings -eval create $wl6501_temp table t ( - i int, f float, c char, - primary key pk(i), unique findex(f), index ck(c)) - engine = innodb row_format = $wl6501_row_fmt - key_block_size = $wl6501_kbs; ---enable_warnings -insert into t values (1, 1.1, 'a'), (2, 2.2, 'b'), (3, 3.3, 'c'); -select * from t; -#check table t; -# -set session debug = "+d,ib_err_trunc_assigning_undo_log"; ---error ER_GET_ERRNO -truncate table t; -set session debug = "-d,ib_err_trunc_assigning_undo_log"; -# -#check table t; -select * from t; -drop table t; - -#----------------------------------------------------------------------------- -# -# 2. Error while preparing for truncate. -# ---echo "2. Error while preparing for truncate." -eval set global innodb_file_per_table = $wl6501_file_per_table; ---disable_warnings -eval create $wl6501_temp table t ( - i int, f float, c char, - primary key pk(i), unique findex(f), index ck(c)) - engine = innodb row_format = $wl6501_row_fmt - key_block_size = $wl6501_kbs; ---enable_warnings -insert into t values (1, 1.1, 'a'), (2, 2.2, 'b'), (3, 3.3, 'c'); -select * from t; -#check table t; -# -set session debug = "+d,ib_err_trunc_preparing_for_truncate"; ---error ER_GET_ERRNO -truncate table t; -set session debug = "-d,ib_err_trunc_preparing_for_truncate"; -# -#check table t; -select * from t; -drop table t; - -#----------------------------------------------------------------------------- -# -# 3. Error while dropping/creating indexes -# ---echo "3. Error while dropping/creating indexes" -eval set global innodb_file_per_table = $wl6501_file_per_table; ---disable_warnings -eval create $wl6501_temp table t ( - i int, f float, c char, - primary key pk(i), unique findex(f), index ck(c)) - engine = innodb row_format = $wl6501_row_fmt - key_block_size = $wl6501_kbs; ---enable_warnings -insert into t values (1, 1.1, 'a'), (2, 2.2, 'b'), (3, 3.3, 'c'); -select * from t; -#check table t; -# -set session debug = "+d,ib_err_trunc_drop_index"; ---error ER_GET_ERRNO -truncate table t; -set session debug = "-d,ib_err_trunc_drop_index"; -# -#check table t; ---error ER_TABLE_CORRUPT, 1030 -select * from t; -drop table t; -# -# -eval set global innodb_file_per_table = $wl6501_file_per_table; ---disable_warnings -eval create $wl6501_temp table t ( - i int, f float, c char, - primary key pk(i), unique findex(f), index ck(c)) - engine = innodb row_format = $wl6501_row_fmt - key_block_size = $wl6501_kbs; ---enable_warnings -insert into t values (1, 1.1, 'a'), (2, 2.2, 'b'), (3, 3.3, 'c'); -select * from t; -#check table t; -# -set session debug = "+d,ib_err_trunc_create_index"; ---error ER_GET_ERRNO -truncate table t; -set session debug = "-d,ib_err_trunc_create_index"; -# -#check table t; ---error ER_TABLE_CORRUPT, 1030 -select * from t; -drop table t; -# -# -eval set global innodb_file_per_table = $wl6501_file_per_table; ---disable_warnings -eval create temporary table t ( - i int, f float, c char, - primary key pk(i), unique findex(f), index ck(c)) - engine = innodb row_format = $wl6501_row_fmt - key_block_size = $wl6501_kbs; ---enable_warnings -insert into t values (1, 1.1, 'a'), (2, 2.2, 'b'), (3, 3.3, 'c'); -select * from t; -#check table t; -# -set session debug = "+d,ib_err_trunc_temp_recreate_index"; ---error ER_GET_ERRNO -truncate table t; -set session debug = "-d,ib_err_trunc_temp_recreate_index"; -# -#check table t; ---error ER_TABLE_CORRUPT, 1030 -select * from t; -drop table t; - -#----------------------------------------------------------------------------- -# -# 4. Error while completing truncate of table involving FTS. -# ---echo "4. Error while completing truncate of table involving FTS." -eval set global innodb_file_per_table = $wl6501_file_per_table; ---disable_warnings -eval create $wl6501_temp table t (i int, f float, c char(100), - primary key pk(i), index fk(f), fulltext index ck(c)) - engine=innodb row_format=$wl6501_row_fmt - key_block_size=$wl6501_kbs; ---enable_warnings -insert into t values (1, 1.1, 'mysql is now oracle company'), - (2, 2.2, 'innodb is part of mysql'), - (3, 3.3, 'innodb is default storage engine of mysql'); -select * from t; -#check table t; -# -set session debug = "+d,ib_err_trunc_during_fts_trunc"; ---error ER_GET_ERRNO -truncate table t; -set session debug = "-d,ib_err_trunc_during_fts_trunc"; -# -#check table t; ---error ER_TABLE_CORRUPT, 1030 -select * from t; -drop table t; - -#----------------------------------------------------------------------------- -# -# 5. Error while updating sys-tables. -# ---echo "5. Error while updating sys-tables." -eval set global innodb_file_per_table = $wl6501_file_per_table; ---disable_warnings -eval create $wl6501_temp table t (i int, f float, c char(100), - primary key pk(i), index fk(f), fulltext index ck(c)) - engine=innodb row_format=$wl6501_row_fmt - key_block_size=$wl6501_kbs; ---enable_warnings -insert into t values (1, 1.1, 'mysql is now oracle company'), - (2, 2.2, 'innodb is part of mysql'), - (3, 3.3, 'innodb is default storage engine of mysql'); -select * from t order by i; -#check table t; -# -set session debug = "+d,ib_err_trunc_during_sys_table_update"; ---error ER_GET_ERRNO -truncate table t; -set session debug = "-d,ib_err_trunc_during_sys_table_update"; -# -#check table t; ---error ER_TABLE_CORRUPT, 1030 -select * from t order by i; -drop table t; - -#----------------------------------------------------------------------------- -# -# remove test-bed -# -eval set global innodb_file_per_table = $per_table; From 85aea5a12b711c3d94e1bbeb737f267590a0e3aa Mon Sep 17 00:00:00 2001 From: Sergei Petrunia Date: Mon, 15 Jan 2018 16:50:18 +0300 Subject: [PATCH 065/106] Update .result for rocksdb.rpl_row_triggers (not the whole test works yet) --- .../rocksdb/r/rpl_row_triggers.result | 50 +++++++++++++++++-- 1 file changed, 47 insertions(+), 3 deletions(-) diff --git a/storage/rocksdb/mysql-test/rocksdb/r/rpl_row_triggers.result b/storage/rocksdb/mysql-test/rocksdb/r/rpl_row_triggers.result index 1d3cd7db641e0..69acc4a92e8cd 100644 --- a/storage/rocksdb/mysql-test/rocksdb/r/rpl_row_triggers.result +++ b/storage/rocksdb/mysql-test/rocksdb/r/rpl_row_triggers.result @@ -1,12 +1,12 @@ include/master-slave.inc -Warnings: -Note #### Sending passwords in plain text without SSL/TLS is extremely insecure. -Note #### Storing MySQL user name or password information in the master info repository is not secure and is therefore not recommended. Please consider using the USER and PASSWORD connection options for START SLAVE; see the 'START SLAVE Syntax' in the MySQL Manual for more information. [connection master] # Test of row replication with triggers on the slave side +connection master; CREATE TABLE t1 (C1 CHAR(1) primary key, C2 CHAR(1)); SELECT * FROM t1; C1 C2 +connection slave; +connection slave; SET @old_slave_exec_mode= @@global.slave_exec_mode; SET @old_slave_run_triggers_for_rbr= @@global.slave_run_triggers_for_rbr; SET @@global.slave_exec_mode= IDEMPOTENT; @@ -38,8 +38,11 @@ i0 0 i1 0 u0 0 u1 0 +connection master; # INSERT triggers test insert into t1 values ('a','b'); +connection slave; +connection slave; SELECT * FROM t2 order by id; id cnt o n d0 0 @@ -48,8 +51,11 @@ i0 1 a i1 1 a u0 0 u1 0 +connection master; # UPDATE triggers test update t1 set C1= 'd'; +connection slave; +connection slave; SELECT * FROM t2 order by id; id cnt o n d0 0 @@ -58,8 +64,11 @@ i0 1 a i1 1 a u0 1 a d u1 1 a d +connection master; # DELETE triggers test delete from t1 where C1='d'; +connection slave; +connection slave; SELECT * FROM t2 order by id; id cnt o n d0 1 d @@ -78,7 +87,10 @@ i0 2 0 i1 2 0 u0 1 a d u1 1 a d +connection master; insert into t1 values ('0','1'); +connection slave; +connection slave; SELECT * FROM t2 order by id; id cnt o n d0 1 d @@ -90,32 +102,44 @@ u1 2 0 0 # INSERT triggers which cause also DELETE test # (insert duplicate row in table referenced by foreign key) insert into t1 values ('1','1'); +connection master; drop table if exists t1; +connection slave; +connection slave; SET @@global.slave_exec_mode= @old_slave_exec_mode; SET @@global.slave_run_triggers_for_rbr= @old_slave_run_triggers_for_rbr; drop table t2; +connection master; CREATE TABLE t1 (i INT); CREATE TABLE t2 (i INT); +connection slave; SET @old_slave_run_triggers_for_rbr= @@global.slave_run_triggers_for_rbr; SET GLOBAL slave_run_triggers_for_rbr=YES; CREATE TRIGGER tr AFTER INSERT ON t1 FOR EACH ROW INSERT INTO t2 VALUES (new.i); +connection master; BEGIN; INSERT INTO t1 VALUES (1); INSERT INTO t1 VALUES (2); COMMIT; +connection slave; select * from t2; i 1 2 SET @@global.slave_run_triggers_for_rbr= @old_slave_run_triggers_for_rbr; +connection master; drop tables t2,t1; +connection slave; # Triggers on slave do not work if master has some +connection master; CREATE TABLE t1 (C1 CHAR(1) primary key, C2 CHAR(1)); SELECT * FROM t1; C1 C2 create trigger t1_dummy before delete on t1 for each row set @dummy= 1; +connection slave; +connection slave; SET @old_slave_exec_mode= @@global.slave_exec_mode; SET @old_slave_run_triggers_for_rbr= @@global.slave_run_triggers_for_rbr; SET @@global.slave_exec_mode= IDEMPOTENT; @@ -145,8 +169,11 @@ i0 0 i1 0 u0 0 u1 0 +connection master; # INSERT triggers test insert into t1 values ('a','b'); +connection slave; +connection slave; SELECT * FROM t2 order by id; id cnt o n d0 0 @@ -155,8 +182,11 @@ i0 0 i1 0 u0 0 u1 0 +connection master; # UPDATE triggers test update t1 set C1= 'd'; +connection slave; +connection slave; SELECT * FROM t2 order by id; id cnt o n d0 0 @@ -165,8 +195,11 @@ i0 0 i1 0 u0 0 u1 0 +connection master; # DELETE triggers test delete from t1 where C1='d'; +connection slave; +connection slave; SELECT * FROM t2 order by id; id cnt o n d0 0 @@ -185,7 +218,10 @@ i0 1 0 i1 1 0 u0 0 u1 0 +connection master; insert into t1 values ('0','1'); +connection slave; +connection slave; SELECT * FROM t2 order by id; id cnt o n d0 0 @@ -197,22 +233,30 @@ u1 0 # INSERT triggers which cause also DELETE test # (insert duplicate row in table referenced by foreign key) insert into t1 values ('1','1'); +connection master; drop table if exists t1; +connection slave; +connection slave; SET @@global.slave_exec_mode= @old_slave_exec_mode; SET @@global.slave_run_triggers_for_rbr= @old_slave_run_triggers_for_rbr; drop table t2; # # MDEV-5513: Trigger is applied to the rows after first one # +connection master; create table t1 (a int, b int); create table tlog (a int auto_increment primary key); set sql_log_bin=0; create trigger tr1 after insert on t1 for each row insert into tlog values (null); set sql_log_bin=1; +connection slave; +connection slave; set @slave_run_triggers_for_rbr.saved = @@slave_run_triggers_for_rbr; set global slave_run_triggers_for_rbr=1; create trigger tr2 before insert on t1 for each row set new.b = new.a; +connection master; insert into t1 values (1,10),(2,20),(3,30); +connection slave; select * from t1; a b 1 10 From 4794e5b091c384e4d94b3c5f3c1629cba388d9f4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Mon, 15 Jan 2018 16:19:46 +0200 Subject: [PATCH 066/106] Fix a test that always failed on --embedded --- .../suite/innodb/r/innodb-lru-force-no-free-page.result | 5 +++-- .../suite/innodb/t/innodb-lru-force-no-free-page.test | 7 ++++--- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/mysql-test/suite/innodb/r/innodb-lru-force-no-free-page.result b/mysql-test/suite/innodb/r/innodb-lru-force-no-free-page.result index d095f4e32e731..dba01945de306 100644 --- a/mysql-test/suite/innodb/r/innodb-lru-force-no-free-page.result +++ b/mysql-test/suite/innodb/r/innodb-lru-force-no-free-page.result @@ -1,9 +1,10 @@ -call mtr.add_suppression("\\[Warning\\] InnoDB: Difficult to find free blocks in the buffer pool."); +call mtr.add_suppression("InnoDB: Difficult to find free blocks in the buffer pool"); +SET @saved_debug = @@SESSION.debug_dbug; SET SESSION debug_dbug="+d,ib_lru_force_no_free_page"; CREATE TABLE t1 (j LONGBLOB) ENGINE = InnoDB; BEGIN; INSERT INTO t1 VALUES (repeat('abcdefghijklmnopqrstuvwxyz',200)); COMMIT; -SET SESSION debug_dbug=""; +SET debug_dbug = @saved_debug; DROP TABLE t1; FOUND /InnoDB: Difficult to find free blocks / in mysqld.1.err diff --git a/mysql-test/suite/innodb/t/innodb-lru-force-no-free-page.test b/mysql-test/suite/innodb/t/innodb-lru-force-no-free-page.test index e358446eca944..fe04f4b770509 100644 --- a/mysql-test/suite/innodb/t/innodb-lru-force-no-free-page.test +++ b/mysql-test/suite/innodb/t/innodb-lru-force-no-free-page.test @@ -1,8 +1,10 @@ --source include/have_innodb.inc --source include/have_debug.inc +--source include/not_embedded.inc -call mtr.add_suppression("\\[Warning\\] InnoDB: Difficult to find free blocks in the buffer pool."); +call mtr.add_suppression("InnoDB: Difficult to find free blocks in the buffer pool"); +SET @saved_debug = @@SESSION.debug_dbug; SET SESSION debug_dbug="+d,ib_lru_force_no_free_page"; CREATE TABLE t1 (j LONGBLOB) ENGINE = InnoDB; @@ -10,7 +12,7 @@ BEGIN; INSERT INTO t1 VALUES (repeat('abcdefghijklmnopqrstuvwxyz',200)); COMMIT; -SET SESSION debug_dbug=""; +SET debug_dbug = @saved_debug; DROP TABLE t1; @@ -21,4 +23,3 @@ let SEARCH_RANGE= -50000; let SEARCH_FILE= $MYSQLTEST_VARDIR/log/mysqld.1.err; --let SEARCH_PATTERN=InnoDB: Difficult to find free blocks --source include/search_pattern_in_file.inc - From 9c6fc7b644da912a0e68c720f2234a8554bbfeb9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Mon, 15 Jan 2018 16:38:16 +0200 Subject: [PATCH 067/106] Fix -Wsign-compare introduced by Compilation speed (#546) --- sql/key.cc | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/sql/key.cc b/sql/key.cc index af473d85847bb..0040647b5c793 100644 --- a/sql/key.cc +++ b/sql/key.cc @@ -1,4 +1,5 @@ /* Copyright (c) 2000, 2010, Oracle and/or its affiliates. All rights reserved. + Copyright (c) 2018, MariaDB This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by @@ -132,7 +133,7 @@ void key_copy(uchar *to_key, uchar *from_record, KEY *key_info, Don't copy data for null values The -1 below is to subtract the null byte which is already handled */ - length= MY_MIN(key_length, key_part->store_length-1); + length= MY_MIN(key_length, uint(key_part->store_length)-1); if (with_zerofill) bzero((char*) to_key, length); continue; @@ -202,7 +203,7 @@ void key_restore(uchar *to_record, const uchar *from_key, KEY *key_info, Don't copy data for null bytes The -1 below is to subtract the null byte which is already handled */ - length= MY_MIN(key_length, key_part->store_length-1); + length= MY_MIN(key_length, uint(key_part->store_length)-1); continue; } } From abbce9ed560d16bd76a84b30d4f4f731f0b56dc0 Mon Sep 17 00:00:00 2001 From: Monty Date: Fri, 22 Dec 2017 15:03:24 +0200 Subject: [PATCH 068/106] Fixed compiler warnings in guess_malloc_library --- mysys/guess_malloc_library.c | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/mysys/guess_malloc_library.c b/mysys/guess_malloc_library.c index 2e640757e1112..70e3c7ca0f291 100644 --- a/mysys/guess_malloc_library.c +++ b/mysys/guess_malloc_library.c @@ -20,8 +20,13 @@ #include #include +typedef const char* (*tc_version_type)(int*, int*, const char**); +typedef int (*mallctl_type)(const char*, void*, size_t*, void*, size_t); + char *guess_malloc_library() { + tc_version_type tc_version_func; + mallctl_type mallctl_func; #ifndef HAVE_DLOPEN return (char*) MALLOC_LIBRARY; #else @@ -33,9 +38,7 @@ char *guess_malloc_library() } /* tcmalloc */ - typedef const char* (*tc_version_type)(int*, int*, const char**); - tc_version_type tc_version_func = - (tc_version_type) dlsym(RTLD_DEFAULT, "tc_version"); + tc_version_func= (tc_version_type) dlsym(RTLD_DEFAULT, "tc_version"); if (tc_version_func) { int major, minor; @@ -45,9 +48,7 @@ char *guess_malloc_library() } /* jemalloc */ - typedef int (*mallctl_type)(const char*, void*, size_t*, void*, size_t); - mallctl_type mallctl_func = - (mallctl_type) dlsym(RTLD_DEFAULT, "mallctl"); + mallctl_func= (mallctl_type) dlsym(RTLD_DEFAULT, "mallctl"); if (mallctl_func) { char *ver; From 3fdd390791ac91ecdc545fd1686a7ed43e8f1945 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Mon, 15 Jan 2018 19:02:38 +0200 Subject: [PATCH 069/106] MDEV-14441 InnoDB hangs when setting innodb_adaptive_hash_index=OFF during UPDATE This race condition is a regression caused by MDEV-12121. btr_cur_update_in_place(): Determine block->index!=NULL only once in order to determine whether an adaptive hash index bucket needs to be exclusively locked and unlocked. If we evaluated block->index multiple times, and the adaptive hash index was disabled before we locked the adaptive hash index, then we would never release the adaptive hash index bucket latch, which would eventually lead to InnoDB hanging. --- storage/innobase/btr/btr0cur.cc | 48 ++++++++++++++++++--------------- 1 file changed, 26 insertions(+), 22 deletions(-) diff --git a/storage/innobase/btr/btr0cur.cc b/storage/innobase/btr/btr0cur.cc index 6acab88128846..2830a38fa4a43 100644 --- a/storage/innobase/btr/btr0cur.cc +++ b/storage/innobase/btr/btr0cur.cc @@ -3,7 +3,7 @@ Copyright (c) 1994, 2016, Oracle and/or its affiliates. All Rights Reserved. Copyright (c) 2008, Google Inc. Copyright (c) 2012, Facebook Inc. -Copyright (c) 2015, 2017, MariaDB Corporation. +Copyright (c) 2015, 2018, MariaDB Corporation. Portions of this file contain modifications contributed and copyrighted by Google, Inc. Those modifications are gratefully acknowledged and are described @@ -3692,34 +3692,38 @@ btr_cur_update_in_place( || row_get_rec_trx_id(rec, index, offsets)); #ifdef BTR_CUR_HASH_ADAPT - if (block->index) { - /* TO DO: Can we skip this if none of the fields - index->search_info->curr_n_fields - are being updated? */ - - /* The function row_upd_changes_ord_field_binary works only - if the update vector was built for a clustered index, we must - NOT call it if index is secondary */ - - if (!dict_index_is_clust(index) - || row_upd_changes_ord_field_binary(index, update, thr, - NULL, NULL)) { - - /* Remove possible hash index pointer to this record */ - btr_search_update_hash_on_delete(cursor); + { + rw_lock_t* ahi_latch = block->index + ? btr_get_search_latch(block->index) : NULL; + if (ahi_latch) { + /* TO DO: Can we skip this if none of the fields + index->search_info->curr_n_fields + are being updated? */ + + /* The function row_upd_changes_ord_field_binary + does not work on a secondary index. */ + + if (!dict_index_is_clust(index) + || row_upd_changes_ord_field_binary( + index, update, thr, NULL, NULL)) { + + /* Remove possible hash index pointer + to this record */ + btr_search_update_hash_on_delete(cursor); + } } - btr_search_x_lock(index); - } + rw_lock_x_lock(ahi_latch); - assert_block_ahi_valid(block); + assert_block_ahi_valid(block); #endif /* BTR_CUR_HASH_ADAPT */ - row_upd_rec_in_place(rec, index, offsets, update, page_zip); + row_upd_rec_in_place(rec, index, offsets, update, page_zip); #ifdef BTR_CUR_HASH_ADAPT - if (block->index) { - btr_search_x_unlock(index); + if (ahi_latch) { + rw_lock_x_unlock(ahi_latch); + } } #endif /* BTR_CUR_HASH_ADAPT */ From 458e33cfbc53d5e3e6e2857688ae98efcef1faf1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Mon, 15 Jan 2018 17:06:26 +0200 Subject: [PATCH 070/106] MDEV-14441 Deadlock due to InnoDB adaptive hash index This is not fixing the reported problem, but a potential problem that was introduced in MDEV-11369. row_sel_try_search_shortcut(), row_sel_try_search_shortcut_for_mysql(): When an adaptive hash index search lands on top of rec_is_default_row(), we must skip the candidate and perform a normal search. This is because the adaptive hash index latch only protects the record from being deleted but does not prevent concurrent inserts into the page. Therefore, it is not safe to dereference the next-record pointer. --- storage/innobase/row/row0sel.cc | 24 +++--------------------- 1 file changed, 3 insertions(+), 21 deletions(-) diff --git a/storage/innobase/row/row0sel.cc b/storage/innobase/row/row0sel.cc index 1395faaad9668..083ffba33f860 100644 --- a/storage/innobase/row/row0sel.cc +++ b/storage/innobase/row/row0sel.cc @@ -2,7 +2,7 @@ Copyright (c) 1997, 2017, Oracle and/or its affiliates. All Rights Reserved. Copyright (c) 2008, Google Inc. -Copyright (c) 2015, 2017, MariaDB Corporation. +Copyright (c) 2015, 2018, MariaDB Corporation. Portions of this file contain modifications contributed and copyrighted by Google, Inc. Those modifications are gratefully acknowledged and are described @@ -1487,20 +1487,11 @@ row_sel_try_search_shortcut( rec = btr_pcur_get_rec(&(plan->pcur)); - if (!page_rec_is_user_rec(rec)) { + if (!page_rec_is_user_rec(rec) || rec_is_default_row(rec, index)) { return(SEL_RETRY); } - if (rec_is_default_row(rec, index)) { - /* Skip the 'default row' pseudo-record. */ - if (!btr_pcur_move_to_next_user_rec(&plan->pcur, mtr)) { - return(SEL_RETRY); - } - - rec = btr_pcur_get_rec(&plan->pcur); - } - ut_ad(plan->mode == PAGE_CUR_GE); /* As the cursor is now placed on a user record after a search with @@ -3908,20 +3899,11 @@ row_sel_try_search_shortcut_for_mysql( BTR_SEARCH_LEAF, pcur, RW_S_LATCH, mtr); rec = btr_pcur_get_rec(pcur); - if (!page_rec_is_user_rec(rec)) { + if (!page_rec_is_user_rec(rec) || rec_is_default_row(rec, index)) { return(SEL_RETRY); } - if (rec_is_default_row(rec, index)) { - /* Skip the 'default row' pseudo-record. */ - if (!btr_pcur_move_to_next_user_rec(pcur, mtr)) { - return(SEL_RETRY); - } - - rec = btr_pcur_get_rec(pcur); - } - /* As the cursor is now placed on a user record after a search with the mode PAGE_CUR_GE, the up_match field in the cursor tells how many fields in the user record matched to the search tuple */ From 12f804acfa45dac5af04dd7da37670c50f4d8a90 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Mon, 15 Jan 2018 18:08:59 +0200 Subject: [PATCH 071/106] MDEV-14441 Deadlock due to InnoDB adaptive hash index This is mere code clean-up; the reported problem was already fixed in commit 3fdd390791ac91ecdc545fd1686a7ed43e8f1945. row_sel(): Remove the variable search_latch_locked. row_sel_try_search_shortcut(): Remove the parameter search_latch_locked, which was always passed as nonzero. row_sel_try_search_shortcut(), row_sel_try_search_shortcut_for_mysql(): Do not expect the caller to acquire the AHI latch. Instead, acquire and release it inside this function. row_search_mvcc(): Remove a bogus condition on mysql_n_tables_locked. When the btr_search_latch was split into an array of latches in MySQL 5.7.8 as part of the Oracle Bug#20985298 fix, the "caching" of the latch across storage engine API calls was removed, and thus it is unnecessary to avoid adaptive hash index searches during INSERT...SELECT. --- storage/innobase/row/row0sel.cc | 148 ++++++++------------------------ 1 file changed, 36 insertions(+), 112 deletions(-) diff --git a/storage/innobase/row/row0sel.cc b/storage/innobase/row/row0sel.cc index 083ffba33f860..c0d56bf0bfa2a 100644 --- a/storage/innobase/row/row0sel.cc +++ b/storage/innobase/row/row0sel.cc @@ -1462,33 +1462,24 @@ row_sel_try_search_shortcut( sel_node_t* node, /*!< in: select node for a consistent read */ plan_t* plan, /*!< in: plan for a unique search in clustered index */ - ibool search_latch_locked, - /*!< in: whether the search holds latch on - search system. */ mtr_t* mtr) /*!< in: mtr */ { - dict_index_t* index; - rec_t* rec; - mem_heap_t* heap = NULL; - ulint offsets_[REC_OFFS_NORMAL_SIZE]; - ulint* offsets = offsets_; - ulint ret; - rec_offs_init(offsets_); - - index = plan->index; + dict_index_t* index = plan->index; ut_ad(node->read_view); ut_ad(plan->unique_search); ut_ad(!plan->must_get_clust); - ut_ad(!search_latch_locked - || rw_lock_own(btr_get_search_latch(index), RW_LOCK_S)); - row_sel_open_pcur(plan, search_latch_locked, mtr); + rw_lock_t* ahi_latch = btr_get_search_latch(index); + rw_lock_s_lock(ahi_latch); - rec = btr_pcur_get_rec(&(plan->pcur)); + row_sel_open_pcur(plan, TRUE, mtr); - if (!page_rec_is_user_rec(rec) || rec_is_default_row(rec, index)) { + const rec_t* rec = btr_pcur_get_rec(&(plan->pcur)); + if (!page_rec_is_user_rec(rec) || rec_is_default_row(rec, index)) { +retry: + rw_lock_s_unlock(ahi_latch); return(SEL_RETRY); } @@ -1499,36 +1490,34 @@ row_sel_try_search_shortcut( fields in the user record matched to the search tuple */ if (btr_pcur_get_up_match(&(plan->pcur)) < plan->n_exact_match) { - +exhausted: + rw_lock_s_unlock(ahi_latch); return(SEL_EXHAUSTED); } /* This is a non-locking consistent read: if necessary, fetch a previous version of the record */ + mem_heap_t* heap = NULL; + ulint offsets_[REC_OFFS_NORMAL_SIZE]; + ulint* offsets = offsets_; + rec_offs_init(offsets_); offsets = rec_get_offsets(rec, index, offsets, true, ULINT_UNDEFINED, &heap); if (dict_index_is_clust(index)) { if (!lock_clust_rec_cons_read_sees(rec, index, offsets, node->read_view)) { - ret = SEL_RETRY; - goto func_exit; + goto retry; } } else if (!srv_read_only_mode && !lock_sec_rec_cons_read_sees( rec, index, node->read_view)) { - - ret = SEL_RETRY; - goto func_exit; + goto retry; } - /* Test the deleted flag. */ - if (rec_get_deleted_flag(rec, dict_table_is_comp(plan->table))) { - - ret = SEL_EXHAUSTED; - goto func_exit; + goto exhausted; } /* Fetch the columns needed in test conditions. The index @@ -1542,20 +1531,18 @@ row_sel_try_search_shortcut( /* Test the rest of search conditions */ if (!row_sel_test_other_conds(plan)) { - - ret = SEL_EXHAUSTED; - goto func_exit; + goto exhausted; } ut_ad(plan->pcur.latch_mode == BTR_SEARCH_LEAF); plan->n_rows_fetched++; - ret = SEL_FOUND; -func_exit: + rw_lock_s_unlock(ahi_latch); + if (UNIV_LIKELY_NULL(heap)) { mem_heap_free(heap); } - return(ret); + return(SEL_FOUND); } #endif /* BTR_CUR_HASH_ADAPT */ @@ -1602,12 +1589,6 @@ row_sel( ut_ad(thr->run_node == node); -#ifdef BTR_CUR_HASH_ADAPT - ibool search_latch_locked = FALSE; -#else /* BTR_CUR_HASH_ADAPT */ -# define search_latch_locked false -#endif /* BTR_CUR_HASH_ADAPT */ - if (node->read_view) { /* In consistent reads, we try to do with the hash index and not to use the buffer page get. This is to reduce memory bus @@ -1656,33 +1637,14 @@ row_sel( #ifdef BTR_CUR_HASH_ADAPT if (consistent_read && plan->unique_search && !plan->pcur_is_open && !plan->must_get_clust) { - if (!search_latch_locked) { - btr_search_s_lock(index); - - search_latch_locked = TRUE; - } else if (rw_lock_get_writer(btr_get_search_latch(index)) - == RW_LOCK_X_WAIT) { - - /* There is an x-latch request waiting: release the - s-latch for a moment; as an s-latch here is often - kept for some 10 searches before being released, - a waiting x-latch request would block other threads - from acquiring an s-latch for a long time, lowering - performance significantly in multiprocessors. */ - - btr_search_s_unlock(index); - btr_search_s_lock(index); - } - - switch (row_sel_try_search_shortcut(node, plan, - search_latch_locked, - &mtr)) { + switch (row_sel_try_search_shortcut(node, plan, &mtr)) { case SEL_FOUND: goto next_table; case SEL_EXHAUSTED: goto table_exhausted; default: ut_ad(0); + /* fall through */ case SEL_RETRY: break; } @@ -1692,19 +1654,13 @@ row_sel( mtr_commit(&mtr); mtr_start(&mtr); } - - if (search_latch_locked) { - btr_search_s_unlock(index); - - search_latch_locked = FALSE; - } #endif /* BTR_CUR_HASH_ADAPT */ if (!plan->pcur_is_open) { /* Evaluate the expressions to build the search tuple and open the cursor */ - row_sel_open_pcur(plan, search_latch_locked, &mtr); + row_sel_open_pcur(plan, FALSE, &mtr); cursor_just_opened = TRUE; @@ -2107,8 +2063,6 @@ row_sel( } next_rec: - ut_ad(!search_latch_locked); - if (mtr_has_extra_clust_latch) { /* We must commit &mtr if we are moving to the next @@ -2146,8 +2100,6 @@ row_sel( plan->cursor_at_end = TRUE; } else { - ut_ad(!search_latch_locked); - plan->stored_cursor_rec_processed = TRUE; btr_pcur_store_position(&(plan->pcur), &mtr); @@ -2238,8 +2190,6 @@ row_sel( inserted new records which should have appeared in the result set, which would result in the phantom problem. */ - ut_ad(!search_latch_locked); - plan->stored_cursor_rec_processed = FALSE; btr_pcur_store_position(&(plan->pcur), &mtr); @@ -2256,7 +2206,6 @@ row_sel( plan->stored_cursor_rec_processed = TRUE; - ut_ad(!search_latch_locked); btr_pcur_store_position(&(plan->pcur), &mtr); mtr_commit(&mtr); @@ -2270,7 +2219,6 @@ row_sel( /* See the note at stop_for_a_while: the same holds for this case */ ut_ad(!btr_pcur_is_before_first_on_page(&plan->pcur) || !node->asc); - ut_ad(!search_latch_locked); plan->stored_cursor_rec_processed = FALSE; btr_pcur_store_position(&(plan->pcur), &mtr); @@ -2278,11 +2226,6 @@ row_sel( mtr_commit(&mtr); func_exit: -#ifdef BTR_CUR_HASH_ADAPT - if (search_latch_locked) { - btr_search_s_unlock(index); - } -#endif /* BTR_CUR_HASH_ADAPT */ ut_ad(!sync_check_iterate(dict_sync_check())); if (heap != NULL) { @@ -3895,12 +3838,15 @@ row_sel_try_search_shortcut_for_mysql( ut_ad(dict_index_is_clust(index)); ut_ad(!prebuilt->templ_contains_blob); + rw_lock_t* ahi_latch = btr_get_search_latch(index); + rw_lock_s_lock(ahi_latch); btr_pcur_open_with_no_init(index, search_tuple, PAGE_CUR_GE, BTR_SEARCH_LEAF, pcur, RW_S_LATCH, mtr); rec = btr_pcur_get_rec(pcur); if (!page_rec_is_user_rec(rec) || rec_is_default_row(rec, index)) { - +retry: + rw_lock_s_unlock(ahi_latch); return(SEL_RETRY); } @@ -3909,7 +3855,8 @@ row_sel_try_search_shortcut_for_mysql( fields in the user record matched to the search tuple */ if (btr_pcur_get_up_match(pcur) < dtuple_get_n_fields(search_tuple)) { - +exhausted: + rw_lock_s_unlock(ahi_latch); return(SEL_EXHAUSTED); } @@ -3921,20 +3868,19 @@ row_sel_try_search_shortcut_for_mysql( if (!lock_clust_rec_cons_read_sees( rec, index, *offsets, trx_get_read_view(trx))) { - - return(SEL_RETRY); + goto retry; } if (rec_get_deleted_flag(rec, dict_table_is_comp(index->table))) { /* In delete-marked records, DB_TRX_ID must always refer to an existing undo log record. */ ut_ad(row_get_rec_trx_id(rec, index, *offsets)); - - return(SEL_EXHAUSTED); + goto exhausted; } *out_rec = rec; + rw_lock_s_unlock(ahi_latch); return(SEL_FOUND); } #endif /* BTR_CUR_HASH_ADAPT */ @@ -4323,24 +4269,14 @@ row_search_mvcc( mode = PAGE_CUR_GE; - if (trx->mysql_n_tables_locked == 0 - && prebuilt->select_lock_type == LOCK_NONE + if (prebuilt->select_lock_type == LOCK_NONE && trx->isolation_level > TRX_ISO_READ_UNCOMMITTED && MVCC::is_view_active(trx->read_view)) { /* This is a SELECT query done as a consistent read, and the read view has already been allocated: let us try a search shortcut through the hash - index. - NOTE that we must also test that - mysql_n_tables_locked == 0, because this might - also be INSERT INTO ... SELECT ... or - CREATE TABLE ... SELECT ... . Our algorithm is - NOT prepared to inserts interleaved with the SELECT, - and if we try that, we can deadlock on the adaptive - hash index semaphore! */ - - rw_lock_s_lock(btr_get_search_latch(index)); + index. */ switch (row_sel_try_search_shortcut_for_mysql( &rec, prebuilt, &offsets, &heap, @@ -4388,27 +4324,17 @@ row_search_mvcc( shortcut_match: mtr_commit(&mtr); - /* NOTE that we do NOT store the cursor position */ - err = DB_SUCCESS; - - rw_lock_s_unlock(btr_get_search_latch(index)); - goto func_exit; case SEL_EXHAUSTED: shortcut_mismatch: mtr_commit(&mtr); - - err = DB_RECORD_NOT_FOUND; - - rw_lock_s_unlock(btr_get_search_latch(index)); - /* NOTE that we do NOT store the cursor position */ - + err = DB_RECORD_NOT_FOUND; goto func_exit; case SEL_RETRY: @@ -4420,8 +4346,6 @@ row_search_mvcc( mtr_commit(&mtr); mtr_start(&mtr); - - rw_lock_s_unlock(btr_get_search_latch(index)); } } #endif /* BTR_CUR_HASH_ADAPT */ From 542ad0fa3f1e72a83190f8ba95c7ad616213fc9d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Mon, 15 Jan 2018 17:58:16 +0200 Subject: [PATCH 072/106] btr_search_check_guess(): Remove the parameter 'mode' Also, use 32-bit native reads to read the 32-bit aligned FIL_PAGE_PREV and FIL_PAGE_NEXT reads, to compare them to the byte order agnostic pattern FIL_NULL (0xffffffff). --- storage/innobase/btr/btr0sea.cc | 30 ++++++++++++------------------ 1 file changed, 12 insertions(+), 18 deletions(-) diff --git a/storage/innobase/btr/btr0sea.cc b/storage/innobase/btr/btr0sea.cc index 6613e4c59f89e..c154af9ecccd2 100644 --- a/storage/innobase/btr/btr0sea.cc +++ b/storage/innobase/btr/btr0sea.cc @@ -2,7 +2,7 @@ Copyright (c) 1996, 2016, Oracle and/or its affiliates. All Rights Reserved. Copyright (c) 2008, Google Inc. -Copyright (c) 2017, MariaDB Corporation. +Copyright (c) 2017, 2018, MariaDB Corporation. Portions of this file contain modifications contributed and copyrighted by Google, Inc. Those modifications are gratefully acknowledged and are described @@ -791,16 +791,14 @@ TRUE, then cursor->up_match and cursor->low_match both have sensible values. previous record to check our guess! @param[in] tuple data tuple @param[in] mode PAGE_CUR_L, PAGE_CUR_LE, PAGE_CUR_G, PAGE_CUR_GE -@param[in] mtr mini transaction -@return TRUE if success */ +@return whether a match was found */ static -ibool +bool btr_search_check_guess( btr_cur_t* cursor, - ibool can_only_compare_to_cursor_rec, + bool can_only_compare_to_cursor_rec, const dtuple_t* tuple, - ulint mode, - mtr_t* mtr) + ulint mode) { rec_t* rec; ulint n_unique; @@ -862,14 +860,13 @@ btr_search_check_guess( match = 0; if ((mode == PAGE_CUR_G) || (mode == PAGE_CUR_GE)) { - rec_t* prev_rec; - ut_ad(!page_rec_is_infimum(rec)); - prev_rec = page_rec_get_prev(rec); + const rec_t* prev_rec = page_rec_get_prev(rec); if (page_rec_is_infimum(prev_rec)) { - success = btr_page_get_prev(page_align(prev_rec), mtr) + success = *reinterpret_cast( + page_align(prev_rec) + FIL_PAGE_PREV) == FIL_NULL; goto exit_func; @@ -884,17 +881,14 @@ btr_search_check_guess( } else { success = cmp >= 0; } - - goto exit_func; } else { - rec_t* next_rec; - ut_ad(!page_rec_is_supremum(rec)); - next_rec = page_rec_get_next(rec); + const rec_t* next_rec = page_rec_get_next(rec); if (page_rec_is_supremum(next_rec)) { - if (btr_page_get_next(page_align(next_rec), mtr) + if (*reinterpret_cast( + page_align(next_rec) + FIL_PAGE_NEXT) == FIL_NULL) { cursor->up_match = 0; @@ -1094,7 +1088,7 @@ btr_search_guess_on_hash( if (index_id != btr_page_get_index_id(block->frame) || !btr_search_check_guess(cursor, has_search_latch, - tuple, mode, mtr)) { + tuple, mode)) { if (!has_search_latch) { btr_leaf_page_release(block, latch_mode, mtr); From 4beb699a361cd28c605e94029330fd3701769f33 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Mon, 15 Jan 2018 17:35:34 +0200 Subject: [PATCH 073/106] MDEV-14952 Avoid repeated calls to btr_get_search_latch() btr_cur_search_to_nth_level(), row_sel(): Do not bother to yield to waiting exclusive lock requests on the adaptive hash index latch. When the btr_search_latch was split into an array of latches in MySQL 5.7.8 as part of the Oracle Bug#20985298 fix, the "caching" of the latch across storage engine API calls was removed. Thus, X-lock requests should have a good chance of becoming served, and starvation should not be possible. btr_search_guess_on_hash(): Clean up a debug assertion. --- storage/innobase/btr/btr0cur.cc | 4 +--- storage/innobase/btr/btr0sea.cc | 3 +-- 2 files changed, 2 insertions(+), 5 deletions(-) diff --git a/storage/innobase/btr/btr0cur.cc b/storage/innobase/btr/btr0cur.cc index 74940123d1a37..b6b3cb041ea8b 100644 --- a/storage/innobase/btr/btr0cur.cc +++ b/storage/innobase/btr/btr0cur.cc @@ -3,7 +3,7 @@ Copyright (c) 1994, 2016, Oracle and/or its affiliates. All Rights Reserved. Copyright (c) 2008, Google Inc. Copyright (c) 2012, Facebook Inc. -Copyright (c) 2015, 2017, MariaDB Corporation. +Copyright (c) 2015, 2018, MariaDB Corporation. Portions of this file contain modifications contributed and copyrighted by Google, Inc. Those modifications are gratefully acknowledged and are described @@ -1129,8 +1129,6 @@ btr_cur_search_to_nth_level( && btr_search_enabled && !modify_external && !(tuple->info_bits & REC_INFO_MIN_REC_FLAG) - && rw_lock_get_writer(btr_get_search_latch(index)) - == RW_LOCK_NOT_LOCKED && btr_search_guess_on_hash(index, info, tuple, mode, latch_mode, cursor, has_search_latch, mtr)) { diff --git a/storage/innobase/btr/btr0sea.cc b/storage/innobase/btr/btr0sea.cc index c154af9ecccd2..eeb43f513de3d 100644 --- a/storage/innobase/btr/btr0sea.cc +++ b/storage/innobase/btr/btr0sea.cc @@ -1021,8 +1021,7 @@ btr_search_guess_on_hash( } } - ut_ad(rw_lock_get_writer(btr_get_search_latch(index)) != RW_LOCK_X); - ut_ad(rw_lock_get_reader_count(btr_get_search_latch(index)) > 0); + ut_ad(rw_lock_own(btr_get_search_latch(index), RW_LOCK_S)); rec = (rec_t*) ha_search_and_get_data( btr_get_search_table(index), fold); From 0664d633e415d04a90f343dbaf954ae756a474b3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Mon, 15 Jan 2018 19:51:09 +0200 Subject: [PATCH 074/106] MDEV-14952 Avoid repeated calls to btr_get_search_latch() btr_cur_search_to_nth_level(), btr_search_guess_on_hash(), btr_pcur_open_with_no_init_func(), row_sel_open_pcur(): Replace the parameter has_search_latch with the ahi_latch (passed as NULL if the caller does not hold the latch). btr_search_update_hash_node_on_insert(), btr_search_update_hash_on_insert(), btr_search_build_page_hash_index(): Add the parameter ahi_latch. btr_search_x_lock(), btr_search_x_unlock(), btr_search_s_lock(), btr_search_s_unlock(): Remove. --- storage/innobase/btr/btr0cur.cc | 53 +++-- storage/innobase/btr/btr0pcur.cc | 6 +- storage/innobase/btr/btr0sea.cc | 314 +++++++++++++-------------- storage/innobase/include/btr0cur.h | 29 ++- storage/innobase/include/btr0pcur.h | 27 ++- storage/innobase/include/btr0pcur.ic | 34 +-- storage/innobase/include/btr0sea.h | 55 ++--- storage/innobase/include/btr0sea.ic | 47 +--- storage/innobase/row/row0sel.cc | 30 ++- 9 files changed, 270 insertions(+), 325 deletions(-) diff --git a/storage/innobase/btr/btr0cur.cc b/storage/innobase/btr/btr0cur.cc index b6b3cb041ea8b..cda0636146f00 100644 --- a/storage/innobase/btr/btr0cur.cc +++ b/storage/innobase/btr/btr0cur.cc @@ -919,8 +919,7 @@ search tuple should be performed in the B-tree. InnoDB does an insert immediately after the cursor. Thus, the cursor may end up on a user record, or on a page infimum record. */ dberr_t -btr_cur_search_to_nth_level( -/*========================*/ +btr_cur_search_to_nth_level_func( dict_index_t* index, /*!< in: index */ ulint level, /*!< in: the tree level of search */ const dtuple_t* tuple, /*!< in: data tuple; NOTE: n_fields_cmp in @@ -935,17 +934,16 @@ btr_cur_search_to_nth_level( cursor->left_block is used to store a pointer to the left neighbor page, in the cases BTR_SEARCH_PREV and BTR_MODIFY_PREV; - NOTE that if has_search_latch - is != 0, we maybe do not have a latch set - on the cursor page, we assume - the caller uses his search latch - to protect the record! */ + NOTE that if ahi_latch, we might not have a + cursor page latch, we assume that ahi_latch + protects the record! */ btr_cur_t* cursor, /*!< in/out: tree cursor; the cursor page is s- or x-latched, but see also above! */ - ulint has_search_latch, - /*!< in: info on the latch mode the - caller currently has on search system: - RW_S_LATCH, or 0 */ +#ifdef BTR_CUR_HASH_ADAPT + rw_lock_t* ahi_latch, + /*!< in: currently held btr_search_latch + (in RW_S_LATCH mode), or NULL */ +#endif /* BTR_CUR_HASH_ADAPT */ const char* file, /*!< in: file name */ unsigned line, /*!< in: line where called */ mtr_t* mtr, /*!< in: mtr */ @@ -1123,7 +1121,7 @@ btr_cur_search_to_nth_level( && mode != PAGE_CUR_LE_OR_EXTENDS # endif /* PAGE_CUR_LE_OR_EXTENDS */ && !dict_index_is_spatial(index) - /* If !has_search_latch, we do a dirty read of + /* If !ahi_latch, we do a dirty read of btr_search_enabled below, and btr_search_guess_on_hash() will have to check it again. */ && btr_search_enabled @@ -1131,7 +1129,7 @@ btr_cur_search_to_nth_level( && !(tuple->info_bits & REC_INFO_MIN_REC_FLAG) && btr_search_guess_on_hash(index, info, tuple, mode, latch_mode, cursor, - has_search_latch, mtr)) { + ahi_latch, mtr)) { /* Search using the hash index succeeded */ @@ -1152,10 +1150,12 @@ btr_cur_search_to_nth_level( /* If the hash search did not succeed, do binary search down the tree */ - if (has_search_latch) { +#ifdef BTR_CUR_HASH_ADAPT + if (ahi_latch) { /* Release possible search latch to obey latching order */ - btr_search_s_unlock(index); + rw_lock_s_unlock(ahi_latch); } +#endif /* BTR_CUR_HASH_ADAPT */ /* Store the position of the tree latch we push to mtr so that we know how to release it when we have latched leaf node(s) */ @@ -2222,15 +2222,17 @@ btr_cur_search_to_nth_level( ut_free(prev_tree_savepoints); } - if (has_search_latch) { - btr_search_s_lock(index); - } - if (mbr_adj) { /* remember that we will need to adjust parent MBR */ cursor->rtr_info->mbr_adj = true; } +#ifdef BTR_CUR_HASH_ADAPT + if (ahi_latch) { + rw_lock_s_lock(ahi_latch); + } +#endif /* BTR_CUR_HASH_ADAPT */ + DBUG_RETURN(err); } @@ -3300,10 +3302,14 @@ btr_cur_optimistic_insert( ut_ad(entry->info_bits == REC_INFO_DEFAULT_ROW); ut_ad(index->is_instant()); ut_ad(flags == BTR_NO_LOCKING_FLAG); - } else if (!reorg && cursor->flag == BTR_CUR_HASH) { - btr_search_update_hash_node_on_insert(cursor); } else { - btr_search_update_hash_on_insert(cursor); + rw_lock_t* ahi_latch = btr_get_search_latch(index); + if (!reorg && cursor->flag == BTR_CUR_HASH) { + btr_search_update_hash_node_on_insert( + cursor, ahi_latch); + } else { + btr_search_update_hash_on_insert(cursor, ahi_latch); + } } #endif /* BTR_CUR_HASH_ADAPT */ @@ -3509,7 +3515,8 @@ btr_cur_pessimistic_insert( ut_ad((flags & ~BTR_KEEP_IBUF_BITMAP) == BTR_NO_LOCKING_FLAG); } else { - btr_search_update_hash_on_insert(cursor); + btr_search_update_hash_on_insert( + cursor, btr_get_search_latch(index)); } #endif /* BTR_CUR_HASH_ADAPT */ if (inherit && !(flags & BTR_NO_LOCKING_FLAG)) { diff --git a/storage/innobase/btr/btr0pcur.cc b/storage/innobase/btr/btr0pcur.cc index 2b85c764a3ba5..56f6da81dedf1 100644 --- a/storage/innobase/btr/btr0pcur.cc +++ b/storage/innobase/btr/btr0pcur.cc @@ -353,7 +353,11 @@ btr_pcur_restore_position_func( } btr_pcur_open_with_no_init_func(index, tuple, mode, latch_mode, - cursor, 0, file, line, mtr); + cursor, +#ifdef BTR_CUR_HASH_ADAPT + NULL, +#endif /* BTR_CUR_HASH_ADAPT */ + file, line, mtr); /* Restore the old search mode */ cursor->search_mode = old_mode; diff --git a/storage/innobase/btr/btr0sea.cc b/storage/innobase/btr/btr0sea.cc index eeb43f513de3d..1a70dce1c7b5f 100644 --- a/storage/innobase/btr/btr0sea.cc +++ b/storage/innobase/btr/btr0sea.cc @@ -48,7 +48,7 @@ Search system is protected by array of latches. */ char btr_search_enabled = true; /** Number of adaptive hash index partition. */ -ulong btr_ahi_parts = 8; +ulong btr_ahi_parts; #ifdef UNIV_SEARCH_PERF_STAT /** Number of successful adaptive hash index lookups */ @@ -177,23 +177,6 @@ btr_search_get_n_fields( return(btr_search_get_n_fields(cursor->n_fields, cursor->n_bytes)); } -/********************************************************************//** -Builds a hash index on a page with the given parameters. If the page already -has a hash index with different parameters, the old hash index is removed. -If index is non-NULL, this function checks if n_fields and n_bytes are -sensible values, and does not build a hash index if not. */ -static -void -btr_search_build_page_hash_index( -/*=============================*/ - dict_index_t* index, /*!< in: index for which to build, or NULL if - not known */ - buf_block_t* block, /*!< in: index page, s- or x-latched */ - ulint n_fields,/*!< in: hash this many full fields */ - ulint n_bytes,/*!< in: hash this many bytes from the next - field */ - ibool left_side);/*!< in: hash for searches from left side? */ - /** This function should be called before reserving any btr search mutex, if the intended operation might add nodes to the search system hash table. Because of the latching order, once we have reserved the btr search system @@ -224,8 +207,9 @@ btr_search_check_free_space_in_heap(dict_index_t* index) if (heap->free_block == NULL) { buf_block_t* block = buf_block_alloc(NULL); + rw_lock_t* ahi_latch = btr_get_search_latch(index); - btr_search_x_lock(index); + rw_lock_x_lock(ahi_latch); if (btr_search_enabled && heap->free_block == NULL) { @@ -234,7 +218,7 @@ btr_search_check_free_space_in_heap(dict_index_t* index) buf_block_free(block); } - btr_search_x_unlock(index); + rw_lock_x_unlock(ahi_latch); } } @@ -457,12 +441,10 @@ btr_search_info_get_ref_count( ut_ad(info); - ut_ad(!rw_lock_own(btr_get_search_latch(index), RW_LOCK_S)); - ut_ad(!rw_lock_own(btr_get_search_latch(index), RW_LOCK_X)); - - btr_search_s_lock(index); + rw_lock_t* ahi_latch = btr_get_search_latch(index); + rw_lock_s_lock(ahi_latch); ret = info->ref_count; - btr_search_s_unlock(index); + rw_lock_s_unlock(ahi_latch); return(ret); } @@ -723,61 +705,6 @@ btr_search_update_hash_ref( } } -/** Updates the search info. -@param[in,out] info search info -@param[in] cursor cursor which was just positioned */ -void -btr_search_info_update_slow( - btr_search_t* info, - btr_cur_t* cursor) -{ - buf_block_t* block; - ibool build_index; - - ut_ad(!rw_lock_own(btr_get_search_latch(cursor->index), RW_LOCK_S)); - ut_ad(!rw_lock_own(btr_get_search_latch(cursor->index), RW_LOCK_X)); - - block = btr_cur_get_block(cursor); - - /* NOTE that the following two function calls do NOT protect - info or block->n_fields etc. with any semaphore, to save CPU time! - We cannot assume the fields are consistent when we return from - those functions! */ - - btr_search_info_update_hash(info, cursor); - - build_index = btr_search_update_block_hash_info(info, block, cursor); - - if (build_index || (cursor->flag == BTR_CUR_HASH_FAIL)) { - - btr_search_check_free_space_in_heap(cursor->index); - } - - if (cursor->flag == BTR_CUR_HASH_FAIL) { - /* Update the hash node reference, if appropriate */ - -#ifdef UNIV_SEARCH_PERF_STAT - btr_search_n_hash_fail++; -#endif /* UNIV_SEARCH_PERF_STAT */ - - btr_search_x_lock(cursor->index); - - btr_search_update_hash_ref(info, block, cursor); - - btr_search_x_unlock(cursor->index); - } - - if (build_index) { - /* Note that since we did not protect block->n_fields etc. - with any semaphore, the values can be inconsistent. We have - to check inside the function call that they make sense. */ - btr_search_build_page_hash_index(cursor->index, block, - block->n_fields, - block->n_bytes, - block->left_side); - } -} - /** Checks if a guessed position for a tree cursor is right. Note that if mode is PAGE_CUR_LE, which is used in inserts, and the function returns TRUE, then cursor->up_match and cursor->low_match both have sensible values. @@ -947,12 +874,11 @@ both have sensible values. we assume the caller uses his search latch to protect the record! @param[out] cursor tree cursor -@param[in] has_search_latch - latch mode the caller currently has on - search system: RW_S/X_LATCH or 0 +@param[in] ahi_latch the adaptive hash index latch being held, + or NULL @param[in] mtr mini transaction -@return TRUE if succeeded */ -ibool +@return whether the search succeeded */ +bool btr_search_guess_on_hash( dict_index_t* index, btr_search_t* info, @@ -960,7 +886,7 @@ btr_search_guess_on_hash( ulint mode, ulint latch_mode, btr_cur_t* cursor, - ulint has_search_latch, + rw_lock_t* ahi_latch, mtr_t* mtr) { const rec_t* rec; @@ -970,6 +896,8 @@ btr_search_guess_on_hash( btr_cur_t cursor2; btr_pcur_t pcur; #endif + ut_ad(!ahi_latch || rw_lock_own(ahi_latch, RW_LOCK_S) + || rw_lock_own(ahi_latch, RW_LOCK_X)); if (!btr_search_enabled) { return(FALSE); @@ -977,6 +905,7 @@ btr_search_guess_on_hash( ut_ad(index && info && tuple && cursor && mtr); ut_ad(!dict_index_is_ibuf(index)); + ut_ad(!ahi_latch || ahi_latch == btr_get_search_latch(index)); ut_ad((latch_mode == BTR_SEARCH_LEAF) || (latch_mode == BTR_MODIFY_LEAF)); @@ -1009,27 +938,27 @@ btr_search_guess_on_hash( cursor->fold = fold; cursor->flag = BTR_CUR_HASH; - if (!has_search_latch) { - btr_search_s_lock(index); - - if (!btr_search_enabled) { - btr_search_s_unlock(index); + rw_lock_t* use_latch = ahi_latch ? NULL : btr_get_search_latch(index); - btr_search_failure(info, cursor); + if (use_latch) { + rw_lock_s_lock(use_latch); - return(FALSE); + if (!btr_search_enabled) { + goto fail; } + } else { + ut_ad(btr_search_enabled); } ut_ad(rw_lock_own(btr_get_search_latch(index), RW_LOCK_S)); rec = (rec_t*) ha_search_and_get_data( - btr_get_search_table(index), fold); + btr_get_search_table(index), fold); if (rec == NULL) { - - if (!has_search_latch) { - btr_search_s_unlock(index); + if (use_latch) { +fail: + rw_lock_s_unlock(use_latch); } btr_search_failure(info, cursor); @@ -1039,22 +968,15 @@ btr_search_guess_on_hash( buf_block_t* block = buf_block_from_ahi(rec); - if (!has_search_latch) { + if (use_latch) { if (!buf_page_get_known_nowait( latch_mode, block, BUF_MAKE_YOUNG, __FILE__, __LINE__, mtr)) { - - if (!has_search_latch) { - btr_search_s_unlock(index); - } - - btr_search_failure(info, cursor); - - return(FALSE); + goto fail; } - btr_search_s_unlock(index); + rw_lock_s_unlock(use_latch); buf_block_dbg_add_level(block, SYNC_TREE_NODE_FROM_HASH); } @@ -1063,7 +985,7 @@ btr_search_guess_on_hash( ut_ad(buf_block_get_state(block) == BUF_BLOCK_REMOVE_HASH); - if (!has_search_latch) { + if (!ahi_latch) { btr_leaf_page_release(block, latch_mode, mtr); } @@ -1085,11 +1007,9 @@ btr_search_guess_on_hash( record to determine if our guess for the cursor position is right. */ if (index_id != btr_page_get_index_id(block->frame) - || !btr_search_check_guess(cursor, - has_search_latch, - tuple, mode)) { + || !btr_search_check_guess(cursor, !!ahi_latch, tuple, mode)) { - if (!has_search_latch) { + if (!ahi_latch) { btr_leaf_page_release(block, latch_mode, mtr); } @@ -1110,7 +1030,7 @@ btr_search_guess_on_hash( info->last_hash_succ = FALSE; /* Currently, does not work if the following fails: */ - ut_ad(!has_search_latch); + ut_ad(!ahi_latch); btr_leaf_page_release(block, latch_mode, mtr); @@ -1143,7 +1063,7 @@ btr_search_guess_on_hash( #ifdef UNIV_SEARCH_PERF_STAT btr_search_n_succ++; #endif - if (!has_search_latch && buf_page_peek_if_too_old(&block->page)) { + if (!ahi_latch && buf_page_peek_if_too_old(&block->page)) { buf_page_make_young(&block->page); } @@ -1191,6 +1111,8 @@ btr_search_drop_page_hash_index(buf_block_t* block) /* This debug check uses a dirty read that could theoretically cause false positives while buf_pool_clear_hash_index() is executing. */ assert_block_ahi_valid(block); + ut_ad(!btr_search_own_any(RW_LOCK_S)); + ut_ad(!btr_search_own_any(RW_LOCK_X)); if (index == NULL) { return; @@ -1214,9 +1136,6 @@ btr_search_drop_page_hash_index(buf_block_t* block) % btr_ahi_parts; latch = btr_search_latches[ahi_slot]; - ut_ad(!btr_search_own_any(RW_LOCK_S)); - ut_ad(!btr_search_own_any(RW_LOCK_X)); - rw_lock_s_lock(latch); assert_block_ahi_valid(block); @@ -1419,6 +1338,7 @@ If index is non-NULL, this function checks if n_fields and n_bytes are sensible, and does not build a hash index if not. @param[in,out] index index for which to build. @param[in,out] block index page, s-/x- latched. +@param[in,out] ahi_latch the adaptive search latch @param[in] n_fields hash this many full fields @param[in] n_bytes hash this many bytes of the next field @param[in] left_side hash for searches from left side */ @@ -1427,12 +1347,11 @@ void btr_search_build_page_hash_index( dict_index_t* index, buf_block_t* block, + rw_lock_t* ahi_latch, ulint n_fields, ulint n_bytes, ibool left_side) { - hash_table_t* table; - page_t* page; const rec_t* rec; const rec_t* next_rec; ulint fold; @@ -1454,6 +1373,7 @@ btr_search_build_page_hash_index( } rec_offs_init(offsets_); + ut_ad(ahi_latch == btr_get_search_latch(index)); ut_ad(index); ut_ad(block->page.id.space() == index->space); ut_a(!dict_index_is_ibuf(index)); @@ -1463,20 +1383,17 @@ btr_search_build_page_hash_index( ut_ad(rw_lock_own(&(block->lock), RW_LOCK_S) || rw_lock_own(&(block->lock), RW_LOCK_X)); - btr_search_s_lock(index); - - table = btr_get_search_table(index); - page = buf_block_get_frame(block); + rw_lock_s_lock(ahi_latch); - if (block->index && ((block->curr_n_fields != n_fields) - || (block->curr_n_bytes != n_bytes) - || (block->curr_left_side != left_side))) { + const bool rebuild = block->index + && (block->curr_n_fields != n_fields + || block->curr_n_bytes != n_bytes + || block->curr_left_side != left_side); - btr_search_s_unlock(index); + rw_lock_s_unlock(ahi_latch); + if (rebuild) { btr_search_drop_page_hash_index(block); - } else { - btr_search_s_unlock(index); } /* Check that the values for hash index build are sensible */ @@ -1491,6 +1408,7 @@ btr_search_build_page_hash_index( return; } + page_t* page = buf_block_get_frame(block); n_recs = page_get_n_recs(page); if (n_recs == 0) { @@ -1573,7 +1491,8 @@ btr_search_build_page_hash_index( btr_search_check_free_space_in_heap(index); - btr_search_x_lock(index); + hash_table_t* table = btr_get_search_table(index); + rw_lock_x_lock(ahi_latch); if (!btr_search_enabled) { goto exit_func; @@ -1611,7 +1530,7 @@ btr_search_build_page_hash_index( MONITOR_INC_VALUE(MONITOR_ADAPTIVE_HASH_ROW_ADDED, n_cached); exit_func: assert_block_ahi_valid(block); - btr_search_x_unlock(index); + rw_lock_x_unlock(ahi_latch); ut_free(folds); ut_free(recs); @@ -1620,6 +1539,60 @@ btr_search_build_page_hash_index( } } +/** Updates the search info. +@param[in,out] info search info +@param[in,out] cursor cursor which was just positioned */ +void +btr_search_info_update_slow(btr_search_t* info, btr_cur_t* cursor) +{ + rw_lock_t* ahi_latch = btr_get_search_latch(cursor->index); + + ut_ad(!rw_lock_own(ahi_latch, RW_LOCK_S)); + ut_ad(!rw_lock_own(ahi_latch, RW_LOCK_X)); + + buf_block_t* block = btr_cur_get_block(cursor); + + /* NOTE that the following two function calls do NOT protect + info or block->n_fields etc. with any semaphore, to save CPU time! + We cannot assume the fields are consistent when we return from + those functions! */ + + btr_search_info_update_hash(info, cursor); + + bool build_index = btr_search_update_block_hash_info( + info, block, cursor); + + if (build_index || (cursor->flag == BTR_CUR_HASH_FAIL)) { + + btr_search_check_free_space_in_heap(cursor->index); + } + + if (cursor->flag == BTR_CUR_HASH_FAIL) { + /* Update the hash node reference, if appropriate */ + +#ifdef UNIV_SEARCH_PERF_STAT + btr_search_n_hash_fail++; +#endif /* UNIV_SEARCH_PERF_STAT */ + + rw_lock_x_lock(ahi_latch); + + btr_search_update_hash_ref(info, block, cursor); + + rw_lock_x_unlock(ahi_latch); + } + + if (build_index) { + /* Note that since we did not protect block->n_fields etc. + with any semaphore, the values can be inconsistent. We have + to check inside the function call that they make sense. */ + btr_search_build_page_hash_index(cursor->index, block, + ahi_latch, + block->n_fields, + block->n_bytes, + block->left_side); + } +} + /** Move or delete hash entries for moved records, usually in a page split. If new_block is already hashed, then any hash index for block is dropped. If new_block is not hashed, and block is hashed, then a new hash index is @@ -1638,19 +1611,27 @@ btr_search_move_or_delete_hash_entries( return; } + dict_index_t* index = block->index; + if (!index) { + index = new_block->index; + } else { + ut_ad(!new_block->index || index == new_block->index); + } assert_block_ahi_valid(block); assert_block_ahi_valid(new_block); + rw_lock_t* ahi_latch = index ? btr_get_search_latch(index) : NULL; + if (new_block->index) { btr_search_drop_page_hash_index(block); return; } - dict_index_t* index = block->index; if (!index) { return; } - btr_search_s_lock(index); + + rw_lock_s_lock(ahi_latch); if (block->index) { ulint n_fields = block->curr_n_fields; @@ -1661,19 +1642,20 @@ btr_search_move_or_delete_hash_entries( new_block->n_bytes = block->curr_n_bytes; new_block->left_side = left_side; - btr_search_s_unlock(index); + rw_lock_s_unlock(ahi_latch); ut_a(n_fields > 0 || n_bytes > 0); btr_search_build_page_hash_index( - index, new_block, n_fields, n_bytes, left_side); + index, new_block, ahi_latch, + n_fields, n_bytes, left_side); ut_ad(n_fields == block->curr_n_fields); ut_ad(n_bytes == block->curr_n_bytes); ut_ad(left_side == block->curr_left_side); return; } - btr_search_s_unlock(index); + rw_lock_s_unlock(ahi_latch); } /** Updates the page hash index when a single record is deleted from a page. @@ -1728,7 +1710,9 @@ btr_search_update_hash_on_delete(btr_cur_t* cursor) mem_heap_free(heap); } - btr_search_x_lock(index); + rw_lock_t* ahi_latch = btr_get_search_latch(index); + + rw_lock_x_lock(ahi_latch); assert_block_ahi_valid(block); if (block->index) { @@ -1744,21 +1728,25 @@ btr_search_update_hash_on_delete(btr_cur_t* cursor) assert_block_ahi_valid(block); } - btr_search_x_unlock(index); + rw_lock_x_unlock(ahi_latch); } /** Updates the page hash index when a single record is inserted on a page. @param[in] cursor cursor which was positioned to the place to insert using btr_cur_search_, and the new record has been - inserted next to the cursor. */ + inserted next to the cursor. +@param[in] ahi_latch the adaptive hash index latch */ void -btr_search_update_hash_node_on_insert(btr_cur_t* cursor) +btr_search_update_hash_node_on_insert(btr_cur_t* cursor, rw_lock_t* ahi_latch) { hash_table_t* table; buf_block_t* block; dict_index_t* index; rec_t* rec; + ut_ad(ahi_latch == btr_get_search_latch(cursor->index)); + ut_ad(!btr_search_own_any(RW_LOCK_S)); + ut_ad(!btr_search_own_any(RW_LOCK_X)); #ifdef MYSQL_INDEX_DISABLE_AHI if (cursor->index->disable_ahi) return; #endif @@ -1781,8 +1769,7 @@ btr_search_update_hash_node_on_insert(btr_cur_t* cursor) ut_a(cursor->index == index); ut_a(!dict_index_is_ibuf(index)); - - btr_search_x_lock(index); + rw_lock_x_lock(ahi_latch); if (!block->index) { @@ -1806,11 +1793,11 @@ btr_search_update_hash_node_on_insert(btr_cur_t* cursor) func_exit: assert_block_ahi_valid(block); - btr_search_x_unlock(index); + rw_lock_x_unlock(ahi_latch); } else { - btr_search_x_unlock(index); + rw_lock_x_unlock(ahi_latch); - btr_search_update_hash_on_insert(cursor); + btr_search_update_hash_on_insert(cursor, ahi_latch); } } @@ -1818,9 +1805,10 @@ btr_search_update_hash_node_on_insert(btr_cur_t* cursor) @param[in,out] cursor cursor which was positioned to the place to insert using btr_cur_search_..., and the new record has been inserted next - to the cursor */ + to the cursor +@param[in] ahi_latch the adaptive hash index latch */ void -btr_search_update_hash_on_insert(btr_cur_t* cursor) +btr_search_update_hash_on_insert(btr_cur_t* cursor, rw_lock_t* ahi_latch) { hash_table_t* table; buf_block_t* block; @@ -1834,13 +1822,16 @@ btr_search_update_hash_on_insert(btr_cur_t* cursor) ulint n_fields; ulint n_bytes; ibool left_side; - ibool locked = FALSE; + bool locked = false; mem_heap_t* heap = NULL; ulint offsets_[REC_OFFS_NORMAL_SIZE]; ulint* offsets = offsets_; rec_offs_init(offsets_); + ut_ad(ahi_latch == btr_get_search_latch(cursor->index)); ut_ad(page_is_leaf(btr_cur_get_page(cursor))); + ut_ad(!btr_search_own_any(RW_LOCK_S)); + ut_ad(!btr_search_own_any(RW_LOCK_X)); #ifdef MYSQL_INDEX_DISABLE_AHI if (cursor->index->disable_ahi) return; #endif @@ -1899,10 +1890,8 @@ btr_search_update_hash_on_insert(btr_cur_t* cursor) fold = rec_fold(rec, offsets, n_fields, n_bytes, index->id); } else { if (left_side) { - - btr_search_x_lock(index); - - locked = TRUE; + locked = true; + rw_lock_x_lock(ahi_latch); if (!btr_search_enabled) { goto function_exit; @@ -1917,10 +1906,8 @@ btr_search_update_hash_on_insert(btr_cur_t* cursor) if (fold != ins_fold) { if (!locked) { - - btr_search_x_lock(index); - - locked = TRUE; + locked = true; + rw_lock_x_lock(ahi_latch); if (!btr_search_enabled) { goto function_exit; @@ -1938,11 +1925,9 @@ btr_search_update_hash_on_insert(btr_cur_t* cursor) if (page_rec_is_supremum(next_rec)) { if (!left_side) { - if (!locked) { - btr_search_x_lock(index); - - locked = TRUE; + locked = true; + rw_lock_x_lock(ahi_latch); if (!btr_search_enabled) { goto function_exit; @@ -1958,10 +1943,8 @@ btr_search_update_hash_on_insert(btr_cur_t* cursor) if (ins_fold != next_fold) { if (!locked) { - - btr_search_x_lock(index); - - locked = TRUE; + locked = true; + rw_lock_x_lock(ahi_latch); if (!btr_search_enabled) { goto function_exit; @@ -1980,8 +1963,9 @@ btr_search_update_hash_on_insert(btr_cur_t* cursor) mem_heap_free(heap); } if (locked) { - btr_search_x_unlock(index); + rw_lock_x_unlock(ahi_latch); } + ut_ad(!rw_lock_own(ahi_latch, RW_LOCK_X)); } #if defined UNIV_AHI_DEBUG || defined UNIV_DEBUG diff --git a/storage/innobase/include/btr0cur.h b/storage/innobase/include/btr0cur.h index 76f13325e2ad8..8d8fe0bc2366b 100644 --- a/storage/innobase/include/btr0cur.h +++ b/storage/innobase/include/btr0cur.h @@ -1,7 +1,7 @@ /***************************************************************************** Copyright (c) 1994, 2016, Oracle and/or its affiliates. All Rights Reserved. -Copyright (c) 2017, MariaDB Corporation. +Copyright (c) 2017, 2018, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -177,8 +177,7 @@ Note that if mode is PAGE_CUR_LE, which is used in inserts, then cursor->up_match and cursor->low_match both will have sensible values. If mode is PAGE_CUR_GE, then up_match will a have a sensible value. */ dberr_t -btr_cur_search_to_nth_level( -/*========================*/ +btr_cur_search_to_nth_level_func( dict_index_t* index, /*!< in: index */ ulint level, /*!< in: the tree level of search */ const dtuple_t* tuple, /*!< in: data tuple; NOTE: n_fields_cmp in @@ -197,23 +196,29 @@ btr_cur_search_to_nth_level( cursor->left_block is used to store a pointer to the left neighbor page, in the cases BTR_SEARCH_PREV and BTR_MODIFY_PREV; - NOTE that if has_search_latch - is != 0, we maybe do not have a latch set - on the cursor page, we assume - the caller uses his search latch - to protect the record! */ + NOTE that if ahi_latch, we might not have a + cursor page latch, we assume that ahi_latch + protects the record! */ btr_cur_t* cursor, /*!< in/out: tree cursor; the cursor page is s- or x-latched, but see also above! */ - ulint has_search_latch, - /*!< in: latch mode the caller - currently has on search system: - RW_S_LATCH, or 0 */ +#ifdef BTR_CUR_HASH_ADAPT + rw_lock_t* ahi_latch, + /*!< in: currently held btr_search_latch + (in RW_S_LATCH mode), or NULL */ +#endif /* BTR_CUR_HASH_ADAPT */ const char* file, /*!< in: file name */ unsigned line, /*!< in: line where called */ mtr_t* mtr, /*!< in/out: mini-transaction */ ib_uint64_t autoinc = 0); /*!< in: PAGE_ROOT_AUTO_INC to be written (0 if none) */ +#ifdef BTR_CUR_HASH_ADAPT +# define btr_cur_search_to_nth_level(i,l,t,m,lm,c,a,fi,li,mtr) \ + btr_cur_search_to_nth_level_func(i,l,t,m,lm,c,a,fi,li,mtr) +#else /* BTR_CUR_HASH_ADAPT */ +# define btr_cur_search_to_nth_level(i,l,t,m,lm,c,a,fi,li,mtr) \ + btr_cur_search_to_nth_level_func(i,l,t,m,lm,c,fi,li,mtr) +#endif /* BTR_CUR_HASH_ADAPT */ /*****************************************************************//** Opens a cursor at either end of an index. diff --git a/storage/innobase/include/btr0pcur.h b/storage/innobase/include/btr0pcur.h index fab934ca0ee64..1d8690a3c90cd 100644 --- a/storage/innobase/include/btr0pcur.h +++ b/storage/innobase/include/btr0pcur.h @@ -1,7 +1,7 @@ /***************************************************************************** Copyright (c) 1996, 2016, Oracle and/or its affiliates. All Rights Reserved. -Copyright (c) 2017, MariaDB Corporation. +Copyright (c) 2017, 2018, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -136,20 +136,25 @@ btr_pcur_open_with_no_init_func( may end up on the previous page of the record! */ ulint latch_mode,/*!< in: BTR_SEARCH_LEAF, ...; - NOTE that if has_search_latch != 0 then - we maybe do not acquire a latch on the cursor - page, but assume that the caller uses his - btr search latch to protect the record! */ + NOTE that if ahi_latch then we might not + acquire a cursor page latch, but assume + that the ahi_latch protects the record! */ btr_pcur_t* cursor, /*!< in: memory buffer for persistent cursor */ - ulint has_search_latch, - /*!< in: latch mode the caller - currently has on search system: - RW_S_LATCH, or 0 */ +#ifdef BTR_CUR_HASH_ADAPT + rw_lock_t* ahi_latch, + /*!< in: adaptive hash index latch held + by the caller, or NULL if none */ +#endif /* BTR_CUR_HASH_ADAPT */ const char* file, /*!< in: file name */ unsigned line, /*!< in: line where called */ mtr_t* mtr); /*!< in: mtr */ -#define btr_pcur_open_with_no_init(ix,t,md,l,cur,has,m) \ - btr_pcur_open_with_no_init_func(ix,t,md,l,cur,has,__FILE__,__LINE__,m) +#ifdef BTR_CUR_HASH_ADAPT +# define btr_pcur_open_with_no_init(ix,t,md,l,cur,ahi,m) \ + btr_pcur_open_with_no_init_func(ix,t,md,l,cur,ahi,__FILE__,__LINE__,m) +#else /* BTR_CUR_HASH_ADAPT */ +# define btr_pcur_open_with_no_init(ix,t,md,l,cur,ahi,m) \ + btr_pcur_open_with_no_init_func(ix,t,md,l,cur,__FILE__,__LINE__,m) +#endif /* BTR_CUR_HASH_ADAPT */ /*****************************************************************//** Opens a persistent cursor at either end of an index. */ diff --git a/storage/innobase/include/btr0pcur.ic b/storage/innobase/include/btr0pcur.ic index 4490942a2bb2b..cbe1d76b09455 100644 --- a/storage/innobase/include/btr0pcur.ic +++ b/storage/innobase/include/btr0pcur.ic @@ -1,7 +1,7 @@ /***************************************************************************** Copyright (c) 1996, 2015, Oracle and/or its affiliates. All Rights Reserved. -Copyright (c) 2015, 2017, MariaDB Corporation. +Copyright (c) 2015, 2018, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -454,9 +454,12 @@ btr_pcur_open_low( ut_ad(!dict_index_is_spatial(index)); - err = btr_cur_search_to_nth_level( - index, level, tuple, mode, latch_mode, - btr_cursor, 0, file, line, mtr, autoinc); + err = btr_cur_search_to_nth_level_func( + index, level, tuple, mode, latch_mode, btr_cursor, +#ifdef BTR_CUR_HASH_ADAPT + NULL, +#endif /* BTR_CUR_HASH_ADAPT */ + file, line, mtr, autoinc); if (err != DB_SUCCESS) { ib::warn() << " Error code: " << err @@ -491,15 +494,15 @@ btr_pcur_open_with_no_init_func( may end up on the previous page of the record! */ ulint latch_mode,/*!< in: BTR_SEARCH_LEAF, ...; - NOTE that if has_search_latch != 0 then - we maybe do not acquire a latch on the cursor - page, but assume that the caller uses his - btr search latch to protect the record! */ + NOTE that if ahi_latch then we might not + acquire a cursor page latch, but assume + that the ahi_latch protects the record! */ btr_pcur_t* cursor, /*!< in: memory buffer for persistent cursor */ - ulint has_search_latch, - /*!< in: latch mode the caller - currently has on search system: - RW_S_LATCH, or 0 */ +#ifdef BTR_CUR_HASH_ADAPT + rw_lock_t* ahi_latch, + /*!< in: adaptive hash index latch held + by the caller, or NULL if none */ +#endif /* BTR_CUR_HASH_ADAPT */ const char* file, /*!< in: file name */ unsigned line, /*!< in: line where called */ mtr_t* mtr) /*!< in: mtr */ @@ -514,9 +517,12 @@ btr_pcur_open_with_no_init_func( btr_cursor = btr_pcur_get_btr_cur(cursor); - err = btr_cur_search_to_nth_level( + err = btr_cur_search_to_nth_level_func( index, 0, tuple, mode, latch_mode, btr_cursor, - has_search_latch, file, line, mtr); +#ifdef BTR_CUR_HASH_ADAPT + ahi_latch, +#endif /* BTR_CUR_HASH_ADAPT */ + file, line, mtr); cursor->pos_state = BTR_PCUR_IS_POSITIONED; diff --git a/storage/innobase/include/btr0sea.h b/storage/innobase/include/btr0sea.h index bd1a72fc3ace7..cddf846438df7 100644 --- a/storage/innobase/include/btr0sea.h +++ b/storage/innobase/include/btr0sea.h @@ -1,7 +1,7 @@ /***************************************************************************** Copyright (c) 1996, 2016, Oracle and/or its affiliates. All Rights Reserved. -Copyright (c) 2017, MariaDB Corporation. +Copyright (c) 2017, 2018, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -91,12 +91,11 @@ both have sensible values. we assume the caller uses his search latch to protect the record! @param[out] cursor tree cursor -@param[in] has_search_latch - latch mode the caller currently has on - search system: RW_S/X_LATCH or 0 +@param[in] ahi_latch the adaptive hash index latch being held, + or NULL @param[in] mtr mini transaction -@return TRUE if succeeded */ -ibool +@return whether the search succeeded */ +bool btr_search_guess_on_hash( dict_index_t* index, btr_search_t* info, @@ -104,7 +103,7 @@ btr_search_guess_on_hash( ulint mode, ulint latch_mode, btr_cur_t* cursor, - ulint has_search_latch, + rw_lock_t* ahi_latch, mtr_t* mtr); /** Move or delete hash entries for moved records, usually in a page split. @@ -140,17 +139,19 @@ btr_search_drop_page_hash_when_freed( /** Updates the page hash index when a single record is inserted on a page. @param[in] cursor cursor which was positioned to the place to insert using btr_cur_search_, and the new record has been - inserted next to the cursor. */ + inserted next to the cursor. +@param[in] ahi_latch the adaptive hash index latch */ void -btr_search_update_hash_node_on_insert(btr_cur_t* cursor); +btr_search_update_hash_node_on_insert(btr_cur_t* cursor, rw_lock_t* ahi_latch); /** Updates the page hash index when a single record is inserted on a page. -@param[in] cursor cursor which was positioned to the +@param[in,out] cursor cursor which was positioned to the place to insert using btr_cur_search_..., and the new record has been inserted next - to the cursor */ + to the cursor +@param[in] ahi_latch the adaptive hash index latch */ void -btr_search_update_hash_on_insert(btr_cur_t* cursor); +btr_search_update_hash_on_insert(btr_cur_t* cursor, rw_lock_t* ahi_latch); /** Updates the page hash index when a single record is deleted from a page. @param[in] cursor cursor which was positioned on the record to delete @@ -163,18 +164,6 @@ btr_search_update_hash_on_delete(btr_cur_t* cursor); bool btr_search_validate(); -/** X-Lock the search latch (corresponding to given index) -@param[in] index index handler */ -UNIV_INLINE -void -btr_search_x_lock(const dict_index_t* index); - -/** X-Unlock the search latch (corresponding to given index) -@param[in] index index handler */ -UNIV_INLINE -void -btr_search_x_unlock(const dict_index_t* index); - /** Lock all search latches in exclusive mode. */ UNIV_INLINE void @@ -185,18 +174,6 @@ UNIV_INLINE void btr_search_x_unlock_all(); -/** S-Lock the search latch (corresponding to given index) -@param[in] index index handler */ -UNIV_INLINE -void -btr_search_s_lock(const dict_index_t* index); - -/** S-Unlock the search latch (corresponding to given index) -@param[in] index index handler */ -UNIV_INLINE -void -btr_search_s_unlock(const dict_index_t* index); - /** Lock all search latches in shared mode. */ UNIV_INLINE void @@ -243,15 +220,11 @@ btr_get_search_table(const dict_index_t* index); #else /* BTR_CUR_HASH_ADAPT */ # define btr_search_sys_create(size) # define btr_search_drop_page_hash_index(block) -# define btr_search_s_lock(index) -# define btr_search_s_unlock(index) # define btr_search_s_lock_all(index) # define btr_search_s_unlock_all(index) -# define btr_search_x_lock(index) -# define btr_search_x_unlock(index) # define btr_search_info_update(index, cursor) # define btr_search_move_or_delete_hash_entries(new_block, block) -# define btr_search_update_hash_on_insert(cursor) +# define btr_search_update_hash_on_insert(cursor, ahi_latch) # define btr_search_update_hash_on_delete(cursor) # define btr_search_sys_resize(hash_size) #endif /* BTR_CUR_HASH_ADAPT */ diff --git a/storage/innobase/include/btr0sea.ic b/storage/innobase/include/btr0sea.ic index b5a7536a2b454..65b953961dfcd 100644 --- a/storage/innobase/include/btr0sea.ic +++ b/storage/innobase/include/btr0sea.ic @@ -1,6 +1,7 @@ /***************************************************************************** Copyright (c) 1996, 2015, Oracle and/or its affiliates. All Rights Reserved. +Copyright (c) 2018, MariaDB Corporation. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software @@ -45,13 +46,11 @@ btr_search_info_create(mem_heap_t* heap) } #ifdef BTR_CUR_HASH_ADAPT -/*********************************************************************//** -Updates the search info. */ +/** Updates the search info. +@param[in,out] info search info +@param[in,out] cursor cursor which was just positioned */ void -btr_search_info_update_slow( -/*========================*/ - btr_search_t* info, /*!< in/out: search info */ - btr_cur_t* cursor);/*!< in: cursor which was just positioned */ +btr_search_info_update_slow(btr_search_t* info, btr_cur_t* cursor); /*********************************************************************//** Updates the search info. */ @@ -87,24 +86,6 @@ btr_search_info_update( btr_search_info_update_slow(info, cursor); } -/** X-Lock the search latch (corresponding to given index) -@param[in] index index handler */ -UNIV_INLINE -void -btr_search_x_lock(const dict_index_t* index) -{ - rw_lock_x_lock(btr_get_search_latch(index)); -} - -/** X-Unlock the search latch (corresponding to given index) -@param[in] index index handler */ -UNIV_INLINE -void -btr_search_x_unlock(const dict_index_t* index) -{ - rw_lock_x_unlock(btr_get_search_latch(index)); -} - /** Lock all search latches in exclusive mode. */ UNIV_INLINE void @@ -125,24 +106,6 @@ btr_search_x_unlock_all() } } -/** S-Lock the search latch (corresponding to given index) -@param[in] index index handler */ -UNIV_INLINE -void -btr_search_s_lock(const dict_index_t* index) -{ - rw_lock_s_lock(btr_get_search_latch(index)); -} - -/** S-Unlock the search latch (corresponding to given index) -@param[in] index index handler */ -UNIV_INLINE -void -btr_search_s_unlock(const dict_index_t* index) -{ - rw_lock_s_unlock(btr_get_search_latch(index)); -} - /** Lock all search latches in shared mode. */ UNIV_INLINE void diff --git a/storage/innobase/row/row0sel.cc b/storage/innobase/row/row0sel.cc index c0d56bf0bfa2a..d01b870fc09f4 100644 --- a/storage/innobase/row/row0sel.cc +++ b/storage/innobase/row/row0sel.cc @@ -1272,24 +1272,19 @@ static void row_sel_open_pcur( /*==============*/ - plan_t* plan, /*!< in: table plan */ - ibool search_latch_locked, - /*!< in: TRUE if the thread currently - has the search latch locked in - s-mode */ - mtr_t* mtr) /*!< in: mtr */ + plan_t* plan, /*!< in: table plan */ +#ifdef BTR_CUR_HASH_ADAPT + rw_lock_t* ahi_latch, + /*!< in: the adaptive hash index latch */ +#endif /* BTR_CUR_HASH_ADAPT */ + mtr_t* mtr) /*!< in/out: mini-transaction */ { dict_index_t* index; func_node_t* cond; que_node_t* exp; ulint n_fields; - ulint has_search_latch = 0; /* RW_S_LATCH or 0 */ ulint i; - if (search_latch_locked) { - has_search_latch = RW_S_LATCH; - } - index = plan->index; /* Calculate the value of the search tuple: the exact match columns @@ -1325,7 +1320,7 @@ row_sel_open_pcur( btr_pcur_open_with_no_init(index, plan->tuple, plan->mode, BTR_SEARCH_LEAF, &plan->pcur, - has_search_latch, mtr); + ahi_latch, mtr); } else { /* Open the cursor to the start or the end of the index (FALSE: no init) */ @@ -1473,7 +1468,7 @@ row_sel_try_search_shortcut( rw_lock_t* ahi_latch = btr_get_search_latch(index); rw_lock_s_lock(ahi_latch); - row_sel_open_pcur(plan, TRUE, mtr); + row_sel_open_pcur(plan, ahi_latch, mtr); const rec_t* rec = btr_pcur_get_rec(&(plan->pcur)); @@ -1659,8 +1654,11 @@ row_sel( if (!plan->pcur_is_open) { /* Evaluate the expressions to build the search tuple and open the cursor */ - - row_sel_open_pcur(plan, FALSE, &mtr); + row_sel_open_pcur(plan, +#ifdef BTR_CUR_HASH_ADAPT + NULL, +#endif /* BTR_CUR_HASH_ADAPT */ + &mtr); cursor_just_opened = TRUE; @@ -3841,7 +3839,7 @@ row_sel_try_search_shortcut_for_mysql( rw_lock_t* ahi_latch = btr_get_search_latch(index); rw_lock_s_lock(ahi_latch); btr_pcur_open_with_no_init(index, search_tuple, PAGE_CUR_GE, - BTR_SEARCH_LEAF, pcur, RW_S_LATCH, mtr); + BTR_SEARCH_LEAF, pcur, ahi_latch, mtr); rec = btr_pcur_get_rec(pcur); if (!page_rec_is_user_rec(rec) || rec_is_default_row(rec, index)) { From 0292cd0a27418fabe4660fa6f2a92a0952b06826 Mon Sep 17 00:00:00 2001 From: Sergei Petrunia Date: Mon, 15 Jan 2018 21:08:00 +0300 Subject: [PATCH 075/106] Better explanation why rpl_row_triggers is disabled. --- storage/rocksdb/mysql-test/rocksdb/t/disabled.def | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/storage/rocksdb/mysql-test/rocksdb/t/disabled.def b/storage/rocksdb/mysql-test/rocksdb/t/disabled.def index c2ca6efe46c79..60f658ec7e13e 100644 --- a/storage/rocksdb/mysql-test/rocksdb/t/disabled.def +++ b/storage/rocksdb/mysql-test/rocksdb/t/disabled.def @@ -59,7 +59,7 @@ bytes_written: Needs I_S.TABLE_STATISTICS.IO_WRITE_BYTES trx_info_rpl : MariaRocks: @@rpl_skip_tx_api doesn't work, yet. rpl_read_free: MDEV-10976 lock_wait_timeout_stats: MDEV-13404 - +rpl_row_triggers : Requires read-free slave. ## ## Test failures (in buildbot or else where) @@ -74,5 +74,4 @@ information_schema: MDEV-14372: unstable testcase ## mysqlbinlog_gtid_skip_empty_trans_rocksdb : MariaRocks: requires GTIDs -rpl_row_triggers : MariaRocks: requires GTIDs From f5e158183cb72de5bfc1cc7a9b6864a85c6517fd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Tue, 16 Jan 2018 07:46:51 +0200 Subject: [PATCH 076/106] Follow-up fix to MDEV-14441: Correct a misplaced condition btr_cur_update_in_place(): The call rw_lock_x_lock(ahi_latch) must of course be inside the if (ahi_latch) condition. This is a mistake that I made when backporting the fix-under-development from 10.3. --- storage/innobase/btr/btr0cur.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/storage/innobase/btr/btr0cur.cc b/storage/innobase/btr/btr0cur.cc index 2830a38fa4a43..8146667f7c26a 100644 --- a/storage/innobase/btr/btr0cur.cc +++ b/storage/innobase/btr/btr0cur.cc @@ -3711,9 +3711,9 @@ btr_cur_update_in_place( to this record */ btr_search_update_hash_on_delete(cursor); } - } - rw_lock_x_lock(ahi_latch); + rw_lock_x_lock(ahi_latch); + } assert_block_ahi_valid(block); #endif /* BTR_CUR_HASH_ADAPT */ From da27dc995ee407cb56761393bc5dcdd48456d851 Mon Sep 17 00:00:00 2001 From: Vladislav Vaintroub Date: Tue, 16 Jan 2018 09:18:22 +0000 Subject: [PATCH 077/106] Fix Windows build --- sql/rpl_gtid.cc | 2 +- sql/set_var.cc | 2 +- sql/sql_class.h | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/sql/rpl_gtid.cc b/sql/rpl_gtid.cc index 9f48f908102bb..78e6165a5517b 100644 --- a/sql/rpl_gtid.cc +++ b/sql/rpl_gtid.cc @@ -582,7 +582,7 @@ rpl_slave_state::record_gtid(THD *thd, const rpl_gtid *gtid, uint64 sub_id, bool table_opened= false; TABLE *table; list_element *delete_list= 0, *next, *cur, **next_ptr_ptr, **best_ptr_ptr; - uint64_t best_sub_id; + uint64 best_sub_id; element *elem; ulonglong thd_saved_option= thd->variables.option_bits; Query_tables_list lex_backup; diff --git a/sql/set_var.cc b/sql/set_var.cc index 311b33bc0dd9d..097124a0b3af3 100644 --- a/sql/set_var.cc +++ b/sql/set_var.cc @@ -1323,7 +1323,7 @@ resolve_engine_list_item(THD *thd, plugin_ref *list, uint32 *idx, { LEX_CSTRING item_str; plugin_ref ref; - uint32_t i; + uint32 i; THD *thd_or_null = (temp_copy ? thd : NULL); item_str.str= pos; diff --git a/sql/sql_class.h b/sql/sql_class.h index 8747d658aea77..bb957e9ea201f 100644 --- a/sql/sql_class.h +++ b/sql/sql_class.h @@ -833,7 +833,7 @@ typedef struct system_status_var ulonglong table_open_cache_overflows; double last_query_cost; double cpu_time, busy_time; - uint32_t threads_running; + uint32 threads_running; /* Don't initialize */ /* Memory used for thread local storage */ int64 max_local_memory_used; From e260c6a087484973625445787d05e8b0f7ffe5b5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Otto=20Kek=C3=A4l=C3=A4inen?= Date: Fri, 12 Jan 2018 12:04:01 +0000 Subject: [PATCH 078/106] Fix commit 9631d933fbc2ec99874 so Travis CI doesn't fail TokuDB and Mroonga are not built on Travis CI to save time. Previously their packages were not created due to a hack in debian/rules, that got removed in commit 9631d933fbc2ec99874. This change complements that commit. --- debian/autobake-deb.sh | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/debian/autobake-deb.sh b/debian/autobake-deb.sh index 1a709eb82848b..2158d0d0e29b2 100755 --- a/debian/autobake-deb.sh +++ b/debian/autobake-deb.sh @@ -87,6 +87,12 @@ then sed '/Package: mariadb-plugin-aws-key-management-10.3/,+15d' -i debian/control fi +# Mroonga, TokuDB never built on Travis CI anyway, see build flags above +if [[ $TRAVIS ]] +then + sed -i -e "/Package: mariadb-plugin-tokudb/,+17d" debian/control + sed -i -e "/Package: mariadb-plugin-mroonga/,+16d" debian/control +fi # Adjust changelog, add new version echo "Incrementing changelog and starting build scripts" From 094938255d1926f16b0f74bcbd7c3e967eaef2ed Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Otto=20Kek=C3=A4l=C3=A4inen?= Date: Fri, 12 Jan 2018 12:05:30 +0000 Subject: [PATCH 079/106] Deb: sync architecture restrictions from packaging in Debian official --- debian/control | 4 ++-- debian/rules | 7 +------ 2 files changed, 3 insertions(+), 8 deletions(-) diff --git a/debian/control b/debian/control index fe9a10a2cb8b0..bbea53788ca85 100644 --- a/debian/control +++ b/debian/control @@ -500,7 +500,7 @@ Description: OQGraph storage engine for MariaDB This package contains the OQGraph plugin for MariaDB. Package: mariadb-plugin-tokudb -Architecture: any +Architecture: amd64 Depends: mariadb-server-10.3 (= ${binary:Version}), ${misc:Depends}, ${shlibs:Depends} @@ -519,7 +519,7 @@ Description: TokuDB storage engine for MariaDB This package contains the TokuDB plugin for MariaDB. Package: mariadb-plugin-mroonga -Architecture: any +Architecture: any-alpha any-amd64 any-arm any-arm64 any-i386 any-ia64 any-mips64el any-mips64r6el any-mipsel any-mipsr6el any-nios2 any-powerpcel any-ppc64el any-sh3 any-sh4 any-tilegx Depends: mariadb-server-10.3 (= ${binary:Version}), ${misc:Depends}, ${shlibs:Depends} diff --git a/debian/rules b/debian/rules index 76433d47576e0..456af8b066472 100755 --- a/debian/rules +++ b/debian/rules @@ -39,7 +39,7 @@ else endif # Ignore test suite exit code on unstable platforms -ifneq (,$(filter $(ARCH), mips mipsel)) +ifneq (,$(filter $(ARCH),mips mipsel mips64el alpha powerpc sh4 hurd-i386 sparc64 kfreebsd-i386 kfreebsd-amd64)) TESTSUITE_FAIL_CMD:=true else TESTSUITE_FAIL_CMD:=exit 1 @@ -114,11 +114,6 @@ override_dh_auto_install: dh_testdir dh_testroot -# Skip TokuDB if arch is not amd64 -ifneq ($(ARCH), amd64) - sed -i -e "/Package: mariadb-plugin-tokudb/,+18d" debian/control -endif - # If libthrift-dev was available (manually installed, as it is # not in Debian) and ha_cassandra.so was thus built, create package, # otherwise skip it. From 846517d9b704a91c557e0992f149f5890ae796ca Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Otto=20Kek=C3=A4l=C3=A4inen?= Date: Thu, 11 Jan 2018 09:46:09 +0000 Subject: [PATCH 080/106] Deb: Recover include/mariadb wrongly removed in commit 9b10b4181f70634c The libmariadb-dev package must include this directory so that downstream developers can intentionally include MariaDB libraries as they want. This matches the contents of the official Debian libmariadb-dev package. --- debian/libmariadb-dev.install | 1 + 1 file changed, 1 insertion(+) diff --git a/debian/libmariadb-dev.install b/debian/libmariadb-dev.install index e62aad1b43f76..7aac84d0000c4 100644 --- a/debian/libmariadb-dev.install +++ b/debian/libmariadb-dev.install @@ -1,5 +1,6 @@ usr/bin/mysql_config usr/include/mysql +usr/include/mariadb usr/lib/*/libmariadb.so usr/lib/*/libmysqlclient.so usr/lib/*/libmariadbclient.so From 9719fb4beffd0cec057be63f43beb018099c77d2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Otto=20Kek=C3=A4l=C3=A4inen?= Date: Thu, 11 Jan 2018 10:46:04 +0000 Subject: [PATCH 081/106] Deb: Re-rename *.files into *.install The correct format for new Debian packages is to define install lists as *.install files. This has already been fixed for 10.3, but it when merging commit c5a525bda2b64dd008598afa51f3b7c6347e24fc from 10.2, this change was overlooked and a file of the old format slipped in. --- debian/{mariadb-backup-10.3.files => mariadb-backup-10.3.install} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename debian/{mariadb-backup-10.3.files => mariadb-backup-10.3.install} (100%) diff --git a/debian/mariadb-backup-10.3.files b/debian/mariadb-backup-10.3.install similarity index 100% rename from debian/mariadb-backup-10.3.files rename to debian/mariadb-backup-10.3.install From 503fcfb80f62da4a02912404ca724868edc607e3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Otto=20Kek=C3=A4l=C3=A4inen?= Date: Thu, 11 Jan 2018 10:51:03 +0000 Subject: [PATCH 082/106] Deb: Re-remove unnecessary version suffixes from package names There is no point in having version suffixes in plugin packages. They will via their control file anyway depend on the correct server or client version. Having version suffices has no benefit and it just complicates upgrades when a new major version name package needs to replace all possible previous version. This issue was overlooked when merging commit 844a5759462d11d5e4ae83d951b796647621e2d0 from 10.2 into 10.3, so it needs to be manually fixed now. --- debian/autobake-deb.sh | 5 ++++- debian/control | 2 +- ...0.2.install => mariadb-plugin-aws-key-management.install} | 0 3 files changed, 5 insertions(+), 2 deletions(-) rename debian/{mariadb-plugin-aws-key-management-10.2.install => mariadb-plugin-aws-key-management.install} (100%) diff --git a/debian/autobake-deb.sh b/debian/autobake-deb.sh index 2158d0d0e29b2..5948229ac28fb 100755 --- a/debian/autobake-deb.sh +++ b/debian/autobake-deb.sh @@ -82,9 +82,12 @@ if [[ $GCCVERSION -lt 40800 ]] || [[ $(arch) =~ i[346]86 ]] || [[ $TRAVIS ]] then sed '/Package: mariadb-plugin-rocksdb/,+13d' -i debian/control fi + +# AWS SDK requires c++11 -capable compiler +# Minimal supported versions are g++ 4.8 and clang 3.3. if [[ $GCCVERSION -lt 40800 ]] || [[ $TRAVIS ]] then - sed '/Package: mariadb-plugin-aws-key-management-10.3/,+15d' -i debian/control + sed '/Package: mariadb-plugin-aws-key-management/,+14d' -i debian/control fi # Mroonga, TokuDB never built on Travis CI anyway, see build flags above diff --git a/debian/control b/debian/control index bbea53788ca85..8c0874648ce25 100644 --- a/debian/control +++ b/debian/control @@ -626,7 +626,7 @@ Description: CrackLib Password Validation Plugin for MariaDB This password validation plugin uses cracklib to allow only sufficiently secure (as defined by cracklib) user passwords in MariaDB. -Package: mariadb-plugin-aws-key-management-10.3 +Package: mariadb-plugin-aws-key-management Section: database Architecture: any Breaks: mariadb-aws-key-management-10.1, diff --git a/debian/mariadb-plugin-aws-key-management-10.2.install b/debian/mariadb-plugin-aws-key-management.install similarity index 100% rename from debian/mariadb-plugin-aws-key-management-10.2.install rename to debian/mariadb-plugin-aws-key-management.install From 90f1f72622cec1c8407cfffc67a7ec94f9cbe6ae Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Otto=20Kek=C3=A4l=C3=A4inen?= Date: Thu, 11 Jan 2018 12:49:21 +0000 Subject: [PATCH 083/106] Deb: Add missing files to packages - tokuft_logprint should be shipped in the TokuDB plugin - mysql_install_db is needed for installing/upgrading the datadir so it must be in the server core package so users can roll their own upgrades --- debian/mariadb-plugin-tokudb.install | 1 + debian/mariadb-server-10.3.install | 1 - debian/mariadb-server-10.3.manpages | 1 - debian/mariadb-server-core-10.3.install | 1 + debian/mariadb-server-core-10.3.manpages | 1 + 5 files changed, 3 insertions(+), 2 deletions(-) diff --git a/debian/mariadb-plugin-tokudb.install b/debian/mariadb-plugin-tokudb.install index 3c366ee0c4481..36beb14885f98 100644 --- a/debian/mariadb-plugin-tokudb.install +++ b/debian/mariadb-plugin-tokudb.install @@ -1,4 +1,5 @@ etc/mysql/conf.d/tokudb.cnf etc/mysql/mariadb.conf.d +usr/bin/tokuft_logprint usr/bin/tokuftdump usr/lib/mysql/plugin/ha_tokudb.so usr/share/doc/mariadb-server-10.3/README.md usr/share/doc/mariadb-plugin-tokudb/README.md diff --git a/debian/mariadb-server-10.3.install b/debian/mariadb-server-10.3.install index 604179c7dca44..487837e9582c3 100644 --- a/debian/mariadb-server-10.3.install +++ b/debian/mariadb-server-10.3.install @@ -19,7 +19,6 @@ usr/bin/myisamchk usr/bin/myisamlog usr/bin/myisampack usr/bin/mysql_convert_table_format -usr/bin/mysql_install_db usr/bin/mysql_plugin usr/bin/mysql_secure_installation usr/bin/mysql_setpermission diff --git a/debian/mariadb-server-10.3.manpages b/debian/mariadb-server-10.3.manpages index 1825b7655da47..ebe72c2459208 100644 --- a/debian/mariadb-server-10.3.manpages +++ b/debian/mariadb-server-10.3.manpages @@ -14,7 +14,6 @@ debian/tmp/usr/share/man/man1/mysql_convert_table_format.1 debian/tmp/usr/share/man/man1/mysqld_multi.1 debian/tmp/usr/share/man/man1/mysqld_safe.1 debian/tmp/usr/share/man/man1/mysqlhotcopy.1 -debian/tmp/usr/share/man/man1/mysql_install_db.1 debian/tmp/usr/share/man/man1/mysql_plugin.1 debian/tmp/usr/share/man/man1/mysql_secure_installation.1 debian/tmp/usr/share/man/man1/mysql_setpermission.1 diff --git a/debian/mariadb-server-core-10.3.install b/debian/mariadb-server-core-10.3.install index d882bd53f52a0..6aadea0adc855 100644 --- a/debian/mariadb-server-core-10.3.install +++ b/debian/mariadb-server-core-10.3.install @@ -1,4 +1,5 @@ usr/bin/innochecksum +usr/bin/mysql_install_db usr/bin/mysql_upgrade usr/sbin/mysqld usr/share/mysql/charsets diff --git a/debian/mariadb-server-core-10.3.manpages b/debian/mariadb-server-core-10.3.manpages index 234108cde50cf..04187d4cdf240 100644 --- a/debian/mariadb-server-core-10.3.manpages +++ b/debian/mariadb-server-core-10.3.manpages @@ -1,3 +1,4 @@ debian/tmp/usr/share/man/man8/mysqld.8 debian/tmp/usr/share/man/man1/innochecksum.1 +debian/tmp/usr/share/man/man1/mysql_install_db.1 debian/tmp/usr/share/man/man1/mysql_upgrade.1 From 65db816c325eab6b28e05617b91ec6df9aecd0cc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Otto=20Kek=C3=A4l=C3=A4inen?= Date: Thu, 11 Jan 2018 14:59:46 +0000 Subject: [PATCH 084/106] Deb: Split libmysqlclient19/20 compat links into a separate package We already ship libmysqlclient18 that replaces any existing package with the same name and install symlinks that point to out libmariadb.so. Instead of creating more packages, we add just one more that does the same and provides both libmysqlclient19 and -20 replacements symlinks. This is a clean solution, because systems that only use (and are built against) libmariadb will not need them and if somebody has done a mistake in the linker parameters, they will spot it and can easily update the filename to libmariadb. --- debian/control | 27 +++++++++++++++++++++------ debian/libmariadb3-compat.install | 2 ++ debian/libmariadb3.install | 2 -- 3 files changed, 23 insertions(+), 8 deletions(-) create mode 100644 debian/libmariadb3-compat.install diff --git a/debian/control b/debian/control index 8c0874648ce25..bca2393a1aa48 100644 --- a/debian/control +++ b/debian/control @@ -50,10 +50,6 @@ Conflicts: mariadb-galera-server-10.0 (<< 10.0.5), mariadb-server-5.3, mariadb-server-5.5 (<< 5.5.33), libmariadbclient18 (<< 10.2.0) -Replaces: libmysqlclient19, - libmysqlclient20 -Provides: libmysqlclient19, - libmysqlclient20 Description: MariaDB database client library MariaDB is a fast, stable and true multi-user, multi-threaded SQL database server. SQL (Structured Query Language) is the most popular database query @@ -62,6 +58,25 @@ Description: MariaDB database client library . This package includes the client library. +Package: libmariadb3-compat +Architecture: any +Section: libs +Depends: mariadb-common, libmariadb3, ${misc:Depends}, ${shlibs:Depends} +Breaks: libmysqlclient19, + libmysqlclient20 +Replaces: libmysqlclient19, + libmysqlclient20 +Provides: libmysqlclient19, + libmysqlclient20 +Description: MariaDB database client library MySQL compat package + MariaDB is a fast, stable and true multi-user, multi-threaded SQL database + server. SQL (Structured Query Language) is the most popular database query + language in the world. The main goals of MariaDB are speed, robustness and + ease of use. + . + This package includes the client runtime libraries that simulate and replace + the equivalents found in MySQL 5.6 and 5.7 (mysqlclient19 and 20). + Package: libmariadbclient18 Section: libs Architecture: any @@ -74,7 +89,7 @@ Description: Virtual package to satisfy external depends language in the world. The main goals of MariaDB are speed, robustness and ease of use. . - This package provides compatibility symlinks for libmariadb3 + This package provides compatibility symlinks for libmariadb3. Package: libmysqlclient18 Section: libs @@ -88,7 +103,7 @@ Description: Virtual package to satisfy external depends language in the world. The main goals of MariaDB are speed, robustness and ease of use. . - This package provides compatibility symlinks for libmariadb3 + This package provides compatibility symlinks for libmariadb3. Package: libmariadb-dev Architecture: any diff --git a/debian/libmariadb3-compat.install b/debian/libmariadb3-compat.install new file mode 100644 index 0000000000000..4e0d631d6c83c --- /dev/null +++ b/debian/libmariadb3-compat.install @@ -0,0 +1,2 @@ +usr/lib/*/libmysqlclient.so.19 +usr/lib/*/libmysqlclient.so.20 diff --git a/debian/libmariadb3.install b/debian/libmariadb3.install index 58e2bd1dcb213..55e2f3f4193c0 100644 --- a/debian/libmariadb3.install +++ b/debian/libmariadb3.install @@ -1,5 +1,3 @@ -usr/lib/*/libmysqlclient.so.19 -usr/lib/*/libmysqlclient.so.20 usr/lib/*/libmariadb.so.* usr/lib/mysql/plugin/dialog.so usr/lib/mysql/plugin/mysql_clear_password.so From 9ef2268eda401f65fc1a93cad37b9382a91b6f3b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Otto=20Kek=C3=A4l=C3=A4inen?= Date: Thu, 11 Jan 2018 15:48:17 +0000 Subject: [PATCH 085/106] Deb: Simplify libmariadb.so.3 links No path is required. These files always reside in the same directory anyway. --- debian/rules | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/debian/rules b/debian/rules index 456af8b066472..3a9ff27a82578 100755 --- a/debian/rules +++ b/debian/rules @@ -148,13 +148,13 @@ override_dh_auto_install: install -D -m 644 debian/mariadb-server-10.3.py $(TMP)/usr/share/apport/package-hooks/source_mariadb-10.3.py # Install libmariadbclient18 compatibility links - ln -s /usr/lib/$(DEB_HOST_MULTIARCH)/libmariadb.so.3 $(TMP)/usr/lib/$(DEB_HOST_MULTIARCH)/libmariadbclient.so - ln -s /usr/lib/$(DEB_HOST_MULTIARCH)/libmariadb.so.3 $(TMP)/usr/lib/$(DEB_HOST_MULTIARCH)/libmariadbclient.so.18 + ln -s libmariadb.so.3 $(TMP)/usr/lib/$(DEB_HOST_MULTIARCH)/libmariadbclient.so + ln -s libmariadb.so.3 $(TMP)/usr/lib/$(DEB_HOST_MULTIARCH)/libmariadbclient.so.18 # Install libmysqlclientclientXX compatibility links - ln -s /usr/lib/$(DEB_HOST_MULTIARCH)/libmariadb.so.3 $(TMP)/usr/lib/$(DEB_HOST_MULTIARCH)/libmysqlclient.so.18 - ln -s /usr/lib/$(DEB_HOST_MULTIARCH)/libmariadb.so.3 $(TMP)/usr/lib/$(DEB_HOST_MULTIARCH)/libmysqlclient.so.19 - ln -s /usr/lib/$(DEB_HOST_MULTIARCH)/libmariadb.so.3 $(TMP)/usr/lib/$(DEB_HOST_MULTIARCH)/libmysqlclient.so.20 + ln -s libmariadb.so.3 $(TMP)/usr/lib/$(DEB_HOST_MULTIARCH)/libmysqlclient.so.18 + ln -s libmariadb.so.3 $(TMP)/usr/lib/$(DEB_HOST_MULTIARCH)/libmysqlclient.so.19 + ln -s libmariadb.so.3 $(TMP)/usr/lib/$(DEB_HOST_MULTIARCH)/libmysqlclient.so.20 touch $@ From be59851ad98c21cd388a3a5ff6d088602291c6d1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Otto=20Kek=C3=A4l=C3=A4inen?= Date: Thu, 11 Jan 2018 15:11:16 +0000 Subject: [PATCH 086/106] Deb: Match official Debian package libmariadb-dev-compat Using this package any source that expects MySQL Client libraries should build without changes. Sources that expect MariaDB Connector C should build using only libmariadb-dev. Current MariaDB Connector/C does not build quite identically in this source tree compared to how it build in Debian official. The remaining differences are tracked in MDEV-14921. --- debian/control | 34 +++++++++++++++++++++++----- debian/libmariadb-dev-compat.install | 4 ++++ debian/libmariadb-dev-compat.links | 2 ++ debian/libmariadb-dev.install | 4 +--- debian/libmariadb-dev.links | 2 ++ debian/libmariadbd-dev.install | 1 - 6 files changed, 37 insertions(+), 10 deletions(-) create mode 100644 debian/libmariadb-dev-compat.install create mode 100644 debian/libmariadb-dev-compat.links create mode 100644 debian/libmariadb-dev.links diff --git a/debian/control b/debian/control index bca2393a1aa48..f86849e57413d 100644 --- a/debian/control +++ b/debian/control @@ -112,22 +112,44 @@ Depends: libmariadb3 (= ${binary:Version}), zlib1g-dev, ${misc:Depends}, ${shlibs:Depends} -Breaks: libmariadbclient-dev, libmysqlclient-dev -Replaces: libmariadbclient-dev, libmysqlclient-dev -Conflicts: libmariadbclient16-dev, +Breaks: libmariadbclient-dev +Replaces: libmariadbclient-dev +Conflicts: libmariadbclient16-dev +Provides: libmariadbclient-dev +Description: MariaDB database development files + MariaDB is a fast, stable and true multi-user, multi-threaded SQL database + server. SQL (Structured Query Language) is the most popular database query + language in the world. The main goals of MariaDB are speed, robustness and + ease of use. + . + This package includes development libraries and header files. To allow sources + expecting the MariaDB Connector/C to build. Sources that expect the MySQL + Client libraries should use files from the libmariadb-dev-compat package. + +Package: libmariadb-dev-compat +Architecture: any +Multi-Arch: same +Section: libdevel +Priority: extra +Depends: libmariadb-dev (= ${binary:Version}), ${misc:Depends} +Conflicts: libmariadb-client-lgpl-dev-compat, + libmariadbclient-dev-compat, + libmysqlclient-dev, libmysqlclient10-dev, libmysqlclient12-dev, libmysqlclient14-dev, libmysqlclient15-dev, libmysqlclient16-dev -Provides: libmariadbclient-dev, libmariadbclient-dev-compat, libmysqlclient-dev -Description: MariaDB database development files +Provides: libmariadbclient-dev-compat, libmariadb-client-lgpl-dev-compat, libmysqlclient-dev +Replaces: libmariadbclient-dev-compat, libmariadb-client-lgpl-dev-compat, libmysqlclient-dev +Description: MariaDB Connector/C, compatibility symlinks MariaDB is a fast, stable and true multi-user, multi-threaded SQL database server. SQL (Structured Query Language) is the most popular database query language in the world. The main goals of MariaDB are speed, robustness and ease of use. . - This package includes development libraries and header files. + This package includes compatibility symlinks to allow sources expecting the + MySQL client libraries to be built against MariaDB Connector/C. Package: libmariadbd19 Architecture: any diff --git a/debian/libmariadb-dev-compat.install b/debian/libmariadb-dev-compat.install new file mode 100644 index 0000000000000..7fe10c50f697a --- /dev/null +++ b/debian/libmariadb-dev-compat.install @@ -0,0 +1,4 @@ +usr/lib/*/libmysqlclient.a +usr/lib/*/libmysqlclient.so +usr/lib/*/libmysqlclient_r.a +usr/lib/*/libmysqlclient_r.so diff --git a/debian/libmariadb-dev-compat.links b/debian/libmariadb-dev-compat.links new file mode 100644 index 0000000000000..5079841768103 --- /dev/null +++ b/debian/libmariadb-dev-compat.links @@ -0,0 +1,2 @@ +usr/bin/mariadb_config usr/bin/mysql_config +usr/share/pkgconfig/mariadb.pc usr/share/pkgconfig/mysqlclient.pc diff --git a/debian/libmariadb-dev.install b/debian/libmariadb-dev.install index 7aac84d0000c4..ae26a1e10e99d 100644 --- a/debian/libmariadb-dev.install +++ b/debian/libmariadb-dev.install @@ -1,8 +1,6 @@ -usr/bin/mysql_config +usr/bin/mariadb_config usr/include/mysql -usr/include/mariadb usr/lib/*/libmariadb.so -usr/lib/*/libmysqlclient.so usr/lib/*/libmariadbclient.so usr/lib/*/libmariadbclient.a usr/lib/*/libmysqlservices.a diff --git a/debian/libmariadb-dev.links b/debian/libmariadb-dev.links new file mode 100644 index 0000000000000..14d60027b8a4a --- /dev/null +++ b/debian/libmariadb-dev.links @@ -0,0 +1,2 @@ +usr/include/mysql usr/include/mariadb +usr/share/man/man1/mysql_config.1.gz usr/share/man/man1/mariadb_config.1.gz diff --git a/debian/libmariadbd-dev.install b/debian/libmariadbd-dev.install index 6fe225b15ba11..4608bf6e838cc 100644 --- a/debian/libmariadbd-dev.install +++ b/debian/libmariadbd-dev.install @@ -1,3 +1,2 @@ -usr/bin/mariadb_config usr/lib/*/libmysqld.a usr/lib/*/libmysqld.so From 5b3bf57be251929e2a7e32ab4499559934c6ccbe Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Otto=20Kek=C3=A4l=C3=A4inen?= Date: Thu, 11 Jan 2018 17:58:41 +0000 Subject: [PATCH 087/106] Deb: Install missing man pages --- debian/mariadb-plugin-tokudb.manpages | 2 ++ debian/mariadb-server-10.3.manpages | 9 +++++++++ 2 files changed, 11 insertions(+) create mode 100644 debian/mariadb-plugin-tokudb.manpages diff --git a/debian/mariadb-plugin-tokudb.manpages b/debian/mariadb-plugin-tokudb.manpages new file mode 100644 index 0000000000000..102a593b8095e --- /dev/null +++ b/debian/mariadb-plugin-tokudb.manpages @@ -0,0 +1,2 @@ +debian/tmp/usr/share/man/man1/tokuft_logdump.1 +debian/tmp/usr/share/man/man1/tokuftdump.1 diff --git a/debian/mariadb-server-10.3.manpages b/debian/mariadb-server-10.3.manpages index ebe72c2459208..a05b695585eb2 100644 --- a/debian/mariadb-server-10.3.manpages +++ b/debian/mariadb-server-10.3.manpages @@ -3,6 +3,9 @@ debian/tmp/usr/share/man/man1/aria_dump_log.1 debian/tmp/usr/share/man/man1/aria_ftdump.1 debian/tmp/usr/share/man/man1/aria_pack.1 debian/tmp/usr/share/man/man1/aria_read_log.1 +debian/tmp/usr/share/man/man1/galera_new_cluster.1 +debian/tmp/usr/share/man/man1/galera_recovery.1 +debian/tmp/usr/share/man/man1/mariadb-service-convert.1 debian/tmp/usr/share/man/man1/msql2mysql.1 debian/tmp/usr/share/man/man1/myisamchk.1 debian/tmp/usr/share/man/man1/myisam_ftdump.1 @@ -18,7 +21,13 @@ debian/tmp/usr/share/man/man1/mysql_plugin.1 debian/tmp/usr/share/man/man1/mysql_secure_installation.1 debian/tmp/usr/share/man/man1/mysql_setpermission.1 debian/tmp/usr/share/man/man1/mysql_tzinfo_to_sql.1 +debian/tmp/usr/share/man/man1/mysqld_safe_helper.1 debian/tmp/usr/share/man/man1/perror.1 debian/tmp/usr/share/man/man1/replace.1 debian/tmp/usr/share/man/man1/resolveip.1 debian/tmp/usr/share/man/man1/resolve_stack_dump.1 +debian/tmp/usr/share/man/man1/wsrep_sst_common.1 +debian/tmp/usr/share/man/man1/wsrep_sst_mysqldump.1 +debian/tmp/usr/share/man/man1/wsrep_sst_rsync.1 +debian/tmp/usr/share/man/man1/wsrep_sst_xtrabackup-v2.1 +debian/tmp/usr/share/man/man1/wsrep_sst_xtrabackup.1 From ed00fd2bced5a7d52e3a26f95aa8acdb45eb49da Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Otto=20Kek=C3=A4l=C3=A4inen?= Date: Thu, 11 Jan 2018 20:04:09 +0000 Subject: [PATCH 088/106] Deb: switch from usr/include/mysql to usr/include/mariadb Matching libmariadb-dev package contents in official Debian repositories, the MariaDB Connector/C files should go into a folder using the 'mariadb' name. For compatibility with sources that expect to find 'mysql' stuff, create a symlink from include/mysql to include/mariadb. --- cmake/install_layout.cmake | 2 +- debian/libmariadb-dev-compat.links | 1 + debian/libmariadb-dev.install | 2 +- debian/libmariadb-dev.links | 1 - 4 files changed, 3 insertions(+), 3 deletions(-) diff --git a/cmake/install_layout.cmake b/cmake/install_layout.cmake index 5484691e94aba..63e2419951b46 100644 --- a/cmake/install_layout.cmake +++ b/cmake/install_layout.cmake @@ -176,7 +176,7 @@ SET(INSTALL_SYSCONF2DIR_DEB "/etc/mysql/conf.d") SET(INSTALL_LIBDIR_DEB "lib") SET(INSTALL_PLUGINDIR_DEB "lib/mysql/plugin") # -SET(INSTALL_INCLUDEDIR_DEB "include/mysql") +SET(INSTALL_INCLUDEDIR_DEB "include/mariadb") # SET(INSTALL_DOCDIR_DEB "share/doc") SET(INSTALL_DOCREADMEDIR_DEB "share/doc") diff --git a/debian/libmariadb-dev-compat.links b/debian/libmariadb-dev-compat.links index 5079841768103..11e0dbce2251a 100644 --- a/debian/libmariadb-dev-compat.links +++ b/debian/libmariadb-dev-compat.links @@ -1,2 +1,3 @@ usr/bin/mariadb_config usr/bin/mysql_config +usr/include/mariadb usr/include/mysql usr/share/pkgconfig/mariadb.pc usr/share/pkgconfig/mysqlclient.pc diff --git a/debian/libmariadb-dev.install b/debian/libmariadb-dev.install index ae26a1e10e99d..5025ae4a47c0f 100644 --- a/debian/libmariadb-dev.install +++ b/debian/libmariadb-dev.install @@ -1,5 +1,5 @@ usr/bin/mariadb_config -usr/include/mysql +usr/include/mariadb usr/lib/*/libmariadb.so usr/lib/*/libmariadbclient.so usr/lib/*/libmariadbclient.a diff --git a/debian/libmariadb-dev.links b/debian/libmariadb-dev.links index 14d60027b8a4a..6a7bf707ca7e3 100644 --- a/debian/libmariadb-dev.links +++ b/debian/libmariadb-dev.links @@ -1,2 +1 @@ -usr/include/mysql usr/include/mariadb usr/share/man/man1/mysql_config.1.gz usr/share/man/man1/mariadb_config.1.gz From a41192def8ab67f4420e437ebd26646023a9bcca Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Otto=20Kek=C3=A4l=C3=A4inen?= Date: Thu, 11 Jan 2018 20:27:40 +0000 Subject: [PATCH 089/106] Deb: Fix spelling etc Lintian complaints --- debian/control | 15 +++++++++------ debian/mariadb-client-10.3.README.Debian | 2 +- debian/mariadb-server-10.3.README.Debian | 8 ++++---- 3 files changed, 14 insertions(+), 11 deletions(-) diff --git a/debian/control b/debian/control index f86849e57413d..680b6e7de3049 100644 --- a/debian/control +++ b/debian/control @@ -83,13 +83,14 @@ Architecture: any Depends: libmariadb3 (= ${binary:Version}), ${misc:Depends} Replaces: libmariadbclient18 Provides: libmariadbclient18 -Description: Virtual package to satisfy external depends +Description: Virtual package to satisfy external libmariadbclient18 depends MariaDB is a fast, stable and true multi-user, multi-threaded SQL database server. SQL (Structured Query Language) is the most popular database query language in the world. The main goals of MariaDB are speed, robustness and ease of use. . - This package provides compatibility symlinks for libmariadb3. + This package provides compatibility symlinks for binaries that expect to find + libmariadbclient.so.18 will automatically use libmariadb.so.3 instead. Package: libmysqlclient18 Section: libs @@ -97,13 +98,14 @@ Architecture: any Depends: libmariadb3 (= ${binary:Version}), ${misc:Depends} Replaces: libmysqlclient18 Provides: libmysqlclient18 -Description: Virtual package to satisfy external depends +Description: Virtual package to satisfy external libmysqlclient18 depends MariaDB is a fast, stable and true multi-user, multi-threaded SQL database server. SQL (Structured Query Language) is the most popular database query language in the world. The main goals of MariaDB are speed, robustness and ease of use. . - This package provides compatibility symlinks for libmariadb3. + This package provides compatibility symlinks for binaries that expect to find + libmysqlclient.so.18 will automatically use libmariadb.so.3 instead. Package: libmariadb-dev Architecture: any @@ -626,6 +628,7 @@ Replaces: mariadb-gssapi-server-10.1, mariadb-gssapi-server-10.2, mariadb-gssapi-server-10.3 Description: GSSAPI authentication plugin for MariaDB server + This package contains the server components. Package: mariadb-plugin-gssapi-client Architecture: any @@ -640,9 +643,9 @@ Replaces: mariadb-gssapi-client-10.1, mariadb-gssapi-client-10.2, mariadb-gssapi-client-10.3 Description: GSSAPI authentication plugin for MariaDB client + This package contains the client components. Package: mariadb-backup-10.3 -Section: database Architecture: any Breaks: mariadb-backup-10.1, mariadb-backup-10.2 @@ -652,6 +655,7 @@ Depends: mariadb-server-10.3, ${misc:Depends}, ${shlibs:Depends} Description: Backup tool for MariaDB server + This backup tool is guaranteed to be compatible with MariaDB. Package: mariadb-plugin-cracklib-password-check Architecture: any @@ -664,7 +668,6 @@ Description: CrackLib Password Validation Plugin for MariaDB sufficiently secure (as defined by cracklib) user passwords in MariaDB. Package: mariadb-plugin-aws-key-management -Section: database Architecture: any Breaks: mariadb-aws-key-management-10.1, mariadb-aws-key-management-10.2 diff --git a/debian/mariadb-client-10.3.README.Debian b/debian/mariadb-client-10.3.README.Debian index b245638f9c95c..64f0f50995107 100644 --- a/debian/mariadb-client-10.3.README.Debian +++ b/debian/mariadb-client-10.3.README.Debian @@ -1,4 +1,4 @@ FAQ: -Q: My completition is gone, why? +Q: My completion is gone, why? A: You have "no-auto-rehash" in the "[mysql]" section of /etc/mysql/my.cnf! diff --git a/debian/mariadb-server-10.3.README.Debian b/debian/mariadb-server-10.3.README.Debian index be2e33d705deb..ca81cc357c4f1 100644 --- a/debian/mariadb-server-10.3.README.Debian +++ b/debian/mariadb-server-10.3.README.Debian @@ -42,7 +42,7 @@ https://mariadb.com/kb It is strongly recommended you create an admin users for your database adminstration needs. -If your your local unix account is the one you want to have local super user +If your local unix account is the one you want to have local super user access on your database with you can create the following account that will only work for the local unix user connecting to the database locally. @@ -57,14 +57,14 @@ the DB server over the network: sudo /usr/bin/mysql -e "GRANT ALL ON *.* TO 'USERNAME'@'%' IDENTIFIED BY 'password' WITH GRANT OPTION" -Scripts should run as a user have have the required grants and be identified via unix_socket. +Scripts should run as a user have the required grants and be identified via unix_socket. If you are too tired to type the password in every time and unix_socket auth doesn't suit your needs, you can store it in the file $HOME/.my.cnf. It should -be chmod 0600 (-rw------- username username .my.cnf) to ensure that nobody else +be chmod 0600 (-rw------- username usergroup .my.cnf) to ensure that nobody else can read it. Every other configuration parameter can be stored there, too. -For more information in the MariaDB manual in/usr/share/doc/mariadb-doc or +For more information in the MariaDB manual in/usr/share/doc/mariadb-doc or https://mariadb.com/kb/en/configuring-mariadb-with-mycnf/. ATTENTION: It is necessary, that a ~/.my.cnf from root always contains a "user" From 711e8c56346d0f9a4717ea2b2e0c7a98b6c247a9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Otto=20Kek=C3=A4l=C3=A4inen?= Date: Fri, 12 Jan 2018 13:06:14 +0000 Subject: [PATCH 090/106] MDEV-12642: Build deb source packages on buildbot, just not on Travis-CI --- debian/autobake-deb.sh | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/debian/autobake-deb.sh b/debian/autobake-deb.sh index 5948229ac28fb..fca3482e64b99 100755 --- a/debian/autobake-deb.sh +++ b/debian/autobake-deb.sh @@ -111,12 +111,20 @@ dch -b -D ${CODENAME} -v "${UPSTREAM}${PATCHLEVEL}~${CODENAME}" "Automatic build echo "Creating package version ${UPSTREAM}${PATCHLEVEL}~${CODENAME} ... " +# On Travis CI, use -b to build binary only packages as there is no need to +# waste time on generating the source package. +if [[ $TRAVIS ]] +then + BUILDPACKAGE_FLAGS="-b" +fi + # Build the package # Pass -I so that .git and other unnecessary temporary and source control files # will be ignored by dpkg-source when creating the tar.gz source package. -# Use -b to build binary only packages as there is no need to waste time on -# generating the source package. -fakeroot dpkg-buildpackage -us -uc -I -b +fakeroot dpkg-buildpackage -us -uc -I $BUILDPACKAGE_FLAGS + +# If the step above fails due to missing dependencies, you can manually run +# sudo mk-build-deps debian/control -r -i # Don't log package contents on Travis-CI to save time and log size if [[ ! $TRAVIS ]] From dbc9cfa52b822ebd0542faf09e0a561b638a637b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Otto=20Kek=C3=A4l=C3=A4inen?= Date: Fri, 12 Jan 2018 19:35:01 +0000 Subject: [PATCH 091/106] Deb: .manpages files should only contain man pages unique to deb packages All man pages that stem from the actual sources should be installed along everything else in .install files. --- debian/libmariadb-dev.install | 1 + debian/libmariadb-dev.manpages | 1 - debian/mariadb-client-10.3.install | 10 +++++++ debian/mariadb-client-10.3.manpages | 10 ------- debian/mariadb-client-core-10.3.install | 2 ++ debian/mariadb-client-core-10.3.manpages | 2 -- debian/mariadb-plugin-tokudb.install | 2 ++ debian/mariadb-plugin-tokudb.manpages | 2 -- debian/mariadb-server-10.3.install | 33 ++++++++++++++++++++++++ debian/mariadb-server-10.3.manpages | 33 ------------------------ debian/mariadb-server-core-10.3.install | 4 +++ debian/mariadb-server-core-10.3.manpages | 4 --- debian/mariadb-test.install | 4 +++ debian/mariadb-test.manpages | 4 --- 14 files changed, 56 insertions(+), 56 deletions(-) delete mode 100644 debian/libmariadb-dev.manpages delete mode 100644 debian/mariadb-client-core-10.3.manpages delete mode 100644 debian/mariadb-plugin-tokudb.manpages delete mode 100644 debian/mariadb-server-10.3.manpages delete mode 100644 debian/mariadb-server-core-10.3.manpages delete mode 100644 debian/mariadb-test.manpages diff --git a/debian/libmariadb-dev.install b/debian/libmariadb-dev.install index 5025ae4a47c0f..f33febf0e50ca 100644 --- a/debian/libmariadb-dev.install +++ b/debian/libmariadb-dev.install @@ -6,3 +6,4 @@ usr/lib/*/libmariadbclient.a usr/lib/*/libmysqlservices.a usr/share/aclocal/mysql.m4 usr/share/pkgconfig/mariadb.pc +usr/share/man/man1/mysql_config.1 diff --git a/debian/libmariadb-dev.manpages b/debian/libmariadb-dev.manpages deleted file mode 100644 index 3aac7f4b4a92c..0000000000000 --- a/debian/libmariadb-dev.manpages +++ /dev/null @@ -1 +0,0 @@ -debian/tmp/usr/share/man/man1/mysql_config.1 diff --git a/debian/mariadb-client-10.3.install b/debian/mariadb-client-10.3.install index 271f2a2f417ec..75ee09db78327 100644 --- a/debian/mariadb-client-10.3.install +++ b/debian/mariadb-client-10.3.install @@ -10,3 +10,13 @@ usr/bin/mysqldumpslow usr/bin/mysqlimport usr/bin/mysqlshow usr/bin/mysqlslap +usr/share/man/man1/mysqlaccess.1 +usr/share/man/man1/mysqladmin.1 +usr/share/man/man1/mysqldump.1 +usr/share/man/man1/mysqldumpslow.1 +usr/share/man/man1/mysql_find_rows.1 +usr/share/man/man1/mysql_fix_extensions.1 +usr/share/man/man1/mysqlimport.1 +usr/share/man/man1/mysqlshow.1 +usr/share/man/man1/mysqlslap.1 +usr/share/man/man1/mysql_waitpid.1 diff --git a/debian/mariadb-client-10.3.manpages b/debian/mariadb-client-10.3.manpages index 504619785bcd2..6f3e2bc188ccb 100644 --- a/debian/mariadb-client-10.3.manpages +++ b/debian/mariadb-client-10.3.manpages @@ -1,12 +1,2 @@ debian/additions/innotop/innotop.1 -debian/tmp/usr/share/man/man1/mysqlaccess.1 -debian/tmp/usr/share/man/man1/mysqladmin.1 -debian/tmp/usr/share/man/man1/mysqldump.1 -debian/tmp/usr/share/man/man1/mysqldumpslow.1 -debian/tmp/usr/share/man/man1/mysql_find_rows.1 -debian/tmp/usr/share/man/man1/mysql_fix_extensions.1 -debian/tmp/usr/share/man/man1/mysqlimport.1 debian/additions/mysqlreport.1 -debian/tmp/usr/share/man/man1/mysqlshow.1 -debian/tmp/usr/share/man/man1/mysqlslap.1 -debian/tmp/usr/share/man/man1/mysql_waitpid.1 diff --git a/debian/mariadb-client-core-10.3.install b/debian/mariadb-client-core-10.3.install index 6308d769930c2..a2781309439dd 100644 --- a/debian/mariadb-client-core-10.3.install +++ b/debian/mariadb-client-core-10.3.install @@ -1,2 +1,4 @@ usr/bin/mysql usr/bin/mysqlcheck +usr/share/man/man1/mysql.1 +usr/share/man/man1/mysqlcheck.1 diff --git a/debian/mariadb-client-core-10.3.manpages b/debian/mariadb-client-core-10.3.manpages deleted file mode 100644 index 2be91d81f9a7c..0000000000000 --- a/debian/mariadb-client-core-10.3.manpages +++ /dev/null @@ -1,2 +0,0 @@ -debian/tmp/usr/share/man/man1/mysql.1 -debian/tmp/usr/share/man/man1/mysqlcheck.1 diff --git a/debian/mariadb-plugin-tokudb.install b/debian/mariadb-plugin-tokudb.install index 36beb14885f98..d15db03b3e567 100644 --- a/debian/mariadb-plugin-tokudb.install +++ b/debian/mariadb-plugin-tokudb.install @@ -3,3 +3,5 @@ usr/bin/tokuft_logprint usr/bin/tokuftdump usr/lib/mysql/plugin/ha_tokudb.so usr/share/doc/mariadb-server-10.3/README.md usr/share/doc/mariadb-plugin-tokudb/README.md +usr/share/man/man1/tokuft_logdump.1 +usr/share/man/man1/tokuftdump.1 diff --git a/debian/mariadb-plugin-tokudb.manpages b/debian/mariadb-plugin-tokudb.manpages deleted file mode 100644 index 102a593b8095e..0000000000000 --- a/debian/mariadb-plugin-tokudb.manpages +++ /dev/null @@ -1,2 +0,0 @@ -debian/tmp/usr/share/man/man1/tokuft_logdump.1 -debian/tmp/usr/share/man/man1/tokuftdump.1 diff --git a/debian/mariadb-server-10.3.install b/debian/mariadb-server-10.3.install index 487837e9582c3..ac1ab6dd99094 100644 --- a/debian/mariadb-server-10.3.install +++ b/debian/mariadb-server-10.3.install @@ -65,3 +65,36 @@ usr/share/mysql/mysql_system_tables.sql usr/share/mysql/mysql_system_tables_data.sql usr/share/mysql/mysql_test_data_timezone.sql usr/share/mysql/wsrep_notify +usr/share/man/man1/aria_chk.1 +usr/share/man/man1/aria_dump_log.1 +usr/share/man/man1/aria_ftdump.1 +usr/share/man/man1/aria_pack.1 +usr/share/man/man1/aria_read_log.1 +usr/share/man/man1/galera_new_cluster.1 +usr/share/man/man1/galera_recovery.1 +usr/share/man/man1/mariadb-service-convert.1 +usr/share/man/man1/msql2mysql.1 +usr/share/man/man1/myisamchk.1 +usr/share/man/man1/myisam_ftdump.1 +usr/share/man/man1/myisamlog.1 +usr/share/man/man1/myisampack.1 +usr/share/man/man1/my_print_defaults.1 +usr/share/man/man1/mysqlbinlog.1 +usr/share/man/man1/mysql_convert_table_format.1 +usr/share/man/man1/mysqld_multi.1 +usr/share/man/man1/mysqld_safe.1 +usr/share/man/man1/mysqlhotcopy.1 +usr/share/man/man1/mysql_plugin.1 +usr/share/man/man1/mysql_secure_installation.1 +usr/share/man/man1/mysql_setpermission.1 +usr/share/man/man1/mysql_tzinfo_to_sql.1 +usr/share/man/man1/mysqld_safe_helper.1 +usr/share/man/man1/perror.1 +usr/share/man/man1/replace.1 +usr/share/man/man1/resolveip.1 +usr/share/man/man1/resolve_stack_dump.1 +usr/share/man/man1/wsrep_sst_common.1 +usr/share/man/man1/wsrep_sst_mysqldump.1 +usr/share/man/man1/wsrep_sst_rsync.1 +usr/share/man/man1/wsrep_sst_xtrabackup-v2.1 +usr/share/man/man1/wsrep_sst_xtrabackup.1 diff --git a/debian/mariadb-server-10.3.manpages b/debian/mariadb-server-10.3.manpages deleted file mode 100644 index a05b695585eb2..0000000000000 --- a/debian/mariadb-server-10.3.manpages +++ /dev/null @@ -1,33 +0,0 @@ -debian/tmp/usr/share/man/man1/aria_chk.1 -debian/tmp/usr/share/man/man1/aria_dump_log.1 -debian/tmp/usr/share/man/man1/aria_ftdump.1 -debian/tmp/usr/share/man/man1/aria_pack.1 -debian/tmp/usr/share/man/man1/aria_read_log.1 -debian/tmp/usr/share/man/man1/galera_new_cluster.1 -debian/tmp/usr/share/man/man1/galera_recovery.1 -debian/tmp/usr/share/man/man1/mariadb-service-convert.1 -debian/tmp/usr/share/man/man1/msql2mysql.1 -debian/tmp/usr/share/man/man1/myisamchk.1 -debian/tmp/usr/share/man/man1/myisam_ftdump.1 -debian/tmp/usr/share/man/man1/myisamlog.1 -debian/tmp/usr/share/man/man1/myisampack.1 -debian/tmp/usr/share/man/man1/my_print_defaults.1 -debian/tmp/usr/share/man/man1/mysqlbinlog.1 -debian/tmp/usr/share/man/man1/mysql_convert_table_format.1 -debian/tmp/usr/share/man/man1/mysqld_multi.1 -debian/tmp/usr/share/man/man1/mysqld_safe.1 -debian/tmp/usr/share/man/man1/mysqlhotcopy.1 -debian/tmp/usr/share/man/man1/mysql_plugin.1 -debian/tmp/usr/share/man/man1/mysql_secure_installation.1 -debian/tmp/usr/share/man/man1/mysql_setpermission.1 -debian/tmp/usr/share/man/man1/mysql_tzinfo_to_sql.1 -debian/tmp/usr/share/man/man1/mysqld_safe_helper.1 -debian/tmp/usr/share/man/man1/perror.1 -debian/tmp/usr/share/man/man1/replace.1 -debian/tmp/usr/share/man/man1/resolveip.1 -debian/tmp/usr/share/man/man1/resolve_stack_dump.1 -debian/tmp/usr/share/man/man1/wsrep_sst_common.1 -debian/tmp/usr/share/man/man1/wsrep_sst_mysqldump.1 -debian/tmp/usr/share/man/man1/wsrep_sst_rsync.1 -debian/tmp/usr/share/man/man1/wsrep_sst_xtrabackup-v2.1 -debian/tmp/usr/share/man/man1/wsrep_sst_xtrabackup.1 diff --git a/debian/mariadb-server-core-10.3.install b/debian/mariadb-server-core-10.3.install index 6aadea0adc855..6244abd119372 100644 --- a/debian/mariadb-server-core-10.3.install +++ b/debian/mariadb-server-core-10.3.install @@ -27,3 +27,7 @@ usr/share/mysql/slovak usr/share/mysql/spanish usr/share/mysql/swedish usr/share/mysql/ukrainian +usr/share/man/man8/mysqld.8 +usr/share/man/man1/innochecksum.1 +usr/share/man/man1/mysql_install_db.1 +usr/share/man/man1/mysql_upgrade.1 diff --git a/debian/mariadb-server-core-10.3.manpages b/debian/mariadb-server-core-10.3.manpages deleted file mode 100644 index 04187d4cdf240..0000000000000 --- a/debian/mariadb-server-core-10.3.manpages +++ /dev/null @@ -1,4 +0,0 @@ -debian/tmp/usr/share/man/man8/mysqld.8 -debian/tmp/usr/share/man/man1/innochecksum.1 -debian/tmp/usr/share/man/man1/mysql_install_db.1 -debian/tmp/usr/share/man/man1/mysql_upgrade.1 diff --git a/debian/mariadb-test.install b/debian/mariadb-test.install index 605620dc28c7c..6c59a78c2d783 100644 --- a/debian/mariadb-test.install +++ b/debian/mariadb-test.install @@ -26,3 +26,7 @@ usr/share/mysql/mysql-test/mysql-test-run.pl usr/share/mysql/mysql-test/purify.supp usr/share/mysql/mysql-test/suite.pm usr/share/mysql/mysql-test/valgrind.supp +usr/share/man/man1/mysql_client_test.1 +usr/share/man/man1/mysql_client_test_embedded.1 +usr/share/man/man1/mysqltest.1 +usr/share/man/man1/mysqltest_embedded.1 diff --git a/debian/mariadb-test.manpages b/debian/mariadb-test.manpages deleted file mode 100644 index 33ce8710ea7fe..0000000000000 --- a/debian/mariadb-test.manpages +++ /dev/null @@ -1,4 +0,0 @@ -debian/tmp/usr/share/man/man1/mysql_client_test.1 -debian/tmp/usr/share/man/man1/mysql_client_test_embedded.1 -debian/tmp/usr/share/man/man1/mysqltest.1 -debian/tmp/usr/share/man/man1/mysqltest_embedded.1 From b5edb4ca3a39c46e0109cf888aeb6ca54a6fc9a9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Otto=20Kek=C3=A4l=C3=A4inen?= Date: Fri, 12 Jan 2018 16:56:55 +0000 Subject: [PATCH 092/106] Deb: Add missing files into correct packages and rest in not-installed There was a lot of files generated by the sources that were not installed in any package. This fixes most of those issues, but not all. Files still outside any package are in the not-installed, which is used by dh_install. Also make sure all configuration files are installed to the correct location that matches layout used by Debian official packages. --- debian/libmariadb3.install | 2 + debian/mariadb-common.install | 2 +- .../mariadb-plugin-aws-key-management.install | 3 +- debian/mariadb-plugin-connect.install | 2 +- debian/mariadb-plugin-mroonga.install | 2 + debian/mariadb-server-10.3.install | 1 + debian/mariadb-test.install | 3 + debian/not-installed | 127 ++++++++++++++++++ 8 files changed, 139 insertions(+), 3 deletions(-) create mode 100644 debian/not-installed diff --git a/debian/libmariadb3.install b/debian/libmariadb3.install index 55e2f3f4193c0..68d763ae40413 100644 --- a/debian/libmariadb3.install +++ b/debian/libmariadb3.install @@ -1,3 +1,5 @@ usr/lib/*/libmariadb.so.* usr/lib/mysql/plugin/dialog.so usr/lib/mysql/plugin/mysql_clear_password.so +usr/lib/mysql/plugin/client_ed25519.so +usr/lib/mysql/plugin/sha256_password.so diff --git a/debian/mariadb-common.install b/debian/mariadb-common.install index 78dbc22b9f69f..611c7d4d36ee5 100644 --- a/debian/mariadb-common.install +++ b/debian/mariadb-common.install @@ -1 +1 @@ -debian/additions/mariadb.cnf etc/mysql/conf.d +debian/additions/mariadb.cnf etc/mysql/ diff --git a/debian/mariadb-plugin-aws-key-management.install b/debian/mariadb-plugin-aws-key-management.install index ed966b4115dbb..655735cb8ac1a 100644 --- a/debian/mariadb-plugin-aws-key-management.install +++ b/debian/mariadb-plugin-aws-key-management.install @@ -1,2 +1,3 @@ usr/lib/mysql/plugin/aws_key_management.so -debian/additions/enable_encryption.preset etc/mysql/conf.d/ +debian/additions/enable_encryption.preset etc/mysql/mariadb.conf.d +etc/mysql/conf.d/aws_key_management.cnf etc/mysql/mariadb.conf.d diff --git a/debian/mariadb-plugin-connect.install b/debian/mariadb-plugin-connect.install index 8a7aee412dfad..22d73c7df05dc 100644 --- a/debian/mariadb-plugin-connect.install +++ b/debian/mariadb-plugin-connect.install @@ -1,2 +1,2 @@ -etc/mysql/conf.d/connect.cnf +etc/mysql/conf.d/connect.cnf etc/mysql/mariadb.conf.d usr/lib/mysql/plugin/ha_connect.so diff --git a/debian/mariadb-plugin-mroonga.install b/debian/mariadb-plugin-mroonga.install index c28fde2fd181c..5cb81419e5e3e 100644 --- a/debian/mariadb-plugin-mroonga.install +++ b/debian/mariadb-plugin-mroonga.install @@ -1,3 +1,5 @@ usr/lib/mysql/plugin/ha_mroonga.so usr/share/mysql/mroonga/install.sql usr/share/mysql/mroonga/uninstall.sql +usr/share/mysql/mroonga/COPYING +usr/share/mysql/mroonga/AUTHORS diff --git a/debian/mariadb-server-10.3.install b/debian/mariadb-server-10.3.install index ac1ab6dd99094..c0af11c5cc0e6 100644 --- a/debian/mariadb-server-10.3.install +++ b/debian/mariadb-server-10.3.install @@ -41,6 +41,7 @@ usr/bin/wsrep_sst_mariabackup usr/lib/mysql/plugin/auth_pam.so usr/lib/mysql/plugin/auth_socket.so usr/lib/mysql/plugin/file_key_management.so +usr/lib/mysql/plugin/auth_ed25519.so usr/lib/mysql/plugin/ha_archive.so usr/lib/mysql/plugin/ha_blackhole.so usr/lib/mysql/plugin/ha_federated.so diff --git a/debian/mariadb-test.install b/debian/mariadb-test.install index 6c59a78c2d783..4a3ef436634cc 100644 --- a/debian/mariadb-test.install +++ b/debian/mariadb-test.install @@ -26,6 +26,9 @@ usr/share/mysql/mysql-test/mysql-test-run.pl usr/share/mysql/mysql-test/purify.supp usr/share/mysql/mysql-test/suite.pm usr/share/mysql/mysql-test/valgrind.supp +usr/share/mysql/mysql-test/dgcov.pl +usr/share/man/man1/mysql-stress-test.pl.1 +usr/share/man/man1/mysql-test-run.pl.1 usr/share/man/man1/mysql_client_test.1 usr/share/man/man1/mysql_client_test_embedded.1 usr/share/man/man1/mysqltest.1 diff --git a/debian/not-installed b/debian/not-installed new file mode 100644 index 0000000000000..c151cf753e683 --- /dev/null +++ b/debian/not-installed @@ -0,0 +1,127 @@ +lib/systemd/system/mariadb.service # Installed by rules file +lib/systemd/system/mariadb@.service # Installed by rules file +usr/bin/mysql_config # We already have the MariaDB variant +usr/bin/mysql_embedded # Huge 500 MB file. Not intended for distribution via any disto package. +usr/bin/mytop # Mytop is distributed from a separate source package +usr/lib/sysusers.d/sysusers.conf +usr/lib/tmpfiles.d/tmpfiles.conf +usr/lib/mysql/plugin/JavaWrappers.jar # These are only built if JNI/libjawt.so is installed from e.g. openjdk-8-jre-headless +usr/lib/mysql/plugin/JdbcInterface.jar # These are only built if JNI/libjawt.so is installed from e.g. openjdk-8-jre-headless +usr/share/doc/mariadb-server-10.3/COPYING +usr/share/doc/mariadb-server-10.3/COPYING.AGPLv3 +usr/share/doc/mariadb-server-10.3/COPYING.GPLv2 +usr/share/doc/mariadb-server-10.3/COPYING.thirdparty +usr/share/doc/mariadb-server-10.3/CREDITS +usr/share/doc/mariadb-server-10.3/EXCEPTIONS-CLIENT +usr/share/doc/mariadb-server-10.3/INSTALL-BINARY +usr/share/doc/mariadb-server-10.3/PATENTS +usr/share/doc/mariadb-server-10.3/README-wsrep +usr/share/groonga/COPYING +usr/share/groonga-normalizer-mysql/lgpl-2.0.txt +usr/share/groonga-normalizer-mysql/README.md +usr/share/groonga/README.md +usr/share/man/man1/my_safe_process.1 +usr/share/man/man1/mysql.server.1 +usr/share/mysql/binary-configure +usr/share/mysql/magic +usr/share/mysql/maria_add_gis_sp.sql +usr/share/mysql/mysqld_multi.server +usr/share/mysql/mysql-log-rotate +usr/share/mysql/mysql.server +usr/share/mysql/mysql-test/mtr # Already created by mariadb-test.links +usr/share/mysql/mysql-test/mysql-test-run # Already created by mariadb-test.links +usr/share/mysql/mysql_to_mariadb.sql +usr/share/mysql/policy/apparmor/README # In MariaDB we don't want to use AppArmor at the moment +usr/share/mysql/policy/apparmor/usr.sbin.mysqld # In MariaDB we don't want to use AppArmor at the moment +usr/share/mysql/policy/apparmor/usr.sbin.mysqld.local # In MariaDB we don't want to use AppArmor at the moment +usr/share/mysql/policy/selinux/mariadb-server.fc # In MariaDB we don't want to use SELinux at the moment +usr/share/mysql/policy/selinux/mariadb-server.te # In MariaDB we don't want to use SELinux at the moment +usr/share/mysql/policy/selinux/mariadb.te # In MariaDB we don't want to use SELinux at the moment +usr/share/mysql/policy/selinux/README # In MariaDB we don't want to use SELinux at the moment +usr/share/mysql/systemd/mariadb.service # Installed by rules file +usr/share/mysql/systemd/mariadb@.service # Installed by rules file +usr/share/mysql/systemd/use_galera_new_cluster.conf +usr/share/mysql/wsrep.cnf +usr/sql-bench/bench-count-distinct +usr/sql-bench/bench-init.pl # SQL-bench is distributed from a separate source package +usr/sql-bench/compare-results +usr/sql-bench/copy-db +usr/sql-bench/crash-me +usr/sql-bench/Data/ATIS/aircraft.txt +usr/sql-bench/Data/ATIS/airline.txt +usr/sql-bench/Data/ATIS/airport_service.txt +usr/sql-bench/Data/ATIS/airport.txt +usr/sql-bench/Data/ATIS/city.txt +usr/sql-bench/Data/ATIS/class_of_service.txt +usr/sql-bench/Data/ATIS/code_description.txt +usr/sql-bench/Data/ATIS/compound_class.txt +usr/sql-bench/Data/ATIS/connect_leg.txt +usr/sql-bench/Data/ATIS/date_day.txt +usr/sql-bench/Data/ATIS/day_name.txt +usr/sql-bench/Data/ATIS/dual_carrier.txt +usr/sql-bench/Data/ATIS/fare.txt +usr/sql-bench/Data/ATIS/fconnection.txt +usr/sql-bench/Data/ATIS/flight_class.txt +usr/sql-bench/Data/ATIS/flight_day.txt +usr/sql-bench/Data/ATIS/flight_fare.txt +usr/sql-bench/Data/ATIS/flight.txt +usr/sql-bench/Data/ATIS/food_service.txt +usr/sql-bench/Data/ATIS/ground_service.txt +usr/sql-bench/Data/ATIS/month_name.txt +usr/sql-bench/Data/ATIS/restrict_carrier.txt +usr/sql-bench/Data/ATIS/restrict_class.txt +usr/sql-bench/Data/ATIS/restriction.txt +usr/sql-bench/Data/ATIS/state.txt +usr/sql-bench/Data/ATIS/stop1.txt +usr/sql-bench/Data/ATIS/stop.txt +usr/sql-bench/Data/ATIS/time_interval.txt +usr/sql-bench/Data/ATIS/time_zone.txt +usr/sql-bench/Data/ATIS/transport.txt +usr/sql-bench/Data/Wisconsin/onek.data +usr/sql-bench/Data/Wisconsin/tenk.data +usr/sql-bench/graph-compare-results +usr/sql-bench/innotest1 +usr/sql-bench/innotest1a +usr/sql-bench/innotest1b +usr/sql-bench/innotest2 +usr/sql-bench/innotest2a +usr/sql-bench/innotest2b +usr/sql-bench/limits/access.cfg +usr/sql-bench/limits/access_odbc.cfg +usr/sql-bench/limits/Adabas.cfg +usr/sql-bench/limits/db2.cfg +usr/sql-bench/limits/empress.cfg +usr/sql-bench/limits/frontbase.cfg +usr/sql-bench/limits/Informix.cfg +usr/sql-bench/limits/interbase.cfg +usr/sql-bench/limits/interbase-dialect1.cfg +usr/sql-bench/limits/interbase-dialect3.cfg +usr/sql-bench/limits/interbase-superserver.cfg +usr/sql-bench/limits/mimer.cfg +usr/sql-bench/limits/msql.cfg +usr/sql-bench/limits/ms-sql65.cfg +usr/sql-bench/limits/ms-sql.cfg +usr/sql-bench/limits/mysql-3.22.cfg +usr/sql-bench/limits/mysql-3.23.cfg +usr/sql-bench/limits/mysql-4.0.cfg +usr/sql-bench/limits/mysql-4.1.cfg +usr/sql-bench/limits/mysql.cfg +usr/sql-bench/limits/oracle.cfg +usr/sql-bench/limits/pg.cfg +usr/sql-bench/limits/solid.cfg +usr/sql-bench/limits/solid-nt4.cfg +usr/sql-bench/limits/sybase.cfg +usr/sql-bench/myisam.cnf +usr/sql-bench/README +usr/sql-bench/run-all-tests +usr/sql-bench/server-cfg +usr/sql-bench/test-alter-table +usr/sql-bench/test-ATIS +usr/sql-bench/test-big-tables +usr/sql-bench/test-connect +usr/sql-bench/test-create +usr/sql-bench/test-insert +usr/sql-bench/test-select +usr/sql-bench/test-table-elimination +usr/sql-bench/test-transactions +usr/sql-bench/test-wisconsin From 72768cf188536d2d29ff7e25f273be5981c01997 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Otto=20Kek=C3=A4l=C3=A4inen?= Date: Fri, 12 Jan 2018 20:27:22 +0000 Subject: [PATCH 093/106] Deb: Warn if sources contain a file that is not installed anywhere --- debian/rules | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/debian/rules b/debian/rules index 3a9ff27a82578..b4ef2e5da6db7 100755 --- a/debian/rules +++ b/debian/rules @@ -177,7 +177,11 @@ override_dh_installcron-arch: get-orig-source: uscan --force-download --verbose +# If a file is not supposed to be included anywhere, add it to the not-installed +# file and document the reason. Note that dh_install supports the above mentioned +# white list file only starting from Debian Stretch and Ubuntu Xenial. +# To find more, grep build logs for 'but is not installed to anywhere'. %: - dh $@ --parallel --with dpatch --with systemd + dh $@ --parallel --with dpatch --with systemd --list-missing # vim: ts=8 From 3862d8bc89b9011c03738e3b48e444b692ae901a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Otto=20Kek=C3=A4l=C3=A4inen?= Date: Fri, 12 Jan 2018 22:56:53 +0000 Subject: [PATCH 094/106] Deb: wrap-and-sort -a --- debian/control | 86 ++++++++++++------- debian/libmariadb-dev.install | 4 +- debian/libmariadb3.install | 2 +- debian/mariadb-client-10.3.docs | 2 +- debian/mariadb-client-10.3.install | 6 +- debian/mariadb-client-10.3.links | 4 +- .../mariadb-plugin-aws-key-management.install | 2 +- debian/mariadb-plugin-mroonga.install | 4 +- debian/mariadb-plugin-rocksdb.install | 2 +- debian/mariadb-server-10.3.install | 34 ++++---- debian/mariadb-server-core-10.3.install | 8 +- debian/mariadb-test.install | 14 +-- debian/mariadb-test.links | 2 +- debian/mysql-common.install | 2 +- 14 files changed, 99 insertions(+), 73 deletions(-) diff --git a/debian/control b/debian/control index 680b6e7de3049..adc92a5808715 100644 --- a/debian/control +++ b/debian/control @@ -13,17 +13,17 @@ Build-Depends: bison, libaio-dev [linux-any], libboost-dev, libcrack2-dev (>= 2.9.0), + libcurl3-dev, libjemalloc-dev (>= 3.0.0~) [linux-any], libjudy-dev, libkrb5-dev, - libcurl3-dev, libncurses5-dev (>= 5.0-6~), libnuma-dev, libpam0g-dev, libpcre3-dev (>= 2:8.35-3.2~), libreadline-gplv2-dev, - libssl-dev | libssl1.0-dev, libsnappy-dev, + libssl-dev | libssl1.0-dev, libsystemd-dev, libxml2-dev, lsb-release, @@ -41,15 +41,17 @@ Vcs-Browser: https://github.com/MariaDB/server/ Package: libmariadb3 Architecture: any Section: libs -Depends: mariadb-common, ${misc:Depends}, ${shlibs:Depends} -Conflicts: mariadb-galera-server-10.0 (<< 10.0.5), +Depends: mariadb-common, + ${misc:Depends}, + ${shlibs:Depends} +Conflicts: libmariadbclient18 (<< 10.2.0), + mariadb-galera-server-10.0 (<< 10.0.5), mariadb-galera-server-5.5 (<< 5.5.33), mariadb-server-10.0 (<< 10.0.5), mariadb-server-5.1, mariadb-server-5.2, mariadb-server-5.3, - mariadb-server-5.5 (<< 5.5.33), - libmariadbclient18 (<< 10.2.0) + mariadb-server-5.5 (<< 5.5.33) Description: MariaDB database client library MariaDB is a fast, stable and true multi-user, multi-threaded SQL database server. SQL (Structured Query Language) is the most popular database query @@ -61,13 +63,16 @@ Description: MariaDB database client library Package: libmariadb3-compat Architecture: any Section: libs -Depends: mariadb-common, libmariadb3, ${misc:Depends}, ${shlibs:Depends} +Depends: libmariadb3, + mariadb-common, + ${misc:Depends}, + ${shlibs:Depends} Breaks: libmysqlclient19, - libmysqlclient20 + libmysqlclient20 Replaces: libmysqlclient19, - libmysqlclient20 + libmysqlclient20 Provides: libmysqlclient19, - libmysqlclient20 + libmysqlclient20 Description: MariaDB database client library MySQL compat package MariaDB is a fast, stable and true multi-user, multi-threaded SQL database server. SQL (Structured Query Language) is the most popular database query @@ -80,7 +85,8 @@ Description: MariaDB database client library MySQL compat package Package: libmariadbclient18 Section: libs Architecture: any -Depends: libmariadb3 (= ${binary:Version}), ${misc:Depends} +Depends: libmariadb3 (= ${binary:Version}), + ${misc:Depends} Replaces: libmariadbclient18 Provides: libmariadbclient18 Description: Virtual package to satisfy external libmariadbclient18 depends @@ -95,7 +101,8 @@ Description: Virtual package to satisfy external libmariadbclient18 depends Package: libmysqlclient18 Section: libs Architecture: any -Depends: libmariadb3 (= ${binary:Version}), ${misc:Depends} +Depends: libmariadb3 (= ${binary:Version}), + ${misc:Depends} Replaces: libmysqlclient18 Provides: libmysqlclient18 Description: Virtual package to satisfy external libmysqlclient18 depends @@ -133,7 +140,8 @@ Architecture: any Multi-Arch: same Section: libdevel Priority: extra -Depends: libmariadb-dev (= ${binary:Version}), ${misc:Depends} +Depends: libmariadb-dev (= ${binary:Version}), + ${misc:Depends} Conflicts: libmariadb-client-lgpl-dev-compat, libmariadbclient-dev-compat, libmysqlclient-dev, @@ -142,8 +150,12 @@ Conflicts: libmariadb-client-lgpl-dev-compat, libmysqlclient14-dev, libmysqlclient15-dev, libmysqlclient16-dev -Provides: libmariadbclient-dev-compat, libmariadb-client-lgpl-dev-compat, libmysqlclient-dev -Replaces: libmariadbclient-dev-compat, libmariadb-client-lgpl-dev-compat, libmysqlclient-dev +Provides: libmariadb-client-lgpl-dev-compat, + libmariadbclient-dev-compat, + libmysqlclient-dev +Replaces: libmariadb-client-lgpl-dev-compat, + libmariadbclient-dev-compat, + libmysqlclient-dev Description: MariaDB Connector/C, compatibility symlinks MariaDB is a fast, stable and true multi-user, multi-threaded SQL database server. SQL (Structured Query Language) is the most popular database query @@ -156,7 +168,8 @@ Description: MariaDB Connector/C, compatibility symlinks Package: libmariadbd19 Architecture: any Section: libs -Depends: ${misc:Depends}, ${shlibs:Depends} +Depends: ${misc:Depends}, + ${shlibs:Depends} Multi-Arch: same Breaks: libmariadbd-dev (<< ${source:Version}) Replaces: libmariadbd-dev (<< ${source:Version}) @@ -174,9 +187,9 @@ Section: libdevel Provides: libmysqld-dev Pre-Depends: ${misc:Pre-Depends} Depends: libmariadb-dev (= ${binary:Version}), - libmariadbd19 (= ${binary:Version}), - ${misc:Depends}, - ${shlibs:Depends} + libmariadbd19 (= ${binary:Version}), + ${misc:Depends}, + ${shlibs:Depends} Breaks: libmysqld-dev Replaces: libmysqld-dev Description: MariaDB embedded database, development files @@ -189,7 +202,8 @@ Description: MariaDB embedded database, development files Package: mysql-common Architecture: all -Depends: ${misc:Depends}, ${shlibs:Depends} +Depends: ${misc:Depends}, + ${shlibs:Depends} Description: MariaDB database common files (e.g. /etc/mysql/my.cnf) MariaDB is a fast, stable and true multi-user, multi-threaded SQL database server. SQL (Structured Query Language) is the most popular database query @@ -201,7 +215,9 @@ Description: MariaDB database common files (e.g. /etc/mysql/my.cnf) Package: mariadb-common Architecture: all -Depends: mysql-common, ${misc:Depends}, ${shlibs:Depends} +Depends: mysql-common, + ${misc:Depends}, + ${shlibs:Depends} Description: MariaDB database common files (e.g. /etc/mysql/conf.d/mariadb.cnf) MariaDB is a fast, stable and true multi-user, multi-threaded SQL database server. SQL (Structured Query Language) is the most popular database query @@ -323,7 +339,9 @@ Provides: default-mysql-client, mysql-client-5.6, mysql-client-5.7, virtual-mysql-client -Recommends: libdbd-mysql-perl (>= 1.2202), libdbi-perl, libterm-readkey-perl +Recommends: libdbd-mysql-perl (>= 1.2202), + libdbi-perl, + libterm-readkey-perl Description: MariaDB database client binaries MariaDB is a fast, stable and true multi-user, multi-threaded SQL database server. SQL (Structured Query Language) is the most popular database query @@ -392,9 +410,14 @@ Description: MariaDB database core server files Package: mariadb-server-10.3 Architecture: any -Suggests: mailx, mariadb-test, netcat-openbsd, tinyca +Suggests: mailx, + mariadb-test, + netcat-openbsd, + tinyca Recommends: libhtml-template-perl -Pre-Depends: adduser (>= 3.40), debconf, mariadb-common (>= ${source:Version}) +Pre-Depends: adduser (>= 3.40), + debconf, + mariadb-common (>= ${source:Version}) Depends: bsdutils, coreutils, findutils, @@ -455,7 +478,8 @@ Replaces: libmariadbclient-dev (<< 5.5.0), mysql-server-5.6, mysql-server-5.7, virtual-mysql-server -Provides: default-mysql-server, virtual-mysql-server +Provides: default-mysql-server, + virtual-mysql-server Description: MariaDB database server binaries MariaDB is a fast, stable and true multi-user, multi-threaded SQL database server. SQL (Structured Query Language) is the most popular database query @@ -466,7 +490,8 @@ Description: MariaDB database server binaries Package: mariadb-server Architecture: all -Depends: mariadb-server-10.3 (>= ${source:Version}), ${misc:Depends} +Depends: mariadb-server-10.3 (>= ${source:Version}), + ${misc:Depends} Description: MariaDB database server (metapackage depending on the latest version) This is an empty package that depends on the current "best" version of mariadb-server (currently mariadb-server-10.3), as determined by the MariaDB @@ -481,7 +506,8 @@ Description: MariaDB database server (metapackage depending on the latest versio Package: mariadb-client Architecture: all -Depends: mariadb-client-10.3 (>= ${source:Version}), ${misc:Depends} +Depends: mariadb-client-10.3 (>= ${source:Version}), + ${misc:Depends} Description: MariaDB database client (metapackage depending on the latest version) This is an empty package that depends on the current "best" version of mariadb-client (currently mariadb-client-10.3), as determined by the MariaDB @@ -650,7 +676,7 @@ Architecture: any Breaks: mariadb-backup-10.1, mariadb-backup-10.2 Replaces: mariadb-backup-10.1, - mariadb-backup-10.2 + mariadb-backup-10.2 Depends: mariadb-server-10.3, ${misc:Depends}, ${shlibs:Depends} @@ -673,8 +699,8 @@ Breaks: mariadb-aws-key-management-10.1, mariadb-aws-key-management-10.2 Replaces: mariadb-aws-key-management-10.1, mariadb-aws-key-management-10.2 -Depends: mariadb-server-10.3, - libcurl3, +Depends: libcurl3, + mariadb-server-10.3, ${misc:Depends}, ${shlibs:Depends} Description: Amazon Web Service Key Management Service Plugin for MariaDB diff --git a/debian/libmariadb-dev.install b/debian/libmariadb-dev.install index f33febf0e50ca..72798dec379ba 100644 --- a/debian/libmariadb-dev.install +++ b/debian/libmariadb-dev.install @@ -1,9 +1,9 @@ usr/bin/mariadb_config usr/include/mariadb usr/lib/*/libmariadb.so -usr/lib/*/libmariadbclient.so usr/lib/*/libmariadbclient.a +usr/lib/*/libmariadbclient.so usr/lib/*/libmysqlservices.a usr/share/aclocal/mysql.m4 -usr/share/pkgconfig/mariadb.pc usr/share/man/man1/mysql_config.1 +usr/share/pkgconfig/mariadb.pc diff --git a/debian/libmariadb3.install b/debian/libmariadb3.install index 68d763ae40413..8636166a493cc 100644 --- a/debian/libmariadb3.install +++ b/debian/libmariadb3.install @@ -1,5 +1,5 @@ usr/lib/*/libmariadb.so.* +usr/lib/mysql/plugin/client_ed25519.so usr/lib/mysql/plugin/dialog.so usr/lib/mysql/plugin/mysql_clear_password.so -usr/lib/mysql/plugin/client_ed25519.so usr/lib/mysql/plugin/sha256_password.so diff --git a/debian/mariadb-client-10.3.docs b/debian/mariadb-client-10.3.docs index 8117d689011e1..c09092629c313 100644 --- a/debian/mariadb-client-10.3.docs +++ b/debian/mariadb-client-10.3.docs @@ -1,2 +1,2 @@ -debian/additions/innotop/changelog.innotop README.md +debian/additions/innotop/changelog.innotop diff --git a/debian/mariadb-client-10.3.install b/debian/mariadb-client-10.3.install index 75ee09db78327..945bf77c68959 100644 --- a/debian/mariadb-client-10.3.install +++ b/debian/mariadb-client-10.3.install @@ -10,13 +10,13 @@ usr/bin/mysqldumpslow usr/bin/mysqlimport usr/bin/mysqlshow usr/bin/mysqlslap +usr/share/man/man1/mysql_find_rows.1 +usr/share/man/man1/mysql_fix_extensions.1 +usr/share/man/man1/mysql_waitpid.1 usr/share/man/man1/mysqlaccess.1 usr/share/man/man1/mysqladmin.1 usr/share/man/man1/mysqldump.1 usr/share/man/man1/mysqldumpslow.1 -usr/share/man/man1/mysql_find_rows.1 -usr/share/man/man1/mysql_fix_extensions.1 usr/share/man/man1/mysqlimport.1 usr/share/man/man1/mysqlshow.1 usr/share/man/man1/mysqlslap.1 -usr/share/man/man1/mysql_waitpid.1 diff --git a/debian/mariadb-client-10.3.links b/debian/mariadb-client-10.3.links index 0b86e87f2e92b..4a5049692469d 100644 --- a/debian/mariadb-client-10.3.links +++ b/debian/mariadb-client-10.3.links @@ -1,6 +1,6 @@ -usr/bin/mysqlcheck usr/bin/mysqlrepair usr/bin/mysqlcheck usr/bin/mysqlanalyze usr/bin/mysqlcheck usr/bin/mysqloptimize -usr/share/man/man1/mysqlcheck.1.gz usr/share/man/man1/mysqlrepair.1.gz +usr/bin/mysqlcheck usr/bin/mysqlrepair usr/share/man/man1/mysqlcheck.1.gz usr/share/man/man1/mysqlanalyze.1.gz usr/share/man/man1/mysqlcheck.1.gz usr/share/man/man1/mysqloptimize.1.gz +usr/share/man/man1/mysqlcheck.1.gz usr/share/man/man1/mysqlrepair.1.gz diff --git a/debian/mariadb-plugin-aws-key-management.install b/debian/mariadb-plugin-aws-key-management.install index 655735cb8ac1a..199d4c87950d1 100644 --- a/debian/mariadb-plugin-aws-key-management.install +++ b/debian/mariadb-plugin-aws-key-management.install @@ -1,3 +1,3 @@ -usr/lib/mysql/plugin/aws_key_management.so debian/additions/enable_encryption.preset etc/mysql/mariadb.conf.d etc/mysql/conf.d/aws_key_management.cnf etc/mysql/mariadb.conf.d +usr/lib/mysql/plugin/aws_key_management.so diff --git a/debian/mariadb-plugin-mroonga.install b/debian/mariadb-plugin-mroonga.install index 5cb81419e5e3e..fedcf62eef0be 100644 --- a/debian/mariadb-plugin-mroonga.install +++ b/debian/mariadb-plugin-mroonga.install @@ -1,5 +1,5 @@ usr/lib/mysql/plugin/ha_mroonga.so +usr/share/mysql/mroonga/AUTHORS +usr/share/mysql/mroonga/COPYING usr/share/mysql/mroonga/install.sql usr/share/mysql/mroonga/uninstall.sql -usr/share/mysql/mroonga/COPYING -usr/share/mysql/mroonga/AUTHORS diff --git a/debian/mariadb-plugin-rocksdb.install b/debian/mariadb-plugin-rocksdb.install index ee45a822e0c42..224ef5501c58e 100644 --- a/debian/mariadb-plugin-rocksdb.install +++ b/debian/mariadb-plugin-rocksdb.install @@ -1,4 +1,4 @@ etc/mysql/conf.d/rocksdb.cnf etc/mysql/mariadb.conf.d -usr/lib/mysql/plugin/ha_rocksdb.so usr/bin/mysql_ldb usr/bin/sst_dump +usr/lib/mysql/plugin/ha_rocksdb.so diff --git a/debian/mariadb-server-10.3.install b/debian/mariadb-server-10.3.install index c0af11c5cc0e6..d8de5512b0595 100644 --- a/debian/mariadb-server-10.3.install +++ b/debian/mariadb-server-10.3.install @@ -33,15 +33,15 @@ usr/bin/replace usr/bin/resolve_stack_dump usr/bin/resolveip usr/bin/wsrep_sst_common +usr/bin/wsrep_sst_mariabackup usr/bin/wsrep_sst_mysqldump usr/bin/wsrep_sst_rsync usr/bin/wsrep_sst_xtrabackup usr/bin/wsrep_sst_xtrabackup-v2 -usr/bin/wsrep_sst_mariabackup +usr/lib/mysql/plugin/auth_ed25519.so usr/lib/mysql/plugin/auth_pam.so usr/lib/mysql/plugin/auth_socket.so usr/lib/mysql/plugin/file_key_management.so -usr/lib/mysql/plugin/auth_ed25519.so usr/lib/mysql/plugin/ha_archive.so usr/lib/mysql/plugin/ha_blackhole.so usr/lib/mysql/plugin/ha_federated.so @@ -58,14 +58,6 @@ usr/lib/mysql/plugin/sql_errlog.so usr/lib/mysql/plugin/wsrep_info.so usr/share/apport/package-hooks/source_mariadb-10.3.py usr/share/doc/mariadb-server-10.3/mysqld.sym.gz -usr/share/mysql/errmsg-utf8.txt -usr/share/mysql/fill_help_tables.sql -usr/share/mysql/maria_add_gis_sp_bootstrap.sql -usr/share/mysql/mysql_performance_tables.sql -usr/share/mysql/mysql_system_tables.sql -usr/share/mysql/mysql_system_tables_data.sql -usr/share/mysql/mysql_test_data_timezone.sql -usr/share/mysql/wsrep_notify usr/share/man/man1/aria_chk.1 usr/share/man/man1/aria_dump_log.1 usr/share/man/man1/aria_ftdump.1 @@ -75,27 +67,35 @@ usr/share/man/man1/galera_new_cluster.1 usr/share/man/man1/galera_recovery.1 usr/share/man/man1/mariadb-service-convert.1 usr/share/man/man1/msql2mysql.1 -usr/share/man/man1/myisamchk.1 +usr/share/man/man1/my_print_defaults.1 usr/share/man/man1/myisam_ftdump.1 +usr/share/man/man1/myisamchk.1 usr/share/man/man1/myisamlog.1 usr/share/man/man1/myisampack.1 -usr/share/man/man1/my_print_defaults.1 -usr/share/man/man1/mysqlbinlog.1 usr/share/man/man1/mysql_convert_table_format.1 -usr/share/man/man1/mysqld_multi.1 -usr/share/man/man1/mysqld_safe.1 -usr/share/man/man1/mysqlhotcopy.1 usr/share/man/man1/mysql_plugin.1 usr/share/man/man1/mysql_secure_installation.1 usr/share/man/man1/mysql_setpermission.1 usr/share/man/man1/mysql_tzinfo_to_sql.1 +usr/share/man/man1/mysqlbinlog.1 +usr/share/man/man1/mysqld_multi.1 +usr/share/man/man1/mysqld_safe.1 usr/share/man/man1/mysqld_safe_helper.1 +usr/share/man/man1/mysqlhotcopy.1 usr/share/man/man1/perror.1 usr/share/man/man1/replace.1 -usr/share/man/man1/resolveip.1 usr/share/man/man1/resolve_stack_dump.1 +usr/share/man/man1/resolveip.1 usr/share/man/man1/wsrep_sst_common.1 usr/share/man/man1/wsrep_sst_mysqldump.1 usr/share/man/man1/wsrep_sst_rsync.1 usr/share/man/man1/wsrep_sst_xtrabackup-v2.1 usr/share/man/man1/wsrep_sst_xtrabackup.1 +usr/share/mysql/errmsg-utf8.txt +usr/share/mysql/fill_help_tables.sql +usr/share/mysql/maria_add_gis_sp_bootstrap.sql +usr/share/mysql/mysql_performance_tables.sql +usr/share/mysql/mysql_system_tables.sql +usr/share/mysql/mysql_system_tables_data.sql +usr/share/mysql/mysql_test_data_timezone.sql +usr/share/mysql/wsrep_notify diff --git a/debian/mariadb-server-core-10.3.install b/debian/mariadb-server-core-10.3.install index 6244abd119372..46c116b618de9 100644 --- a/debian/mariadb-server-core-10.3.install +++ b/debian/mariadb-server-core-10.3.install @@ -2,6 +2,10 @@ usr/bin/innochecksum usr/bin/mysql_install_db usr/bin/mysql_upgrade usr/sbin/mysqld +usr/share/man/man1/innochecksum.1 +usr/share/man/man1/mysql_install_db.1 +usr/share/man/man1/mysql_upgrade.1 +usr/share/man/man8/mysqld.8 usr/share/mysql/charsets usr/share/mysql/czech usr/share/mysql/danish @@ -27,7 +31,3 @@ usr/share/mysql/slovak usr/share/mysql/spanish usr/share/mysql/swedish usr/share/mysql/ukrainian -usr/share/man/man8/mysqld.8 -usr/share/man/man1/innochecksum.1 -usr/share/man/man1/mysql_install_db.1 -usr/share/man/man1/mysql_upgrade.1 diff --git a/debian/mariadb-test.install b/debian/mariadb-test.install index 4a3ef436634cc..a3532bde594b5 100644 --- a/debian/mariadb-test.install +++ b/debian/mariadb-test.install @@ -16,9 +16,16 @@ usr/lib/mysql/plugin/mypluglib.so usr/lib/mysql/plugin/qa_auth_client.so usr/lib/mysql/plugin/qa_auth_interface.so usr/lib/mysql/plugin/qa_auth_server.so +usr/share/man/man1/mysql-stress-test.pl.1 +usr/share/man/man1/mysql-test-run.pl.1 +usr/share/man/man1/mysql_client_test.1 +usr/share/man/man1/mysql_client_test_embedded.1 +usr/share/man/man1/mysqltest.1 +usr/share/man/man1/mysqltest_embedded.1 usr/share/mysql/mysql-test/README usr/share/mysql/mysql-test/README-gcov usr/share/mysql/mysql-test/README.stress +usr/share/mysql/mysql-test/dgcov.pl usr/share/mysql/mysql-test/disabled.def usr/share/mysql/mysql-test/lib usr/share/mysql/mysql-test/mysql-stress-test.pl @@ -26,10 +33,3 @@ usr/share/mysql/mysql-test/mysql-test-run.pl usr/share/mysql/mysql-test/purify.supp usr/share/mysql/mysql-test/suite.pm usr/share/mysql/mysql-test/valgrind.supp -usr/share/mysql/mysql-test/dgcov.pl -usr/share/man/man1/mysql-stress-test.pl.1 -usr/share/man/man1/mysql-test-run.pl.1 -usr/share/man/man1/mysql_client_test.1 -usr/share/man/man1/mysql_client_test_embedded.1 -usr/share/man/man1/mysqltest.1 -usr/share/man/man1/mysqltest_embedded.1 diff --git a/debian/mariadb-test.links b/debian/mariadb-test.links index 082680fe5eda4..884b25a81da4b 100644 --- a/debian/mariadb-test.links +++ b/debian/mariadb-test.links @@ -1,2 +1,2 @@ -usr/share/mysql/mysql-test/mysql-test-run.pl usr/share/mysql/mysql-test/mysql-test-run usr/share/mysql/mysql-test/mysql-test-run.pl usr/share/mysql/mysql-test/mtr +usr/share/mysql/mysql-test/mysql-test-run.pl usr/share/mysql/mysql-test/mysql-test-run diff --git a/debian/mysql-common.install b/debian/mysql-common.install index 5ff685f42fffb..264df6118229f 100644 --- a/debian/mysql-common.install +++ b/debian/mysql-common.install @@ -1,2 +1,2 @@ debian/additions/my.cnf etc/mysql -usr/share/mysql-common/internal-use-only \ No newline at end of file +usr/share/mysql-common/internal-use-only From 1d522353c8c2436eea986d00aa155ab00395ba04 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Otto=20Kek=C3=A4l=C3=A4inen?= Date: Fri, 12 Jan 2018 20:32:12 +0000 Subject: [PATCH 095/106] Deb: wrap-and-sort -a the lib* packages in debian/control This way all the libraries are listed in a logical order and it will later be easier to compare the control file contents to downstream versions. --- debian/control | 120 ++++++++++++++++++++++++------------------------- 1 file changed, 60 insertions(+), 60 deletions(-) diff --git a/debian/control b/debian/control index adc92a5808715..9ace4dc8841c5 100644 --- a/debian/control +++ b/debian/control @@ -38,6 +38,57 @@ Homepage: http://mariadb.org/ Vcs-Git: https://github.com/MariaDB/server.git Vcs-Browser: https://github.com/MariaDB/server/ +Package: libmariadb-dev +Architecture: any +Section: libdevel +Depends: libmariadb3 (= ${binary:Version}), + zlib1g-dev, + ${misc:Depends}, + ${shlibs:Depends} +Breaks: libmariadbclient-dev +Replaces: libmariadbclient-dev +Conflicts: libmariadbclient16-dev +Provides: libmariadbclient-dev +Description: MariaDB database development files + MariaDB is a fast, stable and true multi-user, multi-threaded SQL database + server. SQL (Structured Query Language) is the most popular database query + language in the world. The main goals of MariaDB are speed, robustness and + ease of use. + . + This package includes development libraries and header files. To allow sources + expecting the MariaDB Connector/C to build. Sources that expect the MySQL + Client libraries should use files from the libmariadb-dev-compat package. + +Package: libmariadb-dev-compat +Architecture: any +Multi-Arch: same +Section: libdevel +Priority: extra +Depends: libmariadb-dev (= ${binary:Version}), + ${misc:Depends} +Conflicts: libmariadb-client-lgpl-dev-compat, + libmariadbclient-dev-compat, + libmysqlclient-dev, + libmysqlclient10-dev, + libmysqlclient12-dev, + libmysqlclient14-dev, + libmysqlclient15-dev, + libmysqlclient16-dev +Provides: libmariadb-client-lgpl-dev-compat, + libmariadbclient-dev-compat, + libmysqlclient-dev +Replaces: libmariadb-client-lgpl-dev-compat, + libmariadbclient-dev-compat, + libmysqlclient-dev +Description: MariaDB Connector/C, compatibility symlinks + MariaDB is a fast, stable and true multi-user, multi-threaded SQL database + server. SQL (Structured Query Language) is the most popular database query + language in the world. The main goals of MariaDB are speed, robustness and + ease of use. + . + This package includes compatibility symlinks to allow sources expecting the + MySQL client libraries to be built against MariaDB Connector/C. + Package: libmariadb3 Architecture: any Section: libs @@ -114,56 +165,24 @@ Description: Virtual package to satisfy external libmysqlclient18 depends This package provides compatibility symlinks for binaries that expect to find libmysqlclient.so.18 will automatically use libmariadb.so.3 instead. -Package: libmariadb-dev +Package: libmariadbd-dev Architecture: any Section: libdevel -Depends: libmariadb3 (= ${binary:Version}), - zlib1g-dev, +Provides: libmysqld-dev +Pre-Depends: ${misc:Pre-Depends} +Depends: libmariadb-dev (= ${binary:Version}), + libmariadbd19 (= ${binary:Version}), ${misc:Depends}, ${shlibs:Depends} -Breaks: libmariadbclient-dev -Replaces: libmariadbclient-dev -Conflicts: libmariadbclient16-dev -Provides: libmariadbclient-dev -Description: MariaDB database development files - MariaDB is a fast, stable and true multi-user, multi-threaded SQL database - server. SQL (Structured Query Language) is the most popular database query - language in the world. The main goals of MariaDB are speed, robustness and - ease of use. - . - This package includes development libraries and header files. To allow sources - expecting the MariaDB Connector/C to build. Sources that expect the MySQL - Client libraries should use files from the libmariadb-dev-compat package. - -Package: libmariadb-dev-compat -Architecture: any -Multi-Arch: same -Section: libdevel -Priority: extra -Depends: libmariadb-dev (= ${binary:Version}), - ${misc:Depends} -Conflicts: libmariadb-client-lgpl-dev-compat, - libmariadbclient-dev-compat, - libmysqlclient-dev, - libmysqlclient10-dev, - libmysqlclient12-dev, - libmysqlclient14-dev, - libmysqlclient15-dev, - libmysqlclient16-dev -Provides: libmariadb-client-lgpl-dev-compat, - libmariadbclient-dev-compat, - libmysqlclient-dev -Replaces: libmariadb-client-lgpl-dev-compat, - libmariadbclient-dev-compat, - libmysqlclient-dev -Description: MariaDB Connector/C, compatibility symlinks +Breaks: libmysqld-dev +Replaces: libmysqld-dev +Description: MariaDB embedded database, development files MariaDB is a fast, stable and true multi-user, multi-threaded SQL database server. SQL (Structured Query Language) is the most popular database query language in the world. The main goals of MariaDB are speed, robustness and ease of use. . - This package includes compatibility symlinks to allow sources expecting the - MySQL client libraries to be built against MariaDB Connector/C. + This package includes the embedded server library development and header files. Package: libmariadbd19 Architecture: any @@ -181,25 +200,6 @@ Description: MariaDB embedded database, shared library . This package includes a shared library for embedded MariaDB applications -Package: libmariadbd-dev -Architecture: any -Section: libdevel -Provides: libmysqld-dev -Pre-Depends: ${misc:Pre-Depends} -Depends: libmariadb-dev (= ${binary:Version}), - libmariadbd19 (= ${binary:Version}), - ${misc:Depends}, - ${shlibs:Depends} -Breaks: libmysqld-dev -Replaces: libmysqld-dev -Description: MariaDB embedded database, development files - MariaDB is a fast, stable and true multi-user, multi-threaded SQL database - server. SQL (Structured Query Language) is the most popular database query - language in the world. The main goals of MariaDB are speed, robustness and - ease of use. - . - This package includes the embedded server library development and header files. - Package: mysql-common Architecture: all Depends: ${misc:Depends}, From 33ecf8345d357cbf71bde2b03b641a612db7781f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Tue, 16 Jan 2018 13:55:45 +0200 Subject: [PATCH 096/106] Follow-up fix to MDEV-14441: Fix a potential race condition btr_cur_update_in_place(): Read block->index only once, so that it cannot change to NULL after the first read. When block->index != NULL, it must be equal to index. --- storage/innobase/btr/btr0cur.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/storage/innobase/btr/btr0cur.cc b/storage/innobase/btr/btr0cur.cc index 8146667f7c26a..139e06ab9964e 100644 --- a/storage/innobase/btr/btr0cur.cc +++ b/storage/innobase/btr/btr0cur.cc @@ -3694,7 +3694,7 @@ btr_cur_update_in_place( #ifdef BTR_CUR_HASH_ADAPT { rw_lock_t* ahi_latch = block->index - ? btr_get_search_latch(block->index) : NULL; + ? btr_get_search_latch(index) : NULL; if (ahi_latch) { /* TO DO: Can we skip this if none of the fields index->search_info->curr_n_fields From be85c2dc889b668382106071e712213cd3b1cbcf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Tue, 16 Jan 2018 13:57:30 +0200 Subject: [PATCH 097/106] Mariabackup --prepare: Do not access transactions or data dictionary innobase_start_or_create_for_mysql(): Only start the data dictionary and transaction subsystems in normal server startup and during mariabackup --export. --- storage/innobase/srv/srv0start.cc | 29 ++++++++++++++++++++++------- 1 file changed, 22 insertions(+), 7 deletions(-) diff --git a/storage/innobase/srv/srv0start.cc b/storage/innobase/srv/srv0start.cc index 7bfd1715e9b7c..f45e4c98e8113 100644 --- a/storage/innobase/srv/srv0start.cc +++ b/storage/innobase/srv/srv0start.cc @@ -2246,17 +2246,32 @@ innobase_start_or_create_for_mysql() recv_sys->dblwr.pages.clear(); - if (err == DB_SUCCESS) { - /* Initialize the change buffer. */ - err = dict_boot(); - } - if (err != DB_SUCCESS) { return(srv_init_abort(err)); } - /* This must precede recv_apply_hashed_log_recs(true). */ - trx_sys_init_at_db_start(); + switch (srv_operation) { + case SRV_OPERATION_NORMAL: + case SRV_OPERATION_RESTORE_EXPORT: + /* Initialize the change buffer. */ + err = dict_boot(); + if (err != DB_SUCCESS) { + return(srv_init_abort(err)); + } + /* This must precede + recv_apply_hashed_log_recs(true). */ + trx_sys_init_at_db_start(); + break; + case SRV_OPERATION_RESTORE_DELTA: + case SRV_OPERATION_BACKUP: + ut_ad(!"wrong mariabackup mode"); + /* fall through */ + case SRV_OPERATION_RESTORE: + /* mariabackup --prepare only deals with + the redo log and the data files, not with + transactions or the data dictionary. */ + break; + } if (srv_force_recovery < SRV_FORCE_NO_LOG_REDO) { /* Apply the hashed log records to the From 2281fcf38a9f833e053d99564f0cd766e393dba2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Tue, 16 Jan 2018 13:42:33 +0200 Subject: [PATCH 098/106] Follow-up fix to MDEV-14952 for Mariabackup innodb_init_param(): Initialize btr_ahi_parts=1 for Mariabackup. btr_search_enabled: Let the adaptive hash index be disabled in Mariabackup. This would potentially only matter during --export, and --export performs a table scan, not many index lookups. --- extra/mariabackup/xtrabackup.cc | 3 +++ storage/innobase/btr/btr0sea.cc | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/extra/mariabackup/xtrabackup.cc b/extra/mariabackup/xtrabackup.cc index 20919cf3a818f..d914932170298 100644 --- a/extra/mariabackup/xtrabackup.cc +++ b/extra/mariabackup/xtrabackup.cc @@ -1461,6 +1461,9 @@ innodb_init_param(void) srv_page_size = 0; srv_page_size_shift = 0; +#ifdef BTR_CUR_HASH_ADAPT + btr_ahi_parts = 1; +#endif /* BTR_CUR_HASH_ADAPT */ if (innobase_page_size != (1LL << 14)) { int n_shift = (int)get_bit_shift((ulint) innobase_page_size); diff --git a/storage/innobase/btr/btr0sea.cc b/storage/innobase/btr/btr0sea.cc index 1a70dce1c7b5f..3ca50e00b42c5 100644 --- a/storage/innobase/btr/btr0sea.cc +++ b/storage/innobase/btr/btr0sea.cc @@ -45,7 +45,7 @@ Created 2/17/1996 Heikki Tuuri /** Is search system enabled. Search system is protected by array of latches. */ -char btr_search_enabled = true; +char btr_search_enabled; /** Number of adaptive hash index partition. */ ulong btr_ahi_parts; From d87531a6a053fdf8bc828857d9cdc11a97026ad6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Tue, 16 Jan 2018 14:05:06 +0200 Subject: [PATCH 099/106] Follow-up to MDEV-14952: Remove some more btr_get_search_latch() Replace some !rw_lock_own() assertions with the stronger !btr_search_own_any(). Remove some redundant btr_get_search_latch() calls. btr_search_update_hash_ref(): Remove a duplicated assertion. btr_search_build_page_hash_index(): Remove a duplicated assertion. rw_lock_s_lock() asserts that the latch is not being held. btr_search_disable_ref_count(): Remove an assertion. The only caller is acquiring all adaptive hash index latches. --- storage/innobase/btr/btr0sea.cc | 32 ++++++++++++----------------- storage/innobase/include/btr0sea.ic | 4 ++-- 2 files changed, 15 insertions(+), 21 deletions(-) diff --git a/storage/innobase/btr/btr0sea.cc b/storage/innobase/btr/btr0sea.cc index 3ca50e00b42c5..ccd3e2f129f58 100644 --- a/storage/innobase/btr/btr0sea.cc +++ b/storage/innobase/btr/btr0sea.cc @@ -189,13 +189,13 @@ will not guarantee success. @param[in] index index handler */ static void -btr_search_check_free_space_in_heap(dict_index_t* index) +btr_search_check_free_space_in_heap(const dict_index_t* index) { hash_table_t* table; mem_heap_t* heap; - ut_ad(!rw_lock_own(btr_get_search_latch(index), RW_LOCK_S)); - ut_ad(!rw_lock_own(btr_get_search_latch(index), RW_LOCK_X)); + ut_ad(!btr_search_own_any(RW_LOCK_S)); + ut_ad(!btr_search_own_any(RW_LOCK_X)); table = btr_get_search_table(index); @@ -345,9 +345,6 @@ btr_search_disable_ref_count( for (index = dict_table_get_first_index(table); index != NULL; index = dict_table_get_next_index(index)) { - - ut_ad(rw_lock_own(btr_get_search_latch(index), RW_LOCK_X)); - index->search_info->ref_count = 0; } } @@ -464,8 +461,8 @@ btr_search_info_update_hash( ulint n_unique; int cmp; - ut_ad(!rw_lock_own(btr_get_search_latch(index), RW_LOCK_S)); - ut_ad(!rw_lock_own(btr_get_search_latch(index), RW_LOCK_X)); + ut_ad(!btr_search_own_any(RW_LOCK_S)); + ut_ad(!btr_search_own_any(RW_LOCK_X)); if (dict_index_is_ibuf(index)) { /* So many deletes are performed on an insert buffer tree @@ -572,14 +569,14 @@ semaphore, to save CPU time! Do not assume the fields are consistent. @param[in,out] block buffer block @param[in] cursor cursor */ static -ibool +bool btr_search_update_block_hash_info( btr_search_t* info, buf_block_t* block, const btr_cur_t* cursor) { - ut_ad(!rw_lock_own(btr_get_search_latch(cursor->index), RW_LOCK_S)); - ut_ad(!rw_lock_own(btr_get_search_latch(cursor->index), RW_LOCK_X)); + ut_ad(!btr_search_own_any(RW_LOCK_S)); + ut_ad(!btr_search_own_any(RW_LOCK_X)); ut_ad(rw_lock_own(&block->lock, RW_LOCK_S) || rw_lock_own(&block->lock, RW_LOCK_X)); @@ -626,11 +623,11 @@ btr_search_update_block_hash_info( /* Build a new hash index on the page */ - return(TRUE); + return(true); } } - return(FALSE); + return(false); } /** Updates a hash node reference when it has been unsuccessfully used in a @@ -670,8 +667,8 @@ btr_search_update_hash_ref( } ut_ad(block->page.id.space() == index->space); - ut_a(index == cursor->index); - ut_a(!dict_index_is_ibuf(index)); + ut_ad(index == cursor->index); + ut_ad(!dict_index_is_ibuf(index)); if ((info->n_hash_potential > 0) && (block->curr_n_fields == info->n_fields) @@ -696,7 +693,6 @@ btr_search_update_hash_ref( if (UNIV_LIKELY_NULL(heap)) { mem_heap_free(heap); } - ut_ad(rw_lock_own(btr_get_search_latch(index), RW_LOCK_X)); ha_insert_for_fold(btr_get_search_table(index), fold, block, rec); @@ -948,10 +944,9 @@ btr_search_guess_on_hash( } } else { ut_ad(btr_search_enabled); + ut_ad(rw_lock_own(ahi_latch, RW_LOCK_S)); } - ut_ad(rw_lock_own(btr_get_search_latch(index), RW_LOCK_S)); - rec = (rec_t*) ha_search_and_get_data( btr_get_search_table(index), fold); @@ -1379,7 +1374,6 @@ btr_search_build_page_hash_index( ut_a(!dict_index_is_ibuf(index)); ut_ad(page_is_leaf(block->frame)); - ut_ad(!rw_lock_own(btr_get_search_latch(index), RW_LOCK_X)); ut_ad(rw_lock_own(&(block->lock), RW_LOCK_S) || rw_lock_own(&(block->lock), RW_LOCK_X)); diff --git a/storage/innobase/include/btr0sea.ic b/storage/innobase/include/btr0sea.ic index 65b953961dfcd..e0052a98639eb 100644 --- a/storage/innobase/include/btr0sea.ic +++ b/storage/innobase/include/btr0sea.ic @@ -61,8 +61,8 @@ btr_search_info_update( dict_index_t* index, /*!< in: index of the cursor */ btr_cur_t* cursor) /*!< in: cursor which was just positioned */ { - ut_ad(!rw_lock_own(btr_get_search_latch(index), RW_LOCK_S)); - ut_ad(!rw_lock_own(btr_get_search_latch(index), RW_LOCK_X)); + ut_ad(!btr_search_own_any(RW_LOCK_S)); + ut_ad(!btr_search_own_any(RW_LOCK_X)); if (dict_index_is_spatial(index) || !btr_search_enabled) { return; From 81378b394763e56d57c4a4fc3c20244cc0ee9cc5 Mon Sep 17 00:00:00 2001 From: Alexander Barkov Date: Tue, 16 Jan 2018 16:09:51 +0400 Subject: [PATCH 100/106] Moving a change_list related methods from THD to Item_change_list 1. Moving the following methods from THD to Item_change_list: nocheck_register_item_tree_change() check_and_register_item_tree_change() rollback_item_tree_changes() as they work only with the "change_list" member and don't require anything else from THD. 2. Deriving THD from Item_change_list This change will help to fix "MDEV-14603 signal 11 with short stacktrace" easier. --- sql/sp_head.cc | 8 ++++---- sql/sql_class.cc | 14 +++++++++----- sql/sql_class.h | 37 +++++++++++++++++++++++-------------- sql/sql_parse.cc | 2 +- sql/sql_prepare.cc | 8 ++++---- 5 files changed, 41 insertions(+), 28 deletions(-) diff --git a/sql/sp_head.cc b/sql/sp_head.cc index 7816af398c29c..8b05d14f2dec6 100644 --- a/sql/sp_head.cc +++ b/sql/sp_head.cc @@ -1249,7 +1249,7 @@ sp_head::execute(THD *thd, bool merge_da_on_success) We should also save Item tree change list to avoid rollback something too early in the calling query. */ - thd->change_list.move_elements_to(&old_change_list); + thd->Item_change_list::move_elements_to(&old_change_list); /* Cursors will use thd->packet, so they may corrupt data which was prepared for sending by upper level. OTOH cursors in the same routine can share this @@ -1389,8 +1389,8 @@ sp_head::execute(THD *thd, bool merge_da_on_success) /* Restore all saved */ thd->server_status= (thd->server_status & ~status_backup_mask) | old_server_status; old_packet.swap(thd->packet); - DBUG_ASSERT(thd->change_list.is_empty()); - old_change_list.move_elements_to(&thd->change_list); + DBUG_ASSERT(thd->Item_change_list::is_empty()); + old_change_list.move_elements_to(thd); thd->lex= old_lex; thd->set_query_id(old_query_id); DBUG_ASSERT(!thd->derived_tables); @@ -2976,7 +2976,7 @@ sp_lex_keeper::reset_lex_and_exec_core(THD *thd, uint *nextp, bool parent_modified_non_trans_table= thd->transaction.stmt.modified_non_trans_table; thd->transaction.stmt.modified_non_trans_table= FALSE; DBUG_ASSERT(!thd->derived_tables); - DBUG_ASSERT(thd->change_list.is_empty()); + DBUG_ASSERT(thd->Item_change_list::is_empty()); /* Use our own lex. We should not save old value since it is saved/restored in diff --git a/sql/sql_class.cc b/sql/sql_class.cc index c8eb8ed31289b..d0b45441d83b4 100644 --- a/sql/sql_class.cc +++ b/sql/sql_class.cc @@ -2605,8 +2605,10 @@ struct Item_change_record: public ilink thd->mem_root (due to possible set_n_backup_active_arena called for thd). */ -void THD::nocheck_register_item_tree_change(Item **place, Item *old_value, - MEM_ROOT *runtime_memroot) +void +Item_change_list::nocheck_register_item_tree_change(Item **place, + Item *old_value, + MEM_ROOT *runtime_memroot) { Item_change_record *change; DBUG_ENTER("THD::nocheck_register_item_tree_change"); @@ -2647,8 +2649,10 @@ void THD::nocheck_register_item_tree_change(Item **place, Item *old_value, changes to substitute the same reference at both locations L1 and L2. */ -void THD::check_and_register_item_tree_change(Item **place, Item **new_value, - MEM_ROOT *runtime_memroot) +void +Item_change_list::check_and_register_item_tree_change(Item **place, + Item **new_value, + MEM_ROOT *runtime_memroot) { Item_change_record *change; I_List_iterator it(change_list); @@ -2663,7 +2667,7 @@ void THD::check_and_register_item_tree_change(Item **place, Item **new_value, } -void THD::rollback_item_tree_changes() +void Item_change_list::rollback_item_tree_changes() { I_List_iterator it(change_list); Item_change_record *change; diff --git a/sql/sql_class.h b/sql/sql_class.h index e5aaba9b4075f..bcd43cd62cd46 100644 --- a/sql/sql_class.h +++ b/sql/sql_class.h @@ -1279,7 +1279,21 @@ class Security_context { */ struct Item_change_record; -typedef I_List Item_change_list; +class Item_change_list +{ + I_List change_list; +public: + void nocheck_register_item_tree_change(Item **place, Item *old_value, + MEM_ROOT *runtime_memroot); + void check_and_register_item_tree_change(Item **place, Item **new_value, + MEM_ROOT *runtime_memroot); + void rollback_item_tree_changes(); + void move_elements_to(Item_change_list *to) + { + change_list.move_elements_to(&to->change_list); + } + bool is_empty() { return change_list.is_empty(); } +}; /** @@ -2040,6 +2054,14 @@ void dbug_serve_apcs(THD *thd, int n_calls); */ class THD :public Statement, + /* + This is to track items changed during execution of a prepared + statement/stored procedure. It's created by + nocheck_register_item_tree_change() in memory root of THD, + and freed in rollback_item_tree_changes(). + For conventional execution it's always empty. + */ + public Item_change_list, public MDL_context_owner, public Open_tables_state { @@ -2510,14 +2532,6 @@ class THD :public Statement, #ifdef SIGNAL_WITH_VIO_CLOSE Vio* active_vio; #endif - /* - This is to track items changed during execution of a prepared - statement/stored procedure. It's created by - nocheck_register_item_tree_change() in memory root of THD, and freed in - rollback_item_tree_changes(). For conventional execution it's always - empty. - */ - Item_change_list change_list; /* A permanent memory area of the statement. For conventional @@ -3599,11 +3613,6 @@ class THD :public Statement, */ memcpy((char*) place, new_value, sizeof(*new_value)); } - void nocheck_register_item_tree_change(Item **place, Item *old_value, - MEM_ROOT *runtime_memroot); - void check_and_register_item_tree_change(Item **place, Item **new_value, - MEM_ROOT *runtime_memroot); - void rollback_item_tree_changes(); /* Cleanup statement parse state (parse tree, lex) and execution diff --git a/sql/sql_parse.cc b/sql/sql_parse.cc index 6d068eb99ac5e..21abc1a248c2a 100644 --- a/sql/sql_parse.cc +++ b/sql/sql_parse.cc @@ -7910,7 +7910,7 @@ void mysql_parse(THD *thd, char *rawbuf, uint length, sp_cache_enforce_limit(thd->sp_func_cache, stored_program_cache_size); thd->end_statement(); thd->cleanup_after_query(); - DBUG_ASSERT(thd->change_list.is_empty()); + DBUG_ASSERT(thd->Item_change_list::is_empty()); } else { diff --git a/sql/sql_prepare.cc b/sql/sql_prepare.cc index 390b70877f5f3..4e847fb9ff3c1 100644 --- a/sql/sql_prepare.cc +++ b/sql/sql_prepare.cc @@ -3930,7 +3930,7 @@ bool Prepared_statement::prepare(const char *packet, uint packet_len) If called from a stored procedure, ensure that we won't rollback external changes when cleaning up after validation. */ - DBUG_ASSERT(thd->change_list.is_empty()); + DBUG_ASSERT(thd->Item_change_list::is_empty()); /* Marker used to release metadata locks acquired while the prepared @@ -4407,7 +4407,7 @@ Prepared_statement::execute_server_runnable(Server_runnable *server_runnable) bool error; Query_arena *save_stmt_arena= thd->stmt_arena; Item_change_list save_change_list; - thd->change_list.move_elements_to(&save_change_list); + thd->Item_change_list::move_elements_to(&save_change_list); state= STMT_CONVENTIONAL_EXECUTION; @@ -4426,7 +4426,7 @@ Prepared_statement::execute_server_runnable(Server_runnable *server_runnable) thd->restore_backup_statement(this, &stmt_backup); thd->stmt_arena= save_stmt_arena; - save_change_list.move_elements_to(&thd->change_list); + save_change_list.move_elements_to(thd); /* Items and memory will freed in destructor */ @@ -4654,7 +4654,7 @@ bool Prepared_statement::execute(String *expanded_query, bool open_cursor) If the free_list is not empty, we'll wrongly free some externally allocated items when cleaning up after execution of this statement. */ - DBUG_ASSERT(thd->change_list.is_empty()); + DBUG_ASSERT(thd->Item_change_list::is_empty()); /* The only case where we should have items in the thd->free_list is From f44017384afa7a10df9866423291fc764046043f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Tue, 16 Jan 2018 20:02:38 +0200 Subject: [PATCH 101/106] MDEV-14968 On upgrade, InnoDB reports "started; log sequence number 0" srv_prepare_to_delete_redo_log_files(): Initialize srv_start_lsn. --- storage/innobase/srv/srv0start.cc | 1 + 1 file changed, 1 insertion(+) diff --git a/storage/innobase/srv/srv0start.cc b/storage/innobase/srv/srv0start.cc index f45e4c98e8113..d3b7fb6bf947f 100644 --- a/storage/innobase/srv/srv0start.cc +++ b/storage/innobase/srv/srv0start.cc @@ -1446,6 +1446,7 @@ srv_prepare_to_delete_redo_log_files( << " bytes; LSN=" << flushed_lsn; } + srv_start_lsn = flushed_lsn; /* Flush the old log files. */ log_mutex_exit(); From edb637591016bc6ecde4c0e98f829425174a6118 Mon Sep 17 00:00:00 2001 From: Sergei Golubchik Date: Sat, 13 Jan 2018 12:15:46 +0100 Subject: [PATCH 102/106] compilation warning on windows --- sql/partition_element.h | 2 +- sql/partition_info.cc | 2 +- sql/partition_info.h | 6 +++--- sql/sql_yacc.yy | 2 +- sql/table.h | 6 +----- storage/innobase/include/row0mysql.h | 2 +- 6 files changed, 8 insertions(+), 12 deletions(-) diff --git a/sql/partition_element.h b/sql/partition_element.h index 535c320c626f8..fdf2cb3900d38 100644 --- a/sql/partition_element.h +++ b/sql/partition_element.h @@ -202,7 +202,7 @@ class partition_element :public Sql_alloc engine_type(NULL), connect_string(null_clex_str), part_state(PART_NORMAL), nodegroup_id(UNDEF_NODEGROUP), has_null_value(FALSE), signed_flag(FALSE), max_value(FALSE), - id(UINT32_MAX), + id(UINT_MAX32), empty(true) {} partition_element(partition_element *part_elem) diff --git a/sql/partition_info.cc b/sql/partition_info.cc index 65df87bee041f..819ba69a1475b 100644 --- a/sql/partition_info.cc +++ b/sql/partition_info.cc @@ -1001,7 +1001,7 @@ bool partition_info::vers_setup_expression(THD * thd, uint32 alter_add) continue; } /* Newly added element is inserted before AS_OF_NOW. */ - if (el->id == UINT32_MAX || el->type() == partition_element::CURRENT) + if (el->id == UINT_MAX32 || el->type() == partition_element::CURRENT) { DBUG_ASSERT(table && table->s); Vers_min_max_stats *stat_trx_end= new (&table->s->mem_root) diff --git a/sql/partition_info.h b/sql/partition_info.h index 23e94661e6881..67a62874fcd97 100644 --- a/sql/partition_info.h +++ b/sql/partition_info.h @@ -57,11 +57,11 @@ struct Vers_part_info : public Sql_alloc { if (now_part) { - DBUG_ASSERT(now_part->id != UINT32_MAX); + DBUG_ASSERT(now_part->id != UINT_MAX32); DBUG_ASSERT(now_part->type() == partition_element::CURRENT); DBUG_ASSERT(!fully || (bool) hist_part); DBUG_ASSERT(!hist_part || ( - hist_part->id != UINT32_MAX && + hist_part->id != UINT_MAX32 && hist_part->type() == partition_element::HISTORY)); return true; } @@ -406,7 +406,7 @@ class partition_info : public Sql_alloc { DBUG_ASSERT(table && table->s); DBUG_ASSERT(vers_info && vers_info->initialized()); - DBUG_ASSERT(table->s->hist_part_id != UINT32_MAX); + DBUG_ASSERT(table->s->hist_part_id != UINT_MAX32); if (table->s->hist_part_id == vers_info->hist_part->id) return vers_info->hist_part; diff --git a/sql/sql_yacc.yy b/sql/sql_yacc.yy index 895248f4824dd..4d55ebf8bea2b 100644 --- a/sql/sql_yacc.yy +++ b/sql/sql_yacc.yy @@ -5500,7 +5500,7 @@ opt_part_values: else { part_info->vers_init_info(thd); - elem->id= UINT32_MAX; + elem->id= UINT_MAX32; } DBUG_ASSERT(part_info->vers_info); if (part_info->vers_info->now_part) diff --git a/sql/table.h b/sql/table.h index 3882bfad27916..48e9bbb42ce12 100644 --- a/sql/table.h +++ b/sql/table.h @@ -583,10 +583,6 @@ enum vers_sys_type_t VERS_TRX_ID }; -#ifndef UINT32_MAX -#define UINT32_MAX (4294967295U) -#endif - /** This structure is shared between different table objects. There is one instance of table share per one table in the database. @@ -791,7 +787,7 @@ struct TABLE_SHARE void vers_init() { - hist_part_id= UINT32_MAX; + hist_part_id= UINT_MAX32; busy_rotation= false; stat_trx= NULL; stat_serial= 0; diff --git a/storage/innobase/include/row0mysql.h b/storage/innobase/include/row0mysql.h index 2ba23b6e68c23..407e1705a0fa5 100644 --- a/storage/innobase/include/row0mysql.h +++ b/storage/innobase/include/row0mysql.h @@ -869,7 +869,7 @@ struct row_prebuilt_t { { ut_ad(col < n_template); ut_ad(mysql_template); - for (int i = col; i < n_template; ++i) { + for (ulint i = col; i < n_template; ++i) { const mysql_row_templ_t* templ = &mysql_template[i]; if (!templ->is_virtual && templ->col_no == col) { return templ; From 1ea2b2956b8204e58a515f8828f8749f321a0692 Mon Sep 17 00:00:00 2001 From: Sergei Golubchik Date: Sun, 14 Jan 2018 12:37:55 +0100 Subject: [PATCH 103/106] Revert "MDEV-14786 Server crashes in Item_cond::transform on 2nd execution of SP querying from a view [fixes #436]" This reverts commit 7069071d7de774dcf28f73b6a968bcb730a12885 And add a test to show that optimization steps that a) are repeated for every execution b) create new items cannot be done on the statement arena --- mysql-test/r/sp-big.result | 29 +++++++++++++++++++++++++-- mysql-test/t/sp-big.test | 40 +++++++++++++++++++++++++++++++++----- sql/sql_select.cc | 4 ---- 3 files changed, 62 insertions(+), 11 deletions(-) diff --git a/mysql-test/r/sp-big.result b/mysql-test/r/sp-big.result index 9765508859c28..e12136eb36d7e 100644 --- a/mysql-test/r/sp-big.result +++ b/mysql-test/r/sp-big.result @@ -1,5 +1,3 @@ -drop procedure if exists test.longprocedure; -drop table if exists t1; create table t1 (a int); insert into t1 values (1),(2),(3); length @@ -60,3 +58,30 @@ f1 This is a test case for for Bug#9819 drop procedure p1; drop table t1, t2; +create table t1 ( +`id1` int unsigned not null default '0', +`id2` int unsigned not null default '0', +`link_type` int unsigned not null default '0', +`visibility` tinyint not null default '0', +`data` varchar(255) not null default '', +`time` int unsigned not null default '0', +`version` int unsigned not null default '0', +primary key (id1, link_type, visibility, id2) +) default collate=latin1_bin; +create procedure select_test() +begin +declare id1_cond int; +set id1_cond = 1; +while id1_cond <= 10000 do +select count(*) as cnt from (select id1 from t1 force index (primary) where id1 = id1_cond and link_type = 1 and visibility = 1 order by id2 desc) as t into @cnt; +set id1_cond = id1_cond + 1; +end while; +end// +insert t1 select seq, seq, 1, 1, seq, seq, seq from seq_1_to_2000; +set @before=unix_timestamp(); +call select_test(); +select unix_timestamp() - @before < 60; +unix_timestamp() - @before < 60 +1 +drop procedure select_test; +drop table t1; diff --git a/mysql-test/t/sp-big.test b/mysql-test/t/sp-big.test index 6541e546e431f..4220541697eb7 100644 --- a/mysql-test/t/sp-big.test +++ b/mysql-test/t/sp-big.test @@ -1,11 +1,7 @@ # # Bug #11602: SP with very large body not handled well # - ---disable_warnings -drop procedure if exists test.longprocedure; -drop table if exists t1; ---enable_warnings +source include/have_sequence.inc; create table t1 (a int); insert into t1 values (1),(2),(3); @@ -85,3 +81,37 @@ select f1 from t1 limit 1; select f1 from t2 limit 1; drop procedure p1; drop table t1, t2; + +# +# Loops with many iterations +# (Item_equal must be created in the execution arena) +# +create table t1 ( + `id1` int unsigned not null default '0', + `id2` int unsigned not null default '0', + `link_type` int unsigned not null default '0', + `visibility` tinyint not null default '0', + `data` varchar(255) not null default '', + `time` int unsigned not null default '0', + `version` int unsigned not null default '0', + primary key (id1, link_type, visibility, id2) +) default collate=latin1_bin; + +delimiter //; +create procedure select_test() +begin + declare id1_cond int; + set id1_cond = 1; + while id1_cond <= 10000 do + select count(*) as cnt from (select id1 from t1 force index (primary) where id1 = id1_cond and link_type = 1 and visibility = 1 order by id2 desc) as t into @cnt; + set id1_cond = id1_cond + 1; + end while; +end// +delimiter ;// + +insert t1 select seq, seq, 1, 1, seq, seq, seq from seq_1_to_2000; +set @before=unix_timestamp(); +call select_test(); +select unix_timestamp() - @before < 60; +drop procedure select_test; +drop table t1; diff --git a/sql/sql_select.cc b/sql/sql_select.cc index 1781b7ccfcb8d..0527eef4fdee1 100644 --- a/sql/sql_select.cc +++ b/sql/sql_select.cc @@ -14677,8 +14677,6 @@ static COND* substitute_for_best_equal_field(THD *thd, JOIN_TAB *context_tab, Item_equal *item_equal; COND *org_cond= cond; // Return this in case of fatal error - Query_arena_stmt on_stmt_arena(thd); - if (cond->type() == Item::COND_ITEM) { List *cond_list= ((Item_cond*) cond)->argument_list(); @@ -15800,8 +15798,6 @@ optimize_cond(JOIN *join, COND *conds, THD *thd= join->thd; DBUG_ENTER("optimize_cond"); - Query_arena_stmt on_stmt_arena(thd); - if (!conds) { *cond_value= Item::COND_TRUE; From 715a507e3368451b824f211dea34a55c5d4dac1d Mon Sep 17 00:00:00 2001 From: Sergei Golubchik Date: Sun, 14 Jan 2018 16:03:35 +0100 Subject: [PATCH 104/106] MDEV-14786 Server crashes in Item_cond::transform on 2nd execution of SP querying from a view instead of skipping invalid items in setup_conds(), don't pass them into a JOIN at all (test case in versioning.select2) --- sql/sql_base.cc | 4 ---- sql/sql_union.cc | 3 ++- 2 files changed, 2 insertions(+), 5 deletions(-) diff --git a/sql/sql_base.cc b/sql/sql_base.cc index 6178dd3ac6a1f..45efa6330b0bf 100644 --- a/sql/sql_base.cc +++ b/sql/sql_base.cc @@ -7997,10 +7997,6 @@ int setup_conds(THD *thd, TABLE_LIST *tables, List &leaves, TABLE_LIST *derived= select_lex->master_unit()->derived; DBUG_ENTER("setup_conds"); - /* Do not fix conditions for the derived tables that have been merged */ - if (derived && derived->merged) - DBUG_RETURN(0); - select_lex->is_item_list_lookup= 0; thd->mark_used_columns= MARK_COLUMNS_READ; diff --git a/sql/sql_union.cc b/sql/sql_union.cc index 82745b619295b..dda57cc5b4de1 100644 --- a/sql/sql_union.cc +++ b/sql/sql_union.cc @@ -645,6 +645,7 @@ bool st_select_lex_unit::prepare_join(THD *thd_arg, SELECT_LEX *sl, bool is_union_select) { DBUG_ENTER("st_select_lex_unit::prepare_join"); + TABLE_LIST *derived= sl->master_unit()->derived; bool can_skip_order_by; sl->options|= SELECT_NO_UNLOCK; JOIN *join= new JOIN(thd_arg, sl->item_list, @@ -660,7 +661,7 @@ bool st_select_lex_unit::prepare_join(THD *thd_arg, SELECT_LEX *sl, saved_error= join->prepare(sl->table_list.first, sl->with_wild, - sl->where, + (derived && derived->merged ? NULL : sl->where), (can_skip_order_by ? 0 : sl->order_list.elements) + sl->group_list.elements, From 04eef79bf93fc825cf2651633bcbcc21b20e3a83 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Wed, 17 Jan 2018 11:28:02 +0200 Subject: [PATCH 105/106] Do not define unused function mark_blocks_free() Follow-up to commit 9ec19b9b41804d8d4491f0f58ac06ae417e02ffa --- mysys/my_alloc.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/mysys/my_alloc.c b/mysys/my_alloc.c index 3df731279986f..b2a1264bd2bfb 100644 --- a/mysys/my_alloc.c +++ b/mysys/my_alloc.c @@ -325,7 +325,8 @@ void *multi_alloc_root(MEM_ROOT *root, ...) #define TRASH_MEM(X) TRASH(((char*)(X) + ((X)->size-(X)->left)), (X)->left) -/* Mark all data in blocks free for reusage */ +#if !(defined(HAVE_valgrind) && defined(EXTRA_DEBUG)) +/** Mark all data in blocks free for reusage */ static inline void mark_blocks_free(MEM_ROOT* root) { @@ -355,6 +356,7 @@ static inline void mark_blocks_free(MEM_ROOT* root) root->first_block_usage= 0; root->block_num= 4; } +#endif /* From 656f66def27b7a2cf42a28f873f1eeef0416aa71 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20M=C3=A4kel=C3=A4?= Date: Wed, 17 Jan 2018 15:53:30 +0200 Subject: [PATCH 106/106] Follow-up fix to MDEV-14585 Automatically remove #sql- tables in InnoDB dictionary during recovery If InnoDB is killed while ALTER TABLE...ALGORITHM=COPY is in progress, after recovery there could be undo log records some records that were inserted into an intermediate copy of the table. Due to these undo log records, InnoDB would resurrect locks at recovery, and the intermediate table would be locked while we are trying to drop it. This would cause a call to row_rename_table_for_mysql(), either from row_mysql_drop_garbage_tables() or from the rollback of a RENAME operation that was part of the ALTER TABLE. row_rename_table_for_mysql(): Do not attempt to parse FOREIGN KEY constraints when renaming from #sql-something to #sql-something-else, because it does not make any sense. row_drop_table_for_mysql(): When deferring DROP TABLE due to locks, do not rename the table if its name already starts with the #sql- prefix, which is what row_mysql_drop_garbage_tables() uses. Previously, the too strict prefix #sql-ib was used, and some tables were renamed unnecessarily. --- .../suite/innodb/r/rename_table_debug.result | 31 +++++++++++++-- .../suite/innodb/t/rename_table_debug.test | 39 ++++++++++++++++++- storage/innobase/row/row0mysql.cc | 4 +- 3 files changed, 66 insertions(+), 8 deletions(-) diff --git a/mysql-test/suite/innodb/r/rename_table_debug.result b/mysql-test/suite/innodb/r/rename_table_debug.result index 912ed9de48b09..7c9b961dee554 100644 --- a/mysql-test/suite/innodb/r/rename_table_debug.result +++ b/mysql-test/suite/innodb/r/rename_table_debug.result @@ -1,5 +1,5 @@ -CREATE TABLE t1 (a INT UNSIGNED PRIMARY KEY) ENGINE=InnoDB; -INSERT INTO t1 VALUES(42); +CREATE TABLE t1 (a SERIAL, b INT, c INT, d INT) ENGINE=InnoDB; +INSERT INTO t1 () VALUES (); connect con1,localhost,root,,test; SET DEBUG_SYNC='before_rename_table_commit SIGNAL renamed WAIT_FOR ever'; RENAME TABLE t1 TO t2; @@ -7,6 +7,29 @@ connection default; SET DEBUG_SYNC='now WAIT_FOR renamed'; disconnect con1; SELECT * FROM t1; -a -42 +a b c d +1 NULL NULL NULL +BEGIN; +COMMIT; +UPDATE t1 SET b=a%7, c=a%11, d=a%13; +SET DEBUG_DBUG='+d,crash_commit_before'; +ALTER TABLE t1 +ADD INDEX(b,c,d,a),ADD INDEX(b,c,a,d),ADD INDEX(b,a,c,d),ADD INDEX(b,a,d,c), +ADD INDEX(b,d,a,c),ADD INDEX(b,d,c,a),ADD INDEX(a,b,c,d),ADD INDEX(a,b,d,c), +ADD INDEX(a,c,b,d),ADD INDEX(a,c,d,b),ADD INDEX(a,d,b,c),ADD INDEX(a,d,c,b), +ADD INDEX(c,a,b,d),ADD INDEX(c,a,d,b),ADD INDEX(c,b,a,d),ADD INDEX(c,b,d,a), +ADD INDEX(c,d,a,b),ADD INDEX(c,d,b,a),ADD INDEX(d,a,b,c),ADD INDEX(d,a,c,b), +ADD INDEX(d,b,a,c),ADD INDEX(d,b,c,a),ADD INDEX(d,c,a,b),ADD INDEX(d,c,b,a), +ADD INDEX(a,b,c), ADD INDEX(a,c,b), ADD INDEX(a,c,d), ADD INDEX(a,d,c), +ADD INDEX(a,b,d), ADD INDEX(a,d,b), ADD INDEX(b,c,d), ADD INDEX(b,d,c), +ALGORITHM=COPY; +ERROR HY000: Lost connection to MySQL server during query +CHECK TABLE t1; +Table Op Msg_type Msg_text +test.t1 check status OK +SELECT COUNT(*) FROM t1; +COUNT(*) +1000 DROP TABLE t1; +SET GLOBAL innodb_background_drop_list_empty= +@@GLOBAL.innodb_background_drop_list_empty; diff --git a/mysql-test/suite/innodb/t/rename_table_debug.test b/mysql-test/suite/innodb/t/rename_table_debug.test index 4620f7bef221f..20af12dc15cb9 100644 --- a/mysql-test/suite/innodb/t/rename_table_debug.test +++ b/mysql-test/suite/innodb/t/rename_table_debug.test @@ -3,8 +3,10 @@ --source include/have_debug_sync.inc --source include/not_embedded.inc -CREATE TABLE t1 (a INT UNSIGNED PRIMARY KEY) ENGINE=InnoDB; -INSERT INTO t1 VALUES(42); +LET $datadir= `SELECT @@datadir`; + +CREATE TABLE t1 (a SERIAL, b INT, c INT, d INT) ENGINE=InnoDB; +INSERT INTO t1 () VALUES (); --connect (con1,localhost,root,,test) SET DEBUG_SYNC='before_rename_table_commit SIGNAL renamed WAIT_FOR ever'; @@ -16,4 +18,37 @@ SET DEBUG_SYNC='now WAIT_FOR renamed'; --source include/restart_mysqld.inc --disconnect con1 SELECT * FROM t1; + +let $c = 999; +BEGIN; +--disable_query_log +while ($c) { +INSERT INTO t1() VALUES(); +dec $c; +} +--enable_query_log +COMMIT; +UPDATE t1 SET b=a%7, c=a%11, d=a%13; + +--source include/expect_crash.inc +SET DEBUG_DBUG='+d,crash_commit_before'; +--error 2013 +ALTER TABLE t1 +ADD INDEX(b,c,d,a),ADD INDEX(b,c,a,d),ADD INDEX(b,a,c,d),ADD INDEX(b,a,d,c), +ADD INDEX(b,d,a,c),ADD INDEX(b,d,c,a),ADD INDEX(a,b,c,d),ADD INDEX(a,b,d,c), +ADD INDEX(a,c,b,d),ADD INDEX(a,c,d,b),ADD INDEX(a,d,b,c),ADD INDEX(a,d,c,b), +ADD INDEX(c,a,b,d),ADD INDEX(c,a,d,b),ADD INDEX(c,b,a,d),ADD INDEX(c,b,d,a), +ADD INDEX(c,d,a,b),ADD INDEX(c,d,b,a),ADD INDEX(d,a,b,c),ADD INDEX(d,a,c,b), +ADD INDEX(d,b,a,c),ADD INDEX(d,b,c,a),ADD INDEX(d,c,a,b),ADD INDEX(d,c,b,a), +ADD INDEX(a,b,c), ADD INDEX(a,c,b), ADD INDEX(a,c,d), ADD INDEX(a,d,c), +ADD INDEX(a,b,d), ADD INDEX(a,d,b), ADD INDEX(b,c,d), ADD INDEX(b,d,c), +ALGORITHM=COPY; +--source include/start_mysqld.inc +CHECK TABLE t1; +SELECT COUNT(*) FROM t1; DROP TABLE t1; +# MDEV-11415 TODO: remove the following +SET GLOBAL innodb_background_drop_list_empty= +@@GLOBAL.innodb_background_drop_list_empty; +# Work around missing crash recovery at the SQL layer. +--remove_files_wildcard $datadir/test #sql-*.frm diff --git a/storage/innobase/row/row0mysql.cc b/storage/innobase/row/row0mysql.cc index 8f9b5296015a1..c3c6e66b3e47d 100644 --- a/storage/innobase/row/row0mysql.cc +++ b/storage/innobase/row/row0mysql.cc @@ -3712,7 +3712,7 @@ row_drop_table_for_mysql( if (table->n_foreign_key_checks_running > 0) { defer: - if (!strstr(table->name.m_name, "/" TEMP_FILE_PREFIX_INNODB)) { + if (!strstr(table->name.m_name, "/" TEMP_FILE_PREFIX)) { heap = mem_heap_create(FN_REFLEN); const char* tmp_name = dict_mem_create_temporary_tablename( @@ -4477,7 +4477,7 @@ row_rename_table_for_mysql( goto funct_exit; - } else if (new_is_tmp) { + } else if (!old_is_tmp && new_is_tmp) { /* MySQL is doing an ALTER TABLE command and it renames the original table to a temporary table name. We want to preserve the original foreign key constraint definitions despite the