From e48ac39c6644114c674f600dbdc4b34788e47091 Mon Sep 17 00:00:00 2001 From: BohuTANG Date: Sat, 3 Oct 2015 17:05:22 +0800 Subject: [PATCH] FT-701 keep all FT files consistency before next checkpoint for backup [summary] When we do hot backup like below: 1) take checkpoint lock 2) lock table for read 3) wirte binary log postion 4) copy redo log to backup dir 5) unlock tables 6) start to copy datas files 6.1) write data by another client 6.2) copy file1 to backup dir 7) release checkpoint lock Between 6.1) and 6.2) the FT may does ft_close and update FT header to new, the new writes are visible in the backup instance, this broke the consistency. It would be better for TokuDB to update FT header only in checkpoint, keep all FT files consistency when we are backing up. Copyright (c) 2015, BohuTANG All rights reserved. Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. --- ft/cachetable/cachetable-internal.h | 3 + ft/cachetable/cachetable.cc | 22 ++++ ft/cachetable/cachetable.h | 4 + ft/ft.cc | 14 ++- ft/tests/ft-in-backup-test.cc | 155 ++++++++++++++++++++++++++++ src/ydb.cc | 10 +- 6 files changed, 204 insertions(+), 4 deletions(-) create mode 100644 ft/tests/ft-in-backup-test.cc diff --git a/ft/cachetable/cachetable-internal.h b/ft/cachetable/cachetable-internal.h index dc6aec922..0a8d94c01 100644 --- a/ft/cachetable/cachetable-internal.h +++ b/ft/cachetable/cachetable-internal.h @@ -411,6 +411,8 @@ class checkpointer { void add_background_job(); void remove_background_job(); void end_checkpoint(void (*testcallback_f)(void*), void* testextra); + void begin_backup(); + void end_backup(); TOKULOGGER get_logger(); // used during begin_checkpoint void increment_num_txns(); @@ -600,6 +602,7 @@ struct cachetable { KIBBUTZ client_kibbutz; // pool of worker threads and jobs to do asynchronously for the client. KIBBUTZ ct_kibbutz; // pool of worker threads and jobs to do asynchronously for the cachetable KIBBUTZ checkpointing_kibbutz; // small pool for checkpointing cloned pairs + bool in_backup; // we are in back up or NOT, default is false char *env_dir; }; diff --git a/ft/cachetable/cachetable.cc b/ft/cachetable/cachetable.cc index 5bba977de..a00cafb26 100644 --- a/ft/cachetable/cachetable.cc +++ b/ft/cachetable/cachetable.cc @@ -250,6 +250,7 @@ int toku_cachetable_create_ex(CACHETABLE *ct_result, long size_limit, CACHETABLE XCALLOC(ct); ct->list.init(); ct->cf_list.init(); + ct->in_backup = false; int num_processors = toku_os_get_number_active_processors(); int checkpointing_nworkers = (num_processors/4) ? num_processors/4 : 1; @@ -2749,6 +2750,27 @@ void toku_cachetable_end_checkpoint(CHECKPOINTER cp, TOKULOGGER UU(logger), cp->end_checkpoint(testcallback_f, testextra); } +// in_backup begin +void toku_cachetable_begin_backup(CACHETABLE ct) +{ + ct->cf_list.read_lock(); + ct->in_backup = true; + ct->cf_list.read_unlock(); +} + +void toku_cachetable_end_backup(CACHETABLE ct) +{ + ct->cf_list.read_lock(); + ct->in_backup = false; + ct->cf_list.read_unlock(); +} + +bool toku_cachefile_in_backup(CACHEFILE cf) +{ + return cf->cachetable->in_backup; +} +// in_backup end + TOKULOGGER toku_cachefile_logger (CACHEFILE cf) { return cf->cachetable->cp.get_logger(); } diff --git a/ft/cachetable/cachetable.h b/ft/cachetable/cachetable.h index 148326562..8a20f2b1e 100644 --- a/ft/cachetable/cachetable.h +++ b/ft/cachetable/cachetable.h @@ -147,6 +147,10 @@ void toku_cachetable_begin_checkpoint (CHECKPOINTER cp, struct tokulogger *logge void toku_cachetable_end_checkpoint(CHECKPOINTER cp, struct tokulogger *logger, void (*testcallback_f)(void*), void * testextra); +void toku_cachetable_begin_backup(CACHETABLE ct); +void toku_cachetable_end_backup(CACHETABLE ct); +// cachefile is in backup or not +bool toku_cachefile_in_backup(CACHEFILE cf); // Shuts down checkpoint thread // Requires no locks be held that are taken by the checkpoint function diff --git a/ft/ft.cc b/ft/ft.cc index 2a0fb6f68..97303b59c 100644 --- a/ft/ft.cc +++ b/ft/ft.cc @@ -282,7 +282,13 @@ static void ft_note_pin_by_checkpoint (CACHEFILE UU(cachefile), void *header_v) FT ft = (FT) header_v; toku_ft_grab_reflock(ft); assert(!ft->pinned_by_checkpoint); + + // because we add in_backup for cachetable, with this case: + // a cachefile in_backup is true and then do a checkpoint + // this assertion fails since nobody references to the ft +#if 0 assert(toku_ft_needed_unlocked(ft)); +#endif ft->pinned_by_checkpoint = true; toku_ft_release_reflock(ft); } @@ -902,8 +908,12 @@ toku_ft_remove_reference(FT ft, bool oplsn_valid, LSN oplsn, remove_ft_ref_callb assert(!needed); } if (!needed) { - // close header - toku_ft_evict_from_memory(ft, oplsn_valid, oplsn); + // close header if we are not in backup + // if close header during backup, the FT files would be inconsistency + bool in_backup = toku_cachefile_in_backup(ft->cf); + if (!in_backup) { + toku_ft_evict_from_memory(ft, oplsn_valid, oplsn); + } } toku_ft_open_close_unlock(); diff --git a/ft/tests/ft-in-backup-test.cc b/ft/tests/ft-in-backup-test.cc new file mode 100644 index 000000000..863272dbe --- /dev/null +++ b/ft/tests/ft-in-backup-test.cc @@ -0,0 +1,155 @@ +/* -*- mode: C++; c-basic-offset: 4; indent-tabs-mode: nil -*- */ +// vim: ft=cpp:expandtab:ts=8:sw=4:softtabstop=4: +#ident "$Id$" +/*====== +This file is part of PerconaFT. + + +Copyright (c) 2006, 2015, Percona and/or its affiliates. All rights reserved. + + PerconaFT is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License, version 2, + as published by the Free Software Foundation. + + PerconaFT is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with PerconaFT. If not, see . + +---------------------------------------- + + PerconaFT is free software: you can redistribute it and/or modify + it under the terms of the GNU Affero General Public License, version 3, + as published by the Free Software Foundation. + + PerconaFT is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Affero General Public License for more details. + + You should have received a copy of the GNU Affero General Public License + along with PerconaFT. If not, see . +======= */ + +#ident "Copyright (c) 2006, 2015, Percona and/or its affiliates. All rights reserved." + +#include "test.h" +#include "cachetable/checkpoint.h" + +static TOKUTXN const null_txn = 0; +static const char *fname = TOKU_TEST_FILENAME; + +/* test for_backup in ft_close */ +static void test_in_backup() { + int r; + CACHETABLE ct; + FT_HANDLE ft; + unlink(fname); + + toku_cachetable_create(&ct, 0, ZERO_LSN, nullptr); + CHECKPOINTER cp = toku_cachetable_get_checkpointer(ct); + + // TEST1 : for normal + r = toku_open_ft_handle(fname, 1, &ft, 1<<12, 1<<9, TOKU_DEFAULT_COMPRESSION_METHOD, ct, null_txn, toku_builtin_compare_fun); + assert_zero(r); + r = toku_close_ft_handle_nolsn(ft, 0); + assert_zero(r); + + r = toku_open_ft_handle(fname, 0, &ft, 1<<12, 1<<9, TOKU_DEFAULT_COMPRESSION_METHOD, ct, null_txn, toku_builtin_compare_fun); + assert_zero(r); + { + DBT k,v; + toku_ft_insert(ft, toku_fill_dbt(&k, "hello", 6), toku_fill_dbt(&v, "there", 6), null_txn); + } + r = toku_close_ft_handle_nolsn(ft, 0); + assert_zero(r); + + r = toku_open_ft_handle(fname, 0, &ft, 1<<12, 1<<9, TOKU_DEFAULT_COMPRESSION_METHOD, ct, null_txn, toku_builtin_compare_fun); + assert_zero(r); + ft_lookup_and_check_nodup(ft, "hello", "there"); + r = toku_close_ft_handle_nolsn(ft, 0); + assert_zero(r); + toku_cachetable_close(&ct); + + // TEST2: in fly without checkpoint test + toku_cachetable_create(&ct, 0, ZERO_LSN, nullptr); + cp = toku_cachetable_get_checkpointer(ct); + r = toku_open_ft_handle(fname, 0, &ft, 1<<12, 1<<9, TOKU_DEFAULT_COMPRESSION_METHOD, ct, null_txn, toku_builtin_compare_fun); + assert_zero(r); + + toku_cachetable_begin_backup(ct); + // this key/value just in fly since we are in backing up + { + DBT k,v; + toku_ft_insert(ft, toku_fill_dbt(&k, "halou", 6), toku_fill_dbt(&v, "not there", 10), null_txn); + } + r = toku_close_ft_handle_nolsn(ft, 0); + assert_zero(r); + + r = toku_open_ft_handle(fname, 0, &ft, 1<<12, 1<<9, TOKU_DEFAULT_COMPRESSION_METHOD, ct, null_txn, toku_builtin_compare_fun); + assert_zero(r); + ft_lookup_and_check_nodup(ft, "halou", "not there"); + + // because we are in backup, so the FT header is stale after cachefile&cachetable closed + // here has a leak for this ft evicts from memroy, but that makes sense + r = toku_close_ft_handle_nolsn(ft, 0); + assert_zero(r); + toku_cachetable_end_backup(ct); + toku_cachetable_close(&ct); + + // check the in fly key/value, it shouldn't exist + toku_cachetable_create(&ct, 0, ZERO_LSN, nullptr); + r = toku_open_ft_handle(fname, 0, &ft, 1<<12, 1<<9, TOKU_DEFAULT_COMPRESSION_METHOD, ct, null_txn, toku_builtin_compare_fun); + assert_zero(r); + ft_lookup_and_fail_nodup(ft, (char*)"halou"); + r = toku_close_ft_handle_nolsn(ft, 0); + assert_zero(r); + toku_cachetable_close(&ct); + + // TEST3: in fly with checkpoint test + toku_cachetable_create(&ct, 0, ZERO_LSN, nullptr); + cp = toku_cachetable_get_checkpointer(ct); + r = toku_open_ft_handle(fname, 0, &ft, 1<<12, 1<<9, TOKU_DEFAULT_COMPRESSION_METHOD, ct, null_txn, toku_builtin_compare_fun); + assert_zero(r); + + toku_cachetable_begin_backup(ct); + // this key/value just in fly since we are in backup + { + DBT k,v; + toku_ft_insert(ft, toku_fill_dbt(&k, "halou1", 7), toku_fill_dbt(&v, "not there", 10), null_txn); + } + r = toku_close_ft_handle_nolsn(ft, 0); + assert_zero(r); + + r = toku_open_ft_handle(fname, 0, &ft, 1<<12, 1<<9, TOKU_DEFAULT_COMPRESSION_METHOD, ct, null_txn, toku_builtin_compare_fun); + assert_zero(r); + ft_lookup_and_check_nodup(ft, "halou1", "not there"); + + // because we are in backup, so the FT header is stale after cachefile&cachetable closed + r = toku_close_ft_handle_nolsn(ft, 0); + assert_zero(r); + toku_cachetable_end_backup(ct); + r = toku_checkpoint(cp, NULL, NULL, NULL, NULL, NULL, CLIENT_CHECKPOINT); + assert_zero(r); + toku_cachetable_close(&ct); + + // check after checkpoint + toku_cachetable_create(&ct, 0, ZERO_LSN, nullptr); + r = toku_open_ft_handle(fname, 0, &ft, 1<<12, 1<<9, TOKU_DEFAULT_COMPRESSION_METHOD, ct, null_txn, toku_builtin_compare_fun); + assert_zero(r); + ft_lookup_and_check_nodup(ft, "halou1", "not there"); + r = toku_close_ft_handle_nolsn(ft, 0); + assert_zero(r); + toku_cachetable_close(&ct); +} + +int +test_main (int argc , const char *argv[]) { + default_parse_args(argc, argv); + test_in_backup(); + if (verbose) printf("test ok\n"); + return 0; +} diff --git a/src/ydb.cc b/src/ydb.cc index 88c6c86f2..afc249bfe 100644 --- a/src/ydb.cc +++ b/src/ydb.cc @@ -1623,7 +1623,10 @@ env_checkpointing_postpone(DB_ENV * env) { HANDLE_PANICKED_ENV(env); int r = 0; if (!env_opened(env)) r = EINVAL; - else toku_checkpoint_safe_client_lock(); + else { + toku_checkpoint_safe_client_lock(); + toku_cachetable_begin_backup(env->i->cachetable); + } return r; } @@ -1632,7 +1635,10 @@ env_checkpointing_resume(DB_ENV * env) { HANDLE_PANICKED_ENV(env); int r = 0; if (!env_opened(env)) r = EINVAL; - else toku_checkpoint_safe_client_unlock(); + else { + toku_cachetable_end_backup(env->i->cachetable); + toku_checkpoint_safe_client_unlock(); + } return r; }