diff --git a/ft/cachetable/cachetable-internal.h b/ft/cachetable/cachetable-internal.h
index dc6aec922..0a8d94c01 100644
--- a/ft/cachetable/cachetable-internal.h
+++ b/ft/cachetable/cachetable-internal.h
@@ -411,6 +411,8 @@ class checkpointer {
void add_background_job();
void remove_background_job();
void end_checkpoint(void (*testcallback_f)(void*), void* testextra);
+ void begin_backup();
+ void end_backup();
TOKULOGGER get_logger();
// used during begin_checkpoint
void increment_num_txns();
@@ -600,6 +602,7 @@ struct cachetable {
KIBBUTZ client_kibbutz; // pool of worker threads and jobs to do asynchronously for the client.
KIBBUTZ ct_kibbutz; // pool of worker threads and jobs to do asynchronously for the cachetable
KIBBUTZ checkpointing_kibbutz; // small pool for checkpointing cloned pairs
+ bool in_backup; // we are in back up or NOT, default is false
char *env_dir;
};
diff --git a/ft/cachetable/cachetable.cc b/ft/cachetable/cachetable.cc
index 5bba977de..a00cafb26 100644
--- a/ft/cachetable/cachetable.cc
+++ b/ft/cachetable/cachetable.cc
@@ -250,6 +250,7 @@ int toku_cachetable_create_ex(CACHETABLE *ct_result, long size_limit,
CACHETABLE XCALLOC(ct);
ct->list.init();
ct->cf_list.init();
+ ct->in_backup = false;
int num_processors = toku_os_get_number_active_processors();
int checkpointing_nworkers = (num_processors/4) ? num_processors/4 : 1;
@@ -2749,6 +2750,27 @@ void toku_cachetable_end_checkpoint(CHECKPOINTER cp, TOKULOGGER UU(logger),
cp->end_checkpoint(testcallback_f, testextra);
}
+// in_backup begin
+void toku_cachetable_begin_backup(CACHETABLE ct)
+{
+ ct->cf_list.read_lock();
+ ct->in_backup = true;
+ ct->cf_list.read_unlock();
+}
+
+void toku_cachetable_end_backup(CACHETABLE ct)
+{
+ ct->cf_list.read_lock();
+ ct->in_backup = false;
+ ct->cf_list.read_unlock();
+}
+
+bool toku_cachefile_in_backup(CACHEFILE cf)
+{
+ return cf->cachetable->in_backup;
+}
+// in_backup end
+
TOKULOGGER toku_cachefile_logger (CACHEFILE cf) {
return cf->cachetable->cp.get_logger();
}
diff --git a/ft/cachetable/cachetable.h b/ft/cachetable/cachetable.h
index 148326562..8a20f2b1e 100644
--- a/ft/cachetable/cachetable.h
+++ b/ft/cachetable/cachetable.h
@@ -147,6 +147,10 @@ void toku_cachetable_begin_checkpoint (CHECKPOINTER cp, struct tokulogger *logge
void toku_cachetable_end_checkpoint(CHECKPOINTER cp, struct tokulogger *logger,
void (*testcallback_f)(void*), void * testextra);
+void toku_cachetable_begin_backup(CACHETABLE ct);
+void toku_cachetable_end_backup(CACHETABLE ct);
+// cachefile is in backup or not
+bool toku_cachefile_in_backup(CACHEFILE cf);
// Shuts down checkpoint thread
// Requires no locks be held that are taken by the checkpoint function
diff --git a/ft/ft.cc b/ft/ft.cc
index 2a0fb6f68..97303b59c 100644
--- a/ft/ft.cc
+++ b/ft/ft.cc
@@ -282,7 +282,13 @@ static void ft_note_pin_by_checkpoint (CACHEFILE UU(cachefile), void *header_v)
FT ft = (FT) header_v;
toku_ft_grab_reflock(ft);
assert(!ft->pinned_by_checkpoint);
+
+ // because we add in_backup for cachetable, with this case:
+ // a cachefile in_backup is true and then do a checkpoint
+ // this assertion fails since nobody references to the ft
+#if 0
assert(toku_ft_needed_unlocked(ft));
+#endif
ft->pinned_by_checkpoint = true;
toku_ft_release_reflock(ft);
}
@@ -902,8 +908,12 @@ toku_ft_remove_reference(FT ft, bool oplsn_valid, LSN oplsn, remove_ft_ref_callb
assert(!needed);
}
if (!needed) {
- // close header
- toku_ft_evict_from_memory(ft, oplsn_valid, oplsn);
+ // close header if we are not in backup
+ // if close header during backup, the FT files would be inconsistency
+ bool in_backup = toku_cachefile_in_backup(ft->cf);
+ if (!in_backup) {
+ toku_ft_evict_from_memory(ft, oplsn_valid, oplsn);
+ }
}
toku_ft_open_close_unlock();
diff --git a/ft/tests/ft-in-backup-test.cc b/ft/tests/ft-in-backup-test.cc
new file mode 100644
index 000000000..863272dbe
--- /dev/null
+++ b/ft/tests/ft-in-backup-test.cc
@@ -0,0 +1,155 @@
+/* -*- mode: C++; c-basic-offset: 4; indent-tabs-mode: nil -*- */
+// vim: ft=cpp:expandtab:ts=8:sw=4:softtabstop=4:
+#ident "$Id$"
+/*======
+This file is part of PerconaFT.
+
+
+Copyright (c) 2006, 2015, Percona and/or its affiliates. All rights reserved.
+
+ PerconaFT is free software: you can redistribute it and/or modify
+ it under the terms of the GNU General Public License, version 2,
+ as published by the Free Software Foundation.
+
+ PerconaFT is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ GNU General Public License for more details.
+
+ You should have received a copy of the GNU General Public License
+ along with PerconaFT. If not, see .
+
+----------------------------------------
+
+ PerconaFT is free software: you can redistribute it and/or modify
+ it under the terms of the GNU Affero General Public License, version 3,
+ as published by the Free Software Foundation.
+
+ PerconaFT is distributed in the hope that it will be useful,
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ GNU Affero General Public License for more details.
+
+ You should have received a copy of the GNU Affero General Public License
+ along with PerconaFT. If not, see .
+======= */
+
+#ident "Copyright (c) 2006, 2015, Percona and/or its affiliates. All rights reserved."
+
+#include "test.h"
+#include "cachetable/checkpoint.h"
+
+static TOKUTXN const null_txn = 0;
+static const char *fname = TOKU_TEST_FILENAME;
+
+/* test for_backup in ft_close */
+static void test_in_backup() {
+ int r;
+ CACHETABLE ct;
+ FT_HANDLE ft;
+ unlink(fname);
+
+ toku_cachetable_create(&ct, 0, ZERO_LSN, nullptr);
+ CHECKPOINTER cp = toku_cachetable_get_checkpointer(ct);
+
+ // TEST1 : for normal
+ r = toku_open_ft_handle(fname, 1, &ft, 1<<12, 1<<9, TOKU_DEFAULT_COMPRESSION_METHOD, ct, null_txn, toku_builtin_compare_fun);
+ assert_zero(r);
+ r = toku_close_ft_handle_nolsn(ft, 0);
+ assert_zero(r);
+
+ r = toku_open_ft_handle(fname, 0, &ft, 1<<12, 1<<9, TOKU_DEFAULT_COMPRESSION_METHOD, ct, null_txn, toku_builtin_compare_fun);
+ assert_zero(r);
+ {
+ DBT k,v;
+ toku_ft_insert(ft, toku_fill_dbt(&k, "hello", 6), toku_fill_dbt(&v, "there", 6), null_txn);
+ }
+ r = toku_close_ft_handle_nolsn(ft, 0);
+ assert_zero(r);
+
+ r = toku_open_ft_handle(fname, 0, &ft, 1<<12, 1<<9, TOKU_DEFAULT_COMPRESSION_METHOD, ct, null_txn, toku_builtin_compare_fun);
+ assert_zero(r);
+ ft_lookup_and_check_nodup(ft, "hello", "there");
+ r = toku_close_ft_handle_nolsn(ft, 0);
+ assert_zero(r);
+ toku_cachetable_close(&ct);
+
+ // TEST2: in fly without checkpoint test
+ toku_cachetable_create(&ct, 0, ZERO_LSN, nullptr);
+ cp = toku_cachetable_get_checkpointer(ct);
+ r = toku_open_ft_handle(fname, 0, &ft, 1<<12, 1<<9, TOKU_DEFAULT_COMPRESSION_METHOD, ct, null_txn, toku_builtin_compare_fun);
+ assert_zero(r);
+
+ toku_cachetable_begin_backup(ct);
+ // this key/value just in fly since we are in backing up
+ {
+ DBT k,v;
+ toku_ft_insert(ft, toku_fill_dbt(&k, "halou", 6), toku_fill_dbt(&v, "not there", 10), null_txn);
+ }
+ r = toku_close_ft_handle_nolsn(ft, 0);
+ assert_zero(r);
+
+ r = toku_open_ft_handle(fname, 0, &ft, 1<<12, 1<<9, TOKU_DEFAULT_COMPRESSION_METHOD, ct, null_txn, toku_builtin_compare_fun);
+ assert_zero(r);
+ ft_lookup_and_check_nodup(ft, "halou", "not there");
+
+ // because we are in backup, so the FT header is stale after cachefile&cachetable closed
+ // here has a leak for this ft evicts from memroy, but that makes sense
+ r = toku_close_ft_handle_nolsn(ft, 0);
+ assert_zero(r);
+ toku_cachetable_end_backup(ct);
+ toku_cachetable_close(&ct);
+
+ // check the in fly key/value, it shouldn't exist
+ toku_cachetable_create(&ct, 0, ZERO_LSN, nullptr);
+ r = toku_open_ft_handle(fname, 0, &ft, 1<<12, 1<<9, TOKU_DEFAULT_COMPRESSION_METHOD, ct, null_txn, toku_builtin_compare_fun);
+ assert_zero(r);
+ ft_lookup_and_fail_nodup(ft, (char*)"halou");
+ r = toku_close_ft_handle_nolsn(ft, 0);
+ assert_zero(r);
+ toku_cachetable_close(&ct);
+
+ // TEST3: in fly with checkpoint test
+ toku_cachetable_create(&ct, 0, ZERO_LSN, nullptr);
+ cp = toku_cachetable_get_checkpointer(ct);
+ r = toku_open_ft_handle(fname, 0, &ft, 1<<12, 1<<9, TOKU_DEFAULT_COMPRESSION_METHOD, ct, null_txn, toku_builtin_compare_fun);
+ assert_zero(r);
+
+ toku_cachetable_begin_backup(ct);
+ // this key/value just in fly since we are in backup
+ {
+ DBT k,v;
+ toku_ft_insert(ft, toku_fill_dbt(&k, "halou1", 7), toku_fill_dbt(&v, "not there", 10), null_txn);
+ }
+ r = toku_close_ft_handle_nolsn(ft, 0);
+ assert_zero(r);
+
+ r = toku_open_ft_handle(fname, 0, &ft, 1<<12, 1<<9, TOKU_DEFAULT_COMPRESSION_METHOD, ct, null_txn, toku_builtin_compare_fun);
+ assert_zero(r);
+ ft_lookup_and_check_nodup(ft, "halou1", "not there");
+
+ // because we are in backup, so the FT header is stale after cachefile&cachetable closed
+ r = toku_close_ft_handle_nolsn(ft, 0);
+ assert_zero(r);
+ toku_cachetable_end_backup(ct);
+ r = toku_checkpoint(cp, NULL, NULL, NULL, NULL, NULL, CLIENT_CHECKPOINT);
+ assert_zero(r);
+ toku_cachetable_close(&ct);
+
+ // check after checkpoint
+ toku_cachetable_create(&ct, 0, ZERO_LSN, nullptr);
+ r = toku_open_ft_handle(fname, 0, &ft, 1<<12, 1<<9, TOKU_DEFAULT_COMPRESSION_METHOD, ct, null_txn, toku_builtin_compare_fun);
+ assert_zero(r);
+ ft_lookup_and_check_nodup(ft, "halou1", "not there");
+ r = toku_close_ft_handle_nolsn(ft, 0);
+ assert_zero(r);
+ toku_cachetable_close(&ct);
+}
+
+int
+test_main (int argc , const char *argv[]) {
+ default_parse_args(argc, argv);
+ test_in_backup();
+ if (verbose) printf("test ok\n");
+ return 0;
+}
diff --git a/src/ydb.cc b/src/ydb.cc
index 88c6c86f2..afc249bfe 100644
--- a/src/ydb.cc
+++ b/src/ydb.cc
@@ -1623,7 +1623,10 @@ env_checkpointing_postpone(DB_ENV * env) {
HANDLE_PANICKED_ENV(env);
int r = 0;
if (!env_opened(env)) r = EINVAL;
- else toku_checkpoint_safe_client_lock();
+ else {
+ toku_checkpoint_safe_client_lock();
+ toku_cachetable_begin_backup(env->i->cachetable);
+ }
return r;
}
@@ -1632,7 +1635,10 @@ env_checkpointing_resume(DB_ENV * env) {
HANDLE_PANICKED_ENV(env);
int r = 0;
if (!env_opened(env)) r = EINVAL;
- else toku_checkpoint_safe_client_unlock();
+ else {
+ toku_cachetable_end_backup(env->i->cachetable);
+ toku_checkpoint_safe_client_unlock();
+ }
return r;
}