From e442e9d295f513828ffd3fa116bf5b9d193e9155 Mon Sep 17 00:00:00 2001 From: Yu Zhang Date: Fri, 20 May 2022 18:26:02 -0700 Subject: [PATCH 1/5] Add timestamp support to CompactedDBImpl --- HISTORY.md | 1 + db/db_impl/compacted_db_impl.cc | 88 ++++++-- db/db_impl/compacted_db_impl.h | 18 +- db/db_readonly_with_timestamp_test.cc | 306 ++++++++++++++++++++++++++ 4 files changed, 387 insertions(+), 26 deletions(-) diff --git a/HISTORY.md b/HISTORY.md index 2b98bcaaffa..26cfbf56ed7 100644 --- a/HISTORY.md +++ b/HISTORY.md @@ -31,6 +31,7 @@ * Add an option, `CompressionOptions::use_zstd_dict_trainer`, to indicate whether zstd dictionary trainer should be used for generating zstd compression dictionaries. The default value of this option is true for backward compatibility. When this option is set to false, zstd API `ZDICT_finalizeDictionary` is used to generate compression dictionaries. * Seek API which positions itself every LevelIterator on the correct data block in the correct SST file which can be parallelized if ReadOptions.async_io option is enabled. * Add new stat number_async_seek in PerfContext that indicates number of async calls made by seek to prefetch data. +* Add support for user-defined timestamps to read only DB. ### Bug Fixes * RocksDB calls FileSystem::Poll API during FilePrefetchBuffer destruction which impacts performance as it waits for read requets completion which is not needed anymore. Calling FileSystem::AbortIO to abort those requests instead fixes that performance issue. diff --git a/db/db_impl/compacted_db_impl.cc b/db/db_impl/compacted_db_impl.cc index e1c061c2751..c8755e4bba9 100644 --- a/db/db_impl/compacted_db_impl.cc +++ b/db/db_impl/compacted_db_impl.cc @@ -40,16 +40,35 @@ size_t CompactedDBImpl::FindFile(const Slice& key) { Status CompactedDBImpl::Get(const ReadOptions& options, ColumnFamilyHandle*, const Slice& key, PinnableSlice* value) { + return Get(options, nullptr, key, value, /*timestamp*/ nullptr); +} + +Status CompactedDBImpl::Get(const ReadOptions& options, ColumnFamilyHandle*, + const Slice& key, PinnableSlice* value, + std::string* timestamp) { assert(user_comparator_); - if (options.timestamp || user_comparator_->timestamp_size()) { - // TODO: support timestamp - return Status::NotSupported(); + if (options.timestamp) { + const Status s = + FailIfTsSizesMismatch(DefaultColumnFamily(), *(options.timestamp)); + if (!s.ok()) { + return s; + } + } else { + const Status s = FailIfCfHasTs(DefaultColumnFamily()); + if (!s.ok()) { + return s; + } } + GetWithTimestampReadCallback read_cb(kMaxSequenceNumber); + std::string* ts = + user_comparator_->timestamp_size() > 0 ? timestamp : nullptr; + LookupKey lkey(key, kMaxSequenceNumber, options.timestamp); GetContext get_context(user_comparator_, nullptr, nullptr, nullptr, - GetContext::kNotFound, key, value, nullptr, nullptr, - nullptr, true, nullptr, nullptr); - LookupKey lkey(key, kMaxSequenceNumber); - Status s = files_.files[FindFile(key)].fd.table_reader->Get( + GetContext::kNotFound, lkey.user_key(), value, ts, + nullptr, nullptr, true, nullptr, nullptr, nullptr, + nullptr, &read_cb); + + Status s = files_.files[FindFile(lkey.user_key())].fd.table_reader->Get( options, lkey.internal_key(), &get_context, nullptr); if (!s.ok() && !s.IsNotFound()) { return s; @@ -60,37 +79,62 @@ Status CompactedDBImpl::Get(const ReadOptions& options, ColumnFamilyHandle*, return Status::NotFound(); } -std::vector CompactedDBImpl::MultiGet(const ReadOptions& options, - const std::vector&, +std::vector CompactedDBImpl::MultiGet( + const ReadOptions& options, const std::vector&, const std::vector& keys, std::vector* values) { + return MultiGet(options, keys, values, /*timestamps*/ nullptr); +} + +std::vector CompactedDBImpl::MultiGet( + const ReadOptions& options, const std::vector&, + const std::vector& keys, std::vector* values, + std::vector* timestamps) { assert(user_comparator_); - if (user_comparator_->timestamp_size() || options.timestamp) { - // TODO: support timestamp - return std::vector(keys.size(), Status::NotSupported()); + size_t num_keys = keys.size(); + + if (options.timestamp) { + Status s = + FailIfTsSizesMismatch(DefaultColumnFamily(), *(options.timestamp)); + if (!s.ok()) { + return std::vector(num_keys, s); + } + } else { + Status s = FailIfCfHasTs(DefaultColumnFamily()); + if (!s.ok()) { + return std::vector(num_keys, s); + } } + + GetWithTimestampReadCallback read_cb(kMaxSequenceNumber); autovector reader_list; for (const auto& key : keys) { - const FdWithKeyRange& f = files_.files[FindFile(key)]; - if (user_comparator_->Compare(key, ExtractUserKey(f.smallest_key)) < 0) { + LookupKey lkey(key, kMaxSequenceNumber, options.timestamp); + const FdWithKeyRange& f = files_.files[FindFile(lkey.user_key())]; + if (user_comparator_->Compare(lkey.internal_key(), + ExtractUserKey(f.smallest_key)) < 0) { reader_list.push_back(nullptr); } else { - LookupKey lkey(key, kMaxSequenceNumber); f.fd.table_reader->Prepare(lkey.internal_key()); reader_list.push_back(f.fd.table_reader); } } - - std::vector statuses(keys.size(), Status::NotFound()); - values->resize(keys.size()); + std::vector statuses(num_keys, Status::NotFound()); + values->resize(num_keys); + if (timestamps) { + timestamps->resize(num_keys); + } int idx = 0; for (auto* r : reader_list) { if (r != nullptr) { PinnableSlice pinnable_val; std::string& value = (*values)[idx]; - GetContext get_context(user_comparator_, nullptr, nullptr, nullptr, - GetContext::kNotFound, keys[idx], &pinnable_val, - nullptr, nullptr, nullptr, true, nullptr, nullptr); - LookupKey lkey(keys[idx], kMaxSequenceNumber); + LookupKey lkey(keys[idx], kMaxSequenceNumber, options.timestamp); + std::string* timestamp = timestamps ? &(*timestamps)[idx] : nullptr; + GetContext get_context( + user_comparator_, nullptr, nullptr, nullptr, GetContext::kNotFound, + lkey.user_key(), &pinnable_val, + user_comparator_->timestamp_size() > 0 ? timestamp : nullptr, nullptr, + nullptr, true, nullptr, nullptr, nullptr, nullptr, &read_cb); Status s = r->Get(options, lkey.internal_key(), &get_context, nullptr); assert(static_cast(idx) < statuses.size()); if (!s.ok() && !s.IsNotFound()) { diff --git a/db/db_impl/compacted_db_impl.h b/db/db_impl/compacted_db_impl.h index 4cf00785e2f..c2b4dec0a6a 100644 --- a/db/db_impl/compacted_db_impl.h +++ b/db/db_impl/compacted_db_impl.h @@ -28,12 +28,22 @@ class CompactedDBImpl : public DBImpl { virtual Status Get(const ReadOptions& options, ColumnFamilyHandle* column_family, const Slice& key, PinnableSlice* value) override; + + Status Get(const ReadOptions& options, ColumnFamilyHandle* column_family, + const Slice& key, PinnableSlice* value, + std::string* timestamp) override; + using DB::MultiGet; virtual std::vector MultiGet( - const ReadOptions& options, - const std::vector&, - const std::vector& keys, std::vector* values) - override; + const ReadOptions& options, const std::vector&, + const std::vector& keys, + std::vector* values) override; + + std::vector MultiGet(const ReadOptions& options, + const std::vector&, + const std::vector& keys, + std::vector* values, + std::vector* timestamps) override; using DBImpl::Put; virtual Status Put(const WriteOptions& /*options*/, diff --git a/db/db_readonly_with_timestamp_test.cc b/db/db_readonly_with_timestamp_test.cc index 908e791eed8..4d7de9dffaa 100644 --- a/db/db_readonly_with_timestamp_test.cc +++ b/db/db_readonly_with_timestamp_test.cc @@ -320,6 +320,312 @@ TEST_F(DBReadOnlyTestWithTimestamp, Close(); } + +TEST_F(DBReadOnlyTestWithTimestamp, CompactedDBGetReadTimestampSizeMismatch) { + const int kNumKeysPerFile = 128; + const uint64_t kMaxKey = 1024; + Options options = CurrentOptions(); + options.env = env_; + options.create_if_missing = true; + options.disable_auto_compactions = true; + const size_t kTimestampSize = Timestamp(0, 0).size(); + TestComparator test_cmp(kTimestampSize); + options.comparator = &test_cmp; + options.memtable_factory.reset( + test::NewSpecialSkipListFactory(kNumKeysPerFile)); + DestroyAndReopen(options); + std::string write_timestamp = Timestamp(1, 0); + WriteOptions write_opts; + for (uint64_t key = 0; key <= kMaxKey; ++key) { + Status s = db_->Put(write_opts, Key1(key), write_timestamp, + "value" + std::to_string(0)); + ASSERT_OK(s); + } + ASSERT_OK(db_->Flush(FlushOptions())); + ASSERT_OK(db_->CompactRange(CompactRangeOptions(), nullptr, nullptr)); + Close(); + + // Reopen the database in read only mode as a Compacted DB to test its + // timestamp support. + options.max_open_files = -1; + ASSERT_OK(ReadOnlyReopen(options)); + ASSERT_EQ(0, NumTableFilesAtLevel(0)); + + ReadOptions read_opts; + std::string different_size_read_timestamp; + PutFixed32(&different_size_read_timestamp, 2); + Slice different_size_read_ts = different_size_read_timestamp; + read_opts.timestamp = &different_size_read_ts; + for (uint64_t key = 0; key <= kMaxKey; ++key) { + std::string value_from_get; + std::string timestamp; + ASSERT_TRUE(db_->Get(read_opts, Key1(key), &value_from_get, ×tamp) + .IsInvalidArgument()); + } + Close(); +} + +TEST_F(DBReadOnlyTestWithTimestamp, + CompactedDBGetReadTimestampSpecifiedWithoutWriteTimestamp) { + const int kNumKeysPerFile = 128; + const uint64_t kMaxKey = 1024; + Options options = CurrentOptions(); + options.env = env_; + options.create_if_missing = true; + options.disable_auto_compactions = true; + options.memtable_factory.reset( + test::NewSpecialSkipListFactory(kNumKeysPerFile)); + DestroyAndReopen(options); + WriteOptions write_opts; + for (uint64_t key = 0; key <= kMaxKey; ++key) { + Status s = db_->Put(write_opts, Key1(key), "value" + std::to_string(0)); + ASSERT_OK(s); + } + ASSERT_OK(db_->Flush(FlushOptions())); + ASSERT_OK(db_->CompactRange(CompactRangeOptions(), nullptr, nullptr)); + Close(); + + // Reopen the database in read only mode as a Compacted DB to test its + // timestamp support. + options.max_open_files = -1; + ASSERT_OK(ReadOnlyReopen(options)); + ASSERT_EQ(0, NumTableFilesAtLevel(0)); + + ReadOptions read_opts; + const std::string read_timestamp = Timestamp(2, 0); + Slice read_ts = read_timestamp; + read_opts.timestamp = &read_ts; + for (uint64_t key = 0; key <= kMaxKey; ++key) { + std::string value_from_get; + std::string timestamp; + ASSERT_TRUE(db_->Get(read_opts, Key1(key), &value_from_get, ×tamp) + .IsInvalidArgument()); + } + Close(); +} + +TEST_F(DBReadOnlyTestWithTimestamp, CompactedDBGet) { + const int kNumKeysPerFile = 128; + const uint64_t kMaxKey = 1024; + Options options = CurrentOptions(); + options.env = env_; + options.create_if_missing = true; + options.disable_auto_compactions = true; + const size_t kTimestampSize = Timestamp(0, 0).size(); + TestComparator test_cmp(kTimestampSize); + options.comparator = &test_cmp; + options.memtable_factory.reset( + test::NewSpecialSkipListFactory(kNumKeysPerFile)); + DestroyAndReopen(options); + const std::vector start_keys = {1, 0}; + const std::vector write_timestamps = {Timestamp(1, 0), + Timestamp(3, 0)}; + const std::vector read_timestamps = {Timestamp(2, 0), + Timestamp(4, 0)}; + for (size_t i = 0; i < write_timestamps.size(); ++i) { + WriteOptions write_opts; + for (uint64_t key = start_keys[i]; key <= kMaxKey; ++key) { + Status s = db_->Put(write_opts, Key1(key), write_timestamps[i], + "value" + std::to_string(i)); + ASSERT_OK(s); + } + ASSERT_OK(db_->Flush(FlushOptions())); + } + ASSERT_OK(db_->CompactRange(CompactRangeOptions(), nullptr, nullptr)); + Close(); + + // Reopen the database in read only mode as a Compacted DB to test its + // timestamp support. + options.max_open_files = -1; + ASSERT_OK(ReadOnlyReopen(options)); + ASSERT_EQ(0, NumTableFilesAtLevel(0)); + + for (size_t i = 0; i < read_timestamps.size(); ++i) { + ReadOptions read_opts; + Slice read_ts = read_timestamps[i]; + std::string read_ts_str = read_ts.ToString(); + read_opts.timestamp = &read_ts; + int count = 0; + for (uint64_t key = start_keys[i]; key <= kMaxKey; ++key, ++count) { + std::string value_from_get; + std::string timestamp; + ASSERT_OK(db_->Get(read_opts, Key1(key), &value_from_get, ×tamp)); + ASSERT_EQ("value" + std::to_string(i), value_from_get); + ASSERT_EQ(write_timestamps[i], timestamp); + } + size_t expected_count = kMaxKey - start_keys[i] + 1; + ASSERT_EQ(expected_count, count); + } + Close(); +} + +TEST_F(DBReadOnlyTestWithTimestamp, + CompactedDBMultiGetReadTimestampSizeMismatch) { + const int kNumKeysPerFile = 128; + const uint64_t kMaxKey = 3; + Options options = CurrentOptions(); + options.env = env_; + options.create_if_missing = true; + options.disable_auto_compactions = true; + const size_t kTimestampSize = Timestamp(0, 0).size(); + TestComparator test_cmp(kTimestampSize); + options.comparator = &test_cmp; + options.memtable_factory.reset( + test::NewSpecialSkipListFactory(kNumKeysPerFile)); + DestroyAndReopen(options); + std::string write_timestamp = Timestamp(1, 0); + WriteOptions write_opts; + for (uint64_t key = 0; key <= kMaxKey; ++key) { + Status s = db_->Put(write_opts, Key1(key), write_timestamp, + "value" + std::to_string(0)); + ASSERT_OK(s); + } + ASSERT_OK(db_->Flush(FlushOptions())); + ASSERT_OK(db_->CompactRange(CompactRangeOptions(), nullptr, nullptr)); + Close(); + + // Reopen the database in read only mode as a Compacted DB to test its + // timestamp support. + options.max_open_files = -1; + ASSERT_OK(ReadOnlyReopen(options)); + ASSERT_EQ(0, NumTableFilesAtLevel(0)); + + ReadOptions read_opts; + std::string different_size_read_timestamp; + PutFixed32(&different_size_read_timestamp, 2); + Slice different_size_read_ts = different_size_read_timestamp; + read_opts.timestamp = &different_size_read_ts; + std::vector key_strs; + std::vector keys; + for (uint64_t key = 0; key <= kMaxKey; ++key) { + key_strs.push_back(Key1(key)); + } + for (const auto& key_str : key_strs) { + keys.emplace_back(key_str); + } + std::vector values; + std::vector timestamps; + std::vector status_list = + db_->MultiGet(read_opts, keys, &values, ×tamps); + for (const auto& status : status_list) { + ASSERT_TRUE(status.IsInvalidArgument()); + } + Close(); +} + +TEST_F(DBReadOnlyTestWithTimestamp, + CompactedDBMultiGetReadTimestampSpecifiedWithoutWriteTimestamp) { + const int kNumKeysPerFile = 128; + const uint64_t kMaxKey = 3; + Options options = CurrentOptions(); + options.env = env_; + options.create_if_missing = true; + options.disable_auto_compactions = true; + options.memtable_factory.reset( + test::NewSpecialSkipListFactory(kNumKeysPerFile)); + DestroyAndReopen(options); + WriteOptions write_opts; + for (uint64_t key = 0; key <= kMaxKey; ++key) { + Status s = db_->Put(write_opts, Key1(key), "value" + std::to_string(0)); + ASSERT_OK(s); + } + ASSERT_OK(db_->Flush(FlushOptions())); + ASSERT_OK(db_->CompactRange(CompactRangeOptions(), nullptr, nullptr)); + Close(); + + // Reopen the database in read only mode as a Compacted DB to test its + // timestamp support. + options.max_open_files = -1; + ASSERT_OK(ReadOnlyReopen(options)); + ASSERT_EQ(0, NumTableFilesAtLevel(0)); + + ReadOptions read_opts; + std::string read_timestamp = Timestamp(2, 0); + Slice read_ts = read_timestamp; + std::string read_ts_str = read_ts.ToString(); + read_opts.timestamp = &read_ts; + std::vector key_strs; + std::vector keys; + for (uint64_t key = 0; key <= kMaxKey; ++key) { + key_strs.push_back(Key1(key)); + } + for (const auto& key_str : key_strs) { + keys.emplace_back(key_str); + } + std::vector values; + std::vector timestamps; + std::vector status_list = + db_->MultiGet(read_opts, keys, &values, ×tamps); + for (const auto& status : status_list) { + ASSERT_TRUE(status.IsInvalidArgument()); + } + Close(); +} + +TEST_F(DBReadOnlyTestWithTimestamp, CompactedDBMultiGet) { + const int kNumKeysPerFile = 128; + const uint64_t kMaxKey = 4; + Options options = CurrentOptions(); + options.env = env_; + options.create_if_missing = true; + options.disable_auto_compactions = true; + const size_t kTimestampSize = Timestamp(0, 0).size(); + TestComparator test_cmp(kTimestampSize); + options.comparator = &test_cmp; + options.memtable_factory.reset( + test::NewSpecialSkipListFactory(kNumKeysPerFile)); + DestroyAndReopen(options); + const std::vector start_keys = {1, 0}; + const std::vector write_timestamps = {Timestamp(1, 0), + Timestamp(3, 0)}; + const std::vector read_timestamps = {Timestamp(2, 0), + Timestamp(4, 0)}; + for (size_t i = 0; i < write_timestamps.size(); ++i) { + WriteOptions write_opts; + for (uint64_t key = start_keys[i]; key <= kMaxKey; ++key) { + Status s = db_->Put(write_opts, Key1(key), write_timestamps[i], + "value" + std::to_string(i)); + ASSERT_OK(s); + } + } + ASSERT_OK(db_->Flush(FlushOptions())); + ASSERT_OK(db_->CompactRange(CompactRangeOptions(), nullptr, nullptr)); + Close(); + + // Reopen the database in read only mode as a Compacted DB to test its + // timestamp support. + options.max_open_files = -1; + ASSERT_OK(ReadOnlyReopen(options)); + ASSERT_EQ(0, NumTableFilesAtLevel(0)); + for (size_t i = 0; i < write_timestamps.size(); ++i) { + ReadOptions read_opts; + Slice read_ts = read_timestamps[i]; + std::string read_ts_str = read_ts.ToString(); + read_opts.timestamp = &read_ts; + std::vector key_strs; + std::vector keys; + for (uint64_t key = start_keys[i]; key <= kMaxKey; ++key) { + key_strs.push_back(Key1(key)); + } + for (const auto& key_str : key_strs) { + keys.emplace_back(key_str); + } + size_t batch_size = kMaxKey - start_keys[i] + 1; + std::vector values; + std::vector timestamps; + std::vector status_list = + db_->MultiGet(read_opts, keys, &values, ×tamps); + ASSERT_EQ(batch_size, values.size()); + ASSERT_EQ(batch_size, timestamps.size()); + for (uint64_t idx = 0; idx < values.size(); ++idx) { + ASSERT_EQ("value" + std::to_string(i), values[idx]); + ASSERT_EQ(write_timestamps[i], timestamps[idx]); + ASSERT_OK(status_list[idx]); + } + } + + Close(); +} #endif // !ROCKSDB_LITE } // namespace ROCKSDB_NAMESPACE From 15bdd7483aac417ea2ff5a8c719b096d97381858 Mon Sep 17 00:00:00 2001 From: Yu Zhang Date: Mon, 23 May 2022 12:29:11 -0700 Subject: [PATCH 2/5] Fix comparator's usage for filtering out files and add test case --- db/db_impl/compacted_db_impl.cc | 22 +++- db/db_impl/compacted_db_impl.h | 3 + db/db_readonly_with_timestamp_test.cc | 169 +++++++++++++++++++++++++- 3 files changed, 186 insertions(+), 8 deletions(-) diff --git a/db/db_impl/compacted_db_impl.cc b/db/db_impl/compacted_db_impl.cc index c8755e4bba9..ccb366c5333 100644 --- a/db/db_impl/compacted_db_impl.cc +++ b/db/db_impl/compacted_db_impl.cc @@ -40,7 +40,8 @@ size_t CompactedDBImpl::FindFile(const Slice& key) { Status CompactedDBImpl::Get(const ReadOptions& options, ColumnFamilyHandle*, const Slice& key, PinnableSlice* value) { - return Get(options, nullptr, key, value, /*timestamp*/ nullptr); + return Get(options, /*column_family*/ nullptr, key, value, + /*timestamp*/ nullptr); } Status CompactedDBImpl::Get(const ReadOptions& options, ColumnFamilyHandle*, @@ -68,8 +69,16 @@ Status CompactedDBImpl::Get(const ReadOptions& options, ColumnFamilyHandle*, nullptr, nullptr, true, nullptr, nullptr, nullptr, nullptr, &read_cb); - Status s = files_.files[FindFile(lkey.user_key())].fd.table_reader->Get( - options, lkey.internal_key(), &get_context, nullptr); + const FdWithKeyRange& f = files_.files[FindFile(lkey.user_key())]; + if (user_comparator_->CompareWithoutTimestamp( + key, /*a_has_ts=*/false, + ExtractUserKeyAndStripTimestamp(f.smallest_key, + user_comparator_->timestamp_size()), + /*b_has_ts=*/false) < 0) { + return Status::NotFound(); + } + Status s = f.fd.table_reader->Get(options, lkey.internal_key(), &get_context, + nullptr); if (!s.ok() && !s.IsNotFound()) { return s; } @@ -110,8 +119,11 @@ std::vector CompactedDBImpl::MultiGet( for (const auto& key : keys) { LookupKey lkey(key, kMaxSequenceNumber, options.timestamp); const FdWithKeyRange& f = files_.files[FindFile(lkey.user_key())]; - if (user_comparator_->Compare(lkey.internal_key(), - ExtractUserKey(f.smallest_key)) < 0) { + if (user_comparator_->CompareWithoutTimestamp( + key, /*a_has_ts=*/false, + ExtractUserKeyAndStripTimestamp(f.smallest_key, + user_comparator_->timestamp_size()), + /*b_has_ts=*/false) < 0) { reader_list.push_back(nullptr); } else { f.fd.table_reader->Prepare(lkey.internal_key()); diff --git a/db/db_impl/compacted_db_impl.h b/db/db_impl/compacted_db_impl.h index c2b4dec0a6a..7a83a3cc5f9 100644 --- a/db/db_impl/compacted_db_impl.h +++ b/db/db_impl/compacted_db_impl.h @@ -34,6 +34,9 @@ class CompactedDBImpl : public DBImpl { std::string* timestamp) override; using DB::MultiGet; + // Note that CompactedDBImpl::MultiGet is not the optimized version of + // MultiGet to use. + // TODO: optimize CompactedDBImpl::MultiGet, see DBImpl::MultiGet for details. virtual std::vector MultiGet( const ReadOptions& options, const std::vector&, const std::vector& keys, diff --git a/db/db_readonly_with_timestamp_test.cc b/db/db_readonly_with_timestamp_test.cc index 4d7de9dffaa..b0231f601b7 100644 --- a/db/db_readonly_with_timestamp_test.cc +++ b/db/db_readonly_with_timestamp_test.cc @@ -101,6 +101,46 @@ TEST_F(DBReadOnlyTestWithTimestamp, Close(); } +TEST_F(DBReadOnlyTestWithTimestamp, + IteratorAndGetWriteWithTimestampReadWithoutTimestamp) { + const int kNumKeysPerFile = 128; + const uint64_t kMaxKey = 1024; + Options options = CurrentOptions(); + options.env = env_; + options.create_if_missing = true; + const size_t kTimestampSize = Timestamp(0, 0).size(); + TestComparator test_cmp(kTimestampSize); + options.comparator = &test_cmp; + options.memtable_factory.reset( + test::NewSpecialSkipListFactory(kNumKeysPerFile)); + DestroyAndReopen(options); + const std::string write_timestamp = Timestamp(1, 0); + WriteOptions write_opts; + for (uint64_t key = 0; key <= kMaxKey; ++key) { + Status s = db_->Put(write_opts, Key1(key), write_timestamp, + "value" + std::to_string(key)); + ASSERT_OK(s); + } + + // Reopen the database in read only mode to test its timestamp support. + Close(); + ASSERT_OK(ReadOnlyReopen(options)); + ReadOptions read_opts; + { + std::unique_ptr iter(db_->NewIterator(read_opts)); + ASSERT_FALSE(iter->Valid()); + ASSERT_TRUE(iter->status().IsInvalidArgument()); + } + + for (uint64_t key = 0; key <= kMaxKey; ++key) { + std::string value_from_get; + ASSERT_TRUE( + db_->Get(read_opts, Key1(key), &value_from_get).IsInvalidArgument()); + } + + Close(); +} + TEST_F(DBReadOnlyTestWithTimestamp, IteratorAndGet) { const int kNumKeysPerFile = 128; const uint64_t kMaxKey = 1024; @@ -321,6 +361,41 @@ TEST_F(DBReadOnlyTestWithTimestamp, Close(); } +TEST_F(DBReadOnlyTestWithTimestamp, + IteratorsWriteWithTimestampReadWithoutTimestamp) { + const int kNumKeysPerFile = 128; + const uint64_t kMaxKey = 1024; + Options options = CurrentOptions(); + options.env = env_; + options.create_if_missing = true; + const size_t kTimestampSize = Timestamp(0, 0).size(); + TestComparator test_cmp(kTimestampSize); + options.comparator = &test_cmp; + options.memtable_factory.reset( + test::NewSpecialSkipListFactory(kNumKeysPerFile)); + DestroyAndReopen(options); + const std::string write_timestamp = Timestamp(1, 0); + WriteOptions write_opts; + for (uint64_t key = 0; key <= kMaxKey; ++key) { + Status s = db_->Put(write_opts, Key1(key), write_timestamp, + "value" + std::to_string(key)); + ASSERT_OK(s); + } + + // Reopen the database in read only mode to test its timestamp support. + Close(); + ASSERT_OK(ReadOnlyReopen(options)); + ReadOptions read_opts; + { + std::vector iters; + ASSERT_TRUE( + db_->NewIterators(read_opts, {db_->DefaultColumnFamily()}, &iters) + .IsInvalidArgument()); + } + + Close(); +} + TEST_F(DBReadOnlyTestWithTimestamp, CompactedDBGetReadTimestampSizeMismatch) { const int kNumKeysPerFile = 128; const uint64_t kMaxKey = 1024; @@ -404,6 +479,46 @@ TEST_F(DBReadOnlyTestWithTimestamp, Close(); } +TEST_F(DBReadOnlyTestWithTimestamp, + CompactedDBGetWriteWithTimestampReadWithoutTimestamp) { + const int kNumKeysPerFile = 128; + const uint64_t kMaxKey = 1024; + Options options = CurrentOptions(); + options.env = env_; + options.create_if_missing = true; + options.disable_auto_compactions = true; + const size_t kTimestampSize = Timestamp(0, 0).size(); + TestComparator test_cmp(kTimestampSize); + options.comparator = &test_cmp; + options.memtable_factory.reset( + test::NewSpecialSkipListFactory(kNumKeysPerFile)); + DestroyAndReopen(options); + std::string write_timestamp = Timestamp(1, 0); + WriteOptions write_opts; + for (uint64_t key = 0; key <= kMaxKey; ++key) { + Status s = db_->Put(write_opts, Key1(key), write_timestamp, + "value" + std::to_string(0)); + ASSERT_OK(s); + } + ASSERT_OK(db_->Flush(FlushOptions())); + ASSERT_OK(db_->CompactRange(CompactRangeOptions(), nullptr, nullptr)); + Close(); + + // Reopen the database in read only mode as a Compacted DB to test its + // timestamp support. + options.max_open_files = -1; + ASSERT_OK(ReadOnlyReopen(options)); + ASSERT_EQ(0, NumTableFilesAtLevel(0)); + + ReadOptions read_opts; + for (uint64_t key = 0; key <= kMaxKey; ++key) { + std::string value_from_get; + ASSERT_TRUE( + db_->Get(read_opts, Key1(key), &value_from_get).IsInvalidArgument()); + } + Close(); +} + TEST_F(DBReadOnlyTestWithTimestamp, CompactedDBGet) { const int kNumKeysPerFile = 128; const uint64_t kMaxKey = 1024; @@ -462,7 +577,7 @@ TEST_F(DBReadOnlyTestWithTimestamp, CompactedDBGet) { TEST_F(DBReadOnlyTestWithTimestamp, CompactedDBMultiGetReadTimestampSizeMismatch) { const int kNumKeysPerFile = 128; - const uint64_t kMaxKey = 3; + const uint64_t kMaxKey = 1024; Options options = CurrentOptions(); options.env = env_; options.create_if_missing = true; @@ -516,7 +631,7 @@ TEST_F(DBReadOnlyTestWithTimestamp, TEST_F(DBReadOnlyTestWithTimestamp, CompactedDBMultiGetReadTimestampSpecifiedWithoutWriteTimestamp) { const int kNumKeysPerFile = 128; - const uint64_t kMaxKey = 3; + const uint64_t kMaxKey = 1024; Options options = CurrentOptions(); options.env = env_; options.create_if_missing = true; @@ -562,9 +677,57 @@ TEST_F(DBReadOnlyTestWithTimestamp, Close(); } +TEST_F(DBReadOnlyTestWithTimestamp, + CompactedDBMultiGetWriteWithTimestampReadWithoutTimestamp) { + const int kNumKeysPerFile = 128; + const uint64_t kMaxKey = 1024; + Options options = CurrentOptions(); + options.env = env_; + options.create_if_missing = true; + options.disable_auto_compactions = true; + const size_t kTimestampSize = Timestamp(0, 0).size(); + TestComparator test_cmp(kTimestampSize); + options.comparator = &test_cmp; + options.memtable_factory.reset( + test::NewSpecialSkipListFactory(kNumKeysPerFile)); + DestroyAndReopen(options); + std::string write_timestamp = Timestamp(1, 0); + WriteOptions write_opts; + for (uint64_t key = 0; key <= kMaxKey; ++key) { + Status s = db_->Put(write_opts, Key1(key), write_timestamp, + "value" + std::to_string(0)); + ASSERT_OK(s); + } + ASSERT_OK(db_->Flush(FlushOptions())); + ASSERT_OK(db_->CompactRange(CompactRangeOptions(), nullptr, nullptr)); + Close(); + + // Reopen the database in read only mode as a Compacted DB to test its + // timestamp support. + options.max_open_files = -1; + ASSERT_OK(ReadOnlyReopen(options)); + ASSERT_EQ(0, NumTableFilesAtLevel(0)); + + ReadOptions read_opts; + std::vector key_strs; + std::vector keys; + for (uint64_t key = 0; key <= kMaxKey; ++key) { + key_strs.push_back(Key1(key)); + } + for (const auto& key_str : key_strs) { + keys.emplace_back(key_str); + } + std::vector values; + std::vector status_list = db_->MultiGet(read_opts, keys, &values); + for (const auto& status : status_list) { + ASSERT_TRUE(status.IsInvalidArgument()); + } + Close(); +} + TEST_F(DBReadOnlyTestWithTimestamp, CompactedDBMultiGet) { const int kNumKeysPerFile = 128; - const uint64_t kMaxKey = 4; + const uint64_t kMaxKey = 1024; Options options = CurrentOptions(); options.env = env_; options.create_if_missing = true; From 9ddb9f493edf415a792861b42c8c9abd6e4dd2c2 Mon Sep 17 00:00:00 2001 From: Yu Zhang Date: Mon, 23 May 2022 14:39:19 -0700 Subject: [PATCH 3/5] Add DBOpenedAsCompactedDB check for compacted db test cases --- db/db_readonly_with_timestamp_test.cc | 63 ++++++++++++++++----------- 1 file changed, 38 insertions(+), 25 deletions(-) diff --git a/db/db_readonly_with_timestamp_test.cc b/db/db_readonly_with_timestamp_test.cc index b0231f601b7..46c2cb54bf2 100644 --- a/db/db_readonly_with_timestamp_test.cc +++ b/db/db_readonly_with_timestamp_test.cc @@ -15,6 +15,26 @@ class DBReadOnlyTestWithTimestamp : public DBBasicTestWithTimestampBase { public: DBReadOnlyTestWithTimestamp() : DBBasicTestWithTimestampBase("db_readonly_test_with_timestamp") {} + + protected: + void CheckDBOpenedAsCompactedDB() { + VersionSet* const versions = dbfull()->GetVersionSet(); + ASSERT_NE(versions, nullptr); + + ColumnFamilyData* const cfd = versions->GetColumnFamilySet()->GetDefault(); + ASSERT_NE(cfd, nullptr); + + Version* const current = cfd->current(); + ASSERT_NE(current, nullptr); + + const VersionStorageInfo* const storage_info = current->storage_info(); + ASSERT_NE(storage_info, nullptr); + + // Only 1 L0 file. + ASSERT_EQ(1, NumTableFilesAtLevel(0)); + // L0 is the max level. + ASSERT_EQ(storage_info->num_non_empty_levels(), 1); + } }; #ifndef ROCKSDB_LITE @@ -397,7 +417,7 @@ TEST_F(DBReadOnlyTestWithTimestamp, } TEST_F(DBReadOnlyTestWithTimestamp, CompactedDBGetReadTimestampSizeMismatch) { - const int kNumKeysPerFile = 128; + const int kNumKeysPerFile = 1026; const uint64_t kMaxKey = 1024; Options options = CurrentOptions(); options.env = env_; @@ -417,14 +437,13 @@ TEST_F(DBReadOnlyTestWithTimestamp, CompactedDBGetReadTimestampSizeMismatch) { ASSERT_OK(s); } ASSERT_OK(db_->Flush(FlushOptions())); - ASSERT_OK(db_->CompactRange(CompactRangeOptions(), nullptr, nullptr)); Close(); // Reopen the database in read only mode as a Compacted DB to test its // timestamp support. options.max_open_files = -1; ASSERT_OK(ReadOnlyReopen(options)); - ASSERT_EQ(0, NumTableFilesAtLevel(0)); + CheckDBOpenedAsCompactedDB(); ReadOptions read_opts; std::string different_size_read_timestamp; @@ -442,7 +461,7 @@ TEST_F(DBReadOnlyTestWithTimestamp, CompactedDBGetReadTimestampSizeMismatch) { TEST_F(DBReadOnlyTestWithTimestamp, CompactedDBGetReadTimestampSpecifiedWithoutWriteTimestamp) { - const int kNumKeysPerFile = 128; + const int kNumKeysPerFile = 1026; const uint64_t kMaxKey = 1024; Options options = CurrentOptions(); options.env = env_; @@ -457,14 +476,13 @@ TEST_F(DBReadOnlyTestWithTimestamp, ASSERT_OK(s); } ASSERT_OK(db_->Flush(FlushOptions())); - ASSERT_OK(db_->CompactRange(CompactRangeOptions(), nullptr, nullptr)); Close(); // Reopen the database in read only mode as a Compacted DB to test its // timestamp support. options.max_open_files = -1; ASSERT_OK(ReadOnlyReopen(options)); - ASSERT_EQ(0, NumTableFilesAtLevel(0)); + CheckDBOpenedAsCompactedDB(); ReadOptions read_opts; const std::string read_timestamp = Timestamp(2, 0); @@ -481,7 +499,7 @@ TEST_F(DBReadOnlyTestWithTimestamp, TEST_F(DBReadOnlyTestWithTimestamp, CompactedDBGetWriteWithTimestampReadWithoutTimestamp) { - const int kNumKeysPerFile = 128; + const int kNumKeysPerFile = 1026; const uint64_t kMaxKey = 1024; Options options = CurrentOptions(); options.env = env_; @@ -501,14 +519,13 @@ TEST_F(DBReadOnlyTestWithTimestamp, ASSERT_OK(s); } ASSERT_OK(db_->Flush(FlushOptions())); - ASSERT_OK(db_->CompactRange(CompactRangeOptions(), nullptr, nullptr)); Close(); // Reopen the database in read only mode as a Compacted DB to test its // timestamp support. options.max_open_files = -1; ASSERT_OK(ReadOnlyReopen(options)); - ASSERT_EQ(0, NumTableFilesAtLevel(0)); + CheckDBOpenedAsCompactedDB(); ReadOptions read_opts; for (uint64_t key = 0; key <= kMaxKey; ++key) { @@ -520,7 +537,7 @@ TEST_F(DBReadOnlyTestWithTimestamp, } TEST_F(DBReadOnlyTestWithTimestamp, CompactedDBGet) { - const int kNumKeysPerFile = 128; + const int kNumKeysPerFile = 1026 * 2; const uint64_t kMaxKey = 1024; Options options = CurrentOptions(); options.env = env_; @@ -544,16 +561,15 @@ TEST_F(DBReadOnlyTestWithTimestamp, CompactedDBGet) { "value" + std::to_string(i)); ASSERT_OK(s); } - ASSERT_OK(db_->Flush(FlushOptions())); } - ASSERT_OK(db_->CompactRange(CompactRangeOptions(), nullptr, nullptr)); + ASSERT_OK(db_->Flush(FlushOptions())); Close(); // Reopen the database in read only mode as a Compacted DB to test its // timestamp support. options.max_open_files = -1; ASSERT_OK(ReadOnlyReopen(options)); - ASSERT_EQ(0, NumTableFilesAtLevel(0)); + CheckDBOpenedAsCompactedDB(); for (size_t i = 0; i < read_timestamps.size(); ++i) { ReadOptions read_opts; @@ -576,7 +592,7 @@ TEST_F(DBReadOnlyTestWithTimestamp, CompactedDBGet) { TEST_F(DBReadOnlyTestWithTimestamp, CompactedDBMultiGetReadTimestampSizeMismatch) { - const int kNumKeysPerFile = 128; + const int kNumKeysPerFile = 1026; const uint64_t kMaxKey = 1024; Options options = CurrentOptions(); options.env = env_; @@ -596,14 +612,13 @@ TEST_F(DBReadOnlyTestWithTimestamp, ASSERT_OK(s); } ASSERT_OK(db_->Flush(FlushOptions())); - ASSERT_OK(db_->CompactRange(CompactRangeOptions(), nullptr, nullptr)); Close(); // Reopen the database in read only mode as a Compacted DB to test its // timestamp support. options.max_open_files = -1; ASSERT_OK(ReadOnlyReopen(options)); - ASSERT_EQ(0, NumTableFilesAtLevel(0)); + CheckDBOpenedAsCompactedDB(); ReadOptions read_opts; std::string different_size_read_timestamp; @@ -630,7 +645,7 @@ TEST_F(DBReadOnlyTestWithTimestamp, TEST_F(DBReadOnlyTestWithTimestamp, CompactedDBMultiGetReadTimestampSpecifiedWithoutWriteTimestamp) { - const int kNumKeysPerFile = 128; + const int kNumKeysPerFile = 1026; const uint64_t kMaxKey = 1024; Options options = CurrentOptions(); options.env = env_; @@ -645,14 +660,13 @@ TEST_F(DBReadOnlyTestWithTimestamp, ASSERT_OK(s); } ASSERT_OK(db_->Flush(FlushOptions())); - ASSERT_OK(db_->CompactRange(CompactRangeOptions(), nullptr, nullptr)); Close(); // Reopen the database in read only mode as a Compacted DB to test its // timestamp support. options.max_open_files = -1; ASSERT_OK(ReadOnlyReopen(options)); - ASSERT_EQ(0, NumTableFilesAtLevel(0)); + CheckDBOpenedAsCompactedDB(); ReadOptions read_opts; std::string read_timestamp = Timestamp(2, 0); @@ -679,7 +693,7 @@ TEST_F(DBReadOnlyTestWithTimestamp, TEST_F(DBReadOnlyTestWithTimestamp, CompactedDBMultiGetWriteWithTimestampReadWithoutTimestamp) { - const int kNumKeysPerFile = 128; + const int kNumKeysPerFile = 1026; const uint64_t kMaxKey = 1024; Options options = CurrentOptions(); options.env = env_; @@ -699,14 +713,13 @@ TEST_F(DBReadOnlyTestWithTimestamp, ASSERT_OK(s); } ASSERT_OK(db_->Flush(FlushOptions())); - ASSERT_OK(db_->CompactRange(CompactRangeOptions(), nullptr, nullptr)); Close(); // Reopen the database in read only mode as a Compacted DB to test its // timestamp support. options.max_open_files = -1; ASSERT_OK(ReadOnlyReopen(options)); - ASSERT_EQ(0, NumTableFilesAtLevel(0)); + CheckDBOpenedAsCompactedDB(); ReadOptions read_opts; std::vector key_strs; @@ -726,7 +739,7 @@ TEST_F(DBReadOnlyTestWithTimestamp, } TEST_F(DBReadOnlyTestWithTimestamp, CompactedDBMultiGet) { - const int kNumKeysPerFile = 128; + const int kNumKeysPerFile = 1026 * 2; const uint64_t kMaxKey = 1024; Options options = CurrentOptions(); options.env = env_; @@ -752,14 +765,14 @@ TEST_F(DBReadOnlyTestWithTimestamp, CompactedDBMultiGet) { } } ASSERT_OK(db_->Flush(FlushOptions())); - ASSERT_OK(db_->CompactRange(CompactRangeOptions(), nullptr, nullptr)); Close(); // Reopen the database in read only mode as a Compacted DB to test its // timestamp support. options.max_open_files = -1; ASSERT_OK(ReadOnlyReopen(options)); - ASSERT_EQ(0, NumTableFilesAtLevel(0)); + CheckDBOpenedAsCompactedDB(); + for (size_t i = 0; i < write_timestamps.size(); ++i) { ReadOptions read_opts; Slice read_ts = read_timestamps[i]; From e1ecdfba29c31013b6d1c828bf86b542bd2c25db Mon Sep 17 00:00:00 2001 From: Yu Zhang Date: Mon, 23 May 2022 16:06:38 -0700 Subject: [PATCH 4/5] Add test cases with check CheckDBOpenedAsCompactedDBWithOnlyLmaxFiles for Get and MultiGet --- db/db_readonly_with_timestamp_test.cc | 172 +++++++++++++++++++++++--- 1 file changed, 158 insertions(+), 14 deletions(-) diff --git a/db/db_readonly_with_timestamp_test.cc b/db/db_readonly_with_timestamp_test.cc index 46c2cb54bf2..99168dd8030 100644 --- a/db/db_readonly_with_timestamp_test.cc +++ b/db/db_readonly_with_timestamp_test.cc @@ -17,7 +17,8 @@ class DBReadOnlyTestWithTimestamp : public DBBasicTestWithTimestampBase { : DBBasicTestWithTimestampBase("db_readonly_test_with_timestamp") {} protected: - void CheckDBOpenedAsCompactedDB() { +#ifndef ROCKSDB_LITE + void CheckDBOpenedAsCompactedDBWithOneLevel0File() { VersionSet* const versions = dbfull()->GetVersionSet(); ASSERT_NE(versions, nullptr); @@ -35,6 +36,33 @@ class DBReadOnlyTestWithTimestamp : public DBBasicTestWithTimestampBase { // L0 is the max level. ASSERT_EQ(storage_info->num_non_empty_levels(), 1); } + + void CheckDBOpenedAsCompactedDBWithOnlyLmaxFiles() { + VersionSet* const versions = dbfull()->GetVersionSet(); + ASSERT_NE(versions, nullptr); + + ColumnFamilyData* const cfd = versions->GetColumnFamilySet()->GetDefault(); + ASSERT_NE(cfd, nullptr); + + Version* const current = cfd->current(); + ASSERT_NE(current, nullptr); + + const VersionStorageInfo* const storage_info = current->storage_info(); + ASSERT_NE(storage_info, nullptr); + + // L0 has no files. + ASSERT_EQ(0, NumTableFilesAtLevel(0)); + + // All other levels except the max level have no files. + for (int i = 1; i < storage_info->num_non_empty_levels() - 1; ++i) { + ASSERT_FALSE(storage_info->LevelFilesBrief(i).num_files > 0); + } + + // Lmax has files. + int max_level = storage_info->num_non_empty_levels() - 1; + ASSERT_TRUE(storage_info->LevelFilesBrief(max_level).num_files > 0); + } +#endif // !ROCKSDB_LITE }; #ifndef ROCKSDB_LITE @@ -443,7 +471,7 @@ TEST_F(DBReadOnlyTestWithTimestamp, CompactedDBGetReadTimestampSizeMismatch) { // timestamp support. options.max_open_files = -1; ASSERT_OK(ReadOnlyReopen(options)); - CheckDBOpenedAsCompactedDB(); + CheckDBOpenedAsCompactedDBWithOneLevel0File(); ReadOptions read_opts; std::string different_size_read_timestamp; @@ -482,7 +510,7 @@ TEST_F(DBReadOnlyTestWithTimestamp, // timestamp support. options.max_open_files = -1; ASSERT_OK(ReadOnlyReopen(options)); - CheckDBOpenedAsCompactedDB(); + CheckDBOpenedAsCompactedDBWithOneLevel0File(); ReadOptions read_opts; const std::string read_timestamp = Timestamp(2, 0); @@ -525,7 +553,7 @@ TEST_F(DBReadOnlyTestWithTimestamp, // timestamp support. options.max_open_files = -1; ASSERT_OK(ReadOnlyReopen(options)); - CheckDBOpenedAsCompactedDB(); + CheckDBOpenedAsCompactedDBWithOneLevel0File(); ReadOptions read_opts; for (uint64_t key = 0; key <= kMaxKey; ++key) { @@ -536,7 +564,7 @@ TEST_F(DBReadOnlyTestWithTimestamp, Close(); } -TEST_F(DBReadOnlyTestWithTimestamp, CompactedDBGet) { +TEST_F(DBReadOnlyTestWithTimestamp, CompactedDBGetWithOnlyOneL0File) { const int kNumKeysPerFile = 1026 * 2; const uint64_t kMaxKey = 1024; Options options = CurrentOptions(); @@ -569,12 +597,65 @@ TEST_F(DBReadOnlyTestWithTimestamp, CompactedDBGet) { // timestamp support. options.max_open_files = -1; ASSERT_OK(ReadOnlyReopen(options)); - CheckDBOpenedAsCompactedDB(); + CheckDBOpenedAsCompactedDBWithOneLevel0File(); + + for (size_t i = 0; i < read_timestamps.size(); ++i) { + ReadOptions read_opts; + Slice read_ts = read_timestamps[i]; + read_opts.timestamp = &read_ts; + int count = 0; + for (uint64_t key = start_keys[i]; key <= kMaxKey; ++key, ++count) { + std::string value_from_get; + std::string timestamp; + ASSERT_OK(db_->Get(read_opts, Key1(key), &value_from_get, ×tamp)); + ASSERT_EQ("value" + std::to_string(i), value_from_get); + ASSERT_EQ(write_timestamps[i], timestamp); + } + size_t expected_count = kMaxKey - start_keys[i] + 1; + ASSERT_EQ(expected_count, count); + } + Close(); +} + +TEST_F(DBReadOnlyTestWithTimestamp, CompactedDBGetWithOnlyLmaxFiles) { + const int kNumKeysPerFile = 128; + const uint64_t kMaxKey = 1024; + Options options = CurrentOptions(); + options.env = env_; + options.create_if_missing = true; + options.disable_auto_compactions = true; + const size_t kTimestampSize = Timestamp(0, 0).size(); + TestComparator test_cmp(kTimestampSize); + options.comparator = &test_cmp; + options.memtable_factory.reset( + test::NewSpecialSkipListFactory(kNumKeysPerFile)); + DestroyAndReopen(options); + const std::vector start_keys = {1, 0}; + const std::vector write_timestamps = {Timestamp(1, 0), + Timestamp(3, 0)}; + const std::vector read_timestamps = {Timestamp(2, 0), + Timestamp(4, 0)}; + for (size_t i = 0; i < write_timestamps.size(); ++i) { + WriteOptions write_opts; + for (uint64_t key = start_keys[i]; key <= kMaxKey; ++key) { + Status s = db_->Put(write_opts, Key1(key), write_timestamps[i], + "value" + std::to_string(i)); + ASSERT_OK(s); + } + } + ASSERT_OK(db_->Flush(FlushOptions())); + ASSERT_OK(db_->CompactRange(CompactRangeOptions(), nullptr, nullptr)); + Close(); + + // Reopen the database in read only mode as a Compacted DB to test its + // timestamp support. + options.max_open_files = -1; + ASSERT_OK(ReadOnlyReopen(options)); + CheckDBOpenedAsCompactedDBWithOnlyLmaxFiles(); for (size_t i = 0; i < read_timestamps.size(); ++i) { ReadOptions read_opts; Slice read_ts = read_timestamps[i]; - std::string read_ts_str = read_ts.ToString(); read_opts.timestamp = &read_ts; int count = 0; for (uint64_t key = start_keys[i]; key <= kMaxKey; ++key, ++count) { @@ -618,7 +699,7 @@ TEST_F(DBReadOnlyTestWithTimestamp, // timestamp support. options.max_open_files = -1; ASSERT_OK(ReadOnlyReopen(options)); - CheckDBOpenedAsCompactedDB(); + CheckDBOpenedAsCompactedDBWithOneLevel0File(); ReadOptions read_opts; std::string different_size_read_timestamp; @@ -666,12 +747,11 @@ TEST_F(DBReadOnlyTestWithTimestamp, // timestamp support. options.max_open_files = -1; ASSERT_OK(ReadOnlyReopen(options)); - CheckDBOpenedAsCompactedDB(); + CheckDBOpenedAsCompactedDBWithOneLevel0File(); ReadOptions read_opts; std::string read_timestamp = Timestamp(2, 0); Slice read_ts = read_timestamp; - std::string read_ts_str = read_ts.ToString(); read_opts.timestamp = &read_ts; std::vector key_strs; std::vector keys; @@ -719,7 +799,7 @@ TEST_F(DBReadOnlyTestWithTimestamp, // timestamp support. options.max_open_files = -1; ASSERT_OK(ReadOnlyReopen(options)); - CheckDBOpenedAsCompactedDB(); + CheckDBOpenedAsCompactedDBWithOneLevel0File(); ReadOptions read_opts; std::vector key_strs; @@ -738,7 +818,7 @@ TEST_F(DBReadOnlyTestWithTimestamp, Close(); } -TEST_F(DBReadOnlyTestWithTimestamp, CompactedDBMultiGet) { +TEST_F(DBReadOnlyTestWithTimestamp, CompactedDBMultiGetWithOnlyOneL0File) { const int kNumKeysPerFile = 1026 * 2; const uint64_t kMaxKey = 1024; Options options = CurrentOptions(); @@ -771,12 +851,76 @@ TEST_F(DBReadOnlyTestWithTimestamp, CompactedDBMultiGet) { // timestamp support. options.max_open_files = -1; ASSERT_OK(ReadOnlyReopen(options)); - CheckDBOpenedAsCompactedDB(); + CheckDBOpenedAsCompactedDBWithOneLevel0File(); + + for (size_t i = 0; i < write_timestamps.size(); ++i) { + ReadOptions read_opts; + Slice read_ts = read_timestamps[i]; + read_opts.timestamp = &read_ts; + std::vector key_strs; + std::vector keys; + for (uint64_t key = start_keys[i]; key <= kMaxKey; ++key) { + key_strs.push_back(Key1(key)); + } + for (const auto& key_str : key_strs) { + keys.emplace_back(key_str); + } + size_t batch_size = kMaxKey - start_keys[i] + 1; + std::vector values; + std::vector timestamps; + std::vector status_list = + db_->MultiGet(read_opts, keys, &values, ×tamps); + ASSERT_EQ(batch_size, values.size()); + ASSERT_EQ(batch_size, timestamps.size()); + for (uint64_t idx = 0; idx < values.size(); ++idx) { + ASSERT_EQ("value" + std::to_string(i), values[idx]); + ASSERT_EQ(write_timestamps[i], timestamps[idx]); + ASSERT_OK(status_list[idx]); + } + } + + Close(); +} + +TEST_F(DBReadOnlyTestWithTimestamp, CompactedDBMultiGetWithOnlyLmaxFiles) { + const int kNumKeysPerFile = 128; + const uint64_t kMaxKey = 1024; + Options options = CurrentOptions(); + options.env = env_; + options.create_if_missing = true; + options.disable_auto_compactions = true; + const size_t kTimestampSize = Timestamp(0, 0).size(); + TestComparator test_cmp(kTimestampSize); + options.comparator = &test_cmp; + options.memtable_factory.reset( + test::NewSpecialSkipListFactory(kNumKeysPerFile)); + DestroyAndReopen(options); + const std::vector start_keys = {1, 0}; + const std::vector write_timestamps = {Timestamp(1, 0), + Timestamp(3, 0)}; + const std::vector read_timestamps = {Timestamp(2, 0), + Timestamp(4, 0)}; + for (size_t i = 0; i < write_timestamps.size(); ++i) { + WriteOptions write_opts; + for (uint64_t key = start_keys[i]; key <= kMaxKey; ++key) { + Status s = db_->Put(write_opts, Key1(key), write_timestamps[i], + "value" + std::to_string(i)); + ASSERT_OK(s); + } + } + ASSERT_OK(db_->Flush(FlushOptions())); + ASSERT_OK(db_->CompactRange(CompactRangeOptions(), nullptr, nullptr)); + Close(); + + // Reopen the database in read only mode as a Compacted DB to test its + // timestamp support. + options.max_open_files = -1; + ASSERT_OK(ReadOnlyReopen(options)); + CheckDBOpenedAsCompactedDBWithOnlyLmaxFiles(); for (size_t i = 0; i < write_timestamps.size(); ++i) { ReadOptions read_opts; Slice read_ts = read_timestamps[i]; - std::string read_ts_str = read_ts.ToString(); read_opts.timestamp = &read_ts; std::vector key_strs; std::vector keys; From 293e8ee164297f878ffe52ff641f0b566f8814cd Mon Sep 17 00:00:00 2001 From: Yu Zhang Date: Mon, 23 May 2022 17:15:01 -0700 Subject: [PATCH 5/5] Reword Lmax to HighestNonEmptyLevel in tests --- db/db_readonly_with_timestamp_test.cc | 21 ++++++++++++--------- 1 file changed, 12 insertions(+), 9 deletions(-) diff --git a/db/db_readonly_with_timestamp_test.cc b/db/db_readonly_with_timestamp_test.cc index 99168dd8030..3f53e780632 100644 --- a/db/db_readonly_with_timestamp_test.cc +++ b/db/db_readonly_with_timestamp_test.cc @@ -37,7 +37,7 @@ class DBReadOnlyTestWithTimestamp : public DBBasicTestWithTimestampBase { ASSERT_EQ(storage_info->num_non_empty_levels(), 1); } - void CheckDBOpenedAsCompactedDBWithOnlyLmaxFiles() { + void CheckDBOpenedAsCompactedDBWithOnlyHighestNonEmptyLevelFiles() { VersionSet* const versions = dbfull()->GetVersionSet(); ASSERT_NE(versions, nullptr); @@ -53,14 +53,15 @@ class DBReadOnlyTestWithTimestamp : public DBBasicTestWithTimestampBase { // L0 has no files. ASSERT_EQ(0, NumTableFilesAtLevel(0)); - // All other levels except the max level have no files. + // All other levels have no files except the highest level with files. for (int i = 1; i < storage_info->num_non_empty_levels() - 1; ++i) { ASSERT_FALSE(storage_info->LevelFilesBrief(i).num_files > 0); } - // Lmax has files. - int max_level = storage_info->num_non_empty_levels() - 1; - ASSERT_TRUE(storage_info->LevelFilesBrief(max_level).num_files > 0); + // The highest level with files have some files. + int highest_non_empty_level = storage_info->num_non_empty_levels() - 1; + ASSERT_TRUE( + storage_info->LevelFilesBrief(highest_non_empty_level).num_files > 0); } #endif // !ROCKSDB_LITE }; @@ -617,7 +618,8 @@ TEST_F(DBReadOnlyTestWithTimestamp, CompactedDBGetWithOnlyOneL0File) { Close(); } -TEST_F(DBReadOnlyTestWithTimestamp, CompactedDBGetWithOnlyLmaxFiles) { +TEST_F(DBReadOnlyTestWithTimestamp, + CompactedDBGetWithOnlyHighestNonEmptyLevelFiles) { const int kNumKeysPerFile = 128; const uint64_t kMaxKey = 1024; Options options = CurrentOptions(); @@ -651,7 +653,7 @@ TEST_F(DBReadOnlyTestWithTimestamp, CompactedDBGetWithOnlyLmaxFiles) { // timestamp support. options.max_open_files = -1; ASSERT_OK(ReadOnlyReopen(options)); - CheckDBOpenedAsCompactedDBWithOnlyLmaxFiles(); + CheckDBOpenedAsCompactedDBWithOnlyHighestNonEmptyLevelFiles(); for (size_t i = 0; i < read_timestamps.size(); ++i) { ReadOptions read_opts; @@ -882,7 +884,8 @@ TEST_F(DBReadOnlyTestWithTimestamp, CompactedDBMultiGetWithOnlyOneL0File) { Close(); } -TEST_F(DBReadOnlyTestWithTimestamp, CompactedDBMultiGetWithOnlyLmaxFiles) { +TEST_F(DBReadOnlyTestWithTimestamp, + CompactedDBMultiGetWithOnlyHighestNonEmptyLevelFiles) { const int kNumKeysPerFile = 128; const uint64_t kMaxKey = 1024; Options options = CurrentOptions(); @@ -916,7 +919,7 @@ TEST_F(DBReadOnlyTestWithTimestamp, CompactedDBMultiGetWithOnlyLmaxFiles) { // timestamp support. options.max_open_files = -1; ASSERT_OK(ReadOnlyReopen(options)); - CheckDBOpenedAsCompactedDBWithOnlyLmaxFiles(); + CheckDBOpenedAsCompactedDBWithOnlyHighestNonEmptyLevelFiles(); for (size_t i = 0; i < write_timestamps.size(); ++i) { ReadOptions read_opts;