diff --git a/cachelib/adaptor/rocks_secondary_cache/CMakeLists.txt b/cachelib/adaptor/rocks_secondary_cache/CMakeLists.txt new file mode 100644 index 0000000000..498ab77c44 --- /dev/null +++ b/cachelib/adaptor/rocks_secondary_cache/CMakeLists.txt @@ -0,0 +1,13 @@ +cmake_minimum_required(VERSION 3.4) + +find_package(cachelib REQUIRED) + +# Suppresses errors from folly exceptions +set(cachelib_SOURCES CachelibWrapper.cpp PARENT_SCOPE) +set(cachelib_INCLUDE_PATHS ${CACHELIB_INCLUDE_DIR}/.. PARENT_SCOPE) +set(cachelib_COMPILE_FLAGS -Wno-error=class-memaccess -Wno-error=sign-compare -Wno-error=deprecated-declarationsPARENT_SCOPE) +#set(cachelib_LIBS "cachelib_allocator" "cachelib_shm" "cachelib_navy" "cachelib_common" "thriftcpp2" "thriftfrozen2" "thriftmetadata" "thriftanyrep" "thrifttype" "thrifttyperep" "thriftannotation" "thriftprotocol" "async" "wangle" "fizz" "sodium" "rt" PARENT_SCOPE) +set(cachelib_LIBS "cachelib_allocator" "cachelib_shm" "cachelib_navy" "cachelib_common" "thriftcpp2" "thriftfrozen2" "thriftmetadata" "thriftanyrep" "thrifttype" "thrifttyperep" "thriftannotation" "thriftprotocol" "async" "wangle" "fizz" "folly" "folly_test_util" "fmtd" "glogd" "sodium" "boost_context" "rt" PARENT_SCOPE) +set(cachelib_FUNC register_CachelibObjects PARENT_SCOPE) +set(cachelib_LINK_PATHS ${CMAKE_PREFIX_PATH}/lib PARENT_SCOPE) +set(cachelib_TESTS tests/CachelibWrapperTest.cpp PARENT_SCOPE) diff --git a/cachelib/adaptor/rocks_secondary_cache/CachelibWrapper.cpp b/cachelib/adaptor/rocks_secondary_cache/CachelibWrapper.cpp index eee8435f6e..5094e29b80 100644 --- a/cachelib/adaptor/rocks_secondary_cache/CachelibWrapper.cpp +++ b/cachelib/adaptor/rocks_secondary_cache/CachelibWrapper.cpp @@ -14,18 +14,19 @@ * limitations under the License. */ -#include "cachelib/adaptor/rocks_secondary_cache/CachelibWrapper.h" +#include "CachelibWrapper.h" -#include "cachelib/facebook/utils/FbInternalRuntimeUpdateWrapper.h" #include "folly/init/Init.h" #include "folly/synchronization/Rcu.h" +#include "rocksdb/convenience.h" #include "rocksdb/version.h" +#include "rocksdb/utilities/object_registry.h" +#include "rocksdb/utilities/options_type.h" namespace facebook { namespace rocks_secondary_cache { #define FB_CACHE_MAX_ITEM_SIZE 4 << 20 -using ApiWrapper = cachelib::FbInternalRuntimeUpdateWrapper; namespace { // We use a separate RCU domain since read side critical sections can block @@ -129,8 +130,68 @@ class RocksCachelibWrapperHandle : public rocksdb::SecondaryCacheResultHandle { } } }; + +static std::unordered_map +rocks_cachelib_type_info = { +#ifndef ROCKSDB_LITE + {"cachename", {offsetof(struct RocksCachelibOptions, cacheName), ROCKSDB_NAMESPACE::OptionType::kString}}, + {"filename", {offsetof(struct RocksCachelibOptions, fileName), ROCKSDB_NAMESPACE::OptionType::kString}}, + {"size", {offsetof(struct RocksCachelibOptions, size), ROCKSDB_NAMESPACE::OptionType::kSizeT}}, + {"block_size", {offsetof(struct RocksCachelibOptions, blockSize), ROCKSDB_NAMESPACE::OptionType::kSizeT}}, + {"region_size",{offsetof(struct RocksCachelibOptions, regionSize), ROCKSDB_NAMESPACE::OptionType::kSizeT}}, + {"policy", {offsetof(struct RocksCachelibOptions, admPolicy), ROCKSDB_NAMESPACE::OptionType::kString}}, + {"probability",{offsetof(struct RocksCachelibOptions, admProbability), ROCKSDB_NAMESPACE::OptionType::kDouble}}, + {"max_write_rate",{offsetof(struct RocksCachelibOptions, maxWriteRate), ROCKSDB_NAMESPACE::OptionType::kUInt64T}}, + {"admission_write_rate",{offsetof(struct RocksCachelibOptions, admissionWriteRate), ROCKSDB_NAMESPACE::OptionType::kUInt64T}}, + {"volatile_size",{offsetof(struct RocksCachelibOptions, volatileSize), ROCKSDB_NAMESPACE::OptionType::kSizeT}}, + {"bucket_power", {offsetof(struct RocksCachelibOptions, bktPower), ROCKSDB_NAMESPACE::OptionType::kUInt32T}}, + {"lock_power", {offsetof(struct RocksCachelibOptions, lockPower), ROCKSDB_NAMESPACE::OptionType::kUInt32T}}, +#endif // ROCKSDB_LITE +}; } // namespace +RocksCachelibWrapper::RocksCachelibWrapper(const RocksCachelibOptions& options) + : options_(options), cache_(nullptr) { + RegisterOptions(&options_, &rocks_cachelib_type_info); +} + +ROCKSDB_NAMESPACE::Status RocksCachelibWrapper::PrepareOptions(const ROCKSDB_NAMESPACE::ConfigOptions& opts) { + FbCache* cache = cache_.load(); + + if (!cache) { + cachelib::PoolId defaultPool; + FbCacheConfig config; + NvmCacheConfig nvmConfig; + + nvmConfig.navyConfig.setBlockSize(options_.blockSize); + nvmConfig.navyConfig.setSimpleFile(options_.fileName, + options_.size, + /*truncateFile=*/true); + nvmConfig.navyConfig.blockCache().setRegionSize(options_.regionSize); + if (options_.admPolicy == "random") { + nvmConfig.navyConfig.enableRandomAdmPolicy().setAdmProbability( + options_.admProbability); + } else { + nvmConfig.navyConfig.enableDynamicRandomAdmPolicy() + .setMaxWriteRate(options_.maxWriteRate) + .setAdmWriteRate(options_.admissionWriteRate); + } + nvmConfig.enableFastNegativeLookups = true; + + config.setCacheSize(options_.volatileSize) + .setCacheName(options_.cacheName) + .setAccessConfig( + {options_.bktPower /* bucket power */, options_.lockPower /* lock power */}) + .enableNvmCache(nvmConfig) + .validate(); // will throw if bad config + auto new_cache = std::make_unique(config); + pool_ = + new_cache->addPool("default", new_cache->getCacheMemoryStats().cacheSize); + cache_.store(new_cache.release()); + } + return SecondaryCache::PrepareOptions(opts); +} + RocksCachelibWrapper::~RocksCachelibWrapper() { Close(); } rocksdb::Status RocksCachelibWrapper::Insert( @@ -225,64 +286,31 @@ void RocksCachelibWrapper::Close() { // sections already started to finish, and then delete the cache cache_.store(nullptr); GetRcuDomain().synchronize(); - admin_.reset(); delete cache; } } -bool RocksCachelibWrapper::UpdateMaxWriteRateForDynamicRandom( - uint64_t maxRate) { - FbCache* cache = cache_.load(); - bool ret = false; - if (cache) { - ret = ApiWrapper::updateMaxRateForDynamicRandomAP(*cache, maxRate); - } - return ret; -} - // Global cache object and a default cache pool std::unique_ptr NewRocksCachelibWrapper( const RocksCachelibOptions& opts) { - std::unique_ptr cache; - std::unique_ptr admin; - cachelib::PoolId defaultPool; - FbCacheConfig config; - NvmCacheConfig nvmConfig; - - nvmConfig.navyConfig.setBlockSize(opts.blockSize); - nvmConfig.navyConfig.setSimpleFile(opts.fileName, - opts.size, - /*truncateFile=*/true); - nvmConfig.navyConfig.blockCache().setRegionSize(opts.regionSize); - if (opts.admPolicy == "random") { - nvmConfig.navyConfig.enableRandomAdmPolicy().setAdmProbability( - opts.admProbability); - } else { - nvmConfig.navyConfig.enableDynamicRandomAdmPolicy() - .setMaxWriteRate(opts.maxWriteRate) - .setAdmWriteRate(opts.admissionWriteRate); - } - nvmConfig.enableFastNegativeLookups = true; - - config.setCacheSize(opts.volatileSize) - .setCacheName(opts.cacheName) - .setAccessConfig( - {opts.bktPower /* bucket power */, opts.lockPower /* lock power */}) - .enableNvmCache(nvmConfig) - .validate(); // will throw if bad config - cache = std::make_unique(config); - defaultPool = - cache->addPool("default", cache->getCacheMemoryStats().cacheSize); - - if (opts.fb303Stats) { - cachelib::CacheAdmin::Config adminConfig; - adminConfig.oncall = opts.oncallName; - admin = std::make_unique(*cache, adminConfig); - } - - return std::unique_ptr(new RocksCachelibWrapper( - std::move(cache), std::move(admin), std::move(defaultPool))); + std::unique_ptr secondary = std::make_unique(opts); + assert(secondary->PrepareOptions(ROCKSDB_NAMESPACE::ConfigOptions()).ok()); + return secondary; } +#ifndef ROCKSDB_LITE +int register_CachelibObjects(ROCKSDB_NAMESPACE::ObjectLibrary& library, const std::string&) { + library.AddFactory(RocksCachelibWrapper::kClassName(), + [](const std::string& uri, std::unique_ptr* guard, + std::string* /*errmsg*/) { + RocksCachelibOptions options; + guard->reset(new RocksCachelibWrapper(options)); + return guard->get(); + }); + return 1; +} +#endif // ROCKSDB_LITE } // namespace rocks_secondary_cache } // namespace facebook + + diff --git a/cachelib/adaptor/rocks_secondary_cache/CachelibWrapper.h b/cachelib/adaptor/rocks_secondary_cache/CachelibWrapper.h index 9b1565034d..05b5b456af 100644 --- a/cachelib/adaptor/rocks_secondary_cache/CachelibWrapper.h +++ b/cachelib/adaptor/rocks_secondary_cache/CachelibWrapper.h @@ -16,15 +16,20 @@ #pragma once #include "cachelib/allocator/CacheAllocator.h" -#include "cachelib/facebook/admin/CacheAdmin.h" #include "rocksdb/secondary_cache.h" #include "rocksdb/types.h" #include "rocksdb/version.h" +namespace ROCKSDB_NAMESPACE { +class ObjectLibrary; +} // namespace ROCKSDB_NAMESPACE + namespace facebook { namespace rocks_secondary_cache { // Options structure for configuring a Cachelib SecondaryCache instance struct RocksCachelibOptions { + static const char *kName() { return "RocksCachelibOptions"; } + // A name for the use case std::string cacheName; @@ -82,37 +87,33 @@ using FbCacheReadHandle = typename FbCache::ReadHandle; using FbCacheItem = typename FbCache::Item; // The RocksCachelibWrapper is a concrete implementation of -// rocksdb::SecondaryCache. It can be allocated using +// ROCKSDB_NAMESPACE::SecondaryCache. It can be allocated using // NewRocksCachelibWrapper() and the resulting pointer -// can be passed in rocksdb::LRUCacheOptions to -// rocksdb::NewLRUCache(). +// can be passed in ROCKSDB_NAMESPACE::LRUCacheOptions to +// ROCKSDB_NAMESPACE::NewLRUCache(). // // Users can also cast a pointer to it and call methods on // it directly, especially custom methods that may be added // in the future. For example - -// std::unique_ptr cache = +// std::unique_ptr cache = // NewRocksCachelibWrapper(opts); // static_cast(cache.get())->Erase(key); -class RocksCachelibWrapper : public rocksdb::SecondaryCache { +class RocksCachelibWrapper : public ROCKSDB_NAMESPACE::SecondaryCache { public: - RocksCachelibWrapper(std::unique_ptr&& cache, - std::unique_ptr&& admin, - cachelib::PoolId pool) - : cache_(std::move(cache).release()), - admin_(std::move(admin)), - pool_(pool) {} + RocksCachelibWrapper(const RocksCachelibOptions& options); ~RocksCachelibWrapper() override; - const char* Name() const override { return "RocksCachelibWrapper"; } + static const char* kClassName() { return "RocksCachelibWrapper"; } + const char* Name() const override { return kClassName(); } - rocksdb::Status Insert( - const rocksdb::Slice& key, + ROCKSDB_NAMESPACE::Status Insert( + const ROCKSDB_NAMESPACE::Slice& key, void* value, - const rocksdb::Cache::CacheItemHelper* helper) override; + const ROCKSDB_NAMESPACE::Cache::CacheItemHelper* helper) override; - std::unique_ptr Lookup( - const rocksdb::Slice& key, - const rocksdb::Cache::CreateCallback& create_cb, + std::unique_ptr Lookup( + const ROCKSDB_NAMESPACE::Slice& key, + const ROCKSDB_NAMESPACE::Cache::CreateCallback& create_cb, bool wait #if ROCKSDB_MAJOR > 7 || (ROCKSDB_MAJOR == 7 && ROCKSDB_MINOR >= 7) , @@ -125,14 +126,15 @@ class RocksCachelibWrapper : public rocksdb::SecondaryCache { bool SupportForceErase() const override { return false; } #endif - void Erase(const rocksdb::Slice& key) override; + void Erase(const ROCKSDB_NAMESPACE::Slice& key) override; void WaitAll( - std::vector handles) override; + std::vector handles) override; // TODO std::string GetPrintableOptions() const override { return ""; } - + ROCKSDB_NAMESPACE::Status PrepareOptions(const ROCKSDB_NAMESPACE::ConfigOptions& /*options*/) override; + // Calling Close() persists the cachelib state to the file and frees the // cachelib object. After calling Close(), subsequent lookups will fail, // and subsequent inserts will be silently ignored. Close() is not thread @@ -140,19 +142,20 @@ class RocksCachelibWrapper : public rocksdb::SecondaryCache { // ongoing lookups and inserts by other threads to be quiesced. void Close(); - // If the admPolicy in RocksCachelibOptions was set to "dynamic_random", - // then this function can be called to update the max write rate for that - // policy. - bool UpdateMaxWriteRateForDynamicRandom(uint64_t maxRate); - private: + RocksCachelibOptions options_; std::atomic cache_; - std::unique_ptr admin_; cachelib::PoolId pool_; }; -// Allocate a new Cache instance with a rocksdb::TieredCache wrapper around it -extern std::unique_ptr NewRocksCachelibWrapper( +// Allocate a new Cache instance with a ROCKSDB_NAMESPACE::TieredCache wrapper around it +extern std::unique_ptr NewRocksCachelibWrapper( const RocksCachelibOptions& opts); +#ifndef ROCKSDB_LITE +extern "C" { +int register_CachelibObjects(ROCKSDB_NAMESPACE::ObjectLibrary& library, const std::string&); +} // extern "C" + +#endif // ROCKSDB_LITE } // namespace rocks_secondary_cache } // namespace facebook diff --git a/cachelib/adaptor/rocks_secondary_cache/README.md b/cachelib/adaptor/rocks_secondary_cache/README.md new file mode 100644 index 0000000000..09cb067caa --- /dev/null +++ b/cachelib/adaptor/rocks_secondary_cache/README.md @@ -0,0 +1,44 @@ +# RocksDB CacheLib Secondary Cache + +This directory contains files and tests for building Cachelib as a SecondaryCache for RocksDB. + +# Build + +Currently the build is only supported via cmake. + +First, build cachelib through the normal build procedure. + +To build under RocksDB, link this directory adaptor/rocksdb_secondary_cache) to the plugins/cachelib directory under RocksDB: +``` +$ln -s .../secondary_cache .../plugins/cachelib +``` +This will allow RocksDB to find and build the CacheLib plugin code. + +Next, under the RocksDB build directory, instruct RocksDB to build and include the cachelib plugin: +``` +$cmake -DROCKSDB_PLUGINS=cachelib -DCMAKE_PREFIX_PATH= -DCMAKE_MODULE_PATH="" .. +``` +where the prefix path points to the opt/cachelib directory and module path points to the cachelib/cmake directory. + +Finally, build RocksDB using "make". + +# Using the RocksDB Cachelib Secondary Cache + +The Secondary Cache can be created via either the SecondaryCache::CreateFromString or NewRocksCachelibWrapper APIs. +``` +SecondaryCache::CreateFromString(..., "id=RocksCachelibWrapper; ..."); + +RoksCachelibOptions options; +NewRocksCachelibWrapper(options); +``` + +When using CreateFromString API, the options can be specified as name-value pairs on the command line. The mapping between the field names and options is found in CacheLibWrapper.cpp. + +# Issues and TODO + +* There are incompatibilities between RocksDB and Cachelib on the version of GTEST required and supported. In order to successfully build, you must use the GTEST version in Cachelib and remove the gtest from the RocksDB build. This requirement is true whether or not you are building tests. In the RocksDB build, you must remove GTEST from the "include_directories" and "add_subdirectories" and add " -Wno-error=deprecated-declarations" to the CMAKE_CXX_FLAGS. You may also need to add "-Wno-error=sign-compare" to build the RocksDB tests, depending on your compiler. + +* RocksDB does not currently enable plugins to build their own tests (see PR11052). When the CachelibWrapperTest is added, additional link libraries may be required (See the CMakeLists.txt file). No investigation as to the reason has been undertaken. + + + diff --git a/cachelib/adaptor/rocks_secondary_cache/tests/CachelibWrapperTest.cpp b/cachelib/adaptor/rocks_secondary_cache/tests/CachelibWrapperTest.cpp index 2bd72b9cad..6a566aabfd 100644 --- a/cachelib/adaptor/rocks_secondary_cache/tests/CachelibWrapperTest.cpp +++ b/cachelib/adaptor/rocks_secondary_cache/tests/CachelibWrapperTest.cpp @@ -14,8 +14,11 @@ * limitations under the License. */ -#include -#include +#include "plugin/cachelib/CachelibWrapper.h" +#include "rocksdb/convenience.h" +#include "cachelib/common/Utils.h" +#include "test_util/testharness.h" + #include #include #include @@ -60,7 +63,7 @@ class CachelibWrapperTest : public ::testing::Test { CachelibWrapperTest() : fail_create_(false) { RocksCachelibOptions opts; - path_ = files::FileUtil::recreateRandomTempDir("CachelibWrapperTest"); + path_ = ROCKSDB_NAMESPACE::test::TmpDir(); opts.volatileSize = kVolatileSize; opts.cacheName = "CachelibWrapperTest"; opts.fileName = path_ + "/cachelib_wrapper_test_file"; @@ -162,7 +165,7 @@ TEST_F(CachelibWrapperTest, BasicTest) { ASSERT_EQ(cache()->Insert("k2", &item2, &CachelibWrapperTest::helper_), Status::OK()); - std::unique_ptr handle; + std::unique_ptr handle; bool is_in_sec_cache{false}; handle = cache()->Lookup("k2", test_item_creator, true #if ROCKSDB_MAJOR > 7 || (ROCKSDB_MAJOR == 7 && ROCKSDB_MINOR >= 7) @@ -192,7 +195,7 @@ TEST_F(CachelibWrapperTest, BasicTest) { } TEST_F(CachelibWrapperTest, BasicFailTest) { - std::unique_ptr handle; + std::unique_ptr handle; bool is_in_sec_cache{false}; handle = cache()->Lookup("k1", test_item_creator, true #if ROCKSDB_MAJOR > 7 || (ROCKSDB_MAJOR == 7 && ROCKSDB_MINOR >= 7) @@ -525,6 +528,7 @@ TEST_F(CachelibWrapperTest, WaitAllWhileCloseTest) { pthread_mutex_destroy(&mu); } +#if 0 TEST_F(CachelibWrapperTest, UpdateMaxRateTest) { RocksCachelibOptions opts; opts.volatileSize = kVolatileSize; @@ -539,14 +543,15 @@ TEST_F(CachelibWrapperTest, UpdateMaxRateTest) { ASSERT_TRUE(static_cast(sec_cache.get()) ->UpdateMaxWriteRateForDynamicRandom(32 << 20)); } - +#endif + TEST_F(CachelibWrapperTest, LargeItemTest) { std::string str1 = RandomString(8 << 20); TestItem item1(str1.data(), str1.length()); ASSERT_EQ(cache()->Insert("k1", &item1, &CachelibWrapperTest::helper_), Status::InvalidArgument()); - std::unique_ptr handle; + std::unique_ptr handle; bool is_in_sec_cache{false}; handle = cache()->Lookup("k1", test_item_creator, true, #if ROCKSDB_MAJOR > 7 || (ROCKSDB_MAJOR == 7 && ROCKSDB_MINOR >= 7) @@ -557,5 +562,49 @@ TEST_F(CachelibWrapperTest, LargeItemTest) { handle.reset(); } +#ifndef ROCKSDB_LITE +TEST_F(CachelibWrapperTest, CreateFromString) { + ROCKSDB_NAMESPACE::ConfigOptions opts; + opts.invoke_prepare_options = false; + std::shared_ptr scache; + std::string props = "cachename=foo;" + "filename=bar;" + "size=10000;" + "block_size=512;" + "region_size=4096;" + "volatile_size=1024;" + "policy=unknown;" + "probability=0.42;" + "max_write_rate=11;" + "admission_write_rate=22;" + "bucket_power=48;" + "lock_power=24;"; + + ASSERT_EQ(ROCKSDB_NAMESPACE::SecondaryCache::CreateFromString(opts, + props + "id=" + RocksCachelibWrapper::kClassName(), + &scache), + ROCKSDB_NAMESPACE::Status::OK()); + auto rco = scache->GetOptions(); + ASSERT_NE(rco, nullptr); + ASSERT_EQ(rco->cacheName, "foo"); + ASSERT_EQ(rco->fileName, "bar"); + ASSERT_EQ(rco->size, 10000); + ASSERT_EQ(rco->blockSize, 512); + ASSERT_EQ(rco->regionSize, 4096); + ASSERT_EQ(rco->admProbability, 0.42); + ASSERT_EQ(rco->admPolicy, "unknown"); + ASSERT_EQ(rco->maxWriteRate, 11); + ASSERT_EQ(rco->admissionWriteRate, 22); + ASSERT_EQ(rco->volatileSize, 1024); + ASSERT_EQ(rco->bktPower, 48); + ASSERT_EQ(rco->lockPower, 24); + +} +#endif // ROCKSDB_LITE } // namespace rocks_secondary_cache } // namespace facebook + +int main(int argc, char** argv) { + ::testing::InitGoogleTest(&argc, argv); + return RUN_ALL_TESTS(); +}