[Core] Support roughly estimate the memory usage of an ObjList (#217)

core/objlist.hpp

@@ -20,6 +20,8 @@
 #include <unordered_map>
 #include <vector>
 
+#include "boost/random.hpp"
+
 #include "base/disk_store.hpp"
 #include "base/exception.hpp"
 #include "base/serialization.hpp"
@@ -281,6 +283,30 @@ class ObjList : public ObjListBase {
         del_bitmap_.swap(tmp_bool);
     }
 
+    size_t estimated_storage_size(const double sample_rate = 0.005) {
+        if (this->get_vector_size() == 0)
+            return 0;
+        const size_t sample_num = this->get_vector_size() * sample_rate + 1;
+        BinStream bs;
+
+        // sample
+        std::unordered_set<size_t> sample_container;
+        boost::random::mt19937 generator;
+        boost::random::uniform_real_distribution<double> distribution(0.0, 1.0);
+        while (sample_container.size() < sample_num) {
+            size_t index = distribution(generator) * objlist_data_.get_vector_size();
+            sample_container.insert(index);
+        }
+
+        // log the size
+        for (auto iter = sample_container.begin(); iter != sample_container.end(); ++iter)
+            bs << objlist_data_.data_[*iter];
+
+        std::vector<ObjT>& v = objlist_data_.data_;
+        size_t ret = bs.size() * sizeof(char) * v.capacity() / sample_num;
+        return ret;
+    }
+
    protected:
     ObjListData<ObjT> objlist_data_;
     size_t sorted_size_ = 0;

core/objlist_unittest.cpp

@@ -1,5 +1,6 @@
 #include "core/objlist.hpp"
 
+#include <algorithm>
 #include <string>
 #include <vector>
 
@@ -168,5 +169,21 @@ TEST_F(TestObjList, WriteAndRead) {
         EXPECT_EQ(list_to_read.get_del(i), false);
 }
 
+TEST_F(TestObjList, EstimatedStorage) {
+    const size_t len = 1000 * 1000 * 10;
+    ObjList<Obj> test_list;
+    for (size_t i = 0; i < len; ++i) {
+        Obj obj(i);
+        test_list.add_object(std::move(obj));
+    }
+
+    const double sample_rate = 0.005;
+    size_t estimated_storage = test_list.estimated_storage_size(sample_rate);
+    size_t real_storage = test_list.get_data().capacity() * sizeof(Obj);  // just in this class Obj case
+    double diff_time =
+        double(std::max(real_storage, estimated_storage)) / double(std::min(real_storage, estimated_storage));
+    EXPECT_EQ(diff_time, 1);
+}
+
 }  // namespace
 }  // namespace husky