DLIB (de)serialization : enhanced STL container support (#2185)

* [DLIB]  STL containers

* [DLIB]  STL containers

* [DLIB] applied code corrections suggested by code review

* [DLIB] applied code corrections suggested by code review

* [DLIB] applied code corrections suggested by code review
This commit is contained in:
pfeatherstone 2020-09-25 13:27:30 +01:00 committed by GitHub
parent 5408b17f74
commit 4125a7bb1f
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
2 changed files with 737 additions and 14 deletions

View File

@ -68,16 +68,27 @@
This file provides serialization support to the following object types:
- The C++ base types (NOT including pointer types)
- The C++ base types (NOT including raw pointer)
- std::string
- std::wstring
- std::vector
- std::list
- std::forward_list
- std::array
- std::deque
- std::map
- std::unordered_map
- std::multimap
- std::unordered_multimap
- std::set
- std::unordered_set
- std::multiset
- std::unordered_multiset
- std::pair
- std::tuple
- std::complex
- std::unique_ptr
- std::shared_ptr
- dlib::uint64
- dlib::int64
- float_details
@ -87,16 +98,27 @@
- Google protocol buffer objects.
This file provides deserialization support to the following object types:
- The C++ base types (NOT including pointer types)
- The C++ base types (NOT including raw pointers)
- std::string
- std::wstring
- std::vector
- std::list
- std::forward_list
- std::array
- std::deque
- std::map
- std::unordered_map
- std::multimap
- std::unordered_multimap
- std::set
- std::unordered_set
- std::multiset
- std::unordered_multiset
- std::pair
- std::tuple
- std::complex
- std::unique_ptr
- std::shared_ptr
- dlib::uint64
- dlib::int64
- float_details
@ -197,12 +219,17 @@
#include <fstream>
#include <string>
#include <vector>
#include <list>
#include <forward_list>
#include <array>
#include <deque>
#include <complex>
#include <map>
#include <unordered_map>
#include <tuple>
#include <memory>
#include <set>
#include <unordered_set>
#include <limits>
#include <type_traits>
#include <utility>
@ -757,6 +784,42 @@ namespace dlib
std::map<domain, range, compare, alloc>& item,
std::istream& in
);
template <typename domain, typename range, typename hash, typename keyEqual, typename alloc>
void serialize (
const std::unordered_map<domain, range, hash, keyEqual, alloc>& item,
std::ostream& out
);
template <typename domain, typename range, typename hash, typename keyEqual, typename alloc>
void deserialize (
std::unordered_map<domain, range, hash, keyEqual, alloc>& item,
std::istream& in
);
template <typename domain, typename range, typename compare, typename alloc>
void serialize (
const std::multimap<domain,range, compare, alloc>& item,
std::ostream& out
);
template <typename domain, typename range, typename compare, typename alloc>
void deserialize (
std::multimap<domain, range, compare, alloc>& item,
std::istream& in
);
template <typename domain, typename range, typename hash, typename keyEqual, typename alloc>
void serialize (
const std::unordered_multimap<domain, range, hash, keyEqual, alloc>& item,
std::ostream& out
);
template <typename domain, typename range, typename hash, typename keyEqual, typename alloc>
void deserialize (
std::unordered_multimap<domain, range, hash, keyEqual, alloc>& item,
std::istream& in
);
template <typename domain, typename compare, typename alloc>
void serialize (
@ -769,6 +832,42 @@ namespace dlib
std::set<domain, compare, alloc>& item,
std::istream& in
);
template <typename domain, typename hash, typename keyEqual, typename alloc>
void serialize (
const std::unordered_set<domain, hash, keyEqual, alloc>& item,
std::ostream& out
);
template <typename domain, typename hash, typename keyEqual, typename alloc>
void deserialize (
std::unordered_set<domain, hash, keyEqual, alloc>& item,
std::istream& in
);
template <typename domain, typename compare, typename alloc>
void serialize (
const std::multiset<domain, compare, alloc>& item,
std::ostream& out
);
template <typename domain, typename compare, typename alloc>
void deserialize (
std::multiset<domain, compare, alloc>& item,
std::istream& in
);
template <typename domain, typename hash, typename keyEqual, typename alloc>
void serialize (
const std::unordered_multiset<domain, hash, keyEqual, alloc>& item,
std::ostream& out
);
template <typename domain, typename hash, typename keyEqual, typename alloc>
void deserialize (
std::unordered_multiset<domain, hash, keyEqual, alloc>& item,
std::istream& in
);
template <typename T, typename alloc>
void serialize (
@ -781,6 +880,30 @@ namespace dlib
std::vector<T,alloc>& item,
std::istream& in
);
template <typename T, typename alloc>
void serialize (
const std::list<T,alloc>& item,
std::ostream& out
);
template <typename T, typename alloc>
void deserialize (
std::list<T,alloc>& item,
std::istream& in
);
template <typename T, typename alloc>
void serialize (
const std::forward_list<T,alloc>& item,
std::ostream& out
);
template <typename T, typename alloc>
void deserialize (
std::forward_list<T,alloc>& item,
std::istream& in
);
template <typename T, typename alloc>
void serialize (
@ -793,7 +916,43 @@ namespace dlib
std::deque<T,alloc>& item,
std::istream& in
);
template <typename... Types>
void serialize (
const std::tuple<Types...>& item,
std::ostream& out
);
template <typename... Types>
void deserialize (
std::tuple<Types...>& item,
std::istream& in
);
template <typename T, typename deleter>
void serialize (
const std::unique_ptr<T, deleter>& item,
std::ostream& out
);
template <typename T, typename deleter>
void deserialize (
std::unique_ptr<T, deleter>& item,
std::istream& in
);
template <typename T>
void serialize (
const std::shared_ptr<T>& item,
std::ostream& out
);
template <typename T>
void deserialize (
std::shared_ptr<T>& item,
std::istream& in
);
inline void serialize (
const std::string& item,
std::ostream& out
@ -933,6 +1092,88 @@ namespace dlib
// ----------------------------------------------------------------------------------------
template<std::size_t I = 0, typename FuncT, typename... Tp>
inline typename std::enable_if<I == sizeof...(Tp), void>::type
for_each_in_tuple(std::tuple<Tp...>&, FuncT)
{}
template<std::size_t I = 0, typename FuncT, typename... Tp>
inline typename std::enable_if<I < sizeof...(Tp), void>::type
for_each_in_tuple(std::tuple<Tp...>& t, FuncT f)
{
f(std::get<I>(t));
for_each_in_tuple<I + 1, FuncT, Tp...>(t, f);
}
template<std::size_t I = 0, typename FuncT, typename... Tp>
inline typename std::enable_if<I == sizeof...(Tp), void>::type
for_each_in_tuple(const std::tuple<Tp...>&, FuncT)
{}
template<std::size_t I = 0, typename FuncT, typename... Tp>
inline typename std::enable_if<I < sizeof...(Tp), void>::type
for_each_in_tuple(const std::tuple<Tp...>& t, FuncT f)
{
f(std::get<I>(t));
for_each_in_tuple<I + 1, FuncT, Tp...>(t, f);
}
struct serialize_tuple_helper
{
serialize_tuple_helper(std::ostream& out_) : out(out_) {}
template<typename T>
void operator()(const T& item)
{
serialize(item, out);
}
std::ostream& out;
};
struct deserialize_tuple_helper
{
deserialize_tuple_helper(std::istream& in_) : in(in_) {}
template<typename T>
void operator()(T& item)
{
deserialize(item, in);
}
std::istream& in;
};
template <typename... Types>
void serialize (
const std::tuple<Types...>& item,
std::ostream& out
)
{
try
{
for_each_in_tuple(item, serialize_tuple_helper(out));
}
catch (serialization_error& e)
{ throw serialization_error(e.info + "\n while serializing object of type std::tuple"); }
}
template <typename... Types>
void deserialize (
std::tuple<Types...>& item,
std::istream& in
)
{
try
{
for_each_in_tuple(item, deserialize_tuple_helper(in));
}
catch (serialization_error& e)
{ throw serialization_error(e.info + "\n while deserializing object of type std::tuple"); }
}
// ----------------------------------------------------------------------------------------
template <typename domain, typename range, typename compare, typename alloc>
void serialize (
const std::map<domain,range, compare, alloc>& item,
@ -983,6 +1224,141 @@ namespace dlib
// ----------------------------------------------------------------------------------------
template <typename domain, typename range, typename hash, typename keyEqual, typename alloc>
void serialize (
const std::unordered_map<domain, range, hash, keyEqual, alloc>& item,
std::ostream& out
)
{
try
{
serialize(item.size(),out);
for (const auto& x : item)
{
serialize(x.first,out);
serialize(x.second,out);
}
}
catch (serialization_error& e)
{ throw serialization_error(e.info + "\n while serializing object of type std::unordered_map"); }
}
template <typename domain, typename range, typename hash, typename keyEqual, typename alloc>
void deserialize (
std::unordered_map<domain, range, hash, keyEqual, alloc>& item,
std::istream& in
)
{
try
{
item.clear();
std::size_t size;
deserialize(size,in);
domain d;
range r;
for (unsigned long i = 0; i < size; ++i)
{
deserialize(d,in);
deserialize(r,in);
item[d] = r;
}
}
catch (serialization_error& e)
{ throw serialization_error(e.info + "\n while deserializing object of type std::unordered_map"); }
}
// ----------------------------------------------------------------------------------------
template <typename domain, typename range, typename compare, typename alloc>
void serialize (
const std::multimap<domain,range, compare, alloc>& item,
std::ostream& out
)
{
try
{
serialize(item.size(),out);
for (const auto& x : item)
{
serialize(x.first,out);
serialize(x.second,out);
}
}
catch (serialization_error& e)
{ throw serialization_error(e.info + "\n while serializing object of type std::multimap"); }
}
template <typename domain, typename range, typename compare, typename alloc>
void deserialize (
std::multimap<domain, range, compare, alloc>& item,
std::istream& in
)
{
try
{
item.clear();
std::size_t size;
deserialize(size,in);
domain d;
range r;
for (unsigned long i = 0; i < size; ++i)
{
deserialize(d,in);
deserialize(r,in);
item.insert({d,r});
}
}
catch (serialization_error& e)
{ throw serialization_error(e.info + "\n while deserializing object of type std::multimap"); }
}
// ----------------------------------------------------------------------------------------
template <typename domain, typename range, typename hash, typename keyEqual, typename alloc>
void serialize (
const std::unordered_multimap<domain, range, hash, keyEqual, alloc>& item,
std::ostream& out
)
{
try
{
serialize(item.size(),out);
for (const auto& x : item)
{
serialize(x.first,out);
serialize(x.second,out);
}
}
catch (serialization_error& e)
{ throw serialization_error(e.info + "\n while serializing object of type std::unordered_multimap"); }
}
template <typename domain, typename range, typename hash, typename keyEqual, typename alloc>
void deserialize (
std::unordered_multimap<domain, range, hash, keyEqual, alloc>& item,
std::istream& in
)
{
try
{
item.clear();
std::size_t size;
deserialize(size,in);
domain d;
range r;
for (unsigned long i = 0; i < size; ++i)
{
deserialize(d,in);
deserialize(r,in);
item.insert({d,r});
}
}
catch (serialization_error& e)
{ throw serialization_error(e.info + "\n while deserializing object of type std::unordered_multimap"); }
}
// ----------------------------------------------------------------------------------------
template <typename domain, typename compare, typename alloc>
void serialize (
const std::set<domain, compare, alloc>& item,
@ -1028,6 +1404,126 @@ namespace dlib
{ throw serialization_error(e.info + "\n while deserializing object of type std::set"); }
}
// ----------------------------------------------------------------------------------------
template <typename domain, typename hash, typename keyEqual, typename alloc>
void serialize (
const std::unordered_set<domain, hash, keyEqual, alloc>& item,
std::ostream& out
)
{
try
{
serialize(item.size(),out);
for (const auto& x : item)
serialize(x,out);
}
catch (serialization_error& e)
{ throw serialization_error(e.info + "\n while serializing object of type std::unordered_set"); }
}
template <typename domain, typename hash, typename keyEqual, typename alloc>
void deserialize (
std::unordered_set<domain, hash, keyEqual, alloc>& item,
std::istream& in
)
{
try
{
item.clear();
std::size_t size;
deserialize(size,in);
domain d;
for (unsigned long i = 0; i < size; ++i)
{
deserialize(d,in);
item.insert(d);
}
}
catch (serialization_error& e)
{ throw serialization_error(e.info + "\n while deserializing object of type std::unordered_set"); }
}
// ----------------------------------------------------------------------------------------
template <typename domain, typename compare, typename alloc>
void serialize (
const std::multiset<domain, compare, alloc>& item,
std::ostream& out
)
{
try
{
serialize(item.size(),out);
for (const auto& x : item)
serialize(x,out);
}
catch (serialization_error& e)
{ throw serialization_error(e.info + "\n while serializing object of type std::multiset"); }
}
template <typename domain, typename compare, typename alloc>
void deserialize (
std::multiset<domain, compare, alloc>& item,
std::istream& in
)
{
try
{
item.clear();
std::size_t size;
deserialize(size,in);
domain d;
for (unsigned long i = 0; i < size; ++i)
{
deserialize(d,in);
item.insert(d);
}
}
catch (serialization_error& e)
{ throw serialization_error(e.info + "\n while deserializing object of type std::multiset"); }
}
// ----------------------------------------------------------------------------------------
template <typename domain, typename hash, typename keyEqual, typename alloc>
void serialize (
const std::unordered_multiset<domain, hash, keyEqual, alloc>& item,
std::ostream& out
)
{
try
{
serialize(item.size(),out);
for (const auto& x : item)
serialize(x,out);
}
catch (serialization_error& e)
{ throw serialization_error(e.info + "\n while serializing object of type std::unordered_multiset"); }
}
template <typename domain, typename hash, typename keyEqual, typename alloc>
void deserialize (
std::unordered_multiset<domain, hash, keyEqual, alloc>& item,
std::istream& in
)
{
try
{
item.clear();
std::size_t size;
deserialize(size,in);
domain d;
for (unsigned long i = 0; i < size; ++i)
{
deserialize(d,in);
item.insert(d);
}
}
catch (serialization_error& e)
{ throw serialization_error(e.info + "\n while deserializing object of type std::unordered_multiset"); }
}
// ----------------------------------------------------------------------------------------
template <typename alloc>
@ -1179,6 +1675,80 @@ namespace dlib
// ----------------------------------------------------------------------------------------
template <typename T, typename alloc>
void serialize (
const std::list<T,alloc>& item,
std::ostream& out
)
{
try
{
const unsigned long size = static_cast<unsigned long>(item.size());
serialize(size,out);
for (const auto& x : item)
serialize(x, out);
}
catch (serialization_error& e)
{ throw serialization_error(e.info + "\n while serializing object of type std::list"); }
}
template <typename T, typename alloc>
void deserialize (
std::list<T,alloc>& item,
std::istream& in
)
{
try
{
unsigned long size;
deserialize(size, in);
item.resize(size);
for (auto& x : item)
deserialize(x, in);
}
catch (serialization_error& e)
{ throw serialization_error(e.info + "\n while deserializing object of type std::list"); }
}
// ----------------------------------------------------------------------------------------
template <typename T, typename alloc>
void serialize (
const std::forward_list<T,alloc>& item,
std::ostream& out
)
{
try
{
const unsigned long size = std::distance(item.begin(), item.end());
serialize(size,out);
for (const auto& x : item)
serialize(x, out);
}
catch (serialization_error& e)
{ throw serialization_error(e.info + "\n while serializing object of type std::forward_list"); }
}
template <typename T, typename alloc>
void deserialize (
std::forward_list<T,alloc>& item,
std::istream& in
)
{
try
{
unsigned long size;
deserialize(size,in);
item.resize(size);
for (auto& x : item)
deserialize(x,in);
}
catch (serialization_error& e)
{ throw serialization_error(e.info + "\n while deserializing object of type std::forward_list"); }
}
// ----------------------------------------------------------------------------------------
template <typename T, typename alloc>
void serialize (
const std::deque<T,alloc>& item,
@ -1592,6 +2162,92 @@ namespace dlib
}
}
// ----------------------------------------------------------------------------------------
template <typename T, typename deleter>
void serialize (
const std::unique_ptr<T, deleter>& item,
std::ostream& out
)
{
try
{
bool is_non_empty = item != nullptr;
serialize(is_non_empty, out);
if (is_non_empty)
serialize(*item, out);
}
catch (serialization_error& e)
{
throw serialization_error(e.info + "\n while serializing an object of type std::unique_ptr");
}
}
template <typename T, typename deleter>
void deserialize (
std::unique_ptr<T, deleter>& item,
std::istream& in
)
{
try
{
//when deserializing unique_ptr, this is fresh state, so reset the pointers, even if item is non-empty
bool is_non_empty;
deserialize(is_non_empty, in);
item.reset(is_non_empty ? new T() : nullptr); //can't use make_unique since dlib does not use C++14 as a minimum requirement.
if (is_non_empty)
deserialize(*item, in);
}
catch (serialization_error& e)
{
throw serialization_error(e.info + "\n while deserializing an object of type std::unique_ptr");
}
}
// ----------------------------------------------------------------------------------------
template <typename T>
void serialize (
const std::shared_ptr<T>& item,
std::ostream& out
)
{
try
{
bool is_non_empty = item != nullptr;
serialize(is_non_empty, out);
if (is_non_empty)
serialize(*item, out);
}
catch (serialization_error& e)
{
throw serialization_error(e.info + "\n while serializing an object of type std::shared_ptr");
}
}
template <typename T>
void deserialize (
std::shared_ptr<T>& item,
std::istream& in
)
{
try
{
//when deserializing shared_ptr, this is fresh state, so reset the pointers, even if item is non-empty
bool is_non_empty;
deserialize(is_non_empty, in);
item = is_non_empty ? std::make_shared<T>() : nullptr;
if (is_non_empty)
deserialize(*item, in);
}
catch (serialization_error& e)
{
throw serialization_error(e.info + "\n while deserializing an object of type std::shared_ptr");
}
}
// ----------------------------------------------------------------------------------------
class proxy_serialize

View File

@ -401,18 +401,46 @@ namespace
dlib::deserialize(item.b_false,in);
}
template<typename T>
bool pointers_values_equal(const unique_ptr<T>& l, const unique_ptr<T>& r)
{
return l && r ? *l == *r : l == r;
}
template<typename T>
bool pointers_values_equal(const shared_ptr<T>& l, const shared_ptr<T>& r)
{
return l && r ? *l == *r : l == r;
}
struct my_custom_type
{
int a;
float b;
std::vector<float> c;
std::list<string> d;
std::forward_list<string> e;
std::pair<int,string> f;
std::tuple<int,string,float> g;
std::map<string,int> h;
std::unordered_map<string, int> i;
std::multimap<string, int> j;
std::unordered_multimap<string, int> k;
std::set<string> l;
std::unordered_set<string> m;
std::multiset<string> n;
std::unordered_multiset<string> o;
std::shared_ptr<string> ptr_shared1;
std::shared_ptr<string> ptr_shared2;
bool operator==(const my_custom_type& rhs) const
{
return std::tie(a,b,c) == std::tie(rhs.a, rhs.b, rhs.c);
{
return std::tie(a,b,c,d,e,f,g,h,i,j,k,l,m,n,o) == std::tie(rhs.a,rhs.b,rhs.c,rhs.d,rhs.e,rhs.f,rhs.g,rhs.h,rhs.i,rhs.j,rhs.k,rhs.l,rhs.m,rhs.n,rhs.o)
&& pointers_values_equal(ptr_shared1, rhs.ptr_shared1)
&& pointers_values_equal(ptr_shared2, rhs.ptr_shared2);
}
DLIB_DEFINE_DEFAULT_SERIALIZATION(my_custom_type, a, b, c);
DLIB_DEFINE_DEFAULT_SERIALIZATION(my_custom_type, a, b, c, d, e, f, g, h, i, j, k, l, m, n, o, ptr_shared1, ptr_shared2);
};
struct my_custom_type_array
@ -1056,10 +1084,41 @@ namespace
void test_macros_and_serializers()
{
std::unique_ptr<string> uptr1, uptr2, uptr3, uptr4;
uptr1.reset(new string("hello from uptr1"));
my_custom_type t1, t2, t3, t4;
t1.a = 1;
t1.b = 2.5;
t1.c.resize(1024);
t1.c = {1.f, 2.f, 3.f, 4.f, 5.f};
t1.d.push_back("hello from back of list");
t1.d.push_back("world from back of list");
t1.d.push_front("world from front of list");
t1.d.push_front("hello from front of list");
t1.e.push_front("world from forward_list");
t1.e.push_front("hello from forward_list");
t1.f = make_pair(2, "hello from pair");
std::get<0>(t1.g) = 2;
std::get<1>(t1.g) = "hello from tuple";
std::get<2>(t1.g) = 1.4142;
t1.h["key"] = 15;
t1.i["key"] = 16;
t1.i.insert({"inserted key", 17});
t1.j.insert({"key", 21});
t1.j.insert({"key", 22});
t1.j.insert({"inserted key", 23});
t1.j.insert({"inserted key", 24});
t1.j.insert({"key", 25});
t1.j.insert({"key", 26});
t1.k.insert({"inserted key", 27});
t1.k.insert({"inserted key", 28});
t1.l.insert("hello from set");
t1.m.insert("hello from unordered_set");
t1.n.insert("hello from multiset");
t1.n.insert("hello from multiset");
t1.o.insert("hello from unordered_multiset");
t1.o.insert("hello from unordered_multiset");
t1.ptr_shared1 = make_shared<string>("hello from shared_ptr");
t2.a = 2;
t2.b = 4.0;
@ -1070,43 +1129,51 @@ namespace
v1.v.push_back(t2);
{
dlib::serialize("serialization_test_macros.dat") << t1 << t2 << v1;
dlib::deserialize("serialization_test_macros.dat") >> t3 >> t4 >> v2;
dlib::serialize("serialization_test_macros.dat") << t1 << t2 << v1 << uptr1 << uptr2;
dlib::deserialize("serialization_test_macros.dat") >> t3 >> t4 >> v2 >> uptr3 >> uptr4;
DLIB_TEST(t1 == t3);
DLIB_TEST(t2 == t4);
DLIB_TEST(v1 == v2);
DLIB_TEST(pointers_values_equal(uptr1, uptr3));
DLIB_TEST(pointers_values_equal(uptr2, uptr4));
}
{
std::stringstream ss;
dlib::serialize(ss) << t1 << t2 << v1;
dlib::deserialize(ss) >> t3 >> t4 >> v2;
dlib::serialize(ss) << t1 << t2 << v1 << uptr1 << uptr2;
dlib::deserialize(ss) >> t3 >> t4 >> v2 >> uptr3 >> uptr4;
DLIB_TEST(t1 == t3);
DLIB_TEST(t2 == t4);
DLIB_TEST(v1 == v2);
DLIB_TEST(pointers_values_equal(uptr1, uptr3));
DLIB_TEST(pointers_values_equal(uptr2, uptr4));
}
{
std::ostringstream sout;
dlib::serialize(sout) << t1 << t2 << v1;
dlib::serialize(sout) << t1 << t2 << v1 << uptr1 << uptr2;
std::istringstream sin(sout.str());
dlib::deserialize(sin) >> t3 >> t4 >> v2;
dlib::deserialize(sin) >> t3 >> t4 >> v2 >> uptr3 >> uptr4;
DLIB_TEST(t1 == t3);
DLIB_TEST(t2 == t4);
DLIB_TEST(v1 == v2);
DLIB_TEST(pointers_values_equal(uptr1, uptr3));
DLIB_TEST(pointers_values_equal(uptr2, uptr4));
}
{
std::vector<char> buf;
dlib::serialize(buf) << t1 << t2 << v1;
dlib::deserialize(buf) >> t3 >> t4 >> v2;
dlib::serialize(buf) << t1 << t2 << v1 << uptr1 << uptr2;
dlib::deserialize(buf) >> t3 >> t4 >> v2 >> uptr3 >> uptr4;
DLIB_TEST(t1 == t3);
DLIB_TEST(t2 == t4);
DLIB_TEST(v1 == v2);
DLIB_TEST(pointers_values_equal(uptr1, uptr3));
DLIB_TEST(pointers_values_equal(uptr2, uptr4));
}
}