From 9d60949a3a937f33acfc64db51bb087098db000e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A0=20Arrufat?= <1671644+arrufat@users.noreply.github.com> Date: Sat, 12 Sep 2020 20:55:24 +0900 Subject: [PATCH] Add scale_prev layer (#2171) * Add scale_prev layer * remove comment and fix gradient * add test for scale_ and scale_prev_ layers --- dlib/dnn/layers.h | 122 +++++++++++++++++++++++++++++++++++++ dlib/dnn/layers_abstract.h | 74 ++++++++++++++++++++++ dlib/test/dnn.cpp | 53 ++++++++++++++++ 3 files changed, 249 insertions(+) diff --git a/dlib/dnn/layers.h b/dlib/dnn/layers.h index 2abeb1fc1..c596cbcd8 100644 --- a/dlib/dnn/layers.h +++ b/dlib/dnn/layers.h @@ -2723,6 +2723,128 @@ namespace dlib using scale9_ = scale_; using scale10_ = scale_; +// ---------------------------------------------------------------------------------------- + + template < + template class tag + > + class scale_prev_ + { + public: + const static unsigned long id = tag_id::id; + + scale_prev_() + { + } + + template + void setup (const SUBNET& /*sub*/) + { + } + + template + void forward(const SUBNET& sub, resizable_tensor& output) + { + auto&& src = sub.get_output(); + auto&& scales = layer(sub).get_output(); + DLIB_CASSERT(scales.num_samples() == src.num_samples() && + scales.k() == src.k() && + scales.nr() == 1 && + scales.nc() == 1, + "scales.k(): " << scales.k() << + "\nsrc.k(): " << src.k() + ); + + output.copy_size(src); + tt::scale_channels(false, output, src, scales); + } + + template + void backward(const tensor& gradient_input, SUBNET& sub, tensor& /*params_grad*/) + { + auto&& src = sub.get_output(); + auto&& scales = layer(sub).get_output(); + tt::scale_channels(true, sub.get_gradient_input(), gradient_input, scales); + + if (reshape_src.num_samples() != src.num_samples()) + { + reshape_scales = alias_tensor(src.num_samples()*src.k()); + reshape_src = alias_tensor(src.num_samples()*src.k(),src.nr()*src.nc()); + } + + auto&& scales_grad = layer(sub).get_gradient_input(); + auto sgrad = reshape_scales(scales_grad); + tt::dot_prods(true, sgrad, reshape_src(src), reshape_src(gradient_input)); + } + + const tensor& get_layer_params() const { return params; } + tensor& get_layer_params() { return params; } + + inline dpoint map_input_to_output (const dpoint& p) const { return p; } + inline dpoint map_output_to_input (const dpoint& p) const { return p; } + + friend void serialize(const scale_prev_& item, std::ostream& out) + { + serialize("scale_prev_", out); + serialize(item.reshape_scales, out); + serialize(item.reshape_src, out); + } + + friend void deserialize(scale_prev_& item, std::istream& in) + { + std::string version; + deserialize(version, in); + if (version != "scale_prev_") + throw serialization_error("Unexpected version '"+version+"' found while deserializing dlib::scale_prev_."); + deserialize(item.reshape_scales, in); + deserialize(item.reshape_src, in); + } + + friend std::ostream& operator<<(std::ostream& out, const scale_prev_& /*item*/) + { + out << "scale_prev"<\n"; + } + + private: + alias_tensor reshape_scales; + alias_tensor reshape_src; + resizable_tensor params; + }; + + template < + template class tag, + typename SUBNET + > + using scale_prev = add_layer, SUBNET>; + + template using scale_prev1 = scale_prev; + template using scale_prev2 = scale_prev; + template using scale_prev3 = scale_prev; + template using scale_prev4 = scale_prev; + template using scale_prev5 = scale_prev; + template using scale_prev6 = scale_prev; + template using scale_prev7 = scale_prev; + template using scale_prev8 = scale_prev; + template using scale_prev9 = scale_prev; + template using scale_prev10 = scale_prev; + + using scale_prev1_ = scale_prev_; + using scale_prev2_ = scale_prev_; + using scale_prev3_ = scale_prev_; + using scale_prev4_ = scale_prev_; + using scale_prev5_ = scale_prev_; + using scale_prev6_ = scale_prev_; + using scale_prev7_ = scale_prev_; + using scale_prev8_ = scale_prev_; + using scale_prev9_ = scale_prev_; + using scale_prev10_ = scale_prev_; + // ---------------------------------------------------------------------------------------- class relu_ diff --git a/dlib/dnn/layers_abstract.h b/dlib/dnn/layers_abstract.h index a3e0b5de3..c74145234 100644 --- a/dlib/dnn/layers_abstract.h +++ b/dlib/dnn/layers_abstract.h @@ -2656,6 +2656,80 @@ namespace dlib // ---------------------------------------------------------------------------------------- + template < + template class tag + > + class scale_prev_ + { + /*! + WHAT THIS OBJECT REPRESENTS + This is an implementation of the EXAMPLE_COMPUTATIONAL_LAYER_ interface + defined above. This layer scales the output channels of the tagged layer + by multiplying it with the output of the previous layer. It is excatly the + same as the scale_ layer, but with the inputs swapped, which is useful since + it allows mapping between inputs and outputs of this layer. To be specific: + - Let INPUT == sub.get_output() + - Let SCALES == layer(sub).get_output() + - This layer takes INPUT and SCALES as input. + - The output of this layer has the same dimensions as INPUT. + - This layer requires: + - SCALES.num_samples() == INPUT.num_samples() + - SCALES.k() == INPUT.k() + - SCALES.nr() == 1 + - SCALES.nc() == 1 + - The output tensor is produced by pointwise multiplying SCALES with + INPUT at each spatial location. Therefore, if OUT is the output of + this layer then we would have: + OUT(n,k,r,c) == INPUT(n,k,r,c)*SCALES(n,k) + !*/ + + public: + scale_prev_( + ); + + template void setup (const SUBNET& sub); + template void forward(const SUBNET& sub, resizable_tensor& output); + template void backward(const tensor& gradient_input, SUBNET& sub, tensor& params_grad); + dpoint map_input_to_output(dpoint p) const; + dpoint map_output_to_input(dpoint p) const; + const tensor& get_layer_params() const; + tensor& get_layer_params(); + /*! + These functions are implemented as described in the EXAMPLE_COMPUTATIONAL_LAYER_ interface. + !*/ + }; + + + template < + template class tag, + typename SUBNET + > + using scale_prev = add_layer, SUBNET>; + + // Here we add some convenient aliases for using scale_prev_ with the tag layers. + template using scale_prev1 = scale_prev; + template using scale_prev2 = scale_prev; + template using scale_prev3 = scale_prev; + template using scale_prev4 = scale_prev; + template using scale_prev5 = scale_prev; + template using scale_prev6 = scale_prev; + template using scale_prev7 = scale_prev; + template using scale_prev8 = scale_prev; + template using scale_prev9 = scale_prev; + template using scale_prev10 = scale_prev; + using scale_prev1_ = scale_prev_; + using scale_prev2_ = scale_prev_; + using scale_prev3_ = scale_prev_; + using scale_prev4_ = scale_prev_; + using scale_prev5_ = scale_prev_; + using scale_prev6_ = scale_prev_; + using scale_prev7_ = scale_prev_; + using scale_prev8_ = scale_prev_; + using scale_prev9_ = scale_prev_; + using scale_prev10_ = scale_prev_; + + // ---------------------------------------------------------------------------------------- + template< template class... TAG_TYPES > diff --git a/dlib/test/dnn.cpp b/dlib/test/dnn.cpp index 160a5b38c..5c1f5c133 100644 --- a/dlib/test/dnn.cpp +++ b/dlib/test/dnn.cpp @@ -3706,6 +3706,58 @@ namespace } + void test_layers_scale_and_scale_prev() + { + print_spinner(); + using net_type1 = scale1>>>; + using net_type2 = scale_prev2>>>>>; + + dlib::tt::tensor_rand rnd; + dlib::resizable_tensor x(1, 3, 64, 64); + rnd.fill_gaussian(x); + net_type1 net1; + net_type2 net2; + net1.forward(x); + net2.forward(x); + + // make sure both convolutional layers have the same weights + layer<3>(net2).layer_details() = layer<1>(net1).layer_details(); + const auto& params1 = layer<1>(net1).layer_details().get_layer_params(); + const auto& params2 = layer<3>(net2).layer_details().get_layer_params(); + DLIB_CASSERT(params1.size() == params2.size()); + for (size_t i = 0; i < params1.size(); ++i) + { + DLIB_CASSERT(*(params1.begin() + i) == *(params2.begin() + i)); + } + net2.forward(x); + + // make sure both outputs are the same + const auto& out1 = net1.get_output(); + const auto& out2 = net2.get_output(); + DLIB_TEST(out1.size() == out2.size()); + for (size_t i = 0; i < out1.size(); ++i) + { + DLIB_TEST(*(out1.begin() + i) == *(out2.begin() + i)); + } + + // make sure gradients are the same (within some precision) + const double epsilon = 1e-4; + dlib::resizable_tensor gradient(out1); + rnd.fill_gaussian(gradient); + + net1.back_propagate_error(x, gradient); + const auto& grad1 = layer<1>(net1).get_parameter_gradient(); + + net2.back_propagate_error(x, gradient); + const auto& grad2 = layer<3>(net2).get_parameter_gradient(); + + DLIB_TEST(grad1.size() == grad2.size()); + for (size_t i = 0; i < grad1.size(); ++i) + { + DLIB_TEST(::std::abs(*(grad1.begin() + i) - *(grad2.begin() + i)) < epsilon); + } + } + // ---------------------------------------------------------------------------------------- // This test really just checks if the mmod loss goes negative when a whole lot of overlapping @@ -3887,6 +3939,7 @@ namespace test_loss_dot(); test_loss_multimulticlass_log(); test_loss_mmod(); + test_layers_scale_and_scale_prev(); } void perform_test()