From bb7d2ef558e79201eb80d0f970f6a83d7296777e Mon Sep 17 00:00:00 2001 From: Davis King Date: Sat, 3 Sep 2016 07:52:55 -0400 Subject: [PATCH] Added input_tensor_to_output_tensor() and output_tensor_to_input_tensor() along with the mapping functions necessary at each layer to support these routines. --- dlib/dnn/layers.h | 80 +++++++++++++++++++++++++++++++++++ dlib/dnn/layers_abstract.h | 34 +++++++++++++++ dlib/dnn/utilities.h | 58 +++++++++++++++++++++++++ dlib/dnn/utilities_abstract.h | 37 ++++++++++++++++ 4 files changed, 209 insertions(+) diff --git a/dlib/dnn/layers.h b/dlib/dnn/layers.h index 8e5ec1893..212de2061 100644 --- a/dlib/dnn/layers.h +++ b/dlib/dnn/layers.h @@ -69,6 +69,23 @@ namespace dlib void set_bias_learning_rate_multiplier(double val) { bias_learning_rate_multiplier = val; } void set_bias_weight_decay_multiplier(double val) { bias_weight_decay_multiplier = val; } + inline point map_input_to_output ( + point p + ) const + { + p.x() = (p.x()+padding_x()-nc()/2)/stride_x(); + p.y() = (p.y()+padding_y()-nr()/2)/stride_y(); + return p; + } + + inline point map_output_to_input ( + point p + ) const + { + p.x() = p.x()*stride_x() - padding_x() + nc()/2; + p.y() = p.y()*stride_y() - padding_y() + nr()/2; + return p; + } con_ ( const con_& item @@ -317,6 +334,24 @@ namespace dlib long padding_y() const { return padding_y_; } long padding_x() const { return padding_x_; } + inline point map_input_to_output ( + point p + ) const + { + p.x() = (p.x()+padding_x()-nc()/2)/stride_x(); + p.y() = (p.y()+padding_y()-nr()/2)/stride_y(); + return p; + } + + inline point map_output_to_input ( + point p + ) const + { + p.x() = p.x()*stride_x() - padding_x() + nc()/2; + p.y() = p.y()*stride_y() - padding_y() + nr()/2; + return p; + } + max_pool_ ( const max_pool_& item ) : @@ -496,6 +531,24 @@ namespace dlib long padding_y() const { return padding_y_; } long padding_x() const { return padding_x_; } + inline point map_input_to_output ( + point p + ) const + { + p.x() = (p.x()+padding_x()-nc()/2)/stride_x(); + p.y() = (p.y()+padding_y()-nr()/2)/stride_y(); + return p; + } + + inline point map_output_to_input ( + point p + ) const + { + p.x() = p.x()*stride_x() - padding_x() + nc()/2; + p.y() = p.y()*stride_y() - padding_y() + nr()/2; + return p; + } + avg_pool_ ( const avg_pool_& item ) : @@ -683,6 +736,9 @@ namespace dlib void set_bias_learning_rate_multiplier(double val) { bias_learning_rate_multiplier = val; } void set_bias_weight_decay_multiplier(double val) { bias_weight_decay_multiplier = val; } + inline point map_input_to_output (const point& p) const { return p; } + inline point map_output_to_input (const point& p) const { return p; } + template void setup (const SUBNET& sub) @@ -1143,6 +1199,9 @@ namespace dlib tt::multiply(true, data_grad, mask, gradient_input); } + inline point map_input_to_output (const point& p) const { return p; } + inline point map_output_to_input (const point& p) const { return p; } + const tensor& get_layer_params() const { return params; } tensor& get_layer_params() { return params; } @@ -1219,6 +1278,9 @@ namespace dlib tt::affine_transform(output, input, val); } + inline point map_input_to_output (const point& p) const { return p; } + inline point map_output_to_input (const point& p) const { return p; } + void backward_inplace( const tensor& gradient_input, tensor& data_grad, @@ -1324,6 +1386,9 @@ namespace dlib layer_mode get_mode() const { return mode; } + inline point map_input_to_output (const point& p) const { return p; } + inline point map_output_to_input (const point& p) const { return p; } + template void setup (const SUBNET& sub) { @@ -1584,6 +1649,9 @@ namespace dlib tt::relu_gradient(data_grad, computed_output, gradient_input); } + inline point map_input_to_output (const point& p) const { return p; } + inline point map_output_to_input (const point& p) const { return p; } + const tensor& get_layer_params() const { return params; } tensor& get_layer_params() { return params; } @@ -1661,6 +1729,9 @@ namespace dlib gradient_input, params, params_grad); } + inline point map_input_to_output (const point& p) const { return p; } + inline point map_output_to_input (const point& p) const { return p; } + const tensor& get_layer_params() const { return params; } tensor& get_layer_params() { return params; } @@ -1733,6 +1804,9 @@ namespace dlib tt::sigmoid_gradient(data_grad, computed_output, gradient_input); } + inline point map_input_to_output (const point& p) const { return p; } + inline point map_output_to_input (const point& p) const { return p; } + const tensor& get_layer_params() const { return params; } tensor& get_layer_params() { return params; } @@ -1783,6 +1857,9 @@ namespace dlib { } + inline point map_input_to_output (const point& p) const { return p; } + inline point map_output_to_input (const point& p) const { return p; } + void forward_inplace(const tensor& input, tensor& output) { tt::tanh(output, input); @@ -1994,6 +2071,9 @@ namespace dlib impl::concat_helper_impl::split(gradient_input, sub, 0); } + point map_input_to_output(point p) const; + point map_output_to_input(point p) const; + const tensor& get_layer_params() const { return params; } tensor& get_layer_params() { return params; } diff --git a/dlib/dnn/layers_abstract.h b/dlib/dnn/layers_abstract.h index faa5e5eef..b52d3b5b8 100644 --- a/dlib/dnn/layers_abstract.h +++ b/dlib/dnn/layers_abstract.h @@ -348,6 +348,16 @@ namespace dlib - returns the parameters that define the behavior of forward(). !*/ + + point map_input_to_output(point p) const; + point map_output_to_input(point p) const; + /*! + These two functions are optional. If provided, they should map between + (column,row) coordinates in input and output tensors of forward(). Providing + these functions allows you to use global utility functions like + input_tensor_to_output_tensor(). + !*/ + }; std::ostream& operator<<(std::ostream& out, const EXAMPLE_COMPUTATIONAL_LAYER_& item); @@ -740,6 +750,8 @@ namespace dlib template void setup (const SUBNET& sub); template void forward(const SUBNET& sub, resizable_tensor& output); template void backward(const tensor& gradient_input, SUBNET& sub, tensor& params_grad); + point map_input_to_output(point p) const; + point map_output_to_input(point p) const; const tensor& get_layer_params() const; tensor& get_layer_params(); /*! @@ -798,6 +810,8 @@ namespace dlib template void setup (const SUBNET& sub); void forward_inplace(const tensor& input, tensor& output); void backward_inplace(const tensor& gradient_input, tensor& data_grad, tensor& params_grad); + point map_input_to_output(point p) const; + point map_output_to_input(point p) const; const tensor& get_layer_params() const; tensor& get_layer_params(); /*! @@ -850,6 +864,8 @@ namespace dlib template void setup (const SUBNET& sub); void forward_inplace(const tensor& input, tensor& output); void backward_inplace(const tensor& gradient_input, tensor& data_grad, tensor& params_grad); + point map_input_to_output(point p) const; + point map_output_to_input(point p) const; const tensor& get_layer_params() const; tensor& get_layer_params(); /*! @@ -1048,6 +1064,8 @@ namespace dlib template void setup (const SUBNET& sub); template void forward(const SUBNET& sub, resizable_tensor& output); template void backward(const tensor& gradient_input, SUBNET& sub, tensor& params_grad); + point map_input_to_output(point p) const; + point map_output_to_input(point p) const; const tensor& get_layer_params() const; tensor& get_layer_params(); /*! @@ -1139,6 +1157,8 @@ namespace dlib template void setup (const SUBNET& sub); void forward_inplace(const tensor& input, tensor& output); void backward_inplace(const tensor& computed_output, const tensor& gradient_input, tensor& data_grad, tensor& params_grad); + point map_input_to_output(point p) const; + point map_output_to_input(point p) const; const tensor& get_layer_params() const; tensor& get_layer_params(); /*! @@ -1273,6 +1293,8 @@ namespace dlib template void setup (const SUBNET& sub); template void forward(const SUBNET& sub, resizable_tensor& output); template void backward(const tensor& computed_output, const tensor& gradient_input, SUBNET& sub, tensor& params_grad); + point map_input_to_output(point p) const; + point map_output_to_input(point p) const; const tensor& get_layer_params() const; tensor& get_layer_params(); /*! @@ -1416,6 +1438,8 @@ namespace dlib template void setup (const SUBNET& sub); template void forward(const SUBNET& sub, resizable_tensor& output); template void backward(const tensor& computed_output, const tensor& gradient_input, SUBNET& sub, tensor& params_grad); + point map_input_to_output(point p) const; + point map_output_to_input(point p) const; const tensor& get_layer_params() const; tensor& get_layer_params(); /*! @@ -1461,6 +1485,8 @@ namespace dlib template void setup (const SUBNET& sub); void forward_inplace(const tensor& input, tensor& output); void backward_inplace(const tensor& computed_output, const tensor& gradient_input, tensor& data_grad, tensor& params_grad); + point map_input_to_output(point p) const; + point map_output_to_input(point p) const; const tensor& get_layer_params() const; tensor& get_layer_params(); /*! @@ -1514,6 +1540,8 @@ namespace dlib template void setup (const SUBNET& sub); void forward_inplace(const tensor& input, tensor& output); void backward_inplace(const tensor& computed_output, const tensor& gradient_input, tensor& data_grad, tensor& params_grad); + point map_input_to_output(point p) const; + point map_output_to_input(point p) const; const tensor& get_layer_params() const; tensor& get_layer_params(); /*! @@ -1545,6 +1573,8 @@ namespace dlib template void setup (const SUBNET& sub); void forward_inplace(const tensor& input, tensor& output); void backward_inplace(const tensor& computed_output, const tensor& gradient_input, tensor& data_grad, tensor& params_grad); + point map_input_to_output(point p) const; + point map_output_to_input(point p) const; const tensor& get_layer_params() const; tensor& get_layer_params(); /*! @@ -1578,6 +1608,8 @@ namespace dlib template void setup (const SUBNET& sub); void forward_inplace(const tensor& input, tensor& output); void backward_inplace(const tensor& computed_output, const tensor& gradient_input, tensor& data_grad, tensor& params_grad); + point map_input_to_output(point p) const; + point map_output_to_input(point p) const; const tensor& get_layer_params() const; tensor& get_layer_params(); /*! @@ -1729,6 +1761,8 @@ namespace dlib template void setup (const SUBNET& sub); template void forward(const SUBNET& sub, resizable_tensor& output); template void backward(const tensor& gradient_input, SUBNET& sub, tensor& params_grad); + point map_input_to_output(point p) const; + point map_output_to_input(point p) const; const tensor& get_layer_params() const; tensor& get_layer_params(); /*! diff --git a/dlib/dnn/utilities.h b/dlib/dnn/utilities.h index 116d5bbad..4a915f02d 100644 --- a/dlib/dnn/utilities.h +++ b/dlib/dnn/utilities.h @@ -5,6 +5,7 @@ #include "core.h" #include "utilities_abstract.h" +#include "../geometry.h" namespace dlib { @@ -107,6 +108,63 @@ namespace dlib out << "\n"; } +// ---------------------------------------------------------------------------------------- + + namespace impl + { + + class visitor_net_map_input_to_output + { + public: + + visitor_net_map_input_to_output(point& p_) : p(p_) {} + + point& p; + + template + void operator()(size_t idx, const layer_type& net) + { + p = net.layer_details().map_input_to_output(p); + } + }; + + class visitor_net_map_output_to_input + { + public: + visitor_net_map_output_to_input(point& p_) : p(p_) {} + + point& p; + + template + void operator()(size_t idx, const layer_type& net) + { + p = net.layer_details().map_output_to_input(p); + } + }; + } + + template + inline point input_tensor_to_output_tensor( + const net_type& net, + point p + ) + { + impl::visitor_net_map_input_to_output temp(p); + visit_layers_backwards_range<0,net_type::num_layers-1>(net, temp); + return p; + } + + template + inline point output_tensor_to_input_tensor( + const net_type& net, + point p + ) + { + impl::visitor_net_map_output_to_input temp(p); + visit_layers_range<0,net_type::num_layers-1>(net, temp); + return p; + } + // ---------------------------------------------------------------------------------------- } diff --git a/dlib/dnn/utilities_abstract.h b/dlib/dnn/utilities_abstract.h index 77e8d05df..ecffb506f 100644 --- a/dlib/dnn/utilities_abstract.h +++ b/dlib/dnn/utilities_abstract.h @@ -4,6 +4,7 @@ #ifdef DLIB_DNn_UTILITIES_ABSTRACT_H_ #include "core_abstract.h" +#include "../geometry/vector_abstract.h" namespace dlib { @@ -55,6 +56,42 @@ namespace dlib stream. !*/ +// ---------------------------------------------------------------------------------------- + + template + point input_tensor_to_output_tensor( + const net_type& net, + point p + ); + /*! + requires + - net_type is an object of type add_layer, add_skip_layer, or add_tag_layer. + - All layers in the net must provide map_input_to_output() functions. + ensures + - Given a point (i.e. a row,column coordinate) in the input tensor given to + net, this function returns the corresponding point in the output tensor + net.get_output(). This kind of mapping is useful when working with fully + convolutional networks as you will often want to know what parts of the + output feature maps correspond to what parts of the input. + !*/ + +// ---------------------------------------------------------------------------------------- + + template + point output_tensor_to_input_tensor( + const net_type& net, + point p + ); + /*! + requires + - net_type is an object of type add_layer, add_skip_layer, or add_tag_layer. + - All layers in the net must provide map_output_to_input() functions. + ensures + - This function provides the reverse mapping of input_tensor_to_output_tensor(). + That is, given a point in net.get_output(), what is the corresponding point + in the input tensor? + !*/ + // ---------------------------------------------------------------------------------------- }