From 5074850356e76a4c601cbbdc7d434f59c313d9e3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Adri=C3=A0=20Arrufat?= <1671644+arrufat@users.noreply.github.com> Date: Wed, 14 Oct 2020 21:17:30 +0900 Subject: [PATCH] fix backtracking when losses stay at inf (fixes #2206) (#2209) * fix backtracking when losses stay at inf * always backtrack when there is an inf value --- dlib/dnn/trainer.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dlib/dnn/trainer.h b/dlib/dnn/trainer.h index 9f004af42..0de0b6f42 100644 --- a/dlib/dnn/trainer.h +++ b/dlib/dnn/trainer.h @@ -1113,10 +1113,10 @@ namespace dlib while (previous_loss_values_to_keep_until_disk_sync.size() > 2 * gradient_updates_since_last_sync) previous_loss_values_to_keep_until_disk_sync.pop_front(); - // Always retry if there are any nan values + // Always retry if there are any nan or inf values for (auto x : previous_loss_values_to_keep_until_disk_sync) { - if (std::isnan(x)) + if (std::isnan(x) || std::isinf(x)) return true; }