From a49a42d61a8ede8aa0167cba1d3e7b93e136c0ba Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Adri=C3=A0=20Arrufat?= <adria.arrufat@outlook.com>
Date: Mon, 7 Jun 2021 01:00:56 +0900
Subject: [PATCH 01/62] Initial version of the loss_yolo layer, still not
 working

---
 dlib/dnn/loss.h                | 482 +++++++++++++++++++++++++++++++++
 examples/CMakeLists.txt        |   1 +
 examples/dnn_yolo_train_ex.cpp | 256 +++++++++++++++++
 3 files changed, 739 insertions(+)
 create mode 100644 examples/dnn_yolo_train_ex.cpp

diff --git a/dlib/dnn/loss.h b/dlib/dnn/loss.h
index fff8445e42..682f44e1bb 100644
--- a/dlib/dnn/loss.h
+++ b/dlib/dnn/loss.h
@@ -3447,6 +3447,488 @@ namespace dlib
 
 // ----------------------------------------------------------------------------------------
 
+    using yolo_rect = mmod_rect;
+    inline bool operator<(const yolo_rect& lhs, const yolo_rect& rhs)
+    {
+        return lhs.detection_confidence < rhs.detection_confidence;
+    }
+
+    struct yolo_options
+    {
+    public:
+        struct anchor_box_details
+        {
+            anchor_box_details() = default;
+            anchor_box_details(unsigned long w, unsigned long h) : width(w), height(h) {}
+
+            unsigned long width = 0;
+            unsigned long height = 0;
+
+            friend inline void serialize(const anchor_box_details& item, std::ostream& out)
+            {
+                int version = 0;
+                serialize(version, out);
+                serialize(item.width, out);
+                serialize(item.height, out);
+            }
+
+            friend inline void deserialize(anchor_box_details& item, std::istream& in)
+            {
+                int version = 0;
+                deserialize(version, in);
+                deserialize(item.width, in);
+                deserialize(item.height, in);
+            }
+        };
+
+        yolo_options() = default;
+
+        template <template <typename> class TAG_TYPE>
+        void add_anchors(const std::vector<anchor_box_details>& boxes)
+        {
+            anchors[tag_id<TAG_TYPE>::id] = boxes;
+        }
+
+        // map between the stride and the anchor boxes
+        std::unordered_map<int, std::vector<anchor_box_details>> anchors;
+        std::vector<std::string> labels;
+        double confidence_threshold = 0.25;
+        double ignore_iou_threshold = 0.7;
+        test_box_overlap overlaps_nms = test_box_overlap(0.45, 1.0);
+        test_box_overlap overlaps_ignore = test_box_overlap(0.5, 1.0);
+        double lambda_obj = 1.0;
+        double lambda_noobj = 1.0;
+        double lambda_bbr = 0.75;
+        double lambda_cls = 1.0;
+
+    };
+
+    inline void serialize(const yolo_options& item, std::ostream& out)
+    {
+        int version = 1;
+        serialize(version, out);
+        serialize(item.anchors, out);
+        serialize(item.confidence_threshold, out);
+        serialize(item.ignore_iou_threshold, out);
+        serialize(item.overlaps_nms, out);
+        serialize(item.overlaps_ignore, out);
+        serialize(item.lambda_obj, out);
+        serialize(item.lambda_bbr, out);
+        serialize(item.lambda_cls, out);
+    }
+
+    inline void deserialize(yolo_options& item, std::istream& in)
+    {
+        int version = 0;
+        deserialize(version, in);
+        if (version != 1)
+            throw serialization_error("Unexpected version found while deserializing dlib::yolo_options.");
+        deserialize(item.anchors, in);
+        deserialize(item.confidence_threshold, in);
+        deserialize(item.ignore_iou_threshold, in);
+        deserialize(item.overlaps_nms, in);
+        deserialize(item.overlaps_ignore, in);
+        deserialize(item.lambda_obj, in);
+        deserialize(item.lambda_bbr, in);
+        deserialize(item.lambda_cls, in);
+    }
+
+    inline std::ostream& operator<<(std::ostream& out, const std::unordered_map<int, std::vector<yolo_options::anchor_box_details>>& anchors)
+    {
+        out << "anchors: " << anchors.size();
+        return out;
+    }
+
+    namespace impl
+    {
+        template <template <typename> class TAG_TYPE, template <typename> class... TAG_TYPES>
+        struct yolo_helper_impl
+        {
+            constexpr static size_t tag_count()
+            {
+                return 1 + yolo_helper_impl<TAG_TYPES...>::tag_count();
+            }
+
+            static void list_tags(std::ostream& out)
+            {
+                out << tag_id<TAG_TYPE>::id << (tag_count() > 1 ? "," : "");
+                yolo_helper_impl<TAG_TYPES...>::list_tags(out);
+            }
+
+            template <typename SUBNET>
+            static void tensor_to_dets(
+                const tensor& input,
+                const SUBNET& sub,
+                const long n,
+                const yolo_options& options,
+                std::vector<yolo_rect>& dets
+            )
+            {
+                yolo_helper_impl<TAG_TYPE>::tensor_to_dets(input, sub, n, options, dets);
+                yolo_helper_impl<TAG_TYPES...>::tensor_to_dets(input, sub, n, options, dets);
+            }
+
+            template <
+                typename const_label_iterator,
+                typename SUBNET
+            >
+            static void tensor_to_grad(
+                const tensor& input_tensor,
+                const_label_iterator truth,
+                SUBNET& sub,
+                const long n,
+                const yolo_options& options,
+                double& loss
+            )
+            {
+                yolo_helper_impl<TAG_TYPE>::tensor_to_grad(input_tensor, truth, sub, n, options, loss);
+                yolo_helper_impl<TAG_TYPES...>::tensor_to_grad(input_tensor, truth, sub, n, options, loss);
+            }
+        };
+
+        template <template <typename> class TAG_TYPE>
+        struct yolo_helper_impl<TAG_TYPE>
+        {
+            constexpr static size_t tag_count() { return 1; }
+
+            static void list_tags(std::ostream& out) { out << tag_id<TAG_TYPE>::id; }
+
+            template <typename SUBNET>
+            static void tensor_to_dets(
+                const tensor& input_tensor,
+                const SUBNET& sub,
+                const long n,
+                const yolo_options& options,
+                std::vector<yolo_rect>& dets
+            )
+            {
+                DLIB_CASSERT(sub.sample_expansion_factor() == 1, sub.sample_expansion_factor());
+                const tensor& output_tensor = layer<TAG_TYPE>(sub).get_output();
+                const auto stride_x = static_cast<double>(input_tensor.nc()) / output_tensor.nc();
+                const auto stride_y = static_cast<double>(input_tensor.nr()) / output_tensor.nr();
+                const auto& anchors = options.anchors.at(tag_id<TAG_TYPE>::id);
+                const long num_feats = output_tensor.k() / anchors.size();
+                const long num_classes = num_feats - 5;
+                const float* const out_data = output_tensor.host();
+
+                for (size_t a = 0; a < anchors.size(); ++a)
+                {
+                    for (long r = 0; r < output_tensor.nr(); ++r)
+                    {
+                        for (long c = 0; c < output_tensor.nc(); ++c)
+                        {
+                            const float obj = out_data[tensor_index(output_tensor, n, a * num_feats + 4, r, c)];
+                            if (obj > options.confidence_threshold)
+                            {
+                                const auto x = out_data[tensor_index(output_tensor, n, a * num_feats + 0, r, c)];
+                                const auto y = out_data[tensor_index(output_tensor, n, a * num_feats + 1, r, c)];
+                                const auto w = out_data[tensor_index(output_tensor, n, a * num_feats + 2, r, c)];
+                                const auto h = out_data[tensor_index(output_tensor, n, a * num_feats + 3, r, c)];
+                                yolo_rect det(centered_drect(dpoint((x + c) * stride_x, (y + r) * stride_y),
+                                                             w / (1 - w) * anchors[a].width,
+                                                             h / (1 - h) * anchors[a].height));
+                                for (long k = 0; k < num_classes; ++k)
+                                {
+                                    const float conf = out_data[tensor_index(output_tensor, n, a * num_feats + 5 + k, r, c)];
+                                    if (conf > det.detection_confidence)
+                                    {
+                                        det.detection_confidence = conf;
+                                        det.label = options.labels[k];
+                                    }
+                                }
+                                det.detection_confidence *= obj;
+                                if (det.detection_confidence > options.confidence_threshold)
+                                    dets.push_back(std::move(det));
+                            }
+                        }
+                    }
+                }
+            }
+
+            static inline double compute_iou(const yolo_rect& a, const yolo_rect& b)
+            {
+                return a.rect.intersect(b.rect).area() / static_cast<double>((a.rect + b.rect).area());
+            }
+
+            template <
+                typename const_label_iterator,
+                typename SUBNET
+            >
+            static void tensor_to_grad(
+                const tensor& input_tensor,
+                const_label_iterator truth,
+                SUBNET& sub,
+                const long n,
+                const yolo_options& options,
+                double& loss
+            )
+            {
+                DLIB_CASSERT(sub.sample_expansion_factor() == 1, sub.sample_expansion_factor());
+                const tensor& output_tensor = layer<TAG_TYPE>(sub).get_output();
+                const auto stride_x = static_cast<double>(input_tensor.nc()) / output_tensor.nc();
+                const auto stride_y = static_cast<double>(input_tensor.nr()) / output_tensor.nr();
+                const auto& anchors = options.anchors.at(tag_id<TAG_TYPE>::id);
+                const long num_feats = output_tensor.k() / anchors.size();
+                const long num_classes = num_feats - 5;
+                const float* const out_data = output_tensor.host();
+                tensor& grad = layer<TAG_TYPE>(sub).get_gradient_input();
+                float* g = grad.host();
+                const double scale_loss = 1.0 / (output_tensor.num_samples() * output_tensor.nr() * output_tensor.nc());
+                const double scale_grad = 1.0 / output_tensor.num_samples();
+
+                // Compute the objectness loss for all grid cells
+                for (long r = 0; r < output_tensor.nr(); ++r)
+                {
+                    for (long c = 0; c < output_tensor.nc(); ++c)
+                    {
+                        for (size_t a = 0; a < anchors.size(); ++a)
+                        {
+                            const auto x_idx = tensor_index(output_tensor, n, a * num_feats + 0, r, c);
+                            const auto y_idx = tensor_index(output_tensor, n, a * num_feats + 1, r, c);
+                            const auto w_idx = tensor_index(output_tensor, n, a * num_feats + 2, r, c);
+                            const auto h_idx = tensor_index(output_tensor, n, a * num_feats + 3, r, c);
+                            const auto o_idx = tensor_index(output_tensor, n, a * num_feats + 4, r, c);
+
+                            // The prediction at r, c for anchor a
+                            const yolo_rect pred(centered_drect(
+                                dpoint((out_data[x_idx] + c) * stride_x, (out_data[y_idx] + r) * stride_y),
+                                out_data[w_idx] / (1 - out_data[w_idx]) * anchors[a].width,
+                                out_data[h_idx] / (1 - out_data[h_idx]) * anchors[a].height
+                            ));
+
+                            // Find the best IoU for all ground truth boxes
+                            double best_iou = 0;
+                            for (const yolo_rect& truth_box : *truth)
+                            {
+                                if (truth_box.ignore)
+                                    continue;
+                                best_iou = std::max(best_iou, compute_iou(truth_box, pred));
+                            }
+
+                            // Only incur loss for the boxes that are below a certain IoU threshold
+                            if (best_iou < options.ignore_iou_threshold)
+                            {
+                                loss += -scale_loss * options.lambda_noobj * safe_log(1 - out_data[o_idx]);
+                                g[o_idx] = scale_grad * options.lambda_noobj * out_data[o_idx];
+                            }
+                        }
+                    }
+                }
+
+                // Now find the best anchor box for each truth box
+                for (const yolo_rect& truth_box : *truth)
+                {
+                    if (truth_box.ignore)
+                        continue;
+                    const dpoint t_center = dcenter(truth_box);
+                    double best_iou = 0;
+                    size_t best_a = 0;
+                    size_t best_tag_id = 0;
+                    for (const auto& item : options.anchors)
+                    {
+                        const auto tag_id = item.first;
+                        const auto details = item.second;
+                        for (size_t a = 0; a < details.size(); ++a)
+                        {
+                            const yolo_rect anchor(centered_drect(t_center, details[a].width, details[a].height));
+                            const double iou = compute_iou(truth_box, anchor);
+                            if (iou > best_iou)
+                            {
+                                best_iou = iou;
+                                best_a = a;
+                                best_tag_id = tag_id;
+                            }
+                        }
+                    }
+
+                    // Skip if the best anchor is not from the current stride
+                    if (best_tag_id != tag_id<TAG_TYPE>::id)
+                        continue;
+
+                    const long c = t_center.x() / stride_x;
+                    const long r = t_center.y() / stride_y;
+                    const auto x_idx = tensor_index(output_tensor, n, best_a * num_feats + 0, r, c);
+                    const auto y_idx = tensor_index(output_tensor, n, best_a * num_feats + 1, r, c);
+                    const auto w_idx = tensor_index(output_tensor, n, best_a * num_feats + 2, r, c);
+                    const auto h_idx = tensor_index(output_tensor, n, best_a * num_feats + 3, r, c);
+                    const auto o_idx = tensor_index(output_tensor, n, best_a * num_feats + 4, r, c);
+
+                    // This grid cell should detect an object
+                    loss += -scale_loss * options.lambda_obj * safe_log(out_data[o_idx]);
+                    g[o_idx] = scale_grad * options.lambda_obj * (out_data[o_idx] - 1);
+
+                    const double tx = t_center.x() / stride_x - c;
+                    const double ty = t_center.y() / stride_y - r;
+                    const double tw = truth_box.rect.width() / static_cast<double>(anchors[best_a].width + truth_box.rect.width());
+                    const double th = truth_box.rect.height() / static_cast<double>(anchors[best_a].height + truth_box.rect.height());
+                    const double dx = out_data[x_idx] - tx;
+                    const double dy = out_data[y_idx] - ty;
+                    const double dw = out_data[w_idx] - tw;
+                    const double dh = out_data[h_idx] - th;
+
+                    // Compute MSE loss
+                    loss += scale_loss * options.lambda_bbr * (dx * dx + dy * dy + dw * dw + dh * dh);
+
+                    // Compute the gradient
+                    g[x_idx] = scale_grad * options.lambda_bbr * dx;
+                    g[y_idx] = scale_grad * options.lambda_bbr * dy;
+                    g[w_idx] = scale_grad * options.lambda_bbr * dw;
+                    g[h_idx] = scale_grad * options.lambda_bbr * dh;
+
+                    // Compute binary cross-entropy loss
+                    for (long k = 0; k < num_classes; ++k)
+                    {
+                        const auto c_idx = tensor_index(output_tensor, n, best_a * num_feats + 5 + k, r, c);
+                        if (truth_box.label == options.labels[k])
+                        {
+                            loss += -scale_loss * options.lambda_cls * safe_log(out_data[c_idx]);
+                            g[c_idx] = scale_grad * options.lambda_cls * (out_data[c_idx] - 1);
+                        }
+                        else
+                        {
+                            loss += -scale_loss * options.lambda_cls * safe_log(1 - out_data[c_idx]);
+                            g[c_idx] = scale_grad * options.lambda_cls * out_data[c_idx];
+                        }
+                    }
+                }
+            }
+        };
+
+    }  // namespace impl
+
+    template <template <typename> class... TAG_TYPES>
+    class loss_yolo_
+    {
+        static void list_tags(std::ostream& out) { impl::yolo_helper_impl<TAG_TYPES...>::list_tags(out); }
+
+    public:
+
+        typedef std::vector<yolo_rect> training_label_type;
+        typedef std::vector<yolo_rect> output_label_type;
+
+        constexpr static size_t tag_count() { return impl::yolo_helper_impl<TAG_TYPES...>::tag_count(); }
+
+        loss_yolo_() {};
+
+        loss_yolo_(const yolo_options& options) : options(options) { }
+
+        template <
+            typename SUB_TYPE,
+            typename label_iterator
+            >
+        void to_label (
+            const tensor& input_tensor,
+            const SUB_TYPE& sub,
+            label_iterator iter
+        ) const
+        {
+            DLIB_CASSERT(sub.sample_expansion_factor() == 1, sub.sample_expansion_factor());
+            std::vector<yolo_rect> dets_accum;
+            output_label_type final_dets;
+            for (long i = 0; i < input_tensor.num_samples(); ++i)
+            {
+                dets_accum.clear();
+                impl::yolo_helper_impl<TAG_TYPES...>::tensor_to_dets(input_tensor, sub, i, options, dets_accum);
+
+                // Do non-max suppression
+                std::sort(dets_accum.rbegin(), dets_accum.rend());
+                final_dets.clear();
+                for (size_t j = 0; j < dets_accum.size(); ++j)
+                {
+                    if (overlaps_any_box_nms(final_dets, dets_accum[j]))
+                        continue;
+
+                    final_dets.push_back(dets_accum[j]);
+                }
+
+                *iter++ = std::move(final_dets);
+            }
+        }
+
+        template <
+            typename const_label_iterator,
+            typename SUBNET
+        >
+        double compute_loss_value_and_gradient (
+            const tensor& input_tensor,
+            const_label_iterator truth,
+            SUBNET& sub
+        ) const
+        {
+            DLIB_CASSERT(input_tensor.num_samples() != 0);
+            DLIB_CASSERT(sub.sample_expansion_factor() == 1);
+            double loss = 0;
+            for (long i = 0; i < input_tensor.num_samples(); ++i)
+            {
+                impl::yolo_helper_impl<TAG_TYPES...>::tensor_to_grad(input_tensor, truth, sub, i, options, loss);
+            }
+            return loss;
+        }
+
+        void adjust_threshold(float conf_thresh) { options.confidence_threshold = conf_thresh; }
+
+        friend void serialize(const loss_yolo_& item, std::ostream& out)
+        {
+            serialize("loss_yolo_", out);
+            size_t count = tag_count();
+            serialize(count, out);
+            serialize(item.options, out);
+        }
+
+        friend void deserialize(loss_yolo_& item, std::istream& in)
+        {
+            std::string version;
+            deserialize(version, in);
+            if (version != "loss_yolo_")
+                throw serialization_error("Unexpected version found while deserializing dlib::loss_yolo_.");
+            size_t count = 0;
+            deserialize(count, in);
+            if (count != tag_count())
+                throw serialization_error("Invalid number of detection tags " + std::to_string(count) +
+                                          ", while deserializing dlib::loss_yolo_, expecting " +
+                                          std::to_string(tag_count()) + "tags instead.");
+            deserialize(item.options, in);
+        }
+
+        friend std::ostream& operator<<(std::ostream& out, const loss_yolo_& )
+        {
+            out << "loss_yolo\t (" << tag_count() << " output tags: ";
+            list_tags(out);
+            out << ")";
+            return out;
+        }
+
+    private:
+
+        yolo_options options;
+
+        inline bool overlaps_any_box_nms (
+            const std::vector<yolo_rect>& boxes,
+            const yolo_rect& box,
+            const bool classwise = false
+        ) const
+        {
+            for (const auto& b : boxes)
+            {
+                if (options.overlaps_nms(b.rect, box.rect))
+                {
+                    if (classwise)
+                    {
+                        if (b.label == box.label)
+                            return true;
+                    }
+                    else
+                    {
+                        return true;
+                    }
+                }
+            }
+            return false;
+        }
+    };
+
+    template <template <typename> class TAG_1, template <typename> class TAG_2, template <typename> class TAG_3, typename SUBNET>
+    using loss_yolo = add_loss_layer<loss_yolo_<TAG_1, TAG_2, TAG_3>, SUBNET>;
 }
 
 #endif // DLIB_DNn_LOSS_H_
diff --git a/examples/CMakeLists.txt b/examples/CMakeLists.txt
index 893f8a3c79..e3857d6969 100644
--- a/examples/CMakeLists.txt
+++ b/examples/CMakeLists.txt
@@ -155,6 +155,7 @@ if (NOT USING_OLD_VISUAL_STUDIO_COMPILER)
    add_example(dnn_instance_segmentation_train_ex)
    add_example(dnn_metric_learning_on_images_ex)
    add_gui_example(dnn_dcgan_train_ex)
+   add_example(dnn_yolo_train_ex)
 endif()
 
 
diff --git a/examples/dnn_yolo_train_ex.cpp b/examples/dnn_yolo_train_ex.cpp
new file mode 100644
index 0000000000..de12d3af84
--- /dev/null
+++ b/examples/dnn_yolo_train_ex.cpp
@@ -0,0 +1,256 @@
+#include <dlib/cmd_line_parser.h>
+#include <dlib/data_io.h>
+#include <dlib/dir_nav.h>
+#include <dlib/dnn.h>
+#include <dlib/image_io.h>
+
+using namespace std;
+using namespace dlib;
+
+namespace darknet
+{
+    // backbone tags
+    template <typename SUBNET> using btag8 = add_tag_layer<8008, SUBNET>;
+    template <typename SUBNET> using btag16 = add_tag_layer<8016, SUBNET>;
+    template <typename SUBNET> using bskip8 = add_skip_layer<btag8, SUBNET>;
+    template <typename SUBNET> using bskip16 = add_skip_layer<btag16, SUBNET>;
+    // neck tags
+    template <typename SUBNET> using ntag8 = add_tag_layer<6008, SUBNET>;
+    template <typename SUBNET> using ntag16 = add_tag_layer<6016, SUBNET>;
+    template <typename SUBNET> using ntag32 = add_tag_layer<6032, SUBNET>;
+    template <typename SUBNET> using nskip8 = add_skip_layer<ntag8, SUBNET>;
+    template <typename SUBNET> using nskip16 = add_skip_layer<ntag16, SUBNET>;
+    template <typename SUBNET> using nskip32 = add_skip_layer<ntag32, SUBNET>;
+    // head tags
+    template <typename SUBNET> using htag8 = add_tag_layer<7008, SUBNET>;
+    template <typename SUBNET> using htag16 = add_tag_layer<7016, SUBNET>;
+    template <typename SUBNET> using htag32 = add_tag_layer<7032, SUBNET>;
+    template <typename SUBNET> using hskip8 = add_skip_layer<htag8, SUBNET>;
+    template <typename SUBNET> using hskip16 = add_skip_layer<htag16, SUBNET>;
+    template <typename SUBNET> using hskip32 = add_skip_layer<htag32, SUBNET>;
+    // yolo tags
+    template <typename SUBNET> using ytag8 = add_tag_layer<4008, SUBNET>;
+    template <typename SUBNET> using ytag16 = add_tag_layer<4016, SUBNET>;
+    template <typename SUBNET> using ytag32 = add_tag_layer<4032, SUBNET>;
+
+    template <template <typename> class ACT, template <typename> class BN>
+    struct def
+    {
+        template <long nf, long ks, int s, typename SUBNET>
+        using conblock = ACT<BN<add_layer<con_<nf, ks, ks, s, s, ks/2, ks/2>, SUBNET>>>;
+
+        template <long nf1, long nf2, typename SUBNET>
+        using residual = add_prev1<
+                         conblock<nf1, 3, 1,
+                         conblock<nf2, 1, 1,
+                    tag1<SUBNET>>>>;
+
+        template <long nf, long factor, typename SUBNET>
+        using conblock5 = conblock<nf, 1, 1,
+                          conblock<nf * factor, 3, 1,
+                          conblock<nf, 1, 1,
+                          conblock<nf * factor, 3, 1,
+                          conblock<nf, 1, 1, SUBNET>>>>>;
+
+        // the residual block introduced in YOLOv3 (with bottleneck)
+        template <long nf, typename SUBNET> using resv3 = residual<nf, nf / 2, SUBNET>;
+        // the residual block introduced in YOLOv4 (without bottleneck)
+        template <long nf, typename SUBNET> using resv4 = residual<nf, nf, SUBNET>;
+
+        template <typename SUBNET> using resv3_64= resv3<64, SUBNET>;
+        template <typename SUBNET> using resv3_128 = resv3<128, SUBNET>;
+        template <typename SUBNET> using resv3_256 = resv3<256, SUBNET>;
+        template <typename SUBNET> using resv3_512 = resv3<512, SUBNET>;
+        template <typename SUBNET> using resv3_1024 = resv3<1024, SUBNET>;
+
+        template <typename INPUT>
+        using backbone53 = repeat<4, resv3_1024, conblock<1024, 3, 2,
+                    btag16<repeat<8, resv3_512,  conblock<512, 3, 2,
+                     btag8<repeat<8, resv3_256,  conblock<256, 3, 2,
+                           repeat<2, resv3_128,  conblock<128, 3, 2,
+                                     resv3_64<   conblock<64, 3, 2,
+                                                 conblock<32, 3, 1,
+                                                 INPUT>>>>>>>>>>>>>;
+
+
+        template <long nf, int classes, template <typename> class YTAG, template <typename> class NTAG, typename SUBNET>
+        using yolo = YTAG<sig<con<3 * (classes + 5), 1, 1, 1, 1,
+                     conblock<nf, 3, 1,
+                NTAG<conblock5<nf / 2, 2,
+                     SUBNET>>>>>>;
+
+        template <long num_classes>
+        using yolov3 = yolo<256, num_classes, ytag8, ntag8,
+                       concat2<htag8, btag8,
+                 htag8<upsample<2, conblock<128, 1, 1,
+                       nskip16<
+                       yolo<512, num_classes, ytag16, ntag16,
+                       concat2<htag16, btag16,
+                htag16<upsample<2, conblock<256, 1, 1,
+                       nskip32<
+                       yolo<1024, num_classes, ytag32, ntag32,
+                       backbone53<tag1<input_rgb_image>>
+                       >>>>>>>>>>>>>;
+
+    };
+
+    using yolov3_train = def<leaky_relu, bn_con>::yolov3<80>;
+    using yolov3_infer = def<leaky_relu, affine>::yolov3<80>;
+
+    template <typename net_type>
+    void setup_detector(net_type& net, int num_classes = 80, size_t img_size = 416)
+    {
+        // remove bias
+        disable_duplicative_biases(net);
+        // remove mean from input image
+        layer<net_type::num_layers - 1>(net) = input_rgb_image(0, 0, 0);
+        // setup leaky relus
+        visit_computational_layers(net, [](leaky_relu_& l) { l = leaky_relu_(0.1); });
+        // set the number of filters
+        layer<ytag8, 2>(net).layer_details().set_num_filters(3 * (num_classes + 5));
+        layer<ytag16, 2>(net).layer_details().set_num_filters(3 * (num_classes + 5));
+        layer<ytag32, 2>(net).layer_details().set_num_filters(3 * (num_classes + 5));
+        // allocate the network
+        matrix<rgb_pixel> image(img_size, img_size);
+        net(image);
+    }
+
+}
+
+using net_type = dlib::loss_yolo<darknet::ytag8, darknet::ytag16, darknet::ytag32, darknet::yolov3_train>;
+
+int main(const int argc, const char** argv)
+try
+{
+    command_line_parser parser;
+    parser.add_option("size", "image size for training (default: 416)", 1);
+    parser.add_option("learning-rate", "initial learning rate (default: 0.001)", 1);
+    parser.add_option("batch-size", "mini batch size (default: 8)", 1);
+    parser.add_option("warmup", "number of warmup steps (default: 5000)", 1);
+    parser.add_option("steps", "number of training steps (defaule: 500000)", 1);
+    parser.add_option("workers", "number of worker threads to load data (default: 4)", 1);
+    parser.add_option("gpus", "number of GPUs to run the training on (default: 1)", 1);
+    parser.set_group_name("Help Options");
+    parser.add_option("h", "alias of --help");
+    parser.add_option("help", "display this message and exit");
+    parser.parse(argc, argv);
+    if (parser.number_of_arguments() == 0 || parser.option("h") || parser.option("help"))
+    {
+        parser.print_options();
+        cout << "Give the path to a folder containing the training.xml file." << endl;
+        return 0;
+    }
+    const double learning_rate = get_option(parser, "learning-rate", 0.001);
+    const size_t batch_size = get_option(parser, "batch-size", 8);
+    const size_t warmup = get_option(parser, "warmup", 5000);
+    const size_t num_steps = get_option(parser, "steps", 100000);
+    const size_t image_size = get_option(parser, "size", 416);
+    const size_t num_workers = get_option(parser, "workers", 4);
+    const size_t num_gpus = get_option(parser, "gpus", 1);
+    const std::string data_directory = parser[0];
+    image_dataset_metadata::dataset dataset;
+    image_dataset_metadata::load_image_dataset_metadata(dataset, data_directory + "/training.xml");
+    std::cout << "# images: " << dataset.images.size() << std::endl;
+    std::map<std::string, size_t> labels;
+    size_t num_objects = 0;
+    for (const auto& im : dataset.images)
+    {
+        for (const auto& b : im.boxes)
+        {
+            labels[b.label]++;
+            ++num_objects;
+        }
+    }
+    std::cout << "# labels: " << labels.size() << std::endl;
+
+    yolo_options options;
+    for (const auto& label : labels)
+    {
+        std::cout <<  " - " << label.first << ": " << label.second;
+        std::cout << " (" << (100.0*label.second)/num_objects << "%)\n";
+        options.labels.push_back(label.first);
+    }
+    options.confidence_threshold = 0.25;
+    options.add_anchors<darknet::ytag8>({{10, 13}, {16, 30}, {33, 23}});
+    options.add_anchors<darknet::ytag16>({{30, 61}, {62, 45}, {59, 119}});
+    options.add_anchors<darknet::ytag32>({{116, 90}, {156, 198}, {373, 326}});
+    options.overlaps_nms = dlib::test_box_overlap(0.45);
+    net_type net(options);
+    darknet::setup_detector(net, options.labels.size(), image_size);
+
+    // Cosine scheduler with warm-up:
+    // - learning_rate is the highest learning rate value, e.g. 0.01
+    // - warmup: number of steps to linearly increase the learning rate
+    // - steps: maximum number of steps of the training session
+    const matrix<double> learning_rate_schedule = learning_rate * join_rows(
+        linspace(0, 1, warmup),
+        ((1 + cos(pi / (num_steps - warmup) * linspace(0, num_steps - warmup, num_steps - warmup))) / 2)
+    ) + std::numeric_limits<double>::epsilon();  // this prevents the learning rates from being 0
+
+    std::vector<int> gpus(num_gpus);
+    std::iota(gpus.begin(), gpus.end(), 0);
+    dnn_trainer<net_type> trainer(net, sgd(0.0005, 0.9), gpus);
+    trainer.be_verbose();
+    trainer.set_mini_batch_size(batch_size);
+    trainer.set_learning_rate_schedule(learning_rate_schedule);
+    trainer.set_synchronization_file("yolov3_sync", std::chrono::minutes(15));
+    std::cout << trainer;
+    std::cout << "  burnin: " << warmup << std::endl;
+    std::cout << "  #steps: " << num_steps << std::endl;
+
+    dlib::pipe<std::pair<matrix<rgb_pixel>, std::vector<yolo_rect>>> train_data(1000);
+    auto loader = [&dataset, &data_directory, &train_data, &image_size](time_t seed) {
+        dlib::rand rnd(time(nullptr) + seed);
+        matrix<rgb_pixel> image, letterbox;
+        std::pair<matrix<rgb_pixel>, std::vector<yolo_rect>> temp;
+        while (train_data.is_enabled())
+        {
+            std::vector<yolo_rect> boxes;
+            auto idx = rnd.get_random_32bit_number() % dataset.images.size();
+            load_image(image, data_directory + "/" + dataset.images[idx].filename);
+
+            const auto tform = rectangle_transform(letterbox_image(image, letterbox, image_size));
+            for (const auto& box : dataset.images[idx].boxes)
+            {
+                boxes.push_back(yolo_rect(tform(box.rect), 1, box.label));
+            }
+            // here you should do more data augmentation
+            disturb_colors(image, rnd);
+            temp.first = letterbox;
+            temp.second = boxes;
+            train_data.enqueue(temp);
+        }
+    };
+
+    std::vector<std::thread> data_loaders;
+    for (size_t i = 0; i < num_workers; ++i)
+        data_loaders.emplace_back([loader, i]() { loader(i + 1); });
+
+    std::vector<matrix<rgb_pixel>> images;
+    std::vector<std::vector<yolo_rect>> bboxes;
+    while (trainer.get_train_one_step_calls() < num_steps)
+    {
+        images.clear();
+        bboxes.clear();
+        std::pair<matrix<rgb_pixel>, std::vector<yolo_rect>> temp;
+        while (images.size() < trainer.get_mini_batch_size())
+        {
+            train_data.dequeue(temp);
+            images.push_back(std::move(temp.first));
+            bboxes.push_back(std::move(temp.second));
+        }
+        trainer.train_one_step(images, bboxes);
+    }
+
+    trainer.get_net();
+    train_data.disable();
+    for (auto& worker : data_loaders)
+        worker.join();
+
+    serialize("yolov3.dnn") << net;
+}
+catch (const std::exception& e)
+{
+    std::cout << e.what() << '\n';
+    return EXIT_FAILURE;
+}

From dccbaca627a4126bcb394dcade97127bd9ea187e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Adri=C3=A0=20Arrufat?= <adria.arrufat@outlook.com>
Date: Mon, 7 Jun 2021 11:17:40 +0900
Subject: [PATCH 02/62] add option to visualize the detections of the last
 training synchronization file

---
 examples/CMakeLists.txt        |   2 +-
 examples/dnn_yolo_train_ex.cpp | 109 ++++++++++++++++++++++++---------
 2 files changed, 81 insertions(+), 30 deletions(-)

diff --git a/examples/CMakeLists.txt b/examples/CMakeLists.txt
index e3857d6969..6d5e479f57 100644
--- a/examples/CMakeLists.txt
+++ b/examples/CMakeLists.txt
@@ -155,7 +155,7 @@ if (NOT USING_OLD_VISUAL_STUDIO_COMPILER)
    add_example(dnn_instance_segmentation_train_ex)
    add_example(dnn_metric_learning_on_images_ex)
    add_gui_example(dnn_dcgan_train_ex)
-   add_example(dnn_yolo_train_ex)
+   add_gui_example(dnn_yolo_train_ex)
 endif()
 
 
diff --git a/examples/dnn_yolo_train_ex.cpp b/examples/dnn_yolo_train_ex.cpp
index de12d3af84..5556394782 100644
--- a/examples/dnn_yolo_train_ex.cpp
+++ b/examples/dnn_yolo_train_ex.cpp
@@ -1,7 +1,7 @@
 #include <dlib/cmd_line_parser.h>
 #include <dlib/data_io.h>
-#include <dlib/dir_nav.h>
 #include <dlib/dnn.h>
+#include <dlib/gui_widgets.h>
 #include <dlib/image_io.h>
 
 using namespace std;
@@ -27,7 +27,6 @@ namespace darknet
     template <typename SUBNET> using htag32 = add_tag_layer<7032, SUBNET>;
     template <typename SUBNET> using hskip8 = add_skip_layer<htag8, SUBNET>;
     template <typename SUBNET> using hskip16 = add_skip_layer<htag16, SUBNET>;
-    template <typename SUBNET> using hskip32 = add_skip_layer<htag32, SUBNET>;
     // yolo tags
     template <typename SUBNET> using ytag8 = add_tag_layer<4008, SUBNET>;
     template <typename SUBNET> using ytag16 = add_tag_layer<4016, SUBNET>;
@@ -72,7 +71,6 @@ namespace darknet
                                                  conblock<32, 3, 1,
                                                  INPUT>>>>>>>>>>>>>;
 
-
         template <long nf, int classes, template <typename> class YTAG, template <typename> class NTAG, typename SUBNET>
         using yolo = YTAG<sig<con<3 * (classes + 5), 1, 1, 1, 1,
                      conblock<nf, 3, 1,
@@ -98,23 +96,33 @@ namespace darknet
     using yolov3_infer = def<leaky_relu, affine>::yolov3<80>;
 
     template <typename net_type>
-    void setup_detector(net_type& net, int num_classes = 80, size_t img_size = 416)
+    void setup_detector(net_type& net, const yolo_options& options, long image_size)
     {
         // remove bias
         disable_duplicative_biases(net);
-        // remove mean from input image
-        layer<net_type::num_layers - 1>(net) = input_rgb_image(0, 0, 0);
+        // do not remove the mean from the input image
+        input_layer(net) = input_rgb_image(0, 0, 0);
         // setup leaky relus
         visit_computational_layers(net, [](leaky_relu_& l) { l = leaky_relu_(0.1); });
-        // set the number of filters
-        layer<ytag8, 2>(net).layer_details().set_num_filters(3 * (num_classes + 5));
-        layer<ytag16, 2>(net).layer_details().set_num_filters(3 * (num_classes + 5));
-        layer<ytag32, 2>(net).layer_details().set_num_filters(3 * (num_classes + 5));
+        // set the number of filters (they are located after the tag and sig layers)
+        layer<ytag8, 2>(net).layer_details().set_num_filters(3 * (options.labels.size() + 5));
+        layer<ytag16, 2>(net).layer_details().set_num_filters(3 * (options.labels.size() + 5));
+        layer<ytag32, 2>(net).layer_details().set_num_filters(3 * (options.labels.size() + 5));
         // allocate the network
-        matrix<rgb_pixel> image(img_size, img_size);
+        matrix<rgb_pixel> image(image_size, image_size);
         net(image);
     }
+}
+
+rectangle_transform preprocess_image(const matrix<rgb_pixel>& image, matrix<rgb_pixel>& output, const long image_size)
+{
+    return rectangle_transform(inv(letterbox_image(image, output, image_size)));
+}
 
+void postprocess_detections(const rectangle_transform& tform, std::vector<yolo_rect>& detections)
+{
+    for (auto& d : detections)
+        d.rect = tform(d.rect);
 }
 
 using net_type = dlib::loss_yolo<darknet::ytag8, darknet::ytag16, darknet::ytag32, darknet::yolov3_train>;
@@ -130,6 +138,7 @@ try
     parser.add_option("steps", "number of training steps (defaule: 500000)", 1);
     parser.add_option("workers", "number of worker threads to load data (default: 4)", 1);
     parser.add_option("gpus", "number of GPUs to run the training on (default: 1)", 1);
+    parser.add_option("test", "test the detector with a custom threshold (default: 0.01)", 1);
     parser.set_group_name("Help Options");
     parser.add_option("h", "alias of --help");
     parser.add_option("help", "display this message and exit");
@@ -147,11 +156,14 @@ try
     const size_t image_size = get_option(parser, "size", 416);
     const size_t num_workers = get_option(parser, "workers", 4);
     const size_t num_gpus = get_option(parser, "gpus", 1);
-    const std::string data_directory = parser[0];
+
+    const string data_directory = parser[0];
+    const string sync_file_name = "yolov3_sync";
+
     image_dataset_metadata::dataset dataset;
     image_dataset_metadata::load_image_dataset_metadata(dataset, data_directory + "/training.xml");
     std::cout << "# images: " << dataset.images.size() << std::endl;
-    std::map<std::string, size_t> labels;
+    std::map<string, size_t> labels;
     size_t num_objects = 0;
     for (const auto& im : dataset.images)
     {
@@ -161,22 +173,23 @@ try
             ++num_objects;
         }
     }
-    std::cout << "# labels: " << labels.size() << std::endl;
+    cout << "# labels: " << labels.size() << endl;
 
     yolo_options options;
     for (const auto& label : labels)
     {
-        std::cout <<  " - " << label.first << ": " << label.second;
-        std::cout << " (" << (100.0*label.second)/num_objects << "%)\n";
+        cout <<  " - " << label.first << ": " << label.second;
+        cout << " (" << (100.0*label.second)/num_objects << "%)\n";
         options.labels.push_back(label.first);
     }
     options.confidence_threshold = 0.25;
+    options.ignore_iou_threshold = 0.5;
     options.add_anchors<darknet::ytag8>({{10, 13}, {16, 30}, {33, 23}});
     options.add_anchors<darknet::ytag16>({{30, 61}, {62, 45}, {59, 119}});
     options.add_anchors<darknet::ytag32>({{116, 90}, {156, 198}, {373, 326}});
-    options.overlaps_nms = dlib::test_box_overlap(0.45);
+    options.overlaps_nms = test_box_overlap(0.45);
     net_type net(options);
-    darknet::setup_detector(net, options.labels.size(), image_size);
+    darknet::setup_detector(net, options, image_size);
 
     // Cosine scheduler with warm-up:
     // - learning_rate is the highest learning rate value, e.g. 0.01
@@ -185,18 +198,56 @@ try
     const matrix<double> learning_rate_schedule = learning_rate * join_rows(
         linspace(0, 1, warmup),
         ((1 + cos(pi / (num_steps - warmup) * linspace(0, num_steps - warmup, num_steps - warmup))) / 2)
-    ) + std::numeric_limits<double>::epsilon();  // this prevents the learning rates from being 0
+    ) + numeric_limits<double>::epsilon();  // this prevents the learning rates from being 0
 
     std::vector<int> gpus(num_gpus);
-    std::iota(gpus.begin(), gpus.end(), 0);
+    iota(gpus.begin(), gpus.end(), 0);
     dnn_trainer<net_type> trainer(net, sgd(0.0005, 0.9), gpus);
     trainer.be_verbose();
     trainer.set_mini_batch_size(batch_size);
     trainer.set_learning_rate_schedule(learning_rate_schedule);
-    trainer.set_synchronization_file("yolov3_sync", std::chrono::minutes(15));
-    std::cout << trainer;
-    std::cout << "  burnin: " << warmup << std::endl;
-    std::cout << "  #steps: " << num_steps << std::endl;
+    trainer.set_synchronization_file(sync_file_name, chrono::minutes(15));
+    cout << trainer;
+    cout << "  burnin: " << warmup << endl;
+    cout << "  #steps: " << num_steps << endl;
+
+    if (parser.option("test"))
+    {
+        if (!file_exists(sync_file_name))
+        {
+            cout << "Could not find file " << sync_file_name << endl;
+            return EXIT_FAILURE;
+        }
+        const double threshold = get_option(parser, "test", 0.01);
+        {
+            net_type temp;
+            dnn_trainer<net_type> trainer(temp);
+            trainer.set_synchronization_file(sync_file_name);
+            trainer.get_net();
+            net.subnet() = temp.subnet();
+        }
+        net.loss_details().adjust_threshold(threshold);
+        image_window win;
+        matrix<rgb_pixel> image, resized;
+        for (const auto& im : dataset.images)
+        {
+            win.clear_overlay();
+            load_image(image, data_directory + "/" + im.filename);
+            win.set_title(im.filename);
+            win.set_image(image);
+            const auto tform = preprocess_image(image, resized, image_size);
+            auto detections = net(resized);
+            postprocess_detections(tform, detections);
+            cout << "# detections: " << detections.size() << endl;
+            for (const auto& det : detections)
+            {
+                win.add_overlay(det.rect, rgb_pixel(255, 0, 0), det.label);
+                cout << det.label << ": " << det.rect << " " << det.detection_confidence << endl;
+            }
+            cin.get();
+        }
+        return EXIT_SUCCESS;
+    }
 
     dlib::pipe<std::pair<matrix<rgb_pixel>, std::vector<yolo_rect>>> train_data(1000);
     auto loader = [&dataset, &data_directory, &train_data, &image_size](time_t seed) {
@@ -222,7 +273,7 @@ try
         }
     };
 
-    std::vector<std::thread> data_loaders;
+    std::vector<thread> data_loaders;
     for (size_t i = 0; i < num_workers; ++i)
         data_loaders.emplace_back([loader, i]() { loader(i + 1); });
 
@@ -232,12 +283,12 @@ try
     {
         images.clear();
         bboxes.clear();
-        std::pair<matrix<rgb_pixel>, std::vector<yolo_rect>> temp;
+        pair<matrix<rgb_pixel>, std::vector<yolo_rect>> temp;
         while (images.size() < trainer.get_mini_batch_size())
         {
             train_data.dequeue(temp);
-            images.push_back(std::move(temp.first));
-            bboxes.push_back(std::move(temp.second));
+            images.push_back(move(temp.first));
+            bboxes.push_back(move(temp.second));
         }
         trainer.train_one_step(images, bboxes);
     }
@@ -251,6 +302,6 @@ try
 }
 catch (const std::exception& e)
 {
-    std::cout << e.what() << '\n';
+    cout << e.what() << endl;
     return EXIT_FAILURE;
 }

From 66cd401126906e600816faee38c881f4c16201d4 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Adri=C3=A0=20Arrufat?= <adria.arrufat@outlook.com>
Date: Wed, 9 Jun 2021 01:08:36 +0900
Subject: [PATCH 03/62] wip: add more data augmentation, remove tag from input
 and use the same scale for coordinates as the official implementation

---
 dlib/dnn/loss.h                | 13 ++++----
 examples/dnn_yolo_train_ex.cpp | 61 ++++++++++++++++++++++++++--------
 2 files changed, 55 insertions(+), 19 deletions(-)

diff --git a/dlib/dnn/loss.h b/dlib/dnn/loss.h
index 682f44e1bb..0941892fd3 100644
--- a/dlib/dnn/loss.h
+++ b/dlib/dnn/loss.h
@@ -3498,7 +3498,7 @@ namespace dlib
         test_box_overlap overlaps_ignore = test_box_overlap(0.5, 1.0);
         double lambda_obj = 1.0;
         double lambda_noobj = 1.0;
-        double lambda_bbr = 0.75;
+        double lambda_bbr = 1.0;
         double lambda_cls = 1.0;
 
     };
@@ -3766,14 +3766,15 @@ namespace dlib
                     const double dw = out_data[w_idx] - tw;
                     const double dh = out_data[h_idx] - th;
 
+                    const double scale = 2 - truth_box.rect.area() / static_cast<double>(input_tensor.nr() * input_tensor.nc());
                     // Compute MSE loss
-                    loss += scale_loss * options.lambda_bbr * (dx * dx + dy * dy + dw * dw + dh * dh);
+                    loss += scale_loss * options.lambda_bbr * scale *(dx * dx + dy * dy + dw * dw + dh * dh);
 
                     // Compute the gradient
-                    g[x_idx] = scale_grad * options.lambda_bbr * dx;
-                    g[y_idx] = scale_grad * options.lambda_bbr * dy;
-                    g[w_idx] = scale_grad * options.lambda_bbr * dw;
-                    g[h_idx] = scale_grad * options.lambda_bbr * dh;
+                    g[x_idx] = scale_grad * options.lambda_bbr * scale * dx;
+                    g[y_idx] = scale_grad * options.lambda_bbr * scale * dy;
+                    g[w_idx] = scale_grad * options.lambda_bbr * scale * dw;
+                    g[h_idx] = scale_grad * options.lambda_bbr * scale * dh;
 
                     // Compute binary cross-entropy loss
                     for (long k = 0; k < num_classes; ++k)
diff --git a/examples/dnn_yolo_train_ex.cpp b/examples/dnn_yolo_train_ex.cpp
index 5556394782..54f289b924 100644
--- a/examples/dnn_yolo_train_ex.cpp
+++ b/examples/dnn_yolo_train_ex.cpp
@@ -3,6 +3,7 @@
 #include <dlib/dnn.h>
 #include <dlib/gui_widgets.h>
 #include <dlib/image_io.h>
+#include <tools/imglab/src/metadata_editor.h>
 
 using namespace std;
 using namespace dlib;
@@ -87,8 +88,7 @@ namespace darknet
                 htag16<upsample<2, conblock<256, 1, 1,
                        nskip32<
                        yolo<1024, num_classes, ytag32, ntag32,
-                       backbone53<tag1<input_rgb_image>>
-                       >>>>>>>>>>>>>;
+                       backbone53<input_rgb_image>>>>>>>>>>>>>>;
 
     };
 
@@ -125,7 +125,7 @@ void postprocess_detections(const rectangle_transform& tform, std::vector<yolo_r
         d.rect = tform(d.rect);
 }
 
-using net_type = dlib::loss_yolo<darknet::ytag8, darknet::ytag16, darknet::ytag32, darknet::yolov3_train>;
+using net_type = loss_yolo<darknet::ytag8, darknet::ytag16, darknet::ytag32, darknet::yolov3_train>;
 
 int main(const int argc, const char** argv)
 try
@@ -138,7 +138,8 @@ try
     parser.add_option("steps", "number of training steps (defaule: 500000)", 1);
     parser.add_option("workers", "number of worker threads to load data (default: 4)", 1);
     parser.add_option("gpus", "number of GPUs to run the training on (default: 1)", 1);
-    parser.add_option("test", "test the detector with a custom threshold (default: 0.01)", 1);
+    parser.add_option("test", "test the detector with a threshold (default: 0.01)", 1);
+    parser.add_option("visualize", "visualize data augmentation instead of training");
     parser.set_group_name("Help Options");
     parser.add_option("h", "alias of --help");
     parser.add_option("help", "display this message and exit");
@@ -190,6 +191,7 @@ try
     options.overlaps_nms = test_box_overlap(0.45);
     net_type net(options);
     darknet::setup_detector(net, options, image_size);
+    cerr << net << endl;
 
     // Cosine scheduler with warm-up:
     // - learning_rate is the highest learning rate value, e.g. 0.01
@@ -206,6 +208,8 @@ try
     trainer.be_verbose();
     trainer.set_mini_batch_size(batch_size);
     trainer.set_learning_rate_schedule(learning_rate_schedule);
+    // trainer.set_learning_rate(learning_rate);
+    // trainer.set_iterations_without_progress_threshold(10000);
     trainer.set_synchronization_file(sync_file_name, chrono::minutes(15));
     cout << trainer;
     cout << "  burnin: " << warmup << endl;
@@ -229,6 +233,7 @@ try
         net.loss_details().adjust_threshold(threshold);
         image_window win;
         matrix<rgb_pixel> image, resized;
+        color_mapper string_to_color;
         for (const auto& im : dataset.images)
         {
             win.clear_overlay();
@@ -241,7 +246,7 @@ try
             cout << "# detections: " << detections.size() << endl;
             for (const auto& det : detections)
             {
-                win.add_overlay(det.rect, rgb_pixel(255, 0, 0), det.label);
+                win.add_overlay(det.rect, string_to_color(det.label), det.label);
                 cout << det.label << ": " << det.rect << " " << det.detection_confidence << endl;
             }
             cin.get();
@@ -252,23 +257,37 @@ try
     dlib::pipe<std::pair<matrix<rgb_pixel>, std::vector<yolo_rect>>> train_data(1000);
     auto loader = [&dataset, &data_directory, &train_data, &image_size](time_t seed) {
         dlib::rand rnd(time(nullptr) + seed);
-        matrix<rgb_pixel> image, letterbox;
+        matrix<rgb_pixel> image, rotated;
         std::pair<matrix<rgb_pixel>, std::vector<yolo_rect>> temp;
         while (train_data.is_enabled())
         {
-            std::vector<yolo_rect> boxes;
             auto idx = rnd.get_random_32bit_number() % dataset.images.size();
             load_image(image, data_directory + "/" + dataset.images[idx].filename);
 
-            const auto tform = rectangle_transform(letterbox_image(image, letterbox, image_size));
+            // Some data augmentation
+            rectangle_transform tform = rotate_image(
+                image,
+                rotated,
+                rnd.get_double_in_range(-5 * pi / 180, 5 * pi / 180),
+                interpolate_bilinear());
             for (const auto& box : dataset.images[idx].boxes)
             {
-                boxes.push_back(yolo_rect(tform(box.rect), 1, box.label));
+                temp.second.emplace_back(tform(box.rect), 1, box.label);
             }
-            // here you should do more data augmentation
-            disturb_colors(image, rnd);
-            temp.first = letterbox;
-            temp.second = boxes;
+
+            tform = letterbox_image(rotated, temp.first, image_size);
+            for (auto& box : temp.second)
+                box.rect = tform(box.rect);
+
+            if (rnd.get_random_double() > 0.5)
+            {
+                tform = flip_image_left_right(temp.first);
+                for (auto& box : temp.second)
+                    box.rect = tform(box.rect);
+            }
+
+            disturb_colors(temp.first, rnd);
+
             train_data.enqueue(temp);
         }
     };
@@ -277,6 +296,22 @@ try
     for (size_t i = 0; i < num_workers; ++i)
         data_loaders.emplace_back([loader, i]() { loader(i + 1); });
 
+    if (parser.option("visualize"))
+    {
+        image_window win;
+        color_mapper string_to_color;
+        while (true)
+        {
+            std::pair<matrix<rgb_pixel>, std::vector<yolo_rect>> temp;
+            train_data.dequeue(temp);
+            win.clear_overlay();
+            win.set_image(temp.first);
+            for (const auto& r : temp.second)
+                win.add_overlay(r.rect, string_to_color(r.label), r.label);
+            cin.get();
+        }
+    }
+
     std::vector<matrix<rgb_pixel>> images;
     std::vector<std::vector<yolo_rect>> bboxes;
     while (trainer.get_train_one_step_calls() < num_steps)

From 337b1224a31ad6fd05628ff3d5016014430fff41 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Adri=C3=A0=20Arrufat?= <adria.arrufat@outlook.com>
Date: Fri, 11 Jun 2021 14:10:57 +0900
Subject: [PATCH 04/62] wip: use a custom yolo_rect with drectangle instead of
 rectangle

---
 dlib/dnn/loss.h | 63 +++++++++++++++++++++++++++++++++++++++++--------
 1 file changed, 53 insertions(+), 10 deletions(-)

diff --git a/dlib/dnn/loss.h b/dlib/dnn/loss.h
index 0941892fd3..323fec1247 100644
--- a/dlib/dnn/loss.h
+++ b/dlib/dnn/loss.h
@@ -3447,10 +3447,52 @@ namespace dlib
 
 // ----------------------------------------------------------------------------------------
 
-    using yolo_rect = mmod_rect;
-    inline bool operator<(const yolo_rect& lhs, const yolo_rect& rhs)
+    struct yolo_rect
     {
-        return lhs.detection_confidence < rhs.detection_confidence;
+        yolo_rect() = default;
+        yolo_rect(const drectangle& r) : rect(r) {}
+        yolo_rect(const drectangle& r, double score) : rect(r),detection_confidence(score) {}
+        yolo_rect(const drectangle& r, double score, const std::string& label) : rect(r),detection_confidence(score), label(label) {}
+        yolo_rect(const mmod_rect& r) : rect(r.rect), detection_confidence(r.detection_confidence), ignore(r.ignore), label(r.label) {}
+
+        drectangle rect;
+        double detection_confidence = 0;
+        bool ignore = false;
+        std::string label;
+
+        operator rectangle() const { return rect; }
+        bool operator == (const yolo_rect& rhs) const
+        {
+            return rect == rhs.rect
+                   && detection_confidence == rhs.detection_confidence
+                   && ignore == rhs.ignore
+                   && label == rhs.label;
+        }
+        bool operator<(const yolo_rect& rhs) const
+        {
+            return detection_confidence < rhs.detection_confidence;
+        }
+    };
+
+    inline void serialize(const yolo_rect& item, std::ostream& out)
+    {
+        int version = 1;
+        serialize(version, out);
+        serialize(item.rect, out);
+        serialize(item.detection_confidence, out);
+        serialize(item.ignore, out);
+        serialize(item.label, out);
+    }
+
+    inline void deserialize(yolo_rect& item, std::istream& in)
+    {
+        int version = 0;
+        deserialize(version, in);
+        if (version != 1)
+            throw serialization_error("Unexpected version found while deserializing dlib::yolo_rect");
+        deserialize(item.rect, in);
+        deserialize(item.detection_confidence, in);
+        deserialize(item.ignore, in);
     }
 
     struct yolo_options
@@ -3494,6 +3536,7 @@ namespace dlib
         std::vector<std::string> labels;
         double confidence_threshold = 0.25;
         double ignore_iou_threshold = 0.7;
+        bool classwise_nms = false;
         test_box_overlap overlaps_nms = test_box_overlap(0.45, 1.0);
         test_box_overlap overlaps_ignore = test_box_overlap(0.5, 1.0);
         double lambda_obj = 1.0;
@@ -3510,6 +3553,7 @@ namespace dlib
         serialize(item.anchors, out);
         serialize(item.confidence_threshold, out);
         serialize(item.ignore_iou_threshold, out);
+        serialize(item.classwise_nms, out);
         serialize(item.overlaps_nms, out);
         serialize(item.overlaps_ignore, out);
         serialize(item.lambda_obj, out);
@@ -3526,6 +3570,7 @@ namespace dlib
         deserialize(item.anchors, in);
         deserialize(item.confidence_threshold, in);
         deserialize(item.ignore_iou_threshold, in);
+        deserialize(item.classwise_nms, in);
         deserialize(item.overlaps_nms, in);
         deserialize(item.overlaps_ignore, in);
         deserialize(item.lambda_obj, in);
@@ -3759,8 +3804,8 @@ namespace dlib
 
                     const double tx = t_center.x() / stride_x - c;
                     const double ty = t_center.y() / stride_y - r;
-                    const double tw = truth_box.rect.width() / static_cast<double>(anchors[best_a].width + truth_box.rect.width());
-                    const double th = truth_box.rect.height() / static_cast<double>(anchors[best_a].height + truth_box.rect.height());
+                    const double tw = truth_box.rect.width() / (anchors[best_a].width + truth_box.rect.width());
+                    const double th = truth_box.rect.height() / (anchors[best_a].height + truth_box.rect.height());
                     const double dx = out_data[x_idx] - tx;
                     const double dy = out_data[y_idx] - ty;
                     const double dw = out_data[w_idx] - tw;
@@ -3794,8 +3839,7 @@ namespace dlib
                 }
             }
         };
-
-    }  // namespace impl
+    }
 
     template <template <typename> class... TAG_TYPES>
     class loss_yolo_
@@ -3905,15 +3949,14 @@ namespace dlib
 
         inline bool overlaps_any_box_nms (
             const std::vector<yolo_rect>& boxes,
-            const yolo_rect& box,
-            const bool classwise = false
+            const yolo_rect& box
         ) const
         {
             for (const auto& b : boxes)
             {
                 if (options.overlaps_nms(b.rect, box.rect))
                 {
-                    if (classwise)
+                    if (options.classwise_nms)
                     {
                         if (b.label == box.label)
                             return true;

From 9324c6659924a2ce5728e0d092f8077914cd49db Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Adri=C3=A0=20Arrufat?= <adria.arrufat@outlook.com>
Date: Sun, 13 Jun 2021 20:40:18 +0900
Subject: [PATCH 05/62] wip: rename to tensor_to_loss, builtin iou and same
 scale for grad and loss

---
 dlib/dnn/loss.h | 50 ++++++++++++++++++++++---------------------------
 1 file changed, 22 insertions(+), 28 deletions(-)

diff --git a/dlib/dnn/loss.h b/dlib/dnn/loss.h
index 323fec1247..6a5ee0e19b 100644
--- a/dlib/dnn/loss.h
+++ b/dlib/dnn/loss.h
@@ -3617,7 +3617,7 @@ namespace dlib
                 typename const_label_iterator,
                 typename SUBNET
             >
-            static void tensor_to_grad(
+            static void tensor_to_loss(
                 const tensor& input_tensor,
                 const_label_iterator truth,
                 SUBNET& sub,
@@ -3626,8 +3626,8 @@ namespace dlib
                 double& loss
             )
             {
-                yolo_helper_impl<TAG_TYPE>::tensor_to_grad(input_tensor, truth, sub, n, options, loss);
-                yolo_helper_impl<TAG_TYPES...>::tensor_to_grad(input_tensor, truth, sub, n, options, loss);
+                yolo_helper_impl<TAG_TYPE>::tensor_to_loss(input_tensor, truth, sub, n, options, loss);
+                yolo_helper_impl<TAG_TYPES...>::tensor_to_loss(input_tensor, truth, sub, n, options, loss);
             }
         };
 
@@ -3690,16 +3690,11 @@ namespace dlib
                 }
             }
 
-            static inline double compute_iou(const yolo_rect& a, const yolo_rect& b)
-            {
-                return a.rect.intersect(b.rect).area() / static_cast<double>((a.rect + b.rect).area());
-            }
-
             template <
                 typename const_label_iterator,
                 typename SUBNET
             >
-            static void tensor_to_grad(
+            static void tensor_to_loss(
                 const tensor& input_tensor,
                 const_label_iterator truth,
                 SUBNET& sub,
@@ -3718,8 +3713,7 @@ namespace dlib
                 const float* const out_data = output_tensor.host();
                 tensor& grad = layer<TAG_TYPE>(sub).get_gradient_input();
                 float* g = grad.host();
-                const double scale_loss = 1.0 / (output_tensor.num_samples() * output_tensor.nr() * output_tensor.nc());
-                const double scale_grad = 1.0 / output_tensor.num_samples();
+                const double scale = 1.0 / (output_tensor.num_samples() * output_tensor.nr() * output_tensor.nc());
 
                 // Compute the objectness loss for all grid cells
                 for (long r = 0; r < output_tensor.nr(); ++r)
@@ -3747,14 +3741,14 @@ namespace dlib
                             {
                                 if (truth_box.ignore)
                                     continue;
-                                best_iou = std::max(best_iou, compute_iou(truth_box, pred));
+                                best_iou = std::max(best_iou, box_intersection_over_union(truth_box.rect, pred.rect));
                             }
 
                             // Only incur loss for the boxes that are below a certain IoU threshold
                             if (best_iou < options.ignore_iou_threshold)
                             {
-                                loss += -scale_loss * options.lambda_noobj * safe_log(1 - out_data[o_idx]);
-                                g[o_idx] = scale_grad * options.lambda_noobj * out_data[o_idx];
+                                loss += -scale * options.lambda_noobj * safe_log(1 - out_data[o_idx]);
+                                g[o_idx] = scale * options.lambda_noobj * out_data[o_idx];
                             }
                         }
                     }
@@ -3776,7 +3770,7 @@ namespace dlib
                         for (size_t a = 0; a < details.size(); ++a)
                         {
                             const yolo_rect anchor(centered_drect(t_center, details[a].width, details[a].height));
-                            const double iou = compute_iou(truth_box, anchor);
+                            const double iou = box_intersection_over_union(truth_box.rect, anchor.rect);
                             if (iou > best_iou)
                             {
                                 best_iou = iou;
@@ -3799,8 +3793,8 @@ namespace dlib
                     const auto o_idx = tensor_index(output_tensor, n, best_a * num_feats + 4, r, c);
 
                     // This grid cell should detect an object
-                    loss += -scale_loss * options.lambda_obj * safe_log(out_data[o_idx]);
-                    g[o_idx] = scale_grad * options.lambda_obj * (out_data[o_idx] - 1);
+                    loss += -scale * options.lambda_obj * safe_log(out_data[o_idx]);
+                    g[o_idx] = scale * options.lambda_obj * (out_data[o_idx] - 1);
 
                     const double tx = t_center.x() / stride_x - c;
                     const double ty = t_center.y() / stride_y - r;
@@ -3811,15 +3805,15 @@ namespace dlib
                     const double dw = out_data[w_idx] - tw;
                     const double dh = out_data[h_idx] - th;
 
-                    const double scale = 2 - truth_box.rect.area() / static_cast<double>(input_tensor.nr() * input_tensor.nc());
+                    const double scale_bbr = 2 - truth_box.rect.area() / (input_tensor.nr() * input_tensor.nc());
                     // Compute MSE loss
-                    loss += scale_loss * options.lambda_bbr * scale *(dx * dx + dy * dy + dw * dw + dh * dh);
+                    loss += scale * options.lambda_bbr * scale_bbr * (dx * dx + dy * dy + dw * dw + dh * dh);
 
                     // Compute the gradient
-                    g[x_idx] = scale_grad * options.lambda_bbr * scale * dx;
-                    g[y_idx] = scale_grad * options.lambda_bbr * scale * dy;
-                    g[w_idx] = scale_grad * options.lambda_bbr * scale * dw;
-                    g[h_idx] = scale_grad * options.lambda_bbr * scale * dh;
+                    g[x_idx] = scale * options.lambda_bbr * scale_bbr * dx;
+                    g[y_idx] = scale * options.lambda_bbr * scale_bbr * dy;
+                    g[w_idx] = scale * options.lambda_bbr * scale_bbr * dw;
+                    g[h_idx] = scale * options.lambda_bbr * scale_bbr * dh;
 
                     // Compute binary cross-entropy loss
                     for (long k = 0; k < num_classes; ++k)
@@ -3827,13 +3821,13 @@ namespace dlib
                         const auto c_idx = tensor_index(output_tensor, n, best_a * num_feats + 5 + k, r, c);
                         if (truth_box.label == options.labels[k])
                         {
-                            loss += -scale_loss * options.lambda_cls * safe_log(out_data[c_idx]);
-                            g[c_idx] = scale_grad * options.lambda_cls * (out_data[c_idx] - 1);
+                            loss += -scale * options.lambda_cls * safe_log(out_data[c_idx]);
+                            g[c_idx] = scale * options.lambda_cls * (out_data[c_idx] - 1);
                         }
                         else
                         {
-                            loss += -scale_loss * options.lambda_cls * safe_log(1 - out_data[c_idx]);
-                            g[c_idx] = scale_grad * options.lambda_cls * out_data[c_idx];
+                            loss += -scale * options.lambda_cls * safe_log(1 - out_data[c_idx]);
+                            g[c_idx] = scale * options.lambda_cls * out_data[c_idx];
                         }
                     }
                 }
@@ -3905,7 +3899,7 @@ namespace dlib
             double loss = 0;
             for (long i = 0; i < input_tensor.num_samples(); ++i)
             {
-                impl::yolo_helper_impl<TAG_TYPES...>::tensor_to_grad(input_tensor, truth, sub, i, options, loss);
+                impl::yolo_helper_impl<TAG_TYPES...>::tensor_to_loss(input_tensor, truth, sub, i, options, loss);
             }
             return loss;
         }

From e557be36377878322a0941f234b834e06edb8d6e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Adri=C3=A0=20Arrufat?= <adria.arrufat@outlook.com>
Date: Tue, 22 Jun 2021 21:57:32 +0900
Subject: [PATCH 06/62] wip: increment truth iterator after each sample

---
 dlib/dnn/loss.h | 28 +++++++++++++++-------------
 1 file changed, 15 insertions(+), 13 deletions(-)

diff --git a/dlib/dnn/loss.h b/dlib/dnn/loss.h
index 6a5ee0e19b..afba356b54 100644
--- a/dlib/dnn/loss.h
+++ b/dlib/dnn/loss.h
@@ -3535,7 +3535,7 @@ namespace dlib
         std::unordered_map<int, std::vector<anchor_box_details>> anchors;
         std::vector<std::string> labels;
         double confidence_threshold = 0.25;
-        double ignore_iou_threshold = 0.7;
+        double truth_match_iou_threshold = 0.7;
         bool classwise_nms = false;
         test_box_overlap overlaps_nms = test_box_overlap(0.45, 1.0);
         test_box_overlap overlaps_ignore = test_box_overlap(0.5, 1.0);
@@ -3552,7 +3552,7 @@ namespace dlib
         serialize(version, out);
         serialize(item.anchors, out);
         serialize(item.confidence_threshold, out);
-        serialize(item.ignore_iou_threshold, out);
+        serialize(item.truth_match_iou_threshold, out);
         serialize(item.classwise_nms, out);
         serialize(item.overlaps_nms, out);
         serialize(item.overlaps_ignore, out);
@@ -3569,7 +3569,7 @@ namespace dlib
             throw serialization_error("Unexpected version found while deserializing dlib::yolo_options.");
         deserialize(item.anchors, in);
         deserialize(item.confidence_threshold, in);
-        deserialize(item.ignore_iou_threshold, in);
+        deserialize(item.truth_match_iou_threshold, in);
         deserialize(item.classwise_nms, in);
         deserialize(item.overlaps_nms, in);
         deserialize(item.overlaps_ignore, in);
@@ -3601,23 +3601,23 @@ namespace dlib
             }
 
             template <typename SUBNET>
-            static void tensor_to_dets(
-                const tensor& input,
+            static void tensor_to_dets (
+                const tensor& input_tensor,
                 const SUBNET& sub,
                 const long n,
                 const yolo_options& options,
                 std::vector<yolo_rect>& dets
             )
             {
-                yolo_helper_impl<TAG_TYPE>::tensor_to_dets(input, sub, n, options, dets);
-                yolo_helper_impl<TAG_TYPES...>::tensor_to_dets(input, sub, n, options, dets);
+                yolo_helper_impl<TAG_TYPE>::tensor_to_dets(input_tensor, sub, n, options, dets);
+                yolo_helper_impl<TAG_TYPES...>::tensor_to_dets(input_tensor, sub, n, options, dets);
             }
 
             template <
                 typename const_label_iterator,
                 typename SUBNET
             >
-            static void tensor_to_loss(
+            static void tensor_to_loss (
                 const tensor& input_tensor,
                 const_label_iterator truth,
                 SUBNET& sub,
@@ -3639,7 +3639,7 @@ namespace dlib
             static void list_tags(std::ostream& out) { out << tag_id<TAG_TYPE>::id; }
 
             template <typename SUBNET>
-            static void tensor_to_dets(
+            static void tensor_to_dets (
                 const tensor& input_tensor,
                 const SUBNET& sub,
                 const long n,
@@ -3694,7 +3694,7 @@ namespace dlib
                 typename const_label_iterator,
                 typename SUBNET
             >
-            static void tensor_to_loss(
+            static void tensor_to_loss (
                 const tensor& input_tensor,
                 const_label_iterator truth,
                 SUBNET& sub,
@@ -3713,7 +3713,7 @@ namespace dlib
                 const float* const out_data = output_tensor.host();
                 tensor& grad = layer<TAG_TYPE>(sub).get_gradient_input();
                 float* g = grad.host();
-                const double scale = 1.0 / (output_tensor.num_samples() * output_tensor.nr() * output_tensor.nc());
+                const double scale = 1.0; // (output_tensor.num_samples() * output_tensor.nr() * output_tensor.nc());
 
                 // Compute the objectness loss for all grid cells
                 for (long r = 0; r < output_tensor.nr(); ++r)
@@ -3745,7 +3745,7 @@ namespace dlib
                             }
 
                             // Only incur loss for the boxes that are below a certain IoU threshold
-                            if (best_iou < options.ignore_iou_threshold)
+                            if (best_iou < options.truth_match_iou_threshold)
                             {
                                 loss += -scale * options.lambda_noobj * safe_log(1 - out_data[o_idx]);
                                 g[o_idx] = scale * options.lambda_noobj * out_data[o_idx];
@@ -3805,6 +3805,7 @@ namespace dlib
                     const double dw = out_data[w_idx] - tw;
                     const double dh = out_data[h_idx] - th;
 
+                    // Penalize regression error according to the truth size
                     const double scale_bbr = 2 - truth_box.rect.area() / (input_tensor.nr() * input_tensor.nc());
                     // Compute MSE loss
                     loss += scale * options.lambda_bbr * scale_bbr * (dx * dx + dy * dy + dw * dw + dh * dh);
@@ -3900,8 +3901,9 @@ namespace dlib
             for (long i = 0; i < input_tensor.num_samples(); ++i)
             {
                 impl::yolo_helper_impl<TAG_TYPES...>::tensor_to_loss(input_tensor, truth, sub, i, options, loss);
+                ++truth;
             }
-            return loss;
+            return loss / input_tensor.num_samples();
         }
 
         void adjust_threshold(float conf_thresh) { options.confidence_threshold = conf_thresh; }

From 5842b0dfe8c8d3e1f14a7e19b51e66721d10680a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Adri=C3=A0=20Arrufat?= <adria.arrufat@outlook.com>
Date: Wed, 23 Jun 2021 12:19:01 +0900
Subject: [PATCH 07/62] use the L2 loss for display purposes and serialize
 labels

---
 dlib/dnn/loss.h | 60 +++++++++++++++++++------------------------------
 1 file changed, 23 insertions(+), 37 deletions(-)

diff --git a/dlib/dnn/loss.h b/dlib/dnn/loss.h
index afba356b54..f3869b6904 100644
--- a/dlib/dnn/loss.h
+++ b/dlib/dnn/loss.h
@@ -3540,8 +3540,7 @@ namespace dlib
         test_box_overlap overlaps_nms = test_box_overlap(0.45, 1.0);
         test_box_overlap overlaps_ignore = test_box_overlap(0.5, 1.0);
         double lambda_obj = 1.0;
-        double lambda_noobj = 1.0;
-        double lambda_bbr = 1.0;
+        double lambda_box = 1.0;
         double lambda_cls = 1.0;
 
     };
@@ -3551,13 +3550,14 @@ namespace dlib
         int version = 1;
         serialize(version, out);
         serialize(item.anchors, out);
+        serialize(item.labels, out);
         serialize(item.confidence_threshold, out);
         serialize(item.truth_match_iou_threshold, out);
         serialize(item.classwise_nms, out);
         serialize(item.overlaps_nms, out);
         serialize(item.overlaps_ignore, out);
         serialize(item.lambda_obj, out);
-        serialize(item.lambda_bbr, out);
+        serialize(item.lambda_box, out);
         serialize(item.lambda_cls, out);
     }
 
@@ -3568,13 +3568,14 @@ namespace dlib
         if (version != 1)
             throw serialization_error("Unexpected version found while deserializing dlib::yolo_options.");
         deserialize(item.anchors, in);
+        deserialize(item.labels, in);
         deserialize(item.confidence_threshold, in);
         deserialize(item.truth_match_iou_threshold, in);
         deserialize(item.classwise_nms, in);
         deserialize(item.overlaps_nms, in);
         deserialize(item.overlaps_ignore, in);
         deserialize(item.lambda_obj, in);
-        deserialize(item.lambda_bbr, in);
+        deserialize(item.lambda_box, in);
         deserialize(item.lambda_cls, in);
     }
 
@@ -3713,7 +3714,6 @@ namespace dlib
                 const float* const out_data = output_tensor.host();
                 tensor& grad = layer<TAG_TYPE>(sub).get_gradient_input();
                 float* g = grad.host();
-                const double scale = 1.0; // (output_tensor.num_samples() * output_tensor.nr() * output_tensor.nc());
 
                 // Compute the objectness loss for all grid cells
                 for (long r = 0; r < output_tensor.nr(); ++r)
@@ -3744,12 +3744,9 @@ namespace dlib
                                 best_iou = std::max(best_iou, box_intersection_over_union(truth_box.rect, pred.rect));
                             }
 
-                            // Only incur loss for the boxes that are below a certain IoU threshold
+                            // Incur loss for the boxes that are below a certain IoU threshold with any truth box
                             if (best_iou < options.truth_match_iou_threshold)
-                            {
-                                loss += -scale * options.lambda_noobj * safe_log(1 - out_data[o_idx]);
-                                g[o_idx] = scale * options.lambda_noobj * out_data[o_idx];
-                            }
+                                g[o_idx] = out_data[o_idx];
                         }
                     }
                 }
@@ -3793,45 +3790,34 @@ namespace dlib
                     const auto o_idx = tensor_index(output_tensor, n, best_a * num_feats + 4, r, c);
 
                     // This grid cell should detect an object
-                    loss += -scale * options.lambda_obj * safe_log(out_data[o_idx]);
-                    g[o_idx] = scale * options.lambda_obj * (out_data[o_idx] - 1);
+                    g[o_idx] = options.lambda_obj * (out_data[o_idx] - 1);
 
+                    // Get the truth box target values
                     const double tx = t_center.x() / stride_x - c;
                     const double ty = t_center.y() / stride_y - r;
                     const double tw = truth_box.rect.width() / (anchors[best_a].width + truth_box.rect.width());
                     const double th = truth_box.rect.height() / (anchors[best_a].height + truth_box.rect.height());
-                    const double dx = out_data[x_idx] - tx;
-                    const double dy = out_data[y_idx] - ty;
-                    const double dw = out_data[w_idx] - tw;
-                    const double dh = out_data[h_idx] - th;
-
-                    // Penalize regression error according to the truth size
-                    const double scale_bbr = 2 - truth_box.rect.area() / (input_tensor.nr() * input_tensor.nc());
-                    // Compute MSE loss
-                    loss += scale * options.lambda_bbr * scale_bbr * (dx * dx + dy * dy + dw * dw + dh * dh);
-
-                    // Compute the gradient
-                    g[x_idx] = scale * options.lambda_bbr * scale_bbr * dx;
-                    g[y_idx] = scale * options.lambda_bbr * scale_bbr * dy;
-                    g[w_idx] = scale * options.lambda_bbr * scale_bbr * dw;
-                    g[h_idx] = scale * options.lambda_bbr * scale_bbr * dh;
-
-                    // Compute binary cross-entropy loss
+
+                    // Scale regression error according to the truth size
+                    const double scale_box = 2 - truth_box.rect.area() / (input_tensor.nr() * input_tensor.nc());
+
+                    // Compute the gradient for the box coordinates
+                    g[x_idx] = options.lambda_box * scale_box * (out_data[x_idx] - tx);
+                    g[y_idx] = options.lambda_box * scale_box * (out_data[y_idx] - ty);
+                    g[w_idx] = options.lambda_box * scale_box * (out_data[w_idx] - tw);
+                    g[h_idx] = options.lambda_box * scale_box * (out_data[h_idx] - th);
+
+                    // Compute the classification error
                     for (long k = 0; k < num_classes; ++k)
                     {
                         const auto c_idx = tensor_index(output_tensor, n, best_a * num_feats + 5 + k, r, c);
                         if (truth_box.label == options.labels[k])
-                        {
-                            loss += -scale * options.lambda_cls * safe_log(out_data[c_idx]);
-                            g[c_idx] = scale * options.lambda_cls * (out_data[c_idx] - 1);
-                        }
+                            g[c_idx] = options.lambda_cls * (out_data[c_idx] - 1);
                         else
-                        {
-                            loss += -scale * options.lambda_cls * safe_log(1 - out_data[c_idx]);
-                            g[c_idx] = scale * options.lambda_cls * out_data[c_idx];
-                        }
+                            g[c_idx] = options.lambda_cls * out_data[c_idx];
                     }
                 }
+                loss += length_squared(rowm(mat(grad), n));
             }
         };
     }

From e82b8f5183bbe4067a675811db099fa65fd6588a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Adri=C3=A0=20Arrufat?= <adria.arrufat@outlook.com>
Date: Wed, 23 Jun 2021 12:43:55 +0900
Subject: [PATCH 08/62] update training code

---
 examples/dnn_yolo_train_ex.cpp | 144 +++++++++++++++------------------
 1 file changed, 65 insertions(+), 79 deletions(-)

diff --git a/examples/dnn_yolo_train_ex.cpp b/examples/dnn_yolo_train_ex.cpp
index 54f289b924..92e95c3a69 100644
--- a/examples/dnn_yolo_train_ex.cpp
+++ b/examples/dnn_yolo_train_ex.cpp
@@ -37,13 +37,10 @@ namespace darknet
     struct def
     {
         template <long nf, long ks, int s, typename SUBNET>
-        using conblock = ACT<BN<add_layer<con_<nf, ks, ks, s, s, ks/2, ks/2>, SUBNET>>>;
+        using conblock = ACT<BN<add_layer<con_<nf, ks, ks, s, s, ks / 2, ks / 2>, SUBNET>>>;
 
-        template <long nf1, long nf2, typename SUBNET>
-        using residual = add_prev1<
-                         conblock<nf1, 3, 1,
-                         conblock<nf2, 1, 1,
-                    tag1<SUBNET>>>>;
+        template <long nf, typename SUBNET>
+        using residual = add_prev1<conblock<nf, 3, 1, conblock<nf / 2, 1, 1, tag1<SUBNET>>>>;
 
         template <long nf, long factor, typename SUBNET>
         using conblock5 = conblock<nf, 1, 1,
@@ -52,65 +49,59 @@ namespace darknet
                           conblock<nf * factor, 3, 1,
                           conblock<nf, 1, 1, SUBNET>>>>>;
 
-        // the residual block introduced in YOLOv3 (with bottleneck)
-        template <long nf, typename SUBNET> using resv3 = residual<nf, nf / 2, SUBNET>;
-        // the residual block introduced in YOLOv4 (without bottleneck)
-        template <long nf, typename SUBNET> using resv4 = residual<nf, nf, SUBNET>;
-
-        template <typename SUBNET> using resv3_64= resv3<64, SUBNET>;
-        template <typename SUBNET> using resv3_128 = resv3<128, SUBNET>;
-        template <typename SUBNET> using resv3_256 = resv3<256, SUBNET>;
-        template <typename SUBNET> using resv3_512 = resv3<512, SUBNET>;
-        template <typename SUBNET> using resv3_1024 = resv3<1024, SUBNET>;
+        template <typename SUBNET> using res_64 = residual<64, SUBNET>;
+        template <typename SUBNET> using res_128 = residual<128, SUBNET>;
+        template <typename SUBNET> using res_256 = residual<256, SUBNET>;
+        template <typename SUBNET> using res_512 = residual<512, SUBNET>;
+        template <typename SUBNET> using res_1024 = residual<1024, SUBNET>;
 
         template <typename INPUT>
-        using backbone53 = repeat<4, resv3_1024, conblock<1024, 3, 2,
-                    btag16<repeat<8, resv3_512,  conblock<512, 3, 2,
-                     btag8<repeat<8, resv3_256,  conblock<256, 3, 2,
-                           repeat<2, resv3_128,  conblock<128, 3, 2,
-                                     resv3_64<   conblock<64, 3, 2,
-                                                 conblock<32, 3, 1,
-                                                 INPUT>>>>>>>>>>>>>;
-
-        template <long nf, int classes, template <typename> class YTAG, template <typename> class NTAG, typename SUBNET>
-        using yolo = YTAG<sig<con<3 * (classes + 5), 1, 1, 1, 1,
+        using backbone53 = repeat<4, res_1024, conblock<1024, 3, 2,
+                    btag16<repeat<8, res_512,  conblock<512, 3, 2,
+                     btag8<repeat<8, res_256,  conblock<256, 3, 2,
+                           repeat<2, res_128,  conblock<128, 3, 2,
+                                     res_64<   conblock<64, 3, 2,
+                                               conblock<32, 3, 1,
+                                               INPUT>>>>>>>>>>>>>;
+
+        template <long num_classes, long nf, template <typename> class YTAG, template <typename> class NTAG, typename SUBNET>
+        using yolo = YTAG<sig<con<3 * (num_classes + 5), 1, 1, 1, 1,
                      conblock<nf, 3, 1,
                 NTAG<conblock5<nf / 2, 2,
                      SUBNET>>>>>>;
 
         template <long num_classes>
-        using yolov3 = yolo<256, num_classes, ytag8, ntag8,
+        using yolov3 = yolo<num_classes, 256, ytag8, ntag8,
                        concat2<htag8, btag8,
                  htag8<upsample<2, conblock<128, 1, 1,
                        nskip16<
-                       yolo<512, num_classes, ytag16, ntag16,
+                       yolo<num_classes, 512, ytag16, ntag16,
                        concat2<htag16, btag16,
                 htag16<upsample<2, conblock<256, 1, 1,
                        nskip32<
-                       yolo<1024, num_classes, ytag32, ntag32,
+                       yolo<num_classes, 1024, ytag32, ntag32,
                        backbone53<input_rgb_image>>>>>>>>>>>>>>;
 
     };
 
-    using yolov3_train = def<leaky_relu, bn_con>::yolov3<80>;
-    using yolov3_infer = def<leaky_relu, affine>::yolov3<80>;
+    using yolov3_train_type = loss_yolo<ytag8, ytag16, ytag32, def<leaky_relu, bn_con>::yolov3<80>>;
+    using yolov3_infer_type = loss_yolo<ytag8, ytag16, ytag32, def<leaky_relu, affine>::yolov3<80>>;
 
-    template <typename net_type>
-    void setup_detector(net_type& net, const yolo_options& options, long image_size)
+    void setup_detector(yolov3_train_type& net, const yolo_options& options)
     {
-        // remove bias
+        // remove bias from bn inputs
         disable_duplicative_biases(net);
-        // do not remove the mean from the input image
-        input_layer(net) = input_rgb_image(0, 0, 0);
         // setup leaky relus
         visit_computational_layers(net, [](leaky_relu_& l) { l = leaky_relu_(0.1); });
-        // set the number of filters (they are located after the tag and sig layers)
-        layer<ytag8, 2>(net).layer_details().set_num_filters(3 * (options.labels.size() + 5));
-        layer<ytag16, 2>(net).layer_details().set_num_filters(3 * (options.labels.size() + 5));
-        layer<ytag32, 2>(net).layer_details().set_num_filters(3 * (options.labels.size() + 5));
-        // allocate the network
-        matrix<rgb_pixel> image(image_size, image_size);
-        net(image);
+        // enlarge the batch normalization stats window
+        set_all_bn_running_stats_window_sizes(net, 1000);
+        // set the number of filters for detection layers (they are located after the tag and sig layers)
+        const long nfo1 = options.anchors.at(tag_id<ytag8>::id).size() * (options.labels.size() + 5);
+        const long nfo2 = options.anchors.at(tag_id<ytag16>::id).size() * (options.labels.size() + 5);
+        const long nfo3 = options.anchors.at(tag_id<ytag32>::id).size() * (options.labels.size() + 5);
+        layer<ytag8, 2>(net).layer_details().set_num_filters(nfo1);
+        layer<ytag16, 2>(net).layer_details().set_num_filters(nfo2);
+        layer<ytag32, 2>(net).layer_details().set_num_filters(nfo3);
     }
 }
 
@@ -125,8 +116,6 @@ void postprocess_detections(const rectangle_transform& tform, std::vector<yolo_r
         d.rect = tform(d.rect);
 }
 
-using net_type = loss_yolo<darknet::ytag8, darknet::ytag16, darknet::ytag32, darknet::yolov3_train>;
-
 int main(const int argc, const char** argv)
 try
 {
@@ -134,8 +123,8 @@ try
     parser.add_option("size", "image size for training (default: 416)", 1);
     parser.add_option("learning-rate", "initial learning rate (default: 0.001)", 1);
     parser.add_option("batch-size", "mini batch size (default: 8)", 1);
-    parser.add_option("warmup", "number of warmup steps (default: 5000)", 1);
-    parser.add_option("steps", "number of training steps (defaule: 500000)", 1);
+    parser.add_option("burnin", "learning rate burnin steps (default: 1000)", 1);
+    parser.add_option("patience", "number of steps without progress (default: 10000)", 1);
     parser.add_option("workers", "number of worker threads to load data (default: 4)", 1);
     parser.add_option("gpus", "number of GPUs to run the training on (default: 1)", 1);
     parser.add_option("test", "test the detector with a threshold (default: 0.01)", 1);
@@ -151,9 +140,9 @@ try
         return 0;
     }
     const double learning_rate = get_option(parser, "learning-rate", 0.001);
+    const size_t patience = get_option(parser, "patience", 10000);
     const size_t batch_size = get_option(parser, "batch-size", 8);
-    const size_t warmup = get_option(parser, "warmup", 5000);
-    const size_t num_steps = get_option(parser, "steps", 100000);
+    const size_t burnin = get_option(parser, "burnin", 1000);
     const size_t image_size = get_option(parser, "size", 416);
     const size_t num_workers = get_option(parser, "workers", 4);
     const size_t num_gpus = get_option(parser, "gpus", 1);
@@ -183,37 +172,25 @@ try
         cout << " (" << (100.0*label.second)/num_objects << "%)\n";
         options.labels.push_back(label.first);
     }
-    options.confidence_threshold = 0.25;
-    options.ignore_iou_threshold = 0.5;
+    options.truth_match_iou_threshold = 0.7;
     options.add_anchors<darknet::ytag8>({{10, 13}, {16, 30}, {33, 23}});
     options.add_anchors<darknet::ytag16>({{30, 61}, {62, 45}, {59, 119}});
     options.add_anchors<darknet::ytag32>({{116, 90}, {156, 198}, {373, 326}});
-    options.overlaps_nms = test_box_overlap(0.45);
-    net_type net(options);
-    darknet::setup_detector(net, options, image_size);
+    darknet::yolov3_train_type net(options);
+    darknet::setup_detector(net, options);
     cerr << net << endl;
 
-    // Cosine scheduler with warm-up:
-    // - learning_rate is the highest learning rate value, e.g. 0.01
-    // - warmup: number of steps to linearly increase the learning rate
-    // - steps: maximum number of steps of the training session
-    const matrix<double> learning_rate_schedule = learning_rate * join_rows(
-        linspace(0, 1, warmup),
-        ((1 + cos(pi / (num_steps - warmup) * linspace(0, num_steps - warmup, num_steps - warmup))) / 2)
-    ) + numeric_limits<double>::epsilon();  // this prevents the learning rates from being 0
+    const matrix<double> learning_rate_schedule = learning_rate * pow(linspace(1e-12, 1, burnin), 4);
 
     std::vector<int> gpus(num_gpus);
     iota(gpus.begin(), gpus.end(), 0);
-    dnn_trainer<net_type> trainer(net, sgd(0.0005, 0.9), gpus);
+    dnn_trainer<darknet::yolov3_train_type> trainer(net, sgd(0.0005, 0.9), gpus);
     trainer.be_verbose();
     trainer.set_mini_batch_size(batch_size);
     trainer.set_learning_rate_schedule(learning_rate_schedule);
-    // trainer.set_learning_rate(learning_rate);
-    // trainer.set_iterations_without_progress_threshold(10000);
     trainer.set_synchronization_file(sync_file_name, chrono::minutes(15));
+    trainer.set_mini_batch_size(batch_size);
     cout << trainer;
-    cout << "  burnin: " << warmup << endl;
-    cout << "  #steps: " << num_steps << endl;
 
     if (parser.option("test"))
     {
@@ -223,13 +200,6 @@ try
             return EXIT_FAILURE;
         }
         const double threshold = get_option(parser, "test", 0.01);
-        {
-            net_type temp;
-            dnn_trainer<net_type> trainer(temp);
-            trainer.set_synchronization_file(sync_file_name);
-            trainer.get_net();
-            net.subnet() = temp.subnet();
-        }
         net.loss_details().adjust_threshold(threshold);
         image_window win;
         matrix<rgb_pixel> image, resized;
@@ -271,9 +241,7 @@ try
                 rnd.get_double_in_range(-5 * pi / 180, 5 * pi / 180),
                 interpolate_bilinear());
             for (const auto& box : dataset.images[idx].boxes)
-            {
                 temp.second.emplace_back(tform(box.rect), 1, box.label);
-            }
 
             tform = letterbox_image(rotated, temp.first, image_size);
             for (auto& box : temp.second)
@@ -287,7 +255,6 @@ try
             }
 
             disturb_colors(temp.first, rnd);
-
             train_data.enqueue(temp);
         }
     };
@@ -314,7 +281,8 @@ try
 
     std::vector<matrix<rgb_pixel>> images;
     std::vector<std::vector<yolo_rect>> bboxes;
-    while (trainer.get_train_one_step_calls() < num_steps)
+
+    const auto train = [&images, &bboxes, &train_data, &trainer]()
     {
         images.clear();
         bboxes.clear();
@@ -326,7 +294,24 @@ try
             bboxes.push_back(move(temp.second));
         }
         trainer.train_one_step(images, bboxes);
-    }
+    };
+
+    std::cerr << "training started with " << burnin << " burn-in steps" << std::endl;
+    while (trainer.get_train_one_step_calls() < burnin)
+        train();
+
+    std::cerr << "burn-in finished" << std::endl;
+    trainer.get_net();
+    trainer.set_learning_rate(learning_rate);
+    trainer.set_min_learning_rate(learning_rate * 1e-3);
+    trainer.set_learning_rate_shrink_factor(0.1);
+    trainer.set_iterations_without_progress_threshold(patience);
+    std::cerr << trainer << std::endl;
+
+    while (trainer.get_learning_rate() >= trainer.get_min_learning_rate())
+        train();
+
+    std::cerr << "training done" << std::endl;
 
     trainer.get_net();
     train_data.disable();
@@ -334,6 +319,7 @@ try
         worker.join();
 
     serialize("yolov3.dnn") << net;
+    return EXIT_SUCCESS;
 }
 catch (const std::exception& e)
 {

From 729223a396cbe8932910a3e7201198cfa1bfa8c3 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Adri=C3=A0=20Arrufat?= <adria.arrufat@outlook.com>
Date: Wed, 23 Jun 2021 17:24:32 +0900
Subject: [PATCH 09/62] use options.overlaps_ignore field

---
 dlib/dnn/loss.h | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/dlib/dnn/loss.h b/dlib/dnn/loss.h
index f3869b6904..0dc60a57c2 100644
--- a/dlib/dnn/loss.h
+++ b/dlib/dnn/loss.h
@@ -3739,7 +3739,7 @@ namespace dlib
                             double best_iou = 0;
                             for (const yolo_rect& truth_box : *truth)
                             {
-                                if (truth_box.ignore)
+                                if (truth_box.ignore || options.overlaps_ignore(truth_box.rect, pred.rect))
                                     continue;
                                 best_iou = std::max(best_iou, box_intersection_over_union(truth_box.rect, pred.rect));
                             }

From 4a129f5e37114406597842877594adfa0fe4d68f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Adri=C3=A0=20Arrufat?= <adria.arrufat@outlook.com>
Date: Wed, 23 Jun 2021 17:37:08 +0900
Subject: [PATCH 10/62] make random_cropper work with yolo_rect types

---
 dlib/image_transforms/random_cropper.h        | 46 ++++++++------
 .../random_cropper_abstract.h                 | 48 ++++++++------
 examples/dnn_yolo_train_ex.cpp                | 62 ++++++++++++++-----
 3 files changed, 103 insertions(+), 53 deletions(-)

diff --git a/dlib/image_transforms/random_cropper.h b/dlib/image_transforms/random_cropper.h
index 2c754b6080..f19f9cc0ef 100644
--- a/dlib/image_transforms/random_cropper.h
+++ b/dlib/image_transforms/random_cropper.h
@@ -105,14 +105,15 @@ namespace dlib
         }
 
         template <
-            typename array_type
+            typename array_type,
+            typename rectangle_type
             >
         void operator() (
             size_t num_crops,
             const array_type& images,
-            const std::vector<std::vector<mmod_rect>>& rects,
+            const std::vector<std::vector<rectangle_type>>& rects,
             array_type& crops,
-            std::vector<std::vector<mmod_rect>>& crop_rects
+            std::vector<std::vector<rectangle_type>>& crop_rects
         )
         {
             DLIB_CASSERT(images.size() == rects.size());
@@ -122,14 +123,15 @@ namespace dlib
         }
 
         template <
-            typename array_type
+            typename array_type,
+            typename rectangle_type
             >
         void append (
             size_t num_crops,
             const array_type& images,
-            const std::vector<std::vector<mmod_rect>>& rects,
+            const std::vector<std::vector<rectangle_type>>& rects,
             array_type& crops,
-            std::vector<std::vector<mmod_rect>>& crop_rects
+            std::vector<std::vector<rectangle_type>>& crop_rects
         )
         {
             DLIB_CASSERT(images.size() == rects.size());
@@ -145,13 +147,14 @@ namespace dlib
 
         template <
             typename array_type,
-            typename image_type
+            typename image_type,
+            typename rectangle_type
             >
         void operator() (
             const array_type& images,
-            const std::vector<std::vector<mmod_rect>>& rects,
+            const std::vector<std::vector<rectangle_type>>& rects,
             image_type& crop,
-            std::vector<mmod_rect>& crop_rects
+            std::vector<rectangle_type>& crop_rects
         )
         {
             DLIB_CASSERT(images.size() == rects.size());
@@ -163,27 +166,29 @@ namespace dlib
         }
 
         template <
-            typename image_type1
+            typename image_type1,
+            typename rectangle_type
             >
         image_type1 operator() (
             const image_type1& img
         )
         {
             image_type1 crop;
-            std::vector<mmod_rect> junk1, junk2;
+            std::vector<rectangle_type> junk1, junk2;
             (*this)(img, junk1, crop, junk2);
             return crop;
         }
 
         template <
             typename image_type1,
-            typename image_type2
+            typename image_type2,
+            typename rectangle_type
             >
         void operator() (
             const image_type1& img,
-            const std::vector<mmod_rect>& rects,
+            const std::vector<rectangle_type>& rects,
             image_type2& crop,
-            std::vector<mmod_rect>& crop_rects
+            std::vector<rectangle_type>& crop_rects
         )
         {
             DLIB_CASSERT(num_rows(img)*num_columns(img) != 0);
@@ -230,10 +235,13 @@ namespace dlib
 
     private:
 
-        template <typename image_type1>
+        template <
+            typename image_type1,
+            typename rectangle_type
+            >
         void make_crop_plan (
             const image_type1& img,
-            const std::vector<mmod_rect>& rects,
+            const std::vector<rectangle_type>& rects,
             chip_details& crop_plan,
             bool& should_flip_crop
         )
@@ -285,8 +293,9 @@ namespace dlib
             crop_plan = chip_details(crop_rect, dims, angle);
         }
 
+        template <typename rectangle_type>
         bool has_non_ignored_box (
-            const std::vector<mmod_rect>& rects
+            const std::vector<rectangle_type>& rects
         ) const
         {
             for (auto&& b : rects)
@@ -297,8 +306,9 @@ namespace dlib
             return false;
         }
 
+        template <typename rectangle_type>
         size_t randomly_pick_rect (
-            const std::vector<mmod_rect>& rects
+            const std::vector<rectangle_type>& rects
         ) 
         {
             DLIB_CASSERT(has_non_ignored_box(rects));
diff --git a/dlib/image_transforms/random_cropper_abstract.h b/dlib/image_transforms/random_cropper_abstract.h
index 7603a1c475..065b34a3d2 100644
--- a/dlib/image_transforms/random_cropper_abstract.h
+++ b/dlib/image_transforms/random_cropper_abstract.h
@@ -19,8 +19,8 @@ namespace dlib
                 This object is a tool for extracting random crops of objects from a set of
                 images.  The crops are randomly jittered in scale, translation, and
                 rotation but more or less centered on objects specified by mmod_rect
-                objects.
-                
+                objects (or other rectangle types with a compatible interface).
+
             THREAD SAFETY
                 It is safe for multiple threads to make concurrent calls to this object's
                 operator() methods.
@@ -152,7 +152,7 @@ namespace dlib
                   the longest edge of the object (i.e. either its height or width,
                   whichever is longer) is at least #get_min_object_length_long_dim() pixels
                   in length.  When we say "object" here we are referring specifically to
-                  the rectangle in the mmod_rect output by the cropper.
+                  the rectangle in the rectangle_type output by the cropper.
         !*/
 
         long get_min_object_length_short_dim (
@@ -163,7 +163,7 @@ namespace dlib
                   the shortest edge of the object (i.e. either its height or width,
                   whichever is shorter) is at least #get_min_object_length_short_dim()
                   pixels in length.  When we say "object" here we are referring
-                  specifically to the rectangle in the mmod_rect output by the cropper.
+                  specifically to the rectangle in the rectangle_type output by the cropper.
         !*/
 
         void set_min_object_size (
@@ -200,14 +200,15 @@ namespace dlib
         !*/
 
         template <
-            typename array_type
+            typename array_type,
+            typename rectangle_type
             >
         void append (
             size_t num_crops,
             const array_type& images,
-            const std::vector<std::vector<mmod_rect>>& rects,
+            const std::vector<std::vector<rectangle_type>>& rects,
             array_type& crops,
-            std::vector<std::vector<mmod_rect>>& crop_rects
+            std::vector<std::vector<rectangle_type>>& crop_rects
         );
         /*!
             requires
@@ -218,6 +219,8 @@ namespace dlib
                 - array_type is a type with an interface compatible with dlib::array or
                   std::vector and it must in turn contain image objects that implement the
                   interface defined in dlib/image_processing/generic_image.h 
+                - rectangle_type is a type with an interface compatible with mmod_rect, such
+                  as yolo_rect.
             ensures
                 - Randomly extracts num_crops chips from images and appends them to the end
                   of crops.  We also copy the object metadata for each extracted crop and
@@ -230,14 +233,15 @@ namespace dlib
         !*/
 
         template <
-            typename array_type
+            typename array_type,
+            typename rectangle_type
             >
         void operator() (
             size_t num_crops,
             const array_type& images,
-            const std::vector<std::vector<mmod_rect>>& rects,
+            const std::vector<std::vector<rectangle_type>>& rects,
             array_type& crops,
-            std::vector<std::vector<mmod_rect>>& crop_rects
+            std::vector<std::vector<rectangle_type>>& crop_rects
         );
         /*!
             requires
@@ -247,6 +251,8 @@ namespace dlib
                 - array_type is a type with an interface compatible with dlib::array or
                   std::vector and it must in turn contain image objects that implement the
                   interface defined in dlib/image_processing/generic_image.h 
+                - rectangle_type is a type with an interface compatible with mmod_rect, such
+                  as yolo_rect.
             ensures
                 - Randomly extracts num_crops chips from images.  We also copy the object
                   metadata for each extracted crop and store it into #crop_rects.  In
@@ -259,13 +265,14 @@ namespace dlib
 
         template <
             typename array_type,
-            typename image_type
+            typename image_type,
+            typename rectangle_type
             >
         void operator() (
             const array_type& images,
-            const std::vector<std::vector<mmod_rect>>& rects,
+            const std::vector<std::vector<rectangle_type>>& rects,
             image_type& crop,
-            std::vector<mmod_rect>& crop_rects
+            std::vector<rectangle_type>& crop_rects
         );
         /*!
             requires
@@ -277,6 +284,8 @@ namespace dlib
                 - array_type is a type with an interface compatible with dlib::array or
                   std::vector and it must in turn contain image objects that implement the
                   interface defined in dlib/image_processing/generic_image.h 
+                - rectangle_type is a type with an interface compatible with mmod_rect, such
+                  as yolo_rect.
             ensures
                 - Selects a random image and creates a random crop from it.  Specifically,
                   we pick a random index IDX < images.size() and then execute 
@@ -285,13 +294,14 @@ namespace dlib
 
         template <
             typename image_type1,
-            typename image_type2
+            typename image_type2,
+            typename rectangle_type
             >
         void operator() (
             const image_type1& img,
-            const std::vector<mmod_rect>& rects,
+            const std::vector<rectangle_type>& rects,
             image_type2& crop,
-            std::vector<mmod_rect>& crop_rects
+            std::vector<rectangle_type>& crop_rects
         );
         /*!
             requires
@@ -300,9 +310,11 @@ namespace dlib
                   dlib/image_processing/generic_image.h 
                 - image_type2 == an image object that implements the interface defined in
                   dlib/image_processing/generic_image.h 
+                - rectangle_type is a type with an interface compatible with mmod_rect, such
+                  as yolo_rect.
             ensures
-                - Extracts a random crop from img and copies over the mmod_rect objects in
-                  rects to #crop_rects if they are contained inside the crop.  Moreover,
+                - Extracts a random crop from img and copies over the rectangle_type objects
+                  in rects to #crop_rects if they are contained inside the crop.  Moreover,
                   rectangles are marked as ignore if they aren't completely contained
                   inside the crop.
                 - #crop_rects.size() <= rects.size()
diff --git a/examples/dnn_yolo_train_ex.cpp b/examples/dnn_yolo_train_ex.cpp
index 92e95c3a69..a55ba7f560 100644
--- a/examples/dnn_yolo_train_ex.cpp
+++ b/examples/dnn_yolo_train_ex.cpp
@@ -188,8 +188,8 @@ try
     trainer.be_verbose();
     trainer.set_mini_batch_size(batch_size);
     trainer.set_learning_rate_schedule(learning_rate_schedule);
-    trainer.set_synchronization_file(sync_file_name, chrono::minutes(15));
     trainer.set_mini_batch_size(batch_size);
+    trainer.set_synchronization_file(sync_file_name, chrono::minutes(15));
     cout << trainer;
 
     if (parser.option("test"))
@@ -229,31 +229,49 @@ try
         dlib::rand rnd(time(nullptr) + seed);
         matrix<rgb_pixel> image, rotated;
         std::pair<matrix<rgb_pixel>, std::vector<yolo_rect>> temp;
+        random_cropper cropper;
+        cropper.set_seed(time_t(nullptr) + seed);
+        cropper.set_chip_dims(image_size, image_size);
+        cropper.set_max_object_size(0.9);
+        cropper.set_min_object_size(10, 10);
+        cropper.set_max_rotation_degrees(10);
+        cropper.set_translate_amount(0.1);
+        cropper.set_randomly_flip(true);
+        cropper.set_background_crops_fraction(0);
         while (train_data.is_enabled())
         {
-            auto idx = rnd.get_random_32bit_number() % dataset.images.size();
+            const auto idx = rnd.get_random_32bit_number() % dataset.images.size();
             load_image(image, data_directory + "/" + dataset.images[idx].filename);
-
-            // Some data augmentation
-            rectangle_transform tform = rotate_image(
-                image,
-                rotated,
-                rnd.get_double_in_range(-5 * pi / 180, 5 * pi / 180),
-                interpolate_bilinear());
             for (const auto& box : dataset.images[idx].boxes)
-                temp.second.emplace_back(tform(box.rect), 1, box.label);
-
-            tform = letterbox_image(rotated, temp.first, image_size);
-            for (auto& box : temp.second)
-                box.rect = tform(box.rect);
+                temp.second.emplace_back(box.rect, 1, box.label);
 
             if (rnd.get_random_double() > 0.5)
             {
-                tform = flip_image_left_right(temp.first);
+                // Some data augmentation
+                rectangle_transform tform = rotate_image(
+                    image,
+                    rotated,
+                    rnd.get_double_in_range(-5 * pi / 180, 5 * pi / 180),
+                    interpolate_bilinear());
                 for (auto& box : temp.second)
                     box.rect = tform(box.rect);
-            }
 
+                tform = letterbox_image(rotated, temp.first, image_size);
+                for (auto& box : temp.second)
+                    box.rect = tform(box.rect);
+
+                if (rnd.get_random_double() > 0.5)
+                {
+                    tform = flip_image_left_right(temp.first);
+                    for (auto& box : temp.second)
+                        box.rect = tform(box.rect);
+                }
+            }
+            else
+            {
+                std::vector<yolo_rect> boxes = temp.second;
+                cropper(image, boxes, temp.first, temp.second);
+            }
             disturb_colors(temp.first, rnd);
             train_data.enqueue(temp);
         }
@@ -274,7 +292,17 @@ try
             win.clear_overlay();
             win.set_image(temp.first);
             for (const auto& r : temp.second)
-                win.add_overlay(r.rect, string_to_color(r.label), r.label);
+            {
+                auto color = string_to_color(r.label);
+                if (r.ignore)
+                {
+                    color.alpha = 128;
+                    win.add_overlay(r.rect.tl_corner(), r.rect.br_corner(), color);
+                    win.add_overlay(r.rect.tr_corner(), r.rect.bl_corner(), color);
+                }
+                win.add_overlay(r.rect, color, r.label);
+            }
+            cout << "Press enter to visualize the next training sample.";
             cin.get();
         }
     }

From dfd56c14473277895317d5cdb7cf177ad4d0fc5a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Adri=C3=A0=20Arrufat?= <adria.arrufat@outlook.com>
Date: Wed, 23 Jun 2021 18:03:42 +0900
Subject: [PATCH 11/62] add some comments to the example training program

---
 examples/dnn_yolo_train_ex.cpp | 18 ++++++++++++++++--
 1 file changed, 16 insertions(+), 2 deletions(-)

diff --git a/examples/dnn_yolo_train_ex.cpp b/examples/dnn_yolo_train_ex.cpp
index a55ba7f560..9d6d094a49 100644
--- a/examples/dnn_yolo_train_ex.cpp
+++ b/examples/dnn_yolo_train_ex.cpp
@@ -172,7 +172,13 @@ try
         cout << " (" << (100.0*label.second)/num_objects << "%)\n";
         options.labels.push_back(label.first);
     }
+    // When computing the objectness loss in YOLO, predictions that do not have an IoU
+    // with any ground truth box of at least options.truth_match_iou_threshold, will be
+    // treated as not capable of detecting an object, an therefore incur loss.
+    // Predictions about this threshold will be ignored, i.e. will not affect the loss.
     options.truth_match_iou_threshold = 0.7;
+    // These are the anchors computed on COCO dataset by the official DarkNet code
+    // using K-Means clustering.
     options.add_anchors<darknet::ytag8>({{10, 13}, {16, 30}, {33, 23}});
     options.add_anchors<darknet::ytag16>({{30, 61}, {62, 45}, {59, 119}});
     options.add_anchors<darknet::ytag32>({{116, 90}, {156, 198}, {373, 326}});
@@ -180,18 +186,23 @@ try
     darknet::setup_detector(net, options);
     cerr << net << endl;
 
+    // The training process can be unstable at the beginning.  For this reason, we exponentially
+    // increase the learning rate during the first burnin steps.
     const matrix<double> learning_rate_schedule = learning_rate * pow(linspace(1e-12, 1, burnin), 4);
 
+    // In case we have several GPUs, we can tell the dnn_trainer to make use of them.
     std::vector<int> gpus(num_gpus);
     iota(gpus.begin(), gpus.end(), 0);
     dnn_trainer<darknet::yolov3_train_type> trainer(net, sgd(0.0005, 0.9), gpus);
     trainer.be_verbose();
     trainer.set_mini_batch_size(batch_size);
     trainer.set_learning_rate_schedule(learning_rate_schedule);
-    trainer.set_mini_batch_size(batch_size);
     trainer.set_synchronization_file(sync_file_name, chrono::minutes(15));
     cout << trainer;
 
+    // If the training has started and a synchronization file has already been saved to disk,
+    // we can re-run this program with the --test option and a confidence threshold to see
+    // how the training is going.
     if (parser.option("test"))
     {
         if (!file_exists(sync_file_name))
@@ -224,6 +235,7 @@ try
         return EXIT_SUCCESS;
     }
 
+    // Create some data loaders which will load the data, and perform som data augmentation.
     dlib::pipe<std::pair<matrix<rgb_pixel>, std::vector<yolo_rect>>> train_data(1000);
     auto loader = [&dataset, &data_directory, &train_data, &image_size](time_t seed) {
         dlib::rand rnd(time(nullptr) + seed);
@@ -247,7 +259,6 @@ try
 
             if (rnd.get_random_double() > 0.5)
             {
-                // Some data augmentation
                 rectangle_transform tform = rotate_image(
                     image,
                     rotated,
@@ -281,6 +292,8 @@ try
     for (size_t i = 0; i < num_workers; ++i)
         data_loaders.emplace_back([loader, i]() { loader(i + 1); });
 
+    // It is always a good idea to visualize the training samples.  By passing the --visualize
+    // flag, we can see the training samples that will be fed to the dnn_trainer.
     if (parser.option("visualize"))
     {
         image_window win;
@@ -310,6 +323,7 @@ try
     std::vector<matrix<rgb_pixel>> images;
     std::vector<std::vector<yolo_rect>> bboxes;
 
+    // The main training loop, that we will reuse for the warmup and the rest of the training.
     const auto train = [&images, &bboxes, &train_data, &trainer]()
     {
         images.clear();

From 9ea1a02f88366781ca074427bb491189ce33057a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Adri=C3=A0=20Arrufat?= <adria.arrufat@outlook.com>
Date: Wed, 23 Jun 2021 18:16:52 +0900
Subject: [PATCH 12/62] do not use overlaps_ignore

After visualizing some data, it does not make sense to use this feature
for YOLO detectors.
---
 dlib/dnn/loss.h | 5 +----
 1 file changed, 1 insertion(+), 4 deletions(-)

diff --git a/dlib/dnn/loss.h b/dlib/dnn/loss.h
index 0dc60a57c2..1788260b4a 100644
--- a/dlib/dnn/loss.h
+++ b/dlib/dnn/loss.h
@@ -3538,7 +3538,6 @@ namespace dlib
         double truth_match_iou_threshold = 0.7;
         bool classwise_nms = false;
         test_box_overlap overlaps_nms = test_box_overlap(0.45, 1.0);
-        test_box_overlap overlaps_ignore = test_box_overlap(0.5, 1.0);
         double lambda_obj = 1.0;
         double lambda_box = 1.0;
         double lambda_cls = 1.0;
@@ -3555,7 +3554,6 @@ namespace dlib
         serialize(item.truth_match_iou_threshold, out);
         serialize(item.classwise_nms, out);
         serialize(item.overlaps_nms, out);
-        serialize(item.overlaps_ignore, out);
         serialize(item.lambda_obj, out);
         serialize(item.lambda_box, out);
         serialize(item.lambda_cls, out);
@@ -3573,7 +3571,6 @@ namespace dlib
         deserialize(item.truth_match_iou_threshold, in);
         deserialize(item.classwise_nms, in);
         deserialize(item.overlaps_nms, in);
-        deserialize(item.overlaps_ignore, in);
         deserialize(item.lambda_obj, in);
         deserialize(item.lambda_box, in);
         deserialize(item.lambda_cls, in);
@@ -3739,7 +3736,7 @@ namespace dlib
                             double best_iou = 0;
                             for (const yolo_rect& truth_box : *truth)
                             {
-                                if (truth_box.ignore || options.overlaps_ignore(truth_box.rect, pred.rect))
+                                if (truth_box.ignore)
                                     continue;
                                 best_iou = std::max(best_iou, box_intersection_over_union(truth_box.rect, pred.rect));
                             }

From fe2d7e57f47990ba0b18f22a7ca8ca28011eab41 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Adri=C3=A0=20Arrufat?= <adria.arrufat@outlook.com>
Date: Fri, 25 Jun 2021 15:38:27 +0900
Subject: [PATCH 13/62] add mAP computation

---
 dlib/dnn/loss.h                |  6 ++-
 examples/dnn_yolo_train_ex.cpp | 73 +++++++++++++++++++++++++++++++---
 2 files changed, 72 insertions(+), 7 deletions(-)

diff --git a/dlib/dnn/loss.h b/dlib/dnn/loss.h
index 1788260b4a..2838b2c584 100644
--- a/dlib/dnn/loss.h
+++ b/dlib/dnn/loss.h
@@ -3889,7 +3889,11 @@ namespace dlib
             return loss / input_tensor.num_samples();
         }
 
-        void adjust_threshold(float conf_thresh) { options.confidence_threshold = conf_thresh; }
+        void adjust_threshold(double conf_thresh) { options.confidence_threshold = conf_thresh; }
+        void adjust_nms(double iou_thresh, double percent_covered_thresh = 1)
+        {
+            options.overlaps_nms = test_box_overlap(iou_thresh, percent_covered_thresh);
+        }
 
         friend void serialize(const loss_yolo_& item, std::ostream& out)
         {
diff --git a/examples/dnn_yolo_train_ex.cpp b/examples/dnn_yolo_train_ex.cpp
index 9d6d094a49..e477b36f98 100644
--- a/examples/dnn_yolo_train_ex.cpp
+++ b/examples/dnn_yolo_train_ex.cpp
@@ -129,6 +129,7 @@ try
     parser.add_option("gpus", "number of GPUs to run the training on (default: 1)", 1);
     parser.add_option("test", "test the detector with a threshold (default: 0.01)", 1);
     parser.add_option("visualize", "visualize data augmentation instead of training");
+    parser.add_option("map", "compute the mean average precision");
     parser.set_group_name("Help Options");
     parser.add_option("h", "alias of --help");
     parser.add_option("help", "display this message and exit");
@@ -152,7 +153,7 @@ try
 
     image_dataset_metadata::dataset dataset;
     image_dataset_metadata::load_image_dataset_metadata(dataset, data_directory + "/training.xml");
-    std::cout << "# images: " << dataset.images.size() << std::endl;
+    cout << "# images: " << dataset.images.size() << endl;
     std::map<string, size_t> labels;
     size_t num_objects = 0;
     for (const auto& im : dataset.images)
@@ -184,7 +185,6 @@ try
     options.add_anchors<darknet::ytag32>({{116, 90}, {156, 198}, {373, 326}});
     darknet::yolov3_train_type net(options);
     darknet::setup_detector(net, options);
-    cerr << net << endl;
 
     // The training process can be unstable at the beginning.  For this reason, we exponentially
     // increase the learning rate during the first burnin steps.
@@ -235,6 +235,67 @@ try
         return EXIT_SUCCESS;
     }
 
+    if (parser.option("map"))
+    {
+        image_dataset_metadata::dataset dataset;
+        image_dataset_metadata::load_image_dataset_metadata(dataset, data_directory + "/testing.xml");
+        if (!file_exists(sync_file_name))
+        {
+            cout << "Could not find file " << sync_file_name << endl;
+            return EXIT_FAILURE;
+        }
+        matrix<rgb_pixel> image, resized;
+        std::map<std::string, std::vector<std::pair<double, bool>>> hits;
+        std::map<std::string, unsigned long> missing;
+        net.loss_details().adjust_threshold(0.005);
+        for (auto& im : dataset.images)
+        {
+            load_image(image, data_directory + "/" + im.filename);
+            const auto tform = preprocess_image(image, resized, image_size);
+            auto dets = net(resized);
+            postprocess_detections(tform, dets);
+            std::vector<bool> used(dets.size(), false);
+            // true positives: truths matched by detections
+            for (size_t i = 0; i < im.boxes.size(); ++i)
+            {
+                bool found_match = false;
+                for (size_t j = 0; j < dets.size(); ++j)
+                {
+                    if (used[j])
+                        continue;
+                    if (im.boxes[i].label == dets[j].label &&
+                        box_intersection_over_union(drectangle(im.boxes[i].rect), dets[j].rect) > 0.5)
+                    {
+                        used[j] = true;
+                        found_match = true;
+                        hits[dets[j].label].emplace_back(dets[j].detection_confidence, true);
+                        break;
+                    }
+                }
+                // false negatives: truths not matched
+                if (!found_match)
+                    missing[im.boxes[i].label]++;
+            }
+            // false positives: detections not matched
+            for (size_t i = 0; i < dets.size(); ++i)
+            {
+                if (!used[i])
+                    hits[dets[i].label].emplace_back(dets[i].detection_confidence, false);
+            }
+        }
+        double map = 0;
+        for (auto& item : hits)
+        {
+            std::sort(item.second.rbegin(), item.second.rend());
+            const double ap = average_precision(item.second, missing[item.first]);
+            cout << rpad(item.first + ": ", 16, " ") << ap * 100 << '%' << endl;
+            map += ap;
+        }
+        cout << rpad(string("mAP: "), 16, " ") << map / hits.size() * 100 << '%' << endl;
+        return EXIT_SUCCESS;
+    }
+
+
     // Create some data loaders which will load the data, and perform som data augmentation.
     dlib::pipe<std::pair<matrix<rgb_pixel>, std::vector<yolo_rect>>> train_data(1000);
     auto loader = [&dataset, &data_directory, &train_data, &image_size](time_t seed) {
@@ -338,22 +399,22 @@ try
         trainer.train_one_step(images, bboxes);
     };
 
-    std::cerr << "training started with " << burnin << " burn-in steps" << std::endl;
+    cout << "training started with " << burnin << " burn-in steps" << endl;
     while (trainer.get_train_one_step_calls() < burnin)
         train();
 
-    std::cerr << "burn-in finished" << std::endl;
+    cout << "burn-in finished" << endl;
     trainer.get_net();
     trainer.set_learning_rate(learning_rate);
     trainer.set_min_learning_rate(learning_rate * 1e-3);
     trainer.set_learning_rate_shrink_factor(0.1);
     trainer.set_iterations_without_progress_threshold(patience);
-    std::cerr << trainer << std::endl;
+    cout << trainer << endl;
 
     while (trainer.get_learning_rate() >= trainer.get_min_learning_rate())
         train();
 
-    std::cerr << "training done" << std::endl;
+    cout << "training done" << endl;
 
     trainer.get_net();
     train_data.disable();

From c35db674d699afbaf62d526a2fea1494be2a8846 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Adri=C3=A0=20Arrufat?= <adria.arrufat@outlook.com>
Date: Fri, 25 Jun 2021 15:51:03 +0900
Subject: [PATCH 14/62] print progress while computing map

---
 examples/dnn_yolo_train_ex.cpp | 27 +++++++++++++++------------
 1 file changed, 15 insertions(+), 12 deletions(-)

diff --git a/examples/dnn_yolo_train_ex.cpp b/examples/dnn_yolo_train_ex.cpp
index e477b36f98..aed383df1f 100644
--- a/examples/dnn_yolo_train_ex.cpp
+++ b/examples/dnn_yolo_train_ex.cpp
@@ -248,40 +248,43 @@ try
         std::map<std::string, std::vector<std::pair<double, bool>>> hits;
         std::map<std::string, unsigned long> missing;
         net.loss_details().adjust_threshold(0.005);
-        for (auto& im : dataset.images)
+        cout << "computing mean average precision for " << dataset.images.size() << " images..." << endl;
+        for (size_t i = 0; i < dataset.images.size(); ++i)
         {
+            const auto& im = dataset.images[i];
             load_image(image, data_directory + "/" + im.filename);
             const auto tform = preprocess_image(image, resized, image_size);
             auto dets = net(resized);
             postprocess_detections(tform, dets);
             std::vector<bool> used(dets.size(), false);
             // true positives: truths matched by detections
-            for (size_t i = 0; i < im.boxes.size(); ++i)
+            for (size_t t = 0; t < im.boxes.size(); ++t)
             {
                 bool found_match = false;
-                for (size_t j = 0; j < dets.size(); ++j)
+                for (size_t d = 0; d < dets.size(); ++d)
                 {
-                    if (used[j])
+                    if (used[d])
                         continue;
-                    if (im.boxes[i].label == dets[j].label &&
-                        box_intersection_over_union(drectangle(im.boxes[i].rect), dets[j].rect) > 0.5)
+                    if (im.boxes[t].label == dets[d].label &&
+                        box_intersection_over_union(drectangle(im.boxes[t].rect), dets[d].rect) > 0.5)
                     {
-                        used[j] = true;
+                        used[d] = true;
                         found_match = true;
-                        hits[dets[j].label].emplace_back(dets[j].detection_confidence, true);
+                        hits[dets[d].label].emplace_back(dets[d].detection_confidence, true);
                         break;
                     }
                 }
                 // false negatives: truths not matched
                 if (!found_match)
-                    missing[im.boxes[i].label]++;
+                    missing[im.boxes[t].label]++;
             }
             // false positives: detections not matched
-            for (size_t i = 0; i < dets.size(); ++i)
+            for (size_t d = 0; d < dets.size(); ++d)
             {
-                if (!used[i])
-                    hits[dets[i].label].emplace_back(dets[i].detection_confidence, false);
+                if (!used[d])
+                    hits[dets[d].label].emplace_back(dets[d].detection_confidence, false);
             }
+            cout << "progress: " << i << '/' << dataset.images.size() << "\t\t\t\r" << flush;
         }
         double map = 0;
         for (auto& item : hits)

From df8d64cf9e5fb263cb2488e4d82f4150dec157de Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Adri=C3=A0=20Arrufat?= <adria.arrufat@outlook.com>
Date: Fri, 25 Jun 2021 17:10:56 +0900
Subject: [PATCH 15/62] use explicit type

---
 dlib/dnn/loss.h | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/dlib/dnn/loss.h b/dlib/dnn/loss.h
index 2838b2c584..a3dea8d037 100644
--- a/dlib/dnn/loss.h
+++ b/dlib/dnn/loss.h
@@ -3847,7 +3847,7 @@ namespace dlib
         {
             DLIB_CASSERT(sub.sample_expansion_factor() == 1, sub.sample_expansion_factor());
             std::vector<yolo_rect> dets_accum;
-            output_label_type final_dets;
+            std::vector<yolo_rect> final_dets;
             for (long i = 0; i < input_tensor.num_samples(); ++i)
             {
                 dets_accum.clear();

From 9d703a2c991255bdc3b7cd4834b7c705929d41a6 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Adri=C3=A0=20Arrufat?= <adria.arrufat@outlook.com>
Date: Sat, 26 Jun 2021 19:16:19 +0900
Subject: [PATCH 16/62] fix mAP computation and initialize color mapper for all
 labels

---
 examples/dnn_yolo_train_ex.cpp | 17 +++++++++++------
 1 file changed, 11 insertions(+), 6 deletions(-)

diff --git a/examples/dnn_yolo_train_ex.cpp b/examples/dnn_yolo_train_ex.cpp
index aed383df1f..8e500dcf40 100644
--- a/examples/dnn_yolo_train_ex.cpp
+++ b/examples/dnn_yolo_train_ex.cpp
@@ -167,11 +167,13 @@ try
     cout << "# labels: " << labels.size() << endl;
 
     yolo_options options;
+    color_mapper string_to_color;
     for (const auto& label : labels)
     {
         cout <<  " - " << label.first << ": " << label.second;
         cout << " (" << (100.0*label.second)/num_objects << "%)\n";
         options.labels.push_back(label.first);
+        string_to_color(label.first);
     }
     // When computing the objectness loss in YOLO, predictions that do not have an IoU
     // with any ground truth box of at least options.truth_match_iou_threshold, will be
@@ -214,7 +216,6 @@ try
         net.loss_details().adjust_threshold(threshold);
         image_window win;
         matrix<rgb_pixel> image, resized;
-        color_mapper string_to_color;
         for (const auto& im : dataset.images)
         {
             win.clear_overlay();
@@ -247,6 +248,11 @@ try
         matrix<rgb_pixel> image, resized;
         std::map<std::string, std::vector<std::pair<double, bool>>> hits;
         std::map<std::string, unsigned long> missing;
+        for (const auto& label : options.labels)
+        {
+            hits[label] = std::vector<std::pair<double, bool>>();
+            missing[label] = 0;
+        }
         net.loss_details().adjust_threshold(0.005);
         cout << "computing mean average precision for " << dataset.images.size() << " images..." << endl;
         for (size_t i = 0; i < dataset.images.size(); ++i)
@@ -270,19 +276,19 @@ try
                     {
                         used[d] = true;
                         found_match = true;
-                        hits[dets[d].label].emplace_back(dets[d].detection_confidence, true);
+                        hits.at(dets[d].label).emplace_back(dets[d].detection_confidence, true);
                         break;
                     }
                 }
                 // false negatives: truths not matched
                 if (!found_match)
-                    missing[im.boxes[t].label]++;
+                    missing.at(im.boxes[t].label)++;
             }
             // false positives: detections not matched
             for (size_t d = 0; d < dets.size(); ++d)
             {
                 if (!used[d])
-                    hits[dets[d].label].emplace_back(dets[d].detection_confidence, false);
+                    hits.at(dets[d].label).emplace_back(dets[d].detection_confidence, false);
             }
             cout << "progress: " << i << '/' << dataset.images.size() << "\t\t\t\r" << flush;
         }
@@ -311,7 +317,7 @@ try
         cropper.set_max_object_size(0.9);
         cropper.set_min_object_size(10, 10);
         cropper.set_max_rotation_degrees(10);
-        cropper.set_translate_amount(0.1);
+        cropper.set_translate_amount(0.5);
         cropper.set_randomly_flip(true);
         cropper.set_background_crops_fraction(0);
         while (train_data.is_enabled())
@@ -361,7 +367,6 @@ try
     if (parser.option("visualize"))
     {
         image_window win;
-        color_mapper string_to_color;
         while (true)
         {
             std::pair<matrix<rgb_pixel>, std::vector<yolo_rect>> temp;

From 05ac6ff66042a82fb3fcbd621c26d543acde5d01 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Adri=C3=A0=20Arrufat?= <adria.arrufat@outlook.com>
Date: Sat, 26 Jun 2021 23:05:34 +0900
Subject: [PATCH 17/62] allow multi label detections

---
 dlib/dnn/loss.h | 12 ++++++++++--
 1 file changed, 10 insertions(+), 2 deletions(-)

diff --git a/dlib/dnn/loss.h b/dlib/dnn/loss.h
index a3dea8d037..36db373fd1 100644
--- a/dlib/dnn/loss.h
+++ b/dlib/dnn/loss.h
@@ -3459,6 +3459,7 @@ namespace dlib
         double detection_confidence = 0;
         bool ignore = false;
         std::string label;
+        std::vector<std::pair<double, std::string>> labels;
 
         operator rectangle() const { return rect; }
         bool operator == (const yolo_rect& rhs) const
@@ -3482,6 +3483,7 @@ namespace dlib
         serialize(item.detection_confidence, out);
         serialize(item.ignore, out);
         serialize(item.label, out);
+        serialize(item.labels, out);
     }
 
     inline void deserialize(yolo_rect& item, std::istream& in)
@@ -3493,6 +3495,8 @@ namespace dlib
         deserialize(item.rect, in);
         deserialize(item.detection_confidence, in);
         deserialize(item.ignore, in);
+        deserialize(item.label, in);
+        deserialize(item.labels, in);
     }
 
     struct yolo_options
@@ -3672,16 +3676,20 @@ namespace dlib
                                                              h / (1 - h) * anchors[a].height));
                                 for (long k = 0; k < num_classes; ++k)
                                 {
-                                    const float conf = out_data[tensor_index(output_tensor, n, a * num_feats + 5 + k, r, c)];
+                                    const float conf = out_data[tensor_index(output_tensor, n, a * num_feats + 5 + k, r, c)] * obj;
                                     if (conf > det.detection_confidence)
                                     {
                                         det.detection_confidence = conf;
                                         det.label = options.labels[k];
                                     }
+                                    if (conf > options.confidence_threshold)
+                                        det.labels.emplace_back(conf, options.labels[k]);
                                 }
-                                det.detection_confidence *= obj;
                                 if (det.detection_confidence > options.confidence_threshold)
+                                {
+                                    std::sort(det.labels.rbegin(), det.labels.rend());
                                     dets.push_back(std::move(det));
+                                }
                             }
                         }
                     }

From 7f9c7164958c68603f0dda4b66893d3a026a1445 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Adri=C3=A0=20Arrufat?= <adria.arrufat@outlook.com>
Date: Sat, 26 Jun 2021 23:21:53 +0900
Subject: [PATCH 18/62] simplify multilabel processing output

---
 dlib/dnn/loss.h | 9 +++------
 1 file changed, 3 insertions(+), 6 deletions(-)

diff --git a/dlib/dnn/loss.h b/dlib/dnn/loss.h
index 36db373fd1..ead37a5af1 100644
--- a/dlib/dnn/loss.h
+++ b/dlib/dnn/loss.h
@@ -3677,17 +3677,14 @@ namespace dlib
                                 for (long k = 0; k < num_classes; ++k)
                                 {
                                     const float conf = out_data[tensor_index(output_tensor, n, a * num_feats + 5 + k, r, c)] * obj;
-                                    if (conf > det.detection_confidence)
-                                    {
-                                        det.detection_confidence = conf;
-                                        det.label = options.labels[k];
-                                    }
                                     if (conf > options.confidence_threshold)
                                         det.labels.emplace_back(conf, options.labels[k]);
                                 }
-                                if (det.detection_confidence > options.confidence_threshold)
+                                if (!det.labels.empty() && det.labels[0].first > options.confidence_threshold)
                                 {
                                     std::sort(det.labels.rbegin(), det.labels.rend());
+                                    det.detection_confidence = det.labels[0].first;
+                                    det.label = det.labels[0].second;
                                     dets.push_back(std::move(det));
                                 }
                             }

From c07910cc5f96a2eb94e413f1756b31400e8e90cd Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Adri=C3=A0=20Arrufat?= <adria.arrufat@outlook.com>
Date: Sun, 27 Jun 2021 02:12:46 +0900
Subject: [PATCH 19/62] add documentation

---
 dlib/dnn/loss.h                               | 136 +++++++--------
 dlib/dnn/loss_abstract.h                      | 162 ++++++++++++++++++
 dlib/image_processing/full_object_detection.h |  54 ++++++
 .../full_object_detection_abstract.h          |  44 +++++
 dlib/test/test.png                            | Bin 0 -> 76 bytes
 examples/dnn_yolo_train_ex.cpp                |  54 ++++--
 6 files changed, 366 insertions(+), 84 deletions(-)
 create mode 100644 dlib/test/test.png

diff --git a/dlib/dnn/loss.h b/dlib/dnn/loss.h
index ead37a5af1..036e5c268a 100644
--- a/dlib/dnn/loss.h
+++ b/dlib/dnn/loss.h
@@ -3447,58 +3447,6 @@ namespace dlib
 
 // ----------------------------------------------------------------------------------------
 
-    struct yolo_rect
-    {
-        yolo_rect() = default;
-        yolo_rect(const drectangle& r) : rect(r) {}
-        yolo_rect(const drectangle& r, double score) : rect(r),detection_confidence(score) {}
-        yolo_rect(const drectangle& r, double score, const std::string& label) : rect(r),detection_confidence(score), label(label) {}
-        yolo_rect(const mmod_rect& r) : rect(r.rect), detection_confidence(r.detection_confidence), ignore(r.ignore), label(r.label) {}
-
-        drectangle rect;
-        double detection_confidence = 0;
-        bool ignore = false;
-        std::string label;
-        std::vector<std::pair<double, std::string>> labels;
-
-        operator rectangle() const { return rect; }
-        bool operator == (const yolo_rect& rhs) const
-        {
-            return rect == rhs.rect
-                   && detection_confidence == rhs.detection_confidence
-                   && ignore == rhs.ignore
-                   && label == rhs.label;
-        }
-        bool operator<(const yolo_rect& rhs) const
-        {
-            return detection_confidence < rhs.detection_confidence;
-        }
-    };
-
-    inline void serialize(const yolo_rect& item, std::ostream& out)
-    {
-        int version = 1;
-        serialize(version, out);
-        serialize(item.rect, out);
-        serialize(item.detection_confidence, out);
-        serialize(item.ignore, out);
-        serialize(item.label, out);
-        serialize(item.labels, out);
-    }
-
-    inline void deserialize(yolo_rect& item, std::istream& in)
-    {
-        int version = 0;
-        deserialize(version, in);
-        if (version != 1)
-            throw serialization_error("Unexpected version found while deserializing dlib::yolo_rect");
-        deserialize(item.rect, in);
-        deserialize(item.detection_confidence, in);
-        deserialize(item.ignore, in);
-        deserialize(item.label, in);
-        deserialize(item.labels, in);
-    }
-
     struct yolo_options
     {
     public:
@@ -3536,10 +3484,10 @@ namespace dlib
         }
 
         // map between the stride and the anchor boxes
-        std::unordered_map<int, std::vector<anchor_box_details>> anchors;
+        std::map<int, std::vector<anchor_box_details>> anchors;
         std::vector<std::string> labels;
         double confidence_threshold = 0.25;
-        double truth_match_iou_threshold = 0.7;
+        double iou_ignore_threshold = 0.7;
         bool classwise_nms = false;
         test_box_overlap overlaps_nms = test_box_overlap(0.45, 1.0);
         double lambda_obj = 1.0;
@@ -3555,7 +3503,7 @@ namespace dlib
         serialize(item.anchors, out);
         serialize(item.labels, out);
         serialize(item.confidence_threshold, out);
-        serialize(item.truth_match_iou_threshold, out);
+        serialize(item.iou_ignore_threshold, out);
         serialize(item.classwise_nms, out);
         serialize(item.overlaps_nms, out);
         serialize(item.lambda_obj, out);
@@ -3572,7 +3520,7 @@ namespace dlib
         deserialize(item.anchors, in);
         deserialize(item.labels, in);
         deserialize(item.confidence_threshold, in);
-        deserialize(item.truth_match_iou_threshold, in);
+        deserialize(item.iou_ignore_threshold, in);
         deserialize(item.classwise_nms, in);
         deserialize(item.overlaps_nms, in);
         deserialize(item.lambda_obj, in);
@@ -3580,9 +3528,25 @@ namespace dlib
         deserialize(item.lambda_cls, in);
     }
 
-    inline std::ostream& operator<<(std::ostream& out, const std::unordered_map<int, std::vector<yolo_options::anchor_box_details>>& anchors)
+    inline std::ostream& operator<<(std::ostream& out, const std::map<int, std::vector<yolo_options::anchor_box_details>>& anchors)
     {
-        out << "anchors: " << anchors.size();
+        // write anchor boxes grouped by tag id
+        size_t tag_count = 0;
+        for (const auto& i : anchors)
+        {
+            const auto& tag_id = i.first;
+            const auto& details = i.second;
+            if (tag_count++ > 0)
+                out << ";";
+            out << "tag" << tag_id << ":";
+            size_t anchor_count = 0;
+            for (size_t a = 0; a < details.size(); ++a)
+            {
+                out << details[a].width << "x" << details[a].height;
+                if (a + 1 < details.size())
+                    out << ",";
+            }
+        }
         return out;
     }
 
@@ -3608,11 +3572,12 @@ namespace dlib
                 const SUBNET& sub,
                 const long n,
                 const yolo_options& options,
+                const double adjust_threshold,
                 std::vector<yolo_rect>& dets
             )
             {
-                yolo_helper_impl<TAG_TYPE>::tensor_to_dets(input_tensor, sub, n, options, dets);
-                yolo_helper_impl<TAG_TYPES...>::tensor_to_dets(input_tensor, sub, n, options, dets);
+                yolo_helper_impl<TAG_TYPE>::tensor_to_dets(input_tensor, sub, n, options, adjust_threshold, dets);
+                yolo_helper_impl<TAG_TYPES...>::tensor_to_dets(input_tensor, sub, n, options, adjust_threshold, dets);
             }
 
             template <
@@ -3646,14 +3611,16 @@ namespace dlib
                 const SUBNET& sub,
                 const long n,
                 const yolo_options& options,
+                const double adjust_threshold,
                 std::vector<yolo_rect>& dets
             )
             {
                 DLIB_CASSERT(sub.sample_expansion_factor() == 1, sub.sample_expansion_factor());
+                const auto& anchors = options.anchors.at(tag_id<TAG_TYPE>::id);
                 const tensor& output_tensor = layer<TAG_TYPE>(sub).get_output();
+                DLIB_CASSERT(output_tensor.k() == anchors.size() * (options.labels.size() + 5));
                 const auto stride_x = static_cast<double>(input_tensor.nc()) / output_tensor.nc();
                 const auto stride_y = static_cast<double>(input_tensor.nr()) / output_tensor.nr();
-                const auto& anchors = options.anchors.at(tag_id<TAG_TYPE>::id);
                 const long num_feats = output_tensor.k() / anchors.size();
                 const long num_classes = num_feats - 5;
                 const float* const out_data = output_tensor.host();
@@ -3665,7 +3632,7 @@ namespace dlib
                         for (long c = 0; c < output_tensor.nc(); ++c)
                         {
                             const float obj = out_data[tensor_index(output_tensor, n, a * num_feats + 4, r, c)];
-                            if (obj > options.confidence_threshold)
+                            if (obj > adjust_threshold)
                             {
                                 const auto x = out_data[tensor_index(output_tensor, n, a * num_feats + 0, r, c)];
                                 const auto y = out_data[tensor_index(output_tensor, n, a * num_feats + 1, r, c)];
@@ -3677,10 +3644,10 @@ namespace dlib
                                 for (long k = 0; k < num_classes; ++k)
                                 {
                                     const float conf = out_data[tensor_index(output_tensor, n, a * num_feats + 5 + k, r, c)] * obj;
-                                    if (conf > options.confidence_threshold)
+                                    if (conf > adjust_threshold)
                                         det.labels.emplace_back(conf, options.labels[k]);
                                 }
-                                if (!det.labels.empty() && det.labels[0].first > options.confidence_threshold)
+                                if (!det.labels.empty() && det.labels[0].first > adjust_threshold)
                                 {
                                     std::sort(det.labels.rbegin(), det.labels.rend());
                                     det.detection_confidence = det.labels[0].first;
@@ -3708,13 +3675,16 @@ namespace dlib
             {
                 DLIB_CASSERT(sub.sample_expansion_factor() == 1, sub.sample_expansion_factor());
                 const tensor& output_tensor = layer<TAG_TYPE>(sub).get_output();
+                const auto& anchors = options.anchors.at(tag_id<TAG_TYPE>::id);
+                DLIB_CASSERT(output_tensor.k() == anchors.size() * (options.labels.size() + 5));
                 const auto stride_x = static_cast<double>(input_tensor.nc()) / output_tensor.nc();
                 const auto stride_y = static_cast<double>(input_tensor.nr()) / output_tensor.nr();
-                const auto& anchors = options.anchors.at(tag_id<TAG_TYPE>::id);
                 const long num_feats = output_tensor.k() / anchors.size();
                 const long num_classes = num_feats - 5;
                 const float* const out_data = output_tensor.host();
                 tensor& grad = layer<TAG_TYPE>(sub).get_gradient_input();
+                DLIB_CASSERT(input_tensor.num_samples() == grad.num_samples());
+                DLIB_CASSERT(input_tensor.num_samples() == output_tensor.num_samples());
                 float* g = grad.host();
 
                 // Compute the objectness loss for all grid cells
@@ -3747,7 +3717,7 @@ namespace dlib
                             }
 
                             // Incur loss for the boxes that are below a certain IoU threshold with any truth box
-                            if (best_iou < options.truth_match_iou_threshold)
+                            if (best_iou < options.iou_ignore_threshold)
                                 g[o_idx] = out_data[o_idx];
                         }
                     }
@@ -3847,16 +3817,16 @@ namespace dlib
         void to_label (
             const tensor& input_tensor,
             const SUB_TYPE& sub,
-            label_iterator iter
+            label_iterator iter,
+            double adjust_threshold = 0.25
         ) const
         {
-            DLIB_CASSERT(sub.sample_expansion_factor() == 1, sub.sample_expansion_factor());
             std::vector<yolo_rect> dets_accum;
             std::vector<yolo_rect> final_dets;
             for (long i = 0; i < input_tensor.num_samples(); ++i)
             {
                 dets_accum.clear();
-                impl::yolo_helper_impl<TAG_TYPES...>::tensor_to_dets(input_tensor, sub, i, options, dets_accum);
+                impl::yolo_helper_impl<TAG_TYPES...>::tensor_to_dets(input_tensor, sub, i, options, adjust_threshold, dets_accum);
 
                 // Do non-max suppression
                 std::sort(dets_accum.rbegin(), dets_accum.rend());
@@ -3884,7 +3854,6 @@ namespace dlib
         ) const
         {
             DLIB_CASSERT(input_tensor.num_samples() != 0);
-            DLIB_CASSERT(sub.sample_expansion_factor() == 1);
             double loss = 0;
             for (long i = 0; i < input_tensor.num_samples(); ++i)
             {
@@ -3894,7 +3863,6 @@ namespace dlib
             return loss / input_tensor.num_samples();
         }
 
-        void adjust_threshold(double conf_thresh) { options.confidence_threshold = conf_thresh; }
         void adjust_nms(double iou_thresh, double percent_covered_thresh = 1)
         {
             options.overlaps_nms = test_box_overlap(iou_thresh, percent_covered_thresh);
@@ -3923,14 +3891,34 @@ namespace dlib
             deserialize(item.options, in);
         }
 
-        friend std::ostream& operator<<(std::ostream& out, const loss_yolo_& )
+        friend std::ostream& operator<<(std::ostream& out, const loss_yolo_& item)
         {
-            out << "loss_yolo\t (" << tag_count() << " output tags: ";
-            list_tags(out);
+            out << "loss_yolo\t (";
+            const auto& opts = item.options;
+            out << "num outputs:" << tag_count();
+            out << ", anchor boxes:(" << opts.anchors << ")";
+            out << ", labels:(";
+            for (size_t i = 0; i < opts.labels.size(); ++i)
+            {
+                out << opts.labels[i];
+                if (i + 1 < opts.labels.size())
+                    out << ",";
+            }
+            out << ")";
+            out << ", lambda_obj:" << opts.lambda_obj;
+            out << ", lambda_box:" << opts.lambda_box;
+            out << ", lambda_cls:" << opts.lambda_cls;
+            out << ", overlaps_nms:(" << opts.overlaps_nms.get_iou_thresh() << "," << opts.overlaps_nms.get_percent_covered_thresh() << ")";
+            out << ", classwise_nms:" << std::boolalpha << opts.classwise_nms;
             out << ")";
             return out;
         }
 
+        friend void to_xml(const loss_yolo_& /*item*/, std::ostream& out)
+        {
+            out << "<loss_yolo/>";
+        }
+
     private:
 
         yolo_options options;
diff --git a/dlib/dnn/loss_abstract.h b/dlib/dnn/loss_abstract.h
index c285779ba6..7aea09777e 100644
--- a/dlib/dnn/loss_abstract.h
+++ b/dlib/dnn/loss_abstract.h
@@ -1852,6 +1852,168 @@ namespace dlib
     template <typename SUBNET>
     using loss_dot = add_loss_layer<loss_dot_, SUBNET>;
 
+// ----------------------------------------------------------------------------------------
+
+    struct yolo_options
+    {
+        /*!
+            WHAT THIS OBJECT REPRESENTS
+                This object contains all the parameters that control the behavior of loss_yolo_.
+        !*/
+    public:
+        struct anchor_box_details
+        {
+            anchor_box_details() = default;
+            anchor_box_details(unsigned long w, unsigned long h) : width(w), height(h) {}
+
+            unsigned long width = 0;
+            unsigned long height = 0;
+
+            friend inline void serialize(const anchor_box_details& item, std::ostream& out);
+            friend inline void deserialize(anchor_box_details& item, std::istream& in);
+        };
+
+        yolo_options() = default;
+
+        // This kind of object detector is a multi-scale object detector with bounding box regression
+        // for anchor boxes.  The anchors field determinase which anchors will be used at the output
+        // pointed by the tag layer whose id is the key of the map.
+        std::unordered_map<int, std::vector<anchor_box_details>> anchors;
+
+        template <template <typename> class TAG_TYPE>
+        void add_anchors(
+            const std::vector<anchor_box_details>& boxes
+        );
+        /*!
+            ensures
+                - acnhors.at(tag_id<TAG_TYPE>::id) == boxes
+        !*/
+
+        // This field contains the labels of all the possible object this detector can find.
+        std::vector<std::string> labels;
+        // When computing the objectness loss, any detection that has an IoU above iou_ignore_threshold
+        // will be ignored and not incur any loss
+        double iou_ignore_threshold = 0.7;
+        // Tell the non-max Supression whether to take classes into account
+        bool classwise_nms = false;
+        // When doing non-max suppression, we use overlaps_nms to decide if a box overlaps
+        // an already output detection and should therefore be thrown out.
+        test_box_overlap overlaps_nms = test_box_overlap(0.45, 1.0);
+        // These parameters control how we penalize different kinds of mistakes: notably the objectness loss,
+        // the box (bounding box regression) loss, and the class loss.
+        double lambda_obj = 1.0;
+        double lambda_box = 1.0;
+        double lambda_cls = 1.0;
+
+    };
+
+    void serialize(const yolo_options& item, std::ostream& out)
+    void deserialize(yolo_options& item, std::istream& in)
+
+// ----------------------------------------------------------------------------------------
+
+    template <template <typename> class... TAG_TYPES>
+    class loss_yolo_
+    {
+        /*!
+            WHAT THIS OBJECT REPRESENTS
+                This object implements the loss layer interface defined above by
+                EXAMPLE_LOSS_LAYER_.  In particular, it implements the YOLO detection
+                loss defined in the paper:
+                    YOLOv3: An Incremental Improvement by Joseph Redmon and Ali Farhadi.
+
+                This means you use this loss if you want to detect the locations of objects
+                in images.
+
+                It should also be noted that this loss layer requires tag layers as template
+                parameters, which in turn require a subnetwork to be of type:
+                layer<TAG_TYPE>(net).subnet(): sig<con<(num_classes + 5) * num_anchors), SUBNET>>
+
+                Where num_classes is the number of categories that the detector is trained on,
+                and num_anchors is the number of priors or anchor boxes at the output pointed
+                by the tag layer. The number 5 corresponds to the objectness and the 4 coordinates
+                for performing bounding box regression.
+        !*/
+
+    public:
+
+        typedef std::vector<mmod_rect> training_label_type;
+        typedef std::vector<mmod_rect> output_label_type;
+
+        loss_mmod_(
+        );
+        /*!
+            ensures
+                - #get_options() == yolo_options()
+        !*/
+
+        loss_yolo_(
+            mmod_options options_
+        );
+        /*!
+            ensures
+                - #get_options() == options_
+        !*/
+
+        const yolo_options& get_options (
+        ) const;
+        /*!
+            ensures
+                - returns the options object that defines the general behavior of this loss layer.
+        !*/
+
+        template <
+            typename SUB_TYPE,
+            typename label_iterator
+            >
+        void to_label (
+            const tensor& input_tensor,
+            const SUB_TYPE& sub,
+            label_iterator iter,
+            double adjust_threshold = 0
+        ) const;
+        /*!
+            This function has the same interface as EXAMPLE_LOSS_LAYER_::to_label() except
+            it has the additional calling requirements that: 
+                - sub.get_output().k() == 1
+                - sub.get_output().num_samples() == input_tensor.num_samples()
+                - sub.sample_expansion_factor() == 1
+            Also, the output labels are std::vectors of mmod_rects where, for each mmod_rect R,
+            we have the following interpretations:
+                - R.rect == the location of an object in the image.
+                - R.detection_confidence the score for the object, the bigger the score the
+                  more confident the detector is that an object is really there.  Only
+                  objects with a detection_confidence > adjust_threshold are output.  So if
+                  you want to output more objects (that are also of less confidence) you
+                  can call to_label() with a smaller value of adjust_threshold.
+                - R.ignore == false (this value is unused by to_label()).
+        !*/
+
+        template <
+            typename const_label_iterator,
+            typename SUBNET
+            >
+        double compute_loss_value_and_gradient (
+            const tensor& input_tensor,
+            const_label_iterator truth, 
+            SUBNET& sub
+        ) const;
+        /*!
+            This function has the same interface as EXAMPLE_LOSS_LAYER_::compute_loss_value_and_gradient() 
+            except it has the additional calling requirements that: 
+                - sub.get_output().k() == 1
+                - sub.get_output().num_samples() == input_tensor.num_samples()
+                - sub.sample_expansion_factor() == 1
+            Also, the loss value returned is roughly equal to the average number of
+            mistakes made per image.  This is the sum of false alarms and missed
+            detections, weighted by the loss weights for these types of mistakes specified
+            in the mmod_options.
+        !*/
+    };
+
+    template <typename SUBNET>
+    using loss_mmod = add_loss_layer<loss_mmod_, SUBNET>;
+
 // ----------------------------------------------------------------------------------------
 
 }
diff --git a/dlib/image_processing/full_object_detection.h b/dlib/image_processing/full_object_detection.h
index 1dfc99b2d8..5c6d112dfa 100644
--- a/dlib/image_processing/full_object_detection.h
+++ b/dlib/image_processing/full_object_detection.h
@@ -183,6 +183,60 @@ namespace dlib
             item.label = "";
     }
 
+// ----------------------------------------------------------------------------------------
+
+    struct yolo_rect
+    {
+        yolo_rect() = default;
+        yolo_rect(const drectangle& r) : rect(r) {}
+        yolo_rect(const drectangle& r, double score) : rect(r),detection_confidence(score) {}
+        yolo_rect(const drectangle& r, double score, const std::string& label) : rect(r),detection_confidence(score), label(label) {}
+        yolo_rect(const mmod_rect& r) : rect(r.rect), detection_confidence(r.detection_confidence), ignore(r.ignore), label(r.label) {}
+
+        drectangle rect;
+        double detection_confidence = 0;
+        bool ignore = false;
+        std::string label;
+        std::vector<std::pair<double, std::string>> labels;
+
+        operator rectangle() const { return rect; }
+        bool operator == (const yolo_rect& rhs) const
+        {
+            return rect == rhs.rect
+                   && detection_confidence == rhs.detection_confidence
+                   && ignore == rhs.ignore
+                   && label == rhs.label;
+        }
+        bool operator<(const yolo_rect& rhs) const
+        {
+            return detection_confidence < rhs.detection_confidence;
+        }
+    };
+
+    inline void serialize(const yolo_rect& item, std::ostream& out)
+    {
+        int version = 1;
+        serialize(version, out);
+        serialize(item.rect, out);
+        serialize(item.detection_confidence, out);
+        serialize(item.ignore, out);
+        serialize(item.label, out);
+        serialize(item.labels, out);
+    }
+
+    inline void deserialize(yolo_rect& item, std::istream& in)
+    {
+        int version = 0;
+        deserialize(version, in);
+        if (version != 1)
+            throw serialization_error("Unexpected version found while deserializing dlib::yolo_rect");
+        deserialize(item.rect, in);
+        deserialize(item.detection_confidence, in);
+        deserialize(item.ignore, in);
+        deserialize(item.label, in);
+        deserialize(item.labels, in);
+    }
+
 // ----------------------------------------------------------------------------------------
 
 }
diff --git a/dlib/image_processing/full_object_detection_abstract.h b/dlib/image_processing/full_object_detection_abstract.h
index 099ee01b0c..411089303d 100644
--- a/dlib/image_processing/full_object_detection_abstract.h
+++ b/dlib/image_processing/full_object_detection_abstract.h
@@ -194,6 +194,50 @@ namespace dlib
         provides serialization support
     !*/
 
+// ----------------------------------------------------------------------------------------
+
+    struct yolo_rect
+    {
+        /*!
+            WHAT THIS OBJECT REPRESENTS
+                This is a simple struct that is used to give training data and receive detections
+                from the YOLO Detection loss layer loss_yolo_ object.
+        !*/
+
+        yolo_rect() = default;
+        yolo_rect(const drectangle& r) : rect(r) {}
+        yolo_rect(const drectangle& r, double score) : rect(r),detection_confidence(score) {}
+        yolo_rect(const drectangle& r, double score, const std::string& label) : rect(r),detection_confidence(score), label(label) {}
+        yolo_rect(const mmod_rect& r) : rect(r.rect), detection_confidence(r.detection_confidence), ignore(r.ignore), label(r.label) {}
+
+        drectangle rect;
+        double detection_confidence = 0;
+        bool ignore = false;
+        std::string label;
+        // YOLO detectors are multi label detectors: this field will contain all confidences and labels for a particular detection
+        std::vector<std::pair<double, std::string>> labels;
+
+        operator rectangle() const { return rect; }
+        bool operator == (const yolo_rect& rhs) const;
+        /*!
+            ensures
+                - returns true if and only if rect == rhs.rect && detection_confidence == rhs.detection_confidence && label == rhs.label.
+        !*/
+
+        bool operator<(const yolo_rect& rhs) const
+        /*!
+            ensures
+                - returns true if and only if detection_confidence < rhs.detection_confidence.
+        !*/
+
+    };
+
+    inline void serialize(const yolo_rect& item, std::ostream& out);
+    inline void deserialize(yolo_rect& item, std::istream& in);
+    /*!
+        provides serialization support
+    !*/
+
 // ----------------------------------------------------------------------------------------
 
 }
diff --git a/dlib/test/test.png b/dlib/test/test.png
new file mode 100644
index 0000000000000000000000000000000000000000..b25fb669289da50e94093b5a6944d54c99bc5a45
GIT binary patch
literal 76
zcmeAS@N?(olHy`uVBq!ia0vp^{6Ngd0VEi%K9n>DQo^1tjv*W~lT#90SZh|qOjA6^
Y&af+jacxM&)@+b6Pgg&ebxsLQ05N6}^#A|>

literal 0
HcmV?d00001

diff --git a/examples/dnn_yolo_train_ex.cpp b/examples/dnn_yolo_train_ex.cpp
index 8e500dcf40..39f2021064 100644
--- a/examples/dnn_yolo_train_ex.cpp
+++ b/examples/dnn_yolo_train_ex.cpp
@@ -1,3 +1,19 @@
+// The contents of this file are in the public domain. See LICENSE_FOR_EXAMPLE_PROGRAMS.txt
+/*
+    This is an example illustrating the use of the deep learning tools from the dlib C++
+    Library.  I'm assuming you have already read the dnn_introduction_ex.cpp, the
+    dnn_introduction2_ex.cpp and the dnn_introduction3_ex.cpp examples.  In this example
+    program we are going to show how one can train a YOLO detector.  In particular, we will train
+    the YOLOv3 model like the one introduced in this paper:
+    "YOLOv3: An Incremental Improvement" by Joseph Redmon and  Ali Farhadi.
+
+    This example program will work with any imglab dataset, such as:
+        - faces: http://dlib.net/files/data/dlib_face_detection_dataset-2016-09-30.tar.gz
+        - vehicles: http://dlib.net/files/data/dlib_rear_end_vehicles_v1.tar
+    Just uncompress the dataset and give the directory containing the training.xml and testing.xml
+    files as an argument to this program.
+*/
+
 #include <dlib/cmd_line_parser.h>
 #include <dlib/data_io.h>
 #include <dlib/dnn.h>
@@ -8,6 +24,9 @@
 using namespace std;
 using namespace dlib;
 
+// In the darknet namespace we define:
+// - the network architecture: DarkNet53 backbone and detection head for YOLO.
+// - a helper function to setup the detector: change the number of classes, etc.
 namespace darknet
 {
     // backbone tags
@@ -64,6 +83,12 @@ namespace darknet
                                                conblock<32, 3, 1,
                                                INPUT>>>>>>>>>>>>>;
 
+        // This is the layer that will be passed to the loss layer to get the detections from the network.
+        // The main thing to pay attention to when defining the YOLO output layer is that it should be
+        // a tag layer, followed by a sigmoid layer and a 1x1 convolution.  The tag layer should be unique
+        // in the whole network definition, as the loss layer will use it to get the outputs.  The number of
+        // filters in the convolutional layer should be (1 + 4 + num_classes) * num_anchors at that output.
+        // The 1 corresponds to the objectness in the loss layer and the 4 to the bounding box coordinates.
         template <long num_classes, long nf, template <typename> class YTAG, template <typename> class NTAG, typename SUBNET>
         using yolo = YTAG<sig<con<3 * (num_classes + 5), 1, 1, 1, 1,
                      conblock<nf, 3, 1,
@@ -105,11 +130,14 @@ namespace darknet
     }
 }
 
+// In this example, YOLO expects square images, and we choose to transform them by letterboxing them.
 rectangle_transform preprocess_image(const matrix<rgb_pixel>& image, matrix<rgb_pixel>& output, const long image_size)
 {
     return rectangle_transform(inv(letterbox_image(image, output, image_size)));
 }
 
+// YOLO outputs the bounding boxes in the coordinate system of the input (letterboxed) image, so we need to convert them
+// back to the original image.
 void postprocess_detections(const rectangle_transform& tform, std::vector<yolo_rect>& detections)
 {
     for (auto& d : detections)
@@ -176,12 +204,12 @@ try
         string_to_color(label.first);
     }
     // When computing the objectness loss in YOLO, predictions that do not have an IoU
-    // with any ground truth box of at least options.truth_match_iou_threshold, will be
+    // with any ground truth box of at least options.iou_ignore_threshold, will be
     // treated as not capable of detecting an object, an therefore incur loss.
-    // Predictions about this threshold will be ignored, i.e. will not affect the loss.
-    options.truth_match_iou_threshold = 0.7;
-    // These are the anchors computed on COCO dataset by the official DarkNet code
-    // using K-Means clustering.
+    // Predictions above this threshold will be ignored, i.e. will not contribute to the
+    // loss. Good values are 0.7 or 0.5.
+    options.iou_ignore_threshold = 0.7;
+    // These are the anchors computed on COCO dataset, presented in the YOLOv3 paper.
     options.add_anchors<darknet::ytag8>({{10, 13}, {16, 30}, {33, 23}});
     options.add_anchors<darknet::ytag16>({{30, 61}, {62, 45}, {59, 119}});
     options.add_anchors<darknet::ytag32>({{116, 90}, {156, 198}, {373, 326}});
@@ -195,6 +223,8 @@ try
     // In case we have several GPUs, we can tell the dnn_trainer to make use of them.
     std::vector<int> gpus(num_gpus);
     iota(gpus.begin(), gpus.end(), 0);
+    // We initialize the trainer here, as it will be used in several contexts, depending on the
+    // arguments passed the the program.
     dnn_trainer<darknet::yolov3_train_type> trainer(net, sgd(0.0005, 0.9), gpus);
     trainer.be_verbose();
     trainer.set_mini_batch_size(batch_size);
@@ -213,7 +243,6 @@ try
             return EXIT_FAILURE;
         }
         const double threshold = get_option(parser, "test", 0.01);
-        net.loss_details().adjust_threshold(threshold);
         image_window win;
         matrix<rgb_pixel> image, resized;
         for (const auto& im : dataset.images)
@@ -223,7 +252,7 @@ try
             win.set_title(im.filename);
             win.set_image(image);
             const auto tform = preprocess_image(image, resized, image_size);
-            auto detections = net(resized);
+            auto detections = net.process(resized, threshold);
             postprocess_detections(tform, detections);
             cout << "# detections: " << detections.size() << endl;
             for (const auto& det : detections)
@@ -236,6 +265,9 @@ try
         return EXIT_SUCCESS;
     }
 
+    // If the training has started and a synchronization file has already been saved to disk,
+    // we can re-run this program with the --map option to compute the mean average precision
+    // on the test set.
     if (parser.option("map"))
     {
         image_dataset_metadata::dataset dataset;
@@ -253,14 +285,13 @@ try
             hits[label] = std::vector<std::pair<double, bool>>();
             missing[label] = 0;
         }
-        net.loss_details().adjust_threshold(0.005);
         cout << "computing mean average precision for " << dataset.images.size() << " images..." << endl;
         for (size_t i = 0; i < dataset.images.size(); ++i)
         {
             const auto& im = dataset.images[i];
             load_image(image, data_directory + "/" + im.filename);
             const auto tform = preprocess_image(image, resized, image_size);
-            auto dets = net(resized);
+            auto dets = net.process(resized, 0.005);
             postprocess_detections(tform, dets);
             std::vector<bool> used(dets.size(), false);
             // true positives: truths matched by detections
@@ -307,7 +338,8 @@ try
 
     // Create some data loaders which will load the data, and perform som data augmentation.
     dlib::pipe<std::pair<matrix<rgb_pixel>, std::vector<yolo_rect>>> train_data(1000);
-    auto loader = [&dataset, &data_directory, &train_data, &image_size](time_t seed) {
+    const auto loader = [&dataset, &data_directory, &train_data, &image_size](time_t seed)
+    {
         dlib::rand rnd(time(nullptr) + seed);
         matrix<rgb_pixel> image, rotated;
         std::pair<matrix<rgb_pixel>, std::vector<yolo_rect>> temp;
@@ -327,6 +359,7 @@ try
             for (const auto& box : dataset.images[idx].boxes)
                 temp.second.emplace_back(box.rect, 1, box.label);
 
+            // We alternate between augmenting the full image and random cropping
             if (rnd.get_random_double() > 0.5)
             {
                 rectangle_transform tform = rotate_image(
@@ -376,6 +409,7 @@ try
             for (const auto& r : temp.second)
             {
                 auto color = string_to_color(r.label);
+                // make semi-transparent and cross-out the ignored boxes
                 if (r.ignore)
                 {
                     color.alpha = 128;

From f6cae91780555bb02e54b7b1c523bbe8aaf12e96 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Adri=C3=A0=20Arrufat?= <adria.arrufat@outlook.com>
Date: Sun, 27 Jun 2021 02:21:34 +0900
Subject: [PATCH 20/62] remove test image added by mistake

---
 dlib/test/test.png | Bin 76 -> 0 bytes
 1 file changed, 0 insertions(+), 0 deletions(-)
 delete mode 100644 dlib/test/test.png

diff --git a/dlib/test/test.png b/dlib/test/test.png
deleted file mode 100644
index b25fb669289da50e94093b5a6944d54c99bc5a45..0000000000000000000000000000000000000000
GIT binary patch
literal 0
HcmV?d00001

literal 76
zcmeAS@N?(olHy`uVBq!ia0vp^{6Ngd0VEi%K9n>DQo^1tjv*W~lT#90SZh|qOjA6^
Y&af+jacxM&)@+b6Pgg&ebxsLQ05N6}^#A|>


From adfbabcb66fab3dba4445c65008d42e9f00e0247 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Adri=C3=A0=20Arrufat?= <adria.arrufat@outlook.com>
Date: Sun, 27 Jun 2021 09:21:53 +0900
Subject: [PATCH 21/62] fix documentation, loss details printing and
 compilation on windows

---
 dlib/dnn/loss.h                |  6 ++++--
 dlib/dnn/loss_abstract.h       | 39 ++++++++++++++++++++++------------
 examples/dnn_yolo_train_ex.cpp |  2 +-
 3 files changed, 31 insertions(+), 16 deletions(-)

diff --git a/dlib/dnn/loss.h b/dlib/dnn/loss.h
index 036e5c268a..f195014dcd 100644
--- a/dlib/dnn/loss.h
+++ b/dlib/dnn/loss.h
@@ -3895,9 +3895,11 @@ namespace dlib
         {
             out << "loss_yolo\t (";
             const auto& opts = item.options;
-            out << "num outputs:" << tag_count();
+            out << tag_count() << " output" << (tag_count() > 1 ? "s" : "") << ":(";
+            list_tags(out);
+            out << ")";
             out << ", anchor boxes:(" << opts.anchors << ")";
-            out << ", labels:(";
+            out << ", " << opts.labels.size() << " label" << (opts.labels.size() > 1 ? "s" : "") << ":(";
             for (size_t i = 0; i < opts.labels.size(); ++i)
             {
                 out << opts.labels[i];
diff --git a/dlib/dnn/loss_abstract.h b/dlib/dnn/loss_abstract.h
index 7aea09777e..d5958090f1 100644
--- a/dlib/dnn/loss_abstract.h
+++ b/dlib/dnn/loss_abstract.h
@@ -1937,10 +1937,10 @@ namespace dlib
 
     public:
 
-        typedef std::vector<mmod_rect> training_label_type;
-        typedef std::vector<mmod_rect> output_label_type;
+        typedef std::vector<yolo_rect> training_label_type;
+        typedef std::vector<yolo_rect> output_label_type;
 
-        loss_mmod_(
+        loss_yolo_(
         );
         /*!
             ensures
@@ -1948,7 +1948,7 @@ namespace dlib
         !*/
 
         loss_yolo_(
-            mmod_options options_
+            yolo_options options_
         );
         /*!
             ensures
@@ -1970,7 +1970,7 @@ namespace dlib
             const tensor& input_tensor,
             const SUB_TYPE& sub,
             label_iterator iter,
-            double adjust_threshold = 0
+            double adjust_threshold = 0.25
         ) const;
         /*!
             This function has the same interface as EXAMPLE_LOSS_LAYER_::to_label() except
@@ -1978,14 +1978,19 @@ namespace dlib
                 - sub.get_output().k() == 1
                 - sub.get_output().num_samples() == input_tensor.num_samples()
                 - sub.sample_expansion_factor() == 1
-            Also, the output labels are std::vectors of mmod_rects where, for each mmod_rect R,
+            Also, the output labels are std::vectors of yolo_rects where, for each yolo_rect R,
             we have the following interpretations:
                 - R.rect == the location of an object in the image.
-                - R.detection_confidence the score for the object, the bigger the score the
-                  more confident the detector is that an object is really there.  Only
+                - R.detection_confidence == the score for the object, between 0 and 1.  Only
                   objects with a detection_confidence > adjust_threshold are output.  So if
                   you want to output more objects (that are also of less confidence) you
                   can call to_label() with a smaller value of adjust_threshold.
+                - R.label == the label of the detected object.
+                - R.labels == a std::vector<double, std::string> containing all the confidence values
+                  and labels that have a detection score > adjust_threshold, since this loss allows
+                  for multi-label outputs.  Note that the followin is true:
+                      - R.labels[0].first == R.detection_confidence
+                      - R.labels[0].second == R.label
                 - R.ignore == false (this value is unused by to_label()).
         !*/
 
@@ -2004,15 +2009,23 @@ namespace dlib
                 - sub.get_output().k() == 1
                 - sub.get_output().num_samples() == input_tensor.num_samples()
                 - sub.sample_expansion_factor() == 1
-            Also, the loss value returned is roughly equal to the average number of
-            mistakes made per image.  This is the sum of false alarms and missed
-            detections, weighted by the loss weights for these types of mistakes specified
-            in the mmod_options.
+            Also, the loss value returned corresponts to the L2 loss.
+        !*/
+
+        void adjust_nms
+        (
+            double iou_thresh,
+            double percent_covered_thresh = 1
+        );
+        /*!
+            ensures
+                - #get_options().overlaps_nms == test_box_overlap(iou_thresh, percent_covered_thresh)
         !*/
+
     };
 
     template <typename SUBNET>
-    using loss_mmod = add_loss_layer<loss_mmod_, SUBNET>;
+    using loss_yolo = add_loss_layer<loss_yolo_, SUBNET>;
 
 // ----------------------------------------------------------------------------------------
 
diff --git a/examples/dnn_yolo_train_ex.cpp b/examples/dnn_yolo_train_ex.cpp
index 39f2021064..c23346809f 100644
--- a/examples/dnn_yolo_train_ex.cpp
+++ b/examples/dnn_yolo_train_ex.cpp
@@ -344,7 +344,7 @@ try
         matrix<rgb_pixel> image, rotated;
         std::pair<matrix<rgb_pixel>, std::vector<yolo_rect>> temp;
         random_cropper cropper;
-        cropper.set_seed(time_t(nullptr) + seed);
+        cropper.set_seed(time(nullptr) + seed);
         cropper.set_chip_dims(image_size, image_size);
         cropper.set_max_object_size(0.9);
         cropper.set_min_object_size(10, 10);

From 23c021ebcd1892d4273c4d1a88dbc8c45883439f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Adri=C3=A0=20Arrufat?= <adria.arrufat@outlook.com>
Date: Sun, 27 Jun 2021 12:48:47 +0900
Subject: [PATCH 22/62] add optional height and width fields to
 image_dataset_metadata::image

---
 dlib/data_io/image_dataset_metadata.cpp | 11 ++++++++++-
 dlib/data_io/image_dataset_metadata.h   |  4 +++-
 2 files changed, 13 insertions(+), 2 deletions(-)

diff --git a/dlib/data_io/image_dataset_metadata.cpp b/dlib/data_io/image_dataset_metadata.cpp
index 390ef6a0a2..e1ca410898 100644
--- a/dlib/data_io/image_dataset_metadata.cpp
+++ b/dlib/data_io/image_dataset_metadata.cpp
@@ -60,7 +60,13 @@ namespace dlib
             fout << "<images>\n";
             for (unsigned long i = 0; i < images.size(); ++i)
             {
-                fout << "  <image file='" << images[i].filename << "'>\n";
+                fout << "  <image file='" << images[i].filename << "'";
+                if (images[i].width != 0 && images[i].height != 0)
+                {
+                    fout << " width='" << images[i].width << "'";
+                    fout << " height='" << images[i].height << "'";
+                }
+                fout << ">\n";
 
                 // save all the boxes
                 for (unsigned long j = 0; j < images[i].boxes.size(); ++j)
@@ -251,6 +257,9 @@ namespace dlib
 
                         if (atts.is_in_list("file")) temp_image.filename = atts["file"];
                         else throw dlib::error("<image> missing required attribute 'file'");
+
+                        if (atts.is_in_list("width")) temp_image.width = sa = atts["width"];
+                        if (atts.is_in_list("height")) temp_image.height = sa = atts["height"];
                     }
 
                     ts.push_back(name);
diff --git a/dlib/data_io/image_dataset_metadata.h b/dlib/data_io/image_dataset_metadata.h
index 3dac29ba68..ef8abe2ca9 100644
--- a/dlib/data_io/image_dataset_metadata.h
+++ b/dlib/data_io/image_dataset_metadata.h
@@ -101,7 +101,7 @@ namespace dlib
         {
             /*!
                 WHAT THIS OBJECT REPRESENTS
-                    This object represents an annotated image.   
+                    This object represents an annotated image.
             !*/
 
             image() {}
@@ -109,6 +109,8 @@ namespace dlib
 
             std::string filename;
             std::vector<box> boxes;
+            unsigned long width = 0;
+            unsigned long height = 0;
         };
 
     // ------------------------------------------------------------------------------------

From 1211bc3b79105bf3f29dcf231a2a84df6f9b0b8c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Adri=C3=A0=20Arrufat?= <adria.arrufat@outlook.com>
Date: Sun, 27 Jun 2021 15:57:03 +0900
Subject: [PATCH 23/62] add simple anchor boxes computation using K-Means
 clustering

---
 examples/dnn_yolo_train_ex.cpp | 50 ++++++++++++++++++++++++++++++++++
 1 file changed, 50 insertions(+)

diff --git a/examples/dnn_yolo_train_ex.cpp b/examples/dnn_yolo_train_ex.cpp
index c23346809f..89eed753f9 100644
--- a/examples/dnn_yolo_train_ex.cpp
+++ b/examples/dnn_yolo_train_ex.cpp
@@ -158,6 +158,7 @@ try
     parser.add_option("test", "test the detector with a threshold (default: 0.01)", 1);
     parser.add_option("visualize", "visualize data augmentation instead of training");
     parser.add_option("map", "compute the mean average precision");
+    parser.add_option("anchors", "compute <arg1> anchor boxes using K-Means", 1);
     parser.set_group_name("Help Options");
     parser.add_option("h", "alias of --help");
     parser.add_option("help", "display this message and exit");
@@ -203,6 +204,55 @@ try
         options.labels.push_back(label.first);
         string_to_color(label.first);
     }
+
+    // If default anchor boxes don't fit well your data, you should recompute them.
+    // Here's an simple way to do it using K-Means clustering.  Note that the approach
+    //  shown below is suboptimal, since it doesn't group the bounding boxes by size.
+    //  Grouping the bounding boxes by size and computing the K-Means on each group
+    //  would make more sense, since each stride of the network is meant to output a
+    //  boxes at a particular size.
+    if (parser.option("anchors"))
+    {
+        const auto num_clusers = std::stoul(parser.option("anchors").argument());
+        std::vector<matrix<double, 2, 1>> samples;
+        // First we need to rescale the bounding boxes to match the image size at training time.
+        for (const auto& image_info : dataset.images)
+        {
+            const auto scale = image_size / std::max<double>(image_info.width, image_info.height);
+            for (const auto& box : image_info.boxes)
+            {
+                matrix<double, 2, 1> sample;
+                sample(0) = box.rect.width() * scale;
+                sample(1) = box.rect.height() * scale;
+                samples.push_back(move(sample));
+            }
+        }
+        // Now we can compute K-Means clustering
+        randomize_samples(samples);
+        cout << "Computing anchors for " << samples.size() << " samples" << endl;
+        std::vector<matrix<double, 2, 1>> anchors;
+        pick_initial_centers(num_clusers, anchors, samples);
+        find_clusters_using_kmeans(samples, anchors);
+        std::sort(anchors.begin(), anchors.end(), [](const auto& a, const auto& b){ return a(0)*a(1) < b(0)*b(1); });
+        for (const auto& c : anchors)
+            cout << round(c(0)) << 'x' << round(c(1)) << endl;
+        // And check the average IoU of the newly computed anchor boxes and the training samples.
+        double average_iou;
+        for (const auto& s : samples)
+        {
+            drectangle sample = centered_drect(dpoint(0, 0), s(0), s(1));
+            double best_iou = 0;
+            for (const auto& a : anchors)
+            {
+                drectangle anchor = centered_drect(dpoint(0, 0), a(0), a(1));
+                best_iou = std::max(best_iou, box_intersection_over_union(sample, anchor));
+            }
+            average_iou += best_iou;
+        }
+        cout << "Average IoU: " << average_iou / samples.size() << endl;
+        return EXIT_SUCCESS;
+    }
+
     // When computing the objectness loss in YOLO, predictions that do not have an IoU
     // with any ground truth box of at least options.iou_ignore_threshold, will be
     // treated as not capable of detecting an object, an therefore incur loss.

From 2499947e735471054a16f4b7c23590613f25d04d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Adri=C3=A0=20Arrufat?= <adria.arrufat@outlook.com>
Date: Sun, 27 Jun 2021 16:03:07 +0900
Subject: [PATCH 24/62] fix comment

---
 examples/dnn_yolo_train_ex.cpp | 11 ++++++-----
 1 file changed, 6 insertions(+), 5 deletions(-)

diff --git a/examples/dnn_yolo_train_ex.cpp b/examples/dnn_yolo_train_ex.cpp
index 89eed753f9..b6fb64eba7 100644
--- a/examples/dnn_yolo_train_ex.cpp
+++ b/examples/dnn_yolo_train_ex.cpp
@@ -205,12 +205,13 @@ try
         string_to_color(label.first);
     }
 
-    // If default anchor boxes don't fit well your data, you should recompute them.
+    // If the default anchor boxes don't fit well your data, you should recompute them.
     // Here's an simple way to do it using K-Means clustering.  Note that the approach
-    //  shown below is suboptimal, since it doesn't group the bounding boxes by size.
-    //  Grouping the bounding boxes by size and computing the K-Means on each group
-    //  would make more sense, since each stride of the network is meant to output a
-    //  boxes at a particular size.
+    // shown below is suboptimal, since it doesn't group the bounding boxes by size.
+    // Grouping the bounding boxes by size and computing the K-Means on each group
+    // would make more sense, since each stride of the network is meant to output a
+    // boxes at a particular size, but that is very specific to the network architecture
+    // and the dataset itself.
     if (parser.option("anchors"))
     {
         const auto num_clusers = std::stoul(parser.option("anchors").argument());

From a2959ce0c9d8061188c10e263e9d37ec7d8e7371 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Adri=C3=A0=20Arrufat?= <adria.arrufat@outlook.com>
Date: Sun, 27 Jun 2021 20:30:17 +0900
Subject: [PATCH 25/62] remove unused variable

---
 dlib/dnn/loss.h | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/dlib/dnn/loss.h b/dlib/dnn/loss.h
index f195014dcd..d10c54928d 100644
--- a/dlib/dnn/loss.h
+++ b/dlib/dnn/loss.h
@@ -3539,7 +3539,6 @@ namespace dlib
             if (tag_count++ > 0)
                 out << ";";
             out << "tag" << tag_id << ":";
-            size_t anchor_count = 0;
             for (size_t a = 0; a < details.size(); ++a)
             {
                 out << details[a].width << "x" << details[a].height;
@@ -3863,6 +3862,8 @@ namespace dlib
             return loss / input_tensor.num_samples();
         }
 
+        const yolo_options& get_options() const { return options; }
+
         void adjust_nms(double iou_thresh, double percent_covered_thresh = 1)
         {
             options.overlaps_nms = test_box_overlap(iou_thresh, percent_covered_thresh);

From 7474ccc690b3a18c7863530ed1ce0e6deef08ecb Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Adri=C3=A0=20Arrufat?= <adria.arrufat@outlook.com>
Date: Sun, 27 Jun 2021 21:13:17 +0900
Subject: [PATCH 26/62] fix maybe uninitialized value

---
 examples/dnn_yolo_train_ex.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/examples/dnn_yolo_train_ex.cpp b/examples/dnn_yolo_train_ex.cpp
index b6fb64eba7..0fff520021 100644
--- a/examples/dnn_yolo_train_ex.cpp
+++ b/examples/dnn_yolo_train_ex.cpp
@@ -238,7 +238,7 @@ try
         for (const auto& c : anchors)
             cout << round(c(0)) << 'x' << round(c(1)) << endl;
         // And check the average IoU of the newly computed anchor boxes and the training samples.
-        double average_iou;
+        double average_iou = 0;
         for (const auto& s : samples)
         {
             drectangle sample = centered_drect(dpoint(0, 0), s(0), s(1));

From c96b9de5cd0982d7e2047ad8f7a99af5b64ba0c5 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Adri=C3=A0=20Arrufat?= <adria.arrufat@outlook.com>
Date: Sun, 27 Jun 2021 21:17:26 +0900
Subject: [PATCH 27/62] fix sign compare warning

---
 dlib/dnn/loss.h | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/dlib/dnn/loss.h b/dlib/dnn/loss.h
index d10c54928d..7614d56746 100644
--- a/dlib/dnn/loss.h
+++ b/dlib/dnn/loss.h
@@ -3617,7 +3617,7 @@ namespace dlib
                 DLIB_CASSERT(sub.sample_expansion_factor() == 1, sub.sample_expansion_factor());
                 const auto& anchors = options.anchors.at(tag_id<TAG_TYPE>::id);
                 const tensor& output_tensor = layer<TAG_TYPE>(sub).get_output();
-                DLIB_CASSERT(output_tensor.k() == anchors.size() * (options.labels.size() + 5));
+                DLIB_CASSERT(static_cast<size_t>(output_tensor.k()) == anchors.size() * (options.labels.size() + 5));
                 const auto stride_x = static_cast<double>(input_tensor.nc()) / output_tensor.nc();
                 const auto stride_y = static_cast<double>(input_tensor.nr()) / output_tensor.nr();
                 const long num_feats = output_tensor.k() / anchors.size();
@@ -3675,7 +3675,7 @@ namespace dlib
                 DLIB_CASSERT(sub.sample_expansion_factor() == 1, sub.sample_expansion_factor());
                 const tensor& output_tensor = layer<TAG_TYPE>(sub).get_output();
                 const auto& anchors = options.anchors.at(tag_id<TAG_TYPE>::id);
-                DLIB_CASSERT(output_tensor.k() == anchors.size() * (options.labels.size() + 5));
+                DLIB_CASSERT(static_cast<size_t>(output_tensor.k()) == anchors.size() * (options.labels.size() + 5));
                 const auto stride_x = static_cast<double>(input_tensor.nc()) / output_tensor.nc();
                 const auto stride_y = static_cast<double>(input_tensor.nr()) / output_tensor.nr();
                 const long num_feats = output_tensor.k() / anchors.size();

From 9becb23308bc027eb9f87c2e9ce53ac0bf671833 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Adri=C3=A0=20Arrufat?= <adria.arrufat@outlook.com>
Date: Mon, 28 Jun 2021 10:00:26 +0900
Subject: [PATCH 28/62] print iou_ignore_threshold

---
 dlib/dnn/loss.h | 1 +
 1 file changed, 1 insertion(+)

diff --git a/dlib/dnn/loss.h b/dlib/dnn/loss.h
index 7614d56746..225ef3d12f 100644
--- a/dlib/dnn/loss.h
+++ b/dlib/dnn/loss.h
@@ -3908,6 +3908,7 @@ namespace dlib
                     out << ",";
             }
             out << ")";
+            out << ", iou_ignore_threshold: " << opts.iou_ignore_threshold;
             out << ", lambda_obj:" << opts.lambda_obj;
             out << ", lambda_box:" << opts.lambda_box;
             out << ", lambda_cls:" << opts.lambda_cls;

From c63738a3be9d50e3e32f3ee056a739342f1a171b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Adri=C3=A0=20Arrufat?= <adria.arrufat@outlook.com>
Date: Mon, 28 Jun 2021 11:20:59 +0900
Subject: [PATCH 29/62] remove unused field

---
 dlib/dnn/loss.h | 3 ---
 1 file changed, 3 deletions(-)

diff --git a/dlib/dnn/loss.h b/dlib/dnn/loss.h
index 225ef3d12f..24c9d47021 100644
--- a/dlib/dnn/loss.h
+++ b/dlib/dnn/loss.h
@@ -3486,7 +3486,6 @@ namespace dlib
         // map between the stride and the anchor boxes
         std::map<int, std::vector<anchor_box_details>> anchors;
         std::vector<std::string> labels;
-        double confidence_threshold = 0.25;
         double iou_ignore_threshold = 0.7;
         bool classwise_nms = false;
         test_box_overlap overlaps_nms = test_box_overlap(0.45, 1.0);
@@ -3502,7 +3501,6 @@ namespace dlib
         serialize(version, out);
         serialize(item.anchors, out);
         serialize(item.labels, out);
-        serialize(item.confidence_threshold, out);
         serialize(item.iou_ignore_threshold, out);
         serialize(item.classwise_nms, out);
         serialize(item.overlaps_nms, out);
@@ -3519,7 +3517,6 @@ namespace dlib
             throw serialization_error("Unexpected version found while deserializing dlib::yolo_options.");
         deserialize(item.anchors, in);
         deserialize(item.labels, in);
-        deserialize(item.confidence_threshold, in);
         deserialize(item.iou_ignore_threshold, in);
         deserialize(item.classwise_nms, in);
         deserialize(item.overlaps_nms, in);

From 9a0f0a346abc94b5d37a64b21d6e11aca53ca030 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Adri=C3=A0=20Arrufat?= <adria.arrufat@outlook.com>
Date: Mon, 28 Jun 2021 13:05:34 +0900
Subject: [PATCH 30/62] improve loss formatting

---
 dlib/dnn/loss.h | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/dlib/dnn/loss.h b/dlib/dnn/loss.h
index 24c9d47021..c15ecafbbd 100644
--- a/dlib/dnn/loss.h
+++ b/dlib/dnn/loss.h
@@ -3599,7 +3599,7 @@ namespace dlib
         {
             constexpr static size_t tag_count() { return 1; }
 
-            static void list_tags(std::ostream& out) { out << tag_id<TAG_TYPE>::id; }
+            static void list_tags(std::ostream& out) { out << "tag" << tag_id<TAG_TYPE>::id; }
 
             template <typename SUBNET>
             static void tensor_to_dets (
@@ -3893,11 +3893,11 @@ namespace dlib
         {
             out << "loss_yolo\t (";
             const auto& opts = item.options;
-            out << tag_count() << " output" << (tag_count() > 1 ? "s" : "") << ":(";
+            out << tag_count() << " output" << (tag_count() != 1 ? "s" : "") << ":(";
             list_tags(out);
             out << ")";
             out << ", anchor boxes:(" << opts.anchors << ")";
-            out << ", " << opts.labels.size() << " label" << (opts.labels.size() > 1 ? "s" : "") << ":(";
+            out << ", " << opts.labels.size() << " label" << (opts.labels.size() != 1 ? "s" : "") << ":(";
             for (size_t i = 0; i < opts.labels.size(); ++i)
             {
                 out << opts.labels[i];

From 1fbc41bcd8f125a05a1e39fd8c977e97662f9367 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Adri=C3=A0=20Arrufat?= <adria.arrufat@outlook.com>
Date: Mon, 28 Jun 2021 17:01:23 +0900
Subject: [PATCH 31/62] fix tag output formatting

---
 dlib/dnn/loss.h | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/dlib/dnn/loss.h b/dlib/dnn/loss.h
index c15ecafbbd..358dbcd9da 100644
--- a/dlib/dnn/loss.h
+++ b/dlib/dnn/loss.h
@@ -3558,7 +3558,7 @@ namespace dlib
 
             static void list_tags(std::ostream& out)
             {
-                out << tag_id<TAG_TYPE>::id << (tag_count() > 1 ? "," : "");
+                out << "tag" << tag_id<TAG_TYPE>::id << (tag_count() > 1 ? "," : "");
                 yolo_helper_impl<TAG_TYPES...>::list_tags(out);
             }
 

From 3fba65fae39bb3d111a47a4905517082262a9bd2 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Adri=C3=A0=20Arrufat?= <adria.arrufat@outlook.com>
Date: Fri, 2 Jul 2021 12:18:40 +0900
Subject: [PATCH 32/62] fix documentation and add classwise to adjust_nms

---
 dlib/dnn/loss.h          |  3 ++-
 dlib/dnn/loss_abstract.h | 26 ++++++++++++++------------
 2 files changed, 16 insertions(+), 13 deletions(-)

diff --git a/dlib/dnn/loss.h b/dlib/dnn/loss.h
index 358dbcd9da..bb86aba692 100644
--- a/dlib/dnn/loss.h
+++ b/dlib/dnn/loss.h
@@ -3861,9 +3861,10 @@ namespace dlib
 
         const yolo_options& get_options() const { return options; }
 
-        void adjust_nms(double iou_thresh, double percent_covered_thresh = 1)
+        void adjust_nms(double iou_thresh, double percent_covered_thresh = 1, bool classwise = false)
         {
             options.overlaps_nms = test_box_overlap(iou_thresh, percent_covered_thresh);
+            options.classwise_nms = classwise;
         }
 
         friend void serialize(const loss_yolo_& item, std::ostream& out)
diff --git a/dlib/dnn/loss_abstract.h b/dlib/dnn/loss_abstract.h
index d5958090f1..1f3e068100 100644
--- a/dlib/dnn/loss_abstract.h
+++ b/dlib/dnn/loss_abstract.h
@@ -1876,7 +1876,7 @@ namespace dlib
         yolo_options() = default;
 
         // This kind of object detector is a multi-scale object detector with bounding box regression
-        // for anchor boxes.  The anchors field determinase which anchors will be used at the output
+        // for anchor boxes.  The anchors field determines which anchors will be used at the output
         // pointed by the tag layer whose id is the key of the map.
         std::unordered_map<int, std::vector<anchor_box_details>> anchors;
 
@@ -1886,19 +1886,19 @@ namespace dlib
         );
         /*!
             ensures
-                - acnhors.at(tag_id<TAG_TYPE>::id) == boxes
+                - anchors.at(tag_id<TAG_TYPE>::id) == boxes
         !*/
 
         // This field contains the labels of all the possible object this detector can find.
         std::vector<std::string> labels;
         // When computing the objectness loss, any detection that has an IoU above iou_ignore_threshold
-        // will be ignored and not incur any loss
+        // with a ground truth box will not incur any loss.
         double iou_ignore_threshold = 0.7;
-        // Tell the non-max Supression whether to take classes into account
-        bool classwise_nms = false;
         // When doing non-max suppression, we use overlaps_nms to decide if a box overlaps
         // an already output detection and should therefore be thrown out.
         test_box_overlap overlaps_nms = test_box_overlap(0.45, 1.0);
+        // Tell the non-max Suppression whether to take classes into account
+        bool classwise_nms = false;
         // These parameters control how we penalize different kinds of mistakes: notably the objectness loss,
         // the box (bounding box regression) loss, and the class loss.
         double lambda_obj = 1.0;
@@ -1931,7 +1931,7 @@ namespace dlib
 
                 Where num_classes is the number of categories that the detector is trained on,
                 and num_anchors is the number of priors or anchor boxes at the output pointed
-                by the tag layer. The number 5 corresponds to the objectness and the 4 coordinates
+                by the tag layer. The number 5 corresponds to the objectness plus the 4 coordinates
                 for performing bounding box regression.
         !*/
 
@@ -1974,8 +1974,8 @@ namespace dlib
         ) const;
         /*!
             This function has the same interface as EXAMPLE_LOSS_LAYER_::to_label() except
-            it has the additional calling requirements that: 
-                - sub.get_output().k() == 1
+            it has the additional calling requirements that:
+                - layer<TAG_TYPE>(sub).get_output().k() == options.anchors.at(tag_id<TAG_TYPE>::id).size() * (5 + options.labels.size());
                 - sub.get_output().num_samples() == input_tensor.num_samples()
                 - sub.sample_expansion_factor() == 1
             Also, the output labels are std::vectors of yolo_rects where, for each yolo_rect R,
@@ -1988,7 +1988,7 @@ namespace dlib
                 - R.label == the label of the detected object.
                 - R.labels == a std::vector<double, std::string> containing all the confidence values
                   and labels that have a detection score > adjust_threshold, since this loss allows
-                  for multi-label outputs.  Note that the followin is true:
+                  for multi-label outputs.  Note that the following is true:
                       - R.labels[0].first == R.detection_confidence
                       - R.labels[0].second == R.label
                 - R.ignore == false (this value is unused by to_label()).
@@ -2006,20 +2006,22 @@ namespace dlib
         /*!
             This function has the same interface as EXAMPLE_LOSS_LAYER_::compute_loss_value_and_gradient() 
             except it has the additional calling requirements that: 
-                - sub.get_output().k() == 1
+                - layer<TAG_TYPE>(sub).get_output().k() == options.anchors.at(tag_id<TAG_TYPE>::id).size() * (5 + options.labels.size());
                 - sub.get_output().num_samples() == input_tensor.num_samples()
                 - sub.sample_expansion_factor() == 1
-            Also, the loss value returned corresponts to the L2 loss.
+            Also, the loss value returned corresponds to the L2 loss.
         !*/
 
         void adjust_nms
         (
             double iou_thresh,
-            double percent_covered_thresh = 1
+            double percent_covered_thresh = 1,
+            bool classwise = false
         );
         /*!
             ensures
                 - #get_options().overlaps_nms == test_box_overlap(iou_thresh, percent_covered_thresh)
+                - #get_options().classwise_nms == classwise
         !*/
 
     };

From 01157787a500ad5921a0035b4a6491aefbf9363c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Adri=C3=A0=20Arrufat?= <adria.arrufat@outlook.com>
Date: Fri, 2 Jul 2021 17:25:57 +0900
Subject: [PATCH 33/62] minor formatting fix in loss

---
 dlib/dnn/loss.h | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/dlib/dnn/loss.h b/dlib/dnn/loss.h
index bb86aba692..2085abf427 100644
--- a/dlib/dnn/loss.h
+++ b/dlib/dnn/loss.h
@@ -3897,7 +3897,7 @@ namespace dlib
             out << tag_count() << " output" << (tag_count() != 1 ? "s" : "") << ":(";
             list_tags(out);
             out << ")";
-            out << ", anchor boxes:(" << opts.anchors << ")";
+            out << ", anchor_boxes:(" << opts.anchors << ")";
             out << ", " << opts.labels.size() << " label" << (opts.labels.size() != 1 ? "s" : "") << ":(";
             for (size_t i = 0; i < opts.labels.size(); ++i)
             {

From 9723dd46c4088b991f16db83f88992dc286479b0 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Adri=C3=A0=20Arrufat?= <adria.arrufat@outlook.com>
Date: Sat, 3 Jul 2021 23:01:28 +0900
Subject: [PATCH 34/62] add option to allow objects to fall outside of the
 crops output by the random_cropper

---
 dlib/image_transforms/random_cropper.h        | 19 ++++++++++++++++---
 .../random_cropper_abstract.h                 | 18 ++++++++++++++++++
 examples/dnn_yolo_train_ex.cpp                |  1 +
 3 files changed, 35 insertions(+), 3 deletions(-)

diff --git a/dlib/image_transforms/random_cropper.h b/dlib/image_transforms/random_cropper.h
index f19f9cc0ef..7c3543007a 100644
--- a/dlib/image_transforms/random_cropper.h
+++ b/dlib/image_transforms/random_cropper.h
@@ -23,6 +23,7 @@ namespace dlib
         double max_object_size = 0.7; // cropped object will be at most this fraction of the size of the image.
         double background_crops_fraction = 0.5;
         double translate_amount = 0.10;
+        double min_object_coverage = 1.0;
 
         std::mutex rnd_mutex;
         dlib::rand rnd;
@@ -104,6 +105,16 @@ namespace dlib
             max_object_size = value; 
         }
 
+        double get_min_object_coverage (
+        ) const { return min_object_coverage; }
+        void set_min_object_coverage (
+            double value
+        )
+        {
+            DLIB_CASSERT(0 < value && value <= 1);
+            min_object_coverage = value;
+        }
+
         template <
             typename array_type,
             typename rectangle_type
@@ -207,12 +218,14 @@ namespace dlib
                 // map to crop
                 rect.rect = tform(rect.rect);
 
+                const double intersection = get_rect(crop).intersect(rect.rect).area();
+
                 // if the rect is at least partly in the crop
-                if (get_rect(crop).intersect(rect.rect).area() != 0)
+                if (intersection != 0)
                 {
                     // set to ignore if not totally in the crop or if too small.
-                    if (!get_rect(crop).contains(rect.rect) || 
-                        ((long)rect.rect.height() < min_object_length_long_dim  && (long)rect.rect.width() < min_object_length_long_dim) || 
+                    if (intersection / rect.rect.area() < min_object_coverage ||
+                        ((long)rect.rect.height() < min_object_length_long_dim  && (long)rect.rect.width() < min_object_length_long_dim) ||
                         ((long)rect.rect.height() < min_object_length_short_dim || (long)rect.rect.width() < min_object_length_short_dim))
                     {
                         rect.ignore = true;
diff --git a/dlib/image_transforms/random_cropper_abstract.h b/dlib/image_transforms/random_cropper_abstract.h
index 065b34a3d2..6304905d1a 100644
--- a/dlib/image_transforms/random_cropper_abstract.h
+++ b/dlib/image_transforms/random_cropper_abstract.h
@@ -199,6 +199,24 @@ namespace dlib
                 - #get_max_object_size() == value
         !*/
 
+        double get_min_object_coverage (
+        ) const;
+        /*!
+            ensures
+                - When a chip is extracted, any object whose area covered is smaller than
+                  get_min_object_coverage() will have its ignore field set to true.
+        !*/
+
+        void set_min_object_coverage (
+            double value
+        );
+        /*!
+            requires
+                - 0 < value <= 1
+            ensures
+                - #get_min_object_coverage() == value
+        !*/
+
         template <
             typename array_type,
             typename rectangle_type
diff --git a/examples/dnn_yolo_train_ex.cpp b/examples/dnn_yolo_train_ex.cpp
index 0fff520021..76b396ccea 100644
--- a/examples/dnn_yolo_train_ex.cpp
+++ b/examples/dnn_yolo_train_ex.cpp
@@ -403,6 +403,7 @@ try
         cropper.set_translate_amount(0.5);
         cropper.set_randomly_flip(true);
         cropper.set_background_crops_fraction(0);
+        cropper.set_min_object_coverage(0.8);
         while (train_data.is_enabled())
         {
             const auto idx = rnd.get_random_32bit_number() % dataset.images.size();

From 3c0e6c212d96b25fff74e8a7201bf0aa6cbbffe9 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Adri=C3=A0=20Arrufat?= <adria.arrufat@outlook.com>
Date: Tue, 6 Jul 2021 12:31:09 +0900
Subject: [PATCH 35/62] add iou_anchor_threshold

This allows changing the behavior of the network while training.
Setting it to 1 (disable) will have the standard YOLO behavior:
match only the best anchor with a truth.  However, setting it to
a lower value will make any anchor with an IoU > iou_anchor_threshold
with a ground truth be updated as well.
See this thread for more details:
https://github.com/AlexeyAB/darknet/issues/7087
---
 dlib/dnn/loss.h                | 102 +++++++++++++++++++++------------
 dlib/dnn/loss_abstract.h       |  20 +++++--
 examples/dnn_yolo_train_ex.cpp |   3 +
 3 files changed, 82 insertions(+), 43 deletions(-)

diff --git a/dlib/dnn/loss.h b/dlib/dnn/loss.h
index 2085abf427..bfa03eab72 100644
--- a/dlib/dnn/loss.h
+++ b/dlib/dnn/loss.h
@@ -3487,8 +3487,9 @@ namespace dlib
         std::map<int, std::vector<anchor_box_details>> anchors;
         std::vector<std::string> labels;
         double iou_ignore_threshold = 0.7;
-        bool classwise_nms = false;
+        double iou_anchor_threshold = 1.0;
         test_box_overlap overlaps_nms = test_box_overlap(0.45, 1.0);
+        bool classwise_nms = false;
         double lambda_obj = 1.0;
         double lambda_box = 1.0;
         double lambda_cls = 1.0;
@@ -3502,6 +3503,7 @@ namespace dlib
         serialize(item.anchors, out);
         serialize(item.labels, out);
         serialize(item.iou_ignore_threshold, out);
+        serialize(item.iou_anchor_threshold, out);
         serialize(item.classwise_nms, out);
         serialize(item.overlaps_nms, out);
         serialize(item.lambda_obj, out);
@@ -3518,6 +3520,7 @@ namespace dlib
         deserialize(item.anchors, in);
         deserialize(item.labels, in);
         deserialize(item.iou_ignore_threshold, in);
+        deserialize(item.iou_anchor_threshold, in);
         deserialize(item.classwise_nms, in);
         deserialize(item.overlaps_nms, in);
         deserialize(item.lambda_obj, in);
@@ -3745,46 +3748,70 @@ namespace dlib
                         }
                     }
 
-                    // Skip if the best anchor is not from the current stride
-                    if (best_tag_id != tag_id<TAG_TYPE>::id)
-                        continue;
+                    // Helper function to compute objectness, bounding box regression and classification gradients
+                    const auto compute_grads = [&] (const size_t anchor_idx)
+                    {
+                        const long c = t_center.x() / stride_x;
+                        const long r = t_center.y() / stride_y;
+                        const auto x_idx = tensor_index(output_tensor, n, anchor_idx * num_feats + 0, r, c);
+                        const auto y_idx = tensor_index(output_tensor, n, anchor_idx * num_feats + 1, r, c);
+                        const auto w_idx = tensor_index(output_tensor, n, anchor_idx * num_feats + 2, r, c);
+                        const auto h_idx = tensor_index(output_tensor, n, anchor_idx * num_feats + 3, r, c);
+                        const auto o_idx = tensor_index(output_tensor, n, anchor_idx * num_feats + 4, r, c);
+
+                        // This grid cell should detect an object
+                        g[o_idx] = options.lambda_obj * (out_data[o_idx] - 1);
+
+                        // Get the truth box target values
+                        const double tx = t_center.x() / stride_x - c;
+                        const double ty = t_center.y() / stride_y - r;
+                        const double tw = truth_box.rect.width() / (anchors[anchor_idx].width + truth_box.rect.width());
+                        const double th = truth_box.rect.height() / (anchors[anchor_idx].height + truth_box.rect.height());
+
+                        // Scale regression error according to the truth size
+                        const double scale_box = 2 - truth_box.rect.area() / (input_tensor.nr() * input_tensor.nc());
+
+                        // Compute the gradient for the box coordinates
+                        g[x_idx] = options.lambda_box * scale_box * (out_data[x_idx] - tx);
+                        g[y_idx] = options.lambda_box * scale_box * (out_data[y_idx] - ty);
+                        g[w_idx] = options.lambda_box * scale_box * (out_data[w_idx] - tw);
+                        g[h_idx] = options.lambda_box * scale_box * (out_data[h_idx] - th);
+
+                        // Compute the classification error
+                        for (long k = 0; k < num_classes; ++k)
+                        {
+                            const auto c_idx = tensor_index(output_tensor, n, anchor_idx * num_feats + 5 + k, r, c);
+                            if (truth_box.label == options.labels[k])
+                                g[c_idx] = options.lambda_cls * (out_data[c_idx] - 1);
+                            else
+                                g[c_idx] = options.lambda_cls * out_data[c_idx];
+                        }
+                    };
 
-                    const long c = t_center.x() / stride_x;
-                    const long r = t_center.y() / stride_y;
-                    const auto x_idx = tensor_index(output_tensor, n, best_a * num_feats + 0, r, c);
-                    const auto y_idx = tensor_index(output_tensor, n, best_a * num_feats + 1, r, c);
-                    const auto w_idx = tensor_index(output_tensor, n, best_a * num_feats + 2, r, c);
-                    const auto h_idx = tensor_index(output_tensor, n, best_a * num_feats + 3, r, c);
-                    const auto o_idx = tensor_index(output_tensor, n, best_a * num_feats + 4, r, c);
-
-                    // This grid cell should detect an object
-                    g[o_idx] = options.lambda_obj * (out_data[o_idx] - 1);
-
-                    // Get the truth box target values
-                    const double tx = t_center.x() / stride_x - c;
-                    const double ty = t_center.y() / stride_y - r;
-                    const double tw = truth_box.rect.width() / (anchors[best_a].width + truth_box.rect.width());
-                    const double th = truth_box.rect.height() / (anchors[best_a].height + truth_box.rect.height());
-
-                    // Scale regression error according to the truth size
-                    const double scale_box = 2 - truth_box.rect.area() / (input_tensor.nr() * input_tensor.nc());
-
-                    // Compute the gradient for the box coordinates
-                    g[x_idx] = options.lambda_box * scale_box * (out_data[x_idx] - tx);
-                    g[y_idx] = options.lambda_box * scale_box * (out_data[y_idx] - ty);
-                    g[w_idx] = options.lambda_box * scale_box * (out_data[w_idx] - tw);
-                    g[h_idx] = options.lambda_box * scale_box * (out_data[h_idx] - th);
-
-                    // Compute the classification error
-                    for (long k = 0; k < num_classes; ++k)
+                    // Update the best anchor if it's from the current stride
+                    if (best_tag_id == tag_id<TAG_TYPE>::id)
                     {
-                        const auto c_idx = tensor_index(output_tensor, n, best_a * num_feats + 5 + k, r, c);
-                        if (truth_box.label == options.labels[k])
-                            g[c_idx] = options.lambda_cls * (out_data[c_idx] - 1);
-                        else
-                            g[c_idx] = options.lambda_cls * out_data[c_idx];
+                        compute_grads(best_a);
+                    }
+
+                    // Update other anchors according to IoU threshold option
+                    if (options.iou_anchor_threshold < 1.0)
+                    {
+                        for (size_t a = 0; a < anchors.size(); ++a)
+                        {
+                            // Skip if we already updated this anchor
+                            if (best_tag_id == tag_id<TAG_TYPE>::id && a == best_a)
+                                continue;
+
+                            const yolo_rect anchor(centered_drect(t_center, anchors[a].width, anchors[a].height));
+                            const double iou = box_intersection_over_union(truth_box.rect, anchor.rect);
+                            if (iou > options.iou_anchor_threshold)
+                                compute_grads(a);
+                        }
                     }
                 }
+
+                // Compute the L2 loss
                 loss += length_squared(rowm(mat(grad), n));
             }
         };
@@ -3907,6 +3934,7 @@ namespace dlib
             }
             out << ")";
             out << ", iou_ignore_threshold: " << opts.iou_ignore_threshold;
+            out << ", iou_anchor_threshold: " << opts.iou_anchor_threshold;
             out << ", lambda_obj:" << opts.lambda_obj;
             out << ", lambda_box:" << opts.lambda_box;
             out << ", lambda_cls:" << opts.lambda_cls;
diff --git a/dlib/dnn/loss_abstract.h b/dlib/dnn/loss_abstract.h
index 1f3e068100..4eeb90fd38 100644
--- a/dlib/dnn/loss_abstract.h
+++ b/dlib/dnn/loss_abstract.h
@@ -1875,9 +1875,9 @@ namespace dlib
 
         yolo_options() = default;
 
-        // This kind of object detector is a multi-scale object detector with bounding box regression
-        // for anchor boxes.  The anchors field determines which anchors will be used at the output
-        // pointed by the tag layer whose id is the key of the map.
+        // This kind of object detector is a multi-scale object detector with bounding box
+        // regression for anchor boxes.  The anchors field determines which anchors will be
+        // used at the output pointed by the tag layer whose id is the key of the map.
         std::unordered_map<int, std::vector<anchor_box_details>> anchors;
 
         template <template <typename> class TAG_TYPE>
@@ -1891,16 +1891,24 @@ namespace dlib
 
         // This field contains the labels of all the possible object this detector can find.
         std::vector<std::string> labels;
-        // When computing the objectness loss, any detection that has an IoU above iou_ignore_threshold
-        // with a ground truth box will not incur any loss.
+        // When computing the objectness loss, any detection that has an IoU above
+        // iou_ignore_threshold with a ground truth box will not incur any loss.
         double iou_ignore_threshold = 0.7;
+        // When computing the YOLO loss (objectness + bounding box regression + classification),
+        // only the best match between a truth and an anchor are used.  Setting iou_anchor_threshold
+        // to 1 will make the model use only the best anchor for each ground truth, so
+        // other anchors can be used for other ground truths in the same cell (useful for
+        // detecting objects in crowds).  However, if the other anchors have an IoU with a
+        // truth above iou_anchor_threshold, they will also be updated.  This will match
+        // all anchors whose iou with a ground truth is above iou_anchor_threshold.
+        double iou_anchor_threshold = 1.0;
         // When doing non-max suppression, we use overlaps_nms to decide if a box overlaps
         // an already output detection and should therefore be thrown out.
         test_box_overlap overlaps_nms = test_box_overlap(0.45, 1.0);
         // Tell the non-max Suppression whether to take classes into account
         bool classwise_nms = false;
         // These parameters control how we penalize different kinds of mistakes: notably the objectness loss,
-        // the box (bounding box regression) loss, and the class loss.
+        // the box (bounding box regression) loss, and the classification loss.
         double lambda_obj = 1.0;
         double lambda_box = 1.0;
         double lambda_cls = 1.0;
diff --git a/examples/dnn_yolo_train_ex.cpp b/examples/dnn_yolo_train_ex.cpp
index 76b396ccea..a642c0d264 100644
--- a/examples/dnn_yolo_train_ex.cpp
+++ b/examples/dnn_yolo_train_ex.cpp
@@ -260,6 +260,9 @@ try
     // Predictions above this threshold will be ignored, i.e. will not contribute to the
     // loss. Good values are 0.7 or 0.5.
     options.iou_ignore_threshold = 0.7;
+    // By setting this to a value < 1, we are telling the model to update all the predictions
+    // as long as the anchor box has an IoU > 0.2 with a ground truth.
+    options.iou_anchor_threshold = 0.2;
     // These are the anchors computed on COCO dataset, presented in the YOLOv3 paper.
     options.add_anchors<darknet::ytag8>({{10, 13}, {16, 30}, {33, 23}});
     options.add_anchors<darknet::ytag16>({{30, 61}, {62, 45}, {59, 119}});

From 8a7ebb63a34b788c98341ff2df70f915c52ce732 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Adri=C3=A0=20Arrufat?= <adria.arrufat@outlook.com>
Date: Tue, 6 Jul 2021 22:12:54 +0900
Subject: [PATCH 36/62] simplify anchor update logic

---
 dlib/dnn/loss.h | 89 +++++++++++++++++++------------------------------
 1 file changed, 35 insertions(+), 54 deletions(-)

diff --git a/dlib/dnn/loss.h b/dlib/dnn/loss.h
index bfa03eab72..4b804d48a8 100644
--- a/dlib/dnn/loss.h
+++ b/dlib/dnn/loss.h
@@ -3748,65 +3748,46 @@ namespace dlib
                         }
                     }
 
-                    // Helper function to compute objectness, bounding box regression and classification gradients
-                    const auto compute_grads = [&] (const size_t anchor_idx)
+                    for (size_t a = 0; a < anchors.size(); ++a)
                     {
-                        const long c = t_center.x() / stride_x;
-                        const long r = t_center.y() / stride_y;
-                        const auto x_idx = tensor_index(output_tensor, n, anchor_idx * num_feats + 0, r, c);
-                        const auto y_idx = tensor_index(output_tensor, n, anchor_idx * num_feats + 1, r, c);
-                        const auto w_idx = tensor_index(output_tensor, n, anchor_idx * num_feats + 2, r, c);
-                        const auto h_idx = tensor_index(output_tensor, n, anchor_idx * num_feats + 3, r, c);
-                        const auto o_idx = tensor_index(output_tensor, n, anchor_idx * num_feats + 4, r, c);
-
-                        // This grid cell should detect an object
-                        g[o_idx] = options.lambda_obj * (out_data[o_idx] - 1);
-
-                        // Get the truth box target values
-                        const double tx = t_center.x() / stride_x - c;
-                        const double ty = t_center.y() / stride_y - r;
-                        const double tw = truth_box.rect.width() / (anchors[anchor_idx].width + truth_box.rect.width());
-                        const double th = truth_box.rect.height() / (anchors[anchor_idx].height + truth_box.rect.height());
-
-                        // Scale regression error according to the truth size
-                        const double scale_box = 2 - truth_box.rect.area() / (input_tensor.nr() * input_tensor.nc());
-
-                        // Compute the gradient for the box coordinates
-                        g[x_idx] = options.lambda_box * scale_box * (out_data[x_idx] - tx);
-                        g[y_idx] = options.lambda_box * scale_box * (out_data[y_idx] - ty);
-                        g[w_idx] = options.lambda_box * scale_box * (out_data[w_idx] - tw);
-                        g[h_idx] = options.lambda_box * scale_box * (out_data[h_idx] - th);
-
-                        // Compute the classification error
-                        for (long k = 0; k < num_classes; ++k)
+                        // Update best anchor if it's from the current stride, and optionally other anchors
+                        if ((best_tag_id == tag_id<TAG_TYPE>::id && best_a = a) || options.iou_anchor_threshold < 1)
                         {
-                            const auto c_idx = tensor_index(output_tensor, n, anchor_idx * num_feats + 5 + k, r, c);
-                            if (truth_box.label == options.labels[k])
-                                g[c_idx] = options.lambda_cls * (out_data[c_idx] - 1);
-                            else
-                                g[c_idx] = options.lambda_cls * out_data[c_idx];
-                        }
-                    };
+                            const long c = t_center.x() / stride_x;
+                            const long r = t_center.y() / stride_y;
+                            const auto x_idx = tensor_index(output_tensor, n, a * num_feats + 0, r, c);
+                            const auto y_idx = tensor_index(output_tensor, n, a * num_feats + 1, r, c);
+                            const auto w_idx = tensor_index(output_tensor, n, a * num_feats + 2, r, c);
+                            const auto h_idx = tensor_index(output_tensor, n, a * num_feats + 3, r, c);
+                            const auto o_idx = tensor_index(output_tensor, n, a * num_feats + 4, r, c);
 
-                    // Update the best anchor if it's from the current stride
-                    if (best_tag_id == tag_id<TAG_TYPE>::id)
-                    {
-                        compute_grads(best_a);
-                    }
+                            // This grid cell should detect an object
+                            g[o_idx] = options.lambda_obj * (out_data[o_idx] - 1);
 
-                    // Update other anchors according to IoU threshold option
-                    if (options.iou_anchor_threshold < 1.0)
-                    {
-                        for (size_t a = 0; a < anchors.size(); ++a)
-                        {
-                            // Skip if we already updated this anchor
-                            if (best_tag_id == tag_id<TAG_TYPE>::id && a == best_a)
-                                continue;
+                            // Get the truth box target values
+                            const double tx = t_center.x() / stride_x - c;
+                            const double ty = t_center.y() / stride_y - r;
+                            const double tw = truth_box.rect.width() / (anchors[a].width + truth_box.rect.width());
+                            const double th = truth_box.rect.height() / (anchors[a].height + truth_box.rect.height());
 
-                            const yolo_rect anchor(centered_drect(t_center, anchors[a].width, anchors[a].height));
-                            const double iou = box_intersection_over_union(truth_box.rect, anchor.rect);
-                            if (iou > options.iou_anchor_threshold)
-                                compute_grads(a);
+                            // Scale regression error according to the truth size
+                            const double scale_box = 2 - truth_box.rect.area() / (input_tensor.nr() * input_tensor.nc());
+
+                            // Compute the gradient for the box coordinates
+                            g[x_idx] = options.lambda_box * scale_box * (out_data[x_idx] - tx);
+                            g[y_idx] = options.lambda_box * scale_box * (out_data[y_idx] - ty);
+                            g[w_idx] = options.lambda_box * scale_box * (out_data[w_idx] - tw);
+                            g[h_idx] = options.lambda_box * scale_box * (out_data[h_idx] - th);
+
+                            // Compute the classification error
+                            for (long k = 0; k < num_classes; ++k)
+                            {
+                                const auto c_idx = tensor_index(output_tensor, n, a * num_feats + 5 + k, r, c);
+                                if (truth_box.label == options.labels[k])
+                                    g[c_idx] = options.lambda_cls * (out_data[c_idx] - 1);
+                                else
+                                    g[c_idx] = options.lambda_cls * out_data[c_idx];
+                            }
                         }
                     }
                 }

From 45498c8b336375b21d38efdea3a8b027f2163ca3 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Adri=C3=A0=20Arrufat?= <adria.arrufat@outlook.com>
Date: Tue, 6 Jul 2021 22:20:10 +0900
Subject: [PATCH 37/62] oops, fix typo

---
 dlib/dnn/loss.h | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/dlib/dnn/loss.h b/dlib/dnn/loss.h
index 4b804d48a8..0542da8bcc 100644
--- a/dlib/dnn/loss.h
+++ b/dlib/dnn/loss.h
@@ -3751,7 +3751,7 @@ namespace dlib
                     for (size_t a = 0; a < anchors.size(); ++a)
                     {
                         // Update best anchor if it's from the current stride, and optionally other anchors
-                        if ((best_tag_id == tag_id<TAG_TYPE>::id && best_a = a) || options.iou_anchor_threshold < 1)
+                        if ((best_tag_id == tag_id<TAG_TYPE>::id && best_a == a) || options.iou_anchor_threshold < 1)
                         {
                             const long c = t_center.x() / stride_x;
                             const long r = t_center.y() / stride_y;

From 3d2ae145e9f0c7402e1b6d2ed9f0d8eb7d8751f8 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Adri=C3=A0=20Arrufat?= <adria.arrufat@outlook.com>
Date: Tue, 6 Jul 2021 23:01:00 +0900
Subject: [PATCH 38/62] regression fix: do not update other anchors with low
 IoU

---
 dlib/dnn/loss.h | 10 ++++++++++
 1 file changed, 10 insertions(+)

diff --git a/dlib/dnn/loss.h b/dlib/dnn/loss.h
index 0542da8bcc..afbe8fcec3 100644
--- a/dlib/dnn/loss.h
+++ b/dlib/dnn/loss.h
@@ -3753,6 +3753,16 @@ namespace dlib
                         // Update best anchor if it's from the current stride, and optionally other anchors
                         if ((best_tag_id == tag_id<TAG_TYPE>::id && best_a == a) || options.iou_anchor_threshold < 1)
                         {
+
+                            // do not update other anchors if they have low IoU
+                            if (!(best_tag_id == tag_id<TAG_TYPE>::id && best_a == a))
+                            {
+                                const yolo_rect anchor(centered_drect(t_center, anchors[a].width, anchors[a].height));
+                                const double iou = box_intersection_over_union(truth_box.rect, anchor.rect);
+                                if (iou < options.iou_anchor_threshold)
+                                    continue;
+                            }
+
                             const long c = t_center.x() / stride_x;
                             const long r = t_center.y() / stride_y;
                             const auto x_idx = tensor_index(output_tensor, n, a * num_feats + 0, r, c);

From 32021ae110acf48cb5760db13c16024276b63d04 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Adri=C3=A0=20Arrufat?= <adria.arrufat@outlook.com>
Date: Wed, 7 Jul 2021 00:12:58 +0900
Subject: [PATCH 39/62] make NMS classwise by default

---
 dlib/dnn/loss.h          | 4 ++--
 dlib/dnn/loss_abstract.h | 4 ++--
 2 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/dlib/dnn/loss.h b/dlib/dnn/loss.h
index afbe8fcec3..629c287d96 100644
--- a/dlib/dnn/loss.h
+++ b/dlib/dnn/loss.h
@@ -3489,7 +3489,7 @@ namespace dlib
         double iou_ignore_threshold = 0.7;
         double iou_anchor_threshold = 1.0;
         test_box_overlap overlaps_nms = test_box_overlap(0.45, 1.0);
-        bool classwise_nms = false;
+        bool classwise_nms = true;
         double lambda_obj = 1.0;
         double lambda_box = 1.0;
         double lambda_cls = 1.0;
@@ -3879,7 +3879,7 @@ namespace dlib
 
         const yolo_options& get_options() const { return options; }
 
-        void adjust_nms(double iou_thresh, double percent_covered_thresh = 1, bool classwise = false)
+        void adjust_nms(double iou_thresh, double percent_covered_thresh = 1, bool classwise = true)
         {
             options.overlaps_nms = test_box_overlap(iou_thresh, percent_covered_thresh);
             options.classwise_nms = classwise;
diff --git a/dlib/dnn/loss_abstract.h b/dlib/dnn/loss_abstract.h
index 4eeb90fd38..47ba203a06 100644
--- a/dlib/dnn/loss_abstract.h
+++ b/dlib/dnn/loss_abstract.h
@@ -1906,7 +1906,7 @@ namespace dlib
         // an already output detection and should therefore be thrown out.
         test_box_overlap overlaps_nms = test_box_overlap(0.45, 1.0);
         // Tell the non-max Suppression whether to take classes into account
-        bool classwise_nms = false;
+        bool classwise_nms = true;
         // These parameters control how we penalize different kinds of mistakes: notably the objectness loss,
         // the box (bounding box regression) loss, and the classification loss.
         double lambda_obj = 1.0;
@@ -2024,7 +2024,7 @@ namespace dlib
         (
             double iou_thresh,
             double percent_covered_thresh = 1,
-            bool classwise = false
+            bool classwise = true
         );
         /*!
             ensures

From 00c3c437dc68fc07675a313075d532126c269404 Mon Sep 17 00:00:00 2001
From: Davis King <davis@dlib.net>
Date: Sat, 10 Jul 2021 09:11:15 -0400
Subject: [PATCH 40/62] Make imglab populate the new width and height image
 fields

---
 tools/imglab/src/cluster.cpp         | 2 ++
 tools/imglab/src/main.cpp            | 6 +++++-
 tools/imglab/src/metadata_editor.cpp | 9 +++++++++
 3 files changed, 16 insertions(+), 1 deletion(-)

diff --git a/tools/imglab/src/cluster.cpp b/tools/imglab/src/cluster.cpp
index 7a15584d5e..359e32b3d8 100644
--- a/tools/imglab/src/cluster.cpp
+++ b/tools/imglab/src/cluster.cpp
@@ -247,6 +247,8 @@ int cluster_dataset(
         {
             idata[i].first = std::numeric_limits<double>::infinity();
             idata[i].second.filename = data.images[i].filename;
+            idata[i].second.width = data.images[i].width;
+            idata[i].second.height = data.images[i].height;
             if (!has_non_ignored_boxes(data.images[i]))
                 continue;
 
diff --git a/tools/imglab/src/main.cpp b/tools/imglab/src/main.cpp
index 0b470e0871..250ceb646b 100644
--- a/tools/imglab/src/main.cpp
+++ b/tools/imglab/src/main.cpp
@@ -21,7 +21,7 @@
 #include <dlib/dir_nav.h>
 
 
-const char* VERSION = "1.17";
+const char* VERSION = "1.18";
 
 
 
@@ -332,6 +332,8 @@ void rotate_dataset(const command_line_parser& parser)
 
         load_image(img, metadata.images[i].filename);
         const point_transform_affine tran = rotate_image(img, temp, angle*pi/180);
+        metadata.images[i].width = temp.nc();
+        metadata.images[i].height = temp.nr();
         if (parser.option("jpg"))
         {
             filename = to_jpg_name(filename);
@@ -447,6 +449,8 @@ int resample_dataset(const command_line_parser& parser)
             std::ostringstream sout;
             sout << hex << murmur_hash3_128bit(&chip[0][0], chip.size()*sizeof(chip[0][0])).second;
             dimg.filename = data.images[i].filename + "_RESAMPLED_"+sout.str()+".png";
+            dimg.width = chip.nc();
+            dimg.height = chip.nr();
 
             if (parser.option("jpg"))
             {
diff --git a/tools/imglab/src/metadata_editor.cpp b/tools/imglab/src/metadata_editor.cpp
index 84d946452f..794f5a5b37 100644
--- a/tools/imglab/src/metadata_editor.cpp
+++ b/tools/imglab/src/metadata_editor.cpp
@@ -224,7 +224,12 @@ void propagate_boxes(
 
     array2d<rgb_pixel> img1, img2;
     dlib::load_image(img1, data.images[prev].filename);
+    data.images[prev].width = img1.nc();
+    data.images[prev].height = img1.nr();
+
     dlib::load_image(img2, data.images[next].filename);
+    data.images[next].width = img2.nc();
+    data.images[next].height = img2.nr();
     for (unsigned long i = 0; i < data.images[prev].boxes.size(); ++i)
     {
         correlation_tracker tracker;
@@ -513,6 +518,8 @@ load_image(
     try
     {
         dlib::load_image(img, metadata.images[idx].filename);
+        metadata.images[idx].width = img.nc();
+        metadata.images[idx].height = img.nr();
         set_title(metadata.name + " #"+cast_to_string(idx)+": " +metadata.images[idx].filename);
     }
     catch (exception& e)
@@ -543,6 +550,8 @@ load_image_and_set_size(
     try
     {
         dlib::load_image(img, metadata.images[idx].filename);
+        metadata.images[idx].width = img.nc();
+        metadata.images[idx].height = img.nr();
         set_title(metadata.name + " #"+cast_to_string(idx)+": " +metadata.images[idx].filename);
     }
     catch (exception& e)

From 040c83f3ed1da97a7934754c1010e3a633559bcb Mon Sep 17 00:00:00 2001
From: Davis King <davis@dlib.net>
Date: Sat, 10 Jul 2021 09:12:33 -0400
Subject: [PATCH 41/62] update example xml files to have width and height
 annotations

---
 examples/faces/testing.xml                      | 12 ++++++------
 examples/faces/testing_with_face_landmarks.xml  | 15 +++++++--------
 examples/faces/training.xml                     |  8 ++++----
 examples/faces/training_with_face_landmarks.xml | 13 ++++++-------
 4 files changed, 23 insertions(+), 25 deletions(-)

diff --git a/examples/faces/testing.xml b/examples/faces/testing.xml
index f7ef446c31..45174b565b 100644
--- a/examples/faces/testing.xml
+++ b/examples/faces/testing.xml
@@ -4,7 +4,7 @@
 <name>Testing faces</name>
 <comment>These are images from the PASCAL VOC 2011 dataset.</comment>
 <images>
-  <image file='2008_002470.jpg'>
+  <image file='2008_002470.jpg' width='500' height='332'>
     <box top='181' left='274' width='52' height='53'/>
     <box top='156' left='55' width='44' height='44'/>
     <box top='166' left='146' width='37' height='37'/>
@@ -12,12 +12,12 @@
     <box top='74' left='233' width='44' height='44'/>
     <box top='86' left='178' width='37' height='37'/>
   </image>
-  <image file='2008_002506.jpg'>
+  <image file='2008_002506.jpg' width='500' height='375'>
     <box top='78' left='329' width='109' height='109'/>
     <box top='95' left='224' width='91' height='91'/>
     <box top='65' left='125' width='90' height='91'/>
   </image>
-  <image file='2008_004176.jpg'>
+  <image file='2008_004176.jpg' width='480' height='438'>
     <box top='230' left='206' width='37' height='37'/>
     <box top='118' left='162' width='37' height='37'/>
     <box top='82' left='190' width='37' height='37'/>
@@ -26,7 +26,7 @@
     <box top='86' left='110' width='37' height='37'/>
     <box top='102' left='282' width='37' height='37'/>
   </image>
-  <image file='2008_007676.jpg'>
+  <image file='2008_007676.jpg' width='500' height='334'>
     <box top='62' left='226' width='37' height='37'/>
     <box top='113' left='194' width='44' height='44'/>
     <box top='130' left='262' width='37' height='37'/>
@@ -35,9 +35,9 @@
     <box top='141' left='107' width='52' height='53'/>
     <box top='84' left='137' width='44' height='44'/>
   </image>
-  <image file='2009_004587.jpg'>
+  <image file='2009_004587.jpg' width='400' height='500'>
     <box top='46' left='154' width='75' height='76'/>
     <box top='280' left='266' width='63' height='63'/>
   </image>
 </images>
-</dataset>
+</dataset>
\ No newline at end of file
diff --git a/examples/faces/testing_with_face_landmarks.xml b/examples/faces/testing_with_face_landmarks.xml
index 7589561b12..e0d2d92c01 100644
--- a/examples/faces/testing_with_face_landmarks.xml
+++ b/examples/faces/testing_with_face_landmarks.xml
@@ -5,10 +5,9 @@
 <comment>These are images from the PASCAL VOC 2011 dataset.
    The face landmarks are from dlib's shape_predictor_68_face_landmarks.dat
    landmarking model.  The model uses the 68 landmark scheme used by the iBUG
-   300-W dataset.
-</comment>
+   300-W dataset.</comment>
 <images>
-  <image file='2008_002470.jpg'>
+  <image file='2008_002470.jpg' width='500' height='332'>
     <box top='181' left='274' width='52' height='53'>
       <part name='00' x='277' y='194'/>
       <part name='01' x='278' y='200'/>
@@ -430,7 +429,7 @@
       <part name='67' x='196' y='112'/>
     </box>
   </image>
-  <image file='2008_002506.jpg'>
+  <image file='2008_002506.jpg' width='500' height='375'>
     <box top='78' left='329' width='109' height='109'>
       <part name='00' x='342' y='134'/>
       <part name='01' x='345' y='145'/>
@@ -642,7 +641,7 @@
       <part name='67' x='163' y='133'/>
     </box>
   </image>
-  <image file='2008_004176.jpg'>
+  <image file='2008_004176.jpg' width='480' height='438'>
     <box top='230' left='206' width='37' height='37'>
       <part name='00' x='206' y='241'/>
       <part name='01' x='206' y='245'/>
@@ -1134,7 +1133,7 @@
       <part name='67' x='294' y='126'/>
     </box>
   </image>
-  <image file='2008_007676.jpg'>
+  <image file='2008_007676.jpg' width='500' height='334'>
     <box top='62' left='226' width='37' height='37'>
       <part name='00' x='223' y='72'/>
       <part name='01' x='224' y='77'/>
@@ -1626,7 +1625,7 @@
       <part name='67' x='160' y='115'/>
     </box>
   </image>
-  <image file='2009_004587.jpg'>
+  <image file='2009_004587.jpg' width='400' height='500'>
     <box top='46' left='154' width='75' height='76'>
       <part name='00' x='147' y='74'/>
       <part name='01' x='147' y='84'/>
@@ -1769,4 +1768,4 @@
     </box>
   </image>
 </images>
-</dataset>
+</dataset>
\ No newline at end of file
diff --git a/examples/faces/training.xml b/examples/faces/training.xml
index b7e1d00710..3b5577620f 100644
--- a/examples/faces/training.xml
+++ b/examples/faces/training.xml
@@ -4,7 +4,7 @@
 <name>Training faces</name>
 <comment>These are images from the PASCAL VOC 2011 dataset.</comment>
 <images>
-  <image file='2007_007763.jpg'>
+  <image file='2007_007763.jpg' width='500' height='375'>
     <box top='90' left='194' width='37' height='37'/>
     <box top='114' left='158' width='37' height='37'/>
     <box top='89' left='381' width='45' height='44'/>
@@ -13,7 +13,7 @@
     <box top='86' left='294' width='37' height='37'/>
     <box top='233' left='309' width='45' height='44'/>
   </image>
-  <image file='2008_002079.jpg'>
+  <image file='2008_002079.jpg' width='500' height='375'>
     <box top='166' left='407' width='37' height='37'/>
     <box top='134' left='122' width='37' height='37'/>
     <box top='138' left='346' width='37' height='37'/>
@@ -21,11 +21,11 @@
     <box top='134' left='62' width='37' height='37'/>
     <box top='194' left='41' width='44' height='44'/>
   </image>
-  <image file='2008_001009.jpg'>
+  <image file='2008_001009.jpg' width='360' height='480'>
     <box top='79' left='145' width='76' height='76'/>
     <box top='214' left='125' width='90' height='91'/>
   </image>
-  <image file='2008_001322.jpg'>
+  <image file='2008_001322.jpg' width='500' height='375'>
     <box top='162' left='104' width='76' height='76'/>
     <box top='218' left='232' width='63' height='63'/>
     <box top='155' left='344' width='90' height='90'/>
diff --git a/examples/faces/training_with_face_landmarks.xml b/examples/faces/training_with_face_landmarks.xml
index b87e75350f..6647d348af 100644
--- a/examples/faces/training_with_face_landmarks.xml
+++ b/examples/faces/training_with_face_landmarks.xml
@@ -5,10 +5,9 @@
 <comment>These are images from the PASCAL VOC 2011 dataset.
    The face landmarks are from dlib's shape_predictor_68_face_landmarks.dat
    landmarking model.  The model uses the 68 landmark scheme used by the iBUG
-   300-W dataset.
-</comment>
+   300-W dataset.</comment>
 <images>
-  <image file='2007_007763.jpg'>
+  <image file='2007_007763.jpg' width='500' height='375'>
     <box top='90' left='194' width='37' height='37'>
       <part name='00' x='201' y='107'/>
       <part name='01' x='201' y='110'/>
@@ -500,7 +499,7 @@
       <part name='67' x='323' y='267'/>
     </box>
   </image>
-  <image file='2008_002079.jpg'>
+  <image file='2008_002079.jpg' width='500' height='375'>
     <box top='166' left='406' width='37' height='37'>
       <part name='00' x='412' y='179'/>
       <part name='01' x='411' y='183'/>
@@ -922,7 +921,7 @@
       <part name='67' x='68' y='227'/>
     </box>
   </image>
-  <image file='2008_001009.jpg'>
+  <image file='2008_001009.jpg' width='360' height='480'>
     <box top='79' left='145' width='76' height='76'>
       <part name='00' x='145' y='115'/>
       <part name='01' x='148' y='124'/>
@@ -1064,7 +1063,7 @@
       <part name='67' x='168' y='280'/>
     </box>
   </image>
-  <image file='2008_001322.jpg'>
+  <image file='2008_001322.jpg' width='500' height='375'>
     <box top='162' left='104' width='76' height='76'>
       <part name='00' x='106' y='183'/>
       <part name='01' x='106' y='193'/>
@@ -1277,4 +1276,4 @@
     </box>
   </image>
 </images>
-</dataset>
+</dataset>
\ No newline at end of file

From 0eeb9de9ce04fac7558ae0328a59b03eef6b2120 Mon Sep 17 00:00:00 2001
From: Davis King <davis@dlib.net>
Date: Sat, 10 Jul 2021 09:13:02 -0400
Subject: [PATCH 42/62] Adjust example so it builds in C++11 and slightly
 simplified code.

---
 examples/dnn_yolo_train_ex.cpp | 28 +++++++++++++---------------
 1 file changed, 13 insertions(+), 15 deletions(-)

diff --git a/examples/dnn_yolo_train_ex.cpp b/examples/dnn_yolo_train_ex.cpp
index a642c0d264..528809b29f 100644
--- a/examples/dnn_yolo_train_ex.cpp
+++ b/examples/dnn_yolo_train_ex.cpp
@@ -158,7 +158,7 @@ try
     parser.add_option("test", "test the detector with a threshold (default: 0.01)", 1);
     parser.add_option("visualize", "visualize data augmentation instead of training");
     parser.add_option("map", "compute the mean average precision");
-    parser.add_option("anchors", "compute <arg1> anchor boxes using K-Means", 1);
+    parser.add_option("anchors", "Do nothing but compute <arg1> anchor boxes using K-Means and print their shapes.", 1);
     parser.set_group_name("Help Options");
     parser.add_option("h", "alias of --help");
     parser.add_option("help", "display this message and exit");
@@ -205,47 +205,45 @@ try
         string_to_color(label.first);
     }
 
-    // If the default anchor boxes don't fit well your data, you should recompute them.
+    // If the default anchor boxes don't fit your data well you should recompute them.
     // Here's an simple way to do it using K-Means clustering.  Note that the approach
     // shown below is suboptimal, since it doesn't group the bounding boxes by size.
     // Grouping the bounding boxes by size and computing the K-Means on each group
-    // would make more sense, since each stride of the network is meant to output a
+    // would make more sense, since each stride of the network is meant to output
     // boxes at a particular size, but that is very specific to the network architecture
     // and the dataset itself.
     if (parser.option("anchors"))
     {
         const auto num_clusers = std::stoul(parser.option("anchors").argument());
-        std::vector<matrix<double, 2, 1>> samples;
+        std::vector<dpoint> samples;
         // First we need to rescale the bounding boxes to match the image size at training time.
         for (const auto& image_info : dataset.images)
         {
             const auto scale = image_size / std::max<double>(image_info.width, image_info.height);
             for (const auto& box : image_info.boxes)
             {
-                matrix<double, 2, 1> sample;
-                sample(0) = box.rect.width() * scale;
-                sample(1) = box.rect.height() * scale;
-                samples.push_back(move(sample));
+                dpoint sample(box.rect.width(), box.rect.height());
+                samples.push_back(sample*scale);
             }
         }
         // Now we can compute K-Means clustering
         randomize_samples(samples);
         cout << "Computing anchors for " << samples.size() << " samples" << endl;
-        std::vector<matrix<double, 2, 1>> anchors;
+        std::vector<dpoint> anchors;
         pick_initial_centers(num_clusers, anchors, samples);
         find_clusters_using_kmeans(samples, anchors);
-        std::sort(anchors.begin(), anchors.end(), [](const auto& a, const auto& b){ return a(0)*a(1) < b(0)*b(1); });
-        for (const auto& c : anchors)
+        std::sort(anchors.begin(), anchors.end(), [](const dpoint& a, const dpoint& b){ return prod(a) < prod(b); });
+        for (const dpoint& c : anchors)
             cout << round(c(0)) << 'x' << round(c(1)) << endl;
         // And check the average IoU of the newly computed anchor boxes and the training samples.
         double average_iou = 0;
-        for (const auto& s : samples)
+        for (const dpoint& s : samples)
         {
-            drectangle sample = centered_drect(dpoint(0, 0), s(0), s(1));
+            drectangle sample = centered_drect(dpoint(0, 0), s.x(), s.y());
             double best_iou = 0;
-            for (const auto& a : anchors)
+            for (const dpoint& a : anchors)
             {
-                drectangle anchor = centered_drect(dpoint(0, 0), a(0), a(1));
+                drectangle anchor = centered_drect(dpoint(0, 0), a.x(), a.y());
                 best_iou = std::max(best_iou, box_intersection_over_union(sample, anchor));
             }
             average_iou += best_iou;

From f0fff724c89215193caf64c88c147b74e513543b Mon Sep 17 00:00:00 2001
From: Davis King <davis@dlib.net>
Date: Sat, 10 Jul 2021 10:10:49 -0400
Subject: [PATCH 43/62] Add --add-width-height-metadata to imglab

---
 tools/imglab/src/main.cpp | 43 ++++++++++++++++++++++++++++++++++++++-
 1 file changed, 42 insertions(+), 1 deletion(-)

diff --git a/tools/imglab/src/main.cpp b/tools/imglab/src/main.cpp
index 250ceb646b..c8c2ea082f 100644
--- a/tools/imglab/src/main.cpp
+++ b/tools/imglab/src/main.cpp
@@ -361,6 +361,32 @@ void rotate_dataset(const command_line_parser& parser)
 
 // ----------------------------------------------------------------------------------------
 
+void add_width_and_height_metadata(const command_line_parser& parser)
+{
+    for (unsigned long i = 0; i < parser.number_of_arguments(); ++i) {
+        image_dataset_metadata::dataset metadata;
+        const string datasource = parser[i];
+        load_image_dataset_metadata(metadata,datasource);
+
+        // Set the current directory to be the one that contains the
+        // metadata file. We do this because the file might contain
+        // file paths which are relative to this folder.
+        set_current_dir(get_parent_directory(file(datasource)));
+
+        parallel_for(0, metadata.images.size(), [&](long i) 
+        {
+            array2d<rgb_pixel> img;
+            load_image(img, metadata.images[i].filename);
+            metadata.images[i].width = img.nc();
+            metadata.images[i].height = img.nr();
+        });
+
+        save_image_dataset_metadata(metadata, datasource);
+    }
+}
+
+// ----------------------------------------------------------------------------------------
+
 int resample_dataset(const command_line_parser& parser)
 {
     if (parser.number_of_arguments() != 1)
@@ -592,6 +618,8 @@ int main(int argc, char** argv)
                                         "The parts are instead simply mirrored to the flipped dataset.", 1);
         parser.add_option("rotate", "Read an XML image dataset and output a copy that is rotated counter clockwise by <arg> degrees. "
                                   "The output is saved to an XML file prefixed with rotated_<arg>.",1);
+        parser.add_option("add-width-height-metadata", "Open the given xml files and set the width and height image metadata fields "
+                            "for every image. This involves loading each image to find these values.");
         parser.add_option("cluster", "Cluster all the objects in an XML file into <arg> different clusters (pass 0 to find automatically) and save "
                                      "the results as cluster_###.xml and cluster_###.jpg files.",1);
         parser.add_option("ignore", "Mark boxes labeled as <arg> as ignored.  The resulting XML file is output as a separate file and the original is not modified.",1);
@@ -616,7 +644,7 @@ int main(int argc, char** argv)
         const char* singles[] = {"h","c","r","l","files","convert","parts","rmdiff", "rmtrunc", "rmdupes", "seed", "shuffle", "split", "add", 
                                  "flip-basic", "flip", "rotate", "tile", "size", "cluster", "resample", "min-object-size", "rmempty",
                                  "crop-size", "cropped-object-size", "rmlabel", "rm-other-labels", "rm-if-overlaps", "sort-num-objects", 
-                                 "one-object-per-image", "jpg", "rmignore", "sort", "split-train-test", "box-images"};
+                                 "one-object-per-image", "jpg", "rmignore", "sort", "split-train-test", "box-images", "add-width-height-metadata"};
         parser.check_one_time_options(singles);
         const char* c_sub_ops[] = {"r", "convert"};
         parser.check_sub_options("c", c_sub_ops);
@@ -641,6 +669,7 @@ int main(int argc, char** argv)
         parser.check_incompatible_options("c", "flip-basic");
         parser.check_incompatible_options("flip", "flip-basic");
         parser.check_incompatible_options("c", "rotate");
+        parser.check_incompatible_options("c", "add-width-height-metadata");
         parser.check_incompatible_options("c", "rename");
         parser.check_incompatible_options("c", "ignore");
         parser.check_incompatible_options("c", "parts");
@@ -654,6 +683,7 @@ int main(int argc, char** argv)
         parser.check_incompatible_options("l", "flip");
         parser.check_incompatible_options("l", "flip-basic");
         parser.check_incompatible_options("l", "rotate");
+        parser.check_incompatible_options("l", "add-width-height-metadata");
         parser.check_incompatible_options("files", "rename");
         parser.check_incompatible_options("files", "ignore");
         parser.check_incompatible_options("files", "add");
@@ -661,22 +691,27 @@ int main(int argc, char** argv)
         parser.check_incompatible_options("files", "flip");
         parser.check_incompatible_options("files", "flip-basic");
         parser.check_incompatible_options("files", "rotate");
+        parser.check_incompatible_options("files", "add-width-height-metadata");
         parser.check_incompatible_options("add", "flip");
         parser.check_incompatible_options("add", "flip-basic");
         parser.check_incompatible_options("add", "rotate");
+        parser.check_incompatible_options("add", "add-width-height-metadata");
         parser.check_incompatible_options("add", "tile");
         parser.check_incompatible_options("flip", "tile");
         parser.check_incompatible_options("flip-basic", "tile");
         parser.check_incompatible_options("rotate", "tile");
+        parser.check_incompatible_options("add-width-height-metadata", "tile");
         parser.check_incompatible_options("cluster", "tile");
         parser.check_incompatible_options("resample", "tile");
         parser.check_incompatible_options("flip", "cluster");
         parser.check_incompatible_options("flip-basic", "cluster");
         parser.check_incompatible_options("rotate", "cluster");
+        parser.check_incompatible_options("add-width-height-metadata", "cluster");
         parser.check_incompatible_options("add", "cluster");
         parser.check_incompatible_options("flip", "resample");
         parser.check_incompatible_options("flip-basic", "resample");
         parser.check_incompatible_options("rotate", "resample");
+        parser.check_incompatible_options("add-width-height-metadata", "resample");
         parser.check_incompatible_options("add", "resample");
         parser.check_incompatible_options("shuffle", "tile");
         parser.check_incompatible_options("sort-num-objects", "tile");
@@ -742,6 +777,12 @@ int main(int argc, char** argv)
             return EXIT_SUCCESS;
         }
 
+        if (parser.option("add-width-height-metadata"))
+        {
+            add_width_and_height_metadata(parser);
+            return EXIT_SUCCESS;
+        }
+
         if (parser.option("v"))
         {
             cout << "imglab v" << VERSION 

From 4f80d8d77e24ee768bb7d61de333eb8bd519b883 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Adri=C3=A0=20Arrufat?= <adria.arrufat@outlook.com>
Date: Mon, 12 Jul 2021 10:04:34 +0900
Subject: [PATCH 44/62] remove redundant check

---
 dlib/dnn/loss.h | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/dlib/dnn/loss.h b/dlib/dnn/loss.h
index 629c287d96..af7a17ff14 100644
--- a/dlib/dnn/loss.h
+++ b/dlib/dnn/loss.h
@@ -3646,7 +3646,7 @@ namespace dlib
                                     if (conf > adjust_threshold)
                                         det.labels.emplace_back(conf, options.labels[k]);
                                 }
-                                if (!det.labels.empty() && det.labels[0].first > adjust_threshold)
+                                if (!det.labels.empty())
                                 {
                                     std::sort(det.labels.rbegin(), det.labels.rend());
                                     det.detection_confidence = det.labels[0].first;

From 6e51086a07c7e68d0a1e2fc8d8363c591cc738db Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Adri=C3=A0=20Arrufat?= <adria.arrufat@outlook.com>
Date: Tue, 13 Jul 2021 23:17:45 +0900
Subject: [PATCH 45/62] update comment about iou_anchor_threshold

---
 dlib/dnn/loss_abstract.h | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/dlib/dnn/loss_abstract.h b/dlib/dnn/loss_abstract.h
index 47ba203a06..71c2b1fb25 100644
--- a/dlib/dnn/loss_abstract.h
+++ b/dlib/dnn/loss_abstract.h
@@ -1895,12 +1895,12 @@ namespace dlib
         // iou_ignore_threshold with a ground truth box will not incur any loss.
         double iou_ignore_threshold = 0.7;
         // When computing the YOLO loss (objectness + bounding box regression + classification),
-        // only the best match between a truth and an anchor are used.  Setting iou_anchor_threshold
-        // to 1 will make the model use only the best anchor for each ground truth, so
-        // other anchors can be used for other ground truths in the same cell (useful for
-        // detecting objects in crowds).  However, if the other anchors have an IoU with a
-        // truth above iou_anchor_threshold, they will also be updated.  This will match
-        // all anchors whose iou with a ground truth is above iou_anchor_threshold.
+        // the best match between a truth and an anchor is always used, regardless of their IoU.
+        // However, if other anchors have an IoU with a truth above iou_anchor_threshold, they
+        // will also be updated.  This will update all those anchors to match the same truth.
+        // Setting iou_anchor_threshold to 1 will make the model use only the best anchor for
+        // each ground truth, so other anchors can be used for other ground truths in the same
+        // cell (useful for detecting objects in crowds).
         double iou_anchor_threshold = 1.0;
         // When doing non-max suppression, we use overlaps_nms to decide if a box overlaps
         // an already output detection and should therefore be thrown out.

From ffec8e7bb07d850383adaeb6ce111b5b5ced0a4c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Adri=C3=A0=20Arrufat?= <adria.arrufat@outlook.com>
Date: Tue, 13 Jul 2021 23:23:42 +0900
Subject: [PATCH 46/62] clarify more comment about iou_anchor_threshold

---
 dlib/dnn/loss_abstract.h | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/dlib/dnn/loss_abstract.h b/dlib/dnn/loss_abstract.h
index 71c2b1fb25..61496cad72 100644
--- a/dlib/dnn/loss_abstract.h
+++ b/dlib/dnn/loss_abstract.h
@@ -1895,12 +1895,12 @@ namespace dlib
         // iou_ignore_threshold with a ground truth box will not incur any loss.
         double iou_ignore_threshold = 0.7;
         // When computing the YOLO loss (objectness + bounding box regression + classification),
-        // the best match between a truth and an anchor is always used, regardless of their IoU.
+        // the best match between a truth and an anchor is always used, regardless of the IoU.
         // However, if other anchors have an IoU with a truth above iou_anchor_threshold, they
-        // will also be updated.  This will update all those anchors to match the same truth.
-        // Setting iou_anchor_threshold to 1 will make the model use only the best anchor for
-        // each ground truth, so other anchors can be used for other ground truths in the same
-        // cell (useful for detecting objects in crowds).
+        // will also be updated to match the same truth.  Setting iou_anchor_threshold to 1 will
+        // make the model use only the best anchor for each ground truth, so other anchors can be
+        // used for other ground truths in the same cell (useful for detecting objects in crowds).
+        // This setting is meant to be used with "high capacity" models, not small ones.
         double iou_anchor_threshold = 1.0;
         // When doing non-max suppression, we use overlaps_nms to decide if a box overlaps
         // an already output detection and should therefore be thrown out.

From 74af45291ded82c799904457202736f4fab3b5ec Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Adri=C3=A0=20Arrufat?= <adria.arrufat@outlook.com>
Date: Wed, 28 Jul 2021 00:18:31 +0900
Subject: [PATCH 47/62] use long, images have nr and nc defined as long (avoids
 warnings)

---
 dlib/data_io/image_dataset_metadata.h | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/dlib/data_io/image_dataset_metadata.h b/dlib/data_io/image_dataset_metadata.h
index ef8abe2ca9..7f3ede426a 100644
--- a/dlib/data_io/image_dataset_metadata.h
+++ b/dlib/data_io/image_dataset_metadata.h
@@ -109,8 +109,8 @@ namespace dlib
 
             std::string filename;
             std::vector<box> boxes;
-            unsigned long width = 0;
-            unsigned long height = 0;
+            long width = 0;
+            long height = 0;
         };
 
     // ------------------------------------------------------------------------------------

From f888553d21f06e371991414f59e3a093a7a5b6ac Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Adri=C3=A0=20Arrufat?=
 <1671644+arrufat@users.noreply.github.com>
Date: Thu, 29 Jul 2021 10:48:32 +0900
Subject: [PATCH 48/62] Update dlib/dnn/loss_abstract.h

Co-authored-by: Davis E. King <davis@dlib.net>
---
 dlib/dnn/loss_abstract.h | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/dlib/dnn/loss_abstract.h b/dlib/dnn/loss_abstract.h
index 61496cad72..e9129995e4 100644
--- a/dlib/dnn/loss_abstract.h
+++ b/dlib/dnn/loss_abstract.h
@@ -1889,7 +1889,7 @@ namespace dlib
                 - anchors.at(tag_id<TAG_TYPE>::id) == boxes
         !*/
 
-        // This field contains the labels of all the possible object this detector can find.
+        // This field contains the labels of all the possible objects this detector can find.
         std::vector<std::string> labels;
         // When computing the objectness loss, any detection that has an IoU above
         // iou_ignore_threshold with a ground truth box will not incur any loss.
@@ -2042,4 +2042,3 @@ namespace dlib
 }
 
 #endif // DLIB_DNn_LOSS_ABSTRACT_H_
-

From 000ffae46351acddc741108487d0b3684fbb5d52 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Adri=C3=A0=20Arrufat?=
 <1671644+arrufat@users.noreply.github.com>
Date: Thu, 29 Jul 2021 10:49:23 +0900
Subject: [PATCH 49/62] Update dlib/dnn/loss_abstract.h

Co-authored-by: Davis E. King <davis@dlib.net>
---
 dlib/dnn/loss_abstract.h | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/dlib/dnn/loss_abstract.h b/dlib/dnn/loss_abstract.h
index e9129995e4..7ff6dc86f5 100644
--- a/dlib/dnn/loss_abstract.h
+++ b/dlib/dnn/loss_abstract.h
@@ -1896,8 +1896,8 @@ namespace dlib
         double iou_ignore_threshold = 0.7;
         // When computing the YOLO loss (objectness + bounding box regression + classification),
         // the best match between a truth and an anchor is always used, regardless of the IoU.
-        // However, if other anchors have an IoU with a truth above iou_anchor_threshold, they
-        // will also be updated to match the same truth.  Setting iou_anchor_threshold to 1 will
+        // However, if other anchors have an IoU with a truth box above iou_anchor_threshold, they
+        // will also experience loss against that truth box as well.  Setting iou_anchor_threshold to 1 will
         // make the model use only the best anchor for each ground truth, so other anchors can be
         // used for other ground truths in the same cell (useful for detecting objects in crowds).
         // This setting is meant to be used with "high capacity" models, not small ones.

From 9d5c03236195032d837bbbec2de9c94237901570 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Adri=C3=A0=20Arrufat?=
 <1671644+arrufat@users.noreply.github.com>
Date: Thu, 29 Jul 2021 10:49:32 +0900
Subject: [PATCH 50/62] Update dlib/dnn/loss_abstract.h

Co-authored-by: Davis E. King <davis@dlib.net>
---
 dlib/dnn/loss_abstract.h | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/dlib/dnn/loss_abstract.h b/dlib/dnn/loss_abstract.h
index 7ff6dc86f5..efff26bce9 100644
--- a/dlib/dnn/loss_abstract.h
+++ b/dlib/dnn/loss_abstract.h
@@ -1899,7 +1899,7 @@ namespace dlib
         // However, if other anchors have an IoU with a truth box above iou_anchor_threshold, they
         // will also experience loss against that truth box as well.  Setting iou_anchor_threshold to 1 will
         // make the model use only the best anchor for each ground truth, so other anchors can be
-        // used for other ground truths in the same cell (useful for detecting objects in crowds).
+        // used for other ground truth boxes in the same cell (useful for detecting objects in crowds).
         // This setting is meant to be used with "high capacity" models, not small ones.
         double iou_anchor_threshold = 1.0;
         // When doing non-max suppression, we use overlaps_nms to decide if a box overlaps

From 8822c1b7e78048011a77fdf5c8910df7a8a6bf92 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Adri=C3=A0=20Arrufat?=
 <1671644+arrufat@users.noreply.github.com>
Date: Thu, 29 Jul 2021 10:49:50 +0900
Subject: [PATCH 51/62] Update dlib/dnn/loss_abstract.h

Co-authored-by: Davis E. King <davis@dlib.net>
---
 dlib/dnn/loss_abstract.h | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/dlib/dnn/loss_abstract.h b/dlib/dnn/loss_abstract.h
index efff26bce9..982ba69e59 100644
--- a/dlib/dnn/loss_abstract.h
+++ b/dlib/dnn/loss_abstract.h
@@ -1994,7 +1994,7 @@ namespace dlib
                   you want to output more objects (that are also of less confidence) you
                   can call to_label() with a smaller value of adjust_threshold.
                 - R.label == the label of the detected object.
-                - R.labels == a std::vector<double, std::string> containing all the confidence values
+                - R.labels == a std::vector<std::pair<double, std::string>> containing all the confidence values
                   and labels that have a detection score > adjust_threshold, since this loss allows
                   for multi-label outputs.  Note that the following is true:
                       - R.labels[0].first == R.detection_confidence

From c747def4fb840d424e20f33150c8f61bcc432bc9 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Adri=C3=A0=20Arrufat?=
 <1671644+arrufat@users.noreply.github.com>
Date: Thu, 29 Jul 2021 10:50:44 +0900
Subject: [PATCH 52/62] Update
 dlib/image_processing/full_object_detection_abstract.h

Co-authored-by: Davis E. King <davis@dlib.net>
---
 dlib/image_processing/full_object_detection_abstract.h | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/dlib/image_processing/full_object_detection_abstract.h b/dlib/image_processing/full_object_detection_abstract.h
index 411089303d..1f71413d4a 100644
--- a/dlib/image_processing/full_object_detection_abstract.h
+++ b/dlib/image_processing/full_object_detection_abstract.h
@@ -218,7 +218,7 @@ namespace dlib
         std::vector<std::pair<double, std::string>> labels;
 
         operator rectangle() const { return rect; }
-        bool operator == (const yolo_rect& rhs) const;
+        bool operator== (const yolo_rect& rhs) const;
         /*!
             ensures
                 - returns true if and only if rect == rhs.rect && detection_confidence == rhs.detection_confidence && label == rhs.label.
@@ -244,4 +244,3 @@ namespace dlib
 
 #endif // DLIB_FULL_OBJECT_DeTECTION_ABSTRACT_Hh_
 
-

From 66fe53eeee88e343a7bb4522a07d6bf0b47b207f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Adri=C3=A0=20Arrufat?=
 <1671644+arrufat@users.noreply.github.com>
Date: Thu, 29 Jul 2021 10:51:18 +0900
Subject: [PATCH 53/62] Update dlib/image_transforms/random_cropper_abstract.h

Co-authored-by: Davis E. King <davis@dlib.net>
---
 dlib/image_transforms/random_cropper_abstract.h | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/dlib/image_transforms/random_cropper_abstract.h b/dlib/image_transforms/random_cropper_abstract.h
index 6304905d1a..1f25024852 100644
--- a/dlib/image_transforms/random_cropper_abstract.h
+++ b/dlib/image_transforms/random_cropper_abstract.h
@@ -203,8 +203,8 @@ namespace dlib
         ) const;
         /*!
             ensures
-                - When a chip is extracted, any object whose area covered is smaller than
-                  get_min_object_coverage() will have its ignore field set to true.
+                - When a chip is extracted, any object that has less than get_min_object_coverage() fraction of its 
+                   total area contained within the crop will have its ignore field set to true.
         !*/
 
         void set_min_object_coverage (
@@ -373,4 +373,3 @@ namespace dlib
 
 #endif // DLIB_RaNDOM_CROPPER_ABSTRACT_H_
 
-

From 281b9419fd605212e55dab0c9a128c322a3d8ff3 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Adri=C3=A0=20Arrufat?=
 <1671644+arrufat@users.noreply.github.com>
Date: Thu, 29 Jul 2021 10:51:29 +0900
Subject: [PATCH 54/62] Update examples/dnn_yolo_train_ex.cpp

Co-authored-by: Davis E. King <davis@dlib.net>
---
 examples/dnn_yolo_train_ex.cpp | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/examples/dnn_yolo_train_ex.cpp b/examples/dnn_yolo_train_ex.cpp
index 528809b29f..825f7e8898 100644
--- a/examples/dnn_yolo_train_ex.cpp
+++ b/examples/dnn_yolo_train_ex.cpp
@@ -206,7 +206,7 @@ try
     }
 
     // If the default anchor boxes don't fit your data well you should recompute them.
-    // Here's an simple way to do it using K-Means clustering.  Note that the approach
+    // Here's a simple way to do it using K-Means clustering.  Note that the approach
     // shown below is suboptimal, since it doesn't group the bounding boxes by size.
     // Grouping the bounding boxes by size and computing the K-Means on each group
     // would make more sense, since each stride of the network is meant to output

From a6954ee9e59deda1bc57a7b3cc36e60a91e5cb2b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Adri=C3=A0=20Arrufat?=
 <1671644+arrufat@users.noreply.github.com>
Date: Thu, 29 Jul 2021 10:56:05 +0900
Subject: [PATCH 55/62] Update examples/dnn_yolo_train_ex.cpp

Co-authored-by: Davis E. King <davis@dlib.net>
---
 examples/dnn_yolo_train_ex.cpp | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/examples/dnn_yolo_train_ex.cpp b/examples/dnn_yolo_train_ex.cpp
index 825f7e8898..917c5a1fcf 100644
--- a/examples/dnn_yolo_train_ex.cpp
+++ b/examples/dnn_yolo_train_ex.cpp
@@ -255,8 +255,7 @@ try
     // When computing the objectness loss in YOLO, predictions that do not have an IoU
     // with any ground truth box of at least options.iou_ignore_threshold, will be
     // treated as not capable of detecting an object, an therefore incur loss.
-    // Predictions above this threshold will be ignored, i.e. will not contribute to the
-    // loss. Good values are 0.7 or 0.5.
+    // Similarly, predictions above this threshold are considered correct predictions by the loss. Typical settings for this threshold are in the range 0.5 to 0.7.
     options.iou_ignore_threshold = 0.7;
     // By setting this to a value < 1, we are telling the model to update all the predictions
     // as long as the anchor box has an IoU > 0.2 with a ground truth.

From 5efdb0c9dc28f6c49dec862b38bd558b83c34ebc Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Adri=C3=A0=20Arrufat?=
 <1671644+arrufat@users.noreply.github.com>
Date: Thu, 29 Jul 2021 10:57:34 +0900
Subject: [PATCH 56/62] Update dlib/dnn/loss_abstract.h

Co-authored-by: Davis E. King <davis@dlib.net>
---
 dlib/dnn/loss_abstract.h | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/dlib/dnn/loss_abstract.h b/dlib/dnn/loss_abstract.h
index 982ba69e59..383f16af00 100644
--- a/dlib/dnn/loss_abstract.h
+++ b/dlib/dnn/loss_abstract.h
@@ -2020,8 +2020,7 @@ namespace dlib
             Also, the loss value returned corresponds to the L2 loss.
         !*/
 
-        void adjust_nms
-        (
+        void adjust_nms (
             double iou_thresh,
             double percent_covered_thresh = 1,
             bool classwise = true

From c671f05461b0bd6ab6fbd98ec3acc0800c725b29 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Adri=C3=A0=20Arrufat?= <adria.arrufat@outlook.com>
Date: Thu, 29 Jul 2021 11:18:14 +0900
Subject: [PATCH 57/62] update random_cropper docs about
 get_min_object_coverage

---
 dlib/image_transforms/random_cropper_abstract.h | 1 +
 1 file changed, 1 insertion(+)

diff --git a/dlib/image_transforms/random_cropper_abstract.h b/dlib/image_transforms/random_cropper_abstract.h
index 1f25024852..8fafdaebc5 100644
--- a/dlib/image_transforms/random_cropper_abstract.h
+++ b/dlib/image_transforms/random_cropper_abstract.h
@@ -40,6 +40,7 @@ namespace dlib
                 - #get_max_object_size() == 0.7
                 - #get_background_crops_fraction() == 0.5
                 - #get_translate_amount() == 0.1
+                - #get_min_object_coverage == 1.0
         !*/
 
         void set_seed (

From b5815b3f34b023303df92f94d8a1432535f0c93a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Adri=C3=A0=20Arrufat?= <adria.arrufat@outlook.com>
Date: Thu, 29 Jul 2021 11:42:23 +0900
Subject: [PATCH 58/62] improve classwise_nms documentation

---
 dlib/dnn/loss_abstract.h | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/dlib/dnn/loss_abstract.h b/dlib/dnn/loss_abstract.h
index 383f16af00..8a37c61a05 100644
--- a/dlib/dnn/loss_abstract.h
+++ b/dlib/dnn/loss_abstract.h
@@ -1905,7 +1905,7 @@ namespace dlib
         // When doing non-max suppression, we use overlaps_nms to decide if a box overlaps
         // an already output detection and should therefore be thrown out.
         test_box_overlap overlaps_nms = test_box_overlap(0.45, 1.0);
-        // Tell the non-max Suppression whether to take classes into account
+        // When set to true, NMS will only be applied between objects with the same class label.
         bool classwise_nms = true;
         // These parameters control how we penalize different kinds of mistakes: notably the objectness loss,
         // the box (bounding box regression) loss, and the classification loss.

From 1902a608b6909483dec7a37111d4b6e781d8ac68 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Adri=C3=A0=20Arrufat?= <adria.arrufat@outlook.com>
Date: Thu, 29 Jul 2021 22:11:44 +0900
Subject: [PATCH 59/62] fix adjust_nms documentation

---
 dlib/dnn/loss_abstract.h | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/dlib/dnn/loss_abstract.h b/dlib/dnn/loss_abstract.h
index 8a37c61a05..e34d781a24 100644
--- a/dlib/dnn/loss_abstract.h
+++ b/dlib/dnn/loss_abstract.h
@@ -2027,8 +2027,8 @@ namespace dlib
         );
         /*!
             ensures
-                - #get_options().overlaps_nms == test_box_overlap(iou_thresh, percent_covered_thresh)
-                - #get_options().classwise_nms == classwise
+                - #get_options().overlaps_nms = test_box_overlap(iou_thresh, percent_covered_thresh)
+                - #get_options().classwise_nms = classwise
         !*/
 
     };

From 7c0b425741c3015676724a4789dc00c503705ad9 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Adri=C3=A0=20Arrufat?= <adria.arrufat@outlook.com>
Date: Fri, 30 Jul 2021 08:17:56 +0900
Subject: [PATCH 60/62] Revert "fix adjust_nms documentation"

This reverts commit 1902a608b6909483dec7a37111d4b6e781d8ac68.
---
 dlib/dnn/loss_abstract.h | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/dlib/dnn/loss_abstract.h b/dlib/dnn/loss_abstract.h
index e34d781a24..8a37c61a05 100644
--- a/dlib/dnn/loss_abstract.h
+++ b/dlib/dnn/loss_abstract.h
@@ -2027,8 +2027,8 @@ namespace dlib
         );
         /*!
             ensures
-                - #get_options().overlaps_nms = test_box_overlap(iou_thresh, percent_covered_thresh)
-                - #get_options().classwise_nms = classwise
+                - #get_options().overlaps_nms == test_box_overlap(iou_thresh, percent_covered_thresh)
+                - #get_options().classwise_nms == classwise
         !*/
 
     };

From fdd49cfa5ba3c3a9979833cc91c19eba8c997192 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Adri=C3=A0=20Arrufat?= <adria.arrufat@outlook.com>
Date: Fri, 30 Jul 2021 08:18:17 +0900
Subject: [PATCH 61/62] split line

---
 examples/dnn_yolo_train_ex.cpp | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/examples/dnn_yolo_train_ex.cpp b/examples/dnn_yolo_train_ex.cpp
index 917c5a1fcf..73ef3e5fe0 100644
--- a/examples/dnn_yolo_train_ex.cpp
+++ b/examples/dnn_yolo_train_ex.cpp
@@ -255,7 +255,8 @@ try
     // When computing the objectness loss in YOLO, predictions that do not have an IoU
     // with any ground truth box of at least options.iou_ignore_threshold, will be
     // treated as not capable of detecting an object, an therefore incur loss.
-    // Similarly, predictions above this threshold are considered correct predictions by the loss. Typical settings for this threshold are in the range 0.5 to 0.7.
+    // Similarly, predictions above this threshold are considered correct predictions
+    // by the loss.  Typical settings for this threshold are in the range 0.5 to 0.7.
     options.iou_ignore_threshold = 0.7;
     // By setting this to a value < 1, we are telling the model to update all the predictions
     // as long as the anchor box has an IoU > 0.2 with a ground truth.

From 4793705a67704d9bf93d94439d3e598dc97a82d9 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Adri=C3=A0=20Arrufat?= <adria.arrufat@outlook.com>
Date: Fri, 30 Jul 2021 09:03:53 +0900
Subject: [PATCH 62/62] fix description about the returned loss value

---
 dlib/dnn/loss_abstract.h | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/dlib/dnn/loss_abstract.h b/dlib/dnn/loss_abstract.h
index 8a37c61a05..c527eec6cc 100644
--- a/dlib/dnn/loss_abstract.h
+++ b/dlib/dnn/loss_abstract.h
@@ -2017,7 +2017,7 @@ namespace dlib
                 - layer<TAG_TYPE>(sub).get_output().k() == options.anchors.at(tag_id<TAG_TYPE>::id).size() * (5 + options.labels.size());
                 - sub.get_output().num_samples() == input_tensor.num_samples()
                 - sub.sample_expansion_factor() == 1
-            Also, the loss value returned corresponds to the L2 loss.
+            Also, the loss value returned corresponds to the squared norm of the error gradient.
         !*/
 
         void adjust_nms (