Merge pull request #160 from HyperInspire/feature/hyplmk2

Feature/hyplmk2

CMakeLists.txt

@@ -10,7 +10,7 @@ set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -O3")
 # Current version
 set(INSPIRE_FACE_VERSION_MAJOR 1)
 set(INSPIRE_FACE_VERSION_MINOR 1)
-set(INSPIRE_FACE_VERSION_PATCH 12)
+set(INSPIRE_FACE_VERSION_PATCH 13)
 
 # Converts the version number to a string
 string(CONCAT INSPIRE_FACE_VERSION_MAJOR_STR ${INSPIRE_FACE_VERSION_MAJOR})

README.md

@@ -14,6 +14,8 @@ Please contact [[email protected]](mailto:[email protected]?subject=In
 
 ## Change Logs
 
+**`2025-02-20`** Upgrade the face landmark model.
+
 **`2025-01-21`** Update all models to t3 and add tool to convert cosine similarity to percentage.
 
 **`2025-01-08`** Support inference on Rockchip devices **RK3566/RK3568** NPU.
@@ -563,11 +565,11 @@ For different scenarios, we currently provide several Packs, each containing mul
 
 | Name | Supported Devices | Note | Last Update | Link |
 | --- | --- | --- | --- | --- |
-| Pikachu | CPU | Lightweight edge-side models | Jan 21, 2025 | [Download](https://github.com/HyperInspire/InspireFace/releases/download/v1.x/Pikachu) |
-| Megatron | CPU, GPU | Mobile and server models | Jan 21, 2025 | [Download](https://github.com/HyperInspire/InspireFace/releases/download/v1.x/Megatron) |
-| Gundam-RV1109 | RKNPU | Supports RK1109 and RK1126 | Jan 21, 2025 | [Download](https://github.com/HyperInspire/InspireFace/releases/download/v1.x/Gundam_RV1109) |
-| Gundam-RV1106 | RKNPU | Supports RV1103 and RV1106 | Jan 21, 2025 | [Download](https://github.com/HyperInspire/InspireFace/releases/download/v1.x/Gundam_RV1106) |
-| Gundam-RK356X | RKNPU | Supports RK3566 and RK3568 | Jan 21, 2025 | [Download](https://github.com/HyperInspire/InspireFace/releases/download/v1.x/Gundam_RK356X) |
+| Pikachu | CPU | Lightweight edge-side models | Feb 20, 2025 | [Download](https://github.com/HyperInspire/InspireFace/releases/download/v1.x/Pikachu) |
+| Megatron | CPU, GPU | Mobile and server models | Feb 20, 2025 | [Download](https://github.com/HyperInspire/InspireFace/releases/download/v1.x/Megatron) |
+| Gundam-RV1109 | RKNPU | Supports RK1109 and RK1126 | Feb 20, 2025 | [Download](https://github.com/HyperInspire/InspireFace/releases/download/v1.x/Gundam_RV1109) |
+| Gundam-RV1106 | RKNPU | Supports RV1103 and RV1106 | Feb 20, 2025 | [Download](https://github.com/HyperInspire/InspireFace/releases/download/v1.x/Gundam_RV1106) |
+| Gundam-RK356X | RKNPU | Supports RK3566 and RK3568 | Feb 20, 2025 | [Download](https://github.com/HyperInspire/InspireFace/releases/download/v1.x/Gundam_RK356X) |
 
 ## Acknowledgement
 

cpp/inspireface/c_api/inspireface.h

@@ -461,7 +461,7 @@ HYPER_CAPI_EXPORT extern HResult HFSessionSetFilterMinimumFacePixelSize(HFSessio
 HYPER_CAPI_EXPORT extern HResult HFSessionSetFaceDetectThreshold(HFSession session, HFloat threshold);
 
 /**
- * @brief Set the track mode smooth ratio in the session. default value is  0.025
+ * @brief Set the track mode smooth ratio in the session. default value is  0.05
  *
  * @param session Handle to the session.
  * @param ratio The smooth ratio value.
@@ -470,7 +470,7 @@ HYPER_CAPI_EXPORT extern HResult HFSessionSetFaceDetectThreshold(HFSession sessi
 HYPER_CAPI_EXPORT extern HResult HFSessionSetTrackModeSmoothRatio(HFSession session, HFloat ratio);
 
 /**
- * @brief Set the track mode num smooth cache frame in the session. default value is 15
+ * @brief Set the track mode num smooth cache frame in the session. default value is 5
  *
  * @param session Handle to the session.
  * @param num The num smooth cache frame value.

cpp/inspireface/pipeline_module/liveness/order_of_hyper_landmark.h

@@ -11,10 +11,10 @@
 namespace inspire {
 
 // HyperLandmark left eye contour points sequence of dense facial landmarks.
-const std::vector<int> HLMK_LEFT_EYE_POINTS_INDEX = {1, 34, 53, 59, 67, 3, 12, 94};
+const std::vector<int> HLMK_LEFT_EYE_POINTS_INDEX = {51, 52, 53, 54, 55, 56, 57, 58};
 
 // HyperLandmark right eye contour points sequence of dense facial landmarks.
-const std::vector<int> HLMK_RIGHT_EYE_POINTS_INDEX = {27, 104, 41, 85, 20, 47, 43, 51};
+const std::vector<int> HLMK_RIGHT_EYE_POINTS_INDEX = {59, 60, 61, 62, 63, 64, 65, 66};
 
 }  // namespace inspire
 

cpp/inspireface/track_module/face_track_module.h

@@ -209,9 +209,9 @@ class INSPIRE_API FaceTrackModule {
 
     bool m_detect_mode_landmark_{true};  ///< Detect mode landmark
 
-    int m_track_mode_num_smooth_cache_frame_ = 15;  ///< Track mode number of smooth cache frame
+    int m_track_mode_num_smooth_cache_frame_ = 5;  ///< Track mode number of smooth cache frame
 
-    float m_track_mode_smooth_ratio_ = 0.025;  ///< Track mode smooth ratio
+    float m_track_mode_smooth_ratio_ = 0.05;  ///< Track mode smooth ratio
 };
 
 }  // namespace inspire

cpp/inspireface/track_module/landmark/face_landmark_adapt.h

@@ -38,13 +38,13 @@ class INSPIRE_API FaceLandmarkAdapt : public AnyNetAdapter {
     int getInputSize() const;
 
 public:
-    const static int LEFT_EYE_CENTER = 55;     ///< Landmark index for the center of the left eye.
-    const static int RIGHT_EYE_CENTER = 105;   ///< Landmark index for the center of the right eye.
-    const static int NOSE_CORNER = 69;         ///< Landmark index for the tip of the nose.
-    const static int MOUTH_LEFT_CORNER = 45;   ///< Landmark index for the left corner of the mouth.
-    const static int MOUTH_RIGHT_CORNER = 50;  ///< Landmark index for the right corner of the mouth.
-    const static int MOUTH_LOWER = 37 - 1;
-    const static int MOUTH_UPPER = 104 - 1;
+    const static int LEFT_EYE_CENTER = 67;      ///< Landmark index for the center of the left eye.
+    const static int RIGHT_EYE_CENTER = 68;     ///< Landmark index for the center of the right eye.
+    const static int NOSE_CORNER = 100;         ///< Landmark index for the tip of the nose.
+    const static int MOUTH_LEFT_CORNER = 104;   ///< Landmark index for the left corner of the mouth.
+    const static int MOUTH_RIGHT_CORNER = 105;  ///< Landmark index for the right corner of the mouth.
+    const static int MOUTH_LOWER = 84;
+    const static int MOUTH_UPPER = 87;
 
     const static int NUM_OF_LANDMARK = 106;  ///< Total number of landmarks detected.
 

cpp/inspireface/track_module/landmark/mean_shape.h

@@ -1,28 +1,43 @@
 /**
  * Created by Jingyu Yan
- * @date 2024-10-01
+ * @date 2025-02-19
  */
 #pragma once
 #ifndef INSPIREFACE_MEAN_SHAPE_H
 #define INSPIREFACE_MEAN_SHAPE_H
 
 static const float mean_shape[212] = {
-  57.32878,   105.83315, 29.624239, 20.165985, 67.11997,   68.87743,  34.962475,  24.871325, 70.813126, 72.67922,  9.729976,   60.237606, 8.554062,
-  52.910946,  7.9402304, 45.916294, 7.893721,  39.038574,  7.975547,  31.956337,  8.622486,  25.029036, 9.702873,  18.02224,   29.999453, 24.584417,
-  101.480095, 18.32498,  102.96178, 31.837185, 102.421196, 25.293413, 103.14877,  45.68892,  102.83123, 38.913933, 101.509834, 59.725216, 18.334711,
-  15.045836,  87.70989,  23.169832, 57.21172,  19.708687,  57.577744, 34.00675,   57.35939,  26.938309, 66.14159,  9.411763,   66.75376,  68.22819,
-  62.6334,    62.411827, 69.79199,  23.472647, 48.240788,  8.405983,  31.455082,  5.2922187, 64.99532,  74.57111,  43.865818,  48.00997,  57.690235,
-  75.386116,  69.78392,  65.17174,  34.54513,  18.694391,  49.58706,  23.827042,  57.785107, 68.08823,  45.6225,   65.31815,   57.99898,  63.48456,
-  53.4715,    62.43662,  48.237335, 68.25542,  78.643456,  18.564125, 73.371895,  69.14687,  79.056885, 25.003506, 67.02467,   13.268747, 38.6335,
-  69.56436,   57.690876, 51.854816, 83.83572,  24.79103,   89.265,    12.73436,   99.282196, 73.33285,  76.34923,  69.55592,   74.20933,  24.599825,
-  78.53577,   20.896051, 39.997128, 19.68584,  74.3311,    11.654106, 35.248596,  21.433992, 100.43374, 66.47729,  28.646503,  97.14609,  40.153236,
-  11.322219,  44.286064, 23.392376, 47.92698,  12.827263,  41.805626, 69.43811,   24.885767, 12.71509,  48.393387, 68.740715,  50.14667,  74.66772,
-  44.07503,   72.71468,  12.846613, 74.02932,  39.782177,  24.16436,  102.569374, 52.513325, 57.747818, 41.270073, 82.500984,  5.628833,  82.15605,
-  11.161637,  35.225296, 21.377424, 73.433525, 6.3903227,  96.06538,  15.276841,  90.67236,  8.234125,  48.962955, 105.35969,  41.72264,  103.72105,
-  34.76933,   100.86308, 40.689857, 5.962051,  23.708202,  92.38355,  19.15793,   86.72503,  15.390202, 80.73963,  32.348423,  11.188513, 23.406786,
-  8.145785,   83.923935, 19.972424, 67.38307,  50.0823,    62.78393,  50.88386,   48.079323, 50.001537, 52.845192, 50.883617,  46.965893, 39.609715,
-  64.93429,   23.805267, 68.55757,  39.412674, 72.076805,  47.75158,  25.989338,  23.418095, 72.78181,  102.99326, 79.36011,   99.85451,  65.49673,
-  105.025246, 93.91416,  86.0099,   96.90711,  79.99802,   84.915344, 96.01808,   90.13561,  91.4032,   11.011357, 67.08702,   58.06053,  68.5939,
-  73.52952,   19.782192, 78.52056,  20.81598};
+  108.38947968990857, 25.872507850440652, 107.83061691599185, 32.740667888401575, 107.39614893771035, 39.61981246234094,  106.91122964874452,
+  46.49403692042678,  106.2010288700007,  53.34741525135486,  105.09197874965457, 60.15582154225913,  103.42464830472807, 66.83258235371402,
+  101.06766895276914, 73.25011372147983,  97.9868709520959,   79.30544663628336,  94.26571910204586,  84.93503057741032,  90.00562918536812,
+  90.1122202354656,   85.32941316890714,  94.87185192884652,  80.35160378755248,  99.24044649063815,  75.077051388358,    103.10840866085465,
+  69.44018790369117,  106.28129102952379, 63.371791205040466, 108.56111173383506, 56.80262016834233,  109.7499051441407,  50.10379888471675,
+  108.6992507479294,  43.90788570414131,  106.50137834485915, 38.14868418493501,  103.37055610413495, 32.759996698129285, 99.52105297615987,
+  27.680053930061195, 95.16092929087473,  22.917297681524964, 90.39786780114727,  18.575706289743966, 85.20751276433494,  14.768101387475808,
+  79.56267473535601,  11.595841658352757, 73.49136080613575,  9.138881618312933,  67.05404266869571,  7.365347520342082,  60.353342780036044,
+  6.170510409991614,  53.51182231740061,  5.413249465696331,  46.61201199457635,  4.919113585044149,  39.68424569622725,  4.508607756550588,
+  32.75335234242255,  4.0021445204797725, 25.844283881249446, 20.10333643336724,  11.931315790409663, 24.650849944400893, 7.726547826795972,
+  30.408737910838845, 5.739425286951978,  36.63709825167494,  5.7990141975975735, 42.5960288130762,   7.734380599398077,  47.54591990037575,
+  11.374600124045191, 40.6701702686607,   11.684019226473147, 33.713706506191905, 11.418272465358037, 26.812784451308616, 11.269863943170142,
+  93.07932357431682,  11.92899217593448,  88.67053327362375,  7.753896364911641,  83.02397300977135,  5.806492520461124,  76.89097637117361,
+  5.898778431596352,  71.0228769585926,   7.842751904864862,  66.17079582408657,  11.45044902205504,  72.93048624555809,  11.673920920047593,
+  79.77352047369524,  11.42746145492823,  86.54226883746315,  11.312143719439764, 26.190494438734053, 24.325682223561593, 30.366775767967628,
+  21.29695831806158,  35.6070030786359,   20.300764805001755, 40.84233026971642,  21.32760476469489,  44.56320847446876,  24.863093217233942,
+  40.15727240006,     26.299853886839397, 35.25656863819322,  26.745706888418976, 30.437323925963305, 26.09743522459145,  87.12625473052692,
+  24.281244176246002, 83.10628533147951,  21.29726688067164,  77.9629162209028,   20.330772078573286, 72.7718115646307,   21.34368525497503,
+  69.02681716843543,  24.8625797889175,   73.45785066951291,  26.264874015322185, 78.3288579738154,   26.70498208986356,  83.03448398701677,
+  26.040749214404023, 35.98974721945182,  23.296285437719114, 77.6104064849568,   23.28902786261666,  35.42762213737087,  23.907137413710117,
+  78.1019097047022,   23.890769184322778, 73.64132732704913,  75.09299946772475,  68.33809472897121,  71.1431381125012,   61.45715432795517,
+  68.5255727062771,   56.88494393202458,  69.54893655228194,  52.23001938967138,  68.53591299060278,  45.1946880963144,   71.15044515105585,
+  39.531841299807645, 75.07763099178555,  43.88786976326114,  79.83475120739998,  49.515567201234155, 83.26256423798866,  56.81551845731522,
+  84.56820600073104,  63.98463972206808,  83.17902074650735,  69.46501161287217,  79.75710932436276,  65.0011806029609,   76.95904955102614,
+  56.80326229443618,  77.9217550745768,   48.61522852368347,  76.98346163901918,  48.66939742480621,  73.65781445636614,  56.856195386353555,
+  73.62930599992023,  64.98468095979847,  73.66401999131368,  63.45803902648853,  25.14206608791347,  65.950699532727,    38.42145308657457,
+  71.15236199888838,  51.7461970201108,   68.35896800514699,  56.90136467351805,  63.49285164236126,  55.15790554274934,  56.91186565985643,
+  58.48273034863865,  50.38652736379088,  55.156095329634354, 45.490615257275394, 57.00613038907359,  42.508457049067346, 51.765068617716196,
+  47.67302806353165,  38.441738574750175, 50.168926238237994, 25.17549045412592,  56.97274142951366,  48.885497519339204, 56.70557388413479,
+  21.19000556177852,  56.89002821964203,  39.52139553527144,  56.85088564345258,  30.32367503455182,  39.531841299807645, 75.07763099178555,
+  73.64132732704913,  75.09299946772475,
+};
 
 #endif  // INSPIREFACE_MEAN_SHAPE_H

cpp/sample/CMakeLists.txt

@@ -81,6 +81,13 @@ set_target_properties(FeatureHubSample PROPERTIES
         RUNTIME_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}/sample/"
 )
 
+add_executable(LandmarkSample source/landmark_sample.cpp)
+target_link_libraries(LandmarkSample InspireFace ${ext})
+set_target_properties(LandmarkSample PROPERTIES
+        RUNTIME_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}/sample/"
+)
+
+
 # Platform watershed
 if (ISF_BUILD_LINUX_ARM7 OR ISF_BUILD_LINUX_AARCH64)
     # Typically this is an embedded system or development board scenario where some GUI-related functions are not supported

cpp/sample/source/landmark_sample.cpp

@@ -0,0 +1,35 @@
+#include <iostream>
+#include <inspirecv/inspirecv.h>
+#include "inspireface/initialization_module/launch.h"
+#include <inspireface/middleware/inspirecv_image_process.h>
+#include "inspireface/track_module/landmark/face_landmark_adapt.h"
+int main() {
+    std::string expansion_path = "";
+    INSPIRE_LAUNCH->Load("test_res/pack/Pikachu-t4");
+    auto archive = INSPIRE_LAUNCH->getMArchive();
+
+    inspire::InspireModel lmkModel;
+    auto ret = archive.LoadModel("landmark", lmkModel);
+    if (ret != 0) {
+        INSPIRE_LOGE("Load %s error: %d", "landmark", ret);
+        return -1;
+    }
+
+    inspire::FaceLandmarkAdapt lmk;
+    lmk.loadData(lmkModel, lmkModel.modelType);
+
+    auto image = inspirecv::Image::Create("test_res/data/crop/crop.png");
+    auto data = image.Resize(112, 112);
+    auto lmk_out = lmk(data);
+    std::vector<inspirecv::Point2i> landmarks_output(inspire::FaceLandmarkAdapt::NUM_OF_LANDMARK);
+    for (int i = 0; i < inspire::FaceLandmarkAdapt::NUM_OF_LANDMARK; ++i) {
+        float x = lmk_out[i * 2 + 0] * image.Width();
+        float y = lmk_out[i * 2 + 1] * image.Height();
+        landmarks_output[i] = inspirecv::Point<int>(x, y);
+    }
+
+    for (int i = 0; i < landmarks_output.size(); ++i) {
+        image.DrawCircle(landmarks_output[i], 5, {0, 0, 255});
+    }
+    image.Write("crop_lmk.png");
+}

python/inspireface/modules/utils/resource.py

@@ -3,6 +3,14 @@
 from pathlib import Path
 import urllib.request
 import ssl
+import hashlib
+
+def get_file_hash_sha256(file_path):
+    sha256 = hashlib.sha256()
+    with open(file_path, 'rb') as f:
+        for chunk in iter(lambda: f.read(4096), b''):
+            sha256.update(chunk)
+    return sha256.hexdigest()
 
 class ResourceManager:
     def __init__(self):
@@ -19,11 +27,13 @@ def __init__(self):
         self._MODEL_LIST = {
             "Pikachu": {
                 "url": "https://github.com/HyperInspire/InspireFace/releases/download/v1.x/Pikachu",
-                "filename": "Pikachu"
+                "filename": "Pikachu",
+                "md5": "f2983a2d884902229c1443fdc921b8e5f49cf2daba8a4f103cd127910dc9e7cd"
             },
             "Megatron": {
                 "url": "https://github.com/HyperInspire/InspireFace/releases/download/v1.x/Megatron",
-                "filename": "Megatron"
+                "filename": "Megatron",
+                "md5": "28f2284c5e7cf53b0e152ff524a416c966ab21e724002643b1304aedc4af6b06"
             }
         }
 
@@ -44,10 +54,14 @@ def get_model(self, name: str, re_download: bool = False) -> str:
         model_info = self._MODEL_LIST[name]
         model_file = self.models_dir / model_info["filename"]
         downloading_flag = model_file.with_suffix('.downloading')
-
+        
         # Check if model exists and is complete
         if model_file.exists() and not downloading_flag.exists() and not re_download:
-            return str(model_file)
+            current_hash = get_file_hash_sha256(model_file)
+            if current_hash == model_info["md5"]:
+                return str(model_file)
+            else:
+                print(f"Model file hash mismatch for '{name}'. Re-downloading...")
 
         # Start download
         try:

python/sample_face_track_from_video.py

@@ -138,7 +138,7 @@ def case_face_tracker_from_video(source, show, out):
 
         if show:
             cv2.imshow("Face Tracker", frame)
-            if cv2.waitKey(1) & 0xFF == ord('q'):
+            if cv2.waitKey(25) & 0xFF == ord('q'):
                 break  # Exit loop if 'q' is pressed.
 
         if out:

tools/get_model_md5.py

@@ -0,0 +1,24 @@
+import hashlib
+import os
+import click
+
+file_list = [
+    "Pikachu",
+    "Megatron",
+]
+
+def get_file_hash_sha256(file_path):
+    sha256 = hashlib.sha256()
+    with open(file_path, 'rb') as f:
+        for chunk in iter(lambda: f.read(4096), b''):
+            sha256.update(chunk)
+    return sha256.hexdigest()
+
+@click.command()
+@click.argument('model_dir', default="test_res/pack/")
+def main(model_dir):
+    for file in file_list:
+        print(f"{file}: {get_file_hash_sha256(os.path.join(model_dir, file))}")
+
+if __name__ == "__main__":
+    main()