Add job document comparator to ignore pre-signed difference in s3 url (…

…#473) * Modified condition to create default sample shadow directory * Updated versions in build.yml for aws credential fetch * Updated build.yml to revert version of configure-aws-credentials to v1 * Updated aws-credentials fetching mechanism version in build.yml * Add job document comparator to ignore pre-signed difference in s3 url * Remove unit test for the compareJobsDocument and make it private --------- Co-authored-by: Ubuntu <[email protected]>
awslabs · Nov 14, 2024 · ce63afe · ce63afe
1 parent 8ee8047
commit ce63afe
Show file tree

Hide file tree

Showing 3 changed files with 58 additions and 5 deletions.
diff --git a/source/jobs/JobsFeature.cpp b/source/jobs/JobsFeature.cpp
@@ -527,9 +527,50 @@ void JobsFeature::copyJobsNotification(Iotjobs::JobExecutionData job)
     latestJobsNotification.ExecutionNumber = job.ExecutionNumber.value();
 }
 
+
+bool JobsFeature::compareJobDocuments(const Aws::Crt::JsonObject& job1, const Aws::Crt::JsonObject& job2) {
+    std::string str1 = job1.View().WriteCompact().c_str();
+    std::string str2 = job2.View().WriteCompact().c_str();
+
+    // Regular expression to match S3 URLs and capture the non-presigned parts
+    std::regex s3UrlRegex(R"((https://[^.\s"]+\.s3[.-](?:[^.\s"]+\.)?amazonaws\.com/[^?\s"]+)(\?[^"\s]+)?)");
+
+    // Function to replace only the pre-signed portion of S3 URLs
+    auto processPresignedUrls = [&s3UrlRegex](std::string& s) -> int {
+        int count = 0;
+        std::string result;
+        std::sregex_iterator it(s.begin(), s.end(), s3UrlRegex);
+        std::sregex_iterator end;
+
+        size_t lastPos = 0;
+        for (; it != end; ++it) {
+            count++;
+            result += s.substr(lastPos, it->position() - lastPos);
+            result += it->str(1);
+            // Add the non-presigned part of the URL
+            result += "?aws:iot:s3-presigned-suffix:PLACEHOLDER"; 
+            // Replace only the presigned portion
+            lastPos = it->position() + it->length();
+        }
+        result += s.substr(lastPos);
+
+        s = result;
+        return count;
+    };
+
+    int count1 = processPresignedUrls(str1);
+    int count2 = processPresignedUrls(str2);
+
+    // If the number of pre-signed URLs differs, the documents are different
+    if (count1 != count2) {
+        return false;
+    }
+    return str1 == str2;
+}
 bool JobsFeature::isDuplicateNotification(JobExecutionData job)
 {
     unique_lock<mutex> readLatestNotificationLock(latestJobsNotificationLock);
+
     if (!latestJobsNotification.JobId.has_value())
     {
         // We have not seen a job yet
@@ -543,10 +584,9 @@ bool JobsFeature::isDuplicateNotification(JobExecutionData job)
         return false;
     }
 
-    if (strcmp(
-            job.JobDocument.value().View().WriteCompact().c_str(),
-            latestJobsNotification.JobDocument.value().View().WriteCompact().c_str()) != 0)
-    {
+    if (!compareJobDocuments(
+            job.JobDocument.value(),
+            latestJobsNotification.JobDocument.value())) {
         LOG_DEBUG(TAG, "Job document differs");
         return false;
     }

diff --git a/source/jobs/JobsFeature.h b/source/jobs/JobsFeature.h
@@ -15,6 +15,7 @@
 #include "IotJobsClientWrapper.h"
 #include "JobDocument.h"
 #include "JobEngine.h"
+#include <regex>
 
 namespace Aws
 {
@@ -313,6 +314,17 @@ namespace Aws
                      */
                     bool isDuplicateNotification(Iotjobs::JobExecutionData job);
 
+                    /**
+                     * \brief Compares two job documents, ignoring differences in pre-signed URLs.
+                     *
+                     * @param job1 The first job document as a JsonObject
+                     * @param job2 The second job document as a JsonObject
+                     * @return true if the documents are equivalent (ignoring pre-signed URLs), false otherwise
+                     */
+
+                    bool compareJobDocuments(const Aws::Crt::JsonObject& job1, const Aws::Crt::JsonObject& job2);
+
+
                     /**
                      * \brief Stores information about a job notification
                      *

diff --git a/test/jobs/TestJobsFeature.cpp b/test/jobs/TestJobsFeature.cpp
@@ -15,6 +15,7 @@
 #include <aws/iotjobs/RejectedError.h>
 #include <aws/iotjobs/StartNextJobExecutionResponse.h>
 #include <aws/iotjobs/UpdateJobExecutionResponse.h>
+#include <aws/crt/Api.h>
 
 using namespace std;
 using namespace testing;
@@ -709,4 +710,4 @@ TEST_F(TestJobsFeature, InvalidJobDocument)
 
     jobsMock->init(std::shared_ptr<Mqtt::MqttConnection>(), notifier, config);
     jobsMock->invokeRunJobs();
-}
+}