feat(web-scraping): introduce Web Page Content trackers

secutils-dev · Nov 17, 2023 · b879bf1 · b879bf1
1 parent a18521b
commit b879bf1
Show file tree

Hide file tree

Showing 39 changed files with 5,203 additions and 5,144 deletions.
diff --git a/.sqlx/query-0f9416568287e47a6eeb632ba3c49a2763bf20c824ec0ec4882520c8dbc20c51.json b/.sqlx/query-0f9416568287e47a6eeb632ba3c49a2763bf20c824ec0ec4882520c8dbc20c51.json
diff --git a/...c17335c4e8601a03b98bd540c095c983049d.json → ...d4ae3c6a671b50724017b37e1a42e47fedd3.json b/...c17335c4e8601a03b98bd540c095c983049d.json → ...d4ae3c6a671b50724017b37e1a42e47fedd3.json
diff --git a/.sqlx/query-74922f5fa35e20e168eda9a00aa5a004572e0c75abad03034bc65216954266d7.json b/.sqlx/query-74922f5fa35e20e168eda9a00aa5a004572e0c75abad03034bc65216954266d7.json
diff --git a/.sqlx/query-c7681bf27d43ae8b64967e4ef1006816a207886df229136a8bdbc8d71625cc3b.json b/.sqlx/query-c7681bf27d43ae8b64967e4ef1006816a207886df229136a8bdbc8d71625cc3b.json
diff --git a/.sqlx/query-cd134aef09a6920a1e327ab9962735cf3f0b4faca50db9cd17555018ce7a080a.json b/.sqlx/query-cd134aef09a6920a1e327ab9962735cf3f0b4faca50db9cd17555018ce7a080a.json
diff --git a/...50b82e60ba5c36dba9c113865c4d25aefdf8.json → ...2a28eb24e034bbec16e76cb95869c4e0a8bd.json b/...50b82e60ba5c36dba9c113865c4d25aefdf8.json → ...2a28eb24e034bbec16e76cb95869c4e0a8bd.json
diff --git a/assets/templates/web_page_content_tracker_changes_email.hbs b/assets/templates/web_page_content_tracker_changes_email.hbs
@@ -0,0 +1,33 @@
+<!DOCTYPE html>
+<html lang="en">
+<head>
+  <title>"{{tracker_name}}" content tracker detected changes</title>
+  <meta charset="utf-8">
+  <meta name="viewport" content="width=device-width, initial-scale=1">
+  {{> email_styles}}
+  <style>
+    .navigate-link {
+      display: block;
+      width: 250px;
+      margin: auto;
+      padding: 10px 20px;
+      text-align: center;
+      text-decoration: none;
+      color: #5e1d3f;
+      background-color: #fed047;
+      border-radius: 5px;
+      font-weight: bold;
+    }
+  </style>
+</head>
+<body>
+<div class="container">
+  <h1>"{{tracker_name}}" content tracker detected changes: "{{content}}"</h1>
+  <p>To learn more, visit the <b>Resources Trackers</b> page:</p>
+  <a class="navigate-link" href="{{back_link}}">Web Scraping → Resources Trackers</a>
+  <p>If the button above doesn't work, you can navigate to the following URL directly: </p>
+  <p>{{back_link}}</p>
+  <a href="{{home_link}}"><img src="cid:secutils-logo" alt="Secutils.dev logo" width="89" height="14" /></a>
+</div>
+</body>
+</html>
diff --git a/...lates/resources_tracker_changes_email.hbs → ..._page_resources_tracker_changes_email.hbs b/...lates/resources_tracker_changes_email.hbs → ..._page_resources_tracker_changes_email.hbs
diff --git a/migrations/20231109215110_web_scraping.sql b/migrations/20231109215110_web_scraping.sql
@@ -10,13 +10,13 @@ CREATE TABLE IF NOT EXISTS user_data_web_scraping_trackers
     id              BLOB PRIMARY KEY,
     name            TEXT NOT NULL COLLATE NOCASE,
     url             TEXT NOT NULL,
-    kind            INTEGER NOT NULL,
+    kind            BLOB NOT NULL,
     schedule        TEXT,
     job_id          BLOB UNIQUE,
     data            BLOB NOT NULL,
     created_at      INTEGER NOT NULL,
     user_id         INTEGER NOT NULL REFERENCES users(id) ON DELETE CASCADE,
-    UNIQUE          (name, user_id)
+    UNIQUE          (name, kind, user_id)
 ) STRICT;
 
 -- Create table to store web page trackers history.

diff --git a/src/config/scheduler_jobs_config.rs b/src/config/scheduler_jobs_config.rs
@@ -3,10 +3,10 @@ use cron::Schedule;
 /// Configuration for the Secutils.dev scheduler jobs.
 #[derive(Clone, Debug)]
 pub struct SchedulerJobsConfig {
-    /// The schedule to use for the `ResourcesTrackersSchedule` job.
-    pub resources_trackers_schedule: Schedule,
-    /// The schedule to use for the `ResourcesTrackersFetch` job.
-    pub resources_trackers_fetch: Schedule,
+    /// The schedule to use for the `WebPageTrackersSchedule` job.
+    pub web_page_trackers_schedule: Schedule,
+    /// The schedule to use for the `WebPageTrackersFetch` job.
+    pub web_page_trackers_fetch: Schedule,
     /// The schedule to use for the `NotificationsSend` job.
     pub notifications_send: Schedule,
 }
diff --git a/src/main.rs b/src/main.rs
@@ -100,21 +100,21 @@ fn process_command(version: &str, matches: ArgMatches) -> Result<(), anyhow::Err
             search_index_version: 3,
         },
         jobs: SchedulerJobsConfig {
-            resources_trackers_schedule: matches
-                .get_one::<String>("JOBS_RESOURCES_TRACKERS_SCHEDULE")
+            web_page_trackers_schedule: matches
+                .get_one::<String>("JOBS_WEB_PAGE_TRACKERS_SCHEDULE")
                 .ok_or_else(|| {
-                    anyhow!("<JOBS_RESOURCES_TRACKERS_SCHEDULE> argument is not provided.")
+                    anyhow!("<JOBS_WEB_PAGE_TRACKERS_SCHEDULE> argument is not provided.")
                 })
                 .and_then(|schedule| {
                     Schedule::try_from(schedule.as_str())
-                        .with_context(|| "Cannot parse resources trackers schedule job schedule.")
+                        .with_context(|| "Cannot parse web page trackers schedule job schedule.")
                 })?,
-            resources_trackers_fetch: matches
-                .get_one::<String>("JOBS_RESOURCES_TRACKERS_FETCH")
-                .ok_or_else(|| anyhow!("<JOBS_RESOURCES_TRACKERS_FETCH> argument is not provided."))
+            web_page_trackers_fetch: matches
+                .get_one::<String>("JOBS_WEB_PAGE_TRACKERS_FETCH")
+                .ok_or_else(|| anyhow!("<JOBS_WEB_PAGE_TRACKERS_FETCH> argument is not provided."))
                 .and_then(|schedule| {
                     Schedule::try_from(schedule.as_str())
-                        .with_context(|| "Cannot parse resources trackers fetch job schedule.")
+                        .with_context(|| "Cannot parse web page trackers fetch job schedule.")
                 })?,
             notifications_send: matches
                 .get_one::<String>("JOBS_NOTIFICATIONS_SEND")
@@ -254,20 +254,20 @@ fn main() -> Result<(), anyhow::Error> {
                 .help("The URL to access the Web Scraper component."),
         )
         .arg(
-            Arg::new("JOBS_RESOURCES_TRACKERS_SCHEDULE")
-                .long("jobs-resources-trackers-schedule")
+            Arg::new("JOBS_WEB_PAGE_TRACKERS_SCHEDULE")
+                .long("jobs-web-page-trackers-schedule")
                 .global(true)
-                .env("SECUTILS_JOBS_RESOURCES_TRACKERS_SCHEDULE")
+                .env("SECUTILS_JOBS_WEB_PAGE_TRACKERS_SCHEDULE")
                 .default_value("0 * * * * * *")
-                .help("The cron schedule to use for the resources trackers schedule job."),
+                .help("The cron schedule to use for the web page trackers schedule job."),
         )
         .arg(
-            Arg::new("JOBS_RESOURCES_TRACKERS_FETCH")
-                .long("jobs-resources-trackers-fetch")
+            Arg::new("JOBS_WEB_PAGE_TRACKERS_FETCH")
+                .long("jobs-web-page-trackers-fetch")
                 .global(true)
-                .env("SECUTILS_JOBS_RESOURCES_TRACKERS_FETCH")
+                .env("SECUTILS_JOBS_WEB_PAGE_TRACKERS_FETCH")
                 .default_value("0 * * * * * *")
-                .help("The cron schedule to use for the resources trackers fetch job."),
+                .help("The cron schedule to use for the web page trackers fetch job."),
         ).arg(
         Arg::new("JOBS_NOTIFICATIONS_SEND")
             .long("jobs-notifications-send")
@@ -480,8 +480,8 @@ mod tests {
                 search_index_version: 3,
             },
             jobs: SchedulerJobsConfig {
-                resources_trackers_schedule: Schedule::try_from("0 * 0 * * * *")?,
-                resources_trackers_fetch: Schedule::try_from("0 * 1 * * * *")?,
+                web_page_trackers_schedule: Schedule::try_from("0 * 0 * * * *")?,
+                web_page_trackers_fetch: Schedule::try_from("0 * 1 * * * *")?,
                 notifications_send: Schedule::try_from("0 * 2 * * * *")?,
             },
         })