Fix Ubuntu AWS kernel crawler (again)

We skipped some linux-aws-headers packages before. Also, collapsing the regexes halves the crawler time.
draios · Mar 11, 2020 · ef8b0e0 · ef8b0e0
1 parent 839cd51
commit ef8b0e0
Showing 1 changed file with 2 additions and 16 deletions.
diff --git a/probe-builder/kernel-crawler.py b/probe-builder/kernel-crawler.py
@@ -133,29 +133,15 @@ def sqlite_column(row, col):
             "root" : "https://mirrors.kernel.org/ubuntu/pool/main/l/",
             "discovery_pattern" : "/html/body//a[regex:test(@href, 'linux-aws.*/')]/@href",
             "subdirs" : [""],
-            "page_pattern" : "/html/body//a[regex:test(@href, '^linux-(image|(aws-.*)?headers)-[3-9].*-aws.*amd64.deb$')]/@href"
-        },
-
-        {
-            "root" : "https://mirrors.kernel.org/ubuntu/pool/main/l/",
-            "discovery_pattern" : "/html/body//a[regex:test(@href, 'linux-aws.*/')]/@href",
-            "subdirs" : [""],
-            "page_pattern" : "/html/body//a[regex:test(@href, '^linux-(aws-.*)?headers-[3-9].*_all.deb$')]/@href"
+            "page_pattern" : "/html/body//a[regex:test(@href, '^linux-(image|(aws-.*)?headers|modules)-[3-9].*(all|amd64).deb$')]/@href"
         },
 
         {
             "root" : "http://security.ubuntu.com/ubuntu/pool/main/l/",
             "discovery_pattern" : "/html/body//a[regex:test(@href, 'linux-aws.*/')]/@href",
             "subdirs" : [""],
-            "page_pattern" : "/html/body//a[regex:test(@href, '^linux-(image|(aws-.*)?headers)-[3-9].*-aws.*amd64.deb$')]/@href"
+            "page_pattern" : "/html/body//a[regex:test(@href, '^linux-(image|(aws-.*)?headers|modules)-[3-9].*(all|amd64).deb$')]/@href"
         },
-
-        {
-            "root" : "http://security.ubuntu.com/ubuntu/pool/main/l/",
-            "discovery_pattern" : "/html/body//a[regex:test(@href, 'linux-aws.*/')]/@href",
-            "subdirs" : [""],
-            "page_pattern" : "/html/body//a[regex:test(@href, '^linux-modules-[3-9].*-aws.*amd64.deb$')]/@href"
-        }
     ],
 
     "Fedora" : [