chore: improve version fixture generators for local usage (#953)

This updates the generators used to build fixtures for `semantic` in a
few different ways particularly if you're running them locally:
- you can use `VERSION_GENERATOR_PRINT` to control if successes,
failures, or everything is outputted
  - if any comparisons fail, then the generator will exit with code 1
- the generated fixture file will now assuredly have a final newline, in
accordance with our editor config

---

this is the first of a couple of updates to the generators based on what
I've already done in `osv-detector`

scripts/generators/GenerateMavenVersions.java

@@ -133,12 +133,14 @@ public static boolean compareVers(String version1, String op, String version2) {
     throw new RuntimeException("unsupported comparison operator " + op);
   }
 
-  public static void compareVersions(List<String> lines, String select) {
-    lines.forEach(line -> {
+  public static boolean compareVersions(List<String> lines, String select) {
+    boolean didAnyFail = false;
+
+    for(String line : lines) {
       line = line.trim();
 
       if(line.isEmpty() || line.startsWith("#") || line.startsWith("//")) {
-        return;
+        continue;
       }
 
       String[] parts = line.split(" ");
@@ -148,22 +150,28 @@ public static void compareVersions(List<String> lines, String select) {
 
       boolean r = compareVers(v1, op, v2);
 
+      if(!r) {
+        didAnyFail = true;
+      }
+
       if(select.equals("failures") && r) {
-        return;
+        continue;
       }
 
       if(select.equals("successes") && !r) {
-        return;
+        continue;
       }
 
       String color = r ? "\033[92m" : "\033[91m";
       String rs = r ? "T" : "F";
 
       System.out.printf("%s%s\033[0m: \033[93m%s\033[0m\n", color, rs, line);
-    });
+    }
+
+    return didAnyFail;
   }
 
-  public static void compareVersionsInFile(String filepath, String select) throws IOException {
+  public static boolean compareVersionsInFile(String filepath, String select) throws IOException {
     List<String> lines = new ArrayList<>();
 
     try(BufferedReader br = new BufferedReader(new FileReader(filepath))) {
@@ -175,7 +183,7 @@ public static void compareVersionsInFile(String filepath, String select) throws
       }
     }
 
-    compareVersions(lines, select);
+    return compareVersions(lines, select);
   }
 
   public static List<String> generateVersionCompares(List<String> versions) {
@@ -198,12 +206,30 @@ public static List<String> generatePackageCompares(Map<String, List<String>> pac
              .collect(Collectors.toList());
   }
 
+  public static String getSelectFilter() {
+    // set this to either "failures" or "successes" to only have those comparison results
+    // printed; setting it to anything else will have all comparison results printed
+    String value = System.getenv("VERSION_GENERATOR_PRINT");
+
+    if(value == null) {
+      return "failures";
+    }
+
+    return value;
+  }
+
   public static void main(String[] args) throws IOException {
-    String outfile = "maven-versions-generated.txt";
+    String outfile = "internal/semantic/fixtures/maven-versions-generated.txt";
     Map<String, List<String>> packages = fetchPackageVersions();
 
     writeToFile(outfile, generatePackageCompares(packages));
 
-    compareVersionsInFile(outfile, "failures");
+    String show = getSelectFilter();
+
+    boolean didAnyFail = compareVersionsInFile(outfile, show);
+
+    if(didAnyFail) {
+      System.exit(1);
+    }
   }
 }

scripts/generators/generate-debian-versions.py

@@ -1,10 +1,11 @@
 #!/usr/bin/env python3
 
-import zipfile
-import operator
-import urllib.request
 import json
+import operator
+import os
 import subprocess
+import urllib.request
+import zipfile
 from pathlib import Path
 
 
@@ -121,6 +122,8 @@ def compare(v1, relate, v2):
 
 
 def compare_versions(lines, select="all"):
+  has_any_failed = False
+
   for line in lines:
     line = line.strip()
 
@@ -131,6 +134,9 @@ def compare_versions(lines, select="all"):
 
     r = compare(DebianVersion(v1), op, DebianVersion(v2))
 
+    if not r:
+      has_any_failed = r
+
     if select == "failures" and r:
       continue
 
@@ -140,12 +146,13 @@ def compare_versions(lines, select="all"):
     color = '\033[92m' if r else '\033[91m'
     rs = "T" if r else "F"
     print(f"{color}{rs}\033[0m: \033[93m{line}\033[0m")
+  return has_any_failed
 
 
 def compare_versions_in_file(filepath, select="all"):
   with open(filepath) as f:
     lines = f.readlines()
-    compare_versions(lines, select)
+    return compare_versions(lines, select)
 
 
 def generate_version_compares(versions):
@@ -184,5 +191,13 @@ def fetch_packages_versions():
 packs = fetch_packages_versions()
 with open(outfile, "w") as f:
   f.writelines(generate_package_compares(packs))
+  f.write("\n")
+
+# set this to either "failures" or "successes" to only have those comparison results
+# printed; setting it to anything else will have all comparison results printed
+show = os.environ.get("VERSION_GENERATOR_PRINT", "failures")
+
+did_any_fail = compare_versions_in_file(outfile, show)
 
-compare_versions_in_file(outfile, "failures")
+if did_any_fail:
+  sys.exit(1)

scripts/generators/generate-packagist-versions.php

@@ -132,8 +132,10 @@ function generatePackageCompares(array $packages): array
   return array_merge(...$comparisons);
 }
 
-function compareVersions(array $lines, string $select = "all"): void
+function compareVersions(array $lines, string $select = "all"): bool
 {
+  $hasAnyFailed = false;
+
   foreach ($lines as $line) {
     $line = trim($line);
 
@@ -145,6 +147,10 @@ function compareVersions(array $lines, string $select = "all"): void
 
     $r = version_compare($v1, $v2, $op);
 
+    if (!$r) {
+      $hasAnyFailed = true;
+    }
+
     if ($select === "failures" && $r === true) {
       continue;
     }
@@ -157,13 +163,23 @@ function compareVersions(array $lines, string $select = "all"): void
     $rs    = $r ? "T" : "F";
     echo "$color$rs\033[0m: \033[93m$line\033[0m\n";
   }
+
+  return $hasAnyFailed;
 }
 
-$outfile = "packagist-versions-generated.txt";
+$outfile = "internal/semantic/fixtures/packagist-versions-generated.txt";
 
 /** @noinspection PhpUnhandledExceptionInspection */
 $packages = fetchPackageVersions();
 
-file_put_contents($outfile, implode("\n", array_unique(generatePackageCompares($packages))));
+file_put_contents($outfile, implode("\n", array_unique(generatePackageCompares($packages))) . "\n");
 
-compareVersions(explode("\n", file_get_contents($outfile)), "failures");
+// set this to either "failures" or "successes" to only have those comparison results
+// printed; setting it to anything else will have all comparison results printed
+$show = getenv("VERSION_GENERATOR_PRINT") ?: "failures";
+
+$didAnyFail = compareVersions(explode("\n", file_get_contents($outfile)), $show);
+
+if ($didAnyFail === true) {
+  exit(1);
+}

scripts/generators/generate-pypi-versions.py

@@ -1,10 +1,11 @@
 #!/usr/bin/env python3
 
-import packaging.version
-import zipfile
+import json
 import operator
+import os
+import packaging.version
 import urllib.request
-import json
+import zipfile
 
 
 # this requires you run "pip install packaging" - have to be careful about versions too
@@ -43,6 +44,8 @@ def compare(v1, relate, v2):
 
 
 def compare_versions(lines, select="all"):
+  has_any_failed = False
+
   for line in lines:
     line = line.strip()
 
@@ -53,6 +56,9 @@ def compare_versions(lines, select="all"):
 
     r = compare(packaging.version.parse(v1), op, packaging.version.parse(v2))
 
+    if not r:
+      has_any_failed = True
+
     if select == "failures" and r:
       continue
 
@@ -62,12 +68,13 @@ def compare_versions(lines, select="all"):
     color = '\033[92m' if r else '\033[91m'
     rs = "T" if r else "F"
     print(f"{color}{rs}\033[0m: \033[93m{line}\033[0m")
+  return has_any_failed
 
 
 def compare_versions_in_file(filepath, select="all"):
   with open(filepath) as f:
     lines = f.readlines()
-    compare_versions(lines, select)
+    return compare_versions(lines, select)
 
 
 def generate_version_compares(versions):
@@ -106,5 +113,13 @@ def fetch_packages_versions():
 packs = fetch_packages_versions()
 with open(outfile, "w") as f:
   f.writelines(generate_package_compares(packs))
+  f.write("\n")
+
+# set this to either "failures" or "successes" to only have those comparison results
+# printed; setting it to anything else will have all comparison results printed
+show = os.environ.get("VERSION_GENERATOR_PRINT", "failures")
+
+did_any_fail = compare_versions_in_file(outfile, show)
 
-compare_versions_in_file(outfile, "failures")
+if did_any_fail:
+  sys.exit(1)

scripts/generators/generate-rubygems-versions.rb

@@ -35,7 +35,10 @@ def compare_version(v1, op, v2)
 end
 
 # @param [Array<String>] lines
+# @return [Boolean]
 def compare_versions(lines, select = :all)
+  has_any_failed = false
+
   lines.each do |line|
     line = line.strip
 
@@ -48,13 +51,17 @@ def compare_versions(lines, select = :all)
 
     r = compare_version(v1, op, v2)
 
+    has_any_failed = true unless r
+
     next if select == :failures && r == true
     next if select == :successes && r != true
 
     color = r ? "\033[92m" : "\033[91m"
     rs = r ? "T" : "F"
     puts "#{color}#{rs}\033[0m: \033[93m#{line}\033[0m"
   end
+
+  has_any_failed
 end
 
 def compare_versions_in_file(filepath, select = :all)
@@ -96,6 +103,12 @@ def fetch_packages_versions
 
 packs = fetch_packages_versions
 
-File.open(outfile, "w") { |f| f.write(generate_package_compares(packs).uniq.join("\n")) }
+File.open(outfile, "w") { |f| f.write(generate_package_compares(packs).uniq.join("\n") + "\n") }
+
+# set this to either "failures" or "successes" to only have those comparison results
+# printed; setting it to anything else will have all comparison results printed
+show = ENV.fetch("VERSION_GENERATOR_PRINT", :failures).to_sym
+
+did_any_fail = compare_versions_in_file(outfile, show)
 
-compare_versions_in_file(outfile, :failures)
+exit(1) if did_any_fail