[WIP] Update Larky to enable parity (#55)

* Checkpoint * Fixes #28 * Add struct constructor enhancement and allow EvalException to propogate upwards * Fixes #51 * Updated larky to use re2.split() to match the python re module * wrap regular expression in namedtuple * regular expression tests fully working * Allow building java 1.8 with java 11 and allow building with java8 as well * Restructure the larky project. Fixes #18.
verygoodsecurity · Mar 30, 2021 · 377a976 · 377a976
1 parent d62dd8c
commit 377a976
Show file tree

Hide file tree

Showing 39 changed files with 1,734 additions and 234 deletions.
diff --git a/larky/pom.xml b/larky/pom.xml
@@ -10,22 +10,6 @@
     <modelVersion>4.0.0</modelVersion>
 
     <artifactId>larky</artifactId>
-    <build>
-        <plugins>
-            <plugin>
-                <groupId>org.apache.maven.plugins</groupId>
-                <artifactId>maven-compiler-plugin</artifactId>
-                <configuration>
-                    <source>11</source>
-                    <target>11</target>
-                </configuration>
-            </plugin>
-        </plugins>
-    </build>
-
-    <properties>
-        <maven.compiler.parameters>true</maven.compiler.parameters>
-    </properties>
 
     <dependencies>
 

diff --git a/larky/src/main/java/com/verygood/security/larky/Larky.java b/larky/src/main/java/com/verygood/security/larky/Larky.java
@@ -1,6 +1,8 @@
 package com.verygood.security.larky;
 
 
+import static java.nio.charset.StandardCharsets.UTF_8;
+
 import com.google.common.annotations.VisibleForTesting;
 
 import net.starlark.java.eval.EvalException;
@@ -17,16 +19,16 @@
 import net.starlark.java.syntax.SyntaxError;
 
 import java.io.BufferedReader;
+import java.io.BufferedWriter;
 import java.io.FileOutputStream;
 import java.io.IOException;
 import java.io.InputStreamReader;
+import java.nio.charset.Charset;
 import java.nio.file.Files;
 import java.nio.file.Paths;
 import java.nio.file.StandardOpenOption;
 import java.time.Duration;
 
-import static java.nio.charset.StandardCharsets.UTF_8;
-
 public class Larky {
 
   private static final String START_PROMPT = ">> ";
@@ -143,7 +145,11 @@ static int execute(ParserInput input) {
   }
 
   static void writeOutput(String outputFile, StarlarkValue returnValue) throws IOException {
-    Files.writeString(Paths.get(outputFile), returnValue.toString(), StandardOpenOption.CREATE);
+    try (BufferedWriter bw = Files.newBufferedWriter(Paths.get(outputFile),
+                                                     Charset.defaultCharset(),
+                                                     StandardOpenOption.CREATE)) {
+      bw.write(returnValue.toString());
+    }
   }
 
   public static void main(String[] args) throws Exception {

diff --git a/larky/src/main/java/com/verygood/security/larky/ModuleSupplier.java b/larky/src/main/java/com/verygood/security/larky/ModuleSupplier.java
@@ -20,17 +20,18 @@
 import com.google.common.collect.ImmutableMap;
 import com.google.common.collect.ImmutableSet;
 
-import com.verygood.security.larky.nativelib.LarkyGlobals;
-import com.verygood.security.larky.nativelib.PythonBuiltins;
-import com.verygood.security.larky.nativelib.std.C99Math;
-import com.verygood.security.larky.nativelib.std.Hashlib;
+import com.verygood.security.larky.modules.ProtoBufModule;
+import com.verygood.security.larky.modules.globals.LarkyGlobals;
+import com.verygood.security.larky.modules.globals.PythonBuiltins;
+import com.verygood.security.larky.modules.C99MathModule;
+import com.verygood.security.larky.modules.HashModule;
+import com.verygood.security.larky.modules.JsonModule;
+import com.verygood.security.larky.modules.RegexModule;
+import com.verygood.security.larky.modules.testing.AssertionsModule;
+import com.verygood.security.larky.modules.testing.UnittestModule;
 
 import net.starlark.java.annot.StarlarkBuiltin;
 import net.starlark.java.eval.StarlarkValue;
-import com.verygood.security.larky.nativelib.std.Json;
-import com.verygood.security.larky.nativelib.std.Proto;
-import com.verygood.security.larky.nativelib.test.LarkyAssertions;
-import com.verygood.security.larky.nativelib.test.UnittestModule;
 
 import java.util.Map;
 import java.util.function.Function;
@@ -46,15 +47,16 @@ public class ModuleSupplier {
   );
 
   public static final ImmutableSet<StarlarkValue> STD_MODULES = ImmutableSet.of(
-      Json.INSTANCE,
-      Proto.INSTANCE,
-      Hashlib.INSTANCE,
-      C99Math.INSTANCE
+      JsonModule.INSTANCE,
+      ProtoBufModule.INSTANCE,
+      HashModule.INSTANCE,
+      C99MathModule.INSTANCE,
+      RegexModule.INSTANCE
   );
 
   public static final ImmutableSet<StarlarkValue> TEST_MODULES = ImmutableSet.of(
       UnittestModule.INSTANCE,
-      LarkyAssertions.INSTANCE
+      AssertionsModule.INSTANCE
   );
 
   private final Map<String, Object> environment;

diff --git a/...security/larky/nativelib/std/C99Math.java → ...security/larky/modules/C99MathModule.java b/...security/larky/nativelib/std/C99Math.java → ...security/larky/modules/C99MathModule.java
@@ -1,4 +1,4 @@
-package com.verygood.security.larky.nativelib.std;
+package com.verygood.security.larky.modules;
 
 import com.google.common.math.DoubleMath;
 
@@ -14,9 +14,9 @@
     name = "c99math",
     category = "BUILTIN",
     doc = "This module provides access to the mathematical functions defined by the C99 standard")
-public class C99Math implements StarlarkValue {
+public class C99MathModule implements StarlarkValue {
 
-  public static final C99Math INSTANCE = new C99Math();
+  public static final C99MathModule INSTANCE = new C99MathModule();
 
   @StarlarkMethod(name = "PI", doc = "a constant pi", structField = true)
   public StarlarkFloat PI_CONSTANT() {

diff --git a/...security/larky/nativelib/std/Hashlib.java → ...od/security/larky/modules/HashModule.java b/...security/larky/nativelib/std/Hashlib.java → ...od/security/larky/modules/HashModule.java
@@ -1,4 +1,4 @@
-package com.verygood.security.larky.nativelib.std;
+package com.verygood.security.larky.modules;
 
 import com.google.common.hash.HashCode;
 import com.google.common.hash.Hashing;
@@ -14,9 +14,9 @@
     name = "hashlib",
     category = "BUILTIN",
     doc = "This module implements a common interface to many different secure hash and message digest algorithms.")
-public class Hashlib implements StarlarkValue {
+public class HashModule implements StarlarkValue {
 
-  public static final Hashlib INSTANCE = new Hashlib();
+  public static final HashModule INSTANCE = new HashModule();
 
   @StarlarkMethod(
       name = "md5",

diff --git a/...od/security/larky/nativelib/std/Json.java → ...od/security/larky/modules/JsonModule.java b/...od/security/larky/nativelib/std/Json.java → ...od/security/larky/modules/JsonModule.java
@@ -12,10 +12,8 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-package com.verygood.security.larky.nativelib.std;
+package com.verygood.security.larky.modules;
 
-import java.util.Arrays;
-import java.util.Map;
 import net.starlark.java.annot.Param;
 import net.starlark.java.annot.StarlarkBuiltin;
 import net.starlark.java.annot.StarlarkMethod;
@@ -31,25 +29,66 @@
 import net.starlark.java.eval.StarlarkValue;
 import net.starlark.java.eval.Structure;
 
+import java.util.Arrays;
+import java.util.Map;
+
 // Tests at //src/test/java/net/starlark/java/eval:testdata/json.sky
 
 /**
- * Json defines the Starlark {@code json} module, which provides functions for encoding/decoding
+ * JsonModule defines the Starlark {@code json} module, which provides functions for encoding/decoding
  * Starlark values as JSON (https://tools.ietf.org/html/rfc8259).
  */
 @StarlarkBuiltin(
     name = "json",
     category = "core.lib",
     doc = "Module json is a Starlark module of JSON-related functions.")
-public final class Json implements StarlarkValue {
-
-  private Json() {}
+public final class JsonModule implements StarlarkValue {
+
+  //@formatter:off
+  private static final String _METHOD_ENCODE_DOCUMENTATION =
+      "<p>The encode function accepts one required positional argument, which it converts to"
+    + " JSON by cases:\n"
+    + "<ul>\n"
+    + "<li>None, True, and False are converted to 'null', 'true', and 'false',"
+    + " respectively.\n"
+    + "<li>An int, no matter how large, is encoded as a decimal integer. Some decoders"
+    + " may not be able to decode very large integers.\n"
+    + "<li>A float is encoded using a decimal point or an exponent or both, even if its"
+    + " numeric value is an integer. It is an error to encode a non-finite "
+    + " floating-point value.\n"
+    + "<li>A string value is encoded as a JSON string literal that denotes the value. "
+    + " Each unpaired surrogate is replaced by U+FFFD.\n"
+    + "<li>A dict is encoded as a JSON object, in key order.  It is an error if any key"
+    + " is not a string.\n"
+    + "<li>A list or tuple is encoded as a JSON array.\n"
+    + "<li>A struct-like value is encoded as a JSON object, in field name order.\n"
+    + "</ul>\n"
+    + "An application-defined type may define its own JSON encoding.\n"
+    + "Encoding any other value yields an error.\n";
+  //@formatter:on
+
+  //@formatter:off
+  private static final String _METHOD_DECODE_DOCUMENTATION =
+      "The decode function accepts one positional parameter, a JSON string.\n"
+          + "It returns the Starlark value that the string denotes.\n"
+          + "<ul>"
+          + "<li>'null', 'true', and 'false' are parsed as None, True, and False.\n"
+          + "<li>Numbers are parsed as int, or as a float if they contain"
+          + " a decimal point or an exponent. Although JSON has no syntax "
+          + " for non-finite values, very large values may be decoded as infinity.\n"
+          + "<li>a JSON object is parsed as a new unfrozen Starlark dict."
+          + " Keys must be unique strings.\n"
+          + "<li>a JSON array is parsed as new unfrozen Starlark list.\n"
+          + "</ul>\n"
+          + "Decoding fails if x is not a valid JSON encoding.\n";
+  //@formatter:on
+  private JsonModule() {}
 
   /**
    * The module instance. You may wish to add this to your predeclared environment under the name
    * "json".
    */
-  public static final Json INSTANCE = new Json();
+  public static final JsonModule INSTANCE = new JsonModule();
 
   /** An interface for StarlarkValue subclasses to define their own JSON encoding. */
   public interface Encodable {
@@ -100,6 +139,14 @@ public String encode(Object x) throws EvalException {
     return enc.out.toString();
   }
 
+  @StarlarkMethod(
+      name = "dumps",
+      doc =_METHOD_ENCODE_DOCUMENTATION,
+      parameters = {@Param(name = "x")})
+  public String dumps(Object x) throws EvalException {
+    return encode(x);
+  }
+
   private static final class Encoder {
 
     private final StringBuilder out = new StringBuilder();
@@ -297,6 +344,15 @@ public Object decode(String x, StarlarkThread thread) throws EvalException {
     return new Decoder(thread.mutability(), x).decode();
   }
 
+  @StarlarkMethod(
+      name = "loads",
+      doc =_METHOD_DECODE_DOCUMENTATION,
+      parameters = {@Param(name = "x")},
+      useStarlarkThread = true)
+  public Object loads(String x, StarlarkThread thread) throws EvalException {
+    return decode(x, thread);
+  }
+
   private static final class Decoder {
 
     // The decoder necessarily makes certain representation choices

diff --git a/...d/security/larky/nativelib/std/Proto.java → ...ecurity/larky/modules/ProtoBufModule.java b/...d/security/larky/nativelib/std/Proto.java → ...ecurity/larky/modules/ProtoBufModule.java
@@ -1,4 +1,4 @@
-package com.verygood.security.larky.nativelib.std;
+package com.verygood.security.larky.modules;
 
 import net.starlark.java.annot.Param;
 import net.starlark.java.annot.StarlarkBuiltin;
@@ -16,7 +16,7 @@
 import java.util.Map;
 
 /**
- * Proto defines the "proto" Starlark module of utilities for protocol message processing.
+ * ProtoBufModule defines the "proto" Starlark module of utilities for protocol message processing.
  *
  * This file is copied currently from Bazel's:
  *   com.google.devtools.build.lib.packages.StarlarkLibrary
@@ -25,13 +25,13 @@
     name = "proto",
     category = "BUILTIN",
     doc = "A module for protocol message processing.")
-public final class Proto implements StarlarkValue {
+public final class ProtoBufModule implements StarlarkValue {
 
   // Note: in due course this is likely to move to net.starlark.java.lib.proto.
   // Do not add functions that would not belong there!
   // Functions related to running the protocol compiler belong in proto_common.
 
-  public static final Proto INSTANCE = new Proto();
+  public static final ProtoBufModule INSTANCE = new ProtoBufModule();
 
   @StarlarkMethod(
       name = "encode_text",

diff --git a/...rygood/security/larky/nativelib/README.md → ...verygood/security/larky/modules/README.md b/...rygood/security/larky/nativelib/README.md → ...verygood/security/larky/modules/README.md
@@ -10,3 +10,25 @@ In order to ensure that Larky is compatible with Python (besides the obvious `lo
 
 As a result, globals should not be accessed directly. Instead, access Larky native functions and methods via the [`Larky` stdlib namespace](https://github.com/verygoodsecurity/starlarky/blob/master/larky/src/main/resources/stdlib/larky.star). Again, Do not access these libraries directly, but access them through Larky StdLib via the [`larky` namespace](https://github.com/verygoodsecurity/starlarky/blob/master/larky/src/main/resources/stdlib/larky.star). 
 
+### How does one emulate a while loop?
+```python
+    while pos <= finish:
+       # do stuff
+```
+
+emulate it by:
+
+```python
+    for _while_ in range(1000):  # "while pos <= finish" is the same as:
+        if pos > finish:         # for _while_ in range(xxx):
+            break                # if pos > finish: break
+```
+
+Obviously, range can take a larger number to emulate infinity.
+
+### Native Module
+
+Source files for standard library _extension_ modules.
+
+These are *NOT* built-in modules, but are basically extension wrappers that help
+implement the standard library. 
diff --git a/larky/src/main/java/com/verygood/security/larky/modules/RegexModule.java b/larky/src/main/java/com/verygood/security/larky/modules/RegexModule.java
@@ -0,0 +1,33 @@
+package com.verygood.security.larky.modules;
+
+import com.verygood.security.larky.modules.re.RegexPattern;
+
+import net.starlark.java.annot.StarlarkBuiltin;
+import net.starlark.java.annot.StarlarkMethod;
+import net.starlark.java.eval.StarlarkValue;
+
+
+@StarlarkBuiltin(
+    name = "re2j",
+    category = "BUILTIN",
+    doc = "This module provides access to the linear regular expression matching engine.\n" +
+        "\n" +
+        "This package provides an implementation of regular expression matching based on Russ Cox's linear-time RE2 algorithm.\n" +
+        "\n" +
+        "The API presented by com.google.re2j mimics that of java.util.regex.Matcher and java.util.regex.Pattern. While not identical, they are similar enough that most users can switch implementations simply by changing their imports.\n" +
+        "\n" +
+        "The syntax of the regular expressions accepted is the same general syntax used by Perl, Python, and other languages. More precisely, it is the syntax accepted by the C++ and Go implementations of RE2 described at https://github.com/google/re2/wiki/Syntax, except for \\C (match any byte), which is not supported because in this implementation, the matcher's input is conceptually a stream of Unicode code points, not bytes.\n" +
+        "\n" +
+        "The current API is rather small and intended for compatibility with java.util.regex, but the underlying implementation supports some additional features, such as the ability to process input character streams encoded as UTF-8 byte arrays. These may be exposed in a future release if there is sufficient interest." +
+        "\n" +
+        "More on syntax here: https://github.com/google/re2/wiki/Syntax")
+public class RegexModule implements StarlarkValue {
+
+  public static final RegexModule INSTANCE = new RegexModule();
+
+  private static final RegexPattern _Pattern = new RegexPattern();
+
+  @StarlarkMethod(name = "Pattern", doc = "pattern", structField = true)
+  public static RegexPattern Pattern() { return _Pattern; }
+
+}
diff --git a/...ecurity/larky/nativelib/LarkyGlobals.java → ...y/larky/modules/globals/LarkyGlobals.java b/...ecurity/larky/nativelib/LarkyGlobals.java → ...y/larky/modules/globals/LarkyGlobals.java
@@ -1,9 +1,10 @@
-package com.verygood.security.larky.nativelib;
+package com.verygood.security.larky.modules.globals;
 
 import com.verygood.security.larky.annot.Library;
 import com.verygood.security.larky.annot.StarlarkConstructor;
-import com.verygood.security.larky.stdtypes.structs.Partial;
-import com.verygood.security.larky.stdtypes.structs.SimpleStruct;
+import com.verygood.security.larky.modules.types.Property;
+import com.verygood.security.larky.modules.types.Partial;
+import com.verygood.security.larky.modules.types.structs.SimpleStruct;
 
 import net.starlark.java.annot.Param;
 import net.starlark.java.annot.ParamType;
@@ -107,8 +108,8 @@ public Partial partial(StarlarkFunction function, Tuple args, Dict<String, Objec
           @Param(name = "kwargs", defaultValue = "{}", doc = "Dictionary of arguments."),
       useStarlarkThread = true
     )
-  public LarkyProperty property(StarlarkCallable getter, Object setter, Tuple args, Dict<String, Object> kwargs, StarlarkThread thread)  {
-    return LarkyProperty.builder()
+  public Property property(StarlarkCallable getter, Object setter, Tuple args, Dict<String, Object> kwargs, StarlarkThread thread)  {
+      return Property.builder()
         .thread(thread)
         .fget(getter)
         .fset(setter != Starlark.NONE ? (StarlarkCallable) setter : null)