Implement a wrapper around c-kzg jni bindings (#6520)

* Implement a wrapper around c-kzg jni bindings

* Add more tests + fix empty commitments/blobs

* Don't copy trusted setup to temporary file when not needed

* Fix windows path translation

infrastructure/kzg/build.gradle

@@ -1,5 +1,12 @@
+repositories {
+  maven { url "https://artifacts.consensys.net/public/maven/maven/" }
+}
+
 dependencies {
   implementation 'org.apache.tuweni:tuweni-bytes'
   implementation 'org.apache.tuweni:tuweni-ssz'
+  implementation("tech.pegasys:jc-kzg-4844:develop")
   implementation 'commons-io:commons-io'
+
+  implementation project(":infrastructure:io")
 }

infrastructure/kzg/src/main/java/tech/pegasys/teku/kzg/KZG.java

@@ -0,0 +1,131 @@
+/*
+ * Copyright ConsenSys Software Inc., 2022
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
+ * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations under the License.
+ */
+
+package tech.pegasys.teku.kzg;
+
+import java.io.File;
+import java.io.FileNotFoundException;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.net.URISyntaxException;
+import java.net.URL;
+import java.nio.file.Paths;
+import java.util.List;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
+import org.apache.tuweni.bytes.Bytes;
+import org.apache.tuweni.bytes.Bytes32;
+import tech.pegasys.teku.infrastructure.io.resource.ResourceLoader;
+import tech.pegasys.teku.kzg.impl.KZG4844;
+import tech.pegasys.teku.kzg.impl.KzgException;
+import tech.pegasys.teku.kzg.impl.ckzg.CkzgLoader;
+
+/**
+ * Implements the standard KZG functions needed for the EIP-4844 specification.
+ *
+ * <p>This package strives to implement the KZG standard as used in the Eth2 specification and is
+ * the entry-point for all KZG operations in Teku. Do not rely on any of the classes used by this
+ * one conforming to the specification or standard.
+ */
+public final class KZG {
+  private static final Logger LOG = LogManager.getLogger();
+  private static final String FILE_SCHEME = "file";
+
+  private static KZG4844 kzgImpl;
+
+  static {
+    resetKzgImplementation();
+  }
+
+  public static void setKzgImplementation(final KZG4844 kzgImpl) {
+    KZG.kzgImpl = kzgImpl;
+  }
+
+  public static void resetKzgImplementation() {
+    if (CkzgLoader.INSTANCE.isPresent()) {
+      kzgImpl = CkzgLoader.INSTANCE.get();
+      LOG.info("KZG: loaded CKZG library");
+    } else {
+      throw new KzgException("Failed to load CKZG library.");
+    }
+  }
+
+  public static KZG4844 getKzgImpl() {
+    return kzgImpl;
+  }
+
+  public static void resetTrustedSetup() {
+    try {
+      kzgImpl.resetTrustedSetup();
+    } catch (final KzgException ex) {
+      LOG.trace("Trying to reset KZG trusted setup which is not loaded");
+    }
+  }
+
+  public static void loadTrustedSetup(final URL url) {
+    final String filePath;
+    try {
+      filePath = copyResourceToTempFileIfNeeded(url);
+    } catch (final IOException ex) {
+      throw new KzgException(
+          String.format("Failed to copy trusted setup '%s' to temporary file", url), ex);
+    }
+    kzgImpl.loadTrustedSetup(filePath);
+  }
+
+  public static KZGProof computeAggregateKzgProof(final List<Bytes> blobs) {
+    return kzgImpl.computeAggregateKzgProof(blobs);
+  }
+
+  public static boolean verifyAggregateKzgProof(
+      final List<Bytes> blobs, final List<KZGCommitment> kzgCommitments, final KZGProof kzgProof) {
+    return kzgImpl.verifyAggregateKzgProof(blobs, kzgCommitments, kzgProof);
+  }
+
+  public static KZGCommitment blobToKzgCommitment(final Bytes blob) {
+    return kzgImpl.blobToKzgCommitment(blob);
+  }
+
+  public static boolean verifyKzgProof(
+      final KZGCommitment kzgCommitment,
+      final Bytes32 z,
+      final Bytes32 y,
+      final KZGProof kzgProof) {
+    return kzgImpl.verifyKzgProof(kzgCommitment, z, y, kzgProof);
+  }
+
+  private static String copyResourceToTempFileIfNeeded(final URL url) throws IOException {
+    try {
+      if (url.toURI().getScheme().equals(FILE_SCHEME)) {
+        // Platform-agnostic safe way to get path
+        return Paths.get(url.toURI()).toFile().getPath();
+      }
+    } catch (final URISyntaxException ex) {
+      throw new KzgException(String.format("%s is incorrect file path", url), ex);
+    }
+
+    final Bytes resource =
+        ResourceLoader.urlOrFile("application/octet-stream")
+            .loadBytes(url.toExternalForm())
+            .orElseThrow(() -> new FileNotFoundException("Not found"));
+
+    File temp = File.createTempFile("resource", ".tmp");
+    temp.deleteOnExit();
+
+    try (final FileOutputStream out = new FileOutputStream(temp)) {
+      out.write(resource.toArray());
+    }
+
+    return temp.getAbsolutePath();
+  }
+}

infrastructure/kzg/src/main/java/tech/pegasys/teku/kzg/impl/KZG4844.java

@@ -0,0 +1,80 @@
+/*
+ * Copyright ConsenSys Software Inc., 2022
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
+ * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations under the License.
+ */
+
+package tech.pegasys.teku.kzg.impl;
+
+import java.util.List;
+import org.apache.tuweni.bytes.Bytes;
+import org.apache.tuweni.bytes.Bytes32;
+import tech.pegasys.teku.kzg.KZGCommitment;
+import tech.pegasys.teku.kzg.KZGProof;
+
+public interface KZG4844 {
+
+  /**
+   * Free the current trusted setup
+   *
+   * @throws KzgException if no trusted setup has been loaded.
+   */
+  void resetTrustedSetup() throws KzgException;
+
+  /**
+   * Loads the trusted setup from a file. Once loaded, the same setup will be used for all the
+   * calls. To load a new setup, reset the current one by calling {@link #resetTrustedSetup()} and
+   * then load the new one. If no trusted setup has been loaded, all other API calls will throw an
+   * exception.
+   *
+   * @param path a path to a trusted setup file in filesystem
+   * @throws KzgException if file not found or arguments from file are incorrect
+   */
+  void loadTrustedSetup(String path) throws KzgException;
+
+  /**
+   * Calculates aggregated proof for the given blobs
+   *
+   * @param blobs Blobs
+   * @return the aggregated proof
+   */
+  KZGProof computeAggregateKzgProof(List<Bytes> blobs) throws KzgException;
+
+  /**
+   * Verify aggregated proof and commitments for the given blobs
+   *
+   * @param blobs Blobs
+   * @param kzgCommitments KZG Commitments
+   * @param kzgProof The proof that needs verifying
+   * @return true if the proof is valid and false otherwise
+   */
+  boolean verifyAggregateKzgProof(
+      List<Bytes> blobs, List<KZGCommitment> kzgCommitments, KZGProof kzgProof) throws KzgException;
+
+  /**
+   * Calculates commitment for a given blob
+   *
+   * @param blob Blob
+   * @return the commitment
+   */
+  KZGCommitment blobToKzgCommitment(Bytes blob) throws KzgException;
+
+  /**
+   * Verify the proof by point evaluation for the given commitment
+   *
+   * @param kzgCommitment KZG Commitment
+   * @param z Z
+   * @param y Y
+   * @param kzgProof The proof that needs verifying
+   * @return true if the proof is valid and false otherwise
+   */
+  boolean verifyKzgProof(KZGCommitment kzgCommitment, Bytes32 z, Bytes32 y, KZGProof kzgProof)
+      throws KzgException;
+}

infrastructure/kzg/src/main/java/tech/pegasys/teku/kzg/impl/KzgException.java

@@ -0,0 +1,25 @@
+/*
+ * Copyright ConsenSys Software Inc., 2022
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
+ * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations under the License.
+ */
+
+package tech.pegasys.teku.kzg.impl;
+
+public class KzgException extends IllegalArgumentException {
+
+  public KzgException(String message) {
+    super(message);
+  }
+
+  public KzgException(String message, Throwable cause) {
+    super(message, cause);
+  }
+}

infrastructure/kzg/src/main/java/tech/pegasys/teku/kzg/impl/ckzg/CkzgKZG4844.java

@@ -0,0 +1,108 @@
+/*
+ * Copyright ConsenSys Software Inc., 2022
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
+ * an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations under the License.
+ */
+
+package tech.pegasys.teku.kzg.impl.ckzg;
+
+import ethereum.ckzg4844.CKzg4844JNI;
+import java.util.List;
+import org.apache.tuweni.bytes.Bytes;
+import org.apache.tuweni.bytes.Bytes32;
+import org.apache.tuweni.bytes.Bytes48;
+import tech.pegasys.teku.kzg.KZGCommitment;
+import tech.pegasys.teku.kzg.KZGProof;
+import tech.pegasys.teku.kzg.impl.KZG4844;
+import tech.pegasys.teku.kzg.impl.KzgException;
+
+/**
+ * Wrapper around JNI C-KZG library implementing stripped down KZG specification needed for EIP-4844
+ */
+public final class CkzgKZG4844 implements KZG4844 {
+
+  @Override
+  public void resetTrustedSetup() throws KzgException {
+    try {
+      CKzg4844JNI.freeTrustedSetup();
+    } catch (final Exception ex) {
+      throw new KzgException("Failed to reset trusted setup", ex);
+    }
+  }
+
+  @Override
+  public void loadTrustedSetup(final String path) throws KzgException {
+    try {
+      CKzg4844JNI.loadTrustedSetup(path);
+    } catch (final Exception ex) {
+      throw new KzgException(String.format("Failed to load trusted setup: %s", path), ex);
+    }
+  }
+
+  @Override
+  public KZGProof computeAggregateKzgProof(final List<Bytes> blobs) throws KzgException {
+    try {
+      final byte[] result =
+          CKzg4844JNI.computeAggregateKzgProof(
+              blobs.stream().reduce(Bytes::wrap).orElse(Bytes.EMPTY).toArray(), blobs.size());
+      return KZGProof.fromBytesCompressed(Bytes48.wrap(result));
+    } catch (final Exception ex) {
+      throw new KzgException("Failed to compute aggregated KZG Proof for Blobs", ex);
+    }
+  }
+
+  @Override
+  public boolean verifyAggregateKzgProof(
+      final List<Bytes> blobs, final List<KZGCommitment> kzgCommitments, final KZGProof kzgProof)
+      throws KzgException {
+    try {
+      return CKzg4844JNI.verifyAggregateKzgProof(
+          blobs.stream().reduce(Bytes::wrap).orElse(Bytes.EMPTY).toArray(),
+          kzgCommitments.stream()
+              .map(kzgCommitment -> (Bytes) kzgCommitment.getBytesCompressed())
+              .reduce(Bytes::wrap)
+              .orElse(Bytes.EMPTY)
+              .toArray(),
+          blobs.size(),
+          kzgProof.getBytesCompressed().toArray());
+    } catch (final Exception ex) {
+      throw new KzgException(
+          String.format("Failed to verify blobs against KZG Proof %s", kzgProof), ex);
+    }
+  }
+
+  @Override
+  public KZGCommitment blobToKzgCommitment(final Bytes blob) throws KzgException {
+    try {
+      return KZGCommitment.fromBytesCompressed(
+          Bytes48.wrap(CKzg4844JNI.blobToKzgCommitment(blob.toArray())));
+    } catch (final Exception ex) {
+      throw new KzgException("Failed to produce KZG Commitment from Blob", ex);
+    }
+  }
+
+  @Override
+  public boolean verifyKzgProof(
+      final KZGCommitment kzgCommitment, final Bytes32 z, final Bytes32 y, final KZGProof kzgProof)
+      throws KzgException {
+    try {
+      return CKzg4844JNI.verifyKzgProof(
+          kzgCommitment.getBytesCompressed().toArray(),
+          z.toArray(),
+          y.toArray(),
+          kzgProof.getBytesCompressed().toArray());
+    } catch (final Exception ex) {
+      throw new KzgException(
+          String.format(
+              "Failed to verify KZG Commitment %s against KZG Proof %s", kzgCommitment, kzgProof),
+          ex);
+    }
+  }
+}