Feature/uc 1395 introduce new hashed trackle msg pack version

pom.xml

@@ -20,7 +20,7 @@
 
     <groupId>com.ubirch</groupId>
     <artifactId>ubirch-protocol-java</artifactId>
-    <version>2.1.4-SNAPSHOT</version>
+    <version>2.1.5-SNAPSHOT</version>
     <packaging>jar</packaging>
 
     <licenses>
@@ -74,14 +74,14 @@
 
         <ubirch-crypto.version>2.0.6</ubirch-crypto.version>
         <jackson.version>2.13.3</jackson.version>
-        <msgpack-core.version>0.8.16</msgpack-core.version>
-        <commons-codec.version>1.11</commons-codec.version>
+        <msgpack-core.version>0.9.1</msgpack-core.version>
+        <commons-codec.version>1.13</commons-codec.version>
         <checkstyle.version>10.3</checkstyle.version>
         <maven-checkstyle-plugin.version>3.1.2</maven-checkstyle-plugin.version>
 
         <!-- Logging -->
-        <slf4j-api.version>1.7.25</slf4j-api.version>
-        <logback-classic.version>1.2.3</logback-classic.version>
+        <slf4j-api.version>1.7.36</slf4j-api.version>
+        <logback-classic.version>1.2.11</logback-classic.version>
         <junit.jupiter.version>5.2.0</junit.jupiter.version>
         <junit.platform.version>1.2.0</junit.platform.version>
     </properties>

src/main/java/com/ubirch/protocol/ProtocolMessage.java

@@ -47,22 +47,44 @@ public class ProtocolMessage {
 
     @JsonView(ProtocolMessageViews.Default.class)
     protected int version = 0;
+
+    /**
+     * This uuid represents the id of the device registered at Ubirch console and is
+     * also used to identify the key retrieved by the id-service to verify the UPP.
+     */
     @JsonInclude(NON_NULL)
     @JsonView(ProtocolMessageViews.Default.class)
     protected UUID uuid = null;
     @JsonInclude(NON_NULL)
     @JsonView(ProtocolMessageViews.Default.class)
     protected byte[] chain = null;
+
+    /**
+     * The hint represents the payload type of the ubirch msgPack variants as described in this documentation:
+     * https://github.com/ubirch/ubirch-protocol/blob/master/README_PAYLOAD.md#payload-types
+     */
     @JsonInclude(NON_NULL)
     @JsonView(ProtocolMessageViews.Default.class)
     protected int hint = 0;
 
+    /**
+     * Signed contains usually everything of the msgPack except for the signature.
+     * Only in the case of the hashed trackle message packet (hint = 0x56) it only
+     * contains the already SHA-512 hashed payload, as we don't want to include the
+     * original signed payload in this ubirch msgPack version.
+     */
     @JsonInclude(NON_NULL)
     @JsonView(ProtocolMessageViews.WithSignedData.class)
     protected byte[] signed;
+
     @JsonInclude(NON_NULL)
     @JsonView(ProtocolMessageViews.Default.class)
     protected byte[] signature = null;
+
+    /**
+     * The payload here describes the specific field payload of the ubirch protocol basic message format,
+     * as described here: https://github.com/ubirch/ubirch-protocol#basic-message-format
+     */
     @JsonInclude(NON_NULL)
     @JsonView(ProtocolMessageViews.Default.class)
     protected JsonNode payload;

src/main/java/com/ubirch/protocol/codec/DecoderUtil.java

@@ -0,0 +1,82 @@
+package com.ubirch.protocol.codec;
+
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.node.*;
+import org.msgpack.core.ExtensionTypeHeader;
+import org.msgpack.core.MessageFormat;
+import org.msgpack.core.MessageNeverUsedFormatException;
+import org.msgpack.core.MessageUnpacker;
+import org.msgpack.value.ImmutableStringValue;
+import org.msgpack.value.ValueFactory;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+public final class DecoderUtil {
+
+    private DecoderUtil() {
+        //not called
+    }
+
+    /**
+     * This method analyzes of what type the payload field is and retrieves the value as a JsonNode.
+     * This JsonNode can be of different types, e.g. BooleanNode, BinaryNode, TextNode, ...
+     */
+    public static JsonNode decodePayload(MessageUnpacker unpacker) throws IOException {
+        MessageFormat mf = unpacker.getNextFormat();
+        switch (mf.getValueType()) {
+            case NIL:
+                unpacker.unpackNil();
+                return NullNode.getInstance();
+            case BOOLEAN:
+                return BooleanNode.valueOf(unpacker.unpackBoolean());
+            case INTEGER:
+                if (mf == MessageFormat.UINT64) {
+                    return BigIntegerNode.valueOf(unpacker.unpackBigInteger());
+                }
+                return LongNode.valueOf(unpacker.unpackLong());
+            case FLOAT:
+                return DoubleNode.valueOf(unpacker.unpackDouble());
+            case STRING: {
+                int length = unpacker.unpackRawStringHeader();
+                ImmutableStringValue stringValue = ValueFactory.newString(unpacker.readPayload(length), true);
+                if (stringValue.isRawValue()) {
+                    return BinaryNode.valueOf(stringValue.asRawValue().asByteArray());
+                } else {
+                    return TextNode.valueOf(stringValue.asString());
+                }
+            }
+            case BINARY: {
+                int length = unpacker.unpackBinaryHeader();
+                return BinaryNode.valueOf(unpacker.readPayload(length));
+            }
+            case ARRAY: {
+                int size = unpacker.unpackArrayHeader();
+                List<JsonNode> array = new ArrayList<>(size);
+                for (int i = 0; i < size; i++) {
+                    array.add(decodePayload(unpacker));
+                }
+                return new ArrayNode(null, array);
+            }
+            case MAP: {
+                int size = unpacker.unpackMapHeader();
+                Map<String, JsonNode> kvs = new HashMap<>(size);
+                for (int i = 0; i < size; i++) {
+                    JsonNode kn = decodePayload(unpacker);
+                    String key = kn.isBinary() ? new String(kn.binaryValue()) : kn.asText();
+                    kvs.put(key, decodePayload(unpacker));
+                }
+                return new ObjectNode(null, kvs);
+            }
+            case EXTENSION: {
+                ExtensionTypeHeader extHeader = unpacker.unpackExtensionTypeHeader();
+                return BinaryNode.valueOf(unpacker.readPayload(extHeader.getLength()));
+            }
+            default:
+                throw new MessageNeverUsedFormatException("Unknown value type");
+        }
+    }
+}

src/main/java/com/ubirch/protocol/codec/EncoderUtil.java

@@ -0,0 +1,78 @@
+package com.ubirch.protocol.codec;
+
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.databind.node.*;
+import com.ubirch.protocol.ProtocolException;
+import com.ubirch.protocol.ProtocolMessage;
+import org.msgpack.core.MessagePacker;
+import org.msgpack.jackson.dataformat.MessagePackFactory;
+
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+import java.util.Base64;
+
+public final class EncoderUtil {
+
+    private EncoderUtil() {
+        //not called
+    }
+
+    public static void packVersion(MessagePacker packer, ProtocolMessage pm) throws IOException {
+        packer.packInt(pm.getVersion());
+    }
+
+    public static void packUUID(MessagePacker packer, ProtocolMessage pm) throws IOException {
+        packer.packBinaryHeader(16).addPayload(UUIDUtil.uuidToBytes(pm.getUUID()));
+    }
+
+    public static void packChain(MessagePacker packer, ProtocolMessage pm) throws IOException {
+        switch (pm.getVersion()) {
+            case ProtocolMessage.CHAINED:
+                packer.packBinaryHeader(64);
+                byte[] chainSignature = pm.getChain();
+                if (chainSignature == null) {
+                    packer.addPayload(new byte[64]);
+                } else {
+                    packer.addPayload(chainSignature);
+                }
+                break;
+            case ProtocolMessage.SIGNED:
+                break;
+            default:
+                throw new ProtocolException(String.format("unknown protocol version: 0x%x", pm.getVersion()));
+        }
+    }
+
+    public static void packHint(MessagePacker packer, ProtocolMessage pm) throws IOException {
+        packer.packInt(pm.getHint());
+    }
+
+    /**
+     * This method packs the msgPack depending on the JsonNode type.
+     */
+    public static void packPayload(MessagePacker packer, ProtocolMessage pm, ByteArrayOutputStream out) throws IOException {
+        // https://gitlab.com/ubirch/ubirch-kafka-envelope/-/blob/master/src/main/scala/com/ubirch/kafka/package.scala#L166
+        // json4s
+        // ------
+        // To be able to return the payload as just bytes and not as base64 values, we have to
+        // explicitly try to decode and pack the data in the msgpack.
+        // There seems to be a limitation with the way json4s handles binary nodes.
+        // https://gitlab.com/ubirch/ubirch-kafka-envelope/-/blob/master/src/main/scala/com/ubirch/kafka/package.scala#L166
+        if (pm.getPayload() instanceof BinaryNode) {
+            packer.packBinaryHeader(pm.getPayload().binaryValue().length);
+            packer.writePayload(pm.getPayload().binaryValue());
+        } else if (pm.getPayload() instanceof TextNode) {
+            try {
+                byte[] bytes = Base64.getDecoder().decode(pm.getPayload().asText());
+                packer.packBinaryHeader(bytes.length).addPayload(bytes);
+            } catch (Exception e)  {
+                ObjectMapper mapper = new ObjectMapper(new MessagePackFactory());
+                mapper.writeValue(out, pm.getPayload());
+            }
+        } else {
+            ObjectMapper mapper = new ObjectMapper(new MessagePackFactory());
+            mapper.writeValue(out, pm.getPayload());
+        }
+    }
+
+}

src/main/java/com/ubirch/protocol/codec/MsgPackProtocolDecoder.java

@@ -17,18 +17,22 @@
 package com.ubirch.protocol.codec;
 
 import com.fasterxml.jackson.databind.DeserializationFeature;
-import com.fasterxml.jackson.databind.JsonNode;
 import com.fasterxml.jackson.databind.ObjectMapper;
-import com.fasterxml.jackson.databind.node.*;
 import com.ubirch.protocol.ProtocolException;
 import com.ubirch.protocol.ProtocolMessage;
-import org.msgpack.core.*;
+import org.apache.commons.codec.binary.Hex;
+import org.msgpack.core.MessagePack;
+import org.msgpack.core.MessagePackException;
+import org.msgpack.core.MessageUnpacker;
 import org.msgpack.jackson.dataformat.MessagePackFactory;
-import org.msgpack.value.*;
+import org.msgpack.value.ValueType;
 
 import java.io.ByteArrayInputStream;
 import java.io.IOException;
-import java.util.*;
+import java.util.Arrays;
+
+import static com.ubirch.protocol.codec.DecoderUtil.decodePayload;
+import static com.ubirch.protocol.codec.ProtocolHints.HASHED_TRACKLE_MSG_PACK_HINT;
 
 /**
  * The default msgpack ubirch protocol decoder.
@@ -63,7 +67,6 @@ public static MsgPackProtocolDecoder getDecoder() {
     public ProtocolMessage decode(byte[] message) throws ProtocolException {
         boolean legacyPayloadDecoding = false;
         ByteArrayInputStream in = new ByteArrayInputStream(message);
-
         MessageUnpacker unpacker = MessagePack.newDefaultUnpacker(in);
         ProtocolMessage pm = new ProtocolMessage();
         try {
@@ -101,9 +104,14 @@ public ProtocolMessage decode(byte[] message) throws ProtocolException {
                 }
 
                 // finally store the signed data and signature for later verification
-                pm.setSigned(Arrays.copyOfRange(message, 0, (int) unpacker.getTotalReadBytes()));
+                if (pm.getHint() == HASHED_TRACKLE_MSG_PACK_HINT) {
+                    // in hashed trackle msg packs the payload contains the signed SHA-512 hash
+                    pm.setSigned(pm.getPayload().binaryValue());
+                } else {
+                    // in all other UPPs all bytes including the payload except for the signature are signed
+                    pm.setSigned(Arrays.copyOfRange(message, 0, (int) unpacker.getTotalReadBytes()));
+                }
                 pm.setSignature(unpacker.readPayload(unpacker.unpackRawStringHeader()));
-
                 return pm;
             } else {
                 throw new ProtocolException(String.format("unknown msgpack envelope format: %s[%d]", envelopeType.name(), envelopeLength));
@@ -115,8 +123,19 @@ public ProtocolMessage decode(byte[] message) throws ProtocolException {
         }
     }
 
+    public boolean isHashedTrackleMsgType(byte[] message) {
+        if (message.length > 133) {
+            byte[] hint = Arrays.copyOfRange(message, message.length - 133, message.length - 132);
+            return Hex.encodeHexString(hint).equals(Integer.toHexString(HASHED_TRACKLE_MSG_PACK_HINT));
+        } else {
+            return false;
+        }
+    }
+
     /**
-     * Extracts the signed part and the signature out of the message pack without materializing.
+     * Extracts the signed part and the signature out of the message pack without materializing the other fields
+     * of the msgPack.
+     *
      * @param message the raw protocol message in msgpack format
      * @return an array of arrays where the first element is the signed data and the second element is the signature.
      * @throws ProtocolException if the fast extraction fails
@@ -132,11 +151,21 @@ public byte[][] getDataToVerifyAndSignature(byte[] message) throws ProtocolExcep
             if (envelopeLength > 4 && envelopeLength < 7) {
 
                 //We skip through the values up to the signature.
-                for (int i = 0; i < envelopeLength - 1; i++) {
+                for (int i = 0; i < envelopeLength - 3; i++) {
+                    unpacker.skipValue();
+                }
+
+                byte[] signedBytes;
+                if (unpacker.unpackInt() == HASHED_TRACKLE_MSG_PACK_HINT) {
+                    // in hashed trackle msg packs the payload contains the signed SHA-512 hash
+                    int length = unpacker.unpackBinaryHeader();
+                    signedBytes = unpacker.readPayload(length);
+                } else {
+                    // in all other UPPs all bytes including the payload except for the signature are signed
                     unpacker.skipValue();
+                    signedBytes = Arrays.copyOfRange(message, 0, (int) unpacker.getTotalReadBytes());
                 }
 
-                byte[] signedBytes = Arrays.copyOfRange(message, 0, (int) unpacker.getTotalReadBytes());
                 byte[] signatureBytes = unpacker.readPayload(unpacker.unpackRawStringHeader());
 
                 return new byte[][]{signedBytes, signatureBytes};
@@ -150,58 +179,4 @@ public byte[][] getDataToVerifyAndSignature(byte[] message) throws ProtocolExcep
         }
     }
 
-    private JsonNode decodePayload(MessageUnpacker unpacker) throws IOException {
-        MessageFormat mf = unpacker.getNextFormat();
-        switch (mf.getValueType()) {
-            case NIL:
-                unpacker.unpackNil();
-                return NullNode.getInstance();
-            case BOOLEAN:
-                return BooleanNode.valueOf(unpacker.unpackBoolean());
-            case INTEGER:
-                if (mf == MessageFormat.UINT64) {
-                    return BigIntegerNode.valueOf(unpacker.unpackBigInteger());
-                }
-                return LongNode.valueOf(unpacker.unpackLong());
-            case FLOAT:
-                return DoubleNode.valueOf(unpacker.unpackDouble());
-            case STRING: {
-                int length = unpacker.unpackRawStringHeader();
-                ImmutableStringValue stringValue = ValueFactory.newString(unpacker.readPayload(length), true);
-                if (stringValue.isRawValue()) {
-                    return BinaryNode.valueOf(stringValue.asRawValue().asByteArray());
-                } else {
-                    return TextNode.valueOf(stringValue.asString());
-                }
-            }
-            case BINARY: {
-                int length = unpacker.unpackBinaryHeader();
-                return BinaryNode.valueOf(unpacker.readPayload(length));
-            }
-            case ARRAY: {
-                int size = unpacker.unpackArrayHeader();
-                List<JsonNode> array = new ArrayList<>(size);
-                for (int i = 0; i < size; i++) {
-                    array.add(decodePayload(unpacker));
-                }
-                return new ArrayNode(null, array);
-            }
-            case MAP: {
-                int size = unpacker.unpackMapHeader();
-                Map<String, JsonNode> kvs = new HashMap<>(size);
-                for (int i = 0; i < size; i++) {
-                    JsonNode kn = decodePayload(unpacker);
-                    String key = kn.isBinary() ? new String(kn.binaryValue()) : kn.asText();
-                    kvs.put(key, decodePayload(unpacker));
-                }
-                return new ObjectNode(null, kvs);
-            }
-            case EXTENSION: {
-                ExtensionTypeHeader extHeader = unpacker.unpackExtensionTypeHeader();
-                return BinaryNode.valueOf(unpacker.readPayload(extHeader.getLength()));
-            }
-            default:
-                throw new MessageNeverUsedFormatException("Unknown value type");
-        }
-    }
 }