Initial Skeleton for elastic DAO / Metadata Serializer

.gitignore

@@ -4,10 +4,11 @@ bin
 .gradle/
 .idea/
 .cache/
+out/
 .settings/
 build
 *.iws
 *.iml
 *.ipr
 kibana/
-data/
+data/

analyzer/common/build.gradle

@@ -4,6 +4,8 @@ plugins {
 
 dependencies {
   compile project(':model')
-  compile 'org.rocksdb:rocksdbjni:5.13.2'
-  compile group: 'org.xerial', name: 'sqlite-jdbc', version: '3.23.1'
+  compile project(':logging')
+//  compile 'org.rocksdb:rocksdbjni:5.13.2'
+//  compile group: 'org.xerial', name: 'sqlite-jdbc', version: '3.23.1'
+  testCompile 'junit:junit:4.12'
 }

analyzer/common/src/main/java/castro/analyzer/FSMetadataReader.java

@@ -0,0 +1,44 @@
+package castro.analyzer;
+
+import castro.metadata.Node;
+import com.google.protobuf.CodedInputStream;
+import com.google.protobuf.ExtensionRegistryLite;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.FileNotFoundException;
+import java.io.IOException;
+
+public class FSMetadataReader implements MetadataReader {
+  private Logger logger = LoggerFactory.getLogger(getClass());
+
+  private FileInputStream rawNodeStream;
+  private CodedInputStream nodeStream;
+
+  public FSMetadataReader(File baseDir) {
+    try {
+      this.rawNodeStream = new FileInputStream(new File(baseDir, "node"));
+      this.nodeStream = CodedInputStream.newInstance(rawNodeStream);
+    } catch (FileNotFoundException e) {
+      logger.error("Initialization Error", e);
+    }
+  }
+
+  @Override
+  public Node readNode() throws IOException {
+      var builder = Node.newBuilder();
+      nodeStream.readMessage(builder, ExtensionRegistryLite.getEmptyRegistry());
+      return builder.build();
+  }
+
+  @Override
+  public void close() {
+    try {
+      rawNodeStream.close();
+    } catch (IOException e) {
+      logger.error("Serializer close error", e);
+    }
+  }
+}

analyzer/common/src/main/java/castro/analyzer/FSMetadataWriter.java

@@ -0,0 +1,51 @@
+package castro.analyzer;
+
+import castro.metadata.Node;
+import com.google.protobuf.CodedOutputStream;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.File;
+import java.io.FileOutputStream;
+import java.io.IOException;
+
+public class FSMetadataWriter implements MetadataWriter {
+  private Logger logger = LoggerFactory.getLogger(getClass());
+
+  private FileOutputStream rawNodeStream;
+  private CodedOutputStream nodeStream;
+
+  public FSMetadataWriter(File baseDir) {
+    try {
+      var file = new File(baseDir, "node");
+      if (!file.exists()) {
+        file.createNewFile();
+      }
+      this.rawNodeStream = new FileOutputStream(file);
+      this.nodeStream = CodedOutputStream.newInstance(rawNodeStream);
+    } catch (IOException e) {
+      logger.error("Initialization Error", e);
+    }
+  }
+
+  @Override
+  public MetadataWriter writeNode(Node node) {
+    try {
+      nodeStream.writeMessageNoTag(node);
+    } catch (IOException e) {
+      logger.error("Node Write Error", e);
+    }
+    return this;
+  }
+
+
+  @Override
+  public void close() {
+    try {
+      nodeStream.flush();
+      rawNodeStream.close();
+    } catch (IOException e) {
+      logger.error("Serializer close error", e);
+    }
+  }
+}

analyzer/common/src/main/java/castro/analyzer/MetadataReader.java

@@ -0,0 +1,10 @@
+package castro.analyzer;
+
+import castro.metadata.Node;
+
+import java.io.IOException;
+
+interface MetadataReader {
+  Node readNode() throws IOException;
+  void close();
+}

analyzer/common/src/main/java/castro/analyzer/MetadataWriter.java

@@ -0,0 +1,8 @@
+package castro.analyzer;
+
+import castro.metadata.Node;
+
+interface MetadataWriter {
+  MetadataWriter writeNode(Node node);
+  void close();
+}

analyzer/common/src/test/java/castro/analyzer/MetadataWriteReadTest.java

@@ -0,0 +1,44 @@
+package castro.analyzer;
+
+import castro.metadata.Node;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.TestName;
+
+import java.io.File;
+import java.io.IOException;
+
+import static junit.framework.TestCase.assertEquals;
+import static junit.framework.TestCase.assertTrue;
+
+public class MetadataWriteReadTest {
+
+  @Rule
+  public TestName testName = new TestName();
+
+
+  @Test
+  public void checkNodeWriteRead() throws IOException {
+    var file = File.createTempFile(testName.getMethodName(), "");
+    file.delete();
+    file.mkdir();
+    assertTrue(file.isDirectory());
+
+
+    Node n1 = Node.newBuilder().setIdentifier("1").build();
+    Node n2 = Node.newBuilder().setIdentifier("2").setQname("str").build();
+
+    var writer = new FSMetadataWriter(file);
+    writer.writeNode(n1).writeNode(n2).close();
+
+    var reader = new FSMetadataReader(file);
+    Node n3 = reader.readNode();
+    Node n4 = reader.readNode();
+
+    assertEquals(n1, n3);
+    assertEquals(n2, n4);
+
+    reader.close();
+  }
+
+}

backend/build.gradle

@@ -10,5 +10,5 @@ task backend(dependsOn: run)
 dependencies {
   compile project(":model")
 
-  compile "org.eclipse.jgit:org.eclipse.jgit:5.0.0.201805301535-rc2"
+  compile "org.eclipse.jgit:org.eclipse.jgit:$jgitVersion"
 }

backend/src/main/java/castro/backend/Main.java

@@ -1,7 +1,12 @@
 package castro.backend;
 
+import castro.metadata.Node;
+
 class Main {
   public static void main(String[] args) {
+//    Node.QNAME_FIELD_NUMBER
+//    Node.getDescriptor().findFieldByNumber(Node.QNAME_FIELD_NUMBER).getName()
+
     System.out.println("Hello Java Backend!");
   }
 }

elasticsearch/build.gradle

@@ -0,0 +1,12 @@
+plugins {
+  id 'java'
+}
+
+dependencies {
+  compile project(":model")
+  compile project(":logging")
+  compile 'org.elasticsearch.client:elasticsearch-rest-high-level-client:6.2.4'
+
+  // Check if lite dep could be used
+  compile "com.google.protobuf:protobuf-java:${protocVersion}"
+}

elasticsearch/src/main/java/castro/elasticsearch/dao/ElasticsearchDao.java

@@ -0,0 +1,35 @@
+package castro.elasticsearch.dao;
+
+import com.google.protobuf.GeneratedMessageV3;
+import org.elasticsearch.client.RestHighLevelClient;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+
+public abstract class ElasticsearchDao<T extends GeneratedMessageV3> {
+  private Logger logger = LoggerFactory.getLogger(getClass());
+
+  // Old TransportClient is deprecated
+  private RestHighLevelClient client;
+
+  protected ElasticsearchDao(RestHighLevelClient client) {
+    this.client = client;
+  }
+
+  void save(T t) {
+//    client.
+  }
+
+  T findById(String id) {
+    return null;
+  }
+
+  void close() {
+    try {
+      client.close();
+    } catch (IOException e) {
+      logger.error("Close error", e);
+    }
+  }
+}

elasticsearch/src/main/java/castro/elasticsearch/dao/NodeDao.java

@@ -0,0 +1,14 @@
+package castro.elasticsearch.dao;
+
+import castro.metadata.Node;
+import org.elasticsearch.client.RestHighLevelClient;
+
+public class NodeDao extends ElasticsearchDao<Node> {
+  protected NodeDao(RestHighLevelClient client) {
+    super(client);
+  }
+
+  public Node findByQname(String qname) {
+    return null;
+  }
+}

gradle.properties

@@ -5,3 +5,4 @@ nodeVersion = 8.11.0
 yarnVersion = 1.6.0
 
 protocVersion = 3.5.1
+jgitVersion = 5.0.0.201805301535-rc2

logging/build.gradle

@@ -0,0 +1,7 @@
+plugins {
+  id 'java'
+}
+
+dependencies {
+  compile 'ch.qos.logback:logback-classic:1.2.3'
+}

model/src/Pipeline.proto

@@ -0,0 +1,9 @@
+syntax = "proto3";
+
+option java_package = "castro.pipeline";
+option java_multiple_files = true;
+
+message PipelineConfig {
+  string repositoryId = 1;
+  string revision = 2;
+}

pipeline/build.gradle

@@ -5,6 +5,8 @@ plugins {
 
 dependencies {
   compile project(':model')
+  compile "org.eclipse.jgit:org.eclipse.jgit:$jgitVersion"
+
 }
 
 mainClassName = "castro.pipeline.Main"

pipeline/src/main/java/castro/pipeline/Clone.java

@@ -0,0 +1,7 @@
+package castro.pipeline;
+
+public class Clone {
+  void runStep(PipelineConfig config) {
+
+  }
+}

settings.gradle

@@ -1,3 +1,6 @@
+include 'logging'
+include 'elasticsearch'
+
 include 'backend'
 include 'model'
 include 'pipeline'