Skip to content

Commit

Permalink
Initialize the spark tool of GraphAr and implement the Info and Index…
Browse files Browse the repository at this point in the history 
…Generator (#45)
  • Loading branch information
@acezen
acezen authored Dec 16, 2022
1 parent 47951d9 commit 4488167
Show file tree
Hide file tree
Showing 12 changed files with 1,072 additions and 1 deletion.
2 changes: 2 additions & 0 deletions .github/workflows/ci.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@ on:
- '**.rst'
- 'docs/**'
- 'misc/**'
- 'spark/**'
pull_request:
branches:
- main
Expand All @@ -21,6 +22,7 @@ on:
- '**.rst'
- 'docs/**'
- 'misc/**'
- 'spark/**'

concurrency:
group: ${{ github.repository }}-${{ github.event.number || github.head_ref || github.sha }}-${{ github.workflow }}
Expand Down
41 changes: 41 additions & 0 deletions .github/workflows/spark.yaml
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,41 @@
name: GraphAr Spark CI

on:
# Trigger the workflow on push or pull request,
# but only for the main branch
push:
branches:
- main
paths:
- 'spark/**'
pull_request:
branches:
- main
paths:
- 'spark/**'

concurrency:
group: ${{ github.repository }}-${{ github.event.number || github.head_ref || github.sha }}-${{ github.workflow }}
cancel-in-progress: true

jobs:
GraphAr-spark:
runs-on: ubuntu-20.04
steps:
- uses: actions/checkout@v2
with:
submodules: true

- name: Build GraphAr Spark
run: |
export JAVA_HOME=${JAVA_HOME_11_X64}
pushd spark
mvn clean package
popd
- name: Run test
run: |
export JAVA_HOME=${JAVA_HOME_11_X64}
pushd spark
mvn test
popd
2 changes: 1 addition & 1 deletion .gitignore
View file
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
/build/
spark/generator/target/
spark/target/
.vscode
.idea
.DS_store
Expand Down
125 changes: 125 additions & 0 deletions spark/pom.xml
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,125 @@
<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>

<groupId>com.alibaba</groupId>
<artifactId>graphar</artifactId>
<version>0.1.0-SNAPSHOT</version>

<properties>
<sbt.project.name>graphar</sbt.project.name>
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
<project.reporting.outputEncoding>UTF-8</project.reporting.outputEncoding>
<scala.version>2.12.10</scala.version>
<scala.binary.version>2.12</scala.binary.version>
<PermGen>512m</PermGen>
<MaxPermGen>1024m</MaxPermGen>
<spark.version>3.1.1</spark.version>
<maven.compiler.release>8</maven.compiler.release>
<maven.compiler.source>1.8</maven.compiler.source>
<maven.compiler.target>1.8</maven.compiler.target>
</properties>
<dependencies>
<dependency>
<groupId>org.apache.spark</groupId>
<artifactId>spark-core_${scala.binary.version}</artifactId>
<version>${spark.version}</version>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>org.apache.spark</groupId>
<artifactId>spark-streaming_${scala.binary.version}</artifactId>
<version>${spark.version}</version>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>org.apache.spark</groupId>
<artifactId>spark-mllib_${scala.binary.version}</artifactId>
<version>${spark.version}</version>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>org.apache.spark</groupId>
<artifactId>spark-sql_${scala.binary.version}</artifactId>
<version>${spark.version}</version>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>org.apache.spark</groupId>
<artifactId>spark-hive_${scala.binary.version}</artifactId>
<version>${spark.version}</version>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>org.scalatest</groupId>
<artifactId>scalatest_${scala.binary.version}</artifactId>
<version>3.1.1</version>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>org.scala-lang</groupId>
<artifactId>scala-library</artifactId>
<version>${scala.version}</version>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>org.yaml</groupId>
<artifactId>snakeyaml</artifactId>
<version>1.26</version>
</dependency>
</dependencies>
<build>
<plugins>
<plugin>
<groupId>org.scala-tools</groupId>
<artifactId>maven-scala-plugin</artifactId>
<version>2.15.2</version>
<configuration>
<scalaVersion>${scala.version}</scalaVersion>
<args>
<arg>-target:jvm-1.8</arg>
</args>
<jvmArgs>
<jvmArg>-Xss4096K</jvmArg>
</jvmArgs>
</configuration>
<executions>
<execution>
<id>scala-compile</id>
<goals>
<goal>compile</goal>
</goals>
<configuration>
<excludes>
<exclude>META-INF/*.SF</exclude>
<exclude>META-INF/*.DSA</exclude>
<exclude>META-INF/*.RSA</exclude>
</excludes>
</configuration>
</execution>
<execution>
<id>scala-test-compile</id>
<goals>
<goal>testCompile</goal>
</goals>
</execution>
</executions>
</plugin>
<plugin>
<groupId>org.scalatest</groupId>
<artifactId>scalatest-maven-plugin</artifactId>
<version>2.0.0</version>
<executions>
<execution>
<id>test</id>
<goals>
<goal>test</goal>
</goals>
</execution>
</executions>
</plugin>
</plugins>
</build>
<packaging>jar</packaging>
</project>
14 changes: 14 additions & 0 deletions spark/src/main/java/com/alibaba/graphar/GeneralParams.java
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,14 @@
package com.alibaba.graphar;

public class GeneralParams {
// column name
public static final String vertexIndexCol = "_graphArVertexIndex";
public static final String srcIndexCol = "_graphArSrcIndex";
public static final String dstIndexCol = "_graphArDstIndex";
public static final String offsetCol = "_graphArOffset";
public static final String primaryCol = "_graphArPrimary";
public static final String vertexChunkIndexCol = "_graphArVertexChunkIndex";
public static final String edgeIndexCol = "_graphArEdgeIndex";
public static final String regularSeperator = "_";
}

Loading

0 comments on commit 4488167

Please sign in to comment.